From 4184e4912ca69d4f18a800144539af3b37c6a663 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 12 Mar 2024 17:44:28 +0100 Subject: [PATCH 0001/1702] dt-bindings: pinctrl: samsung: drop unused header with register constants The bindings header for Samsung pin controller DTS pin values (holding register values in fact) was deprecated in v6.1 kernel in commit 9d9292576810 ("dt-bindings: pinctrl: samsung: deprecate header with register constants"). This was enough of time for users to switch to in-DTS headers, so drop the bindings header. Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20240312164428.692552-1-krzysztof.kozlowski@linaro.org Signed-off-by: Krzysztof Kozlowski --- MAINTAINERS | 1 - include/dt-bindings/pinctrl/samsung.h | 95 --------------------------- 2 files changed, 96 deletions(-) delete mode 100644 include/dt-bindings/pinctrl/samsung.h diff --git a/MAINTAINERS b/MAINTAINERS index aa3b947fb0801..643dba658346b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17489,7 +17489,6 @@ C: irc://irc.libera.chat/linux-exynos T: git git://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git F: Documentation/devicetree/bindings/pinctrl/samsung,pinctrl*yaml F: drivers/pinctrl/samsung/ -F: include/dt-bindings/pinctrl/samsung.h PIN CONTROLLER - SINGLE M: Tony Lindgren diff --git a/include/dt-bindings/pinctrl/samsung.h b/include/dt-bindings/pinctrl/samsung.h deleted file mode 100644 index d1da5ff68d0c3..0000000000000 --- a/include/dt-bindings/pinctrl/samsung.h +++ /dev/null @@ -1,95 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Samsung's Exynos pinctrl bindings - * - * Copyright (c) 2016 Samsung Electronics Co., Ltd. - * http://www.samsung.com - * Author: Krzysztof Kozlowski - */ - -#ifndef __DT_BINDINGS_PINCTRL_SAMSUNG_H__ -#define __DT_BINDINGS_PINCTRL_SAMSUNG_H__ - -/* - * These bindings are deprecated, because they do not match the actual - * concept of bindings but rather contain pure register values. - * Instead include the header in the DTS source directory. - */ -#warning "These bindings are deprecated. Instead use the header in the DTS source directory." - -#define EXYNOS_PIN_PULL_NONE 0 -#define EXYNOS_PIN_PULL_DOWN 1 -#define EXYNOS_PIN_PULL_UP 3 - -#define S3C64XX_PIN_PULL_NONE 0 -#define S3C64XX_PIN_PULL_DOWN 1 -#define S3C64XX_PIN_PULL_UP 2 - -/* Pin function in power down mode */ -#define EXYNOS_PIN_PDN_OUT0 0 -#define EXYNOS_PIN_PDN_OUT1 1 -#define EXYNOS_PIN_PDN_INPUT 2 -#define EXYNOS_PIN_PDN_PREV 3 - -/* Drive strengths for Exynos3250, Exynos4 (all) and Exynos5250 */ -#define EXYNOS4_PIN_DRV_LV1 0 -#define EXYNOS4_PIN_DRV_LV2 2 -#define EXYNOS4_PIN_DRV_LV3 1 -#define EXYNOS4_PIN_DRV_LV4 3 - -/* Drive strengths for Exynos5260 */ -#define EXYNOS5260_PIN_DRV_LV1 0 -#define EXYNOS5260_PIN_DRV_LV2 1 -#define EXYNOS5260_PIN_DRV_LV4 2 -#define EXYNOS5260_PIN_DRV_LV6 3 - -/* - * Drive strengths for Exynos5410, Exynos542x, Exynos5800 and Exynos850 (except - * GPIO_HSI block) - */ -#define EXYNOS5420_PIN_DRV_LV1 0 -#define EXYNOS5420_PIN_DRV_LV2 1 -#define EXYNOS5420_PIN_DRV_LV3 2 -#define EXYNOS5420_PIN_DRV_LV4 3 - -/* Drive strengths for Exynos5433 */ -#define EXYNOS5433_PIN_DRV_FAST_SR1 0 -#define EXYNOS5433_PIN_DRV_FAST_SR2 1 -#define EXYNOS5433_PIN_DRV_FAST_SR3 2 -#define EXYNOS5433_PIN_DRV_FAST_SR4 3 -#define EXYNOS5433_PIN_DRV_FAST_SR5 4 -#define EXYNOS5433_PIN_DRV_FAST_SR6 5 -#define EXYNOS5433_PIN_DRV_SLOW_SR1 8 -#define EXYNOS5433_PIN_DRV_SLOW_SR2 9 -#define EXYNOS5433_PIN_DRV_SLOW_SR3 0xa -#define EXYNOS5433_PIN_DRV_SLOW_SR4 0xb -#define EXYNOS5433_PIN_DRV_SLOW_SR5 0xc -#define EXYNOS5433_PIN_DRV_SLOW_SR6 0xf - -/* Drive strengths for Exynos850 GPIO_HSI block */ -#define EXYNOS850_HSI_PIN_DRV_LV1 0 /* 1x */ -#define EXYNOS850_HSI_PIN_DRV_LV1_5 1 /* 1.5x */ -#define EXYNOS850_HSI_PIN_DRV_LV2 2 /* 2x */ -#define EXYNOS850_HSI_PIN_DRV_LV2_5 3 /* 2.5x */ -#define EXYNOS850_HSI_PIN_DRV_LV3 4 /* 3x */ -#define EXYNOS850_HSI_PIN_DRV_LV4 5 /* 4x */ - -#define EXYNOS_PIN_FUNC_INPUT 0 -#define EXYNOS_PIN_FUNC_OUTPUT 1 -#define EXYNOS_PIN_FUNC_2 2 -#define EXYNOS_PIN_FUNC_3 3 -#define EXYNOS_PIN_FUNC_4 4 -#define EXYNOS_PIN_FUNC_5 5 -#define EXYNOS_PIN_FUNC_6 6 -#define EXYNOS_PIN_FUNC_EINT 0xf -#define EXYNOS_PIN_FUNC_F EXYNOS_PIN_FUNC_EINT - -/* Drive strengths for Exynos7 FSYS1 block */ -#define EXYNOS7_FSYS1_PIN_DRV_LV1 0 -#define EXYNOS7_FSYS1_PIN_DRV_LV2 4 -#define EXYNOS7_FSYS1_PIN_DRV_LV3 2 -#define EXYNOS7_FSYS1_PIN_DRV_LV4 6 -#define EXYNOS7_FSYS1_PIN_DRV_LV5 1 -#define EXYNOS7_FSYS1_PIN_DRV_LV6 5 - -#endif /* __DT_BINDINGS_PINCTRL_SAMSUNG_H__ */ From 5a6cb47ef05a48cec778207fff6c189de8f27a0e Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Mon, 25 Mar 2024 12:12:40 +0800 Subject: [PATCH 0002/1702] fs: quota: use group allocation of per-cpu counters API Use group allocation of per-cpu counters api to accelerate dquot_init() and simplify code. Signed-off-by: Kefeng Wang Signed-off-by: Jan Kara Message-Id: <20240325041240.53537-1-wangkefeng.wang@huawei.com> --- fs/quota/dquot.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c index dacbee455c034..808544f74e5e2 100644 --- a/fs/quota/dquot.c +++ b/fs/quota/dquot.c @@ -3016,11 +3016,10 @@ static int __init dquot_init(void) if (!dquot_hash) panic("Cannot create dquot hash table"); - for (i = 0; i < _DQST_DQSTAT_LAST; i++) { - ret = percpu_counter_init(&dqstats.counter[i], 0, GFP_KERNEL); - if (ret) - panic("Cannot create dquot stat counters"); - } + ret = percpu_counter_init_many(dqstats.counter, 0, GFP_KERNEL, + _DQST_DQSTAT_LAST); + if (ret) + panic("Cannot create dquot stat counters"); /* Find power-of-two hlist_heads which can fit into allocation */ nr_hash = (1UL << order) * PAGE_SIZE / sizeof(struct hlist_head); From 1b17a46c9243e9421ee1ac6d628604bbc4ae2201 Mon Sep 17 00:00:00 2001 From: Eric Sandeen Date: Thu, 7 Mar 2024 12:56:33 -0600 Subject: [PATCH 0003/1702] isofs: convert isofs to use the new mount API This also renames iso9660_options to isofs_options, for consistency. Signed-off-by: Eric Sandeen cc: Jan Kara Signed-off-by: Jan Kara Message-Id: <409e28da-9c19-427a-acfb-78788f0a23b8@redhat.com> --- fs/isofs/inode.c | 473 ++++++++++++++++++++++++----------------------- 1 file changed, 240 insertions(+), 233 deletions(-) diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c index 2a616a9f289da..93b1077a380a8 100644 --- a/fs/isofs/inode.c +++ b/fs/isofs/inode.c @@ -21,11 +21,12 @@ #include #include #include -#include #include #include #include #include +#include +#include #include "isofs.h" #include "zisofs.h" @@ -110,10 +111,10 @@ static void destroy_inodecache(void) kmem_cache_destroy(isofs_inode_cachep); } -static int isofs_remount(struct super_block *sb, int *flags, char *data) +static int isofs_reconfigure(struct fs_context *fc) { - sync_filesystem(sb); - if (!(*flags & SB_RDONLY)) + sync_filesystem(fc->root->d_sb); + if (!(fc->sb_flags & SB_RDONLY)) return -EROFS; return 0; } @@ -123,7 +124,6 @@ static const struct super_operations isofs_sops = { .free_inode = isofs_free_inode, .put_super = isofs_put_super, .statfs = isofs_statfs, - .remount_fs = isofs_remount, .show_options = isofs_show_options, }; @@ -145,7 +145,7 @@ static const struct dentry_operations isofs_dentry_ops[] = { #endif }; -struct iso9660_options{ +struct isofs_options{ unsigned int rock:1; unsigned int joliet:1; unsigned int cruft:1; @@ -289,197 +289,161 @@ isofs_dentry_cmpi_ms(const struct dentry *dentry, #endif enum { - Opt_block, Opt_check_r, Opt_check_s, Opt_cruft, Opt_gid, Opt_ignore, - Opt_iocharset, Opt_map_a, Opt_map_n, Opt_map_o, Opt_mode, Opt_nojoliet, - Opt_norock, Opt_sb, Opt_session, Opt_uid, Opt_unhide, Opt_utf8, Opt_err, - Opt_nocompress, Opt_hide, Opt_showassoc, Opt_dmode, Opt_overriderockperm, + Opt_block, Opt_check, Opt_cruft, Opt_gid, Opt_ignore, Opt_iocharset, + Opt_map, Opt_mode, Opt_nojoliet, Opt_norock, Opt_sb, Opt_session, + Opt_uid, Opt_unhide, Opt_utf8, Opt_err, Opt_nocompress, Opt_hide, + Opt_showassoc, Opt_dmode, Opt_overriderockperm, }; -static const match_table_t tokens = { - {Opt_norock, "norock"}, - {Opt_nojoliet, "nojoliet"}, - {Opt_unhide, "unhide"}, - {Opt_hide, "hide"}, - {Opt_showassoc, "showassoc"}, - {Opt_cruft, "cruft"}, - {Opt_utf8, "utf8"}, - {Opt_iocharset, "iocharset=%s"}, - {Opt_map_a, "map=acorn"}, - {Opt_map_a, "map=a"}, - {Opt_map_n, "map=normal"}, - {Opt_map_n, "map=n"}, - {Opt_map_o, "map=off"}, - {Opt_map_o, "map=o"}, - {Opt_session, "session=%u"}, - {Opt_sb, "sbsector=%u"}, - {Opt_check_r, "check=relaxed"}, - {Opt_check_r, "check=r"}, - {Opt_check_s, "check=strict"}, - {Opt_check_s, "check=s"}, - {Opt_uid, "uid=%u"}, - {Opt_gid, "gid=%u"}, - {Opt_mode, "mode=%u"}, - {Opt_dmode, "dmode=%u"}, - {Opt_overriderockperm, "overriderockperm"}, - {Opt_block, "block=%u"}, - {Opt_ignore, "conv=binary"}, - {Opt_ignore, "conv=b"}, - {Opt_ignore, "conv=text"}, - {Opt_ignore, "conv=t"}, - {Opt_ignore, "conv=mtext"}, - {Opt_ignore, "conv=m"}, - {Opt_ignore, "conv=auto"}, - {Opt_ignore, "conv=a"}, - {Opt_nocompress, "nocompress"}, - {Opt_err, NULL} +static const struct constant_table isofs_param_map[] = { + {"acorn", 'a'}, + {"a", 'a'}, + {"normal", 'n'}, + {"n", 'n'}, + {"off", 'o'}, + {"o", 'o'}, + {} }; -static int parse_options(char *options, struct iso9660_options *popt) -{ - char *p; - int option; - unsigned int uv; - - popt->map = 'n'; - popt->rock = 1; - popt->joliet = 1; - popt->cruft = 0; - popt->hide = 0; - popt->showassoc = 0; - popt->check = 'u'; /* unset */ - popt->nocompress = 0; - popt->blocksize = 1024; - popt->fmode = popt->dmode = ISOFS_INVALID_MODE; - popt->uid_set = 0; - popt->gid_set = 0; - popt->gid = GLOBAL_ROOT_GID; - popt->uid = GLOBAL_ROOT_UID; - popt->iocharset = NULL; - popt->overriderockperm = 0; - popt->session=-1; - popt->sbsector=-1; - if (!options) - return 1; - - while ((p = strsep(&options, ",")) != NULL) { - int token; - substring_t args[MAX_OPT_ARGS]; - unsigned n; - - if (!*p) - continue; +static const struct constant_table isofs_param_check[] = { + {"relaxed", 'r'}, + {"r", 'r'}, + {"strict", 's'}, + {"s", 's'}, + {} +}; - token = match_token(p, tokens, args); - switch (token) { - case Opt_norock: - popt->rock = 0; - break; - case Opt_nojoliet: - popt->joliet = 0; - break; - case Opt_hide: - popt->hide = 1; - break; - case Opt_unhide: - case Opt_showassoc: - popt->showassoc = 1; - break; - case Opt_cruft: - popt->cruft = 1; - break; +static const struct fs_parameter_spec isofs_param_spec[] = { + fsparam_flag ("norock", Opt_norock), + fsparam_flag ("nojoliet", Opt_nojoliet), + fsparam_flag ("unhide", Opt_unhide), + fsparam_flag ("hide", Opt_hide), + fsparam_flag ("showassoc", Opt_showassoc), + fsparam_flag ("cruft", Opt_cruft), + fsparam_flag ("utf8", Opt_utf8), + fsparam_string ("iocharset", Opt_iocharset), + fsparam_enum ("map", Opt_map, isofs_param_map), + fsparam_u32 ("session", Opt_session), + fsparam_u32 ("sbsector", Opt_sb), + fsparam_enum ("check", Opt_check, isofs_param_check), + fsparam_u32 ("uid", Opt_uid), + fsparam_u32 ("gid", Opt_gid), + /* Note: mode/dmode historically accepted %u not strictly %o */ + fsparam_u32 ("mode", Opt_mode), + fsparam_u32 ("dmode", Opt_dmode), + fsparam_flag ("overriderockperm", Opt_overriderockperm), + fsparam_u32 ("block", Opt_block), + fsparam_string ("conv", Opt_ignore), + fsparam_flag ("nocompress", Opt_nocompress), + {} +}; + +static int isofs_parse_param(struct fs_context *fc, + struct fs_parameter *param) +{ + struct isofs_options *popt = fc->fs_private; + struct fs_parse_result result; + int opt; + kuid_t uid; + kgid_t gid; + unsigned int n; + + /* There are no remountable options */ + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) + return 0; + + opt = fs_parse(fc, isofs_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_norock: + popt->rock = 0; + break; + case Opt_nojoliet: + popt->joliet = 0; + break; + case Opt_hide: + popt->hide = 1; + break; + case Opt_unhide: + case Opt_showassoc: + popt->showassoc = 1; + break; + case Opt_cruft: + popt->cruft = 1; + break; #ifdef CONFIG_JOLIET - case Opt_utf8: - kfree(popt->iocharset); - popt->iocharset = kstrdup("utf8", GFP_KERNEL); - if (!popt->iocharset) - return 0; - break; - case Opt_iocharset: - kfree(popt->iocharset); - popt->iocharset = match_strdup(&args[0]); - if (!popt->iocharset) - return 0; - break; + case Opt_utf8: + kfree(popt->iocharset); + popt->iocharset = kstrdup("utf8", GFP_KERNEL); + if (!popt->iocharset) + return -ENOMEM; + break; + case Opt_iocharset: + kfree(popt->iocharset); + popt->iocharset = kstrdup(param->string, GFP_KERNEL); + if (!popt->iocharset) + return -ENOMEM; + break; #endif - case Opt_map_a: - popt->map = 'a'; - break; - case Opt_map_o: - popt->map = 'o'; - break; - case Opt_map_n: - popt->map = 'n'; - break; - case Opt_session: - if (match_int(&args[0], &option)) - return 0; - n = option; - /* - * Track numbers are supposed to be in range 1-99, the - * mount option starts indexing at 0. - */ - if (n >= 99) - return 0; - popt->session = n + 1; - break; - case Opt_sb: - if (match_int(&args[0], &option)) - return 0; - popt->sbsector = option; - break; - case Opt_check_r: - popt->check = 'r'; - break; - case Opt_check_s: - popt->check = 's'; - break; - case Opt_ignore: - break; - case Opt_uid: - if (match_uint(&args[0], &uv)) - return 0; - popt->uid = make_kuid(current_user_ns(), uv); - if (!uid_valid(popt->uid)) - return 0; - popt->uid_set = 1; - break; - case Opt_gid: - if (match_uint(&args[0], &uv)) - return 0; - popt->gid = make_kgid(current_user_ns(), uv); - if (!gid_valid(popt->gid)) - return 0; - popt->gid_set = 1; - break; - case Opt_mode: - if (match_int(&args[0], &option)) - return 0; - popt->fmode = option; - break; - case Opt_dmode: - if (match_int(&args[0], &option)) - return 0; - popt->dmode = option; - break; - case Opt_overriderockperm: - popt->overriderockperm = 1; - break; - case Opt_block: - if (match_int(&args[0], &option)) - return 0; - n = option; - if (n != 512 && n != 1024 && n != 2048) - return 0; - popt->blocksize = n; - break; - case Opt_nocompress: - popt->nocompress = 1; - break; - default: - return 0; - } + case Opt_map: + popt->map = result.uint_32; + break; + case Opt_session: + n = result.uint_32; + /* + * Track numbers are supposed to be in range 1-99, the + * mount option starts indexing at 0. + */ + if (n >= 99) + return -EINVAL; + popt->session = n + 1; + break; + case Opt_sb: + popt->sbsector = result.uint_32; + break; + case Opt_check: + popt->check = result.uint_32; + break; + case Opt_ignore: + break; + case Opt_uid: + uid = make_kuid(current_user_ns(), result.uint_32); + if (!uid_valid(uid)) + return -EINVAL; + popt->uid = uid; + popt->uid_set = 1; + break; + case Opt_gid: + gid = make_kgid(current_user_ns(), result.uint_32); + if (!gid_valid(gid)) + return -EINVAL; + popt->gid = gid; + popt->gid_set = 1; + break; + case Opt_mode: + popt->fmode = result.uint_32; + break; + case Opt_dmode: + popt->dmode = result.uint_32; + break; + case Opt_overriderockperm: + popt->overriderockperm = 1; + break; + case Opt_block: + n = result.uint_32; + if (n != 512 && n != 1024 && n != 2048) + return -EINVAL; + popt->blocksize = n; + break; + case Opt_nocompress: + popt->nocompress = 1; + break; + default: + return -EINVAL; } - return 1; + return 0; } /* @@ -615,7 +579,7 @@ static bool rootdir_empty(struct super_block *sb, unsigned long block) /* * Initialize the superblock and read the root inode. */ -static int isofs_fill_super(struct super_block *s, void *data, int silent) +static int isofs_fill_super(struct super_block *s, struct fs_context *fc) { struct buffer_head *bh = NULL, *pri_bh = NULL; struct hs_primary_descriptor *h_pri = NULL; @@ -623,7 +587,7 @@ static int isofs_fill_super(struct super_block *s, void *data, int silent) struct iso_supplementary_descriptor *sec = NULL; struct iso_directory_record *rootp; struct inode *inode; - struct iso9660_options opt; + struct isofs_options *opt = fc->fs_private; struct isofs_sb_info *sbi; unsigned long first_data_zone; int joliet_level = 0; @@ -631,15 +595,13 @@ static int isofs_fill_super(struct super_block *s, void *data, int silent) int orig_zonesize; int table, error = -EINVAL; unsigned int vol_desc_start; + int silent = fc->sb_flags & SB_SILENT; sbi = kzalloc(sizeof(*sbi), GFP_KERNEL); if (!sbi) return -ENOMEM; s->s_fs_info = sbi; - if (!parse_options((char *)data, &opt)) - goto out_freesbi; - /* * First of all, get the hardware blocksize for this device. * If we don't know what it is, or the hardware blocksize is @@ -655,14 +617,14 @@ static int isofs_fill_super(struct super_block *s, void *data, int silent) bdev_logical_block_size(s->s_bdev)); goto out_freesbi; } - opt.blocksize = sb_min_blocksize(s, opt.blocksize); + opt->blocksize = sb_min_blocksize(s, opt->blocksize); sbi->s_high_sierra = 0; /* default is iso9660 */ - sbi->s_session = opt.session; - sbi->s_sbsector = opt.sbsector; + sbi->s_session = opt->session; + sbi->s_sbsector = opt->sbsector; - vol_desc_start = (opt.sbsector != -1) ? - opt.sbsector : isofs_get_last_session(s,opt.session); + vol_desc_start = (opt->sbsector != -1) ? + opt->sbsector : isofs_get_last_session(s, opt->session); for (iso_blknum = vol_desc_start+16; iso_blknum < vol_desc_start+100; iso_blknum++) { @@ -696,7 +658,7 @@ static int isofs_fill_super(struct super_block *s, void *data, int silent) else if (isonum_711(vdp->type) == ISO_VD_SUPPLEMENTARY) { sec = (struct iso_supplementary_descriptor *)vdp; if (sec->escape[0] == 0x25 && sec->escape[1] == 0x2f) { - if (opt.joliet) { + if (opt->joliet) { if (sec->escape[2] == 0x40) joliet_level = 1; else if (sec->escape[2] == 0x43) @@ -721,7 +683,7 @@ static int isofs_fill_super(struct super_block *s, void *data, int silent) goto out_freebh; sbi->s_high_sierra = 1; - opt.rock = 0; + opt->rock = 0; h_pri = (struct hs_primary_descriptor *)vdp; goto root_found; } @@ -749,7 +711,7 @@ static int isofs_fill_super(struct super_block *s, void *data, int silent) goto out_freebh; } - if (joliet_level && (!pri || !opt.rock)) { + if (joliet_level && (!pri || !opt->rock)) { /* This is the case of Joliet with the norock mount flag. * A disc with both Joliet and Rock Ridge is handled later */ @@ -780,7 +742,7 @@ static int isofs_fill_super(struct super_block *s, void *data, int silent) * blocks that were 512 bytes (which should only very rarely * happen.) */ - if (orig_zonesize < opt.blocksize) + if (orig_zonesize < opt->blocksize) goto out_bad_size; /* RDE: convert log zone size to bit shift */ @@ -865,10 +827,10 @@ static int isofs_fill_super(struct super_block *s, void *data, int silent) #ifdef CONFIG_JOLIET if (joliet_level) { - char *p = opt.iocharset ? opt.iocharset : CONFIG_NLS_DEFAULT; + char *p = opt->iocharset ? opt->iocharset : CONFIG_NLS_DEFAULT; if (strcmp(p, "utf8") != 0) { - sbi->s_nls_iocharset = opt.iocharset ? - load_nls(opt.iocharset) : load_nls_default(); + sbi->s_nls_iocharset = opt->iocharset ? + load_nls(opt->iocharset) : load_nls_default(); if (!sbi->s_nls_iocharset) goto out_freesbi; } @@ -876,29 +838,29 @@ static int isofs_fill_super(struct super_block *s, void *data, int silent) #endif s->s_op = &isofs_sops; s->s_export_op = &isofs_export_ops; - sbi->s_mapping = opt.map; - sbi->s_rock = (opt.rock ? 2 : 0); + sbi->s_mapping = opt->map; + sbi->s_rock = (opt->rock ? 2 : 0); sbi->s_rock_offset = -1; /* initial offset, will guess until SP is found*/ - sbi->s_cruft = opt.cruft; - sbi->s_hide = opt.hide; - sbi->s_showassoc = opt.showassoc; - sbi->s_uid = opt.uid; - sbi->s_gid = opt.gid; - sbi->s_uid_set = opt.uid_set; - sbi->s_gid_set = opt.gid_set; - sbi->s_nocompress = opt.nocompress; - sbi->s_overriderockperm = opt.overriderockperm; + sbi->s_cruft = opt->cruft; + sbi->s_hide = opt->hide; + sbi->s_showassoc = opt->showassoc; + sbi->s_uid = opt->uid; + sbi->s_gid = opt->gid; + sbi->s_uid_set = opt->uid_set; + sbi->s_gid_set = opt->gid_set; + sbi->s_nocompress = opt->nocompress; + sbi->s_overriderockperm = opt->overriderockperm; /* * It would be incredibly stupid to allow people to mark every file * on the disk as suid, so we merely allow them to set the default * permissions. */ - if (opt.fmode != ISOFS_INVALID_MODE) - sbi->s_fmode = opt.fmode & 0777; + if (opt->fmode != ISOFS_INVALID_MODE) + sbi->s_fmode = opt->fmode & 0777; else sbi->s_fmode = ISOFS_INVALID_MODE; - if (opt.dmode != ISOFS_INVALID_MODE) - sbi->s_dmode = opt.dmode & 0777; + if (opt->dmode != ISOFS_INVALID_MODE) + sbi->s_dmode = opt->dmode & 0777; else sbi->s_dmode = ISOFS_INVALID_MODE; @@ -960,12 +922,12 @@ static int isofs_fill_super(struct super_block *s, void *data, int silent) } } - if (opt.check == 'u') { + if (opt->check == 'u') { /* Only Joliet is case insensitive by default */ if (joliet_level) - opt.check = 'r'; + opt->check = 'r'; else - opt.check = 's'; + opt->check = 's'; } sbi->s_joliet_level = joliet_level; @@ -980,9 +942,9 @@ static int isofs_fill_super(struct super_block *s, void *data, int silent) table = 0; if (joliet_level) table += 2; - if (opt.check == 'r') + if (opt->check == 'r') table++; - sbi->s_check = opt.check; + sbi->s_check = opt->check; if (table) s->s_d_op = &isofs_dentry_ops[table - 1]; @@ -994,7 +956,7 @@ static int isofs_fill_super(struct super_block *s, void *data, int silent) goto out_no_inode; } - kfree(opt.iocharset); + kfree(opt->iocharset); return 0; @@ -1023,7 +985,7 @@ static int isofs_fill_super(struct super_block *s, void *data, int silent) goto out_freebh; out_bad_size: printk(KERN_WARNING "ISOFS: Logical zone size(%d) < hardware blocksize(%u)\n", - orig_zonesize, opt.blocksize); + orig_zonesize, opt->blocksize); goto out_freebh; out_unknown_format: if (!silent) @@ -1033,7 +995,7 @@ static int isofs_fill_super(struct super_block *s, void *data, int silent) brelse(bh); brelse(pri_bh); out_freesbi: - kfree(opt.iocharset); + kfree(opt->iocharset); kfree(sbi); s->s_fs_info = NULL; return error; @@ -1567,18 +1529,63 @@ struct inode *__isofs_iget(struct super_block *sb, return inode; } -static struct dentry *isofs_mount(struct file_system_type *fs_type, - int flags, const char *dev_name, void *data) +static int isofs_get_tree(struct fs_context *fc) { - return mount_bdev(fs_type, flags, dev_name, data, isofs_fill_super); + return get_tree_bdev(fc, isofs_fill_super); +} + +static void isofs_free_fc(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static const struct fs_context_operations isofs_context_ops = { + .parse_param = isofs_parse_param, + .get_tree = isofs_get_tree, + .reconfigure = isofs_reconfigure, + .free = isofs_free_fc, +}; + +static int isofs_init_fs_context(struct fs_context *fc) +{ + struct isofs_options *opt; + + opt = kzalloc(sizeof(*opt), GFP_KERNEL); + if (!opt) + return -ENOMEM; + + opt->map = 'n'; + opt->rock = 1; + opt->joliet = 1; + opt->cruft = 0; + opt->hide = 0; + opt->showassoc = 0; + opt->check = 'u'; /* unset */ + opt->nocompress = 0; + opt->blocksize = 1024; + opt->fmode = opt->dmode = ISOFS_INVALID_MODE; + opt->uid_set = 0; + opt->gid_set = 0; + opt->gid = GLOBAL_ROOT_GID; + opt->uid = GLOBAL_ROOT_UID; + opt->iocharset = NULL; + opt->overriderockperm = 0; + opt->session = -1; + opt->sbsector = -1; + + fc->fs_private = opt; + fc->ops = &isofs_context_ops; + + return 0; } static struct file_system_type iso9660_fs_type = { .owner = THIS_MODULE, .name = "iso9660", - .mount = isofs_mount, .kill_sb = kill_block_super, .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = isofs_init_fs_context, + .parameters = isofs_param_spec, }; MODULE_ALIAS_FS("iso9660"); MODULE_ALIAS("iso9660"); From 8777446a3f2dd6dab0859e4aedef6e0240955fdb Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 15 Mar 2024 09:19:49 +0000 Subject: [PATCH 0004/1702] udf: Remove second semicolon There is a statement with two semicolons. Remove the second one, it is redundant. Signed-off-by: Colin Ian King Signed-off-by: Jan Kara Message-Id: <20240315091949.2430585-1-colin.i.king@gmail.com> --- fs/udf/super.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/udf/super.c b/fs/udf/super.c index 2217f7ed7a495..ba6b747a3830b 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -630,7 +630,7 @@ static int udf_parse_param(struct fs_context *fc, struct fs_parameter *param) if (!uopt->nls_map) { errorf(fc, "iocharset %s not found", param->string); - return -EINVAL;; + return -EINVAL; } } break; From e6595224464b692ddae193d783402130d1625147 Mon Sep 17 00:00:00 2001 From: Nikita Kiryushin Date: Thu, 14 Mar 2024 16:36:56 +0300 Subject: [PATCH 0005/1702] fanotify: remove unneeded sub-zero check for unsigned value Unsigned size_t len in copy_fid_info_to_user is checked for negative value. This check is redundant as it is always false. Found by Linux Verification Center (linuxtesting.org) with SVACE. Fixes: 5e469c830fdb ("fanotify: copy event fid info to user") Signed-off-by: Nikita Kiryushin Signed-off-by: Jan Kara Message-Id: --- fs/notify/fanotify/fanotify_user.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index fbdc63cc10d92..4201723357cfb 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -502,7 +502,7 @@ static int copy_fid_info_to_user(__kernel_fsid_t *fsid, struct fanotify_fh *fh, } /* Pad with 0's */ - WARN_ON_ONCE(len < 0 || len >= FANOTIFY_EVENT_ALIGN); + WARN_ON_ONCE(len >= FANOTIFY_EVENT_ALIGN); if (len > 0 && clear_user(buf, len)) return -EFAULT; From ad6d17e10306a66fb40985da77889bc28c2a5c1b Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Wed, 13 Mar 2024 19:28:55 +0100 Subject: [PATCH 0006/1702] dt-bindings: display: samsung,exynos5-dp: convert to DT Schema Convert Samsung Exynos5250/5420 SoC Display Port Controller bindings to DT schema with a change: add power-domains, already used in DTS. This Display Port controller is actually variant of Analogix Display Port bridge, however new DT Schema does not reference analogix,dp.yaml, because of incompatibilities in the driver. The analogix,dp.yaml expects two ports, input and output, but Linux Exynos DP DRM driver and DTS use only one port: output. Signed-off-by: Krzysztof Kozlowski Reviewed-by: Conor Dooley Link: https://lore.kernel.org/r/20240313182855.14140-1-krzysztof.kozlowski@linaro.org Signed-off-by: Rob Herring --- .../bindings/display/exynos/exynos_dp.txt | 112 ------------ .../display/samsung/samsung,exynos5-dp.yaml | 163 ++++++++++++++++++ 2 files changed, 163 insertions(+), 112 deletions(-) delete mode 100644 Documentation/devicetree/bindings/display/exynos/exynos_dp.txt create mode 100644 Documentation/devicetree/bindings/display/samsung/samsung,exynos5-dp.yaml diff --git a/Documentation/devicetree/bindings/display/exynos/exynos_dp.txt b/Documentation/devicetree/bindings/display/exynos/exynos_dp.txt deleted file mode 100644 index 3a401590320fa..0000000000000 --- a/Documentation/devicetree/bindings/display/exynos/exynos_dp.txt +++ /dev/null @@ -1,112 +0,0 @@ -The Exynos display port interface should be configured based on -the type of panel connected to it. - -We use two nodes: - -dp-controller node - -dptx-phy node(defined inside dp-controller node) - -For the DP-PHY initialization, we use the dptx-phy node. -Required properties for dptx-phy: deprecated, use phys and phy-names - -reg: deprecated - Base address of DP PHY register. - -samsung,enable-mask: deprecated - The bit-mask used to enable/disable DP PHY. - -For the Panel initialization, we read data from dp-controller node. -Required properties for dp-controller: - -compatible: - should be "samsung,exynos5-dp". - -reg: - physical base address of the controller and length - of memory mapped region. - -interrupts: - interrupt combiner values. - -clocks: - from common clock binding: handle to dp clock. - -clock-names: - from common clock binding: Shall be "dp". - -phys: - from general PHY binding: the phandle for the PHY device. - -phy-names: - from general PHY binding: Should be "dp". - -Optional properties for dp-controller: - -interlaced: - interlace scan mode. - Progressive if defined, Interlaced if not defined - -vsync-active-high: - VSYNC polarity configuration. - High if defined, Low if not defined - -hsync-active-high: - HSYNC polarity configuration. - High if defined, Low if not defined - -samsung,hpd-gpio: - Hotplug detect GPIO. - Indicates which GPIO should be used for hotplug - detection - -video interfaces: Device node can contain video interface port - nodes according to [1]. - - display-timings: timings for the connected panel as described by - Documentation/devicetree/bindings/display/panel/display-timing.txt - -For the below properties, please refer to Analogix DP binding document: - * Documentation/devicetree/bindings/display/bridge/analogix,dp.yaml - -phys (required) - -phy-names (required) - -hpd-gpios (optional) - force-hpd (optional) - -Deprecated properties for DisplayPort: --interlaced: deprecated prop that can parsed from drm_display_mode. --vsync-active-high: deprecated prop that can parsed from drm_display_mode. --hsync-active-high: deprecated prop that can parsed from drm_display_mode. --samsung,ycbcr-coeff: deprecated prop that can parsed from drm_display_mode. --samsung,dynamic-range: deprecated prop that can parsed from drm_display_mode. --samsung,color-space: deprecated prop that can parsed from drm_display_info. --samsung,color-depth: deprecated prop that can parsed from drm_display_info. --samsung,link-rate: deprecated prop that can reading from monitor by dpcd method. --samsung,lane-count: deprecated prop that can reading from monitor by dpcd method. --samsung,hpd-gpio: deprecated name for hpd-gpios. - -------------------------------------------------------------------------------- - -Example: - -SOC specific portion: - dp-controller { - compatible = "samsung,exynos5-dp"; - reg = <0x145b0000 0x10000>; - interrupts = <10 3>; - interrupt-parent = <&combiner>; - clocks = <&clock 342>; - clock-names = "dp"; - - phys = <&dp_phy>; - phy-names = "dp"; - }; - -Board Specific portion: - dp-controller { - display-timings { - native-mode = <&lcd_timing>; - lcd_timing: 1366x768 { - clock-frequency = <70589280>; - hactive = <1366>; - vactive = <768>; - hfront-porch = <40>; - hback-porch = <40>; - hsync-len = <32>; - vback-porch = <10>; - vfront-porch = <12>; - vsync-len = <6>; - }; - }; - - ports { - port@0 { - dp_out: endpoint { - remote-endpoint = <&bridge_in>; - }; - }; - }; - }; diff --git a/Documentation/devicetree/bindings/display/samsung/samsung,exynos5-dp.yaml b/Documentation/devicetree/bindings/display/samsung/samsung,exynos5-dp.yaml new file mode 100644 index 0000000000000..dda9097a79116 --- /dev/null +++ b/Documentation/devicetree/bindings/display/samsung/samsung,exynos5-dp.yaml @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/samsung/samsung,exynos5-dp.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Samsung Exynos5250/Exynos5420 SoC Display Port + +maintainers: + - Inki Dae + - Seung-Woo Kim + - Kyungmin Park + - Krzysztof Kozlowski + +properties: + compatible: + const: samsung,exynos5-dp + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-names: + items: + - const: dp + + display-timings: + $ref: /schemas/display/panel/display-timings.yaml# + + interrupts: + maxItems: 1 + + hpd-gpios: + description: + Hotplug detect GPIO. + Indicates which GPIO should be used for hotplug detection + + phys: + maxItems: 1 + + phy-names: + items: + - const: dp + + power-domains: + maxItems: 1 + + interlaced: + type: boolean + deprecated: true + description: + Interlace scan mode. Progressive if defined, interlaced if not defined. + + vsync-active-high: + type: boolean + deprecated: true + description: + VSYNC polarity configuration. High if defined, low if not defined + + hsync-active-high: + type: boolean + deprecated: true + description: + HSYNC polarity configuration. High if defined, low if not defined + + ports: + $ref: /schemas/graph.yaml#/properties/ports + + properties: + port: + $ref: /schemas/graph.yaml#/properties/port + description: + Port node with one endpoint connected to a dp-connector node. + + required: + - port + + samsung,hpd-gpios: + maxItems: 1 + deprecated: true + + samsung,ycbcr-coeff: + $ref: /schemas/types.yaml#/definitions/uint32 + deprecated: true + description: + Deprecated prop that can parsed from drm_display_mode. + + samsung,dynamic-range: + $ref: /schemas/types.yaml#/definitions/uint32 + deprecated: true + description: + Deprecated prop that can parsed from drm_display_mode. + + samsung,color-space: + $ref: /schemas/types.yaml#/definitions/uint32 + deprecated: true + description: + Deprecated prop that can parsed from drm_display_info. + + samsung,color-depth: + $ref: /schemas/types.yaml#/definitions/uint32 + deprecated: true + description: + Deprecated prop that can parsed from drm_display_info. + + samsung,link-rate: + $ref: /schemas/types.yaml#/definitions/uint32 + deprecated: true + description: + Deprecated prop that can reading from monitor by dpcd method. + + samsung,lane-count: + $ref: /schemas/types.yaml#/definitions/uint32 + deprecated: true + description: + Deprecated prop that can reading from monitor by dpcd method. + +required: + - compatible + - reg + - clocks + - clock-names + - interrupts + - phys + - phy-names + +additionalProperties: false + +examples: + - | + #include + #include + #include + + dp-controller@145b0000 { + compatible = "samsung,exynos5-dp"; + reg = <0x145b0000 0x1000>; + clocks = <&clock CLK_DP>; + clock-names = "dp"; + interrupts = <10 3>; + interrupt-parent = <&combiner>; + phys = <&dp_phy>; + phy-names = "dp"; + pinctrl-0 = <&dp_hpd>; + pinctrl-names = "default"; + power-domains = <&pd_disp1>; + + samsung,color-space = <0>; + samsung,color-depth = <1>; + samsung,link-rate = <0x0a>; + samsung,lane-count = <2>; + hpd-gpios = <&gpx0 7 GPIO_ACTIVE_HIGH>; + + ports { + port { + dp_out: endpoint { + remote-endpoint = <&bridge_in>; + }; + }; + }; + }; From 57b7d5d315e2b4193199228574d613640d3f5b50 Mon Sep 17 00:00:00 2001 From: Dharma Balasubiramani Date: Mon, 18 Mar 2024 11:10:13 +0530 Subject: [PATCH 0007/1702] dt-bindings: display: atmel,lcdc: convert to dtschema Convert the atmel,lcdc bindings to DT schema. Changes during conversion: add missing clocks and clock-names properties. Signed-off-by: Dharma Balasubiramani Link: https://lore.kernel.org/r/20240318-lcdc-fb-v4-1-c533c7c2c706@microchip.com Signed-off-by: Rob Herring --- .../bindings/display/atmel,lcdc-display.yaml | 103 ++++++++++++++++++ .../bindings/display/atmel,lcdc.txt | 87 --------------- .../bindings/display/atmel,lcdc.yaml | 70 ++++++++++++ 3 files changed, 173 insertions(+), 87 deletions(-) create mode 100644 Documentation/devicetree/bindings/display/atmel,lcdc-display.yaml delete mode 100644 Documentation/devicetree/bindings/display/atmel,lcdc.txt create mode 100644 Documentation/devicetree/bindings/display/atmel,lcdc.yaml diff --git a/Documentation/devicetree/bindings/display/atmel,lcdc-display.yaml b/Documentation/devicetree/bindings/display/atmel,lcdc-display.yaml new file mode 100644 index 0000000000000..a5cf040ab4ea4 --- /dev/null +++ b/Documentation/devicetree/bindings/display/atmel,lcdc-display.yaml @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/atmel,lcdc-display.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Microchip's LCDC Display + +maintainers: + - Nicolas Ferre + - Dharma Balasubiramani + +description: + The LCD Controller (LCDC) consists of logic for transferring LCD image data + from an external display buffer to a TFT LCD panel. The LCDC has one display + input buffer per layer that fetches pixels through the single bus host + interface and a look-up table to allow palletized display configurations. The + LCDC is programmable on a per layer basis, and supports different LCD + resolutions, window sizes, image formats and pixel depths. + +# We need a select here since this schema is applicable only for nodes with the +# following properties + +select: + anyOf: + - required: [ 'atmel,dmacon' ] + - required: [ 'atmel,lcdcon2' ] + - required: [ 'atmel,guard-time' ] + +properties: + atmel,dmacon: + $ref: /schemas/types.yaml#/definitions/uint32 + description: dma controller configuration + + atmel,lcdcon2: + $ref: /schemas/types.yaml#/definitions/uint32 + description: lcd controller configuration + + atmel,guard-time: + $ref: /schemas/types.yaml#/definitions/uint32 + description: lcd guard time (Delay in frame periods) + maximum: 127 + + bits-per-pixel: + $ref: /schemas/types.yaml#/definitions/uint32 + description: lcd panel bit-depth. + enum: [1, 2, 4, 8, 16, 24, 32] + + atmel,lcdcon-backlight: + $ref: /schemas/types.yaml#/definitions/flag + description: enable backlight + + atmel,lcdcon-backlight-inverted: + $ref: /schemas/types.yaml#/definitions/flag + description: invert backlight PWM polarity + + atmel,lcd-wiring-mode: + $ref: /schemas/types.yaml#/definitions/string + description: lcd wiring mode "RGB" or "BRG" + enum: + - RGB + - BRG + + atmel,power-control-gpio: + description: gpio to power on or off the LCD (as many as needed) + maxItems: 1 + + display-timings: + $ref: panel/display-timings.yaml# + +required: + - atmel,dmacon + - atmel,lcdcon2 + - atmel,guard-time + - bits-per-pixel + +additionalProperties: false + +examples: + - | + display: panel { + bits-per-pixel = <32>; + atmel,lcdcon-backlight; + atmel,dmacon = <0x1>; + atmel,lcdcon2 = <0x80008002>; + atmel,guard-time = <9>; + atmel,lcd-wiring-mode = "RGB"; + + display-timings { + native-mode = <&timing0>; + timing0: timing0 { + clock-frequency = <9000000>; + hactive = <480>; + vactive = <272>; + hback-porch = <1>; + hfront-porch = <1>; + vback-porch = <40>; + vfront-porch = <1>; + hsync-len = <45>; + vsync-len = <1>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/display/atmel,lcdc.txt b/Documentation/devicetree/bindings/display/atmel,lcdc.txt deleted file mode 100644 index b5e355ada2fab..0000000000000 --- a/Documentation/devicetree/bindings/display/atmel,lcdc.txt +++ /dev/null @@ -1,87 +0,0 @@ -Atmel LCDC Framebuffer ------------------------------------------------------ - -Required properties: -- compatible : - "atmel,at91sam9261-lcdc" , - "atmel,at91sam9263-lcdc" , - "atmel,at91sam9g10-lcdc" , - "atmel,at91sam9g45-lcdc" , - "atmel,at91sam9g45es-lcdc" , - "atmel,at91sam9rl-lcdc" , -- reg : Should contain 1 register ranges(address and length). - Can contain an additional register range(address and length) - for fixed framebuffer memory. Useful for dedicated memories. -- interrupts : framebuffer controller interrupt -- display: a phandle pointing to the display node - -Required nodes: -- display: a display node is required to initialize the lcd panel - This should be in the board dts. -- default-mode: a videomode within the display with timing parameters - as specified below. - -Optional properties: -- lcd-supply: Regulator for LCD supply voltage. - -Example: - - fb0: fb@00500000 { - compatible = "atmel,at91sam9g45-lcdc"; - reg = <0x00500000 0x1000>; - interrupts = <23 3 0>; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_fb>; - display = <&display0>; - #address-cells = <1>; - #size-cells = <1>; - - }; - -Example for fixed framebuffer memory: - - fb0: fb@00500000 { - compatible = "atmel,at91sam9263-lcdc"; - reg = <0x00700000 0x1000 0x70000000 0x200000>; - [...] - }; - -Atmel LCDC Display ------------------------------------------------------ -Required properties (as per of_videomode_helper): - - - atmel,dmacon: dma controller configuration - - atmel,lcdcon2: lcd controller configuration - - atmel,guard-time: lcd guard time (Delay in frame periods) - - bits-per-pixel: lcd panel bit-depth. - -Optional properties (as per of_videomode_helper): - - atmel,lcdcon-backlight: enable backlight - - atmel,lcdcon-backlight-inverted: invert backlight PWM polarity - - atmel,lcd-wiring-mode: lcd wiring mode "RGB" or "BRG" - - atmel,power-control-gpio: gpio to power on or off the LCD (as many as needed) - -Example: - display0: display { - bits-per-pixel = <32>; - atmel,lcdcon-backlight; - atmel,dmacon = <0x1>; - atmel,lcdcon2 = <0x80008002>; - atmel,guard-time = <9>; - atmel,lcd-wiring-mode = <1>; - - display-timings { - native-mode = <&timing0>; - timing0: timing0 { - clock-frequency = <9000000>; - hactive = <480>; - vactive = <272>; - hback-porch = <1>; - hfront-porch = <1>; - vback-porch = <40>; - vfront-porch = <1>; - hsync-len = <45>; - vsync-len = <1>; - }; - }; - }; diff --git a/Documentation/devicetree/bindings/display/atmel,lcdc.yaml b/Documentation/devicetree/bindings/display/atmel,lcdc.yaml new file mode 100644 index 0000000000000..1b6f7e3950060 --- /dev/null +++ b/Documentation/devicetree/bindings/display/atmel,lcdc.yaml @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/atmel,lcdc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Microchip's LCDC Framebuffer + +maintainers: + - Nicolas Ferre + - Dharma Balasubiramani + +description: + The LCDC works with a framebuffer, which is a section of memory that contains + a complete frame of data representing pixel values for the display. The LCDC + reads the pixel data from the framebuffer and sends it to the LCD panel to + render the image. + +properties: + compatible: + enum: + - atmel,at91sam9261-lcdc + - atmel,at91sam9263-lcdc + - atmel,at91sam9g10-lcdc + - atmel,at91sam9g45-lcdc + - atmel,at91sam9g45es-lcdc + - atmel,at91sam9rl-lcdc + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 2 + + clock-names: + items: + - const: hclk + - const: lcdc_clk + + display: + $ref: /schemas/types.yaml#/definitions/phandle + description: A phandle pointing to the display node. + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + - display + +additionalProperties: false + +examples: + - | + #include + #include + fb@500000 { + compatible = "atmel,at91sam9g45-lcdc"; + reg = <0x00500000 0x1000>; + interrupts = <23 IRQ_TYPE_LEVEL_HIGH 0>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_fb>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 23>, <&pmc PMC_TYPE_PERIPHERAL 23>; + clock-names = "hclk", "lcdc_clk"; + display = <&display>; + }; From 070c1470ae24317e7b19bd3882b300b6d69922a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Wed, 6 Mar 2024 20:37:04 +0100 Subject: [PATCH 0008/1702] power: supply: test-power: implement charge_behaviour property MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To validate the special formatting of the "charge_behaviour" sysfs property add it to the example driver. Signed-off-by: Thomas Weißschuh Link: https://lore.kernel.org/r/20240306-power_supply-charge_behaviour_prop-v3-1-d04cf1f5f0af@weissschuh.net Signed-off-by: Sebastian Reichel --- drivers/power/supply/test_power.c | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/drivers/power/supply/test_power.c b/drivers/power/supply/test_power.c index 0d0a77584c5db..442ceb7795e1d 100644 --- a/drivers/power/supply/test_power.c +++ b/drivers/power/supply/test_power.c @@ -35,6 +35,8 @@ static int battery_capacity = 50; static int battery_voltage = 3300; static int battery_charge_counter = -1000; static int battery_current = -1600; +static enum power_supply_charge_behaviour battery_charge_behaviour = + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO; static bool module_initialized; @@ -123,6 +125,9 @@ static int test_power_get_battery_property(struct power_supply *psy, case POWER_SUPPLY_PROP_CURRENT_NOW: val->intval = battery_current; break; + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + val->intval = battery_charge_behaviour; + break; default: pr_info("%s: some properties deliberately report errors.\n", __func__); @@ -131,6 +136,31 @@ static int test_power_get_battery_property(struct power_supply *psy, return 0; } +static int test_power_battery_property_is_writeable(struct power_supply *psy, + enum power_supply_property psp) +{ + return psp == POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR; +} + +static int test_power_set_battery_property(struct power_supply *psy, + enum power_supply_property psp, + const union power_supply_propval *val) +{ + switch (psp) { + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + if (val->intval < 0 || + val->intval >= BITS_PER_TYPE(typeof(psy->desc->charge_behaviours)) || + !(BIT(val->intval) & psy->desc->charge_behaviours)) { + return -EINVAL; + } + battery_charge_behaviour = val->intval; + break; + default: + return -EINVAL; + } + return 0; +} + static enum power_supply_property test_power_ac_props[] = { POWER_SUPPLY_PROP_ONLINE, }; @@ -156,6 +186,7 @@ static enum power_supply_property test_power_battery_props[] = { POWER_SUPPLY_PROP_VOLTAGE_NOW, POWER_SUPPLY_PROP_CURRENT_AVG, POWER_SUPPLY_PROP_CURRENT_NOW, + POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, }; static char *test_power_ac_supplied_to[] = { @@ -178,6 +209,11 @@ static const struct power_supply_desc test_power_desc[] = { .properties = test_power_battery_props, .num_properties = ARRAY_SIZE(test_power_battery_props), .get_property = test_power_get_battery_property, + .set_property = test_power_set_battery_property, + .property_is_writeable = test_power_battery_property_is_writeable, + .charge_behaviours = BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO) + | BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE) + | BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE), }, [TEST_USB] = { .name = "test_usb", From ee745e4736fbf33079d0d0808e1343c2280fd59a Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Thu, 29 Feb 2024 22:38:38 +0800 Subject: [PATCH 0009/1702] f2fs: support .shutdown in f2fs_sops Support .shutdown callback in f2fs_sops, then, it can be called to shut down the file system when underlying block device is marked dead. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/f2fs.h | 2 ++ fs/f2fs/file.c | 70 ++++++++++++++++++++++++++++++------------------- fs/f2fs/super.c | 6 +++++ 3 files changed, 51 insertions(+), 27 deletions(-) diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index fced2b7652f40..adfce581026b8 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -3497,6 +3497,8 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr); int f2fs_truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end); void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count); +int f2fs_do_shutdown(struct f2fs_sb_info *sbi, unsigned int flag, + bool readonly); int f2fs_precache_extents(struct inode *inode); int f2fs_fileattr_get(struct dentry *dentry, struct fileattr *fa); int f2fs_fileattr_set(struct mnt_idmap *idmap, diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 1761ad125f97a..f4347db5b03e5 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -2224,34 +2224,13 @@ static int f2fs_ioc_abort_atomic_write(struct file *filp) return ret; } -static int f2fs_ioc_shutdown(struct file *filp, unsigned long arg) +int f2fs_do_shutdown(struct f2fs_sb_info *sbi, unsigned int flag, + bool readonly) { - struct inode *inode = file_inode(filp); - struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct super_block *sb = sbi->sb; - __u32 in; int ret = 0; - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; - - if (get_user(in, (__u32 __user *)arg)) - return -EFAULT; - - if (in != F2FS_GOING_DOWN_FULLSYNC) { - ret = mnt_want_write_file(filp); - if (ret) { - if (ret == -EROFS) { - ret = 0; - f2fs_stop_checkpoint(sbi, false, - STOP_CP_REASON_SHUTDOWN); - trace_f2fs_shutdown(sbi, in, ret); - } - return ret; - } - } - - switch (in) { + switch (flag) { case F2FS_GOING_DOWN_FULLSYNC: ret = bdev_freeze(sb->s_bdev); if (ret) @@ -2290,6 +2269,9 @@ static int f2fs_ioc_shutdown(struct file *filp, unsigned long arg) goto out; } + if (readonly) + goto out; + f2fs_stop_gc_thread(sbi); f2fs_stop_discard_thread(sbi); @@ -2298,10 +2280,44 @@ static int f2fs_ioc_shutdown(struct file *filp, unsigned long arg) f2fs_update_time(sbi, REQ_TIME); out: - if (in != F2FS_GOING_DOWN_FULLSYNC) - mnt_drop_write_file(filp); - trace_f2fs_shutdown(sbi, in, ret); + trace_f2fs_shutdown(sbi, flag, ret); + + return ret; +} + +static int f2fs_ioc_shutdown(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + __u32 in; + int ret; + bool need_drop = false, readonly = false; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (get_user(in, (__u32 __user *)arg)) + return -EFAULT; + + if (in != F2FS_GOING_DOWN_FULLSYNC) { + ret = mnt_want_write_file(filp); + if (ret) { + if (ret != -EROFS) + return ret; + + /* fallback to nosync shutdown for readonly fs */ + in = F2FS_GOING_DOWN_NOSYNC; + readonly = true; + } else { + need_drop = true; + } + } + + ret = f2fs_do_shutdown(sbi, in, readonly); + + if (need_drop) + mnt_drop_write_file(filp); return ret; } diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index a6867f26f1418..7c45929671ad6 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -2547,6 +2547,11 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data) return err; } +static void f2fs_shutdown(struct super_block *sb) +{ + f2fs_do_shutdown(F2FS_SB(sb), F2FS_GOING_DOWN_NOSYNC, false); +} + #ifdef CONFIG_QUOTA static bool f2fs_need_recovery(struct f2fs_sb_info *sbi) { @@ -3146,6 +3151,7 @@ static const struct super_operations f2fs_sops = { .unfreeze_fs = f2fs_unfreeze, .statfs = f2fs_statfs, .remount_fs = f2fs_remount, + .shutdown = f2fs_shutdown, }; #ifdef CONFIG_FS_ENCRYPTION From 002b831076722f3a966a58defd1dc6976a46c109 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 26 Feb 2024 11:59:45 +0100 Subject: [PATCH 0010/1702] dt-bindings: clock: r9a07g043-cpg: Annotate RZ/G2UL-only core clocks The M2 (CRU main clock), M3 (LCDC Video Clock), and AT (Cortex-A55 Debug clock) core clocks are only present on RZ/G2UL, not on RZ/Five. Annotate this in the comments, like is already done for module clocks and resets. Signed-off-by: Geert Uytterhoeven Reviewed-by: Lad Prabhakar Acked-by: Rob Herring Link: https://lore.kernel.org/r/ffcdcd479c76b92f67481836a33ec86e97f85634.1708944903.git.geert+renesas@glider.be --- include/dt-bindings/clock/r9a07g043-cpg.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/dt-bindings/clock/r9a07g043-cpg.h b/include/dt-bindings/clock/r9a07g043-cpg.h index 77cde8effdc73..a64139fec8152 100644 --- a/include/dt-bindings/clock/r9a07g043-cpg.h +++ b/include/dt-bindings/clock/r9a07g043-cpg.h @@ -16,15 +16,15 @@ #define R9A07G043_CLK_SD0 5 #define R9A07G043_CLK_SD1 6 #define R9A07G043_CLK_M0 7 -#define R9A07G043_CLK_M2 8 -#define R9A07G043_CLK_M3 9 +#define R9A07G043_CLK_M2 8 /* RZ/G2UL Only */ +#define R9A07G043_CLK_M3 9 /* RZ/G2UL Only */ #define R9A07G043_CLK_HP 10 #define R9A07G043_CLK_TSU 11 #define R9A07G043_CLK_ZT 12 #define R9A07G043_CLK_P0 13 #define R9A07G043_CLK_P1 14 #define R9A07G043_CLK_P2 15 -#define R9A07G043_CLK_AT 16 +#define R9A07G043_CLK_AT 16 /* RZ/G2UL Only */ #define R9A07G043_OSCCLK 17 #define R9A07G043_CLK_P0_DIV2 18 From c69fe2ae2e7f203bbfbd6dd1757d3b290bce8509 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 6 Mar 2024 11:52:04 +0100 Subject: [PATCH 0011/1702] clk: renesas: r8a779h0: Add thermal clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the module clock used by the Thermal Sensor/Chip Internal Voltage Monitor/Core Voltage Monitor (THS/CIVM/CVM) on the Renesas R-Car V4M (R8A779H0) SoC. Based on a patch in the BSP by Cong Dang. Signed-off-by: Geert Uytterhoeven Reviewed-by: Niklas Söderlund Link: https://lore.kernel.org/r/befac3e8342cd552f580d34be863ef84403c541f.1709722056.git.geert+renesas@glider.be --- drivers/clk/renesas/r8a779h0-cpg-mssr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/clk/renesas/r8a779h0-cpg-mssr.c b/drivers/clk/renesas/r8a779h0-cpg-mssr.c index 71f67a1c86d80..5c48e645f0c31 100644 --- a/drivers/clk/renesas/r8a779h0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a779h0-cpg-mssr.c @@ -192,6 +192,7 @@ static const struct mssr_mod_clk r8a779h0_mod_clks[] = { DEF_MOD("pfc0", 915, R8A779H0_CLK_CP), DEF_MOD("pfc1", 916, R8A779H0_CLK_CP), DEF_MOD("pfc2", 917, R8A779H0_CLK_CP), + DEF_MOD("tsc2:tsc1", 919, R8A779H0_CLK_CL16M), }; /* From b8ae9d344d09b73361493054fbde15b9f5ebe91a Mon Sep 17 00:00:00 2001 From: Paul Barker Date: Wed, 20 Mar 2024 08:28:30 +0000 Subject: [PATCH 0012/1702] clk: renesas: r9a07g043: Mark mod_clks and resets arrays as const The r9a07g043_mod_clks and r9a07g043_resets arrays describe the module clocks and reset signals (respectively) in this SoC and do not change at runtime. Signed-off-by: Paul Barker Reviewed-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20240320082831.9666-1-paul.barker.ct@bp.renesas.com Signed-off-by: Geert Uytterhoeven --- drivers/clk/renesas/r9a07g043-cpg.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/clk/renesas/r9a07g043-cpg.c b/drivers/clk/renesas/r9a07g043-cpg.c index 33532673d25d7..e36d2ec2c0f54 100644 --- a/drivers/clk/renesas/r9a07g043-cpg.c +++ b/drivers/clk/renesas/r9a07g043-cpg.c @@ -149,7 +149,7 @@ static const struct cpg_core_clk r9a07g043_core_clks[] __initconst = { #endif }; -static struct rzg2l_mod_clk r9a07g043_mod_clks[] = { +static const struct rzg2l_mod_clk r9a07g043_mod_clks[] = { #ifdef CONFIG_ARM64 DEF_MOD("gic", R9A07G043_GIC600_GICCLK, R9A07G043_CLK_P1, 0x514, 0), @@ -282,7 +282,7 @@ static struct rzg2l_mod_clk r9a07g043_mod_clks[] = { 0x5ac, 0), }; -static struct rzg2l_reset r9a07g043_resets[] = { +static const struct rzg2l_reset r9a07g043_resets[] = { #ifdef CONFIG_ARM64 DEF_RST(R9A07G043_GIC600_GICRESET_N, 0x814, 0), DEF_RST(R9A07G043_GIC600_DBG_GICRESET_N, 0x814, 1), From e56321e48db4c57c9f389592ef6bbfb24affd0fd Mon Sep 17 00:00:00 2001 From: Paul Barker Date: Wed, 20 Mar 2024 08:28:31 +0000 Subject: [PATCH 0013/1702] clk: renesas: r9a07g044: Mark resets array as const The r9a07g044_resets array describes the reset signals in this SoC and does not change at runtime. Signed-off-by: Paul Barker Reviewed-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20240320082831.9666-2-paul.barker.ct@bp.renesas.com Signed-off-by: Geert Uytterhoeven --- drivers/clk/renesas/r9a07g044-cpg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clk/renesas/r9a07g044-cpg.c b/drivers/clk/renesas/r9a07g044-cpg.c index 48404cafea3f5..f6df3f7a31b54 100644 --- a/drivers/clk/renesas/r9a07g044-cpg.c +++ b/drivers/clk/renesas/r9a07g044-cpg.c @@ -368,7 +368,7 @@ static const struct { #endif }; -static struct rzg2l_reset r9a07g044_resets[] = { +static const struct rzg2l_reset r9a07g044_resets[] = { DEF_RST(R9A07G044_GIC600_GICRESET_N, 0x814, 0), DEF_RST(R9A07G044_GIC600_DBG_GICRESET_N, 0x814, 1), DEF_RST(R9A07G044_IA55_RESETN, 0x818, 0), From 7fa37084061fef80dab81bc062c6ec0fa8c26b2d Mon Sep 17 00:00:00 2001 From: Sam Protsenko Date: Thu, 29 Feb 2024 19:51:16 -0600 Subject: [PATCH 0014/1702] clk: samsung: Implement manual PLL control for ARM64 SoCs Some ARM64 Exynos chips are capable to control PLL clocks automatically. For those chips, whether the PLL is controlled automatically or manually is chosen in PLL_CON1 register with next bits: [28] ENABLE_AUTOMATIC_CLKGATING [1] MANUAL_PLL_CTRL [0] AUTO_PLL_CTRL The bl2 bootloader sets 0x10000001 value for some PLL_CON1 registers, which means any attempt to control those PLLs manually (e.g. disabling/enabling those PLLs or changing MUX parent clocks) would lead to PLL lock timeout with error message like this: Could not lock PLL ... At the moment, all Samsung clock drivers implement manual clock control. So in order to make it possible to control PLLs, corresponding PLL_CON1 registers should be set to 0x2 first. Some older ARM64 chips don't implement the automatic clock control though. It also might be desirable to configure some PLLs for manual control, while keeping the default configuration for the rest. So it'd convenient to choose this PLL mode for each CMU separately. Introduce .manual_plls field to CMU structure to choose the PLL control mode. Because it'll be initialized with "false" in all existing CMU structures by default, it won't affect any existing clock drivers, allowing for this feature to be enabled gradually when it's needed with no change for the rest of users. In case .manual_plls is set, set PLL_CON1 registers to manual control, akin to what's already done for gate clocks in exynos_arm64_init_clocks(). Of course, PLL_CON1 registers should be added to corresponding struct samsung_cmu_info::clk_regs array to make sure they get initialized. No functional change. This patch adds a feature, but doesn't enable it for any users. Signed-off-by: Sam Protsenko Link: https://lore.kernel.org/r/20240301015118.30072-1-semen.protsenko@linaro.org Signed-off-by: Krzysztof Kozlowski --- drivers/clk/samsung/clk-exynos-arm64.c | 56 +++++++++++++++++++------- drivers/clk/samsung/clk.h | 4 ++ 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/drivers/clk/samsung/clk-exynos-arm64.c b/drivers/clk/samsung/clk-exynos-arm64.c index 6fb7194df7aba..bf7de21f329ec 100644 --- a/drivers/clk/samsung/clk-exynos-arm64.c +++ b/drivers/clk/samsung/clk-exynos-arm64.c @@ -17,10 +17,17 @@ #include "clk-exynos-arm64.h" +/* PLL register bits */ +#define PLL_CON1_MANUAL BIT(1) + /* Gate register bits */ #define GATE_MANUAL BIT(20) #define GATE_ENABLE_HWACG BIT(28) +/* PLL_CONx_PLL register offsets range */ +#define PLL_CON_OFF_START 0x100 +#define PLL_CON_OFF_END 0x600 + /* Gate register offsets range */ #define GATE_OFF_START 0x2000 #define GATE_OFF_END 0x2fff @@ -38,17 +45,36 @@ struct exynos_arm64_cmu_data { struct samsung_clk_provider *ctx; }; +/* Check if the register offset is a GATE register */ +static bool is_gate_reg(unsigned long off) +{ + return off >= GATE_OFF_START && off <= GATE_OFF_END; +} + +/* Check if the register offset is a PLL_CONx register */ +static bool is_pll_conx_reg(unsigned long off) +{ + return off >= PLL_CON_OFF_START && off <= PLL_CON_OFF_END; +} + +/* Check if the register offset is a PLL_CON1 register */ +static bool is_pll_con1_reg(unsigned long off) +{ + return is_pll_conx_reg(off) && (off & 0xf) == 0x4 && !(off & 0x10); +} + /** * exynos_arm64_init_clocks - Set clocks initial configuration - * @np: CMU device tree node with "reg" property (CMU addr) - * @reg_offs: Register offsets array for clocks to init - * @reg_offs_len: Number of register offsets in reg_offs array + * @np: CMU device tree node with "reg" property (CMU addr) + * @cmu: CMU data * - * Set manual control mode for all gate clocks. + * Set manual control mode for all gate and PLL clocks. */ static void __init exynos_arm64_init_clocks(struct device_node *np, - const unsigned long *reg_offs, size_t reg_offs_len) + const struct samsung_cmu_info *cmu) { + const unsigned long *reg_offs = cmu->clk_regs; + size_t reg_offs_len = cmu->nr_clk_regs; void __iomem *reg_base; size_t i; @@ -60,14 +86,14 @@ static void __init exynos_arm64_init_clocks(struct device_node *np, void __iomem *reg = reg_base + reg_offs[i]; u32 val; - /* Modify only gate clock registers */ - if (reg_offs[i] < GATE_OFF_START || reg_offs[i] > GATE_OFF_END) - continue; - - val = readl(reg); - val |= GATE_MANUAL; - val &= ~GATE_ENABLE_HWACG; - writel(val, reg); + if (cmu->manual_plls && is_pll_con1_reg(reg_offs[i])) { + writel(PLL_CON1_MANUAL, reg); + } else if (is_gate_reg(reg_offs[i])) { + val = readl(reg); + val |= GATE_MANUAL; + val &= ~GATE_ENABLE_HWACG; + writel(val, reg); + } } iounmap(reg_base); @@ -177,7 +203,7 @@ void __init exynos_arm64_register_cmu(struct device *dev, pr_err("%s: could not enable bus clock %s; err = %d\n", __func__, cmu->clk_name, err); - exynos_arm64_init_clocks(np, cmu->clk_regs, cmu->nr_clk_regs); + exynos_arm64_init_clocks(np, cmu); samsung_cmu_register_one(np, cmu); } @@ -224,7 +250,7 @@ int __init exynos_arm64_register_cmu_pm(struct platform_device *pdev, __func__, cmu->clk_name, ret); if (set_manual) - exynos_arm64_init_clocks(np, cmu->clk_regs, cmu->nr_clk_regs); + exynos_arm64_init_clocks(np, cmu); reg_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(reg_base)) diff --git a/drivers/clk/samsung/clk.h b/drivers/clk/samsung/clk.h index a763309e6f129..a70bd7cce39fd 100644 --- a/drivers/clk/samsung/clk.h +++ b/drivers/clk/samsung/clk.h @@ -330,6 +330,7 @@ struct samsung_clock_reg_cache { * @suspend_regs: list of clock registers to set before suspend * @nr_suspend_regs: count of clock registers in @suspend_regs * @clk_name: name of the parent clock needed for CMU register access + * @manual_plls: Enable manual control for PLL clocks */ struct samsung_cmu_info { const struct samsung_pll_clock *pll_clks; @@ -354,6 +355,9 @@ struct samsung_cmu_info { const struct samsung_clk_reg_dump *suspend_regs; unsigned int nr_suspend_regs; const char *clk_name; + + /* ARM64 Exynos CMUs */ + bool manual_plls; }; struct samsung_clk_provider *samsung_clk_init(struct device *dev, From dedf87341ad66fa6889fedcf610b6941d2d3bcb6 Mon Sep 17 00:00:00 2001 From: Sam Protsenko Date: Thu, 29 Feb 2024 19:51:17 -0600 Subject: [PATCH 0015/1702] clk: samsung: exynos850: Add CMU_CPUCL0 and CMU_CPUCL1 Implement support for CPU clock management units: - CMU_CPUCL0: clocks for cluster 0: 4 x Cortex-A55 (cpu0..cpu3) - CMU_CPUCL1: clocks for cluster 1: 4 x Cortex-A55 (cpu4..cpu7) CPU PLLs are generating main CPU clocks for each cluster, and there are alternate ("switch") clocks that can be used temporarily while re-configuring the PLL for a new rate. ACLK, ATCLK, PCLKDBG and PERIPHCLK clocks are driving corresponding buses. CLK_CLUSTERx_SCLK are actual leaf CPU clocks and should be used to change CPU rates. Also some CoreSight clocks can be derived from DBG_USER (debug clock). PLL table was extracted from ECT table. ECT stands for "Exynos Characteristic Table", it's a Samsung specific binary data populated by BL2 bootloader in RAM at 0x90000000 address, containing PLL tables for various CMUs and other hardware specific information. The particular PLL type used in CMU_CPUCL0 and CMU_CPUCL1 (pll0822x) is an integer PLL with middle FVCO. The equation to calculate its output rate is: fout = fin * M / (P*2^S) where: fin = 26 MHz (OSCCLK frequency) M = 64..1023 P = 1..63 S = 0..6 The PLL table tries to keep "P" value low to reduce the locking time, which for pll0822x is "t = P * 150" (in OSCCLK cycles). Signed-off-by: Sam Protsenko Link: https://lore.kernel.org/r/20240301015118.30072-2-semen.protsenko@linaro.org Signed-off-by: Krzysztof Kozlowski --- drivers/clk/samsung/clk-exynos850.c | 440 +++++++++++++++++++++++++++- 1 file changed, 439 insertions(+), 1 deletion(-) diff --git a/drivers/clk/samsung/clk-exynos850.c b/drivers/clk/samsung/clk-exynos850.c index 82cfa22c07888..6215471c4ac6d 100644 --- a/drivers/clk/samsung/clk-exynos850.c +++ b/drivers/clk/samsung/clk-exynos850.c @@ -14,13 +14,16 @@ #include #include "clk.h" +#include "clk-cpu.h" #include "clk-exynos-arm64.h" /* NOTE: Must be equal to the last clock ID increased by one */ -#define CLKS_NR_TOP (CLK_DOUT_G3D_SWITCH + 1) +#define CLKS_NR_TOP (CLK_DOUT_CPUCL1_SWITCH + 1) #define CLKS_NR_APM (CLK_GOUT_SYSREG_APM_PCLK + 1) #define CLKS_NR_AUD (CLK_GOUT_AUD_CMU_AUD_PCLK + 1) #define CLKS_NR_CMGP (CLK_GOUT_SYSREG_CMGP_PCLK + 1) +#define CLKS_NR_CPUCL0 (CLK_CLUSTER0_SCLK + 1) +#define CLKS_NR_CPUCL1 (CLK_CLUSTER1_SCLK + 1) #define CLKS_NR_G3D (CLK_GOUT_G3D_SYSREG_PCLK + 1) #define CLKS_NR_HSI (CLK_GOUT_HSI_CMU_HSI_PCLK + 1) #define CLKS_NR_IS (CLK_GOUT_IS_SYSREG_PCLK + 1) @@ -47,6 +50,10 @@ #define CLK_CON_MUX_MUX_CLKCMU_CORE_CCI 0x1018 #define CLK_CON_MUX_MUX_CLKCMU_CORE_MMC_EMBD 0x101c #define CLK_CON_MUX_MUX_CLKCMU_CORE_SSS 0x1020 +#define CLK_CON_MUX_MUX_CLKCMU_CPUCL0_DBG 0x1024 +#define CLK_CON_MUX_MUX_CLKCMU_CPUCL0_SWITCH 0x1028 +#define CLK_CON_MUX_MUX_CLKCMU_CPUCL1_DBG 0x102c +#define CLK_CON_MUX_MUX_CLKCMU_CPUCL1_SWITCH 0x1030 #define CLK_CON_MUX_MUX_CLKCMU_DPU 0x1034 #define CLK_CON_MUX_MUX_CLKCMU_G3D_SWITCH 0x1038 #define CLK_CON_MUX_MUX_CLKCMU_HSI_BUS 0x103c @@ -69,6 +76,10 @@ #define CLK_CON_DIV_CLKCMU_CORE_CCI 0x1824 #define CLK_CON_DIV_CLKCMU_CORE_MMC_EMBD 0x1828 #define CLK_CON_DIV_CLKCMU_CORE_SSS 0x182c +#define CLK_CON_DIV_CLKCMU_CPUCL0_DBG 0x1830 +#define CLK_CON_DIV_CLKCMU_CPUCL0_SWITCH 0x1834 +#define CLK_CON_DIV_CLKCMU_CPUCL1_DBG 0x1838 +#define CLK_CON_DIV_CLKCMU_CPUCL1_SWITCH 0x183c #define CLK_CON_DIV_CLKCMU_DPU 0x1840 #define CLK_CON_DIV_CLKCMU_G3D_SWITCH 0x1844 #define CLK_CON_DIV_CLKCMU_HSI_BUS 0x1848 @@ -97,6 +108,10 @@ #define CLK_CON_GAT_GATE_CLKCMU_CORE_CCI 0x2020 #define CLK_CON_GAT_GATE_CLKCMU_CORE_MMC_EMBD 0x2024 #define CLK_CON_GAT_GATE_CLKCMU_CORE_SSS 0x2028 +#define CLK_CON_GAT_GATE_CLKCMU_CPUCL0_DBG 0x202c +#define CLK_CON_GAT_GATE_CLKCMU_CPUCL0_SWITCH 0x2030 +#define CLK_CON_GAT_GATE_CLKCMU_CPUCL1_DBG 0x2034 +#define CLK_CON_GAT_GATE_CLKCMU_CPUCL1_SWITCH 0x2038 #define CLK_CON_GAT_GATE_CLKCMU_DPU 0x203c #define CLK_CON_GAT_GATE_CLKCMU_G3D_SWITCH 0x2040 #define CLK_CON_GAT_GATE_CLKCMU_HSI_BUS 0x2044 @@ -130,6 +145,10 @@ static const unsigned long top_clk_regs[] __initconst = { CLK_CON_MUX_MUX_CLKCMU_CORE_CCI, CLK_CON_MUX_MUX_CLKCMU_CORE_MMC_EMBD, CLK_CON_MUX_MUX_CLKCMU_CORE_SSS, + CLK_CON_MUX_MUX_CLKCMU_CPUCL0_DBG, + CLK_CON_MUX_MUX_CLKCMU_CPUCL0_SWITCH, + CLK_CON_MUX_MUX_CLKCMU_CPUCL1_DBG, + CLK_CON_MUX_MUX_CLKCMU_CPUCL1_SWITCH, CLK_CON_MUX_MUX_CLKCMU_DPU, CLK_CON_MUX_MUX_CLKCMU_G3D_SWITCH, CLK_CON_MUX_MUX_CLKCMU_HSI_BUS, @@ -152,6 +171,10 @@ static const unsigned long top_clk_regs[] __initconst = { CLK_CON_DIV_CLKCMU_CORE_CCI, CLK_CON_DIV_CLKCMU_CORE_MMC_EMBD, CLK_CON_DIV_CLKCMU_CORE_SSS, + CLK_CON_DIV_CLKCMU_CPUCL0_DBG, + CLK_CON_DIV_CLKCMU_CPUCL0_SWITCH, + CLK_CON_DIV_CLKCMU_CPUCL1_DBG, + CLK_CON_DIV_CLKCMU_CPUCL1_SWITCH, CLK_CON_DIV_CLKCMU_DPU, CLK_CON_DIV_CLKCMU_G3D_SWITCH, CLK_CON_DIV_CLKCMU_HSI_BUS, @@ -180,6 +203,10 @@ static const unsigned long top_clk_regs[] __initconst = { CLK_CON_GAT_GATE_CLKCMU_CORE_CCI, CLK_CON_GAT_GATE_CLKCMU_CORE_MMC_EMBD, CLK_CON_GAT_GATE_CLKCMU_CORE_SSS, + CLK_CON_GAT_GATE_CLKCMU_CPUCL0_DBG, + CLK_CON_GAT_GATE_CLKCMU_CPUCL0_SWITCH, + CLK_CON_GAT_GATE_CLKCMU_CPUCL1_DBG, + CLK_CON_GAT_GATE_CLKCMU_CPUCL1_SWITCH, CLK_CON_GAT_GATE_CLKCMU_DPU, CLK_CON_GAT_GATE_CLKCMU_G3D_SWITCH, CLK_CON_GAT_GATE_CLKCMU_HSI_BUS, @@ -234,6 +261,14 @@ PNAME(mout_core_mmc_embd_p) = { "oscclk", "dout_shared0_div2", "oscclk", "oscclk" }; PNAME(mout_core_sss_p) = { "dout_shared0_div3", "dout_shared1_div3", "dout_shared0_div4", "dout_shared1_div4" }; +/* List of parent clocks for Muxes in CMU_TOP: for CMU_CPUCL0 */ +PNAME(mout_cpucl0_switch_p) = { "fout_shared0_pll", "fout_shared1_pll", + "dout_shared0_div2", "dout_shared1_div2" }; +PNAME(mout_cpucl0_dbg_p) = { "dout_shared0_div4", "dout_shared1_div4" }; +/* List of parent clocks for Muxes in CMU_TOP: for CMU_CPUCL1 */ +PNAME(mout_cpucl1_switch_p) = { "fout_shared0_pll", "fout_shared1_pll", + "dout_shared0_div2", "dout_shared1_div2" }; +PNAME(mout_cpucl1_dbg_p) = { "dout_shared0_div4", "dout_shared1_div4" }; /* List of parent clocks for Muxes in CMU_TOP: for CMU_G3D */ PNAME(mout_g3d_switch_p) = { "dout_shared0_div2", "dout_shared1_div2", "dout_shared0_div3", "dout_shared1_div3" }; @@ -300,6 +335,18 @@ static const struct samsung_mux_clock top_mux_clks[] __initconst = { MUX(CLK_MOUT_CORE_SSS, "mout_core_sss", mout_core_sss_p, CLK_CON_MUX_MUX_CLKCMU_CORE_SSS, 0, 2), + /* CPUCL0 */ + MUX(CLK_MOUT_CPUCL0_DBG, "mout_cpucl0_dbg", mout_cpucl0_dbg_p, + CLK_CON_MUX_MUX_CLKCMU_CPUCL0_DBG, 0, 1), + MUX(CLK_MOUT_CPUCL0_SWITCH, "mout_cpucl0_switch", mout_cpucl0_switch_p, + CLK_CON_MUX_MUX_CLKCMU_CPUCL0_SWITCH, 0, 2), + + /* CPUCL1 */ + MUX(CLK_MOUT_CPUCL1_DBG, "mout_cpucl1_dbg", mout_cpucl1_dbg_p, + CLK_CON_MUX_MUX_CLKCMU_CPUCL1_DBG, 0, 1), + MUX(CLK_MOUT_CPUCL1_SWITCH, "mout_cpucl1_switch", mout_cpucl1_switch_p, + CLK_CON_MUX_MUX_CLKCMU_CPUCL1_SWITCH, 0, 2), + /* DPU */ MUX(CLK_MOUT_DPU, "mout_dpu", mout_dpu_p, CLK_CON_MUX_MUX_CLKCMU_DPU, 0, 2), @@ -378,6 +425,18 @@ static const struct samsung_div_clock top_div_clks[] __initconst = { DIV(CLK_DOUT_CORE_SSS, "dout_core_sss", "gout_core_sss", CLK_CON_DIV_CLKCMU_CORE_SSS, 0, 4), + /* CPUCL0 */ + DIV(CLK_DOUT_CPUCL0_DBG, "dout_cpucl0_dbg", "gout_cpucl0_dbg", + CLK_CON_DIV_CLKCMU_CPUCL0_DBG, 0, 3), + DIV(CLK_DOUT_CPUCL0_SWITCH, "dout_cpucl0_switch", "gout_cpucl0_switch", + CLK_CON_DIV_CLKCMU_CPUCL0_SWITCH, 0, 3), + + /* CPUCL1 */ + DIV(CLK_DOUT_CPUCL1_DBG, "dout_cpucl1_dbg", "gout_cpucl1_dbg", + CLK_CON_DIV_CLKCMU_CPUCL1_DBG, 0, 3), + DIV(CLK_DOUT_CPUCL1_SWITCH, "dout_cpucl1_switch", "gout_cpucl1_switch", + CLK_CON_DIV_CLKCMU_CPUCL1_SWITCH, 0, 3), + /* DPU */ DIV(CLK_DOUT_DPU, "dout_dpu", "gout_dpu", CLK_CON_DIV_CLKCMU_DPU, 0, 4), @@ -442,6 +501,18 @@ static const struct samsung_gate_clock top_gate_clks[] __initconst = { GATE(CLK_GOUT_AUD, "gout_aud", "mout_aud", CLK_CON_GAT_GATE_CLKCMU_AUD, 21, 0, 0), + /* CPUCL0 */ + GATE(CLK_GOUT_CPUCL0_DBG, "gout_cpucl0_dbg", "mout_cpucl0_dbg", + CLK_CON_GAT_GATE_CLKCMU_CPUCL0_DBG, 21, 0, 0), + GATE(CLK_GOUT_CPUCL0_SWITCH, "gout_cpucl0_switch", "mout_cpucl0_switch", + CLK_CON_GAT_GATE_CLKCMU_CPUCL0_SWITCH, 21, 0, 0), + + /* CPUCL1 */ + GATE(CLK_GOUT_CPUCL1_DBG, "gout_cpucl1_dbg", "mout_cpucl1_dbg", + CLK_CON_GAT_GATE_CLKCMU_CPUCL1_DBG, 21, 0, 0), + GATE(CLK_GOUT_CPUCL1_SWITCH, "gout_cpucl1_switch", "mout_cpucl1_switch", + CLK_CON_GAT_GATE_CLKCMU_CPUCL1_SWITCH, 21, 0, 0), + /* DPU */ GATE(CLK_GOUT_DPU, "gout_dpu", "mout_dpu", CLK_CON_GAT_GATE_CLKCMU_DPU, 21, 0, 0), @@ -1030,6 +1101,373 @@ static const struct samsung_cmu_info cmgp_cmu_info __initconst = { .clk_name = "gout_clkcmu_cmgp_bus", }; +/* ---- CMU_CPUCL0 ---------------------------------------------------------- */ + +/* Register Offset definitions for CMU_CPUCL0 (0x10900000) */ +#define PLL_LOCKTIME_PLL_CPUCL0 0x0000 +#define PLL_CON0_PLL_CPUCL0 0x0100 +#define PLL_CON1_PLL_CPUCL0 0x0104 +#define PLL_CON3_PLL_CPUCL0 0x010c +#define PLL_CON0_MUX_CLKCMU_CPUCL0_DBG_USER 0x0600 +#define PLL_CON0_MUX_CLKCMU_CPUCL0_SWITCH_USER 0x0610 +#define CLK_CON_MUX_MUX_CLK_CPUCL0_PLL 0x100c +#define CLK_CON_DIV_DIV_CLK_CLUSTER0_ACLK 0x1800 +#define CLK_CON_DIV_DIV_CLK_CLUSTER0_ATCLK 0x1808 +#define CLK_CON_DIV_DIV_CLK_CLUSTER0_PCLKDBG 0x180c +#define CLK_CON_DIV_DIV_CLK_CLUSTER0_PERIPHCLK 0x1810 +#define CLK_CON_DIV_DIV_CLK_CPUCL0_CMUREF 0x1814 +#define CLK_CON_DIV_DIV_CLK_CPUCL0_CPU 0x1818 +#define CLK_CON_DIV_DIV_CLK_CPUCL0_PCLK 0x181c +#define CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_ATCLK 0x2000 +#define CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_PCLK 0x2004 +#define CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_PERIPHCLK 0x2008 +#define CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_SCLK 0x200c +#define CLK_CON_GAT_CLK_CPUCL0_CMU_CPUCL0_PCLK 0x2010 +#define CLK_CON_GAT_GATE_CLK_CPUCL0_CPU 0x2020 + +static const unsigned long cpucl0_clk_regs[] __initconst = { + PLL_LOCKTIME_PLL_CPUCL0, + PLL_CON0_PLL_CPUCL0, + PLL_CON1_PLL_CPUCL0, + PLL_CON3_PLL_CPUCL0, + PLL_CON0_MUX_CLKCMU_CPUCL0_DBG_USER, + PLL_CON0_MUX_CLKCMU_CPUCL0_SWITCH_USER, + CLK_CON_MUX_MUX_CLK_CPUCL0_PLL, + CLK_CON_DIV_DIV_CLK_CLUSTER0_ACLK, + CLK_CON_DIV_DIV_CLK_CLUSTER0_ATCLK, + CLK_CON_DIV_DIV_CLK_CLUSTER0_PCLKDBG, + CLK_CON_DIV_DIV_CLK_CLUSTER0_PERIPHCLK, + CLK_CON_DIV_DIV_CLK_CPUCL0_CMUREF, + CLK_CON_DIV_DIV_CLK_CPUCL0_CPU, + CLK_CON_DIV_DIV_CLK_CPUCL0_PCLK, + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_ATCLK, + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_PCLK, + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_PERIPHCLK, + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_SCLK, + CLK_CON_GAT_CLK_CPUCL0_CMU_CPUCL0_PCLK, + CLK_CON_GAT_GATE_CLK_CPUCL0_CPU, +}; + +/* List of parent clocks for Muxes in CMU_CPUCL0 */ +PNAME(mout_pll_cpucl0_p) = { "oscclk", "fout_cpucl0_pll" }; +PNAME(mout_cpucl0_switch_user_p) = { "oscclk", "dout_cpucl0_switch" }; +PNAME(mout_cpucl0_dbg_user_p) = { "oscclk", "dout_cpucl0_dbg" }; +PNAME(mout_cpucl0_pll_p) = { "mout_pll_cpucl0", + "mout_cpucl0_switch_user" }; + +static const struct samsung_pll_rate_table cpu_pll_rates[] __initconst = { + PLL_35XX_RATE(26 * MHZ, 2210000000U, 255, 3, 0), + PLL_35XX_RATE(26 * MHZ, 2106000000U, 243, 3, 0), + PLL_35XX_RATE(26 * MHZ, 2002000000U, 231, 3, 0), + PLL_35XX_RATE(26 * MHZ, 1846000000U, 213, 3, 0), + PLL_35XX_RATE(26 * MHZ, 1742000000U, 201, 3, 0), + PLL_35XX_RATE(26 * MHZ, 1586000000U, 183, 3, 0), + PLL_35XX_RATE(26 * MHZ, 1456000000U, 168, 3, 0), + PLL_35XX_RATE(26 * MHZ, 1300000000U, 150, 3, 0), + PLL_35XX_RATE(26 * MHZ, 1157000000U, 267, 3, 1), + PLL_35XX_RATE(26 * MHZ, 1053000000U, 243, 3, 1), + PLL_35XX_RATE(26 * MHZ, 949000000U, 219, 3, 1), + PLL_35XX_RATE(26 * MHZ, 806000000U, 186, 3, 1), + PLL_35XX_RATE(26 * MHZ, 650000000U, 150, 3, 1), + PLL_35XX_RATE(26 * MHZ, 546000000U, 252, 3, 2), + PLL_35XX_RATE(26 * MHZ, 442000000U, 204, 3, 2), + PLL_35XX_RATE(26 * MHZ, 351000000U, 162, 3, 2), + PLL_35XX_RATE(26 * MHZ, 247000000U, 114, 3, 2), + PLL_35XX_RATE(26 * MHZ, 182000000U, 168, 3, 3), + PLL_35XX_RATE(26 * MHZ, 130000000U, 120, 3, 3), +}; + +static const struct samsung_pll_clock cpucl0_pll_clks[] __initconst = { + PLL(pll_0822x, CLK_FOUT_CPUCL0_PLL, "fout_cpucl0_pll", "oscclk", + PLL_LOCKTIME_PLL_CPUCL0, PLL_CON3_PLL_CPUCL0, cpu_pll_rates), +}; + +static const struct samsung_mux_clock cpucl0_mux_clks[] __initconst = { + MUX_F(CLK_MOUT_PLL_CPUCL0, "mout_pll_cpucl0", mout_pll_cpucl0_p, + PLL_CON0_PLL_CPUCL0, 4, 1, + CLK_SET_RATE_PARENT | CLK_RECALC_NEW_RATES, 0), + MUX_F(CLK_MOUT_CPUCL0_SWITCH_USER, "mout_cpucl0_switch_user", + mout_cpucl0_switch_user_p, + PLL_CON0_MUX_CLKCMU_CPUCL0_SWITCH_USER, 4, 1, + CLK_SET_RATE_PARENT, 0), + MUX(CLK_MOUT_CPUCL0_DBG_USER, "mout_cpucl0_dbg_user", + mout_cpucl0_dbg_user_p, + PLL_CON0_MUX_CLKCMU_CPUCL0_DBG_USER, 4, 1), + MUX_F(CLK_MOUT_CPUCL0_PLL, "mout_cpucl0_pll", mout_cpucl0_pll_p, + CLK_CON_MUX_MUX_CLK_CPUCL0_PLL, 0, 1, CLK_SET_RATE_PARENT, 0), +}; + +static const struct samsung_div_clock cpucl0_div_clks[] __initconst = { + DIV_F(CLK_DOUT_CPUCL0_CPU, "dout_cpucl0_cpu", "mout_cpucl0_pll", + CLK_CON_DIV_DIV_CLK_CPUCL0_CPU, 0, 1, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CPUCL0_CMUREF, "dout_cpucl0_cmuref", "dout_cpucl0_cpu", + CLK_CON_DIV_DIV_CLK_CPUCL0_CMUREF, 0, 3, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CPUCL0_PCLK, "dout_cpucl0_pclk", "dout_cpucl0_cpu", + CLK_CON_DIV_DIV_CLK_CPUCL0_PCLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + + /* EMBEDDED_CMU_CPUCL0 */ + DIV_F(CLK_DOUT_CLUSTER0_ACLK, "dout_cluster0_aclk", "gout_cluster0_cpu", + CLK_CON_DIV_DIV_CLK_CLUSTER0_ACLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CLUSTER0_ATCLK, "dout_cluster0_atclk", + "gout_cluster0_cpu", CLK_CON_DIV_DIV_CLK_CLUSTER0_ATCLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CLUSTER0_PCLKDBG, "dout_cluster0_pclkdbg", + "gout_cluster0_cpu", CLK_CON_DIV_DIV_CLK_CLUSTER0_PCLKDBG, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CLUSTER0_PERIPHCLK, "dout_cluster0_periphclk", + "gout_cluster0_cpu", CLK_CON_DIV_DIV_CLK_CLUSTER0_PERIPHCLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), +}; + +static const struct samsung_gate_clock cpucl0_gate_clks[] __initconst = { + GATE(CLK_GOUT_CPUCL0_CMU_CPUCL0_PCLK, "gout_cpucl0_cmu_cpucl0_pclk", + "dout_cpucl0_pclk", + CLK_CON_GAT_CLK_CPUCL0_CMU_CPUCL0_PCLK, 21, CLK_IGNORE_UNUSED, 0), + + /* EMBEDDED_CMU_CPUCL0 */ + GATE(CLK_GOUT_CLUSTER0_CPU, "gout_cluster0_cpu", "dout_cpucl0_cpu", + CLK_CON_GAT_GATE_CLK_CPUCL0_CPU, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER0_SCLK, "gout_cluster0_sclk", "gout_cluster0_cpu", + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_SCLK, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER0_ATCLK, "gout_cluster0_atclk", + "dout_cluster0_atclk", + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_ATCLK, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER0_PERIPHCLK, "gout_cluster0_periphclk", + "dout_cluster0_periphclk", + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_PERIPHCLK, 21, + CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER0_PCLK, "gout_cluster0_pclk", + "dout_cluster0_pclkdbg", + CLK_CON_GAT_CLK_CPUCL0_CLUSTER0_PCLK, 21, CLK_IGNORE_UNUSED, 0), +}; + +/* + * Each parameter is going to be written into the corresponding DIV register. So + * the actual divider value for each parameter will be 1/(param+1). All these + * parameters must be in the range of 0..15, as the divider range for all of + * these DIV clocks is 1..16. The default values for these dividers is + * (1, 3, 3, 1). + */ +#define E850_CPU_DIV0(aclk, atclk, pclkdbg, periphclk) \ + (((aclk) << 16) | ((atclk) << 12) | ((pclkdbg) << 8) | \ + ((periphclk) << 4)) + +static const struct exynos_cpuclk_cfg_data exynos850_cluster_clk_d[] __initconst += { + { 2210000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 2106000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 2002000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 1846000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 1742000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 1586000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 1456000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 1300000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 1157000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 1053000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 949000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 806000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 650000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 546000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 442000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 351000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 247000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 182000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 130000, E850_CPU_DIV0(1, 3, 3, 1) }, + { 0 } +}; + +static const struct samsung_cpu_clock cpucl0_cpu_clks[] __initconst = { + CPU_CLK(CLK_CLUSTER0_SCLK, "cluster0_clk", CLK_MOUT_PLL_CPUCL0, + CLK_MOUT_CPUCL0_SWITCH_USER, 0, 0x0, CPUCLK_LAYOUT_E850_CL0, + exynos850_cluster_clk_d), +}; + +static const struct samsung_cmu_info cpucl0_cmu_info __initconst = { + .pll_clks = cpucl0_pll_clks, + .nr_pll_clks = ARRAY_SIZE(cpucl0_pll_clks), + .mux_clks = cpucl0_mux_clks, + .nr_mux_clks = ARRAY_SIZE(cpucl0_mux_clks), + .div_clks = cpucl0_div_clks, + .nr_div_clks = ARRAY_SIZE(cpucl0_div_clks), + .gate_clks = cpucl0_gate_clks, + .nr_gate_clks = ARRAY_SIZE(cpucl0_gate_clks), + .cpu_clks = cpucl0_cpu_clks, + .nr_cpu_clks = ARRAY_SIZE(cpucl0_cpu_clks), + .nr_clk_ids = CLKS_NR_CPUCL0, + .clk_regs = cpucl0_clk_regs, + .nr_clk_regs = ARRAY_SIZE(cpucl0_clk_regs), + .clk_name = "dout_cpucl0_switch", + .manual_plls = true, +}; + +static void __init exynos850_cmu_cpucl0_init(struct device_node *np) +{ + exynos_arm64_register_cmu(NULL, np, &cpucl0_cmu_info); +} + +/* Register CMU_CPUCL0 early, as CPU clocks should be available ASAP */ +CLK_OF_DECLARE(exynos850_cmu_cpucl0, "samsung,exynos850-cmu-cpucl0", + exynos850_cmu_cpucl0_init); + +/* ---- CMU_CPUCL1 ---------------------------------------------------------- */ + +/* Register Offset definitions for CMU_CPUCL1 (0x10800000) */ +#define PLL_LOCKTIME_PLL_CPUCL1 0x0000 +#define PLL_CON0_PLL_CPUCL1 0x0100 +#define PLL_CON1_PLL_CPUCL1 0x0104 +#define PLL_CON3_PLL_CPUCL1 0x010c +#define PLL_CON0_MUX_CLKCMU_CPUCL1_DBG_USER 0x0600 +#define PLL_CON0_MUX_CLKCMU_CPUCL1_SWITCH_USER 0x0610 +#define CLK_CON_MUX_MUX_CLK_CPUCL1_PLL 0x1000 +#define CLK_CON_DIV_DIV_CLK_CLUSTER1_ACLK 0x1800 +#define CLK_CON_DIV_DIV_CLK_CLUSTER1_ATCLK 0x1808 +#define CLK_CON_DIV_DIV_CLK_CLUSTER1_PCLKDBG 0x180c +#define CLK_CON_DIV_DIV_CLK_CLUSTER1_PERIPHCLK 0x1810 +#define CLK_CON_DIV_DIV_CLK_CPUCL1_CMUREF 0x1814 +#define CLK_CON_DIV_DIV_CLK_CPUCL1_CPU 0x1818 +#define CLK_CON_DIV_DIV_CLK_CPUCL1_PCLK 0x181c +#define CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_ATCLK 0x2000 +#define CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_PCLK 0x2004 +#define CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_PERIPHCLK 0x2008 +#define CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_SCLK 0x200c +#define CLK_CON_GAT_CLK_CPUCL1_CMU_CPUCL1_PCLK 0x2010 +#define CLK_CON_GAT_GATE_CLK_CPUCL1_CPU 0x2020 + +static const unsigned long cpucl1_clk_regs[] __initconst = { + PLL_LOCKTIME_PLL_CPUCL1, + PLL_CON0_PLL_CPUCL1, + PLL_CON1_PLL_CPUCL1, + PLL_CON3_PLL_CPUCL1, + PLL_CON0_MUX_CLKCMU_CPUCL1_DBG_USER, + PLL_CON0_MUX_CLKCMU_CPUCL1_SWITCH_USER, + CLK_CON_MUX_MUX_CLK_CPUCL1_PLL, + CLK_CON_DIV_DIV_CLK_CLUSTER1_ACLK, + CLK_CON_DIV_DIV_CLK_CLUSTER1_ATCLK, + CLK_CON_DIV_DIV_CLK_CLUSTER1_PCLKDBG, + CLK_CON_DIV_DIV_CLK_CLUSTER1_PERIPHCLK, + CLK_CON_DIV_DIV_CLK_CPUCL1_CMUREF, + CLK_CON_DIV_DIV_CLK_CPUCL1_CPU, + CLK_CON_DIV_DIV_CLK_CPUCL1_PCLK, + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_ATCLK, + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_PCLK, + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_PERIPHCLK, + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_SCLK, + CLK_CON_GAT_CLK_CPUCL1_CMU_CPUCL1_PCLK, + CLK_CON_GAT_GATE_CLK_CPUCL1_CPU, +}; + +/* List of parent clocks for Muxes in CMU_CPUCL0 */ +PNAME(mout_pll_cpucl1_p) = { "oscclk", "fout_cpucl1_pll" }; +PNAME(mout_cpucl1_switch_user_p) = { "oscclk", "dout_cpucl1_switch" }; +PNAME(mout_cpucl1_dbg_user_p) = { "oscclk", "dout_cpucl1_dbg" }; +PNAME(mout_cpucl1_pll_p) = { "mout_pll_cpucl1", + "mout_cpucl1_switch_user" }; + +static const struct samsung_pll_clock cpucl1_pll_clks[] __initconst = { + PLL(pll_0822x, CLK_FOUT_CPUCL1_PLL, "fout_cpucl1_pll", "oscclk", + PLL_LOCKTIME_PLL_CPUCL1, PLL_CON3_PLL_CPUCL1, cpu_pll_rates), +}; + +static const struct samsung_mux_clock cpucl1_mux_clks[] __initconst = { + MUX_F(CLK_MOUT_PLL_CPUCL1, "mout_pll_cpucl1", mout_pll_cpucl1_p, + PLL_CON0_PLL_CPUCL1, 4, 1, + CLK_SET_RATE_PARENT | CLK_RECALC_NEW_RATES, 0), + MUX_F(CLK_MOUT_CPUCL1_SWITCH_USER, "mout_cpucl1_switch_user", + mout_cpucl1_switch_user_p, + PLL_CON0_MUX_CLKCMU_CPUCL1_SWITCH_USER, 4, 1, + CLK_SET_RATE_PARENT, 0), + MUX(CLK_MOUT_CPUCL1_DBG_USER, "mout_cpucl1_dbg_user", + mout_cpucl1_dbg_user_p, + PLL_CON0_MUX_CLKCMU_CPUCL1_DBG_USER, 4, 1), + MUX_F(CLK_MOUT_CPUCL1_PLL, "mout_cpucl1_pll", mout_cpucl1_pll_p, + CLK_CON_MUX_MUX_CLK_CPUCL1_PLL, 0, 1, CLK_SET_RATE_PARENT, 0), +}; + +static const struct samsung_div_clock cpucl1_div_clks[] __initconst = { + DIV_F(CLK_DOUT_CPUCL1_CPU, "dout_cpucl1_cpu", "mout_cpucl1_pll", + CLK_CON_DIV_DIV_CLK_CPUCL1_CPU, 0, 1, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CPUCL1_CMUREF, "dout_cpucl1_cmuref", "dout_cpucl1_cpu", + CLK_CON_DIV_DIV_CLK_CPUCL1_CMUREF, 0, 3, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CPUCL1_PCLK, "dout_cpucl1_pclk", "dout_cpucl1_cpu", + CLK_CON_DIV_DIV_CLK_CPUCL1_PCLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + + /* EMBEDDED_CMU_CPUCL1 */ + DIV_F(CLK_DOUT_CLUSTER1_ACLK, "dout_cluster1_aclk", "gout_cluster1_cpu", + CLK_CON_DIV_DIV_CLK_CLUSTER1_ACLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CLUSTER1_ATCLK, "dout_cluster1_atclk", + "gout_cluster1_cpu", CLK_CON_DIV_DIV_CLK_CLUSTER1_ATCLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CLUSTER1_PCLKDBG, "dout_cluster1_pclkdbg", + "gout_cluster1_cpu", CLK_CON_DIV_DIV_CLK_CLUSTER1_PCLKDBG, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), + DIV_F(CLK_DOUT_CLUSTER1_PERIPHCLK, "dout_cluster1_periphclk", + "gout_cluster1_cpu", CLK_CON_DIV_DIV_CLK_CLUSTER1_PERIPHCLK, 0, 4, + CLK_GET_RATE_NOCACHE, CLK_DIVIDER_READ_ONLY), +}; + +static const struct samsung_gate_clock cpucl1_gate_clks[] __initconst = { + GATE(CLK_GOUT_CPUCL1_CMU_CPUCL1_PCLK, "gout_cpucl1_cmu_cpucl1_pclk", + "dout_cpucl1_pclk", + CLK_CON_GAT_CLK_CPUCL1_CMU_CPUCL1_PCLK, 21, CLK_IGNORE_UNUSED, 0), + + /* EMBEDDED_CMU_CPUCL1 */ + GATE(CLK_GOUT_CLUSTER1_CPU, "gout_cluster1_cpu", "dout_cpucl1_cpu", + CLK_CON_GAT_GATE_CLK_CPUCL1_CPU, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER1_SCLK, "gout_cluster1_sclk", "gout_cluster1_cpu", + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_SCLK, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER1_ATCLK, "gout_cluster1_atclk", + "dout_cluster1_atclk", + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_ATCLK, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER1_PERIPHCLK, "gout_cluster1_periphclk", + "dout_cluster1_periphclk", + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_PERIPHCLK, 21, + CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_CLUSTER1_PCLK, "gout_cluster1_pclk", + "dout_cluster1_pclkdbg", + CLK_CON_GAT_CLK_CPUCL1_CLUSTER1_PCLK, 21, CLK_IGNORE_UNUSED, 0), +}; + +static const struct samsung_cpu_clock cpucl1_cpu_clks[] __initconst = { + CPU_CLK(CLK_CLUSTER1_SCLK, "cluster1_clk", CLK_MOUT_PLL_CPUCL1, + CLK_MOUT_CPUCL1_SWITCH_USER, 0, 0x0, CPUCLK_LAYOUT_E850_CL1, + exynos850_cluster_clk_d), +}; + +static const struct samsung_cmu_info cpucl1_cmu_info __initconst = { + .pll_clks = cpucl1_pll_clks, + .nr_pll_clks = ARRAY_SIZE(cpucl1_pll_clks), + .mux_clks = cpucl1_mux_clks, + .nr_mux_clks = ARRAY_SIZE(cpucl1_mux_clks), + .div_clks = cpucl1_div_clks, + .nr_div_clks = ARRAY_SIZE(cpucl1_div_clks), + .gate_clks = cpucl1_gate_clks, + .nr_gate_clks = ARRAY_SIZE(cpucl1_gate_clks), + .cpu_clks = cpucl1_cpu_clks, + .nr_cpu_clks = ARRAY_SIZE(cpucl1_cpu_clks), + .nr_clk_ids = CLKS_NR_CPUCL1, + .clk_regs = cpucl1_clk_regs, + .nr_clk_regs = ARRAY_SIZE(cpucl1_clk_regs), + .clk_name = "dout_cpucl1_switch", + .manual_plls = true, +}; + +static void __init exynos850_cmu_cpucl1_init(struct device_node *np) +{ + exynos_arm64_register_cmu(NULL, np, &cpucl1_cmu_info); +} + +/* Register CMU_CPUCL1 early, as CPU clocks should be available ASAP */ +CLK_OF_DECLARE(exynos850_cmu_cpucl1, "samsung,exynos850-cmu-cpucl1", + exynos850_cmu_cpucl1_init); + /* ---- CMU_G3D ------------------------------------------------------------- */ /* Register Offset definitions for CMU_G3D (0x11400000) */ From 98784a9d398ea5513cf571835be7e45885ba49ab Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 12 Mar 2024 19:50:35 +0100 Subject: [PATCH 0016/1702] dt-bindings: clock: samsung,s3c6400-clock: convert to DT Schema Convert Samsung S3C6400/S3C6410 SoC clock controller bindings to DT schema. Acked-by: Conor Dooley Reviewed-by: Stephen Boyd Link: https://lore.kernel.org/r/20240312185035.720491-1-krzysztof.kozlowski@linaro.org Signed-off-by: Krzysztof Kozlowski --- .../bindings/clock/samsung,s3c6400-clock.yaml | 57 ++++++++++++++ .../bindings/clock/samsung,s3c64xx-clock.txt | 76 ------------------- 2 files changed, 57 insertions(+), 76 deletions(-) create mode 100644 Documentation/devicetree/bindings/clock/samsung,s3c6400-clock.yaml delete mode 100644 Documentation/devicetree/bindings/clock/samsung,s3c64xx-clock.txt diff --git a/Documentation/devicetree/bindings/clock/samsung,s3c6400-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,s3c6400-clock.yaml new file mode 100644 index 0000000000000..0fcc0c963f8f4 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/samsung,s3c6400-clock.yaml @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/samsung,s3c6400-clock.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Samsung S3C6400 SoC clock controller + +maintainers: + - Krzysztof Kozlowski + +description: | + There are several clocks that are generated outside the SoC. It is expected + that they are defined using standard clock bindings with following + clock-output-names and/or provided as clock inputs to this clock controller: + - "fin_pll" - PLL input clock (xtal/extclk) - required, + - "xusbxti" - USB xtal - required, + - "iiscdclk0" - I2S0 codec clock - optional, + - "iiscdclk1" - I2S1 codec clock - optional, + - "iiscdclk2" - I2S2 codec clock - optional, + - "pcmcdclk0" - PCM0 codec clock - optional, + - "pcmcdclk1" - PCM1 codec clock - optional, only S3C6410. + + All available clocks are defined as preprocessor macros in + include/dt-bindings/clock/samsung,s3c64xx-clock.h header. + +properties: + compatible: + enum: + - samsung,s3c6400-clock + - samsung,s3c6410-clock + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + "#clock-cells": + const: 1 + +required: + - compatible + - reg + - clocks + - "#clock-cells" + +additionalProperties: false + +examples: + - | + clock-controller@7e00f000 { + compatible = "samsung,s3c6410-clock"; + reg = <0x7e00f000 0x1000>; + #clock-cells = <1>; + clocks = <&fin_pll>; + }; diff --git a/Documentation/devicetree/bindings/clock/samsung,s3c64xx-clock.txt b/Documentation/devicetree/bindings/clock/samsung,s3c64xx-clock.txt deleted file mode 100644 index 872ee8e0f0412..0000000000000 --- a/Documentation/devicetree/bindings/clock/samsung,s3c64xx-clock.txt +++ /dev/null @@ -1,76 +0,0 @@ -* Samsung S3C64xx Clock Controller - -The S3C64xx clock controller generates and supplies clock to various controllers -within the SoC. The clock binding described here is applicable to all SoCs in -the S3C64xx family. - -Required Properties: - -- compatible: should be one of the following. - - "samsung,s3c6400-clock" - controller compatible with S3C6400 SoC. - - "samsung,s3c6410-clock" - controller compatible with S3C6410 SoC. - -- reg: physical base address of the controller and length of memory mapped - region. - -- #clock-cells: should be 1. - -Each clock is assigned an identifier and client nodes can use this identifier -to specify the clock which they consume. Some of the clocks are available only -on a particular S3C64xx SoC and this is specified where applicable. - -All available clocks are defined as preprocessor macros in -dt-bindings/clock/samsung,s3c64xx-clock.h header and can be used in device -tree sources. - -External clocks: - -There are several clocks that are generated outside the SoC. It is expected -that they are defined using standard clock bindings with following -clock-output-names: - - "fin_pll" - PLL input clock (xtal/extclk) - required, - - "xusbxti" - USB xtal - required, - - "iiscdclk0" - I2S0 codec clock - optional, - - "iiscdclk1" - I2S1 codec clock - optional, - - "iiscdclk2" - I2S2 codec clock - optional, - - "pcmcdclk0" - PCM0 codec clock - optional, - - "pcmcdclk1" - PCM1 codec clock - optional, only S3C6410. - -Example: Clock controller node: - - clock: clock-controller@7e00f000 { - compatible = "samsung,s3c6410-clock"; - reg = <0x7e00f000 0x1000>; - #clock-cells = <1>; - }; - -Example: Required external clocks: - - fin_pll: clock-fin-pll { - compatible = "fixed-clock"; - clock-output-names = "fin_pll"; - clock-frequency = <12000000>; - #clock-cells = <0>; - }; - - xusbxti: clock-xusbxti { - compatible = "fixed-clock"; - clock-output-names = "xusbxti"; - clock-frequency = <48000000>; - #clock-cells = <0>; - }; - -Example: UART controller node that consumes the clock generated by the clock - controller (refer to the standard clock bindings for information about - "clocks" and "clock-names" properties): - - uart0: serial@7f005000 { - compatible = "samsung,s3c6400-uart"; - reg = <0x7f005000 0x100>; - interrupt-parent = <&vic1>; - interrupts = <5>; - clock-names = "uart", "clk_uart_baud2", - "clk_uart_baud3"; - clocks = <&clock PCLK_UART0>, <&clocks PCLK_UART0>, - <&clock SCLK_UART>; - }; From 4d69c58ef2e419550f02f988ee022fca564f7dd7 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 25 Mar 2024 19:36:02 -0600 Subject: [PATCH 0017/1702] fsnotify: Avoid -Wflex-array-member-not-at-end warning Use the `DEFINE_FLEX()` helper for an on-stack definition of a flexible structure where the size of the flexible-array member is known at compile-time, and refactor the rest of the code, accordingly. So, with these changes, fix the following warning: fs/notify/fdinfo.c:45:36: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end] Link: https://github.com/KSPP/linux/issues/202 Signed-off-by: Gustavo A. R. Silva Signed-off-by: Jan Kara Message-Id: --- fs/notify/fdinfo.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/fs/notify/fdinfo.c b/fs/notify/fdinfo.c index 5c430736ec12c..dec553034027e 100644 --- a/fs/notify/fdinfo.c +++ b/fs/notify/fdinfo.c @@ -41,29 +41,25 @@ static void show_fdinfo(struct seq_file *m, struct file *f, #if defined(CONFIG_EXPORTFS) static void show_mark_fhandle(struct seq_file *m, struct inode *inode) { - struct { - struct file_handle handle; - u8 pad[MAX_HANDLE_SZ]; - } f; + DEFINE_FLEX(struct file_handle, f, f_handle, handle_bytes, MAX_HANDLE_SZ); int size, ret, i; - f.handle.handle_bytes = sizeof(f.pad); - size = f.handle.handle_bytes >> 2; + size = f->handle_bytes >> 2; - ret = exportfs_encode_fid(inode, (struct fid *)f.handle.f_handle, &size); + ret = exportfs_encode_fid(inode, (struct fid *)f->f_handle, &size); if ((ret == FILEID_INVALID) || (ret < 0)) { WARN_ONCE(1, "Can't encode file handler for inotify: %d\n", ret); return; } - f.handle.handle_type = ret; - f.handle.handle_bytes = size * sizeof(u32); + f->handle_type = ret; + f->handle_bytes = size * sizeof(u32); seq_printf(m, "fhandle-bytes:%x fhandle-type:%x f_handle:", - f.handle.handle_bytes, f.handle.handle_type); + f->handle_bytes, f->handle_type); - for (i = 0; i < f.handle.handle_bytes; i++) - seq_printf(m, "%02x", (int)f.handle.f_handle[i]); + for (i = 0; i < f->handle_bytes; i++) + seq_printf(m, "%02x", (int)f->f_handle[i]); } #else static void show_mark_fhandle(struct seq_file *m, struct inode *inode) From 3127f1010c9b27d925e83081d413ea7fc361abb0 Mon Sep 17 00:00:00 2001 From: Yeongjin Gil Date: Fri, 22 Mar 2024 13:16:39 +0900 Subject: [PATCH 0018/1702] f2fs: Prevent s_writer rw_sem count mismatch in f2fs_evict_inode If f2fs_evict_inode is called between freeze_super and thaw_super, the s_writer rwsem count may become negative, resulting in hang. CPU1 CPU2 f2fs_resize_fs() f2fs_evict_inode() f2fs_freeze set SBI_IS_FREEZING skip sb_start_intwrite f2fs_unfreeze clear SBI_IS_FREEZING sb_end_intwrite To solve this problem, the call to sb_end_write is determined by whether sb_start_intwrite is called, rather than the current freezing status. Reviewed-by: Sungjong Seo Reviewed-by: Sunmin Jeong Signed-off-by: Yeongjin Gil Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/inode.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index c26effdce9aa7..12b1fef31f437 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -804,6 +804,7 @@ void f2fs_evict_inode(struct inode *inode) struct f2fs_inode_info *fi = F2FS_I(inode); nid_t xnid = fi->i_xattr_nid; int err = 0; + bool freeze_protected = false; f2fs_abort_atomic_write(inode, true); @@ -843,8 +844,10 @@ void f2fs_evict_inode(struct inode *inode) f2fs_remove_ino_entry(sbi, inode->i_ino, UPDATE_INO); f2fs_remove_ino_entry(sbi, inode->i_ino, FLUSH_INO); - if (!is_sbi_flag_set(sbi, SBI_IS_FREEZING)) + if (!is_sbi_flag_set(sbi, SBI_IS_FREEZING)) { sb_start_intwrite(inode->i_sb); + freeze_protected = true; + } set_inode_flag(inode, FI_NO_ALLOC); i_size_write(inode, 0); retry: @@ -887,7 +890,7 @@ void f2fs_evict_inode(struct inode *inode) if (dquot_initialize_needed(inode)) set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); } - if (!is_sbi_flag_set(sbi, SBI_IS_FREEZING)) + if (freeze_protected) sb_end_intwrite(inode->i_sb); no_delete: dquot_drop(inode); From 92c556ed6318e13c16746495a8d4513129eb9b0f Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Fri, 22 Mar 2024 22:59:55 +0800 Subject: [PATCH 0019/1702] f2fs: fix to detect inconsistent nat entry during truncation As Roman Smirnov reported as below: " There is a possible bug in f2fs_truncate_inode_blocks(): if (err < 0 && err != -ENOENT) goto fail; ... offset[1] = 0; offset[0]++; nofs += err; If err = -ENOENT then nofs will sum with an error code, which is strange behaviour. Also if nofs < ENOENT this will cause an overflow. err will be equal to -ENOENT with the following call stack: truncate_nodes() f2fs_get_node_page() __get_node_page() read_node_page() " If nat is corrupted, truncate_nodes() may return -ENOENT, and f2fs_truncate_inode_blocks() doesn't handle such error correctly, fix it. Reported-by: Roman Smirnov Closes: https://lore.kernel.org/linux-f2fs-devel/085b27fd2b364a3c8c3a9ca77363e246@omp.ru Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/node.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index b3de6d6cdb021..bb57bbaff7b4f 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -1187,7 +1187,17 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from) default: BUG(); } - if (err < 0 && err != -ENOENT) + if (err == -ENOENT) { + set_sbi_flag(F2FS_P_SB(page), SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); + f2fs_err_ratelimited(sbi, + "truncate node fail, ino:%lu, nid:%u, " + "offset[0]:%d, offset[1]:%d, nofs:%d", + inode->i_ino, dn.nid, offset[0], + offset[1], nofs); + err = 0; + } + if (err < 0) goto fail; if (offset[1] == 0 && ri->i_nid[offset[0] - NODE_DIR1_BLOCK]) { From ac7805be6cffd5a16f86872c4e56c28a1433b222 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Mon, 25 Mar 2024 23:27:25 +0800 Subject: [PATCH 0020/1702] f2fs: introduce map_is_mergeable() for cleanup No logic changes. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/data.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index d9494b5fc7c18..08815394223a3 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -1507,6 +1507,22 @@ static bool f2fs_map_blocks_cached(struct inode *inode, return true; } +static bool map_is_mergeable(struct f2fs_sb_info *sbi, + struct f2fs_map_blocks *map, + block_t blkaddr, int flag, int bidx, + int ofs) +{ + if (map->m_multidev_dio && map->m_bdev != FDEV(bidx).bdev) + return false; + if (map->m_pblk != NEW_ADDR && blkaddr == (map->m_pblk + ofs)) + return true; + if (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) + return true; + if (flag == F2FS_GET_BLOCK_PRE_DIO) + return true; + return false; +} + /* * f2fs_map_blocks() tries to find or build mapping relationship which * maps continuous logical blocks to physical blocks, and return such @@ -1653,12 +1669,7 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) if (map->m_multidev_dio) map->m_bdev = FDEV(bidx).bdev; - } else if ((map->m_pblk != NEW_ADDR && - blkaddr == (map->m_pblk + ofs)) || - (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) || - flag == F2FS_GET_BLOCK_PRE_DIO) { - if (map->m_multidev_dio && map->m_bdev != FDEV(bidx).bdev) - goto sync_out; + } else if (map_is_mergeable(sbi, map, blkaddr, flag, bidx, ofs)) { ofs++; map->m_len++; } else { From ea217fefef8c9632486a943a029b01cee717dce5 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 11 Mar 2024 16:08:33 +0200 Subject: [PATCH 0021/1702] pinctrl: pxa2xx: Make use of struct pinfunction Since pin control provides a generic data type for the pin function, use it in the driver. Signed-off-by: Andy Shevchenko Message-ID: <20240311140833.1168742-1-andriy.shevchenko@linux.intel.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/pxa/pinctrl-pxa2xx.c | 24 ++++++++++-------------- drivers/pinctrl/pxa/pinctrl-pxa2xx.h | 8 +------- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/drivers/pinctrl/pxa/pinctrl-pxa2xx.c b/drivers/pinctrl/pxa/pinctrl-pxa2xx.c index d2568dab8c782..f24bf49fa82bc 100644 --- a/drivers/pinctrl/pxa/pinctrl-pxa2xx.c +++ b/drivers/pinctrl/pxa/pinctrl-pxa2xx.c @@ -109,7 +109,7 @@ static const char *pxa2xx_pmx_get_func_name(struct pinctrl_dev *pctldev, unsigned function) { struct pxa_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev); - struct pxa_pinctrl_function *pf = pctl->functions + function; + struct pinfunction *pf = pctl->functions + function; return pf->name; } @@ -127,7 +127,7 @@ static int pxa2xx_pmx_get_func_groups(struct pinctrl_dev *pctldev, unsigned * const num_groups) { struct pxa_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev); - struct pxa_pinctrl_function *pf = pctl->functions + function; + struct pinfunction *pf = pctl->functions + function; *groups = pf->groups; *num_groups = pf->ngroups; @@ -249,11 +249,11 @@ static struct pinctrl_desc pxa2xx_pinctrl_desc = { .pmxops = &pxa2xx_pinmux_ops, }; -static const struct pxa_pinctrl_function * -pxa2xx_find_function(struct pxa_pinctrl *pctl, const char *fname, - const struct pxa_pinctrl_function *functions) +static const struct pinfunction *pxa2xx_find_function(struct pxa_pinctrl *pctl, + const char *fname, + const struct pinfunction *functions) { - const struct pxa_pinctrl_function *func; + const struct pinfunction *func; for (func = functions; func->name; func++) if (!strcmp(fname, func->name)) @@ -264,8 +264,8 @@ pxa2xx_find_function(struct pxa_pinctrl *pctl, const char *fname, static int pxa2xx_build_functions(struct pxa_pinctrl *pctl) { + struct pinfunction *functions; int i; - struct pxa_pinctrl_function *functions; struct pxa_desc_function *df; /* @@ -296,9 +296,9 @@ static int pxa2xx_build_functions(struct pxa_pinctrl *pctl) static int pxa2xx_build_groups(struct pxa_pinctrl *pctl) { int i, j, ngroups; - struct pxa_pinctrl_function *func; struct pxa_desc_function *df; - char **gtmp; + struct pinfunction *func; + const char **gtmp; gtmp = devm_kmalloc_array(pctl->dev, pctl->npins, sizeof(*gtmp), GFP_KERNEL); @@ -316,13 +316,9 @@ static int pxa2xx_build_groups(struct pxa_pinctrl *pctl) pctl->ppins[j].pin.name; func = pctl->functions + i; func->ngroups = ngroups; - func->groups = - devm_kmalloc_array(pctl->dev, ngroups, - sizeof(char *), GFP_KERNEL); + func->groups = devm_kmemdup(pctl->dev, gtmp, ngroups * sizeof(*gtmp), GFP_KERNEL); if (!func->groups) return -ENOMEM; - - memcpy(func->groups, gtmp, ngroups * sizeof(*gtmp)); } devm_kfree(pctl->dev, gtmp); diff --git a/drivers/pinctrl/pxa/pinctrl-pxa2xx.h b/drivers/pinctrl/pxa/pinctrl-pxa2xx.h index d86d47dbbc947..a0bdcec551585 100644 --- a/drivers/pinctrl/pxa/pinctrl-pxa2xx.h +++ b/drivers/pinctrl/pxa/pinctrl-pxa2xx.h @@ -57,12 +57,6 @@ struct pxa_pinctrl_group { unsigned pin; }; -struct pxa_pinctrl_function { - const char *name; - const char **groups; - unsigned ngroups; -}; - struct pxa_pinctrl { spinlock_t lock; void __iomem **base_gafr; @@ -76,7 +70,7 @@ struct pxa_pinctrl { unsigned ngroups; struct pxa_pinctrl_group *groups; unsigned nfuncs; - struct pxa_pinctrl_function *functions; + struct pinfunction *functions; char *name; }; From 76c22f094153478dc89735c9bd259ad99f933913 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 11 Mar 2024 16:22:48 +0200 Subject: [PATCH 0022/1702] pinctrl: pxa2xx: Make use of struct pingroup Since pin control provides a generic data type for the pin group, use it in the driver. Signed-off-by: Andy Shevchenko Message-ID: <20240311142346.1261203-1-andriy.shevchenko@linux.intel.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/pxa/pinctrl-pxa2xx.c | 31 ++++++++++++++-------------- drivers/pinctrl/pxa/pinctrl-pxa2xx.h | 7 +------ 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/drivers/pinctrl/pxa/pinctrl-pxa2xx.c b/drivers/pinctrl/pxa/pinctrl-pxa2xx.c index f24bf49fa82bc..9e34b92ff5f2d 100644 --- a/drivers/pinctrl/pxa/pinctrl-pxa2xx.c +++ b/drivers/pinctrl/pxa/pinctrl-pxa2xx.c @@ -32,7 +32,7 @@ static const char *pxa2xx_pctrl_get_group_name(struct pinctrl_dev *pctldev, unsigned tgroup) { struct pxa_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev); - struct pxa_pinctrl_group *group = pctl->groups + tgroup; + struct pingroup *group = pctl->groups + tgroup; return group->name; } @@ -43,10 +43,10 @@ static int pxa2xx_pctrl_get_group_pins(struct pinctrl_dev *pctldev, unsigned *num_pins) { struct pxa_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev); - struct pxa_pinctrl_group *group = pctl->groups + tgroup; + struct pingroup *group = pctl->groups + tgroup; - *pins = (unsigned *)&group->pin; - *num_pins = 1; + *pins = group->pins; + *num_pins = group->npins; return 0; } @@ -139,20 +139,18 @@ static int pxa2xx_pmx_set_mux(struct pinctrl_dev *pctldev, unsigned function, unsigned tgroup) { struct pxa_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev); - struct pxa_pinctrl_group *group = pctl->groups + tgroup; + struct pingroup *g = pctl->groups + tgroup; + unsigned int pin = g->pins[0]; struct pxa_desc_function *df; - int pin, shift; unsigned long flags; void __iomem *gafr, *gpdr; + int shift; u32 val; - - df = pxa_desc_by_func_group(pctl, group->name, - (pctl->functions + function)->name); + df = pxa_desc_by_func_group(pctl, g->name, (pctl->functions + function)->name); if (!df) return -EINVAL; - pin = group->pin; gafr = pctl->base_gafr[pin / 16]; gpdr = pctl->base_gpdr[pin / 32]; shift = (pin % 16) << 1; @@ -186,9 +184,9 @@ static int pxa2xx_pconf_group_get(struct pinctrl_dev *pctldev, unsigned long *config) { struct pxa_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev); - struct pxa_pinctrl_group *g = pctl->groups + group; + struct pingroup *g = pctl->groups + group; + unsigned int pin = g->pins[0]; unsigned long flags; - unsigned pin = g->pin; void __iomem *pgsr = pctl->base_pgsr[pin / 32]; u32 val; @@ -208,9 +206,9 @@ static int pxa2xx_pconf_group_set(struct pinctrl_dev *pctldev, unsigned num_configs) { struct pxa_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev); - struct pxa_pinctrl_group *g = pctl->groups + group; + struct pingroup *g = pctl->groups + group; + unsigned int pin = g->pins[0]; unsigned long flags; - unsigned pin = g->pin; void __iomem *pgsr = pctl->base_pgsr[pin / 32]; int i, is_set = 0; u32 val; @@ -328,8 +326,8 @@ static int pxa2xx_build_groups(struct pxa_pinctrl *pctl) static int pxa2xx_build_state(struct pxa_pinctrl *pctl, const struct pxa_desc_pin *ppins, int npins) { - struct pxa_pinctrl_group *group; struct pinctrl_pin_desc *pins; + struct pingroup *group; int ret, i; pctl->npins = npins; @@ -353,7 +351,8 @@ static int pxa2xx_build_state(struct pxa_pinctrl *pctl, for (i = 0; i < npins; i++) { group = pctl->groups + i; group->name = ppins[i].pin.name; - group->pin = ppins[i].pin.number; + group->pins = &ppins[i].pin.number; + group->npins = 1; } ret = pxa2xx_build_functions(pctl); diff --git a/drivers/pinctrl/pxa/pinctrl-pxa2xx.h b/drivers/pinctrl/pxa/pinctrl-pxa2xx.h index a0bdcec551585..b292b79efdf85 100644 --- a/drivers/pinctrl/pxa/pinctrl-pxa2xx.h +++ b/drivers/pinctrl/pxa/pinctrl-pxa2xx.h @@ -52,11 +52,6 @@ struct pxa_desc_pin { struct pxa_desc_function *functions; }; -struct pxa_pinctrl_group { - const char *name; - unsigned pin; -}; - struct pxa_pinctrl { spinlock_t lock; void __iomem **base_gafr; @@ -68,7 +63,7 @@ struct pxa_pinctrl { unsigned npins; const struct pxa_desc_pin *ppins; unsigned ngroups; - struct pxa_pinctrl_group *groups; + struct pingroup *groups; unsigned nfuncs; struct pinfunction *functions; char *name; From 85b02bc0785b5e6326814af7e1b7528ce3a488ea Mon Sep 17 00:00:00 2001 From: Stefan Wahren Date: Thu, 7 Mar 2024 08:01:12 +0100 Subject: [PATCH 0023/1702] pinctrl: bcm2835: Implement bcm2835_pinconf_get Even the driver already has implemented pin_dbg_show, it could be helpful to implement pin_conf_get for a more generic behavior. Contrary to the BCM2711, the BCM2835 SOC doesn't allow to read the bias config, so the implementation is limited to the basics. Keep ENOTSUPP here, because it's only used internally. Signed-off-by: Stefan Wahren Reviewed-by: Chen-Yu Tsai Message-ID: <20240307070113.4888-2-wahrenst@gmx.net> Signed-off-by: Linus Walleij --- drivers/pinctrl/bcm/pinctrl-bcm2835.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/drivers/pinctrl/bcm/pinctrl-bcm2835.c index 1489191a213fe..5d2b188a1ef42 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm2835.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm2835.c @@ -1003,8 +1003,27 @@ static const struct pinmux_ops bcm2835_pmx_ops = { static int bcm2835_pinconf_get(struct pinctrl_dev *pctldev, unsigned pin, unsigned long *config) { - /* No way to read back config in HW */ - return -ENOTSUPP; + enum pin_config_param param = pinconf_to_config_param(*config); + struct bcm2835_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev); + enum bcm2835_fsel fsel = bcm2835_pinctrl_fsel_get(pc, pin); + u32 val; + + /* No way to read back bias config in HW */ + + switch (param) { + case PIN_CONFIG_OUTPUT: + if (fsel != BCM2835_FSEL_GPIO_OUT) + return -EINVAL; + + val = bcm2835_gpio_get_bit(pc, GPLEV0, pin); + *config = pinconf_to_config_packed(param, val); + break; + + default: + return -ENOTSUPP; + } + + return 0; } static void bcm2835_pull_config_set(struct bcm2835_pinctrl *pc, From d54e4cda297241406bed37af3f836d242184ec84 Mon Sep 17 00:00:00 2001 From: Stefan Wahren Date: Thu, 7 Mar 2024 08:01:13 +0100 Subject: [PATCH 0024/1702] pinctrl: bcm2835: Implement bcm2711_pinconf_get The BCM2711 allows to read the bias config. So implement pin_conf_get accordingly. The pull resistor values has been taken from the BCM2711/7211 datasheet. This implementation assumes that BCM7211 behaves the same way. Signed-off-by: Stefan Wahren Reviewed-by: Florian Fainelli Tested-by: Florian Fainelli Message-ID: <20240307070113.4888-3-wahrenst@gmx.net> Signed-off-by: Linus Walleij --- drivers/pinctrl/bcm/pinctrl-bcm2835.c | 41 ++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/drivers/pinctrl/bcm/pinctrl-bcm2835.c index 5d2b188a1ef42..f5a9372d43bde 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm2835.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm2835.c @@ -1098,6 +1098,45 @@ static const struct pinconf_ops bcm2835_pinconf_ops = { .pin_config_set = bcm2835_pinconf_set, }; +static int bcm2711_pinconf_get(struct pinctrl_dev *pctldev, unsigned pin, + unsigned long *config) +{ + struct bcm2835_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev); + enum pin_config_param param = pinconf_to_config_param(*config); + u32 offset, shift, val; + + offset = PUD_2711_REG_OFFSET(pin); + shift = PUD_2711_REG_SHIFT(pin); + val = bcm2835_gpio_rd(pc, GP_GPIO_PUP_PDN_CNTRL_REG0 + (offset * 4)); + + switch (param) { + case PIN_CONFIG_BIAS_DISABLE: + if (((val >> shift) & PUD_2711_MASK) != BCM2711_PULL_NONE) + return -EINVAL; + + break; + + case PIN_CONFIG_BIAS_PULL_UP: + if (((val >> shift) & PUD_2711_MASK) != BCM2711_PULL_UP) + return -EINVAL; + + *config = pinconf_to_config_packed(param, 50000); + break; + + case PIN_CONFIG_BIAS_PULL_DOWN: + if (((val >> shift) & PUD_2711_MASK) != BCM2711_PULL_DOWN) + return -EINVAL; + + *config = pinconf_to_config_packed(param, 50000); + break; + + default: + return bcm2835_pinconf_get(pctldev, pin, config); + } + + return 0; +} + static void bcm2711_pull_config_set(struct bcm2835_pinctrl *pc, unsigned int pin, unsigned int arg) { @@ -1165,7 +1204,7 @@ static int bcm2711_pinconf_set(struct pinctrl_dev *pctldev, static const struct pinconf_ops bcm2711_pinconf_ops = { .is_generic = true, - .pin_config_get = bcm2835_pinconf_get, + .pin_config_get = bcm2711_pinconf_get, .pin_config_set = bcm2711_pinconf_set, }; From 842ecb5fcf8de17dfde11d4ec5f41930f0076887 Mon Sep 17 00:00:00 2001 From: Tengfei Fan Date: Tue, 12 Mar 2024 10:58:07 +0800 Subject: [PATCH 0025/1702] dt-bindings: pinctrl: qcom: update functions to match with driver Some functions were consolidated in the SM4450 pinctrl driver, but they had not been updated in the binding file before the previous SM4450 pinctrl patch series was merged. Update functions in this binding file to match with SM4450 pinctrl driver. Some functions need to be consolidated and some functions need to be removed. The following functions are removed: - atest_char0, atest_char1, atest_char2, atest_char3 - atest_usb00, atest_usb01, atest_usb02, atest_usb03 - audio_ref - cci_async - cci_timer0, cci_timer1, cci_timer2, cci_timer3, cci_timer4 - cmu_rng0, cmu_rng1, cmu_rng2, cmu_rng3 - coex_uart1 - cri_trng0, cri_trng1 - dbg_out - ddr_pxi0, ddr_pxi1 - dp0_hot - gcc_gp1, gcc_gp2, gcc_gp3 - ibi_i3c - jitter_bist - mdp_vsync0, mdp_vsync1, mdp_vsync2, mdp_vsync3 - mi2s0_data0, mi2s0_data1, mi2s0_sck, mi2s0_ws, mi2s2_data0, mi2s2_data1, mi2s2_sck, mi2s2_ws, mi2s_mclk0, mi2s_mclk1 - nav_gpio0, nav_gpio1, nav_gpio2 - phase_flag0, phase_flag1, phase_flag10, phase_flag11, phase_flag12, phase_flag13, phase_flag14, phase_flag15, phase_flag16, phase_flag17, phase_flag18, phase_flag19, phase_flag2, phase_flag20, phase_flag21, phase_flag22, phase_flag23, phase_flag24, phase_flag25, phase_flag26, phase_flag27, phase_flag28, phase_flag29, phase_flag3, phase_flag30, phase_flag31, phase_flag4, phase_flag5, phase_flag6, phase_flag7, phase_flag8, phase_flag9 - pll_bist, pll_clk - prng_rosc0, prng_rosc1, prng_rosc2, prng_rosc3 - qdss_gpio0, qdss_gpio1, qdss_gpio10, qdss_gpio11, qdss_gpio12, qdss_gpio13, qdss_gpio14, qdss_gpio15, qdss_gpio2, qdss_gpio3, qdss_gpio4, qdss_gpio5, qdss_gpio6, qdss_gpio7, qdss_gpio8, qdss_gpio9 - qlink0_wmss - qup0_se5, qup0_se6, qup0_se7, qup1_se5, qup1_se6 - sd_write - tb_trig - tgu_ch0, tgu_ch1, tgu_ch2, tgu_ch3 - tmess_prng0, tmess_prng1, tmess_prng2, tmess_prng3 - tsense_pwm1, tsense_pwm2 - uim0_clk, uim0_data, uim0_present, uim0_reset, uim1_clk, uim1_data, uim1_present, uim1_reset - usb0_hs, usb0_phy - vsense_trigger The following functions are added: - atest_char - atest_usb0 - audio_ref_clk - cci - cci_async_in0 - cmu_rng - coex_uart1_rx, coex_uart1_tx - dbg_out_clk - ddr_pxi0_test, ddr_pxi1_test - gcc_gp1_clk, gcc_gp2_clk, gcc_gp3_clk - ibi_i3c_qup0, ibi_i3c_qup1 - jitter_bist_ref - mdp_vsync - nav - phase_flag - pll_bist_sync, pll_clk_aux - prng_rosc - qlink0_wmss_reset - sd_write_protect - tb_trig_sdc1, tb_trig_sdc2 - tgu_ch0_trigout, tgu_ch1_trigout, tgu_ch2_trigout, tgu_ch3_trigout - tmess_prng - tsense_pwm1_out, tsense_pwm2_out - uim0, uim1 - usb0_hs_ac, usb0_phy_ps - vsense_trigger_mirnat - wlan1_adc_dtest0, wlan1_adc_dtest1 Fixes: 7bf8b78f86db ("dt-bindings: pinctrl: qcom: Add SM4450 pinctrl") Signed-off-by: Tengfei Fan Acked-by: Krzysztof Kozlowski Message-ID: <20240312025807.26075-3-quic_tengfan@quicinc.com> Signed-off-by: Linus Walleij --- .../bindings/pinctrl/qcom,sm4450-tlmm.yaml | 52 +++++++------------ 1 file changed, 18 insertions(+), 34 deletions(-) diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm4450-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm4450-tlmm.yaml index bb675c8ec220f..1b941b276b3f8 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,sm4450-tlmm.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm4450-tlmm.yaml @@ -72,40 +72,24 @@ $defs: description: Specify the alternative function to be configured for the specified pins. - enum: [ gpio, atest_char, atest_char0, atest_char1, atest_char2, - atest_char3, atest_usb0, atest_usb00, atest_usb01, atest_usb02, - atest_usb03, audio_ref, cam_mclk, cci_async, cci_i2c, - cci_timer0, cci_timer1, cci_timer2, cci_timer3, cci_timer4, - cmu_rng0, cmu_rng1, cmu_rng2, cmu_rng3, coex_uart1, cri_trng, - cri_trng0, cri_trng1, dbg_out, ddr_bist, ddr_pxi0, ddr_pxi1, - dp0_hot, gcc_gp1, gcc_gp2, gcc_gp3, host2wlan_sol, ibi_i3c, - jitter_bist, mdp_vsync, mdp_vsync0, mdp_vsync1, mdp_vsync2, - mdp_vsync3, mi2s0_data0, mi2s0_data1, mi2s0_sck, mi2s0_ws, - mi2s2_data0, mi2s2_data1, mi2s2_sck, mi2s2_ws, mi2s_mclk0, - mi2s_mclk1, nav_gpio0, nav_gpio1, nav_gpio2, pcie0_clk, - phase_flag0, phase_flag1, phase_flag10, phase_flag11, - phase_flag12, phase_flag13, phase_flag14, phase_flag15, - phase_flag16, phase_flag17, phase_flag18, phase_flag19, - phase_flag2, phase_flag20, phase_flag21, phase_flag22, - phase_flag23, phase_flag24, phase_flag25, phase_flag26, - phase_flag27, phase_flag28, phase_flag29, phase_flag3, - phase_flag30, phase_flag31, phase_flag4, phase_flag5, - phase_flag6, phase_flag7, phase_flag8, phase_flag9, - pll_bist, pll_clk, prng_rosc0, prng_rosc1, prng_rosc2, - prng_rosc3, qdss_cti, qdss_gpio, qdss_gpio0, qdss_gpio1, - qdss_gpio10, qdss_gpio11, qdss_gpio12, qdss_gpio13, qdss_gpio14, - qdss_gpio15, qdss_gpio2, qdss_gpio3, qdss_gpio4, qdss_gpio5, - qdss_gpio6, qdss_gpio7, qdss_gpio8, qdss_gpio9, qlink0_enable, - qlink0_request, qlink0_wmss, qlink1_enable, qlink1_request, - qlink1_wmss, qlink2_enable, qlink2_request, qlink2_wmss, - qup0_se0, qup0_se1, qup0_se2, qup0_se3, qup0_se4, qup0_se5, - qup0_se6, qup0_se7, qup1_se0, qup1_se1, qup1_se2, qup1_se3, - qup1_se4, qup1_se5, qup1_se6, sd_write, tb_trig, tgu_ch0, - tgu_ch1, tgu_ch2, tgu_ch3, tmess_prng0, tmess_prng1, - tmess_prng2, tmess_prng3, tsense_pwm1, tsense_pwm2, uim0_clk, - uim0_data, uim0_present, uim0_reset, uim1_clk, uim1_data, - uim1_present, uim1_reset, usb0_hs, usb0_phy, vfr_0, vfr_1, - vsense_trigger ] + enum: [ gpio, atest_char, atest_usb0, audio_ref_clk, cam_mclk, + cci_async_in0, cci_i2c, cci, cmu_rng, coex_uart1_rx, + coex_uart1_tx, cri_trng, dbg_out_clk, ddr_bist, + ddr_pxi0_test, ddr_pxi1_test, gcc_gp1_clk, gcc_gp2_clk, + gcc_gp3_clk, host2wlan_sol, ibi_i3c_qup0, ibi_i3c_qup1, + jitter_bist_ref, mdp_vsync0_out, mdp_vsync1_out, + mdp_vsync2_out, mdp_vsync3_out, mdp_vsync, nav, + pcie0_clk_req, phase_flag, pll_bist_sync, pll_clk_aux, + prng_rosc, qdss_cti_trig0, qdss_cti_trig1, qdss_gpio, + qlink0_enable, qlink0_request, qlink0_wmss_reset, + qup0_se0, qup0_se1, qup0_se2, qup0_se3, qup0_se4, + qup1_se0, qup1_se1, qup1_se2, qup1_se2_l2, qup1_se3, + qup1_se4, sd_write_protect, tb_trig_sdc1, tb_trig_sdc2, + tgu_ch0_trigout, tgu_ch1_trigout, tgu_ch2_trigout, + tgu_ch3_trigout, tmess_prng, tsense_pwm1_out, + tsense_pwm2_out, uim0, uim1, usb0_hs_ac, usb0_phy_ps, + vfr_0_mira, vfr_0_mirb, vfr_1, vsense_trigger_mirnat, + wlan1_adc_dtest0, wlan1_adc_dtest1 ] required: - pins From ae6f9707d704c6a919caa0aa959970003e1391a0 Mon Sep 17 00:00:00 2001 From: David Collins Date: Tue, 26 Mar 2024 15:06:27 -0700 Subject: [PATCH 0026/1702] dt-bindings: pinctrl: qcom,pmic-gpio: Add PMXR2230 and PM6450 support Update the Qualcomm Technologies, Inc. PMIC GPIO binding documentation to include compatible strings for PMXR2230 and PM6450 PMICs. Signed-off-by: David Collins Signed-off-by: Anjelique Melendez Acked-by: Krzystof Kozlowski Acked-by: Krzysztof Kozlowski Message-ID: <20240326220628.2392802-2-quic_amelende@quicinc.com> Signed-off-by: Linus Walleij --- .../devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml index 3f8ad07c7cfdc..2b17d244f0511 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml @@ -24,6 +24,7 @@ properties: - qcom,pm6150-gpio - qcom,pm6150l-gpio - qcom,pm6350-gpio + - qcom,pm6450-gpio - qcom,pm7250b-gpio - qcom,pm7325-gpio - qcom,pm7550ba-gpio @@ -72,6 +73,7 @@ properties: - qcom,pmx55-gpio - qcom,pmx65-gpio - qcom,pmx75-gpio + - qcom,pmxr2230-gpio - enum: - qcom,spmi-gpio @@ -198,6 +200,7 @@ allOf: contains: enum: - qcom,pm6350-gpio + - qcom,pm6450-gpio - qcom,pm8350c-gpio then: properties: @@ -261,6 +264,7 @@ allOf: - qcom,pmc8180c-gpio - qcom,pmp8074-gpio - qcom,pms405-gpio + - qcom,pmxr2230-gpio then: properties: gpio-line-names: @@ -417,6 +421,7 @@ $defs: - gpio1-gpio10 for pm6150 - gpio1-gpio12 for pm6150l - gpio1-gpio9 for pm6350 + - gpio1-gpio9 for pm6450 - gpio1-gpio12 for pm7250b - gpio1-gpio10 for pm7325 - gpio1-gpio8 for pm7550ba @@ -464,6 +469,7 @@ $defs: and gpio11) - gpio1-gpio16 for pmx65 - gpio1-gpio16 for pmx75 + - gpio1-gpio12 for pmxr2230 items: pattern: "^gpio([0-9]+)$" From 6acc46f8c065f5282be8577db696d098442f808f Mon Sep 17 00:00:00 2001 From: Anjelique Melendez Date: Tue, 26 Mar 2024 15:06:28 -0700 Subject: [PATCH 0027/1702] dt-bindings: pinctrl: qcom,pmic-gpio: Add PMIH0108 and PMD8028 support Update the Qualcomm Technologies, Inc. PMIC GPIO binding documentation to include compatible strings for PMIH0108 and PMD8028 PMICs. Signed-off-by: Anjelique Melendez Reviewed-by: Krzysztof Kozlowski Message-ID: <20240326220628.2392802-3-quic_amelende@quicinc.com> Signed-off-by: Linus Walleij --- .../bindings/pinctrl/qcom,pmic-gpio.yaml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml index 2b17d244f0511..a786357ed1afd 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml @@ -57,10 +57,12 @@ properties: - qcom,pma8084-gpio - qcom,pmc8180-gpio - qcom,pmc8180c-gpio + - qcom,pmd8028-gpio - qcom,pmi632-gpio - qcom,pmi8950-gpio - qcom,pmi8994-gpio - qcom,pmi8998-gpio + - qcom,pmih0108-gpio - qcom,pmk8350-gpio - qcom,pmk8550-gpio - qcom,pmm8155au-gpio @@ -143,6 +145,7 @@ allOf: - qcom,pm8005-gpio - qcom,pm8450-gpio - qcom,pm8916-gpio + - qcom,pmd8028-gpio - qcom,pmk8350-gpio - qcom,pmr735a-gpio - qcom,pmr735b-gpio @@ -304,6 +307,21 @@ allOf: minItems: 1 maxItems: 7 + - if: + properties: + comptaible: + contains: + enum: + - qcom,pmih0108-gpio + then: + properties: + gpio-line-names: + minItems: 18 + maxItems: 18 + gpio-reserved-ranges: + minItems: 1 + maxItems: 9 + - if: properties: compatible: @@ -452,9 +470,11 @@ $defs: - gpio1-gpio22 for pm8994 - gpio1-gpio26 for pm8998 - gpio1-gpio22 for pma8084 + - gpio1-gpio4 for pmd8028 - gpio1-gpio8 for pmi632 - gpio1-gpio2 for pmi8950 - gpio1-gpio10 for pmi8994 + - gpio1-gpio18 for pmih0108 - gpio1-gpio4 for pmk8350 - gpio1-gpio6 for pmk8550 - gpio1-gpio10 for pmm8155au From e5c7b013cbcc7a4d91333811bd8524112f9c22a5 Mon Sep 17 00:00:00 2001 From: Anjelique Melendez Date: Tue, 26 Mar 2024 15:06:29 -0700 Subject: [PATCH 0028/1702] pinctrl: qcom: spmi-gpio: Add PMXR2230 and PM6450 support Add support for qcom,pmxr2230-gpio and qcom,pm6450-gpio. Signed-off-by: Anjelique Melendez Reviewed-by: Konrad Dybcio Message-ID: <20240326220628.2392802-4-quic_amelende@quicinc.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/qcom/pinctrl-spmi-gpio.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c index f4e2c88a7c822..54ffb7e1189a6 100644 --- a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c +++ b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c @@ -1202,6 +1202,7 @@ static const struct of_device_id pmic_gpio_of_match[] = { { .compatible = "qcom,pm6150-gpio", .data = (void *) 10 }, { .compatible = "qcom,pm6150l-gpio", .data = (void *) 12 }, { .compatible = "qcom,pm6350-gpio", .data = (void *) 9 }, + { .compatible = "qcom,pm6450-gpio", .data = (void *) 9 }, { .compatible = "qcom,pm7250b-gpio", .data = (void *) 12 }, { .compatible = "qcom,pm7325-gpio", .data = (void *) 10 }, { .compatible = "qcom,pm7550ba-gpio", .data = (void *) 8}, @@ -1253,6 +1254,7 @@ static const struct of_device_id pmic_gpio_of_match[] = { { .compatible = "qcom,pmx55-gpio", .data = (void *) 11 }, { .compatible = "qcom,pmx65-gpio", .data = (void *) 16 }, { .compatible = "qcom,pmx75-gpio", .data = (void *) 16 }, + { .compatible = "qcom,pmxr2230-gpio", .data = (void *) 12 }, { }, }; From b5658c7ab942cc9c49a164dd8e027096b3e3c0f4 Mon Sep 17 00:00:00 2001 From: Anjelique Melendez Date: Tue, 26 Mar 2024 15:06:30 -0700 Subject: [PATCH 0029/1702] pinctrl: qcom: spmi-gpio: Add PMIH0108 and PMD8028 support Add support for qcom,pmih0108-gpio and qcom,pmd8028-gpio. Signed-off-by: Anjelique Melendez Reviewed-by: Konrad Dybcio Message-ID: <20240326220628.2392802-5-quic_amelende@quicinc.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/qcom/pinctrl-spmi-gpio.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c index 54ffb7e1189a6..4e80c7204e5fb 100644 --- a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c +++ b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c @@ -1235,10 +1235,12 @@ static const struct of_device_id pmic_gpio_of_match[] = { { .compatible = "qcom,pm8994-gpio", .data = (void *) 22 }, { .compatible = "qcom,pm8998-gpio", .data = (void *) 26 }, { .compatible = "qcom,pma8084-gpio", .data = (void *) 22 }, + { .compatible = "qcom,pmd8028-gpio", .data = (void *) 4 }, { .compatible = "qcom,pmi632-gpio", .data = (void *) 8 }, { .compatible = "qcom,pmi8950-gpio", .data = (void *) 2 }, { .compatible = "qcom,pmi8994-gpio", .data = (void *) 10 }, { .compatible = "qcom,pmi8998-gpio", .data = (void *) 14 }, + { .compatible = "qcom,pmih0108-gpio", .data = (void *) 18 }, { .compatible = "qcom,pmk8350-gpio", .data = (void *) 4 }, { .compatible = "qcom,pmk8550-gpio", .data = (void *) 6 }, { .compatible = "qcom,pmm8155au-gpio", .data = (void *) 10 }, From 85dfe458376c01282261bb3555f623f2afb1b203 Mon Sep 17 00:00:00 2001 From: Thomas Richard Date: Mon, 4 Mar 2024 16:35:45 +0100 Subject: [PATCH 0030/1702] pinctrl: pinctrl-single: move suspend()/resume() callbacks to noirq The goal is to extend the active period of pinctrl. Some devices may need active pinctrl after suspend() and/or before resume(). So move suspend()/resume() to suspend_noirq()/resume_noirq() in order to have active pinctrl until suspend_noirq() (included), and from resume_noirq() (included). The deprecated API has been removed to use the new one (dev_pm_ops struct). No need to check the pointer returned by dev_get_drvdata(), as platform_set_drvdata() is called during the probe. Reviewed-by: Linus Walleij Reviewed-by: Andy Shevchenko Signed-off-by: Thomas Richard Reviewed-by: Tony Lindgren Reviewed-by: Dhruva Gole Message-ID: <20240102-j7200-pcie-s2r-v4-2-6f1f53390c85@bootlin.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-single.c | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c index 19cc0db771a5a..0dd4b0e11adf7 100644 --- a/drivers/pinctrl/pinctrl-single.c +++ b/drivers/pinctrl/pinctrl-single.c @@ -1625,7 +1625,6 @@ static int pcs_irq_init_chained_handler(struct pcs_device *pcs, return 0; } -#ifdef CONFIG_PM static int pcs_save_context(struct pcs_device *pcs) { int i, mux_bytes; @@ -1690,14 +1689,9 @@ static void pcs_restore_context(struct pcs_device *pcs) } } -static int pinctrl_single_suspend(struct platform_device *pdev, - pm_message_t state) +static int pinctrl_single_suspend_noirq(struct device *dev) { - struct pcs_device *pcs; - - pcs = platform_get_drvdata(pdev); - if (!pcs) - return -EINVAL; + struct pcs_device *pcs = dev_get_drvdata(dev); if (pcs->flags & PCS_CONTEXT_LOSS_OFF) { int ret; @@ -1710,20 +1704,19 @@ static int pinctrl_single_suspend(struct platform_device *pdev, return pinctrl_force_sleep(pcs->pctl); } -static int pinctrl_single_resume(struct platform_device *pdev) +static int pinctrl_single_resume_noirq(struct device *dev) { - struct pcs_device *pcs; - - pcs = platform_get_drvdata(pdev); - if (!pcs) - return -EINVAL; + struct pcs_device *pcs = dev_get_drvdata(dev); if (pcs->flags & PCS_CONTEXT_LOSS_OFF) pcs_restore_context(pcs); return pinctrl_force_default(pcs->pctl); } -#endif + +static DEFINE_NOIRQ_DEV_PM_OPS(pinctrl_single_pm_ops, + pinctrl_single_suspend_noirq, + pinctrl_single_resume_noirq); /** * pcs_quirk_missing_pinctrl_cells - handle legacy binding @@ -1986,11 +1979,8 @@ static struct platform_driver pcs_driver = { .driver = { .name = DRIVER_NAME, .of_match_table = pcs_of_match, + .pm = pm_sleep_ptr(&pinctrl_single_pm_ops), }, -#ifdef CONFIG_PM - .suspend = pinctrl_single_suspend, - .resume = pinctrl_single_resume, -#endif }; module_platform_driver(pcs_driver); From b6e2c6548074d77e15af8fb6a5a7d03578f04b2f Mon Sep 17 00:00:00 2001 From: Dmitry Rokosov Date: Wed, 20 Mar 2024 18:54:45 +0300 Subject: [PATCH 0031/1702] clk: meson: a1: peripherals: determine maximum register in regmap config When the max_register value is not set, the regmap debugfs 'registers' file does not display the entire range of the regmap. Signed-off-by: Dmitry Rokosov Link: https://lore.kernel.org/r/20240320155512.3544-2-ddrokosov@salutedevices.com Signed-off-by: Jerome Brunet --- drivers/clk/meson/a1-peripherals.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/clk/meson/a1-peripherals.c b/drivers/clk/meson/a1-peripherals.c index e2a1f12f9175c..621af1e6e4b2b 100644 --- a/drivers/clk/meson/a1-peripherals.c +++ b/drivers/clk/meson/a1-peripherals.c @@ -2187,6 +2187,7 @@ static struct regmap_config a1_periphs_regmap_cfg = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .max_register = DMC_CLK_CTRL, }; static struct meson_clk_hw_data a1_periphs_clks = { From acc628adc363d441a4bbd05d92ed30253db006ff Mon Sep 17 00:00:00 2001 From: Dmitry Rokosov Date: Wed, 20 Mar 2024 18:54:46 +0300 Subject: [PATCH 0032/1702] clk: meson: a1: pll: determine maximum register in regmap config When the max_register value is not set, the regmap debugfs 'registers' file does not display the entire range of the regmap. Signed-off-by: Dmitry Rokosov Link: https://lore.kernel.org/r/20240320155512.3544-3-ddrokosov@salutedevices.com Signed-off-by: Jerome Brunet --- drivers/clk/meson/a1-pll.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/clk/meson/a1-pll.c b/drivers/clk/meson/a1-pll.c index 4325e8a6a3ef5..90b0aeeb049c6 100644 --- a/drivers/clk/meson/a1-pll.c +++ b/drivers/clk/meson/a1-pll.c @@ -299,6 +299,7 @@ static struct regmap_config a1_pll_regmap_cfg = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .max_register = ANACTRL_HIFIPLL_STS, }; static struct meson_clk_hw_data a1_pll_clks = { From 32fba1c16576324f7783d7c27b03689fa7bd9998 Mon Sep 17 00:00:00 2001 From: Dmitry Rokosov Date: Wed, 20 Mar 2024 18:54:47 +0300 Subject: [PATCH 0033/1702] clk: meson: s4: peripherals: determine maximum register in regmap config When the max_register value is not set, the regmap debugfs 'registers' file does not display the entire range of the regmap. Signed-off-by: Dmitry Rokosov Link: https://lore.kernel.org/r/20240320155512.3544-4-ddrokosov@salutedevices.com Signed-off-by: Jerome Brunet --- drivers/clk/meson/s4-peripherals.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/clk/meson/s4-peripherals.c b/drivers/clk/meson/s4-peripherals.c index 6c35de3d536fd..1fceb93faf13d 100644 --- a/drivers/clk/meson/s4-peripherals.c +++ b/drivers/clk/meson/s4-peripherals.c @@ -3751,6 +3751,7 @@ static struct regmap_config clkc_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .max_register = CLKCTRL_DEMOD_CLK_CTRL, }; static struct meson_clk_hw_data s4_periphs_clks = { From 5995a2f26f83f09d128a56a51df6d5631a274040 Mon Sep 17 00:00:00 2001 From: Dmitry Rokosov Date: Wed, 20 Mar 2024 18:54:48 +0300 Subject: [PATCH 0034/1702] clk: meson: s4: pll: determine maximum register in regmap config When the max_register value is not set, the regmap debugfs 'registers' file does not display the entire range of the regmap. Signed-off-by: Dmitry Rokosov Link: https://lore.kernel.org/r/20240320155512.3544-5-ddrokosov@salutedevices.com Signed-off-by: Jerome Brunet --- drivers/clk/meson/s4-pll.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/clk/meson/s4-pll.c b/drivers/clk/meson/s4-pll.c index 8dfaeccaadc22..c8a95842b14c6 100644 --- a/drivers/clk/meson/s4-pll.c +++ b/drivers/clk/meson/s4-pll.c @@ -798,6 +798,7 @@ static struct regmap_config clkc_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .max_register = ANACTRL_HDMIPLL_CTRL0, }; static struct meson_clk_hw_data s4_pll_clks = { From 16182ac30a68204aeb9fb373f5cb53f995251e85 Mon Sep 17 00:00:00 2001 From: Dmitry Rokosov Date: Thu, 28 Mar 2024 22:57:29 +0300 Subject: [PATCH 0035/1702] clk: meson: pll: print out pll name when unable to lock it In most meson systems, multiple PLLs are present, making it difficult to identify the specific PLL that fails to lock. To address this issue, print out the name of the PLL that cannot be locked. Signed-off-by: Dmitry Rokosov Link: https://lore.kernel.org/r/20240328195733.30572-1-ddrokosov@salutedevices.com Signed-off-by: Jerome Brunet --- drivers/clk/meson/clk-pll.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/clk/meson/clk-pll.c b/drivers/clk/meson/clk-pll.c index 6fa7639a30503..78d17b2415afc 100644 --- a/drivers/clk/meson/clk-pll.c +++ b/drivers/clk/meson/clk-pll.c @@ -436,8 +436,8 @@ static int meson_clk_pll_set_rate(struct clk_hw *hw, unsigned long rate, ret = meson_clk_pll_enable(hw); if (ret) { - pr_warn("%s: pll did not lock, trying to restore old rate %lu\n", - __func__, old_rate); + pr_warn("%s: pll %s didn't lock, trying to set old rate %lu\n", + __func__, clk_hw_get_name(hw), old_rate); /* * FIXME: Do we really need/want this HACK ? * It looks unsafe. what happens if the clock gets into a From 91b623cda43e449a49177ba99b6723f551a4bfbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Fri, 29 Mar 2024 09:18:29 +0100 Subject: [PATCH 0036/1702] power: supply: core: simplify charge_behaviour formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function power_supply_show_charge_behaviour() is not needed and can be removed completely. Removing the function also saves a spurious read of the property from the driver on each call. The convulted logic was a leftover from an earlier patch revision. Some restructuring made this cleanup possible. Suggested-by: Hans de Goede Link: https://lore.kernel.org/all/9e035ae4-cb07-4f84-8336-1a0050855bea@redhat.com/ Fixes: 4e61f1e9d58f ("power: supply: core: fix charge_behaviour formatting") Signed-off-by: Thomas Weißschuh Reviewed-by: Hans de Goede Link: https://lore.kernel.org/r/20240329-power-supply-simplify-v1-1-416f1002739f@weissschuh.net Signed-off-by: Sebastian Reichel --- drivers/power/supply/power_supply_sysfs.c | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/drivers/power/supply/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c index 0d2c3724d0bc0..b86e11bdc07ef 100644 --- a/drivers/power/supply/power_supply_sysfs.c +++ b/drivers/power/supply/power_supply_sysfs.c @@ -271,23 +271,6 @@ static ssize_t power_supply_show_usb_type(struct device *dev, return count; } -static ssize_t power_supply_show_charge_behaviour(struct device *dev, - struct power_supply *psy, - union power_supply_propval *value, - char *buf) -{ - int ret; - - ret = power_supply_get_property(psy, - POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, - value); - if (ret < 0) - return ret; - - return power_supply_charge_behaviour_show(dev, psy->desc->charge_behaviours, - value->intval, buf); -} - static ssize_t power_supply_show_property(struct device *dev, struct device_attribute *attr, char *buf) { @@ -321,7 +304,8 @@ static ssize_t power_supply_show_property(struct device *dev, &value, buf); break; case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: - ret = power_supply_show_charge_behaviour(dev, psy, &value, buf); + ret = power_supply_charge_behaviour_show(dev, psy->desc->charge_behaviours, + value.intval, buf); break; case POWER_SUPPLY_PROP_MODEL_NAME ... POWER_SUPPLY_PROP_SERIAL_NUMBER: ret = sysfs_emit(buf, "%s\n", value.strval); From 33e62cd7b4c281cd737c62e5d8c4f0e602a8c5c5 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Wed, 27 Mar 2024 15:42:23 +0800 Subject: [PATCH 0037/1702] f2fs: multidev: fix to recognize valid zero block address As reported by Yi Zhang in mailing list [1], kernel warning was catched during zbd/010 test as below: ./check zbd/010 zbd/010 (test gap zone support with F2FS) [failed] runtime ... 3.752s something found in dmesg: [ 4378.146781] run blktests zbd/010 at 2024-02-18 11:31:13 [ 4378.192349] null_blk: module loaded [ 4378.209860] null_blk: disk nullb0 created [ 4378.413285] scsi_debug:sdebug_driver_probe: scsi_debug: trim poll_queues to 0. poll_q/nr_hw = (0/1) [ 4378.422334] scsi host15: scsi_debug: version 0191 [20210520] dev_size_mb=1024, opts=0x0, submit_queues=1, statistics=0 [ 4378.434922] scsi 15:0:0:0: Direct-Access-ZBC Linux scsi_debug 0191 PQ: 0 ANSI: 7 [ 4378.443343] scsi 15:0:0:0: Power-on or device reset occurred [ 4378.449371] sd 15:0:0:0: Attached scsi generic sg5 type 20 [ 4378.449418] sd 15:0:0:0: [sdf] Host-managed zoned block device ... (See '/mnt/tests/gitlab.com/api/v4/projects/19168116/repository/archive.zip/storage/blktests/blk/blktests/results/nodev/zbd/010.dmesg' WARNING: CPU: 22 PID: 44011 at fs/iomap/iter.c:51 CPU: 22 PID: 44011 Comm: fio Not tainted 6.8.0-rc3+ #1 RIP: 0010:iomap_iter+0x32b/0x350 Call Trace: __iomap_dio_rw+0x1df/0x830 f2fs_file_read_iter+0x156/0x3d0 [f2fs] aio_read+0x138/0x210 io_submit_one+0x188/0x8c0 __x64_sys_io_submit+0x8c/0x1a0 do_syscall_64+0x86/0x170 entry_SYSCALL_64_after_hwframe+0x6e/0x76 Shinichiro Kawasaki helps to analyse this issue and proposes a potential fixing patch in [2]. Quoted from reply of Shinichiro Kawasaki: "I confirmed that the trigger commit is dbf8e63f48af as Yi reported. I took a look in the commit, but it looks fine to me. So I thought the cause is not in the commit diff. I found the WARN is printed when the f2fs is set up with multiple devices, and read requests are mapped to the very first block of the second device in the direct read path. In this case, f2fs_map_blocks() and f2fs_map_blocks_cached() modify map->m_pblk as the physical block address from each block device. It becomes zero when it is mapped to the first block of the device. However, f2fs_iomap_begin() assumes that map->m_pblk is the physical block address of the whole f2fs, across the all block devices. It compares map->m_pblk against NULL_ADDR == 0, then go into the unexpected branch and sets the invalid iomap->length. The WARN catches the invalid iomap->length. This WARN is printed even for non-zoned block devices, by following steps. - Create two (non-zoned) null_blk devices memory backed with 128MB size each: nullb0 and nullb1. # mkfs.f2fs /dev/nullb0 -c /dev/nullb1 # mount -t f2fs /dev/nullb0 "${mount_dir}" # dd if=/dev/zero of="${mount_dir}/test.dat" bs=1M count=192 # dd if="${mount_dir}/test.dat" of=/dev/null bs=1M count=192 iflag=direct ..." So, the root cause of this issue is: when multi-devices feature is on, f2fs_map_blocks() may return zero blkaddr in non-primary device, which is a verified valid block address, however, f2fs_iomap_begin() treats it as an invalid block address, and then it triggers the warning in iomap framework code. Finally, as discussed, we decide to use a more simple and direct way that checking (map.m_flags & F2FS_MAP_MAPPED) condition instead of (map.m_pblk != NULL_ADDR) to fix this issue. Thanks a lot for the effort of Yi Zhang and Shinichiro Kawasaki on this issue. [1] https://lore.kernel.org/linux-f2fs-devel/CAHj4cs-kfojYC9i0G73PRkYzcxCTex=-vugRFeP40g_URGvnfQ@mail.gmail.com/ [2] https://lore.kernel.org/linux-f2fs-devel/gngdj77k4picagsfdtiaa7gpgnup6fsgwzsltx6milmhegmjff@iax2n4wvrqye/ Reported-by: Yi Zhang Closes: https://lore.kernel.org/linux-f2fs-devel/CAHj4cs-kfojYC9i0G73PRkYzcxCTex=-vugRFeP40g_URGvnfQ@mail.gmail.com/ Tested-by: Shin'ichiro Kawasaki Tested-by: Yi Zhang Fixes: 1517c1a7a445 ("f2fs: implement iomap operations") Fixes: 8d3c1fa3fa5e ("f2fs: don't rely on F2FS_MAP_* in f2fs_iomap_begin") Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 08815394223a3..fa5398ac45052 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -4196,7 +4196,7 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, if (WARN_ON_ONCE(map.m_pblk == COMPRESS_ADDR)) return -EINVAL; - if (map.m_pblk != NULL_ADDR) { + if (map.m_flags & F2FS_MAP_MAPPED) { iomap->length = blks_to_bytes(inode, map.m_len); iomap->type = IOMAP_MAPPED; iomap->flags |= IOMAP_F_MERGED; From 9f0f6bf4271488b3d3a290ba119a0e0a08df2cc6 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Mon, 25 Mar 2024 23:27:26 +0800 Subject: [PATCH 0038/1702] f2fs: support to map continuous holes or preallocated address This patch supports to map continuous holes or preallocated addresses to improve performace of lookuping mapping info during read DIO. [testcase 1] xfs_io -f /mnt/f2fs/hole -c "truncate 1m" -c "fsync" xfs_io -d /mnt/f2fs/hole -c "pread -b 1m 0 1m" [before] f2fs_direct_IO_enter: dev = (253,16), ino = 6 pos = 0 len = 1048576 ki_flags = 20000 ki_ioprio = 0 rw = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 0, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 1, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 2, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 3, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 4, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 5, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 6, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 7, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 8, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 9, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 10, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 11, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 12, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 13, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 14, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 15, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 16, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 ...... f2fs_direct_IO_exit: dev = (253,16), ino = 6 pos = 0 len = 1048576 rw = 0 ret = 1048576 [after] f2fs_direct_IO_enter: dev = (253,16), ino = 6 pos = 0 len = 1048576 ki_flags = 20000 ki_ioprio = 0 rw = 0 f2fs_map_blocks: dev = (253,16), ino = 6, file offset = 0, start blkaddr = 0x0, len = 0x100, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_direct_IO_exit: dev = (253,16), ino = 6 pos = 0 len = 1048576 rw = 0 ret = 1048576 [testcase 2] xfs_io -f /mnt/f2fs/preallocated -c "falloc 0 1m" -c "fsync" xfs_io -d /mnt/f2fs/preallocated -c "pread -b 1m 0 1m" [before] f2fs_direct_IO_enter: dev = (253,16), ino = 11 pos = 0 len = 1048576 ki_flags = 20000 ki_ioprio = 0 rw = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 0, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 1, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 2, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 3, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 4, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 5, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 6, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 7, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 8, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 9, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 10, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 11, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 12, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 13, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 14, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 15, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 16, start blkaddr = 0x0, len = 0x0, flags = 0, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 ...... f2fs_direct_IO_exit: dev = (253,16), ino = 11 pos = 0 len = 1048576 rw = 0 ret = 1048576 [after] f2fs_direct_IO_enter: dev = (253,16), ino = 11 pos = 0 len = 1048576 ki_flags = 20000 ki_ioprio = 0 rw = 0 f2fs_map_blocks: dev = (253,16), ino = 11, file offset = 0, start blkaddr = 0xffffffff, len = 0x100, flags = 4, seg_type = 1, may_create = 0, multidevice = 0, flag = 3, err = 0 f2fs_direct_IO_exit: dev = (253,16), ino = 11 pos = 0 len = 1048576 rw = 0 ret = 1048576 Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/data.c | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index fa5398ac45052..0d88649c60a50 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -1520,6 +1520,9 @@ static bool map_is_mergeable(struct f2fs_sb_info *sbi, return true; if (flag == F2FS_GET_BLOCK_PRE_DIO) return true; + if (flag == F2FS_GET_BLOCK_DIO && + map->m_pblk == NULL_ADDR && blkaddr == NULL_ADDR) + return true; return false; } @@ -1644,6 +1647,10 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) goto sync_out; } break; + case F2FS_GET_BLOCK_DIO: + if (map->m_next_pgofs) + *map->m_next_pgofs = pgofs + 1; + break; default: /* for defragment case */ if (map->m_next_pgofs) @@ -1662,7 +1669,8 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) /* reserved delalloc block should be mapped for fiemap. */ if (blkaddr == NEW_ADDR) map->m_flags |= F2FS_MAP_DELALLOC; - map->m_flags |= F2FS_MAP_MAPPED; + if (flag != F2FS_GET_BLOCK_DIO || !is_hole) + map->m_flags |= F2FS_MAP_MAPPED; map->m_pblk = blkaddr; map->m_len = 1; @@ -4191,12 +4199,13 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, * We should never see delalloc or compressed extents here based on * prior flushing and checks. */ - if (WARN_ON_ONCE(map.m_pblk == NEW_ADDR)) - return -EINVAL; if (WARN_ON_ONCE(map.m_pblk == COMPRESS_ADDR)) return -EINVAL; if (map.m_flags & F2FS_MAP_MAPPED) { + if (WARN_ON_ONCE(map.m_pblk == NEW_ADDR)) + return -EINVAL; + iomap->length = blks_to_bytes(inode, map.m_len); iomap->type = IOMAP_MAPPED; iomap->flags |= IOMAP_F_MERGED; @@ -4205,9 +4214,17 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, } else { if (flags & IOMAP_WRITE) return -ENOTBLK; - iomap->length = blks_to_bytes(inode, next_pgofs) - - iomap->offset; - iomap->type = IOMAP_HOLE; + + if (map.m_pblk == NULL_ADDR) { + iomap->length = blks_to_bytes(inode, next_pgofs) - + iomap->offset; + iomap->type = IOMAP_HOLE; + } else if (map.m_pblk == NEW_ADDR) { + iomap->length = blks_to_bytes(inode, map.m_len); + iomap->type = IOMAP_UNWRITTEN; + } else { + f2fs_bug_on(F2FS_I_SB(inode), 1); + } iomap->addr = IOMAP_NULL_ADDR; } From d3876e34e7e789e2cbdd782360fef2a777391082 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Tue, 26 Mar 2024 19:28:45 +0800 Subject: [PATCH 0039/1702] f2fs: fix to wait on page writeback in __clone_blkaddrs() In below race condition, dst page may become writeback status in __clone_blkaddrs(), it needs to wait writeback before update, fix it. Thread A GC Thread - f2fs_move_file_range - filemap_write_and_wait_range(dst) - gc_data_segment - f2fs_down_write(dst) - move_data_page - set_page_writeback(dst_page) - f2fs_submit_page_write - f2fs_up_write(dst) - f2fs_down_write(dst) - __exchange_data_block - __clone_blkaddrs - f2fs_get_new_data_page - memcpy_page Fixes: 0a2aa8fbb969 ("f2fs: refactor __exchange_data_block for speed up") Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index f4347db5b03e5..719064730f9a6 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -1325,6 +1325,9 @@ static int __clone_blkaddrs(struct inode *src_inode, struct inode *dst_inode, f2fs_put_page(psrc, 1); return PTR_ERR(pdst); } + + f2fs_wait_on_page_writeback(pdst, DATA, true, true); + memcpy_page(pdst, 0, psrc, 0, PAGE_SIZE); set_page_dirty(pdst); set_page_private_gcing(pdst); From ac5eecf481c29942eb9a862e758c0c8b68090c33 Mon Sep 17 00:00:00 2001 From: Yunlei He Date: Tue, 26 Mar 2024 14:10:43 +0800 Subject: [PATCH 0040/1702] f2fs: remove clear SB_INLINECRYPT flag in default_options In f2fs_remount, SB_INLINECRYPT flag will be clear and re-set. If create new file or open file during this gap, these files will not use inlinecrypt. Worse case, it may lead to data corruption if wrappedkey_v0 is enable. Thread A: Thread B: -f2fs_remount -f2fs_file_open or f2fs_new_inode -default_options <- clear SB_INLINECRYPT flag -fscrypt_select_encryption_impl -parse_options <- set SB_INLINECRYPT again Signed-off-by: Yunlei He Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/super.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 7c45929671ad6..df9765b41dac1 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -2132,8 +2132,6 @@ static void default_options(struct f2fs_sb_info *sbi, bool remount) F2FS_OPTION(sbi).memory_mode = MEMORY_MODE_NORMAL; F2FS_OPTION(sbi).errors = MOUNT_ERRORS_CONTINUE; - sbi->sb->s_flags &= ~SB_INLINECRYPT; - set_opt(sbi, INLINE_XATTR); set_opt(sbi, INLINE_DATA); set_opt(sbi, INLINE_DENTRY); From 04ee3a0b44e3d18cf6b0c712d14b98624877fd26 Mon Sep 17 00:00:00 2001 From: Jaewon Kim Date: Thu, 28 Mar 2024 18:10:00 +0900 Subject: [PATCH 0041/1702] clk: samsung: exynosautov9: fix wrong pll clock id value All PLL id values of CMU_TOP were incorrectly set to FOUT_SHARED0_PLL. It modified to the correct PLL clock id value. Fixes: 6587c62f69dc ("clk: samsung: add top clock support for Exynos Auto v9 SoC") Signed-off-by: Jaewon Kim Reviewed-by: Sam Protsenko Link: https://lore.kernel.org/r/20240328091000.17660-1-jaewon02.kim@samsung.com Signed-off-by: Krzysztof Kozlowski --- drivers/clk/samsung/clk-exynosautov9.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/clk/samsung/clk-exynosautov9.c b/drivers/clk/samsung/clk-exynosautov9.c index e9c06eb93e666..f04bacacab2cb 100644 --- a/drivers/clk/samsung/clk-exynosautov9.c +++ b/drivers/clk/samsung/clk-exynosautov9.c @@ -352,13 +352,13 @@ static const struct samsung_pll_clock top_pll_clks[] __initconst = { /* CMU_TOP_PURECLKCOMP */ PLL(pll_0822x, FOUT_SHARED0_PLL, "fout_shared0_pll", "oscclk", PLL_LOCKTIME_PLL_SHARED0, PLL_CON3_PLL_SHARED0, NULL), - PLL(pll_0822x, FOUT_SHARED0_PLL, "fout_shared1_pll", "oscclk", + PLL(pll_0822x, FOUT_SHARED1_PLL, "fout_shared1_pll", "oscclk", PLL_LOCKTIME_PLL_SHARED1, PLL_CON3_PLL_SHARED1, NULL), - PLL(pll_0822x, FOUT_SHARED0_PLL, "fout_shared2_pll", "oscclk", + PLL(pll_0822x, FOUT_SHARED2_PLL, "fout_shared2_pll", "oscclk", PLL_LOCKTIME_PLL_SHARED2, PLL_CON3_PLL_SHARED2, NULL), - PLL(pll_0822x, FOUT_SHARED0_PLL, "fout_shared3_pll", "oscclk", + PLL(pll_0822x, FOUT_SHARED3_PLL, "fout_shared3_pll", "oscclk", PLL_LOCKTIME_PLL_SHARED3, PLL_CON3_PLL_SHARED3, NULL), - PLL(pll_0822x, FOUT_SHARED0_PLL, "fout_shared4_pll", "oscclk", + PLL(pll_0822x, FOUT_SHARED4_PLL, "fout_shared4_pll", "oscclk", PLL_LOCKTIME_PLL_SHARED4, PLL_CON3_PLL_SHARED4, NULL), }; From 0f8678c34cbfdc63569a9b0ede1fe235ec6ec693 Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Mon, 1 Apr 2024 11:00:49 +0800 Subject: [PATCH 0042/1702] power: supply: cros_usbpd: provide ID table for avoiding fallback match Instead of using fallback driver name match, provide ID table[1] for the primary match. [1]: https://elixir.bootlin.com/linux/v6.8/source/drivers/base/platform.c#L1353 Reviewed-by: Benson Leung Reviewed-by: Prashant Malani Reviewed-by: Krzysztof Kozlowski Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20240401030052.2887845-4-tzungbi@kernel.org Signed-off-by: Sebastian Reichel --- drivers/power/supply/cros_usbpd-charger.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/power/supply/cros_usbpd-charger.c b/drivers/power/supply/cros_usbpd-charger.c index b6c96376776a9..8008e31c0c098 100644 --- a/drivers/power/supply/cros_usbpd-charger.c +++ b/drivers/power/supply/cros_usbpd-charger.c @@ -5,6 +5,7 @@ * Copyright (c) 2014 - 2018 Google, Inc */ +#include #include #include #include @@ -711,16 +712,22 @@ static int cros_usbpd_charger_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(cros_usbpd_charger_pm_ops, NULL, cros_usbpd_charger_resume); +static const struct platform_device_id cros_usbpd_charger_id[] = { + { DRV_NAME, 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, cros_usbpd_charger_id); + static struct platform_driver cros_usbpd_charger_driver = { .driver = { .name = DRV_NAME, .pm = &cros_usbpd_charger_pm_ops, }, - .probe = cros_usbpd_charger_probe + .probe = cros_usbpd_charger_probe, + .id_table = cros_usbpd_charger_id, }; module_platform_driver(cros_usbpd_charger_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("ChromeOS EC USBPD charger"); -MODULE_ALIAS("platform:" DRV_NAME); From d6486a13665e9df5b503a375e18226e1824f21d3 Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Mon, 1 Apr 2024 11:00:50 +0800 Subject: [PATCH 0043/1702] power: supply: cros_pchg: provide ID table for avoiding fallback match Instead of using fallback driver name match, provide ID table[1] for the primary match. [1]: https://elixir.bootlin.com/linux/v6.8/source/drivers/base/platform.c#L1353 Reviewed-by: Benson Leung Reviewed-by: Krzysztof Kozlowski Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20240401030052.2887845-5-tzungbi@kernel.org Signed-off-by: Sebastian Reichel --- drivers/power/supply/cros_peripheral_charger.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/power/supply/cros_peripheral_charger.c b/drivers/power/supply/cros_peripheral_charger.c index a204f2355be48..d406f2a784497 100644 --- a/drivers/power/supply/cros_peripheral_charger.c +++ b/drivers/power/supply/cros_peripheral_charger.c @@ -5,6 +5,7 @@ * Copyright 2020 Google LLC. */ +#include #include #include #include @@ -367,16 +368,22 @@ static int __maybe_unused cros_pchg_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(cros_pchg_pm_ops, NULL, cros_pchg_resume); +static const struct platform_device_id cros_pchg_id[] = { + { DRV_NAME, 0 }, + {} +}; +MODULE_DEVICE_TABLE(platform, cros_pchg_id); + static struct platform_driver cros_pchg_driver = { .driver = { .name = DRV_NAME, .pm = &cros_pchg_pm_ops, }, - .probe = cros_pchg_probe + .probe = cros_pchg_probe, + .id_table = cros_pchg_id, }; module_platform_driver(cros_pchg_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("ChromeOS EC peripheral device charger"); -MODULE_ALIAS("platform:" DRV_NAME); From c32c617de8076d8fb2a16a4a2f3b5da5f3df398d Mon Sep 17 00:00:00 2001 From: Andrew Davis Date: Mon, 25 Mar 2024 15:31:24 -0500 Subject: [PATCH 0044/1702] power: supply: bq27xxx: Move temperature reading out of update loop Most of the functions that read values return a status and put the value itself in an a function parameter. Update temperature reading to match. As temp is not checked for changes as part of the update loop, remove the read of the temperature from the periodic update loop. This saves I2C/1W bandwidth. It also means we do not have to cache it, fresh values are read when requested. Signed-off-by: Andrew Davis Link: https://lore.kernel.org/r/20240325203129.150030-1-afd@ti.com Signed-off-by: Sebastian Reichel --- drivers/power/supply/bq27xxx_battery.c | 17 ++++++++++------- include/linux/power/bq27xxx_battery.h | 1 - 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index abca568344686..00b2a56039ac0 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1652,10 +1652,11 @@ static int bq27xxx_battery_read_energy(struct bq27xxx_device_info *di) } /* - * Return the battery temperature in tenths of degree Kelvin + * Return the battery temperature in tenths of degree Celsius * Or < 0 if something fails. */ -static int bq27xxx_battery_read_temperature(struct bq27xxx_device_info *di) +static int bq27xxx_battery_read_temperature(struct bq27xxx_device_info *di, + union power_supply_propval *val) { int temp; @@ -1668,7 +1669,12 @@ static int bq27xxx_battery_read_temperature(struct bq27xxx_device_info *di) if (di->opts & BQ27XXX_O_ZERO) temp = 5 * temp / 2; - return temp; + /* Convert decidegree Kelvin to Celsius */ + temp -= 2731; + + val->intval = temp; + + return 0; } /* @@ -1851,7 +1857,6 @@ static void bq27xxx_battery_update_unlocked(struct bq27xxx_device_info *di) if ((cache.flags & 0xff) == 0xff) cache.flags = -1; /* read error */ if (cache.flags >= 0) { - cache.temperature = bq27xxx_battery_read_temperature(di); if (di->regs[BQ27XXX_REG_TTE] != INVALID_REG_ADDR) cache.time_to_empty = bq27xxx_battery_read_time(di, BQ27XXX_REG_TTE); if (di->regs[BQ27XXX_REG_TTECP] != INVALID_REG_ADDR) @@ -2038,9 +2043,7 @@ static int bq27xxx_battery_get_property(struct power_supply *psy, ret = bq27xxx_battery_capacity_level(di, val); break; case POWER_SUPPLY_PROP_TEMP: - ret = bq27xxx_simple_value(di->cache.temperature, val); - if (ret == 0) - val->intval -= 2731; /* convert decidegree k to c */ + ret = bq27xxx_battery_read_temperature(di, val); break; case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW: ret = bq27xxx_simple_value(di->cache.time_to_empty, val); diff --git a/include/linux/power/bq27xxx_battery.h b/include/linux/power/bq27xxx_battery.h index b9e5bd2b42d36..64b2749d9562b 100644 --- a/include/linux/power/bq27xxx_battery.h +++ b/include/linux/power/bq27xxx_battery.h @@ -47,7 +47,6 @@ struct bq27xxx_access_methods { }; struct bq27xxx_reg_cache { - int temperature; int time_to_empty; int time_to_empty_avg; int time_to_full; From 651a620aa4d49f5647e21e55fc71bb049bc03389 Mon Sep 17 00:00:00 2001 From: Andrew Davis Date: Mon, 25 Mar 2024 15:31:25 -0500 Subject: [PATCH 0045/1702] power: supply: bq27xxx: Move time reading out of update loop Most of the functions that read values return a status and put the value itself in an a function parameter. Update time reading to match. As time is not checked for changes as part of the update loop, remove the read of the this from the periodic update loop. This saves I2C/1W bandwidth. It also means we do not have to cache it, fresh values are read when requested. Signed-off-by: Andrew Davis Link: https://lore.kernel.org/r/20240325203129.150030-2-afd@ti.com Signed-off-by: Sebastian Reichel --- drivers/power/supply/bq27xxx_battery.c | 20 ++++++++------------ include/linux/power/bq27xxx_battery.h | 3 --- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 00b2a56039ac0..dcfe3c10e7ed3 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1696,7 +1696,8 @@ static int bq27xxx_battery_read_cyct(struct bq27xxx_device_info *di) * Read a time register. * Return < 0 if something fails. */ -static int bq27xxx_battery_read_time(struct bq27xxx_device_info *di, u8 reg) +static int bq27xxx_battery_read_time(struct bq27xxx_device_info *di, u8 reg, + union power_supply_propval *val) { int tval; @@ -1710,7 +1711,9 @@ static int bq27xxx_battery_read_time(struct bq27xxx_device_info *di, u8 reg) if (tval == 65535) return -ENODATA; - return tval * 60; + val->intval = tval * 60; + + return 0; } /* @@ -1857,13 +1860,6 @@ static void bq27xxx_battery_update_unlocked(struct bq27xxx_device_info *di) if ((cache.flags & 0xff) == 0xff) cache.flags = -1; /* read error */ if (cache.flags >= 0) { - if (di->regs[BQ27XXX_REG_TTE] != INVALID_REG_ADDR) - cache.time_to_empty = bq27xxx_battery_read_time(di, BQ27XXX_REG_TTE); - if (di->regs[BQ27XXX_REG_TTECP] != INVALID_REG_ADDR) - cache.time_to_empty_avg = bq27xxx_battery_read_time(di, BQ27XXX_REG_TTECP); - if (di->regs[BQ27XXX_REG_TTF] != INVALID_REG_ADDR) - cache.time_to_full = bq27xxx_battery_read_time(di, BQ27XXX_REG_TTF); - cache.charge_full = bq27xxx_battery_read_fcc(di); cache.capacity = bq27xxx_battery_read_soc(di); if (di->regs[BQ27XXX_REG_AE] != INVALID_REG_ADDR) @@ -2046,13 +2042,13 @@ static int bq27xxx_battery_get_property(struct power_supply *psy, ret = bq27xxx_battery_read_temperature(di, val); break; case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW: - ret = bq27xxx_simple_value(di->cache.time_to_empty, val); + ret = bq27xxx_battery_read_time(di, BQ27XXX_REG_TTE, val); break; case POWER_SUPPLY_PROP_TIME_TO_EMPTY_AVG: - ret = bq27xxx_simple_value(di->cache.time_to_empty_avg, val); + ret = bq27xxx_battery_read_time(di, BQ27XXX_REG_TTECP, val); break; case POWER_SUPPLY_PROP_TIME_TO_FULL_NOW: - ret = bq27xxx_simple_value(di->cache.time_to_full, val); + ret = bq27xxx_battery_read_time(di, BQ27XXX_REG_TTF, val); break; case POWER_SUPPLY_PROP_TECHNOLOGY: if (di->opts & BQ27XXX_O_MUL_CHEM) diff --git a/include/linux/power/bq27xxx_battery.h b/include/linux/power/bq27xxx_battery.h index 64b2749d9562b..e89ef989a5752 100644 --- a/include/linux/power/bq27xxx_battery.h +++ b/include/linux/power/bq27xxx_battery.h @@ -47,9 +47,6 @@ struct bq27xxx_access_methods { }; struct bq27xxx_reg_cache { - int time_to_empty; - int time_to_empty_avg; - int time_to_full; int charge_full; int cycle_count; int capacity; From 8d846335204f25a2247e5e88e39e1604b6ecc133 Mon Sep 17 00:00:00 2001 From: Andrew Davis Date: Mon, 25 Mar 2024 15:31:26 -0500 Subject: [PATCH 0046/1702] power: supply: bq27xxx: Move charge reading out of update loop Most of the functions that read values return a status and put the value itself in an a function parameter. Update charge reading to match. As charge state is not checked for changes as part of the update loop, remove the read of this from the periodic update loop. This saves I2C/1W bandwidth. It also means we do not have to cache it, fresh values are read when requested. Signed-off-by: Andrew Davis Link: https://lore.kernel.org/r/20240325203129.150030-3-afd@ti.com Signed-off-by: Sebastian Reichel --- drivers/power/supply/bq27xxx_battery.c | 29 +++++++++++++++----------- include/linux/power/bq27xxx_battery.h | 1 - 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index dcfe3c10e7ed3..799ee0aa9ef79 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1545,7 +1545,8 @@ static int bq27xxx_battery_read_soc(struct bq27xxx_device_info *di) * Return a battery charge value in µAh * Or < 0 if something fails. */ -static int bq27xxx_battery_read_charge(struct bq27xxx_device_info *di, u8 reg) +static int bq27xxx_battery_read_charge(struct bq27xxx_device_info *di, u8 reg, + union power_supply_propval *val) { int charge; @@ -1561,34 +1562,39 @@ static int bq27xxx_battery_read_charge(struct bq27xxx_device_info *di, u8 reg) else charge *= 1000; - return charge; + val->intval = charge; + + return 0; } /* * Return the battery Nominal available capacity in µAh * Or < 0 if something fails. */ -static inline int bq27xxx_battery_read_nac(struct bq27xxx_device_info *di) +static inline int bq27xxx_battery_read_nac(struct bq27xxx_device_info *di, + union power_supply_propval *val) { - return bq27xxx_battery_read_charge(di, BQ27XXX_REG_NAC); + return bq27xxx_battery_read_charge(di, BQ27XXX_REG_NAC, val); } /* * Return the battery Remaining Capacity in µAh * Or < 0 if something fails. */ -static inline int bq27xxx_battery_read_rc(struct bq27xxx_device_info *di) +static inline int bq27xxx_battery_read_rc(struct bq27xxx_device_info *di, + union power_supply_propval *val) { - return bq27xxx_battery_read_charge(di, BQ27XXX_REG_RC); + return bq27xxx_battery_read_charge(di, BQ27XXX_REG_RC, val); } /* * Return the battery Full Charge Capacity in µAh * Or < 0 if something fails. */ -static inline int bq27xxx_battery_read_fcc(struct bq27xxx_device_info *di) +static inline int bq27xxx_battery_read_fcc(struct bq27xxx_device_info *di, + union power_supply_propval *val) { - return bq27xxx_battery_read_charge(di, BQ27XXX_REG_FCC); + return bq27xxx_battery_read_charge(di, BQ27XXX_REG_FCC, val); } /* @@ -1860,7 +1866,6 @@ static void bq27xxx_battery_update_unlocked(struct bq27xxx_device_info *di) if ((cache.flags & 0xff) == 0xff) cache.flags = -1; /* read error */ if (cache.flags >= 0) { - cache.charge_full = bq27xxx_battery_read_fcc(di); cache.capacity = bq27xxx_battery_read_soc(di); if (di->regs[BQ27XXX_REG_AE] != INVALID_REG_ADDR) cache.energy = bq27xxx_battery_read_energy(di); @@ -2058,12 +2063,12 @@ static int bq27xxx_battery_get_property(struct power_supply *psy, break; case POWER_SUPPLY_PROP_CHARGE_NOW: if (di->regs[BQ27XXX_REG_NAC] != INVALID_REG_ADDR) - ret = bq27xxx_simple_value(bq27xxx_battery_read_nac(di), val); + ret = bq27xxx_battery_read_nac(di, val); else - ret = bq27xxx_simple_value(bq27xxx_battery_read_rc(di), val); + ret = bq27xxx_battery_read_rc(di, val); break; case POWER_SUPPLY_PROP_CHARGE_FULL: - ret = bq27xxx_simple_value(di->cache.charge_full, val); + ret = bq27xxx_battery_read_fcc(di, val); break; case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN: ret = bq27xxx_battery_read_dcap(di, val); diff --git a/include/linux/power/bq27xxx_battery.h b/include/linux/power/bq27xxx_battery.h index e89ef989a5752..1c67fa46013b3 100644 --- a/include/linux/power/bq27xxx_battery.h +++ b/include/linux/power/bq27xxx_battery.h @@ -47,7 +47,6 @@ struct bq27xxx_access_methods { }; struct bq27xxx_reg_cache { - int charge_full; int cycle_count; int capacity; int energy; From 39cf1c4cd03254218a23ef955bd534e19328f618 Mon Sep 17 00:00:00 2001 From: Andrew Davis Date: Mon, 25 Mar 2024 15:31:27 -0500 Subject: [PATCH 0047/1702] power: supply: bq27xxx: Move energy reading out of update loop Most of the functions that read values return a status and put the value itself in an a function parameter. Update energy reading to match. As energy is not checked for changes as part of the update loop, remove the read of this from the periodic update loop. This saves I2C/1W bandwidth. It also means we do not have to cache it, fresh values are read when requested. Signed-off-by: Andrew Davis Link: https://lore.kernel.org/r/20240325203129.150030-4-afd@ti.com Signed-off-by: Sebastian Reichel --- drivers/power/supply/bq27xxx_battery.c | 11 ++++++----- include/linux/power/bq27xxx_battery.h | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 799ee0aa9ef79..eae2b9a60f407 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1639,7 +1639,8 @@ static int bq27xxx_battery_read_dcap(struct bq27xxx_device_info *di, * Return the battery Available energy in µWh * Or < 0 if something fails. */ -static int bq27xxx_battery_read_energy(struct bq27xxx_device_info *di) +static int bq27xxx_battery_read_energy(struct bq27xxx_device_info *di, + union power_supply_propval *val) { int ae; @@ -1654,7 +1655,9 @@ static int bq27xxx_battery_read_energy(struct bq27xxx_device_info *di) else ae *= 1000; - return ae; + val->intval = ae; + + return 0; } /* @@ -1867,8 +1870,6 @@ static void bq27xxx_battery_update_unlocked(struct bq27xxx_device_info *di) cache.flags = -1; /* read error */ if (cache.flags >= 0) { cache.capacity = bq27xxx_battery_read_soc(di); - if (di->regs[BQ27XXX_REG_AE] != INVALID_REG_ADDR) - cache.energy = bq27xxx_battery_read_energy(di); di->cache.flags = cache.flags; cache.health = bq27xxx_battery_read_health(di); if (di->regs[BQ27XXX_REG_CYCT] != INVALID_REG_ADDR) @@ -2084,7 +2085,7 @@ static int bq27xxx_battery_get_property(struct power_supply *psy, ret = bq27xxx_simple_value(di->cache.cycle_count, val); break; case POWER_SUPPLY_PROP_ENERGY_NOW: - ret = bq27xxx_simple_value(di->cache.energy, val); + ret = bq27xxx_battery_read_energy(di, val); break; case POWER_SUPPLY_PROP_POWER_AVG: ret = bq27xxx_battery_pwr_avg(di, val); diff --git a/include/linux/power/bq27xxx_battery.h b/include/linux/power/bq27xxx_battery.h index 1c67fa46013b3..5c75abf3cf069 100644 --- a/include/linux/power/bq27xxx_battery.h +++ b/include/linux/power/bq27xxx_battery.h @@ -49,7 +49,6 @@ struct bq27xxx_access_methods { struct bq27xxx_reg_cache { int cycle_count; int capacity; - int energy; int flags; int health; }; From 656489ac90f25f92190a1dd5c4e5c5293bd70323 Mon Sep 17 00:00:00 2001 From: Andrew Davis Date: Mon, 25 Mar 2024 15:31:28 -0500 Subject: [PATCH 0048/1702] power: supply: bq27xxx: Move cycle count reading out of update loop Most of the functions that read values return a status and put the value itself in an a function parameter. Update cycle count reading to match. As cycle count is not checked for changes as part of the update loop, remove the read of this from the periodic update loop. This saves I2C/1W bandwidth. It also means we do not have to cache it, fresh values are read when requested. Signed-off-by: Andrew Davis Link: https://lore.kernel.org/r/20240325203129.150030-5-afd@ti.com Signed-off-by: Sebastian Reichel --- drivers/power/supply/bq27xxx_battery.c | 11 ++++++----- include/linux/power/bq27xxx_battery.h | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index eae2b9a60f407..fe1b0af143ca7 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1690,7 +1690,8 @@ static int bq27xxx_battery_read_temperature(struct bq27xxx_device_info *di, * Return the battery Cycle count total * Or < 0 if something fails. */ -static int bq27xxx_battery_read_cyct(struct bq27xxx_device_info *di) +static int bq27xxx_battery_read_cyct(struct bq27xxx_device_info *di, + union power_supply_propval *val) { int cyct; @@ -1698,7 +1699,9 @@ static int bq27xxx_battery_read_cyct(struct bq27xxx_device_info *di) if (cyct < 0) dev_err(di->dev, "error reading cycle count total\n"); - return cyct; + val->intval = cyct; + + return 0; } /* @@ -1872,8 +1875,6 @@ static void bq27xxx_battery_update_unlocked(struct bq27xxx_device_info *di) cache.capacity = bq27xxx_battery_read_soc(di); di->cache.flags = cache.flags; cache.health = bq27xxx_battery_read_health(di); - if (di->regs[BQ27XXX_REG_CYCT] != INVALID_REG_ADDR) - cache.cycle_count = bq27xxx_battery_read_cyct(di); /* * On gauges with signed current reporting the current must be @@ -2082,7 +2083,7 @@ static int bq27xxx_battery_get_property(struct power_supply *psy, case POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN: return -EINVAL; case POWER_SUPPLY_PROP_CYCLE_COUNT: - ret = bq27xxx_simple_value(di->cache.cycle_count, val); + ret = bq27xxx_battery_read_cyct(di, val); break; case POWER_SUPPLY_PROP_ENERGY_NOW: ret = bq27xxx_battery_read_energy(di, val); diff --git a/include/linux/power/bq27xxx_battery.h b/include/linux/power/bq27xxx_battery.h index 5c75abf3cf069..d743270799d75 100644 --- a/include/linux/power/bq27xxx_battery.h +++ b/include/linux/power/bq27xxx_battery.h @@ -47,7 +47,6 @@ struct bq27xxx_access_methods { }; struct bq27xxx_reg_cache { - int cycle_count; int capacity; int flags; int health; From 50f0ff7c8cc4c1d10fabc4b3b3f3b9e942b08187 Mon Sep 17 00:00:00 2001 From: Andrew Davis Date: Mon, 25 Mar 2024 15:31:29 -0500 Subject: [PATCH 0049/1702] power: supply: bq27xxx: Move health reading out of update loop Most of the functions that read values return a status and put the value itself in an a function parameter. Update health reading to match. As health is not checked for changes as part of the update loop, remove the read of this from the periodic update loop. This saves I2C/1W bandwidth. It also means we do not have to cache it, fresh values are read when requested. Signed-off-by: Andrew Davis Link: https://lore.kernel.org/r/20240325203129.150030-6-afd@ti.com Signed-off-by: Sebastian Reichel --- drivers/power/supply/bq27xxx_battery.c | 30 +++++++++++++++----------- include/linux/power/bq27xxx_battery.h | 1 - 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index fe1b0af143ca7..750fda543308c 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1777,19 +1777,26 @@ static bool bq27xxx_battery_capacity_inaccurate(struct bq27xxx_device_info *di, return false; } -static int bq27xxx_battery_read_health(struct bq27xxx_device_info *di) +static int bq27xxx_battery_read_health(struct bq27xxx_device_info *di, + union power_supply_propval *val) { + int health; + /* Unlikely but important to return first */ if (unlikely(bq27xxx_battery_overtemp(di, di->cache.flags))) - return POWER_SUPPLY_HEALTH_OVERHEAT; - if (unlikely(bq27xxx_battery_undertemp(di, di->cache.flags))) - return POWER_SUPPLY_HEALTH_COLD; - if (unlikely(bq27xxx_battery_dead(di, di->cache.flags))) - return POWER_SUPPLY_HEALTH_DEAD; - if (unlikely(bq27xxx_battery_capacity_inaccurate(di, di->cache.flags))) - return POWER_SUPPLY_HEALTH_CALIBRATION_REQUIRED; - - return POWER_SUPPLY_HEALTH_GOOD; + health = POWER_SUPPLY_HEALTH_OVERHEAT; + else if (unlikely(bq27xxx_battery_undertemp(di, di->cache.flags))) + health = POWER_SUPPLY_HEALTH_COLD; + else if (unlikely(bq27xxx_battery_dead(di, di->cache.flags))) + health = POWER_SUPPLY_HEALTH_DEAD; + else if (unlikely(bq27xxx_battery_capacity_inaccurate(di, di->cache.flags))) + health = POWER_SUPPLY_HEALTH_CALIBRATION_REQUIRED; + else + health = POWER_SUPPLY_HEALTH_GOOD; + + val->intval = health; + + return 0; } static bool bq27xxx_battery_is_full(struct bq27xxx_device_info *di, int flags) @@ -1874,7 +1881,6 @@ static void bq27xxx_battery_update_unlocked(struct bq27xxx_device_info *di) if (cache.flags >= 0) { cache.capacity = bq27xxx_battery_read_soc(di); di->cache.flags = cache.flags; - cache.health = bq27xxx_battery_read_health(di); /* * On gauges with signed current reporting the current must be @@ -2092,7 +2098,7 @@ static int bq27xxx_battery_get_property(struct power_supply *psy, ret = bq27xxx_battery_pwr_avg(di, val); break; case POWER_SUPPLY_PROP_HEALTH: - ret = bq27xxx_simple_value(di->cache.health, val); + ret = bq27xxx_battery_read_health(di, val); break; case POWER_SUPPLY_PROP_MANUFACTURER: val->strval = BQ27XXX_MANUFACTURER; diff --git a/include/linux/power/bq27xxx_battery.h b/include/linux/power/bq27xxx_battery.h index d743270799d75..5180dc9f17064 100644 --- a/include/linux/power/bq27xxx_battery.h +++ b/include/linux/power/bq27xxx_battery.h @@ -49,7 +49,6 @@ struct bq27xxx_access_methods { struct bq27xxx_reg_cache { int capacity; int flags; - int health; }; struct bq27xxx_device_info { From f0697bf078368d765b9e9ceef1dac0d5eb69b4b6 Mon Sep 17 00:00:00 2001 From: Boshi Yu Date: Mon, 11 Mar 2024 19:38:19 +0800 Subject: [PATCH 0050/1702] RDMA/erdma: Allocate doorbell records from dma pool Currently, the 8 byte doorbell record is allocated along with the queue buffer, which may result in waste of dma space when the queue buffer is page aligned. To address this issue, we introduce a dma pool named db_pool and allocate doorbell record from it. Reviewed-by: Cheng Xu Signed-off-by: Boshi Yu Link: https://lore.kernel.org/r/20240311113821.22482-2-boshiyu@alibaba-inc.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/erdma/erdma.h | 7 +- drivers/infiniband/hw/erdma/erdma_cmdq.c | 102 +++++++++++++--------- drivers/infiniband/hw/erdma/erdma_eq.c | 55 +++++++----- drivers/infiniband/hw/erdma/erdma_main.c | 15 +++- drivers/infiniband/hw/erdma/erdma_verbs.c | 85 ++++++++++-------- drivers/infiniband/hw/erdma/erdma_verbs.h | 4 + 6 files changed, 167 insertions(+), 101 deletions(-) diff --git a/drivers/infiniband/hw/erdma/erdma.h b/drivers/infiniband/hw/erdma/erdma.h index 5df401a30cb9b..e116263a608fe 100644 --- a/drivers/infiniband/hw/erdma/erdma.h +++ b/drivers/infiniband/hw/erdma/erdma.h @@ -34,6 +34,7 @@ struct erdma_eq { void __iomem *db; u64 *db_record; + dma_addr_t db_record_dma_addr; }; struct erdma_cmdq_sq { @@ -49,6 +50,7 @@ struct erdma_cmdq_sq { u16 wqebb_cnt; u64 *db_record; + dma_addr_t db_record_dma_addr; }; struct erdma_cmdq_cq { @@ -62,6 +64,7 @@ struct erdma_cmdq_cq { u32 cmdsn; u64 *db_record; + dma_addr_t db_record_dma_addr; atomic64_t armed_num; }; @@ -177,9 +180,6 @@ enum { ERDMA_RES_CNT = 2, }; -#define ERDMA_EXTRA_BUFFER_SIZE ERDMA_DB_SIZE -#define WARPPED_BUFSIZE(size) ((size) + ERDMA_EXTRA_BUFFER_SIZE) - struct erdma_dev { struct ib_device ibdev; struct net_device *netdev; @@ -213,6 +213,7 @@ struct erdma_dev { atomic_t num_ctx; struct list_head cep_list; + struct dma_pool *db_pool; struct dma_pool *resp_pool; }; diff --git a/drivers/infiniband/hw/erdma/erdma_cmdq.c b/drivers/infiniband/hw/erdma/erdma_cmdq.c index a151a7bdd5049..c2c666040949e 100644 --- a/drivers/infiniband/hw/erdma/erdma_cmdq.c +++ b/drivers/infiniband/hw/erdma/erdma_cmdq.c @@ -89,20 +89,19 @@ static int erdma_cmdq_sq_init(struct erdma_dev *dev) { struct erdma_cmdq *cmdq = &dev->cmdq; struct erdma_cmdq_sq *sq = &cmdq->sq; - u32 buf_size; sq->wqebb_cnt = SQEBB_COUNT(ERDMA_CMDQ_SQE_SIZE); sq->depth = cmdq->max_outstandings * sq->wqebb_cnt; - buf_size = sq->depth << SQEBB_SHIFT; - - sq->qbuf = - dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), - &sq->qbuf_dma_addr, GFP_KERNEL); + sq->qbuf = dma_alloc_coherent(&dev->pdev->dev, sq->depth << SQEBB_SHIFT, + &sq->qbuf_dma_addr, GFP_KERNEL); if (!sq->qbuf) return -ENOMEM; - sq->db_record = (u64 *)(sq->qbuf + buf_size); + sq->db_record = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, + &sq->db_record_dma_addr); + if (!sq->db_record) + goto err_out; spin_lock_init(&sq->lock); @@ -112,29 +111,35 @@ static int erdma_cmdq_sq_init(struct erdma_dev *dev) lower_32_bits(sq->qbuf_dma_addr)); erdma_reg_write32(dev, ERDMA_REGS_CMDQ_DEPTH_REG, sq->depth); erdma_reg_write64(dev, ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG, - sq->qbuf_dma_addr + buf_size); + sq->db_record_dma_addr); return 0; + +err_out: + dma_free_coherent(&dev->pdev->dev, sq->depth << SQEBB_SHIFT, + sq->qbuf, sq->qbuf_dma_addr); + + return -ENOMEM; } static int erdma_cmdq_cq_init(struct erdma_dev *dev) { struct erdma_cmdq *cmdq = &dev->cmdq; struct erdma_cmdq_cq *cq = &cmdq->cq; - u32 buf_size; cq->depth = cmdq->sq.depth; - buf_size = cq->depth << CQE_SHIFT; - - cq->qbuf = - dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), - &cq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO); + cq->qbuf = dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, + &cq->qbuf_dma_addr, + GFP_KERNEL | __GFP_ZERO); if (!cq->qbuf) return -ENOMEM; spin_lock_init(&cq->lock); - cq->db_record = (u64 *)(cq->qbuf + buf_size); + cq->db_record = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, + &cq->db_record_dma_addr); + if (!cq->db_record) + goto err_out; atomic64_set(&cq->armed_num, 0); @@ -143,23 +148,26 @@ static int erdma_cmdq_cq_init(struct erdma_dev *dev) erdma_reg_write32(dev, ERDMA_REGS_CMDQ_CQ_ADDR_L_REG, lower_32_bits(cq->qbuf_dma_addr)); erdma_reg_write64(dev, ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG, - cq->qbuf_dma_addr + buf_size); + cq->db_record_dma_addr); return 0; + +err_out: + dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, cq->qbuf, + cq->qbuf_dma_addr); + + return -ENOMEM; } static int erdma_cmdq_eq_init(struct erdma_dev *dev) { struct erdma_cmdq *cmdq = &dev->cmdq; struct erdma_eq *eq = &cmdq->eq; - u32 buf_size; eq->depth = cmdq->max_outstandings; - buf_size = eq->depth << EQE_SHIFT; - - eq->qbuf = - dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), - &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO); + eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, + &eq->qbuf_dma_addr, + GFP_KERNEL | __GFP_ZERO); if (!eq->qbuf) return -ENOMEM; @@ -167,7 +175,10 @@ static int erdma_cmdq_eq_init(struct erdma_dev *dev) atomic64_set(&eq->event_num, 0); eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG; - eq->db_record = (u64 *)(eq->qbuf + buf_size); + eq->db_record = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, + &eq->db_record_dma_addr); + if (!eq->db_record) + goto err_out; erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_H_REG, upper_32_bits(eq->qbuf_dma_addr)); @@ -175,9 +186,15 @@ static int erdma_cmdq_eq_init(struct erdma_dev *dev) lower_32_bits(eq->qbuf_dma_addr)); erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_DEPTH_REG, eq->depth); erdma_reg_write64(dev, ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG, - eq->qbuf_dma_addr + buf_size); + eq->db_record_dma_addr); return 0; + +err_out: + dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf, + eq->qbuf_dma_addr); + + return -ENOMEM; } int erdma_cmdq_init(struct erdma_dev *dev) @@ -211,17 +228,19 @@ int erdma_cmdq_init(struct erdma_dev *dev) return 0; err_destroy_cq: - dma_free_coherent(&dev->pdev->dev, - (cmdq->cq.depth << CQE_SHIFT) + - ERDMA_EXTRA_BUFFER_SIZE, + dma_free_coherent(&dev->pdev->dev, cmdq->cq.depth << CQE_SHIFT, cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr); + dma_pool_free(dev->db_pool, cmdq->cq.db_record, + cmdq->cq.db_record_dma_addr); + err_destroy_sq: - dma_free_coherent(&dev->pdev->dev, - (cmdq->sq.depth << SQEBB_SHIFT) + - ERDMA_EXTRA_BUFFER_SIZE, + dma_free_coherent(&dev->pdev->dev, cmdq->sq.depth << SQEBB_SHIFT, cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr); + dma_pool_free(dev->db_pool, cmdq->sq.db_record, + cmdq->sq.db_record_dma_addr); + return err; } @@ -238,18 +257,23 @@ void erdma_cmdq_destroy(struct erdma_dev *dev) clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state); - dma_free_coherent(&dev->pdev->dev, - (cmdq->eq.depth << EQE_SHIFT) + - ERDMA_EXTRA_BUFFER_SIZE, + dma_free_coherent(&dev->pdev->dev, cmdq->eq.depth << EQE_SHIFT, cmdq->eq.qbuf, cmdq->eq.qbuf_dma_addr); - dma_free_coherent(&dev->pdev->dev, - (cmdq->sq.depth << SQEBB_SHIFT) + - ERDMA_EXTRA_BUFFER_SIZE, + + dma_pool_free(dev->db_pool, cmdq->eq.db_record, + cmdq->eq.db_record_dma_addr); + + dma_free_coherent(&dev->pdev->dev, cmdq->sq.depth << SQEBB_SHIFT, cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr); - dma_free_coherent(&dev->pdev->dev, - (cmdq->cq.depth << CQE_SHIFT) + - ERDMA_EXTRA_BUFFER_SIZE, + + dma_pool_free(dev->db_pool, cmdq->sq.db_record, + cmdq->sq.db_record_dma_addr); + + dma_free_coherent(&dev->pdev->dev, cmdq->cq.depth << CQE_SHIFT, cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr); + + dma_pool_free(dev->db_pool, cmdq->cq.db_record, + cmdq->cq.db_record_dma_addr); } static void *get_next_valid_cmdq_cqe(struct erdma_cmdq *cmdq) diff --git a/drivers/infiniband/hw/erdma/erdma_eq.c b/drivers/infiniband/hw/erdma/erdma_eq.c index ea47cb21fdb8c..809c33628f38c 100644 --- a/drivers/infiniband/hw/erdma/erdma_eq.c +++ b/drivers/infiniband/hw/erdma/erdma_eq.c @@ -83,14 +83,12 @@ void erdma_aeq_event_handler(struct erdma_dev *dev) int erdma_aeq_init(struct erdma_dev *dev) { struct erdma_eq *eq = &dev->aeq; - u32 buf_size; eq->depth = ERDMA_DEFAULT_EQ_DEPTH; - buf_size = eq->depth << EQE_SHIFT; - eq->qbuf = - dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), - &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO); + eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, + &eq->qbuf_dma_addr, + GFP_KERNEL | __GFP_ZERO); if (!eq->qbuf) return -ENOMEM; @@ -99,7 +97,10 @@ int erdma_aeq_init(struct erdma_dev *dev) atomic64_set(&eq->notify_num, 0); eq->db = dev->func_bar + ERDMA_REGS_AEQ_DB_REG; - eq->db_record = (u64 *)(eq->qbuf + buf_size); + eq->db_record = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, + &eq->db_record_dma_addr); + if (!eq->db_record) + goto err_out; erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_H_REG, upper_32_bits(eq->qbuf_dma_addr)); @@ -107,18 +108,25 @@ int erdma_aeq_init(struct erdma_dev *dev) lower_32_bits(eq->qbuf_dma_addr)); erdma_reg_write32(dev, ERDMA_REGS_AEQ_DEPTH_REG, eq->depth); erdma_reg_write64(dev, ERDMA_AEQ_DB_HOST_ADDR_REG, - eq->qbuf_dma_addr + buf_size); + eq->db_record_dma_addr); return 0; + +err_out: + dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf, + eq->qbuf_dma_addr); + + return -ENOMEM; } void erdma_aeq_destroy(struct erdma_dev *dev) { struct erdma_eq *eq = &dev->aeq; - dma_free_coherent(&dev->pdev->dev, - WARPPED_BUFSIZE(eq->depth << EQE_SHIFT), eq->qbuf, + dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf, eq->qbuf_dma_addr); + + dma_pool_free(dev->db_pool, eq->db_record, eq->db_record_dma_addr); } void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb) @@ -209,7 +217,6 @@ static void erdma_free_ceq_irq(struct erdma_dev *dev, u16 ceqn) static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq) { struct erdma_cmdq_create_eq_req req; - dma_addr_t db_info_dma_addr; erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, CMDQ_OPCODE_CREATE_EQ); @@ -219,9 +226,8 @@ static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq) req.qtype = ERDMA_EQ_TYPE_CEQ; /* Vector index is the same as EQN. */ req.vector_idx = eqn; - db_info_dma_addr = eq->qbuf_dma_addr + (eq->depth << EQE_SHIFT); - req.db_dma_addr_l = lower_32_bits(db_info_dma_addr); - req.db_dma_addr_h = upper_32_bits(db_info_dma_addr); + req.db_dma_addr_l = lower_32_bits(eq->db_record_dma_addr); + req.db_dma_addr_h = upper_32_bits(eq->db_record_dma_addr); return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); } @@ -229,12 +235,12 @@ static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq) static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn) { struct erdma_eq *eq = &dev->ceqs[ceqn].eq; - u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT; int ret; - eq->qbuf = - dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), - &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO); + eq->depth = ERDMA_DEFAULT_EQ_DEPTH; + eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, + &eq->qbuf_dma_addr, + GFP_KERNEL | __GFP_ZERO); if (!eq->qbuf) return -ENOMEM; @@ -242,10 +248,17 @@ static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn) atomic64_set(&eq->event_num, 0); atomic64_set(&eq->notify_num, 0); - eq->depth = ERDMA_DEFAULT_EQ_DEPTH; eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG + (ceqn + 1) * ERDMA_DB_SIZE; - eq->db_record = (u64 *)(eq->qbuf + buf_size); + + eq->db_record = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, + &eq->db_record_dma_addr); + if (!eq->db_record) { + dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, + eq->qbuf, eq->qbuf_dma_addr); + return -ENOMEM; + } + eq->ci = 0; dev->ceqs[ceqn].dev = dev; @@ -259,7 +272,6 @@ static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn) static void erdma_ceq_uninit_one(struct erdma_dev *dev, u16 ceqn) { struct erdma_eq *eq = &dev->ceqs[ceqn].eq; - u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT; struct erdma_cmdq_destroy_eq_req req; int err; @@ -276,8 +288,9 @@ static void erdma_ceq_uninit_one(struct erdma_dev *dev, u16 ceqn) if (err) return; - dma_free_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), eq->qbuf, + dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf, eq->qbuf_dma_addr); + dma_pool_free(dev->db_pool, eq->db_record, eq->db_record_dma_addr); } int erdma_ceqs_init(struct erdma_dev *dev) diff --git a/drivers/infiniband/hw/erdma/erdma_main.c b/drivers/infiniband/hw/erdma/erdma_main.c index 472939172f0c6..7080f8a71ec43 100644 --- a/drivers/infiniband/hw/erdma/erdma_main.c +++ b/drivers/infiniband/hw/erdma/erdma_main.c @@ -178,16 +178,26 @@ static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev) if (!dev->resp_pool) return -ENOMEM; + dev->db_pool = dma_pool_create("erdma_db_pool", &pdev->dev, + ERDMA_DB_SIZE, ERDMA_DB_SIZE, 0); + if (!dev->db_pool) { + ret = -ENOMEM; + goto destroy_resp_pool; + } + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(ERDMA_PCI_WIDTH)); if (ret) - goto destroy_pool; + goto destroy_db_pool; dma_set_max_seg_size(&pdev->dev, UINT_MAX); return 0; -destroy_pool: +destroy_db_pool: + dma_pool_destroy(dev->db_pool); + +destroy_resp_pool: dma_pool_destroy(dev->resp_pool); return ret; @@ -195,6 +205,7 @@ static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev) static void erdma_device_uninit(struct erdma_dev *dev) { + dma_pool_destroy(dev->db_pool); dma_pool_destroy(dev->resp_pool); } diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 23dfc01603f8a..b78ddca1483ea 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -76,10 +76,8 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp) req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr; req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr; - req.sq_db_info_dma_addr = qp->kern_qp.sq_buf_dma_addr + - (qp->attrs.sq_size << SQEBB_SHIFT); - req.rq_db_info_dma_addr = qp->kern_qp.rq_buf_dma_addr + - (qp->attrs.rq_size << RQE_SHIFT); + req.sq_db_info_dma_addr = qp->kern_qp.sq_db_info_dma_addr; + req.rq_db_info_dma_addr = qp->kern_qp.rq_db_info_dma_addr; } else { user_qp = &qp->user_qp; req.sq_cqn_mtt_cfg = FIELD_PREP( @@ -209,8 +207,7 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq) ERDMA_MR_MTT_0LEVEL); req.first_page_offset = 0; - req.cq_db_info_addr = - cq->kern_cq.qbuf_dma_addr + (cq->depth << CQE_SHIFT); + req.cq_db_info_addr = cq->kern_cq.db_record_dma_addr; } else { mem = &cq->user_cq.qbuf_mem; req.cfg0 |= @@ -482,16 +479,24 @@ static void free_kernel_qp(struct erdma_qp *qp) vfree(qp->kern_qp.rwr_tbl); if (qp->kern_qp.sq_buf) - dma_free_coherent( - &dev->pdev->dev, - WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT), - qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); + dma_free_coherent(&dev->pdev->dev, + qp->attrs.sq_size << SQEBB_SHIFT, + qp->kern_qp.sq_buf, + qp->kern_qp.sq_buf_dma_addr); + + if (qp->kern_qp.sq_db_info) + dma_pool_free(dev->db_pool, qp->kern_qp.sq_db_info, + qp->kern_qp.sq_db_info_dma_addr); if (qp->kern_qp.rq_buf) - dma_free_coherent( - &dev->pdev->dev, - WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT), - qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); + dma_free_coherent(&dev->pdev->dev, + qp->attrs.rq_size << RQE_SHIFT, + qp->kern_qp.rq_buf, + qp->kern_qp.rq_buf_dma_addr); + + if (qp->kern_qp.rq_db_info) + dma_pool_free(dev->db_pool, qp->kern_qp.rq_db_info, + qp->kern_qp.rq_db_info_dma_addr); } static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, @@ -516,20 +521,27 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, if (!kqp->swr_tbl || !kqp->rwr_tbl) goto err_out; - size = (qp->attrs.sq_size << SQEBB_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE; + size = qp->attrs.sq_size << SQEBB_SHIFT; kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size, &kqp->sq_buf_dma_addr, GFP_KERNEL); if (!kqp->sq_buf) goto err_out; - size = (qp->attrs.rq_size << RQE_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE; + kqp->sq_db_info = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, + &kqp->sq_db_info_dma_addr); + if (!kqp->sq_db_info) + goto err_out; + + size = qp->attrs.rq_size << RQE_SHIFT; kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size, &kqp->rq_buf_dma_addr, GFP_KERNEL); if (!kqp->rq_buf) goto err_out; - kqp->sq_db_info = kqp->sq_buf + (qp->attrs.sq_size << SQEBB_SHIFT); - kqp->rq_db_info = kqp->rq_buf + (qp->attrs.rq_size << RQE_SHIFT); + kqp->rq_db_info = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, + &kqp->rq_db_info_dma_addr); + if (!kqp->rq_db_info) + goto err_out; return 0; @@ -1237,9 +1249,10 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) return err; if (rdma_is_kernel_res(&cq->ibcq.res)) { - dma_free_coherent(&dev->pdev->dev, - WARPPED_BUFSIZE(cq->depth << CQE_SHIFT), + dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); + dma_pool_free(dev->db_pool, cq->kern_cq.db_record, + cq->kern_cq.db_record_dma_addr); } else { erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); put_mtt_entries(dev, &cq->user_cq.qbuf_mem); @@ -1279,16 +1292,7 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) wait_for_completion(&qp->safe_free); if (rdma_is_kernel_res(&qp->ibqp.res)) { - vfree(qp->kern_qp.swr_tbl); - vfree(qp->kern_qp.rwr_tbl); - dma_free_coherent( - &dev->pdev->dev, - WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT), - qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); - dma_free_coherent( - &dev->pdev->dev, - WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT), - qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); + free_kernel_qp(qp); } else { put_mtt_entries(dev, &qp->user_qp.sq_mem); put_mtt_entries(dev, &qp->user_qp.rq_mem); @@ -1600,19 +1604,27 @@ static int erdma_init_kernel_cq(struct erdma_cq *cq) struct erdma_dev *dev = to_edev(cq->ibcq.device); cq->kern_cq.qbuf = - dma_alloc_coherent(&dev->pdev->dev, - WARPPED_BUFSIZE(cq->depth << CQE_SHIFT), + dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL); if (!cq->kern_cq.qbuf) return -ENOMEM; - cq->kern_cq.db_record = - (u64 *)(cq->kern_cq.qbuf + (cq->depth << CQE_SHIFT)); + cq->kern_cq.db_record = dma_pool_zalloc( + dev->db_pool, GFP_KERNEL, &cq->kern_cq.db_record_dma_addr); + if (!cq->kern_cq.db_record) + goto err_out; + spin_lock_init(&cq->kern_cq.lock); /* use default cqdb addr */ cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET; return 0; + +err_out: + dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, + cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); + + return -ENOMEM; } int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, @@ -1676,9 +1688,10 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); put_mtt_entries(dev, &cq->user_cq.qbuf_mem); } else { - dma_free_coherent(&dev->pdev->dev, - WARPPED_BUFSIZE(depth << CQE_SHIFT), + dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT, cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); + dma_pool_free(dev->db_pool, cq->kern_cq.db_record, + cq->kern_cq.db_record_dma_addr); } err_out_xa: diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h index db6018529ccc3..b02ffdc8c811d 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.h +++ b/drivers/infiniband/hw/erdma/erdma_verbs.h @@ -170,6 +170,9 @@ struct erdma_kqp { void *sq_db_info; void *rq_db_info; + dma_addr_t sq_db_info_dma_addr; + dma_addr_t rq_db_info_dma_addr; + u8 sig_all; }; @@ -247,6 +250,7 @@ struct erdma_kcq_info { spinlock_t lock; u8 __iomem *db; u64 *db_record; + dma_addr_t db_record_dma_addr; }; struct erdma_ucq_info { From fdb09ed15f272adb7c0403f7a6f9b4db3959284d Mon Sep 17 00:00:00 2001 From: Boshi Yu Date: Mon, 11 Mar 2024 19:38:20 +0800 Subject: [PATCH 0051/1702] RDMA/erdma: Unify the names related to doorbell records There exist two different names for the doorbell records: db_info and db_record. We use dbrec for cpu address of the doorbell record and dbrec_dma for dma address of the doorbell recordi uniformly. Reviewed-by: Cheng Xu Signed-off-by: Boshi Yu Link: https://lore.kernel.org/r/20240311113821.22482-3-boshiyu@alibaba-inc.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/erdma/erdma.h | 12 ++--- drivers/infiniband/hw/erdma/erdma_cmdq.c | 43 ++++++--------- drivers/infiniband/hw/erdma/erdma_cq.c | 2 +- drivers/infiniband/hw/erdma/erdma_eq.c | 23 ++++---- drivers/infiniband/hw/erdma/erdma_hw.h | 6 +-- drivers/infiniband/hw/erdma/erdma_qp.c | 4 +- drivers/infiniband/hw/erdma/erdma_verbs.c | 64 +++++++++++------------ drivers/infiniband/hw/erdma/erdma_verbs.h | 18 +++---- 8 files changed, 79 insertions(+), 93 deletions(-) diff --git a/drivers/infiniband/hw/erdma/erdma.h b/drivers/infiniband/hw/erdma/erdma.h index e116263a608fe..c8bd698e21b04 100644 --- a/drivers/infiniband/hw/erdma/erdma.h +++ b/drivers/infiniband/hw/erdma/erdma.h @@ -33,8 +33,8 @@ struct erdma_eq { atomic64_t notify_num; void __iomem *db; - u64 *db_record; - dma_addr_t db_record_dma_addr; + u64 *dbrec; + dma_addr_t dbrec_dma; }; struct erdma_cmdq_sq { @@ -49,8 +49,8 @@ struct erdma_cmdq_sq { u16 wqebb_cnt; - u64 *db_record; - dma_addr_t db_record_dma_addr; + u64 *dbrec; + dma_addr_t dbrec_dma; }; struct erdma_cmdq_cq { @@ -63,8 +63,8 @@ struct erdma_cmdq_cq { u32 ci; u32 cmdsn; - u64 *db_record; - dma_addr_t db_record_dma_addr; + u64 *dbrec; + dma_addr_t dbrec_dma; atomic64_t armed_num; }; diff --git a/drivers/infiniband/hw/erdma/erdma_cmdq.c b/drivers/infiniband/hw/erdma/erdma_cmdq.c index c2c666040949e..0ac2683cfccf6 100644 --- a/drivers/infiniband/hw/erdma/erdma_cmdq.c +++ b/drivers/infiniband/hw/erdma/erdma_cmdq.c @@ -14,7 +14,7 @@ static void arm_cmdq_cq(struct erdma_cmdq *cmdq) FIELD_PREP(ERDMA_CQDB_CMDSN_MASK, cmdq->cq.cmdsn) | FIELD_PREP(ERDMA_CQDB_IDX_MASK, cmdq->cq.cmdsn); - *cmdq->cq.db_record = db_data; + *cmdq->cq.dbrec = db_data; writeq(db_data, dev->func_bar + ERDMA_CMDQ_CQDB_REG); atomic64_inc(&cmdq->cq.armed_num); @@ -25,7 +25,7 @@ static void kick_cmdq_db(struct erdma_cmdq *cmdq) struct erdma_dev *dev = container_of(cmdq, struct erdma_dev, cmdq); u64 db_data = FIELD_PREP(ERDMA_CMD_HDR_WQEBB_INDEX_MASK, cmdq->sq.pi); - *cmdq->sq.db_record = db_data; + *cmdq->sq.dbrec = db_data; writeq(db_data, dev->func_bar + ERDMA_CMDQ_SQDB_REG); } @@ -98,9 +98,8 @@ static int erdma_cmdq_sq_init(struct erdma_dev *dev) if (!sq->qbuf) return -ENOMEM; - sq->db_record = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, - &sq->db_record_dma_addr); - if (!sq->db_record) + sq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &sq->dbrec_dma); + if (!sq->dbrec) goto err_out; spin_lock_init(&sq->lock); @@ -110,8 +109,7 @@ static int erdma_cmdq_sq_init(struct erdma_dev *dev) erdma_reg_write32(dev, ERDMA_REGS_CMDQ_SQ_ADDR_L_REG, lower_32_bits(sq->qbuf_dma_addr)); erdma_reg_write32(dev, ERDMA_REGS_CMDQ_DEPTH_REG, sq->depth); - erdma_reg_write64(dev, ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG, - sq->db_record_dma_addr); + erdma_reg_write64(dev, ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG, sq->dbrec_dma); return 0; @@ -136,9 +134,8 @@ static int erdma_cmdq_cq_init(struct erdma_dev *dev) spin_lock_init(&cq->lock); - cq->db_record = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, - &cq->db_record_dma_addr); - if (!cq->db_record) + cq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &cq->dbrec_dma); + if (!cq->dbrec) goto err_out; atomic64_set(&cq->armed_num, 0); @@ -147,8 +144,7 @@ static int erdma_cmdq_cq_init(struct erdma_dev *dev) upper_32_bits(cq->qbuf_dma_addr)); erdma_reg_write32(dev, ERDMA_REGS_CMDQ_CQ_ADDR_L_REG, lower_32_bits(cq->qbuf_dma_addr)); - erdma_reg_write64(dev, ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG, - cq->db_record_dma_addr); + erdma_reg_write64(dev, ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG, cq->dbrec_dma); return 0; @@ -175,9 +171,8 @@ static int erdma_cmdq_eq_init(struct erdma_dev *dev) atomic64_set(&eq->event_num, 0); eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG; - eq->db_record = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, - &eq->db_record_dma_addr); - if (!eq->db_record) + eq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &eq->dbrec_dma); + if (!eq->dbrec) goto err_out; erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_H_REG, @@ -185,8 +180,7 @@ static int erdma_cmdq_eq_init(struct erdma_dev *dev) erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_L_REG, lower_32_bits(eq->qbuf_dma_addr)); erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_DEPTH_REG, eq->depth); - erdma_reg_write64(dev, ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG, - eq->db_record_dma_addr); + erdma_reg_write64(dev, ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG, eq->dbrec_dma); return 0; @@ -231,15 +225,13 @@ int erdma_cmdq_init(struct erdma_dev *dev) dma_free_coherent(&dev->pdev->dev, cmdq->cq.depth << CQE_SHIFT, cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr); - dma_pool_free(dev->db_pool, cmdq->cq.db_record, - cmdq->cq.db_record_dma_addr); + dma_pool_free(dev->db_pool, cmdq->cq.dbrec, cmdq->cq.dbrec_dma); err_destroy_sq: dma_free_coherent(&dev->pdev->dev, cmdq->sq.depth << SQEBB_SHIFT, cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr); - dma_pool_free(dev->db_pool, cmdq->sq.db_record, - cmdq->sq.db_record_dma_addr); + dma_pool_free(dev->db_pool, cmdq->sq.dbrec, cmdq->sq.dbrec_dma); return err; } @@ -260,20 +252,17 @@ void erdma_cmdq_destroy(struct erdma_dev *dev) dma_free_coherent(&dev->pdev->dev, cmdq->eq.depth << EQE_SHIFT, cmdq->eq.qbuf, cmdq->eq.qbuf_dma_addr); - dma_pool_free(dev->db_pool, cmdq->eq.db_record, - cmdq->eq.db_record_dma_addr); + dma_pool_free(dev->db_pool, cmdq->eq.dbrec, cmdq->eq.dbrec_dma); dma_free_coherent(&dev->pdev->dev, cmdq->sq.depth << SQEBB_SHIFT, cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr); - dma_pool_free(dev->db_pool, cmdq->sq.db_record, - cmdq->sq.db_record_dma_addr); + dma_pool_free(dev->db_pool, cmdq->sq.dbrec, cmdq->sq.dbrec_dma); dma_free_coherent(&dev->pdev->dev, cmdq->cq.depth << CQE_SHIFT, cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr); - dma_pool_free(dev->db_pool, cmdq->cq.db_record, - cmdq->cq.db_record_dma_addr); + dma_pool_free(dev->db_pool, cmdq->cq.dbrec, cmdq->cq.dbrec_dma); } static void *get_next_valid_cmdq_cqe(struct erdma_cmdq *cmdq) diff --git a/drivers/infiniband/hw/erdma/erdma_cq.c b/drivers/infiniband/hw/erdma/erdma_cq.c index c1cb5568eab2d..70f89f0162aa7 100644 --- a/drivers/infiniband/hw/erdma/erdma_cq.c +++ b/drivers/infiniband/hw/erdma/erdma_cq.c @@ -26,7 +26,7 @@ static void notify_cq(struct erdma_cq *cq, u8 solcitied) FIELD_PREP(ERDMA_CQDB_CMDSN_MASK, cq->kern_cq.cmdsn) | FIELD_PREP(ERDMA_CQDB_CI_MASK, cq->kern_cq.ci); - *cq->kern_cq.db_record = db_data; + *cq->kern_cq.dbrec = db_data; writeq(db_data, cq->kern_cq.db); } diff --git a/drivers/infiniband/hw/erdma/erdma_eq.c b/drivers/infiniband/hw/erdma/erdma_eq.c index 809c33628f38c..0a4746e6d05c2 100644 --- a/drivers/infiniband/hw/erdma/erdma_eq.c +++ b/drivers/infiniband/hw/erdma/erdma_eq.c @@ -13,7 +13,7 @@ void notify_eq(struct erdma_eq *eq) u64 db_data = FIELD_PREP(ERDMA_EQDB_CI_MASK, eq->ci) | FIELD_PREP(ERDMA_EQDB_ARM_MASK, 1); - *eq->db_record = db_data; + *eq->dbrec = db_data; writeq(db_data, eq->db); atomic64_inc(&eq->notify_num); @@ -97,9 +97,8 @@ int erdma_aeq_init(struct erdma_dev *dev) atomic64_set(&eq->notify_num, 0); eq->db = dev->func_bar + ERDMA_REGS_AEQ_DB_REG; - eq->db_record = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, - &eq->db_record_dma_addr); - if (!eq->db_record) + eq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &eq->dbrec_dma); + if (!eq->dbrec) goto err_out; erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_H_REG, @@ -107,8 +106,7 @@ int erdma_aeq_init(struct erdma_dev *dev) erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_L_REG, lower_32_bits(eq->qbuf_dma_addr)); erdma_reg_write32(dev, ERDMA_REGS_AEQ_DEPTH_REG, eq->depth); - erdma_reg_write64(dev, ERDMA_AEQ_DB_HOST_ADDR_REG, - eq->db_record_dma_addr); + erdma_reg_write64(dev, ERDMA_AEQ_DB_HOST_ADDR_REG, eq->dbrec_dma); return 0; @@ -126,7 +124,7 @@ void erdma_aeq_destroy(struct erdma_dev *dev) dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf, eq->qbuf_dma_addr); - dma_pool_free(dev->db_pool, eq->db_record, eq->db_record_dma_addr); + dma_pool_free(dev->db_pool, eq->dbrec, eq->dbrec_dma); } void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb) @@ -226,8 +224,8 @@ static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq) req.qtype = ERDMA_EQ_TYPE_CEQ; /* Vector index is the same as EQN. */ req.vector_idx = eqn; - req.db_dma_addr_l = lower_32_bits(eq->db_record_dma_addr); - req.db_dma_addr_h = upper_32_bits(eq->db_record_dma_addr); + req.db_dma_addr_l = lower_32_bits(eq->dbrec_dma); + req.db_dma_addr_h = upper_32_bits(eq->dbrec_dma); return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); } @@ -251,9 +249,8 @@ static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn) eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG + (ceqn + 1) * ERDMA_DB_SIZE; - eq->db_record = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, - &eq->db_record_dma_addr); - if (!eq->db_record) { + eq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &eq->dbrec_dma); + if (!eq->dbrec) { dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf, eq->qbuf_dma_addr); return -ENOMEM; @@ -290,7 +287,7 @@ static void erdma_ceq_uninit_one(struct erdma_dev *dev, u16 ceqn) dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf, eq->qbuf_dma_addr); - dma_pool_free(dev->db_pool, eq->db_record, eq->db_record_dma_addr); + dma_pool_free(dev->db_pool, eq->dbrec, eq->dbrec_dma); } int erdma_ceqs_init(struct erdma_dev *dev) diff --git a/drivers/infiniband/hw/erdma/erdma_hw.h b/drivers/infiniband/hw/erdma/erdma_hw.h index 3212a12227606..05978f3b14759 100644 --- a/drivers/infiniband/hw/erdma/erdma_hw.h +++ b/drivers/infiniband/hw/erdma/erdma_hw.h @@ -240,7 +240,7 @@ struct erdma_cmdq_create_cq_req { u32 qbuf_addr_l; u32 qbuf_addr_h; u32 cfg1; - u64 cq_db_info_addr; + u64 cq_dbrec_dma; u32 first_page_offset; u32 cfg2; }; @@ -335,8 +335,8 @@ struct erdma_cmdq_create_qp_req { u64 rq_buf_addr; u32 sq_mtt_cfg; u32 rq_mtt_cfg; - u64 sq_db_info_dma_addr; - u64 rq_db_info_dma_addr; + u64 sq_dbrec_dma; + u64 rq_dbrec_dma; u64 sq_mtt_entry[3]; u64 rq_mtt_entry[3]; diff --git a/drivers/infiniband/hw/erdma/erdma_qp.c b/drivers/infiniband/hw/erdma/erdma_qp.c index 6d0330badd68e..4d1f9114cd97c 100644 --- a/drivers/infiniband/hw/erdma/erdma_qp.c +++ b/drivers/infiniband/hw/erdma/erdma_qp.c @@ -492,7 +492,7 @@ static void kick_sq_db(struct erdma_qp *qp, u16 pi) u64 db_data = FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, QP_ID(qp)) | FIELD_PREP(ERDMA_SQE_HDR_WQEBB_INDEX_MASK, pi); - *(u64 *)qp->kern_qp.sq_db_info = db_data; + *(u64 *)qp->kern_qp.sq_dbrec = db_data; writeq(db_data, qp->kern_qp.hw_sq_db); } @@ -557,7 +557,7 @@ static int erdma_post_recv_one(struct erdma_qp *qp, return -EINVAL; } - *(u64 *)qp->kern_qp.rq_db_info = *(u64 *)rqe; + *(u64 *)qp->kern_qp.rq_dbrec = *(u64 *)rqe; writeq(*(u64 *)rqe, qp->kern_qp.hw_rq_db); qp->kern_qp.rwr_tbl[qp->kern_qp.rq_pi & (qp->attrs.rq_size - 1)] = diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index b78ddca1483ea..40c9b6e46b82b 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -76,8 +76,8 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp) req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr; req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr; - req.sq_db_info_dma_addr = qp->kern_qp.sq_db_info_dma_addr; - req.rq_db_info_dma_addr = qp->kern_qp.rq_db_info_dma_addr; + req.sq_dbrec_dma = qp->kern_qp.sq_dbrec_dma; + req.rq_dbrec_dma = qp->kern_qp.rq_dbrec_dma; } else { user_qp = &qp->user_qp; req.sq_cqn_mtt_cfg = FIELD_PREP( @@ -105,8 +105,8 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp) assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg, &req.rq_buf_addr, req.rq_mtt_entry); - req.sq_db_info_dma_addr = user_qp->sq_db_info_dma_addr; - req.rq_db_info_dma_addr = user_qp->rq_db_info_dma_addr; + req.sq_dbrec_dma = user_qp->sq_dbrec_dma; + req.rq_dbrec_dma = user_qp->rq_dbrec_dma; if (uctx->ext_db.enable) { req.sq_cqn_mtt_cfg |= @@ -207,7 +207,7 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq) ERDMA_MR_MTT_0LEVEL); req.first_page_offset = 0; - req.cq_db_info_addr = cq->kern_cq.db_record_dma_addr; + req.cq_dbrec_dma = cq->kern_cq.dbrec_dma; } else { mem = &cq->user_cq.qbuf_mem; req.cfg0 |= @@ -230,7 +230,7 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq) mem->mtt_nents); req.first_page_offset = mem->page_offset; - req.cq_db_info_addr = cq->user_cq.db_info_dma_addr; + req.cq_dbrec_dma = cq->user_cq.dbrec_dma; if (uctx->ext_db.enable) { req.cfg1 |= FIELD_PREP( @@ -484,9 +484,9 @@ static void free_kernel_qp(struct erdma_qp *qp) qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); - if (qp->kern_qp.sq_db_info) - dma_pool_free(dev->db_pool, qp->kern_qp.sq_db_info, - qp->kern_qp.sq_db_info_dma_addr); + if (qp->kern_qp.sq_dbrec) + dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec, + qp->kern_qp.sq_dbrec_dma); if (qp->kern_qp.rq_buf) dma_free_coherent(&dev->pdev->dev, @@ -494,9 +494,9 @@ static void free_kernel_qp(struct erdma_qp *qp) qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); - if (qp->kern_qp.rq_db_info) - dma_pool_free(dev->db_pool, qp->kern_qp.rq_db_info, - qp->kern_qp.rq_db_info_dma_addr); + if (qp->kern_qp.rq_dbrec) + dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec, + qp->kern_qp.rq_dbrec_dma); } static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, @@ -527,9 +527,9 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, if (!kqp->sq_buf) goto err_out; - kqp->sq_db_info = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, - &kqp->sq_db_info_dma_addr); - if (!kqp->sq_db_info) + kqp->sq_dbrec = + dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma); + if (!kqp->sq_dbrec) goto err_out; size = qp->attrs.rq_size << RQE_SHIFT; @@ -538,9 +538,9 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, if (!kqp->rq_buf) goto err_out; - kqp->rq_db_info = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, - &kqp->rq_db_info_dma_addr); - if (!kqp->rq_db_info) + kqp->rq_dbrec = + dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma); + if (!kqp->rq_dbrec) goto err_out; return 0; @@ -876,9 +876,9 @@ erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx, } static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, - u64 va, u32 len, u64 db_info_va) + u64 va, u32 len, u64 dbrec_va) { - dma_addr_t db_info_dma_addr; + dma_addr_t dbrec_dma; u32 rq_offset; int ret; @@ -901,14 +901,14 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, if (ret) goto put_sq_mtt; - ret = erdma_map_user_dbrecords(uctx, db_info_va, + ret = erdma_map_user_dbrecords(uctx, dbrec_va, &qp->user_qp.user_dbr_page, - &db_info_dma_addr); + &dbrec_dma); if (ret) goto put_rq_mtt; - qp->user_qp.sq_db_info_dma_addr = db_info_dma_addr; - qp->user_qp.rq_db_info_dma_addr = db_info_dma_addr + ERDMA_DB_SIZE; + qp->user_qp.sq_dbrec_dma = dbrec_dma; + qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE; return 0; @@ -1251,8 +1251,8 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) if (rdma_is_kernel_res(&cq->ibcq.res)) { dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); - dma_pool_free(dev->db_pool, cq->kern_cq.db_record, - cq->kern_cq.db_record_dma_addr); + dma_pool_free(dev->db_pool, cq->kern_cq.dbrec, + cq->kern_cq.dbrec_dma); } else { erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); put_mtt_entries(dev, &cq->user_cq.qbuf_mem); @@ -1592,7 +1592,7 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va, &cq->user_cq.user_dbr_page, - &cq->user_cq.db_info_dma_addr); + &cq->user_cq.dbrec_dma); if (ret) put_mtt_entries(dev, &cq->user_cq.qbuf_mem); @@ -1609,9 +1609,9 @@ static int erdma_init_kernel_cq(struct erdma_cq *cq) if (!cq->kern_cq.qbuf) return -ENOMEM; - cq->kern_cq.db_record = dma_pool_zalloc( - dev->db_pool, GFP_KERNEL, &cq->kern_cq.db_record_dma_addr); - if (!cq->kern_cq.db_record) + cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, + &cq->kern_cq.dbrec_dma); + if (!cq->kern_cq.dbrec) goto err_out; spin_lock_init(&cq->kern_cq.lock); @@ -1690,8 +1690,8 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, } else { dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT, cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); - dma_pool_free(dev->db_pool, cq->kern_cq.db_record, - cq->kern_cq.db_record_dma_addr); + dma_pool_free(dev->db_pool, cq->kern_cq.dbrec, + cq->kern_cq.dbrec_dma); } err_out_xa: diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h index b02ffdc8c811d..4f02ba06b210a 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.h +++ b/drivers/infiniband/hw/erdma/erdma_verbs.h @@ -140,8 +140,8 @@ struct erdma_uqp { struct erdma_mem sq_mem; struct erdma_mem rq_mem; - dma_addr_t sq_db_info_dma_addr; - dma_addr_t rq_db_info_dma_addr; + dma_addr_t sq_dbrec_dma; + dma_addr_t rq_dbrec_dma; struct erdma_user_dbrecords_page *user_dbr_page; @@ -167,11 +167,11 @@ struct erdma_kqp { void *rq_buf; dma_addr_t rq_buf_dma_addr; - void *sq_db_info; - void *rq_db_info; + void *sq_dbrec; + void *rq_dbrec; - dma_addr_t sq_db_info_dma_addr; - dma_addr_t rq_db_info_dma_addr; + dma_addr_t sq_dbrec_dma; + dma_addr_t rq_dbrec_dma; u8 sig_all; }; @@ -249,14 +249,14 @@ struct erdma_kcq_info { spinlock_t lock; u8 __iomem *db; - u64 *db_record; - dma_addr_t db_record_dma_addr; + u64 *dbrec; + dma_addr_t dbrec_dma; }; struct erdma_ucq_info { struct erdma_mem qbuf_mem; struct erdma_user_dbrecords_page *user_dbr_page; - dma_addr_t db_info_dma_addr; + dma_addr_t dbrec_dma; }; struct erdma_cq { From df0e16bab5c7f13d083484e0ab7488cc7ca510f1 Mon Sep 17 00:00:00 2001 From: Boshi Yu Date: Mon, 11 Mar 2024 19:38:21 +0800 Subject: [PATCH 0052/1702] RDMA/erdma: Remove unnecessary __GFP_ZERO flag The dma_alloc_coherent() interface automatically zero the memory returned. Thus, we do not need to specify the __GFP_ZERO flag explicitly when we call dma_alloc_coherent(). Reviewed-by: Cheng Xu Signed-off-by: Boshi Yu Link: https://lore.kernel.org/r/20240311113821.22482-4-boshiyu@alibaba-inc.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/erdma/erdma_cmdq.c | 6 ++---- drivers/infiniband/hw/erdma/erdma_eq.c | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/drivers/infiniband/hw/erdma/erdma_cmdq.c b/drivers/infiniband/hw/erdma/erdma_cmdq.c index 0ac2683cfccf6..43ff40b5a09d9 100644 --- a/drivers/infiniband/hw/erdma/erdma_cmdq.c +++ b/drivers/infiniband/hw/erdma/erdma_cmdq.c @@ -127,8 +127,7 @@ static int erdma_cmdq_cq_init(struct erdma_dev *dev) cq->depth = cmdq->sq.depth; cq->qbuf = dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, - &cq->qbuf_dma_addr, - GFP_KERNEL | __GFP_ZERO); + &cq->qbuf_dma_addr, GFP_KERNEL); if (!cq->qbuf) return -ENOMEM; @@ -162,8 +161,7 @@ static int erdma_cmdq_eq_init(struct erdma_dev *dev) eq->depth = cmdq->max_outstandings; eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, - &eq->qbuf_dma_addr, - GFP_KERNEL | __GFP_ZERO); + &eq->qbuf_dma_addr, GFP_KERNEL); if (!eq->qbuf) return -ENOMEM; diff --git a/drivers/infiniband/hw/erdma/erdma_eq.c b/drivers/infiniband/hw/erdma/erdma_eq.c index 0a4746e6d05c2..84ccdd8144c9e 100644 --- a/drivers/infiniband/hw/erdma/erdma_eq.c +++ b/drivers/infiniband/hw/erdma/erdma_eq.c @@ -87,8 +87,7 @@ int erdma_aeq_init(struct erdma_dev *dev) eq->depth = ERDMA_DEFAULT_EQ_DEPTH; eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, - &eq->qbuf_dma_addr, - GFP_KERNEL | __GFP_ZERO); + &eq->qbuf_dma_addr, GFP_KERNEL); if (!eq->qbuf) return -ENOMEM; @@ -237,8 +236,7 @@ static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn) eq->depth = ERDMA_DEFAULT_EQ_DEPTH; eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, - &eq->qbuf_dma_addr, - GFP_KERNEL | __GFP_ZERO); + &eq->qbuf_dma_addr, GFP_KERNEL); if (!eq->qbuf) return -ENOMEM; From ca537a34775c103f7b14d7bbd976403f1d1525d8 Mon Sep 17 00:00:00 2001 From: Wenchao Hao Date: Mon, 18 Mar 2024 17:23:20 +0800 Subject: [PATCH 0053/1702] RDMA/restrack: Fix potential invalid address access struct rdma_restrack_entry's kern_name was set to KBUILD_MODNAME in ib_create_cq(), while if the module exited but forgot del this rdma_restrack_entry, it would cause a invalid address access in rdma_restrack_clean() when print the owner of this rdma_restrack_entry. These code is used to help find one forgotten PD release in one of the ULPs. But it is not needed anymore, so delete them. Signed-off-by: Wenchao Hao Link: https://lore.kernel.org/r/20240318092320.1215235-1-haowenchao2@huawei.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/restrack.c | 51 +----------------------------- 1 file changed, 1 insertion(+), 50 deletions(-) diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c index 01a499a8b88db..438ed35881752 100644 --- a/drivers/infiniband/core/restrack.c +++ b/drivers/infiniband/core/restrack.c @@ -37,22 +37,6 @@ int rdma_restrack_init(struct ib_device *dev) return 0; } -static const char *type2str(enum rdma_restrack_type type) -{ - static const char * const names[RDMA_RESTRACK_MAX] = { - [RDMA_RESTRACK_PD] = "PD", - [RDMA_RESTRACK_CQ] = "CQ", - [RDMA_RESTRACK_QP] = "QP", - [RDMA_RESTRACK_CM_ID] = "CM_ID", - [RDMA_RESTRACK_MR] = "MR", - [RDMA_RESTRACK_CTX] = "CTX", - [RDMA_RESTRACK_COUNTER] = "COUNTER", - [RDMA_RESTRACK_SRQ] = "SRQ", - }; - - return names[type]; -}; - /** * rdma_restrack_clean() - clean resource tracking * @dev: IB device @@ -60,47 +44,14 @@ static const char *type2str(enum rdma_restrack_type type) void rdma_restrack_clean(struct ib_device *dev) { struct rdma_restrack_root *rt = dev->res; - struct rdma_restrack_entry *e; - char buf[TASK_COMM_LEN]; - bool found = false; - const char *owner; int i; for (i = 0 ; i < RDMA_RESTRACK_MAX; i++) { struct xarray *xa = &dev->res[i].xa; - if (!xa_empty(xa)) { - unsigned long index; - - if (!found) { - pr_err("restrack: %s", CUT_HERE); - dev_err(&dev->dev, "BUG: RESTRACK detected leak of resources\n"); - } - xa_for_each(xa, index, e) { - if (rdma_is_kernel_res(e)) { - owner = e->kern_name; - } else { - /* - * There is no need to call get_task_struct here, - * because we can be here only if there are more - * get_task_struct() call than put_task_struct(). - */ - get_task_comm(buf, e->task); - owner = buf; - } - - pr_err("restrack: %s %s object allocated by %s is not freed\n", - rdma_is_kernel_res(e) ? "Kernel" : - "User", - type2str(e->type), owner); - } - found = true; - } + WARN_ON(!xa_empty(xa)); xa_destroy(xa); } - if (found) - pr_err("restrack: %s", CUT_HERE); - kfree(rt); } From 46f5be7cd4bceb3a503c544b3dab7b75fe4bb96b Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Tue, 26 Mar 2024 13:08:05 -0700 Subject: [PATCH 0054/1702] RDMA/mana_ib: Introduce helpers to create and destroy mana queues Intoduce helpers to work with mana ib queues (struct mana_ib_queue). A queue always consists of umem, gdma_region, and id. A queue can become a WQ or a CQ. Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1711483688-24358-2-git-send-email-kotaranov@linux.microsoft.com Reviewed-by: Long Li Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/main.c | 43 ++++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 10 +++++++ 2 files changed, 53 insertions(+) diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index 71e33feee61bb..4524c6b807487 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -237,6 +237,49 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) ibdev_dbg(ibdev, "Failed to destroy doorbell page %d\n", ret); } +int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, + struct mana_ib_queue *queue) +{ + struct ib_umem *umem; + int err; + + queue->umem = NULL; + queue->id = INVALID_QUEUE_ID; + queue->gdma_region = GDMA_INVALID_DMA_REGION; + + umem = ib_umem_get(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(umem)) { + err = PTR_ERR(umem); + ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %d\n", err); + return err; + } + + err = mana_ib_create_zero_offset_dma_region(mdev, umem, &queue->gdma_region); + if (err) { + ibdev_dbg(&mdev->ib_dev, "Failed to create dma region, %d\n", err); + goto free_umem; + } + queue->umem = umem; + + ibdev_dbg(&mdev->ib_dev, + "create_dma_region ret %d gdma_region 0x%llx\n", + err, queue->gdma_region); + + return 0; +free_umem: + ib_umem_release(umem); + return err; +} + +void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue) +{ + /* Ignore return code as there is not much we can do about it. + * The error message is printed inside. + */ + mana_ib_gd_destroy_dma_region(mdev, queue->gdma_region); + ib_umem_release(queue->umem); +} + static int mana_ib_gd_first_dma_region(struct mana_ib_dev *dev, struct gdma_context *gc, diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index f83390eebb7d7..859fd3bfc764f 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -45,6 +45,12 @@ struct mana_ib_adapter_caps { u32 max_inline_data_size; }; +struct mana_ib_queue { + struct ib_umem *umem; + u64 gdma_region; + u64 id; +}; + struct mana_ib_dev { struct ib_device ib_dev; struct gdma_dev *gdma_dev; @@ -169,6 +175,10 @@ int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem, int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, mana_handle_t gdma_region); +int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, + struct mana_ib_queue *queue); +void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue); + struct ib_wq *mana_ib_create_wq(struct ib_pd *pd, struct ib_wq_init_attr *init_attr, struct ib_udata *udata); From 60a7ac0b8bec5df9764b7460ffee91fc981e8a31 Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Tue, 26 Mar 2024 13:08:06 -0700 Subject: [PATCH 0055/1702] RDMA/mana_ib: Use struct mana_ib_queue for CQs Use struct mana_ib_queue and its helpers for CQs Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1711483688-24358-3-git-send-email-kotaranov@linux.microsoft.com Reviewed-by: Long Li Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/cq.c | 52 ++++++---------------------- drivers/infiniband/hw/mana/mana_ib.h | 4 +-- drivers/infiniband/hw/mana/qp.c | 26 +++++++------- 3 files changed, 24 insertions(+), 58 deletions(-) diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index 4a71e678d09c1..c9129218f1be1 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -39,37 +39,13 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, } cq->cqe = attr->cqe; - cq->umem = ib_umem_get(ibdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE, - IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(cq->umem)) { - err = PTR_ERR(cq->umem); - ibdev_dbg(ibdev, "Failed to get umem for create cq, err %d\n", - err); - return err; - } - - err = mana_ib_create_zero_offset_dma_region(mdev, cq->umem, &cq->gdma_region); + err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE, &cq->queue); if (err) { - ibdev_dbg(ibdev, - "Failed to create dma region for create cq, %d\n", - err); - goto err_release_umem; + ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err); + return err; } - ibdev_dbg(ibdev, - "create_dma_region ret %d gdma_region 0x%llx\n", - err, cq->gdma_region); - - /* - * The CQ ID is not known at this time. The ID is generated at create_qp - */ - cq->id = INVALID_QUEUE_ID; - return 0; - -err_release_umem: - ib_umem_release(cq->umem); - return err; } int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) @@ -78,24 +54,16 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) struct ib_device *ibdev = ibcq->device; struct mana_ib_dev *mdev; struct gdma_context *gc; - int err; mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); gc = mdev_to_gc(mdev); - err = mana_ib_gd_destroy_dma_region(mdev, cq->gdma_region); - if (err) { - ibdev_dbg(ibdev, - "Failed to destroy dma region, %d\n", err); - return err; - } - - if (cq->id != INVALID_QUEUE_ID) { - kfree(gc->cq_table[cq->id]); - gc->cq_table[cq->id] = NULL; + if (cq->queue.id != INVALID_QUEUE_ID) { + kfree(gc->cq_table[cq->queue.id]); + gc->cq_table[cq->queue.id] = NULL; } - ib_umem_release(cq->umem); + mana_ib_destroy_queue(mdev, &cq->queue); return 0; } @@ -114,7 +82,7 @@ int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) struct gdma_queue *gdma_cq; /* Create CQ table entry */ - WARN_ON(gc->cq_table[cq->id]); + WARN_ON(gc->cq_table[cq->queue.id]); gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL); if (!gdma_cq) return -ENOMEM; @@ -122,7 +90,7 @@ int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) gdma_cq->cq.context = cq; gdma_cq->type = GDMA_CQ; gdma_cq->cq.callback = mana_ib_cq_handler; - gdma_cq->id = cq->id; - gc->cq_table[cq->id] = gdma_cq; + gdma_cq->id = cq->queue.id; + gc->cq_table[cq->queue.id] = gdma_cq; return 0; } diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index 859fd3bfc764f..6acb5c281c368 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -88,10 +88,8 @@ struct mana_ib_mr { struct mana_ib_cq { struct ib_cq ibcq; - struct ib_umem *umem; + struct mana_ib_queue queue; int cqe; - u64 gdma_region; - u64 id; u32 comp_vector; }; diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 6e7627745c957..d7485ee6a6854 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -197,7 +197,7 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, wq_spec.gdma_region = wq->gdma_region; wq_spec.queue_size = wq->wq_buf_size; - cq_spec.gdma_region = cq->gdma_region; + cq_spec.gdma_region = cq->queue.gdma_region; cq_spec.queue_size = cq->cqe * COMP_ENTRY_SIZE; cq_spec.modr_ctx_id = 0; eq = &mpc->ac->eqs[cq->comp_vector % gc->max_num_queues]; @@ -213,16 +213,16 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, /* The GDMA regions are now owned by the WQ object */ wq->gdma_region = GDMA_INVALID_DMA_REGION; - cq->gdma_region = GDMA_INVALID_DMA_REGION; + cq->queue.gdma_region = GDMA_INVALID_DMA_REGION; wq->id = wq_spec.queue_index; - cq->id = cq_spec.queue_index; + cq->queue.id = cq_spec.queue_index; ibdev_dbg(&mdev->ib_dev, "ret %d rx_object 0x%llx wq id %llu cq id %llu\n", - ret, wq->rx_object, wq->id, cq->id); + ret, wq->rx_object, wq->id, cq->queue.id); - resp.entries[i].cqid = cq->id; + resp.entries[i].cqid = cq->queue.id; resp.entries[i].wqid = wq->id; mana_ind_table[i] = wq->rx_object; @@ -232,7 +232,7 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, if (ret) goto fail; - gdma_cq_allocated[i] = gc->cq_table[cq->id]; + gdma_cq_allocated[i] = gc->cq_table[cq->queue.id]; } resp.num_entries = i; @@ -264,7 +264,7 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, wq = container_of(ibwq, struct mana_ib_wq, ibwq); cq = container_of(ibcq, struct mana_ib_cq, ibcq); - gc->cq_table[cq->id] = NULL; + gc->cq_table[cq->queue.id] = NULL; kfree(gdma_cq_allocated[i]); mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object); @@ -374,7 +374,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, wq_spec.gdma_region = qp->sq_gdma_region; wq_spec.queue_size = ucmd.sq_buf_size; - cq_spec.gdma_region = send_cq->gdma_region; + cq_spec.gdma_region = send_cq->queue.gdma_region; cq_spec.queue_size = send_cq->cqe * COMP_ENTRY_SIZE; cq_spec.modr_ctx_id = 0; eq_vec = send_cq->comp_vector % gc->max_num_queues; @@ -392,10 +392,10 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, /* The GDMA regions are now owned by the WQ object */ qp->sq_gdma_region = GDMA_INVALID_DMA_REGION; - send_cq->gdma_region = GDMA_INVALID_DMA_REGION; + send_cq->queue.gdma_region = GDMA_INVALID_DMA_REGION; qp->sq_id = wq_spec.queue_index; - send_cq->id = cq_spec.queue_index; + send_cq->queue.id = cq_spec.queue_index; /* Create CQ table entry */ err = mana_ib_install_cq_cb(mdev, send_cq); @@ -404,10 +404,10 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, ibdev_dbg(&mdev->ib_dev, "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n", err, - qp->tx_object, qp->sq_id, send_cq->id); + qp->tx_object, qp->sq_id, send_cq->queue.id); resp.sqid = qp->sq_id; - resp.cqid = send_cq->id; + resp.cqid = send_cq->queue.id; resp.tx_vp_offset = pd->tx_vp_offset; err = ib_copy_to_udata(udata, &resp, sizeof(resp)); @@ -422,7 +422,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, err_release_gdma_cq: kfree(gdma_cq); - gc->cq_table[send_cq->id] = NULL; + gc->cq_table[send_cq->queue.id] = NULL; err_destroy_wq_obj: mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object); From 688bac28e3dc9eb795ae8ea5aa40cb637e289faa Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Tue, 26 Mar 2024 13:08:07 -0700 Subject: [PATCH 0056/1702] RDMA/mana_ib: Use struct mana_ib_queue for WQs Use struct mana_ib_queue and its helpers for WQs Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1711483688-24358-4-git-send-email-kotaranov@linux.microsoft.com Reviewed-by: Long Li Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/mana_ib.h | 4 +--- drivers/infiniband/hw/mana/qp.c | 10 ++++----- drivers/infiniband/hw/mana/wq.c | 31 ++++------------------------ 3 files changed, 10 insertions(+), 35 deletions(-) diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index 6acb5c281c368..a8953ee808d9e 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -59,11 +59,9 @@ struct mana_ib_dev { struct mana_ib_wq { struct ib_wq ibwq; - struct ib_umem *umem; + struct mana_ib_queue queue; int wqe; u32 wq_buf_size; - u64 gdma_region; - u64 id; mana_handle_t rx_object; }; diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index d7485ee6a6854..f606caa758395 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -194,7 +194,7 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, ibcq = ibwq->cq; cq = container_of(ibcq, struct mana_ib_cq, ibcq); - wq_spec.gdma_region = wq->gdma_region; + wq_spec.gdma_region = wq->queue.gdma_region; wq_spec.queue_size = wq->wq_buf_size; cq_spec.gdma_region = cq->queue.gdma_region; @@ -212,18 +212,18 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, } /* The GDMA regions are now owned by the WQ object */ - wq->gdma_region = GDMA_INVALID_DMA_REGION; + wq->queue.gdma_region = GDMA_INVALID_DMA_REGION; cq->queue.gdma_region = GDMA_INVALID_DMA_REGION; - wq->id = wq_spec.queue_index; + wq->queue.id = wq_spec.queue_index; cq->queue.id = cq_spec.queue_index; ibdev_dbg(&mdev->ib_dev, "ret %d rx_object 0x%llx wq id %llu cq id %llu\n", - ret, wq->rx_object, wq->id, cq->queue.id); + ret, wq->rx_object, wq->queue.id, cq->queue.id); resp.entries[i].cqid = cq->queue.id; - resp.entries[i].wqid = wq->id; + resp.entries[i].wqid = wq->queue.id; mana_ind_table[i] = wq->rx_object; diff --git a/drivers/infiniband/hw/mana/wq.c b/drivers/infiniband/hw/mana/wq.c index 7c9c699625734..f959f4b9244f1 100644 --- a/drivers/infiniband/hw/mana/wq.c +++ b/drivers/infiniband/hw/mana/wq.c @@ -13,7 +13,6 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd, container_of(pd->device, struct mana_ib_dev, ib_dev); struct mana_ib_create_wq ucmd = {}; struct mana_ib_wq *wq; - struct ib_umem *umem; int err; if (udata->inlen < sizeof(ucmd)) @@ -32,39 +31,18 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd, ibdev_dbg(&mdev->ib_dev, "ucmd wq_buf_addr 0x%llx\n", ucmd.wq_buf_addr); - umem = ib_umem_get(pd->device, ucmd.wq_buf_addr, ucmd.wq_buf_size, - IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(umem)) { - err = PTR_ERR(umem); + err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue); + if (err) { ibdev_dbg(&mdev->ib_dev, - "Failed to get umem for create wq, err %d\n", err); + "Failed to create queue for create wq, %d\n", err); goto err_free_wq; } - wq->umem = umem; wq->wqe = init_attr->max_wr; wq->wq_buf_size = ucmd.wq_buf_size; wq->rx_object = INVALID_MANA_HANDLE; - - err = mana_ib_create_zero_offset_dma_region(mdev, wq->umem, &wq->gdma_region); - if (err) { - ibdev_dbg(&mdev->ib_dev, - "Failed to create dma region for create wq, %d\n", - err); - goto err_release_umem; - } - - ibdev_dbg(&mdev->ib_dev, - "create_dma_region ret %d gdma_region 0x%llx\n", - err, wq->gdma_region); - - /* WQ ID is returned at wq_create time, doesn't know the value yet */ - return &wq->ibwq; -err_release_umem: - ib_umem_release(umem); - err_free_wq: kfree(wq); @@ -86,8 +64,7 @@ int mana_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata) mdev = container_of(ib_dev, struct mana_ib_dev, ib_dev); - mana_ib_gd_destroy_dma_region(mdev, wq->gdma_region); - ib_umem_release(wq->umem); + mana_ib_destroy_queue(mdev, &wq->queue); kfree(wq); From f10242b3da908dc9d4bfa040e6511a5b86522499 Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Tue, 26 Mar 2024 13:08:08 -0700 Subject: [PATCH 0057/1702] RDMA/mana_ib: Use struct mana_ib_queue for RAW QPs Use struct mana_ib_queue and its helpers for RAW QPs Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1711483688-24358-5-git-send-email-kotaranov@linux.microsoft.com Reviewed-by: Long Li Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/mana_ib.h | 8 +--- drivers/infiniband/hw/mana/qp.c | 56 ++++++++-------------------- 2 files changed, 18 insertions(+), 46 deletions(-) diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index a8953ee808d9e..ceca21cef72ab 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -94,12 +94,8 @@ struct mana_ib_cq { struct mana_ib_qp { struct ib_qp ibqp; - /* Work queue info */ - struct ib_umem *sq_umem; - int sqe; - u64 sq_gdma_region; - u64 sq_id; - mana_handle_t tx_object; + mana_handle_t qp_handle; + struct mana_ib_queue raw_sq; /* The port on the IB device, starting with 1 */ u32 port; diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index f606caa758395..ef0a6dc664d0f 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -297,7 +297,6 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, struct mana_obj_spec cq_spec = {}; struct mana_port_context *mpc; struct net_device *ndev; - struct ib_umem *umem; struct mana_eq *eq; int eq_vec; u32 port; @@ -346,32 +345,15 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n", ucmd.sq_buf_addr, ucmd.port); - umem = ib_umem_get(ibpd->device, ucmd.sq_buf_addr, ucmd.sq_buf_size, - IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(umem)) { - err = PTR_ERR(umem); - ibdev_dbg(&mdev->ib_dev, - "Failed to get umem for create qp-raw, err %d\n", - err); - goto err_free_vport; - } - qp->sq_umem = umem; - - err = mana_ib_create_zero_offset_dma_region(mdev, qp->sq_umem, - &qp->sq_gdma_region); + err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq); if (err) { ibdev_dbg(&mdev->ib_dev, - "Failed to create dma region for create qp-raw, %d\n", - err); - goto err_release_umem; + "Failed to create queue for create qp-raw, err %d\n", err); + goto err_free_vport; } - ibdev_dbg(&mdev->ib_dev, - "create_dma_region ret %d gdma_region 0x%llx\n", - err, qp->sq_gdma_region); - /* Create a WQ on the same port handle used by the Ethernet */ - wq_spec.gdma_region = qp->sq_gdma_region; + wq_spec.gdma_region = qp->raw_sq.gdma_region; wq_spec.queue_size = ucmd.sq_buf_size; cq_spec.gdma_region = send_cq->queue.gdma_region; @@ -382,19 +364,19 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, cq_spec.attached_eq = eq->eq->id; err = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_SQ, &wq_spec, - &cq_spec, &qp->tx_object); + &cq_spec, &qp->qp_handle); if (err) { ibdev_dbg(&mdev->ib_dev, "Failed to create wq for create raw-qp, err %d\n", err); - goto err_destroy_dma_region; + goto err_destroy_queue; } /* The GDMA regions are now owned by the WQ object */ - qp->sq_gdma_region = GDMA_INVALID_DMA_REGION; + qp->raw_sq.gdma_region = GDMA_INVALID_DMA_REGION; send_cq->queue.gdma_region = GDMA_INVALID_DMA_REGION; - qp->sq_id = wq_spec.queue_index; + qp->raw_sq.id = wq_spec.queue_index; send_cq->queue.id = cq_spec.queue_index; /* Create CQ table entry */ @@ -403,10 +385,10 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, goto err_destroy_wq_obj; ibdev_dbg(&mdev->ib_dev, - "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n", err, - qp->tx_object, qp->sq_id, send_cq->queue.id); + "ret %d qp->qp_handle 0x%llx sq id %llu cq id %llu\n", err, + qp->qp_handle, qp->raw_sq.id, send_cq->queue.id); - resp.sqid = qp->sq_id; + resp.sqid = qp->raw_sq.id; resp.cqid = send_cq->queue.id; resp.tx_vp_offset = pd->tx_vp_offset; @@ -425,13 +407,10 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, gc->cq_table[send_cq->queue.id] = NULL; err_destroy_wq_obj: - mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object); + mana_destroy_wq_obj(mpc, GDMA_SQ, qp->qp_handle); -err_destroy_dma_region: - mana_ib_gd_destroy_dma_region(mdev, qp->sq_gdma_region); - -err_release_umem: - ib_umem_release(umem); +err_destroy_queue: + mana_ib_destroy_queue(mdev, &qp->raw_sq); err_free_vport: mana_ib_uncfg_vport(mdev, pd, port); @@ -505,12 +484,9 @@ static int mana_ib_destroy_qp_raw(struct mana_ib_qp *qp, struct ib_udata *udata) mpc = netdev_priv(ndev); pd = container_of(ibpd, struct mana_ib_pd, ibpd); - mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object); + mana_destroy_wq_obj(mpc, GDMA_SQ, qp->qp_handle); - if (qp->sq_umem) { - mana_ib_gd_destroy_dma_region(mdev, qp->sq_gdma_region); - ib_umem_release(qp->sq_umem); - } + mana_ib_destroy_queue(mdev, &qp->raw_sq); mana_ib_uncfg_vport(mdev, pd, qp->port); From 9c85158585214770a232ff2efdbdb0e75704743f Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Tue, 12 Mar 2024 10:01:34 +0100 Subject: [PATCH 0058/1702] clk: renesas: r8a779h0: Add SCIF clocks Add the module clocks used by the Serial Communication Interfaces with FIFO (SCIF) on the Renesas R-Car V4M (R8A779H0) SoC. Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/82d731edd4ae4a8cd7683368131095777f4fa172.1709741224.git.geert+renesas@glider.be --- drivers/clk/renesas/r8a779h0-cpg-mssr.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/clk/renesas/r8a779h0-cpg-mssr.c b/drivers/clk/renesas/r8a779h0-cpg-mssr.c index 5c48e645f0c31..4bc35bc912547 100644 --- a/drivers/clk/renesas/r8a779h0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a779h0-cpg-mssr.c @@ -185,6 +185,10 @@ static const struct mssr_mod_clk r8a779h0_mod_clks[] = { DEF_MOD("i2c2", 520, R8A779H0_CLK_S0D6_PER), DEF_MOD("i2c3", 521, R8A779H0_CLK_S0D6_PER), DEF_MOD("rpc-if", 629, R8A779H0_CLK_RPCD2), + DEF_MOD("scif0", 702, R8A779H0_CLK_SASYNCPERD4), + DEF_MOD("scif1", 703, R8A779H0_CLK_SASYNCPERD4), + DEF_MOD("scif3", 704, R8A779H0_CLK_SASYNCPERD4), + DEF_MOD("scif4", 705, R8A779H0_CLK_SASYNCPERD4), DEF_MOD("sdhi0", 706, R8A779H0_CLK_SD0), DEF_MOD("sydm1", 709, R8A779H0_CLK_S0D6_PER), DEF_MOD("sydm2", 710, R8A779H0_CLK_S0D6_PER), From d303735ce5dcd79783e65a3710f7f16d14f8900a Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 22 Mar 2024 14:21:43 +0100 Subject: [PATCH 0059/1702] pinctrl: armada-37xx: remove an unused variable This variable has never been used and can be removed to avoid a W=1 warning: drivers/pinctrl/mvebu/pinctrl-armada-37xx.c:837:6: error: variable 'i' set but not used [-Werror,-Wunused-but-set-variable] 837 | int i = 0; Fixes: 87466ccd9401 ("pinctrl: armada-37xx: Add pin controller support for Armada 37xx") Signed-off-by: Arnd Bergmann Reviewed-by: Andrew Lunn Message-ID: <20240322132205.906729-1-arnd@kernel.org> Signed-off-by: Linus Walleij --- drivers/pinctrl/mvebu/pinctrl-armada-37xx.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c index c34719b7506da..4c4ada06423d7 100644 --- a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c +++ b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c @@ -834,8 +834,6 @@ static int armada_37xx_gpiochip_register(struct platform_device *pdev, static int armada_37xx_add_function(struct armada_37xx_pmx_func *funcs, int *funcsize, const char *name) { - int i = 0; - if (*funcsize <= 0) return -EOVERFLOW; @@ -847,7 +845,6 @@ static int armada_37xx_add_function(struct armada_37xx_pmx_func *funcs, return -EEXIST; } funcs++; - i++; } /* append new unique function */ From 94755a00a4e79236d4bfa6dc671a339828fb38ce Mon Sep 17 00:00:00 2001 From: Justin Stitt Date: Mon, 1 Apr 2024 20:01:51 +0000 Subject: [PATCH 0060/1702] udf: replace deprecated strncpy/strcpy with strscpy strncpy() is deprecated for use on NUL-terminated destination strings [1] and as such we should prefer more robust and less ambiguous string interfaces. Also replace an instance of strcpy() which is also deprecated. s_volume_ident is a NUL-terminated string which is evident from its usage in udf_debug: | udf_debug("volIdent[] = '%s'\n", UDF_SB(sb)->s_volume_ident); s_volume_ident should also be NUL-padded as it is copied out to userspace: | if (copy_to_user((char __user *)arg, | UDF_SB(inode->i_sb)->s_volume_ident, 32)) | return -EFAULT; Considering the above, a suitable replacement is `strscpy_pad` [2] due to the fact that it guarantees both NUL-termination and NUL-padding on the destination buffer. To simplify the code, let's use the new 2-argument version of strscpy_pad() introduced in Commit e6584c3964f2f ("string: Allow 2-argument strscpy()"). Also zero-allocate @outstr so we can safely use a non-@ret length argument. This is just in case udf_dstrCS0toChar() doesn't include the NUL-byte in its return length, we won't truncate @outstr or write garbage bytes either. Link: https://www.kernel.org/doc/html/latest/process/deprecated.html#strncpy-on-nul-terminated-strings [1] Link: https://manpages.debian.org/testing/linux-manual-4.8/strscpy.9.en.html [2] Link: https://github.com/KSPP/linux/issues/90 Cc: linux-hardening@vger.kernel.org Signed-off-by: Justin Stitt Signed-off-by: Jan Kara Message-Id: <20240401-strncpy-fs-udf-super-c-v1-1-80cddab7a281@google.com> --- fs/udf/super.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/udf/super.c b/fs/udf/super.c index ba6b747a3830b..9381a66c6ce58 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -895,7 +895,7 @@ static int udf_load_pvoldesc(struct super_block *sb, sector_t block) int ret; struct timestamp *ts; - outstr = kmalloc(128, GFP_KERNEL); + outstr = kzalloc(128, GFP_KERNEL); if (!outstr) return -ENOMEM; @@ -921,11 +921,11 @@ static int udf_load_pvoldesc(struct super_block *sb, sector_t block) ret = udf_dstrCS0toChar(sb, outstr, 31, pvoldesc->volIdent, 32); if (ret < 0) { - strcpy(UDF_SB(sb)->s_volume_ident, "InvalidName"); + strscpy_pad(UDF_SB(sb)->s_volume_ident, "InvalidName"); pr_warn("incorrect volume identification, setting to " "'InvalidName'\n"); } else { - strncpy(UDF_SB(sb)->s_volume_ident, outstr, ret); + strscpy_pad(UDF_SB(sb)->s_volume_ident, outstr); } udf_debug("volIdent[] = '%s'\n", UDF_SB(sb)->s_volume_ident); From 197080156f27b6bf8cf198e9313dfbb94769c736 Mon Sep 17 00:00:00 2001 From: Zhiguo Niu Date: Wed, 27 Mar 2024 16:53:40 +0800 Subject: [PATCH 0061/1702] f2fs: fix to adjust appropirate defragment pg_end A length that exceeds the real size of the inode may be specified from user, although these out-of-range areas are not mapped, but they still need to be check in while loop, which is unnecessary. Signed-off-by: Zhiguo Niu Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 719064730f9a6..bd55bbb4fa672 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -2626,12 +2626,13 @@ static int f2fs_defragment_range(struct f2fs_sb_info *sbi, bool fragmented = false; int err; - pg_start = range->start >> PAGE_SHIFT; - pg_end = (range->start + range->len) >> PAGE_SHIFT; - f2fs_balance_fs(sbi, true); inode_lock(inode); + pg_start = range->start >> PAGE_SHIFT; + pg_end = min_t(pgoff_t, + (range->start + range->len) >> PAGE_SHIFT, + DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE)); if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) { err = -EINVAL; @@ -2646,8 +2647,9 @@ static int f2fs_defragment_range(struct f2fs_sb_info *sbi, } /* writeback all dirty pages in the range */ - err = filemap_write_and_wait_range(inode->i_mapping, range->start, - range->start + range->len - 1); + err = filemap_write_and_wait_range(inode->i_mapping, + pg_start << PAGE_SHIFT, + (pg_end << PAGE_SHIFT) - 1); if (err) goto out; From e5e8a58023707472e5dbe9bc7b473a8703b401e0 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 29 Mar 2024 12:55:15 +0200 Subject: [PATCH 0062/1702] pinctrl: aw9523: Destroy mutex on ->remove() If aw9523_hw_init() fails on ->remove() the mutex left alive. Destroy it in that case as well. While at it, remove never true check at the beginning of the function. Signed-off-by: Andy Shevchenko Message-ID: <20240329105634.712457-2-andy.shevchenko@gmail.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-aw9523.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/drivers/pinctrl/pinctrl-aw9523.c b/drivers/pinctrl/pinctrl-aw9523.c index 4edd371c469fb..66629af0b88b4 100644 --- a/drivers/pinctrl/pinctrl-aw9523.c +++ b/drivers/pinctrl/pinctrl-aw9523.c @@ -1067,10 +1067,6 @@ static int aw9523_probe(struct i2c_client *client) static void aw9523_remove(struct i2c_client *client) { struct aw9523 *awi = i2c_get_clientdata(client); - int ret; - - if (!awi) - return; /* * If the chip VIO is connected to a regulator that we can turn @@ -1082,10 +1078,8 @@ static void aw9523_remove(struct i2c_client *client) regulator_disable(awi->vio_vreg); } else { mutex_lock(&awi->i2c_lock); - ret = aw9523_hw_init(awi); + aw9523_hw_init(awi); mutex_unlock(&awi->i2c_lock); - if (ret) - return; } mutex_destroy(&awi->i2c_lock); From f91eafcb18e096108cd19d24ab71a0db5bc12416 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 29 Mar 2024 12:55:16 +0200 Subject: [PATCH 0063/1702] pinctrl: aw9523: Use correct error code for not supported functionality The pin control subsystem internally uses ENOTSUPP for the not supported functionality. The checkpatch is false positive about this error code. Signed-off-by: Andy Shevchenko Message-ID: <20240329105634.712457-3-andy.shevchenko@gmail.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-aw9523.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/pinctrl/pinctrl-aw9523.c b/drivers/pinctrl/pinctrl-aw9523.c index 66629af0b88b4..65d523697b731 100644 --- a/drivers/pinctrl/pinctrl-aw9523.c +++ b/drivers/pinctrl/pinctrl-aw9523.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -239,7 +240,7 @@ static int aw9523_pcfg_param_to_reg(enum pin_config_param pcp, int pin, u8 *r) reg = AW9523_REG_OUT_STATE(pin); break; default: - return -EOPNOTSUPP; + return -ENOTSUPP; } *r = reg; @@ -290,7 +291,7 @@ static int aw9523_pconf_get(struct pinctrl_dev *pctldev, unsigned int pin, val = FIELD_GET(AW9523_GCR_GPOMD_MASK, val); break; default: - return -EOPNOTSUPP; + return -ENOTSUPP; } if (val < 1) return -EINVAL; @@ -344,7 +345,7 @@ static int aw9523_pconf_set(struct pinctrl_dev *pctldev, unsigned int pin, case PIN_CONFIG_DRIVE_OPEN_DRAIN: /* Open-Drain is supported only on port 0 */ if (pin >= AW9523_PINS_PER_PORT) { - rc = -EOPNOTSUPP; + rc = -ENOTSUPP; goto end; } mask = AW9523_GCR_GPOMD_MASK; @@ -361,7 +362,7 @@ static int aw9523_pconf_set(struct pinctrl_dev *pctldev, unsigned int pin, val = AW9523_GCR_GPOMD_MASK; break; default: - rc = -EOPNOTSUPP; + rc = -ENOTSUPP; goto end; } From 091655b9285d837db520381924c689bd5dc5d286 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 29 Mar 2024 12:55:17 +0200 Subject: [PATCH 0064/1702] pinctrl: aw9523: Always try both ports in aw9523_gpio_set_multiple() The ports are equivalent from the user's point of view. Don't limit trying them both if writing to one fails. Signed-off-by: Andy Shevchenko Message-ID: <20240329105634.712457-4-andy.shevchenko@gmail.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-aw9523.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/pinctrl/pinctrl-aw9523.c b/drivers/pinctrl/pinctrl-aw9523.c index 65d523697b731..d93640a02d1d3 100644 --- a/drivers/pinctrl/pinctrl-aw9523.c +++ b/drivers/pinctrl/pinctrl-aw9523.c @@ -653,7 +653,7 @@ static void aw9523_gpio_set_multiple(struct gpio_chip *chip, struct aw9523 *awi = gpiochip_get_data(chip); u8 mask_lo, mask_hi, bits_lo, bits_hi; unsigned int reg; - int ret = 0; + int ret; mask_lo = *mask & U8_MAX; mask_hi = (*mask >> 8) & U8_MAX; @@ -663,10 +663,8 @@ static void aw9523_gpio_set_multiple(struct gpio_chip *chip, bits_hi = (*bits >> 8) & U8_MAX; ret = regmap_write_bits(awi->regmap, reg, mask_hi, bits_hi); - if (ret) { + if (ret) dev_warn(awi->dev, "Cannot write port1 out level\n"); - goto out; - } } if (mask_lo) { reg = AW9523_REG_OUT_STATE(0); @@ -675,7 +673,6 @@ static void aw9523_gpio_set_multiple(struct gpio_chip *chip, if (ret) dev_warn(awi->dev, "Cannot write port0 out level\n"); } -out: mutex_unlock(&awi->i2c_lock); } From 418ee9488ff74ab4ada3a539a2840dda9e56f847 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 29 Mar 2024 12:55:18 +0200 Subject: [PATCH 0065/1702] pinctrl: aw9523: Make use of struct pinfunction and PINCTRL_PINFUNCTION() Since pin control provides a generic data type and a macro for the pin function definition, use them in the driver. Signed-off-by: Andy Shevchenko Message-ID: <20240329105634.712457-5-andy.shevchenko@gmail.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-aw9523.c | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/drivers/pinctrl/pinctrl-aw9523.c b/drivers/pinctrl/pinctrl-aw9523.c index d93640a02d1d3..7db901216a94c 100644 --- a/drivers/pinctrl/pinctrl-aw9523.c +++ b/drivers/pinctrl/pinctrl-aw9523.c @@ -66,18 +66,6 @@ struct aw9523_irq { u16 cached_gpio; }; -/* - * struct aw9523_pinmux - Pin mux params - * @name: Name of the mux - * @grps: Groups of the mux - * @num_grps: Number of groups (sizeof array grps) - */ -struct aw9523_pinmux { - const char *name; - const char * const *grps; - const u8 num_grps; -}; - /* * struct aw9523 - Main driver structure * @dev: device handle @@ -158,17 +146,9 @@ static const char * const gpio_pwm_groups[] = { }; /* Warning: Do NOT reorder this array */ -static const struct aw9523_pinmux aw9523_pmx[] = { - { - .name = "pwm", - .grps = gpio_pwm_groups, - .num_grps = ARRAY_SIZE(gpio_pwm_groups), - }, - { - .name = "gpio", - .grps = gpio_pwm_groups, - .num_grps = ARRAY_SIZE(gpio_pwm_groups), - }, +static const struct pinfunction aw9523_pmx[] = { + PINCTRL_PINFUNCTION("pwm", gpio_pwm_groups, ARRAY_SIZE(gpio_pwm_groups)), + PINCTRL_PINFUNCTION("gpio", gpio_pwm_groups, ARRAY_SIZE(gpio_pwm_groups)), }; static int aw9523_pmx_get_funcs_count(struct pinctrl_dev *pctl) @@ -184,10 +164,10 @@ static const char *aw9523_pmx_get_fname(struct pinctrl_dev *pctl, static int aw9523_pmx_get_groups(struct pinctrl_dev *pctl, unsigned int sel, const char * const **groups, - unsigned int * const num_groups) + unsigned int * const ngroups) { - *groups = aw9523_pmx[sel].grps; - *num_groups = aw9523_pmx[sel].num_grps; + *groups = aw9523_pmx[sel].groups; + *ngroups = aw9523_pmx[sel].ngroups; return 0; } From 66413f0468d35adb352c76bc286bf6f6746ba354 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 29 Mar 2024 12:55:19 +0200 Subject: [PATCH 0066/1702] pinctrl: aw9523: Use temporary variable for HW IRQ number There are two different ways on how to get HW IRQ number in some functions. Unify that by using temporary variable and irqd_to_hwirq() call. Signed-off-by: Andy Shevchenko Message-ID: <20240329105634.712457-6-andy.shevchenko@gmail.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-aw9523.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/pinctrl/pinctrl-aw9523.c b/drivers/pinctrl/pinctrl-aw9523.c index 7db901216a94c..0f3361fa9ed88 100644 --- a/drivers/pinctrl/pinctrl-aw9523.c +++ b/drivers/pinctrl/pinctrl-aw9523.c @@ -428,12 +428,12 @@ static int aw9523_gpio_irq_type(struct irq_data *d, unsigned int type) static void aw9523_irq_mask(struct irq_data *d) { struct aw9523 *awi = gpiochip_get_data(irq_data_get_irq_chip_data(d)); - unsigned int n = d->hwirq % AW9523_PINS_PER_PORT; + irq_hw_number_t hwirq = irqd_to_hwirq(d); + unsigned int n = hwirq % AW9523_PINS_PER_PORT; - regmap_update_bits(awi->regmap, - AW9523_REG_INTR_DIS(d->hwirq), + regmap_update_bits(awi->regmap, AW9523_REG_INTR_DIS(hwirq), BIT(n), BIT(n)); - gpiochip_disable_irq(&awi->gpio, irqd_to_hwirq(d)); + gpiochip_disable_irq(&awi->gpio, hwirq); } /* @@ -446,11 +446,11 @@ static void aw9523_irq_mask(struct irq_data *d) static void aw9523_irq_unmask(struct irq_data *d) { struct aw9523 *awi = gpiochip_get_data(irq_data_get_irq_chip_data(d)); - unsigned int n = d->hwirq % AW9523_PINS_PER_PORT; + irq_hw_number_t hwirq = irqd_to_hwirq(d); + unsigned int n = hwirq % AW9523_PINS_PER_PORT; - gpiochip_enable_irq(&awi->gpio, irqd_to_hwirq(d)); - regmap_update_bits(awi->regmap, - AW9523_REG_INTR_DIS(d->hwirq), + gpiochip_enable_irq(&awi->gpio, hwirq); + regmap_update_bits(awi->regmap, AW9523_REG_INTR_DIS(hwirq), BIT(n), 0); } From 4210ef801a248223a0ea5f47b5446081b4925e10 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 29 Mar 2024 12:55:20 +0200 Subject: [PATCH 0067/1702] pinctrl: aw9523: Get rid of redundant ' & U8_MAX' pieces When the variable is declared as u8, no need to perform ' & U8_MAX' as it's implied anyway. Signed-off-by: Andy Shevchenko Message-ID: <20240329105634.712457-7-andy.shevchenko@gmail.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-aw9523.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/pinctrl/pinctrl-aw9523.c b/drivers/pinctrl/pinctrl-aw9523.c index 0f3361fa9ed88..8c615adb3d578 100644 --- a/drivers/pinctrl/pinctrl-aw9523.c +++ b/drivers/pinctrl/pinctrl-aw9523.c @@ -603,7 +603,7 @@ static int aw9523_gpio_get_multiple(struct gpio_chip *chip, mutex_lock(&awi->i2c_lock); /* Port 0 (gpio 0-7) */ - m = *mask & U8_MAX; + m = *mask; if (m) { ret = _aw9523_gpio_get_multiple(awi, 0, &state, m); if (ret) @@ -612,7 +612,7 @@ static int aw9523_gpio_get_multiple(struct gpio_chip *chip, *bits = state; /* Port 1 (gpio 8-15) */ - m = (*mask >> 8) & U8_MAX; + m = *mask >> 8; if (m) { ret = _aw9523_gpio_get_multiple(awi, AW9523_PINS_PER_PORT, &state, m); @@ -635,20 +635,20 @@ static void aw9523_gpio_set_multiple(struct gpio_chip *chip, unsigned int reg; int ret; - mask_lo = *mask & U8_MAX; - mask_hi = (*mask >> 8) & U8_MAX; + mask_lo = *mask; + mask_hi = *mask >> 8; + bits_lo = *bits; + bits_hi = *bits >> 8; + mutex_lock(&awi->i2c_lock); if (mask_hi) { reg = AW9523_REG_OUT_STATE(AW9523_PINS_PER_PORT); - bits_hi = (*bits >> 8) & U8_MAX; - ret = regmap_write_bits(awi->regmap, reg, mask_hi, bits_hi); if (ret) dev_warn(awi->dev, "Cannot write port1 out level\n"); } if (mask_lo) { reg = AW9523_REG_OUT_STATE(0); - bits_lo = *bits & U8_MAX; ret = regmap_write_bits(awi->regmap, reg, mask_lo, bits_lo); if (ret) dev_warn(awi->dev, "Cannot write port0 out level\n"); From 6bf270863ade776485d1c6bdb8f69d642b0e5f64 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 29 Mar 2024 12:55:21 +0200 Subject: [PATCH 0068/1702] pinctrl: aw9523: Remove unused irqchip field in struct aw9523_irq The irqchip field is allocated, assigned but never used. Remove it. Signed-off-by: Andy Shevchenko Message-ID: <20240329105634.712457-8-andy.shevchenko@gmail.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-aw9523.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/drivers/pinctrl/pinctrl-aw9523.c b/drivers/pinctrl/pinctrl-aw9523.c index 8c615adb3d578..f084dad5c88f9 100644 --- a/drivers/pinctrl/pinctrl-aw9523.c +++ b/drivers/pinctrl/pinctrl-aw9523.c @@ -57,12 +57,10 @@ /* * struct aw9523_irq - Interrupt controller structure * @lock: mutex locking for the irq bus - * @irqchip: structure holding irqchip params * @cached_gpio: stores the previous gpio status for bit comparison */ struct aw9523_irq { struct mutex lock; - struct irq_chip *irqchip; u16 cached_gpio; }; @@ -805,21 +803,15 @@ static int aw9523_init_irq(struct aw9523 *awi, int irq) { struct device *dev = awi->dev; struct gpio_irq_chip *girq; - struct irq_chip *irqchip; int ret; if (!device_property_read_bool(dev, "interrupt-controller")) return 0; - irqchip = devm_kzalloc(dev, sizeof(*irqchip), GFP_KERNEL); - if (!irqchip) - return -ENOMEM; - awi->irq = devm_kzalloc(dev, sizeof(*awi->irq), GFP_KERNEL); if (!awi->irq) return -ENOMEM; - awi->irq->irqchip = irqchip; mutex_init(&awi->irq->lock); ret = devm_request_threaded_irq(dev, irq, NULL, aw9523_irq_thread_func, From c567b00cc3d73f3ce4e92126731545d177262090 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 29 Mar 2024 12:55:22 +0200 Subject: [PATCH 0069/1702] pinctrl: aw9523: Make use of dev_err_probe() Simplify the error handling in probe function by switching from dev_err() to dev_err_probe(). Signed-off-by: Andy Shevchenko Message-ID: <20240329105634.712457-9-andy.shevchenko@gmail.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-aw9523.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/pinctrl/pinctrl-aw9523.c b/drivers/pinctrl/pinctrl-aw9523.c index f084dad5c88f9..fe398c59c2d11 100644 --- a/drivers/pinctrl/pinctrl-aw9523.c +++ b/drivers/pinctrl/pinctrl-aw9523.c @@ -816,10 +816,8 @@ static int aw9523_init_irq(struct aw9523 *awi, int irq) ret = devm_request_threaded_irq(dev, irq, NULL, aw9523_irq_thread_func, IRQF_ONESHOT, dev_name(dev), awi); - if (ret) { - dev_err(dev, "Failed to request irq %d\n", irq); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "Failed to request irq %d\n", irq); girq = &awi->gpio.irq; gpio_irq_chip_set_chip(girq, &aw9523_irq_chip); @@ -1016,8 +1014,7 @@ static int aw9523_probe(struct i2c_client *client) awi->pctl = devm_pinctrl_register(dev, pdesc, awi); if (IS_ERR(awi->pctl)) { - ret = PTR_ERR(awi->pctl); - dev_err(dev, "Cannot register pinctrl: %d", ret); + ret = dev_err_probe(dev, PTR_ERR(awi->pctl), "Cannot register pinctrl"); goto err_disable_vregs; } From 7b8b9b5450b89d01e4b8f120b903cee85b529231 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 29 Mar 2024 12:55:23 +0200 Subject: [PATCH 0070/1702] pinctrl: aw9523: Sort headers and group pinctrl/* One header was misplaced and group pinctrl/* ones to show the relation with the pin control subsystem. Signed-off-by: Andy Shevchenko Message-ID: <20240329105634.712457-10-andy.shevchenko@gmail.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-aw9523.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/pinctrl/pinctrl-aw9523.c b/drivers/pinctrl/pinctrl-aw9523.c index fe398c59c2d11..49d5035899227 100644 --- a/drivers/pinctrl/pinctrl-aw9523.c +++ b/drivers/pinctrl/pinctrl-aw9523.c @@ -13,17 +13,18 @@ #include #include #include -#include #include -#include -#include -#include -#include +#include #include #include #include #include +#include +#include +#include +#include + #define AW9523_MAX_FUNCS 2 #define AW9523_NUM_PORTS 2 #define AW9523_PINS_PER_PORT 8 From 4aad0ad20f4ea80180a3e58b04b701728541c0f7 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 29 Mar 2024 12:55:24 +0200 Subject: [PATCH 0071/1702] pinctrl: aw9523: Fix indentation in a few places In the comment, function prototype, and array of strings indentation is kinda broken. Reindent that. Signed-off-by: Andy Shevchenko Message-ID: <20240329105634.712457-11-andy.shevchenko@gmail.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-aw9523.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/pinctrl/pinctrl-aw9523.c b/drivers/pinctrl/pinctrl-aw9523.c index 49d5035899227..b5e1c467625ba 100644 --- a/drivers/pinctrl/pinctrl-aw9523.c +++ b/drivers/pinctrl/pinctrl-aw9523.c @@ -1,8 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Awinic AW9523B i2c pin controller driver - * Copyright (c) 2020, AngeloGioacchino Del Regno - * + * Copyright (c) 2020, AngeloGioacchino Del Regno */ #include @@ -139,9 +138,10 @@ static const struct pinctrl_ops aw9523_pinctrl_ops = { }; static const char * const gpio_pwm_groups[] = { - "gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5", - "gpio6", "gpio7", "gpio8", "gpio9", "gpio10", "gpio11", - "gpio12", "gpio13", "gpio14", "gpio15" + "gpio0", "gpio1", "gpio2", "gpio3", /* 0-3 */ + "gpio4", "gpio5", "gpio6", "gpio7", /* 4-7 */ + "gpio8", "gpio9", "gpio10", "gpio11", /* 8-11 */ + "gpio12", "gpio13", "gpio14", "gpio15", /* 11-15 */ }; /* Warning: Do NOT reorder this array */ @@ -388,8 +388,8 @@ static int aw9523_get_pin_direction(struct regmap *regmap, u8 pin, u8 n) * * Return: Zero for success or negative number for error */ -static int aw9523_get_port_state(struct regmap *regmap, u8 pin, - u8 regbit, unsigned int *state) +static int aw9523_get_port_state(struct regmap *regmap, u8 pin, u8 regbit, + unsigned int *state) { u8 reg; int dir; @@ -984,8 +984,7 @@ static int aw9523_probe(struct i2c_client *client) } mutex_init(&awi->i2c_lock); - lockdep_set_subclass(&awi->i2c_lock, - i2c_adapter_depth(client->adapter)); + lockdep_set_subclass(&awi->i2c_lock, i2c_adapter_depth(client->adapter)); pdesc = devm_kzalloc(dev, sizeof(*pdesc), GFP_KERNEL); if (!pdesc) From e798845d685e0ca652d4fd954a81edd1626f235d Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sat, 30 Mar 2024 22:09:54 +0100 Subject: [PATCH 0072/1702] pinctrl: sunxi: sun9i-a80-r: drop driver owner assignment Core in platform_driver_register() already sets the .owner, so driver does not need to. Signed-off-by: Krzysztof Kozlowski Acked-by: Chen-Yu Tsai Message-ID: <20240330210954.100842-1-krzysztof.kozlowski@linaro.org> Signed-off-by: Linus Walleij --- drivers/pinctrl/sunxi/pinctrl-sun9i-a80-r.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/pinctrl/sunxi/pinctrl-sun9i-a80-r.c b/drivers/pinctrl/sunxi/pinctrl-sun9i-a80-r.c index 919b6a20af832..5b4822f77d2a5 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sun9i-a80-r.c +++ b/drivers/pinctrl/sunxi/pinctrl-sun9i-a80-r.c @@ -169,7 +169,6 @@ static struct platform_driver sun9i_a80_r_pinctrl_driver = { .probe = sun9i_a80_r_pinctrl_probe, .driver = { .name = "sun9i-a80-r-pinctrl", - .owner = THIS_MODULE, .of_match_table = sun9i_a80_r_pinctrl_match, }, }; From d2f277e26f521ccf6fb438463b41dba6123caabe Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Sun, 17 Mar 2024 20:41:45 +0200 Subject: [PATCH 0073/1702] fsnotify: rename fsnotify_{get,put}_sb_connectors() Instead of counting the number of connectors in an sb, we would like to count the number of watched objects per priority group. As a start, create an accessor fsnotify_sb_watched_objects() to s_fsnotify_connectors and rename the fsnotify_{get,put}_sb_connectors() helpers to fsnotify_{get,put}_sb_watchers() to better describes the counter. Increment the counter at the end of fsnotify_attach_connector_to_object() if connector was attached instead of decrementing it on race to connect. This is fine, because fsnotify_delete_sb() cannot be running in parallel to fsnotify_attach_connector_to_object() which requires a reference to a filesystem object. Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara Message-Id: <20240317184154.1200192-2-amir73il@gmail.com> --- fs/notify/fsnotify.c | 4 +- fs/notify/mark.c | 67 ++++++++++++++++++-------------- include/linux/fsnotify.h | 2 +- include/linux/fsnotify_backend.h | 5 +++ 4 files changed, 45 insertions(+), 33 deletions(-) diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c index 2fc105a72a8f6..503e7c75e777f 100644 --- a/fs/notify/fsnotify.c +++ b/fs/notify/fsnotify.c @@ -92,8 +92,8 @@ void fsnotify_sb_delete(struct super_block *sb) fsnotify_unmount_inodes(sb); fsnotify_clear_marks_by_sb(sb); /* Wait for outstanding object references from connectors */ - wait_var_event(&sb->s_fsnotify_connectors, - !atomic_long_read(&sb->s_fsnotify_connectors)); + wait_var_event(fsnotify_sb_watched_objects(sb), + !atomic_long_read(fsnotify_sb_watched_objects(sb))); } /* diff --git a/fs/notify/mark.c b/fs/notify/mark.c index d6944ff86ffab..3bcce2671bd54 100644 --- a/fs/notify/mark.c +++ b/fs/notify/mark.c @@ -116,10 +116,43 @@ __u32 fsnotify_conn_mask(struct fsnotify_mark_connector *conn) return *fsnotify_conn_mask_p(conn); } +static void fsnotify_get_sb_watched_objects(struct super_block *sb) +{ + atomic_long_inc(fsnotify_sb_watched_objects(sb)); +} + +static void fsnotify_put_sb_watched_objects(struct super_block *sb) +{ + if (atomic_long_dec_and_test(fsnotify_sb_watched_objects(sb))) + wake_up_var(fsnotify_sb_watched_objects(sb)); +} + static void fsnotify_get_inode_ref(struct inode *inode) { ihold(inode); - atomic_long_inc(&inode->i_sb->s_fsnotify_connectors); + fsnotify_get_sb_watched_objects(inode->i_sb); +} + +static void fsnotify_put_inode_ref(struct inode *inode) +{ + fsnotify_put_sb_watched_objects(inode->i_sb); + iput(inode); +} + +static void fsnotify_get_sb_watchers(struct fsnotify_mark_connector *conn) +{ + struct super_block *sb = fsnotify_connector_sb(conn); + + if (sb) + fsnotify_get_sb_watched_objects(sb); +} + +static void fsnotify_put_sb_watchers(struct fsnotify_mark_connector *conn) +{ + struct super_block *sb = fsnotify_connector_sb(conn); + + if (sb) + fsnotify_put_sb_watched_objects(sb); } /* @@ -213,31 +246,6 @@ static void fsnotify_connector_destroy_workfn(struct work_struct *work) } } -static void fsnotify_put_inode_ref(struct inode *inode) -{ - struct super_block *sb = inode->i_sb; - - iput(inode); - if (atomic_long_dec_and_test(&sb->s_fsnotify_connectors)) - wake_up_var(&sb->s_fsnotify_connectors); -} - -static void fsnotify_get_sb_connectors(struct fsnotify_mark_connector *conn) -{ - struct super_block *sb = fsnotify_connector_sb(conn); - - if (sb) - atomic_long_inc(&sb->s_fsnotify_connectors); -} - -static void fsnotify_put_sb_connectors(struct fsnotify_mark_connector *conn) -{ - struct super_block *sb = fsnotify_connector_sb(conn); - - if (sb && atomic_long_dec_and_test(&sb->s_fsnotify_connectors)) - wake_up_var(&sb->s_fsnotify_connectors); -} - static void *fsnotify_detach_connector_from_object( struct fsnotify_mark_connector *conn, unsigned int *type) @@ -261,7 +269,7 @@ static void *fsnotify_detach_connector_from_object( fsnotify_conn_sb(conn)->s_fsnotify_mask = 0; } - fsnotify_put_sb_connectors(conn); + fsnotify_put_sb_watchers(conn); rcu_assign_pointer(*(conn->obj), NULL); conn->obj = NULL; conn->type = FSNOTIFY_OBJ_TYPE_DETACHED; @@ -549,8 +557,6 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp, conn->flags = 0; conn->type = obj_type; conn->obj = connp; - conn->flags = 0; - fsnotify_get_sb_connectors(conn); /* * cmpxchg() provides the barrier so that readers of *connp can see @@ -558,10 +564,11 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp, */ if (cmpxchg(connp, NULL, conn)) { /* Someone else created list structure for us */ - fsnotify_put_sb_connectors(conn); kmem_cache_free(fsnotify_mark_connector_cachep, conn); + return 0; } + fsnotify_get_sb_watchers(conn); return 0; } diff --git a/include/linux/fsnotify.h b/include/linux/fsnotify.h index 1a9de119a0f73..e470bb67c9a33 100644 --- a/include/linux/fsnotify.h +++ b/include/linux/fsnotify.h @@ -20,7 +20,7 @@ /* Are there any inode/mount/sb objects that are being watched at all? */ static inline bool fsnotify_sb_has_watchers(struct super_block *sb) { - return atomic_long_read(&sb->s_fsnotify_connectors); + return atomic_long_read(fsnotify_sb_watched_objects(sb)); } /* diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index 8f40c349b2283..d4e3bc55d1745 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -483,6 +483,11 @@ struct fsnotify_mark_connector { struct hlist_head list; }; +static inline atomic_long_t *fsnotify_sb_watched_objects(struct super_block *sb) +{ + return &sb->s_fsnotify_connectors; +} + /* * A mark is simply an object attached to an in core inode which allows an * fsnotify listener to indicate they are either no longer interested in events From f115815d75332f9dabb0d7c29c8f67b0f26889c5 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Sun, 17 Mar 2024 20:41:46 +0200 Subject: [PATCH 0074/1702] fsnotify: create helpers to get sb and connp from object In preparation to passing an object pointer to add/remove/find mark helpers, create helpers to get sb and connp by object type. Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara Message-Id: <20240317184154.1200192-3-amir73il@gmail.com> --- fs/notify/fsnotify.h | 15 +++++++++++++++ fs/notify/mark.c | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/fs/notify/fsnotify.h b/fs/notify/fsnotify.h index fde74eb333cc9..6d9951aac449c 100644 --- a/fs/notify/fsnotify.h +++ b/fs/notify/fsnotify.h @@ -27,6 +27,21 @@ static inline struct super_block *fsnotify_conn_sb( return container_of(conn->obj, struct super_block, s_fsnotify_marks); } +static inline struct super_block *fsnotify_object_sb(void *obj, + enum fsnotify_obj_type obj_type) +{ + switch (obj_type) { + case FSNOTIFY_OBJ_TYPE_INODE: + return ((struct inode *)obj)->i_sb; + case FSNOTIFY_OBJ_TYPE_VFSMOUNT: + return ((struct vfsmount *)obj)->mnt_sb; + case FSNOTIFY_OBJ_TYPE_SB: + return (struct super_block *)obj; + default: + return NULL; + } +} + static inline struct super_block *fsnotify_connector_sb( struct fsnotify_mark_connector *conn) { diff --git a/fs/notify/mark.c b/fs/notify/mark.c index 3bcce2671bd54..d9977af258327 100644 --- a/fs/notify/mark.c +++ b/fs/notify/mark.c @@ -97,6 +97,21 @@ void fsnotify_get_mark(struct fsnotify_mark *mark) refcount_inc(&mark->refcnt); } +static fsnotify_connp_t *fsnotify_object_connp(void *obj, + enum fsnotify_obj_type obj_type) +{ + switch (obj_type) { + case FSNOTIFY_OBJ_TYPE_INODE: + return &((struct inode *)obj)->i_fsnotify_marks; + case FSNOTIFY_OBJ_TYPE_VFSMOUNT: + return &real_mount(obj)->mnt_fsnotify_marks; + case FSNOTIFY_OBJ_TYPE_SB: + return &((struct super_block *)obj)->s_fsnotify_marks; + default: + return NULL; + } +} + static __u32 *fsnotify_conn_mask_p(struct fsnotify_mark_connector *conn) { if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) From 230d97d39ee2eb9030309f04f98615aaeb420dac Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Sun, 17 Mar 2024 20:41:47 +0200 Subject: [PATCH 0075/1702] fsnotify: create a wrapper fsnotify_find_inode_mark() In preparation to passing an object pointer to fsnotify_find_mark(), add a wrapper fsnotify_find_inode_mark() and use it where possible. Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara Message-Id: <20240317184154.1200192-4-amir73il@gmail.com> --- fs/nfsd/filecache.c | 4 ++-- fs/notify/dnotify/dnotify.c | 4 ++-- fs/notify/inotify/inotify_user.c | 2 +- include/linux/fsnotify_backend.h | 7 +++++++ kernel/audit_tree.c | 2 +- kernel/audit_watch.c | 2 +- 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index ddd3e0d9cfa63..ad9083ca144ba 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -159,8 +159,8 @@ nfsd_file_mark_find_or_create(struct nfsd_file *nf, struct inode *inode) do { fsnotify_group_lock(nfsd_file_fsnotify_group); - mark = fsnotify_find_mark(&inode->i_fsnotify_marks, - nfsd_file_fsnotify_group); + mark = fsnotify_find_inode_mark(inode, + nfsd_file_fsnotify_group); if (mark) { nfm = nfsd_file_mark_get(container_of(mark, struct nfsd_file_mark, diff --git a/fs/notify/dnotify/dnotify.c b/fs/notify/dnotify/dnotify.c index 3464fa7e85380..f3669403fabf1 100644 --- a/fs/notify/dnotify/dnotify.c +++ b/fs/notify/dnotify/dnotify.c @@ -162,7 +162,7 @@ void dnotify_flush(struct file *filp, fl_owner_t id) if (!S_ISDIR(inode->i_mode)) return; - fsn_mark = fsnotify_find_mark(&inode->i_fsnotify_marks, dnotify_group); + fsn_mark = fsnotify_find_inode_mark(inode, dnotify_group); if (!fsn_mark) return; dn_mark = container_of(fsn_mark, struct dnotify_mark, fsn_mark); @@ -326,7 +326,7 @@ int fcntl_dirnotify(int fd, struct file *filp, unsigned int arg) fsnotify_group_lock(dnotify_group); /* add the new_fsn_mark or find an old one. */ - fsn_mark = fsnotify_find_mark(&inode->i_fsnotify_marks, dnotify_group); + fsn_mark = fsnotify_find_inode_mark(inode, dnotify_group); if (fsn_mark) { dn_mark = container_of(fsn_mark, struct dnotify_mark, fsn_mark); spin_lock(&fsn_mark->lock); diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c index 85d8fdd553294..4ffc30606e0b9 100644 --- a/fs/notify/inotify/inotify_user.c +++ b/fs/notify/inotify/inotify_user.c @@ -544,7 +544,7 @@ static int inotify_update_existing_watch(struct fsnotify_group *group, int create = (arg & IN_MASK_CREATE); int ret; - fsn_mark = fsnotify_find_mark(&inode->i_fsnotify_marks, group); + fsn_mark = fsnotify_find_inode_mark(inode, group); if (!fsn_mark) return -ENOENT; else if (create) { diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index d4e3bc55d1745..992b57a7e95f4 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -789,6 +789,13 @@ static inline int fsnotify_add_inode_mark_locked(struct fsnotify_mark *mark, FSNOTIFY_OBJ_TYPE_INODE, add_flags); } +static inline struct fsnotify_mark *fsnotify_find_inode_mark( + struct inode *inode, + struct fsnotify_group *group) +{ + return fsnotify_find_mark(&inode->i_fsnotify_marks, group); +} + /* given a group and a mark, flag mark to be freed when all references are dropped */ extern void fsnotify_destroy_mark(struct fsnotify_mark *mark, struct fsnotify_group *group); diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c index 1b07e6f12a07a..f2f38903b2fe3 100644 --- a/kernel/audit_tree.c +++ b/kernel/audit_tree.c @@ -463,7 +463,7 @@ static int tag_chunk(struct inode *inode, struct audit_tree *tree) int n; fsnotify_group_lock(audit_tree_group); - mark = fsnotify_find_mark(&inode->i_fsnotify_marks, audit_tree_group); + mark = fsnotify_find_inode_mark(inode, audit_tree_group); if (!mark) return create_chunk(inode, tree); diff --git a/kernel/audit_watch.c b/kernel/audit_watch.c index 7a98cd176a127..7f358740e9582 100644 --- a/kernel/audit_watch.c +++ b/kernel/audit_watch.c @@ -90,7 +90,7 @@ static inline struct audit_parent *audit_find_parent(struct inode *inode) struct audit_parent *parent = NULL; struct fsnotify_mark *entry; - entry = fsnotify_find_mark(&inode->i_fsnotify_marks, audit_watch_group); + entry = fsnotify_find_inode_mark(inode, audit_watch_group); if (entry) parent = container_of(entry, struct audit_parent, mark); From b5cae086cc2fde629495d988106d70e44c90cb20 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Sun, 17 Mar 2024 20:41:48 +0200 Subject: [PATCH 0076/1702] fanotify: merge two checks regarding add of ignore mark There are two similar checks for adding an ignore mark without FAN_MARK_IGNORED_SURV_MODIFY, one for the old FAN_MARK_IGNORED flag and one for the new FAN_MARK_IGNORE flag. Merge the two checks into a single location. Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara Message-Id: <20240317184154.1200192-5-amir73il@gmail.com> --- fs/notify/fanotify/fanotify_user.c | 35 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index 4201723357cfb..04a0b31f325df 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -1412,18 +1412,6 @@ static int fanotify_add_inode_mark(struct fsnotify_group *group, struct inode *inode, __u32 mask, unsigned int flags, struct fan_fsid *fsid) { - pr_debug("%s: group=%p inode=%p\n", __func__, group, inode); - - /* - * If some other task has this inode open for write we should not add - * an ignore mask, unless that ignore mask is supposed to survive - * modification changes anyway. - */ - if ((flags & FANOTIFY_MARK_IGNORE_BITS) && - !(flags & FAN_MARK_IGNORED_SURV_MODIFY) && - inode_is_open_for_write(inode)) - return 0; - return fanotify_add_mark(group, &inode->i_fsnotify_marks, FSNOTIFY_OBJ_TYPE_INODE, mask, flags, fsid); } @@ -1913,12 +1901,23 @@ static int do_fanotify_mark(int fanotify_fd, unsigned int flags, __u64 mask, else mnt = path.mnt; - ret = mnt ? -EINVAL : -EISDIR; - /* FAN_MARK_IGNORE requires SURV_MODIFY for sb/mount/dir marks */ - if (mark_cmd == FAN_MARK_ADD && ignore == FAN_MARK_IGNORE && - (mnt || S_ISDIR(inode->i_mode)) && - !(flags & FAN_MARK_IGNORED_SURV_MODIFY)) - goto path_put_and_out; + /* + * If some other task has this inode open for write we should not add + * an ignore mask, unless that ignore mask is supposed to survive + * modification changes anyway. + */ + if (mark_cmd == FAN_MARK_ADD && (flags & FANOTIFY_MARK_IGNORE_BITS) && + !(flags & FAN_MARK_IGNORED_SURV_MODIFY)) { + ret = mnt ? -EINVAL : -EISDIR; + /* FAN_MARK_IGNORE requires SURV_MODIFY for sb/mount/dir marks */ + if (ignore == FAN_MARK_IGNORE && + (mnt || S_ISDIR(inode->i_mode))) + goto path_put_and_out; + + ret = 0; + if (inode && inode_is_open_for_write(inode)) + goto path_put_and_out; + } /* Mask out FAN_EVENT_ON_CHILD flag for sb/mount/non-dir marks */ if (mnt || !S_ISDIR(inode->i_mode)) { From 687c217c6aa2c24e6ddf98cc7f86a5b986e5918d Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Sun, 17 Mar 2024 20:41:49 +0200 Subject: [PATCH 0077/1702] fsnotify: pass object pointer and type to fsnotify mark helpers Instead of passing fsnotify_connp_t, pass the pointer to the marked object. Store the object pointer in the connector and move the definition of fsnotify_connp_t to internal fsnotify subsystem API, so it is no longer used by fsnotify backends. Suggested-by: Jan Kara Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara Message-Id: <20240317184154.1200192-6-amir73il@gmail.com> --- fs/notify/fanotify/fanotify_user.c | 95 +++++++----------------------- fs/notify/fsnotify.h | 23 ++++---- fs/notify/mark.c | 28 +++++---- include/linux/fsnotify_backend.h | 33 ++++------- 4 files changed, 59 insertions(+), 120 deletions(-) diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index 04a0b31f325df..925015ee7fbd1 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -1076,7 +1076,7 @@ static __u32 fanotify_mark_remove_from_mask(struct fsnotify_mark *fsn_mark, } static int fanotify_remove_mark(struct fsnotify_group *group, - fsnotify_connp_t *connp, __u32 mask, + void *obj, unsigned int obj_type, __u32 mask, unsigned int flags, __u32 umask) { struct fsnotify_mark *fsn_mark = NULL; @@ -1084,7 +1084,7 @@ static int fanotify_remove_mark(struct fsnotify_group *group, int destroy_mark; fsnotify_group_lock(group); - fsn_mark = fsnotify_find_mark(connp, group); + fsn_mark = fsnotify_find_mark(obj, obj_type, group); if (!fsn_mark) { fsnotify_group_unlock(group); return -ENOENT; @@ -1105,30 +1105,6 @@ static int fanotify_remove_mark(struct fsnotify_group *group, return 0; } -static int fanotify_remove_vfsmount_mark(struct fsnotify_group *group, - struct vfsmount *mnt, __u32 mask, - unsigned int flags, __u32 umask) -{ - return fanotify_remove_mark(group, &real_mount(mnt)->mnt_fsnotify_marks, - mask, flags, umask); -} - -static int fanotify_remove_sb_mark(struct fsnotify_group *group, - struct super_block *sb, __u32 mask, - unsigned int flags, __u32 umask) -{ - return fanotify_remove_mark(group, &sb->s_fsnotify_marks, mask, - flags, umask); -} - -static int fanotify_remove_inode_mark(struct fsnotify_group *group, - struct inode *inode, __u32 mask, - unsigned int flags, __u32 umask) -{ - return fanotify_remove_mark(group, &inode->i_fsnotify_marks, mask, - flags, umask); -} - static bool fanotify_mark_update_flags(struct fsnotify_mark *fsn_mark, unsigned int fan_flags) { @@ -1249,7 +1225,7 @@ static int fanotify_set_mark_fsid(struct fsnotify_group *group, } static struct fsnotify_mark *fanotify_add_new_mark(struct fsnotify_group *group, - fsnotify_connp_t *connp, + void *obj, unsigned int obj_type, unsigned int fan_flags, struct fan_fsid *fsid) @@ -1288,7 +1264,7 @@ static struct fsnotify_mark *fanotify_add_new_mark(struct fsnotify_group *group, fan_mark->fsid.val[0] = fan_mark->fsid.val[1] = 0; } - ret = fsnotify_add_mark_locked(mark, connp, obj_type, 0); + ret = fsnotify_add_mark_locked(mark, obj, obj_type, 0); if (ret) goto out_put_mark; @@ -1344,7 +1320,7 @@ static int fanotify_may_update_existing_mark(struct fsnotify_mark *fsn_mark, } static int fanotify_add_mark(struct fsnotify_group *group, - fsnotify_connp_t *connp, unsigned int obj_type, + void *obj, unsigned int obj_type, __u32 mask, unsigned int fan_flags, struct fan_fsid *fsid) { @@ -1353,9 +1329,9 @@ static int fanotify_add_mark(struct fsnotify_group *group, int ret = 0; fsnotify_group_lock(group); - fsn_mark = fsnotify_find_mark(connp, group); + fsn_mark = fsnotify_find_mark(obj, obj_type, group); if (!fsn_mark) { - fsn_mark = fanotify_add_new_mark(group, connp, obj_type, + fsn_mark = fanotify_add_new_mark(group, obj, obj_type, fan_flags, fsid); if (IS_ERR(fsn_mark)) { fsnotify_group_unlock(group); @@ -1392,30 +1368,6 @@ static int fanotify_add_mark(struct fsnotify_group *group, return ret; } -static int fanotify_add_vfsmount_mark(struct fsnotify_group *group, - struct vfsmount *mnt, __u32 mask, - unsigned int flags, struct fan_fsid *fsid) -{ - return fanotify_add_mark(group, &real_mount(mnt)->mnt_fsnotify_marks, - FSNOTIFY_OBJ_TYPE_VFSMOUNT, mask, flags, fsid); -} - -static int fanotify_add_sb_mark(struct fsnotify_group *group, - struct super_block *sb, __u32 mask, - unsigned int flags, struct fan_fsid *fsid) -{ - return fanotify_add_mark(group, &sb->s_fsnotify_marks, - FSNOTIFY_OBJ_TYPE_SB, mask, flags, fsid); -} - -static int fanotify_add_inode_mark(struct fsnotify_group *group, - struct inode *inode, __u32 mask, - unsigned int flags, struct fan_fsid *fsid) -{ - return fanotify_add_mark(group, &inode->i_fsnotify_marks, - FSNOTIFY_OBJ_TYPE_INODE, mask, flags, fsid); -} - static struct fsnotify_event *fanotify_alloc_overflow_event(void) { struct fanotify_event *oevent; @@ -1738,6 +1690,7 @@ static int do_fanotify_mark(int fanotify_fd, unsigned int flags, __u64 mask, unsigned int mark_cmd = flags & FANOTIFY_MARK_CMD_BITS; unsigned int ignore = flags & FANOTIFY_MARK_IGNORE_BITS; unsigned int obj_type, fid_mode; + void *obj; u32 umask = 0; int ret; @@ -1896,10 +1849,16 @@ static int do_fanotify_mark(int fanotify_fd, unsigned int flags, __u64 mask, } /* inode held in place by reference to path; group by fget on fd */ - if (mark_type == FAN_MARK_INODE) + if (mark_type == FAN_MARK_INODE) { inode = path.dentry->d_inode; - else + obj = inode; + } else { mnt = path.mnt; + if (mark_type == FAN_MARK_MOUNT) + obj = mnt; + else + obj = mnt->mnt_sb; + } /* * If some other task has this inode open for write we should not add @@ -1935,26 +1894,12 @@ static int do_fanotify_mark(int fanotify_fd, unsigned int flags, __u64 mask, /* create/update an inode mark */ switch (mark_cmd) { case FAN_MARK_ADD: - if (mark_type == FAN_MARK_MOUNT) - ret = fanotify_add_vfsmount_mark(group, mnt, mask, - flags, fsid); - else if (mark_type == FAN_MARK_FILESYSTEM) - ret = fanotify_add_sb_mark(group, mnt->mnt_sb, mask, - flags, fsid); - else - ret = fanotify_add_inode_mark(group, inode, mask, - flags, fsid); + ret = fanotify_add_mark(group, obj, obj_type, mask, flags, + fsid); break; case FAN_MARK_REMOVE: - if (mark_type == FAN_MARK_MOUNT) - ret = fanotify_remove_vfsmount_mark(group, mnt, mask, - flags, umask); - else if (mark_type == FAN_MARK_FILESYSTEM) - ret = fanotify_remove_sb_mark(group, mnt->mnt_sb, mask, - flags, umask); - else - ret = fanotify_remove_inode_mark(group, inode, mask, - flags, umask); + ret = fanotify_remove_mark(group, obj, obj_type, mask, flags, + umask); break; default: ret = -EINVAL; diff --git a/fs/notify/fsnotify.h b/fs/notify/fsnotify.h index 6d9951aac449c..b1fec7cd8ee40 100644 --- a/fs/notify/fsnotify.h +++ b/fs/notify/fsnotify.h @@ -9,22 +9,28 @@ #include "../mount.h" +/* + * fsnotify_connp_t is what we embed in objects which connector can be attached + * to. + */ +typedef struct fsnotify_mark_connector __rcu *fsnotify_connp_t; + static inline struct inode *fsnotify_conn_inode( struct fsnotify_mark_connector *conn) { - return container_of(conn->obj, struct inode, i_fsnotify_marks); + return conn->obj; } static inline struct mount *fsnotify_conn_mount( struct fsnotify_mark_connector *conn) { - return container_of(conn->obj, struct mount, mnt_fsnotify_marks); + return real_mount(conn->obj); } static inline struct super_block *fsnotify_conn_sb( struct fsnotify_mark_connector *conn) { - return container_of(conn->obj, struct super_block, s_fsnotify_marks); + return conn->obj; } static inline struct super_block *fsnotify_object_sb(void *obj, @@ -45,16 +51,7 @@ static inline struct super_block *fsnotify_object_sb(void *obj, static inline struct super_block *fsnotify_connector_sb( struct fsnotify_mark_connector *conn) { - switch (conn->type) { - case FSNOTIFY_OBJ_TYPE_INODE: - return fsnotify_conn_inode(conn)->i_sb; - case FSNOTIFY_OBJ_TYPE_VFSMOUNT: - return fsnotify_conn_mount(conn)->mnt.mnt_sb; - case FSNOTIFY_OBJ_TYPE_SB: - return fsnotify_conn_sb(conn); - default: - return NULL; - } + return fsnotify_object_sb(conn->obj, conn->type); } /* destroy all events sitting in this groups notification queue */ diff --git a/fs/notify/mark.c b/fs/notify/mark.c index d9977af258327..1f611e4eab53f 100644 --- a/fs/notify/mark.c +++ b/fs/notify/mark.c @@ -265,6 +265,7 @@ static void *fsnotify_detach_connector_from_object( struct fsnotify_mark_connector *conn, unsigned int *type) { + fsnotify_connp_t *connp = fsnotify_object_connp(conn->obj, conn->type); struct inode *inode = NULL; *type = conn->type; @@ -285,7 +286,7 @@ static void *fsnotify_detach_connector_from_object( } fsnotify_put_sb_watchers(conn); - rcu_assign_pointer(*(conn->obj), NULL); + rcu_assign_pointer(*connp, NULL); conn->obj = NULL; conn->type = FSNOTIFY_OBJ_TYPE_DETACHED; @@ -560,7 +561,7 @@ int fsnotify_compare_groups(struct fsnotify_group *a, struct fsnotify_group *b) } static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp, - unsigned int obj_type) + void *obj, unsigned int obj_type) { struct fsnotify_mark_connector *conn; @@ -571,7 +572,7 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp, INIT_HLIST_HEAD(&conn->list); conn->flags = 0; conn->type = obj_type; - conn->obj = connp; + conn->obj = obj; /* * cmpxchg() provides the barrier so that readers of *connp can see @@ -620,24 +621,25 @@ static struct fsnotify_mark_connector *fsnotify_grab_connector( * to which group and for which inodes. These marks are ordered according to * priority, highest number first, and then by the group's location in memory. */ -static int fsnotify_add_mark_list(struct fsnotify_mark *mark, - fsnotify_connp_t *connp, +static int fsnotify_add_mark_list(struct fsnotify_mark *mark, void *obj, unsigned int obj_type, int add_flags) { struct fsnotify_mark *lmark, *last = NULL; struct fsnotify_mark_connector *conn; + fsnotify_connp_t *connp; int cmp; int err = 0; if (WARN_ON(!fsnotify_valid_obj_type(obj_type))) return -EINVAL; + connp = fsnotify_object_connp(obj, obj_type); restart: spin_lock(&mark->lock); conn = fsnotify_grab_connector(connp); if (!conn) { spin_unlock(&mark->lock); - err = fsnotify_attach_connector_to_object(connp, obj_type); + err = fsnotify_attach_connector_to_object(connp, obj, obj_type); if (err) return err; goto restart; @@ -689,7 +691,7 @@ static int fsnotify_add_mark_list(struct fsnotify_mark *mark, * event types should be delivered to which group. */ int fsnotify_add_mark_locked(struct fsnotify_mark *mark, - fsnotify_connp_t *connp, unsigned int obj_type, + void *obj, unsigned int obj_type, int add_flags) { struct fsnotify_group *group = mark->group; @@ -710,7 +712,7 @@ int fsnotify_add_mark_locked(struct fsnotify_mark *mark, fsnotify_get_mark(mark); /* for g_list */ spin_unlock(&mark->lock); - ret = fsnotify_add_mark_list(mark, connp, obj_type, add_flags); + ret = fsnotify_add_mark_list(mark, obj, obj_type, add_flags); if (ret) goto err; @@ -728,14 +730,14 @@ int fsnotify_add_mark_locked(struct fsnotify_mark *mark, return ret; } -int fsnotify_add_mark(struct fsnotify_mark *mark, fsnotify_connp_t *connp, +int fsnotify_add_mark(struct fsnotify_mark *mark, void *obj, unsigned int obj_type, int add_flags) { int ret; struct fsnotify_group *group = mark->group; fsnotify_group_lock(group); - ret = fsnotify_add_mark_locked(mark, connp, obj_type, add_flags); + ret = fsnotify_add_mark_locked(mark, obj, obj_type, add_flags); fsnotify_group_unlock(group); return ret; } @@ -745,12 +747,16 @@ EXPORT_SYMBOL_GPL(fsnotify_add_mark); * Given a list of marks, find the mark associated with given group. If found * take a reference to that mark and return it, else return NULL. */ -struct fsnotify_mark *fsnotify_find_mark(fsnotify_connp_t *connp, +struct fsnotify_mark *fsnotify_find_mark(void *obj, unsigned int obj_type, struct fsnotify_group *group) { + fsnotify_connp_t *connp = fsnotify_object_connp(obj, obj_type); struct fsnotify_mark_connector *conn; struct fsnotify_mark *mark; + if (!connp) + return NULL; + conn = fsnotify_grab_connector(connp); if (!conn) return NULL; diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index 992b57a7e95f4..face68fcf8507 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -456,13 +456,6 @@ FSNOTIFY_ITER_FUNCS(sb, SB) type < FSNOTIFY_ITER_TYPE_COUNT; \ type++) -/* - * fsnotify_connp_t is what we embed in objects which connector can be attached - * to. fsnotify_connp_t * is how we refer from connector back to object. - */ -struct fsnotify_mark_connector; -typedef struct fsnotify_mark_connector __rcu *fsnotify_connp_t; - /* * Inode/vfsmount/sb point to this structure which tracks all marks attached to * the inode/vfsmount/sb. The reference to inode/vfsmount/sb is held by this @@ -476,7 +469,7 @@ struct fsnotify_mark_connector { unsigned short flags; /* flags [lock] */ union { /* Object pointer [lock] */ - fsnotify_connp_t *obj; + void *obj; /* Used listing heads to free after srcu period expires */ struct fsnotify_mark_connector *destroy_next; }; @@ -763,37 +756,35 @@ extern void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn); extern void fsnotify_init_mark(struct fsnotify_mark *mark, struct fsnotify_group *group); /* Find mark belonging to given group in the list of marks */ -extern struct fsnotify_mark *fsnotify_find_mark(fsnotify_connp_t *connp, - struct fsnotify_group *group); +struct fsnotify_mark *fsnotify_find_mark(void *obj, unsigned int obj_type, + struct fsnotify_group *group); /* attach the mark to the object */ -extern int fsnotify_add_mark(struct fsnotify_mark *mark, - fsnotify_connp_t *connp, unsigned int obj_type, - int add_flags); -extern int fsnotify_add_mark_locked(struct fsnotify_mark *mark, - fsnotify_connp_t *connp, - unsigned int obj_type, int add_flags); +int fsnotify_add_mark(struct fsnotify_mark *mark, void *obj, + unsigned int obj_type, int add_flags); +int fsnotify_add_mark_locked(struct fsnotify_mark *mark, void *obj, + unsigned int obj_type, int add_flags); /* attach the mark to the inode */ static inline int fsnotify_add_inode_mark(struct fsnotify_mark *mark, struct inode *inode, int add_flags) { - return fsnotify_add_mark(mark, &inode->i_fsnotify_marks, - FSNOTIFY_OBJ_TYPE_INODE, add_flags); + return fsnotify_add_mark(mark, inode, FSNOTIFY_OBJ_TYPE_INODE, + add_flags); } static inline int fsnotify_add_inode_mark_locked(struct fsnotify_mark *mark, struct inode *inode, int add_flags) { - return fsnotify_add_mark_locked(mark, &inode->i_fsnotify_marks, - FSNOTIFY_OBJ_TYPE_INODE, add_flags); + return fsnotify_add_mark_locked(mark, inode, FSNOTIFY_OBJ_TYPE_INODE, + add_flags); } static inline struct fsnotify_mark *fsnotify_find_inode_mark( struct inode *inode, struct fsnotify_group *group) { - return fsnotify_find_mark(&inode->i_fsnotify_marks, group); + return fsnotify_find_mark(inode, FSNOTIFY_OBJ_TYPE_INODE, group); } /* given a group and a mark, flag mark to be freed when all references are dropped */ From c9d4603b054f9a63c07c7012040af4c80adb2c60 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Sun, 17 Mar 2024 20:41:50 +0200 Subject: [PATCH 0078/1702] fsnotify: create helper fsnotify_update_sb_watchers() We would like to count watched objects by priority group, so we will need to update the watched object counter after adding/removing marks. Create a helper fsnotify_update_sb_watchers() and call it after attaching/detaching a mark, instead of fsnotify_{get,put}_sb_watchers() only after attaching/detaching a connector. Soon, we will use this helper to count watched objects by the highest watching priority group. Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara Message-Id: <20240317184154.1200192-7-amir73il@gmail.com> --- fs/notify/mark.c | 36 +++++++++++++++++++------------- include/linux/fsnotify_backend.h | 1 + 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/fs/notify/mark.c b/fs/notify/mark.c index 1f611e4eab53f..647c49c204673 100644 --- a/fs/notify/mark.c +++ b/fs/notify/mark.c @@ -154,20 +154,23 @@ static void fsnotify_put_inode_ref(struct inode *inode) iput(inode); } -static void fsnotify_get_sb_watchers(struct fsnotify_mark_connector *conn) +/* + * Grab or drop watched objects reference depending on whether the connector + * is attached and has any marks attached. + */ +static void fsnotify_update_sb_watchers(struct super_block *sb, + struct fsnotify_mark_connector *conn) { - struct super_block *sb = fsnotify_connector_sb(conn); + bool is_watched = conn->flags & FSNOTIFY_CONN_FLAG_IS_WATCHED; + bool has_marks = conn->obj && !hlist_empty(&conn->list); - if (sb) + if (has_marks && !is_watched) { + conn->flags |= FSNOTIFY_CONN_FLAG_IS_WATCHED; fsnotify_get_sb_watched_objects(sb); -} - -static void fsnotify_put_sb_watchers(struct fsnotify_mark_connector *conn) -{ - struct super_block *sb = fsnotify_connector_sb(conn); - - if (sb) + } else if (!has_marks && is_watched) { + conn->flags &= ~FSNOTIFY_CONN_FLAG_IS_WATCHED; fsnotify_put_sb_watched_objects(sb); + } } /* @@ -266,6 +269,7 @@ static void *fsnotify_detach_connector_from_object( unsigned int *type) { fsnotify_connp_t *connp = fsnotify_object_connp(conn->obj, conn->type); + struct super_block *sb = fsnotify_connector_sb(conn); struct inode *inode = NULL; *type = conn->type; @@ -285,10 +289,10 @@ static void *fsnotify_detach_connector_from_object( fsnotify_conn_sb(conn)->s_fsnotify_mask = 0; } - fsnotify_put_sb_watchers(conn); rcu_assign_pointer(*connp, NULL); conn->obj = NULL; conn->type = FSNOTIFY_OBJ_TYPE_DETACHED; + fsnotify_update_sb_watchers(sb, conn); return inode; } @@ -340,6 +344,11 @@ void fsnotify_put_mark(struct fsnotify_mark *mark) objp = fsnotify_detach_connector_from_object(conn, &type); free_conn = true; } else { + struct super_block *sb = fsnotify_connector_sb(conn); + + /* Update watched objects after detaching mark */ + if (sb) + fsnotify_update_sb_watchers(sb, conn); objp = __fsnotify_recalc_mask(conn); type = conn->type; } @@ -581,10 +590,7 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp, if (cmpxchg(connp, NULL, conn)) { /* Someone else created list structure for us */ kmem_cache_free(fsnotify_mark_connector_cachep, conn); - return 0; } - - fsnotify_get_sb_watchers(conn); return 0; } @@ -624,6 +630,7 @@ static struct fsnotify_mark_connector *fsnotify_grab_connector( static int fsnotify_add_mark_list(struct fsnotify_mark *mark, void *obj, unsigned int obj_type, int add_flags) { + struct super_block *sb = fsnotify_object_sb(obj, obj_type); struct fsnotify_mark *lmark, *last = NULL; struct fsnotify_mark_connector *conn; fsnotify_connp_t *connp; @@ -673,6 +680,7 @@ static int fsnotify_add_mark_list(struct fsnotify_mark *mark, void *obj, /* mark should be the last entry. last is the current last entry */ hlist_add_behind_rcu(&mark->obj_list, &last->obj_list); added: + fsnotify_update_sb_watchers(sb, conn); /* * Since connector is attached to object using cmpxchg() we are * guaranteed that connector initialization is fully visible by anyone diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index face68fcf8507..83004d9e07a31 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -465,6 +465,7 @@ FSNOTIFY_ITER_FUNCS(sb, SB) struct fsnotify_mark_connector { spinlock_t lock; unsigned short type; /* Type of object [lock] */ +#define FSNOTIFY_CONN_FLAG_IS_WATCHED 0x01 #define FSNOTIFY_CONN_FLAG_HAS_IREF 0x02 unsigned short flags; /* flags [lock] */ union { From 07a3b8d0bf726a1e49b050bbc6bd72f031e505fe Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Sun, 17 Mar 2024 20:41:51 +0200 Subject: [PATCH 0079/1702] fsnotify: lazy attach fsnotify_sb_info state to sb Define a container struct fsnotify_sb_info to hold per-sb state, including the reference to sb marks connector. Allocate the fsnotify_sb_info state before attaching connector to any object on the sb and free it only when killing sb. This state is going to be used for storing per priority watched objects counters. Suggested-by: Jan Kara Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara Message-Id: <20240317184154.1200192-8-amir73il@gmail.com> --- fs/notify/fsnotify.c | 16 +++++++++++++--- fs/notify/fsnotify.h | 9 ++++++++- fs/notify/mark.c | 32 +++++++++++++++++++++++++++++++- include/linux/fs.h | 8 ++++---- include/linux/fsnotify_backend.h | 17 +++++++++++++++++ 5 files changed, 73 insertions(+), 9 deletions(-) diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c index 503e7c75e777f..fb3f36bc6ea9e 100644 --- a/fs/notify/fsnotify.c +++ b/fs/notify/fsnotify.c @@ -89,11 +89,18 @@ static void fsnotify_unmount_inodes(struct super_block *sb) void fsnotify_sb_delete(struct super_block *sb) { + struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(sb); + + /* Were any marks ever added to any object on this sb? */ + if (!sbinfo) + return; + fsnotify_unmount_inodes(sb); fsnotify_clear_marks_by_sb(sb); /* Wait for outstanding object references from connectors */ wait_var_event(fsnotify_sb_watched_objects(sb), !atomic_long_read(fsnotify_sb_watched_objects(sb))); + kfree(sbinfo); } /* @@ -489,6 +496,7 @@ int fsnotify(__u32 mask, const void *data, int data_type, struct inode *dir, { const struct path *path = fsnotify_data_path(data, data_type); struct super_block *sb = fsnotify_data_sb(data, data_type); + struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(sb); struct fsnotify_iter_info iter_info = {}; struct mount *mnt = NULL; struct inode *inode2 = NULL; @@ -525,7 +533,7 @@ int fsnotify(__u32 mask, const void *data, int data_type, struct inode *dir, * SRCU because we have no references to any objects and do not * need SRCU to keep them "alive". */ - if (!sb->s_fsnotify_marks && + if ((!sbinfo || !sbinfo->sb_marks) && (!mnt || !mnt->mnt_fsnotify_marks) && (!inode || !inode->i_fsnotify_marks) && (!inode2 || !inode2->i_fsnotify_marks)) @@ -552,8 +560,10 @@ int fsnotify(__u32 mask, const void *data, int data_type, struct inode *dir, iter_info.srcu_idx = srcu_read_lock(&fsnotify_mark_srcu); - iter_info.marks[FSNOTIFY_ITER_TYPE_SB] = - fsnotify_first_mark(&sb->s_fsnotify_marks); + if (sbinfo) { + iter_info.marks[FSNOTIFY_ITER_TYPE_SB] = + fsnotify_first_mark(&sbinfo->sb_marks); + } if (mnt) { iter_info.marks[FSNOTIFY_ITER_TYPE_VFSMOUNT] = fsnotify_first_mark(&mnt->mnt_fsnotify_marks); diff --git a/fs/notify/fsnotify.h b/fs/notify/fsnotify.h index b1fec7cd8ee40..2d059f789ee35 100644 --- a/fs/notify/fsnotify.h +++ b/fs/notify/fsnotify.h @@ -54,6 +54,13 @@ static inline struct super_block *fsnotify_connector_sb( return fsnotify_object_sb(conn->obj, conn->type); } +static inline fsnotify_connp_t *fsnotify_sb_marks(struct super_block *sb) +{ + struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(sb); + + return sbinfo ? &sbinfo->sb_marks : NULL; +} + /* destroy all events sitting in this groups notification queue */ extern void fsnotify_flush_notify(struct fsnotify_group *group); @@ -79,7 +86,7 @@ static inline void fsnotify_clear_marks_by_mount(struct vfsmount *mnt) /* run the list of all marks associated with sb and destroy them */ static inline void fsnotify_clear_marks_by_sb(struct super_block *sb) { - fsnotify_destroy_marks(&sb->s_fsnotify_marks); + fsnotify_destroy_marks(fsnotify_sb_marks(sb)); } /* diff --git a/fs/notify/mark.c b/fs/notify/mark.c index 647c49c204673..b2f5d8c9cce11 100644 --- a/fs/notify/mark.c +++ b/fs/notify/mark.c @@ -106,7 +106,7 @@ static fsnotify_connp_t *fsnotify_object_connp(void *obj, case FSNOTIFY_OBJ_TYPE_VFSMOUNT: return &real_mount(obj)->mnt_fsnotify_marks; case FSNOTIFY_OBJ_TYPE_SB: - return &((struct super_block *)obj)->s_fsnotify_marks; + return fsnotify_sb_marks(obj); default: return NULL; } @@ -569,6 +569,26 @@ int fsnotify_compare_groups(struct fsnotify_group *a, struct fsnotify_group *b) return -1; } +static int fsnotify_attach_info_to_sb(struct super_block *sb) +{ + struct fsnotify_sb_info *sbinfo; + + /* sb info is freed on fsnotify_sb_delete() */ + sbinfo = kzalloc(sizeof(*sbinfo), GFP_KERNEL); + if (!sbinfo) + return -ENOMEM; + + /* + * cmpxchg() provides the barrier so that callers of fsnotify_sb_info() + * will observe an initialized structure + */ + if (cmpxchg(&sb->s_fsnotify_info, NULL, sbinfo)) { + /* Someone else created sbinfo for us */ + kfree(sbinfo); + } + return 0; +} + static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp, void *obj, unsigned int obj_type) { @@ -640,6 +660,16 @@ static int fsnotify_add_mark_list(struct fsnotify_mark *mark, void *obj, if (WARN_ON(!fsnotify_valid_obj_type(obj_type))) return -EINVAL; + /* + * Attach the sb info before attaching a connector to any object on sb. + * The sb info will remain attached as long as sb lives. + */ + if (!fsnotify_sb_info(sb)) { + err = fsnotify_attach_info_to_sb(sb); + if (err) + return err; + } + connp = fsnotify_object_connp(obj, obj_type); restart: spin_lock(&mark->lock); diff --git a/include/linux/fs.h b/include/linux/fs.h index 00fc429b0af0f..7f40b592f7119 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -73,6 +73,8 @@ struct fscrypt_inode_info; struct fscrypt_operations; struct fsverity_info; struct fsverity_operations; +struct fsnotify_mark_connector; +struct fsnotify_sb_info; struct fs_context; struct fs_parameter_spec; struct fileattr; @@ -620,8 +622,6 @@ is_uncached_acl(struct posix_acl *acl) #define IOP_XATTR 0x0008 #define IOP_DEFAULT_READLINK 0x0010 -struct fsnotify_mark_connector; - /* * Keep mostly read-only and often accessed (especially for * the RCU path lookup and 'stat' data) fields at the beginning @@ -1249,7 +1249,7 @@ struct super_block { /* * Keep s_fs_info, s_time_gran, s_fsnotify_mask, and - * s_fsnotify_marks together for cache efficiency. They are frequently + * s_fsnotify_info together for cache efficiency. They are frequently * accessed and rarely modified. */ void *s_fs_info; /* Filesystem private info */ @@ -1261,7 +1261,7 @@ struct super_block { time64_t s_time_max; #ifdef CONFIG_FSNOTIFY __u32 s_fsnotify_mask; - struct fsnotify_mark_connector __rcu *s_fsnotify_marks; + struct fsnotify_sb_info *s_fsnotify_info; #endif /* diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index 83004d9e07a31..c9f2b2f6b4938 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -477,6 +477,23 @@ struct fsnotify_mark_connector { struct hlist_head list; }; +/* + * Container for per-sb fsnotify state (sb marks and more). + * Attached lazily on first marked object on the sb and freed when killing sb. + */ +struct fsnotify_sb_info { + struct fsnotify_mark_connector __rcu *sb_marks; +}; + +static inline struct fsnotify_sb_info *fsnotify_sb_info(struct super_block *sb) +{ +#ifdef CONFIG_FSNOTIFY + return READ_ONCE(sb->s_fsnotify_info); +#else + return NULL; +#endif +} + static inline atomic_long_t *fsnotify_sb_watched_objects(struct super_block *sb) { return &sb->s_fsnotify_connectors; From cb5d4f48c10445c97a22af0bd8b9cf0ed6cc8036 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Sun, 17 Mar 2024 20:41:52 +0200 Subject: [PATCH 0080/1702] fsnotify: move s_fsnotify_connectors into fsnotify_sb_info Move the s_fsnotify_connectors counter into the per-sb fsnotify state. Suggested-by: Christian Brauner Signed-off-by: Amir Goldstein Reviewed-by: Christian Brauner Signed-off-by: Jan Kara Message-Id: <20240317184154.1200192-9-amir73il@gmail.com> --- include/linux/fs.h | 6 ------ include/linux/fsnotify.h | 8 +++++++- include/linux/fsnotify_backend.h | 7 ++++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/include/linux/fs.h b/include/linux/fs.h index 7f40b592f7119..c36c2f8fdbe30 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1302,12 +1302,6 @@ struct super_block { /* Number of inodes with nlink == 0 but still referenced */ atomic_long_t s_remove_count; - /* - * Number of inode/mount/sb objects that are being watched, note that - * inodes objects are currently double-accounted. - */ - atomic_long_t s_fsnotify_connectors; - /* Read-only state of the superblock is being changed */ int s_readonly_remount; diff --git a/include/linux/fsnotify.h b/include/linux/fsnotify.h index e470bb67c9a33..48dc657024151 100644 --- a/include/linux/fsnotify.h +++ b/include/linux/fsnotify.h @@ -20,7 +20,13 @@ /* Are there any inode/mount/sb objects that are being watched at all? */ static inline bool fsnotify_sb_has_watchers(struct super_block *sb) { - return atomic_long_read(fsnotify_sb_watched_objects(sb)); + struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(sb); + + /* Were any marks ever added to any object on this sb? */ + if (!sbinfo) + return false; + + return atomic_long_read(&sbinfo->watched_objects); } /* diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index c9f2b2f6b4938..ec592aeadfa3f 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -483,6 +483,11 @@ struct fsnotify_mark_connector { */ struct fsnotify_sb_info { struct fsnotify_mark_connector __rcu *sb_marks; + /* + * Number of inode/mount/sb objects that are being watched in this sb. + * Note that inodes objects are currently double-accounted. + */ + atomic_long_t watched_objects; }; static inline struct fsnotify_sb_info *fsnotify_sb_info(struct super_block *sb) @@ -496,7 +501,7 @@ static inline struct fsnotify_sb_info *fsnotify_sb_info(struct super_block *sb) static inline atomic_long_t *fsnotify_sb_watched_objects(struct super_block *sb) { - return &sb->s_fsnotify_connectors; + return &fsnotify_sb_info(sb)->watched_objects; } /* From 477cf917dd02853ba78a73cdeb6548889e5f8cd7 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Sun, 17 Mar 2024 20:41:53 +0200 Subject: [PATCH 0081/1702] fsnotify: use an enum for group priority constants And use meaningfull names for the constants. Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara Message-Id: <20240317184154.1200192-10-amir73il@gmail.com> --- fs/notify/fanotify/fanotify_user.c | 11 +++++------ include/linux/fsnotify_backend.h | 20 ++++++++++++-------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index 925015ee7fbd1..483a6a1255fb2 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -1516,13 +1516,13 @@ SYSCALL_DEFINE2(fanotify_init, unsigned int, flags, unsigned int, event_f_flags) INIT_LIST_HEAD(&group->fanotify_data.access_list); switch (class) { case FAN_CLASS_NOTIF: - group->priority = FS_PRIO_0; + group->priority = FSNOTIFY_PRIO_NORMAL; break; case FAN_CLASS_CONTENT: - group->priority = FS_PRIO_1; + group->priority = FSNOTIFY_PRIO_CONTENT; break; case FAN_CLASS_PRE_CONTENT: - group->priority = FS_PRIO_2; + group->priority = FSNOTIFY_PRIO_PRE_CONTENT; break; default: fd = -EINVAL; @@ -1774,12 +1774,11 @@ static int do_fanotify_mark(int fanotify_fd, unsigned int flags, __u64 mask, goto fput_and_out; /* - * group->priority == FS_PRIO_0 == FAN_CLASS_NOTIF. These are not - * allowed to set permissions events. + * Permission events require minimum priority FAN_CLASS_CONTENT. */ ret = -EINVAL; if (mask & FANOTIFY_PERM_EVENTS && - group->priority == FS_PRIO_0) + group->priority < FSNOTIFY_PRIO_CONTENT) goto fput_and_out; if (mask & FAN_FS_ERROR && diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index ec592aeadfa3f..fc38587d85643 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -176,6 +176,17 @@ struct fsnotify_event { struct list_head list; }; +/* + * fsnotify group priorities. + * Events are sent in order from highest priority to lowest priority. + */ +enum fsnotify_group_prio { + FSNOTIFY_PRIO_NORMAL = 0, /* normal notifiers, no permissions */ + FSNOTIFY_PRIO_CONTENT, /* fanotify permission events */ + FSNOTIFY_PRIO_PRE_CONTENT, /* fanotify pre-content events */ + __FSNOTIFY_PRIO_NUM +}; + /* * A group is a "thing" that wants to receive notification about filesystem * events. The mask holds the subset of event types this group cares about. @@ -201,14 +212,7 @@ struct fsnotify_group { wait_queue_head_t notification_waitq; /* read() on the notification file blocks on this waitq */ unsigned int q_len; /* events on the queue */ unsigned int max_events; /* maximum events allowed on the list */ - /* - * Valid fsnotify group priorities. Events are send in order from highest - * priority to lowest priority. We default to the lowest priority. - */ - #define FS_PRIO_0 0 /* normal notifiers, no permissions */ - #define FS_PRIO_1 1 /* fanotify content based access control */ - #define FS_PRIO_2 2 /* fanotify pre-content access */ - unsigned int priority; + enum fsnotify_group_prio priority; /* priority for sending events */ bool shutdown; /* group is being shut down, don't queue more events */ #define FSNOTIFY_GROUP_USER 0x01 /* user allocated group */ From a5e57b4d370c6d320e5bfb0c919fe00aee29e039 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Sun, 17 Mar 2024 20:41:54 +0200 Subject: [PATCH 0082/1702] fsnotify: optimize the case of no permission event watchers Commit e43de7f0862b ("fsnotify: optimize the case of no marks of any type") optimized the case where there are no fsnotify watchers on any of the filesystem's objects. It is quite common for a system to have a single local filesystem and it is quite common for the system to have some inotify watches on some config files or directories, so the optimization of no marks at all is often not in effect. Permission event watchers, which require high priority group are more rare, so optimizing the case of no marks og high priority groups can improve performance for more systems, especially for performance sensitive io workloads. Count per-sb watched objects by high priority groups and use that the optimize out the call to __fsnotify_parent() and fsnotify() in fsnotify permission hooks. Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara Message-Id: <20240317184154.1200192-11-amir73il@gmail.com> --- fs/notify/fsnotify.c | 3 +++ fs/notify/mark.c | 30 +++++++++++++++++++++++++++--- include/linux/fsnotify.h | 19 ++++++++++++++++--- include/linux/fsnotify_backend.h | 11 ++++++++--- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c index fb3f36bc6ea9e..2ae965ef37e85 100644 --- a/fs/notify/fsnotify.c +++ b/fs/notify/fsnotify.c @@ -100,6 +100,9 @@ void fsnotify_sb_delete(struct super_block *sb) /* Wait for outstanding object references from connectors */ wait_var_event(fsnotify_sb_watched_objects(sb), !atomic_long_read(fsnotify_sb_watched_objects(sb))); + WARN_ON(fsnotify_sb_has_priority_watchers(sb, FSNOTIFY_PRIO_CONTENT)); + WARN_ON(fsnotify_sb_has_priority_watchers(sb, + FSNOTIFY_PRIO_PRE_CONTENT)); kfree(sbinfo); } diff --git a/fs/notify/mark.c b/fs/notify/mark.c index b2f5d8c9cce11..c3eefa70633c4 100644 --- a/fs/notify/mark.c +++ b/fs/notify/mark.c @@ -161,13 +161,36 @@ static void fsnotify_put_inode_ref(struct inode *inode) static void fsnotify_update_sb_watchers(struct super_block *sb, struct fsnotify_mark_connector *conn) { + struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(sb); bool is_watched = conn->flags & FSNOTIFY_CONN_FLAG_IS_WATCHED; - bool has_marks = conn->obj && !hlist_empty(&conn->list); + struct fsnotify_mark *first_mark = NULL; + unsigned int highest_prio = 0; - if (has_marks && !is_watched) { + if (conn->obj) + first_mark = hlist_entry_safe(conn->list.first, + struct fsnotify_mark, obj_list); + if (first_mark) + highest_prio = first_mark->group->priority; + if (WARN_ON(highest_prio >= __FSNOTIFY_PRIO_NUM)) + highest_prio = 0; + + /* + * If the highest priority of group watching this object is prio, + * then watched object has a reference on counters [0..prio]. + * Update priority >= 1 watched objects counters. + */ + for (unsigned int p = conn->prio + 1; p <= highest_prio; p++) + atomic_long_inc(&sbinfo->watched_objects[p]); + for (unsigned int p = conn->prio; p > highest_prio; p--) + atomic_long_dec(&sbinfo->watched_objects[p]); + conn->prio = highest_prio; + + /* Update priority >= 0 (a.k.a total) watched objects counter */ + BUILD_BUG_ON(FSNOTIFY_PRIO_NORMAL != 0); + if (first_mark && !is_watched) { conn->flags |= FSNOTIFY_CONN_FLAG_IS_WATCHED; fsnotify_get_sb_watched_objects(sb); - } else if (!has_marks && is_watched) { + } else if (!first_mark && is_watched) { conn->flags &= ~FSNOTIFY_CONN_FLAG_IS_WATCHED; fsnotify_put_sb_watched_objects(sb); } @@ -600,6 +623,7 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp, spin_lock_init(&conn->lock); INIT_HLIST_HEAD(&conn->list); conn->flags = 0; + conn->prio = 0; conn->type = obj_type; conn->obj = obj; diff --git a/include/linux/fsnotify.h b/include/linux/fsnotify.h index 48dc657024151..4da80e92f804f 100644 --- a/include/linux/fsnotify.h +++ b/include/linux/fsnotify.h @@ -17,8 +17,9 @@ #include #include -/* Are there any inode/mount/sb objects that are being watched at all? */ -static inline bool fsnotify_sb_has_watchers(struct super_block *sb) +/* Are there any inode/mount/sb objects watched with priority prio or above? */ +static inline bool fsnotify_sb_has_priority_watchers(struct super_block *sb, + int prio) { struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(sb); @@ -26,7 +27,13 @@ static inline bool fsnotify_sb_has_watchers(struct super_block *sb) if (!sbinfo) return false; - return atomic_long_read(&sbinfo->watched_objects); + return atomic_long_read(&sbinfo->watched_objects[prio]); +} + +/* Are there any inode/mount/sb objects that are being watched at all? */ +static inline bool fsnotify_sb_has_watchers(struct super_block *sb) +{ + return fsnotify_sb_has_priority_watchers(sb, 0); } /* @@ -109,6 +116,12 @@ static inline int fsnotify_file(struct file *file, __u32 mask) return 0; path = &file->f_path; + /* Permission events require group prio >= FSNOTIFY_PRIO_CONTENT */ + if (mask & ALL_FSNOTIFY_PERM_EVENTS && + !fsnotify_sb_has_priority_watchers(path->dentry->d_sb, + FSNOTIFY_PRIO_CONTENT)) + return 0; + return fsnotify_parent(path->dentry, mask, path, FSNOTIFY_EVENT_PATH); } diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index fc38587d85643..7f1ab8264e41d 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -468,7 +468,8 @@ FSNOTIFY_ITER_FUNCS(sb, SB) */ struct fsnotify_mark_connector { spinlock_t lock; - unsigned short type; /* Type of object [lock] */ + unsigned char type; /* Type of object [lock] */ + unsigned char prio; /* Highest priority group */ #define FSNOTIFY_CONN_FLAG_IS_WATCHED 0x01 #define FSNOTIFY_CONN_FLAG_HAS_IREF 0x02 unsigned short flags; /* flags [lock] */ @@ -490,8 +491,12 @@ struct fsnotify_sb_info { /* * Number of inode/mount/sb objects that are being watched in this sb. * Note that inodes objects are currently double-accounted. + * + * The value in watched_objects[prio] is the number of objects that are + * watched by groups of priority >= prio, so watched_objects[0] is the + * total number of watched objects in this sb. */ - atomic_long_t watched_objects; + atomic_long_t watched_objects[__FSNOTIFY_PRIO_NUM]; }; static inline struct fsnotify_sb_info *fsnotify_sb_info(struct super_block *sb) @@ -505,7 +510,7 @@ static inline struct fsnotify_sb_info *fsnotify_sb_info(struct super_block *sb) static inline atomic_long_t *fsnotify_sb_watched_objects(struct super_block *sb) { - return &fsnotify_sb_info(sb)->watched_objects; + return &fsnotify_sb_info(sb)->watched_objects[0]; } /* From 3e90417f41f41a7d4a6d5ee0ca216698e9ab4381 Mon Sep 17 00:00:00 2001 From: "Ritesh Harjani (IBM)" Date: Thu, 29 Feb 2024 19:54:13 +0530 Subject: [PATCH 0083/1702] ext2: set FMODE_CAN_ODIRECT instead of a dummy direct_IO method Since commit a2ad63daa88b ("VFS: add FMODE_CAN_ODIRECT file flag") file systems can just set the FMODE_CAN_ODIRECT flag at open time instead of wiring up a dummy direct_IO method to indicate support for direct I/O. Signed-off-by: Ritesh Harjani (IBM) Signed-off-by: Jan Kara Message-Id: <94f78492f55c3f421359fb6e0d8fab6e79ea17b2.1709215665.git.ritesh.list@gmail.com> --- fs/ext2/file.c | 8 +++++++- fs/ext2/inode.c | 2 -- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/fs/ext2/file.c b/fs/ext2/file.c index 4ddc36f4dbd40..10b061ac5bc09 100644 --- a/fs/ext2/file.c +++ b/fs/ext2/file.c @@ -302,6 +302,12 @@ static ssize_t ext2_file_write_iter(struct kiocb *iocb, struct iov_iter *from) return generic_file_write_iter(iocb, from); } +static int ext2_file_open(struct inode *inode, struct file *filp) +{ + filp->f_mode |= FMODE_CAN_ODIRECT; + return dquot_file_open(inode, filp); +} + const struct file_operations ext2_file_operations = { .llseek = generic_file_llseek, .read_iter = ext2_file_read_iter, @@ -311,7 +317,7 @@ const struct file_operations ext2_file_operations = { .compat_ioctl = ext2_compat_ioctl, #endif .mmap = ext2_file_mmap, - .open = dquot_file_open, + .open = ext2_file_open, .release = ext2_release_file, .fsync = ext2_fsync, .get_unmapped_area = thp_get_unmapped_area, diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index f3d570a9302b0..0caa1650cee87 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -965,7 +965,6 @@ const struct address_space_operations ext2_aops = { .write_begin = ext2_write_begin, .write_end = ext2_write_end, .bmap = ext2_bmap, - .direct_IO = noop_direct_IO, .writepages = ext2_writepages, .migrate_folio = buffer_migrate_folio, .is_partially_uptodate = block_is_partially_uptodate, @@ -974,7 +973,6 @@ const struct address_space_operations ext2_aops = { static const struct address_space_operations ext2_dax_aops = { .writepages = ext2_dax_writepages, - .direct_IO = noop_direct_IO, .dirty_folio = noop_dirty_folio, }; From 0cd830ac3be548c920924baa84f782a763167460 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Mon, 1 Apr 2024 17:31:53 +0200 Subject: [PATCH 0084/1702] clk: nxp: Remove an unused field in struct lpc18xx_pll In "struct lpc18xx_pll", the 'lock' field is unused. Remove it. Found with cppcheck, unusedStructMember. Signed-off-by: Christophe JAILLET Link: https://lore.kernel.org/r/6cfb0e5251c3a59a156e70bcf6a0cc74aa764faa.1711985490.git.christophe.jaillet@wanadoo.fr Acked-by: Vladimir Zapolskiy Signed-off-by: Stephen Boyd --- drivers/clk/nxp/clk-lpc18xx-cgu.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/clk/nxp/clk-lpc18xx-cgu.c b/drivers/clk/nxp/clk-lpc18xx-cgu.c index 69ebf65081b81..81efa885069b2 100644 --- a/drivers/clk/nxp/clk-lpc18xx-cgu.c +++ b/drivers/clk/nxp/clk-lpc18xx-cgu.c @@ -250,7 +250,6 @@ static struct lpc18xx_cgu_base_clk lpc18xx_cgu_base_clks[] = { struct lpc18xx_pll { struct clk_hw hw; void __iomem *reg; - spinlock_t *lock; u8 flags; }; From 922c86f8d3ad6c85142be3b8317f0e28a794116c Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Mon, 1 Apr 2024 19:30:05 +0200 Subject: [PATCH 0085/1702] pinctrl: max77620: Remove an unused fields in struct max77620_pin_info and max77620_pctrl_info In "struct max77620_pin_info", the 'pull_config' field is unused. In "struct max77620_pctrl_info", the 'pins_current_opt' field is unused. Remove them. On my x86_64 config, with allmodconfig, this shrinks the struct max77620_pctrl_info from 360 bytes to 296. Found with cppcheck, unusedStructMember. Signed-off-by: Christophe JAILLET Message-ID: <60af8968864ae4a83a76e589b39a2b1e1f65c9db.1711992588.git.christophe.jaillet@wanadoo.fr> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-max77620.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/pinctrl/pinctrl-max77620.c b/drivers/pinctrl/pinctrl-max77620.c index ab723ab4ec1d5..d236daa7c13eb 100644 --- a/drivers/pinctrl/pinctrl-max77620.c +++ b/drivers/pinctrl/pinctrl-max77620.c @@ -88,7 +88,6 @@ struct max77620_pingroup { struct max77620_pin_info { enum max77620_pin_ppdrv drv_type; - int pull_config; }; struct max77620_fps_config { @@ -104,7 +103,6 @@ struct max77620_pctrl_info { struct device *dev; struct pinctrl_dev *pctl; struct regmap *rmap; - int pins_current_opt[MAX77620_GPIO_NR]; const struct max77620_pin_function *functions; unsigned int num_functions; const struct max77620_pingroup *pin_groups; From a95e2bc817feac96f229910c6822c682da887fa1 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Mon, 1 Apr 2024 22:48:55 +0200 Subject: [PATCH 0086/1702] pinctrl: pinctrl-single: Remove some unused fields in struct pcs_function In "struct pcs_function", the 'pgnames' and 'npgnames' fields are unused. This is a left-over from commit 571aec4df5b7 ("pinctrl: single: Use generic pinmux helpers for managing functions"); Remove them. Found with cppcheck, unusedStructMember. Signed-off-by: Christophe JAILLET Message-ID: Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-single.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c index 0dd4b0e11adf7..b4f8aa609b972 100644 --- a/drivers/pinctrl/pinctrl-single.c +++ b/drivers/pinctrl/pinctrl-single.c @@ -81,8 +81,6 @@ struct pcs_conf_type { * @name: pinctrl function name * @vals: register and vals array * @nvals: number of entries in vals array - * @pgnames: array of pingroup names the function uses - * @npgnames: number of pingroup names the function uses * @conf: array of pin configurations * @nconfs: number of pin configurations available * @node: list node @@ -91,8 +89,6 @@ struct pcs_function { const char *name; struct pcs_func_vals *vals; unsigned nvals; - const char **pgnames; - int npgnames; struct pcs_conf_vals *conf; int nconfs; struct list_head node; From b5fe46efc147516a908d2d31bf40eb858ab76d51 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Tue, 19 Mar 2024 12:06:34 +0100 Subject: [PATCH 0087/1702] pinctrl: single: Fix PIN_CONFIG_BIAS_DISABLE handling The pinctrl-single driver handles pin_config_set by looking up the requested setting in a DT-defined lookup table, which defines what bits correspond to each setting. There is no way to add PIN_CONFIG_BIAS_DISABLE entries to the table, since there is instead code to disable the bias by applying the disable values of both the pullup and pulldown entries in the table. However, this code is inside the table-lookup loop, so it would only execute if there is an entry for PIN_CONFIG_BIAS_DISABLE in the table, which can never exist, so this code never runs. This commit lifts the offending code out of the loop, so it just executes directly whenever PIN_CONFIG_BIAS_DISABLE is requested, skippipng the table lookup loop. This also introduces a new `param` variable to make the code slightly more readable. This bug seems to have existed when this code was first merged in commit 9dddb4df90d13 ("pinctrl: single: support generic pinconf"). Earlier versions of this patch did have an entry for PIN_CONFIG_BIAS_DISABLE in the lookup table, but that was removed, which is probably how this bug was introduced. Signed-off-by: Matthijs Kooijman Reviewed-by: Haojian Zhuang Reviewed-by: Tony Lindgren Message-ID: <20240319110633.230329-1-matthijs@stdin.nl> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-single.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c index b4f8aa609b972..a798f31d69542 100644 --- a/drivers/pinctrl/pinctrl-single.c +++ b/drivers/pinctrl/pinctrl-single.c @@ -550,21 +550,30 @@ static int pcs_pinconf_set(struct pinctrl_dev *pctldev, unsigned offset = 0, shift = 0, i, data, ret; u32 arg; int j; + enum pin_config_param param; ret = pcs_get_function(pctldev, pin, &func); if (ret) return ret; for (j = 0; j < num_configs; j++) { + param = pinconf_to_config_param(configs[j]); + + /* BIAS_DISABLE has no entry in the func->conf table */ + if (param == PIN_CONFIG_BIAS_DISABLE) { + /* This just disables all bias entries */ + pcs_pinconf_clear_bias(pctldev, pin); + continue; + } + for (i = 0; i < func->nconfs; i++) { - if (pinconf_to_config_param(configs[j]) - != func->conf[i].param) + if (param != func->conf[i].param) continue; offset = pin * (pcs->width / BITS_PER_BYTE); data = pcs->read(pcs->base + offset); arg = pinconf_to_config_argument(configs[j]); - switch (func->conf[i].param) { + switch (param) { /* 2 parameters */ case PIN_CONFIG_INPUT_SCHMITT: case PIN_CONFIG_DRIVE_STRENGTH: @@ -576,9 +585,6 @@ static int pcs_pinconf_set(struct pinctrl_dev *pctldev, data |= (arg << shift) & func->conf[i].mask; break; /* 4 parameters */ - case PIN_CONFIG_BIAS_DISABLE: - pcs_pinconf_clear_bias(pctldev, pin); - break; case PIN_CONFIG_BIAS_PULL_DOWN: case PIN_CONFIG_BIAS_PULL_UP: if (arg) From 4acfeecd0062821fe66a520635adf65c6db759c3 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 3 Apr 2024 10:06:29 +0200 Subject: [PATCH 0088/1702] clk: ti: dpll: fix incorrect #ifdef checks Building with W=1 shows warnings about unused const variables like this one: drivers/clk/ti/dpll.c:99:29: error: unused variable 'omap3_dpll_core_ck_ops' [-Werror,-Wunused-const-variable] static const struct clk_ops omap3_dpll_core_ck_ops = {}; The problem is that the #ifdef checks for some of the structures in this file have gone out of sync with the code referencing them. Update these to match the current usage. Signed-off-by: Arnd Bergmann Link: https://lore.kernel.org/r/20240403080702.3509288-12-arnd@kernel.org Reviewed-by: Tony Lindgren Signed-off-by: Stephen Boyd --- drivers/clk/ti/dpll.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/clk/ti/dpll.c b/drivers/clk/ti/dpll.c index 403ec81f561b6..3386bd1903df6 100644 --- a/drivers/clk/ti/dpll.c +++ b/drivers/clk/ti/dpll.c @@ -34,8 +34,6 @@ static const struct clk_ops dpll_m4xen_ck_ops = { .save_context = &omap3_core_dpll_save_context, .restore_context = &omap3_core_dpll_restore_context, }; -#else -static const struct clk_ops dpll_m4xen_ck_ops = {}; #endif #if defined(CONFIG_ARCH_OMAP3) || defined(CONFIG_ARCH_OMAP4) || \ @@ -95,11 +93,7 @@ static const struct clk_ops omap3_dpll_core_ck_ops = { .recalc_rate = &omap3_dpll_recalc, .round_rate = &omap2_dpll_round_rate, }; -#else -static const struct clk_ops omap3_dpll_core_ck_ops = {}; -#endif -#ifdef CONFIG_ARCH_OMAP3 static const struct clk_ops omap3_dpll_ck_ops = { .enable = &omap3_noncore_dpll_enable, .disable = &omap3_noncore_dpll_disable, @@ -137,9 +131,13 @@ static const struct clk_ops omap3_dpll_per_ck_ops = { }; #endif +#if defined(CONFIG_ARCH_OMAP4) || defined(CONFIG_SOC_OMAP5) || \ + defined(CONFIG_SOC_DRA7XX) || defined(CONFIG_SOC_AM33XX) || \ + defined(CONFIG_SOC_AM43XX) static const struct clk_ops dpll_x2_ck_ops = { .recalc_rate = &omap3_clkoutx2_recalc, }; +#endif /** * _register_dpll - low level registration of a DPLL clock From cf770af5645a41a753c55a053fa1237105b0964a Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 8 Apr 2024 09:34:24 +0200 Subject: [PATCH 0089/1702] firmware: dmi-id: add a release callback function dmi_class uses kfree() as the .release function, but that now causes a warning with clang-16 as it violates control flow integrity (KCFI) rules: drivers/firmware/dmi-id.c:174:17: error: cast from 'void (*)(const void *)' to 'void (*)(struct device *)' converts to incompatible function type [-Werror,-Wcast-function-type-strict] 174 | .dev_release = (void(*)(struct device *)) kfree, Add an explicit function to call kfree() instead. Fixes: 4f5c791a850e ("DMI-based module autoloading") Link: https://lore.kernel.org/lkml/20240213100238.456912-1-arnd@kernel.org/ Signed-off-by: Arnd Bergmann Signed-off-by: Jean Delvare --- drivers/firmware/dmi-id.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/firmware/dmi-id.c b/drivers/firmware/dmi-id.c index 5f3a3e913d28f..d19c78a78ae3a 100644 --- a/drivers/firmware/dmi-id.c +++ b/drivers/firmware/dmi-id.c @@ -169,9 +169,14 @@ static int dmi_dev_uevent(const struct device *dev, struct kobj_uevent_env *env) return 0; } +static void dmi_dev_release(struct device *dev) +{ + kfree(dev); +} + static struct class dmi_class = { .name = "dmi", - .dev_release = (void(*)(struct device *)) kfree, + .dev_release = dmi_dev_release, .dev_uevent = dmi_dev_uevent, }; From 932640c0f78947c59a6e9c743a17a355165d69d5 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 4 Apr 2024 22:35:21 +0300 Subject: [PATCH 0090/1702] pinctrl: Use DEFINE_SHOW_STORE_ATTRIBUTE() helper for debugfs Use DEFINE_SHOW_STORE_ATTRIBUTE() helper for read-write file to reduce some duplicated code. Signed-off-by: Andy Shevchenko Message-ID: <20240404193521.3581399-1-andriy.shevchenko@linux.intel.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinmux.c | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/drivers/pinctrl/pinmux.c b/drivers/pinctrl/pinmux.c index d924207d629b4..addba55334d94 100644 --- a/drivers/pinctrl/pinmux.c +++ b/drivers/pinctrl/pinmux.c @@ -578,6 +578,7 @@ static int pinmux_functions_show(struct seq_file *s, void *what) return 0; } +DEFINE_SHOW_ATTRIBUTE(pinmux_functions); static int pinmux_pins_show(struct seq_file *s, void *what) { @@ -650,6 +651,7 @@ static int pinmux_pins_show(struct seq_file *s, void *what) return 0; } +DEFINE_SHOW_ATTRIBUTE(pinmux_pins); void pinmux_show_map(struct seq_file *s, const struct pinctrl_map *map) { @@ -672,10 +674,12 @@ void pinmux_show_setting(struct seq_file *s, setting->data.mux.func); } -DEFINE_SHOW_ATTRIBUTE(pinmux_functions); -DEFINE_SHOW_ATTRIBUTE(pinmux_pins); +static int pinmux_select_show(struct seq_file *s, void *unused) +{ + return -EPERM; +} -static ssize_t pinmux_select(struct file *file, const char __user *user_buf, +static ssize_t pinmux_select_write(struct file *file, const char __user *user_buf, size_t len, loff_t *ppos) { struct seq_file *sfile = file->private_data; @@ -749,19 +753,7 @@ static ssize_t pinmux_select(struct file *file, const char __user *user_buf, return ret; } - -static int pinmux_select_open(struct inode *inode, struct file *file) -{ - return single_open(file, NULL, inode->i_private); -} - -static const struct file_operations pinmux_select_ops = { - .owner = THIS_MODULE, - .open = pinmux_select_open, - .write = pinmux_select, - .llseek = no_llseek, - .release = single_release, -}; +DEFINE_SHOW_STORE_ATTRIBUTE(pinmux_select); void pinmux_init_device_debugfs(struct dentry *devroot, struct pinctrl_dev *pctldev) @@ -771,7 +763,7 @@ void pinmux_init_device_debugfs(struct dentry *devroot, debugfs_create_file("pinmux-pins", 0444, devroot, pctldev, &pinmux_pins_fops); debugfs_create_file("pinmux-select", 0200, - devroot, pctldev, &pinmux_select_ops); + devroot, pctldev, &pinmux_select_fops); } #endif /* CONFIG_DEBUG_FS */ From c0516eb4cf04ac61b6fe1f86cc15b2f5f024ee78 Mon Sep 17 00:00:00 2001 From: Thanh Quan Date: Tue, 2 Apr 2024 16:37:52 +0200 Subject: [PATCH 0091/1702] clk: renesas: r8a779h0: Add timer clocks Add the module clocks used by Timer (CMT/TMU) blocks on the Renesas R-Car V4M (R8A779H0) SoC. Signed-off-by: Thanh Quan Signed-off-by: Geert Uytterhoeven Reviewed-by: Wolfram Sang Link: https://lore.kernel.org/r/79a66e8ff84378d7f65d5f55cfb01b9b745edd12.1712068639.git.geert+renesas@glider.be --- drivers/clk/renesas/r8a779h0-cpg-mssr.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/clk/renesas/r8a779h0-cpg-mssr.c b/drivers/clk/renesas/r8a779h0-cpg-mssr.c index 4bc35bc912547..a7d272285db04 100644 --- a/drivers/clk/renesas/r8a779h0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a779h0-cpg-mssr.c @@ -192,7 +192,16 @@ static const struct mssr_mod_clk r8a779h0_mod_clks[] = { DEF_MOD("sdhi0", 706, R8A779H0_CLK_SD0), DEF_MOD("sydm1", 709, R8A779H0_CLK_S0D6_PER), DEF_MOD("sydm2", 710, R8A779H0_CLK_S0D6_PER), + DEF_MOD("tmu0", 713, R8A779H0_CLK_SASYNCRT), + DEF_MOD("tmu1", 714, R8A779H0_CLK_SASYNCPERD2), + DEF_MOD("tmu2", 715, R8A779H0_CLK_SASYNCPERD2), + DEF_MOD("tmu3", 716, R8A779H0_CLK_SASYNCPERD2), + DEF_MOD("tmu4", 717, R8A779H0_CLK_SASYNCPERD2), DEF_MOD("wdt1:wdt0", 907, R8A779H0_CLK_R), + DEF_MOD("cmt0", 910, R8A779H0_CLK_R), + DEF_MOD("cmt1", 911, R8A779H0_CLK_R), + DEF_MOD("cmt2", 912, R8A779H0_CLK_R), + DEF_MOD("cmt3", 913, R8A779H0_CLK_R), DEF_MOD("pfc0", 915, R8A779H0_CLK_CP), DEF_MOD("pfc1", 916, R8A779H0_CLK_CP), DEF_MOD("pfc2", 917, R8A779H0_CLK_CP), From 0611a8e8b475fc5230b9a24d29c8397aaab20b63 Mon Sep 17 00:00:00 2001 From: Or Har-Toov Date: Wed, 3 Apr 2024 13:35:59 +0300 Subject: [PATCH 0092/1702] RDMA/mlx5: Uncacheable mkey has neither rb_key or cache_ent As some mkeys can't be modified with UMR due to some UMR limitations, like the size of translation that can be updated, not all user mkeys can be cached. Fixes: dd1b913fb0d0 ("RDMA/mlx5: Cache all user cacheable mkeys on dereg MR flow") Signed-off-by: Or Har-Toov Link: https://lore.kernel.org/r/f2742dd934ed73b2d32c66afb8e91b823063880c.1712140377.git.leon@kernel.org Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/mlx5_ib.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index a8de35c07c9ef..e74f048650624 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -643,7 +643,7 @@ struct mlx5_ib_mkey { unsigned int ndescs; struct wait_queue_head wait; refcount_t usecount; - /* User Mkey must hold either a rb_key or a cache_ent. */ + /* Cacheable user Mkey must hold either a rb_key or a cache_ent. */ struct mlx5r_cache_rb_key rb_key; struct mlx5_cache_ent *cache_ent; }; From 8c1185fef68cc603b954fece2a434c9f851d6a86 Mon Sep 17 00:00:00 2001 From: Or Har-Toov Date: Wed, 3 Apr 2024 13:36:00 +0300 Subject: [PATCH 0093/1702] RDMA/mlx5: Change check for cacheable mkeys umem can be NULL for user application mkeys in some cases. Therefore umem can't be used for checking if the mkey is cacheable and it is changed for checking a flag that indicates it. Also make sure that all mkeys which are not returned to the cache will be destroyed. Fixes: dd1b913fb0d0 ("RDMA/mlx5: Cache all user cacheable mkeys on dereg MR flow") Signed-off-by: Or Har-Toov Link: https://lore.kernel.org/r/2690bc5c6896bcb937f89af16a1ff0343a7ab3d0.1712140377.git.leon@kernel.org Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/mlx5_ib.h | 1 + drivers/infiniband/hw/mlx5/mr.c | 32 +++++++++++++++++++--------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index e74f048650624..f255a12e26a02 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -646,6 +646,7 @@ struct mlx5_ib_mkey { /* Cacheable user Mkey must hold either a rb_key or a cache_ent. */ struct mlx5r_cache_rb_key rb_key; struct mlx5_cache_ent *cache_ent; + u8 cacheable : 1; }; #define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE) diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index a8ee2ca1f4a17..7f7b1f59b5f05 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1158,6 +1158,7 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd, if (IS_ERR(mr)) return mr; mr->mmkey.rb_key = rb_key; + mr->mmkey.cacheable = true; return mr; } @@ -1168,6 +1169,7 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd, mr->ibmr.pd = pd; mr->umem = umem; mr->page_shift = order_base_2(page_size); + mr->mmkey.cacheable = true; set_mr_fields(dev, mr, umem->length, access_flags, iova); return mr; @@ -1835,6 +1837,23 @@ static int cache_ent_find_and_store(struct mlx5_ib_dev *dev, return ret; } +static int mlx5_revoke_mr(struct mlx5_ib_mr *mr) +{ + struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device); + struct mlx5_cache_ent *ent = mr->mmkey.cache_ent; + + if (mr->mmkey.cacheable && !mlx5r_umr_revoke_mr(mr) && !cache_ent_find_and_store(dev, mr)) + return 0; + + if (ent) { + spin_lock_irq(&ent->mkeys_queue.lock); + ent->in_use--; + mr->mmkey.cache_ent = NULL; + spin_unlock_irq(&ent->mkeys_queue.lock); + } + return destroy_mkey(dev, mr); +} + int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) { struct mlx5_ib_mr *mr = to_mmr(ibmr); @@ -1880,16 +1899,9 @@ int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) } /* Stop DMA */ - if (mr->umem && mlx5r_umr_can_load_pas(dev, mr->umem->length)) - if (mlx5r_umr_revoke_mr(mr) || - cache_ent_find_and_store(dev, mr)) - mr->mmkey.cache_ent = NULL; - - if (!mr->mmkey.cache_ent) { - rc = destroy_mkey(to_mdev(mr->ibmr.device), mr); - if (rc) - return rc; - } + rc = mlx5_revoke_mr(mr); + if (rc) + return rc; if (mr->umem) { bool is_odp = is_odp_mr(mr); From 2ca7e93bc963d9ec2f5c24d117176851454967af Mon Sep 17 00:00:00 2001 From: Or Har-Toov Date: Wed, 3 Apr 2024 13:36:01 +0300 Subject: [PATCH 0094/1702] RDMA/mlx5: Adding remote atomic access flag to updatable flags Currently IB_ACCESS_REMOTE_ATOMIC is blocked from being updated via UMR although in some cases it should be possible. These cases are checked in mlx5r_umr_can_reconfig function. Fixes: ef3642c4f54d ("RDMA/mlx5: Fix error unwinds for rereg_mr") Signed-off-by: Or Har-Toov Link: https://lore.kernel.org/r/24dac73e2fa48cb806f33a932d97f3e402a5ea2c.1712140377.git.leon@kernel.org Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/mr.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 7f7b1f59b5f05..ecc111ed5d86e 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1572,7 +1572,8 @@ static bool can_use_umr_rereg_access(struct mlx5_ib_dev *dev, unsigned int diffs = current_access_flags ^ target_access_flags; if (diffs & ~(IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE | - IB_ACCESS_REMOTE_READ | IB_ACCESS_RELAXED_ORDERING)) + IB_ACCESS_REMOTE_READ | IB_ACCESS_RELAXED_ORDERING | + IB_ACCESS_REMOTE_ATOMIC)) return false; return mlx5r_umr_can_reconfig(dev, current_access_flags, target_access_flags); From ee20cc17e9d8fd85225e18351637460f3482be2f Mon Sep 17 00:00:00 2001 From: Junxian Huang Date: Fri, 15 Mar 2024 17:35:51 +0800 Subject: [PATCH 0095/1702] RDMA/hns: Support DSCP Add support for DSCP configuration. For DSCP, get dscp-prio mapping via hns3 nic driver api .get_dscp_prio() and fill the SL (in WQE for UD or in QPC for RC) with the priority value. The prio-tc mapping is configured to HW by hns3 nic driver. HW will select a corresponding TC according to SL and the prio-tc mapping. Signed-off-by: Junxian Huang Link: https://lore.kernel.org/r/20240315093551.1650088-1-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_ah.c | 33 +++++--- drivers/infiniband/hw/hns/hns_roce_device.h | 6 ++ drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 85 ++++++++++++++++----- drivers/infiniband/hw/hns/hns_roce_qp.c | 13 ++++ include/uapi/rdma/hns-abi.h | 9 ++- 5 files changed, 116 insertions(+), 30 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c index b4209b6aed8d7..3e02c474f59fe 100644 --- a/drivers/infiniband/hw/hns/hns_roce_ah.c +++ b/drivers/infiniband/hw/hns/hns_roce_ah.c @@ -59,8 +59,10 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct hns_roce_dev *hr_dev = to_hr_dev(ibah->device); struct hns_roce_ib_create_ah_resp resp = {}; struct hns_roce_ah *ah = to_hr_ah(ibah); - int ret = 0; - u32 max_sl; + u8 tclass = get_tclass(grh); + u8 priority = 0; + u8 tc_mode = 0; + int ret; if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 && udata) return -EOPNOTSUPP; @@ -74,16 +76,23 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, ah->av.hop_limit = grh->hop_limit; ah->av.flowlabel = grh->flow_label; ah->av.udp_sport = get_ah_udp_sport(ah_attr); - ah->av.tclass = get_tclass(grh); - - ah->av.sl = rdma_ah_get_sl(ah_attr); - max_sl = min_t(u32, MAX_SERVICE_LEVEL, hr_dev->caps.sl_num - 1); - if (unlikely(ah->av.sl > max_sl)) { - ibdev_err_ratelimited(&hr_dev->ib_dev, - "failed to set sl, sl (%u) shouldn't be larger than %u.\n", - ah->av.sl, max_sl); + ah->av.tclass = tclass; + + ret = hr_dev->hw->get_dscp(hr_dev, tclass, &tc_mode, &priority); + if (ret == -EOPNOTSUPP) + ret = 0; + + if (ret && grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) + return ret; + + if (tc_mode == HNAE3_TC_MAP_MODE_DSCP && + grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) + ah->av.sl = priority; + else + ah->av.sl = rdma_ah_get_sl(ah_attr); + + if (!check_sl_valid(hr_dev, ah->av.sl)) return -EINVAL; - } memcpy(ah->av.dgid, grh->dgid.raw, HNS_ROCE_GID_SIZE); memcpy(ah->av.mac, ah_attr->roce.dmac, ETH_ALEN); @@ -99,6 +108,8 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, } if (udata) { + resp.priority = ah->av.sl; + resp.tc_mode = tc_mode; memcpy(resp.dmac, ah_attr->roce.dmac, ETH_ALEN); ret = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp))); diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index c3cbd0a494bfd..78b4d19ff8486 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -645,6 +645,8 @@ struct hns_roce_qp { struct hns_user_mmap_entry *dwqe_mmap_entry; u32 config; enum hns_roce_cong_type cong_type; + u8 tc_mode; + u8 priority; }; struct hns_roce_ib_iboe { @@ -950,6 +952,8 @@ struct hns_roce_hw { int (*query_sccc)(struct hns_roce_dev *hr_dev, u32 qpn, void *buffer); int (*query_hw_counter)(struct hns_roce_dev *hr_dev, u64 *stats, u32 port, int *hw_counters); + int (*get_dscp)(struct hns_roce_dev *hr_dev, u8 dscp, + u8 *tc_mode, u8 *priority); const struct ib_device_ops *hns_roce_dev_ops; const struct ib_device_ops *hns_roce_dev_srq_ops; }; @@ -1292,4 +1296,6 @@ struct hns_user_mmap_entry * hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, size_t length, enum hns_roce_mmap_type mmap_type); +bool check_sl_valid(struct hns_roce_dev *hr_dev, u8 sl); + #endif /* _HNS_ROCE_DEVICE_H */ diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index ba7ae792d279d..423ab66c5856d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -443,10 +443,6 @@ static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, hr_reg_write(ud_sq_wqe, UD_SEND_WQE_HOPLIMIT, ah->av.hop_limit); hr_reg_write(ud_sq_wqe, UD_SEND_WQE_TCLASS, ah->av.tclass); hr_reg_write(ud_sq_wqe, UD_SEND_WQE_FLOW_LABEL, ah->av.flowlabel); - - if (WARN_ON(ah->av.sl > MAX_SERVICE_LEVEL)) - return -EINVAL; - hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, ah->av.sl); ud_sq_wqe->sgid_index = ah->av.gid_index; @@ -4828,6 +4824,69 @@ static int fill_cong_field(struct ib_qp *ibqp, const struct ib_qp_attr *attr, return 0; } +static int hns_roce_hw_v2_get_dscp(struct hns_roce_dev *hr_dev, u8 dscp, + u8 *tc_mode, u8 *priority) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + + if (!ops->get_dscp_prio) + return -EOPNOTSUPP; + + return ops->get_dscp_prio(handle, dscp, tc_mode, priority); +} + +bool check_sl_valid(struct hns_roce_dev *hr_dev, u8 sl) +{ + u32 max_sl; + + max_sl = min_t(u32, MAX_SERVICE_LEVEL, hr_dev->caps.sl_num - 1); + if (unlikely(sl > max_sl)) { + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to set SL(%u). Shouldn't be larger than %u.\n", + sl, max_sl); + return false; + } + + return true; +} + +static int hns_roce_set_sl(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct ib_device *ibdev = &hr_dev->ib_dev; + int ret; + + ret = hns_roce_hw_v2_get_dscp(hr_dev, get_tclass(&attr->ah_attr.grh), + &hr_qp->tc_mode, &hr_qp->priority); + if (ret && ret != -EOPNOTSUPP && + grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { + ibdev_err_ratelimited(ibdev, + "failed to get dscp, ret = %d.\n", ret); + return ret; + } + + if (hr_qp->tc_mode == HNAE3_TC_MAP_MODE_DSCP && + grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) + hr_qp->sl = hr_qp->priority; + else + hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); + + if (!check_sl_valid(hr_dev, hr_qp->sl)) + return -EINVAL; + + hr_reg_write(context, QPC_SL, hr_qp->sl); + hr_reg_clear(qpc_mask, QPC_SL); + + return 0; +} + static int hns_roce_v2_set_path(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int attr_mask, @@ -4843,25 +4902,18 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, int is_roce_protocol; u16 vlan_id = 0xffff; bool is_udp = false; - u32 max_sl; u8 ib_port; u8 hr_port; int ret; - max_sl = min_t(u32, MAX_SERVICE_LEVEL, hr_dev->caps.sl_num - 1); - if (unlikely(sl > max_sl)) { - ibdev_err_ratelimited(ibdev, - "failed to fill QPC, sl (%u) shouldn't be larger than %u.\n", - sl, max_sl); - return -EINVAL; - } - /* * If free_mr_en of qp is set, it means that this qp comes from * free mr. This qp will perform the loopback operation. * In the loopback scenario, only sl needs to be set. */ if (hr_qp->free_mr_en) { + if (!check_sl_valid(hr_dev, sl)) + return -EINVAL; hr_reg_write(context, QPC_SL, sl); hr_reg_clear(qpc_mask, QPC_SL); hr_qp->sl = sl; @@ -4931,11 +4983,7 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw)); - hr_qp->sl = sl; - hr_reg_write(context, QPC_SL, hr_qp->sl); - hr_reg_clear(qpc_mask, QPC_SL); - - return 0; + return hns_roce_set_sl(ibqp, attr, context, qpc_mask); } static bool check_qp_state(enum ib_qp_state cur_state, @@ -6735,6 +6783,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { .query_srqc = hns_roce_v2_query_srqc, .query_sccc = hns_roce_v2_query_sccc, .query_hw_counter = hns_roce_hw_v2_query_counter, + .get_dscp = hns_roce_hw_v2_get_dscp, .hns_roce_dev_ops = &hns_roce_v2_dev_ops, .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops, }; diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index f35a66325d9a6..697230f964b10 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -1386,6 +1386,7 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_ib_modify_qp_resp resp = {}; struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); enum ib_qp_state cur_state, new_state; int ret = -EINVAL; @@ -1427,6 +1428,18 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state, new_state, udata); + if (ret) + goto out; + + if (udata && udata->outlen) { + resp.tc_mode = hr_qp->tc_mode; + resp.priority = hr_qp->sl; + ret = ib_copy_to_udata(udata, &resp, + min(udata->outlen, sizeof(resp))); + if (ret) + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to copy modify qp resp.\n"); + } out: mutex_unlock(&hr_qp->mutex); diff --git a/include/uapi/rdma/hns-abi.h b/include/uapi/rdma/hns-abi.h index 158670da2b2a5..94e861870e270 100644 --- a/include/uapi/rdma/hns-abi.h +++ b/include/uapi/rdma/hns-abi.h @@ -109,6 +109,12 @@ struct hns_roce_ib_create_qp_resp { __aligned_u64 dwqe_mmap_key; }; +struct hns_roce_ib_modify_qp_resp { + __u8 tc_mode; + __u8 priority; + __u8 reserved[6]; +}; + enum { HNS_ROCE_EXSGE_FLAGS = 1 << 0, HNS_ROCE_RQ_INLINE_FLAGS = 1 << 1, @@ -143,7 +149,8 @@ struct hns_roce_ib_alloc_pd_resp { struct hns_roce_ib_create_ah_resp { __u8 dmac[6]; - __u8 reserved[2]; + __u8 priority; + __u8 tc_mode; }; #endif /* HNS_ABI_USER_H */ From 734554fdfce6731b22f0777ec3f1e4a853354883 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 5 Apr 2024 17:52:07 +0100 Subject: [PATCH 0096/1702] iommu/arm-smmu-v3: Retire disable_bypass parameter The disable_bypass parameter has been mostly meaningless for a long time since the introduction of default domains. Its original intent is now fulfilled by the controls users have over the default domain type, and its remaining effect in the brief window between Stream Table initialisation and default domain creation hardly seems worth the complication. Furthermore, thanks to 2-level Stream Tables, disabling disable_bypass (there's another reason not to like it right there) has never guaranteed that any particular StreamID *will* bypass anyway - any device which might actually care about that wants RMRs - so there's not really much lost by taking away that option (which has already been non-default for nearing 6 years now). As part of this, also remove the weird behaviour where we "successfully" probe and register a non-functional SMMU if the DT "#iommu-cells" property is wrong. I have no memory of what possessed me to think that was a good idea at the time, and by now I suspect it's likely to break things worse than simply failing probe would. Signed-off-by: Robin Murphy Reviewed-by: Mostafa Saleh Link: https://lore.kernel.org/r/ea3ac4cd595a81b5511729601b2f7d4668178438.1712335927.git.robin.murphy@arm.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 46 ++++++--------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 41f93c3ab160d..4eb74f0ad13b4 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -30,11 +30,6 @@ #include "arm-smmu-v3.h" #include "../../dma-iommu.h" -static bool disable_bypass = true; -module_param(disable_bypass, bool, 0444); -MODULE_PARM_DESC(disable_bypass, - "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); - static bool disable_msipolling; module_param(disable_msipolling, bool, 0444); MODULE_PARM_DESC(disable_msipolling, @@ -1567,17 +1562,13 @@ static void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, * This can safely directly manipulate the STE memory without a sync sequence * because the STE table has not been installed in the SMMU yet. */ -static void arm_smmu_init_initial_stes(struct arm_smmu_device *smmu, - struct arm_smmu_ste *strtab, +static void arm_smmu_init_initial_stes(struct arm_smmu_ste *strtab, unsigned int nent) { unsigned int i; for (i = 0; i < nent; ++i) { - if (disable_bypass) - arm_smmu_make_abort_ste(strtab); - else - arm_smmu_make_bypass_ste(smmu, strtab); + arm_smmu_make_abort_ste(strtab); strtab++; } } @@ -1605,7 +1596,7 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) return -ENOMEM; } - arm_smmu_init_initial_stes(smmu, desc->l2ptr, 1 << STRTAB_SPLIT); + arm_smmu_init_initial_stes(desc->l2ptr, 1 << STRTAB_SPLIT); arm_smmu_write_strtab_l1_desc(strtab, desc); return 0; } @@ -2915,10 +2906,10 @@ static void arm_smmu_release_device(struct device *dev) iopf_queue_remove_device(master->smmu->evtq.iopf, dev); /* Put the STE back to what arm_smmu_init_strtab() sets */ - if (disable_bypass && !dev->iommu->require_direct) - arm_smmu_attach_dev_blocked(&arm_smmu_blocked_domain, dev); - else + if (dev->iommu->require_direct) arm_smmu_attach_dev_identity(&arm_smmu_identity_domain, dev); + else + arm_smmu_attach_dev_blocked(&arm_smmu_blocked_domain, dev); arm_smmu_disable_pasid(master); arm_smmu_remove_master(master); @@ -3273,7 +3264,7 @@ static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits); cfg->strtab_base_cfg = reg; - arm_smmu_init_initial_stes(smmu, strtab, cfg->num_l1_ents); + arm_smmu_init_initial_stes(strtab, cfg->num_l1_ents); return 0; } @@ -3503,7 +3494,7 @@ static int arm_smmu_device_disable(struct arm_smmu_device *smmu) return ret; } -static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass) +static int arm_smmu_device_reset(struct arm_smmu_device *smmu) { int ret; u32 reg, enables; @@ -3513,7 +3504,6 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass) reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); if (reg & CR0_SMMUEN) { dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n"); - WARN_ON(is_kdump_kernel() && !disable_bypass); arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0); } @@ -3620,14 +3610,8 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass) if (is_kdump_kernel()) enables &= ~(CR0_EVTQEN | CR0_PRIQEN); - /* Enable the SMMU interface, or ensure bypass */ - if (!bypass || disable_bypass) { - enables |= CR0_SMMUEN; - } else { - ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT); - if (ret) - return ret; - } + /* Enable the SMMU interface */ + enables |= CR0_SMMUEN; ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, ARM_SMMU_CR0ACK); if (ret) { @@ -4019,7 +4003,6 @@ static int arm_smmu_device_probe(struct platform_device *pdev) resource_size_t ioaddr; struct arm_smmu_device *smmu; struct device *dev = &pdev->dev; - bool bypass; smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); if (!smmu) @@ -4030,12 +4013,9 @@ static int arm_smmu_device_probe(struct platform_device *pdev) ret = arm_smmu_device_dt_probe(pdev, smmu); } else { ret = arm_smmu_device_acpi_probe(pdev, smmu); - if (ret == -ENODEV) - return ret; } - - /* Set bypass mode according to firmware probing result */ - bypass = !!ret; + if (ret) + return ret; /* Base address */ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -4099,7 +4079,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) arm_smmu_rmr_install_bypass_ste(smmu); /* Reset the device */ - ret = arm_smmu_device_reset(smmu, bypass); + ret = arm_smmu_device_reset(smmu); if (ret) return ret; From fdc69d39e77f88264ee6e8174ff9aaf0953aecd9 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 27 Mar 2024 15:07:49 -0300 Subject: [PATCH 0097/1702] iommu/arm-smmu-v3: Do not allow a SVA domain to be set on the wrong PASID The SVA code is wired to assume that the SVA is programmed onto the mm->pasid. The current core code always does this, so it is fine. Add a check for clarity. Tested-by: Nicolin Chen Tested-by: Shameer Kolothum Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/3-v6-228e7adf25eb+4155-smmuv3_newapi_p2_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 2cd433a9c8a0f..41b44baef15e8 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -569,6 +569,9 @@ static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain, int ret = 0; struct mm_struct *mm = domain->mm; + if (mm_get_enqcmd_pasid(mm) != id) + return -EINVAL; + mutex_lock(&sva_lock); ret = __arm_smmu_sva_bind(dev, id, mm); mutex_unlock(&sva_lock); From 86e5ca098dd9f8c5b80a6205395aea0535018837 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 27 Mar 2024 15:07:50 -0300 Subject: [PATCH 0098/1702] iommu/arm-smmu-v3: Do not ATC invalidate the entire domain At this point we know which master we are going to change the PCI config on, this is the only device we need to invalidate. Switch arm_smmu_atc_inv_domain() for arm_smmu_atc_inv_master(). Tested-by: Nicolin Chen Tested-by: Shameer Kolothum Reviewed-by: Michael Shavit Reviewed-by: Nicolin Chen Reviewed-by: Moritz Fischer Reviewed-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/4-v6-228e7adf25eb+4155-smmuv3_newapi_p2_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 4eb74f0ad13b4..3aad4cb5c2631 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2410,7 +2410,10 @@ static void arm_smmu_enable_ats(struct arm_smmu_master *master, pdev = to_pci_dev(master->dev); atomic_inc(&smmu_domain->nr_ats_masters); - arm_smmu_atc_inv_domain(smmu_domain, IOMMU_NO_PASID, 0, 0); + /* + * ATC invalidation of PASID 0 causes the entire ATC to be flushed. + */ + arm_smmu_atc_inv_master(master); if (pci_enable_ats(pdev, stu)) dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu); } From e8e4398d53f98be7ac48e0bda9ea6e26df24136d Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 27 Mar 2024 15:07:51 -0300 Subject: [PATCH 0099/1702] iommu/arm-smmu-v3: Add a type for the CD entry Instead of passing a naked __le16 * around to represent a CD table entry wrap it in a "struct arm_smmu_cd" with an array of the correct size. This makes it much clearer which functions will comprise the "CD API". Tested-by: Nicolin Chen Tested-by: Shameer Kolothum Reviewed-by: Michael Shavit Reviewed-by: Moritz Fischer Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/5-v6-228e7adf25eb+4155-smmuv3_newapi_p2_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 20 +++++++++++--------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 7 ++++++- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 3aad4cb5c2631..79c18e95dd293 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1209,7 +1209,8 @@ static void arm_smmu_write_cd_l1_desc(__le64 *dst, WRITE_ONCE(*dst, cpu_to_le64(val)); } -static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, u32 ssid) +static struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, + u32 ssid) { __le64 *l1ptr; unsigned int idx; @@ -1218,7 +1219,8 @@ static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, u32 ssid) struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_LINEAR) - return cd_table->cdtab + ssid * CTXDESC_CD_DWORDS; + return (struct arm_smmu_cd *)(cd_table->cdtab + + ssid * CTXDESC_CD_DWORDS); idx = ssid >> CTXDESC_SPLIT; l1_desc = &cd_table->l1_desc[idx]; @@ -1232,7 +1234,7 @@ static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, u32 ssid) arm_smmu_sync_cd(master, ssid, false); } idx = ssid & (CTXDESC_L2_ENTRIES - 1); - return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS; + return &l1_desc->l2ptr[idx]; } int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, @@ -1251,7 +1253,7 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, */ u64 val; bool cd_live; - __le64 *cdptr; + struct arm_smmu_cd *cdptr; struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; struct arm_smmu_device *smmu = master->smmu; @@ -1262,7 +1264,7 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, if (!cdptr) return -ENOMEM; - val = le64_to_cpu(cdptr[0]); + val = le64_to_cpu(cdptr->data[0]); cd_live = !!(val & CTXDESC_CD_0_V); if (!cd) { /* (5) */ @@ -1279,9 +1281,9 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, * this substream's traffic */ } else { /* (1) and (2) */ - cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK); - cdptr[2] = 0; - cdptr[3] = cpu_to_le64(cd->mair); + cdptr->data[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK); + cdptr->data[2] = 0; + cdptr->data[3] = cpu_to_le64(cd->mair); /* * STE may be live, and the SMMU might read dwords of this CD in any @@ -1313,7 +1315,7 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, * field within an aligned 64-bit span of a structure can be altered * without first making the structure invalid. */ - WRITE_ONCE(cdptr[0], cpu_to_le64(val)); + WRITE_ONCE(cdptr->data[0], cpu_to_le64(val)); arm_smmu_sync_cd(master, ssid, true); return 0; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 2a19bb63e5c6d..4b767e0eeeb68 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -283,6 +283,11 @@ struct arm_smmu_ste { #define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12) #define CTXDESC_CD_DWORDS 8 + +struct arm_smmu_cd { + __le64 data[CTXDESC_CD_DWORDS]; +}; + #define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0) #define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6) #define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8) @@ -592,7 +597,7 @@ struct arm_smmu_ctx_desc { }; struct arm_smmu_l1_ctx_desc { - __le64 *l2ptr; + struct arm_smmu_cd *l2ptr; dma_addr_t l2ptr_dma; }; From 0f9b12142be1af8555cfe53c6fbecb8e60a40dac Mon Sep 17 00:00:00 2001 From: Wenjie Qi Date: Sun, 7 Apr 2024 15:21:23 +0800 Subject: [PATCH 0100/1702] f2fs: fix zoned block device information initialization If the max open zones of zoned devices are less than the active logs of F2FS, the device may error due to insufficient zone resources when multiple active logs are being written at the same time. Signed-off-by: Wenjie Qi Signed-off-by: Chao Yu Reviewed-by: Daeho Jeong Signed-off-by: Jaegeuk Kim --- fs/f2fs/f2fs.h | 1 + fs/f2fs/super.c | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index adfce581026b8..e9ef971f4dba6 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -1557,6 +1557,7 @@ struct f2fs_sb_info { #ifdef CONFIG_BLK_DEV_ZONED unsigned int blocks_per_blkz; /* F2FS blocks per zone */ + unsigned int max_open_zones; /* max open zone resources of the zoned device */ #endif /* for node-related operations */ diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index df9765b41dac1..8ac4734c2df60 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -2324,6 +2324,17 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data) if (err) goto restore_opts; +#ifdef CONFIG_BLK_DEV_ZONED + if (f2fs_sb_has_blkzoned(sbi) && + sbi->max_open_zones < F2FS_OPTION(sbi).active_logs) { + f2fs_err(sbi, + "zoned: max open zones %u is too small, need at least %u open zones", + sbi->max_open_zones, F2FS_OPTION(sbi).active_logs); + err = -EINVAL; + goto restore_opts; + } +#endif + /* flush outstanding errors before changing fs state */ flush_work(&sbi->s_error_work); @@ -3866,11 +3877,24 @@ static int init_blkz_info(struct f2fs_sb_info *sbi, int devi) sector_t nr_sectors = bdev_nr_sectors(bdev); struct f2fs_report_zones_args rep_zone_arg; u64 zone_sectors; + unsigned int max_open_zones; int ret; if (!f2fs_sb_has_blkzoned(sbi)) return 0; + if (bdev_is_zoned(FDEV(devi).bdev)) { + max_open_zones = bdev_max_open_zones(bdev); + if (max_open_zones && (max_open_zones < sbi->max_open_zones)) + sbi->max_open_zones = max_open_zones; + if (sbi->max_open_zones < F2FS_OPTION(sbi).active_logs) { + f2fs_err(sbi, + "zoned: max open zones %u is too small, need at least %u open zones", + sbi->max_open_zones, F2FS_OPTION(sbi).active_logs); + return -EINVAL; + } + } + zone_sectors = bdev_zone_sectors(bdev); if (sbi->blocks_per_blkz && sbi->blocks_per_blkz != SECTOR_TO_BLOCK(zone_sectors)) @@ -4184,6 +4208,9 @@ static int f2fs_scan_devices(struct f2fs_sb_info *sbi) logical_blksize = bdev_logical_block_size(sbi->sb->s_bdev); sbi->aligned_blksize = true; +#ifdef CONFIG_BLK_DEV_ZONED + sbi->max_open_zones = UINT_MAX; +#endif for (i = 0; i < max_devices; i++) { if (i == 0) From 575bc7b477e3f6c505f49c3d99d7be965594d640 Mon Sep 17 00:00:00 2001 From: Sascha Hauer Date: Fri, 5 Apr 2024 09:38:36 +0200 Subject: [PATCH 0101/1702] dt-bindings: clock: rockchip: add USB480M_PHY mux The USB480M clock can source from a MUX that selects the clock to come from either of the USB-phy internal 480MHz PLLs. These clocks are provided by the USB phy driver. This adds the define for it. Signed-off-by: Sascha Hauer Acked-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240405-clk-rk3568-usb480m-phy-mux-v1-1-6c89de20a6ff@pengutronix.de Signed-off-by: Heiko Stuebner --- include/dt-bindings/clock/rk3568-cru.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/dt-bindings/clock/rk3568-cru.h b/include/dt-bindings/clock/rk3568-cru.h index d29890865150d..5263085c5b238 100644 --- a/include/dt-bindings/clock/rk3568-cru.h +++ b/include/dt-bindings/clock/rk3568-cru.h @@ -78,6 +78,7 @@ #define CPLL_333M 9 #define ARMCLK 10 #define USB480M 11 +#define USB480M_PHY 12 #define ACLK_CORE_NIU2BUS 18 #define CLK_CORE_PVTM 19 #define CLK_CORE_PVTM_CORE 20 From ca151fd56b5736a7adbdba5675b9d87d70f20b23 Mon Sep 17 00:00:00 2001 From: Shreeya Patel Date: Thu, 28 Mar 2024 04:20:52 +0530 Subject: [PATCH 0102/1702] dt-bindings: reset: Define reset id used for HDMI Receiver Add reset id used for HDMI Receiver in RK3588 SoCs Acked-by: Rob Herring Signed-off-by: Shreeya Patel Link: https://lore.kernel.org/r/20240327225057.672304-2-shreeya.patel@collabora.com Signed-off-by: Heiko Stuebner --- include/dt-bindings/reset/rockchip,rk3588-cru.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/dt-bindings/reset/rockchip,rk3588-cru.h b/include/dt-bindings/reset/rockchip,rk3588-cru.h index d4264db2a07f1..e2fe4bd5f7f01 100644 --- a/include/dt-bindings/reset/rockchip,rk3588-cru.h +++ b/include/dt-bindings/reset/rockchip,rk3588-cru.h @@ -751,4 +751,6 @@ #define SRST_P_TRNG_CHK 658 #define SRST_TRNG_S 659 +#define SRST_A_HDMIRX_BIU 660 + #endif From 007bd99669eae90f23817023dc78dbb38e76437d Mon Sep 17 00:00:00 2001 From: David Jander Date: Fri, 5 Apr 2024 09:38:37 +0200 Subject: [PATCH 0103/1702] clk: rockchip: rk3568: Add missing USB480M_PHY mux The USB480M clock can source from a MUX that selects the clock to come from either of the USB-phy internal 480MHz PLLs. These clocks are provided by the USB phy driver. Signed-off-by: David Jander Link: https://lore.kernel.org/r/20240404-clk-rockchip-rk3568-add-usb480m-phy-mux-v1-1-e8542afd58b9@pengutronix.de Signed-off-by: Sascha Hauer Link: https://lore.kernel.org/r/20240405-clk-rk3568-usb480m-phy-mux-v1-2-6c89de20a6ff@pengutronix.de Signed-off-by: Heiko Stuebner --- drivers/clk/rockchip/clk-rk3568.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/clk/rockchip/clk-rk3568.c b/drivers/clk/rockchip/clk-rk3568.c index 8cb21d10beca2..2d44bcaef046b 100644 --- a/drivers/clk/rockchip/clk-rk3568.c +++ b/drivers/clk/rockchip/clk-rk3568.c @@ -215,6 +215,7 @@ static const struct rockchip_cpuclk_reg_data rk3568_cpuclk_data = { PNAME(mux_pll_p) = { "xin24m" }; PNAME(mux_usb480m_p) = { "xin24m", "usb480m_phy", "clk_rtc_32k" }; +PNAME(mux_usb480m_phy_p) = { "clk_usbphy0_480m", "clk_usbphy1_480m"}; PNAME(mux_armclk_p) = { "apll", "gpll" }; PNAME(clk_i2s0_8ch_tx_p) = { "clk_i2s0_8ch_tx_src", "clk_i2s0_8ch_tx_frac", "i2s0_mclkin", "xin_osc0_half" }; PNAME(clk_i2s0_8ch_rx_p) = { "clk_i2s0_8ch_rx_src", "clk_i2s0_8ch_rx_frac", "i2s0_mclkin", "xin_osc0_half" }; @@ -485,6 +486,9 @@ static struct rockchip_clk_branch rk3568_clk_branches[] __initdata = { MUX(USB480M, "usb480m", mux_usb480m_p, CLK_SET_RATE_PARENT, RK3568_MODE_CON0, 14, 2, MFLAGS), + MUX(USB480M_PHY, "usb480m_phy", mux_usb480m_phy_p, CLK_SET_RATE_PARENT, + RK3568_MISC_CON2, 15, 1, MFLAGS), + /* PD_CORE */ COMPOSITE(0, "sclk_core_src", apll_gpll_npll_p, CLK_IGNORE_UNUSED, RK3568_CLKSEL_CON(2), 8, 2, MFLAGS, 0, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, From 7af67019cd78d028ef377df689ac103d51905518 Mon Sep 17 00:00:00 2001 From: Shreeya Patel Date: Thu, 28 Mar 2024 04:20:53 +0530 Subject: [PATCH 0104/1702] clk: rockchip: rk3588: Add reset line for HDMI Receiver Export hdmirx_biu reset line required by the Synopsys DesignWare HDMIRX Controller. Signed-off-by: Shreeya Patel Link: https://lore.kernel.org/r/20240327225057.672304-3-shreeya.patel@collabora.com Signed-off-by: Heiko Stuebner --- drivers/clk/rockchip/rst-rk3588.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/clk/rockchip/rst-rk3588.c b/drivers/clk/rockchip/rst-rk3588.c index e855bb8d54136..c4ebc01f1c9c4 100644 --- a/drivers/clk/rockchip/rst-rk3588.c +++ b/drivers/clk/rockchip/rst-rk3588.c @@ -577,6 +577,7 @@ static const int rk3588_register_offset[] = { /* SOFTRST_CON59 */ RK3588_CRU_RESET_OFFSET(SRST_A_HDCP1_BIU, 59, 6), + RK3588_CRU_RESET_OFFSET(SRST_A_HDMIRX_BIU, 59, 7), RK3588_CRU_RESET_OFFSET(SRST_A_VO1_BIU, 59, 8), RK3588_CRU_RESET_OFFSET(SRST_H_VOP1_BIU, 59, 9), RK3588_CRU_RESET_OFFSET(SRST_H_VOP1_S_BIU, 59, 10), From bb5aa08572b5313157c093a09d53ebf2efda3dc1 Mon Sep 17 00:00:00 2001 From: Neil Armstrong Date: Wed, 3 Apr 2024 09:46:33 +0200 Subject: [PATCH 0105/1702] clk: meson: add vclk driver The VCLK and VCLK_DIV clocks have supplementary bits. The VCLK gate has a "SOFT RESET" bit to toggle after the whole VCLK sub-tree rate has been set, this is implemented in the gate enable callback. The VCLK_DIV clocks as enable and reset bits used to disable and reset the divider, associated with CLK_SET_RATE_GATE it ensures the rate is set while the divider is disabled and in reset mode. The VCLK_DIV enable bit isn't implemented as a gate since it's part of the divider logic and vendor does this exact sequence to ensure the divider is correctly set. Signed-off-by: Neil Armstrong Link: https://lore.kernel.org/r/20240403-amlogic-v6-4-upstream-dsi-ccf-vim3-v12-2-99ecdfdc87fc@linaro.org Signed-off-by: Jerome Brunet --- drivers/clk/meson/Kconfig | 4 ++ drivers/clk/meson/Makefile | 1 + drivers/clk/meson/vclk.c | 141 +++++++++++++++++++++++++++++++++++++ drivers/clk/meson/vclk.h | 51 ++++++++++++++ 4 files changed, 197 insertions(+) create mode 100644 drivers/clk/meson/vclk.c create mode 100644 drivers/clk/meson/vclk.h diff --git a/drivers/clk/meson/Kconfig b/drivers/clk/meson/Kconfig index 29ffd14d267b6..8a9823789fa35 100644 --- a/drivers/clk/meson/Kconfig +++ b/drivers/clk/meson/Kconfig @@ -30,6 +30,10 @@ config COMMON_CLK_MESON_VID_PLL_DIV tristate select COMMON_CLK_MESON_REGMAP +config COMMON_CLK_MESON_VCLK + tristate + select COMMON_CLK_MESON_REGMAP + config COMMON_CLK_MESON_CLKC_UTILS tristate diff --git a/drivers/clk/meson/Makefile b/drivers/clk/meson/Makefile index 9ee4b954c8961..9ba43fe7a07a8 100644 --- a/drivers/clk/meson/Makefile +++ b/drivers/clk/meson/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_COMMON_CLK_MESON_PLL) += clk-pll.o obj-$(CONFIG_COMMON_CLK_MESON_REGMAP) += clk-regmap.o obj-$(CONFIG_COMMON_CLK_MESON_SCLK_DIV) += sclk-div.o obj-$(CONFIG_COMMON_CLK_MESON_VID_PLL_DIV) += vid-pll-div.o +obj-$(CONFIG_COMMON_CLK_MESON_VCLK) += vclk.o # Amlogic Clock controllers diff --git a/drivers/clk/meson/vclk.c b/drivers/clk/meson/vclk.c new file mode 100644 index 0000000000000..45dc216941eaf --- /dev/null +++ b/drivers/clk/meson/vclk.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2024 Neil Armstrong + */ + +#include +#include "vclk.h" + +/* The VCLK gate has a supplementary reset bit to pulse after ungating */ + +static inline struct meson_vclk_gate_data * +clk_get_meson_vclk_gate_data(struct clk_regmap *clk) +{ + return (struct meson_vclk_gate_data *)clk->data; +} + +static int meson_vclk_gate_enable(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_gate_data *vclk = clk_get_meson_vclk_gate_data(clk); + + meson_parm_write(clk->map, &vclk->enable, 1); + + /* Do a reset pulse */ + meson_parm_write(clk->map, &vclk->reset, 1); + meson_parm_write(clk->map, &vclk->reset, 0); + + return 0; +} + +static void meson_vclk_gate_disable(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_gate_data *vclk = clk_get_meson_vclk_gate_data(clk); + + meson_parm_write(clk->map, &vclk->enable, 0); +} + +static int meson_vclk_gate_is_enabled(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_gate_data *vclk = clk_get_meson_vclk_gate_data(clk); + + return meson_parm_read(clk->map, &vclk->enable); +} + +const struct clk_ops meson_vclk_gate_ops = { + .enable = meson_vclk_gate_enable, + .disable = meson_vclk_gate_disable, + .is_enabled = meson_vclk_gate_is_enabled, +}; +EXPORT_SYMBOL_GPL(meson_vclk_gate_ops); + +/* The VCLK Divider has supplementary reset & enable bits */ + +static inline struct meson_vclk_div_data * +clk_get_meson_vclk_div_data(struct clk_regmap *clk) +{ + return (struct meson_vclk_div_data *)clk->data; +} + +static unsigned long meson_vclk_div_recalc_rate(struct clk_hw *hw, + unsigned long prate) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_div_data *vclk = clk_get_meson_vclk_div_data(clk); + + return divider_recalc_rate(hw, prate, meson_parm_read(clk->map, &vclk->div), + vclk->table, vclk->flags, vclk->div.width); +} + +static int meson_vclk_div_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_div_data *vclk = clk_get_meson_vclk_div_data(clk); + + return divider_determine_rate(hw, req, vclk->table, vclk->div.width, + vclk->flags); +} + +static int meson_vclk_div_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_div_data *vclk = clk_get_meson_vclk_div_data(clk); + int ret; + + ret = divider_get_val(rate, parent_rate, vclk->table, vclk->div.width, + vclk->flags); + if (ret < 0) + return ret; + + meson_parm_write(clk->map, &vclk->div, ret); + + return 0; +}; + +static int meson_vclk_div_enable(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_div_data *vclk = clk_get_meson_vclk_div_data(clk); + + /* Unreset the divider when ungating */ + meson_parm_write(clk->map, &vclk->reset, 0); + meson_parm_write(clk->map, &vclk->enable, 1); + + return 0; +} + +static void meson_vclk_div_disable(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_div_data *vclk = clk_get_meson_vclk_div_data(clk); + + /* Reset the divider when gating */ + meson_parm_write(clk->map, &vclk->enable, 0); + meson_parm_write(clk->map, &vclk->reset, 1); +} + +static int meson_vclk_div_is_enabled(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_vclk_div_data *vclk = clk_get_meson_vclk_div_data(clk); + + return meson_parm_read(clk->map, &vclk->enable); +} + +const struct clk_ops meson_vclk_div_ops = { + .recalc_rate = meson_vclk_div_recalc_rate, + .determine_rate = meson_vclk_div_determine_rate, + .set_rate = meson_vclk_div_set_rate, + .enable = meson_vclk_div_enable, + .disable = meson_vclk_div_disable, + .is_enabled = meson_vclk_div_is_enabled, +}; +EXPORT_SYMBOL_GPL(meson_vclk_div_ops); + +MODULE_DESCRIPTION("Amlogic vclk clock driver"); +MODULE_AUTHOR("Neil Armstrong "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/meson/vclk.h b/drivers/clk/meson/vclk.h new file mode 100644 index 0000000000000..20b0b181db09d --- /dev/null +++ b/drivers/clk/meson/vclk.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2024 Neil Armstrong + */ + +#ifndef __VCLK_H +#define __VCLK_H + +#include "clk-regmap.h" +#include "parm.h" + +/** + * struct meson_vclk_gate_data - vclk_gate regmap backed specific data + * + * @enable: vclk enable field + * @reset: vclk reset field + * @flags: hardware-specific flags + * + * Flags: + * Same as clk_gate except CLK_GATE_HIWORD_MASK which is ignored + */ +struct meson_vclk_gate_data { + struct parm enable; + struct parm reset; + u8 flags; +}; + +extern const struct clk_ops meson_vclk_gate_ops; + +/** + * struct meson_vclk_div_data - vclk_div regmap back specific data + * + * @div: divider field + * @enable: vclk divider enable field + * @reset: vclk divider reset field + * @table: array of value/divider pairs, last entry should have div = 0 + * + * Flags: + * Same as clk_divider except CLK_DIVIDER_HIWORD_MASK which is ignored + */ +struct meson_vclk_div_data { + struct parm div; + struct parm enable; + struct parm reset; + const struct clk_div_table *table; + u8 flags; +}; + +extern const struct clk_ops meson_vclk_div_ops; + +#endif /* __VCLK_H */ From b70cb1a21a54236be24c03787eecc40c451158c3 Mon Sep 17 00:00:00 2001 From: Neil Armstrong Date: Wed, 3 Apr 2024 09:46:34 +0200 Subject: [PATCH 0106/1702] clk: meson: g12a: make VCLK2 and ENCL clock path configurable by CCF In order to setup the DSI clock, let's make the unused VCLK2 clock path configuration via CCF. The nocache option is removed from following clocks: - vclk2_sel - vclk2_input - vclk2_div - vclk2 - vclk_div1 - vclk2_div2_en - vclk2_div4_en - vclk2_div6_en - vclk2_div12_en - vclk2_div2 - vclk2_div4 - vclk2_div6 - vclk2_div12 - cts_encl_sel vclk2 and vclk2_div uses the newly introduced vclk regmap driver to handle the enable and reset bits. In order to set a rate on cts_encl via the vclk2 clock path, the NO_REPARENT flag is set on cts_encl_sel & vclk2_sel in order to keep CCF from selection a parent. The parents of cts_encl_sel & vclk2_sel are expected to be defined in DT or manually set by the display driver at some point. The following clock scheme is to be used for DSI: xtal \_ gp0_pll_dco \_ gp0_pll |- vclk2_sel | \_ vclk2_input | \_ vclk2_div | \_ vclk2 | \_ vclk2_div1 | \_ cts_encl_sel | \_ cts_encl -> to VPU LCD Encoder |- mipi_dsi_pxclk_sel \_ mipi_dsi_pxclk_div \_ mipi_dsi_pxclk -> to DSI controller The mipi_dsi_pxclk_div is set as bypass with a single /1 entry in div_table in order to use the same GP0 for mipi_dsi_pxclk and vclk2_input. The SET_RATE_PARENT is only set on the mipi_dsi_pxclk_sel clock so the DSI bitclock is the reference base clock to calculate the vclk2_div value when pixel clock is set on the cts_encl endpoint. Signed-off-by: Neil Armstrong Link: https://lore.kernel.org/r/20240403-amlogic-v6-4-upstream-dsi-ccf-vim3-v12-3-99ecdfdc87fc@linaro.org Signed-off-by: Jerome Brunet --- drivers/clk/meson/Kconfig | 1 + drivers/clk/meson/g12a.c | 76 ++++++++++++++++++++++++++++----------- 2 files changed, 57 insertions(+), 20 deletions(-) diff --git a/drivers/clk/meson/Kconfig b/drivers/clk/meson/Kconfig index 8a9823789fa35..59a40a49f8e10 100644 --- a/drivers/clk/meson/Kconfig +++ b/drivers/clk/meson/Kconfig @@ -144,6 +144,7 @@ config COMMON_CLK_G12A select COMMON_CLK_MESON_EE_CLKC select COMMON_CLK_MESON_CPU_DYNDIV select COMMON_CLK_MESON_VID_PLL_DIV + select COMMON_CLK_MESON_VCLK select MFD_SYSCON help Support for the clock controller on Amlogic S905D2, S905X2 and S905Y2 diff --git a/drivers/clk/meson/g12a.c b/drivers/clk/meson/g12a.c index 90f4c6103014c..df7e17c850d88 100644 --- a/drivers/clk/meson/g12a.c +++ b/drivers/clk/meson/g12a.c @@ -22,6 +22,7 @@ #include "clk-regmap.h" #include "clk-cpu-dyndiv.h" #include "vid-pll-div.h" +#include "vclk.h" #include "meson-eeclk.h" #include "g12a.h" @@ -3165,7 +3166,7 @@ static struct clk_regmap g12a_vclk2_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = g12a_vclk_parent_hws, .num_parents = ARRAY_SIZE(g12a_vclk_parent_hws), - .flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE, + .flags = CLK_SET_RATE_NO_REPARENT, }, }; @@ -3193,7 +3194,6 @@ static struct clk_regmap g12a_vclk2_input = { .ops = &clk_regmap_gate_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2_sel.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, }, }; @@ -3215,19 +3215,32 @@ static struct clk_regmap g12a_vclk_div = { }; static struct clk_regmap g12a_vclk2_div = { - .data = &(struct clk_regmap_div_data){ - .offset = HHI_VIID_CLK_DIV, - .shift = 0, - .width = 8, + .data = &(struct meson_vclk_div_data){ + .div = { + .reg_off = HHI_VIID_CLK_DIV, + .shift = 0, + .width = 8, + }, + .enable = { + .reg_off = HHI_VIID_CLK_DIV, + .shift = 16, + .width = 1, + }, + .reset = { + .reg_off = HHI_VIID_CLK_DIV, + .shift = 17, + .width = 1, + }, + .flags = CLK_DIVIDER_ROUND_CLOSEST, }, .hw.init = &(struct clk_init_data){ .name = "vclk2_div", - .ops = &clk_regmap_divider_ops, + .ops = &meson_vclk_div_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2_input.hw }, .num_parents = 1, - .flags = CLK_GET_RATE_NOCACHE, + .flags = CLK_SET_RATE_GATE, }, }; @@ -3246,16 +3259,24 @@ static struct clk_regmap g12a_vclk = { }; static struct clk_regmap g12a_vclk2 = { - .data = &(struct clk_regmap_gate_data){ - .offset = HHI_VIID_CLK_CNTL, - .bit_idx = 19, + .data = &(struct meson_vclk_gate_data){ + .enable = { + .reg_off = HHI_VIID_CLK_CNTL, + .shift = 19, + .width = 1, + }, + .reset = { + .reg_off = HHI_VIID_CLK_CNTL, + .shift = 15, + .width = 1, + }, }, .hw.init = &(struct clk_init_data) { .name = "vclk2", - .ops = &clk_regmap_gate_ops, + .ops = &meson_vclk_gate_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2_div.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3339,7 +3360,7 @@ static struct clk_regmap g12a_vclk2_div1 = { .ops = &clk_regmap_gate_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3353,7 +3374,7 @@ static struct clk_regmap g12a_vclk2_div2_en = { .ops = &clk_regmap_gate_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3367,7 +3388,7 @@ static struct clk_regmap g12a_vclk2_div4_en = { .ops = &clk_regmap_gate_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3381,7 +3402,7 @@ static struct clk_regmap g12a_vclk2_div6_en = { .ops = &clk_regmap_gate_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3395,7 +3416,7 @@ static struct clk_regmap g12a_vclk2_div12_en = { .ops = &clk_regmap_gate_ops, .parent_hws = (const struct clk_hw *[]) { &g12a_vclk2.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3461,6 +3482,7 @@ static struct clk_fixed_factor g12a_vclk2_div2 = { &g12a_vclk2_div2_en.hw }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3474,6 +3496,7 @@ static struct clk_fixed_factor g12a_vclk2_div4 = { &g12a_vclk2_div4_en.hw }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3487,6 +3510,7 @@ static struct clk_fixed_factor g12a_vclk2_div6 = { &g12a_vclk2_div6_en.hw }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3500,6 +3524,7 @@ static struct clk_fixed_factor g12a_vclk2_div12 = { &g12a_vclk2_div12_en.hw }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, }, }; @@ -3561,7 +3586,7 @@ static struct clk_regmap g12a_cts_encl_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = g12a_cts_parent_hws, .num_parents = ARRAY_SIZE(g12a_cts_parent_hws), - .flags = CLK_SET_RATE_NO_REPARENT | CLK_GET_RATE_NOCACHE, + .flags = CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, }, }; @@ -3717,15 +3742,26 @@ static struct clk_regmap g12a_mipi_dsi_pxclk_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = g12a_mipi_dsi_pxclk_parent_hws, .num_parents = ARRAY_SIZE(g12a_mipi_dsi_pxclk_parent_hws), - .flags = CLK_SET_RATE_NO_REPARENT, + .flags = CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT, }, }; +/* + * FIXME: Force as bypass by forcing a single /1 table entry, and doensn't on boot value + * when setting a clock whith this node in the clock path, but doesn't garantee the divider + * is at /1 at boot until a rate is set. + */ +static const struct clk_div_table g12a_mipi_dsi_pxclk_div_table[] = { + { .val = 0, .div = 1 }, + { /* sentinel */ }, +}; + static struct clk_regmap g12a_mipi_dsi_pxclk_div = { .data = &(struct clk_regmap_div_data){ .offset = HHI_MIPIDSI_PHY_CLK_CNTL, .shift = 0, .width = 7, + .table = g12a_mipi_dsi_pxclk_div_table, }, .hw.init = &(struct clk_init_data){ .name = "mipi_dsi_pxclk_div", From e0892cb47351b85da8ed368c6e2cdd77614967f3 Mon Sep 17 00:00:00 2001 From: Neil Armstrong Date: Mon, 8 Apr 2024 11:17:58 +0200 Subject: [PATCH 0107/1702] clk: meson: fix module license to GPL only Fix the checkpatch warning: WARNING: Prefer "GPL" over "GPL v2" - see commit bf7fbeeae6db ("module: Cure the MODULE_LICENSE "GPL" vs. "GPL v2" bogosity") Signed-off-by: Neil Armstrong Reviewed-by: Martin Blumenstingl Link: https://lore.kernel.org/r/20240408-amlogic-v6-9-upstream-fix-clk-module-license-v1-1-366ddc0f3db9@linaro.org Signed-off-by: Jerome Brunet --- drivers/clk/meson/axg-aoclk.c | 2 +- drivers/clk/meson/axg-audio.c | 2 +- drivers/clk/meson/axg.c | 2 +- drivers/clk/meson/clk-cpu-dyndiv.c | 2 +- drivers/clk/meson/clk-dualdiv.c | 2 +- drivers/clk/meson/clk-mpll.c | 2 +- drivers/clk/meson/clk-phase.c | 2 +- drivers/clk/meson/clk-pll.c | 2 +- drivers/clk/meson/clk-regmap.c | 2 +- drivers/clk/meson/g12a-aoclk.c | 2 +- drivers/clk/meson/g12a.c | 2 +- drivers/clk/meson/gxbb-aoclk.c | 2 +- drivers/clk/meson/gxbb.c | 2 +- drivers/clk/meson/meson-aoclk.c | 2 +- drivers/clk/meson/meson-eeclk.c | 2 +- drivers/clk/meson/sclk-div.c | 2 +- drivers/clk/meson/vclk.c | 2 +- drivers/clk/meson/vid-pll-div.c | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/drivers/clk/meson/axg-aoclk.c b/drivers/clk/meson/axg-aoclk.c index d80ab4728f7a6..e4d0f46f47f53 100644 --- a/drivers/clk/meson/axg-aoclk.c +++ b/drivers/clk/meson/axg-aoclk.c @@ -340,4 +340,4 @@ static struct platform_driver axg_aoclkc_driver = { }; module_platform_driver(axg_aoclkc_driver); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/axg-audio.c b/drivers/clk/meson/axg-audio.c index ac34829609034..e03a5bf899c0f 100644 --- a/drivers/clk/meson/axg-audio.c +++ b/drivers/clk/meson/axg-audio.c @@ -1877,4 +1877,4 @@ module_platform_driver(axg_audio_driver); MODULE_DESCRIPTION("Amlogic AXG/G12A/SM1 Audio Clock driver"); MODULE_AUTHOR("Jerome Brunet "); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/axg.c b/drivers/clk/meson/axg.c index 5f60f2bcca592..52d610110e447 100644 --- a/drivers/clk/meson/axg.c +++ b/drivers/clk/meson/axg.c @@ -2185,4 +2185,4 @@ static struct platform_driver axg_driver = { }; module_platform_driver(axg_driver); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/clk-cpu-dyndiv.c b/drivers/clk/meson/clk-cpu-dyndiv.c index 8778c149d26ab..aa824b030cb82 100644 --- a/drivers/clk/meson/clk-cpu-dyndiv.c +++ b/drivers/clk/meson/clk-cpu-dyndiv.c @@ -69,4 +69,4 @@ EXPORT_SYMBOL_GPL(meson_clk_cpu_dyndiv_ops); MODULE_DESCRIPTION("Amlogic CPU Dynamic Clock divider"); MODULE_AUTHOR("Neil Armstrong "); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/clk-dualdiv.c b/drivers/clk/meson/clk-dualdiv.c index feae49a8f6dc5..d46c02b51be54 100644 --- a/drivers/clk/meson/clk-dualdiv.c +++ b/drivers/clk/meson/clk-dualdiv.c @@ -140,4 +140,4 @@ EXPORT_SYMBOL_GPL(meson_clk_dualdiv_ro_ops); MODULE_DESCRIPTION("Amlogic dual divider driver"); MODULE_AUTHOR("Neil Armstrong "); MODULE_AUTHOR("Jerome Brunet "); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/clk-mpll.c b/drivers/clk/meson/clk-mpll.c index 20255e129b377..eae9b7dc5a6cc 100644 --- a/drivers/clk/meson/clk-mpll.c +++ b/drivers/clk/meson/clk-mpll.c @@ -177,4 +177,4 @@ EXPORT_SYMBOL_GPL(meson_clk_mpll_ops); MODULE_DESCRIPTION("Amlogic MPLL driver"); MODULE_AUTHOR("Michael Turquette "); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/clk-phase.c b/drivers/clk/meson/clk-phase.c index a6763439f7d2b..ff3f0b1a3ed16 100644 --- a/drivers/clk/meson/clk-phase.c +++ b/drivers/clk/meson/clk-phase.c @@ -183,4 +183,4 @@ EXPORT_SYMBOL_GPL(meson_sclk_ws_inv_ops); MODULE_DESCRIPTION("Amlogic phase driver"); MODULE_AUTHOR("Jerome Brunet "); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/clk-pll.c b/drivers/clk/meson/clk-pll.c index 78d17b2415afc..07db8b5c30001 100644 --- a/drivers/clk/meson/clk-pll.c +++ b/drivers/clk/meson/clk-pll.c @@ -486,4 +486,4 @@ EXPORT_SYMBOL_GPL(meson_clk_pll_ro_ops); MODULE_DESCRIPTION("Amlogic PLL driver"); MODULE_AUTHOR("Carlo Caione "); MODULE_AUTHOR("Jerome Brunet "); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/clk-regmap.c b/drivers/clk/meson/clk-regmap.c index 8ad8977cf1c21..ad116d24f7002 100644 --- a/drivers/clk/meson/clk-regmap.c +++ b/drivers/clk/meson/clk-regmap.c @@ -183,4 +183,4 @@ EXPORT_SYMBOL_GPL(clk_regmap_mux_ro_ops); MODULE_DESCRIPTION("Amlogic regmap backed clock driver"); MODULE_AUTHOR("Jerome Brunet "); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/g12a-aoclk.c b/drivers/clk/meson/g12a-aoclk.c index c6b1d55cd7c86..58976ed8b92a0 100644 --- a/drivers/clk/meson/g12a-aoclk.c +++ b/drivers/clk/meson/g12a-aoclk.c @@ -475,4 +475,4 @@ static struct platform_driver g12a_aoclkc_driver = { }; module_platform_driver(g12a_aoclkc_driver); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/g12a.c b/drivers/clk/meson/g12a.c index df7e17c850d88..56e66ecc306ee 100644 --- a/drivers/clk/meson/g12a.c +++ b/drivers/clk/meson/g12a.c @@ -5614,4 +5614,4 @@ static struct platform_driver g12a_driver = { }; module_platform_driver(g12a_driver); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/gxbb-aoclk.c b/drivers/clk/meson/gxbb-aoclk.c index 4aec1740ac345..dbda563729db8 100644 --- a/drivers/clk/meson/gxbb-aoclk.c +++ b/drivers/clk/meson/gxbb-aoclk.c @@ -300,4 +300,4 @@ static struct platform_driver gxbb_aoclkc_driver = { }, }; module_platform_driver(gxbb_aoclkc_driver); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c index 1b1279d94781e..29507b8c43049 100644 --- a/drivers/clk/meson/gxbb.c +++ b/drivers/clk/meson/gxbb.c @@ -3569,4 +3569,4 @@ static struct platform_driver gxbb_driver = { }; module_platform_driver(gxbb_driver); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/meson-aoclk.c b/drivers/clk/meson/meson-aoclk.c index bf466fef263c3..b8a9d59e67264 100644 --- a/drivers/clk/meson/meson-aoclk.c +++ b/drivers/clk/meson/meson-aoclk.c @@ -89,4 +89,4 @@ int meson_aoclkc_probe(struct platform_device *pdev) return devm_of_clk_add_hw_provider(dev, meson_clk_hw_get, (void *)&data->hw_clks); } EXPORT_SYMBOL_GPL(meson_aoclkc_probe); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/meson-eeclk.c b/drivers/clk/meson/meson-eeclk.c index 845ca8bfa3461..3cbc7f233bba7 100644 --- a/drivers/clk/meson/meson-eeclk.c +++ b/drivers/clk/meson/meson-eeclk.c @@ -58,4 +58,4 @@ int meson_eeclkc_probe(struct platform_device *pdev) return devm_of_clk_add_hw_provider(dev, meson_clk_hw_get, (void *)&data->hw_clks); } EXPORT_SYMBOL_GPL(meson_eeclkc_probe); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/sclk-div.c b/drivers/clk/meson/sclk-div.c index d12c45c4c2616..987f5b06587c6 100644 --- a/drivers/clk/meson/sclk-div.c +++ b/drivers/clk/meson/sclk-div.c @@ -251,4 +251,4 @@ EXPORT_SYMBOL_GPL(meson_sclk_div_ops); MODULE_DESCRIPTION("Amlogic Sample divider driver"); MODULE_AUTHOR("Jerome Brunet "); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/vclk.c b/drivers/clk/meson/vclk.c index 45dc216941eaf..e886df55d6e38 100644 --- a/drivers/clk/meson/vclk.c +++ b/drivers/clk/meson/vclk.c @@ -138,4 +138,4 @@ EXPORT_SYMBOL_GPL(meson_vclk_div_ops); MODULE_DESCRIPTION("Amlogic vclk clock driver"); MODULE_AUTHOR("Neil Armstrong "); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/vid-pll-div.c b/drivers/clk/meson/vid-pll-div.c index daff235bc7633..ee129f86794db 100644 --- a/drivers/clk/meson/vid-pll-div.c +++ b/drivers/clk/meson/vid-pll-div.c @@ -96,4 +96,4 @@ EXPORT_SYMBOL_GPL(meson_vid_pll_div_ro_ops); MODULE_DESCRIPTION("Amlogic video pll divider driver"); MODULE_AUTHOR("Neil Armstrong "); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); From 3b84adf460381169c085e4bc09e7b57e9e16db0a Mon Sep 17 00:00:00 2001 From: Roman Smirnov Date: Wed, 27 Mar 2024 16:27:55 +0300 Subject: [PATCH 0108/1702] udf: udftime: prevent overflow in udf_disk_stamp_to_time() An overflow can occur in a situation where src.centiseconds takes the value of 255. This situation is unlikely, but there is no validation check anywere in the code. Found by Linux Verification Center (linuxtesting.org) with Svace. Suggested-by: Jan Kara Signed-off-by: Roman Smirnov Reviewed-by: Sergey Shtylyov Signed-off-by: Jan Kara Message-Id: <20240327132755.13945-1-r.smirnov@omp.ru> --- fs/udf/udftime.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/fs/udf/udftime.c b/fs/udf/udftime.c index 758163af39c26..78ecc633606fb 100644 --- a/fs/udf/udftime.c +++ b/fs/udf/udftime.c @@ -46,13 +46,18 @@ udf_disk_stamp_to_time(struct timespec64 *dest, struct timestamp src) dest->tv_sec = mktime64(year, src.month, src.day, src.hour, src.minute, src.second); dest->tv_sec -= offset * 60; - dest->tv_nsec = 1000 * (src.centiseconds * 10000 + - src.hundredsOfMicroseconds * 100 + src.microseconds); + /* * Sanitize nanosecond field since reportedly some filesystems are * recorded with bogus sub-second values. */ - dest->tv_nsec %= NSEC_PER_SEC; + if (src.centiseconds < 100 && src.hundredsOfMicroseconds < 100 && + src.microseconds < 100) { + dest->tv_nsec = 1000 * (src.centiseconds * 10000 + + src.hundredsOfMicroseconds * 100 + src.microseconds); + } else { + dest->tv_nsec = 0; + } } void From 1aab318f1e4900fac325e0d55a0591108ea0cfbb Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Wed, 10 Apr 2024 17:53:56 +0200 Subject: [PATCH 0109/1702] clk: qcom: fix module autoloading Add MODULE_DEVICE_TABLE(), so modules could be properly autoloaded based on the alias from of_device_id table. Clocks are considered core components, so usually they are built-in, however these can be built and used as modules on some generic kernel. Signed-off-by: Krzysztof Kozlowski Reviewed-by: Neil Armstrong Reviewed-by: Konrad Dybcio Link: https://lore.kernel.org/r/20240410155356.224098-1-krzk@kernel.org Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/gcc-msm8917.c | 1 + drivers/clk/qcom/gcc-msm8953.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/clk/qcom/gcc-msm8917.c b/drivers/clk/qcom/gcc-msm8917.c index f2dd132e2fb1c..f2b8729e4198d 100644 --- a/drivers/clk/qcom/gcc-msm8917.c +++ b/drivers/clk/qcom/gcc-msm8917.c @@ -3278,6 +3278,7 @@ static const struct of_device_id gcc_msm8917_match_table[] = { { .compatible = "qcom,gcc-qm215", .data = &gcc_qm215_desc }, {}, }; +MODULE_DEVICE_TABLE(of, gcc_msm8917_match_table); static struct platform_driver gcc_msm8917_driver = { .probe = gcc_msm8917_probe, diff --git a/drivers/clk/qcom/gcc-msm8953.c b/drivers/clk/qcom/gcc-msm8953.c index 68359534ff257..7563bff581186 100644 --- a/drivers/clk/qcom/gcc-msm8953.c +++ b/drivers/clk/qcom/gcc-msm8953.c @@ -4227,6 +4227,7 @@ static const struct of_device_id gcc_msm8953_match_table[] = { { .compatible = "qcom,gcc-msm8953" }, {}, }; +MODULE_DEVICE_TABLE(of, gcc_msm8953_match_table); static struct platform_driver gcc_msm8953_driver = { .probe = gcc_msm8953_probe, From 0a382be005cf8e7ac1b6ed25f47c2f599b1ff6af Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Sat, 6 Apr 2024 12:43:41 +0200 Subject: [PATCH 0110/1702] dt-bindings: clock: airoha: add EN7581 binding Introduce Airoha EN7581 entry in Airoha EN7523 clock binding Signed-off-by: Lorenzo Bianconi Link: https://lore.kernel.org/r/99734deb28889e685a764da94418f68b55ee3bdc.1712399981.git.lorenzo@kernel.org Reviewed-by: Krzysztof Kozlowski Signed-off-by: Stephen Boyd --- .../bindings/clock/airoha,en7523-scu.yaml | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/clock/airoha,en7523-scu.yaml b/Documentation/devicetree/bindings/clock/airoha,en7523-scu.yaml index 79b0752faa917..3f42666377332 100644 --- a/Documentation/devicetree/bindings/clock/airoha,en7523-scu.yaml +++ b/Documentation/devicetree/bindings/clock/airoha,en7523-scu.yaml @@ -29,10 +29,13 @@ description: | properties: compatible: items: - - const: airoha,en7523-scu + - enum: + - airoha,en7523-scu + - airoha,en7581-scu reg: - maxItems: 2 + minItems: 2 + maxItems: 3 "#clock-cells": description: @@ -45,6 +48,30 @@ required: - reg - '#clock-cells' +allOf: + - if: + properties: + compatible: + const: airoha,en7523-scu + then: + properties: + reg: + items: + - description: scu base address + - description: misc scu base address + + - if: + properties: + compatible: + const: airoha,en7581-scu + then: + properties: + reg: + items: + - description: scu base address + - description: misc scu base address + - description: pb scu base address + additionalProperties: false examples: From 457e74667f452d7f071ad2b2d9313ec62ebc4b02 Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Sat, 6 Apr 2024 12:43:43 +0200 Subject: [PATCH 0111/1702] clk: en7523: Add en_clk_soc_data data structure Introduce en_clk_soc_data data structure in order to define multiple clk_ops for each supported SoC. This is a preliminary patch to introduce EN7581 clock support. Tested-by: Zhengping Zhang Signed-off-by: Lorenzo Bianconi Link: https://lore.kernel.org/r/562a0da8d7874a02a324687c152c87a1549924bd.1712399981.git.lorenzo@kernel.org Signed-off-by: Stephen Boyd --- drivers/clk/clk-en7523.c | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/drivers/clk/clk-en7523.c b/drivers/clk/clk-en7523.c index 7cde328495e2b..3fac049c25183 100644 --- a/drivers/clk/clk-en7523.c +++ b/drivers/clk/clk-en7523.c @@ -3,8 +3,8 @@ #include #include #include -#include #include +#include #include #define REG_PCI_CONTROL 0x88 @@ -47,6 +47,10 @@ struct en_clk_gate { struct clk_hw hw; }; +struct en_clk_soc_data { + const struct clk_ops pcie_ops; +}; + static const u32 gsw_base[] = { 400000000, 500000000 }; static const u32 emi_base[] = { 333000000, 400000000 }; static const u32 bus_base[] = { 500000000, 540000000 }; @@ -145,11 +149,6 @@ static const struct en_clk_desc en7523_base_clks[] = { } }; -static const struct of_device_id of_match_clk_en7523[] = { - { .compatible = "airoha,en7523-scu", }, - { /* sentinel */ } -}; - static unsigned int en7523_get_base_rate(void __iomem *base, unsigned int i) { const struct en_clk_desc *desc = &en7523_base_clks[i]; @@ -247,14 +246,10 @@ static void en7523_pci_unprepare(struct clk_hw *hw) static struct clk_hw *en7523_register_pcie_clk(struct device *dev, void __iomem *np_base) { - static const struct clk_ops pcie_gate_ops = { - .is_enabled = en7523_pci_is_enabled, - .prepare = en7523_pci_prepare, - .unprepare = en7523_pci_unprepare, - }; + const struct en_clk_soc_data *soc_data = device_get_match_data(dev); struct clk_init_data init = { .name = "pcie", - .ops = &pcie_gate_ops, + .ops = &soc_data->pcie_ops, }; struct en_clk_gate *cg; @@ -264,7 +259,7 @@ static struct clk_hw *en7523_register_pcie_clk(struct device *dev, cg->base = np_base; cg->hw.init = &init; - en7523_pci_unprepare(&cg->hw); + init.ops->unprepare(&cg->hw); if (clk_hw_register(dev, &cg->hw)) return NULL; @@ -333,6 +328,19 @@ static int en7523_clk_probe(struct platform_device *pdev) return r; } +static const struct en_clk_soc_data en7523_data = { + .pcie_ops = { + .is_enabled = en7523_pci_is_enabled, + .prepare = en7523_pci_prepare, + .unprepare = en7523_pci_unprepare, + }, +}; + +static const struct of_device_id of_match_clk_en7523[] = { + { .compatible = "airoha,en7523-scu", .data = &en7523_data }, + { /* sentinel */ } +}; + static struct platform_driver clk_en7523_drv = { .probe = en7523_clk_probe, .driver = { From 66bc47326ce2a319add7e933d9340215711236ac Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Sat, 6 Apr 2024 12:43:44 +0200 Subject: [PATCH 0112/1702] clk: en7523: Add EN7581 support Introduce EN7581 clock support to clk-en7523 driver. Add hw_init callback to en_clk_soc_data data structure. Tested-by: Zhengping Zhang Signed-off-by: Lorenzo Bianconi Link: https://lore.kernel.org/r/57b6e53ed4d2b2e38abff6a3ea56841bad6be8a9.1712399981.git.lorenzo@kernel.org Signed-off-by: Stephen Boyd --- drivers/clk/clk-en7523.c | 157 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 152 insertions(+), 5 deletions(-) diff --git a/drivers/clk/clk-en7523.c b/drivers/clk/clk-en7523.c index 3fac049c25183..ccc3946926712 100644 --- a/drivers/clk/clk-en7523.c +++ b/drivers/clk/clk-en7523.c @@ -10,7 +10,9 @@ #define REG_PCI_CONTROL 0x88 #define REG_PCI_CONTROL_PERSTOUT BIT(29) #define REG_PCI_CONTROL_PERSTOUT1 BIT(26) +#define REG_PCI_CONTROL_REFCLK_EN0 BIT(23) #define REG_PCI_CONTROL_REFCLK_EN1 BIT(22) +#define REG_PCI_CONTROL_PERSTOUT2 BIT(16) #define REG_GSW_CLK_DIV_SEL 0x1b4 #define REG_EMI_CLK_DIV_SEL 0x1b8 #define REG_BUS_CLK_DIV_SEL 0x1bc @@ -18,10 +20,25 @@ #define REG_SPI_CLK_FREQ_SEL 0x1c8 #define REG_NPU_CLK_DIV_SEL 0x1fc #define REG_CRYPTO_CLKSRC 0x200 -#define REG_RESET_CONTROL 0x834 +#define REG_RESET_CONTROL2 0x830 +#define REG_RESET2_CONTROL_PCIE2 BIT(27) +#define REG_RESET_CONTROL1 0x834 #define REG_RESET_CONTROL_PCIEHB BIT(29) #define REG_RESET_CONTROL_PCIE1 BIT(27) #define REG_RESET_CONTROL_PCIE2 BIT(26) +/* EN7581 */ +#define REG_PCIE0_MEM 0x00 +#define REG_PCIE0_MEM_MASK 0x04 +#define REG_PCIE1_MEM 0x08 +#define REG_PCIE1_MEM_MASK 0x0c +#define REG_PCIE2_MEM 0x10 +#define REG_PCIE2_MEM_MASK 0x14 +#define REG_PCIE_RESET_OPEN_DRAIN 0x018c +#define REG_PCIE_RESET_OPEN_DRAIN_MASK GENMASK(2, 0) +#define REG_NP_SCU_PCIC 0x88 +#define REG_NP_SCU_SSTR 0x9c +#define REG_PCIE_XSI0_SEL_MASK GENMASK(14, 13) +#define REG_PCIE_XSI1_SEL_MASK GENMASK(12, 11) struct en_clk_desc { int id; @@ -49,6 +66,8 @@ struct en_clk_gate { struct en_clk_soc_data { const struct clk_ops pcie_ops; + int (*hw_init)(struct platform_device *pdev, void __iomem *base, + void __iomem *np_base); }; static const u32 gsw_base[] = { 400000000, 500000000 }; @@ -211,14 +230,14 @@ static int en7523_pci_prepare(struct clk_hw *hw) usleep_range(1000, 2000); /* Reset to default */ - val = readl(np_base + REG_RESET_CONTROL); + val = readl(np_base + REG_RESET_CONTROL1); mask = REG_RESET_CONTROL_PCIE1 | REG_RESET_CONTROL_PCIE2 | REG_RESET_CONTROL_PCIEHB; - writel(val & ~mask, np_base + REG_RESET_CONTROL); + writel(val & ~mask, np_base + REG_RESET_CONTROL1); usleep_range(1000, 2000); - writel(val | mask, np_base + REG_RESET_CONTROL); + writel(val | mask, np_base + REG_RESET_CONTROL1); msleep(100); - writel(val & ~mask, np_base + REG_RESET_CONTROL); + writel(val & ~mask, np_base + REG_RESET_CONTROL1); usleep_range(5000, 10000); /* Release device */ @@ -259,6 +278,9 @@ static struct clk_hw *en7523_register_pcie_clk(struct device *dev, cg->base = np_base; cg->hw.init = &init; + + if (init.ops->disable) + init.ops->disable(&cg->hw); init.ops->unprepare(&cg->hw); if (clk_hw_register(dev, &cg->hw)) @@ -267,6 +289,111 @@ static struct clk_hw *en7523_register_pcie_clk(struct device *dev, return &cg->hw; } +static int en7581_pci_is_enabled(struct clk_hw *hw) +{ + struct en_clk_gate *cg = container_of(hw, struct en_clk_gate, hw); + u32 val, mask; + + mask = REG_PCI_CONTROL_REFCLK_EN0 | REG_PCI_CONTROL_REFCLK_EN1; + val = readl(cg->base + REG_PCI_CONTROL); + return (val & mask) == mask; +} + +static int en7581_pci_prepare(struct clk_hw *hw) +{ + struct en_clk_gate *cg = container_of(hw, struct en_clk_gate, hw); + void __iomem *np_base = cg->base; + u32 val, mask; + + mask = REG_RESET_CONTROL_PCIE1 | REG_RESET_CONTROL_PCIE2 | + REG_RESET_CONTROL_PCIEHB; + val = readl(np_base + REG_RESET_CONTROL1); + writel(val & ~mask, np_base + REG_RESET_CONTROL1); + val = readl(np_base + REG_RESET_CONTROL2); + writel(val & ~REG_RESET2_CONTROL_PCIE2, np_base + REG_RESET_CONTROL2); + usleep_range(5000, 10000); + + return 0; +} + +static int en7581_pci_enable(struct clk_hw *hw) +{ + struct en_clk_gate *cg = container_of(hw, struct en_clk_gate, hw); + void __iomem *np_base = cg->base; + u32 val, mask; + + mask = REG_PCI_CONTROL_REFCLK_EN0 | REG_PCI_CONTROL_REFCLK_EN1 | + REG_PCI_CONTROL_PERSTOUT1 | REG_PCI_CONTROL_PERSTOUT2 | + REG_PCI_CONTROL_PERSTOUT; + val = readl(np_base + REG_PCI_CONTROL); + writel(val | mask, np_base + REG_PCI_CONTROL); + msleep(250); + + return 0; +} + +static void en7581_pci_unprepare(struct clk_hw *hw) +{ + struct en_clk_gate *cg = container_of(hw, struct en_clk_gate, hw); + void __iomem *np_base = cg->base; + u32 val, mask; + + mask = REG_RESET_CONTROL_PCIE1 | REG_RESET_CONTROL_PCIE2 | + REG_RESET_CONTROL_PCIEHB; + val = readl(np_base + REG_RESET_CONTROL1); + writel(val | mask, np_base + REG_RESET_CONTROL1); + mask = REG_RESET_CONTROL_PCIE1 | REG_RESET_CONTROL_PCIE2; + writel(val | mask, np_base + REG_RESET_CONTROL1); + val = readl(np_base + REG_RESET_CONTROL2); + writel(val | REG_RESET_CONTROL_PCIE2, np_base + REG_RESET_CONTROL2); + msleep(100); +} + +static void en7581_pci_disable(struct clk_hw *hw) +{ + struct en_clk_gate *cg = container_of(hw, struct en_clk_gate, hw); + void __iomem *np_base = cg->base; + u32 val, mask; + + mask = REG_PCI_CONTROL_REFCLK_EN0 | REG_PCI_CONTROL_REFCLK_EN1 | + REG_PCI_CONTROL_PERSTOUT1 | REG_PCI_CONTROL_PERSTOUT2 | + REG_PCI_CONTROL_PERSTOUT; + val = readl(np_base + REG_PCI_CONTROL); + writel(val & ~mask, np_base + REG_PCI_CONTROL); + usleep_range(1000, 2000); +} + +static int en7581_clk_hw_init(struct platform_device *pdev, + void __iomem *base, + void __iomem *np_base) +{ + void __iomem *pb_base; + u32 val; + + pb_base = devm_platform_ioremap_resource(pdev, 2); + if (IS_ERR(pb_base)) + return PTR_ERR(pb_base); + + val = readl(np_base + REG_NP_SCU_SSTR); + val &= ~(REG_PCIE_XSI0_SEL_MASK | REG_PCIE_XSI1_SEL_MASK); + writel(val, np_base + REG_NP_SCU_SSTR); + val = readl(np_base + REG_NP_SCU_PCIC); + writel(val | 3, np_base + REG_NP_SCU_PCIC); + + writel(0x20000000, pb_base + REG_PCIE0_MEM); + writel(0xfc000000, pb_base + REG_PCIE0_MEM_MASK); + writel(0x24000000, pb_base + REG_PCIE1_MEM); + writel(0xfc000000, pb_base + REG_PCIE1_MEM_MASK); + writel(0x28000000, pb_base + REG_PCIE2_MEM); + writel(0xfc000000, pb_base + REG_PCIE2_MEM_MASK); + + val = readl(base + REG_PCIE_RESET_OPEN_DRAIN); + writel(val | REG_PCIE_RESET_OPEN_DRAIN_MASK, + base + REG_PCIE_RESET_OPEN_DRAIN); + + return 0; +} + static void en7523_register_clocks(struct device *dev, struct clk_hw_onecell_data *clk_data, void __iomem *base, void __iomem *np_base) { @@ -299,6 +426,7 @@ static void en7523_register_clocks(struct device *dev, struct clk_hw_onecell_dat static int en7523_clk_probe(struct platform_device *pdev) { struct device_node *node = pdev->dev.of_node; + const struct en_clk_soc_data *soc_data; struct clk_hw_onecell_data *clk_data; void __iomem *base, *np_base; int r; @@ -311,6 +439,13 @@ static int en7523_clk_probe(struct platform_device *pdev) if (IS_ERR(np_base)) return PTR_ERR(np_base); + soc_data = device_get_match_data(&pdev->dev); + if (soc_data->hw_init) { + r = soc_data->hw_init(pdev, base, np_base); + if (r) + return r; + } + clk_data = devm_kzalloc(&pdev->dev, struct_size(clk_data, hws, EN7523_NUM_CLOCKS), GFP_KERNEL); @@ -336,8 +471,20 @@ static const struct en_clk_soc_data en7523_data = { }, }; +static const struct en_clk_soc_data en7581_data = { + .pcie_ops = { + .is_enabled = en7581_pci_is_enabled, + .prepare = en7581_pci_prepare, + .enable = en7581_pci_enable, + .unprepare = en7581_pci_unprepare, + .disable = en7581_pci_disable, + }, + .hw_init = en7581_clk_hw_init, +}; + static const struct of_device_id of_match_clk_en7523[] = { { .compatible = "airoha,en7523-scu", .data = &en7523_data }, + { .compatible = "airoha,en7581-scu", .data = &en7581_data }, { /* sentinel */ } }; From bb7b3c8e7180f36de75cdea200ab7127f93f58cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N=C3=ADcolas=20F=2E=20R=2E=20A=2E=20Prado?= Date: Fri, 8 Mar 2024 15:29:56 -0500 Subject: [PATCH 0113/1702] clk: mediatek: pllfh: Don't log error for missing fhctl node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Support for fhctl clocks in apmixedsys was introduced at a later point and to this moment only one mt6795 based platform has a fhctl DT node present. Therefore the fhctl support in apmixedsys should be seen as optional and not cause an error when it is missing. Change the message's log level to warning. The warning level is chosen so that it will still alert the fact that fhctl support might be unintentionally missing, but without implying that this is necessarily an issue. Even if the FHCTL DT nodes are added to all current platforms moving forward, since those changes won't be backported, this ensures stable kernel releases won't have live with this error. Fixes: d7964de8a8ea ("clk: mediatek: Add new clock driver to handle FHCTL hardware") Signed-off-by: Nícolas F. R. A. Prado Link: https://lore.kernel.org/r/20240308-mtk-fhctl-no-node-error-v1-1-51e446eb149a@collabora.com Reviewed-by: AngeloGioacchino Del Regno Signed-off-by: Stephen Boyd --- drivers/clk/mediatek/clk-pllfh.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clk/mediatek/clk-pllfh.c b/drivers/clk/mediatek/clk-pllfh.c index 3a2b3f90be25d..094ec8a26d668 100644 --- a/drivers/clk/mediatek/clk-pllfh.c +++ b/drivers/clk/mediatek/clk-pllfh.c @@ -68,7 +68,7 @@ void fhctl_parse_dt(const u8 *compatible_node, struct mtk_pllfh_data *pllfhs, node = of_find_compatible_node(NULL, NULL, compatible_node); if (!node) { - pr_err("cannot find \"%s\"\n", compatible_node); + pr_warn("cannot find \"%s\"\n", compatible_node); return; } From 50e45d3b03fd20d663dc8e76fc881ba722403676 Mon Sep 17 00:00:00 2001 From: Inochi Amaoto Date: Sat, 9 Mar 2024 17:02:51 +0800 Subject: [PATCH 0114/1702] dt-bindings: clock: sophgo: Add clock controller of SG2000 series SoC SG2000 series SoC has the same clock as CV1810 series, but the clock related to A53 is functional in SG2000 series. So a new compatible string is needed for the new SoC. Add definition for the clock controller of the SG2000 series SoC. Link: https://github.com/sophgo/sophgo-doc/releases/tag/sg2000-datasheet-v1.0-alpha Acked-by: Krzysztof Kozlowski Signed-off-by: Inochi Amaoto Link: https://lore.kernel.org/r/IA1PR20MB495368F185E018767CC6714ABB262@IA1PR20MB4953.namprd20.prod.outlook.com Signed-off-by: Stephen Boyd --- Documentation/devicetree/bindings/clock/sophgo,cv1800-clk.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/clock/sophgo,cv1800-clk.yaml b/Documentation/devicetree/bindings/clock/sophgo,cv1800-clk.yaml index c1dc24673c0d7..59ef41adb539b 100644 --- a/Documentation/devicetree/bindings/clock/sophgo,cv1800-clk.yaml +++ b/Documentation/devicetree/bindings/clock/sophgo,cv1800-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/sophgo,cv1800-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Sophgo CV1800 Series Clock Controller +title: Sophgo CV1800/SG2000 Series Clock Controller maintainers: - Inochi Amaoto @@ -14,6 +14,7 @@ properties: enum: - sophgo,cv1800-clk - sophgo,cv1810-clk + - sophgo,sg2000-clk reg: maxItems: 1 From 80fd61ec46124eb83b29de3647a565f69979e753 Mon Sep 17 00:00:00 2001 From: Inochi Amaoto Date: Sat, 9 Mar 2024 17:02:52 +0800 Subject: [PATCH 0115/1702] clk: sophgo: Add clock support for CV1800 SoC Add clock definition and driver code for CV1800 SoC. Signed-off-by: Inochi Amaoto Link: https://github.com/milkv-duo/duo-files/blob/6f4e9b8ecb459e017cca1a8df248a19ca70837a3/duo/datasheet/CV180X-Clock-v1.xlsx Link: https://github.com/milkv-duo/duo-files/blob/6f4e9b8ecb459e017cca1a8df248a19ca70837a3/duo/datasheet/CV1800B-CV1801B-Preliminary-Datasheet-full-en.pdf Link: https://lore.kernel.org/r/IA1PR20MB49534F37F802CAF117364D66BB262@IA1PR20MB4953.namprd20.prod.outlook.com Signed-off-by: Stephen Boyd --- drivers/clk/Kconfig | 1 + drivers/clk/Makefile | 1 + drivers/clk/sophgo/Kconfig | 11 + drivers/clk/sophgo/Makefile | 7 + drivers/clk/sophgo/clk-cv1800.c | 1327 ++++++++++++++++++++++++ drivers/clk/sophgo/clk-cv1800.h | 122 +++ drivers/clk/sophgo/clk-cv18xx-common.c | 66 ++ drivers/clk/sophgo/clk-cv18xx-common.h | 81 ++ drivers/clk/sophgo/clk-cv18xx-ip.c | 887 ++++++++++++++++ drivers/clk/sophgo/clk-cv18xx-ip.h | 261 +++++ drivers/clk/sophgo/clk-cv18xx-pll.c | 420 ++++++++ drivers/clk/sophgo/clk-cv18xx-pll.h | 118 +++ 12 files changed, 3302 insertions(+) create mode 100644 drivers/clk/sophgo/Kconfig create mode 100644 drivers/clk/sophgo/Makefile create mode 100644 drivers/clk/sophgo/clk-cv1800.c create mode 100644 drivers/clk/sophgo/clk-cv1800.h create mode 100644 drivers/clk/sophgo/clk-cv18xx-common.c create mode 100644 drivers/clk/sophgo/clk-cv18xx-common.h create mode 100644 drivers/clk/sophgo/clk-cv18xx-ip.c create mode 100644 drivers/clk/sophgo/clk-cv18xx-ip.h create mode 100644 drivers/clk/sophgo/clk-cv18xx-pll.c create mode 100644 drivers/clk/sophgo/clk-cv18xx-pll.h diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index 50af5fc7f5708..bc28502ec3c90 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -489,6 +489,7 @@ source "drivers/clk/rockchip/Kconfig" source "drivers/clk/samsung/Kconfig" source "drivers/clk/sifive/Kconfig" source "drivers/clk/socfpga/Kconfig" +source "drivers/clk/sophgo/Kconfig" source "drivers/clk/sprd/Kconfig" source "drivers/clk/starfive/Kconfig" source "drivers/clk/sunxi/Kconfig" diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index 14fa8d4ecc1fb..4abe16c8ccdfe 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -118,6 +118,7 @@ obj-$(CONFIG_ARCH_ROCKCHIP) += rockchip/ obj-$(CONFIG_COMMON_CLK_SAMSUNG) += samsung/ obj-$(CONFIG_CLK_SIFIVE) += sifive/ obj-y += socfpga/ +obj-y += sophgo/ obj-$(CONFIG_PLAT_SPEAR) += spear/ obj-y += sprd/ obj-$(CONFIG_ARCH_STI) += st/ diff --git a/drivers/clk/sophgo/Kconfig b/drivers/clk/sophgo/Kconfig new file mode 100644 index 0000000000000..1cc49be71bdb2 --- /dev/null +++ b/drivers/clk/sophgo/Kconfig @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 +# common clock support for SOPHGO SoC family. + +config CLK_SOPHGO_CV1800 + tristate "Support for the Sophgo CV1800 series SoCs clock controller" + depends on ARCH_SOPHGO || COMPILE_TEST + help + This driver supports clock controller of Sophgo CV18XX series SoC. + The driver require a 25MHz Oscillator to function generate clock. + It includes PLLs, common clock function and some vendor clock for + IPs of CV18XX series SoC diff --git a/drivers/clk/sophgo/Makefile b/drivers/clk/sophgo/Makefile new file mode 100644 index 0000000000000..a50320764200e --- /dev/null +++ b/drivers/clk/sophgo/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CLK_SOPHGO_CV1800) += clk-sophgo-cv1800.o + +clk-sophgo-cv1800-y += clk-cv1800.o +clk-sophgo-cv1800-y += clk-cv18xx-common.o +clk-sophgo-cv1800-y += clk-cv18xx-ip.o +clk-sophgo-cv1800-y += clk-cv18xx-pll.o diff --git a/drivers/clk/sophgo/clk-cv1800.c b/drivers/clk/sophgo/clk-cv1800.c new file mode 100644 index 0000000000000..6606a2701b122 --- /dev/null +++ b/drivers/clk/sophgo/clk-cv1800.c @@ -0,0 +1,1327 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023 Inochi Amaoto + */ + +#include +#include +#include +#include +#include + +#include "clk-cv1800.h" + +#include "clk-cv18xx-common.h" +#include "clk-cv18xx-ip.h" +#include "clk-cv18xx-pll.h" + +struct cv1800_clk_ctrl; + +struct cv1800_clk_desc { + struct clk_hw_onecell_data *clks_data; + + int (*pre_init)(struct device *dev, void __iomem *base, + struct cv1800_clk_ctrl *ctrl, + const struct cv1800_clk_desc *desc); +}; + +struct cv1800_clk_ctrl { + const struct cv1800_clk_desc *desc; + spinlock_t lock; +}; + +#define CV1800_DIV_FLAG \ + (CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ROUND_CLOSEST) +static const struct clk_parent_data osc_parents[] = { + { .index = 0 }, +}; + +static const struct cv1800_clk_pll_limit pll_limits[] = { + { + .pre_div = _CV1800_PLL_LIMIT(1, 127), + .div = _CV1800_PLL_LIMIT(6, 127), + .post_div = _CV1800_PLL_LIMIT(1, 127), + .ictrl = _CV1800_PLL_LIMIT(0, 7), + .mode = _CV1800_PLL_LIMIT(0, 3), + }, + { + .pre_div = _CV1800_PLL_LIMIT(1, 127), + .div = _CV1800_PLL_LIMIT(6, 127), + .post_div = _CV1800_PLL_LIMIT(1, 127), + .ictrl = _CV1800_PLL_LIMIT(0, 7), + .mode = _CV1800_PLL_LIMIT(0, 3), + }, +}; + +static CV1800_INTEGRAL_PLL(clk_fpll, osc_parents, + REG_FPLL_CSR, + REG_PLL_G6_CTRL, 8, + REG_PLL_G6_STATUS, 2, + pll_limits, + CLK_IS_CRITICAL); + +static CV1800_INTEGRAL_PLL(clk_mipimpll, osc_parents, + REG_MIPIMPLL_CSR, + REG_PLL_G2_CTRL, 0, + REG_PLL_G2_STATUS, 0, + pll_limits, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_mipimpll_parents[] = { + { .hw = &clk_mipimpll.common.hw }, +}; +static const struct clk_parent_data clk_bypass_mipimpll_parents[] = { + { .index = 0 }, + { .hw = &clk_mipimpll.common.hw }, +}; +static const struct clk_parent_data clk_bypass_fpll_parents[] = { + { .index = 0 }, + { .hw = &clk_fpll.common.hw }, +}; + +struct cv1800_clk_pll_synthesizer clk_mpll_synthesizer = { + .en = CV1800_CLK_BIT(REG_PLL_G6_SSC_SYN_CTRL, 2), + .clk_half = CV1800_CLK_BIT(REG_PLL_G6_SSC_SYN_CTRL, 0), + .ctrl = REG_MPLL_SSC_SYN_CTRL, + .set = REG_MPLL_SSC_SYN_SET, +}; +static CV1800_FACTIONAL_PLL(clk_mpll, clk_bypass_mipimpll_parents, + REG_MPLL_CSR, + REG_PLL_G6_CTRL, 0, + REG_PLL_G6_STATUS, 0, + pll_limits, + &clk_mpll_synthesizer, + CLK_IS_CRITICAL); + +struct cv1800_clk_pll_synthesizer clk_tpll_synthesizer = { + .en = CV1800_CLK_BIT(REG_PLL_G6_SSC_SYN_CTRL, 3), + .clk_half = CV1800_CLK_BIT(REG_PLL_G6_SSC_SYN_CTRL, 0), + .ctrl = REG_TPLL_SSC_SYN_CTRL, + .set = REG_TPLL_SSC_SYN_SET, +}; +static CV1800_FACTIONAL_PLL(clk_tpll, clk_bypass_mipimpll_parents, + REG_TPLL_CSR, + REG_PLL_G6_CTRL, 4, + REG_PLL_G6_STATUS, 1, + pll_limits, + &clk_tpll_synthesizer, + CLK_IS_CRITICAL); + +struct cv1800_clk_pll_synthesizer clk_a0pll_synthesizer = { + .en = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 2), + .clk_half = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 0), + .ctrl = REG_A0PLL_SSC_SYN_CTRL, + .set = REG_A0PLL_SSC_SYN_SET, +}; +static CV1800_FACTIONAL_PLL(clk_a0pll, clk_bypass_mipimpll_parents, + REG_A0PLL_CSR, + REG_PLL_G2_CTRL, 4, + REG_PLL_G2_STATUS, 1, + pll_limits, + &clk_a0pll_synthesizer, + CLK_IS_CRITICAL); + +struct cv1800_clk_pll_synthesizer clk_disppll_synthesizer = { + .en = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 3), + .clk_half = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 0), + .ctrl = REG_DISPPLL_SSC_SYN_CTRL, + .set = REG_DISPPLL_SSC_SYN_SET, +}; +static CV1800_FACTIONAL_PLL(clk_disppll, clk_bypass_mipimpll_parents, + REG_DISPPLL_CSR, + REG_PLL_G2_CTRL, 8, + REG_PLL_G2_STATUS, 2, + pll_limits, + &clk_disppll_synthesizer, + CLK_IS_CRITICAL); + +struct cv1800_clk_pll_synthesizer clk_cam0pll_synthesizer = { + .en = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 4), + .clk_half = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 0), + .ctrl = REG_CAM0PLL_SSC_SYN_CTRL, + .set = REG_CAM0PLL_SSC_SYN_SET, +}; +static CV1800_FACTIONAL_PLL(clk_cam0pll, clk_bypass_mipimpll_parents, + REG_CAM0PLL_CSR, + REG_PLL_G2_CTRL, 12, + REG_PLL_G2_STATUS, 3, + pll_limits, + &clk_cam0pll_synthesizer, + CLK_IGNORE_UNUSED); + +struct cv1800_clk_pll_synthesizer clk_cam1pll_synthesizer = { + .en = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 5), + .clk_half = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 0), + .ctrl = REG_CAM1PLL_SSC_SYN_CTRL, + .set = REG_CAM1PLL_SSC_SYN_SET, +}; +static CV1800_FACTIONAL_PLL(clk_cam1pll, clk_bypass_mipimpll_parents, + REG_CAM1PLL_CSR, + REG_PLL_G2_CTRL, 16, + REG_PLL_G2_STATUS, 4, + pll_limits, + &clk_cam1pll_synthesizer, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_cam0pll_parents[] = { + { .hw = &clk_cam0pll.common.hw }, +}; + +/* G2D */ +static CV1800_FIXED_DIV(clk_cam0pll_d2, clk_cam0pll_parents, + REG_CAM0PLL_CLK_CSR, 1, + 2, + CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED); +static CV1800_FIXED_DIV(clk_cam0pll_d3, clk_cam0pll_parents, + REG_CAM0PLL_CLK_CSR, 2, + 3, + CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED); +static CV1800_FIXED_DIV(clk_mipimpll_d3, clk_mipimpll_parents, + REG_MIPIMPLL_CLK_CSR, 2, + 3, + CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED); + +/* TPU */ +static const struct clk_parent_data clk_tpu_parents[] = { + { .index = 0 }, + { .hw = &clk_tpll.common.hw }, + { .hw = &clk_a0pll.common.hw }, + { .hw = &clk_mipimpll.common.hw }, + { .hw = &clk_fpll.common.hw }, +}; + +static CV1800_BYPASS_MUX(clk_tpu, clk_tpu_parents, + REG_CLK_EN_0, 4, + REG_DIV_CLK_TPU, 16, 4, 3, CV1800_DIV_FLAG, + REG_DIV_CLK_TPU, 8, 2, + REG_CLK_BYP_0, 3, + 0); +static CV1800_GATE(clk_tpu_fab, clk_mipimpll_parents, + REG_CLK_EN_0, 5, + 0); + +/* FABRIC_AXI6 */ +static CV1800_BYPASS_DIV(clk_axi6, clk_bypass_fpll_parents, + REG_CLK_EN_2, 2, + REG_DIV_CLK_AXI6, 16, 4, 15, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 20, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_axi6_bus_parents[] = { + { .hw = &clk_axi6.div.common.hw }, +}; +static const struct clk_parent_data clk_bypass_axi6_bus_parents[] = { + { .index = 0 }, + { .hw = &clk_axi6.div.common.hw }, +}; + +/* FABRIC_AXI4 */ +static const struct clk_parent_data clk_axi4_parents[] = { + { .index = 0 }, + { .hw = &clk_fpll.common.hw }, + { .hw = &clk_disppll.common.hw }, +}; + +static CV1800_BYPASS_MUX(clk_axi4, clk_axi4_parents, + REG_CLK_EN_2, 1, + REG_DIV_CLK_AXI4, 16, 4, 5, CV1800_DIV_FLAG, + REG_DIV_CLK_AXI4, 8, 2, + REG_CLK_BYP_0, 19, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_axi4_bus_parents[] = { + { .hw = &clk_axi4.mux.common.hw }, +}; + +/* XTAL_MISC */ +static CV1800_GATE(clk_xtal_misc, osc_parents, + REG_CLK_EN_0, 14, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_timer_parents[] = { + { .hw = &clk_xtal_misc.common.hw }, +}; + +/* TOP */ +static const struct clk_parent_data clk_cam0_200_parents[] = { + { .index = 0 }, + { .index = 0 }, + { .hw = &clk_disppll.common.hw }, +}; + +static CV1800_BYPASS_MUX(clk_cam0_200, clk_cam0_200_parents, + REG_CLK_EN_1, 13, + REG_DIV_CLK_CAM0_200, 16, 4, 1, CV1800_DIV_FLAG, + REG_DIV_CLK_CAM0_200, 8, 2, + REG_CLK_BYP_0, 16, + CLK_IS_CRITICAL); +static CV1800_DIV(clk_1m, osc_parents, + REG_CLK_EN_3, 5, + REG_DIV_CLK_1M, 16, 6, 25, CV1800_DIV_FLAG, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_pm, clk_axi6_bus_parents, + REG_CLK_EN_3, 8, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer0, clk_timer_parents, + REG_CLK_EN_3, 9, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer1, clk_timer_parents, + REG_CLK_EN_3, 10, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer2, clk_timer_parents, + REG_CLK_EN_3, 11, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer3, clk_timer_parents, + REG_CLK_EN_3, 12, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer4, clk_timer_parents, + REG_CLK_EN_3, 13, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer5, clk_timer_parents, + REG_CLK_EN_3, 14, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer6, clk_timer_parents, + REG_CLK_EN_3, 15, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_timer7, clk_timer_parents, + REG_CLK_EN_3, 16, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_parents_1m[] = { + { .hw = &clk_1m.common.hw }, +}; +static const struct clk_parent_data clk_uart_parents[] = { + { .hw = &clk_cam0_200.mux.common.hw }, +}; + +/* AHB ROM */ +static CV1800_GATE(clk_ahb_rom, clk_axi4_bus_parents, + REG_CLK_EN_0, 6, + 0); + +/* RTC */ +static CV1800_GATE(clk_rtc_25m, osc_parents, + REG_CLK_EN_0, 8, + CLK_IS_CRITICAL); +static CV1800_BYPASS_DIV(clk_src_rtc_sys_0, clk_bypass_fpll_parents, + REG_CLK_EN_4, 6, + REG_DIV_CLK_RTCSYS_SRC_0, 16, 4, 5, CV1800_DIV_FLAG, + REG_CLK_BYP_1, 5, + CLK_IS_CRITICAL); + +/* TEMPSEN */ +static CV1800_GATE(clk_tempsen, osc_parents, + REG_CLK_EN_0, 9, + 0); + +/* SARADC */ +static CV1800_GATE(clk_saradc, osc_parents, + REG_CLK_EN_0, 10, + 0); + +/* EFUSE */ +static CV1800_GATE(clk_efuse, osc_parents, + REG_CLK_EN_0, 11, + 0); +static CV1800_GATE(clk_apb_efuse, osc_parents, + REG_CLK_EN_0, 12, + 0); + +/* WDT */ +static CV1800_GATE(clk_apb_wdt, osc_parents, + REG_CLK_EN_1, 7, + CLK_IS_CRITICAL); + +/* WGN */ +static CV1800_GATE(clk_wgn, osc_parents, + REG_CLK_EN_3, 22, + 0); +static CV1800_GATE(clk_wgn0, osc_parents, + REG_CLK_EN_3, 23, + 0); +static CV1800_GATE(clk_wgn1, osc_parents, + REG_CLK_EN_3, 24, + 0); +static CV1800_GATE(clk_wgn2, osc_parents, + REG_CLK_EN_3, 25, + 0); + +/* KEYSCAN */ +static CV1800_GATE(clk_keyscan, osc_parents, + REG_CLK_EN_3, 26, + 0); + +/* EMMC */ +static CV1800_GATE(clk_axi4_emmc, clk_axi4_bus_parents, + REG_CLK_EN_0, 15, + 0); +static CV1800_BYPASS_MUX(clk_emmc, clk_axi4_parents, + REG_CLK_EN_0, 16, + REG_DIV_CLK_EMMC, 16, 5, 15, CV1800_DIV_FLAG, + REG_DIV_CLK_EMMC, 8, 2, + REG_CLK_BYP_0, 5, + 0); +static CV1800_DIV(clk_emmc_100k, clk_parents_1m, + REG_CLK_EN_0, 17, + REG_DIV_CLK_EMMC_100K, 16, 8, 10, CV1800_DIV_FLAG, + 0); + +/* SD */ +static CV1800_GATE(clk_axi4_sd0, clk_axi4_bus_parents, + REG_CLK_EN_0, 18, + 0); +static CV1800_BYPASS_MUX(clk_sd0, clk_axi4_parents, + REG_CLK_EN_0, 19, + REG_DIV_CLK_SD0, 16, 5, 15, CV1800_DIV_FLAG, + REG_DIV_CLK_SD0, 8, 2, + REG_CLK_BYP_0, 6, + 0); +static CV1800_DIV(clk_sd0_100k, clk_parents_1m, + REG_CLK_EN_0, 20, + REG_DIV_CLK_SD0_100K, 16, 8, 10, CV1800_DIV_FLAG, + 0); +static CV1800_GATE(clk_axi4_sd1, clk_axi4_bus_parents, + REG_CLK_EN_0, 21, + 0); +static CV1800_BYPASS_MUX(clk_sd1, clk_axi4_parents, + REG_CLK_EN_0, 22, + REG_DIV_CLK_SD1, 16, 5, 15, CV1800_DIV_FLAG, + REG_DIV_CLK_SD1, 8, 2, + REG_CLK_BYP_0, 7, + 0); +static CV1800_DIV(clk_sd1_100k, clk_parents_1m, + REG_CLK_EN_0, 23, + REG_DIV_CLK_SD1_100K, 16, 8, 10, CV1800_DIV_FLAG, + 0); + +/* SPI NAND */ +static CV1800_BYPASS_MUX(clk_spi_nand, clk_axi4_parents, + REG_CLK_EN_0, 24, + REG_DIV_CLK_SPI_NAND, 16, 5, 8, CV1800_DIV_FLAG, + REG_DIV_CLK_SPI_NAND, 8, 2, + REG_CLK_BYP_0, 8, + 0); + +/* GPIO */ +static CV1800_DIV(clk_gpio_db, clk_parents_1m, + REG_CLK_EN_0, 31, + REG_DIV_CLK_GPIO_DB, 16, 16, 10, CV1800_DIV_FLAG, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_apb_gpio, clk_axi6_bus_parents, + REG_CLK_EN_0, 29, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_apb_gpio_intr, clk_axi6_bus_parents, + REG_CLK_EN_0, 30, + CLK_IS_CRITICAL); + +/* ETH */ +static CV1800_BYPASS_DIV(clk_eth0_500m, clk_bypass_fpll_parents, + REG_CLK_EN_0, 25, + REG_DIV_CLK_GPIO_DB, 16, 4, 3, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 9, + 0); +static CV1800_GATE(clk_axi4_eth0, clk_axi4_bus_parents, + REG_CLK_EN_0, 26, + 0); +static CV1800_BYPASS_DIV(clk_eth1_500m, clk_bypass_fpll_parents, + REG_CLK_EN_0, 27, + REG_DIV_CLK_GPIO_DB, 16, 4, 3, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 10, + 0); +static CV1800_GATE(clk_axi4_eth1, clk_axi4_bus_parents, + REG_CLK_EN_0, 28, + 0); + +/* SF */ +static CV1800_GATE(clk_ahb_sf, clk_axi4_bus_parents, + REG_CLK_EN_1, 0, + 0); +static CV1800_GATE(clk_ahb_sf1, clk_axi4_bus_parents, + REG_CLK_EN_3, 27, + 0); + +/* AUDSRC */ +static CV1800_ACLK(clk_a24m, clk_mipimpll_parents, + REG_APLL_FRAC_DIV_CTRL, 0, + REG_APLL_FRAC_DIV_CTRL, 3, + REG_APLL_FRAC_DIV_CTRL, 1, + REG_APLL_FRAC_DIV_CTRL, 2, + REG_APLL_FRAC_DIV_M, 0, 22, CV1800_DIV_FLAG, + REG_APLL_FRAC_DIV_N, 0, 22, CV1800_DIV_FLAG, + 24576000, + 0); + +static const struct clk_parent_data clk_aud_parents[] = { + { .index = 0 }, + { .hw = &clk_a0pll.common.hw }, + { .hw = &clk_a24m.common.hw }, +}; + +static CV1800_BYPASS_MUX(clk_audsrc, clk_aud_parents, + REG_CLK_EN_4, 1, + REG_DIV_CLK_AUDSRC, 16, 8, 18, CV1800_DIV_FLAG, + REG_DIV_CLK_AUDSRC, 8, 2, + REG_CLK_BYP_1, 2, + 0); +static CV1800_GATE(clk_apb_audsrc, clk_axi4_bus_parents, + REG_CLK_EN_4, 2, + 0); + +/* SDMA */ +static CV1800_GATE(clk_sdma_axi, clk_axi4_bus_parents, + REG_CLK_EN_1, 1, + 0); +static CV1800_BYPASS_MUX(clk_sdma_aud0, clk_aud_parents, + REG_CLK_EN_1, 2, + REG_DIV_CLK_SDMA_AUD0, 16, 8, 18, CV1800_DIV_FLAG, + REG_DIV_CLK_SDMA_AUD0, 8, 2, + REG_CLK_BYP_0, 11, + 0); +static CV1800_BYPASS_MUX(clk_sdma_aud1, clk_aud_parents, + REG_CLK_EN_1, 3, + REG_DIV_CLK_SDMA_AUD1, 16, 8, 18, CV1800_DIV_FLAG, + REG_DIV_CLK_SDMA_AUD1, 8, 2, + REG_CLK_BYP_0, 12, + 0); +static CV1800_BYPASS_MUX(clk_sdma_aud2, clk_aud_parents, + REG_CLK_EN_1, 3, + REG_DIV_CLK_SDMA_AUD2, 16, 8, 18, CV1800_DIV_FLAG, + REG_DIV_CLK_SDMA_AUD2, 8, 2, + REG_CLK_BYP_0, 13, + 0); +static CV1800_BYPASS_MUX(clk_sdma_aud3, clk_aud_parents, + REG_CLK_EN_1, 3, + REG_DIV_CLK_SDMA_AUD3, 16, 8, 18, CV1800_DIV_FLAG, + REG_DIV_CLK_SDMA_AUD3, 8, 2, + REG_CLK_BYP_0, 14, + 0); + +/* SPI */ +static CV1800_GATE(clk_apb_spi0, clk_axi4_bus_parents, + REG_CLK_EN_1, 9, + 0); +static CV1800_GATE(clk_apb_spi1, clk_axi4_bus_parents, + REG_CLK_EN_1, 10, + 0); +static CV1800_GATE(clk_apb_spi2, clk_axi4_bus_parents, + REG_CLK_EN_1, 11, + 0); +static CV1800_GATE(clk_apb_spi3, clk_axi4_bus_parents, + REG_CLK_EN_1, 12, + 0); +static CV1800_BYPASS_DIV(clk_spi, clk_bypass_fpll_parents, + REG_CLK_EN_3, 6, + REG_DIV_CLK_SPI, 16, 6, 8, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 30, + 0); + +/* UART */ +static CV1800_GATE(clk_uart0, clk_uart_parents, + REG_CLK_EN_1, 14, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_apb_uart0, clk_axi4_bus_parents, + REG_CLK_EN_1, 15, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_uart1, clk_uart_parents, + REG_CLK_EN_1, 16, + 0); +static CV1800_GATE(clk_apb_uart1, clk_axi4_bus_parents, + REG_CLK_EN_1, 17, + 0); +static CV1800_GATE(clk_uart2, clk_uart_parents, + REG_CLK_EN_1, 18, + 0); +static CV1800_GATE(clk_apb_uart2, clk_axi4_bus_parents, + REG_CLK_EN_1, 19, + 0); +static CV1800_GATE(clk_uart3, clk_uart_parents, + REG_CLK_EN_1, 20, + 0); +static CV1800_GATE(clk_apb_uart3, clk_axi4_bus_parents, + REG_CLK_EN_1, 21, + 0); +static CV1800_GATE(clk_uart4, clk_uart_parents, + REG_CLK_EN_1, 22, + 0); +static CV1800_GATE(clk_apb_uart4, clk_axi4_bus_parents, + REG_CLK_EN_1, 23, + 0); + +/* I2S */ +static CV1800_GATE(clk_apb_i2s0, clk_axi4_bus_parents, + REG_CLK_EN_1, 24, + 0); +static CV1800_GATE(clk_apb_i2s1, clk_axi4_bus_parents, + REG_CLK_EN_1, 25, + 0); +static CV1800_GATE(clk_apb_i2s2, clk_axi4_bus_parents, + REG_CLK_EN_1, 26, + 0); +static CV1800_GATE(clk_apb_i2s3, clk_axi4_bus_parents, + REG_CLK_EN_1, 27, + 0); + +/* DEBUG */ +static CV1800_GATE(clk_debug, osc_parents, + REG_CLK_EN_0, 13, + CLK_IS_CRITICAL); +static CV1800_BYPASS_DIV(clk_ap_debug, clk_bypass_fpll_parents, + REG_CLK_EN_4, 5, + REG_DIV_CLK_AP_DEBUG, 16, 4, 5, CV1800_DIV_FLAG, + REG_CLK_BYP_1, 4, + CLK_IS_CRITICAL); + +/* DDR */ +static CV1800_GATE(clk_ddr_axi_reg, clk_axi6_bus_parents, + REG_CLK_EN_0, 7, + CLK_IS_CRITICAL); + +/* I2C */ +static CV1800_GATE(clk_apb_i2c, clk_axi4_bus_parents, + REG_CLK_EN_1, 6, + 0); +static CV1800_BYPASS_DIV(clk_i2c, clk_bypass_axi6_bus_parents, + REG_CLK_EN_3, 7, + REG_DIV_CLK_I2C, 16, 4, 1, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 31, + 0); +static CV1800_GATE(clk_apb_i2c0, clk_axi4_bus_parents, + REG_CLK_EN_3, 17, + 0); +static CV1800_GATE(clk_apb_i2c1, clk_axi4_bus_parents, + REG_CLK_EN_3, 18, + 0); +static CV1800_GATE(clk_apb_i2c2, clk_axi4_bus_parents, + REG_CLK_EN_3, 19, + 0); +static CV1800_GATE(clk_apb_i2c3, clk_axi4_bus_parents, + REG_CLK_EN_3, 20, + 0); +static CV1800_GATE(clk_apb_i2c4, clk_axi4_bus_parents, + REG_CLK_EN_3, 21, + 0); + +/* USB */ +static CV1800_GATE(clk_axi4_usb, clk_axi4_bus_parents, + REG_CLK_EN_1, 28, + 0); +static CV1800_GATE(clk_apb_usb, clk_axi4_bus_parents, + REG_CLK_EN_1, 29, + 0); +static CV1800_BYPASS_FIXED_DIV(clk_usb_125m, clk_bypass_fpll_parents, + REG_CLK_EN_1, 30, + 12, + REG_CLK_BYP_0, 17, + CLK_SET_RATE_PARENT); +static CV1800_FIXED_DIV(clk_usb_33k, clk_parents_1m, + REG_CLK_EN_1, 31, + 3, + 0); +static CV1800_BYPASS_FIXED_DIV(clk_usb_12m, clk_bypass_fpll_parents, + REG_CLK_EN_2, 0, + 125, + REG_CLK_BYP_0, 18, + CLK_SET_RATE_PARENT); + +/* VIP SYS */ +static const struct clk_parent_data clk_vip_sys_parents[] = { + { .index = 0 }, + { .hw = &clk_mipimpll.common.hw }, + { .hw = &clk_cam0pll.common.hw }, + { .hw = &clk_disppll.common.hw }, + { .hw = &clk_fpll.common.hw }, +}; + +static CV1800_BYPASS_DIV(clk_dsi_esc, clk_bypass_axi6_bus_parents, + REG_CLK_EN_2, 3, + REG_DIV_CLK_DSI_ESC, 16, 4, 5, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 21, + 0); +static CV1800_BYPASS_MUX(clk_axi_vip, clk_vip_sys_parents, + REG_CLK_EN_2, 4, + REG_DIV_CLK_AXI_VIP, 16, 4, 3, CV1800_DIV_FLAG, + REG_DIV_CLK_AXI_VIP, 8, 2, + REG_CLK_BYP_0, 22, + 0); + +static const struct clk_parent_data clk_axi_vip_bus_parents[] = { + { .hw = &clk_axi_vip.mux.common.hw }, +}; + +static CV1800_BYPASS_MUX(clk_src_vip_sys_0, clk_vip_sys_parents, + REG_CLK_EN_2, 5, + REG_DIV_CLK_SRC_VIP_SYS_0, 16, 4, 6, CV1800_DIV_FLAG, + REG_DIV_CLK_SRC_VIP_SYS_0, 8, 2, + REG_CLK_BYP_0, 23, + 0); +static CV1800_BYPASS_MUX(clk_src_vip_sys_1, clk_vip_sys_parents, + REG_CLK_EN_2, 6, + REG_DIV_CLK_SRC_VIP_SYS_1, 16, 4, 6, CV1800_DIV_FLAG, + REG_DIV_CLK_SRC_VIP_SYS_1, 8, 2, + REG_CLK_BYP_0, 24, + 0); +static CV1800_BYPASS_MUX(clk_src_vip_sys_2, clk_vip_sys_parents, + REG_CLK_EN_3, 29, + REG_DIV_CLK_SRC_VIP_SYS_2, 16, 4, 2, CV1800_DIV_FLAG, + REG_DIV_CLK_SRC_VIP_SYS_2, 8, 2, + REG_CLK_BYP_1, 1, + 0); +static CV1800_GATE(clk_csi_mac0_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 18, + 0); +static CV1800_GATE(clk_csi_mac1_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 19, + 0); +static CV1800_GATE(clk_isp_top_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 20, + 0); +static CV1800_GATE(clk_img_d_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 21, + 0); +static CV1800_GATE(clk_img_v_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 22, + 0); +static CV1800_GATE(clk_sc_top_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 23, + 0); +static CV1800_GATE(clk_sc_d_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 24, + 0); +static CV1800_GATE(clk_sc_v1_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 25, + 0); +static CV1800_GATE(clk_sc_v2_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 26, + 0); +static CV1800_GATE(clk_sc_v3_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 27, + 0); +static CV1800_GATE(clk_dwa_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 28, + 0); +static CV1800_GATE(clk_bt_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 29, + 0); +static CV1800_GATE(clk_disp_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 30, + 0); +static CV1800_GATE(clk_dsi_mac_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_2, 31, + 0); +static CV1800_GATE(clk_lvds0_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_3, 0, + 0); +static CV1800_GATE(clk_lvds1_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_3, 1, + 0); +static CV1800_GATE(clk_csi0_rx_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_3, 2, + 0); +static CV1800_GATE(clk_csi1_rx_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_3, 3, + 0); +static CV1800_GATE(clk_pad_vi_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_3, 4, + 0); +static CV1800_GATE(clk_pad_vi1_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_3, 30, + 0); +static CV1800_GATE(clk_cfg_reg_vip, clk_axi6_bus_parents, + REG_CLK_EN_3, 31, + 0); +static CV1800_GATE(clk_pad_vi2_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 7, + 0); +static CV1800_GATE(clk_csi_be_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 8, + 0); +static CV1800_GATE(clk_vip_ip0, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 9, + 0); +static CV1800_GATE(clk_vip_ip1, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 10, + 0); +static CV1800_GATE(clk_vip_ip2, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 11, + 0); +static CV1800_GATE(clk_vip_ip3, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 12, + 0); +static CV1800_BYPASS_MUX(clk_src_vip_sys_3, clk_vip_sys_parents, + REG_CLK_EN_4, 15, + REG_DIV_CLK_SRC_VIP_SYS_3, 16, 4, 2, CV1800_DIV_FLAG, + REG_DIV_CLK_SRC_VIP_SYS_3, 8, 2, + REG_CLK_BYP_1, 8, + 0); +static CV1800_BYPASS_MUX(clk_src_vip_sys_4, clk_vip_sys_parents, + REG_CLK_EN_4, 16, + REG_DIV_CLK_SRC_VIP_SYS_4, 16, 4, 3, CV1800_DIV_FLAG, + REG_DIV_CLK_SRC_VIP_SYS_4, 8, 2, + REG_CLK_BYP_1, 9, + 0); +static CV1800_GATE(clk_ive_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 17, + 0); +static CV1800_GATE(clk_raw_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 18, + 0); +static CV1800_GATE(clk_osdc_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 19, + 0); +static CV1800_GATE(clk_csi_mac2_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 20, + 0); +static CV1800_GATE(clk_cam0_vip, clk_axi_vip_bus_parents, + REG_CLK_EN_4, 21, + 0); + +/* CAM OUT */ +static const struct clk_parent_data clk_cam_parents[] = { + { .hw = &clk_cam0pll.common.hw }, + { .hw = &clk_cam0pll_d2.common.hw }, + { .hw = &clk_cam0pll_d3.common.hw }, + { .hw = &clk_mipimpll_d3.common.hw }, +}; + +static CV1800_MUX(clk_cam0, clk_cam_parents, + REG_CLK_EN_2, 16, + REG_CLK_CAM0_SRC_DIV, 16, 6, 0, CV1800_DIV_FLAG, + REG_CLK_CAM0_SRC_DIV, 8, 2, + CLK_IGNORE_UNUSED); +static CV1800_MUX(clk_cam1, clk_cam_parents, + REG_CLK_EN_2, 17, + REG_CLK_CAM1_SRC_DIV, 16, 6, 0, CV1800_DIV_FLAG, + REG_CLK_CAM1_SRC_DIV, 8, 2, + CLK_IGNORE_UNUSED); + +/* VIDEO SUBSYS */ +static const struct clk_parent_data clk_axi_video_codec_parents[] = { + { .index = 0 }, + { .hw = &clk_a0pll.common.hw }, + { .hw = &clk_mipimpll.common.hw }, + { .hw = &clk_cam1pll.common.hw }, + { .hw = &clk_fpll.common.hw }, +}; +static const struct clk_parent_data clk_vc_src0_parents[] = { + { .index = 0 }, + { .hw = &clk_disppll.common.hw }, + { .hw = &clk_mipimpll.common.hw }, + { .hw = &clk_cam1pll.common.hw }, + { .hw = &clk_fpll.common.hw }, +}; +static const struct clk_parent_data clk_vc_src1_parents[] = { + { .index = 0 }, + { .hw = &clk_cam1pll.common.hw }, +}; + +static CV1800_BYPASS_MUX(clk_axi_video_codec, clk_axi_video_codec_parents, + REG_CLK_EN_2, 8, + REG_DIV_CLK_AXI_VIDEO_CODEC, 16, 4, 2, CV1800_DIV_FLAG, + REG_DIV_CLK_AXI_VIDEO_CODEC, 8, 2, + REG_CLK_BYP_0, 26, + 0); + +static const struct clk_parent_data clk_axi_video_codec_bus_parents[] = { + { .hw = &clk_axi_video_codec.mux.common.hw }, +}; + +static CV1800_BYPASS_MUX(clk_vc_src0, clk_vc_src0_parents, + REG_CLK_EN_2, 9, + REG_DIV_CLK_VC_SRC0, 16, 4, 2, CV1800_DIV_FLAG, + REG_DIV_CLK_VC_SRC0, 8, 2, + REG_CLK_BYP_0, 27, + 0); + +static CV1800_GATE(clk_h264c, clk_axi_video_codec_bus_parents, + REG_CLK_EN_2, 10, + 0); +static CV1800_GATE(clk_h265c, clk_axi_video_codec_bus_parents, + REG_CLK_EN_2, 11, + 0); +static CV1800_GATE(clk_jpeg, clk_axi_video_codec_bus_parents, + REG_CLK_EN_2, 12, + CLK_IGNORE_UNUSED); +static CV1800_GATE(clk_apb_jpeg, clk_axi6_bus_parents, + REG_CLK_EN_2, 13, + CLK_IGNORE_UNUSED); +static CV1800_GATE(clk_apb_h264c, clk_axi6_bus_parents, + REG_CLK_EN_2, 14, + 0); +static CV1800_GATE(clk_apb_h265c, clk_axi6_bus_parents, + REG_CLK_EN_2, 15, + 0); +static CV1800_BYPASS_FIXED_DIV(clk_vc_src1, clk_vc_src1_parents, + REG_CLK_EN_3, 28, + 2, + REG_CLK_BYP_1, 0, + CLK_SET_RATE_PARENT); +static CV1800_BYPASS_FIXED_DIV(clk_vc_src2, clk_bypass_fpll_parents, + REG_CLK_EN_4, 3, + 3, + REG_CLK_BYP_1, 3, + CLK_SET_RATE_PARENT); + +/* VC SYS */ +static CV1800_GATE(clk_cfg_reg_vc, clk_axi6_bus_parents, + REG_CLK_EN_4, 0, + CLK_IGNORE_UNUSED); + +/* PWM */ +static CV1800_BYPASS_MUX(clk_pwm_src, clk_axi4_parents, + REG_CLK_EN_4, 4, + REG_DIV_CLK_PWM_SRC_0, 16, 6, 10, CV1800_DIV_FLAG, + REG_DIV_CLK_PWM_SRC_0, 8, 2, + REG_CLK_BYP_0, 15, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_pwm_parents[] = { + { .hw = &clk_pwm_src.mux.common.hw }, +}; + +static CV1800_GATE(clk_pwm, clk_pwm_parents, + REG_CLK_EN_1, 8, + CLK_IS_CRITICAL); + +/* C906 */ +static const struct clk_parent_data clk_c906_0_parents[] = { + { .index = 0 }, + { .hw = &clk_tpll.common.hw }, + { .hw = &clk_a0pll.common.hw }, + { .hw = &clk_mipimpll.common.hw }, + { .hw = &clk_mpll.common.hw }, + { .hw = &clk_fpll.common.hw }, +}; +static const struct clk_parent_data clk_c906_1_parents[] = { + { .index = 0 }, + { .hw = &clk_tpll.common.hw }, + { .hw = &clk_a0pll.common.hw }, + { .hw = &clk_disppll.common.hw }, + { .hw = &clk_mpll.common.hw }, + { .hw = &clk_fpll.common.hw }, +}; + +static const s8 clk_c906_parent2sel[] = { + -1, /* osc */ + 0, /* mux 0: clk_tpll(c906_0), clk_tpll(c906_1) */ + 0, /* mux 0: clk_a0pll(c906_0), clk_a0pll(c906_1) */ + 0, /* mux 0: clk_mipimpll(c906_0), clk_disppll(c906_1) */ + 0, /* mux 0: clk_mpll(c906_0), clk_mpll(c906_1) */ + 1 /* mux 1: clk_fpll(c906_0), clk_fpll(c906_1) */ +}; + +static const u8 clk_c906_sel2parent[2][4] = { + [0] = { + 1, + 2, + 3, + 4 + }, + [1] = { + 5, + 5, + 5, + 5 + }, +}; + +static CV1800_MMUX(clk_c906_0, clk_c906_0_parents, + REG_CLK_EN_4, 13, + REG_DIV_CLK_C906_0_0, 16, 4, 1, CV1800_DIV_FLAG, + REG_DIV_CLK_C906_0_1, 16, 4, 2, CV1800_DIV_FLAG, + REG_DIV_CLK_C906_0_0, 8, 2, + REG_DIV_CLK_C906_0_1, 8, 2, + REG_CLK_BYP_1, 6, + REG_CLK_SEL_0, 23, + clk_c906_parent2sel, + clk_c906_sel2parent[0], clk_c906_sel2parent[1], + CLK_IS_CRITICAL | CLK_GET_RATE_NOCACHE); +static CV1800_MMUX(clk_c906_1, clk_c906_1_parents, + REG_CLK_EN_4, 14, + REG_DIV_CLK_C906_1_0, 16, 4, 2, CV1800_DIV_FLAG, + REG_DIV_CLK_C906_1_1, 16, 4, 3, CV1800_DIV_FLAG, + REG_DIV_CLK_C906_1_0, 8, 2, + REG_DIV_CLK_C906_1_1, 8, 2, + REG_CLK_BYP_1, 7, + REG_CLK_SEL_0, 24, + clk_c906_parent2sel, + clk_c906_sel2parent[0], clk_c906_sel2parent[1], + CLK_IS_CRITICAL | CLK_GET_RATE_NOCACHE); + +/* A53 */ +static CV1800_BYPASS_DIV(clk_cpu_axi0, clk_axi4_parents, + REG_CLK_EN_0, 1, + REG_DIV_CLK_CPU_AXI0, 16, 4, 3, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 1, + CLK_IS_CRITICAL); +static CV1800_BYPASS_DIV(clk_cpu_gic, clk_bypass_fpll_parents, + REG_CLK_EN_0, 2, + REG_DIV_CLK_CPU_GIC, 16, 4, 5, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 2, + CLK_IS_CRITICAL); +static CV1800_GATE(clk_xtal_ap, osc_parents, + REG_CLK_EN_0, 3, + CLK_IS_CRITICAL); + +static const struct clk_parent_data clk_a53_parents[] = { + { .index = 0 }, + { .hw = &clk_tpll.common.hw }, + { .hw = &clk_a0pll.common.hw }, + { .hw = &clk_mipimpll.common.hw }, + { .hw = &clk_mpll.common.hw }, + { .hw = &clk_fpll.common.hw }, +}; + +static const s8 clk_a53_parent2sel[] = { + -1, /* osc */ + 0, /* mux 0: clk_tpll */ + 0, /* mux 0: clk_a0pll */ + 0, /* mux 0: clk_mipimpll */ + 0, /* mux 0: clk_mpll */ + 1 /* mux 1: clk_fpll */ +}; + +static const u8 clk_a53_sel2parent[2][4] = { + [0] = { + 1, + 2, + 3, + 4 + }, + [1] = { + 5, + 5, + 5, + 5 + }, +}; + +/* + * Clock for A53 cpu in the CV18XX/SG200X series. + * For CV180X and CV181X series, this clock is not used, but can not + * be set to bypass mode, or the SoC will hang. + */ +static CV1800_MMUX(clk_a53, clk_a53_parents, + REG_CLK_EN_0, 0, + REG_DIV_CLK_A53_0, 16, 4, 1, CV1800_DIV_FLAG, + REG_DIV_CLK_A53_1, 16, 4, 2, CV1800_DIV_FLAG, + REG_DIV_CLK_A53_0, 8, 2, + REG_DIV_CLK_A53_1, 8, 2, + REG_CLK_BYP_0, 0, + REG_CLK_SEL_0, 0, + clk_a53_parent2sel, + clk_a53_sel2parent[0], clk_a53_sel2parent[1], + CLK_IS_CRITICAL | CLK_GET_RATE_NOCACHE); + +static struct clk_hw_onecell_data cv1800_hw_clks = { + .num = CV1800_CLK_MAX, + .hws = { + [CLK_MPLL] = &clk_mpll.common.hw, + [CLK_TPLL] = &clk_tpll.common.hw, + [CLK_FPLL] = &clk_fpll.common.hw, + [CLK_MIPIMPLL] = &clk_mipimpll.common.hw, + [CLK_A0PLL] = &clk_a0pll.common.hw, + [CLK_DISPPLL] = &clk_disppll.common.hw, + [CLK_CAM0PLL] = &clk_cam0pll.common.hw, + [CLK_CAM1PLL] = &clk_cam1pll.common.hw, + + [CLK_MIPIMPLL_D3] = &clk_mipimpll_d3.common.hw, + [CLK_CAM0PLL_D2] = &clk_cam0pll_d2.common.hw, + [CLK_CAM0PLL_D3] = &clk_cam0pll_d3.common.hw, + + [CLK_TPU] = &clk_tpu.mux.common.hw, + [CLK_TPU_FAB] = &clk_tpu_fab.common.hw, + [CLK_AHB_ROM] = &clk_ahb_rom.common.hw, + [CLK_DDR_AXI_REG] = &clk_ddr_axi_reg.common.hw, + [CLK_RTC_25M] = &clk_rtc_25m.common.hw, + [CLK_SRC_RTC_SYS_0] = &clk_src_rtc_sys_0.div.common.hw, + [CLK_TEMPSEN] = &clk_tempsen.common.hw, + [CLK_SARADC] = &clk_saradc.common.hw, + [CLK_EFUSE] = &clk_efuse.common.hw, + [CLK_APB_EFUSE] = &clk_apb_efuse.common.hw, + [CLK_DEBUG] = &clk_debug.common.hw, + [CLK_AP_DEBUG] = &clk_ap_debug.div.common.hw, + [CLK_XTAL_MISC] = &clk_xtal_misc.common.hw, + [CLK_AXI4_EMMC] = &clk_axi4_emmc.common.hw, + [CLK_EMMC] = &clk_emmc.mux.common.hw, + [CLK_EMMC_100K] = &clk_emmc_100k.common.hw, + [CLK_AXI4_SD0] = &clk_axi4_sd0.common.hw, + [CLK_SD0] = &clk_sd0.mux.common.hw, + [CLK_SD0_100K] = &clk_sd0_100k.common.hw, + [CLK_AXI4_SD1] = &clk_axi4_sd1.common.hw, + [CLK_SD1] = &clk_sd1.mux.common.hw, + [CLK_SD1_100K] = &clk_sd1_100k.common.hw, + [CLK_SPI_NAND] = &clk_spi_nand.mux.common.hw, + [CLK_ETH0_500M] = &clk_eth0_500m.div.common.hw, + [CLK_AXI4_ETH0] = &clk_axi4_eth0.common.hw, + [CLK_ETH1_500M] = &clk_eth1_500m.div.common.hw, + [CLK_AXI4_ETH1] = &clk_axi4_eth1.common.hw, + [CLK_APB_GPIO] = &clk_apb_gpio.common.hw, + [CLK_APB_GPIO_INTR] = &clk_apb_gpio_intr.common.hw, + [CLK_GPIO_DB] = &clk_gpio_db.common.hw, + [CLK_AHB_SF] = &clk_ahb_sf.common.hw, + [CLK_AHB_SF1] = &clk_ahb_sf1.common.hw, + [CLK_A24M] = &clk_a24m.common.hw, + [CLK_AUDSRC] = &clk_audsrc.mux.common.hw, + [CLK_APB_AUDSRC] = &clk_apb_audsrc.common.hw, + [CLK_SDMA_AXI] = &clk_sdma_axi.common.hw, + [CLK_SDMA_AUD0] = &clk_sdma_aud0.mux.common.hw, + [CLK_SDMA_AUD1] = &clk_sdma_aud1.mux.common.hw, + [CLK_SDMA_AUD2] = &clk_sdma_aud2.mux.common.hw, + [CLK_SDMA_AUD3] = &clk_sdma_aud3.mux.common.hw, + [CLK_I2C] = &clk_i2c.div.common.hw, + [CLK_APB_I2C] = &clk_apb_i2c.common.hw, + [CLK_APB_I2C0] = &clk_apb_i2c0.common.hw, + [CLK_APB_I2C1] = &clk_apb_i2c1.common.hw, + [CLK_APB_I2C2] = &clk_apb_i2c2.common.hw, + [CLK_APB_I2C3] = &clk_apb_i2c3.common.hw, + [CLK_APB_I2C4] = &clk_apb_i2c4.common.hw, + [CLK_APB_WDT] = &clk_apb_wdt.common.hw, + [CLK_PWM_SRC] = &clk_pwm_src.mux.common.hw, + [CLK_PWM] = &clk_pwm.common.hw, + [CLK_SPI] = &clk_spi.div.common.hw, + [CLK_APB_SPI0] = &clk_apb_spi0.common.hw, + [CLK_APB_SPI1] = &clk_apb_spi1.common.hw, + [CLK_APB_SPI2] = &clk_apb_spi2.common.hw, + [CLK_APB_SPI3] = &clk_apb_spi3.common.hw, + [CLK_1M] = &clk_1m.common.hw, + [CLK_CAM0_200] = &clk_cam0_200.mux.common.hw, + [CLK_PM] = &clk_pm.common.hw, + [CLK_TIMER0] = &clk_timer0.common.hw, + [CLK_TIMER1] = &clk_timer1.common.hw, + [CLK_TIMER2] = &clk_timer2.common.hw, + [CLK_TIMER3] = &clk_timer3.common.hw, + [CLK_TIMER4] = &clk_timer4.common.hw, + [CLK_TIMER5] = &clk_timer5.common.hw, + [CLK_TIMER6] = &clk_timer6.common.hw, + [CLK_TIMER7] = &clk_timer7.common.hw, + [CLK_UART0] = &clk_uart0.common.hw, + [CLK_APB_UART0] = &clk_apb_uart0.common.hw, + [CLK_UART1] = &clk_uart1.common.hw, + [CLK_APB_UART1] = &clk_apb_uart1.common.hw, + [CLK_UART2] = &clk_uart2.common.hw, + [CLK_APB_UART2] = &clk_apb_uart2.common.hw, + [CLK_UART3] = &clk_uart3.common.hw, + [CLK_APB_UART3] = &clk_apb_uart3.common.hw, + [CLK_UART4] = &clk_uart4.common.hw, + [CLK_APB_UART4] = &clk_apb_uart4.common.hw, + [CLK_APB_I2S0] = &clk_apb_i2s0.common.hw, + [CLK_APB_I2S1] = &clk_apb_i2s1.common.hw, + [CLK_APB_I2S2] = &clk_apb_i2s2.common.hw, + [CLK_APB_I2S3] = &clk_apb_i2s3.common.hw, + [CLK_AXI4_USB] = &clk_axi4_usb.common.hw, + [CLK_APB_USB] = &clk_apb_usb.common.hw, + [CLK_USB_125M] = &clk_usb_125m.div.common.hw, + [CLK_USB_33K] = &clk_usb_33k.common.hw, + [CLK_USB_12M] = &clk_usb_12m.div.common.hw, + [CLK_AXI4] = &clk_axi4.mux.common.hw, + [CLK_AXI6] = &clk_axi6.div.common.hw, + [CLK_DSI_ESC] = &clk_dsi_esc.div.common.hw, + [CLK_AXI_VIP] = &clk_axi_vip.mux.common.hw, + [CLK_SRC_VIP_SYS_0] = &clk_src_vip_sys_0.mux.common.hw, + [CLK_SRC_VIP_SYS_1] = &clk_src_vip_sys_1.mux.common.hw, + [CLK_SRC_VIP_SYS_2] = &clk_src_vip_sys_2.mux.common.hw, + [CLK_SRC_VIP_SYS_3] = &clk_src_vip_sys_3.mux.common.hw, + [CLK_SRC_VIP_SYS_4] = &clk_src_vip_sys_4.mux.common.hw, + [CLK_CSI_BE_VIP] = &clk_csi_be_vip.common.hw, + [CLK_CSI_MAC0_VIP] = &clk_csi_mac0_vip.common.hw, + [CLK_CSI_MAC1_VIP] = &clk_csi_mac1_vip.common.hw, + [CLK_CSI_MAC2_VIP] = &clk_csi_mac2_vip.common.hw, + [CLK_CSI0_RX_VIP] = &clk_csi0_rx_vip.common.hw, + [CLK_CSI1_RX_VIP] = &clk_csi1_rx_vip.common.hw, + [CLK_ISP_TOP_VIP] = &clk_isp_top_vip.common.hw, + [CLK_IMG_D_VIP] = &clk_img_d_vip.common.hw, + [CLK_IMG_V_VIP] = &clk_img_v_vip.common.hw, + [CLK_SC_TOP_VIP] = &clk_sc_top_vip.common.hw, + [CLK_SC_D_VIP] = &clk_sc_d_vip.common.hw, + [CLK_SC_V1_VIP] = &clk_sc_v1_vip.common.hw, + [CLK_SC_V2_VIP] = &clk_sc_v2_vip.common.hw, + [CLK_SC_V3_VIP] = &clk_sc_v3_vip.common.hw, + [CLK_DWA_VIP] = &clk_dwa_vip.common.hw, + [CLK_BT_VIP] = &clk_bt_vip.common.hw, + [CLK_DISP_VIP] = &clk_disp_vip.common.hw, + [CLK_DSI_MAC_VIP] = &clk_dsi_mac_vip.common.hw, + [CLK_LVDS0_VIP] = &clk_lvds0_vip.common.hw, + [CLK_LVDS1_VIP] = &clk_lvds1_vip.common.hw, + [CLK_PAD_VI_VIP] = &clk_pad_vi_vip.common.hw, + [CLK_PAD_VI1_VIP] = &clk_pad_vi1_vip.common.hw, + [CLK_PAD_VI2_VIP] = &clk_pad_vi2_vip.common.hw, + [CLK_CFG_REG_VIP] = &clk_cfg_reg_vip.common.hw, + [CLK_VIP_IP0] = &clk_vip_ip0.common.hw, + [CLK_VIP_IP1] = &clk_vip_ip1.common.hw, + [CLK_VIP_IP2] = &clk_vip_ip2.common.hw, + [CLK_VIP_IP3] = &clk_vip_ip3.common.hw, + [CLK_IVE_VIP] = &clk_ive_vip.common.hw, + [CLK_RAW_VIP] = &clk_raw_vip.common.hw, + [CLK_OSDC_VIP] = &clk_osdc_vip.common.hw, + [CLK_CAM0_VIP] = &clk_cam0_vip.common.hw, + [CLK_AXI_VIDEO_CODEC] = &clk_axi_video_codec.mux.common.hw, + [CLK_VC_SRC0] = &clk_vc_src0.mux.common.hw, + [CLK_VC_SRC1] = &clk_vc_src1.div.common.hw, + [CLK_VC_SRC2] = &clk_vc_src2.div.common.hw, + [CLK_H264C] = &clk_h264c.common.hw, + [CLK_APB_H264C] = &clk_apb_h264c.common.hw, + [CLK_H265C] = &clk_h265c.common.hw, + [CLK_APB_H265C] = &clk_apb_h265c.common.hw, + [CLK_JPEG] = &clk_jpeg.common.hw, + [CLK_APB_JPEG] = &clk_apb_jpeg.common.hw, + [CLK_CAM0] = &clk_cam0.common.hw, + [CLK_CAM1] = &clk_cam1.common.hw, + [CLK_WGN] = &clk_wgn.common.hw, + [CLK_WGN0] = &clk_wgn0.common.hw, + [CLK_WGN1] = &clk_wgn1.common.hw, + [CLK_WGN2] = &clk_wgn2.common.hw, + [CLK_KEYSCAN] = &clk_keyscan.common.hw, + [CLK_CFG_REG_VC] = &clk_cfg_reg_vc.common.hw, + [CLK_C906_0] = &clk_c906_0.common.hw, + [CLK_C906_1] = &clk_c906_1.common.hw, + [CLK_A53] = &clk_a53.common.hw, + [CLK_CPU_AXI0] = &clk_cpu_axi0.div.common.hw, + [CLK_CPU_GIC] = &clk_cpu_gic.div.common.hw, + [CLK_XTAL_AP] = &clk_xtal_ap.common.hw, + }, +}; + +static void cv18xx_clk_disable_auto_pd(void __iomem *base) +{ + static const u16 CV1800_PD_CLK[] = { + REG_MIPIMPLL_CLK_CSR, + REG_A0PLL_CLK_CSR, + REG_DISPPLL_CLK_CSR, + REG_CAM0PLL_CLK_CSR, + REG_CAM1PLL_CLK_CSR, + }; + + u32 val; + int i; + + /* disable auto power down */ + for (i = 0; i < ARRAY_SIZE(CV1800_PD_CLK); i++) { + u32 reg = CV1800_PD_CLK[i]; + + val = readl(base + reg); + val |= GENMASK(12, 9); + val &= ~BIT(8); + writel(val, base + reg); + } +} + +static void cv18xx_clk_disable_a53(void __iomem *base) +{ + u32 val = readl(base + REG_CLK_BYP_0); + + /* Set bypass clock for clk_a53 */ + val |= BIT(0); + + /* Set bypass clock for clk_cpu_axi0 */ + val |= BIT(1); + + /* Set bypass clock for clk_cpu_gic */ + val |= BIT(2); + + writel(val, base + REG_CLK_BYP_0); +} + +static int cv1800_pre_init(struct device *dev, void __iomem *base, + struct cv1800_clk_ctrl *ctrl, + const struct cv1800_clk_desc *desc) +{ + u32 val = readl(base + REG_CLK_EN_2); + + /* disable unsupported clk_disp_src_vip */ + val &= ~BIT(7); + + writel(val, base + REG_CLK_EN_2); + + cv18xx_clk_disable_a53(base); + cv18xx_clk_disable_auto_pd(base); + + return 0; +} + +static const struct cv1800_clk_desc cv1800_desc = { + .clks_data = &cv1800_hw_clks, + .pre_init = cv1800_pre_init, +}; + +static int cv1800_clk_init_ctrl(struct device *dev, void __iomem *reg, + struct cv1800_clk_ctrl *ctrl, + const struct cv1800_clk_desc *desc) +{ + int i, ret; + + ctrl->desc = desc; + spin_lock_init(&ctrl->lock); + + for (i = 0; i < desc->clks_data->num; i++) { + struct clk_hw *hw = desc->clks_data->hws[i]; + struct cv1800_clk_common *common; + const char *name; + + if (!hw) + continue; + + name = hw->init->name; + + common = hw_to_cv1800_clk_common(hw); + common->base = reg; + common->lock = &ctrl->lock; + + ret = devm_clk_hw_register(dev, hw); + if (ret) { + dev_err(dev, "Couldn't register clock %d - %s\n", + i, name); + return ret; + } + } + + return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, + desc->clks_data); +} + +static int cv1800_clk_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + void __iomem *reg; + int ret; + const struct cv1800_clk_desc *desc; + struct cv1800_clk_ctrl *ctrl; + + reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg)) + return PTR_ERR(reg); + + desc = device_get_match_data(dev); + if (!desc) { + dev_err(dev, "no match data for platform\n"); + return -EINVAL; + } + + ctrl = devm_kzalloc(dev, sizeof(*ctrl), GFP_KERNEL); + if (!ctrl) + return -ENOMEM; + + if (desc->pre_init) { + ret = desc->pre_init(dev, reg, ctrl, desc); + if (ret) + return ret; + } + + return cv1800_clk_init_ctrl(dev, reg, ctrl, desc); +} + +static const struct of_device_id cv1800_clk_ids[] = { + { .compatible = "sophgo,cv1800-clk", .data = &cv1800_desc }, + { } +}; +MODULE_DEVICE_TABLE(of, cv1800_clk_ids); + +static struct platform_driver cv1800_clk_driver = { + .probe = cv1800_clk_probe, + .driver = { + .name = "cv1800-clk", + .suppress_bind_attrs = true, + .of_match_table = cv1800_clk_ids, + }, +}; +module_platform_driver(cv1800_clk_driver); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sophgo/clk-cv1800.h b/drivers/clk/sophgo/clk-cv1800.h new file mode 100644 index 0000000000000..1b9c04b5ac3a7 --- /dev/null +++ b/drivers/clk/sophgo/clk-cv1800.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023 Inochi Amaoto + */ + +#ifndef _CLK_SOPHGO_CV1800_H_ +#define _CLK_SOPHGO_CV1800_H_ + +#include + +#define CV1800_CLK_MAX (CLK_XTAL_AP + 1) + +#define REG_PLL_G2_CTRL 0x800 +#define REG_PLL_G2_STATUS 0x804 +#define REG_MIPIMPLL_CSR 0x808 +#define REG_A0PLL_CSR 0x80C +#define REG_DISPPLL_CSR 0x810 +#define REG_CAM0PLL_CSR 0x814 +#define REG_CAM1PLL_CSR 0x818 +#define REG_PLL_G2_SSC_SYN_CTRL 0x840 +#define REG_A0PLL_SSC_SYN_CTRL 0x850 +#define REG_A0PLL_SSC_SYN_SET 0x854 +#define REG_A0PLL_SSC_SYN_SPAN 0x858 +#define REG_A0PLL_SSC_SYN_STEP 0x85C +#define REG_DISPPLL_SSC_SYN_CTRL 0x860 +#define REG_DISPPLL_SSC_SYN_SET 0x864 +#define REG_DISPPLL_SSC_SYN_SPAN 0x868 +#define REG_DISPPLL_SSC_SYN_STEP 0x86C +#define REG_CAM0PLL_SSC_SYN_CTRL 0x870 +#define REG_CAM0PLL_SSC_SYN_SET 0x874 +#define REG_CAM0PLL_SSC_SYN_SPAN 0x878 +#define REG_CAM0PLL_SSC_SYN_STEP 0x87C +#define REG_CAM1PLL_SSC_SYN_CTRL 0x880 +#define REG_CAM1PLL_SSC_SYN_SET 0x884 +#define REG_CAM1PLL_SSC_SYN_SPAN 0x888 +#define REG_CAM1PLL_SSC_SYN_STEP 0x88C +#define REG_APLL_FRAC_DIV_CTRL 0x890 +#define REG_APLL_FRAC_DIV_M 0x894 +#define REG_APLL_FRAC_DIV_N 0x898 +#define REG_MIPIMPLL_CLK_CSR 0x8A0 +#define REG_A0PLL_CLK_CSR 0x8A4 +#define REG_DISPPLL_CLK_CSR 0x8A8 +#define REG_CAM0PLL_CLK_CSR 0x8AC +#define REG_CAM1PLL_CLK_CSR 0x8B0 +#define REG_CLK_CAM0_SRC_DIV 0x8C0 +#define REG_CLK_CAM1_SRC_DIV 0x8C4 + +/* top_pll_g6 */ +#define REG_PLL_G6_CTRL 0x900 +#define REG_PLL_G6_STATUS 0x904 +#define REG_MPLL_CSR 0x908 +#define REG_TPLL_CSR 0x90C +#define REG_FPLL_CSR 0x910 +#define REG_PLL_G6_SSC_SYN_CTRL 0x940 +#define REG_DPLL_SSC_SYN_CTRL 0x950 +#define REG_DPLL_SSC_SYN_SET 0x954 +#define REG_DPLL_SSC_SYN_SPAN 0x958 +#define REG_DPLL_SSC_SYN_STEP 0x95C +#define REG_MPLL_SSC_SYN_CTRL 0x960 +#define REG_MPLL_SSC_SYN_SET 0x964 +#define REG_MPLL_SSC_SYN_SPAN 0x968 +#define REG_MPLL_SSC_SYN_STEP 0x96C +#define REG_TPLL_SSC_SYN_CTRL 0x970 +#define REG_TPLL_SSC_SYN_SET 0x974 +#define REG_TPLL_SSC_SYN_SPAN 0x978 +#define REG_TPLL_SSC_SYN_STEP 0x97C + +/* clkgen */ +#define REG_CLK_EN_0 0x000 +#define REG_CLK_EN_1 0x004 +#define REG_CLK_EN_2 0x008 +#define REG_CLK_EN_3 0x00C +#define REG_CLK_EN_4 0x010 +#define REG_CLK_SEL_0 0x020 +#define REG_CLK_BYP_0 0x030 +#define REG_CLK_BYP_1 0x034 + +#define REG_DIV_CLK_A53_0 0x040 +#define REG_DIV_CLK_A53_1 0x044 +#define REG_DIV_CLK_CPU_AXI0 0x048 +#define REG_DIV_CLK_CPU_GIC 0x050 +#define REG_DIV_CLK_TPU 0x054 +#define REG_DIV_CLK_EMMC 0x064 +#define REG_DIV_CLK_EMMC_100K 0x06C +#define REG_DIV_CLK_SD0 0x070 +#define REG_DIV_CLK_SD0_100K 0x078 +#define REG_DIV_CLK_SD1 0x07C +#define REG_DIV_CLK_SD1_100K 0x084 +#define REG_DIV_CLK_SPI_NAND 0x088 +#define REG_DIV_CLK_ETH0_500M 0x08C +#define REG_DIV_CLK_ETH1_500M 0x090 +#define REG_DIV_CLK_GPIO_DB 0x094 +#define REG_DIV_CLK_SDMA_AUD0 0x098 +#define REG_DIV_CLK_SDMA_AUD1 0x09C +#define REG_DIV_CLK_SDMA_AUD2 0x0A0 +#define REG_DIV_CLK_SDMA_AUD3 0x0A4 +#define REG_DIV_CLK_CAM0_200 0x0A8 +#define REG_DIV_CLK_AXI4 0x0B8 +#define REG_DIV_CLK_AXI6 0x0BC +#define REG_DIV_CLK_DSI_ESC 0x0C4 +#define REG_DIV_CLK_AXI_VIP 0x0C8 +#define REG_DIV_CLK_SRC_VIP_SYS_0 0x0D0 +#define REG_DIV_CLK_SRC_VIP_SYS_1 0x0D8 +#define REG_DIV_CLK_DISP_SRC_VIP 0x0E0 +#define REG_DIV_CLK_AXI_VIDEO_CODEC 0x0E4 +#define REG_DIV_CLK_VC_SRC0 0x0EC +#define REG_DIV_CLK_1M 0x0FC +#define REG_DIV_CLK_SPI 0x100 +#define REG_DIV_CLK_I2C 0x104 +#define REG_DIV_CLK_SRC_VIP_SYS_2 0x110 +#define REG_DIV_CLK_AUDSRC 0x118 +#define REG_DIV_CLK_PWM_SRC_0 0x120 +#define REG_DIV_CLK_AP_DEBUG 0x128 +#define REG_DIV_CLK_RTCSYS_SRC_0 0x12C +#define REG_DIV_CLK_C906_0_0 0x130 +#define REG_DIV_CLK_C906_0_1 0x134 +#define REG_DIV_CLK_C906_1_0 0x138 +#define REG_DIV_CLK_C906_1_1 0x13C +#define REG_DIV_CLK_SRC_VIP_SYS_3 0x140 +#define REG_DIV_CLK_SRC_VIP_SYS_4 0x144 + +#endif /* _CLK_SOPHGO_CV1800_H_ */ diff --git a/drivers/clk/sophgo/clk-cv18xx-common.c b/drivers/clk/sophgo/clk-cv18xx-common.c new file mode 100644 index 0000000000000..cbcdd88f0e23f --- /dev/null +++ b/drivers/clk/sophgo/clk-cv18xx-common.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023 Inochi Amaoto + */ + +#include +#include +#include +#include + +#include "clk-cv18xx-common.h" + +int cv1800_clk_setbit(struct cv1800_clk_common *common, + struct cv1800_clk_regbit *field) +{ + u32 mask = BIT(field->shift); + u32 value; + unsigned long flags; + + spin_lock_irqsave(common->lock, flags); + + value = readl(common->base + field->reg); + writel(value | mask, common->base + field->reg); + + spin_unlock_irqrestore(common->lock, flags); + + return 0; +} + +int cv1800_clk_clearbit(struct cv1800_clk_common *common, + struct cv1800_clk_regbit *field) +{ + u32 mask = BIT(field->shift); + u32 value; + unsigned long flags; + + spin_lock_irqsave(common->lock, flags); + + value = readl(common->base + field->reg); + writel(value & ~mask, common->base + field->reg); + + spin_unlock_irqrestore(common->lock, flags); + + return 0; +} + +int cv1800_clk_checkbit(struct cv1800_clk_common *common, + struct cv1800_clk_regbit *field) +{ + return readl(common->base + field->reg) & BIT(field->shift); +} + +#define PLL_LOCK_TIMEOUT_US (200 * 1000) + +void cv1800_clk_wait_for_lock(struct cv1800_clk_common *common, + u32 reg, u32 lock) +{ + void __iomem *addr = common->base + reg; + u32 regval; + + if (!lock) + return; + + WARN_ON(readl_relaxed_poll_timeout(addr, regval, regval & lock, + 100, PLL_LOCK_TIMEOUT_US)); +} diff --git a/drivers/clk/sophgo/clk-cv18xx-common.h b/drivers/clk/sophgo/clk-cv18xx-common.h new file mode 100644 index 0000000000000..2bfda02b20647 --- /dev/null +++ b/drivers/clk/sophgo/clk-cv18xx-common.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023 Inochi Amaoto + */ + +#ifndef _CLK_SOPHGO_CV18XX_IP_H_ +#define _CLK_SOPHGO_CV18XX_IP_H_ + +#include +#include +#include + +struct cv1800_clk_common { + void __iomem *base; + spinlock_t *lock; + struct clk_hw hw; + unsigned long features; +}; + +#define CV1800_CLK_COMMON(_name, _parents, _op, _flags) \ + { \ + .hw.init = CLK_HW_INIT_PARENTS_DATA(_name, _parents, \ + _op, _flags), \ + } + +static inline struct cv1800_clk_common * +hw_to_cv1800_clk_common(struct clk_hw *hw) +{ + return container_of(hw, struct cv1800_clk_common, hw); +} + +struct cv1800_clk_regbit { + u16 reg; + s8 shift; +}; + +struct cv1800_clk_regfield { + u16 reg; + u8 shift; + u8 width; + s16 initval; + unsigned long flags; +}; + +#define CV1800_CLK_BIT(_reg, _shift) \ + { \ + .reg = _reg, \ + .shift = _shift, \ + } + +#define CV1800_CLK_REG(_reg, _shift, _width, _initval, _flags) \ + { \ + .reg = _reg, \ + .shift = _shift, \ + .width = _width, \ + .initval = _initval, \ + .flags = _flags, \ + } + +#define cv1800_clk_regfield_genmask(_reg) \ + GENMASK((_reg)->shift + (_reg)->width - 1, (_reg)->shift) +#define cv1800_clk_regfield_get(_val, _reg) \ + (((_val) >> (_reg)->shift) & GENMASK((_reg)->width - 1, 0)) +#define cv1800_clk_regfield_set(_val, _new, _reg) \ + (((_val) & ~cv1800_clk_regfield_genmask((_reg))) | \ + (((_new) & GENMASK((_reg)->width - 1, 0)) << (_reg)->shift)) + +#define _CV1800_SET_FIELD(_reg, _val, _field) \ + (((_reg) & ~(_field)) | FIELD_PREP((_field), (_val))) + +int cv1800_clk_setbit(struct cv1800_clk_common *common, + struct cv1800_clk_regbit *field); +int cv1800_clk_clearbit(struct cv1800_clk_common *common, + struct cv1800_clk_regbit *field); +int cv1800_clk_checkbit(struct cv1800_clk_common *common, + struct cv1800_clk_regbit *field); + +void cv1800_clk_wait_for_lock(struct cv1800_clk_common *common, + u32 reg, u32 lock); + +#endif // _CLK_SOPHGO_CV18XX_IP_H_ diff --git a/drivers/clk/sophgo/clk-cv18xx-ip.c b/drivers/clk/sophgo/clk-cv18xx-ip.c new file mode 100644 index 0000000000000..805f561725ae1 --- /dev/null +++ b/drivers/clk/sophgo/clk-cv18xx-ip.c @@ -0,0 +1,887 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023 Inochi Amaoto + */ + +#include +#include +#include +#include + +#include "clk-cv18xx-ip.h" + +/* GATE */ +static inline struct cv1800_clk_gate *hw_to_cv1800_clk_gate(struct clk_hw *hw) +{ + struct cv1800_clk_common *common = hw_to_cv1800_clk_common(hw); + + return container_of(common, struct cv1800_clk_gate, common); +} + +static int gate_enable(struct clk_hw *hw) +{ + struct cv1800_clk_gate *gate = hw_to_cv1800_clk_gate(hw); + + return cv1800_clk_setbit(&gate->common, &gate->gate); +} + +static void gate_disable(struct clk_hw *hw) +{ + struct cv1800_clk_gate *gate = hw_to_cv1800_clk_gate(hw); + + cv1800_clk_clearbit(&gate->common, &gate->gate); +} + +static int gate_is_enabled(struct clk_hw *hw) +{ + struct cv1800_clk_gate *gate = hw_to_cv1800_clk_gate(hw); + + return cv1800_clk_checkbit(&gate->common, &gate->gate); +} + +static unsigned long gate_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + return parent_rate; +} + +static long gate_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + return *parent_rate; +} + +static int gate_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + return 0; +} + +const struct clk_ops cv1800_clk_gate_ops = { + .disable = gate_disable, + .enable = gate_enable, + .is_enabled = gate_is_enabled, + + .recalc_rate = gate_recalc_rate, + .round_rate = gate_round_rate, + .set_rate = gate_set_rate, +}; + +/* DIV */ +#define _DIV_EN_CLK_DIV_FACTOR_FIELD BIT(3) + +#define DIV_GET_EN_CLK_DIV_FACTOR(_reg) \ + FIELD_GET(_DIV_EN_CLK_DIV_FACTOR_FIELD, _reg) + +#define DIV_SET_EN_DIV_FACTOR(_reg) \ + _CV1800_SET_FIELD(_reg, 1, _DIV_EN_CLK_DIV_FACTOR_FIELD) + +static inline struct cv1800_clk_div *hw_to_cv1800_clk_div(struct clk_hw *hw) +{ + struct cv1800_clk_common *common = hw_to_cv1800_clk_common(hw); + + return container_of(common, struct cv1800_clk_div, common); +} + +static int div_enable(struct clk_hw *hw) +{ + struct cv1800_clk_div *div = hw_to_cv1800_clk_div(hw); + + return cv1800_clk_setbit(&div->common, &div->gate); +} + +static void div_disable(struct clk_hw *hw) +{ + struct cv1800_clk_div *div = hw_to_cv1800_clk_div(hw); + + cv1800_clk_clearbit(&div->common, &div->gate); +} + +static int div_is_enabled(struct clk_hw *hw) +{ + struct cv1800_clk_div *div = hw_to_cv1800_clk_div(hw); + + return cv1800_clk_checkbit(&div->common, &div->gate); +} + +static int div_helper_set_rate(struct cv1800_clk_common *common, + struct cv1800_clk_regfield *div, + unsigned long val) +{ + unsigned long flags; + u32 reg; + + if (div->width == 0) + return 0; + + spin_lock_irqsave(common->lock, flags); + + reg = readl(common->base + div->reg); + reg = cv1800_clk_regfield_set(reg, val, div); + if (div->initval > 0) + reg = DIV_SET_EN_DIV_FACTOR(reg); + + writel(reg, common->base + div->reg); + + spin_unlock_irqrestore(common->lock, flags); + + return 0; +} + +static u32 div_helper_get_clockdiv(struct cv1800_clk_common *common, + struct cv1800_clk_regfield *div) +{ + u32 clockdiv = 1; + u32 reg; + + if (!div || div->initval < 0 || (div->width == 0 && div->initval <= 0)) + return 1; + + if (div->width == 0 && div->initval > 0) + return div->initval; + + reg = readl(common->base + div->reg); + + if (div->initval == 0 || DIV_GET_EN_CLK_DIV_FACTOR(reg)) + clockdiv = cv1800_clk_regfield_get(reg, div); + else if (div->initval > 0) + clockdiv = div->initval; + + return clockdiv; +} + +static u32 div_helper_round_rate(struct cv1800_clk_regfield *div, + struct clk_hw *hw, struct clk_hw *parent, + unsigned long rate, unsigned long *prate) +{ + if (div->width == 0) { + if (div->initval <= 0) + return DIV_ROUND_UP_ULL(*prate, 1); + else + return DIV_ROUND_UP_ULL(*prate, div->initval); + } + + return divider_round_rate_parent(hw, parent, rate, prate, NULL, + div->width, div->flags); +} + +static long div_round_rate(struct clk_hw *parent, unsigned long *parent_rate, + unsigned long rate, int id, void *data) +{ + struct cv1800_clk_div *div = data; + + return div_helper_round_rate(&div->div, &div->common.hw, parent, + rate, parent_rate); +} + +static bool div_is_better_rate(struct cv1800_clk_common *common, + unsigned long target, unsigned long now, + unsigned long best) +{ + if (common->features & CLK_DIVIDER_ROUND_CLOSEST) + return abs_diff(target, now) < abs_diff(target, best); + + return now <= target && now > best; +} + +static int mux_helper_determine_rate(struct cv1800_clk_common *common, + struct clk_rate_request *req, + long (*round)(struct clk_hw *, + unsigned long *, + unsigned long, + int, + void *), + void *data) +{ + unsigned long best_parent_rate = 0, best_rate = 0; + struct clk_hw *best_parent, *hw = &common->hw; + unsigned int i; + + if (clk_hw_get_flags(hw) & CLK_SET_RATE_NO_REPARENT) { + unsigned long adj_parent_rate; + + best_parent = clk_hw_get_parent(hw); + best_parent_rate = clk_hw_get_rate(best_parent); + + best_rate = round(best_parent, &adj_parent_rate, + req->rate, -1, data); + + goto find; + } + + for (i = 0; i < clk_hw_get_num_parents(hw); i++) { + unsigned long tmp_rate, parent_rate; + struct clk_hw *parent; + + parent = clk_hw_get_parent_by_index(hw, i); + if (!parent) + continue; + + parent_rate = clk_hw_get_rate(parent); + + tmp_rate = round(parent, &parent_rate, req->rate, i, data); + + if (tmp_rate == req->rate) { + best_parent = parent; + best_parent_rate = parent_rate; + best_rate = tmp_rate; + goto find; + } + + if (div_is_better_rate(common, req->rate, + tmp_rate, best_rate)) { + best_parent = parent; + best_parent_rate = parent_rate; + best_rate = tmp_rate; + } + } + + if (best_rate == 0) + return -EINVAL; + +find: + req->best_parent_hw = best_parent; + req->best_parent_rate = best_parent_rate; + req->rate = best_rate; + return 0; +} + +static int div_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct cv1800_clk_div *div = hw_to_cv1800_clk_div(hw); + + return mux_helper_determine_rate(&div->common, req, + div_round_rate, div); +} + +static unsigned long div_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_div *div = hw_to_cv1800_clk_div(hw); + unsigned long val; + + val = div_helper_get_clockdiv(&div->common, &div->div); + if (val == 0) + return 0; + + return divider_recalc_rate(hw, parent_rate, val, NULL, + div->div.flags, div->div.width); +} + +static int div_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct cv1800_clk_div *div = hw_to_cv1800_clk_div(hw); + unsigned long val; + + val = divider_get_val(rate, parent_rate, NULL, + div->div.width, div->div.flags); + + return div_helper_set_rate(&div->common, &div->div, val); +} + +const struct clk_ops cv1800_clk_div_ops = { + .disable = div_disable, + .enable = div_enable, + .is_enabled = div_is_enabled, + + .determine_rate = div_determine_rate, + .recalc_rate = div_recalc_rate, + .set_rate = div_set_rate, +}; + +static inline struct cv1800_clk_bypass_div * +hw_to_cv1800_clk_bypass_div(struct clk_hw *hw) +{ + struct cv1800_clk_div *div = hw_to_cv1800_clk_div(hw); + + return container_of(div, struct cv1800_clk_bypass_div, div); +} + +static long bypass_div_round_rate(struct clk_hw *parent, + unsigned long *parent_rate, + unsigned long rate, int id, void *data) +{ + struct cv1800_clk_bypass_div *div = data; + + if (id == -1) { + if (cv1800_clk_checkbit(&div->div.common, &div->bypass)) + return *parent_rate; + else + return div_round_rate(parent, parent_rate, rate, + -1, &div->div); + } + + if (id == 0) + return *parent_rate; + + return div_round_rate(parent, parent_rate, rate, id - 1, &div->div); +} + +static int bypass_div_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct cv1800_clk_bypass_div *div = hw_to_cv1800_clk_bypass_div(hw); + + return mux_helper_determine_rate(&div->div.common, req, + bypass_div_round_rate, div); +} + +static unsigned long bypass_div_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_bypass_div *div = hw_to_cv1800_clk_bypass_div(hw); + + if (cv1800_clk_checkbit(&div->div.common, &div->bypass)) + return parent_rate; + + return div_recalc_rate(hw, parent_rate); +} + +static int bypass_div_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct cv1800_clk_bypass_div *div = hw_to_cv1800_clk_bypass_div(hw); + + if (cv1800_clk_checkbit(&div->div.common, &div->bypass)) + return 0; + + return div_set_rate(hw, rate, parent_rate); +} + +static u8 bypass_div_get_parent(struct clk_hw *hw) +{ + struct cv1800_clk_bypass_div *div = hw_to_cv1800_clk_bypass_div(hw); + + if (cv1800_clk_checkbit(&div->div.common, &div->bypass)) + return 0; + + return 1; +} + +static int bypass_div_set_parent(struct clk_hw *hw, u8 index) +{ + struct cv1800_clk_bypass_div *div = hw_to_cv1800_clk_bypass_div(hw); + + if (index) + return cv1800_clk_clearbit(&div->div.common, &div->bypass); + + return cv1800_clk_setbit(&div->div.common, &div->bypass); +} + +const struct clk_ops cv1800_clk_bypass_div_ops = { + .disable = div_disable, + .enable = div_enable, + .is_enabled = div_is_enabled, + + .determine_rate = bypass_div_determine_rate, + .recalc_rate = bypass_div_recalc_rate, + .set_rate = bypass_div_set_rate, + + .set_parent = bypass_div_set_parent, + .get_parent = bypass_div_get_parent, +}; + +/* MUX */ +static inline struct cv1800_clk_mux *hw_to_cv1800_clk_mux(struct clk_hw *hw) +{ + struct cv1800_clk_common *common = hw_to_cv1800_clk_common(hw); + + return container_of(common, struct cv1800_clk_mux, common); +} + +static int mux_enable(struct clk_hw *hw) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + + return cv1800_clk_setbit(&mux->common, &mux->gate); +} + +static void mux_disable(struct clk_hw *hw) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + + cv1800_clk_clearbit(&mux->common, &mux->gate); +} + +static int mux_is_enabled(struct clk_hw *hw) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + + return cv1800_clk_checkbit(&mux->common, &mux->gate); +} + +static long mux_round_rate(struct clk_hw *parent, unsigned long *parent_rate, + unsigned long rate, int id, void *data) +{ + struct cv1800_clk_mux *mux = data; + + return div_helper_round_rate(&mux->div, &mux->common.hw, parent, + rate, parent_rate); +} + +static int mux_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + + return mux_helper_determine_rate(&mux->common, req, + mux_round_rate, mux); +} + +static unsigned long mux_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + unsigned long val; + + val = div_helper_get_clockdiv(&mux->common, &mux->div); + if (val == 0) + return 0; + + return divider_recalc_rate(hw, parent_rate, val, NULL, + mux->div.flags, mux->div.width); +} + +static int mux_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + unsigned long val; + + val = divider_get_val(rate, parent_rate, NULL, + mux->div.width, mux->div.flags); + + return div_helper_set_rate(&mux->common, &mux->div, val); +} + +static u8 mux_get_parent(struct clk_hw *hw) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + u32 reg = readl(mux->common.base + mux->mux.reg); + + return cv1800_clk_regfield_get(reg, &mux->mux); +} + +static int _mux_set_parent(struct cv1800_clk_mux *mux, u8 index) +{ + u32 reg; + + reg = readl(mux->common.base + mux->mux.reg); + reg = cv1800_clk_regfield_set(reg, index, &mux->mux); + writel(reg, mux->common.base + mux->mux.reg); + + return 0; +} + +static int mux_set_parent(struct clk_hw *hw, u8 index) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + unsigned long flags; + + spin_lock_irqsave(mux->common.lock, flags); + + _mux_set_parent(mux, index); + + spin_unlock_irqrestore(mux->common.lock, flags); + + return 0; +} + +const struct clk_ops cv1800_clk_mux_ops = { + .disable = mux_disable, + .enable = mux_enable, + .is_enabled = mux_is_enabled, + + .determine_rate = mux_determine_rate, + .recalc_rate = mux_recalc_rate, + .set_rate = mux_set_rate, + + .set_parent = mux_set_parent, + .get_parent = mux_get_parent, +}; + +static inline struct cv1800_clk_bypass_mux * +hw_to_cv1800_clk_bypass_mux(struct clk_hw *hw) +{ + struct cv1800_clk_mux *mux = hw_to_cv1800_clk_mux(hw); + + return container_of(mux, struct cv1800_clk_bypass_mux, mux); +} + +static long bypass_mux_round_rate(struct clk_hw *parent, + unsigned long *parent_rate, + unsigned long rate, int id, void *data) +{ + struct cv1800_clk_bypass_mux *mux = data; + + if (id == -1) { + if (cv1800_clk_checkbit(&mux->mux.common, &mux->bypass)) + return *parent_rate; + else + return mux_round_rate(parent, parent_rate, rate, + -1, &mux->mux); + } + + if (id == 0) + return *parent_rate; + + return mux_round_rate(parent, parent_rate, rate, id - 1, &mux->mux); +} + +static int bypass_mux_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct cv1800_clk_bypass_mux *mux = hw_to_cv1800_clk_bypass_mux(hw); + + return mux_helper_determine_rate(&mux->mux.common, req, + bypass_mux_round_rate, mux); +} + +static unsigned long bypass_mux_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_bypass_mux *mux = hw_to_cv1800_clk_bypass_mux(hw); + + if (cv1800_clk_checkbit(&mux->mux.common, &mux->bypass)) + return parent_rate; + + return mux_recalc_rate(hw, parent_rate); +} + +static int bypass_mux_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct cv1800_clk_bypass_mux *mux = hw_to_cv1800_clk_bypass_mux(hw); + + if (cv1800_clk_checkbit(&mux->mux.common, &mux->bypass)) + return 0; + + return mux_set_rate(hw, rate, parent_rate); +} + +static u8 bypass_mux_get_parent(struct clk_hw *hw) +{ + struct cv1800_clk_bypass_mux *mux = hw_to_cv1800_clk_bypass_mux(hw); + + if (cv1800_clk_checkbit(&mux->mux.common, &mux->bypass)) + return 0; + + return mux_get_parent(hw) + 1; +} + +static int bypass_mux_set_parent(struct clk_hw *hw, u8 index) +{ + struct cv1800_clk_bypass_mux *mux = hw_to_cv1800_clk_bypass_mux(hw); + + if (index == 0) + return cv1800_clk_setbit(&mux->mux.common, &mux->bypass); + + return cv1800_clk_clearbit(&mux->mux.common, &mux->bypass); +} + +const struct clk_ops cv1800_clk_bypass_mux_ops = { + .disable = mux_disable, + .enable = mux_enable, + .is_enabled = mux_is_enabled, + + .determine_rate = bypass_mux_determine_rate, + .recalc_rate = bypass_mux_recalc_rate, + .set_rate = bypass_mux_set_rate, + + .set_parent = bypass_mux_set_parent, + .get_parent = bypass_mux_get_parent, +}; + +/* MMUX */ +static inline struct cv1800_clk_mmux *hw_to_cv1800_clk_mmux(struct clk_hw *hw) +{ + struct cv1800_clk_common *common = hw_to_cv1800_clk_common(hw); + + return container_of(common, struct cv1800_clk_mmux, common); +} + +static u8 mmux_get_parent_id(struct cv1800_clk_mmux *mmux) +{ + struct clk_hw *hw = &mmux->common.hw; + struct clk_hw *parent = clk_hw_get_parent(hw); + unsigned int i; + + for (i = 0; i < clk_hw_get_num_parents(hw); i++) { + if (parent == clk_hw_get_parent_by_index(hw, i)) + return i; + } + + unreachable(); +} + +static int mmux_enable(struct clk_hw *hw) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + + return cv1800_clk_setbit(&mmux->common, &mmux->gate); +} + +static void mmux_disable(struct clk_hw *hw) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + + cv1800_clk_clearbit(&mmux->common, &mmux->gate); +} + +static int mmux_is_enabled(struct clk_hw *hw) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + + return cv1800_clk_checkbit(&mmux->common, &mmux->gate); +} + +static long mmux_round_rate(struct clk_hw *parent, unsigned long *parent_rate, + unsigned long rate, int id, void *data) +{ + struct cv1800_clk_mmux *mmux = data; + s8 div_id; + + if (id == -1) { + if (cv1800_clk_checkbit(&mmux->common, &mmux->bypass)) + return *parent_rate; + + id = mmux_get_parent_id(mmux); + } + + div_id = mmux->parent2sel[id]; + + if (div_id < 0) + return *parent_rate; + + return div_helper_round_rate(&mmux->div[div_id], + &mmux->common.hw, parent, + rate, parent_rate); +} + +static int mmux_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + + return mux_helper_determine_rate(&mmux->common, req, + mmux_round_rate, mmux); +} + +static unsigned long mmux_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + unsigned long val; + struct cv1800_clk_regfield *div; + + if (cv1800_clk_checkbit(&mmux->common, &mmux->bypass)) + return parent_rate; + + if (cv1800_clk_checkbit(&mmux->common, &mmux->clk_sel)) + div = &mmux->div[0]; + else + div = &mmux->div[1]; + + val = div_helper_get_clockdiv(&mmux->common, div); + if (val == 0) + return 0; + + return divider_recalc_rate(hw, parent_rate, val, NULL, + div->flags, div->width); +} + +static int mmux_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + struct cv1800_clk_regfield *div; + unsigned long val; + + if (cv1800_clk_checkbit(&mmux->common, &mmux->bypass)) + return parent_rate; + + if (cv1800_clk_checkbit(&mmux->common, &mmux->clk_sel)) + div = &mmux->div[0]; + else + div = &mmux->div[1]; + + val = divider_get_val(rate, parent_rate, NULL, + div->width, div->flags); + + return div_helper_set_rate(&mmux->common, div, val); +} + +static u8 mmux_get_parent(struct clk_hw *hw) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + struct cv1800_clk_regfield *mux; + u32 reg; + s8 clk_sel; + + if (cv1800_clk_checkbit(&mmux->common, &mmux->bypass)) + return 0; + + if (cv1800_clk_checkbit(&mmux->common, &mmux->clk_sel)) + clk_sel = 0; + else + clk_sel = 1; + mux = &mmux->mux[clk_sel]; + + reg = readl(mmux->common.base + mux->reg); + + return mmux->sel2parent[clk_sel][cv1800_clk_regfield_get(reg, mux)]; +} + +static int mmux_set_parent(struct clk_hw *hw, u8 index) +{ + struct cv1800_clk_mmux *mmux = hw_to_cv1800_clk_mmux(hw); + struct cv1800_clk_regfield *mux; + unsigned long flags; + u32 reg; + s8 clk_sel = mmux->parent2sel[index]; + + if (index == 0 || clk_sel == -1) { + cv1800_clk_setbit(&mmux->common, &mmux->bypass); + goto release; + } + + cv1800_clk_clearbit(&mmux->common, &mmux->bypass); + + if (clk_sel) + cv1800_clk_clearbit(&mmux->common, &mmux->clk_sel); + else + cv1800_clk_setbit(&mmux->common, &mmux->clk_sel); + + spin_lock_irqsave(mmux->common.lock, flags); + + mux = &mmux->mux[clk_sel]; + reg = readl(mmux->common.base + mux->reg); + reg = cv1800_clk_regfield_set(reg, index, mux); + + writel(reg, mmux->common.base + mux->reg); + + spin_unlock_irqrestore(mmux->common.lock, flags); + +release: + return 0; +} + +const struct clk_ops cv1800_clk_mmux_ops = { + .disable = mmux_disable, + .enable = mmux_enable, + .is_enabled = mmux_is_enabled, + + .determine_rate = mmux_determine_rate, + .recalc_rate = mmux_recalc_rate, + .set_rate = mmux_set_rate, + + .set_parent = mmux_set_parent, + .get_parent = mmux_get_parent, +}; + +/* AUDIO CLK */ +static inline struct cv1800_clk_audio * +hw_to_cv1800_clk_audio(struct clk_hw *hw) +{ + struct cv1800_clk_common *common = hw_to_cv1800_clk_common(hw); + + return container_of(common, struct cv1800_clk_audio, common); +} + +static int aclk_enable(struct clk_hw *hw) +{ + struct cv1800_clk_audio *aclk = hw_to_cv1800_clk_audio(hw); + + cv1800_clk_setbit(&aclk->common, &aclk->src_en); + return cv1800_clk_setbit(&aclk->common, &aclk->output_en); +} + +static void aclk_disable(struct clk_hw *hw) +{ + struct cv1800_clk_audio *aclk = hw_to_cv1800_clk_audio(hw); + + cv1800_clk_clearbit(&aclk->common, &aclk->output_en); + cv1800_clk_clearbit(&aclk->common, &aclk->src_en); +} + +static int aclk_is_enabled(struct clk_hw *hw) +{ + struct cv1800_clk_audio *aclk = hw_to_cv1800_clk_audio(hw); + + return cv1800_clk_checkbit(&aclk->common, &aclk->output_en); +} + +static int aclk_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct cv1800_clk_audio *aclk = hw_to_cv1800_clk_audio(hw); + + req->rate = aclk->target_rate; + + return 0; +} + +static unsigned long aclk_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_audio *aclk = hw_to_cv1800_clk_audio(hw); + u64 rate = parent_rate; + u64 factor = 2; + u32 regval; + + if (!cv1800_clk_checkbit(&aclk->common, &aclk->div_en)) + return 0; + + regval = readl(aclk->common.base + aclk->m.reg); + factor *= cv1800_clk_regfield_get(regval, &aclk->m); + + regval = readl(aclk->common.base + aclk->n.reg); + rate *= cv1800_clk_regfield_get(regval, &aclk->n); + + return DIV64_U64_ROUND_UP(rate, factor); +} + +static void aclk_determine_mn(unsigned long parent_rate, unsigned long rate, + u32 *m, u32 *n) +{ + u32 tm = parent_rate / 2; + u32 tn = rate; + u32 tcommon = gcd(tm, tn); + *m = tm / tcommon; + *n = tn / tcommon; +} + +static int aclk_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct cv1800_clk_audio *aclk = hw_to_cv1800_clk_audio(hw); + unsigned long flags; + u32 m, n; + + aclk_determine_mn(parent_rate, rate, + &m, &n); + + spin_lock_irqsave(aclk->common.lock, flags); + + writel(m, aclk->common.base + aclk->m.reg); + writel(n, aclk->common.base + aclk->n.reg); + + cv1800_clk_setbit(&aclk->common, &aclk->div_en); + cv1800_clk_setbit(&aclk->common, &aclk->div_up); + + spin_unlock_irqrestore(aclk->common.lock, flags); + + return 0; +} + +const struct clk_ops cv1800_clk_audio_ops = { + .disable = aclk_disable, + .enable = aclk_enable, + .is_enabled = aclk_is_enabled, + + .determine_rate = aclk_determine_rate, + .recalc_rate = aclk_recalc_rate, + .set_rate = aclk_set_rate, +}; diff --git a/drivers/clk/sophgo/clk-cv18xx-ip.h b/drivers/clk/sophgo/clk-cv18xx-ip.h new file mode 100644 index 0000000000000..b37ba42bfde35 --- /dev/null +++ b/drivers/clk/sophgo/clk-cv18xx-ip.h @@ -0,0 +1,261 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023 Inochi Amaoto + */ + +#ifndef _CLK_SOPHGO_CV1800_IP_H_ +#define _CLK_SOPHGO_CV1800_IP_H_ + +#include "clk-cv18xx-common.h" + +struct cv1800_clk_gate { + struct cv1800_clk_common common; + struct cv1800_clk_regbit gate; +}; + +struct cv1800_clk_div_data { + u32 reg; + u32 mask; + u32 width; + u32 init; + u32 flags; +}; + +struct cv1800_clk_div { + struct cv1800_clk_common common; + struct cv1800_clk_regbit gate; + struct cv1800_clk_regfield div; +}; + +struct cv1800_clk_bypass_div { + struct cv1800_clk_div div; + struct cv1800_clk_regbit bypass; +}; + +struct cv1800_clk_mux { + struct cv1800_clk_common common; + struct cv1800_clk_regbit gate; + struct cv1800_clk_regfield div; + struct cv1800_clk_regfield mux; +}; + +struct cv1800_clk_bypass_mux { + struct cv1800_clk_mux mux; + struct cv1800_clk_regbit bypass; +}; + +struct cv1800_clk_mmux { + struct cv1800_clk_common common; + struct cv1800_clk_regbit gate; + struct cv1800_clk_regfield div[2]; + struct cv1800_clk_regfield mux[2]; + struct cv1800_clk_regbit bypass; + struct cv1800_clk_regbit clk_sel; + const s8 *parent2sel; + const u8 *sel2parent[2]; +}; + +struct cv1800_clk_audio { + struct cv1800_clk_common common; + struct cv1800_clk_regbit src_en; + struct cv1800_clk_regbit output_en; + struct cv1800_clk_regbit div_en; + struct cv1800_clk_regbit div_up; + struct cv1800_clk_regfield m; + struct cv1800_clk_regfield n; + u32 target_rate; +}; + +#define CV1800_GATE(_name, _parent, _gate_reg, _gate_shift, _flags) \ + struct cv1800_clk_gate _name = { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + &cv1800_clk_gate_ops, \ + _flags), \ + .gate = CV1800_CLK_BIT(_gate_reg, _gate_shift), \ + } + +#define _CV1800_DIV(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init, \ + _div_flag, _ops, _flags) \ + { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + _ops, _flags), \ + .gate = CV1800_CLK_BIT(_gate_reg, \ + _gate_shift), \ + .div = CV1800_CLK_REG(_div_reg, _div_shift, \ + _div_width, _div_init, \ + _div_flag), \ + } + +#define _CV1800_FIXED_DIV_FLAG \ + (CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ROUND_CLOSEST) + +#define _CV1800_FIXED_DIV(_name, _parent, _gate_reg, _gate_shift, \ + _fix_div, _ops, _flags) \ + { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + _ops, _flags), \ + .gate = CV1800_CLK_BIT(_gate_reg, \ + _gate_shift), \ + .div = CV1800_CLK_REG(0, 0, 0, \ + _fix_div, \ + _CV1800_FIXED_DIV_FLAG),\ + } + +#define CV1800_DIV(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init, \ + _div_flag, _flags) \ + struct cv1800_clk_div _name = \ + _CV1800_DIV(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init,\ + _div_flag, &cv1800_clk_div_ops, _flags) + +#define CV1800_BYPASS_DIV(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init, \ + _div_flag, _bypass_reg, _bypass_shift, _flags)\ + struct cv1800_clk_bypass_div _name = { \ + .div = _CV1800_DIV(_name, _parent, \ + _gate_reg, _gate_shift, \ + _div_reg, _div_shift, \ + _div_width, _div_init, _div_flag, \ + &cv1800_clk_bypass_div_ops, \ + _flags), \ + .bypass = CV1800_CLK_BIT(_bypass_reg, _bypass_shift), \ + } + +#define CV1800_FIXED_DIV(_name, _parent, _gate_reg, _gate_shift, \ + _fix_div, _flags) \ + struct cv1800_clk_div _name = \ + _CV1800_FIXED_DIV(_name, _parent, \ + _gate_reg, _gate_shift, \ + _fix_div, \ + &cv1800_clk_div_ops, _flags) \ + +#define CV1800_BYPASS_FIXED_DIV(_name, _parent, _gate_reg, _gate_shift, \ + _fix_div, _bypass_reg, _bypass_shift, \ + _flags) \ + struct cv1800_clk_bypass_div _name = { \ + .div = _CV1800_FIXED_DIV(_name, _parent, \ + _gate_reg, _gate_shift, \ + _fix_div, \ + &cv1800_clk_bypass_div_ops, \ + _flags), \ + .bypass = CV1800_CLK_BIT(_bypass_reg, _bypass_shift), \ + } + +#define _CV1800_MUX(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init, \ + _div_flag, \ + _mux_reg, _mux_shift, _mux_width, \ + _ops, _flags) \ + { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + _ops, _flags), \ + .gate = CV1800_CLK_BIT(_gate_reg, \ + _gate_shift), \ + .div = CV1800_CLK_REG(_div_reg, _div_shift, \ + _div_width, _div_init, \ + _div_flag), \ + .mux = CV1800_CLK_REG(_mux_reg, _mux_shift, \ + _mux_width, 0, 0), \ + } + +#define CV1800_MUX(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init, \ + _div_flag, \ + _mux_reg, _mux_shift, _mux_width, _flags) \ + struct cv1800_clk_mux _name = \ + _CV1800_MUX(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init,\ + _div_flag, _mux_reg, _mux_shift, _mux_width,\ + &cv1800_clk_mux_ops, _flags) + +#define CV1800_BYPASS_MUX(_name, _parent, _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, _div_init, \ + _div_flag, \ + _mux_reg, _mux_shift, _mux_width, \ + _bypass_reg, _bypass_shift, _flags) \ + struct cv1800_clk_bypass_mux _name = { \ + .mux = _CV1800_MUX(_name, _parent, \ + _gate_reg, _gate_shift, \ + _div_reg, _div_shift, _div_width, \ + _div_init, _div_flag, \ + _mux_reg, _mux_shift, _mux_width, \ + &cv1800_clk_bypass_mux_ops, \ + _flags), \ + .bypass = CV1800_CLK_BIT(_bypass_reg, _bypass_shift), \ + } + +#define CV1800_MMUX(_name, _parent, _gate_reg, _gate_shift, \ + _div0_reg, _div0_shift, _div0_width, _div0_init, \ + _div0_flag, \ + _div1_reg, _div1_shift, _div1_width, _div1_init, \ + _div1_flag, \ + _mux0_reg, _mux0_shift, _mux0_width, \ + _mux1_reg, _mux1_shift, _mux1_width, \ + _bypass_reg, _bypass_shift, \ + _clk_sel_reg, _clk_sel_shift, \ + _parent2sel, _sel2parent0, _sel2parent1, _flags) \ + struct cv1800_clk_mmux _name = { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + &cv1800_clk_mmux_ops,\ + _flags), \ + .gate = CV1800_CLK_BIT(_gate_reg, _gate_shift),\ + .div = { \ + CV1800_CLK_REG(_div0_reg, _div0_shift, \ + _div0_width, _div0_init, \ + _div0_flag), \ + CV1800_CLK_REG(_div1_reg, _div1_shift, \ + _div1_width, _div1_init, \ + _div1_flag), \ + }, \ + .mux = { \ + CV1800_CLK_REG(_mux0_reg, _mux0_shift, \ + _mux0_width, 0, 0), \ + CV1800_CLK_REG(_mux1_reg, _mux1_shift, \ + _mux1_width, 0, 0), \ + }, \ + .bypass = CV1800_CLK_BIT(_bypass_reg, \ + _bypass_shift), \ + .clk_sel = CV1800_CLK_BIT(_clk_sel_reg, \ + _clk_sel_shift), \ + .parent2sel = _parent2sel, \ + .sel2parent = { _sel2parent0, _sel2parent1 }, \ + } + +#define CV1800_ACLK(_name, _parent, \ + _src_en_reg, _src_en_reg_shift, \ + _output_en_reg, _output_en_shift, \ + _div_en_reg, _div_en_reg_shift, \ + _div_up_reg, _div_up_reg_shift, \ + _m_reg, _m_shift, _m_width, _m_flag, \ + _n_reg, _n_shift, _n_width, _n_flag, \ + _target_rate, _flags) \ + struct cv1800_clk_audio _name = { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + &cv1800_clk_audio_ops,\ + _flags), \ + .src_en = CV1800_CLK_BIT(_src_en_reg, \ + _src_en_reg_shift), \ + .output_en = CV1800_CLK_BIT(_output_en_reg, \ + _output_en_shift), \ + .div_en = CV1800_CLK_BIT(_div_en_reg, \ + _div_en_reg_shift), \ + .div_up = CV1800_CLK_BIT(_div_up_reg, \ + _div_up_reg_shift), \ + .m = CV1800_CLK_REG(_m_reg, _m_shift, \ + _m_width, 0, _m_flag), \ + .n = CV1800_CLK_REG(_n_reg, _n_shift, \ + _n_width, 0, _n_flag), \ + .target_rate = _target_rate, \ + } + +extern const struct clk_ops cv1800_clk_gate_ops; +extern const struct clk_ops cv1800_clk_div_ops; +extern const struct clk_ops cv1800_clk_bypass_div_ops; +extern const struct clk_ops cv1800_clk_mux_ops; +extern const struct clk_ops cv1800_clk_bypass_mux_ops; +extern const struct clk_ops cv1800_clk_mmux_ops; +extern const struct clk_ops cv1800_clk_audio_ops; + +#endif // _CLK_SOPHGO_CV1800_IP_H_ diff --git a/drivers/clk/sophgo/clk-cv18xx-pll.c b/drivers/clk/sophgo/clk-cv18xx-pll.c new file mode 100644 index 0000000000000..c546dad1791c3 --- /dev/null +++ b/drivers/clk/sophgo/clk-cv18xx-pll.c @@ -0,0 +1,420 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023 Inochi Amaoto + */ + +#include +#include +#include +#include + +#include "clk-cv18xx-pll.h" + +static inline struct cv1800_clk_pll *hw_to_cv1800_clk_pll(struct clk_hw *hw) +{ + struct cv1800_clk_common *common = hw_to_cv1800_clk_common(hw); + + return container_of(common, struct cv1800_clk_pll, common); +} + +static unsigned long ipll_calc_rate(unsigned long parent_rate, + unsigned long pre_div_sel, + unsigned long div_sel, + unsigned long post_div_sel) +{ + uint64_t rate = parent_rate; + + rate *= div_sel; + do_div(rate, pre_div_sel * post_div_sel); + + return rate; +} + +static unsigned long ipll_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + u32 value; + + value = readl(pll->common.base + pll->pll_reg); + + return ipll_calc_rate(parent_rate, + PLL_GET_PRE_DIV_SEL(value), + PLL_GET_DIV_SEL(value), + PLL_GET_POST_DIV_SEL(value)); +} + +static int ipll_find_rate(const struct cv1800_clk_pll_limit *limit, + unsigned long prate, unsigned long *rate, + u32 *value) +{ + unsigned long best_rate = 0; + unsigned long trate = *rate; + unsigned long pre_div_sel = 0, div_sel = 0, post_div_sel = 0; + unsigned long pre, div, post; + u32 detected = *value; + unsigned long tmp; + + for_each_pll_limit_range(pre, &limit->pre_div) { + for_each_pll_limit_range(div, &limit->div) { + for_each_pll_limit_range(post, &limit->post_div) { + tmp = ipll_calc_rate(prate, pre, div, post); + + if (tmp > trate) + continue; + + if ((trate - tmp) < (trate - best_rate)) { + best_rate = tmp; + pre_div_sel = pre; + div_sel = div; + post_div_sel = post; + } + } + } + } + + if (best_rate) { + detected = PLL_SET_PRE_DIV_SEL(detected, pre_div_sel); + detected = PLL_SET_POST_DIV_SEL(detected, post_div_sel); + detected = PLL_SET_DIV_SEL(detected, div_sel); + *value = detected; + *rate = best_rate; + return 0; + } + + return -EINVAL; +} + +static int ipll_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) +{ + u32 val; + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + return ipll_find_rate(pll->pll_limit, req->best_parent_rate, + &req->rate, &val); +} + +static void pll_get_mode_ctrl(unsigned long div_sel, + bool (*mode_ctrl_check)(unsigned long, + unsigned long, + unsigned long), + const struct cv1800_clk_pll_limit *limit, + u32 *value) +{ + unsigned long ictrl = 0, mode = 0; + u32 detected = *value; + + for_each_pll_limit_range(mode, &limit->mode) { + for_each_pll_limit_range(ictrl, &limit->ictrl) { + if (mode_ctrl_check(div_sel, ictrl, mode)) { + detected = PLL_SET_SEL_MODE(detected, mode); + detected = PLL_SET_ICTRL(detected, ictrl); + *value = detected; + return; + } + } + } +} + +static bool ipll_check_mode_ctrl_restrict(unsigned long div_sel, + unsigned long ictrl, + unsigned long mode) +{ + unsigned long left_rest = 20 * div_sel; + unsigned long right_rest = 35 * div_sel; + unsigned long test = 184 * (1 + mode) * (1 + ictrl) / 2; + + return test > left_rest && test <= right_rest; +} + +static int ipll_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + u32 regval, detected = 0; + unsigned long flags; + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + ipll_find_rate(pll->pll_limit, parent_rate, &rate, &detected); + pll_get_mode_ctrl(PLL_GET_DIV_SEL(detected), + ipll_check_mode_ctrl_restrict, + pll->pll_limit, &detected); + + spin_lock_irqsave(pll->common.lock, flags); + + regval = readl(pll->common.base + pll->pll_reg); + regval = PLL_COPY_REG(regval, detected); + + writel(regval, pll->common.base + pll->pll_reg); + + spin_unlock_irqrestore(pll->common.lock, flags); + + cv1800_clk_wait_for_lock(&pll->common, pll->pll_status.reg, + BIT(pll->pll_status.shift)); + + return 0; +} + +static int pll_enable(struct clk_hw *hw) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + return cv1800_clk_clearbit(&pll->common, &pll->pll_pwd); +} + +static void pll_disable(struct clk_hw *hw) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + cv1800_clk_setbit(&pll->common, &pll->pll_pwd); +} + +static int pll_is_enable(struct clk_hw *hw) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + return cv1800_clk_checkbit(&pll->common, &pll->pll_pwd) == 0; +} + +const struct clk_ops cv1800_clk_ipll_ops = { + .disable = pll_disable, + .enable = pll_enable, + .is_enabled = pll_is_enable, + + .recalc_rate = ipll_recalc_rate, + .determine_rate = ipll_determine_rate, + .set_rate = ipll_set_rate, +}; + +#define PLL_SYN_FACTOR_DOT_POS 26 +#define PLL_SYN_FACTOR_MINIMUM ((4 << PLL_SYN_FACTOR_DOT_POS) + 1) + +static bool fpll_is_factional_mode(struct cv1800_clk_pll *pll) +{ + return cv1800_clk_checkbit(&pll->common, &pll->pll_syn->en); +} + +static unsigned long fpll_calc_rate(unsigned long parent_rate, + unsigned long pre_div_sel, + unsigned long div_sel, + unsigned long post_div_sel, + unsigned long ssc_syn_set, + bool is_full_parent) +{ + u64 dividend = parent_rate * div_sel; + u64 factor = ssc_syn_set * pre_div_sel * post_div_sel; + unsigned long rate; + + dividend <<= PLL_SYN_FACTOR_DOT_POS - 1; + rate = dividend / factor; + dividend %= factor; + + if (is_full_parent) { + dividend <<= 1; + rate <<= 1; + } + + rate += DIV64_U64_ROUND_CLOSEST(dividend, factor); + + return rate; +} + +static unsigned long fpll_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + u32 value; + bool clk_full; + u32 syn_set; + + if (!fpll_is_factional_mode(pll)) + return ipll_recalc_rate(hw, parent_rate); + + syn_set = readl(pll->common.base + pll->pll_syn->set); + + if (syn_set == 0) + return 0; + + clk_full = cv1800_clk_checkbit(&pll->common, + &pll->pll_syn->clk_half); + + value = readl(pll->common.base + pll->pll_reg); + + return fpll_calc_rate(parent_rate, + PLL_GET_PRE_DIV_SEL(value), + PLL_GET_DIV_SEL(value), + PLL_GET_POST_DIV_SEL(value), + syn_set, clk_full); +} + +static unsigned long fpll_find_synthesizer(unsigned long parent, + unsigned long rate, + unsigned long pre_div, + unsigned long div, + unsigned long post_div, + bool is_full_parent, + u32 *ssc_syn_set) +{ + u32 test_max = U32_MAX, test_min = PLL_SYN_FACTOR_MINIMUM; + unsigned long trate; + + while (test_min < test_max) { + u32 tssc = (test_max + test_min) / 2; + + trate = fpll_calc_rate(parent, pre_div, div, post_div, + tssc, is_full_parent); + + if (trate == rate) { + test_min = tssc; + break; + } + + if (trate > rate) + test_min = tssc + 1; + else + test_max = tssc - 1; + } + + if (trate != 0) + *ssc_syn_set = test_min; + + return trate; +} + +static int fpll_find_rate(struct cv1800_clk_pll *pll, + const struct cv1800_clk_pll_limit *limit, + unsigned long prate, + unsigned long *rate, + u32 *value, u32 *ssc_syn_set) +{ + unsigned long best_rate = 0; + unsigned long pre_div_sel = 0, div_sel = 0, post_div_sel = 0; + unsigned long pre, div, post; + unsigned long trate = *rate; + u32 detected = *value; + unsigned long tmp; + bool clk_full = cv1800_clk_checkbit(&pll->common, + &pll->pll_syn->clk_half); + + for_each_pll_limit_range(pre, &limit->pre_div) { + for_each_pll_limit_range(post, &limit->post_div) { + for_each_pll_limit_range(div, &limit->div) { + tmp = fpll_find_synthesizer(prate, trate, + pre, div, post, + clk_full, + ssc_syn_set); + + if ((trate - tmp) < (trate - best_rate)) { + best_rate = tmp; + pre_div_sel = pre; + div_sel = div; + post_div_sel = post; + } + } + } + } + + if (best_rate) { + detected = PLL_SET_PRE_DIV_SEL(detected, pre_div_sel); + detected = PLL_SET_POST_DIV_SEL(detected, post_div_sel); + detected = PLL_SET_DIV_SEL(detected, div_sel); + *value = detected; + *rate = best_rate; + return 0; + } + + return -EINVAL; +} + +static int fpll_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + u32 val, ssc_syn_set; + + if (!fpll_is_factional_mode(pll)) + return ipll_determine_rate(hw, req); + + fpll_find_rate(pll, &pll->pll_limit[2], req->best_parent_rate, + &req->rate, &val, &ssc_syn_set); + + return 0; +} + +static bool fpll_check_mode_ctrl_restrict(unsigned long div_sel, + unsigned long ictrl, + unsigned long mode) +{ + unsigned long left_rest = 10 * div_sel; + unsigned long right_rest = 24 * div_sel; + unsigned long test = 184 * (1 + mode) * (1 + ictrl) / 2; + + return test > left_rest && test <= right_rest; +} + +static int fpll_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + u32 regval; + u32 detected = 0, detected_ssc = 0; + unsigned long flags; + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + if (!fpll_is_factional_mode(pll)) + return ipll_set_rate(hw, rate, parent_rate); + + fpll_find_rate(pll, &pll->pll_limit[2], parent_rate, + &rate, &detected, &detected_ssc); + pll_get_mode_ctrl(PLL_GET_DIV_SEL(detected), + fpll_check_mode_ctrl_restrict, + pll->pll_limit, &detected); + + spin_lock_irqsave(pll->common.lock, flags); + + writel(detected_ssc, pll->common.base + pll->pll_syn->set); + + regval = readl(pll->common.base + pll->pll_reg); + regval = PLL_COPY_REG(regval, detected); + + writel(regval, pll->common.base + pll->pll_reg); + + spin_unlock_irqrestore(pll->common.lock, flags); + + cv1800_clk_wait_for_lock(&pll->common, pll->pll_status.reg, + BIT(pll->pll_status.shift)); + + return 0; +} + +static u8 fpll_get_parent(struct clk_hw *hw) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + if (fpll_is_factional_mode(pll)) + return 1; + + return 0; +} + +static int fpll_set_parent(struct clk_hw *hw, u8 index) +{ + struct cv1800_clk_pll *pll = hw_to_cv1800_clk_pll(hw); + + if (index) + cv1800_clk_setbit(&pll->common, &pll->pll_syn->en); + else + cv1800_clk_clearbit(&pll->common, &pll->pll_syn->en); + + return 0; +} + +const struct clk_ops cv1800_clk_fpll_ops = { + .disable = pll_disable, + .enable = pll_enable, + .is_enabled = pll_is_enable, + + .recalc_rate = fpll_recalc_rate, + .determine_rate = fpll_determine_rate, + .set_rate = fpll_set_rate, + + .set_parent = fpll_set_parent, + .get_parent = fpll_get_parent, +}; diff --git a/drivers/clk/sophgo/clk-cv18xx-pll.h b/drivers/clk/sophgo/clk-cv18xx-pll.h new file mode 100644 index 0000000000000..7a33f3da2d644 --- /dev/null +++ b/drivers/clk/sophgo/clk-cv18xx-pll.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023 Inochi Amaoto + */ + +#ifndef _CLK_SOPHGO_CV1800_PLL_H_ +#define _CLK_SOPHGO_CV1800_PLL_H_ + +#include "clk-cv18xx-common.h" + +struct cv1800_clk_pll_limit { + struct { + u8 min; + u8 max; + } pre_div, div, post_div, ictrl, mode; +}; + +#define _CV1800_PLL_LIMIT(_min, _max) \ + { \ + .min = _min, \ + .max = _max, \ + } \ + +#define for_each_pll_limit_range(_var, _restrict) \ + for (_var = (_restrict)->min; _var <= (_restrict)->max; _var++) + +struct cv1800_clk_pll_synthesizer { + struct cv1800_clk_regbit en; + struct cv1800_clk_regbit clk_half; + u32 ctrl; + u32 set; +}; + +#define _PLL_PRE_DIV_SEL_FIELD GENMASK(6, 0) +#define _PLL_POST_DIV_SEL_FIELD GENMASK(14, 8) +#define _PLL_SEL_MODE_FIELD GENMASK(16, 15) +#define _PLL_DIV_SEL_FIELD GENMASK(23, 17) +#define _PLL_ICTRL_FIELD GENMASK(26, 24) + +#define _PLL_ALL_FIELD_MASK \ + (_PLL_PRE_DIV_SEL_FIELD | \ + _PLL_POST_DIV_SEL_FIELD | \ + _PLL_SEL_MODE_FIELD | \ + _PLL_DIV_SEL_FIELD | \ + _PLL_ICTRL_FIELD) + +#define PLL_COPY_REG(_dest, _src) \ + (((_dest) & (~_PLL_ALL_FIELD_MASK)) | ((_src) & _PLL_ALL_FIELD_MASK)) + +#define PLL_GET_PRE_DIV_SEL(_reg) \ + FIELD_GET(_PLL_PRE_DIV_SEL_FIELD, (_reg)) +#define PLL_GET_POST_DIV_SEL(_reg) \ + FIELD_GET(_PLL_POST_DIV_SEL_FIELD, (_reg)) +#define PLL_GET_SEL_MODE(_reg) \ + FIELD_GET(_PLL_SEL_MODE_FIELD, (_reg)) +#define PLL_GET_DIV_SEL(_reg) \ + FIELD_GET(_PLL_DIV_SEL_FIELD, (_reg)) +#define PLL_GET_ICTRL(_reg) \ + FIELD_GET(_PLL_ICTRL_FIELD, (_reg)) + +#define PLL_SET_PRE_DIV_SEL(_reg, _val) \ + _CV1800_SET_FIELD((_reg), (_val), _PLL_PRE_DIV_SEL_FIELD) +#define PLL_SET_POST_DIV_SEL(_reg, _val) \ + _CV1800_SET_FIELD((_reg), (_val), _PLL_POST_DIV_SEL_FIELD) +#define PLL_SET_SEL_MODE(_reg, _val) \ + _CV1800_SET_FIELD((_reg), (_val), _PLL_SEL_MODE_FIELD) +#define PLL_SET_DIV_SEL(_reg, _val) \ + _CV1800_SET_FIELD((_reg), (_val), _PLL_DIV_SEL_FIELD) +#define PLL_SET_ICTRL(_reg, _val) \ + _CV1800_SET_FIELD((_reg), (_val), _PLL_ICTRL_FIELD) + +struct cv1800_clk_pll { + struct cv1800_clk_common common; + u32 pll_reg; + struct cv1800_clk_regbit pll_pwd; + struct cv1800_clk_regbit pll_status; + const struct cv1800_clk_pll_limit *pll_limit; + struct cv1800_clk_pll_synthesizer *pll_syn; +}; + +#define CV1800_INTEGRAL_PLL(_name, _parent, _pll_reg, \ + _pll_pwd_reg, _pll_pwd_shift, \ + _pll_status_reg, _pll_status_shift, \ + _pll_limit, _flags) \ + struct cv1800_clk_pll _name = { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + &cv1800_clk_ipll_ops,\ + _flags), \ + .pll_reg = _pll_reg, \ + .pll_pwd = CV1800_CLK_BIT(_pll_pwd_reg, \ + _pll_pwd_shift), \ + .pll_status = CV1800_CLK_BIT(_pll_status_reg, \ + _pll_status_shift), \ + .pll_limit = _pll_limit, \ + .pll_syn = NULL, \ + } + +#define CV1800_FACTIONAL_PLL(_name, _parent, _pll_reg, \ + _pll_pwd_reg, _pll_pwd_shift, \ + _pll_status_reg, _pll_status_shift, \ + _pll_limit, _pll_syn, _flags) \ + struct cv1800_clk_pll _name = { \ + .common = CV1800_CLK_COMMON(#_name, _parent, \ + &cv1800_clk_fpll_ops,\ + _flags), \ + .pll_reg = _pll_reg, \ + .pll_pwd = CV1800_CLK_BIT(_pll_pwd_reg, \ + _pll_pwd_shift), \ + .pll_status = CV1800_CLK_BIT(_pll_status_reg, \ + _pll_status_shift), \ + .pll_limit = _pll_limit, \ + .pll_syn = _pll_syn, \ + } + +extern const struct clk_ops cv1800_clk_ipll_ops; +extern const struct clk_ops cv1800_clk_fpll_ops; + +#endif // _CLK_SOPHGO_CV1800_PLL_H_ From 3b8d2042122846e3e61b184f9d17690d31a885a4 Mon Sep 17 00:00:00 2001 From: Inochi Amaoto Date: Sat, 9 Mar 2024 17:02:53 +0800 Subject: [PATCH 0116/1702] clk: sophgo: Add clock support for CV1810 SoC Add clock definition and init code for CV1810 SoC. Signed-off-by: Inochi Amaoto Link: https://github.com/milkv-duo/duo-files/blob/6f4e9b8ecb459e017cca1a8df248a19ca70837a3/duo/datasheet/CV180X-Clock-v1.xlsx Link: https://lore.kernel.org/r/IA1PR20MB495357FB5EEA1623DAB08C94BB262@IA1PR20MB4953.namprd20.prod.outlook.com Signed-off-by: Stephen Boyd --- drivers/clk/sophgo/clk-cv1800.c | 195 ++++++++++++++++++++++++++++++++ drivers/clk/sophgo/clk-cv1800.h | 1 + 2 files changed, 196 insertions(+) diff --git a/drivers/clk/sophgo/clk-cv1800.c b/drivers/clk/sophgo/clk-cv1800.c index 6606a2701b122..c7d4aa6c83433 100644 --- a/drivers/clk/sophgo/clk-cv1800.c +++ b/drivers/clk/sophgo/clk-cv1800.c @@ -631,6 +631,10 @@ static const struct clk_parent_data clk_vip_sys_parents[] = { { .hw = &clk_disppll.common.hw }, { .hw = &clk_fpll.common.hw }, }; +static const struct clk_parent_data clk_disp_vip_parents[] = { + { .index = 0 }, + { .hw = &clk_disppll.common.hw }, +}; static CV1800_BYPASS_DIV(clk_dsi_esc, clk_bypass_axi6_bus_parents, REG_CLK_EN_2, 3, @@ -660,6 +664,11 @@ static CV1800_BYPASS_MUX(clk_src_vip_sys_1, clk_vip_sys_parents, REG_DIV_CLK_SRC_VIP_SYS_1, 8, 2, REG_CLK_BYP_0, 24, 0); +static CV1800_BYPASS_DIV(clk_disp_src_vip, clk_disp_vip_parents, + REG_CLK_EN_2, 7, + REG_DIV_CLK_DISP_SRC_VIP, 16, 4, 8, CV1800_DIV_FLAG, + REG_CLK_BYP_0, 25, + 0); static CV1800_BYPASS_MUX(clk_src_vip_sys_2, clk_vip_sys_parents, REG_CLK_EN_3, 29, REG_DIV_CLK_SRC_VIP_SYS_2, 16, 4, 2, CV1800_DIV_FLAG, @@ -1243,6 +1252,191 @@ static const struct cv1800_clk_desc cv1800_desc = { .pre_init = cv1800_pre_init, }; +static struct clk_hw_onecell_data cv1810_hw_clks = { + .num = CV1810_CLK_MAX, + .hws = { + [CLK_MPLL] = &clk_mpll.common.hw, + [CLK_TPLL] = &clk_tpll.common.hw, + [CLK_FPLL] = &clk_fpll.common.hw, + [CLK_MIPIMPLL] = &clk_mipimpll.common.hw, + [CLK_A0PLL] = &clk_a0pll.common.hw, + [CLK_DISPPLL] = &clk_disppll.common.hw, + [CLK_CAM0PLL] = &clk_cam0pll.common.hw, + [CLK_CAM1PLL] = &clk_cam1pll.common.hw, + + [CLK_MIPIMPLL_D3] = &clk_mipimpll_d3.common.hw, + [CLK_CAM0PLL_D2] = &clk_cam0pll_d2.common.hw, + [CLK_CAM0PLL_D3] = &clk_cam0pll_d3.common.hw, + + [CLK_TPU] = &clk_tpu.mux.common.hw, + [CLK_TPU_FAB] = &clk_tpu_fab.common.hw, + [CLK_AHB_ROM] = &clk_ahb_rom.common.hw, + [CLK_DDR_AXI_REG] = &clk_ddr_axi_reg.common.hw, + [CLK_RTC_25M] = &clk_rtc_25m.common.hw, + [CLK_SRC_RTC_SYS_0] = &clk_src_rtc_sys_0.div.common.hw, + [CLK_TEMPSEN] = &clk_tempsen.common.hw, + [CLK_SARADC] = &clk_saradc.common.hw, + [CLK_EFUSE] = &clk_efuse.common.hw, + [CLK_APB_EFUSE] = &clk_apb_efuse.common.hw, + [CLK_DEBUG] = &clk_debug.common.hw, + [CLK_AP_DEBUG] = &clk_ap_debug.div.common.hw, + [CLK_XTAL_MISC] = &clk_xtal_misc.common.hw, + [CLK_AXI4_EMMC] = &clk_axi4_emmc.common.hw, + [CLK_EMMC] = &clk_emmc.mux.common.hw, + [CLK_EMMC_100K] = &clk_emmc_100k.common.hw, + [CLK_AXI4_SD0] = &clk_axi4_sd0.common.hw, + [CLK_SD0] = &clk_sd0.mux.common.hw, + [CLK_SD0_100K] = &clk_sd0_100k.common.hw, + [CLK_AXI4_SD1] = &clk_axi4_sd1.common.hw, + [CLK_SD1] = &clk_sd1.mux.common.hw, + [CLK_SD1_100K] = &clk_sd1_100k.common.hw, + [CLK_SPI_NAND] = &clk_spi_nand.mux.common.hw, + [CLK_ETH0_500M] = &clk_eth0_500m.div.common.hw, + [CLK_AXI4_ETH0] = &clk_axi4_eth0.common.hw, + [CLK_ETH1_500M] = &clk_eth1_500m.div.common.hw, + [CLK_AXI4_ETH1] = &clk_axi4_eth1.common.hw, + [CLK_APB_GPIO] = &clk_apb_gpio.common.hw, + [CLK_APB_GPIO_INTR] = &clk_apb_gpio_intr.common.hw, + [CLK_GPIO_DB] = &clk_gpio_db.common.hw, + [CLK_AHB_SF] = &clk_ahb_sf.common.hw, + [CLK_AHB_SF1] = &clk_ahb_sf1.common.hw, + [CLK_A24M] = &clk_a24m.common.hw, + [CLK_AUDSRC] = &clk_audsrc.mux.common.hw, + [CLK_APB_AUDSRC] = &clk_apb_audsrc.common.hw, + [CLK_SDMA_AXI] = &clk_sdma_axi.common.hw, + [CLK_SDMA_AUD0] = &clk_sdma_aud0.mux.common.hw, + [CLK_SDMA_AUD1] = &clk_sdma_aud1.mux.common.hw, + [CLK_SDMA_AUD2] = &clk_sdma_aud2.mux.common.hw, + [CLK_SDMA_AUD3] = &clk_sdma_aud3.mux.common.hw, + [CLK_I2C] = &clk_i2c.div.common.hw, + [CLK_APB_I2C] = &clk_apb_i2c.common.hw, + [CLK_APB_I2C0] = &clk_apb_i2c0.common.hw, + [CLK_APB_I2C1] = &clk_apb_i2c1.common.hw, + [CLK_APB_I2C2] = &clk_apb_i2c2.common.hw, + [CLK_APB_I2C3] = &clk_apb_i2c3.common.hw, + [CLK_APB_I2C4] = &clk_apb_i2c4.common.hw, + [CLK_APB_WDT] = &clk_apb_wdt.common.hw, + [CLK_PWM_SRC] = &clk_pwm_src.mux.common.hw, + [CLK_PWM] = &clk_pwm.common.hw, + [CLK_SPI] = &clk_spi.div.common.hw, + [CLK_APB_SPI0] = &clk_apb_spi0.common.hw, + [CLK_APB_SPI1] = &clk_apb_spi1.common.hw, + [CLK_APB_SPI2] = &clk_apb_spi2.common.hw, + [CLK_APB_SPI3] = &clk_apb_spi3.common.hw, + [CLK_1M] = &clk_1m.common.hw, + [CLK_CAM0_200] = &clk_cam0_200.mux.common.hw, + [CLK_PM] = &clk_pm.common.hw, + [CLK_TIMER0] = &clk_timer0.common.hw, + [CLK_TIMER1] = &clk_timer1.common.hw, + [CLK_TIMER2] = &clk_timer2.common.hw, + [CLK_TIMER3] = &clk_timer3.common.hw, + [CLK_TIMER4] = &clk_timer4.common.hw, + [CLK_TIMER5] = &clk_timer5.common.hw, + [CLK_TIMER6] = &clk_timer6.common.hw, + [CLK_TIMER7] = &clk_timer7.common.hw, + [CLK_UART0] = &clk_uart0.common.hw, + [CLK_APB_UART0] = &clk_apb_uart0.common.hw, + [CLK_UART1] = &clk_uart1.common.hw, + [CLK_APB_UART1] = &clk_apb_uart1.common.hw, + [CLK_UART2] = &clk_uart2.common.hw, + [CLK_APB_UART2] = &clk_apb_uart2.common.hw, + [CLK_UART3] = &clk_uart3.common.hw, + [CLK_APB_UART3] = &clk_apb_uart3.common.hw, + [CLK_UART4] = &clk_uart4.common.hw, + [CLK_APB_UART4] = &clk_apb_uart4.common.hw, + [CLK_APB_I2S0] = &clk_apb_i2s0.common.hw, + [CLK_APB_I2S1] = &clk_apb_i2s1.common.hw, + [CLK_APB_I2S2] = &clk_apb_i2s2.common.hw, + [CLK_APB_I2S3] = &clk_apb_i2s3.common.hw, + [CLK_AXI4_USB] = &clk_axi4_usb.common.hw, + [CLK_APB_USB] = &clk_apb_usb.common.hw, + [CLK_USB_125M] = &clk_usb_125m.div.common.hw, + [CLK_USB_33K] = &clk_usb_33k.common.hw, + [CLK_USB_12M] = &clk_usb_12m.div.common.hw, + [CLK_AXI4] = &clk_axi4.mux.common.hw, + [CLK_AXI6] = &clk_axi6.div.common.hw, + [CLK_DSI_ESC] = &clk_dsi_esc.div.common.hw, + [CLK_AXI_VIP] = &clk_axi_vip.mux.common.hw, + [CLK_SRC_VIP_SYS_0] = &clk_src_vip_sys_0.mux.common.hw, + [CLK_SRC_VIP_SYS_1] = &clk_src_vip_sys_1.mux.common.hw, + [CLK_SRC_VIP_SYS_2] = &clk_src_vip_sys_2.mux.common.hw, + [CLK_SRC_VIP_SYS_3] = &clk_src_vip_sys_3.mux.common.hw, + [CLK_SRC_VIP_SYS_4] = &clk_src_vip_sys_4.mux.common.hw, + [CLK_CSI_BE_VIP] = &clk_csi_be_vip.common.hw, + [CLK_CSI_MAC0_VIP] = &clk_csi_mac0_vip.common.hw, + [CLK_CSI_MAC1_VIP] = &clk_csi_mac1_vip.common.hw, + [CLK_CSI_MAC2_VIP] = &clk_csi_mac2_vip.common.hw, + [CLK_CSI0_RX_VIP] = &clk_csi0_rx_vip.common.hw, + [CLK_CSI1_RX_VIP] = &clk_csi1_rx_vip.common.hw, + [CLK_ISP_TOP_VIP] = &clk_isp_top_vip.common.hw, + [CLK_IMG_D_VIP] = &clk_img_d_vip.common.hw, + [CLK_IMG_V_VIP] = &clk_img_v_vip.common.hw, + [CLK_SC_TOP_VIP] = &clk_sc_top_vip.common.hw, + [CLK_SC_D_VIP] = &clk_sc_d_vip.common.hw, + [CLK_SC_V1_VIP] = &clk_sc_v1_vip.common.hw, + [CLK_SC_V2_VIP] = &clk_sc_v2_vip.common.hw, + [CLK_SC_V3_VIP] = &clk_sc_v3_vip.common.hw, + [CLK_DWA_VIP] = &clk_dwa_vip.common.hw, + [CLK_BT_VIP] = &clk_bt_vip.common.hw, + [CLK_DISP_VIP] = &clk_disp_vip.common.hw, + [CLK_DSI_MAC_VIP] = &clk_dsi_mac_vip.common.hw, + [CLK_LVDS0_VIP] = &clk_lvds0_vip.common.hw, + [CLK_LVDS1_VIP] = &clk_lvds1_vip.common.hw, + [CLK_PAD_VI_VIP] = &clk_pad_vi_vip.common.hw, + [CLK_PAD_VI1_VIP] = &clk_pad_vi1_vip.common.hw, + [CLK_PAD_VI2_VIP] = &clk_pad_vi2_vip.common.hw, + [CLK_CFG_REG_VIP] = &clk_cfg_reg_vip.common.hw, + [CLK_VIP_IP0] = &clk_vip_ip0.common.hw, + [CLK_VIP_IP1] = &clk_vip_ip1.common.hw, + [CLK_VIP_IP2] = &clk_vip_ip2.common.hw, + [CLK_VIP_IP3] = &clk_vip_ip3.common.hw, + [CLK_IVE_VIP] = &clk_ive_vip.common.hw, + [CLK_RAW_VIP] = &clk_raw_vip.common.hw, + [CLK_OSDC_VIP] = &clk_osdc_vip.common.hw, + [CLK_CAM0_VIP] = &clk_cam0_vip.common.hw, + [CLK_AXI_VIDEO_CODEC] = &clk_axi_video_codec.mux.common.hw, + [CLK_VC_SRC0] = &clk_vc_src0.mux.common.hw, + [CLK_VC_SRC1] = &clk_vc_src1.div.common.hw, + [CLK_VC_SRC2] = &clk_vc_src2.div.common.hw, + [CLK_H264C] = &clk_h264c.common.hw, + [CLK_APB_H264C] = &clk_apb_h264c.common.hw, + [CLK_H265C] = &clk_h265c.common.hw, + [CLK_APB_H265C] = &clk_apb_h265c.common.hw, + [CLK_JPEG] = &clk_jpeg.common.hw, + [CLK_APB_JPEG] = &clk_apb_jpeg.common.hw, + [CLK_CAM0] = &clk_cam0.common.hw, + [CLK_CAM1] = &clk_cam1.common.hw, + [CLK_WGN] = &clk_wgn.common.hw, + [CLK_WGN0] = &clk_wgn0.common.hw, + [CLK_WGN1] = &clk_wgn1.common.hw, + [CLK_WGN2] = &clk_wgn2.common.hw, + [CLK_KEYSCAN] = &clk_keyscan.common.hw, + [CLK_CFG_REG_VC] = &clk_cfg_reg_vc.common.hw, + [CLK_C906_0] = &clk_c906_0.common.hw, + [CLK_C906_1] = &clk_c906_1.common.hw, + [CLK_A53] = &clk_a53.common.hw, + [CLK_CPU_AXI0] = &clk_cpu_axi0.div.common.hw, + [CLK_CPU_GIC] = &clk_cpu_gic.div.common.hw, + [CLK_XTAL_AP] = &clk_xtal_ap.common.hw, + [CLK_DISP_SRC_VIP] = &clk_disp_src_vip.div.common.hw, + }, +}; + +static int cv1810_pre_init(struct device *dev, void __iomem *base, + struct cv1800_clk_ctrl *ctrl, + const struct cv1800_clk_desc *desc) +{ + cv18xx_clk_disable_a53(base); + cv18xx_clk_disable_auto_pd(base); + + return 0; +} + +static const struct cv1800_clk_desc cv1810_desc = { + .clks_data = &cv1810_hw_clks, + .pre_init = cv1810_pre_init, +}; + static int cv1800_clk_init_ctrl(struct device *dev, void __iomem *reg, struct cv1800_clk_ctrl *ctrl, const struct cv1800_clk_desc *desc) @@ -1311,6 +1505,7 @@ static int cv1800_clk_probe(struct platform_device *pdev) static const struct of_device_id cv1800_clk_ids[] = { { .compatible = "sophgo,cv1800-clk", .data = &cv1800_desc }, + { .compatible = "sophgo,cv1810-clk", .data = &cv1810_desc }, { } }; MODULE_DEVICE_TABLE(of, cv1800_clk_ids); diff --git a/drivers/clk/sophgo/clk-cv1800.h b/drivers/clk/sophgo/clk-cv1800.h index 1b9c04b5ac3a7..1e7107b5d05eb 100644 --- a/drivers/clk/sophgo/clk-cv1800.h +++ b/drivers/clk/sophgo/clk-cv1800.h @@ -9,6 +9,7 @@ #include #define CV1800_CLK_MAX (CLK_XTAL_AP + 1) +#define CV1810_CLK_MAX (CLK_DISP_SRC_VIP + 1) #define REG_PLL_G2_CTRL 0x800 #define REG_PLL_G2_STATUS 0x804 From 1cce3e61af6a89aa4b01189257fba73e46244b2d Mon Sep 17 00:00:00 2001 From: Inochi Amaoto Date: Sat, 9 Mar 2024 17:02:54 +0800 Subject: [PATCH 0117/1702] clk: sophgo: Add clock support for SG2000 SoC Add init code for SG2000 SoC. Signed-off-by: Inochi Amaoto Link: https://github.com/sophgo/sophgo-doc/releases/tag/sg2000-datasheet-v1.0-alpha Link: https://lore.kernel.org/r/IA1PR20MB49537156E71B64483F15C0F2BB262@IA1PR20MB4953.namprd20.prod.outlook.com Signed-off-by: Stephen Boyd --- drivers/clk/sophgo/clk-cv1800.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/clk/sophgo/clk-cv1800.c b/drivers/clk/sophgo/clk-cv1800.c index c7d4aa6c83433..956de5b21a80a 100644 --- a/drivers/clk/sophgo/clk-cv1800.c +++ b/drivers/clk/sophgo/clk-cv1800.c @@ -1437,6 +1437,20 @@ static const struct cv1800_clk_desc cv1810_desc = { .pre_init = cv1810_pre_init, }; +static int sg2000_pre_init(struct device *dev, void __iomem *base, + struct cv1800_clk_ctrl *ctrl, + const struct cv1800_clk_desc *desc) +{ + cv18xx_clk_disable_auto_pd(base); + + return 0; +} + +static const struct cv1800_clk_desc sg2000_desc = { + .clks_data = &cv1810_hw_clks, + .pre_init = sg2000_pre_init, +}; + static int cv1800_clk_init_ctrl(struct device *dev, void __iomem *reg, struct cv1800_clk_ctrl *ctrl, const struct cv1800_clk_desc *desc) @@ -1506,6 +1520,7 @@ static int cv1800_clk_probe(struct platform_device *pdev) static const struct of_device_id cv1800_clk_ids[] = { { .compatible = "sophgo,cv1800-clk", .data = &cv1800_desc }, { .compatible = "sophgo,cv1810-clk", .data = &cv1810_desc }, + { .compatible = "sophgo,sg2000-clk", .data = &sg2000_desc }, { } }; MODULE_DEVICE_TABLE(of, cv1800_clk_ids); From 0b1bfd15f3f9ffb1e36c88861e9ada3f58ccb234 Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Thu, 11 Apr 2024 10:58:06 +0800 Subject: [PATCH 0118/1702] dt-bindings: clock: Add Loongson-2K expand clock index In the new Loongson-2K family of SoCs, more clock indexes are needed, such as clock gates. The patch adds these clock indexes Signed-off-by: Binbin Zhou Acked-by: Conor Dooley Link: https://lore.kernel.org/r/76844e0e4dae290425f7c8025f7f36810cb3a3a8.1712731524.git.zhoubinbin@loongson.cn Acked-by: Huacai Chen Signed-off-by: Stephen Boyd --- include/dt-bindings/clock/loongson,ls2k-clk.h | 54 ++++++++++++------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/include/dt-bindings/clock/loongson,ls2k-clk.h b/include/dt-bindings/clock/loongson,ls2k-clk.h index 3bc4dfc193c2d..4279ba595f1e0 100644 --- a/include/dt-bindings/clock/loongson,ls2k-clk.h +++ b/include/dt-bindings/clock/loongson,ls2k-clk.h @@ -7,24 +7,40 @@ #ifndef __DT_BINDINGS_CLOCK_LOONGSON2_H #define __DT_BINDINGS_CLOCK_LOONGSON2_H -#define LOONGSON2_REF_100M 0 -#define LOONGSON2_NODE_PLL 1 -#define LOONGSON2_DDR_PLL 2 -#define LOONGSON2_DC_PLL 3 -#define LOONGSON2_PIX0_PLL 4 -#define LOONGSON2_PIX1_PLL 5 -#define LOONGSON2_NODE_CLK 6 -#define LOONGSON2_HDA_CLK 7 -#define LOONGSON2_GPU_CLK 8 -#define LOONGSON2_DDR_CLK 9 -#define LOONGSON2_GMAC_CLK 10 -#define LOONGSON2_DC_CLK 11 -#define LOONGSON2_APB_CLK 12 -#define LOONGSON2_USB_CLK 13 -#define LOONGSON2_SATA_CLK 14 -#define LOONGSON2_PIX0_CLK 15 -#define LOONGSON2_PIX1_CLK 16 -#define LOONGSON2_BOOT_CLK 17 -#define LOONGSON2_CLK_END 18 +#define LOONGSON2_REF_100M 0 +#define LOONGSON2_NODE_PLL 1 +#define LOONGSON2_DDR_PLL 2 +#define LOONGSON2_DC_PLL 3 +#define LOONGSON2_PIX0_PLL 4 +#define LOONGSON2_PIX1_PLL 5 +#define LOONGSON2_NODE_CLK 6 +#define LOONGSON2_HDA_CLK 7 +#define LOONGSON2_GPU_CLK 8 +#define LOONGSON2_DDR_CLK 9 +#define LOONGSON2_GMAC_CLK 10 +#define LOONGSON2_DC_CLK 11 +#define LOONGSON2_APB_CLK 12 +#define LOONGSON2_USB_CLK 13 +#define LOONGSON2_SATA_CLK 14 +#define LOONGSON2_PIX0_CLK 15 +#define LOONGSON2_PIX1_CLK 16 +#define LOONGSON2_BOOT_CLK 17 +#define LOONGSON2_OUT0_GATE 18 +#define LOONGSON2_GMAC_GATE 19 +#define LOONGSON2_RIO_GATE 20 +#define LOONGSON2_DC_GATE 21 +#define LOONGSON2_GPU_GATE 22 +#define LOONGSON2_DDR_GATE 23 +#define LOONGSON2_HDA_GATE 24 +#define LOONGSON2_NODE_GATE 25 +#define LOONGSON2_EMMC_GATE 26 +#define LOONGSON2_PIX0_GATE 27 +#define LOONGSON2_PIX1_GATE 28 +#define LOONGSON2_OUT0_CLK 29 +#define LOONGSON2_RIO_CLK 30 +#define LOONGSON2_EMMC_CLK 31 +#define LOONGSON2_DES_CLK 32 +#define LOONGSON2_I2S_CLK 33 +#define LOONGSON2_MISC_CLK 34 #endif From 9796ec0bd04bb0e70487127d44949ca0554df5d3 Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Thu, 11 Apr 2024 10:58:07 +0800 Subject: [PATCH 0119/1702] clk: clk-loongson2: Refactor driver for adding new platforms The driver only supported loongson-2K1000 at first, but the clock structure of loongson-2K0500 and loongson-2K2000 are actually similar, and I tried to refactor the whole driver to adjust to the addition of the new platform. Briefly, I have divided all clocks into three categories according to their properties and their parent clocks: Independent PLLs, clocks based on frequency scales, and clock dividers. Signed-off-by: Binbin Zhou Link: https://lore.kernel.org/r/fb020d1ca19e6f4cdcc95c87b2748869ca76b8ec.1712731524.git.zhoubinbin@loongson.cn Acked-by: Huacai Chen [sboyd@kernel.org: Drop clk.h include] Signed-off-by: Stephen Boyd --- drivers/clk/clk-loongson2.c | 458 ++++++++++++++++-------------------- 1 file changed, 198 insertions(+), 260 deletions(-) diff --git a/drivers/clk/clk-loongson2.c b/drivers/clk/clk-loongson2.c index bacdcbb287ac6..749cf66aef2d4 100644 --- a/drivers/clk/clk-loongson2.c +++ b/drivers/clk/clk-loongson2.c @@ -13,317 +13,254 @@ #include #include -#define LOONGSON2_PLL_MULT_SHIFT 32 -#define LOONGSON2_PLL_MULT_WIDTH 10 -#define LOONGSON2_PLL_DIV_SHIFT 26 -#define LOONGSON2_PLL_DIV_WIDTH 6 -#define LOONGSON2_APB_FREQSCALE_SHIFT 20 -#define LOONGSON2_APB_FREQSCALE_WIDTH 3 -#define LOONGSON2_USB_FREQSCALE_SHIFT 16 -#define LOONGSON2_USB_FREQSCALE_WIDTH 3 -#define LOONGSON2_SATA_FREQSCALE_SHIFT 12 -#define LOONGSON2_SATA_FREQSCALE_WIDTH 3 -#define LOONGSON2_BOOT_FREQSCALE_SHIFT 8 -#define LOONGSON2_BOOT_FREQSCALE_WIDTH 3 - -static void __iomem *loongson2_pll_base; - static const struct clk_parent_data pdata[] = { - { .fw_name = "ref_100m",}, + { .fw_name = "ref_100m", }, }; -static struct clk_hw *loongson2_clk_register(struct device *dev, - const char *name, - const char *parent_name, - const struct clk_ops *ops, - unsigned long flags) -{ - int ret; - struct clk_hw *hw; - struct clk_init_data init = { }; - - hw = devm_kzalloc(dev, sizeof(*hw), GFP_KERNEL); - if (!hw) - return ERR_PTR(-ENOMEM); - - init.name = name; - init.ops = ops; - init.flags = flags; - init.num_parents = 1; - - if (!parent_name) - init.parent_data = pdata; - else - init.parent_names = &parent_name; - - hw->init = &init; - - ret = devm_clk_hw_register(dev, hw); - if (ret) - hw = ERR_PTR(ret); - - return hw; -} - -static unsigned long loongson2_calc_pll_rate(int offset, unsigned long rate) -{ - u64 val; - u32 mult, div; - - val = readq(loongson2_pll_base + offset); - - mult = (val >> LOONGSON2_PLL_MULT_SHIFT) & - clk_div_mask(LOONGSON2_PLL_MULT_WIDTH); - div = (val >> LOONGSON2_PLL_DIV_SHIFT) & - clk_div_mask(LOONGSON2_PLL_DIV_WIDTH); - - return div_u64((u64)rate * mult, div); -} - -static unsigned long loongson2_node_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - return loongson2_calc_pll_rate(0x0, parent_rate); -} - -static const struct clk_ops loongson2_node_clk_ops = { - .recalc_rate = loongson2_node_recalc_rate, +enum loongson2_clk_type { + CLK_TYPE_PLL, + CLK_TYPE_SCALE, + CLK_TYPE_DIVIDER, + CLK_TYPE_NONE, }; -static unsigned long loongson2_ddr_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - return loongson2_calc_pll_rate(0x10, parent_rate); -} +struct loongson2_clk_provider { + void __iomem *base; + struct device *dev; + struct clk_hw_onecell_data clk_data; + spinlock_t clk_lock; /* protect access to DIV registers */ +}; -static const struct clk_ops loongson2_ddr_clk_ops = { - .recalc_rate = loongson2_ddr_recalc_rate, +struct loongson2_clk_data { + struct clk_hw hw; + void __iomem *reg; + u8 div_shift; + u8 div_width; + u8 mult_shift; + u8 mult_width; }; -static unsigned long loongson2_dc_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - return loongson2_calc_pll_rate(0x20, parent_rate); -} +struct loongson2_clk_board_info { + u8 id; + enum loongson2_clk_type type; + const char *name; + const char *parent_name; + u8 reg_offset; + u8 div_shift; + u8 div_width; + u8 mult_shift; + u8 mult_width; +}; -static const struct clk_ops loongson2_dc_clk_ops = { - .recalc_rate = loongson2_dc_recalc_rate, +#define CLK_DIV(_id, _name, _pname, _offset, _dshift, _dwidth) \ + { \ + .id = _id, \ + .type = CLK_TYPE_DIVIDER, \ + .name = _name, \ + .parent_name = _pname, \ + .reg_offset = _offset, \ + .div_shift = _dshift, \ + .div_width = _dwidth, \ + } + +#define CLK_PLL(_id, _name, _offset, _mshift, _mwidth, \ + _dshift, _dwidth) \ + { \ + .id = _id, \ + .type = CLK_TYPE_PLL, \ + .name = _name, \ + .parent_name = NULL, \ + .reg_offset = _offset, \ + .mult_shift = _mshift, \ + .mult_width = _mwidth, \ + .div_shift = _dshift, \ + .div_width = _dwidth, \ + } + +#define CLK_SCALE(_id, _name, _pname, _offset, \ + _dshift, _dwidth) \ + { \ + .id = _id, \ + .type = CLK_TYPE_SCALE, \ + .name = _name, \ + .parent_name = _pname, \ + .reg_offset = _offset, \ + .div_shift = _dshift, \ + .div_width = _dwidth, \ + } + +static const struct loongson2_clk_board_info ls2k1000_clks[] = { + CLK_PLL(LOONGSON2_NODE_PLL, "pll_node", 0, 32, 10, 26, 6), + CLK_PLL(LOONGSON2_DDR_PLL, "pll_ddr", 0x10, 32, 10, 26, 6), + CLK_PLL(LOONGSON2_DC_PLL, "pll_dc", 0x20, 32, 10, 26, 6), + CLK_PLL(LOONGSON2_PIX0_PLL, "pll_pix0", 0x30, 32, 10, 26, 6), + CLK_PLL(LOONGSON2_PIX1_PLL, "pll_pix1", 0x40, 32, 10, 26, 6), + CLK_DIV(LOONGSON2_NODE_CLK, "clk_node", "pll_node", 0x8, 0, 6), + CLK_DIV(LOONGSON2_DDR_CLK, "clk_ddr", "pll_ddr", 0x18, 0, 6), + CLK_DIV(LOONGSON2_GPU_CLK, "clk_gpu", "pll_ddr", 0x18, 22, 6), + /* + * The hda clk divisor in the upper 32bits and the clk-prodiver + * layer code doesn't support 64bit io operation thus a conversion + * is required that subtract shift by 32 and add 4byte to the hda + * address + */ + CLK_DIV(LOONGSON2_HDA_CLK, "clk_hda", "pll_ddr", 0x22, 12, 7), + CLK_DIV(LOONGSON2_DC_CLK, "clk_dc", "pll_dc", 0x28, 0, 6), + CLK_DIV(LOONGSON2_GMAC_CLK, "clk_gmac", "pll_dc", 0x28, 22, 6), + CLK_DIV(LOONGSON2_PIX0_CLK, "clk_pix0", "pll_pix0", 0x38, 0, 6), + CLK_DIV(LOONGSON2_PIX1_CLK, "clk_pix1", "pll_pix1", 0x38, 0, 6), + CLK_SCALE(LOONGSON2_BOOT_CLK, "clk_boot", NULL, 0x50, 8, 3), + CLK_SCALE(LOONGSON2_SATA_CLK, "clk_sata", "clk_gmac", 0x50, 12, 3), + CLK_SCALE(LOONGSON2_USB_CLK, "clk_usb", "clk_gmac", 0x50, 16, 3), + CLK_SCALE(LOONGSON2_APB_CLK, "clk_apb", "clk_gmac", 0x50, 20, 3), + { /* Sentinel */ }, }; -static unsigned long loongson2_pix0_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) +static inline struct loongson2_clk_data *to_loongson2_clk(struct clk_hw *hw) { - return loongson2_calc_pll_rate(0x30, parent_rate); + return container_of(hw, struct loongson2_clk_data, hw); } -static const struct clk_ops loongson2_pix0_clk_ops = { - .recalc_rate = loongson2_pix0_recalc_rate, -}; - -static unsigned long loongson2_pix1_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) +static inline unsigned long loongson2_rate_part(u64 val, u8 shift, u8 width) { - return loongson2_calc_pll_rate(0x40, parent_rate); + return (val & GENMASK(shift + width - 1, shift)) >> shift; } -static const struct clk_ops loongson2_pix1_clk_ops = { - .recalc_rate = loongson2_pix1_recalc_rate, -}; - -static unsigned long loongson2_calc_rate(unsigned long rate, - int shift, int width) +static unsigned long loongson2_pll_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) { - u64 val; - u32 mult; - - val = readq(loongson2_pll_base + 0x50); + u64 val, mult, div; + struct loongson2_clk_data *clk = to_loongson2_clk(hw); - mult = (val >> shift) & clk_div_mask(width); + val = readq(clk->reg); + mult = loongson2_rate_part(val, clk->mult_shift, clk->mult_width); + div = loongson2_rate_part(val, clk->div_shift, clk->div_width); - return div_u64((u64)rate * (mult + 1), 8); -} - -static unsigned long loongson2_boot_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - return loongson2_calc_rate(parent_rate, - LOONGSON2_BOOT_FREQSCALE_SHIFT, - LOONGSON2_BOOT_FREQSCALE_WIDTH); + return div_u64((u64)parent_rate * mult, div); } -static const struct clk_ops loongson2_boot_clk_ops = { - .recalc_rate = loongson2_boot_recalc_rate, +static const struct clk_ops loongson2_pll_recalc_ops = { + .recalc_rate = loongson2_pll_recalc_rate, }; -static unsigned long loongson2_apb_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) +static unsigned long loongson2_freqscale_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) { - return loongson2_calc_rate(parent_rate, - LOONGSON2_APB_FREQSCALE_SHIFT, - LOONGSON2_APB_FREQSCALE_WIDTH); -} + u64 val, mult; + struct loongson2_clk_data *clk = to_loongson2_clk(hw); -static const struct clk_ops loongson2_apb_clk_ops = { - .recalc_rate = loongson2_apb_recalc_rate, -}; + val = readq(clk->reg); + mult = loongson2_rate_part(val, clk->div_shift, clk->div_width) + 1; -static unsigned long loongson2_usb_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - return loongson2_calc_rate(parent_rate, - LOONGSON2_USB_FREQSCALE_SHIFT, - LOONGSON2_USB_FREQSCALE_WIDTH); + return div_u64((u64)parent_rate * mult, 8); } -static const struct clk_ops loongson2_usb_clk_ops = { - .recalc_rate = loongson2_usb_recalc_rate, +static const struct clk_ops loongson2_freqscale_recalc_ops = { + .recalc_rate = loongson2_freqscale_recalc_rate, }; -static unsigned long loongson2_sata_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) +static struct clk_hw *loongson2_clk_register(struct loongson2_clk_provider *clp, + const struct loongson2_clk_board_info *cld, + const struct clk_ops *ops) { - return loongson2_calc_rate(parent_rate, - LOONGSON2_SATA_FREQSCALE_SHIFT, - LOONGSON2_SATA_FREQSCALE_WIDTH); -} + int ret; + struct clk_hw *hw; + struct loongson2_clk_data *clk; + struct clk_init_data init = { }; -static const struct clk_ops loongson2_sata_clk_ops = { - .recalc_rate = loongson2_sata_recalc_rate, -}; + clk = devm_kzalloc(clp->dev, sizeof(*clk), GFP_KERNEL); + if (!clk) + return ERR_PTR(-ENOMEM); -static inline int loongson2_check_clk_hws(struct clk_hw *clks[], unsigned int count) -{ - unsigned int i; + init.name = cld->name; + init.ops = ops; + init.flags = 0; + init.num_parents = 1; - for (i = 0; i < count; i++) - if (IS_ERR(clks[i])) { - pr_err("Loongson2 clk %u: register failed with %ld\n", - i, PTR_ERR(clks[i])); - return PTR_ERR(clks[i]); - } + if (!cld->parent_name) + init.parent_data = pdata; + else + init.parent_names = &cld->parent_name; + + clk->reg = clp->base + cld->reg_offset; + clk->div_shift = cld->div_shift; + clk->div_width = cld->div_width; + clk->mult_shift = cld->mult_shift; + clk->mult_width = cld->mult_width; + clk->hw.init = &init; - return 0; + hw = &clk->hw; + ret = devm_clk_hw_register(clp->dev, hw); + if (ret) + clk = ERR_PTR(ret); + + return hw; } static int loongson2_clk_probe(struct platform_device *pdev) { - int ret; - struct clk_hw **hws; - struct clk_hw_onecell_data *clk_hw_data; - spinlock_t loongson2_clk_lock; + int i, clks_num = 0; + struct clk_hw *hw; struct device *dev = &pdev->dev; + struct loongson2_clk_provider *clp; + const struct loongson2_clk_board_info *p, *data; - loongson2_pll_base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(loongson2_pll_base)) - return PTR_ERR(loongson2_pll_base); - - clk_hw_data = devm_kzalloc(dev, struct_size(clk_hw_data, hws, LOONGSON2_CLK_END), - GFP_KERNEL); - if (WARN_ON(!clk_hw_data)) - return -ENOMEM; - - clk_hw_data->num = LOONGSON2_CLK_END; - hws = clk_hw_data->hws; - - hws[LOONGSON2_NODE_PLL] = loongson2_clk_register(dev, "node_pll", - NULL, - &loongson2_node_clk_ops, 0); - - hws[LOONGSON2_DDR_PLL] = loongson2_clk_register(dev, "ddr_pll", - NULL, - &loongson2_ddr_clk_ops, 0); + data = device_get_match_data(dev); + if (!data) + return -EINVAL; - hws[LOONGSON2_DC_PLL] = loongson2_clk_register(dev, "dc_pll", - NULL, - &loongson2_dc_clk_ops, 0); + for (p = data; p->name; p++) + clks_num++; - hws[LOONGSON2_PIX0_PLL] = loongson2_clk_register(dev, "pix0_pll", - NULL, - &loongson2_pix0_clk_ops, 0); - - hws[LOONGSON2_PIX1_PLL] = loongson2_clk_register(dev, "pix1_pll", - NULL, - &loongson2_pix1_clk_ops, 0); + clp = devm_kzalloc(dev, struct_size(clp, clk_data.hws, clks_num), + GFP_KERNEL); + if (!clp) + return -ENOMEM; - hws[LOONGSON2_BOOT_CLK] = loongson2_clk_register(dev, "boot", - NULL, - &loongson2_boot_clk_ops, 0); + clp->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(clp->base)) + return PTR_ERR(clp->base); + + spin_lock_init(&clp->clk_lock); + clp->clk_data.num = clks_num + 1; + clp->dev = dev; + + for (i = 0; i < clks_num; i++) { + p = &data[i]; + switch (p->type) { + case CLK_TYPE_PLL: + hw = loongson2_clk_register(clp, p, + &loongson2_pll_recalc_ops); + break; + case CLK_TYPE_SCALE: + hw = loongson2_clk_register(clp, p, + &loongson2_freqscale_recalc_ops); + break; + case CLK_TYPE_DIVIDER: + hw = devm_clk_hw_register_divider(dev, p->name, + p->parent_name, 0, + clp->base + p->reg_offset, + p->div_shift, p->div_width, + CLK_DIVIDER_ONE_BASED, + &clp->clk_lock); + break; + default: + return dev_err_probe(dev, -EINVAL, "Invalid clk type\n"); + } - hws[LOONGSON2_NODE_CLK] = devm_clk_hw_register_divider(dev, "node", - "node_pll", 0, - loongson2_pll_base + 0x8, 0, - 6, CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); + if (IS_ERR(hw)) + return dev_err_probe(dev, PTR_ERR(hw), + "Register clk: %s, type: %u failed!\n", + p->name, p->type); - /* - * The hda clk divisor in the upper 32bits and the clk-prodiver - * layer code doesn't support 64bit io operation thus a conversion - * is required that subtract shift by 32 and add 4byte to the hda - * address - */ - hws[LOONGSON2_HDA_CLK] = devm_clk_hw_register_divider(dev, "hda", - "ddr_pll", 0, - loongson2_pll_base + 0x22, 12, - 7, CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); - - hws[LOONGSON2_GPU_CLK] = devm_clk_hw_register_divider(dev, "gpu", - "ddr_pll", 0, - loongson2_pll_base + 0x18, 22, - 6, CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); - - hws[LOONGSON2_DDR_CLK] = devm_clk_hw_register_divider(dev, "ddr", - "ddr_pll", 0, - loongson2_pll_base + 0x18, 0, - 6, CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); - - hws[LOONGSON2_GMAC_CLK] = devm_clk_hw_register_divider(dev, "gmac", - "dc_pll", 0, - loongson2_pll_base + 0x28, 22, - 6, CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); - - hws[LOONGSON2_DC_CLK] = devm_clk_hw_register_divider(dev, "dc", - "dc_pll", 0, - loongson2_pll_base + 0x28, 0, - 6, CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); - - hws[LOONGSON2_APB_CLK] = loongson2_clk_register(dev, "apb", - "gmac", - &loongson2_apb_clk_ops, 0); - - hws[LOONGSON2_USB_CLK] = loongson2_clk_register(dev, "usb", - "gmac", - &loongson2_usb_clk_ops, 0); - - hws[LOONGSON2_SATA_CLK] = loongson2_clk_register(dev, "sata", - "gmac", - &loongson2_sata_clk_ops, 0); - - hws[LOONGSON2_PIX0_CLK] = clk_hw_register_divider(NULL, "pix0", - "pix0_pll", 0, - loongson2_pll_base + 0x38, 0, 6, - CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); - - hws[LOONGSON2_PIX1_CLK] = clk_hw_register_divider(NULL, "pix1", - "pix1_pll", 0, - loongson2_pll_base + 0x48, 0, 6, - CLK_DIVIDER_ONE_BASED, - &loongson2_clk_lock); - - ret = loongson2_check_clk_hws(hws, LOONGSON2_CLK_END); - if (ret) - return ret; + clp->clk_data.hws[p->id] = hw; + } - return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clk_hw_data); + return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, &clp->clk_data); } static const struct of_device_id loongson2_clk_match_table[] = { - { .compatible = "loongson,ls2k-clk" }, + { .compatible = "loongson,ls2k-clk", .data = &ls2k1000_clks }, { } }; MODULE_DEVICE_TABLE(of, loongson2_clk_match_table); @@ -338,4 +275,5 @@ static struct platform_driver loongson2_clk_driver = { module_platform_driver(loongson2_clk_driver); MODULE_DESCRIPTION("Loongson2 clock driver"); +MODULE_AUTHOR("Loongson Technology Corporation Limited"); MODULE_LICENSE("GPL"); From 5aa9d3a79bc66f2cc9ef9e89ffe7f37a96d47ce8 Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Thu, 11 Apr 2024 10:58:08 +0800 Subject: [PATCH 0120/1702] dt-bindings: clock: loongson2: Add Loongson-2K0500 compatible Add the devicetree compatible for Loongson-2K0500 clocks. Signed-off-by: Binbin Zhou Acked-by: Conor Dooley Link: https://lore.kernel.org/r/c4784102d2bb8bf6982799babe39d5827235461d.1712731524.git.zhoubinbin@loongson.cn Acked-by: Huacai Chen Signed-off-by: Stephen Boyd --- Documentation/devicetree/bindings/clock/loongson,ls2k-clk.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/clock/loongson,ls2k-clk.yaml b/Documentation/devicetree/bindings/clock/loongson,ls2k-clk.yaml index 63a59015987e4..83baee40e2000 100644 --- a/Documentation/devicetree/bindings/clock/loongson,ls2k-clk.yaml +++ b/Documentation/devicetree/bindings/clock/loongson,ls2k-clk.yaml @@ -16,7 +16,8 @@ description: | properties: compatible: enum: - - loongson,ls2k-clk + - loongson,ls2k0500-clk + - loongson,ls2k-clk # This is for Loongson-2K1000 reg: maxItems: 1 From 85d4daa59555182f9c8b9f5fe951049a6d0aed0f Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Thu, 11 Apr 2024 10:58:31 +0800 Subject: [PATCH 0121/1702] clk: clk-loongson2: Add Loongson-2K0500 clock support The Loongson-2K0500 and Loongson-2K1000 clock is similar, we add its support by different configurations. Signed-off-by: Binbin Zhou Link: https://lore.kernel.org/r/f767a1783dc590fcd31ff7d4459df2cf80196de1.1712731524.git.zhoubinbin@loongson.cn Acked-by: Huacai Chen Signed-off-by: Stephen Boyd --- drivers/clk/clk-loongson2.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/drivers/clk/clk-loongson2.c b/drivers/clk/clk-loongson2.c index 749cf66aef2d4..190c24b88a8cc 100644 --- a/drivers/clk/clk-loongson2.c +++ b/drivers/clk/clk-loongson2.c @@ -89,6 +89,27 @@ struct loongson2_clk_board_info { .div_width = _dwidth, \ } +static const struct loongson2_clk_board_info ls2k0500_clks[] = { + CLK_PLL(LOONGSON2_NODE_PLL, "pll_node", 0, 16, 8, 8, 6), + CLK_PLL(LOONGSON2_DDR_PLL, "pll_ddr", 0x8, 16, 8, 8, 6), + CLK_PLL(LOONGSON2_DC_PLL, "pll_soc", 0x10, 16, 8, 8, 6), + CLK_PLL(LOONGSON2_PIX0_PLL, "pll_pix0", 0x18, 16, 8, 8, 6), + CLK_PLL(LOONGSON2_PIX1_PLL, "pll_pix1", 0x20, 16, 8, 8, 6), + CLK_DIV(LOONGSON2_NODE_CLK, "clk_node", "pll_node", 0, 24, 6), + CLK_DIV(LOONGSON2_DDR_CLK, "clk_ddr", "pll_ddr", 0x8, 24, 6), + CLK_DIV(LOONGSON2_HDA_CLK, "clk_hda", "pll_ddr", 0xc, 8, 6), + CLK_DIV(LOONGSON2_GPU_CLK, "clk_gpu", "pll_soc", 0x10, 24, 6), + CLK_DIV(LOONGSON2_DC_CLK, "clk_sb", "pll_soc", 0x14, 0, 6), + CLK_DIV(LOONGSON2_GMAC_CLK, "clk_gmac", "pll_soc", 0x14, 8, 6), + CLK_DIV(LOONGSON2_PIX0_CLK, "clk_pix0", "pll_pix0", 0x18, 24, 6), + CLK_DIV(LOONGSON2_PIX1_CLK, "clk_pix1", "pll_pix1", 0x20, 24, 6), + CLK_SCALE(LOONGSON2_BOOT_CLK, "clk_boot", "clk_sb", 0x28, 8, 3), + CLK_SCALE(LOONGSON2_SATA_CLK, "clk_sata", "clk_sb", 0x28, 12, 3), + CLK_SCALE(LOONGSON2_USB_CLK, "clk_usb", "clk_sb", 0x28, 16, 3), + CLK_SCALE(LOONGSON2_APB_CLK, "clk_apb", "clk_sb", 0x28, 20, 3), + { /* Sentinel */ }, +}; + static const struct loongson2_clk_board_info ls2k1000_clks[] = { CLK_PLL(LOONGSON2_NODE_PLL, "pll_node", 0, 32, 10, 26, 6), CLK_PLL(LOONGSON2_DDR_PLL, "pll_ddr", 0x10, 32, 10, 26, 6), @@ -260,6 +281,7 @@ static int loongson2_clk_probe(struct platform_device *pdev) } static const struct of_device_id loongson2_clk_match_table[] = { + { .compatible = "loongson,ls2k0500-clk", .data = &ls2k0500_clks }, { .compatible = "loongson,ls2k-clk", .data = &ls2k1000_clks }, { } }; From 267325629ee3f04b519c4070a0ac65e86354d94f Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Thu, 11 Apr 2024 10:58:32 +0800 Subject: [PATCH 0122/1702] dt-bindings: clock: loongson2: Add Loongson-2K2000 compatible Add the devicetree compatible for Loongson-2K2000 clocks. Signed-off-by: Binbin Zhou Acked-by: Conor Dooley Link: https://lore.kernel.org/r/4820325406aec322ae7c062e2b03437d0c95e820.1712731524.git.zhoubinbin@loongson.cn Acked-by: Huacai Chen Signed-off-by: Stephen Boyd --- Documentation/devicetree/bindings/clock/loongson,ls2k-clk.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/clock/loongson,ls2k-clk.yaml b/Documentation/devicetree/bindings/clock/loongson,ls2k-clk.yaml index 83baee40e2000..4f79cdb417ab6 100644 --- a/Documentation/devicetree/bindings/clock/loongson,ls2k-clk.yaml +++ b/Documentation/devicetree/bindings/clock/loongson,ls2k-clk.yaml @@ -18,6 +18,7 @@ properties: enum: - loongson,ls2k0500-clk - loongson,ls2k-clk # This is for Loongson-2K1000 + - loongson,ls2k2000-clk reg: maxItems: 1 From 4f149dd4786afd5e5a023e96fb988e4f8b44e8ed Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Thu, 11 Apr 2024 10:58:33 +0800 Subject: [PATCH 0123/1702] clk: clk-loongson2: Add Loongson-2K2000 clock support The Loongson-2K2000 and Loongson-2K1000 clock is similar, we add its support by different configurations. Signed-off-by: Binbin Zhou Link: https://lore.kernel.org/r/8b0b5851783acf8ebe13b50391d15b58cc181613.1712731524.git.zhoubinbin@loongson.cn Acked-by: Huacai Chen Signed-off-by: Stephen Boyd --- drivers/clk/clk-loongson2.c | 72 +++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/drivers/clk/clk-loongson2.c b/drivers/clk/clk-loongson2.c index 190c24b88a8cc..820bb1e9e3b79 100644 --- a/drivers/clk/clk-loongson2.c +++ b/drivers/clk/clk-loongson2.c @@ -21,6 +21,8 @@ enum loongson2_clk_type { CLK_TYPE_PLL, CLK_TYPE_SCALE, CLK_TYPE_DIVIDER, + CLK_TYPE_GATE, + CLK_TYPE_FIXED, CLK_TYPE_NONE, }; @@ -45,11 +47,13 @@ struct loongson2_clk_board_info { enum loongson2_clk_type type; const char *name; const char *parent_name; + unsigned long fixed_rate; u8 reg_offset; u8 div_shift; u8 div_width; u8 mult_shift; u8 mult_width; + u8 bit_idx; }; #define CLK_DIV(_id, _name, _pname, _offset, _dshift, _dwidth) \ @@ -89,6 +93,25 @@ struct loongson2_clk_board_info { .div_width = _dwidth, \ } +#define CLK_GATE(_id, _name, _pname, _offset, _bidx) \ + { \ + .id = _id, \ + .type = CLK_TYPE_GATE, \ + .name = _name, \ + .parent_name = _pname, \ + .reg_offset = _offset, \ + .bit_idx = _bidx, \ + } + +#define CLK_FIXED(_id, _name, _pname, _rate) \ + { \ + .id = _id, \ + .type = CLK_TYPE_FIXED, \ + .name = _name, \ + .parent_name = _pname, \ + .fixed_rate = _rate, \ + } + static const struct loongson2_clk_board_info ls2k0500_clks[] = { CLK_PLL(LOONGSON2_NODE_PLL, "pll_node", 0, 16, 8, 8, 6), CLK_PLL(LOONGSON2_DDR_PLL, "pll_ddr", 0x8, 16, 8, 8, 6), @@ -137,6 +160,44 @@ static const struct loongson2_clk_board_info ls2k1000_clks[] = { { /* Sentinel */ }, }; +static const struct loongson2_clk_board_info ls2k2000_clks[] = { + CLK_PLL(LOONGSON2_DC_PLL, "pll_0", 0, 21, 9, 32, 6), + CLK_PLL(LOONGSON2_DDR_PLL, "pll_1", 0x10, 21, 9, 32, 6), + CLK_PLL(LOONGSON2_NODE_PLL, "pll_2", 0x20, 21, 9, 32, 6), + CLK_PLL(LOONGSON2_PIX0_PLL, "pll_pix0", 0x30, 21, 9, 32, 6), + CLK_PLL(LOONGSON2_PIX1_PLL, "pll_pix1", 0x40, 21, 9, 32, 6), + CLK_GATE(LOONGSON2_OUT0_GATE, "out0_gate", "pll_0", 0, 40), + CLK_GATE(LOONGSON2_GMAC_GATE, "gmac_gate", "pll_0", 0, 41), + CLK_GATE(LOONGSON2_RIO_GATE, "rio_gate", "pll_0", 0, 42), + CLK_GATE(LOONGSON2_DC_GATE, "dc_gate", "pll_1", 0x10, 40), + CLK_GATE(LOONGSON2_DDR_GATE, "ddr_gate", "pll_1", 0x10, 41), + CLK_GATE(LOONGSON2_GPU_GATE, "gpu_gate", "pll_1", 0x10, 42), + CLK_GATE(LOONGSON2_HDA_GATE, "hda_gate", "pll_2", 0x20, 40), + CLK_GATE(LOONGSON2_NODE_GATE, "node_gate", "pll_2", 0x20, 41), + CLK_GATE(LOONGSON2_EMMC_GATE, "emmc_gate", "pll_2", 0x20, 42), + CLK_GATE(LOONGSON2_PIX0_GATE, "pix0_gate", "pll_pix0", 0x30, 40), + CLK_GATE(LOONGSON2_PIX1_GATE, "pix1_gate", "pll_pix1", 0x40, 40), + CLK_DIV(LOONGSON2_OUT0_CLK, "clk_out0", "out0_gate", 0, 0, 6), + CLK_DIV(LOONGSON2_GMAC_CLK, "clk_gmac", "gmac_gate", 0, 7, 6), + CLK_DIV(LOONGSON2_RIO_CLK, "clk_rio", "rio_gate", 0, 14, 6), + CLK_DIV(LOONGSON2_DC_CLK, "clk_dc", "dc_gate", 0x10, 0, 6), + CLK_DIV(LOONGSON2_GPU_CLK, "clk_gpu", "gpu_gate", 0x10, 7, 6), + CLK_DIV(LOONGSON2_DDR_CLK, "clk_ddr", "ddr_gate", 0x10, 14, 6), + CLK_DIV(LOONGSON2_HDA_CLK, "clk_hda", "hda_gate", 0x20, 0, 6), + CLK_DIV(LOONGSON2_NODE_CLK, "clk_node", "node_gate", 0x20, 7, 6), + CLK_DIV(LOONGSON2_EMMC_CLK, "clk_emmc", "emmc_gate", 0x20, 14, 6), + CLK_DIV(LOONGSON2_PIX0_CLK, "clk_pix0", "pll_pix0", 0x30, 0, 6), + CLK_DIV(LOONGSON2_PIX1_CLK, "clk_pix1", "pll_pix1", 0x40, 0, 6), + CLK_SCALE(LOONGSON2_SATA_CLK, "clk_sata", "clk_out0", 0x50, 12, 3), + CLK_SCALE(LOONGSON2_USB_CLK, "clk_usb", "clk_out0", 0x50, 16, 3), + CLK_SCALE(LOONGSON2_APB_CLK, "clk_apb", "clk_node", 0x50, 20, 3), + CLK_SCALE(LOONGSON2_BOOT_CLK, "clk_boot", NULL, 0x50, 23, 3), + CLK_SCALE(LOONGSON2_DES_CLK, "clk_des", "clk_node", 0x50, 40, 3), + CLK_SCALE(LOONGSON2_I2S_CLK, "clk_i2s", "clk_node", 0x50, 44, 3), + CLK_FIXED(LOONGSON2_MISC_CLK, "clk_misc", NULL, 50000000), + { /* Sentinel */ }, +}; + static inline struct loongson2_clk_data *to_loongson2_clk(struct clk_hw *hw) { return container_of(hw, struct loongson2_clk_data, hw); @@ -265,6 +326,16 @@ static int loongson2_clk_probe(struct platform_device *pdev) CLK_DIVIDER_ONE_BASED, &clp->clk_lock); break; + case CLK_TYPE_GATE: + hw = devm_clk_hw_register_gate(dev, p->name, p->parent_name, 0, + clp->base + p->reg_offset, + p->bit_idx, 0, + &clp->clk_lock); + break; + case CLK_TYPE_FIXED: + hw = clk_hw_register_fixed_rate_parent_data(dev, p->name, pdata, + 0, p->fixed_rate); + break; default: return dev_err_probe(dev, -EINVAL, "Invalid clk type\n"); } @@ -283,6 +354,7 @@ static int loongson2_clk_probe(struct platform_device *pdev) static const struct of_device_id loongson2_clk_match_table[] = { { .compatible = "loongson,ls2k0500-clk", .data = &ls2k0500_clks }, { .compatible = "loongson,ls2k-clk", .data = &ls2k1000_clks }, + { .compatible = "loongson,ls2k2000-clk", .data = &ls2k2000_clks }, { } }; MODULE_DEVICE_TABLE(of, loongson2_clk_match_table); From c8fc935f4b198dc6e9871b29f4f3360631d90c8e Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Tue, 9 Apr 2024 07:21:05 -0700 Subject: [PATCH 0124/1702] RDMA/mana_ib: remove useless return values from dbg prints Remove printing ret value on success as it was always 0. Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1712672465-29960-1-git-send-email-kotaranov@linux.microsoft.com Reviewed-by: Long Li Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/main.c | 4 +--- drivers/infiniband/hw/mana/mr.c | 2 +- drivers/infiniband/hw/mana/qp.c | 6 +++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index 4524c6b807487..b31dcff326994 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -261,9 +261,7 @@ int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, } queue->umem = umem; - ibdev_dbg(&mdev->ib_dev, - "create_dma_region ret %d gdma_region 0x%llx\n", - err, queue->gdma_region); + ibdev_dbg(&mdev->ib_dev, "created dma region 0x%llx\n", queue->gdma_region); return 0; free_umem: diff --git a/drivers/infiniband/hw/mana/mr.c b/drivers/infiniband/hw/mana/mr.c index b70b13484f097..4f13423ecdbdf 100644 --- a/drivers/infiniband/hw/mana/mr.c +++ b/drivers/infiniband/hw/mana/mr.c @@ -135,7 +135,7 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, } ibdev_dbg(ibdev, - "create_dma_region ret %d gdma_region %llx\n", err, + "created dma region for user-mr 0x%llx\n", dma_region_handle); mr_params.pd_handle = pd->pd_handle; diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 4cd8f8afe80d1..8fedf6e019251 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -217,8 +217,8 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, cq->queue.id = cq_spec.queue_index; ibdev_dbg(&mdev->ib_dev, - "ret %d rx_object 0x%llx wq id %llu cq id %llu\n", - ret, wq->rx_object, wq->queue.id, cq->queue.id); + "rx_object 0x%llx wq id %llu cq id %llu\n", + wq->rx_object, wq->queue.id, cq->queue.id); resp.entries[i].cqid = cq->queue.id; resp.entries[i].wqid = wq->queue.id; @@ -383,7 +383,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, goto err_destroy_wq_obj; ibdev_dbg(&mdev->ib_dev, - "ret %d qp->qp_handle 0x%llx sq id %llu cq id %llu\n", err, + "qp->qp_handle 0x%llx sq id %llu cq id %llu\n", qp->qp_handle, qp->raw_sq.id, send_cq->queue.id); resp.sqid = qp->raw_sq.id; From dfcdb38b21e4fb92a49acdbdf6afa82c07c8eba0 Mon Sep 17 00:00:00 2001 From: Zhu Yanjun Date: Mon, 8 Apr 2024 16:21:42 +0200 Subject: [PATCH 0125/1702] RDMA/rxe: Return the correct errno In the function __rxe_add_to_pool, the function xa_alloc_cyclic is called. The return value of the function xa_alloc_cyclic is as below: " Return: 0 if the allocation succeeded without wrapping. 1 if the allocation succeeded after wrapping, -ENOMEM if memory could not be allocated or -EBUSY if there are no free entries in @limit. " But now the function __rxe_add_to_pool only returns -EINVAL. All the returned error value should be returned to the caller. Signed-off-by: Zhu Yanjun Link: https://lore.kernel.org/r/20240408142142.792413-1-yanjun.zhu@linux.dev Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_pool.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c index 6215c6de3a840..67567d62195e8 100644 --- a/drivers/infiniband/sw/rxe/rxe_pool.c +++ b/drivers/infiniband/sw/rxe/rxe_pool.c @@ -119,7 +119,7 @@ void rxe_pool_cleanup(struct rxe_pool *pool) int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem, bool sleepable) { - int err; + int err = -EINVAL; gfp_t gfp_flags; if (atomic_inc_return(&pool->num_elem) > pool->max_elem) @@ -147,7 +147,7 @@ int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem, err_cnt: atomic_dec(&pool->num_elem); - return -EINVAL; + return err; } void *rxe_pool_get_index(struct rxe_pool *pool, u32 index) From 3cef9e08b6f43fcc7b7802a1f01bb7d9334d6b6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Tue, 13 Feb 2024 14:00:43 +0100 Subject: [PATCH 0126/1702] dt-bindings: usb: mtk-xhci: add compatible for MT7988 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MT7988 SoC contains two on-SoC XHCI controllers. Add proper binding. Signed-off-by: Rafał Miłecki Acked-by: Conor Dooley Reviewed-by: AngeloGioacchino Del Regno Link: https://lore.kernel.org/r/20240213130044.1976-1-zajec5@gmail.com Signed-off-by: Rob Herring --- Documentation/devicetree/bindings/usb/mediatek,mtk-xhci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/usb/mediatek,mtk-xhci.yaml b/Documentation/devicetree/bindings/usb/mediatek,mtk-xhci.yaml index 924fd3d748a88..ef3143f4b794c 100644 --- a/Documentation/devicetree/bindings/usb/mediatek,mtk-xhci.yaml +++ b/Documentation/devicetree/bindings/usb/mediatek,mtk-xhci.yaml @@ -29,6 +29,7 @@ properties: - mediatek,mt7623-xhci - mediatek,mt7629-xhci - mediatek,mt7986-xhci + - mediatek,mt7988-xhci - mediatek,mt8173-xhci - mediatek,mt8183-xhci - mediatek,mt8186-xhci From a12069a39b33c3b4c57929f5b42c88da681496ba Mon Sep 17 00:00:00 2001 From: Inochi Amaoto Date: Fri, 12 Apr 2024 07:24:38 +0800 Subject: [PATCH 0127/1702] clk: sophgo: Make synthesizer struct static Let all synthesizer structs are static to make the compiler happy. Fixes: 80fd61ec4612 ("clk: sophgo: Add clock support for CV1800 SoC") Signed-off-by: Inochi Amaoto Link: https://lore.kernel.org/r/IA1PR20MB49531E437735A71A163694AEBB052@IA1PR20MB4953.namprd20.prod.outlook.com Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202404120548.y9dZIi0e-lkp@intel.com/ Signed-off-by: Stephen Boyd --- drivers/clk/sophgo/clk-cv1800.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/clk/sophgo/clk-cv1800.c b/drivers/clk/sophgo/clk-cv1800.c index 956de5b21a80a..2da4c24621cfa 100644 --- a/drivers/clk/sophgo/clk-cv1800.c +++ b/drivers/clk/sophgo/clk-cv1800.c @@ -79,7 +79,7 @@ static const struct clk_parent_data clk_bypass_fpll_parents[] = { { .hw = &clk_fpll.common.hw }, }; -struct cv1800_clk_pll_synthesizer clk_mpll_synthesizer = { +static struct cv1800_clk_pll_synthesizer clk_mpll_synthesizer = { .en = CV1800_CLK_BIT(REG_PLL_G6_SSC_SYN_CTRL, 2), .clk_half = CV1800_CLK_BIT(REG_PLL_G6_SSC_SYN_CTRL, 0), .ctrl = REG_MPLL_SSC_SYN_CTRL, @@ -93,7 +93,7 @@ static CV1800_FACTIONAL_PLL(clk_mpll, clk_bypass_mipimpll_parents, &clk_mpll_synthesizer, CLK_IS_CRITICAL); -struct cv1800_clk_pll_synthesizer clk_tpll_synthesizer = { +static struct cv1800_clk_pll_synthesizer clk_tpll_synthesizer = { .en = CV1800_CLK_BIT(REG_PLL_G6_SSC_SYN_CTRL, 3), .clk_half = CV1800_CLK_BIT(REG_PLL_G6_SSC_SYN_CTRL, 0), .ctrl = REG_TPLL_SSC_SYN_CTRL, @@ -107,7 +107,7 @@ static CV1800_FACTIONAL_PLL(clk_tpll, clk_bypass_mipimpll_parents, &clk_tpll_synthesizer, CLK_IS_CRITICAL); -struct cv1800_clk_pll_synthesizer clk_a0pll_synthesizer = { +static struct cv1800_clk_pll_synthesizer clk_a0pll_synthesizer = { .en = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 2), .clk_half = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 0), .ctrl = REG_A0PLL_SSC_SYN_CTRL, @@ -121,7 +121,7 @@ static CV1800_FACTIONAL_PLL(clk_a0pll, clk_bypass_mipimpll_parents, &clk_a0pll_synthesizer, CLK_IS_CRITICAL); -struct cv1800_clk_pll_synthesizer clk_disppll_synthesizer = { +static struct cv1800_clk_pll_synthesizer clk_disppll_synthesizer = { .en = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 3), .clk_half = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 0), .ctrl = REG_DISPPLL_SSC_SYN_CTRL, @@ -135,7 +135,7 @@ static CV1800_FACTIONAL_PLL(clk_disppll, clk_bypass_mipimpll_parents, &clk_disppll_synthesizer, CLK_IS_CRITICAL); -struct cv1800_clk_pll_synthesizer clk_cam0pll_synthesizer = { +static struct cv1800_clk_pll_synthesizer clk_cam0pll_synthesizer = { .en = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 4), .clk_half = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 0), .ctrl = REG_CAM0PLL_SSC_SYN_CTRL, @@ -149,7 +149,7 @@ static CV1800_FACTIONAL_PLL(clk_cam0pll, clk_bypass_mipimpll_parents, &clk_cam0pll_synthesizer, CLK_IGNORE_UNUSED); -struct cv1800_clk_pll_synthesizer clk_cam1pll_synthesizer = { +static struct cv1800_clk_pll_synthesizer clk_cam1pll_synthesizer = { .en = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 5), .clk_half = CV1800_CLK_BIT(REG_PLL_G2_SSC_SYN_CTRL, 0), .ctrl = REG_CAM1PLL_SSC_SYN_CTRL, From 239d5fb3ba79b1c2aa257569e55739a07c161c37 Mon Sep 17 00:00:00 2001 From: Gabriel Fernandez Date: Thu, 11 Apr 2024 11:24:50 +0200 Subject: [PATCH 0128/1702] clk: stm32mp13: use platform device APIs Convert devm_platform_ioremap_resource() and remove unnecessary dependency check with SCMI clock driver. Signed-off-by: Gabriel Fernandez Link: https://lore.kernel.org/r/20240411092453.243633-2-gabriel.fernandez@foss.st.com Signed-off-by: Stephen Boyd --- drivers/clk/stm32/clk-stm32-core.c | 11 +++-- drivers/clk/stm32/clk-stm32mp13.c | 72 +++--------------------------- 2 files changed, 10 insertions(+), 73 deletions(-) diff --git a/drivers/clk/stm32/clk-stm32-core.c b/drivers/clk/stm32/clk-stm32-core.c index 58705fcad334d..1721a3ed73867 100644 --- a/drivers/clk/stm32/clk-stm32-core.c +++ b/drivers/clk/stm32/clk-stm32-core.c @@ -25,7 +25,6 @@ static int stm32_rcc_clock_init(struct device *dev, { const struct stm32_rcc_match_data *data = match->data; struct clk_hw_onecell_data *clk_data = data->hw_clks; - struct device_node *np = dev_of_node(dev); struct clk_hw **hws; int n, max_binding; @@ -64,7 +63,7 @@ static int stm32_rcc_clock_init(struct device *dev, hws[cfg_clock->id] = hw; } - return of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_data); + return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clk_data); } int stm32_rcc_init(struct device *dev, const struct of_device_id *match_data, @@ -638,7 +637,7 @@ struct clk_hw *clk_stm32_mux_register(struct device *dev, mux->lock = lock; mux->clock_data = data->clock_data; - err = clk_hw_register(dev, hw); + err = devm_clk_hw_register(dev, hw); if (err) return ERR_PTR(err); @@ -659,7 +658,7 @@ struct clk_hw *clk_stm32_gate_register(struct device *dev, gate->lock = lock; gate->clock_data = data->clock_data; - err = clk_hw_register(dev, hw); + err = devm_clk_hw_register(dev, hw); if (err) return ERR_PTR(err); @@ -680,7 +679,7 @@ struct clk_hw *clk_stm32_div_register(struct device *dev, div->lock = lock; div->clock_data = data->clock_data; - err = clk_hw_register(dev, hw); + err = devm_clk_hw_register(dev, hw); if (err) return ERR_PTR(err); @@ -701,7 +700,7 @@ struct clk_hw *clk_stm32_composite_register(struct device *dev, composite->lock = lock; composite->clock_data = data->clock_data; - err = clk_hw_register(dev, hw); + err = devm_clk_hw_register(dev, hw); if (err) return ERR_PTR(err); diff --git a/drivers/clk/stm32/clk-stm32mp13.c b/drivers/clk/stm32/clk-stm32mp13.c index d4ecb3c34a1b2..bf81d74917088 100644 --- a/drivers/clk/stm32/clk-stm32mp13.c +++ b/drivers/clk/stm32/clk-stm32mp13.c @@ -1536,77 +1536,16 @@ static const struct of_device_id stm32mp13_match_data[] = { }; MODULE_DEVICE_TABLE(of, stm32mp13_match_data); -static int stm32mp1_rcc_init(struct device *dev) -{ - void __iomem *rcc_base; - int ret = -ENOMEM; - - rcc_base = of_iomap(dev_of_node(dev), 0); - if (!rcc_base) { - dev_err(dev, "%pOFn: unable to map resource", dev_of_node(dev)); - goto out; - } - - ret = stm32_rcc_init(dev, stm32mp13_match_data, rcc_base); -out: - if (ret) { - if (rcc_base) - iounmap(rcc_base); - - of_node_put(dev_of_node(dev)); - } - - return ret; -} - -static int get_clock_deps(struct device *dev) -{ - static const char * const clock_deps_name[] = { - "hsi", "hse", "csi", "lsi", "lse", - }; - size_t deps_size = sizeof(struct clk *) * ARRAY_SIZE(clock_deps_name); - struct clk **clk_deps; - int i; - - clk_deps = devm_kzalloc(dev, deps_size, GFP_KERNEL); - if (!clk_deps) - return -ENOMEM; - - for (i = 0; i < ARRAY_SIZE(clock_deps_name); i++) { - struct clk *clk = of_clk_get_by_name(dev_of_node(dev), - clock_deps_name[i]); - - if (IS_ERR(clk)) { - if (PTR_ERR(clk) != -EINVAL && PTR_ERR(clk) != -ENOENT) - return PTR_ERR(clk); - } else { - /* Device gets a reference count on the clock */ - clk_deps[i] = devm_clk_get(dev, __clk_get_name(clk)); - clk_put(clk); - } - } - - return 0; -} - static int stm32mp1_rcc_clocks_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - int ret = get_clock_deps(dev); + void __iomem *base; - if (!ret) - ret = stm32mp1_rcc_init(dev); - - return ret; -} - -static void stm32mp1_rcc_clocks_remove(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct device_node *child, *np = dev_of_node(dev); + base = devm_platform_ioremap_resource(pdev, 0); + if (WARN_ON(IS_ERR(base))) + return PTR_ERR(base); - for_each_available_child_of_node(np, child) - of_clk_del_provider(child); + return stm32_rcc_init(dev, stm32mp13_match_data, base); } static struct platform_driver stm32mp13_rcc_clocks_driver = { @@ -1615,7 +1554,6 @@ static struct platform_driver stm32mp13_rcc_clocks_driver = { .of_match_table = stm32mp13_match_data, }, .probe = stm32mp1_rcc_clocks_probe, - .remove_new = stm32mp1_rcc_clocks_remove, }; static int __init stm32mp13_clocks_init(void) From df5df1257c9f2af6621976f1e18a088182230d06 Mon Sep 17 00:00:00 2001 From: Gabriel Fernandez Date: Thu, 11 Apr 2024 11:24:51 +0200 Subject: [PATCH 0129/1702] dt-bindings: clocks: stm32mp25: add description of all parents RCC driver uses '.index' to define all parent clocks instead '.names' because the use of a name to define a parent clock is discouraged. This is an ABI change, but the RCC driver has not yet merged, unlike all others drivers besides Linux. Fixes: b5be49db3d47 ("dt-bindings: stm32: add clocks and reset binding for stm32mp25 platform") Signed-off-by: Gabriel Fernandez Acked-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240411092453.243633-3-gabriel.fernandez@foss.st.com Signed-off-by: Stephen Boyd --- .../bindings/clock/st,stm32mp25-rcc.yaml | 170 ++++++++++++++++-- include/dt-bindings/reset/st,stm32mp25-rcc.h | 2 +- 2 files changed, 156 insertions(+), 16 deletions(-) diff --git a/Documentation/devicetree/bindings/clock/st,stm32mp25-rcc.yaml b/Documentation/devicetree/bindings/clock/st,stm32mp25-rcc.yaml index 7732e79a42b90..0929fa7e271b3 100644 --- a/Documentation/devicetree/bindings/clock/st,stm32mp25-rcc.yaml +++ b/Documentation/devicetree/bindings/clock/st,stm32mp25-rcc.yaml @@ -38,14 +38,81 @@ properties: - description: CK_SCMI_MSI Low Power Internal oscillator (~ 4 MHz or ~ 16 MHz) - description: CK_SCMI_LSE Low Speed External oscillator (32 KHz) - description: CK_SCMI_LSI Low Speed Internal oscillator (~ 32 KHz) - - clock-names: - items: - - const: hse - - const: hsi - - const: msi - - const: lse - - const: lsi + - description: CK_SCMI_HSE_DIV2 CK_SCMI_HSE divided by 2 (coud be gated) + - description: CK_SCMI_ICN_HS_MCU High Speed interconnect bus clock + - description: CK_SCMI_ICN_LS_MCU Low Speed interconnect bus clock + - description: CK_SCMI_ICN_SDMMC SDMMC interconnect bus clock + - description: CK_SCMI_ICN_DDR DDR interconnect bus clock + - description: CK_SCMI_ICN_DISPLAY Display interconnect bus clock + - description: CK_SCMI_ICN_HSL HSL interconnect bus clock + - description: CK_SCMI_ICN_NIC NIC interconnect bus clock + - description: CK_SCMI_ICN_VID Video interconnect bus clock + - description: CK_SCMI_FLEXGEN_07 flexgen clock 7 + - description: CK_SCMI_FLEXGEN_08 flexgen clock 8 + - description: CK_SCMI_FLEXGEN_09 flexgen clock 9 + - description: CK_SCMI_FLEXGEN_10 flexgen clock 10 + - description: CK_SCMI_FLEXGEN_11 flexgen clock 11 + - description: CK_SCMI_FLEXGEN_12 flexgen clock 12 + - description: CK_SCMI_FLEXGEN_13 flexgen clock 13 + - description: CK_SCMI_FLEXGEN_14 flexgen clock 14 + - description: CK_SCMI_FLEXGEN_15 flexgen clock 15 + - description: CK_SCMI_FLEXGEN_16 flexgen clock 16 + - description: CK_SCMI_FLEXGEN_17 flexgen clock 17 + - description: CK_SCMI_FLEXGEN_18 flexgen clock 18 + - description: CK_SCMI_FLEXGEN_19 flexgen clock 19 + - description: CK_SCMI_FLEXGEN_20 flexgen clock 20 + - description: CK_SCMI_FLEXGEN_21 flexgen clock 21 + - description: CK_SCMI_FLEXGEN_22 flexgen clock 22 + - description: CK_SCMI_FLEXGEN_23 flexgen clock 23 + - description: CK_SCMI_FLEXGEN_24 flexgen clock 24 + - description: CK_SCMI_FLEXGEN_25 flexgen clock 25 + - description: CK_SCMI_FLEXGEN_26 flexgen clock 26 + - description: CK_SCMI_FLEXGEN_27 flexgen clock 27 + - description: CK_SCMI_FLEXGEN_28 flexgen clock 28 + - description: CK_SCMI_FLEXGEN_29 flexgen clock 29 + - description: CK_SCMI_FLEXGEN_30 flexgen clock 30 + - description: CK_SCMI_FLEXGEN_31 flexgen clock 31 + - description: CK_SCMI_FLEXGEN_32 flexgen clock 32 + - description: CK_SCMI_FLEXGEN_33 flexgen clock 33 + - description: CK_SCMI_FLEXGEN_34 flexgen clock 34 + - description: CK_SCMI_FLEXGEN_35 flexgen clock 35 + - description: CK_SCMI_FLEXGEN_36 flexgen clock 36 + - description: CK_SCMI_FLEXGEN_37 flexgen clock 37 + - description: CK_SCMI_FLEXGEN_38 flexgen clock 38 + - description: CK_SCMI_FLEXGEN_39 flexgen clock 39 + - description: CK_SCMI_FLEXGEN_40 flexgen clock 40 + - description: CK_SCMI_FLEXGEN_41 flexgen clock 41 + - description: CK_SCMI_FLEXGEN_42 flexgen clock 42 + - description: CK_SCMI_FLEXGEN_43 flexgen clock 43 + - description: CK_SCMI_FLEXGEN_44 flexgen clock 44 + - description: CK_SCMI_FLEXGEN_45 flexgen clock 45 + - description: CK_SCMI_FLEXGEN_46 flexgen clock 46 + - description: CK_SCMI_FLEXGEN_47 flexgen clock 47 + - description: CK_SCMI_FLEXGEN_48 flexgen clock 48 + - description: CK_SCMI_FLEXGEN_49 flexgen clock 49 + - description: CK_SCMI_FLEXGEN_50 flexgen clock 50 + - description: CK_SCMI_FLEXGEN_51 flexgen clock 51 + - description: CK_SCMI_FLEXGEN_52 flexgen clock 52 + - description: CK_SCMI_FLEXGEN_53 flexgen clock 53 + - description: CK_SCMI_FLEXGEN_54 flexgen clock 54 + - description: CK_SCMI_FLEXGEN_55 flexgen clock 55 + - description: CK_SCMI_FLEXGEN_56 flexgen clock 56 + - description: CK_SCMI_FLEXGEN_57 flexgen clock 57 + - description: CK_SCMI_FLEXGEN_58 flexgen clock 58 + - description: CK_SCMI_FLEXGEN_59 flexgen clock 59 + - description: CK_SCMI_FLEXGEN_60 flexgen clock 60 + - description: CK_SCMI_FLEXGEN_61 flexgen clock 61 + - description: CK_SCMI_FLEXGEN_62 flexgen clock 62 + - description: CK_SCMI_FLEXGEN_63 flexgen clock 63 + - description: CK_SCMI_ICN_APB1 Peripheral bridge 1 + - description: CK_SCMI_ICN_APB2 Peripheral bridge 2 + - description: CK_SCMI_ICN_APB3 Peripheral bridge 3 + - description: CK_SCMI_ICN_APB4 Peripheral bridge 4 + - description: CK_SCMI_ICN_APBDBG Peripheral bridge for degub + - description: CK_SCMI_TIMG1 Peripheral bridge for timer1 + - description: CK_SCMI_TIMG2 Peripheral bridge for timer2 + - description: CK_SCMI_PLL3 PLL3 clock + - description: clk_dsi_txbyte DSI byte clock required: - compatible @@ -53,7 +120,6 @@ required: - '#clock-cells' - '#reset-cells' - clocks - - clock-names additionalProperties: false @@ -66,11 +132,85 @@ examples: reg = <0x44200000 0x10000>; #clock-cells = <1>; #reset-cells = <1>; - clock-names = "hse", "hsi", "msi", "lse", "lsi"; - clocks = <&scmi_clk CK_SCMI_HSE>, - <&scmi_clk CK_SCMI_HSI>, - <&scmi_clk CK_SCMI_MSI>, - <&scmi_clk CK_SCMI_LSE>, - <&scmi_clk CK_SCMI_LSI>; + clocks = <&scmi_clk CK_SCMI_HSE>, + <&scmi_clk CK_SCMI_HSI>, + <&scmi_clk CK_SCMI_MSI>, + <&scmi_clk CK_SCMI_LSE>, + <&scmi_clk CK_SCMI_LSI>, + <&scmi_clk CK_SCMI_HSE_DIV2>, + <&scmi_clk CK_SCMI_ICN_HS_MCU>, + <&scmi_clk CK_SCMI_ICN_LS_MCU>, + <&scmi_clk CK_SCMI_ICN_SDMMC>, + <&scmi_clk CK_SCMI_ICN_DDR>, + <&scmi_clk CK_SCMI_ICN_DISPLAY>, + <&scmi_clk CK_SCMI_ICN_HSL>, + <&scmi_clk CK_SCMI_ICN_NIC>, + <&scmi_clk CK_SCMI_ICN_VID>, + <&scmi_clk CK_SCMI_FLEXGEN_07>, + <&scmi_clk CK_SCMI_FLEXGEN_08>, + <&scmi_clk CK_SCMI_FLEXGEN_09>, + <&scmi_clk CK_SCMI_FLEXGEN_10>, + <&scmi_clk CK_SCMI_FLEXGEN_11>, + <&scmi_clk CK_SCMI_FLEXGEN_12>, + <&scmi_clk CK_SCMI_FLEXGEN_13>, + <&scmi_clk CK_SCMI_FLEXGEN_14>, + <&scmi_clk CK_SCMI_FLEXGEN_15>, + <&scmi_clk CK_SCMI_FLEXGEN_16>, + <&scmi_clk CK_SCMI_FLEXGEN_17>, + <&scmi_clk CK_SCMI_FLEXGEN_18>, + <&scmi_clk CK_SCMI_FLEXGEN_19>, + <&scmi_clk CK_SCMI_FLEXGEN_20>, + <&scmi_clk CK_SCMI_FLEXGEN_21>, + <&scmi_clk CK_SCMI_FLEXGEN_22>, + <&scmi_clk CK_SCMI_FLEXGEN_23>, + <&scmi_clk CK_SCMI_FLEXGEN_24>, + <&scmi_clk CK_SCMI_FLEXGEN_25>, + <&scmi_clk CK_SCMI_FLEXGEN_26>, + <&scmi_clk CK_SCMI_FLEXGEN_27>, + <&scmi_clk CK_SCMI_FLEXGEN_28>, + <&scmi_clk CK_SCMI_FLEXGEN_29>, + <&scmi_clk CK_SCMI_FLEXGEN_30>, + <&scmi_clk CK_SCMI_FLEXGEN_31>, + <&scmi_clk CK_SCMI_FLEXGEN_32>, + <&scmi_clk CK_SCMI_FLEXGEN_33>, + <&scmi_clk CK_SCMI_FLEXGEN_34>, + <&scmi_clk CK_SCMI_FLEXGEN_35>, + <&scmi_clk CK_SCMI_FLEXGEN_36>, + <&scmi_clk CK_SCMI_FLEXGEN_37>, + <&scmi_clk CK_SCMI_FLEXGEN_38>, + <&scmi_clk CK_SCMI_FLEXGEN_39>, + <&scmi_clk CK_SCMI_FLEXGEN_40>, + <&scmi_clk CK_SCMI_FLEXGEN_41>, + <&scmi_clk CK_SCMI_FLEXGEN_42>, + <&scmi_clk CK_SCMI_FLEXGEN_43>, + <&scmi_clk CK_SCMI_FLEXGEN_44>, + <&scmi_clk CK_SCMI_FLEXGEN_45>, + <&scmi_clk CK_SCMI_FLEXGEN_46>, + <&scmi_clk CK_SCMI_FLEXGEN_47>, + <&scmi_clk CK_SCMI_FLEXGEN_48>, + <&scmi_clk CK_SCMI_FLEXGEN_49>, + <&scmi_clk CK_SCMI_FLEXGEN_50>, + <&scmi_clk CK_SCMI_FLEXGEN_51>, + <&scmi_clk CK_SCMI_FLEXGEN_52>, + <&scmi_clk CK_SCMI_FLEXGEN_53>, + <&scmi_clk CK_SCMI_FLEXGEN_54>, + <&scmi_clk CK_SCMI_FLEXGEN_55>, + <&scmi_clk CK_SCMI_FLEXGEN_56>, + <&scmi_clk CK_SCMI_FLEXGEN_57>, + <&scmi_clk CK_SCMI_FLEXGEN_58>, + <&scmi_clk CK_SCMI_FLEXGEN_59>, + <&scmi_clk CK_SCMI_FLEXGEN_60>, + <&scmi_clk CK_SCMI_FLEXGEN_61>, + <&scmi_clk CK_SCMI_FLEXGEN_62>, + <&scmi_clk CK_SCMI_FLEXGEN_63>, + <&scmi_clk CK_SCMI_ICN_APB1>, + <&scmi_clk CK_SCMI_ICN_APB2>, + <&scmi_clk CK_SCMI_ICN_APB3>, + <&scmi_clk CK_SCMI_ICN_APB4>, + <&scmi_clk CK_SCMI_ICN_APBDBG>, + <&scmi_clk CK_SCMI_TIMG1>, + <&scmi_clk CK_SCMI_TIMG2>, + <&scmi_clk CK_SCMI_PLL3>, + <&clk_dsi_txbyte>; }; ... diff --git a/include/dt-bindings/reset/st,stm32mp25-rcc.h b/include/dt-bindings/reset/st,stm32mp25-rcc.h index d5615930bcc86..748e78ae20bde 100644 --- a/include/dt-bindings/reset/st,stm32mp25-rcc.h +++ b/include/dt-bindings/reset/st,stm32mp25-rcc.h @@ -69,7 +69,7 @@ #define ADC3_R 59 #define ETH1_R 60 #define ETH2_R 61 -#define USB2_R 62 +#define USBH_R 62 #define USB2PHY1_R 63 #define USB2PHY2_R 64 #define USB3DR_R 65 From fd7a1c90ba4c6e618596eb44a69977801f9cccdd Mon Sep 17 00:00:00 2001 From: Gabriel Fernandez Date: Thu, 11 Apr 2024 11:24:52 +0200 Subject: [PATCH 0130/1702] clk: stm32: introduce clocks for STM32MP257 platform This driver is intended for the STM32MP25 clock family and utilizes the stm32-core API, similar to the stm32mp13 clock driver. Signed-off-by: Gabriel Fernandez Link: https://lore.kernel.org/r/20240411092453.243633-4-gabriel.fernandez@foss.st.com Signed-off-by: Stephen Boyd --- drivers/clk/stm32/Kconfig | 7 + drivers/clk/stm32/Makefile | 1 + drivers/clk/stm32/clk-stm32mp25.c | 1875 +++++++++++++++++++++++++++++ drivers/clk/stm32/reset-stm32.c | 59 +- drivers/clk/stm32/reset-stm32.h | 7 + drivers/clk/stm32/stm32mp25_rcc.h | 712 +++++++++++ 6 files changed, 2646 insertions(+), 15 deletions(-) create mode 100644 drivers/clk/stm32/clk-stm32mp25.c create mode 100644 drivers/clk/stm32/stm32mp25_rcc.h diff --git a/drivers/clk/stm32/Kconfig b/drivers/clk/stm32/Kconfig index 3c8493a94a11b..dca409d526522 100644 --- a/drivers/clk/stm32/Kconfig +++ b/drivers/clk/stm32/Kconfig @@ -25,5 +25,12 @@ config COMMON_CLK_STM32MP157 help Support for stm32mp15x SoC family clocks. +config COMMON_CLK_STM32MP257 + bool "Clock driver for stm32mp25x clocks" + depends on ARM64 || COMPILE_TEST + default y + help + Support for stm32mp25x SoC family clocks. + endif diff --git a/drivers/clk/stm32/Makefile b/drivers/clk/stm32/Makefile index 5ced7fe3ddec6..0a627164fccee 100644 --- a/drivers/clk/stm32/Makefile +++ b/drivers/clk/stm32/Makefile @@ -1,2 +1,3 @@ obj-$(CONFIG_COMMON_CLK_STM32MP135) += clk-stm32mp13.o clk-stm32-core.o reset-stm32.o obj-$(CONFIG_COMMON_CLK_STM32MP157) += clk-stm32mp1.o reset-stm32.o +obj-$(CONFIG_COMMON_CLK_STM32MP257) += clk-stm32mp25.o clk-stm32-core.o reset-stm32.o diff --git a/drivers/clk/stm32/clk-stm32mp25.c b/drivers/clk/stm32/clk-stm32mp25.c new file mode 100644 index 0000000000000..210b75b39e503 --- /dev/null +++ b/drivers/clk/stm32/clk-stm32mp25.c @@ -0,0 +1,1875 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) STMicroelectronics 2023 - All Rights Reserved + * Author: Gabriel Fernandez for STMicroelectronics. + */ + +#include +#include + +#include "clk-stm32-core.h" +#include "reset-stm32.h" +#include "stm32mp25_rcc.h" + +#include +#include + +enum { + HSE, + HSI, + MSI, + LSE, + LSI, + HSE_DIV2, + ICN_HS_MCU, + ICN_LS_MCU, + ICN_SDMMC, + ICN_DDR, + ICN_DISPLAY, + ICN_HSL, + ICN_NIC, + ICN_VID, + FLEXGEN_07, + FLEXGEN_08, + FLEXGEN_09, + FLEXGEN_10, + FLEXGEN_11, + FLEXGEN_12, + FLEXGEN_13, + FLEXGEN_14, + FLEXGEN_15, + FLEXGEN_16, + FLEXGEN_17, + FLEXGEN_18, + FLEXGEN_19, + FLEXGEN_20, + FLEXGEN_21, + FLEXGEN_22, + FLEXGEN_23, + FLEXGEN_24, + FLEXGEN_25, + FLEXGEN_26, + FLEXGEN_27, + FLEXGEN_28, + FLEXGEN_29, + FLEXGEN_30, + FLEXGEN_31, + FLEXGEN_32, + FLEXGEN_33, + FLEXGEN_34, + FLEXGEN_35, + FLEXGEN_36, + FLEXGEN_37, + FLEXGEN_38, + FLEXGEN_39, + FLEXGEN_40, + FLEXGEN_41, + FLEXGEN_42, + FLEXGEN_43, + FLEXGEN_44, + FLEXGEN_45, + FLEXGEN_46, + FLEXGEN_47, + FLEXGEN_48, + FLEXGEN_49, + FLEXGEN_50, + FLEXGEN_51, + FLEXGEN_52, + FLEXGEN_53, + FLEXGEN_54, + FLEXGEN_55, + FLEXGEN_56, + FLEXGEN_57, + FLEXGEN_58, + FLEXGEN_59, + FLEXGEN_60, + FLEXGEN_61, + FLEXGEN_62, + FLEXGEN_63, + ICN_APB1, + ICN_APB2, + ICN_APB3, + ICN_APB4, + ICN_APBDBG, + TIMG1, + TIMG2, + PLL3, + DSI_TXBYTE, +}; + +static const struct clk_parent_data adc12_src[] = { + { .index = FLEXGEN_46 }, + { .index = ICN_LS_MCU }, +}; + +static const struct clk_parent_data adc3_src[] = { + { .index = FLEXGEN_47 }, + { .index = ICN_LS_MCU }, + { .index = FLEXGEN_46 }, +}; + +static const struct clk_parent_data usb2phy1_src[] = { + { .index = FLEXGEN_57 }, + { .index = HSE_DIV2 }, +}; + +static const struct clk_parent_data usb2phy2_src[] = { + { .index = FLEXGEN_58 }, + { .index = HSE_DIV2 }, +}; + +static const struct clk_parent_data usb3pciphy_src[] = { + { .index = FLEXGEN_34 }, + { .index = HSE_DIV2 }, +}; + +static struct clk_stm32_gate ck_ker_ltdc; + +static const struct clk_parent_data dsiblane_src[] = { + { .index = DSI_TXBYTE }, + { .hw = &ck_ker_ltdc.hw }, +}; + +static const struct clk_parent_data dsiphy_src[] = { + { .index = FLEXGEN_28 }, + { .index = HSE }, +}; + +static const struct clk_parent_data lvdsphy_src[] = { + { .index = FLEXGEN_32 }, + { .index = HSE }, +}; + +static const struct clk_parent_data dts_src[] = { + { .index = HSI }, + { .index = HSE }, + { .index = MSI }, +}; + +static const struct clk_parent_data mco1_src[] = { + { .index = FLEXGEN_61 }, +}; + +static const struct clk_parent_data mco2_src[] = { + { .index = FLEXGEN_62 }, +}; + +enum enum_mux_cfg { + MUX_ADC12, + MUX_ADC3, + MUX_DSIBLANE, + MUX_DSIPHY, + MUX_DTS, + MUX_LVDSPHY, + MUX_MCO1, + MUX_MCO2, + MUX_USB2PHY1, + MUX_USB2PHY2, + MUX_USB3PCIEPHY, + MUX_NB +}; + +#define MUX_CFG(id, _offset, _shift, _witdh) \ + [id] = { \ + .offset = (_offset), \ + .shift = (_shift), \ + .width = (_witdh), \ + } + +static const struct stm32_mux_cfg stm32mp25_muxes[MUX_NB] = { + MUX_CFG(MUX_ADC12, RCC_ADC12CFGR, 12, 1), + MUX_CFG(MUX_ADC3, RCC_ADC3CFGR, 12, 2), + MUX_CFG(MUX_DSIBLANE, RCC_DSICFGR, 12, 1), + MUX_CFG(MUX_DSIPHY, RCC_DSICFGR, 15, 1), + MUX_CFG(MUX_DTS, RCC_DTSCFGR, 12, 2), + MUX_CFG(MUX_LVDSPHY, RCC_LVDSCFGR, 15, 1), + MUX_CFG(MUX_MCO1, RCC_MCO1CFGR, 0, 1), + MUX_CFG(MUX_MCO2, RCC_MCO2CFGR, 0, 1), + MUX_CFG(MUX_USB2PHY1, RCC_USB2PHY1CFGR, 15, 1), + MUX_CFG(MUX_USB2PHY2, RCC_USB2PHY2CFGR, 15, 1), + MUX_CFG(MUX_USB3PCIEPHY, RCC_USB3PCIEPHYCFGR, 15, 1), +}; + +enum enum_gate_cfg { + GATE_ADC12, + GATE_ADC3, + GATE_ADF1, + GATE_CCI, + GATE_CRC, + GATE_CRYP1, + GATE_CRYP2, + GATE_CSI, + GATE_DCMIPP, + GATE_DSI, + GATE_DTS, + GATE_ETH1, + GATE_ETH1MAC, + GATE_ETH1RX, + GATE_ETH1STP, + GATE_ETH1TX, + GATE_ETH2, + GATE_ETH2MAC, + GATE_ETH2RX, + GATE_ETH2STP, + GATE_ETH2TX, + GATE_ETHSW, + GATE_ETHSWACMCFG, + GATE_ETHSWACMMSG, + GATE_ETHSWMAC, + GATE_ETHSWREF, + GATE_FDCAN, + GATE_GPU, + GATE_HASH, + GATE_HDP, + GATE_I2C1, + GATE_I2C2, + GATE_I2C3, + GATE_I2C4, + GATE_I2C5, + GATE_I2C6, + GATE_I2C7, + GATE_I2C8, + GATE_I3C1, + GATE_I3C2, + GATE_I3C3, + GATE_I3C4, + GATE_IS2M, + GATE_IWDG1, + GATE_IWDG2, + GATE_IWDG3, + GATE_IWDG4, + GATE_IWDG5, + GATE_LPTIM1, + GATE_LPTIM2, + GATE_LPTIM3, + GATE_LPTIM4, + GATE_LPTIM5, + GATE_LPUART1, + GATE_LTDC, + GATE_LVDS, + GATE_MCO1, + GATE_MCO2, + GATE_MDF1, + GATE_OSPIIOM, + GATE_PCIE, + GATE_PKA, + GATE_RNG, + GATE_SAES, + GATE_SAI1, + GATE_SAI2, + GATE_SAI3, + GATE_SAI4, + GATE_SDMMC1, + GATE_SDMMC2, + GATE_SDMMC3, + GATE_SERC, + GATE_SPDIFRX, + GATE_SPI1, + GATE_SPI2, + GATE_SPI3, + GATE_SPI4, + GATE_SPI5, + GATE_SPI6, + GATE_SPI7, + GATE_SPI8, + GATE_TIM1, + GATE_TIM10, + GATE_TIM11, + GATE_TIM12, + GATE_TIM13, + GATE_TIM14, + GATE_TIM15, + GATE_TIM16, + GATE_TIM17, + GATE_TIM2, + GATE_TIM20, + GATE_TIM3, + GATE_TIM4, + GATE_TIM5, + GATE_TIM6, + GATE_TIM7, + GATE_TIM8, + GATE_UART4, + GATE_UART5, + GATE_UART7, + GATE_UART8, + GATE_UART9, + GATE_USART1, + GATE_USART2, + GATE_USART3, + GATE_USART6, + GATE_USBH, + GATE_USB2PHY1, + GATE_USB2PHY2, + GATE_USB3DR, + GATE_USB3PCIEPHY, + GATE_USBTC, + GATE_VDEC, + GATE_VENC, + GATE_VREF, + GATE_WWDG1, + GATE_WWDG2, + GATE_NB +}; + +#define GATE_CFG(id, _offset, _bit_idx, _offset_clr) \ + [id] = { \ + .offset = (_offset), \ + .bit_idx = (_bit_idx), \ + .set_clr = (_offset_clr), \ + } + +static const struct stm32_gate_cfg stm32mp25_gates[GATE_NB] = { + GATE_CFG(GATE_ADC12, RCC_ADC12CFGR, 1, 0), + GATE_CFG(GATE_ADC3, RCC_ADC3CFGR, 1, 0), + GATE_CFG(GATE_ADF1, RCC_ADF1CFGR, 1, 0), + GATE_CFG(GATE_CCI, RCC_CCICFGR, 1, 0), + GATE_CFG(GATE_CRC, RCC_CRCCFGR, 1, 0), + GATE_CFG(GATE_CRYP1, RCC_CRYP1CFGR, 1, 0), + GATE_CFG(GATE_CRYP2, RCC_CRYP2CFGR, 1, 0), + GATE_CFG(GATE_CSI, RCC_CSICFGR, 1, 0), + GATE_CFG(GATE_DCMIPP, RCC_DCMIPPCFGR, 1, 0), + GATE_CFG(GATE_DSI, RCC_DSICFGR, 1, 0), + GATE_CFG(GATE_DTS, RCC_DTSCFGR, 1, 0), + GATE_CFG(GATE_ETH1, RCC_ETH1CFGR, 5, 0), + GATE_CFG(GATE_ETH1MAC, RCC_ETH1CFGR, 1, 0), + GATE_CFG(GATE_ETH1RX, RCC_ETH1CFGR, 10, 0), + GATE_CFG(GATE_ETH1STP, RCC_ETH1CFGR, 4, 0), + GATE_CFG(GATE_ETH1TX, RCC_ETH1CFGR, 8, 0), + GATE_CFG(GATE_ETH2, RCC_ETH2CFGR, 5, 0), + GATE_CFG(GATE_ETH2MAC, RCC_ETH2CFGR, 1, 0), + GATE_CFG(GATE_ETH2RX, RCC_ETH2CFGR, 10, 0), + GATE_CFG(GATE_ETH2STP, RCC_ETH2CFGR, 4, 0), + GATE_CFG(GATE_ETH2TX, RCC_ETH2CFGR, 8, 0), + GATE_CFG(GATE_ETHSW, RCC_ETHSWCFGR, 5, 0), + GATE_CFG(GATE_ETHSWACMCFG, RCC_ETHSWACMCFGR, 1, 0), + GATE_CFG(GATE_ETHSWACMMSG, RCC_ETHSWACMMSGCFGR, 1, 0), + GATE_CFG(GATE_ETHSWMAC, RCC_ETHSWCFGR, 1, 0), + GATE_CFG(GATE_ETHSWREF, RCC_ETHSWCFGR, 21, 0), + GATE_CFG(GATE_FDCAN, RCC_FDCANCFGR, 1, 0), + GATE_CFG(GATE_GPU, RCC_GPUCFGR, 1, 0), + GATE_CFG(GATE_HASH, RCC_HASHCFGR, 1, 0), + GATE_CFG(GATE_HDP, RCC_HDPCFGR, 1, 0), + GATE_CFG(GATE_I2C1, RCC_I2C1CFGR, 1, 0), + GATE_CFG(GATE_I2C2, RCC_I2C2CFGR, 1, 0), + GATE_CFG(GATE_I2C3, RCC_I2C3CFGR, 1, 0), + GATE_CFG(GATE_I2C4, RCC_I2C4CFGR, 1, 0), + GATE_CFG(GATE_I2C5, RCC_I2C5CFGR, 1, 0), + GATE_CFG(GATE_I2C6, RCC_I2C6CFGR, 1, 0), + GATE_CFG(GATE_I2C7, RCC_I2C7CFGR, 1, 0), + GATE_CFG(GATE_I2C8, RCC_I2C8CFGR, 1, 0), + GATE_CFG(GATE_I3C1, RCC_I3C1CFGR, 1, 0), + GATE_CFG(GATE_I3C2, RCC_I3C2CFGR, 1, 0), + GATE_CFG(GATE_I3C3, RCC_I3C3CFGR, 1, 0), + GATE_CFG(GATE_I3C4, RCC_I3C4CFGR, 1, 0), + GATE_CFG(GATE_IS2M, RCC_IS2MCFGR, 1, 0), + GATE_CFG(GATE_IWDG1, RCC_IWDG1CFGR, 1, 0), + GATE_CFG(GATE_IWDG2, RCC_IWDG2CFGR, 1, 0), + GATE_CFG(GATE_IWDG3, RCC_IWDG3CFGR, 1, 0), + GATE_CFG(GATE_IWDG4, RCC_IWDG4CFGR, 1, 0), + GATE_CFG(GATE_IWDG5, RCC_IWDG5CFGR, 1, 0), + GATE_CFG(GATE_LPTIM1, RCC_LPTIM1CFGR, 1, 0), + GATE_CFG(GATE_LPTIM2, RCC_LPTIM2CFGR, 1, 0), + GATE_CFG(GATE_LPTIM3, RCC_LPTIM3CFGR, 1, 0), + GATE_CFG(GATE_LPTIM4, RCC_LPTIM4CFGR, 1, 0), + GATE_CFG(GATE_LPTIM5, RCC_LPTIM5CFGR, 1, 0), + GATE_CFG(GATE_LPUART1, RCC_LPUART1CFGR, 1, 0), + GATE_CFG(GATE_LTDC, RCC_LTDCCFGR, 1, 0), + GATE_CFG(GATE_LVDS, RCC_LVDSCFGR, 1, 0), + GATE_CFG(GATE_MCO1, RCC_MCO1CFGR, 8, 0), + GATE_CFG(GATE_MCO2, RCC_MCO2CFGR, 8, 0), + GATE_CFG(GATE_MDF1, RCC_MDF1CFGR, 1, 0), + GATE_CFG(GATE_OSPIIOM, RCC_OSPIIOMCFGR, 1, 0), + GATE_CFG(GATE_PCIE, RCC_PCIECFGR, 1, 0), + GATE_CFG(GATE_PKA, RCC_PKACFGR, 1, 0), + GATE_CFG(GATE_RNG, RCC_RNGCFGR, 1, 0), + GATE_CFG(GATE_SAES, RCC_SAESCFGR, 1, 0), + GATE_CFG(GATE_SAI1, RCC_SAI1CFGR, 1, 0), + GATE_CFG(GATE_SAI2, RCC_SAI2CFGR, 1, 0), + GATE_CFG(GATE_SAI3, RCC_SAI3CFGR, 1, 0), + GATE_CFG(GATE_SAI4, RCC_SAI4CFGR, 1, 0), + GATE_CFG(GATE_SDMMC1, RCC_SDMMC1CFGR, 1, 0), + GATE_CFG(GATE_SDMMC2, RCC_SDMMC2CFGR, 1, 0), + GATE_CFG(GATE_SDMMC3, RCC_SDMMC3CFGR, 1, 0), + GATE_CFG(GATE_SERC, RCC_SERCCFGR, 1, 0), + GATE_CFG(GATE_SPDIFRX, RCC_SPDIFRXCFGR, 1, 0), + GATE_CFG(GATE_SPI1, RCC_SPI1CFGR, 1, 0), + GATE_CFG(GATE_SPI2, RCC_SPI2CFGR, 1, 0), + GATE_CFG(GATE_SPI3, RCC_SPI3CFGR, 1, 0), + GATE_CFG(GATE_SPI4, RCC_SPI4CFGR, 1, 0), + GATE_CFG(GATE_SPI5, RCC_SPI5CFGR, 1, 0), + GATE_CFG(GATE_SPI6, RCC_SPI6CFGR, 1, 0), + GATE_CFG(GATE_SPI7, RCC_SPI7CFGR, 1, 0), + GATE_CFG(GATE_SPI8, RCC_SPI8CFGR, 1, 0), + GATE_CFG(GATE_TIM1, RCC_TIM1CFGR, 1, 0), + GATE_CFG(GATE_TIM10, RCC_TIM10CFGR, 1, 0), + GATE_CFG(GATE_TIM11, RCC_TIM11CFGR, 1, 0), + GATE_CFG(GATE_TIM12, RCC_TIM12CFGR, 1, 0), + GATE_CFG(GATE_TIM13, RCC_TIM13CFGR, 1, 0), + GATE_CFG(GATE_TIM14, RCC_TIM14CFGR, 1, 0), + GATE_CFG(GATE_TIM15, RCC_TIM15CFGR, 1, 0), + GATE_CFG(GATE_TIM16, RCC_TIM16CFGR, 1, 0), + GATE_CFG(GATE_TIM17, RCC_TIM17CFGR, 1, 0), + GATE_CFG(GATE_TIM2, RCC_TIM2CFGR, 1, 0), + GATE_CFG(GATE_TIM20, RCC_TIM20CFGR, 1, 0), + GATE_CFG(GATE_TIM3, RCC_TIM3CFGR, 1, 0), + GATE_CFG(GATE_TIM4, RCC_TIM4CFGR, 1, 0), + GATE_CFG(GATE_TIM5, RCC_TIM5CFGR, 1, 0), + GATE_CFG(GATE_TIM6, RCC_TIM6CFGR, 1, 0), + GATE_CFG(GATE_TIM7, RCC_TIM7CFGR, 1, 0), + GATE_CFG(GATE_TIM8, RCC_TIM8CFGR, 1, 0), + GATE_CFG(GATE_UART4, RCC_UART4CFGR, 1, 0), + GATE_CFG(GATE_UART5, RCC_UART5CFGR, 1, 0), + GATE_CFG(GATE_UART7, RCC_UART7CFGR, 1, 0), + GATE_CFG(GATE_UART8, RCC_UART8CFGR, 1, 0), + GATE_CFG(GATE_UART9, RCC_UART9CFGR, 1, 0), + GATE_CFG(GATE_USART1, RCC_USART1CFGR, 1, 0), + GATE_CFG(GATE_USART2, RCC_USART2CFGR, 1, 0), + GATE_CFG(GATE_USART3, RCC_USART3CFGR, 1, 0), + GATE_CFG(GATE_USART6, RCC_USART6CFGR, 1, 0), + GATE_CFG(GATE_USBH, RCC_USBHCFGR, 1, 0), + GATE_CFG(GATE_USB2PHY1, RCC_USB2PHY1CFGR, 1, 0), + GATE_CFG(GATE_USB2PHY2, RCC_USB2PHY2CFGR, 1, 0), + GATE_CFG(GATE_USB3DR, RCC_USB3DRCFGR, 1, 0), + GATE_CFG(GATE_USB3PCIEPHY, RCC_USB3PCIEPHYCFGR, 1, 0), + GATE_CFG(GATE_USBTC, RCC_USBTCCFGR, 1, 0), + GATE_CFG(GATE_VDEC, RCC_VDECCFGR, 1, 0), + GATE_CFG(GATE_VENC, RCC_VENCCFGR, 1, 0), + GATE_CFG(GATE_VREF, RCC_VREFCFGR, 1, 0), + GATE_CFG(GATE_WWDG1, RCC_WWDG1CFGR, 1, 0), + GATE_CFG(GATE_WWDG2, RCC_WWDG2CFGR, 1, 0), +}; + +#define CLK_HW_INIT_INDEX(_name, _parent, _ops, _flags) \ + (&(struct clk_init_data) { \ + .flags = _flags, \ + .name = _name, \ + .parent_data = (const struct clk_parent_data[]) { \ + { .index = _parent }, \ + }, \ + .num_parents = 1, \ + .ops = _ops, \ + }) + +/* ADC */ +static struct clk_stm32_gate ck_icn_p_adc12 = { + .gate_id = GATE_ADC12, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_adc12", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_composite ck_ker_adc12 = { + .gate_id = GATE_ADC12, + .mux_id = MUX_ADC12, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_ker_adc12", adc12_src, &clk_stm32_composite_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_adc3 = { + .gate_id = GATE_ADC3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_adc3", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_composite ck_ker_adc3 = { + .gate_id = GATE_ADC3, + .mux_id = MUX_ADC3, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_ker_adc3", adc3_src, &clk_stm32_composite_ops, 0), +}; + +/* ADF */ +static struct clk_stm32_gate ck_icn_p_adf1 = { + .gate_id = GATE_ADF1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_adf1", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_adf1 = { + .gate_id = GATE_ADF1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_adf1", FLEXGEN_42, &clk_stm32_gate_ops, 0), +}; + +/* DCMI */ +static struct clk_stm32_gate ck_icn_p_cci = { + .gate_id = GATE_CCI, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_cci", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +/* CSI-HOST */ +static struct clk_stm32_gate ck_icn_p_csi = { + .gate_id = GATE_CSI, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_csi", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_csi = { + .gate_id = GATE_CSI, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_csi", FLEXGEN_29, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_csitxesc = { + .gate_id = GATE_CSI, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_csitxesc", FLEXGEN_30, &clk_stm32_gate_ops, 0), +}; + +/* CSI-PHY */ +static struct clk_stm32_gate ck_ker_csiphy = { + .gate_id = GATE_CSI, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_csiphy", FLEXGEN_31, &clk_stm32_gate_ops, 0), +}; + +/* DCMIPP */ +static struct clk_stm32_gate ck_icn_p_dcmipp = { + .gate_id = GATE_DCMIPP, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_dcmipp", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +/* CRC */ +static struct clk_stm32_gate ck_icn_p_crc = { + .gate_id = GATE_CRC, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_crc", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +/* CRYP */ +static struct clk_stm32_gate ck_icn_p_cryp1 = { + .gate_id = GATE_CRYP1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_cryp1", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_cryp2 = { + .gate_id = GATE_CRYP2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_cryp2", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +/* DBG & TRACE*/ +/* Trace and debug clocks are managed by SCMI */ + +/* LTDC */ +static struct clk_stm32_gate ck_icn_p_ltdc = { + .gate_id = GATE_LTDC, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_ltdc", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_ltdc = { + .gate_id = GATE_LTDC, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_ltdc", FLEXGEN_27, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +/* DSI */ +static struct clk_stm32_gate ck_icn_p_dsi = { + .gate_id = GATE_DSI, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_dsi", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_composite clk_lanebyte = { + .gate_id = GATE_DSI, + .mux_id = MUX_DSIBLANE, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("clk_lanebyte", dsiblane_src, + &clk_stm32_composite_ops, 0), +}; + +/* LVDS */ +static struct clk_stm32_gate ck_icn_p_lvds = { + .gate_id = GATE_LVDS, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_lvds", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +/* DSI PHY */ +static struct clk_stm32_composite clk_phy_dsi = { + .gate_id = GATE_DSI, + .mux_id = MUX_DSIPHY, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("clk_phy_dsi", dsiphy_src, + &clk_stm32_composite_ops, 0), +}; + +/* LVDS PHY */ +static struct clk_stm32_composite ck_ker_lvdsphy = { + .gate_id = GATE_LVDS, + .mux_id = MUX_LVDSPHY, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_ker_lvdsphy", lvdsphy_src, + &clk_stm32_composite_ops, 0), +}; + +/* DTS */ +static struct clk_stm32_composite ck_ker_dts = { + .gate_id = GATE_DTS, + .mux_id = MUX_DTS, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_ker_dts", dts_src, + &clk_stm32_composite_ops, 0), +}; + +/* ETHERNET */ +static struct clk_stm32_gate ck_icn_p_eth1 = { + .gate_id = GATE_ETH1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_eth1", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth1stp = { + .gate_id = GATE_ETH1STP, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth1stp", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth1 = { + .gate_id = GATE_ETH1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth1", FLEXGEN_54, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth1ptp = { + .gate_id = GATE_ETH1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth1ptp", FLEXGEN_56, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth1mac = { + .gate_id = GATE_ETH1MAC, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth1mac", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth1tx = { + .gate_id = GATE_ETH1TX, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth1tx", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth1rx = { + .gate_id = GATE_ETH1RX, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth1rx", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_eth2 = { + .gate_id = GATE_ETH2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_eth2", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth2stp = { + .gate_id = GATE_ETH2STP, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth2stp", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth2 = { + .gate_id = GATE_ETH2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth2", FLEXGEN_55, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth2ptp = { + .gate_id = GATE_ETH2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth2ptp", FLEXGEN_56, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth2mac = { + .gate_id = GATE_ETH2MAC, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth2mac", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth2tx = { + .gate_id = GATE_ETH2TX, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth2tx", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_eth2rx = { + .gate_id = GATE_ETH2RX, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_eth2rx", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_ethsw = { + .gate_id = GATE_ETHSWMAC, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_ethsw", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_ethsw = { + .gate_id = GATE_ETHSW, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_ethsw", FLEXGEN_54, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_ethswref = { + .gate_id = GATE_ETHSWREF, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_ethswref", FLEXGEN_60, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_ethsw_acm_cfg = { + .gate_id = GATE_ETHSWACMCFG, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_ethsw_acm_cfg", ICN_LS_MCU, + &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_ethsw_acm_msg = { + .gate_id = GATE_ETHSWACMMSG, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_ethsw_acm_msg", ICN_LS_MCU, + &clk_stm32_gate_ops, 0), +}; + +/* FDCAN */ +static struct clk_stm32_gate ck_icn_p_fdcan = { + .gate_id = GATE_FDCAN, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_fdcan", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_fdcan = { + .gate_id = GATE_FDCAN, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_fdcan", FLEXGEN_26, &clk_stm32_gate_ops, 0), +}; + +/* GPU */ +static struct clk_stm32_gate ck_icn_m_gpu = { + .gate_id = GATE_GPU, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_m_gpu", FLEXGEN_59, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_gpu = { + .gate_id = GATE_GPU, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_gpu", PLL3, &clk_stm32_gate_ops, 0), +}; + +/* HASH */ +static struct clk_stm32_gate ck_icn_p_hash = { + .gate_id = GATE_HASH, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_hash", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +/* HDP */ +static struct clk_stm32_gate ck_icn_p_hdp = { + .gate_id = GATE_HDP, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_hdp", ICN_APB3, &clk_stm32_gate_ops, 0), +}; + +/* I2C */ +static struct clk_stm32_gate ck_icn_p_i2c8 = { + .gate_id = GATE_I2C8, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c8", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i2c1 = { + .gate_id = GATE_I2C1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c1", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i2c2 = { + .gate_id = GATE_I2C2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c2", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i2c3 = { + .gate_id = GATE_I2C3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c3", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i2c4 = { + .gate_id = GATE_I2C4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c4", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i2c5 = { + .gate_id = GATE_I2C5, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c5", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i2c6 = { + .gate_id = GATE_I2C6, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c6", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i2c7 = { + .gate_id = GATE_I2C7, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i2c7", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c1 = { + .gate_id = GATE_I2C1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c1", FLEXGEN_12, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c2 = { + .gate_id = GATE_I2C2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c2", FLEXGEN_12, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c3 = { + .gate_id = GATE_I2C3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c3", FLEXGEN_13, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c5 = { + .gate_id = GATE_I2C5, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c5", FLEXGEN_13, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c4 = { + .gate_id = GATE_I2C4, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c4", FLEXGEN_14, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c6 = { + .gate_id = GATE_I2C6, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c6", FLEXGEN_14, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c7 = { + .gate_id = GATE_I2C7, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c7", FLEXGEN_15, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i2c8 = { + .gate_id = GATE_I2C8, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i2c8", FLEXGEN_38, &clk_stm32_gate_ops, 0), +}; + +/* I3C */ +static struct clk_stm32_gate ck_icn_p_i3c1 = { + .gate_id = GATE_I3C1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i3c1", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i3c2 = { + .gate_id = GATE_I3C2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i3c2", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i3c3 = { + .gate_id = GATE_I3C3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i3c3", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_i3c4 = { + .gate_id = GATE_I3C4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_i3c4", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i3c1 = { + .gate_id = GATE_I3C1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i3c1", FLEXGEN_12, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i3c2 = { + .gate_id = GATE_I3C2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i3c2", FLEXGEN_12, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i3c3 = { + .gate_id = GATE_I3C3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i3c3", FLEXGEN_13, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_i3c4 = { + .gate_id = GATE_I3C4, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_i3c4", FLEXGEN_36, &clk_stm32_gate_ops, 0), +}; + +/* I2S */ +static struct clk_stm32_gate ck_icn_p_is2m = { + .gate_id = GATE_IS2M, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_is2m", ICN_APB3, &clk_stm32_gate_ops, 0), +}; + +/* IWDG */ +static struct clk_stm32_gate ck_icn_p_iwdg2 = { + .gate_id = GATE_IWDG2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_iwdg2", ICN_APB3, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_iwdg3 = { + .gate_id = GATE_IWDG3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_iwdg3", ICN_APB3, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_iwdg4 = { + .gate_id = GATE_IWDG4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_iwdg4", ICN_APB3, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_iwdg5 = { + .gate_id = GATE_IWDG5, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_iwdg5", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +/* LPTIM */ +static struct clk_stm32_gate ck_icn_p_lptim1 = { + .gate_id = GATE_LPTIM1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_lptim1", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_lptim2 = { + .gate_id = GATE_LPTIM2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_lptim2", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_lptim3 = { + .gate_id = GATE_LPTIM3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_lptim3", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_lptim4 = { + .gate_id = GATE_LPTIM4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_lptim4", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_lptim5 = { + .gate_id = GATE_LPTIM5, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_lptim5", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_lptim1 = { + .gate_id = GATE_LPTIM1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_lptim1", FLEXGEN_07, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_lptim2 = { + .gate_id = GATE_LPTIM2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_lptim2", FLEXGEN_07, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_lptim3 = { + .gate_id = GATE_LPTIM3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_lptim3", FLEXGEN_40, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_lptim4 = { + .gate_id = GATE_LPTIM4, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_lptim4", FLEXGEN_41, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_lptim5 = { + .gate_id = GATE_LPTIM5, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_lptim5", FLEXGEN_41, &clk_stm32_gate_ops, 0), +}; + +/* LPUART */ +static struct clk_stm32_gate ck_icn_p_lpuart1 = { + .gate_id = GATE_LPUART1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_lpuart1", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_lpuart1 = { + .gate_id = GATE_LPUART1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_lpuart1", FLEXGEN_39, &clk_stm32_gate_ops, 0), +}; + +/* MCO1 & MCO2 */ +static struct clk_stm32_composite ck_mco1 = { + .gate_id = GATE_MCO1, + .mux_id = MUX_MCO1, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_mco1", mco1_src, &clk_stm32_composite_ops, 0), +}; + +static struct clk_stm32_composite ck_mco2 = { + .gate_id = GATE_MCO2, + .mux_id = MUX_MCO2, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_mco2", mco2_src, &clk_stm32_composite_ops, 0), +}; + +/* MDF */ +static struct clk_stm32_gate ck_icn_p_mdf1 = { + .gate_id = GATE_MDF1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_mdf1", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_mdf1 = { + .gate_id = GATE_MDF1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_mdf1", FLEXGEN_23, &clk_stm32_gate_ops, 0), +}; + +/* OSPI */ +static struct clk_stm32_gate ck_icn_p_ospiiom = { + .gate_id = GATE_OSPIIOM, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_ospiiom", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +/* PCIE */ +static struct clk_stm32_gate ck_icn_p_pcie = { + .gate_id = GATE_PCIE, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_pcie", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +/* SAI */ +static struct clk_stm32_gate ck_icn_p_sai1 = { + .gate_id = GATE_SAI1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_sai1", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_sai2 = { + .gate_id = GATE_SAI2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_sai2", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_sai3 = { + .gate_id = GATE_SAI3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_sai3", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_sai4 = { + .gate_id = GATE_SAI4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_sai4", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_sai1 = { + .gate_id = GATE_SAI1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_sai1", FLEXGEN_23, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +static struct clk_stm32_gate ck_ker_sai2 = { + .gate_id = GATE_SAI2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_sai2", FLEXGEN_24, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +static struct clk_stm32_gate ck_ker_sai3 = { + .gate_id = GATE_SAI3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_sai3", FLEXGEN_25, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +static struct clk_stm32_gate ck_ker_sai4 = { + .gate_id = GATE_SAI4, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_sai4", FLEXGEN_25, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +/* SDMMC */ +static struct clk_stm32_gate ck_icn_m_sdmmc1 = { + .gate_id = GATE_SDMMC1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_m_sdmmc1", ICN_SDMMC, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_m_sdmmc2 = { + .gate_id = GATE_SDMMC2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_m_sdmmc2", ICN_SDMMC, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_m_sdmmc3 = { + .gate_id = GATE_SDMMC3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_m_sdmmc3", ICN_SDMMC, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_sdmmc1 = { + .gate_id = GATE_SDMMC1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_sdmmc1", FLEXGEN_51, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_sdmmc2 = { + .gate_id = GATE_SDMMC2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_sdmmc2", FLEXGEN_52, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_sdmmc3 = { + .gate_id = GATE_SDMMC3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_sdmmc3", FLEXGEN_53, &clk_stm32_gate_ops, 0), +}; + +/* SPDIF */ +static struct clk_stm32_gate ck_icn_p_spdifrx = { + .gate_id = GATE_SPDIFRX, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spdifrx", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_spdifrx = { + .gate_id = GATE_SPDIFRX, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spdifrx", FLEXGEN_11, &clk_stm32_gate_ops, 0), +}; + +/* SPI */ +static struct clk_stm32_gate ck_icn_p_spi1 = { + .gate_id = GATE_SPI1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi1", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_spi2 = { + .gate_id = GATE_SPI2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi2", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_spi3 = { + .gate_id = GATE_SPI3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi3", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_spi4 = { + .gate_id = GATE_SPI4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi4", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_spi5 = { + .gate_id = GATE_SPI5, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi5", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_spi6 = { + .gate_id = GATE_SPI6, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi6", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_spi7 = { + .gate_id = GATE_SPI7, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi7", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_spi8 = { + .gate_id = GATE_SPI8, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_spi8", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_spi1 = { + .gate_id = GATE_SPI1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi1", FLEXGEN_16, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +static struct clk_stm32_gate ck_ker_spi2 = { + .gate_id = GATE_SPI2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi2", FLEXGEN_10, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +static struct clk_stm32_gate ck_ker_spi3 = { + .gate_id = GATE_SPI3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi3", FLEXGEN_10, &clk_stm32_gate_ops, + CLK_SET_RATE_PARENT), +}; + +static struct clk_stm32_gate ck_ker_spi4 = { + .gate_id = GATE_SPI4, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi4", FLEXGEN_17, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_spi5 = { + .gate_id = GATE_SPI5, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi5", FLEXGEN_17, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_spi6 = { + .gate_id = GATE_SPI6, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi6", FLEXGEN_18, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_spi7 = { + .gate_id = GATE_SPI7, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi7", FLEXGEN_18, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_spi8 = { + .gate_id = GATE_SPI8, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_spi8", FLEXGEN_37, &clk_stm32_gate_ops, 0), +}; + +/* Timers */ +static struct clk_stm32_gate ck_icn_p_tim2 = { + .gate_id = GATE_TIM2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim2", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim3 = { + .gate_id = GATE_TIM3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim3", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim4 = { + .gate_id = GATE_TIM4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim4", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim5 = { + .gate_id = GATE_TIM5, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim5", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim6 = { + .gate_id = GATE_TIM6, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim6", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim7 = { + .gate_id = GATE_TIM7, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim7", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim10 = { + .gate_id = GATE_TIM10, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim10", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim11 = { + .gate_id = GATE_TIM11, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim11", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim12 = { + .gate_id = GATE_TIM12, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim12", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim13 = { + .gate_id = GATE_TIM13, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim13", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim14 = { + .gate_id = GATE_TIM14, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim14", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim1 = { + .gate_id = GATE_TIM1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim1", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim8 = { + .gate_id = GATE_TIM8, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim8", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim15 = { + .gate_id = GATE_TIM15, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim15", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim16 = { + .gate_id = GATE_TIM16, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim16", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim17 = { + .gate_id = GATE_TIM17, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim17", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_tim20 = { + .gate_id = GATE_TIM20, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_tim20", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim2 = { + .gate_id = GATE_TIM2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim2", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim3 = { + .gate_id = GATE_TIM3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim3", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim4 = { + .gate_id = GATE_TIM4, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim4", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim5 = { + .gate_id = GATE_TIM5, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim5", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim6 = { + .gate_id = GATE_TIM6, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim6", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim7 = { + .gate_id = GATE_TIM7, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim7", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim10 = { + .gate_id = GATE_TIM10, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim10", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim11 = { + .gate_id = GATE_TIM11, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim11", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim12 = { + .gate_id = GATE_TIM12, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim12", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim13 = { + .gate_id = GATE_TIM13, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim13", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim14 = { + .gate_id = GATE_TIM14, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim14", TIMG1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim1 = { + .gate_id = GATE_TIM1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim1", TIMG2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim8 = { + .gate_id = GATE_TIM8, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim8", TIMG2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim15 = { + .gate_id = GATE_TIM15, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim15", TIMG2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim16 = { + .gate_id = GATE_TIM16, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim16", TIMG2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim17 = { + .gate_id = GATE_TIM17, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim17", TIMG2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_tim20 = { + .gate_id = GATE_TIM20, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_tim20", TIMG2, &clk_stm32_gate_ops, 0), +}; + +/* UART/USART */ +static struct clk_stm32_gate ck_icn_p_usart2 = { + .gate_id = GATE_USART2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_usart2", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_usart3 = { + .gate_id = GATE_USART3, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_usart3", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_uart4 = { + .gate_id = GATE_UART4, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_uart4", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_uart5 = { + .gate_id = GATE_UART5, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_uart5", ICN_APB1, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_usart1 = { + .gate_id = GATE_USART1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_usart1", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_usart6 = { + .gate_id = GATE_USART6, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_usart6", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_uart7 = { + .gate_id = GATE_UART7, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_uart7", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_uart8 = { + .gate_id = GATE_UART8, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_uart8", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_uart9 = { + .gate_id = GATE_UART9, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_uart9", ICN_APB2, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_usart2 = { + .gate_id = GATE_USART2, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_usart2", FLEXGEN_08, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_uart4 = { + .gate_id = GATE_UART4, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_uart4", FLEXGEN_08, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_usart3 = { + .gate_id = GATE_USART3, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_usart3", FLEXGEN_09, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_uart5 = { + .gate_id = GATE_UART5, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_uart5", FLEXGEN_09, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_usart1 = { + .gate_id = GATE_USART1, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_usart1", FLEXGEN_19, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_usart6 = { + .gate_id = GATE_USART6, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_usart6", FLEXGEN_20, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_uart7 = { + .gate_id = GATE_UART7, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_uart7", FLEXGEN_21, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_uart8 = { + .gate_id = GATE_UART8, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_uart8", FLEXGEN_21, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_uart9 = { + .gate_id = GATE_UART9, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_uart9", FLEXGEN_22, &clk_stm32_gate_ops, 0), +}; + +/* USB2PHY1 */ +static struct clk_stm32_composite ck_ker_usb2phy1 = { + .gate_id = GATE_USB2PHY1, + .mux_id = MUX_USB2PHY1, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_ker_usb2phy1", usb2phy1_src, + &clk_stm32_composite_ops, 0), +}; + +/* USB2H */ +static struct clk_stm32_gate ck_icn_m_usb2ehci = { + .gate_id = GATE_USBH, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_m_usb2ehci", ICN_HSL, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_m_usb2ohci = { + .gate_id = GATE_USBH, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_m_usb2ohci", ICN_HSL, &clk_stm32_gate_ops, 0), +}; + +/* USB2PHY2 */ +static struct clk_stm32_composite ck_ker_usb2phy2_en = { + .gate_id = GATE_USB2PHY2, + .mux_id = MUX_USB2PHY2, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_ker_usb2phy2_en", usb2phy2_src, + &clk_stm32_composite_ops, 0), +}; + +/* USB3 PCIe COMBOPHY */ +static struct clk_stm32_gate ck_icn_p_usb3pciephy = { + .gate_id = GATE_USB3PCIEPHY, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_usb3pciephy", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_composite ck_ker_usb3pciephy = { + .gate_id = GATE_USB3PCIEPHY, + .mux_id = MUX_USB3PCIEPHY, + .div_id = NO_STM32_DIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("ck_ker_usb3pciephy", usb3pciphy_src, + &clk_stm32_composite_ops, 0), +}; + +/* USB3 DRD */ +static struct clk_stm32_gate ck_icn_m_usb3dr = { + .gate_id = GATE_USB3DR, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_m_usb3dr", ICN_HSL, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_usb2phy2 = { + .gate_id = GATE_USB3DR, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_usb2phy2", FLEXGEN_58, &clk_stm32_gate_ops, 0), +}; + +/* USBTC */ +static struct clk_stm32_gate ck_icn_p_usbtc = { + .gate_id = GATE_USBTC, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_usbtc", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_ker_usbtc = { + .gate_id = GATE_USBTC, + .hw.init = CLK_HW_INIT_INDEX("ck_ker_usbtc", FLEXGEN_35, &clk_stm32_gate_ops, 0), +}; + +/* VDEC / VENC */ +static struct clk_stm32_gate ck_icn_p_vdec = { + .gate_id = GATE_VDEC, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_vdec", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_venc = { + .gate_id = GATE_VENC, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_venc", ICN_APB4, &clk_stm32_gate_ops, 0), +}; + +/* VREF */ +static struct clk_stm32_gate ck_icn_p_vref = { + .gate_id = GATE_VREF, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_vref", ICN_APB3, &clk_stm32_gate_ops, 0), +}; + +/* WWDG */ +static struct clk_stm32_gate ck_icn_p_wwdg1 = { + .gate_id = GATE_WWDG1, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_wwdg1", ICN_APB3, &clk_stm32_gate_ops, 0), +}; + +static struct clk_stm32_gate ck_icn_p_wwdg2 = { + .gate_id = GATE_WWDG2, + .hw.init = CLK_HW_INIT_INDEX("ck_icn_p_wwdg2", ICN_LS_MCU, &clk_stm32_gate_ops, 0), +}; + +#define SECF_NONE -1 + +static const struct clock_config stm32mp25_clock_cfg[] = { + STM32_GATE_CFG(CK_BUS_ETH1, ck_icn_p_eth1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_ETH2, ck_icn_p_eth2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_PCIE, ck_icn_p_pcie, SECF_NONE), + STM32_GATE_CFG(CK_BUS_ETHSW, ck_icn_p_ethsw, SECF_NONE), + STM32_GATE_CFG(CK_BUS_ADC12, ck_icn_p_adc12, SECF_NONE), + STM32_GATE_CFG(CK_BUS_ADC3, ck_icn_p_adc3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_CCI, ck_icn_p_cci, SECF_NONE), + STM32_GATE_CFG(CK_BUS_CRC, ck_icn_p_crc, SECF_NONE), + STM32_GATE_CFG(CK_BUS_MDF1, ck_icn_p_mdf1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_OSPIIOM, ck_icn_p_ospiiom, SECF_NONE), + STM32_GATE_CFG(CK_BUS_HASH, ck_icn_p_hash, SECF_NONE), + STM32_GATE_CFG(CK_BUS_CRYP1, ck_icn_p_cryp1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_CRYP2, ck_icn_p_cryp2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_ADF1, ck_icn_p_adf1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI8, ck_icn_p_spi8, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LPUART1, ck_icn_p_lpuart1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C8, ck_icn_p_i2c8, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LPTIM3, ck_icn_p_lptim3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LPTIM4, ck_icn_p_lptim4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LPTIM5, ck_icn_p_lptim5, SECF_NONE), + STM32_GATE_CFG(CK_BUS_IWDG5, ck_icn_p_iwdg5, SECF_NONE), + STM32_GATE_CFG(CK_BUS_WWDG2, ck_icn_p_wwdg2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I3C4, ck_icn_p_i3c4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SDMMC1, ck_icn_m_sdmmc1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SDMMC2, ck_icn_m_sdmmc2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SDMMC3, ck_icn_m_sdmmc3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USB2OHCI, ck_icn_m_usb2ohci, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USB2EHCI, ck_icn_m_usb2ehci, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USB3DR, ck_icn_m_usb3dr, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM2, ck_icn_p_tim2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM3, ck_icn_p_tim3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM4, ck_icn_p_tim4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM5, ck_icn_p_tim5, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM6, ck_icn_p_tim6, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM7, ck_icn_p_tim7, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM10, ck_icn_p_tim10, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM11, ck_icn_p_tim11, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM12, ck_icn_p_tim12, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM13, ck_icn_p_tim13, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM14, ck_icn_p_tim14, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LPTIM1, ck_icn_p_lptim1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LPTIM2, ck_icn_p_lptim2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI2, ck_icn_p_spi2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI3, ck_icn_p_spi3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPDIFRX, ck_icn_p_spdifrx, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USART2, ck_icn_p_usart2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USART3, ck_icn_p_usart3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_UART4, ck_icn_p_uart4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_UART5, ck_icn_p_uart5, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C1, ck_icn_p_i2c1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C2, ck_icn_p_i2c2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C3, ck_icn_p_i2c3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C4, ck_icn_p_i2c4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C5, ck_icn_p_i2c5, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C6, ck_icn_p_i2c6, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I2C7, ck_icn_p_i2c7, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I3C1, ck_icn_p_i3c1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I3C2, ck_icn_p_i3c2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_I3C3, ck_icn_p_i3c3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM1, ck_icn_p_tim1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM8, ck_icn_p_tim8, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM15, ck_icn_p_tim15, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM16, ck_icn_p_tim16, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM17, ck_icn_p_tim17, SECF_NONE), + STM32_GATE_CFG(CK_BUS_TIM20, ck_icn_p_tim20, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SAI1, ck_icn_p_sai1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SAI2, ck_icn_p_sai2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SAI3, ck_icn_p_sai3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SAI4, ck_icn_p_sai4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USART1, ck_icn_p_usart1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USART6, ck_icn_p_usart6, SECF_NONE), + STM32_GATE_CFG(CK_BUS_UART7, ck_icn_p_uart7, SECF_NONE), + STM32_GATE_CFG(CK_BUS_UART8, ck_icn_p_uart8, SECF_NONE), + STM32_GATE_CFG(CK_BUS_UART9, ck_icn_p_uart9, SECF_NONE), + STM32_GATE_CFG(CK_BUS_FDCAN, ck_icn_p_fdcan, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI1, ck_icn_p_spi1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI4, ck_icn_p_spi4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI5, ck_icn_p_spi5, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI6, ck_icn_p_spi6, SECF_NONE), + STM32_GATE_CFG(CK_BUS_SPI7, ck_icn_p_spi7, SECF_NONE), + STM32_GATE_CFG(CK_BUS_IWDG2, ck_icn_p_iwdg2, SECF_NONE), + STM32_GATE_CFG(CK_BUS_IWDG3, ck_icn_p_iwdg3, SECF_NONE), + STM32_GATE_CFG(CK_BUS_IWDG4, ck_icn_p_iwdg4, SECF_NONE), + STM32_GATE_CFG(CK_BUS_WWDG1, ck_icn_p_wwdg1, SECF_NONE), + STM32_GATE_CFG(CK_BUS_VREF, ck_icn_p_vref, SECF_NONE), + STM32_GATE_CFG(CK_BUS_HDP, ck_icn_p_hdp, SECF_NONE), + STM32_GATE_CFG(CK_BUS_IS2M, ck_icn_p_is2m, SECF_NONE), + STM32_GATE_CFG(CK_BUS_DSI, ck_icn_p_dsi, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LTDC, ck_icn_p_ltdc, SECF_NONE), + STM32_GATE_CFG(CK_BUS_CSI, ck_icn_p_csi, SECF_NONE), + STM32_GATE_CFG(CK_BUS_DCMIPP, ck_icn_p_dcmipp, SECF_NONE), + STM32_GATE_CFG(CK_BUS_LVDS, ck_icn_p_lvds, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USBTC, ck_icn_p_usbtc, SECF_NONE), + STM32_GATE_CFG(CK_BUS_USB3PCIEPHY, ck_icn_p_usb3pciephy, SECF_NONE), + STM32_GATE_CFG(CK_BUS_VDEC, ck_icn_p_vdec, SECF_NONE), + STM32_GATE_CFG(CK_BUS_VENC, ck_icn_p_venc, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM2, ck_ker_tim2, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM3, ck_ker_tim3, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM4, ck_ker_tim4, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM5, ck_ker_tim5, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM6, ck_ker_tim6, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM7, ck_ker_tim7, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM10, ck_ker_tim10, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM11, ck_ker_tim11, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM12, ck_ker_tim12, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM13, ck_ker_tim13, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM14, ck_ker_tim14, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM1, ck_ker_tim1, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM8, ck_ker_tim8, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM15, ck_ker_tim15, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM16, ck_ker_tim16, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM17, ck_ker_tim17, SECF_NONE), + STM32_GATE_CFG(CK_KER_TIM20, ck_ker_tim20, SECF_NONE), + STM32_GATE_CFG(CK_KER_LPTIM1, ck_ker_lptim1, SECF_NONE), + STM32_GATE_CFG(CK_KER_LPTIM2, ck_ker_lptim2, SECF_NONE), + STM32_GATE_CFG(CK_KER_USART2, ck_ker_usart2, SECF_NONE), + STM32_GATE_CFG(CK_KER_UART4, ck_ker_uart4, SECF_NONE), + STM32_GATE_CFG(CK_KER_USART3, ck_ker_usart3, SECF_NONE), + STM32_GATE_CFG(CK_KER_UART5, ck_ker_uart5, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI2, ck_ker_spi2, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI3, ck_ker_spi3, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPDIFRX, ck_ker_spdifrx, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C1, ck_ker_i2c1, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C2, ck_ker_i2c2, SECF_NONE), + STM32_GATE_CFG(CK_KER_I3C1, ck_ker_i3c1, SECF_NONE), + STM32_GATE_CFG(CK_KER_I3C2, ck_ker_i3c2, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C3, ck_ker_i2c3, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C5, ck_ker_i2c5, SECF_NONE), + STM32_GATE_CFG(CK_KER_I3C3, ck_ker_i3c3, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C4, ck_ker_i2c4, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C6, ck_ker_i2c6, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C7, ck_ker_i2c7, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI1, ck_ker_spi1, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI4, ck_ker_spi4, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI5, ck_ker_spi5, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI6, ck_ker_spi6, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI7, ck_ker_spi7, SECF_NONE), + STM32_GATE_CFG(CK_KER_USART1, ck_ker_usart1, SECF_NONE), + STM32_GATE_CFG(CK_KER_USART6, ck_ker_usart6, SECF_NONE), + STM32_GATE_CFG(CK_KER_UART7, ck_ker_uart7, SECF_NONE), + STM32_GATE_CFG(CK_KER_UART8, ck_ker_uart8, SECF_NONE), + STM32_GATE_CFG(CK_KER_UART9, ck_ker_uart9, SECF_NONE), + STM32_GATE_CFG(CK_KER_MDF1, ck_ker_mdf1, SECF_NONE), + STM32_GATE_CFG(CK_KER_SAI1, ck_ker_sai1, SECF_NONE), + STM32_GATE_CFG(CK_KER_SAI2, ck_ker_sai2, SECF_NONE), + STM32_GATE_CFG(CK_KER_SAI3, ck_ker_sai3, SECF_NONE), + STM32_GATE_CFG(CK_KER_SAI4, ck_ker_sai4, SECF_NONE), + STM32_GATE_CFG(CK_KER_FDCAN, ck_ker_fdcan, SECF_NONE), + STM32_GATE_CFG(CK_KER_CSI, ck_ker_csi, SECF_NONE), + STM32_GATE_CFG(CK_KER_CSITXESC, ck_ker_csitxesc, SECF_NONE), + STM32_GATE_CFG(CK_KER_CSIPHY, ck_ker_csiphy, SECF_NONE), + STM32_GATE_CFG(CK_KER_USBTC, ck_ker_usbtc, SECF_NONE), + STM32_GATE_CFG(CK_KER_I3C4, ck_ker_i3c4, SECF_NONE), + STM32_GATE_CFG(CK_KER_SPI8, ck_ker_spi8, SECF_NONE), + STM32_GATE_CFG(CK_KER_I2C8, ck_ker_i2c8, SECF_NONE), + STM32_GATE_CFG(CK_KER_LPUART1, ck_ker_lpuart1, SECF_NONE), + STM32_GATE_CFG(CK_KER_LPTIM3, ck_ker_lptim3, SECF_NONE), + STM32_GATE_CFG(CK_KER_LPTIM4, ck_ker_lptim4, SECF_NONE), + STM32_GATE_CFG(CK_KER_LPTIM5, ck_ker_lptim5, SECF_NONE), + STM32_GATE_CFG(CK_KER_ADF1, ck_ker_adf1, SECF_NONE), + STM32_GATE_CFG(CK_KER_SDMMC1, ck_ker_sdmmc1, SECF_NONE), + STM32_GATE_CFG(CK_KER_SDMMC2, ck_ker_sdmmc2, SECF_NONE), + STM32_GATE_CFG(CK_KER_SDMMC3, ck_ker_sdmmc3, SECF_NONE), + STM32_GATE_CFG(CK_KER_ETH1, ck_ker_eth1, SECF_NONE), + STM32_GATE_CFG(CK_ETH1_STP, ck_ker_eth1stp, SECF_NONE), + STM32_GATE_CFG(CK_KER_ETHSW, ck_ker_ethsw, SECF_NONE), + STM32_GATE_CFG(CK_KER_ETH2, ck_ker_eth2, SECF_NONE), + STM32_GATE_CFG(CK_ETH2_STP, ck_ker_eth2stp, SECF_NONE), + STM32_GATE_CFG(CK_KER_ETH1PTP, ck_ker_eth1ptp, SECF_NONE), + STM32_GATE_CFG(CK_KER_ETH2PTP, ck_ker_eth2ptp, SECF_NONE), + STM32_GATE_CFG(CK_BUS_GPU, ck_icn_m_gpu, SECF_NONE), + STM32_GATE_CFG(CK_KER_GPU, ck_ker_gpu, SECF_NONE), + STM32_GATE_CFG(CK_KER_ETHSWREF, ck_ker_ethswref, SECF_NONE), + STM32_GATE_CFG(CK_BUS_ETHSWACMCFG, ck_icn_p_ethsw_acm_cfg, SECF_NONE), + STM32_GATE_CFG(CK_BUS_ETHSWACMMSG, ck_icn_p_ethsw_acm_msg, SECF_NONE), + STM32_GATE_CFG(CK_ETH1_MAC, ck_ker_eth1mac, SECF_NONE), + STM32_GATE_CFG(CK_ETH1_TX, ck_ker_eth1tx, SECF_NONE), + STM32_GATE_CFG(CK_ETH1_RX, ck_ker_eth1rx, SECF_NONE), + STM32_GATE_CFG(CK_ETH2_MAC, ck_ker_eth2mac, SECF_NONE), + STM32_GATE_CFG(CK_ETH2_TX, ck_ker_eth2tx, SECF_NONE), + STM32_GATE_CFG(CK_ETH2_RX, ck_ker_eth2rx, SECF_NONE), + STM32_COMPOSITE_CFG(CK_MCO1, ck_mco1, SECF_NONE), + STM32_COMPOSITE_CFG(CK_MCO2, ck_mco2, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_ADC12, ck_ker_adc12, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_ADC3, ck_ker_adc3, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_USB2PHY1, ck_ker_usb2phy1, SECF_NONE), + STM32_GATE_CFG(CK_KER_USB2PHY2, ck_ker_usb2phy2, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_USB2PHY2EN, ck_ker_usb2phy2_en, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_USB3PCIEPHY, ck_ker_usb3pciephy, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_DSIBLANE, clk_lanebyte, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_DSIPHY, clk_phy_dsi, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_LVDSPHY, ck_ker_lvdsphy, SECF_NONE), + STM32_COMPOSITE_CFG(CK_KER_DTS, ck_ker_dts, SECF_NONE), + STM32_GATE_CFG(CK_KER_LTDC, ck_ker_ltdc, SECF_NONE), +}; + +#define RESET_MP25(id, _offset, _bit_idx, _set_clr) \ + [id] = &(struct stm32_reset_cfg){ \ + .offset = (_offset), \ + .bit_idx = (_bit_idx), \ + .set_clr = (_set_clr), \ + } + +static const struct stm32_reset_cfg *stm32mp25_reset_cfg[STM32MP25_LAST_RESET] = { + RESET_MP25(TIM1_R, RCC_TIM1CFGR, 0, 0), + RESET_MP25(TIM2_R, RCC_TIM2CFGR, 0, 0), + RESET_MP25(TIM3_R, RCC_TIM3CFGR, 0, 0), + RESET_MP25(TIM4_R, RCC_TIM4CFGR, 0, 0), + RESET_MP25(TIM5_R, RCC_TIM5CFGR, 0, 0), + RESET_MP25(TIM6_R, RCC_TIM6CFGR, 0, 0), + RESET_MP25(TIM7_R, RCC_TIM7CFGR, 0, 0), + RESET_MP25(TIM8_R, RCC_TIM8CFGR, 0, 0), + RESET_MP25(TIM10_R, RCC_TIM10CFGR, 0, 0), + RESET_MP25(TIM11_R, RCC_TIM11CFGR, 0, 0), + RESET_MP25(TIM12_R, RCC_TIM12CFGR, 0, 0), + RESET_MP25(TIM13_R, RCC_TIM13CFGR, 0, 0), + RESET_MP25(TIM14_R, RCC_TIM14CFGR, 0, 0), + RESET_MP25(TIM15_R, RCC_TIM15CFGR, 0, 0), + RESET_MP25(TIM16_R, RCC_TIM16CFGR, 0, 0), + RESET_MP25(TIM17_R, RCC_TIM17CFGR, 0, 0), + RESET_MP25(TIM20_R, RCC_TIM20CFGR, 0, 0), + RESET_MP25(LPTIM1_R, RCC_LPTIM1CFGR, 0, 0), + RESET_MP25(LPTIM2_R, RCC_LPTIM2CFGR, 0, 0), + RESET_MP25(LPTIM3_R, RCC_LPTIM3CFGR, 0, 0), + RESET_MP25(LPTIM4_R, RCC_LPTIM4CFGR, 0, 0), + RESET_MP25(LPTIM5_R, RCC_LPTIM5CFGR, 0, 0), + RESET_MP25(SPI1_R, RCC_SPI1CFGR, 0, 0), + RESET_MP25(SPI2_R, RCC_SPI2CFGR, 0, 0), + RESET_MP25(SPI3_R, RCC_SPI3CFGR, 0, 0), + RESET_MP25(SPI4_R, RCC_SPI4CFGR, 0, 0), + RESET_MP25(SPI5_R, RCC_SPI5CFGR, 0, 0), + RESET_MP25(SPI6_R, RCC_SPI6CFGR, 0, 0), + RESET_MP25(SPI7_R, RCC_SPI7CFGR, 0, 0), + RESET_MP25(SPI8_R, RCC_SPI8CFGR, 0, 0), + RESET_MP25(SPDIFRX_R, RCC_SPDIFRXCFGR, 0, 0), + RESET_MP25(USART1_R, RCC_USART1CFGR, 0, 0), + RESET_MP25(USART2_R, RCC_USART2CFGR, 0, 0), + RESET_MP25(USART3_R, RCC_USART3CFGR, 0, 0), + RESET_MP25(UART4_R, RCC_UART4CFGR, 0, 0), + RESET_MP25(UART5_R, RCC_UART5CFGR, 0, 0), + RESET_MP25(USART6_R, RCC_USART6CFGR, 0, 0), + RESET_MP25(UART7_R, RCC_UART7CFGR, 0, 0), + RESET_MP25(UART8_R, RCC_UART8CFGR, 0, 0), + RESET_MP25(UART9_R, RCC_UART9CFGR, 0, 0), + RESET_MP25(LPUART1_R, RCC_LPUART1CFGR, 0, 0), + RESET_MP25(IS2M_R, RCC_IS2MCFGR, 0, 0), + RESET_MP25(I2C1_R, RCC_I2C1CFGR, 0, 0), + RESET_MP25(I2C2_R, RCC_I2C2CFGR, 0, 0), + RESET_MP25(I2C3_R, RCC_I2C3CFGR, 0, 0), + RESET_MP25(I2C4_R, RCC_I2C4CFGR, 0, 0), + RESET_MP25(I2C5_R, RCC_I2C5CFGR, 0, 0), + RESET_MP25(I2C6_R, RCC_I2C6CFGR, 0, 0), + RESET_MP25(I2C7_R, RCC_I2C7CFGR, 0, 0), + RESET_MP25(I2C8_R, RCC_I2C8CFGR, 0, 0), + RESET_MP25(SAI1_R, RCC_SAI1CFGR, 0, 0), + RESET_MP25(SAI2_R, RCC_SAI2CFGR, 0, 0), + RESET_MP25(SAI3_R, RCC_SAI3CFGR, 0, 0), + RESET_MP25(SAI4_R, RCC_SAI4CFGR, 0, 0), + RESET_MP25(MDF1_R, RCC_MDF1CFGR, 0, 0), + RESET_MP25(MDF2_R, RCC_ADF1CFGR, 0, 0), + RESET_MP25(FDCAN_R, RCC_FDCANCFGR, 0, 0), + RESET_MP25(HDP_R, RCC_HDPCFGR, 0, 0), + RESET_MP25(ADC12_R, RCC_ADC12CFGR, 0, 0), + RESET_MP25(ADC3_R, RCC_ADC3CFGR, 0, 0), + RESET_MP25(ETH1_R, RCC_ETH1CFGR, 0, 0), + RESET_MP25(ETH2_R, RCC_ETH2CFGR, 0, 0), + RESET_MP25(USBH_R, RCC_USBHCFGR, 0, 0), + RESET_MP25(USB2PHY1_R, RCC_USB2PHY1CFGR, 0, 0), + RESET_MP25(USB2PHY2_R, RCC_USB2PHY2CFGR, 0, 0), + RESET_MP25(USB3DR_R, RCC_USB3DRCFGR, 0, 0), + RESET_MP25(USB3PCIEPHY_R, RCC_USB3PCIEPHYCFGR, 0, 0), + RESET_MP25(USBTC_R, RCC_USBTCCFGR, 0, 0), + RESET_MP25(ETHSW_R, RCC_ETHSWCFGR, 0, 0), + RESET_MP25(SDMMC1_R, RCC_SDMMC1CFGR, 0, 0), + RESET_MP25(SDMMC1DLL_R, RCC_SDMMC1CFGR, 16, 0), + RESET_MP25(SDMMC2_R, RCC_SDMMC2CFGR, 0, 0), + RESET_MP25(SDMMC2DLL_R, RCC_SDMMC2CFGR, 16, 0), + RESET_MP25(SDMMC3_R, RCC_SDMMC3CFGR, 0, 0), + RESET_MP25(SDMMC3DLL_R, RCC_SDMMC3CFGR, 16, 0), + RESET_MP25(GPU_R, RCC_GPUCFGR, 0, 0), + RESET_MP25(LTDC_R, RCC_LTDCCFGR, 0, 0), + RESET_MP25(DSI_R, RCC_DSICFGR, 0, 0), + RESET_MP25(LVDS_R, RCC_LVDSCFGR, 0, 0), + RESET_MP25(CSI_R, RCC_CSICFGR, 0, 0), + RESET_MP25(DCMIPP_R, RCC_DCMIPPCFGR, 0, 0), + RESET_MP25(CCI_R, RCC_CCICFGR, 0, 0), + RESET_MP25(VDEC_R, RCC_VDECCFGR, 0, 0), + RESET_MP25(VENC_R, RCC_VENCCFGR, 0, 0), + RESET_MP25(WWDG1_R, RCC_WWDG1CFGR, 0, 0), + RESET_MP25(WWDG2_R, RCC_WWDG2CFGR, 0, 0), + RESET_MP25(VREF_R, RCC_VREFCFGR, 0, 0), + RESET_MP25(DTS_R, RCC_DTSCFGR, 0, 0), + RESET_MP25(CRC_R, RCC_CRCCFGR, 0, 0), + RESET_MP25(SERC_R, RCC_SERCCFGR, 0, 0), + RESET_MP25(OSPIIOM_R, RCC_OSPIIOMCFGR, 0, 0), + RESET_MP25(I3C1_R, RCC_I3C1CFGR, 0, 0), + RESET_MP25(I3C2_R, RCC_I3C2CFGR, 0, 0), + RESET_MP25(I3C3_R, RCC_I3C3CFGR, 0, 0), + RESET_MP25(I3C4_R, RCC_I3C4CFGR, 0, 0), + RESET_MP25(IWDG2_KER_R, RCC_IWDGC1CFGSETR, 18, 1), + RESET_MP25(IWDG4_KER_R, RCC_IWDGC2CFGSETR, 18, 1), + RESET_MP25(RNG_R, RCC_RNGCFGR, 0, 0), + RESET_MP25(PKA_R, RCC_PKACFGR, 0, 0), + RESET_MP25(SAES_R, RCC_SAESCFGR, 0, 0), + RESET_MP25(HASH_R, RCC_HASHCFGR, 0, 0), + RESET_MP25(CRYP1_R, RCC_CRYP1CFGR, 0, 0), + RESET_MP25(CRYP2_R, RCC_CRYP2CFGR, 0, 0), + RESET_MP25(PCIE_R, RCC_PCIECFGR, 0, 0), +}; + +static u16 stm32mp25_cpt_gate[GATE_NB]; + +static struct clk_stm32_clock_data stm32mp25_clock_data = { + .gate_cpt = stm32mp25_cpt_gate, + .gates = stm32mp25_gates, + .muxes = stm32mp25_muxes, +}; + +static struct clk_stm32_reset_data stm32mp25_reset_data = { + .reset_lines = stm32mp25_reset_cfg, + .nr_lines = ARRAY_SIZE(stm32mp25_reset_cfg), +}; + +static const struct stm32_rcc_match_data stm32mp25_data = { + .tab_clocks = stm32mp25_clock_cfg, + .num_clocks = ARRAY_SIZE(stm32mp25_clock_cfg), + .maxbinding = STM32MP25_LAST_CLK, + .clock_data = &stm32mp25_clock_data, + .reset_data = &stm32mp25_reset_data, +}; + +static const struct of_device_id stm32mp25_match_data[] = { + { .compatible = "st,stm32mp25-rcc", .data = &stm32mp25_data, }, + { } +}; +MODULE_DEVICE_TABLE(of, stm32mp25_match_data); + +static int stm32mp25_rcc_clocks_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + void __iomem *base; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + return stm32_rcc_init(dev, stm32mp25_match_data, base); +} + +static struct platform_driver stm32mp25_rcc_clocks_driver = { + .driver = { + .name = "stm32mp25_rcc", + .of_match_table = stm32mp25_match_data, + }, + .probe = stm32mp25_rcc_clocks_probe, +}; + +static int __init stm32mp25_clocks_init(void) +{ + return platform_driver_register(&stm32mp25_rcc_clocks_driver); +} + +core_initcall(stm32mp25_clocks_init); diff --git a/drivers/clk/stm32/reset-stm32.c b/drivers/clk/stm32/reset-stm32.c index 14c2ee1eebee0..5a8f525842ce2 100644 --- a/drivers/clk/stm32/reset-stm32.c +++ b/drivers/clk/stm32/reset-stm32.c @@ -19,6 +19,7 @@ struct stm32_reset_data { struct reset_controller_dev rcdev; void __iomem *membase; u32 clear_offset; + const struct stm32_reset_cfg **reset_lines; }; static inline struct stm32_reset_data * @@ -27,22 +28,46 @@ to_stm32_reset_data(struct reset_controller_dev *rcdev) return container_of(rcdev, struct stm32_reset_data, rcdev); } +static const struct stm32_reset_cfg *stm32_get_reset_line(struct reset_controller_dev *rcdev, + unsigned long id, + struct stm32_reset_cfg *line) +{ + struct stm32_reset_data *data = to_stm32_reset_data(rcdev); + + if (!data->reset_lines) { + int reg_width = sizeof(u32); + int bank = id / (reg_width * BITS_PER_BYTE); + int offset = id % (reg_width * BITS_PER_BYTE); + + line->offset = bank * reg_width; + line->bit_idx = offset; + line->set_clr = (data->clear_offset ? true : false); + + return line; + } + + return data->reset_lines[id]; +} + static int stm32_reset_update(struct reset_controller_dev *rcdev, unsigned long id, bool assert) { struct stm32_reset_data *data = to_stm32_reset_data(rcdev); - int reg_width = sizeof(u32); - int bank = id / (reg_width * BITS_PER_BYTE); - int offset = id % (reg_width * BITS_PER_BYTE); + struct stm32_reset_cfg line_reset; + const struct stm32_reset_cfg *ptr_line; - if (data->clear_offset) { + ptr_line = stm32_get_reset_line(rcdev, id, &line_reset); + if (!ptr_line) + return -EPERM; + + if (ptr_line->set_clr) { void __iomem *addr; - addr = data->membase + (bank * reg_width); + addr = data->membase + ptr_line->offset; if (!assert) addr += data->clear_offset; - writel(BIT(offset), addr); + writel(BIT(ptr_line->bit_idx), addr); } else { unsigned long flags; @@ -50,14 +75,14 @@ static int stm32_reset_update(struct reset_controller_dev *rcdev, spin_lock_irqsave(&data->lock, flags); - reg = readl(data->membase + (bank * reg_width)); + reg = readl(data->membase + ptr_line->offset); if (assert) - reg |= BIT(offset); + reg |= BIT(ptr_line->bit_idx); else - reg &= ~BIT(offset); + reg &= ~BIT(ptr_line->bit_idx); - writel(reg, data->membase + (bank * reg_width)); + writel(reg, data->membase + ptr_line->offset); spin_unlock_irqrestore(&data->lock, flags); } @@ -81,14 +106,17 @@ static int stm32_reset_status(struct reset_controller_dev *rcdev, unsigned long id) { struct stm32_reset_data *data = to_stm32_reset_data(rcdev); - int reg_width = sizeof(u32); - int bank = id / (reg_width * BITS_PER_BYTE); - int offset = id % (reg_width * BITS_PER_BYTE); + struct stm32_reset_cfg line_reset; + const struct stm32_reset_cfg *ptr_line; u32 reg; - reg = readl(data->membase + (bank * reg_width)); + ptr_line = stm32_get_reset_line(rcdev, id, &line_reset); + if (!ptr_line) + return -EPERM; + + reg = readl(data->membase + ptr_line->offset); - return !!(reg & BIT(offset)); + return !!(reg & BIT(ptr_line->bit_idx)); } static const struct reset_control_ops stm32_reset_ops = { @@ -113,6 +141,7 @@ int stm32_rcc_reset_init(struct device *dev, struct clk_stm32_reset_data *data, reset_data->rcdev.ops = &stm32_reset_ops; reset_data->rcdev.of_node = dev_of_node(dev); reset_data->rcdev.nr_resets = data->nr_lines; + reset_data->reset_lines = data->reset_lines; reset_data->clear_offset = data->clear_offset; return reset_controller_register(&reset_data->rcdev); diff --git a/drivers/clk/stm32/reset-stm32.h b/drivers/clk/stm32/reset-stm32.h index 8cf1cc9be480b..f79cad21dfd68 100644 --- a/drivers/clk/stm32/reset-stm32.h +++ b/drivers/clk/stm32/reset-stm32.h @@ -4,8 +4,15 @@ * Author: Gabriel Fernandez for STMicroelectronics. */ +struct stm32_reset_cfg { + u16 offset; + u8 bit_idx; + bool set_clr; +}; + struct clk_stm32_reset_data { const struct reset_control_ops *ops; + const struct stm32_reset_cfg **reset_lines; unsigned int nr_lines; u32 clear_offset; }; diff --git a/drivers/clk/stm32/stm32mp25_rcc.h b/drivers/clk/stm32/stm32mp25_rcc.h new file mode 100644 index 0000000000000..687bc6a786270 --- /dev/null +++ b/drivers/clk/stm32/stm32mp25_rcc.h @@ -0,0 +1,712 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) STMicroelectronics 2023 - All Rights Reserved + * Author: Gabriel Fernandez for STMicroelectronics. + */ + +#ifndef STM32MP25_RCC_H +#define STM32MP25_RCC_H + +#define RCC_SECCFGR0 0x0 +#define RCC_SECCFGR1 0x4 +#define RCC_SECCFGR2 0x8 +#define RCC_SECCFGR3 0xC +#define RCC_PRIVCFGR0 0x10 +#define RCC_PRIVCFGR1 0x14 +#define RCC_PRIVCFGR2 0x18 +#define RCC_PRIVCFGR3 0x1C +#define RCC_RCFGLOCKR0 0x20 +#define RCC_RCFGLOCKR1 0x24 +#define RCC_RCFGLOCKR2 0x28 +#define RCC_RCFGLOCKR3 0x2C +#define RCC_R0CIDCFGR 0x30 +#define RCC_R0SEMCR 0x34 +#define RCC_R1CIDCFGR 0x38 +#define RCC_R1SEMCR 0x3C +#define RCC_R2CIDCFGR 0x40 +#define RCC_R2SEMCR 0x44 +#define RCC_R3CIDCFGR 0x48 +#define RCC_R3SEMCR 0x4C +#define RCC_R4CIDCFGR 0x50 +#define RCC_R4SEMCR 0x54 +#define RCC_R5CIDCFGR 0x58 +#define RCC_R5SEMCR 0x5C +#define RCC_R6CIDCFGR 0x60 +#define RCC_R6SEMCR 0x64 +#define RCC_R7CIDCFGR 0x68 +#define RCC_R7SEMCR 0x6C +#define RCC_R8CIDCFGR 0x70 +#define RCC_R8SEMCR 0x74 +#define RCC_R9CIDCFGR 0x78 +#define RCC_R9SEMCR 0x7C +#define RCC_R10CIDCFGR 0x80 +#define RCC_R10SEMCR 0x84 +#define RCC_R11CIDCFGR 0x88 +#define RCC_R11SEMCR 0x8C +#define RCC_R12CIDCFGR 0x90 +#define RCC_R12SEMCR 0x94 +#define RCC_R13CIDCFGR 0x98 +#define RCC_R13SEMCR 0x9C +#define RCC_R14CIDCFGR 0xA0 +#define RCC_R14SEMCR 0xA4 +#define RCC_R15CIDCFGR 0xA8 +#define RCC_R15SEMCR 0xAC +#define RCC_R16CIDCFGR 0xB0 +#define RCC_R16SEMCR 0xB4 +#define RCC_R17CIDCFGR 0xB8 +#define RCC_R17SEMCR 0xBC +#define RCC_R18CIDCFGR 0xC0 +#define RCC_R18SEMCR 0xC4 +#define RCC_R19CIDCFGR 0xC8 +#define RCC_R19SEMCR 0xCC +#define RCC_R20CIDCFGR 0xD0 +#define RCC_R20SEMCR 0xD4 +#define RCC_R21CIDCFGR 0xD8 +#define RCC_R21SEMCR 0xDC +#define RCC_R22CIDCFGR 0xE0 +#define RCC_R22SEMCR 0xE4 +#define RCC_R23CIDCFGR 0xE8 +#define RCC_R23SEMCR 0xEC +#define RCC_R24CIDCFGR 0xF0 +#define RCC_R24SEMCR 0xF4 +#define RCC_R25CIDCFGR 0xF8 +#define RCC_R25SEMCR 0xFC +#define RCC_R26CIDCFGR 0x100 +#define RCC_R26SEMCR 0x104 +#define RCC_R27CIDCFGR 0x108 +#define RCC_R27SEMCR 0x10C +#define RCC_R28CIDCFGR 0x110 +#define RCC_R28SEMCR 0x114 +#define RCC_R29CIDCFGR 0x118 +#define RCC_R29SEMCR 0x11C +#define RCC_R30CIDCFGR 0x120 +#define RCC_R30SEMCR 0x124 +#define RCC_R31CIDCFGR 0x128 +#define RCC_R31SEMCR 0x12C +#define RCC_R32CIDCFGR 0x130 +#define RCC_R32SEMCR 0x134 +#define RCC_R33CIDCFGR 0x138 +#define RCC_R33SEMCR 0x13C +#define RCC_R34CIDCFGR 0x140 +#define RCC_R34SEMCR 0x144 +#define RCC_R35CIDCFGR 0x148 +#define RCC_R35SEMCR 0x14C +#define RCC_R36CIDCFGR 0x150 +#define RCC_R36SEMCR 0x154 +#define RCC_R37CIDCFGR 0x158 +#define RCC_R37SEMCR 0x15C +#define RCC_R38CIDCFGR 0x160 +#define RCC_R38SEMCR 0x164 +#define RCC_R39CIDCFGR 0x168 +#define RCC_R39SEMCR 0x16C +#define RCC_R40CIDCFGR 0x170 +#define RCC_R40SEMCR 0x174 +#define RCC_R41CIDCFGR 0x178 +#define RCC_R41SEMCR 0x17C +#define RCC_R42CIDCFGR 0x180 +#define RCC_R42SEMCR 0x184 +#define RCC_R43CIDCFGR 0x188 +#define RCC_R43SEMCR 0x18C +#define RCC_R44CIDCFGR 0x190 +#define RCC_R44SEMCR 0x194 +#define RCC_R45CIDCFGR 0x198 +#define RCC_R45SEMCR 0x19C +#define RCC_R46CIDCFGR 0x1A0 +#define RCC_R46SEMCR 0x1A4 +#define RCC_R47CIDCFGR 0x1A8 +#define RCC_R47SEMCR 0x1AC +#define RCC_R48CIDCFGR 0x1B0 +#define RCC_R48SEMCR 0x1B4 +#define RCC_R49CIDCFGR 0x1B8 +#define RCC_R49SEMCR 0x1BC +#define RCC_R50CIDCFGR 0x1C0 +#define RCC_R50SEMCR 0x1C4 +#define RCC_R51CIDCFGR 0x1C8 +#define RCC_R51SEMCR 0x1CC +#define RCC_R52CIDCFGR 0x1D0 +#define RCC_R52SEMCR 0x1D4 +#define RCC_R53CIDCFGR 0x1D8 +#define RCC_R53SEMCR 0x1DC +#define RCC_R54CIDCFGR 0x1E0 +#define RCC_R54SEMCR 0x1E4 +#define RCC_R55CIDCFGR 0x1E8 +#define RCC_R55SEMCR 0x1EC +#define RCC_R56CIDCFGR 0x1F0 +#define RCC_R56SEMCR 0x1F4 +#define RCC_R57CIDCFGR 0x1F8 +#define RCC_R57SEMCR 0x1FC +#define RCC_R58CIDCFGR 0x200 +#define RCC_R58SEMCR 0x204 +#define RCC_R59CIDCFGR 0x208 +#define RCC_R59SEMCR 0x20C +#define RCC_R60CIDCFGR 0x210 +#define RCC_R60SEMCR 0x214 +#define RCC_R61CIDCFGR 0x218 +#define RCC_R61SEMCR 0x21C +#define RCC_R62CIDCFGR 0x220 +#define RCC_R62SEMCR 0x224 +#define RCC_R63CIDCFGR 0x228 +#define RCC_R63SEMCR 0x22C +#define RCC_R64CIDCFGR 0x230 +#define RCC_R64SEMCR 0x234 +#define RCC_R65CIDCFGR 0x238 +#define RCC_R65SEMCR 0x23C +#define RCC_R66CIDCFGR 0x240 +#define RCC_R66SEMCR 0x244 +#define RCC_R67CIDCFGR 0x248 +#define RCC_R67SEMCR 0x24C +#define RCC_R68CIDCFGR 0x250 +#define RCC_R68SEMCR 0x254 +#define RCC_R69CIDCFGR 0x258 +#define RCC_R69SEMCR 0x25C +#define RCC_R70CIDCFGR 0x260 +#define RCC_R70SEMCR 0x264 +#define RCC_R71CIDCFGR 0x268 +#define RCC_R71SEMCR 0x26C +#define RCC_R72CIDCFGR 0x270 +#define RCC_R72SEMCR 0x274 +#define RCC_R73CIDCFGR 0x278 +#define RCC_R73SEMCR 0x27C +#define RCC_R74CIDCFGR 0x280 +#define RCC_R74SEMCR 0x284 +#define RCC_R75CIDCFGR 0x288 +#define RCC_R75SEMCR 0x28C +#define RCC_R76CIDCFGR 0x290 +#define RCC_R76SEMCR 0x294 +#define RCC_R77CIDCFGR 0x298 +#define RCC_R77SEMCR 0x29C +#define RCC_R78CIDCFGR 0x2A0 +#define RCC_R78SEMCR 0x2A4 +#define RCC_R79CIDCFGR 0x2A8 +#define RCC_R79SEMCR 0x2AC +#define RCC_R80CIDCFGR 0x2B0 +#define RCC_R80SEMCR 0x2B4 +#define RCC_R81CIDCFGR 0x2B8 +#define RCC_R81SEMCR 0x2BC +#define RCC_R82CIDCFGR 0x2C0 +#define RCC_R82SEMCR 0x2C4 +#define RCC_R83CIDCFGR 0x2C8 +#define RCC_R83SEMCR 0x2CC +#define RCC_R84CIDCFGR 0x2D0 +#define RCC_R84SEMCR 0x2D4 +#define RCC_R85CIDCFGR 0x2D8 +#define RCC_R85SEMCR 0x2DC +#define RCC_R86CIDCFGR 0x2E0 +#define RCC_R86SEMCR 0x2E4 +#define RCC_R87CIDCFGR 0x2E8 +#define RCC_R87SEMCR 0x2EC +#define RCC_R88CIDCFGR 0x2F0 +#define RCC_R88SEMCR 0x2F4 +#define RCC_R89CIDCFGR 0x2F8 +#define RCC_R89SEMCR 0x2FC +#define RCC_R90CIDCFGR 0x300 +#define RCC_R90SEMCR 0x304 +#define RCC_R91CIDCFGR 0x308 +#define RCC_R91SEMCR 0x30C +#define RCC_R92CIDCFGR 0x310 +#define RCC_R92SEMCR 0x314 +#define RCC_R93CIDCFGR 0x318 +#define RCC_R93SEMCR 0x31C +#define RCC_R94CIDCFGR 0x320 +#define RCC_R94SEMCR 0x324 +#define RCC_R95CIDCFGR 0x328 +#define RCC_R95SEMCR 0x32C +#define RCC_R96CIDCFGR 0x330 +#define RCC_R96SEMCR 0x334 +#define RCC_R97CIDCFGR 0x338 +#define RCC_R97SEMCR 0x33C +#define RCC_R98CIDCFGR 0x340 +#define RCC_R98SEMCR 0x344 +#define RCC_R99CIDCFGR 0x348 +#define RCC_R99SEMCR 0x34C +#define RCC_R100CIDCFGR 0x350 +#define RCC_R100SEMCR 0x354 +#define RCC_R101CIDCFGR 0x358 +#define RCC_R101SEMCR 0x35C +#define RCC_R102CIDCFGR 0x360 +#define RCC_R102SEMCR 0x364 +#define RCC_R103CIDCFGR 0x368 +#define RCC_R103SEMCR 0x36C +#define RCC_R104CIDCFGR 0x370 +#define RCC_R104SEMCR 0x374 +#define RCC_R105CIDCFGR 0x378 +#define RCC_R105SEMCR 0x37C +#define RCC_R106CIDCFGR 0x380 +#define RCC_R106SEMCR 0x384 +#define RCC_R107CIDCFGR 0x388 +#define RCC_R107SEMCR 0x38C +#define RCC_R108CIDCFGR 0x390 +#define RCC_R108SEMCR 0x394 +#define RCC_R109CIDCFGR 0x398 +#define RCC_R109SEMCR 0x39C +#define RCC_R110CIDCFGR 0x3A0 +#define RCC_R110SEMCR 0x3A4 +#define RCC_R111CIDCFGR 0x3A8 +#define RCC_R111SEMCR 0x3AC +#define RCC_R112CIDCFGR 0x3B0 +#define RCC_R112SEMCR 0x3B4 +#define RCC_R113CIDCFGR 0x3B8 +#define RCC_R113SEMCR 0x3BC +#define RCC_GRSTCSETR 0x400 +#define RCC_C1RSTCSETR 0x404 +#define RCC_C1P1RSTCSETR 0x408 +#define RCC_C2RSTCSETR 0x40C +#define RCC_HWRSTSCLRR 0x410 +#define RCC_C1HWRSTSCLRR 0x414 +#define RCC_C2HWRSTSCLRR 0x418 +#define RCC_C1BOOTRSTSSETR 0x41C +#define RCC_C1BOOTRSTSCLRR 0x420 +#define RCC_C2BOOTRSTSSETR 0x424 +#define RCC_C2BOOTRSTSCLRR 0x428 +#define RCC_C1SREQSETR 0x42C +#define RCC_C1SREQCLRR 0x430 +#define RCC_CPUBOOTCR 0x434 +#define RCC_STBYBOOTCR 0x438 +#define RCC_LEGBOOTCR 0x43C +#define RCC_BDCR 0x440 +#define RCC_D3DCR 0x444 +#define RCC_D3DSR 0x448 +#define RCC_RDCR 0x44C +#define RCC_C1MSRDCR 0x450 +#define RCC_PWRLPDLYCR 0x454 +#define RCC_C1CIESETR 0x458 +#define RCC_C1CIFCLRR 0x45C +#define RCC_C2CIESETR 0x460 +#define RCC_C2CIFCLRR 0x464 +#define RCC_IWDGC1FZSETR 0x468 +#define RCC_IWDGC1FZCLRR 0x46C +#define RCC_IWDGC1CFGSETR 0x470 +#define RCC_IWDGC1CFGCLRR 0x474 +#define RCC_IWDGC2FZSETR 0x478 +#define RCC_IWDGC2FZCLRR 0x47C +#define RCC_IWDGC2CFGSETR 0x480 +#define RCC_IWDGC2CFGCLRR 0x484 +#define RCC_IWDGC3CFGSETR 0x488 +#define RCC_IWDGC3CFGCLRR 0x48C +#define RCC_C3CFGR 0x490 +#define RCC_MCO1CFGR 0x494 +#define RCC_MCO2CFGR 0x498 +#define RCC_OCENSETR 0x49C +#define RCC_OCENCLRR 0x4A0 +#define RCC_OCRDYR 0x4A4 +#define RCC_HSICFGR 0x4A8 +#define RCC_MSICFGR 0x4AC +#define RCC_RTCDIVR 0x4B0 +#define RCC_APB1DIVR 0x4B4 +#define RCC_APB2DIVR 0x4B8 +#define RCC_APB3DIVR 0x4BC +#define RCC_APB4DIVR 0x4C0 +#define RCC_APBDBGDIVR 0x4C4 +#define RCC_TIMG1PRER 0x4C8 +#define RCC_TIMG2PRER 0x4CC +#define RCC_LSMCUDIVR 0x4D0 +#define RCC_DDRCPCFGR 0x4D4 +#define RCC_DDRCAPBCFGR 0x4D8 +#define RCC_DDRPHYCAPBCFGR 0x4DC +#define RCC_DDRPHYCCFGR 0x4E0 +#define RCC_DDRCFGR 0x4E4 +#define RCC_DDRITFCFGR 0x4E8 +#define RCC_SYSRAMCFGR 0x4F0 +#define RCC_VDERAMCFGR 0x4F4 +#define RCC_SRAM1CFGR 0x4F8 +#define RCC_SRAM2CFGR 0x4FC +#define RCC_RETRAMCFGR 0x500 +#define RCC_BKPSRAMCFGR 0x504 +#define RCC_LPSRAM1CFGR 0x508 +#define RCC_LPSRAM2CFGR 0x50C +#define RCC_LPSRAM3CFGR 0x510 +#define RCC_OSPI1CFGR 0x514 +#define RCC_OSPI2CFGR 0x518 +#define RCC_FMCCFGR 0x51C +#define RCC_DBGCFGR 0x520 +#define RCC_STM500CFGR 0x524 +#define RCC_ETRCFGR 0x528 +#define RCC_GPIOACFGR 0x52C +#define RCC_GPIOBCFGR 0x530 +#define RCC_GPIOCCFGR 0x534 +#define RCC_GPIODCFGR 0x538 +#define RCC_GPIOECFGR 0x53C +#define RCC_GPIOFCFGR 0x540 +#define RCC_GPIOGCFGR 0x544 +#define RCC_GPIOHCFGR 0x548 +#define RCC_GPIOICFGR 0x54C +#define RCC_GPIOJCFGR 0x550 +#define RCC_GPIOKCFGR 0x554 +#define RCC_GPIOZCFGR 0x558 +#define RCC_HPDMA1CFGR 0x55C +#define RCC_HPDMA2CFGR 0x560 +#define RCC_HPDMA3CFGR 0x564 +#define RCC_LPDMACFGR 0x568 +#define RCC_HSEMCFGR 0x56C +#define RCC_IPCC1CFGR 0x570 +#define RCC_IPCC2CFGR 0x574 +#define RCC_RTCCFGR 0x578 +#define RCC_SYSCPU1CFGR 0x580 +#define RCC_BSECCFGR 0x584 +#define RCC_IS2MCFGR 0x58C +#define RCC_PLL2CFGR1 0x590 +#define RCC_PLL2CFGR2 0x594 +#define RCC_PLL2CFGR3 0x598 +#define RCC_PLL2CFGR4 0x59C +#define RCC_PLL2CFGR5 0x5A0 +#define RCC_PLL2CFGR6 0x5A8 +#define RCC_PLL2CFGR7 0x5AC +#define RCC_PLL3CFGR1 0x5B8 +#define RCC_PLL3CFGR2 0x5BC +#define RCC_PLL3CFGR3 0x5C0 +#define RCC_PLL3CFGR4 0x5C4 +#define RCC_PLL3CFGR5 0x5C8 +#define RCC_PLL3CFGR6 0x5D0 +#define RCC_PLL3CFGR7 0x5D4 +#define RCC_HSIFMONCR 0x5E0 +#define RCC_HSIFVALR 0x5E4 +#define RCC_TIM1CFGR 0x700 +#define RCC_TIM2CFGR 0x704 +#define RCC_TIM3CFGR 0x708 +#define RCC_TIM4CFGR 0x70C +#define RCC_TIM5CFGR 0x710 +#define RCC_TIM6CFGR 0x714 +#define RCC_TIM7CFGR 0x718 +#define RCC_TIM8CFGR 0x71C +#define RCC_TIM10CFGR 0x720 +#define RCC_TIM11CFGR 0x724 +#define RCC_TIM12CFGR 0x728 +#define RCC_TIM13CFGR 0x72C +#define RCC_TIM14CFGR 0x730 +#define RCC_TIM15CFGR 0x734 +#define RCC_TIM16CFGR 0x738 +#define RCC_TIM17CFGR 0x73C +#define RCC_TIM20CFGR 0x740 +#define RCC_LPTIM1CFGR 0x744 +#define RCC_LPTIM2CFGR 0x748 +#define RCC_LPTIM3CFGR 0x74C +#define RCC_LPTIM4CFGR 0x750 +#define RCC_LPTIM5CFGR 0x754 +#define RCC_SPI1CFGR 0x758 +#define RCC_SPI2CFGR 0x75C +#define RCC_SPI3CFGR 0x760 +#define RCC_SPI4CFGR 0x764 +#define RCC_SPI5CFGR 0x768 +#define RCC_SPI6CFGR 0x76C +#define RCC_SPI7CFGR 0x770 +#define RCC_SPI8CFGR 0x774 +#define RCC_SPDIFRXCFGR 0x778 +#define RCC_USART1CFGR 0x77C +#define RCC_USART2CFGR 0x780 +#define RCC_USART3CFGR 0x784 +#define RCC_UART4CFGR 0x788 +#define RCC_UART5CFGR 0x78C +#define RCC_USART6CFGR 0x790 +#define RCC_UART7CFGR 0x794 +#define RCC_UART8CFGR 0x798 +#define RCC_UART9CFGR 0x79C +#define RCC_LPUART1CFGR 0x7A0 +#define RCC_I2C1CFGR 0x7A4 +#define RCC_I2C2CFGR 0x7A8 +#define RCC_I2C3CFGR 0x7AC +#define RCC_I2C4CFGR 0x7B0 +#define RCC_I2C5CFGR 0x7B4 +#define RCC_I2C6CFGR 0x7B8 +#define RCC_I2C7CFGR 0x7BC +#define RCC_I2C8CFGR 0x7C0 +#define RCC_SAI1CFGR 0x7C4 +#define RCC_SAI2CFGR 0x7C8 +#define RCC_SAI3CFGR 0x7CC +#define RCC_SAI4CFGR 0x7D0 +#define RCC_MDF1CFGR 0x7D8 +#define RCC_ADF1CFGR 0x7DC +#define RCC_FDCANCFGR 0x7E0 +#define RCC_HDPCFGR 0x7E4 +#define RCC_ADC12CFGR 0x7E8 +#define RCC_ADC3CFGR 0x7EC +#define RCC_ETH1CFGR 0x7F0 +#define RCC_ETH2CFGR 0x7F4 +#define RCC_USBHCFGR 0x7FC +#define RCC_USB2PHY1CFGR 0x800 +#define RCC_USB2PHY2CFGR 0x804 +#define RCC_USB3DRCFGR 0x808 +#define RCC_USB3PCIEPHYCFGR 0x80C +#define RCC_PCIECFGR 0x810 +#define RCC_USBTCCFGR 0x814 +#define RCC_ETHSWCFGR 0x818 +#define RCC_ETHSWACMCFGR 0x81C +#define RCC_ETHSWACMMSGCFGR 0x820 +#define RCC_STGENCFGR 0x824 +#define RCC_SDMMC1CFGR 0x830 +#define RCC_SDMMC2CFGR 0x834 +#define RCC_SDMMC3CFGR 0x838 +#define RCC_GPUCFGR 0x83C +#define RCC_LTDCCFGR 0x840 +#define RCC_DSICFGR 0x844 +#define RCC_LVDSCFGR 0x850 +#define RCC_CSICFGR 0x858 +#define RCC_DCMIPPCFGR 0x85C +#define RCC_CCICFGR 0x860 +#define RCC_VDECCFGR 0x864 +#define RCC_VENCCFGR 0x868 +#define RCC_RNGCFGR 0x870 +#define RCC_PKACFGR 0x874 +#define RCC_SAESCFGR 0x878 +#define RCC_HASHCFGR 0x87C +#define RCC_CRYP1CFGR 0x880 +#define RCC_CRYP2CFGR 0x884 +#define RCC_IWDG1CFGR 0x888 +#define RCC_IWDG2CFGR 0x88C +#define RCC_IWDG3CFGR 0x890 +#define RCC_IWDG4CFGR 0x894 +#define RCC_IWDG5CFGR 0x898 +#define RCC_WWDG1CFGR 0x89C +#define RCC_WWDG2CFGR 0x8A0 +#define RCC_VREFCFGR 0x8A8 +#define RCC_DTSCFGR 0x8AC +#define RCC_CRCCFGR 0x8B4 +#define RCC_SERCCFGR 0x8B8 +#define RCC_OSPIIOMCFGR 0x8BC +#define RCC_GICV2MCFGR 0x8C0 +#define RCC_I3C1CFGR 0x8C8 +#define RCC_I3C2CFGR 0x8CC +#define RCC_I3C3CFGR 0x8D0 +#define RCC_I3C4CFGR 0x8D4 +#define RCC_MUXSELCFGR 0x1000 +#define RCC_XBAR0CFGR 0x1018 +#define RCC_XBAR1CFGR 0x101C +#define RCC_XBAR2CFGR 0x1020 +#define RCC_XBAR3CFGR 0x1024 +#define RCC_XBAR4CFGR 0x1028 +#define RCC_XBAR5CFGR 0x102C +#define RCC_XBAR6CFGR 0x1030 +#define RCC_XBAR7CFGR 0x1034 +#define RCC_XBAR8CFGR 0x1038 +#define RCC_XBAR9CFGR 0x103C +#define RCC_XBAR10CFGR 0x1040 +#define RCC_XBAR11CFGR 0x1044 +#define RCC_XBAR12CFGR 0x1048 +#define RCC_XBAR13CFGR 0x104C +#define RCC_XBAR14CFGR 0x1050 +#define RCC_XBAR15CFGR 0x1054 +#define RCC_XBAR16CFGR 0x1058 +#define RCC_XBAR17CFGR 0x105C +#define RCC_XBAR18CFGR 0x1060 +#define RCC_XBAR19CFGR 0x1064 +#define RCC_XBAR20CFGR 0x1068 +#define RCC_XBAR21CFGR 0x106C +#define RCC_XBAR22CFGR 0x1070 +#define RCC_XBAR23CFGR 0x1074 +#define RCC_XBAR24CFGR 0x1078 +#define RCC_XBAR25CFGR 0x107C +#define RCC_XBAR26CFGR 0x1080 +#define RCC_XBAR27CFGR 0x1084 +#define RCC_XBAR28CFGR 0x1088 +#define RCC_XBAR29CFGR 0x108C +#define RCC_XBAR30CFGR 0x1090 +#define RCC_XBAR31CFGR 0x1094 +#define RCC_XBAR32CFGR 0x1098 +#define RCC_XBAR33CFGR 0x109C +#define RCC_XBAR34CFGR 0x10A0 +#define RCC_XBAR35CFGR 0x10A4 +#define RCC_XBAR36CFGR 0x10A8 +#define RCC_XBAR37CFGR 0x10AC +#define RCC_XBAR38CFGR 0x10B0 +#define RCC_XBAR39CFGR 0x10B4 +#define RCC_XBAR40CFGR 0x10B8 +#define RCC_XBAR41CFGR 0x10BC +#define RCC_XBAR42CFGR 0x10C0 +#define RCC_XBAR43CFGR 0x10C4 +#define RCC_XBAR44CFGR 0x10C8 +#define RCC_XBAR45CFGR 0x10CC +#define RCC_XBAR46CFGR 0x10D0 +#define RCC_XBAR47CFGR 0x10D4 +#define RCC_XBAR48CFGR 0x10D8 +#define RCC_XBAR49CFGR 0x10DC +#define RCC_XBAR50CFGR 0x10E0 +#define RCC_XBAR51CFGR 0x10E4 +#define RCC_XBAR52CFGR 0x10E8 +#define RCC_XBAR53CFGR 0x10EC +#define RCC_XBAR54CFGR 0x10F0 +#define RCC_XBAR55CFGR 0x10F4 +#define RCC_XBAR56CFGR 0x10F8 +#define RCC_XBAR57CFGR 0x10FC +#define RCC_XBAR58CFGR 0x1100 +#define RCC_XBAR59CFGR 0x1104 +#define RCC_XBAR60CFGR 0x1108 +#define RCC_XBAR61CFGR 0x110C +#define RCC_XBAR62CFGR 0x1110 +#define RCC_XBAR63CFGR 0x1114 +#define RCC_PREDIV0CFGR 0x1118 +#define RCC_PREDIV1CFGR 0x111C +#define RCC_PREDIV2CFGR 0x1120 +#define RCC_PREDIV3CFGR 0x1124 +#define RCC_PREDIV4CFGR 0x1128 +#define RCC_PREDIV5CFGR 0x112C +#define RCC_PREDIV6CFGR 0x1130 +#define RCC_PREDIV7CFGR 0x1134 +#define RCC_PREDIV8CFGR 0x1138 +#define RCC_PREDIV9CFGR 0x113C +#define RCC_PREDIV10CFGR 0x1140 +#define RCC_PREDIV11CFGR 0x1144 +#define RCC_PREDIV12CFGR 0x1148 +#define RCC_PREDIV13CFGR 0x114C +#define RCC_PREDIV14CFGR 0x1150 +#define RCC_PREDIV15CFGR 0x1154 +#define RCC_PREDIV16CFGR 0x1158 +#define RCC_PREDIV17CFGR 0x115C +#define RCC_PREDIV18CFGR 0x1160 +#define RCC_PREDIV19CFGR 0x1164 +#define RCC_PREDIV20CFGR 0x1168 +#define RCC_PREDIV21CFGR 0x116C +#define RCC_PREDIV22CFGR 0x1170 +#define RCC_PREDIV23CFGR 0x1174 +#define RCC_PREDIV24CFGR 0x1178 +#define RCC_PREDIV25CFGR 0x117C +#define RCC_PREDIV26CFGR 0x1180 +#define RCC_PREDIV27CFGR 0x1184 +#define RCC_PREDIV28CFGR 0x1188 +#define RCC_PREDIV29CFGR 0x118C +#define RCC_PREDIV30CFGR 0x1190 +#define RCC_PREDIV31CFGR 0x1194 +#define RCC_PREDIV32CFGR 0x1198 +#define RCC_PREDIV33CFGR 0x119C +#define RCC_PREDIV34CFGR 0x11A0 +#define RCC_PREDIV35CFGR 0x11A4 +#define RCC_PREDIV36CFGR 0x11A8 +#define RCC_PREDIV37CFGR 0x11AC +#define RCC_PREDIV38CFGR 0x11B0 +#define RCC_PREDIV39CFGR 0x11B4 +#define RCC_PREDIV40CFGR 0x11B8 +#define RCC_PREDIV41CFGR 0x11BC +#define RCC_PREDIV42CFGR 0x11C0 +#define RCC_PREDIV43CFGR 0x11C4 +#define RCC_PREDIV44CFGR 0x11C8 +#define RCC_PREDIV45CFGR 0x11CC +#define RCC_PREDIV46CFGR 0x11D0 +#define RCC_PREDIV47CFGR 0x11D4 +#define RCC_PREDIV48CFGR 0x11D8 +#define RCC_PREDIV49CFGR 0x11DC +#define RCC_PREDIV50CFGR 0x11E0 +#define RCC_PREDIV51CFGR 0x11E4 +#define RCC_PREDIV52CFGR 0x11E8 +#define RCC_PREDIV53CFGR 0x11EC +#define RCC_PREDIV54CFGR 0x11F0 +#define RCC_PREDIV55CFGR 0x11F4 +#define RCC_PREDIV56CFGR 0x11F8 +#define RCC_PREDIV57CFGR 0x11FC +#define RCC_PREDIV58CFGR 0x1200 +#define RCC_PREDIV59CFGR 0x1204 +#define RCC_PREDIV60CFGR 0x1208 +#define RCC_PREDIV61CFGR 0x120C +#define RCC_PREDIV62CFGR 0x1210 +#define RCC_PREDIV63CFGR 0x1214 +#define RCC_PREDIVSR1 0x1218 +#define RCC_PREDIVSR2 0x121C +#define RCC_FINDIV0CFGR 0x1224 +#define RCC_FINDIV1CFGR 0x1228 +#define RCC_FINDIV2CFGR 0x122C +#define RCC_FINDIV3CFGR 0x1230 +#define RCC_FINDIV4CFGR 0x1234 +#define RCC_FINDIV5CFGR 0x1238 +#define RCC_FINDIV6CFGR 0x123C +#define RCC_FINDIV7CFGR 0x1240 +#define RCC_FINDIV8CFGR 0x1244 +#define RCC_FINDIV9CFGR 0x1248 +#define RCC_FINDIV10CFGR 0x124C +#define RCC_FINDIV11CFGR 0x1250 +#define RCC_FINDIV12CFGR 0x1254 +#define RCC_FINDIV13CFGR 0x1258 +#define RCC_FINDIV14CFGR 0x125C +#define RCC_FINDIV15CFGR 0x1260 +#define RCC_FINDIV16CFGR 0x1264 +#define RCC_FINDIV17CFGR 0x1268 +#define RCC_FINDIV18CFGR 0x126C +#define RCC_FINDIV19CFGR 0x1270 +#define RCC_FINDIV20CFGR 0x1274 +#define RCC_FINDIV21CFGR 0x1278 +#define RCC_FINDIV22CFGR 0x127C +#define RCC_FINDIV23CFGR 0x1280 +#define RCC_FINDIV24CFGR 0x1284 +#define RCC_FINDIV25CFGR 0x1288 +#define RCC_FINDIV26CFGR 0x128C +#define RCC_FINDIV27CFGR 0x1290 +#define RCC_FINDIV28CFGR 0x1294 +#define RCC_FINDIV29CFGR 0x1298 +#define RCC_FINDIV30CFGR 0x129C +#define RCC_FINDIV31CFGR 0x12A0 +#define RCC_FINDIV32CFGR 0x12A4 +#define RCC_FINDIV33CFGR 0x12A8 +#define RCC_FINDIV34CFGR 0x12AC +#define RCC_FINDIV35CFGR 0x12B0 +#define RCC_FINDIV36CFGR 0x12B4 +#define RCC_FINDIV37CFGR 0x12B8 +#define RCC_FINDIV38CFGR 0x12BC +#define RCC_FINDIV39CFGR 0x12C0 +#define RCC_FINDIV40CFGR 0x12C4 +#define RCC_FINDIV41CFGR 0x12C8 +#define RCC_FINDIV42CFGR 0x12CC +#define RCC_FINDIV43CFGR 0x12D0 +#define RCC_FINDIV44CFGR 0x12D4 +#define RCC_FINDIV45CFGR 0x12D8 +#define RCC_FINDIV46CFGR 0x12DC +#define RCC_FINDIV47CFGR 0x12E0 +#define RCC_FINDIV48CFGR 0x12E4 +#define RCC_FINDIV49CFGR 0x12E8 +#define RCC_FINDIV50CFGR 0x12EC +#define RCC_FINDIV51CFGR 0x12F0 +#define RCC_FINDIV52CFGR 0x12F4 +#define RCC_FINDIV53CFGR 0x12F8 +#define RCC_FINDIV54CFGR 0x12FC +#define RCC_FINDIV55CFGR 0x1300 +#define RCC_FINDIV56CFGR 0x1304 +#define RCC_FINDIV57CFGR 0x1308 +#define RCC_FINDIV58CFGR 0x130C +#define RCC_FINDIV59CFGR 0x1310 +#define RCC_FINDIV60CFGR 0x1314 +#define RCC_FINDIV61CFGR 0x1318 +#define RCC_FINDIV62CFGR 0x131C +#define RCC_FINDIV63CFGR 0x1320 +#define RCC_FINDIVSR1 0x1324 +#define RCC_FINDIVSR2 0x1328 +#define RCC_FCALCOBS0CFGR 0x1340 +#define RCC_FCALCOBS1CFGR 0x1344 +#define RCC_FCALCREFCFGR 0x1348 +#define RCC_FCALCCR1 0x134C +#define RCC_FCALCCR2 0x1354 +#define RCC_FCALCSR 0x1358 +#define RCC_PLL4CFGR1 0x1360 +#define RCC_PLL4CFGR2 0x1364 +#define RCC_PLL4CFGR3 0x1368 +#define RCC_PLL4CFGR4 0x136C +#define RCC_PLL4CFGR5 0x1370 +#define RCC_PLL4CFGR6 0x1378 +#define RCC_PLL4CFGR7 0x137C +#define RCC_PLL5CFGR1 0x1388 +#define RCC_PLL5CFGR2 0x138C +#define RCC_PLL5CFGR3 0x1390 +#define RCC_PLL5CFGR4 0x1394 +#define RCC_PLL5CFGR5 0x1398 +#define RCC_PLL5CFGR6 0x13A0 +#define RCC_PLL5CFGR7 0x13A4 +#define RCC_PLL6CFGR1 0x13B0 +#define RCC_PLL6CFGR2 0x13B4 +#define RCC_PLL6CFGR3 0x13B8 +#define RCC_PLL6CFGR4 0x13BC +#define RCC_PLL6CFGR5 0x13C0 +#define RCC_PLL6CFGR6 0x13C8 +#define RCC_PLL6CFGR7 0x13CC +#define RCC_PLL7CFGR1 0x13D8 +#define RCC_PLL7CFGR2 0x13DC +#define RCC_PLL7CFGR3 0x13E0 +#define RCC_PLL7CFGR4 0x13E4 +#define RCC_PLL7CFGR5 0x13E8 +#define RCC_PLL7CFGR6 0x13F0 +#define RCC_PLL7CFGR7 0x13F4 +#define RCC_PLL8CFGR1 0x1400 +#define RCC_PLL8CFGR2 0x1404 +#define RCC_PLL8CFGR3 0x1408 +#define RCC_PLL8CFGR4 0x140C +#define RCC_PLL8CFGR5 0x1410 +#define RCC_PLL8CFGR6 0x1418 +#define RCC_PLL8CFGR7 0x141C +#define RCC_VERR 0xFFF4 +#define RCC_IDR 0xFFF8 +#define RCC_SIDR 0xFFFC + +#endif /* STM32MP25_RCC_H */ From 2aacaed459b5546910ac38fe4570ffa18e4efd98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Mon, 8 Apr 2024 12:51:28 +0200 Subject: [PATCH 0131/1702] dt-bindings: pinctrl: mediatek: mt7622: add "gpio-ranges" property MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow specifying pin to GPIO mapping. It can be find in in-Linux DTS file for MT7622. Signed-off-by: Rafał Miłecki Acked-by: Krzysztof Kozlowski Message-ID: <20240408105128.30586-1-zajec5@gmail.com> Signed-off-by: Linus Walleij --- .../devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml index bd72a326e6e06..4637f3a40e83e 100644 --- a/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml @@ -34,6 +34,9 @@ properties: the amount of cells must be specified as 2. See the below mentioned gpio binding representation for description of particular cells. + gpio-ranges: + maxItems: 1 + interrupt-controller: true interrupts: From 50dca75e7d34f7ba2c5a57eb767a33f14c42000d Mon Sep 17 00:00:00 2001 From: Luca Weiss Date: Tue, 9 Apr 2024 20:36:36 +0200 Subject: [PATCH 0132/1702] dt-bindings: pinctrl: qcom,pmic-gpio: Allow gpio-hog nodes Allow specifying a GPIO hog, as already used on qcom-msm8974-lge-nexus5-hammerhead.dts. Signed-off-by: Luca Weiss Reviewed-by: Krzysztof Kozlowski Message-ID: <20240409-qcom-pmic-gpio-hog-v2-1-5ff812d2baed@z3ntu.xyz> Signed-off-by: Linus Walleij --- .../devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml index a786357ed1afd..bd9471de0c69f 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml @@ -424,6 +424,10 @@ patternProperties: $ref: "#/$defs/qcom-pmic-gpio-state" additionalProperties: false + "-hog(-[0-9]+)?$": + required: + - gpio-hog + $defs: qcom-pmic-gpio-state: type: object @@ -571,6 +575,7 @@ $defs: examples: - | + #include #include pm8921_gpio: gpio@150 { @@ -594,5 +599,12 @@ examples: power-source = ; }; }; + + otg-hog { + gpio-hog; + gpios = <35 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "otg-gpio"; + }; }; ... From b025dea63cded0d82bccd591fa105d39efc6435d Mon Sep 17 00:00:00 2001 From: Yi Liu Date: Thu, 28 Mar 2024 05:29:57 -0700 Subject: [PATCH 0133/1702] iommu: Undo pasid attachment only for the devices that have succeeded There is no error handling now in __iommu_set_group_pasid(), it relies on its caller to loop all the devices to undo the pasid attachment. This is not self-contained and has drawbacks. It would result in unnecessary remove_dev_pasid() calls on the devices that have not been attached to the new domain. But the remove_dev_pasid() callback would get the new domain from the group->pasid_array. So for such devices, the iommu driver won't find the attachment under the domain, hence unable to do cleanup. This may not be a real problem today. But it depends on the implementation of the underlying iommu driver. e.g. the intel iommu driver would warn for such devices. Such warnings are unnecessary. To solve the above problem, it is necessary to handle the error within __iommu_set_group_pasid(). It only loops the devices that have attached to the new domain, and undo it. Fixes: 16603704559c ("iommu: Add attach/detach_dev_pasid iommu interfaces") Suggested-by: Jason Gunthorpe Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Signed-off-by: Yi Liu Reviewed-by: Lu Baolu Link: https://lore.kernel.org/r/20240328122958.83332-2-yi.l.liu@intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index a95a483def2d2..659a77f7bb833 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -3317,15 +3317,26 @@ EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed); static int __iommu_set_group_pasid(struct iommu_domain *domain, struct iommu_group *group, ioasid_t pasid) { - struct group_device *device; - int ret = 0; + struct group_device *device, *last_gdev; + int ret; for_each_group_device(group, device) { ret = domain->ops->set_dev_pasid(domain, device->dev, pasid); if (ret) - break; + goto err_revert; } + return 0; + +err_revert: + last_gdev = device; + for_each_group_device(group, device) { + const struct iommu_ops *ops = dev_iommu_ops(device->dev); + + if (device == last_gdev) + break; + ops->remove_dev_pasid(device->dev, pasid); + } return ret; } @@ -3383,10 +3394,8 @@ int iommu_attach_device_pasid(struct iommu_domain *domain, } ret = __iommu_set_group_pasid(domain, group, pasid); - if (ret) { - __iommu_remove_group_pasid(group, pasid); + if (ret) xa_erase(&group->pasid_array, pasid); - } out_unlock: mutex_unlock(&group->mutex); return ret; From d2f85a263883b679f87ed8f911746105658e9c47 Mon Sep 17 00:00:00 2001 From: Yi Liu Date: Thu, 28 Mar 2024 05:29:58 -0700 Subject: [PATCH 0134/1702] iommu: Pass domain to remove_dev_pasid() op Existing remove_dev_pasid() callbacks of the underlying iommu drivers get the attached domain from the group->pasid_array. However, the domain stored in group->pasid_array is not always correct in all scenarios. A wrong domain may result in failure in remove_dev_pasid() callback. To avoid such problems, it is more reliable to pass the domain to the remove_dev_pasid() op. Suggested-by: Jason Gunthorpe Signed-off-by: Yi Liu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240328122958.83332-3-yi.l.liu@intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 9 ++------- drivers/iommu/intel/iommu.c | 11 +++-------- drivers/iommu/iommu.c | 9 +++++---- include/linux/iommu.h | 3 ++- 4 files changed, 12 insertions(+), 20 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 41f93c3ab160d..2e1988c861f15 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3053,14 +3053,9 @@ static int arm_smmu_def_domain_type(struct device *dev) return 0; } -static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid) +static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid, + struct iommu_domain *domain) { - struct iommu_domain *domain; - - domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA); - if (WARN_ON(IS_ERR(domain)) || !domain) - return; - arm_smmu_sva_remove_dev_pasid(domain, dev, pasid); } diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 50eb9aed47cc5..45c75a8a0ef56 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4587,19 +4587,15 @@ static int intel_iommu_iotlb_sync_map(struct iommu_domain *domain, return 0; } -static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid) +static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid, + struct iommu_domain *domain) { struct device_domain_info *info = dev_iommu_priv_get(dev); + struct dmar_domain *dmar_domain = to_dmar_domain(domain); struct dev_pasid_info *curr, *dev_pasid = NULL; struct intel_iommu *iommu = info->iommu; - struct dmar_domain *dmar_domain; - struct iommu_domain *domain; unsigned long flags; - domain = iommu_get_domain_for_dev_pasid(dev, pasid, 0); - if (WARN_ON_ONCE(!domain)) - goto out_tear_down; - /* * The SVA implementation needs to handle its own stuffs like the mm * notification. Before consolidating that code into iommu core, let @@ -4610,7 +4606,6 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid) goto out_tear_down; } - dmar_domain = to_dmar_domain(domain); spin_lock_irqsave(&dmar_domain->lock, flags); list_for_each_entry(curr, &dmar_domain->dev_pasids, link_domain) { if (curr->dev == dev && curr->pasid == pasid) { diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 659a77f7bb833..3183b0ed4cdb9 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -3335,20 +3335,21 @@ static int __iommu_set_group_pasid(struct iommu_domain *domain, if (device == last_gdev) break; - ops->remove_dev_pasid(device->dev, pasid); + ops->remove_dev_pasid(device->dev, pasid, domain); } return ret; } static void __iommu_remove_group_pasid(struct iommu_group *group, - ioasid_t pasid) + ioasid_t pasid, + struct iommu_domain *domain) { struct group_device *device; const struct iommu_ops *ops; for_each_group_device(group, device) { ops = dev_iommu_ops(device->dev); - ops->remove_dev_pasid(device->dev, pasid); + ops->remove_dev_pasid(device->dev, pasid, domain); } } @@ -3418,7 +3419,7 @@ void iommu_detach_device_pasid(struct iommu_domain *domain, struct device *dev, struct iommu_group *group = dev->iommu_group; mutex_lock(&group->mutex); - __iommu_remove_group_pasid(group, pasid); + __iommu_remove_group_pasid(group, pasid, domain); WARN_ON(xa_erase(&group->pasid_array, pasid) != domain); mutex_unlock(&group->mutex); } diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 2e925b5eba534..40dd439307e83 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -578,7 +578,8 @@ struct iommu_ops { struct iommu_page_response *msg); int (*def_domain_type)(struct device *dev); - void (*remove_dev_pasid)(struct device *dev, ioasid_t pasid); + void (*remove_dev_pasid)(struct device *dev, ioasid_t pasid, + struct iommu_domain *domain); const struct iommu_domain_ops *default_domain_ops; unsigned long pgsize_bitmap; From 4f8cf60ac18bee62e8c58654a300eb44b96caf09 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 5 Mar 2024 18:52:05 +0000 Subject: [PATCH 0135/1702] reiserfs: Convert to writepages Use buffer_migrate_folio to handle folio migration instead of writing out dirty pages and reading them back in again. Use writepages to write out folios more efficiently. We now only do that wait_on_write_block check once per call to writepages instead of once per page. It would be possible to do one transaction per writeback run, but that's a bit of a big change to do to this old filesystem, so leave it as one transaction per folio (and leave reiserfs supporting only one page per folio). Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Jan Kara Signed-off-by: Jan Kara Message-Id: <20240305185208.1200166-1-willy@infradead.org> --- fs/reiserfs/inode.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fs/reiserfs/inode.c b/fs/reiserfs/inode.c index 1d825459ee6e6..c1daedc50f4c2 100644 --- a/fs/reiserfs/inode.c +++ b/fs/reiserfs/inode.c @@ -2503,8 +2503,8 @@ static int map_block_for_writepage(struct inode *inode, * start/recovery path as __block_write_full_folio, along with special * code to handle reiserfs tails. */ -static int reiserfs_write_full_folio(struct folio *folio, - struct writeback_control *wbc) +static int reiserfs_write_folio(struct folio *folio, + struct writeback_control *wbc, void *data) { struct inode *inode = folio->mapping->host; unsigned long end_index = inode->i_size >> PAGE_SHIFT; @@ -2721,12 +2721,11 @@ static int reiserfs_read_folio(struct file *f, struct folio *folio) return block_read_full_folio(folio, reiserfs_get_block); } -static int reiserfs_writepage(struct page *page, struct writeback_control *wbc) +static int reiserfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) { - struct folio *folio = page_folio(page); - struct inode *inode = folio->mapping->host; - reiserfs_wait_on_write_block(inode->i_sb); - return reiserfs_write_full_folio(folio, wbc); + reiserfs_wait_on_write_block(mapping->host->i_sb); + return write_cache_pages(mapping, wbc, reiserfs_write_folio, NULL); } static void reiserfs_truncate_failed_write(struct inode *inode) @@ -3405,7 +3404,7 @@ int reiserfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, } const struct address_space_operations reiserfs_address_space_operations = { - .writepage = reiserfs_writepage, + .writepages = reiserfs_writepages, .read_folio = reiserfs_read_folio, .readahead = reiserfs_readahead, .release_folio = reiserfs_release_folio, @@ -3415,4 +3414,5 @@ const struct address_space_operations reiserfs_address_space_operations = { .bmap = reiserfs_aop_bmap, .direct_IO = reiserfs_direct_IO, .dirty_folio = reiserfs_dirty_folio, + .migrate_folio = buffer_migrate_folio, }; From 8e2e0a79a3349b6853f572e4be4885b482f7d837 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Fri, 12 Apr 2024 17:49:42 +0800 Subject: [PATCH 0136/1702] quota: fix to propagate error of mark_dquot_dirty() to caller in order to let caller be aware of failure of mark_dquot_dirty(). Signed-off-by: Chao Yu Signed-off-by: Jan Kara Message-Id: <20240412094942.2131243-1-chao@kernel.org> --- fs/quota/dquot.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c index 808544f74e5e2..627eb2f72ef37 100644 --- a/fs/quota/dquot.c +++ b/fs/quota/dquot.c @@ -410,7 +410,7 @@ static inline int mark_all_dquot_dirty(struct dquot __rcu * const *dquots) if (dquot) /* Even in case of error we have to continue */ ret = mark_dquot_dirty(dquot); - if (!err) + if (!err && ret < 0) err = ret; } return err; @@ -1737,7 +1737,7 @@ int __dquot_alloc_space(struct inode *inode, qsize_t number, int flags) if (reserve) goto out_flush_warn; - mark_all_dquot_dirty(dquots); + ret = mark_all_dquot_dirty(dquots); out_flush_warn: srcu_read_unlock(&dquot_srcu, index); flush_warnings(warn); @@ -1786,7 +1786,7 @@ int dquot_alloc_inode(struct inode *inode) warn_put_all: spin_unlock(&inode->i_lock); if (ret == 0) - mark_all_dquot_dirty(dquots); + ret = mark_all_dquot_dirty(dquots); srcu_read_unlock(&dquot_srcu, index); flush_warnings(warn); return ret; @@ -1990,7 +1990,7 @@ int __dquot_transfer(struct inode *inode, struct dquot **transfer_to) qsize_t inode_usage = 1; struct dquot __rcu **dquots; struct dquot *transfer_from[MAXQUOTAS] = {}; - int cnt, index, ret = 0; + int cnt, index, ret = 0, err; char is_valid[MAXQUOTAS] = {}; struct dquot_warn warn_to[MAXQUOTAS]; struct dquot_warn warn_from_inodes[MAXQUOTAS]; @@ -2087,8 +2087,12 @@ int __dquot_transfer(struct inode *inode, struct dquot **transfer_to) * mark_all_dquot_dirty(). */ index = srcu_read_lock(&dquot_srcu); - mark_all_dquot_dirty((struct dquot __rcu **)transfer_from); - mark_all_dquot_dirty((struct dquot __rcu **)transfer_to); + err = mark_all_dquot_dirty((struct dquot __rcu **)transfer_from); + if (err < 0) + ret = err; + err = mark_all_dquot_dirty((struct dquot __rcu **)transfer_to); + if (err < 0) + ret = err; srcu_read_unlock(&dquot_srcu, index); flush_warnings(warn_to); @@ -2098,7 +2102,7 @@ int __dquot_transfer(struct inode *inode, struct dquot **transfer_to) for (cnt = 0; cnt < MAXQUOTAS; cnt++) if (is_valid[cnt]) transfer_to[cnt] = transfer_from[cnt]; - return 0; + return ret; over_quota: /* Back out changes we already did */ for (cnt--; cnt >= 0; cnt--) { @@ -2726,6 +2730,7 @@ static int do_set_dqblk(struct dquot *dquot, struct qc_dqblk *di) struct mem_dqblk *dm = &dquot->dq_dqb; int check_blim = 0, check_ilim = 0; struct mem_dqinfo *dqi = &sb_dqopt(dquot->dq_sb)->info[dquot->dq_id.type]; + int ret; if (di->d_fieldmask & ~VFS_QC_MASK) return -EINVAL; @@ -2807,8 +2812,9 @@ static int do_set_dqblk(struct dquot *dquot, struct qc_dqblk *di) else set_bit(DQ_FAKE_B, &dquot->dq_flags); spin_unlock(&dquot->dq_dqb_lock); - mark_dquot_dirty(dquot); - + ret = mark_dquot_dirty(dquot); + if (ret < 0) + return ret; return 0; } From 9ec2b350160104175241229ac711ffdde8c10078 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Fri, 12 Apr 2024 14:32:12 +0200 Subject: [PATCH 0137/1702] reiserfs: Trim some README bits Remove some bits of README based on authors request. References: https://lore.kernel.org/lkml/b98b29cf-27d9-49e0-b10b-1848399badfd@kittens.ph/ Signed-off-by: Jan Kara --- fs/reiserfs/README | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/fs/reiserfs/README b/fs/reiserfs/README index e2f7a264e3ff0..11e9ecf24b63d 100644 --- a/fs/reiserfs/README +++ b/fs/reiserfs/README @@ -102,19 +102,9 @@ that start on a node aligned boundary (there are reasons to want to node align files), and he invented and implemented indirect items and unformatted nodes as the solution. -Konstantin Shvachko, with the help of the Russian version of a VC, -tried to put me in a position where I was forced into giving control -of the project to him. (Fortunately, as the person paying the money -for all salaries from my dayjob I owned all copyrights, and you can't -really force takeovers of sole proprietorships.) This was something -curious, because he never really understood the value of our project, -why we should do what we do, or why innovation was possible in -general, but he was sure that he ought to be controlling it. Every -innovation had to be forced past him while he was with us. He added -two years to the time required to complete reiserfs, and was a net -loss for me. Mikhail Gilula was a brilliant innovator who also left -in a destructive way that erased the value of his contributions, and -that he was shown much generosity just makes it more painful. +Konstantin Shvachko was taking part in the early days. + +Mikhail Gilula was a brilliant innovator that has shown much generosity. Grigory Zaigralin was an extremely effective system administrator for our group. From efee03a50c2844d78f6fb5e98be1ffd17605dc5b Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Sat, 13 Apr 2024 01:02:08 +0900 Subject: [PATCH 0138/1702] bootconfig: do not put quotes on cmdline items unless necessary When trying to migrate to using bootconfig to embed the kernel's and PID1's command line with the kernel image itself, and so allowing changing that without modifying the bootloader, I noticed that /proc/cmdline changed from e.g. console=ttymxc0,115200n8 cma=128M quiet -- --log-level=notice to console="ttymxc0,115200n8" cma="128M" quiet -- --log-level="notice" The kernel parameters are parsed just fine, and the quotes are indeed stripped from the actual argv[] given to PID1. However, the quoting doesn't really serve any purpose and looks excessive, and might confuse some (naive) userspace tool trying to parse /proc/cmdline. So do not quote the value unless it contains whitespace. Link: https://lore.kernel.org/all/20240308124401.1702046-1-linux@rasmusvillemoes.dk/ Signed-off-by: Rasmus Villemoes Acked-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) --- init/main.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/init/main.c b/init/main.c index 5dcf5274c09c7..96e59e5761ff8 100644 --- a/init/main.c +++ b/init/main.c @@ -327,7 +327,7 @@ static int __init xbc_snprint_cmdline(char *buf, size_t size, { struct xbc_node *knode, *vnode; char *end = buf + size; - const char *val; + const char *val, *q; int ret; xbc_node_for_each_key_value(root, knode, val) { @@ -345,8 +345,9 @@ static int __init xbc_snprint_cmdline(char *buf, size_t size, continue; } xbc_array_for_each_value(vnode, val) { - ret = snprintf(buf, rest(buf, end), "%s=\"%s\" ", - xbc_namebuf, val); + q = strpbrk(val, " \t\r\n") ? "\"" : ""; + ret = snprintf(buf, rest(buf, end), "%s=%s%s%s ", + xbc_namebuf, q, val, q); if (ret < 0) return ret; buf += ret; From cd24bdb06820d12dc84aeb0a47a10ad7ecf4b027 Mon Sep 17 00:00:00 2001 From: Yuntao Wang Date: Sat, 13 Apr 2024 01:02:08 +0900 Subject: [PATCH 0139/1702] init/main.c: Remove redundant space from saved_command_line There is a space at the end of extra_init_args. In the current logic, copying extra_init_args to saved_command_line will cause extra spaces in saved_command_line here or there. Remove the trailing space from extra_init_args to make the string in saved_command_line look more perfect. Link: https://lore.kernel.org/all/20240412032950.12687-1-ytcoode@gmail.com/ Signed-off-by: Yuntao Wang Acked-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) --- init/main.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/init/main.c b/init/main.c index 96e59e5761ff8..02156c3c58798 100644 --- a/init/main.c +++ b/init/main.c @@ -628,8 +628,10 @@ static void __init setup_command_line(char *command_line) if (extra_command_line) xlen = strlen(extra_command_line); - if (extra_init_args) + if (extra_init_args) { + extra_init_args = strim(extra_init_args); /* remove trailing space */ ilen = strlen(extra_init_args) + 4; /* for " -- " */ + } len = xlen + strlen(boot_command_line) + 1; From ddd53363f875eb23b6754362ce0a43f2214f0a83 Mon Sep 17 00:00:00 2001 From: Yuntao Wang Date: Fri, 12 Apr 2024 16:17:33 +0800 Subject: [PATCH 0140/1702] init/main.c: Minor cleanup for the setup_command_line() function This is just a minor cleanup to make the code look a bit cleaner. Link: https://lore.kernel.org/all/20240412081733.35925-3-ytcoode@gmail.com/ Signed-off-by: Yuntao Wang Signed-off-by: Masami Hiramatsu (Google) --- init/main.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/init/main.c b/init/main.c index 02156c3c58798..383796015113e 100644 --- a/init/main.c +++ b/init/main.c @@ -633,11 +633,11 @@ static void __init setup_command_line(char *command_line) ilen = strlen(extra_init_args) + 4; /* for " -- " */ } - len = xlen + strlen(boot_command_line) + 1; + len = xlen + strlen(boot_command_line) + ilen + 1; - saved_command_line = memblock_alloc(len + ilen, SMP_CACHE_BYTES); + saved_command_line = memblock_alloc(len, SMP_CACHE_BYTES); if (!saved_command_line) - panic("%s: Failed to allocate %zu bytes\n", __func__, len + ilen); + panic("%s: Failed to allocate %zu bytes\n", __func__, len); len = xlen + strlen(command_line) + 1; From 15f3df361720de343d125ca01b7b456b8d16327f Mon Sep 17 00:00:00 2001 From: Wadim Mueller Date: Sun, 24 Mar 2024 22:43:24 +0100 Subject: [PATCH 0141/1702] dt-bindings: serial: fsl-linflexuart: add compatible for S32G3 Add a compatible string for the uart binding of NXP S32G3 platforms. Here we use "s32v234-linflexuart" as fallback since the current linflexuart driver can still work on S32G3. Signed-off-by: Wadim Mueller Acked-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240324214329.29988-3-wafgo01@gmail.com Signed-off-by: Rob Herring --- .../devicetree/bindings/serial/fsl,s32-linflexuart.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/serial/fsl,s32-linflexuart.yaml b/Documentation/devicetree/bindings/serial/fsl,s32-linflexuart.yaml index 7a105551fa6a8..4171f524a928c 100644 --- a/Documentation/devicetree/bindings/serial/fsl,s32-linflexuart.yaml +++ b/Documentation/devicetree/bindings/serial/fsl,s32-linflexuart.yaml @@ -23,7 +23,9 @@ properties: oneOf: - const: fsl,s32v234-linflexuart - items: - - const: nxp,s32g2-linflexuart + - enum: + - nxp,s32g2-linflexuart + - nxp,s32g3-linflexuart - const: fsl,s32v234-linflexuart reg: From 7c5dffb3d90c5921b91981cc663e02757d90526e Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Sun, 7 Apr 2024 15:26:03 +0800 Subject: [PATCH 0142/1702] f2fs: compress: fix to relocate check condition in f2fs_{release,reserve}_compress_blocks() Compress flag should be checked after inode lock held to avoid racing w/ f2fs_setflags_common(), fix it. Fixes: 4c8ff7095bef ("f2fs: support data compression") Reported-by: Zhiguo Niu Closes: https://lore.kernel.org/linux-f2fs-devel/CAHJ8P3LdZXLc2rqeYjvymgYHr2+YLuJ0sLG9DdsJZmwO7deuhw@mail.gmail.com Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index bd55bbb4fa672..e4435461aeebb 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -3543,9 +3543,6 @@ static int f2fs_release_compress_blocks(struct file *filp, unsigned long arg) if (!f2fs_sb_has_compression(sbi)) return -EOPNOTSUPP; - if (!f2fs_compressed_file(inode)) - return -EINVAL; - if (f2fs_readonly(sbi->sb)) return -EROFS; @@ -3564,7 +3561,8 @@ static int f2fs_release_compress_blocks(struct file *filp, unsigned long arg) goto out; } - if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) { + if (!f2fs_compressed_file(inode) || + is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) { ret = -EINVAL; goto out; } @@ -3725,9 +3723,6 @@ static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg) if (!f2fs_sb_has_compression(sbi)) return -EOPNOTSUPP; - if (!f2fs_compressed_file(inode)) - return -EINVAL; - if (f2fs_readonly(sbi->sb)) return -EROFS; @@ -3739,7 +3734,8 @@ static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg) inode_lock(inode); - if (!is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) { + if (!f2fs_compressed_file(inode) || + !is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) { ret = -EINVAL; goto unlock_inode; } From bd9ae4ae9e585061acfd4a169f2321706f900246 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Sun, 7 Apr 2024 15:26:04 +0800 Subject: [PATCH 0143/1702] f2fs: compress: fix to relocate check condition in f2fs_ioc_{,de}compress_file() Compress flag should be checked after inode lock held to avoid racing w/ f2fs_setflags_common() , fix it. Fixes: 5fdb322ff2c2 ("f2fs: add F2FS_IOC_DECOMPRESS_FILE and F2FS_IOC_COMPRESS_FILE") Reported-by: Zhiguo Niu Closes: https://lore.kernel.org/linux-f2fs-devel/CAHJ8P3LdZXLc2rqeYjvymgYHr2+YLuJ0sLG9DdsJZmwO7deuhw@mail.gmail.com Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index e4435461aeebb..f88dff065ff18 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -4136,9 +4136,6 @@ static int f2fs_ioc_decompress_file(struct file *filp) if (!(filp->f_mode & FMODE_WRITE)) return -EBADF; - if (!f2fs_compressed_file(inode)) - return -EINVAL; - f2fs_balance_fs(sbi, true); file_start_write(filp); @@ -4149,7 +4146,8 @@ static int f2fs_ioc_decompress_file(struct file *filp) goto out; } - if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) { + if (!f2fs_compressed_file(inode) || + is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) { ret = -EINVAL; goto out; } @@ -4214,9 +4212,6 @@ static int f2fs_ioc_compress_file(struct file *filp) if (!(filp->f_mode & FMODE_WRITE)) return -EBADF; - if (!f2fs_compressed_file(inode)) - return -EINVAL; - f2fs_balance_fs(sbi, true); file_start_write(filp); @@ -4227,7 +4222,8 @@ static int f2fs_ioc_compress_file(struct file *filp) goto out; } - if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) { + if (!f2fs_compressed_file(inode) || + is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) { ret = -EINVAL; goto out; } From 278a6253a673611dbc8ab72a3b34b151a8e75822 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Wed, 3 Apr 2024 22:24:19 +0800 Subject: [PATCH 0144/1702] f2fs: fix to relocate check condition in f2fs_fallocate() compress and pinfile flag should be checked after inode lock held to avoid race condition, fix it. Fixes: 4c8ff7095bef ("f2fs: support data compression") Fixes: 5fed0be8583f ("f2fs: do not allow partial truncation on pinned file") Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index f88dff065ff18..fc3d4d69e72c0 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -1820,15 +1820,6 @@ static long f2fs_fallocate(struct file *file, int mode, (mode & (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE))) return -EOPNOTSUPP; - /* - * Pinned file should not support partial truncation since the block - * can be used by applications. - */ - if ((f2fs_compressed_file(inode) || f2fs_is_pinned_file(inode)) && - (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_COLLAPSE_RANGE | - FALLOC_FL_ZERO_RANGE | FALLOC_FL_INSERT_RANGE))) - return -EOPNOTSUPP; - if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE | FALLOC_FL_INSERT_RANGE)) @@ -1836,6 +1827,17 @@ static long f2fs_fallocate(struct file *file, int mode, inode_lock(inode); + /* + * Pinned file should not support partial truncation since the block + * can be used by applications. + */ + if ((f2fs_compressed_file(inode) || f2fs_is_pinned_file(inode)) && + (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_COLLAPSE_RANGE | + FALLOC_FL_ZERO_RANGE | FALLOC_FL_INSERT_RANGE))) { + ret = -EOPNOTSUPP; + goto out; + } + ret = file_modified(file); if (ret) goto out; From e07230da0500e0919a765037c5e81583b519be2c Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Wed, 3 Apr 2024 22:24:20 +0800 Subject: [PATCH 0145/1702] f2fs: fix to check pinfile flag in f2fs_move_file_range() ioctl(F2FS_IOC_MOVE_RANGE) can truncate or punch hole on pinned file, fix to disallow it. Fixes: 5fed0be8583f ("f2fs: do not allow partial truncation on pinned file") Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index fc3d4d69e72c0..681dfa8775ab4 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -2860,7 +2860,8 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, goto out; } - if (f2fs_compressed_file(src) || f2fs_compressed_file(dst)) { + if (f2fs_compressed_file(src) || f2fs_compressed_file(dst) || + f2fs_is_pinned_file(src) || f2fs_is_pinned_file(dst)) { ret = -EOPNOTSUPP; goto out_unlock; } From 3bdb7f161697e2d5123b89fe1778ef17a44858e7 Mon Sep 17 00:00:00 2001 From: Jaegeuk Kim Date: Wed, 3 Apr 2024 23:07:53 +0000 Subject: [PATCH 0146/1702] f2fs: don't set RO when shutting down f2fs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shutdown does not check the error of thaw_super due to readonly, which causes a deadlock like below. f2fs_ioc_shutdown(F2FS_GOING_DOWN_FULLSYNC) issue_discard_thread - bdev_freeze - freeze_super - f2fs_stop_checkpoint() - f2fs_handle_critical_error - sb_start_write - set RO - waiting - bdev_thaw - thaw_super_locked - return -EINVAL, if sb_rdonly() - f2fs_stop_discard_thread -> wait for kthread_stop(discard_thread); Reported-by: "Light Hsieh (謝明燈)" Reviewed-by: Daeho Jeong Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/super.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 8ac4734c2df60..df32573d1f620 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -4159,9 +4159,15 @@ void f2fs_handle_critical_error(struct f2fs_sb_info *sbi, unsigned char reason, if (shutdown) set_sbi_flag(sbi, SBI_IS_SHUTDOWN); - /* continue filesystem operators if errors=continue */ - if (continue_fs || f2fs_readonly(sb)) + /* + * Continue filesystem operators if errors=continue. Should not set + * RO by shutdown, since RO bypasses thaw_super which can hang the + * system. + */ + if (continue_fs || f2fs_readonly(sb) || shutdown) { + f2fs_warn(sbi, "Stopped filesystem due to reason: %d", reason); return; + } f2fs_warn(sbi, "Remounting filesystem read-only"); /* From b084403cfc3295b59a1b6bcc94efaf870fc3c2c9 Mon Sep 17 00:00:00 2001 From: Daeho Jeong Date: Tue, 9 Apr 2024 16:34:11 -0700 Subject: [PATCH 0147/1702] f2fs: write missing last sum blk of file pinning section While do not allocating a new section in advance for file pinning area, I missed that we should write the sum block for the last segment of a file pinning section. Fixes: 9703d69d9d15 ("f2fs: support file pinning for zoned devices") Signed-off-by: Daeho Jeong Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/segment.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 4fd76e867e0a2..6474b7338e811 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -3559,6 +3559,8 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, if (segment_full) { if (type == CURSEG_COLD_DATA_PINNED && !((curseg->segno + 1) % sbi->segs_per_sec)) { + write_sum_page(sbi, curseg->sum_blk, + GET_SUM_BLOCK(sbi, curseg->segno)); reset_curseg_fields(curseg); goto skip_new_segment; } From fa18d87cb20fbe597047704babae3850a7a65219 Mon Sep 17 00:00:00 2001 From: Zhiguo Niu Date: Wed, 20 Mar 2024 14:22:16 +0800 Subject: [PATCH 0148/1702] f2fs: add REQ_TIME time update for some user behaviors some user behaviors requested filesystem operations, which will cause filesystem not idle. Meanwhile adjust some f2fs_update_time(REQ_TIME) positions. Signed-off-by: Zhiguo Niu Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 681dfa8775ab4..0d2626a6a21fe 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -2375,13 +2375,14 @@ static bool uuid_is_nonzero(__u8 u[16]) static int f2fs_ioc_set_encryption_policy(struct file *filp, unsigned long arg) { struct inode *inode = file_inode(filp); + int ret; if (!f2fs_sb_has_encrypt(F2FS_I_SB(inode))) return -EOPNOTSUPP; + ret = fscrypt_ioctl_set_policy(filp, (const void __user *)arg); f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); - - return fscrypt_ioctl_set_policy(filp, (const void __user *)arg); + return ret; } static int f2fs_ioc_get_encryption_policy(struct file *filp, unsigned long arg) @@ -2809,7 +2810,8 @@ static int f2fs_ioc_defragment(struct file *filp, unsigned long arg) err = f2fs_defragment_range(sbi, filp, &range); mnt_drop_write_file(filp); - f2fs_update_time(sbi, REQ_TIME); + if (range.len) + f2fs_update_time(sbi, REQ_TIME); if (err < 0) return err; @@ -3622,6 +3624,8 @@ static int f2fs_release_compress_blocks(struct file *filp, unsigned long arg) filemap_invalidate_unlock(inode->i_mapping); f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); out: + if (released_blocks) + f2fs_update_time(sbi, REQ_TIME); inode_unlock(inode); mnt_drop_write_file(filp); @@ -3790,6 +3794,8 @@ static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg) f2fs_mark_inode_dirty_sync(inode, true); } unlock_inode: + if (reserved_blocks) + f2fs_update_time(sbi, REQ_TIME); inode_unlock(inode); mnt_drop_write_file(filp); @@ -3986,6 +3992,7 @@ static int f2fs_sec_trim_file(struct file *filp, unsigned long arg) if (len) ret = f2fs_secure_erase(prev_bdev, inode, prev_index, prev_block, len, range.flags); + f2fs_update_time(sbi, REQ_TIME); out: filemap_invalidate_unlock(mapping); f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); @@ -4193,6 +4200,7 @@ static int f2fs_ioc_decompress_file(struct file *filp) if (ret) f2fs_warn(sbi, "%s: The file might be partially decompressed (errno=%d). Please delete the file.", __func__, ret); + f2fs_update_time(sbi, REQ_TIME); out: inode_unlock(inode); file_end_write(filp); @@ -4270,6 +4278,7 @@ static int f2fs_ioc_compress_file(struct file *filp) if (ret) f2fs_warn(sbi, "%s: The file might be partially compressed (errno=%d). Please delete the file.", __func__, ret); + f2fs_update_time(sbi, REQ_TIME); out: inode_unlock(inode); file_end_write(filp); From 16778aea91869756cad3e07c9ce8ef32a660358c Mon Sep 17 00:00:00 2001 From: Jaegeuk Kim Date: Wed, 27 Mar 2024 01:27:02 +0000 Subject: [PATCH 0149/1702] f2fs: use folio_test_writeback Let's convert PageWriteback to folio_test_writeback. Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/compress.c | 2 +- fs/f2fs/data.c | 3 +-- fs/f2fs/f2fs.h | 2 +- fs/f2fs/gc.c | 2 +- fs/f2fs/inline.c | 2 +- fs/f2fs/inode.c | 3 ++- fs/f2fs/node.c | 2 +- fs/f2fs/segment.c | 10 +++++----- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index 8892c82621414..d67c471ab5df6 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -1484,7 +1484,7 @@ static int f2fs_write_raw_pages(struct compress_ctx *cc, if (!PageDirty(cc->rpages[i])) goto continue_unlock; - if (PageWriteback(cc->rpages[i])) { + if (folio_test_writeback(page_folio(cc->rpages[i]))) { if (wbc->sync_mode == WB_SYNC_NONE) goto continue_unlock; f2fs_wait_on_page_writeback(cc->rpages[i], DATA, true, true); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 0d88649c60a50..5d641fac02ba7 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -2704,8 +2704,7 @@ int f2fs_do_write_data_page(struct f2fs_io_info *fio) if (err) { if (fscrypt_inode_uses_fs_layer_crypto(inode)) fscrypt_finalize_bounce_page(&fio->encrypted_page); - if (PageWriteback(page)) - end_page_writeback(page); + end_page_writeback(page); } else { set_inode_flag(inode, FI_UPDATE_WRITE); } diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index e9ef971f4dba6..dd530dc700052 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -4660,7 +4660,7 @@ static inline void f2fs_truncate_meta_inode_pages(struct f2fs_sb_info *sbi, page = find_get_page(META_MAPPING(sbi), blkaddr + i); if (page) { - if (PageWriteback(page)) + if (folio_test_writeback(page_folio(page))) need_submit = true; f2fs_put_page(page, 0); } diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index 8852814dab7f6..ac4cbbe50c2fc 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -1434,7 +1434,7 @@ static int move_data_page(struct inode *inode, block_t bidx, int gc_type, goto out; if (gc_type == BG_GC) { - if (PageWriteback(page)) { + if (folio_test_writeback(page_folio(page))) { err = -EAGAIN; goto out; } diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index ac00423f117b5..3d3218a4b29df 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -164,7 +164,7 @@ int f2fs_convert_inline_page(struct dnode_of_data *dn, struct page *page) return -EFSCORRUPTED; } - f2fs_bug_on(F2FS_P_SB(page), PageWriteback(page)); + f2fs_bug_on(F2FS_P_SB(page), folio_test_writeback(page_folio(page))); f2fs_do_read_inline_data(page, dn->inode_page); set_page_dirty(page); diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index 12b1fef31f437..d7a5a88a1a5eb 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -161,7 +161,8 @@ bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct page *page) if (!f2fs_enable_inode_chksum(sbi, page)) #else if (!f2fs_enable_inode_chksum(sbi, page) || - PageDirty(page) || PageWriteback(page)) + PageDirty(page) || + folio_test_writeback(page_folio(page))) #endif return true; diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index bb57bbaff7b4f..3b9eb56936832 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -1743,7 +1743,7 @@ int f2fs_move_node_page(struct page *node_page, int gc_type) goto release_page; } else { /* set page dirty and write it */ - if (!PageWriteback(node_page)) + if (!folio_test_writeback(page_folio(node_page))) set_page_dirty(node_page); } out_page: diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 6474b7338e811..f0da516ba8dc3 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -3614,13 +3614,13 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, mutex_unlock(&curseg->curseg_mutex); f2fs_up_read(&SM_I(sbi)->curseg_lock); return 0; + out_err: *new_blkaddr = NULL_ADDR; up_write(&sit_i->sentry_lock); mutex_unlock(&curseg->curseg_mutex); f2fs_up_read(&SM_I(sbi)->curseg_lock); return ret; - } void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino, @@ -3662,8 +3662,7 @@ static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) &fio->new_blkaddr, sum, type, fio)) { if (fscrypt_inode_uses_fs_layer_crypto(fio->page->mapping->host)) fscrypt_finalize_bounce_page(&fio->encrypted_page); - if (PageWriteback(fio->page)) - end_page_writeback(fio->page); + end_page_writeback(fio->page); if (f2fs_in_warm_node_list(fio->sbi, fio->page)) f2fs_del_fsync_node_entry(fio->sbi, fio->page); goto out; @@ -3906,7 +3905,7 @@ void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, void f2fs_wait_on_page_writeback(struct page *page, enum page_type type, bool ordered, bool locked) { - if (PageWriteback(page)) { + if (folio_test_writeback(page_folio(page))) { struct f2fs_sb_info *sbi = F2FS_P_SB(page); /* submit cached LFS IO */ @@ -3915,7 +3914,8 @@ void f2fs_wait_on_page_writeback(struct page *page, f2fs_submit_merged_ipu_write(sbi, NULL, page); if (ordered) { wait_on_page_writeback(page); - f2fs_bug_on(sbi, locked && PageWriteback(page)); + f2fs_bug_on(sbi, locked && + folio_test_writeback(page_folio(page))); } else { wait_for_stable_page(page); } From 3fdd89b452c2ea5e2195d6e315bef122769584c9 Mon Sep 17 00:00:00 2001 From: Daeho Jeong Date: Thu, 11 Apr 2024 10:54:10 -0700 Subject: [PATCH 0150/1702] f2fs: prevent writing without fallocate() for pinned files In a case writing without fallocate(), we can't guarantee it's allocated in the conventional area for zoned stroage. To make it consistent across storage devices, we disallow it regardless of storage device types. Signed-off-by: Daeho Jeong Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 0d2626a6a21fe..7fbdb7863778f 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -58,7 +58,7 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) struct inode *inode = file_inode(vmf->vma->vm_file); struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct dnode_of_data dn; - bool need_alloc = true; + bool need_alloc = !f2fs_is_pinned_file(inode); int err = 0; vm_fault_t ret; @@ -115,19 +115,18 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) goto out_sem; } + set_new_dnode(&dn, inode, NULL, NULL, 0); if (need_alloc) { /* block allocation */ - set_new_dnode(&dn, inode, NULL, NULL, 0); err = f2fs_get_block_locked(&dn, page->index); - } - -#ifdef CONFIG_F2FS_FS_COMPRESSION - if (!need_alloc) { - set_new_dnode(&dn, inode, NULL, NULL, 0); + } else { err = f2fs_get_dnode_of_data(&dn, page->index, LOOKUP_NODE); f2fs_put_dnode(&dn); + if (f2fs_is_pinned_file(inode) && + !__is_valid_data_blkaddr(dn.data_blkaddr)) + err = -EIO; } -#endif + if (err) { unlock_page(page); goto out_sem; @@ -3260,7 +3259,7 @@ static int f2fs_ioc_set_pin_file(struct file *filp, unsigned long arg) goto done; } - if (f2fs_sb_has_blkzoned(sbi) && F2FS_HAS_BLOCKS(inode)) { + if (F2FS_HAS_BLOCKS(inode)) { ret = -EFBIG; goto out; } @@ -4823,6 +4822,8 @@ static ssize_t f2fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) bool dio; bool may_need_sync = true; int preallocated; + const loff_t pos = iocb->ki_pos; + const ssize_t count = iov_iter_count(from); ssize_t ret; if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) { @@ -4844,6 +4845,12 @@ static ssize_t f2fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) inode_lock(inode); } + if (f2fs_is_pinned_file(inode) && + !f2fs_overwrite_io(inode, pos, count)) { + ret = -EIO; + goto out_unlock; + } + ret = f2fs_write_checks(iocb, from); if (ret <= 0) goto out_unlock; From b2cf5a1ff236fcd0eb9dcbb188df30b50bb1af0f Mon Sep 17 00:00:00 2001 From: Daeho Jeong Date: Thu, 11 Apr 2024 11:37:53 -0700 Subject: [PATCH 0151/1702] f2fs: allow direct io of pinned files for zoned storage Since the allocation happens in conventional LU for zoned storage, we can allow direct io for that. Signed-off-by: Daeho Jeong Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 7fbdb7863778f..ac1ae85f3cc3d 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -833,7 +833,8 @@ static bool f2fs_force_buffered_io(struct inode *inode, int rw) * for blkzoned device, fallback direct IO to buffered IO, so * all IOs can be serialized by log-structured write. */ - if (f2fs_sb_has_blkzoned(sbi) && (rw == WRITE)) + if (f2fs_sb_has_blkzoned(sbi) && (rw == WRITE) && + !f2fs_is_pinned_file(inode)) return true; if (is_sbi_flag_set(sbi, SBI_CP_DISABLED)) return true; From 34059321f4cf541d69e733cd02882d62c489bb77 Mon Sep 17 00:00:00 2001 From: Yongzhen Zhang Date: Fri, 15 Mar 2024 10:29:23 +0800 Subject: [PATCH 0152/1702] MIPS: BCM47XX: include header for bcm47xx_prom_highmem_init() prototype MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bcm47xx_prom_highmem_init() is a global function declared in arch/mips/bcm47xx/bcm47xx_private.h, but this header is not included before the definition, causing a error: arch/mips/bcm47xx/prom.c:134:13: error: no previous prototype for ‘bcm47xx_prom_highmem_init’ [-Werror=missing-prototypes] 134 | void __init bcm47xx_prom_highmem_init(void) | ^~~~~~~~~~~~~~~~~~~~~~~~~ Signed-off-by: Yongzhen Zhang Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Thomas Bogendoerfer --- arch/mips/bcm47xx/prom.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/mips/bcm47xx/prom.c b/arch/mips/bcm47xx/prom.c index 99a1ba5394e02..7344a5eb990ec 100644 --- a/arch/mips/bcm47xx/prom.c +++ b/arch/mips/bcm47xx/prom.c @@ -35,6 +35,7 @@ #include #include #include +#include "bcm47xx_private.h" static char bcm47xx_system_type[20] = "Broadcom BCM47XX"; From d18419cd66835c29ac732624324b99b43f4cff1c Mon Sep 17 00:00:00 2001 From: Yongzhen Zhang Date: Fri, 15 Mar 2024 10:52:35 +0800 Subject: [PATCH 0153/1702] MIPS: BCM47XX: Declare early_tlb_init() static MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit early_tlb_init() was local to file but not declared static, leading to error: arch/mips/bcm47xx/prom.c:126:6: error: no previous prototype for ‘early_tlb_init’ [-Werror=missing-prototypes] 126 | void early_tlb_init(void) | ^~~~~~~~~~~~~~ Signed-off-by: Yongzhen Zhang Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Thomas Bogendoerfer --- arch/mips/bcm47xx/prom.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/bcm47xx/prom.c b/arch/mips/bcm47xx/prom.c index 7344a5eb990ec..58fb7c2dc3b8a 100644 --- a/arch/mips/bcm47xx/prom.c +++ b/arch/mips/bcm47xx/prom.c @@ -124,7 +124,7 @@ void __init prom_init(void) /* Stripped version of tlb_init, with the call to build_tlb_refill_handler * dropped. Calling it at this stage causes a hang. */ -void early_tlb_init(void) +static void early_tlb_init(void) { write_c0_pagemask(PM_DEFAULT_MASK); write_c0_wired(0); From b796d046433b2042577d8d6c9a5d366e39095c30 Mon Sep 17 00:00:00 2001 From: Yongzhen Zhang Date: Fri, 15 Mar 2024 12:10:43 +0800 Subject: [PATCH 0154/1702] MIPS: RB532: Declare prom_setup_cmdline() and rb532_gpio_init() static MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit early_tlb_init() and rb532_gpio_init() were local to file but not declared static, leading to error: arch/mips/rb532/prom.c:49:13: error: no previous prototype for ‘prom_setup_cmdline’ [-Werror=missing-prototypes] 49 | void __init prom_setup_cmdline(void) | ^~~~~~~~~~~~~~~~~~ arch/mips/rb532/gpio.c:200:12: error: no previous prototype for ‘rb532_gpio_init’ [-Werror=missing-prototypes] 200 | int __init rb532_gpio_init(void) | ^~~~~~~~~~~~~~~ Signed-off-by: Yongzhen Zhang Signed-off-by: Thomas Bogendoerfer --- arch/mips/rb532/gpio.c | 2 +- arch/mips/rb532/prom.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/rb532/gpio.c b/arch/mips/rb532/gpio.c index 29c21b9d42dab..ea6ebfea4a672 100644 --- a/arch/mips/rb532/gpio.c +++ b/arch/mips/rb532/gpio.c @@ -197,7 +197,7 @@ void rb532_gpio_set_func(unsigned gpio) } EXPORT_SYMBOL(rb532_gpio_set_func); -int __init rb532_gpio_init(void) +static int __init rb532_gpio_init(void) { struct resource *r; diff --git a/arch/mips/rb532/prom.c b/arch/mips/rb532/prom.c index b116937155474..b88e89ec58941 100644 --- a/arch/mips/rb532/prom.c +++ b/arch/mips/rb532/prom.c @@ -46,7 +46,7 @@ static inline unsigned long tag2ul(char *arg, const char *tag) return simple_strtoul(num, 0, 10); } -void __init prom_setup_cmdline(void) +static void __init prom_setup_cmdline(void) { static char cmd_line[COMMAND_LINE_SIZE] __initdata; char *cp, *board; From a1b7508cef6208ad06377d2aa05b0f89f7d6b516 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Tue, 26 Mar 2024 11:41:45 +0000 Subject: [PATCH 0155/1702] MIPS: Guard some macros with __ASSEMBLY__ in asm.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are some assembly macros with very generic naming being defined asm.h. They are clashing with other macros from C code. Guard them with __ASSEMBLY__ to prevent futher clashes. Reported-by: Geert Uytterhoeven Link: https://lore.kernel.org/linux-mips/8d78894-dd89-9f4d-52bb-1b873c50be9c@linux-m68k.org/ Signed-off-by: Jiaxun Yang Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/asm.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/mips/include/asm/asm.h b/arch/mips/include/asm/asm.h index 2e99450f42284..87ff609b53fe1 100644 --- a/arch/mips/include/asm/asm.h +++ b/arch/mips/include/asm/asm.h @@ -37,6 +37,7 @@ #define CFI_SECTIONS #endif +#ifdef __ASSEMBLY__ /* * LEAF - declare leaf routine */ @@ -122,6 +123,8 @@ symbol = value #define ASM_PRINT(string) #endif +#endif /* __ASSEMBLY__ */ + /* * Stack alignment */ From 29b83a64df3b42c88c0338696feb6fdcd7f1f3b7 Mon Sep 17 00:00:00 2001 From: Songyang Li Date: Wed, 20 Mar 2024 23:22:00 +0800 Subject: [PATCH 0156/1702] MIPS: Octeon: Add PCIe link status check The standard PCIe configuration read-write interface is used to access the configuration space of the peripheral PCIe devices of the mips processor after the PCIe link surprise down, it can generate kernel panic caused by "Data bus error". So it is necessary to add PCIe link status check for system protection. When the PCIe link is down or in training, assigning a value of 0 to the configuration address can prevent read-write behavior to the configuration space of peripheral PCIe devices, thereby preventing kernel panic. Signed-off-by: Songyang Li Signed-off-by: Thomas Bogendoerfer --- arch/mips/pci/pcie-octeon.c | 6 ++++++ 1 file changed, 6 insertions(+) mode change 100644 => 100755 arch/mips/pci/pcie-octeon.c diff --git a/arch/mips/pci/pcie-octeon.c b/arch/mips/pci/pcie-octeon.c old mode 100644 new mode 100755 index 2583e318e8c6b..b080c7c6cc463 --- a/arch/mips/pci/pcie-octeon.c +++ b/arch/mips/pci/pcie-octeon.c @@ -230,12 +230,18 @@ static inline uint64_t __cvmx_pcie_build_config_addr(int pcie_port, int bus, { union cvmx_pcie_address pcie_addr; union cvmx_pciercx_cfg006 pciercx_cfg006; + union cvmx_pciercx_cfg032 pciercx_cfg032; pciercx_cfg006.u32 = cvmx_pcie_cfgx_read(pcie_port, CVMX_PCIERCX_CFG006(pcie_port)); if ((bus <= pciercx_cfg006.s.pbnum) && (dev != 0)) return 0; + pciercx_cfg032.u32 = + cvmx_pcie_cfgx_read(pcie_port, CVMX_PCIERCX_CFG032(pcie_port)); + if ((pciercx_cfg032.s.dlla == 0) || (pciercx_cfg032.s.lt == 1)) + return 0; + pcie_addr.u64 = 0; pcie_addr.config.upper = 2; pcie_addr.config.io = 1; From f3cac4f8a93bf7f97ba1d4c2eee916fcd1e8885b Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 3 Apr 2024 14:26:33 -0700 Subject: [PATCH 0157/1702] MIPS: Add prototypes for plat_post_relocation() and relocate_kernel() When building malta_defconfig with CONFIG_RELOCATABLE=y, there are two warnings due to missing prototypes for functions only used when that configuration is enabled: arch/mips/kernel/relocate.c:42:12: warning: no previous prototype for 'plat_post_relocation' [-Wmissing-prototypes] 42 | int __weak plat_post_relocation(long offset) | ^~~~~~~~~~~~~~~~~~~~ arch/mips/kernel/relocate.c:324:14: warning: no previous prototype for 'relocate_kernel' [-Wmissing-prototypes] 324 | void *__init relocate_kernel(void) | ^~~~~~~~~~~~~~~ While relocate_kernel() is only called from assembly, it makes sense to keep the prototypes together in C to fix the warnings. Add them to silence the warnings. Signed-off-by: Nathan Chancellor Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/setup.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/mips/include/asm/setup.h b/arch/mips/include/asm/setup.h index 4dce41138bad6..d8077136372c4 100644 --- a/arch/mips/include/asm/setup.h +++ b/arch/mips/include/asm/setup.h @@ -2,6 +2,7 @@ #ifndef _MIPS_SETUP_H #define _MIPS_SETUP_H +#include #include #include @@ -29,4 +30,9 @@ extern void per_cpu_trap_init(bool); extern void cpu_cache_init(void); extern void tlb_init(void); +#ifdef CONFIG_RELOCATABLE +extern void * __init relocate_kernel(void); +extern int plat_post_relocation(long); +#endif + #endif /* __SETUP_H */ From 3eee9ac24cef892e6883b3669544c6101b70c91e Mon Sep 17 00:00:00 2001 From: Justin Swartz Date: Sat, 16 Mar 2024 06:54:29 +0200 Subject: [PATCH 0158/1702] mips: dts: ralink: mt7621: reorder cpu node attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reorder cpu node attributes to fit the DTS Coding Style. Signed-off-by: Justin Swartz Reviewed-by: Arınç ÜNAL Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Sergio Paracuellos Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ralink/mt7621.dtsi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi index 6e95e6f19a6a8..73dad64e11fee 100644 --- a/arch/mips/boot/dts/ralink/mt7621.dtsi +++ b/arch/mips/boot/dts/ralink/mt7621.dtsi @@ -14,15 +14,15 @@ #size-cells = <0>; cpu@0 { - device_type = "cpu"; compatible = "mips,mips1004Kc"; reg = <0>; + device_type = "cpu"; }; cpu@1 { - device_type = "cpu"; compatible = "mips,mips1004Kc"; reg = <1>; + device_type = "cpu"; }; }; From 09e8ff7576ae9dd4996172d3bf18975c73f3dc77 Mon Sep 17 00:00:00 2001 From: Justin Swartz Date: Sat, 16 Mar 2024 06:54:30 +0200 Subject: [PATCH 0159/1702] mips: dts: ralink: mt7621: reorder cpuintc node attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reorder the CPU Interrupt Controller node's attributes to follow what the DTS Coding Style dictates. Signed-off-by: Justin Swartz Reviewed-by: Arınç ÜNAL Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Sergio Paracuellos Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ralink/mt7621.dtsi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi index 73dad64e11fee..ec87e46ba6de6 100644 --- a/arch/mips/boot/dts/ralink/mt7621.dtsi +++ b/arch/mips/boot/dts/ralink/mt7621.dtsi @@ -27,10 +27,12 @@ }; cpuintc: cpuintc { + compatible = "mti,cpu-interrupt-controller"; + #address-cells = <0>; #interrupt-cells = <1>; + interrupt-controller; - compatible = "mti,cpu-interrupt-controller"; }; mmc_fixed_3v3: regulator-3v3 { From df91c0da8096f61543837bc5d033dfdfd5d1c23d Mon Sep 17 00:00:00 2001 From: Justin Swartz Date: Sat, 16 Mar 2024 06:54:31 +0200 Subject: [PATCH 0160/1702] mips: dts: ralink: mt7621: reorder mmc regulator attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reorder the attributes of MMC fixed voltage regulator nodes for the sake of compliance with the DTS style guide. Signed-off-by: Justin Swartz Reviewed-by: Arınç ÜNAL Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Sergio Paracuellos Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ralink/mt7621.dtsi | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi index ec87e46ba6de6..696460b2dca7f 100644 --- a/arch/mips/boot/dts/ralink/mt7621.dtsi +++ b/arch/mips/boot/dts/ralink/mt7621.dtsi @@ -37,20 +37,24 @@ mmc_fixed_3v3: regulator-3v3 { compatible = "regulator-fixed"; - regulator-name = "mmc_power"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; + enable-active-high; + regulator-always-on; + regulator-max-microvolt = <3300000>; + regulator-min-microvolt = <3300000>; + regulator-name = "mmc_power"; }; mmc_fixed_1v8_io: regulator-1v8 { compatible = "regulator-fixed"; - regulator-name = "mmc_io"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; + enable-active-high; + regulator-always-on; + regulator-max-microvolt = <1800000>; + regulator-min-microvolt = <1800000>; + regulator-name = "mmc_io"; }; palmbus: palmbus@1e000000 { From 9938cd312b8f1922b76e9d48a13425d56b597580 Mon Sep 17 00:00:00 2001 From: Justin Swartz Date: Sat, 16 Mar 2024 06:54:32 +0200 Subject: [PATCH 0161/1702] mips: dts: ralink: mt7621: reorder sysc node attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reorder the attributes of the sysc node so that the ralink prefixed attribute is placed after those which lack prefixes. Signed-off-by: Justin Swartz Reviewed-by: Arınç ÜNAL Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Sergio Paracuellos Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ralink/mt7621.dtsi | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi index 696460b2dca7f..d1d4399d730ca 100644 --- a/arch/mips/boot/dts/ralink/mt7621.dtsi +++ b/arch/mips/boot/dts/ralink/mt7621.dtsi @@ -68,12 +68,15 @@ sysc: syscon@0 { compatible = "mediatek,mt7621-sysc", "syscon"; reg = <0x0 0x100>; + #clock-cells = <1>; #reset-cells = <1>; - ralink,memctl = <&memc>; + clock-output-names = "xtal", "cpu", "bus", "50m", "125m", "150m", "250m", "270m"; + + ralink,memctl = <&memc>; }; wdt: watchdog@100 { From 9a4ba656343d4a34626cf0c222ecb6bac8dd1490 Mon Sep 17 00:00:00 2001 From: Justin Swartz Date: Sat, 16 Mar 2024 06:54:33 +0200 Subject: [PATCH 0162/1702] mips: dts: ralink: mt7621: reorder gpio node attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shuffle the attributes of the gpio node to appease the DTS style guide. Signed-off-by: Justin Swartz Reviewed-by: Arınç ÜNAL Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Sergio Paracuellos Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ralink/mt7621.dtsi | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi index d1d4399d730ca..99d47f2867698 100644 --- a/arch/mips/boot/dts/ralink/mt7621.dtsi +++ b/arch/mips/boot/dts/ralink/mt7621.dtsi @@ -86,13 +86,16 @@ }; gpio: gpio@600 { + compatible = "mediatek,mt7621-gpio"; + reg = <0x600 0x100>; + #gpio-cells = <2>; #interrupt-cells = <2>; - compatible = "mediatek,mt7621-gpio"; + gpio-controller; gpio-ranges = <&pinctrl 0 0 95>; + interrupt-controller; - reg = <0x600 0x100>; interrupt-parent = <&gic>; interrupts = ; }; From 9d64db86d129067690aaa5fe6a4572515e98fe53 Mon Sep 17 00:00:00 2001 From: Justin Swartz Date: Sat, 16 Mar 2024 06:54:34 +0200 Subject: [PATCH 0163/1702] mips: dts: ralink: mt7621: reorder i2c node attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rearrange the order of the i2c node's attributes so that they are inline with the DTS style guide. Signed-off-by: Justin Swartz Reviewed-by: Arınç ÜNAL Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Sergio Paracuellos Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ralink/mt7621.dtsi | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi index 99d47f2867698..87a3bcbc0ea58 100644 --- a/arch/mips/boot/dts/ralink/mt7621.dtsi +++ b/arch/mips/boot/dts/ralink/mt7621.dtsi @@ -104,18 +104,19 @@ compatible = "mediatek,mt7621-i2c"; reg = <0x900 0x100>; - clocks = <&sysc MT7621_CLK_I2C>; - clock-names = "i2c"; - resets = <&sysc MT7621_RST_I2C>; - reset-names = "i2c"; - #address-cells = <1>; #size-cells = <0>; - status = "disabled"; + clocks = <&sysc MT7621_CLK_I2C>; + clock-names = "i2c"; pinctrl-names = "default"; pinctrl-0 = <&i2c_pins>; + + resets = <&sysc MT7621_RST_I2C>; + reset-names = "i2c"; + + status = "disabled"; }; memc: memory-controller@5000 { From f5a0fc0a95a0cdbce0aa81431e924a08cec01bc9 Mon Sep 17 00:00:00 2001 From: Justin Swartz Date: Sat, 16 Mar 2024 06:54:35 +0200 Subject: [PATCH 0164/1702] mips: dts: ralink: mt7621: reorder spi0 node attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reorder the attributes of the SPI controller node so that they're aligned with the DTS style guide. Signed-off-by: Justin Swartz Reviewed-by: Arınç ÜNAL Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Sergio Paracuellos Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ralink/mt7621.dtsi | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi index 87a3bcbc0ea58..60dfbae53712c 100644 --- a/arch/mips/boot/dts/ralink/mt7621.dtsi +++ b/arch/mips/boot/dts/ralink/mt7621.dtsi @@ -183,22 +183,22 @@ }; spi0: spi@b00 { - status = "disabled"; - compatible = "ralink,mt7621-spi"; reg = <0xb00 0x100>; - clocks = <&sysc MT7621_CLK_SPI>; - clock-names = "spi"; - - resets = <&sysc MT7621_RST_SPI>; - reset-names = "spi"; - #address-cells = <1>; #size-cells = <0>; + clock-names = "spi"; + clocks = <&sysc MT7621_CLK_SPI>; + pinctrl-names = "default"; pinctrl-0 = <&spi_pins>; + + reset-names = "spi"; + resets = <&sysc MT7621_RST_SPI>; + + status = "disabled"; }; }; From 384f8ef478eb34ce25aab8164ee83f48221a4fff Mon Sep 17 00:00:00 2001 From: Justin Swartz Date: Sat, 16 Mar 2024 06:54:36 +0200 Subject: [PATCH 0165/1702] mips: dts: ralink: mt7621: move pinctrl and sort its children MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the pinctrl node prior to the nodes that feature unit addresses. Sort pinctrl's child nodes into alphabetical order. Signed-off-by: Justin Swartz Reviewed-by: Arınç ÜNAL Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Sergio Paracuellos Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ralink/mt7621.dtsi | 172 +++++++++++++------------- 1 file changed, 86 insertions(+), 86 deletions(-) diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi index 60dfbae53712c..b28aee1e4ca3f 100644 --- a/arch/mips/boot/dts/ralink/mt7621.dtsi +++ b/arch/mips/boot/dts/ralink/mt7621.dtsi @@ -57,6 +57,92 @@ regulator-name = "mmc_io"; }; + pinctrl: pinctrl { + compatible = "ralink,mt7621-pinctrl"; + + i2c_pins: i2c0-pins { + pinmux { + groups = "i2c"; + function = "i2c"; + }; + }; + + mdio_pins: mdio0-pins { + pinmux { + groups = "mdio"; + function = "mdio"; + }; + }; + + nand_pins: nand0-pins { + sdhci-pinmux { + groups = "sdhci"; + function = "nand2"; + }; + + spi-pinmux { + groups = "spi"; + function = "nand1"; + }; + }; + + pcie_pins: pcie0-pins { + pinmux { + groups = "pcie"; + function = "gpio"; + }; + }; + + rgmii1_pins: rgmii1-pins { + pinmux { + groups = "rgmii1"; + function = "rgmii1"; + }; + }; + + rgmii2_pins: rgmii2-pins { + pinmux { + groups = "rgmii2"; + function = "rgmii2"; + }; + }; + + sdhci_pins: sdhci0-pins { + pinmux { + groups = "sdhci"; + function = "sdhci"; + }; + }; + + spi_pins: spi0-pins { + pinmux { + groups = "spi"; + function = "spi"; + }; + }; + + uart1_pins: uart1-pins { + pinmux { + groups = "uart1"; + function = "uart1"; + }; + }; + + uart2_pins: uart2-pins { + pinmux { + groups = "uart2"; + function = "uart2"; + }; + }; + + uart3_pins: uart3-pins { + pinmux { + groups = "uart3"; + function = "uart3"; + }; + }; + }; + palmbus: palmbus@1e000000 { compatible = "palmbus"; reg = <0x1e000000 0x100000>; @@ -202,92 +288,6 @@ }; }; - pinctrl: pinctrl { - compatible = "ralink,mt7621-pinctrl"; - - i2c_pins: i2c0-pins { - pinmux { - groups = "i2c"; - function = "i2c"; - }; - }; - - spi_pins: spi0-pins { - pinmux { - groups = "spi"; - function = "spi"; - }; - }; - - uart1_pins: uart1-pins { - pinmux { - groups = "uart1"; - function = "uart1"; - }; - }; - - uart2_pins: uart2-pins { - pinmux { - groups = "uart2"; - function = "uart2"; - }; - }; - - uart3_pins: uart3-pins { - pinmux { - groups = "uart3"; - function = "uart3"; - }; - }; - - rgmii1_pins: rgmii1-pins { - pinmux { - groups = "rgmii1"; - function = "rgmii1"; - }; - }; - - rgmii2_pins: rgmii2-pins { - pinmux { - groups = "rgmii2"; - function = "rgmii2"; - }; - }; - - mdio_pins: mdio0-pins { - pinmux { - groups = "mdio"; - function = "mdio"; - }; - }; - - pcie_pins: pcie0-pins { - pinmux { - groups = "pcie"; - function = "gpio"; - }; - }; - - nand_pins: nand0-pins { - spi-pinmux { - groups = "spi"; - function = "nand1"; - }; - - sdhci-pinmux { - groups = "sdhci"; - function = "nand2"; - }; - }; - - sdhci_pins: sdhci0-pins { - pinmux { - groups = "sdhci"; - function = "sdhci"; - }; - }; - }; - mmc: mmc@1e130000 { status = "disabled"; From 297fa85fbe96a442aa2c1f0eb062f84e59aa6d6b Mon Sep 17 00:00:00 2001 From: Justin Swartz Date: Sat, 16 Mar 2024 06:54:37 +0200 Subject: [PATCH 0166/1702] mips: dts: ralink: mt7621: reorder mmc node attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shuffle the attributes of the MMC node to meet the guidelines provided by the DTS style guide. Signed-off-by: Justin Swartz Reviewed-by: Arınç ÜNAL Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Sergio Paracuellos Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ralink/mt7621.dtsi | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi index b28aee1e4ca3f..1fbe345bd239d 100644 --- a/arch/mips/boot/dts/ralink/mt7621.dtsi +++ b/arch/mips/boot/dts/ralink/mt7621.dtsi @@ -289,29 +289,33 @@ }; mmc: mmc@1e130000 { - status = "disabled"; - compatible = "mediatek,mt7620-mmc"; reg = <0x1e130000 0x4000>; bus-width = <4>; - max-frequency = <48000000>; - cap-sd-highspeed; - cap-mmc-highspeed; - vmmc-supply = <&mmc_fixed_3v3>; - vqmmc-supply = <&mmc_fixed_1v8_io>; - disable-wp; - pinctrl-names = "default", "state_uhs"; - pinctrl-0 = <&sdhci_pins>; - pinctrl-1 = <&sdhci_pins>; + cap-mmc-highspeed; + cap-sd-highspeed; clocks = <&sysc MT7621_CLK_SHXC>, <&sysc MT7621_CLK_50M>; clock-names = "source", "hclk"; + disable-wp; + interrupt-parent = <&gic>; interrupts = ; + + max-frequency = <48000000>; + + pinctrl-names = "default", "state_uhs"; + pinctrl-0 = <&sdhci_pins>; + pinctrl-1 = <&sdhci_pins>; + + vmmc-supply = <&mmc_fixed_3v3>; + vqmmc-supply = <&mmc_fixed_1v8_io>; + + status = "disabled"; }; usb: usb@1e1c0000 { From a76a20f9e133dcd70d1cedd2705c58dbee33bb36 Mon Sep 17 00:00:00 2001 From: Justin Swartz Date: Sat, 16 Mar 2024 06:54:38 +0200 Subject: [PATCH 0167/1702] mips: dts: ralink: mt7621: reorder gic node attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reorder the attributes of the Global Interrupt Controller node to fit DTS style guidelines. Signed-off-by: Justin Swartz Reviewed-by: Arınç ÜNAL Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Sergio Paracuellos Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ralink/mt7621.dtsi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi index 1fbe345bd239d..8aa9eba686d53 100644 --- a/arch/mips/boot/dts/ralink/mt7621.dtsi +++ b/arch/mips/boot/dts/ralink/mt7621.dtsi @@ -338,15 +338,15 @@ compatible = "mti,gic"; reg = <0x1fbc0000 0x2000>; - interrupt-controller; #interrupt-cells = <3>; + interrupt-controller; mti,reserved-cpu-vectors = <7>; timer { compatible = "mti,gic-timer"; - interrupts = ; clocks = <&sysc MT7621_CLK_CPU>; + interrupts = ; }; }; From 6f04e524442ed4e258ae9c3e68eddf9c901e438c Mon Sep 17 00:00:00 2001 From: Justin Swartz Date: Sat, 16 Mar 2024 06:54:39 +0200 Subject: [PATCH 0168/1702] mips: dts: ralink: mt7621: reorder ethernet node attributes and kids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rearrange attributes and descendents declared under the ethernet node, recursively, to follow the DTS style guide. Signed-off-by: Justin Swartz Reviewed-by: Arınç ÜNAL Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Sergio Paracuellos Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ralink/mt7621.dtsi | 88 +++++++++++++++------------ 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi index 8aa9eba686d53..f6418201bc102 100644 --- a/arch/mips/boot/dts/ralink/mt7621.dtsi +++ b/arch/mips/boot/dts/ralink/mt7621.dtsi @@ -364,46 +364,22 @@ compatible = "mediatek,mt7621-eth"; reg = <0x1e100000 0x10000>; - clocks = <&sysc MT7621_CLK_FE>, <&sysc MT7621_CLK_ETH>; - clock-names = "fe", "ethif"; - #address-cells = <1>; #size-cells = <0>; - resets = <&sysc MT7621_RST_FE>, <&sysc MT7621_RST_ETH>; - reset-names = "fe", "eth"; + clock-names = "fe", "ethif"; + clocks = <&sysc MT7621_CLK_FE>, <&sysc MT7621_CLK_ETH>; interrupt-parent = <&gic>; interrupts = ; - mediatek,ethsys = <&sysc>; - pinctrl-names = "default"; pinctrl-0 = <&mdio_pins>, <&rgmii1_pins>, <&rgmii2_pins>; - gmac0: mac@0 { - compatible = "mediatek,eth-mac"; - reg = <0>; - phy-mode = "trgmii"; - - fixed-link { - speed = <1000>; - full-duplex; - pause; - }; - }; - - gmac1: mac@1 { - compatible = "mediatek,eth-mac"; - reg = <1>; - phy-mode = "rgmii"; + reset-names = "fe", "eth"; + resets = <&sysc MT7621_RST_FE>, <&sysc MT7621_RST_ETH>; - fixed-link { - speed = <1000>; - full-duplex; - pause; - }; - }; + mediatek,ethsys = <&sysc>; mdio: mdio-bus { #address-cells = <1>; @@ -412,73 +388,105 @@ switch0: switch@1f { compatible = "mediatek,mt7621"; reg = <0x1f>; - mediatek,mcm; - resets = <&sysc MT7621_RST_MCM>; - reset-names = "mcm"; - interrupt-controller; + #interrupt-cells = <1>; + interrupt-controller; interrupts = ; + reset-names = "mcm"; + resets = <&sysc MT7621_RST_MCM>; + + mediatek,mcm; + ports { #address-cells = <1>; #size-cells = <0>; port@0 { - status = "disabled"; reg = <0>; label = "swp0"; + status = "disabled"; }; port@1 { - status = "disabled"; reg = <1>; label = "swp1"; + status = "disabled"; }; port@2 { - status = "disabled"; reg = <2>; label = "swp2"; + status = "disabled"; }; port@3 { - status = "disabled"; reg = <3>; label = "swp3"; + status = "disabled"; }; port@4 { - status = "disabled"; reg = <4>; label = "swp4"; + status = "disabled"; }; port@5 { reg = <5>; + ethernet = <&gmac1>; phy-mode = "rgmii"; fixed-link { - speed = <1000>; full-duplex; pause; + speed = <1000>; }; }; port@6 { reg = <6>; + ethernet = <&gmac0>; phy-mode = "trgmii"; fixed-link { - speed = <1000>; full-duplex; pause; + speed = <1000>; }; }; }; }; }; + + gmac0: mac@0 { + compatible = "mediatek,eth-mac"; + reg = <0>; + + phy-mode = "trgmii"; + + fixed-link { + full-duplex; + pause; + speed = <1000>; + }; + }; + + gmac1: mac@1 { + compatible = "mediatek,eth-mac"; + reg = <1>; + + phy-mode = "rgmii"; + + fixed-link { + full-duplex; + pause; + speed = <1000>; + }; + }; + }; pcie: pcie@1e140000 { From fdcb4f10723b5730842bc4419be635065f8e608b Mon Sep 17 00:00:00 2001 From: Justin Swartz Date: Sat, 16 Mar 2024 06:54:40 +0200 Subject: [PATCH 0169/1702] mips: dts: ralink: mt7621: reorder pcie node attributes and children MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reorder the attributes and child nodes of the PCIe Controller node to meet the DTS style guidelines. Signed-off-by: Justin Swartz Reviewed-by: Arınç ÜNAL Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Sergio Paracuellos Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ralink/mt7621.dtsi | 68 +++++++++++++++++---------- 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi index f6418201bc102..aa06d12acacce 100644 --- a/arch/mips/boot/dts/ralink/mt7621.dtsi +++ b/arch/mips/boot/dts/ralink/mt7621.dtsi @@ -495,70 +495,88 @@ <0x1e142000 0x100>, /* pcie port 0 RC control registers */ <0x1e143000 0x100>, /* pcie port 1 RC control registers */ <0x1e144000 0x100>; /* pcie port 2 RC control registers */ + ranges = <0x02000000 0 0x60000000 0x60000000 0 0x10000000>, /* pci memory */ + <0x01000000 0 0x00000000 0x1e160000 0 0x00010000>; /* io space */ + #address-cells = <3>; + #interrupt-cells = <1>; #size-cells = <2>; - pinctrl-names = "default"; - pinctrl-0 = <&pcie_pins>; - device_type = "pci"; - ranges = <0x02000000 0 0x60000000 0x60000000 0 0x10000000>, /* pci memory */ - <0x01000000 0 0x00000000 0x1e160000 0 0x00010000>; /* io space */ - - #interrupt-cells = <1>; - interrupt-map-mask = <0xF800 0 0 0>; - interrupt-map = <0x0000 0 0 0 &gic GIC_SHARED 4 IRQ_TYPE_LEVEL_HIGH>, + interrupt-map-mask = <0xf800 0 0 0>; + interrupt-map = <0x0000 0 0 0 &gic GIC_SHARED 4 IRQ_TYPE_LEVEL_HIGH>, <0x0800 0 0 0 &gic GIC_SHARED 24 IRQ_TYPE_LEVEL_HIGH>, <0x1000 0 0 0 &gic GIC_SHARED 25 IRQ_TYPE_LEVEL_HIGH>; - status = "disabled"; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_pins>; reset-gpios = <&gpio 19 GPIO_ACTIVE_LOW>; + status = "disabled"; + pcie@0,0 { reg = <0x0000 0 0 0 0>; + ranges; + #address-cells = <3>; + #interrupt-cells = <1>; #size-cells = <2>; + + clocks = <&sysc MT7621_CLK_PCIE0>; + device_type = "pci"; - #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0>; interrupt-map = <0 0 0 0 &gic GIC_SHARED 4 IRQ_TYPE_LEVEL_HIGH>; - resets = <&sysc MT7621_RST_PCIE0>; - clocks = <&sysc MT7621_CLK_PCIE0>; - phys = <&pcie0_phy 1>; + phy-names = "pcie-phy0"; - ranges; + phys = <&pcie0_phy 1>; + + resets = <&sysc MT7621_RST_PCIE0>; }; pcie@1,0 { reg = <0x0800 0 0 0 0>; + ranges; + #address-cells = <3>; + #interrupt-cells = <1>; #size-cells = <2>; + + clocks = <&sysc MT7621_CLK_PCIE1>; + device_type = "pci"; - #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0>; interrupt-map = <0 0 0 0 &gic GIC_SHARED 24 IRQ_TYPE_LEVEL_HIGH>; - resets = <&sysc MT7621_RST_PCIE1>; - clocks = <&sysc MT7621_CLK_PCIE1>; - phys = <&pcie0_phy 1>; + phy-names = "pcie-phy1"; - ranges; + phys = <&pcie0_phy 1>; + + resets = <&sysc MT7621_RST_PCIE1>; }; pcie@2,0 { reg = <0x1000 0 0 0 0>; + ranges; + #address-cells = <3>; + #interrupt-cells = <1>; #size-cells = <2>; + + clocks = <&sysc MT7621_CLK_PCIE2>; + device_type = "pci"; - #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0>; interrupt-map = <0 0 0 0 &gic GIC_SHARED 25 IRQ_TYPE_LEVEL_HIGH>; - resets = <&sysc MT7621_RST_PCIE2>; - clocks = <&sysc MT7621_CLK_PCIE2>; - phys = <&pcie2_phy 0>; + phy-names = "pcie-phy2"; - ranges; + phys = <&pcie2_phy 0>; + + resets = <&sysc MT7621_RST_PCIE2>; }; }; From de56f781e5483fb3b3527aa280df2434f0cb2ace Mon Sep 17 00:00:00 2001 From: Justin Swartz Date: Sat, 16 Mar 2024 06:54:41 +0200 Subject: [PATCH 0170/1702] mips: dts: ralink: mt7621: reorder pci?_phy attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reorder the attributes of the PCIe PHY nodes node to match what the DTS style guide recommends. Signed-off-by: Justin Swartz Reviewed-by: Arınç ÜNAL Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Sergio Paracuellos Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ralink/mt7621.dtsi | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi index aa06d12acacce..284811f32929a 100644 --- a/arch/mips/boot/dts/ralink/mt7621.dtsi +++ b/arch/mips/boot/dts/ralink/mt7621.dtsi @@ -583,14 +583,18 @@ pcie0_phy: pcie-phy@1e149000 { compatible = "mediatek,mt7621-pci-phy"; reg = <0x1e149000 0x0700>; - clocks = <&sysc MT7621_CLK_XTAL>; + #phy-cells = <1>; + + clocks = <&sysc MT7621_CLK_XTAL>; }; pcie2_phy: pcie-phy@1e14a000 { compatible = "mediatek,mt7621-pci-phy"; reg = <0x1e14a000 0x0700>; - clocks = <&sysc MT7621_CLK_XTAL>; + #phy-cells = <1>; + + clocks = <&sysc MT7621_CLK_XTAL>; }; }; From b8f8e5a691ba75051a841e68ace5817d5c368fd9 Mon Sep 17 00:00:00 2001 From: Justin Swartz Date: Sat, 16 Mar 2024 06:54:42 +0200 Subject: [PATCH 0171/1702] mips: dts: ralink: mt7621: reorder the attributes of the root node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the compatible attribute of the DTS root node to first place. Signed-off-by: Justin Swartz Reviewed-by: Arınç ÜNAL Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Sergio Paracuellos Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ralink/mt7621.dtsi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi index 284811f32929a..0704eab4a80ba 100644 --- a/arch/mips/boot/dts/ralink/mt7621.dtsi +++ b/arch/mips/boot/dts/ralink/mt7621.dtsi @@ -5,9 +5,10 @@ #include / { + compatible = "mediatek,mt7621-soc"; + #address-cells = <1>; #size-cells = <1>; - compatible = "mediatek,mt7621-soc"; cpus { #address-cells = <1>; From 40e20fbccfb722f219ab8d3ff1edde99e4a7c46c Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sat, 13 Apr 2024 11:49:13 -0700 Subject: [PATCH 0172/1702] MIPS: SGI-IP27: micro-optimize arch_init_irq() The function sets adjasted groups of bits in hub_irq_map by using for-loops. There's a bitmap_set() function dedicated to do this. Because [0, CPU_CALL_B_IRQ] and [NI_BRDCAST_ERR_A, MSC_PANIC_INTR] ranges belong to the same machine word, bitmap_set() would boil down to an inline wrapper in both cases, avoiding generating a loop, whth the associate overhead. Signed-off-by: Yury Norov Signed-off-by: Thomas Bogendoerfer --- arch/mips/sgi-ip27/ip27-irq.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/mips/sgi-ip27/ip27-irq.c b/arch/mips/sgi-ip27/ip27-irq.c index 8f5299b269e7e..dcb14a234b1c3 100644 --- a/arch/mips/sgi-ip27/ip27-irq.c +++ b/arch/mips/sgi-ip27/ip27-irq.c @@ -286,11 +286,8 @@ void __init arch_init_irq(void) * Mark these as reserved right away so they won't be used accidentally * later. */ - for (i = 0; i <= CPU_CALL_B_IRQ; i++) - set_bit(i, hub_irq_map); - - for (i = NI_BRDCAST_ERR_A; i <= MSC_PANIC_INTR; i++) - set_bit(i, hub_irq_map); + bitmap_set(hub_irq_map, 0, CPU_CALL_B_IRQ + 1); + bitmap_set(hub_irq_map, NI_BRDCAST_ERR_A, MSC_PANIC_INTR - NI_BRDCAST_ERR_A + 1); fn = irq_domain_alloc_named_fwnode("HUB"); WARN_ON(fn == NULL); From 06c375053cefc3a2f383d200596abe5ab3fb35f9 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Sat, 13 Apr 2024 00:25:12 +0000 Subject: [PATCH 0173/1702] iommu/vt-d: add wrapper functions for page allocations In order to improve observability and accountability of IOMMU layer, we must account the number of pages that are allocated by functions that are calling directly into buddy allocator. This is achieved by first wrapping the allocation related functions into a separate inline functions in new file: drivers/iommu/iommu-pages.h Convert all page allocation calls under iommu/intel to use these new functions. Signed-off-by: Pasha Tatashin Acked-by: David Rientjes Tested-by: Bagas Sanjaya Link: https://lore.kernel.org/r/20240413002522.1101315-2-pasha.tatashin@soleen.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/dmar.c | 16 +-- drivers/iommu/intel/iommu.c | 47 +++------ drivers/iommu/intel/iommu.h | 2 - drivers/iommu/intel/irq_remapping.c | 16 +-- drivers/iommu/intel/pasid.c | 18 ++-- drivers/iommu/intel/svm.c | 11 +- drivers/iommu/iommu-pages.h | 154 ++++++++++++++++++++++++++++ 7 files changed, 201 insertions(+), 63 deletions(-) create mode 100644 drivers/iommu/iommu-pages.h diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 36d7427b12026..87ad996e5257f 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -32,6 +32,7 @@ #include "iommu.h" #include "../irq_remapping.h" +#include "../iommu-pages.h" #include "perf.h" #include "trace.h" #include "perfmon.h" @@ -1187,7 +1188,7 @@ static void free_iommu(struct intel_iommu *iommu) } if (iommu->qi) { - free_page((unsigned long)iommu->qi->desc); + iommu_free_page(iommu->qi->desc); kfree(iommu->qi->desc_status); kfree(iommu->qi); } @@ -1755,7 +1756,8 @@ static void __dmar_enable_qi(struct intel_iommu *iommu) int dmar_enable_qi(struct intel_iommu *iommu) { struct q_inval *qi; - struct page *desc_page; + void *desc; + int order; if (!ecap_qis(iommu->ecap)) return -ENOENT; @@ -1776,19 +1778,19 @@ int dmar_enable_qi(struct intel_iommu *iommu) * Need two pages to accommodate 256 descriptors of 256 bits each * if the remapping hardware supports scalable mode translation. */ - desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, - !!ecap_smts(iommu->ecap)); - if (!desc_page) { + order = ecap_smts(iommu->ecap) ? 1 : 0; + desc = iommu_alloc_pages_node(iommu->node, GFP_ATOMIC, order); + if (!desc) { kfree(qi); iommu->qi = NULL; return -ENOMEM; } - qi->desc = page_address(desc_page); + qi->desc = desc; qi->desc_status = kcalloc(QI_LENGTH, sizeof(int), GFP_ATOMIC); if (!qi->desc_status) { - free_page((unsigned long) qi->desc); + iommu_free_page(qi->desc); kfree(qi); iommu->qi = NULL; return -ENOMEM; diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index a7ecd90303dc4..daaa7a22595e3 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -27,6 +27,7 @@ #include "iommu.h" #include "../dma-iommu.h" #include "../irq_remapping.h" +#include "../iommu-pages.h" #include "pasid.h" #include "cap_audit.h" #include "perfmon.h" @@ -298,22 +299,6 @@ static int __init intel_iommu_setup(char *str) } __setup("intel_iommu=", intel_iommu_setup); -void *alloc_pgtable_page(int node, gfp_t gfp) -{ - struct page *page; - void *vaddr = NULL; - - page = alloc_pages_node(node, gfp | __GFP_ZERO, 0); - if (page) - vaddr = page_address(page); - return vaddr; -} - -void free_pgtable_page(void *vaddr) -{ - free_page((unsigned long)vaddr); -} - static int domain_type_is_si(struct dmar_domain *domain) { return domain->domain.type == IOMMU_DOMAIN_IDENTITY; @@ -545,7 +530,7 @@ struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus, if (!alloc) return NULL; - context = alloc_pgtable_page(iommu->node, GFP_ATOMIC); + context = iommu_alloc_page_node(iommu->node, GFP_ATOMIC); if (!context) return NULL; @@ -719,17 +704,17 @@ static void free_context_table(struct intel_iommu *iommu) for (i = 0; i < ROOT_ENTRY_NR; i++) { context = iommu_context_addr(iommu, i, 0, 0); if (context) - free_pgtable_page(context); + iommu_free_page(context); if (!sm_supported(iommu)) continue; context = iommu_context_addr(iommu, i, 0x80, 0); if (context) - free_pgtable_page(context); + iommu_free_page(context); } - free_pgtable_page(iommu->root_entry); + iommu_free_page(iommu->root_entry); iommu->root_entry = NULL; } @@ -867,7 +852,7 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, if (!dma_pte_present(pte)) { uint64_t pteval; - tmp_page = alloc_pgtable_page(domain->nid, gfp); + tmp_page = iommu_alloc_page_node(domain->nid, gfp); if (!tmp_page) return NULL; @@ -879,7 +864,7 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, if (cmpxchg64(&pte->val, 0ULL, pteval)) /* Someone else set it while we were thinking; use theirs. */ - free_pgtable_page(tmp_page); + iommu_free_page(tmp_page); else domain_flush_cache(domain, pte, sizeof(*pte)); } @@ -992,7 +977,7 @@ static void dma_pte_free_level(struct dmar_domain *domain, int level, last_pfn < level_pfn + level_size(level) - 1)) { dma_clear_pte(pte); domain_flush_cache(domain, pte, sizeof(*pte)); - free_pgtable_page(level_pte); + iommu_free_page(level_pte); } next: pfn += level_size(level); @@ -1016,7 +1001,7 @@ static void dma_pte_free_pagetable(struct dmar_domain *domain, /* free pgd */ if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) { - free_pgtable_page(domain->pgd); + iommu_free_page(domain->pgd); domain->pgd = NULL; } } @@ -1118,7 +1103,7 @@ static int iommu_alloc_root_entry(struct intel_iommu *iommu) { struct root_entry *root; - root = alloc_pgtable_page(iommu->node, GFP_ATOMIC); + root = iommu_alloc_page_node(iommu->node, GFP_ATOMIC); if (!root) { pr_err("Allocating root entry for %s failed\n", iommu->name); @@ -1841,7 +1826,7 @@ static void domain_exit(struct dmar_domain *domain) LIST_HEAD(freelist); domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist); - put_pages_list(&freelist); + iommu_put_pages_list(&freelist); } if (WARN_ON(!list_empty(&domain->devices))) @@ -2497,7 +2482,7 @@ static int copy_context_table(struct intel_iommu *iommu, if (!old_ce) goto out; - new_ce = alloc_pgtable_page(iommu->node, GFP_KERNEL); + new_ce = iommu_alloc_page_node(iommu->node, GFP_KERNEL); if (!new_ce) goto out_unmap; @@ -3426,7 +3411,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb, start_vpfn, mhp->nr_pages, list_empty(&freelist), 0); rcu_read_unlock(); - put_pages_list(&freelist); + iommu_put_pages_list(&freelist); } break; } @@ -3833,7 +3818,7 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width) domain->max_addr = 0; /* always allocate the top pgd */ - domain->pgd = alloc_pgtable_page(domain->nid, GFP_ATOMIC); + domain->pgd = iommu_alloc_page_node(domain->nid, GFP_ATOMIC); if (!domain->pgd) return -ENOMEM; domain_flush_cache(domain, domain->pgd, PAGE_SIZE); @@ -3987,7 +3972,7 @@ int prepare_domain_attach_device(struct iommu_domain *domain, pte = dmar_domain->pgd; if (dma_pte_present(pte)) { dmar_domain->pgd = phys_to_virt(dma_pte_addr(pte)); - free_pgtable_page(pte); + iommu_free_page(pte); } dmar_domain->agaw--; } @@ -4141,7 +4126,7 @@ static void intel_iommu_tlb_sync(struct iommu_domain *domain, if (dmar_domain->nested_parent) parent_domain_flush(dmar_domain, start_pfn, nrpages, list_empty(&gather->freelist)); - put_pages_list(&gather->freelist); + iommu_put_pages_list(&gather->freelist); } static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 404d2476a8774..8d081d8c6f41d 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1085,8 +1085,6 @@ void domain_update_iommu_cap(struct dmar_domain *domain); int dmar_ir_support(void); -void *alloc_pgtable_page(int node, gfp_t gfp); -void free_pgtable_page(void *vaddr); void iommu_flush_write_buffer(struct intel_iommu *iommu); struct iommu_domain *intel_nested_domain_alloc(struct iommu_domain *parent, const struct iommu_user_data *user_data); diff --git a/drivers/iommu/intel/irq_remapping.c b/drivers/iommu/intel/irq_remapping.c index 566297bc87ddb..39cd9626eb8d0 100644 --- a/drivers/iommu/intel/irq_remapping.c +++ b/drivers/iommu/intel/irq_remapping.c @@ -22,6 +22,7 @@ #include "iommu.h" #include "../irq_remapping.h" +#include "../iommu-pages.h" #include "cap_audit.h" enum irq_mode { @@ -527,7 +528,7 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu) struct ir_table *ir_table; struct fwnode_handle *fn; unsigned long *bitmap; - struct page *pages; + void *ir_table_base; if (iommu->ir_table) return 0; @@ -536,9 +537,9 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu) if (!ir_table) return -ENOMEM; - pages = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, - INTR_REMAP_PAGE_ORDER); - if (!pages) { + ir_table_base = iommu_alloc_pages_node(iommu->node, GFP_KERNEL, + INTR_REMAP_PAGE_ORDER); + if (!ir_table_base) { pr_err("IR%d: failed to allocate pages of order %d\n", iommu->seq_id, INTR_REMAP_PAGE_ORDER); goto out_free_table; @@ -573,7 +574,7 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu) else iommu->ir_domain->msi_parent_ops = &dmar_msi_parent_ops; - ir_table->base = page_address(pages); + ir_table->base = ir_table_base; ir_table->bitmap = bitmap; iommu->ir_table = ir_table; @@ -622,7 +623,7 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu) out_free_bitmap: bitmap_free(bitmap); out_free_pages: - __free_pages(pages, INTR_REMAP_PAGE_ORDER); + iommu_free_pages(ir_table_base, INTR_REMAP_PAGE_ORDER); out_free_table: kfree(ir_table); @@ -643,8 +644,7 @@ static void intel_teardown_irq_remapping(struct intel_iommu *iommu) irq_domain_free_fwnode(fn); iommu->ir_domain = NULL; } - free_pages((unsigned long)iommu->ir_table->base, - INTR_REMAP_PAGE_ORDER); + iommu_free_pages(iommu->ir_table->base, INTR_REMAP_PAGE_ORDER); bitmap_free(iommu->ir_table->bitmap); kfree(iommu->ir_table); iommu->ir_table = NULL; diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 11f0b856d74c0..abce19e2ad6f4 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -20,6 +20,7 @@ #include "iommu.h" #include "pasid.h" +#include "../iommu-pages.h" /* * Intel IOMMU system wide PASID name space: @@ -38,7 +39,7 @@ int intel_pasid_alloc_table(struct device *dev) { struct device_domain_info *info; struct pasid_table *pasid_table; - struct page *pages; + struct pasid_dir_entry *dir; u32 max_pasid = 0; int order, size; @@ -59,14 +60,13 @@ int intel_pasid_alloc_table(struct device *dev) size = max_pasid >> (PASID_PDE_SHIFT - 3); order = size ? get_order(size) : 0; - pages = alloc_pages_node(info->iommu->node, - GFP_KERNEL | __GFP_ZERO, order); - if (!pages) { + dir = iommu_alloc_pages_node(info->iommu->node, GFP_KERNEL, order); + if (!dir) { kfree(pasid_table); return -ENOMEM; } - pasid_table->table = page_address(pages); + pasid_table->table = dir; pasid_table->order = order; pasid_table->max_pasid = 1 << (order + PAGE_SHIFT + 3); info->pasid_table = pasid_table; @@ -97,10 +97,10 @@ void intel_pasid_free_table(struct device *dev) max_pde = pasid_table->max_pasid >> PASID_PDE_SHIFT; for (i = 0; i < max_pde; i++) { table = get_pasid_table_from_pde(&dir[i]); - free_pgtable_page(table); + iommu_free_page(table); } - free_pages((unsigned long)pasid_table->table, pasid_table->order); + iommu_free_pages(pasid_table->table, pasid_table->order); kfree(pasid_table); } @@ -146,7 +146,7 @@ static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) retry: entries = get_pasid_table_from_pde(&dir[dir_index]); if (!entries) { - entries = alloc_pgtable_page(info->iommu->node, GFP_ATOMIC); + entries = iommu_alloc_page_node(info->iommu->node, GFP_ATOMIC); if (!entries) return NULL; @@ -158,7 +158,7 @@ static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) */ if (cmpxchg64(&dir[dir_index].val, 0ULL, (u64)virt_to_phys(entries) | PASID_PTE_PRESENT)) { - free_pgtable_page(entries); + iommu_free_page(entries); goto retry; } if (!ecap_coherent(info->iommu->ecap)) { diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index ee3b469e2da15..71c1b2a0ca169 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -22,6 +22,7 @@ #include "iommu.h" #include "pasid.h" #include "perf.h" +#include "../iommu-pages.h" #include "trace.h" static irqreturn_t prq_event_thread(int irq, void *d); @@ -63,16 +64,14 @@ svm_lookup_device_by_dev(struct intel_svm *svm, struct device *dev) int intel_svm_enable_prq(struct intel_iommu *iommu) { struct iopf_queue *iopfq; - struct page *pages; int irq, ret; - pages = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, PRQ_ORDER); - if (!pages) { + iommu->prq = iommu_alloc_pages_node(iommu->node, GFP_KERNEL, PRQ_ORDER); + if (!iommu->prq) { pr_warn("IOMMU: %s: Failed to allocate page request queue\n", iommu->name); return -ENOMEM; } - iommu->prq = page_address(pages); irq = dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PRQ + iommu->seq_id, iommu->node, iommu); if (irq <= 0) { @@ -117,7 +116,7 @@ int intel_svm_enable_prq(struct intel_iommu *iommu) dmar_free_hwirq(irq); iommu->pr_irq = 0; free_prq: - free_pages((unsigned long)iommu->prq, PRQ_ORDER); + iommu_free_pages(iommu->prq, PRQ_ORDER); iommu->prq = NULL; return ret; @@ -140,7 +139,7 @@ int intel_svm_finish_prq(struct intel_iommu *iommu) iommu->iopf_queue = NULL; } - free_pages((unsigned long)iommu->prq, PRQ_ORDER); + iommu_free_pages(iommu->prq, PRQ_ORDER); iommu->prq = NULL; return 0; diff --git a/drivers/iommu/iommu-pages.h b/drivers/iommu/iommu-pages.h new file mode 100644 index 0000000000000..5a222d0ad25cc --- /dev/null +++ b/drivers/iommu/iommu-pages.h @@ -0,0 +1,154 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2024, Google LLC. + * Pasha Tatashin + */ + +#ifndef __IOMMU_PAGES_H +#define __IOMMU_PAGES_H + +#include +#include +#include + +/* + * All page allocations that should be reported to as "iommu-pagetables" to + * userspace must use one of the functions below. This includes allocations of + * page-tables and other per-iommu_domain configuration structures. + * + * This is necessary for the proper accounting as IOMMU state can be rather + * large, i.e. multiple gigabytes in size. + */ + +/** + * __iommu_alloc_pages - allocate a zeroed page of a given order. + * @gfp: buddy allocator flags + * @order: page order + * + * returns the head struct page of the allocated page. + */ +static inline struct page *__iommu_alloc_pages(gfp_t gfp, int order) +{ + struct page *page; + + page = alloc_pages(gfp | __GFP_ZERO, order); + if (unlikely(!page)) + return NULL; + + return page; +} + +/** + * __iommu_free_pages - free page of a given order + * @page: head struct page of the page + * @order: page order + */ +static inline void __iommu_free_pages(struct page *page, int order) +{ + if (!page) + return; + + __free_pages(page, order); +} + +/** + * iommu_alloc_pages_node - allocate a zeroed page of a given order from + * specific NUMA node. + * @nid: memory NUMA node id + * @gfp: buddy allocator flags + * @order: page order + * + * returns the virtual address of the allocated page + */ +static inline void *iommu_alloc_pages_node(int nid, gfp_t gfp, int order) +{ + struct page *page = alloc_pages_node(nid, gfp | __GFP_ZERO, order); + + if (unlikely(!page)) + return NULL; + + return page_address(page); +} + +/** + * iommu_alloc_pages - allocate a zeroed page of a given order + * @gfp: buddy allocator flags + * @order: page order + * + * returns the virtual address of the allocated page + */ +static inline void *iommu_alloc_pages(gfp_t gfp, int order) +{ + struct page *page = __iommu_alloc_pages(gfp, order); + + if (unlikely(!page)) + return NULL; + + return page_address(page); +} + +/** + * iommu_alloc_page_node - allocate a zeroed page at specific NUMA node. + * @nid: memory NUMA node id + * @gfp: buddy allocator flags + * + * returns the virtual address of the allocated page + */ +static inline void *iommu_alloc_page_node(int nid, gfp_t gfp) +{ + return iommu_alloc_pages_node(nid, gfp, 0); +} + +/** + * iommu_alloc_page - allocate a zeroed page + * @gfp: buddy allocator flags + * + * returns the virtual address of the allocated page + */ +static inline void *iommu_alloc_page(gfp_t gfp) +{ + return iommu_alloc_pages(gfp, 0); +} + +/** + * iommu_free_pages - free page of a given order + * @virt: virtual address of the page to be freed. + * @order: page order + */ +static inline void iommu_free_pages(void *virt, int order) +{ + if (!virt) + return; + + __iommu_free_pages(virt_to_page(virt), order); +} + +/** + * iommu_free_page - free page + * @virt: virtual address of the page to be freed. + */ +static inline void iommu_free_page(void *virt) +{ + iommu_free_pages(virt, 0); +} + +/** + * iommu_put_pages_list - free a list of pages. + * @page: the head of the lru list to be freed. + * + * There are no locking requirement for these pages, as they are going to be + * put on a free list as soon as refcount reaches 0. Pages are put on this LRU + * list once they are removed from the IOMMU page tables. However, they can + * still be access through debugfs. + */ +static inline void iommu_put_pages_list(struct list_head *page) +{ + while (!list_empty(page)) { + struct page *p = list_entry(page->prev, struct page, lru); + + list_del(&p->lru); + put_page(p); + } +} + +#endif /* __IOMMU_PAGES_H */ From 95b18ef9c69157ded5ece1136377cf8123b597f0 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Sat, 13 Apr 2024 00:25:13 +0000 Subject: [PATCH 0174/1702] iommu/dma: use iommu_put_pages_list() to releae freelist Free the IOMMU page tables via iommu_put_pages_list(). The page tables were allocated via iommu_alloc_* functions in architecture specific places, but are released in dma-iommu if the freelist is gathered during map/unmap operations into iommu_iotlb_gather data structure. Currently, only iommu/intel that does that. Signed-off-by: Pasha Tatashin Acked-by: David Rientjes Link: https://lore.kernel.org/r/20240413002522.1101315-3-pasha.tatashin@soleen.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index e4cb26f6a9434..16a7c4a4f3dba 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -32,6 +32,7 @@ #include #include "dma-iommu.h" +#include "iommu-pages.h" struct iommu_dma_msi_page { struct list_head list; @@ -156,7 +157,7 @@ static void fq_ring_free_locked(struct iommu_dma_cookie *cookie, struct iova_fq if (fq->entries[idx].counter >= counter) break; - put_pages_list(&fq->entries[idx].freelist); + iommu_put_pages_list(&fq->entries[idx].freelist); free_iova_fast(&cookie->iovad, fq->entries[idx].iova_pfn, fq->entries[idx].pages); @@ -254,7 +255,7 @@ static void iommu_dma_free_fq_single(struct iova_fq *fq) int idx; fq_ring_for_each(idx, fq) - put_pages_list(&fq->entries[idx].freelist); + iommu_put_pages_list(&fq->entries[idx].freelist); vfree(fq); } @@ -267,7 +268,7 @@ static void iommu_dma_free_fq_percpu(struct iova_fq __percpu *percpu_fq) struct iova_fq *fq = per_cpu_ptr(percpu_fq, cpu); fq_ring_for_each(idx, fq) - put_pages_list(&fq->entries[idx].freelist); + iommu_put_pages_list(&fq->entries[idx].freelist); } free_percpu(percpu_fq); From 75114cbaa136fc50de3a339c85124b20466a7c46 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Sat, 13 Apr 2024 00:25:14 +0000 Subject: [PATCH 0175/1702] iommu/amd: use page allocation function provided by iommu-pages.h Convert iommu/amd/* files to use the new page allocation functions provided in iommu-pages.h. Signed-off-by: Pasha Tatashin Acked-by: David Rientjes Tested-by: Bagas Sanjaya Link: https://lore.kernel.org/r/20240413002522.1101315-4-pasha.tatashin@soleen.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 8 --- drivers/iommu/amd/init.c | 91 ++++++++++++++----------------- drivers/iommu/amd/io_pgtable.c | 13 +++-- drivers/iommu/amd/io_pgtable_v2.c | 18 +++--- drivers/iommu/amd/iommu.c | 11 ++-- 5 files changed, 62 insertions(+), 79 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index f482aab420f78..e014090372063 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -134,14 +134,6 @@ static inline int get_pci_sbdf_id(struct pci_dev *pdev) return PCI_SEG_DEVID_TO_SBDF(seg, devid); } -static inline void *alloc_pgtable_page(int nid, gfp_t gfp) -{ - struct page *page; - - page = alloc_pages_node(nid, gfp | __GFP_ZERO, 0); - return page ? page_address(page) : NULL; -} - /* * This must be called after device probe completes. During probe * use rlookup_amd_iommu() get the iommu. diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index ac6754a85f350..40b3b9cffade3 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -36,6 +36,7 @@ #include "amd_iommu.h" #include "../irq_remapping.h" +#include "../iommu-pages.h" /* * definitions for the ACPI scanning code @@ -649,8 +650,8 @@ static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_ /* Allocate per PCI segment device table */ static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg) { - pci_seg->dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO | GFP_DMA32, - get_order(pci_seg->dev_table_size)); + pci_seg->dev_table = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32, + get_order(pci_seg->dev_table_size)); if (!pci_seg->dev_table) return -ENOMEM; @@ -659,17 +660,16 @@ static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg) static inline void free_dev_table(struct amd_iommu_pci_seg *pci_seg) { - free_pages((unsigned long)pci_seg->dev_table, - get_order(pci_seg->dev_table_size)); + iommu_free_pages(pci_seg->dev_table, + get_order(pci_seg->dev_table_size)); pci_seg->dev_table = NULL; } /* Allocate per PCI segment IOMMU rlookup table. */ static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg) { - pci_seg->rlookup_table = (void *)__get_free_pages( - GFP_KERNEL | __GFP_ZERO, - get_order(pci_seg->rlookup_table_size)); + pci_seg->rlookup_table = iommu_alloc_pages(GFP_KERNEL, + get_order(pci_seg->rlookup_table_size)); if (pci_seg->rlookup_table == NULL) return -ENOMEM; @@ -678,16 +678,15 @@ static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg) static inline void free_rlookup_table(struct amd_iommu_pci_seg *pci_seg) { - free_pages((unsigned long)pci_seg->rlookup_table, - get_order(pci_seg->rlookup_table_size)); + iommu_free_pages(pci_seg->rlookup_table, + get_order(pci_seg->rlookup_table_size)); pci_seg->rlookup_table = NULL; } static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg) { - pci_seg->irq_lookup_table = (void *)__get_free_pages( - GFP_KERNEL | __GFP_ZERO, - get_order(pci_seg->rlookup_table_size)); + pci_seg->irq_lookup_table = iommu_alloc_pages(GFP_KERNEL, + get_order(pci_seg->rlookup_table_size)); kmemleak_alloc(pci_seg->irq_lookup_table, pci_seg->rlookup_table_size, 1, GFP_KERNEL); if (pci_seg->irq_lookup_table == NULL) @@ -699,8 +698,8 @@ static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_se static inline void free_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg) { kmemleak_free(pci_seg->irq_lookup_table); - free_pages((unsigned long)pci_seg->irq_lookup_table, - get_order(pci_seg->rlookup_table_size)); + iommu_free_pages(pci_seg->irq_lookup_table, + get_order(pci_seg->rlookup_table_size)); pci_seg->irq_lookup_table = NULL; } @@ -708,8 +707,8 @@ static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg) { int i; - pci_seg->alias_table = (void *)__get_free_pages(GFP_KERNEL, - get_order(pci_seg->alias_table_size)); + pci_seg->alias_table = iommu_alloc_pages(GFP_KERNEL, + get_order(pci_seg->alias_table_size)); if (!pci_seg->alias_table) return -ENOMEM; @@ -724,8 +723,8 @@ static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg) static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg) { - free_pages((unsigned long)pci_seg->alias_table, - get_order(pci_seg->alias_table_size)); + iommu_free_pages(pci_seg->alias_table, + get_order(pci_seg->alias_table_size)); pci_seg->alias_table = NULL; } @@ -736,8 +735,8 @@ static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg) */ static int __init alloc_command_buffer(struct amd_iommu *iommu) { - iommu->cmd_buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(CMD_BUFFER_SIZE)); + iommu->cmd_buf = iommu_alloc_pages(GFP_KERNEL, + get_order(CMD_BUFFER_SIZE)); return iommu->cmd_buf ? 0 : -ENOMEM; } @@ -845,19 +844,19 @@ static void iommu_disable_command_buffer(struct amd_iommu *iommu) static void __init free_command_buffer(struct amd_iommu *iommu) { - free_pages((unsigned long)iommu->cmd_buf, get_order(CMD_BUFFER_SIZE)); + iommu_free_pages(iommu->cmd_buf, get_order(CMD_BUFFER_SIZE)); } static void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp, size_t size) { int order = get_order(size); - void *buf = (void *)__get_free_pages(gfp, order); + void *buf = iommu_alloc_pages(gfp, order); if (buf && check_feature(FEATURE_SNP) && set_memory_4k((unsigned long)buf, (1 << order))) { - free_pages((unsigned long)buf, order); + iommu_free_pages(buf, order); buf = NULL; } @@ -867,7 +866,7 @@ static void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, /* allocates the memory where the IOMMU will log its events to */ static int __init alloc_event_buffer(struct amd_iommu *iommu) { - iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, + iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL, EVT_BUFFER_SIZE); return iommu->evt_buf ? 0 : -ENOMEM; @@ -901,14 +900,13 @@ static void iommu_disable_event_buffer(struct amd_iommu *iommu) static void __init free_event_buffer(struct amd_iommu *iommu) { - free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE)); + iommu_free_pages(iommu->evt_buf, get_order(EVT_BUFFER_SIZE)); } /* allocates the memory where the IOMMU will log its events to */ static int __init alloc_ppr_log(struct amd_iommu *iommu) { - iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, - PPR_LOG_SIZE); + iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL, PPR_LOG_SIZE); return iommu->ppr_log ? 0 : -ENOMEM; } @@ -937,14 +935,14 @@ static void iommu_enable_ppr_log(struct amd_iommu *iommu) static void __init free_ppr_log(struct amd_iommu *iommu) { - free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE)); + iommu_free_pages(iommu->ppr_log, get_order(PPR_LOG_SIZE)); } static void free_ga_log(struct amd_iommu *iommu) { #ifdef CONFIG_IRQ_REMAP - free_pages((unsigned long)iommu->ga_log, get_order(GA_LOG_SIZE)); - free_pages((unsigned long)iommu->ga_log_tail, get_order(8)); + iommu_free_pages(iommu->ga_log, get_order(GA_LOG_SIZE)); + iommu_free_pages(iommu->ga_log_tail, get_order(8)); #endif } @@ -989,13 +987,11 @@ static int iommu_init_ga_log(struct amd_iommu *iommu) if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) return 0; - iommu->ga_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(GA_LOG_SIZE)); + iommu->ga_log = iommu_alloc_pages(GFP_KERNEL, get_order(GA_LOG_SIZE)); if (!iommu->ga_log) goto err_out; - iommu->ga_log_tail = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(8)); + iommu->ga_log_tail = iommu_alloc_pages(GFP_KERNEL, get_order(8)); if (!iommu->ga_log_tail) goto err_out; @@ -1008,7 +1004,7 @@ static int iommu_init_ga_log(struct amd_iommu *iommu) static int __init alloc_cwwb_sem(struct amd_iommu *iommu) { - iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, 1); + iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL, 1); return iommu->cmd_sem ? 0 : -ENOMEM; } @@ -1016,7 +1012,7 @@ static int __init alloc_cwwb_sem(struct amd_iommu *iommu) static void __init free_cwwb_sem(struct amd_iommu *iommu) { if (iommu->cmd_sem) - free_page((unsigned long)iommu->cmd_sem); + iommu_free_page((void *)iommu->cmd_sem); } static void iommu_enable_xt(struct amd_iommu *iommu) @@ -1081,7 +1077,6 @@ static bool __copy_device_table(struct amd_iommu *iommu) u32 lo, hi, devid, old_devtb_size; phys_addr_t old_devtb_phys; u16 dom_id, dte_v, irq_v; - gfp_t gfp_flag; u64 tmp; /* Each IOMMU use separate device table with the same size */ @@ -1115,9 +1110,8 @@ static bool __copy_device_table(struct amd_iommu *iommu) if (!old_devtb) return false; - gfp_flag = GFP_KERNEL | __GFP_ZERO | GFP_DMA32; - pci_seg->old_dev_tbl_cpy = (void *)__get_free_pages(gfp_flag, - get_order(pci_seg->dev_table_size)); + pci_seg->old_dev_tbl_cpy = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32, + get_order(pci_seg->dev_table_size)); if (pci_seg->old_dev_tbl_cpy == NULL) { pr_err("Failed to allocate memory for copying old device table!\n"); memunmap(old_devtb); @@ -2805,8 +2799,8 @@ static void early_enable_iommus(void) for_each_pci_segment(pci_seg) { if (pci_seg->old_dev_tbl_cpy != NULL) { - free_pages((unsigned long)pci_seg->old_dev_tbl_cpy, - get_order(pci_seg->dev_table_size)); + iommu_free_pages(pci_seg->old_dev_tbl_cpy, + get_order(pci_seg->dev_table_size)); pci_seg->old_dev_tbl_cpy = NULL; } } @@ -2819,8 +2813,8 @@ static void early_enable_iommus(void) pr_info("Copied DEV table from previous kernel.\n"); for_each_pci_segment(pci_seg) { - free_pages((unsigned long)pci_seg->dev_table, - get_order(pci_seg->dev_table_size)); + iommu_free_pages(pci_seg->dev_table, + get_order(pci_seg->dev_table_size)); pci_seg->dev_table = pci_seg->old_dev_tbl_cpy; } @@ -3022,8 +3016,8 @@ static bool __init check_ioapic_information(void) static void __init free_dma_resources(void) { - free_pages((unsigned long)amd_iommu_pd_alloc_bitmap, - get_order(MAX_DOMAIN_ID/8)); + iommu_free_pages(amd_iommu_pd_alloc_bitmap, + get_order(MAX_DOMAIN_ID / 8)); amd_iommu_pd_alloc_bitmap = NULL; free_unity_maps(); @@ -3095,9 +3089,8 @@ static int __init early_amd_iommu_init(void) /* Device table - directly used by all IOMMUs */ ret = -ENOMEM; - amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages( - GFP_KERNEL | __GFP_ZERO, - get_order(MAX_DOMAIN_ID/8)); + amd_iommu_pd_alloc_bitmap = iommu_alloc_pages(GFP_KERNEL, + get_order(MAX_DOMAIN_ID / 8)); if (amd_iommu_pd_alloc_bitmap == NULL) goto out; diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c index 2a0d1e97e52fd..9d9a7fde59e75 100644 --- a/drivers/iommu/amd/io_pgtable.c +++ b/drivers/iommu/amd/io_pgtable.c @@ -22,6 +22,7 @@ #include "amd_iommu_types.h" #include "amd_iommu.h" +#include "../iommu-pages.h" static void v1_tlb_flush_all(void *cookie) { @@ -156,7 +157,7 @@ static bool increase_address_space(struct protection_domain *domain, bool ret = true; u64 *pte; - pte = alloc_pgtable_page(domain->nid, gfp); + pte = iommu_alloc_page_node(domain->nid, gfp); if (!pte) return false; @@ -187,7 +188,7 @@ static bool increase_address_space(struct protection_domain *domain, out: spin_unlock_irqrestore(&domain->lock, flags); - free_page((unsigned long)pte); + iommu_free_page(pte); return ret; } @@ -250,7 +251,7 @@ static u64 *alloc_pte(struct protection_domain *domain, if (!IOMMU_PTE_PRESENT(__pte) || pte_level == PAGE_MODE_NONE) { - page = alloc_pgtable_page(domain->nid, gfp); + page = iommu_alloc_page_node(domain->nid, gfp); if (!page) return NULL; @@ -259,7 +260,7 @@ static u64 *alloc_pte(struct protection_domain *domain, /* pte could have been changed somewhere. */ if (!try_cmpxchg64(pte, &__pte, __npte)) - free_page((unsigned long)page); + iommu_free_page(page); else if (IOMMU_PTE_PRESENT(__pte)) *updated = true; @@ -431,7 +432,7 @@ static int iommu_v1_map_pages(struct io_pgtable_ops *ops, unsigned long iova, } /* Everything flushed out, free pages now */ - put_pages_list(&freelist); + iommu_put_pages_list(&freelist); return ret; } @@ -580,7 +581,7 @@ static void v1_free_pgtable(struct io_pgtable *iop) /* Make changes visible to IOMMUs */ amd_iommu_domain_update(dom); - put_pages_list(&freelist); + iommu_put_pages_list(&freelist); } static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) diff --git a/drivers/iommu/amd/io_pgtable_v2.c b/drivers/iommu/amd/io_pgtable_v2.c index 93489d2db4e8d..78ac37c5ccc1e 100644 --- a/drivers/iommu/amd/io_pgtable_v2.c +++ b/drivers/iommu/amd/io_pgtable_v2.c @@ -18,6 +18,7 @@ #include "amd_iommu_types.h" #include "amd_iommu.h" +#include "../iommu-pages.h" #define IOMMU_PAGE_PRESENT BIT_ULL(0) /* Is present */ #define IOMMU_PAGE_RW BIT_ULL(1) /* Writeable */ @@ -99,11 +100,6 @@ static inline int page_size_to_level(u64 pg_size) return PAGE_MODE_1_LEVEL; } -static inline void free_pgtable_page(u64 *pt) -{ - free_page((unsigned long)pt); -} - static void free_pgtable(u64 *pt, int level) { u64 *p; @@ -125,10 +121,10 @@ static void free_pgtable(u64 *pt, int level) if (level > 2) free_pgtable(p, level - 1); else - free_pgtable_page(p); + iommu_free_page(p); } - free_pgtable_page(pt); + iommu_free_page(pt); } /* Allocate page table */ @@ -156,14 +152,14 @@ static u64 *v2_alloc_pte(int nid, u64 *pgd, unsigned long iova, } if (!IOMMU_PTE_PRESENT(__pte)) { - page = alloc_pgtable_page(nid, gfp); + page = iommu_alloc_page_node(nid, gfp); if (!page) return NULL; __npte = set_pgtable_attr(page); /* pte could have been changed somewhere. */ if (cmpxchg64(pte, __pte, __npte) != __pte) - free_pgtable_page(page); + iommu_free_page(page); else if (IOMMU_PTE_PRESENT(__pte)) *updated = true; @@ -185,7 +181,7 @@ static u64 *v2_alloc_pte(int nid, u64 *pgd, unsigned long iova, if (pg_size == IOMMU_PAGE_SIZE_1G) free_pgtable(__pte, end_level - 1); else if (pg_size == IOMMU_PAGE_SIZE_2M) - free_pgtable_page(__pte); + iommu_free_page(__pte); } return pte; @@ -366,7 +362,7 @@ static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo struct protection_domain *pdom = (struct protection_domain *)cookie; int ias = IOMMU_IN_ADDR_BIT_SIZE; - pgtable->pgd = alloc_pgtable_page(pdom->nid, GFP_ATOMIC); + pgtable->pgd = iommu_alloc_page_node(pdom->nid, GFP_ATOMIC); if (!pgtable->pgd) return NULL; diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index e692217fcb280..922f5d42c18e2 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -42,6 +42,7 @@ #include "amd_iommu.h" #include "../dma-iommu.h" #include "../irq_remapping.h" +#include "../iommu-pages.h" #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) @@ -1728,7 +1729,7 @@ static void free_gcr3_tbl_level1(u64 *tbl) ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK); - free_page((unsigned long)ptr); + iommu_free_page(ptr); } } @@ -1761,7 +1762,7 @@ static void free_gcr3_table(struct gcr3_tbl_info *gcr3_info) /* Free per device domain ID */ domain_id_free(gcr3_info->domid); - free_page((unsigned long)gcr3_info->gcr3_tbl); + iommu_free_page(gcr3_info->gcr3_tbl); gcr3_info->gcr3_tbl = NULL; } @@ -1796,7 +1797,7 @@ static int setup_gcr3_table(struct gcr3_tbl_info *gcr3_info, /* Allocate per device domain ID */ gcr3_info->domid = domain_id_alloc(); - gcr3_info->gcr3_tbl = alloc_pgtable_page(nid, GFP_ATOMIC); + gcr3_info->gcr3_tbl = iommu_alloc_page_node(nid, GFP_ATOMIC); if (gcr3_info->gcr3_tbl == NULL) { domain_id_free(gcr3_info->domid); return -ENOMEM; @@ -2245,7 +2246,7 @@ static void protection_domain_free(struct protection_domain *domain) free_io_pgtable_ops(&domain->iop.iop.ops); if (domain->iop.root) - free_page((unsigned long)domain->iop.root); + iommu_free_page(domain->iop.root); if (domain->id) domain_id_free(domain->id); @@ -2260,7 +2261,7 @@ static int protection_domain_init_v1(struct protection_domain *domain, int mode) BUG_ON(mode < PAGE_MODE_NONE || mode > PAGE_MODE_6_LEVEL); if (mode != PAGE_MODE_NONE) { - pt_root = (void *)get_zeroed_page(GFP_KERNEL); + pt_root = iommu_alloc_page(GFP_KERNEL); if (!pt_root) return -ENOMEM; } From 9a3dd4c1ee7a183229163320eb38f15b17342f78 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Sat, 13 Apr 2024 00:25:15 +0000 Subject: [PATCH 0176/1702] iommu/io-pgtable-arm: use page allocation function provided by iommu-pages.h Convert iommu/io-pgtable-arm.c to use the new page allocation functions provided in iommu-pages.h. Signed-off-by: Pasha Tatashin Acked-by: David Rientjes Tested-by: Bagas Sanjaya Link: https://lore.kernel.org/r/20240413002522.1101315-5-pasha.tatashin@soleen.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable-arm.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index f7828a7aad410..3d23b924cec16 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -21,6 +21,7 @@ #include #include "io-pgtable-arm.h" +#include "iommu-pages.h" #define ARM_LPAE_MAX_ADDR_BITS 52 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 @@ -198,14 +199,10 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, VM_BUG_ON((gfp & __GFP_HIGHMEM)); - if (cfg->alloc) { + if (cfg->alloc) pages = cfg->alloc(cookie, size, gfp); - } else { - struct page *p; - - p = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, order); - pages = p ? page_address(p) : NULL; - } + else + pages = iommu_alloc_pages_node(dev_to_node(dev), gfp, order); if (!pages) return NULL; @@ -233,7 +230,7 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, if (cfg->free) cfg->free(cookie, pages, size); else - free_pages((unsigned long)pages, order); + iommu_free_pages(pages, order); return NULL; } @@ -249,7 +246,7 @@ static void __arm_lpae_free_pages(void *pages, size_t size, if (cfg->free) cfg->free(cookie, pages, size); else - free_pages((unsigned long)pages, get_order(size)); + iommu_free_pages(pages, get_order(size)); } static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries, From 4a0b77e7c899d9a64b597ae8c9624a066e95f555 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Sat, 13 Apr 2024 00:25:16 +0000 Subject: [PATCH 0177/1702] iommu/io-pgtable-dart: use page allocation function provided by iommu-pages.h Convert iommu/io-pgtable-dart.c to use the new page allocation functions provided in iommu-pages.h., and remove unnecessary struct io_pgtable_cfg argument from __dart_alloc_pages(). Signed-off-by: Pasha Tatashin Reviewed-by: Janne Grunau Acked-by: David Rientjes Tested-by: Bagas Sanjaya Link: https://lore.kernel.org/r/20240413002522.1101315-6-pasha.tatashin@soleen.com Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable-dart.c | 37 +++++++++++++-------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/drivers/iommu/io-pgtable-dart.c b/drivers/iommu/io-pgtable-dart.c index 74b1ef2b96bee..ad28031e1e93d 100644 --- a/drivers/iommu/io-pgtable-dart.c +++ b/drivers/iommu/io-pgtable-dart.c @@ -23,6 +23,7 @@ #include #include +#include "iommu-pages.h" #define DART1_MAX_ADDR_BITS 36 @@ -106,18 +107,12 @@ static phys_addr_t iopte_to_paddr(dart_iopte pte, return paddr; } -static void *__dart_alloc_pages(size_t size, gfp_t gfp, - struct io_pgtable_cfg *cfg) +static void *__dart_alloc_pages(size_t size, gfp_t gfp) { int order = get_order(size); - struct page *p; VM_BUG_ON((gfp & __GFP_HIGHMEM)); - p = alloc_pages(gfp | __GFP_ZERO, order); - if (!p) - return NULL; - - return page_address(p); + return iommu_alloc_pages(gfp, order); } static int dart_init_pte(struct dart_io_pgtable *data, @@ -262,13 +257,13 @@ static int dart_map_pages(struct io_pgtable_ops *ops, unsigned long iova, /* no L2 table present */ if (!pte) { - cptep = __dart_alloc_pages(tblsz, gfp, cfg); + cptep = __dart_alloc_pages(tblsz, gfp); if (!cptep) return -ENOMEM; pte = dart_install_table(cptep, ptep, 0, data); if (pte) - free_pages((unsigned long)cptep, get_order(tblsz)); + iommu_free_pages(cptep, get_order(tblsz)); /* L2 table is present (now) */ pte = READ_ONCE(*ptep); @@ -419,8 +414,7 @@ apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) cfg->apple_dart_cfg.n_ttbrs = 1 << data->tbl_bits; for (i = 0; i < cfg->apple_dart_cfg.n_ttbrs; ++i) { - data->pgd[i] = __dart_alloc_pages(DART_GRANULE(data), GFP_KERNEL, - cfg); + data->pgd[i] = __dart_alloc_pages(DART_GRANULE(data), GFP_KERNEL); if (!data->pgd[i]) goto out_free_data; cfg->apple_dart_cfg.ttbr[i] = virt_to_phys(data->pgd[i]); @@ -429,9 +423,10 @@ apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) return &data->iop; out_free_data: - while (--i >= 0) - free_pages((unsigned long)data->pgd[i], - get_order(DART_GRANULE(data))); + while (--i >= 0) { + iommu_free_pages(data->pgd[i], + get_order(DART_GRANULE(data))); + } kfree(data); return NULL; } @@ -439,6 +434,7 @@ apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) static void apple_dart_free_pgtable(struct io_pgtable *iop) { struct dart_io_pgtable *data = io_pgtable_to_data(iop); + int order = get_order(DART_GRANULE(data)); dart_iopte *ptep, *end; int i; @@ -449,15 +445,10 @@ static void apple_dart_free_pgtable(struct io_pgtable *iop) while (ptep != end) { dart_iopte pte = *ptep++; - if (pte) { - unsigned long page = - (unsigned long)iopte_deref(pte, data); - - free_pages(page, get_order(DART_GRANULE(data))); - } + if (pte) + iommu_free_pages(iopte_deref(pte, data), order); } - free_pages((unsigned long)data->pgd[i], - get_order(DART_GRANULE(data))); + iommu_free_pages(data->pgd[i], order); } kfree(data); From fe046f1bf820ab721fbd49aba6e5db39329c9eaa Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Sat, 13 Apr 2024 00:25:17 +0000 Subject: [PATCH 0178/1702] iommu/exynos: use page allocation function provided by iommu-pages.h Convert iommu/exynos-iommu.c to use the new page allocation functions provided in iommu-pages.h. Signed-off-by: Pasha Tatashin Acked-by: David Rientjes Acked-by: Marek Szyprowski Tested-by: Bagas Sanjaya Link: https://lore.kernel.org/r/20240413002522.1101315-7-pasha.tatashin@soleen.com Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index d98c9161948a2..c666ecab955d2 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -22,6 +22,8 @@ #include #include +#include "iommu-pages.h" + typedef u32 sysmmu_iova_t; typedef u32 sysmmu_pte_t; static struct iommu_domain exynos_identity_domain; @@ -900,11 +902,11 @@ static struct iommu_domain *exynos_iommu_domain_alloc_paging(struct device *dev) if (!domain) return NULL; - domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); + domain->pgtable = iommu_alloc_pages(GFP_KERNEL, 2); if (!domain->pgtable) goto err_pgtable; - domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); + domain->lv2entcnt = iommu_alloc_pages(GFP_KERNEL, 1); if (!domain->lv2entcnt) goto err_counter; @@ -930,9 +932,9 @@ static struct iommu_domain *exynos_iommu_domain_alloc_paging(struct device *dev) return &domain->domain; err_lv2ent: - free_pages((unsigned long)domain->lv2entcnt, 1); + iommu_free_pages(domain->lv2entcnt, 1); err_counter: - free_pages((unsigned long)domain->pgtable, 2); + iommu_free_pages(domain->pgtable, 2); err_pgtable: kfree(domain); return NULL; @@ -973,8 +975,8 @@ static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain) phys_to_virt(base)); } - free_pages((unsigned long)domain->pgtable, 2); - free_pages((unsigned long)domain->lv2entcnt, 1); + iommu_free_pages(domain->pgtable, 2); + iommu_free_pages(domain->lv2entcnt, 1); kfree(domain); } From 5404ccaaff357d2d8d40e1a879446da9c9da5879 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Sat, 13 Apr 2024 00:25:18 +0000 Subject: [PATCH 0179/1702] iommu/rockchip: use page allocation function provided by iommu-pages.h Convert iommu/rockchip-iommu.c to use the new page allocation functions provided in iommu-pages.h. Signed-off-by: Pasha Tatashin Acked-by: David Rientjes Tested-by: Bagas Sanjaya Link: https://lore.kernel.org/r/20240413002522.1101315-8-pasha.tatashin@soleen.com Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index da79d9f4cf637..4b369419b32ce 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -26,6 +26,8 @@ #include #include +#include "iommu-pages.h" + /** MMU register offsets */ #define RK_MMU_DTE_ADDR 0x00 /* Directory table address */ #define RK_MMU_STATUS 0x04 @@ -727,14 +729,14 @@ static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, if (rk_dte_is_pt_valid(dte)) goto done; - page_table = (u32 *)get_zeroed_page(GFP_ATOMIC | rk_ops->gfp_flags); + page_table = iommu_alloc_page(GFP_ATOMIC | rk_ops->gfp_flags); if (!page_table) return ERR_PTR(-ENOMEM); pt_dma = dma_map_single(dma_dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE); if (dma_mapping_error(dma_dev, pt_dma)) { dev_err(dma_dev, "DMA mapping error while allocating page table\n"); - free_page((unsigned long)page_table); + iommu_free_page(page_table); return ERR_PTR(-ENOMEM); } @@ -1061,7 +1063,7 @@ static struct iommu_domain *rk_iommu_domain_alloc_paging(struct device *dev) * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. * Allocate one 4 KiB page for each table. */ - rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | rk_ops->gfp_flags); + rk_domain->dt = iommu_alloc_page(GFP_KERNEL | rk_ops->gfp_flags); if (!rk_domain->dt) goto err_free_domain; @@ -1083,7 +1085,7 @@ static struct iommu_domain *rk_iommu_domain_alloc_paging(struct device *dev) return &rk_domain->domain; err_free_dt: - free_page((unsigned long)rk_domain->dt); + iommu_free_page(rk_domain->dt); err_free_domain: kfree(rk_domain); @@ -1104,13 +1106,13 @@ static void rk_iommu_domain_free(struct iommu_domain *domain) u32 *page_table = phys_to_virt(pt_phys); dma_unmap_single(dma_dev, pt_phys, SPAGE_SIZE, DMA_TO_DEVICE); - free_page((unsigned long)page_table); + iommu_free_page(page_table); } } dma_unmap_single(dma_dev, rk_domain->dt_dma, SPAGE_SIZE, DMA_TO_DEVICE); - free_page((unsigned long)rk_domain->dt); + iommu_free_page(rk_domain->dt); kfree(rk_domain); } From cb06b259e166e69eaa07b348202a6205346e2791 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Sat, 13 Apr 2024 00:25:19 +0000 Subject: [PATCH 0180/1702] iommu/sun50i: use page allocation function provided by iommu-pages.h Convert iommu/sun50i-iommu.c to use the new page allocation functions provided in iommu-pages.h. Signed-off-by: Pasha Tatashin Acked-by: David Rientjes Acked-by: Jernej Skrabec Tested-by: Bagas Sanjaya Link: https://lore.kernel.org/r/20240413002522.1101315-9-pasha.tatashin@soleen.com Signed-off-by: Joerg Roedel --- drivers/iommu/sun50i-iommu.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c index decd52cba998a..c519b991749d7 100644 --- a/drivers/iommu/sun50i-iommu.c +++ b/drivers/iommu/sun50i-iommu.c @@ -26,6 +26,8 @@ #include #include +#include "iommu-pages.h" + #define IOMMU_RESET_REG 0x010 #define IOMMU_RESET_RELEASE_ALL 0xffffffff #define IOMMU_ENABLE_REG 0x020 @@ -679,8 +681,7 @@ sun50i_iommu_domain_alloc_paging(struct device *dev) if (!sun50i_domain) return NULL; - sun50i_domain->dt = (u32 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(DT_SIZE)); + sun50i_domain->dt = iommu_alloc_pages(GFP_KERNEL, get_order(DT_SIZE)); if (!sun50i_domain->dt) goto err_free_domain; @@ -702,7 +703,7 @@ static void sun50i_iommu_domain_free(struct iommu_domain *domain) { struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); - free_pages((unsigned long)sun50i_domain->dt, get_order(DT_SIZE)); + iommu_free_pages(sun50i_domain->dt, get_order(DT_SIZE)); sun50i_domain->dt = NULL; kfree(sun50i_domain); From 8e8b4ac5b0ab759fffcd4d802ab3e084e3609191 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Sat, 13 Apr 2024 00:25:20 +0000 Subject: [PATCH 0181/1702] iommu/tegra-smmu: use page allocation function provided by iommu-pages.h Convert iommu/tegra-smmu.c to use the new page allocation functions provided in iommu-pages.h. Signed-off-by: Pasha Tatashin Acked-by: David Rientjes Acked-by: Thierry Reding Tested-by: Bagas Sanjaya Link: https://lore.kernel.org/r/20240413002522.1101315-10-pasha.tatashin@soleen.com Signed-off-by: Joerg Roedel --- drivers/iommu/tegra-smmu.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 14e525bd0d9bb..f86c7ae91814f 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -19,6 +19,8 @@ #include #include +#include "iommu-pages.h" + struct tegra_smmu_group { struct list_head list; struct tegra_smmu *smmu; @@ -282,7 +284,7 @@ static struct iommu_domain *tegra_smmu_domain_alloc_paging(struct device *dev) as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE; - as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO); + as->pd = __iommu_alloc_pages(GFP_KERNEL | __GFP_DMA, 0); if (!as->pd) { kfree(as); return NULL; @@ -290,7 +292,7 @@ static struct iommu_domain *tegra_smmu_domain_alloc_paging(struct device *dev) as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL); if (!as->count) { - __free_page(as->pd); + __iommu_free_pages(as->pd, 0); kfree(as); return NULL; } @@ -298,7 +300,7 @@ static struct iommu_domain *tegra_smmu_domain_alloc_paging(struct device *dev) as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL); if (!as->pts) { kfree(as->count); - __free_page(as->pd); + __iommu_free_pages(as->pd, 0); kfree(as); return NULL; } @@ -599,14 +601,14 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT, DMA_TO_DEVICE); if (dma_mapping_error(smmu->dev, dma)) { - __free_page(page); + __iommu_free_pages(page, 0); return NULL; } if (!smmu_dma_addr_valid(smmu, dma)) { dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT, DMA_TO_DEVICE); - __free_page(page); + __iommu_free_pages(page, 0); return NULL; } @@ -649,7 +651,7 @@ static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) tegra_smmu_set_pde(as, iova, 0); dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE); - __free_page(page); + __iommu_free_pages(page, 0); as->pts[pde] = NULL; } } @@ -688,7 +690,7 @@ static struct page *as_get_pde_page(struct tegra_smmu_as *as, if (gfpflags_allow_blocking(gfp)) spin_unlock_irqrestore(&as->lock, *flags); - page = alloc_page(gfp | __GFP_DMA | __GFP_ZERO); + page = __iommu_alloc_pages(gfp | __GFP_DMA, 0); if (gfpflags_allow_blocking(gfp)) spin_lock_irqsave(&as->lock, *flags); @@ -700,7 +702,7 @@ static struct page *as_get_pde_page(struct tegra_smmu_as *as, */ if (as->pts[pde]) { if (page) - __free_page(page); + __iommu_free_pages(page, 0); page = as->pts[pde]; } From bd3520a93a84cd8c3897283e5891a9106fcf5acc Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Sat, 13 Apr 2024 00:25:21 +0000 Subject: [PATCH 0182/1702] iommu: observability of the IOMMU allocations Add NR_IOMMU_PAGES into node_stat_item that counts number of pages that are allocated by the IOMMU subsystem. The allocations can be view per-node via: /sys/devices/system/node/nodeN/vmstat. For example: $ grep iommu /sys/devices/system/node/node*/vmstat /sys/devices/system/node/node0/vmstat:nr_iommu_pages 106025 /sys/devices/system/node/node1/vmstat:nr_iommu_pages 3464 The value is in page-count, therefore, in the above example the iommu allocations amount to ~428M. Signed-off-by: Pasha Tatashin Acked-by: David Rientjes Tested-by: Bagas Sanjaya Link: https://lore.kernel.org/r/20240413002522.1101315-11-pasha.tatashin@soleen.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu-pages.h | 30 ++++++++++++++++++++++++++++++ include/linux/mmzone.h | 3 +++ mm/vmstat.c | 3 +++ 3 files changed, 36 insertions(+) diff --git a/drivers/iommu/iommu-pages.h b/drivers/iommu/iommu-pages.h index 5a222d0ad25cc..1264b0f6b6c3c 100644 --- a/drivers/iommu/iommu-pages.h +++ b/drivers/iommu/iommu-pages.h @@ -20,6 +20,30 @@ * large, i.e. multiple gigabytes in size. */ +/** + * __iommu_alloc_account - account for newly allocated page. + * @page: head struct page of the page. + * @order: order of the page + */ +static inline void __iommu_alloc_account(struct page *page, int order) +{ + const long pgcnt = 1l << order; + + mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt); +} + +/** + * __iommu_free_account - account a page that is about to be freed. + * @page: head struct page of the page. + * @order: order of the page + */ +static inline void __iommu_free_account(struct page *page, int order) +{ + const long pgcnt = 1l << order; + + mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt); +} + /** * __iommu_alloc_pages - allocate a zeroed page of a given order. * @gfp: buddy allocator flags @@ -35,6 +59,8 @@ static inline struct page *__iommu_alloc_pages(gfp_t gfp, int order) if (unlikely(!page)) return NULL; + __iommu_alloc_account(page, order); + return page; } @@ -48,6 +74,7 @@ static inline void __iommu_free_pages(struct page *page, int order) if (!page) return; + __iommu_free_account(page, order); __free_pages(page, order); } @@ -67,6 +94,8 @@ static inline void *iommu_alloc_pages_node(int nid, gfp_t gfp, int order) if (unlikely(!page)) return NULL; + __iommu_alloc_account(page, order); + return page_address(page); } @@ -147,6 +176,7 @@ static inline void iommu_put_pages_list(struct list_head *page) struct page *p = list_entry(page->prev, struct page, lru); list_del(&p->lru); + __iommu_free_account(p, 0); put_page(p); } } diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index c11b7cde81efa..be17195f6f868 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -206,6 +206,9 @@ enum node_stat_item { #endif NR_PAGETABLE, /* used for pagetables */ NR_SECONDARY_PAGETABLE, /* secondary pagetables, e.g. KVM pagetables */ +#ifdef CONFIG_IOMMU_SUPPORT + NR_IOMMU_PAGES, /* # of pages allocated by IOMMU */ +#endif #ifdef CONFIG_SWAP NR_SWAPCACHE, #endif diff --git a/mm/vmstat.c b/mm/vmstat.c index db79935e4a543..8507c497218b8 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1242,6 +1242,9 @@ const char * const vmstat_text[] = { #endif "nr_page_table_pages", "nr_sec_page_table_pages", +#ifdef CONFIG_IOMMU_SUPPORT + "nr_iommu_pages", +#endif #ifdef CONFIG_SWAP "nr_swapcached", #endif From 212c5c078d83d780cf2873ca931df135771e8bb7 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Sat, 13 Apr 2024 00:25:22 +0000 Subject: [PATCH 0183/1702] iommu: account IOMMU allocated memory In order to be able to limit the amount of memory that is allocated by IOMMU subsystem, the memory must be accounted. Account IOMMU as part of the secondary pagetables as it was discussed at LPC. The value of SecPageTables now contains mmeory allocation by IOMMU and KVM. There is a difference between GFP_ACCOUNT and what NR_IOMMU_PAGES shows. GFP_ACCOUNT is set only where it makes sense to charge to user processes, i.e. IOMMU Page Tables, but there more IOMMU shared data that should not really be charged to a specific process. Signed-off-by: Pasha Tatashin Acked-by: David Rientjes Tested-by: Bagas Sanjaya Link: https://lore.kernel.org/r/20240413002522.1101315-12-pasha.tatashin@soleen.com Signed-off-by: Joerg Roedel --- Documentation/admin-guide/cgroup-v2.rst | 2 +- Documentation/filesystems/proc.rst | 4 ++-- drivers/iommu/iommu-pages.h | 2 ++ include/linux/mmzone.h | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 17e6e95651564..15f80fea8df76 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1432,7 +1432,7 @@ PAGE_SIZE multiple when read back. sec_pagetables Amount of memory allocated for secondary page tables, this currently includes KVM mmu allocations on x86 - and arm64. + and arm64 and IOMMU page tables. percpu (npn) Amount of memory used for storing per-cpu kernel diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst index c6a6b9df21049..707e39280a9a7 100644 --- a/Documentation/filesystems/proc.rst +++ b/Documentation/filesystems/proc.rst @@ -1110,8 +1110,8 @@ KernelStack PageTables Memory consumed by userspace page tables SecPageTables - Memory consumed by secondary page tables, this currently - currently includes KVM mmu allocations on x86 and arm64. + Memory consumed by secondary page tables, this currently includes + KVM mmu and IOMMU allocations on x86 and arm64. NFS_Unstable Always zero. Previous counted pages which had been written to the server, but has not been committed to stable storage. diff --git a/drivers/iommu/iommu-pages.h b/drivers/iommu/iommu-pages.h index 1264b0f6b6c3c..82ebf00330811 100644 --- a/drivers/iommu/iommu-pages.h +++ b/drivers/iommu/iommu-pages.h @@ -30,6 +30,7 @@ static inline void __iommu_alloc_account(struct page *page, int order) const long pgcnt = 1l << order; mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt); + mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt); } /** @@ -42,6 +43,7 @@ static inline void __iommu_free_account(struct page *page, int order) const long pgcnt = 1l << order; mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt); + mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt); } /** diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index be17195f6f868..8f9c9590a42cf 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -205,7 +205,7 @@ enum node_stat_item { NR_KERNEL_SCS_KB, /* measured in KiB */ #endif NR_PAGETABLE, /* used for pagetables */ - NR_SECONDARY_PAGETABLE, /* secondary pagetables, e.g. KVM pagetables */ + NR_SECONDARY_PAGETABLE, /* secondary pagetables, KVM & IOMMU */ #ifdef CONFIG_IOMMU_SUPPORT NR_IOMMU_PAGES, /* # of pages allocated by IOMMU */ #endif From 1c5e3d9bf33b811e1c6dd9081b322004acc4a1fd Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Tue, 9 Apr 2024 13:59:39 -0500 Subject: [PATCH 0184/1702] of: Add a helper to free property struct Freeing a property struct is 3 kfree()'s which is duplicated in multiple spots. Add a helper, __of_prop_free(), and replace all the open coded cases in the DT code. Reviewed-by: Saravana Kannan Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20240409-dt-cleanup-free-v2-1-5b419a4af38d@kernel.org Signed-off-by: Rob Herring --- drivers/of/dynamic.c | 26 ++++++++++++-------------- drivers/of/of_private.h | 1 + drivers/of/overlay.c | 11 +++-------- drivers/of/unittest.c | 12 +++--------- 4 files changed, 19 insertions(+), 31 deletions(-) diff --git a/drivers/of/dynamic.c b/drivers/of/dynamic.c index 3bf27052832f3..af7c57a7a25d7 100644 --- a/drivers/of/dynamic.c +++ b/drivers/of/dynamic.c @@ -305,15 +305,20 @@ int of_detach_node(struct device_node *np) } EXPORT_SYMBOL_GPL(of_detach_node); +void __of_prop_free(struct property *prop) +{ + kfree(prop->name); + kfree(prop->value); + kfree(prop); +} + static void property_list_free(struct property *prop_list) { struct property *prop, *next; for (prop = prop_list; prop != NULL; prop = next) { next = prop->next; - kfree(prop->name); - kfree(prop->value); - kfree(prop); + __of_prop_free(prop); } } @@ -426,9 +431,7 @@ struct property *__of_prop_dup(const struct property *prop, gfp_t allocflags) return new; err_free: - kfree(new->name); - kfree(new->value); - kfree(new); + __of_prop_free(new); return NULL; } @@ -470,9 +473,7 @@ struct device_node *__of_node_dup(const struct device_node *np, if (!new_pp) goto err_prop; if (__of_add_property(node, new_pp)) { - kfree(new_pp->name); - kfree(new_pp->value); - kfree(new_pp); + __of_prop_free(new_pp); goto err_prop; } } @@ -921,11 +922,8 @@ static int of_changeset_add_prop_helper(struct of_changeset *ocs, return -ENOMEM; ret = of_changeset_add_property(ocs, np, new_pp); - if (ret) { - kfree(new_pp->name); - kfree(new_pp->value); - kfree(new_pp); - } + if (ret) + __of_prop_free(new_pp); return ret; } diff --git a/drivers/of/of_private.h b/drivers/of/of_private.h index 485483524b7f1..94fc0aa07af9e 100644 --- a/drivers/of/of_private.h +++ b/drivers/of/of_private.h @@ -123,6 +123,7 @@ extern void *__unflatten_device_tree(const void *blob, * own the devtree lock or work on detached trees only. */ struct property *__of_prop_dup(const struct property *prop, gfp_t allocflags); +void __of_prop_free(struct property *prop); struct device_node *__of_node_dup(const struct device_node *np, const char *full_name); diff --git a/drivers/of/overlay.c b/drivers/of/overlay.c index 2ae7e9d24a645..4d861a75d6946 100644 --- a/drivers/of/overlay.c +++ b/drivers/of/overlay.c @@ -262,9 +262,7 @@ static struct property *dup_and_fixup_symbol_prop( return new_prop; err_free_new_prop: - kfree(new_prop->name); - kfree(new_prop->value); - kfree(new_prop); + __of_prop_free(new_prop); err_free_target_path: kfree(target_path); @@ -361,11 +359,8 @@ static int add_changeset_property(struct overlay_changeset *ovcs, pr_err("WARNING: memory leak will occur if overlay removed, property: %pOF/%s\n", target->np, new_prop->name); - if (ret) { - kfree(new_prop->name); - kfree(new_prop->value); - kfree(new_prop); - } + if (ret) + __of_prop_free(new_prop); return ret; } diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c index 6b5c36b6a7586..a8c01c953a295 100644 --- a/drivers/of/unittest.c +++ b/drivers/of/unittest.c @@ -795,15 +795,11 @@ static void __init of_unittest_property_copy(void) new = __of_prop_dup(&p1, GFP_KERNEL); unittest(new && propcmp(&p1, new), "empty property didn't copy correctly\n"); - kfree(new->value); - kfree(new->name); - kfree(new); + __of_prop_free(new); new = __of_prop_dup(&p2, GFP_KERNEL); unittest(new && propcmp(&p2, new), "non-empty property didn't copy correctly\n"); - kfree(new->value); - kfree(new->name); - kfree(new); + __of_prop_free(new); #endif } @@ -3718,9 +3714,7 @@ static __init void of_unittest_overlay_high_level(void) goto err_unlock; } if (__of_add_property(of_symbols, new_prop)) { - kfree(new_prop->name); - kfree(new_prop->value); - kfree(new_prop); + __of_prop_free(new_prop); /* "name" auto-generated by unflatten */ if (!strcmp(prop->name, "name")) continue; From 40b0f17453fca50165c9d56401be663448e9136c Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Tue, 9 Apr 2024 13:59:40 -0500 Subject: [PATCH 0185/1702] of: Use scope based kfree() cleanups Use the relatively new scope based kfree() cleanup to simplify error handling. Doing so reduces the chances of memory leaks and simplifies error paths by avoiding the need for goto statements. Reviewed-by: Saravana Kannan Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20240409-dt-cleanup-free-v2-2-5b419a4af38d@kernel.org Signed-off-by: Rob Herring --- drivers/of/base.c | 34 ++++++++-------------------------- drivers/of/dynamic.c | 11 ++++------- drivers/of/resolver.c | 35 +++++++++++++---------------------- 3 files changed, 25 insertions(+), 55 deletions(-) diff --git a/drivers/of/base.c b/drivers/of/base.c index 8856c67c466ac..20603d3c9931b 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -16,6 +16,7 @@ #define pr_fmt(fmt) "OF: " fmt +#include #include #include #include @@ -1393,8 +1394,10 @@ int of_parse_phandle_with_args_map(const struct device_node *np, const char *stem_name, int index, struct of_phandle_args *out_args) { - char *cells_name, *map_name = NULL, *mask_name = NULL; - char *pass_name = NULL; + char *cells_name __free(kfree) = kasprintf(GFP_KERNEL, "#%s-cells", stem_name); + char *map_name __free(kfree) = kasprintf(GFP_KERNEL, "%s-map", stem_name); + char *mask_name __free(kfree) = kasprintf(GFP_KERNEL, "%s-map-mask", stem_name); + char *pass_name __free(kfree) = kasprintf(GFP_KERNEL, "%s-map-pass-thru", stem_name); struct device_node *cur, *new = NULL; const __be32 *map, *mask, *pass; static const __be32 dummy_mask[] = { [0 ... MAX_PHANDLE_ARGS] = cpu_to_be32(~0) }; @@ -1407,27 +1410,13 @@ int of_parse_phandle_with_args_map(const struct device_node *np, if (index < 0) return -EINVAL; - cells_name = kasprintf(GFP_KERNEL, "#%s-cells", stem_name); - if (!cells_name) + if (!cells_name || !map_name || !mask_name || !pass_name) return -ENOMEM; - ret = -ENOMEM; - map_name = kasprintf(GFP_KERNEL, "%s-map", stem_name); - if (!map_name) - goto free; - - mask_name = kasprintf(GFP_KERNEL, "%s-map-mask", stem_name); - if (!mask_name) - goto free; - - pass_name = kasprintf(GFP_KERNEL, "%s-map-pass-thru", stem_name); - if (!pass_name) - goto free; - ret = __of_parse_phandle_with_args(np, list_name, cells_name, -1, index, out_args); if (ret) - goto free; + return ret; /* Get the #-cells property */ cur = out_args->np; @@ -1444,8 +1433,7 @@ int of_parse_phandle_with_args_map(const struct device_node *np, /* Get the -map property */ map = of_get_property(cur, map_name, &map_len); if (!map) { - ret = 0; - goto free; + return 0; } map_len /= sizeof(u32); @@ -1521,12 +1509,6 @@ int of_parse_phandle_with_args_map(const struct device_node *np, put: of_node_put(cur); of_node_put(new); -free: - kfree(mask_name); - kfree(map_name); - kfree(cells_name); - kfree(pass_name); - return ret; } EXPORT_SYMBOL(of_parse_phandle_with_args_map); diff --git a/drivers/of/dynamic.c b/drivers/of/dynamic.c index af7c57a7a25d7..43f4e2c93bd25 100644 --- a/drivers/of/dynamic.c +++ b/drivers/of/dynamic.c @@ -9,6 +9,7 @@ #define pr_fmt(fmt) "OF: " fmt +#include #include #include #include @@ -1019,10 +1020,9 @@ int of_changeset_add_prop_u32_array(struct of_changeset *ocs, const u32 *array, size_t sz) { struct property prop; - __be32 *val; - int i, ret; + __be32 *val __free(kfree) = kcalloc(sz, sizeof(__be32), GFP_KERNEL); + int i; - val = kcalloc(sz, sizeof(__be32), GFP_KERNEL); if (!val) return -ENOMEM; @@ -1032,9 +1032,6 @@ int of_changeset_add_prop_u32_array(struct of_changeset *ocs, prop.length = sizeof(u32) * sz; prop.value = (void *)val; - ret = of_changeset_add_prop_helper(ocs, np, &prop); - kfree(val); - - return ret; + return of_changeset_add_prop_helper(ocs, np, &prop); } EXPORT_SYMBOL_GPL(of_changeset_add_prop_u32_array); diff --git a/drivers/of/resolver.c b/drivers/of/resolver.c index b278ab4338ceb..2780928764a4f 100644 --- a/drivers/of/resolver.c +++ b/drivers/of/resolver.c @@ -8,6 +8,7 @@ #define pr_fmt(fmt) "OF: resolver: " fmt +#include #include #include #include @@ -74,11 +75,11 @@ static int update_usages_of_a_phandle_reference(struct device_node *overlay, { struct device_node *refnode; struct property *prop; - char *value, *cur, *end, *node_path, *prop_name, *s; + char *value __free(kfree) = kmemdup(prop_fixup->value, prop_fixup->length, GFP_KERNEL); + char *cur, *end, *node_path, *prop_name, *s; int offset, len; int err = 0; - value = kmemdup(prop_fixup->value, prop_fixup->length, GFP_KERNEL); if (!value) return -ENOMEM; @@ -89,23 +90,19 @@ static int update_usages_of_a_phandle_reference(struct device_node *overlay, node_path = cur; s = strchr(cur, ':'); - if (!s) { - err = -EINVAL; - goto err_fail; - } + if (!s) + return -EINVAL; *s++ = '\0'; prop_name = s; s = strchr(s, ':'); - if (!s) { - err = -EINVAL; - goto err_fail; - } + if (!s) + return -EINVAL; *s++ = '\0'; err = kstrtoint(s, 10, &offset); if (err) - goto err_fail; + return err; refnode = __of_find_node_by_full_path(of_node_get(overlay), node_path); if (!refnode) @@ -117,22 +114,16 @@ static int update_usages_of_a_phandle_reference(struct device_node *overlay, } of_node_put(refnode); - if (!prop) { - err = -ENOENT; - goto err_fail; - } + if (!prop) + return -ENOENT; - if (offset < 0 || offset + sizeof(__be32) > prop->length) { - err = -EINVAL; - goto err_fail; - } + if (offset < 0 || offset + sizeof(__be32) > prop->length) + return -EINVAL; *(__be32 *)(prop->value + offset) = cpu_to_be32(phandle); } -err_fail: - kfree(value); - return err; + return 0; } /* compare nodes taking into account that 'name' strips out the @ part */ From a5737b21057468d7078477b70314091bbb19c735 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Tue, 9 Apr 2024 13:59:41 -0500 Subject: [PATCH 0186/1702] of: Use scope based of_node_put() cleanups Use the relatively new scope based of_node_put() cleanup to simplify function exit handling. Doing so reduces the chances of forgetting an of_node_put() and simplifies error paths by avoiding the need for goto statements. Reviewed-by: Saravana Kannan Reviewed-by: Jonathan Cameron Link: https://lore.kernel.org/r/20240409-dt-cleanup-free-v2-3-5b419a4af38d@kernel.org Signed-off-by: Rob Herring --- drivers/of/address.c | 113 +++++++++++++++--------------------------- drivers/of/property.c | 28 ++++------- 2 files changed, 48 insertions(+), 93 deletions(-) diff --git a/drivers/of/address.c b/drivers/of/address.c index ae46a36059045..d669ce25b5f9c 100644 --- a/drivers/of/address.c +++ b/drivers/of/address.c @@ -486,34 +486,30 @@ static int of_translate_one(struct device_node *parent, struct of_bus *bus, * device that had registered logical PIO mapping, and the return code is * relative to that node. */ -static u64 __of_translate_address(struct device_node *dev, +static u64 __of_translate_address(struct device_node *node, struct device_node *(*get_parent)(const struct device_node *), const __be32 *in_addr, const char *rprop, struct device_node **host) { - struct device_node *parent = NULL; + struct device_node *dev __free(device_node) = of_node_get(node); + struct device_node *parent __free(device_node) = get_parent(dev); struct of_bus *bus, *pbus; __be32 addr[OF_MAX_ADDR_CELLS]; int na, ns, pna, pns; - u64 result = OF_BAD_ADDR; pr_debug("** translation for device %pOF **\n", dev); - /* Increase refcount at current level */ - of_node_get(dev); - *host = NULL; - /* Get parent & match bus type */ - parent = get_parent(dev); + if (parent == NULL) - goto bail; + return OF_BAD_ADDR; bus = of_match_bus(parent); /* Count address cells & copy address locally */ bus->count_cells(dev, &na, &ns); if (!OF_CHECK_COUNTS(na, ns)) { pr_debug("Bad cell count for %pOF\n", dev); - goto bail; + return OF_BAD_ADDR; } memcpy(addr, in_addr, na * 4); @@ -533,8 +529,7 @@ static u64 __of_translate_address(struct device_node *dev, /* If root, we have finished */ if (parent == NULL) { pr_debug("reached root node\n"); - result = of_read_number(addr, na); - break; + return of_read_number(addr, na); } /* @@ -543,11 +538,11 @@ static u64 __of_translate_address(struct device_node *dev, */ iorange = find_io_range_by_fwnode(&dev->fwnode); if (iorange && (iorange->flags != LOGIC_PIO_CPU_MMIO)) { - result = of_read_number(addr + 1, na - 1); + u64 result = of_read_number(addr + 1, na - 1); pr_debug("indirectIO matched(%pOF) 0x%llx\n", dev, result); - *host = of_node_get(dev); - break; + *host = no_free_ptr(dev); + return result; } /* Get new parent bus and counts */ @@ -555,7 +550,7 @@ static u64 __of_translate_address(struct device_node *dev, pbus->count_cells(dev, &pna, &pns); if (!OF_CHECK_COUNTS(pna, pns)) { pr_err("Bad cell count for %pOF\n", dev); - break; + return OF_BAD_ADDR; } pr_debug("parent bus is %s (na=%d, ns=%d) on %pOF\n", @@ -563,7 +558,7 @@ static u64 __of_translate_address(struct device_node *dev, /* Apply bus translation */ if (of_translate_one(dev, bus, pbus, addr, na, ns, pna, rprop)) - break; + return OF_BAD_ADDR; /* Complete the move up one level */ na = pna; @@ -572,11 +567,8 @@ static u64 __of_translate_address(struct device_node *dev, of_dump_addr("one level translation:", addr, na); } - bail: - of_node_put(parent); - of_node_put(dev); - return result; + unreachable(); } u64 of_translate_address(struct device_node *dev, const __be32 *in_addr) @@ -654,19 +646,16 @@ EXPORT_SYMBOL(of_translate_dma_address); const __be32 *of_translate_dma_region(struct device_node *dev, const __be32 *prop, phys_addr_t *start, size_t *length) { - struct device_node *parent; + struct device_node *parent __free(device_node) = __of_get_dma_parent(dev); u64 address, size; int na, ns; - parent = __of_get_dma_parent(dev); if (!parent) return NULL; na = of_bus_n_addr_cells(parent); ns = of_bus_n_size_cells(parent); - of_node_put(parent); - address = of_translate_dma_address(dev, prop); if (address == OF_BAD_ADDR) return NULL; @@ -688,21 +677,19 @@ const __be32 *__of_get_address(struct device_node *dev, int index, int bar_no, { const __be32 *prop; unsigned int psize; - struct device_node *parent; + struct device_node *parent __free(device_node) = of_get_parent(dev); struct of_bus *bus; int onesize, i, na, ns; - /* Get parent & match bus type */ - parent = of_get_parent(dev); if (parent == NULL) return NULL; + + /* match the parent's bus type */ bus = of_match_bus(parent); - if (strcmp(bus->name, "pci") && (bar_no >= 0)) { - of_node_put(parent); + if (strcmp(bus->name, "pci") && (bar_no >= 0)) return NULL; - } + bus->count_cells(dev, &na, &ns); - of_node_put(parent); if (!OF_CHECK_ADDR_COUNT(na)) return NULL; @@ -888,14 +875,13 @@ static u64 of_translate_ioport(struct device_node *dev, const __be32 *in_addr, */ int of_dma_get_range(struct device_node *np, const struct bus_dma_region **map) { - struct device_node *node = of_node_get(np); + struct device_node *node __free(device_node) = of_node_get(np); const __be32 *ranges = NULL; bool found_dma_ranges = false; struct of_range_parser parser; struct of_range range; struct bus_dma_region *r; int len, num_ranges = 0; - int ret = 0; while (node) { ranges = of_get_property(node, "dma-ranges", &len); @@ -905,10 +891,9 @@ int of_dma_get_range(struct device_node *np, const struct bus_dma_region **map) break; /* Once we find 'dma-ranges', then a missing one is an error */ - if (found_dma_ranges && !ranges) { - ret = -ENODEV; - goto out; - } + if (found_dma_ranges && !ranges) + return -ENODEV; + found_dma_ranges = true; node = of_get_next_dma_parent(node); @@ -916,10 +901,8 @@ int of_dma_get_range(struct device_node *np, const struct bus_dma_region **map) if (!node || !ranges) { pr_debug("no dma-ranges found for node(%pOF)\n", np); - ret = -ENODEV; - goto out; + return -ENODEV; } - of_dma_range_parser_init(&parser, node); for_each_of_range(&parser, &range) { if (range.cpu_addr == OF_BAD_ADDR) { @@ -930,16 +913,12 @@ int of_dma_get_range(struct device_node *np, const struct bus_dma_region **map) num_ranges++; } - if (!num_ranges) { - ret = -EINVAL; - goto out; - } + if (!num_ranges) + return -EINVAL; r = kcalloc(num_ranges + 1, sizeof(*r), GFP_KERNEL); - if (!r) { - ret = -ENOMEM; - goto out; - } + if (!r) + return -ENOMEM; /* * Record all info in the generic DMA ranges array for struct device, @@ -957,9 +936,7 @@ int of_dma_get_range(struct device_node *np, const struct bus_dma_region **map) r->size = range.size; r++; } -out: - of_node_put(node); - return ret; + return 0; } #endif /* CONFIG_HAS_DMA */ @@ -1016,24 +993,18 @@ phys_addr_t __init of_dma_get_max_cpu_address(struct device_node *np) */ bool of_dma_is_coherent(struct device_node *np) { - struct device_node *node; - bool is_coherent = dma_default_coherent; - - node = of_node_get(np); + struct device_node *node __free(device_node) = of_node_get(np); while (node) { - if (of_property_read_bool(node, "dma-coherent")) { - is_coherent = true; - break; - } - if (of_property_read_bool(node, "dma-noncoherent")) { - is_coherent = false; - break; - } + if (of_property_read_bool(node, "dma-coherent")) + return true; + + if (of_property_read_bool(node, "dma-noncoherent")) + return false; + node = of_get_next_dma_parent(node); } - of_node_put(node); - return is_coherent; + return dma_default_coherent; } EXPORT_SYMBOL_GPL(of_dma_is_coherent); @@ -1049,20 +1020,14 @@ EXPORT_SYMBOL_GPL(of_dma_is_coherent); */ static bool of_mmio_is_nonposted(struct device_node *np) { - struct device_node *parent; - bool nonposted; - if (!IS_ENABLED(CONFIG_ARCH_APPLE)) return false; - parent = of_get_parent(np); + struct device_node *parent __free(device_node) = of_get_parent(np); if (!parent) return false; - nonposted = of_property_read_bool(parent, "nonposted-mmio"); - - of_node_put(parent); - return nonposted; + return of_property_read_bool(parent, "nonposted-mmio"); } static int __of_address_to_resource(struct device_node *dev, int index, int bar_no, diff --git a/drivers/of/property.c b/drivers/of/property.c index a6358ee99b74b..c1be194412bf6 100644 --- a/drivers/of/property.c +++ b/drivers/of/property.c @@ -40,15 +40,12 @@ */ bool of_graph_is_present(const struct device_node *node) { - struct device_node *ports, *port; + struct device_node *ports __free(device_node) = of_get_child_by_name(node, "ports"); - ports = of_get_child_by_name(node, "ports"); if (ports) node = ports; - port = of_get_child_by_name(node, "port"); - of_node_put(ports); - of_node_put(port); + struct device_node *port __free(device_node) = of_get_child_by_name(node, "port"); return !!port; } @@ -610,25 +607,22 @@ EXPORT_SYMBOL(of_graph_parse_endpoint); */ struct device_node *of_graph_get_port_by_id(struct device_node *parent, u32 id) { - struct device_node *node, *port; + struct device_node *node __free(device_node) = of_get_child_by_name(parent, "ports"); - node = of_get_child_by_name(parent, "ports"); if (node) parent = node; - for_each_child_of_node(parent, port) { + for_each_child_of_node_scoped(parent, port) { u32 port_id = 0; if (!of_node_name_eq(port, "port")) continue; of_property_read_u32(port, "reg", &port_id); if (id == port_id) - break; + return_ptr(port); } - of_node_put(node); - - return port; + return NULL; } EXPORT_SYMBOL(of_graph_get_port_by_id); @@ -655,15 +649,13 @@ struct device_node *of_graph_get_next_endpoint(const struct device_node *parent, * parent port node. */ if (!prev) { - struct device_node *node; + struct device_node *node __free(device_node) = + of_get_child_by_name(parent, "ports"); - node = of_get_child_by_name(parent, "ports"); if (node) parent = node; port = of_get_child_by_name(parent, "port"); - of_node_put(node); - if (!port) { pr_debug("graph: no port node found in %pOF\n", parent); return NULL; @@ -1052,15 +1044,13 @@ static int of_fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode, struct fwnode_endpoint *endpoint) { const struct device_node *node = to_of_node(fwnode); - struct device_node *port_node = of_get_parent(node); + struct device_node *port_node __free(device_node) = of_get_parent(node); endpoint->local_fwnode = fwnode; of_property_read_u32(port_node, "reg", &endpoint->port); of_property_read_u32(node, "reg", &endpoint->id); - of_node_put(port_node); - return 0; } From 5723879c145255d2502a3f8f3a21a16332b24366 Mon Sep 17 00:00:00 2001 From: Frank Oltmanns Date: Sun, 10 Mar 2024 14:21:13 +0100 Subject: [PATCH 0187/1702] clk: sunxi-ng: nkm: Support constraints on m/n ratio and parent rate The Allwinner A64 manual lists the following constraints for the PLL-MIPI clock: - M/N <= 3 - (PLL_VIDEO0)/M >= 24MHz The PLL-MIPI clock is implemented as ccu_nkm. Therefore, add support for these constraints. Reviewed-by: Jernej Skrabec Signed-off-by: Frank Oltmanns Link: https://lore.kernel.org/r/20240310-pinephone-pll-fixes-v4-3-46fc80c83637@oltmanns.dev Signed-off-by: Jernej Skrabec --- drivers/clk/sunxi-ng/ccu_nkm.c | 21 +++++++++++++++++++++ drivers/clk/sunxi-ng/ccu_nkm.h | 2 ++ 2 files changed, 23 insertions(+) diff --git a/drivers/clk/sunxi-ng/ccu_nkm.c b/drivers/clk/sunxi-ng/ccu_nkm.c index 853f84398e2bd..1168d894d636b 100644 --- a/drivers/clk/sunxi-ng/ccu_nkm.c +++ b/drivers/clk/sunxi-ng/ccu_nkm.c @@ -16,6 +16,20 @@ struct _ccu_nkm { unsigned long m, min_m, max_m; }; +static bool ccu_nkm_is_valid_rate(struct ccu_common *common, unsigned long parent, + unsigned long n, unsigned long m) +{ + struct ccu_nkm *nkm = container_of(common, struct ccu_nkm, common); + + if (nkm->max_m_n_ratio && (m > nkm->max_m_n_ratio * n)) + return false; + + if (nkm->min_parent_m_ratio && (parent < nkm->min_parent_m_ratio * m)) + return false; + + return true; +} + static unsigned long ccu_nkm_find_best_with_parent_adj(struct ccu_common *common, struct clk_hw *parent_hw, unsigned long *parent, unsigned long rate, @@ -31,6 +45,10 @@ static unsigned long ccu_nkm_find_best_with_parent_adj(struct ccu_common *common unsigned long tmp_rate, tmp_parent; tmp_parent = clk_hw_round_rate(parent_hw, rate * _m / (_n * _k)); + + if (!ccu_nkm_is_valid_rate(common, tmp_parent, _n, _m)) + continue; + tmp_rate = tmp_parent * _n * _k / _m; if (ccu_is_better_rate(common, rate, tmp_rate, best_rate) || @@ -64,6 +82,9 @@ static unsigned long ccu_nkm_find_best(unsigned long parent, unsigned long rate, for (_k = nkm->min_k; _k <= nkm->max_k; _k++) { for (_n = nkm->min_n; _n <= nkm->max_n; _n++) { for (_m = nkm->min_m; _m <= nkm->max_m; _m++) { + if (!ccu_nkm_is_valid_rate(common, parent, _n, _m)) + continue; + unsigned long tmp_rate; tmp_rate = parent * _n * _k / _m; diff --git a/drivers/clk/sunxi-ng/ccu_nkm.h b/drivers/clk/sunxi-ng/ccu_nkm.h index 6601defb3f387..c409212ee40ec 100644 --- a/drivers/clk/sunxi-ng/ccu_nkm.h +++ b/drivers/clk/sunxi-ng/ccu_nkm.h @@ -27,6 +27,8 @@ struct ccu_nkm { struct ccu_mux_internal mux; unsigned int fixed_post_div; + unsigned long max_m_n_ratio; + unsigned long min_parent_m_ratio; struct ccu_common common; }; From 52b1429e0c51a4ebbdf573ad97c7fb6d34f011a2 Mon Sep 17 00:00:00 2001 From: Frank Oltmanns Date: Sun, 10 Mar 2024 14:21:14 +0100 Subject: [PATCH 0188/1702] clk: sunxi-ng: a64: Add constraints on PLL-MIPI's n/m ratio and parent rate The Allwinner A64 manual lists the following constraints for the PLL-MIPI clock: - M/N <= 3 - (PLL_VIDEO0)/M >= 24MHz Use these constraints. Reviewed-by: Jernej Skrabec Signed-off-by: Frank Oltmanns Link: https://lore.kernel.org/r/20240310-pinephone-pll-fixes-v4-4-46fc80c83637@oltmanns.dev Signed-off-by: Jernej Skrabec --- drivers/clk/sunxi-ng/ccu-sun50i-a64.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c index 8951ffc14ff52..df679dada792a 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c @@ -171,11 +171,13 @@ static struct ccu_nkm pll_mipi_clk = { * user manual, and by experiments the PLL doesn't work without * these bits toggled. */ - .enable = BIT(31) | BIT(23) | BIT(22), - .lock = BIT(28), - .n = _SUNXI_CCU_MULT(8, 4), - .k = _SUNXI_CCU_MULT_MIN(4, 2, 2), - .m = _SUNXI_CCU_DIV(0, 4), + .enable = BIT(31) | BIT(23) | BIT(22), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT(8, 4), + .k = _SUNXI_CCU_MULT_MIN(4, 2, 2), + .m = _SUNXI_CCU_DIV(0, 4), + .max_m_n_ratio = 3, + .min_parent_m_ratio = 24000000, .common = { .reg = 0x040, .hw.init = CLK_HW_INIT("pll-mipi", "pll-video0", From c60f68047e1996bc1b2fb386d3b383aab8900c1b Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Wed, 10 Apr 2024 17:54:20 +0200 Subject: [PATCH 0189/1702] clk: sunxi-ng: fix module autoloading Add MODULE_DEVICE_TABLE(), so modules could be properly autoloaded based on the alias from of_device_id table. Clocks are considered core components, so usually they are built-in, however these can be built and used as modules on some generic kernel. Signed-off-by: Krzysztof Kozlowski Acked-by: Jernej Skrabec Link: https://lore.kernel.org/r/20240410155420.224157-1-krzk@kernel.org Signed-off-by: Jernej Skrabec --- drivers/clk/sunxi-ng/ccu-sun20i-d1-r.c | 1 + drivers/clk/sunxi-ng/ccu-sun20i-d1.c | 1 + drivers/clk/sunxi-ng/ccu-sun4i-a10.c | 1 + drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c | 1 + drivers/clk/sunxi-ng/ccu-sun50i-a100.c | 1 + drivers/clk/sunxi-ng/ccu-sun50i-a64.c | 1 + drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c | 1 + drivers/clk/sunxi-ng/ccu-sun50i-h6.c | 1 + drivers/clk/sunxi-ng/ccu-sun50i-h616.c | 1 + drivers/clk/sunxi-ng/ccu-sun6i-a31.c | 1 + drivers/clk/sunxi-ng/ccu-sun6i-rtc.c | 1 + drivers/clk/sunxi-ng/ccu-sun8i-a23.c | 1 + drivers/clk/sunxi-ng/ccu-sun8i-a33.c | 1 + drivers/clk/sunxi-ng/ccu-sun8i-a83t.c | 1 + drivers/clk/sunxi-ng/ccu-sun8i-de2.c | 1 + drivers/clk/sunxi-ng/ccu-sun8i-h3.c | 1 + drivers/clk/sunxi-ng/ccu-sun8i-r.c | 1 + drivers/clk/sunxi-ng/ccu-sun8i-r40.c | 1 + drivers/clk/sunxi-ng/ccu-sun8i-v3s.c | 1 + drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c | 1 + drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c | 1 + drivers/clk/sunxi-ng/ccu-sun9i-a80.c | 1 + drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c | 1 + 23 files changed, 23 insertions(+) diff --git a/drivers/clk/sunxi-ng/ccu-sun20i-d1-r.c b/drivers/clk/sunxi-ng/ccu-sun20i-d1-r.c index 9d3ffd3fb2c14..0736f6c81269b 100644 --- a/drivers/clk/sunxi-ng/ccu-sun20i-d1-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun20i-d1-r.c @@ -125,6 +125,7 @@ static const struct of_device_id sun20i_d1_r_ccu_ids[] = { { .compatible = "allwinner,sun20i-d1-r-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun20i_d1_r_ccu_ids); static struct platform_driver sun20i_d1_r_ccu_driver = { .probe = sun20i_d1_r_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun20i-d1.c b/drivers/clk/sunxi-ng/ccu-sun20i-d1.c index 48a8fb2c43b74..60756aadfad6a 100644 --- a/drivers/clk/sunxi-ng/ccu-sun20i-d1.c +++ b/drivers/clk/sunxi-ng/ccu-sun20i-d1.c @@ -1394,6 +1394,7 @@ static const struct of_device_id sun20i_d1_ccu_ids[] = { { .compatible = "allwinner,sun20i-d1-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun20i_d1_ccu_ids); static struct platform_driver sun20i_d1_ccu_driver = { .probe = sun20i_d1_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun4i-a10.c b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c index 451ebb7c99a3f..14f5c3da652b0 100644 --- a/drivers/clk/sunxi-ng/ccu-sun4i-a10.c +++ b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c @@ -1481,6 +1481,7 @@ static const struct of_device_id sun4i_a10_ccu_ids[] = { }, { } }; +MODULE_DEVICE_TABLE(of, sun4i_a10_ccu_ids); static struct platform_driver sun4i_a10_ccu_driver = { .probe = sun4i_a10_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c b/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c index fddd6c877cecc..3b983bb59bd94 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a100-r.c @@ -202,6 +202,7 @@ static const struct of_device_id sun50i_a100_r_ccu_ids[] = { { .compatible = "allwinner,sun50i-a100-r-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun50i_a100_r_ccu_ids); static struct platform_driver sun50i_a100_r_ccu_driver = { .probe = sun50i_a100_r_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a100.c b/drivers/clk/sunxi-ng/ccu-sun50i-a100.c index 5f93b5526e13d..38aa6e5f298e9 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-a100.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a100.c @@ -1264,6 +1264,7 @@ static const struct of_device_id sun50i_a100_ccu_ids[] = { { .compatible = "allwinner,sun50i-a100-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun50i_a100_ccu_ids); static struct platform_driver sun50i_a100_ccu_driver = { .probe = sun50i_a100_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c index df679dada792a..56c629186becf 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c @@ -980,6 +980,7 @@ static const struct of_device_id sun50i_a64_ccu_ids[] = { { .compatible = "allwinner,sun50i-a64-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun50i_a64_ccu_ids); static struct platform_driver sun50i_a64_ccu_driver = { .probe = sun50i_a64_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c index 02b28cfc5525e..e2dc29fa99e70 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c @@ -244,6 +244,7 @@ static const struct of_device_id sun50i_h6_r_ccu_ids[] = { }, { } }; +MODULE_DEVICE_TABLE(of, sun50i_h6_r_ccu_ids); static struct platform_driver sun50i_h6_r_ccu_driver = { .probe = sun50i_h6_r_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c index 42568c6161814..fa483bc2437a0 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c @@ -1259,6 +1259,7 @@ static const struct of_device_id sun50i_h6_ccu_ids[] = { { .compatible = "allwinner,sun50i-h6-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun50i_h6_ccu_ids); static struct platform_driver sun50i_h6_ccu_driver = { .probe = sun50i_h6_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h616.c b/drivers/clk/sunxi-ng/ccu-sun50i-h616.c index 21e918582aa56..45aae1ae51784 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h616.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h616.c @@ -1154,6 +1154,7 @@ static const struct of_device_id sun50i_h616_ccu_ids[] = { { .compatible = "allwinner,sun50i-h616-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun50i_h616_ccu_ids); static struct platform_driver sun50i_h616_ccu_driver = { .probe = sun50i_h616_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c index 0762deffb33ca..8cb8cbbdbafbb 100644 --- a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c +++ b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c @@ -1271,6 +1271,7 @@ static const struct of_device_id sun6i_a31_ccu_ids[] = { { .compatible = "allwinner,sun6i-a31-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun6i_a31_ccu_ids); static struct platform_driver sun6i_a31_ccu_driver = { .probe = sun6i_a31_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c b/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c index fdc8ccc586c99..5a98c4e9e667d 100644 --- a/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c +++ b/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c @@ -336,6 +336,7 @@ static const struct of_device_id sun6i_rtc_ccu_match[] = { }, {}, }; +MODULE_DEVICE_TABLE(of, sun6i_rtc_ccu_match); int sun6i_rtc_ccu_probe(struct device *dev, void __iomem *reg) { diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c index e80cc3864e440..e748ad612b8ff 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c @@ -751,6 +751,7 @@ static const struct of_device_id sun8i_a23_ccu_ids[] = { { .compatible = "allwinner,sun8i-a23-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun8i_a23_ccu_ids); static struct platform_driver sun8i_a23_ccu_driver = { .probe = sun8i_a23_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a33.c b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c index d12878a1ba9e9..8a27a1777600d 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-a33.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c @@ -823,6 +823,7 @@ static const struct of_device_id sun8i_a33_ccu_ids[] = { { .compatible = "allwinner,sun8i-a33-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun8i_a33_ccu_ids); static struct platform_driver sun8i_a33_ccu_driver = { .probe = sun8i_a33_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c b/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c index 76cbd9e9e89f6..93eca47935cfc 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c @@ -911,6 +911,7 @@ static const struct of_device_id sun8i_a83t_ccu_ids[] = { { .compatible = "allwinner,sun8i-a83t-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun8i_a83t_ccu_ids); static struct platform_driver sun8i_a83t_ccu_driver = { .probe = sun8i_a83t_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-de2.c b/drivers/clk/sunxi-ng/ccu-sun8i-de2.c index 6a043a0a9dd65..b0b8dba239aec 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-de2.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-de2.c @@ -337,6 +337,7 @@ static const struct of_device_id sunxi_de2_clk_ids[] = { }, { } }; +MODULE_DEVICE_TABLE(of, sunxi_de2_clk_ids); static struct platform_driver sunxi_de2_clk_driver = { .probe = sunxi_de2_clk_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c index 74274c17efb3e..ca5739fa04f70 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c @@ -1082,6 +1082,7 @@ static const struct of_device_id sun8i_h3_ccu_ids[] = { }, { } }; +MODULE_DEVICE_TABLE(of, sun8i_h3_ccu_ids); static struct platform_driver sun8i_h3_ccu_driver = { .probe = sun8i_h3_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r.c b/drivers/clk/sunxi-ng/ccu-sun8i-r.c index 4890a976b1a02..bac7e737db98c 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-r.c @@ -262,6 +262,7 @@ static const struct of_device_id sun8i_r_ccu_ids[] = { }, { } }; +MODULE_DEVICE_TABLE(of, sun8i_r_ccu_ids); static struct platform_driver sun8i_r_ccu_driver = { .probe = sun8i_r_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c index 31eca0d3bc1e7..3774b293e74c1 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c @@ -1363,6 +1363,7 @@ static const struct of_device_id sun8i_r40_ccu_ids[] = { { .compatible = "allwinner,sun8i-r40-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun8i_r40_ccu_ids); static struct platform_driver sun8i_r40_ccu_driver = { .probe = sun8i_r40_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c index f3ce8664b2883..994258a3ad2e2 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c @@ -768,6 +768,7 @@ static const struct of_device_id sun8i_v3s_ccu_ids[] = { }, { } }; +MODULE_DEVICE_TABLE(of, sun8i_v3s_ccu_ids); static struct platform_driver sun8i_v3s_ccu_driver = { .probe = sun8i_v3s_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c index 1d8b1ae1619df..ae7939d3f502b 100644 --- a/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c +++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c @@ -254,6 +254,7 @@ static const struct of_device_id sun9i_a80_de_clk_ids[] = { { .compatible = "allwinner,sun9i-a80-de-clks" }, { } }; +MODULE_DEVICE_TABLE(of, sun9i_a80_de_clk_ids); static struct platform_driver sun9i_a80_de_clk_driver = { .probe = sun9i_a80_de_clk_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c index a0fb0da8f3563..bfa2ff9d52a47 100644 --- a/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c +++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c @@ -127,6 +127,7 @@ static const struct of_device_id sun9i_a80_usb_clk_ids[] = { { .compatible = "allwinner,sun9i-a80-usb-clks" }, { } }; +MODULE_DEVICE_TABLE(of, sun9i_a80_usb_clk_ids); static struct platform_driver sun9i_a80_usb_clk_driver = { .probe = sun9i_a80_usb_clk_probe, diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80.c index 730fd8e280146..c05805e4ad22e 100644 --- a/drivers/clk/sunxi-ng/ccu-sun9i-a80.c +++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80.c @@ -1236,6 +1236,7 @@ static const struct of_device_id sun9i_a80_ccu_ids[] = { { .compatible = "allwinner,sun9i-a80-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, sun9i_a80_ccu_ids); static struct platform_driver sun9i_a80_ccu_driver = { .probe = sun9i_a80_ccu_probe, diff --git a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c index 0d5b60b123b76..76d3d070b2a7b 100644 --- a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c +++ b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c @@ -565,6 +565,7 @@ static const struct of_device_id suniv_f1c100s_ccu_ids[] = { { .compatible = "allwinner,suniv-f1c100s-ccu" }, { } }; +MODULE_DEVICE_TABLE(of, suniv_f1c100s_ccu_ids); static struct platform_driver suniv_f1c100s_ccu_driver = { .probe = suniv_f1c100s_ccu_probe, From 549d3c9a29921f388ef3bcfd1d4f669b7dd4eed2 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:03 -0700 Subject: [PATCH 0190/1702] xfs: pass xfs_buf lookup flags to xfs_*read_agi Allow callers to pass buffer lookup flags to xfs_read_agi and xfs_ialloc_read_agi. This will be used in the next patch to fix a deadlock in the online fsck inode scanner. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_ag.c | 8 ++++---- fs/xfs/libxfs/xfs_ialloc.c | 16 ++++++++++------ fs/xfs/libxfs/xfs_ialloc.h | 5 +++-- fs/xfs/libxfs/xfs_ialloc_btree.c | 4 ++-- fs/xfs/scrub/common.c | 4 ++-- fs/xfs/scrub/fscounters.c | 2 +- fs/xfs/scrub/iscan.c | 2 +- fs/xfs/scrub/repair.c | 6 +++--- fs/xfs/xfs_inode.c | 8 ++++---- fs/xfs/xfs_iwalk.c | 4 ++-- fs/xfs/xfs_log_recover.c | 4 ++-- 11 files changed, 34 insertions(+), 29 deletions(-) diff --git a/fs/xfs/libxfs/xfs_ag.c b/fs/xfs/libxfs/xfs_ag.c index dc1873f76bffd..09fe9412eab49 100644 --- a/fs/xfs/libxfs/xfs_ag.c +++ b/fs/xfs/libxfs/xfs_ag.c @@ -194,7 +194,7 @@ xfs_initialize_perag_data( pag = xfs_perag_get(mp, index); error = xfs_alloc_read_agf(pag, NULL, 0, NULL); if (!error) - error = xfs_ialloc_read_agi(pag, NULL, NULL); + error = xfs_ialloc_read_agi(pag, NULL, 0, NULL); if (error) { xfs_perag_put(pag); return error; @@ -931,7 +931,7 @@ xfs_ag_shrink_space( int error, err2; ASSERT(pag->pag_agno == mp->m_sb.sb_agcount - 1); - error = xfs_ialloc_read_agi(pag, *tpp, &agibp); + error = xfs_ialloc_read_agi(pag, *tpp, 0, &agibp); if (error) return error; @@ -1062,7 +1062,7 @@ xfs_ag_extend_space( ASSERT(pag->pag_agno == pag->pag_mount->m_sb.sb_agcount - 1); - error = xfs_ialloc_read_agi(pag, tp, &bp); + error = xfs_ialloc_read_agi(pag, tp, 0, &bp); if (error) return error; @@ -1119,7 +1119,7 @@ xfs_ag_get_geometry( int error; /* Lock the AG headers. */ - error = xfs_ialloc_read_agi(pag, NULL, &agi_bp); + error = xfs_ialloc_read_agi(pag, NULL, 0, &agi_bp); if (error) return error; error = xfs_alloc_read_agf(pag, NULL, 0, &agf_bp); diff --git a/fs/xfs/libxfs/xfs_ialloc.c b/fs/xfs/libxfs/xfs_ialloc.c index e5ac3e5430c4e..cb37f0007731f 100644 --- a/fs/xfs/libxfs/xfs_ialloc.c +++ b/fs/xfs/libxfs/xfs_ialloc.c @@ -1699,7 +1699,7 @@ xfs_dialloc_good_ag( return false; if (!xfs_perag_initialised_agi(pag)) { - error = xfs_ialloc_read_agi(pag, tp, NULL); + error = xfs_ialloc_read_agi(pag, tp, 0, NULL); if (error) return false; } @@ -1768,7 +1768,7 @@ xfs_dialloc_try_ag( * Then read in the AGI buffer and recheck with the AGI buffer * lock held. */ - error = xfs_ialloc_read_agi(pag, *tpp, &agbp); + error = xfs_ialloc_read_agi(pag, *tpp, 0, &agbp); if (error) return error; @@ -2286,7 +2286,7 @@ xfs_difree( /* * Get the allocation group header. */ - error = xfs_ialloc_read_agi(pag, tp, &agbp); + error = xfs_ialloc_read_agi(pag, tp, 0, &agbp); if (error) { xfs_warn(mp, "%s: xfs_ialloc_read_agi() returned error %d.", __func__, error); @@ -2332,7 +2332,7 @@ xfs_imap_lookup( int error; int i; - error = xfs_ialloc_read_agi(pag, tp, &agbp); + error = xfs_ialloc_read_agi(pag, tp, 0, &agbp); if (error) { xfs_alert(mp, "%s: xfs_ialloc_read_agi() returned error %d, agno %d", @@ -2675,6 +2675,7 @@ int xfs_read_agi( struct xfs_perag *pag, struct xfs_trans *tp, + xfs_buf_flags_t flags, struct xfs_buf **agibpp) { struct xfs_mount *mp = pag->pag_mount; @@ -2684,7 +2685,7 @@ xfs_read_agi( error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp, XFS_AG_DADDR(mp, pag->pag_agno, XFS_AGI_DADDR(mp)), - XFS_FSS_TO_BB(mp, 1), 0, agibpp, &xfs_agi_buf_ops); + XFS_FSS_TO_BB(mp, 1), flags, agibpp, &xfs_agi_buf_ops); if (xfs_metadata_is_sick(error)) xfs_ag_mark_sick(pag, XFS_SICK_AG_AGI); if (error) @@ -2704,6 +2705,7 @@ int xfs_ialloc_read_agi( struct xfs_perag *pag, struct xfs_trans *tp, + int flags, struct xfs_buf **agibpp) { struct xfs_buf *agibp; @@ -2712,7 +2714,9 @@ xfs_ialloc_read_agi( trace_xfs_ialloc_read_agi(pag->pag_mount, pag->pag_agno); - error = xfs_read_agi(pag, tp, &agibp); + error = xfs_read_agi(pag, tp, + (flags & XFS_IALLOC_FLAG_TRYLOCK) ? XBF_TRYLOCK : 0, + &agibp); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_ialloc.h b/fs/xfs/libxfs/xfs_ialloc.h index f1412183bb44b..b549627e3a615 100644 --- a/fs/xfs/libxfs/xfs_ialloc.h +++ b/fs/xfs/libxfs/xfs_ialloc.h @@ -63,10 +63,11 @@ xfs_ialloc_log_agi( struct xfs_buf *bp, /* allocation group header buffer */ uint32_t fields); /* bitmask of fields to log */ -int xfs_read_agi(struct xfs_perag *pag, struct xfs_trans *tp, +int xfs_read_agi(struct xfs_perag *pag, struct xfs_trans *tp, xfs_buf_flags_t flags, struct xfs_buf **agibpp); int xfs_ialloc_read_agi(struct xfs_perag *pag, struct xfs_trans *tp, - struct xfs_buf **agibpp); + int flags, struct xfs_buf **agibpp); +#define XFS_IALLOC_FLAG_TRYLOCK (1U << 0) /* use trylock for buffer locking */ /* * Lookup a record by ino in the btree given by cur. diff --git a/fs/xfs/libxfs/xfs_ialloc_btree.c b/fs/xfs/libxfs/xfs_ialloc_btree.c index cc661fca6ff5b..42e9fd47f6c73 100644 --- a/fs/xfs/libxfs/xfs_ialloc_btree.c +++ b/fs/xfs/libxfs/xfs_ialloc_btree.c @@ -745,7 +745,7 @@ xfs_finobt_count_blocks( struct xfs_btree_cur *cur; int error; - error = xfs_ialloc_read_agi(pag, tp, &agbp); + error = xfs_ialloc_read_agi(pag, tp, 0, &agbp); if (error) return error; @@ -768,7 +768,7 @@ xfs_finobt_read_blocks( struct xfs_agi *agi; int error; - error = xfs_ialloc_read_agi(pag, tp, &agbp); + error = xfs_ialloc_read_agi(pag, tp, 0, &agbp); if (error) return error; diff --git a/fs/xfs/scrub/common.c b/fs/xfs/scrub/common.c index 47a20cf5205f0..a27d33b6f4641 100644 --- a/fs/xfs/scrub/common.c +++ b/fs/xfs/scrub/common.c @@ -445,7 +445,7 @@ xchk_perag_read_headers( { int error; - error = xfs_ialloc_read_agi(sa->pag, sc->tp, &sa->agi_bp); + error = xfs_ialloc_read_agi(sa->pag, sc->tp, 0, &sa->agi_bp); if (error && want_ag_read_header_failure(sc, XFS_SCRUB_TYPE_AGI)) return error; @@ -827,7 +827,7 @@ xchk_iget_agi( * in the iget cache miss path. */ pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, inum)); - error = xfs_ialloc_read_agi(pag, tp, agi_bpp); + error = xfs_ialloc_read_agi(pag, tp, 0, agi_bpp); xfs_perag_put(pag); if (error) return error; diff --git a/fs/xfs/scrub/fscounters.c b/fs/xfs/scrub/fscounters.c index d310737c88236..da2f6729699dd 100644 --- a/fs/xfs/scrub/fscounters.c +++ b/fs/xfs/scrub/fscounters.c @@ -85,7 +85,7 @@ xchk_fscount_warmup( continue; /* Lock both AG headers. */ - error = xfs_ialloc_read_agi(pag, sc->tp, &agi_bp); + error = xfs_ialloc_read_agi(pag, sc->tp, 0, &agi_bp); if (error) break; error = xfs_alloc_read_agf(pag, sc->tp, 0, &agf_bp); diff --git a/fs/xfs/scrub/iscan.c b/fs/xfs/scrub/iscan.c index ec3478bc505ef..66ba0fbd059e0 100644 --- a/fs/xfs/scrub/iscan.c +++ b/fs/xfs/scrub/iscan.c @@ -281,7 +281,7 @@ xchk_iscan_advance( if (!pag) return -ECANCELED; - ret = xfs_ialloc_read_agi(pag, sc->tp, &agi_bp); + ret = xfs_ialloc_read_agi(pag, sc->tp, 0, &agi_bp); if (ret) goto out_pag; diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c index f43dce771cdd2..443e62f724818 100644 --- a/fs/xfs/scrub/repair.c +++ b/fs/xfs/scrub/repair.c @@ -290,7 +290,7 @@ xrep_calc_ag_resblks( icount = pag->pagi_count; } else { /* Try to get the actual counters from disk. */ - error = xfs_ialloc_read_agi(pag, NULL, &bp); + error = xfs_ialloc_read_agi(pag, NULL, 0, &bp); if (!error) { icount = pag->pagi_count; xfs_buf_relse(bp); @@ -908,7 +908,7 @@ xrep_reinit_pagi( ASSERT(xfs_perag_initialised_agi(pag)); clear_bit(XFS_AGSTATE_AGI_INIT, &pag->pag_opstate); - error = xfs_ialloc_read_agi(pag, sc->tp, &bp); + error = xfs_ialloc_read_agi(pag, sc->tp, 0, &bp); if (error) return error; @@ -934,7 +934,7 @@ xrep_ag_init( ASSERT(!sa->pag); - error = xfs_ialloc_read_agi(pag, sc->tp, &sa->agi_bp); + error = xfs_ialloc_read_agi(pag, sc->tp, 0, &sa->agi_bp); if (error) return error; diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index d55b42b2480d6..3e667a19b80ba 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -2167,7 +2167,7 @@ xfs_iunlink( pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino)); /* Get the agi buffer first. It ensures lock ordering on the list. */ - error = xfs_read_agi(pag, tp, &agibp); + error = xfs_read_agi(pag, tp, 0, &agibp); if (error) goto out; @@ -2264,7 +2264,7 @@ xfs_iunlink_remove( trace_xfs_iunlink_remove(ip); /* Get the agi buffer first. It ensures lock ordering on the list. */ - error = xfs_read_agi(pag, tp, &agibp); + error = xfs_read_agi(pag, tp, 0, &agibp); if (error) return error; @@ -3142,7 +3142,7 @@ xfs_rename( pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, inodes[i]->i_ino)); - error = xfs_read_agi(pag, tp, &bp); + error = xfs_read_agi(pag, tp, 0, &bp); xfs_perag_put(pag); if (error) goto out_trans_cancel; @@ -3814,7 +3814,7 @@ xfs_inode_reload_unlinked_bucket( /* Grab the first inode in the list */ pag = xfs_perag_get(mp, agno); - error = xfs_ialloc_read_agi(pag, tp, &agibp); + error = xfs_ialloc_read_agi(pag, tp, 0, &agibp); xfs_perag_put(pag); if (error) return error; diff --git a/fs/xfs/xfs_iwalk.c b/fs/xfs/xfs_iwalk.c index 01b55f03a1026..730c8d48da282 100644 --- a/fs/xfs/xfs_iwalk.c +++ b/fs/xfs/xfs_iwalk.c @@ -268,7 +268,7 @@ xfs_iwalk_ag_start( /* Set up a fresh cursor and empty the inobt cache. */ iwag->nr_recs = 0; - error = xfs_ialloc_read_agi(pag, tp, agi_bpp); + error = xfs_ialloc_read_agi(pag, tp, 0, agi_bpp); if (error) return error; *curpp = xfs_inobt_init_cursor(pag, tp, *agi_bpp); @@ -386,7 +386,7 @@ xfs_iwalk_run_callbacks( } /* ...and recreate the cursor just past where we left off. */ - error = xfs_ialloc_read_agi(iwag->pag, iwag->tp, agi_bpp); + error = xfs_ialloc_read_agi(iwag->pag, iwag->tp, 0, agi_bpp); if (error) return error; *curpp = xfs_inobt_init_cursor(iwag->pag, iwag->tp, *agi_bpp); diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 13f1d2e915405..1b1f0a4cd4947 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2656,7 +2656,7 @@ xlog_recover_clear_agi_bucket( if (error) goto out_error; - error = xfs_read_agi(pag, tp, &agibp); + error = xfs_read_agi(pag, tp, 0, &agibp); if (error) goto out_abort; @@ -2772,7 +2772,7 @@ xlog_recover_iunlink_ag( int bucket; int error; - error = xfs_read_agi(pag, NULL, &agibp); + error = xfs_read_agi(pag, NULL, 0, &agibp); if (error) { /* * AGI is b0rked. Don't process it. From 2afd5276d314d775ae0bdaa6ec22069515bd9c70 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:04 -0700 Subject: [PATCH 0191/1702] xfs: fix an AGI lock acquisition ordering problem in xrep_dinode_findmode While reviewing the next patch which fixes an ABBA deadlock between the AGI and a directory ILOCK, someone asked a question about why we're holding the AGI in the first place. The reason for that is to quiesce the inode structures for that AG while we do a repair. I then realized that the xrep_dinode_findmode invokes xchk_iscan_iter, which walks the inobts (and hence the AGIs) to find all the inodes. This itself is also an ABBA vector, since the damaged inode could be in AG 5, which we hold while we scan AG 0 for directories. 5 -> 0 is not allowed. To address this, modify the iscan to allow trylock of the AGI buffer using the flags argument to xfs_ialloc_read_agi that the previous patch added. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/inode_repair.c | 1 + fs/xfs/scrub/iscan.c | 36 +++++++++++++++++++++++++++++++++++- fs/xfs/scrub/iscan.h | 15 +++++++++++++++ fs/xfs/scrub/trace.h | 10 ++++++++-- 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/fs/xfs/scrub/inode_repair.c b/fs/xfs/scrub/inode_repair.c index eab380e95ef40..35da0193c919e 100644 --- a/fs/xfs/scrub/inode_repair.c +++ b/fs/xfs/scrub/inode_repair.c @@ -356,6 +356,7 @@ xrep_dinode_find_mode( * so there's a real possibility that _iscan_iter can return EBUSY. */ xchk_iscan_start(sc, 5000, 100, &ri->ftype_iscan); + xchk_iscan_set_agi_trylock(&ri->ftype_iscan); ri->ftype_iscan.skip_ino = sc->sm->sm_ino; ri->alleged_ftype = XFS_DIR3_FT_UNKNOWN; while ((error = xchk_iscan_iter(&ri->ftype_iscan, &dp)) == 1) { diff --git a/fs/xfs/scrub/iscan.c b/fs/xfs/scrub/iscan.c index 66ba0fbd059e0..c643b7d79b607 100644 --- a/fs/xfs/scrub/iscan.c +++ b/fs/xfs/scrub/iscan.c @@ -243,6 +243,40 @@ xchk_iscan_finish( mutex_unlock(&iscan->lock); } +/* + * Grab the AGI to advance the inode scan. Returns 0 if *agi_bpp is now set, + * -ECANCELED if the live scan aborted, -EBUSY if the AGI could not be grabbed, + * or the usual negative errno. + */ +STATIC int +xchk_iscan_read_agi( + struct xchk_iscan *iscan, + struct xfs_perag *pag, + struct xfs_buf **agi_bpp) +{ + struct xfs_scrub *sc = iscan->sc; + unsigned long relax; + int ret; + + if (!xchk_iscan_agi_needs_trylock(iscan)) + return xfs_ialloc_read_agi(pag, sc->tp, 0, agi_bpp); + + relax = msecs_to_jiffies(iscan->iget_retry_delay); + do { + ret = xfs_ialloc_read_agi(pag, sc->tp, XFS_IALLOC_FLAG_TRYLOCK, + agi_bpp); + if (ret != -EAGAIN) + return ret; + if (!iscan->iget_timeout || + time_is_before_jiffies(iscan->__iget_deadline)) + return -EBUSY; + + trace_xchk_iscan_agi_retry_wait(iscan); + } while (!schedule_timeout_killable(relax) && + !xchk_iscan_aborted(iscan)); + return -ECANCELED; +} + /* * Advance ino to the next inode that the inobt thinks is allocated, being * careful to jump to the next AG if we've reached the right end of this AG's @@ -281,7 +315,7 @@ xchk_iscan_advance( if (!pag) return -ECANCELED; - ret = xfs_ialloc_read_agi(pag, sc->tp, 0, &agi_bp); + ret = xchk_iscan_read_agi(iscan, pag, &agi_bp); if (ret) goto out_pag; diff --git a/fs/xfs/scrub/iscan.h b/fs/xfs/scrub/iscan.h index 71f657552dfac..5e0e4ed9dea6d 100644 --- a/fs/xfs/scrub/iscan.h +++ b/fs/xfs/scrub/iscan.h @@ -59,6 +59,9 @@ struct xchk_iscan { /* Set if the scan has been aborted due to some event in the fs. */ #define XCHK_ISCAN_OPSTATE_ABORTED (1) +/* Use trylock to acquire the AGI */ +#define XCHK_ISCAN_OPSTATE_TRYLOCK_AGI (2) + static inline bool xchk_iscan_aborted(const struct xchk_iscan *iscan) { @@ -71,6 +74,18 @@ xchk_iscan_abort(struct xchk_iscan *iscan) set_bit(XCHK_ISCAN_OPSTATE_ABORTED, &iscan->__opstate); } +static inline bool +xchk_iscan_agi_needs_trylock(const struct xchk_iscan *iscan) +{ + return test_bit(XCHK_ISCAN_OPSTATE_TRYLOCK_AGI, &iscan->__opstate); +} + +static inline void +xchk_iscan_set_agi_trylock(struct xchk_iscan *iscan) +{ + set_bit(XCHK_ISCAN_OPSTATE_TRYLOCK_AGI, &iscan->__opstate); +} + void xchk_iscan_start(struct xfs_scrub *sc, unsigned int iget_timeout, unsigned int iget_retry_delay, struct xchk_iscan *iscan); void xchk_iscan_teardown(struct xchk_iscan *iscan); diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 5b294be52c558..b1c7c79760d44 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -1300,7 +1300,7 @@ TRACE_EVENT(xchk_iscan_iget_batch, __entry->unavail) ); -TRACE_EVENT(xchk_iscan_iget_retry_wait, +DECLARE_EVENT_CLASS(xchk_iscan_retry_wait_class, TP_PROTO(struct xchk_iscan *iscan), TP_ARGS(iscan), TP_STRUCT__entry( @@ -1326,7 +1326,13 @@ TRACE_EVENT(xchk_iscan_iget_retry_wait, __entry->remaining, __entry->iget_timeout, __entry->retry_delay) -); +) +#define DEFINE_ISCAN_RETRY_WAIT_EVENT(name) \ +DEFINE_EVENT(xchk_iscan_retry_wait_class, name, \ + TP_PROTO(struct xchk_iscan *iscan), \ + TP_ARGS(iscan)) +DEFINE_ISCAN_RETRY_WAIT_EVENT(xchk_iscan_iget_retry_wait); +DEFINE_ISCAN_RETRY_WAIT_EVENT(xchk_iscan_agi_retry_wait); TRACE_EVENT(xchk_nlinks_collect_dirent, TP_PROTO(struct xfs_mount *mp, struct xfs_inode *dp, From 21ad2d03641ae70a7acdcf1212cd135dafb2a798 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:05 -0700 Subject: [PATCH 0192/1702] xfs: fix potential AGI <-> ILOCK ABBA deadlock in xrep_dinode_findmode_walk_directory xfs/399 found the following deadlock when fuzzing core.mode = ones: /proc/20506/task/20558/stack : [<0>] xfs_ilock+0xa0/0x240 [xfs] [<0>] xfs_ilock_data_map_shared+0x1b/0x20 [xfs] [<0>] xrep_dinode_findmode_walk_directory+0x69/0xe0 [xfs] [<0>] xrep_dinode_find_mode+0x103/0x2a0 [xfs] [<0>] xrep_dinode_mode+0x7c/0x120 [xfs] [<0>] xrep_dinode_core+0xed/0x2b0 [xfs] [<0>] xrep_dinode_problems+0x10/0x80 [xfs] [<0>] xrep_inode+0x6c/0xc0 [xfs] [<0>] xrep_attempt+0x64/0x1d0 [xfs] [<0>] xfs_scrub_metadata+0x365/0x840 [xfs] [<0>] xfs_scrubv_metadata+0x282/0x430 [xfs] [<0>] xfs_ioc_scrubv_metadata+0x149/0x1a0 [xfs] [<0>] xfs_file_ioctl+0xc68/0x1780 [xfs] /proc/20506/task/20559/stack : [<0>] xfs_buf_lock+0x3b/0x110 [xfs] [<0>] xfs_buf_find_lock+0x66/0x1c0 [xfs] [<0>] xfs_buf_get_map+0x208/0xc00 [xfs] [<0>] xfs_buf_read_map+0x5d/0x2c0 [xfs] [<0>] xfs_trans_read_buf_map+0x1b0/0x4c0 [xfs] [<0>] xfs_read_agi+0xbd/0x190 [xfs] [<0>] xfs_ialloc_read_agi+0x47/0x160 [xfs] [<0>] xfs_imap_lookup+0x69/0x1f0 [xfs] [<0>] xfs_imap+0x1fc/0x3d0 [xfs] [<0>] xfs_iget+0x357/0xd50 [xfs] [<0>] xchk_dir_actor+0x16e/0x330 [xfs] [<0>] xchk_dir_walk_block+0x164/0x1e0 [xfs] [<0>] xchk_dir_walk+0x13a/0x190 [xfs] [<0>] xchk_directory+0x1a2/0x2b0 [xfs] [<0>] xfs_scrub_metadata+0x2f4/0x840 [xfs] [<0>] xfs_scrubv_metadata+0x282/0x430 [xfs] [<0>] xfs_ioc_scrubv_metadata+0x149/0x1a0 [xfs] [<0>] xfs_file_ioctl+0xc68/0x1780 [xfs] Thread 20558 holds an AGI buffer and is trying to grab the ILOCK of the root directory. Thread 20559 holds the root directory ILOCK and is trying to grab the AGI of an inode that is one of the root directory's children. The AGI held by 20558 is the same buffer that 20559 is trying to acquire. In other words, this is an ABBA deadlock. In general, the lock order is ILOCK and then AGI -- rename does this while preparing for an operation involving whiteouts or renaming files out of existence; and unlink does this when moving an inode to the unlinked list. The only place where we do it in the opposite order is on the child during an icreate, but at that point the child is marked INEW and is not visible to other threads. Work around this deadlock by replacing the blocking ilock attempt with a nonblocking loop that aborts after 30 seconds. Relax for a jiffy after a failed lock attempt. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/inode_repair.c | 49 ++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/fs/xfs/scrub/inode_repair.c b/fs/xfs/scrub/inode_repair.c index 35da0193c919e..097afba3043f8 100644 --- a/fs/xfs/scrub/inode_repair.c +++ b/fs/xfs/scrub/inode_repair.c @@ -282,6 +282,51 @@ xrep_dinode_findmode_dirent( return 0; } +/* Try to lock a directory, or wait a jiffy. */ +static inline int +xrep_dinode_ilock_nowait( + struct xfs_inode *dp, + unsigned int lock_mode) +{ + if (xfs_ilock_nowait(dp, lock_mode)) + return true; + + schedule_timeout_killable(1); + return false; +} + +/* + * Try to lock a directory to look for ftype hints. Since we already hold the + * AGI buffer, we cannot block waiting for the ILOCK because rename can take + * the ILOCK and then try to lock AGIs. + */ +STATIC int +xrep_dinode_trylock_directory( + struct xrep_inode *ri, + struct xfs_inode *dp, + unsigned int *lock_modep) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(30000); + unsigned int lock_mode; + int error = 0; + + do { + if (xchk_should_terminate(ri->sc, &error)) + return error; + + if (xfs_need_iread_extents(&dp->i_df)) + lock_mode = XFS_ILOCK_EXCL; + else + lock_mode = XFS_ILOCK_SHARED; + + if (xrep_dinode_ilock_nowait(dp, lock_mode)) { + *lock_modep = lock_mode; + return 0; + } + } while (!time_is_before_jiffies(deadline)); + return -EBUSY; +} + /* * If this is a directory, walk the dirents looking for any that point to the * scrub target inode. @@ -299,7 +344,9 @@ xrep_dinode_findmode_walk_directory( * Scan the directory to see if there it contains an entry pointing to * the directory that we are repairing. */ - lock_mode = xfs_ilock_data_map_shared(dp); + error = xrep_dinode_trylock_directory(ri, dp, &lock_mode); + if (error) + return error; /* * If this directory is known to be sick, we cannot scan it reliably From 98a778b42514dcd00c0356b2fbfd458636cbff87 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:06 -0700 Subject: [PATCH 0193/1702] xfs: fix error bailout in xrep_abt_build_new_trees Dan Carpenter reports: "Commit 4bdfd7d15747 ("xfs: repair free space btrees") from Dec 15, 2023 (linux-next), leads to the following Smatch static checker warning: fs/xfs/scrub/alloc_repair.c:781 xrep_abt_build_new_trees() warn: missing unwind goto?" That's a bug, so let's fix it. Reported-by: Dan Carpenter Fixes: 4bdfd7d15747 ("xfs: repair free space btrees") Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/alloc_repair.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/xfs/scrub/alloc_repair.c b/fs/xfs/scrub/alloc_repair.c index d421b253923ed..30295898cc8a6 100644 --- a/fs/xfs/scrub/alloc_repair.c +++ b/fs/xfs/scrub/alloc_repair.c @@ -778,7 +778,7 @@ xrep_abt_build_new_trees( error = xrep_bnobt_sort_records(ra); if (error) - return error; + goto err_levels; /* Load the free space by block number tree. */ ra->array_cur = XFARRAY_CURSOR_INIT; From 5302a5c8beb21d01b7b8d92cc73b6871bc27d7bf Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:06 -0700 Subject: [PATCH 0194/1702] xfs: only clear log incompat flags at clean unmount While reviewing the online fsck patchset, someone spied the xfs_swapext_can_use_without_log_assistance function and wondered why we go through this inverted-bitmask dance to avoid setting the XFS_SB_FEAT_INCOMPAT_LOG_SWAPEXT feature. (The same principles apply to the logged extended attribute update feature bit in the since-merged LARP series.) The reason for this dance is that xfs_add_incompat_log_feature is an expensive operation -- it forces the log, pushes the AIL, and then if nobody's beaten us to it, sets the feature bit and issues a synchronous write of the primary superblock. That could be a one-time cost amortized over the life of the filesystem, but the log quiesce and cover operations call xfs_clear_incompat_log_features to remove feature bits opportunistically. On a moderately loaded filesystem this leads to us cycling those bits on and off over and over, which hurts performance. Why do we clear the log incompat bits? Back in ~2020 I think Dave and I had a conversation on IRC[2] about what the log incompat bits represent. IIRC in that conversation we decided that the log incompat bits protect unrecovered log items so that old kernels won't try to recover them and barf. Since a clean log has no protected log items, we could clear the bits at cover/quiesce time. As Dave Chinner pointed out in the thread, clearing log incompat bits at unmount time has positive effects for golden root disk image generator setups, since the generator could be running a newer kernel than what gets written to the golden image -- if there are log incompat fields set in the golden image that was generated by a newer kernel/OS image builder then the provisioning host cannot mount the filesystem even though the log is clean and recovery is unnecessary to mount the filesystem. Given that it's expensive to set log incompat bits, we really only want to do that once per bit per mount. Therefore, I propose that we only clear log incompat bits as part of writing a clean unmount record. Do this by adding an operational state flag to the xfs mount that guards whether or not the feature bit clearing can actually take place. This eliminates the l_incompat_users rwsem that we use to protect a log cleaning operation from clearing a feature bit that a frontend thread is trying to set -- this lock adds another way to fail w.r.t. locking. For the swapext series, I shard that into multiple locks just to work around the lockdep complaints, and that's fugly. Link: https://lore.kernel.org/linux-xfs/20240131230043.GA6180@frogsfrogsfrogs/ Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Dave Chinner --- .../xfs/xfs-online-fsck-design.rst | 3 -- fs/xfs/xfs_log.c | 28 +------------ fs/xfs/xfs_log.h | 2 - fs/xfs/xfs_log_priv.h | 3 -- fs/xfs/xfs_log_recover.c | 15 ------- fs/xfs/xfs_mount.c | 8 +++- fs/xfs/xfs_mount.h | 6 ++- fs/xfs/xfs_xattr.c | 42 +++---------------- 8 files changed, 19 insertions(+), 88 deletions(-) diff --git a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst index 6333697ba3e82..1d161752f09ed 100644 --- a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst +++ b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst @@ -4047,9 +4047,6 @@ series. | one ``struct rw_semaphore`` for each feature. | | The log cleaning code tries to take this rwsem in exclusive mode to | | clear the bit; if the lock attempt fails, the feature bit remains set. | -| Filesystem code signals its intention to use a log incompat feature in a | -| transaction by calling ``xlog_use_incompat_feat``, which takes the rwsem | -| in shared mode. | | The code supporting a log incompat feature should create wrapper | | functions to obtain the log feature and call | | ``xfs_add_incompat_log_feature`` to set the feature bits in the primary | diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 5004f23d344ed..416c154949832 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -1448,7 +1448,7 @@ xfs_log_work_queue( * Clear the log incompat flags if we have the opportunity. * * This only happens if we're about to log the second dummy transaction as part - * of covering the log and we can get the log incompat feature usage lock. + * of covering the log. */ static inline void xlog_clear_incompat( @@ -1463,11 +1463,7 @@ xlog_clear_incompat( if (log->l_covered_state != XLOG_STATE_COVER_DONE2) return; - if (!down_write_trylock(&log->l_incompat_users)) - return; - xfs_clear_incompat_log_features(mp); - up_write(&log->l_incompat_users); } /* @@ -1585,8 +1581,6 @@ xlog_alloc_log( } log->l_sectBBsize = 1 << log2_size; - init_rwsem(&log->l_incompat_users); - xlog_get_iclog_buffer_size(mp, log); spin_lock_init(&log->l_icloglock); @@ -3871,23 +3865,3 @@ xfs_log_check_lsn( return valid; } - -/* - * Notify the log that we're about to start using a feature that is protected - * by a log incompat feature flag. This will prevent log covering from - * clearing those flags. - */ -void -xlog_use_incompat_feat( - struct xlog *log) -{ - down_read(&log->l_incompat_users); -} - -/* Notify the log that we've finished using log incompat features. */ -void -xlog_drop_incompat_feat( - struct xlog *log) -{ - up_read(&log->l_incompat_users); -} diff --git a/fs/xfs/xfs_log.h b/fs/xfs/xfs_log.h index 2728886c29639..d69acf881153d 100644 --- a/fs/xfs/xfs_log.h +++ b/fs/xfs/xfs_log.h @@ -159,8 +159,6 @@ bool xfs_log_check_lsn(struct xfs_mount *, xfs_lsn_t); xfs_lsn_t xlog_grant_push_threshold(struct xlog *log, int need_bytes); bool xlog_force_shutdown(struct xlog *log, uint32_t shutdown_flags); -void xlog_use_incompat_feat(struct xlog *log); -void xlog_drop_incompat_feat(struct xlog *log); int xfs_attr_use_log_assist(struct xfs_mount *mp); #endif /* __XFS_LOG_H__ */ diff --git a/fs/xfs/xfs_log_priv.h b/fs/xfs/xfs_log_priv.h index e30c06ec20e33..43881575cd498 100644 --- a/fs/xfs/xfs_log_priv.h +++ b/fs/xfs/xfs_log_priv.h @@ -450,9 +450,6 @@ struct xlog { xfs_lsn_t l_recovery_lsn; uint32_t l_iclog_roundoff;/* padding roundoff */ - - /* Users of log incompat features should take a read lock. */ - struct rw_semaphore l_incompat_users; }; /* diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 1b1f0a4cd4947..41aec991433c5 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -3496,21 +3496,6 @@ xlog_recover_finish( */ xfs_log_force(log->l_mp, XFS_LOG_SYNC); - /* - * Now that we've recovered the log and all the intents, we can clear - * the log incompat feature bits in the superblock because there's no - * longer anything to protect. We rely on the AIL push to write out the - * updated superblock after everything else. - */ - if (xfs_clear_incompat_log_features(log->l_mp)) { - error = xfs_sync_sb(log->l_mp, false); - if (error < 0) { - xfs_alert(log->l_mp, - "Failed to clear log incompat features on recovery"); - goto out_error; - } - } - xlog_recover_process_iunlinks(log); /* diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index df370eb5dc15e..d37ba10f5fa33 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -1095,6 +1095,11 @@ xfs_unmountfs( "Freespace may not be correct on next mount."); xfs_unmount_check(mp); + /* + * Indicate that it's ok to clear log incompat bits before cleaning + * the log and writing the unmount record. + */ + xfs_set_done_with_log_incompat(mp); xfs_log_unmount(mp); xfs_da_unmount(mp); xfs_uuid_unmount(mp); @@ -1364,7 +1369,8 @@ xfs_clear_incompat_log_features( if (!xfs_has_crc(mp) || !xfs_sb_has_incompat_log_feature(&mp->m_sb, XFS_SB_FEAT_INCOMPAT_LOG_ALL) || - xfs_is_shutdown(mp)) + xfs_is_shutdown(mp) || + !xfs_is_done_with_log_incompat(mp)) return false; /* diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index e880aa48de68b..6ec038b88454c 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -412,6 +412,8 @@ __XFS_HAS_FEAT(nouuid, NOUUID) #define XFS_OPSTATE_WARNED_LARP 9 /* Mount time quotacheck is running */ #define XFS_OPSTATE_QUOTACHECK_RUNNING 10 +/* Do we want to clear log incompat flags? */ +#define XFS_OPSTATE_UNSET_LOG_INCOMPAT 11 #define __XFS_IS_OPSTATE(name, NAME) \ static inline bool xfs_is_ ## name (struct xfs_mount *mp) \ @@ -439,6 +441,7 @@ __XFS_IS_OPSTATE(quotacheck_running, QUOTACHECK_RUNNING) #else # define xfs_is_quotacheck_running(mp) (false) #endif +__XFS_IS_OPSTATE(done_with_log_incompat, UNSET_LOG_INCOMPAT) static inline bool xfs_should_warn(struct xfs_mount *mp, long nr) @@ -457,7 +460,8 @@ xfs_should_warn(struct xfs_mount *mp, long nr) { (1UL << XFS_OPSTATE_WARNED_SCRUB), "wscrub" }, \ { (1UL << XFS_OPSTATE_WARNED_SHRINK), "wshrink" }, \ { (1UL << XFS_OPSTATE_WARNED_LARP), "wlarp" }, \ - { (1UL << XFS_OPSTATE_QUOTACHECK_RUNNING), "quotacheck" } + { (1UL << XFS_OPSTATE_QUOTACHECK_RUNNING), "quotacheck" }, \ + { (1UL << XFS_OPSTATE_UNSET_LOG_INCOMPAT), "unset_log_incompat" } /* * Max and min values for mount-option defined I/O diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c index 364104e1b38ae..4ebf7052eb673 100644 --- a/fs/xfs/xfs_xattr.c +++ b/fs/xfs/xfs_xattr.c @@ -22,10 +22,7 @@ /* * Get permission to use log-assisted atomic exchange of file extents. - * - * Callers must not be running any transactions or hold any inode locks, and - * they must release the permission by calling xlog_drop_incompat_feat - * when they're done. + * Callers must not be running any transactions or hold any ILOCKs. */ static inline int xfs_attr_grab_log_assist( @@ -33,16 +30,7 @@ xfs_attr_grab_log_assist( { int error = 0; - /* - * Protect ourselves from an idle log clearing the logged xattrs log - * incompat feature bit. - */ - xlog_use_incompat_feat(mp->m_log); - - /* - * If log-assisted xattrs are already enabled, the caller can use the - * log assisted swap functions with the log-incompat reference we got. - */ + /* xattr update log intent items are already enabled */ if (xfs_sb_version_haslogxattrs(&mp->m_sb)) return 0; @@ -52,31 +40,19 @@ xfs_attr_grab_log_assist( * a V5 filesystem for the superblock field, but we'll require rmap * or reflink to avoid having to deal with really old kernels. */ - if (!xfs_has_reflink(mp) && !xfs_has_rmapbt(mp)) { - error = -EOPNOTSUPP; - goto drop_incompat; - } + if (!xfs_has_reflink(mp) && !xfs_has_rmapbt(mp)) + return -EOPNOTSUPP; /* Enable log-assisted xattrs. */ error = xfs_add_incompat_log_feature(mp, XFS_SB_FEAT_INCOMPAT_LOG_XATTRS); if (error) - goto drop_incompat; + return error; xfs_warn_mount(mp, XFS_OPSTATE_WARNED_LARP, "EXPERIMENTAL logged extended attributes feature in use. Use at your own risk!"); return 0; -drop_incompat: - xlog_drop_incompat_feat(mp->m_log); - return error; -} - -static inline void -xfs_attr_rele_log_assist( - struct xfs_mount *mp) -{ - xlog_drop_incompat_feat(mp->m_log); } static inline bool @@ -100,7 +76,6 @@ xfs_attr_change( struct xfs_da_args *args) { struct xfs_mount *mp = args->dp->i_mount; - bool use_logging = false; int error; ASSERT(!(args->op_flags & XFS_DA_OP_LOGGED)); @@ -111,14 +86,9 @@ xfs_attr_change( return error; args->op_flags |= XFS_DA_OP_LOGGED; - use_logging = true; } - error = xfs_attr_set(args); - - if (use_logging) - xfs_attr_rele_log_assist(mp); - return error; + return xfs_attr_set(args); } From a4db266a705c5518f3049074fd233b6c57daab00 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:07 -0700 Subject: [PATCH 0195/1702] xfs: move inode lease breaking functions to xfs_inode.c The lease breaking functions operate at the scope of the entire VFS inode, not subranges of a file. Move them to xfs_inode.c since they're already declared in xfs_inode.h. This cleanup moves us closer to having xfs_FOO.h declare only the symbols in xfs_FOO.c. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_file.c | 61 --------------------------------------------- fs/xfs/xfs_inode.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_inode.h | 1 - 3 files changed, 62 insertions(+), 62 deletions(-) diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 632653e00906f..40b778415f5fc 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -861,67 +861,6 @@ xfs_file_write_iter( return xfs_file_buffered_write(iocb, from); } -static void -xfs_wait_dax_page( - struct inode *inode) -{ - struct xfs_inode *ip = XFS_I(inode); - - xfs_iunlock(ip, XFS_MMAPLOCK_EXCL); - schedule(); - xfs_ilock(ip, XFS_MMAPLOCK_EXCL); -} - -int -xfs_break_dax_layouts( - struct inode *inode, - bool *retry) -{ - struct page *page; - - xfs_assert_ilocked(XFS_I(inode), XFS_MMAPLOCK_EXCL); - - page = dax_layout_busy_page(inode->i_mapping); - if (!page) - return 0; - - *retry = true; - return ___wait_var_event(&page->_refcount, - atomic_read(&page->_refcount) == 1, TASK_INTERRUPTIBLE, - 0, 0, xfs_wait_dax_page(inode)); -} - -int -xfs_break_layouts( - struct inode *inode, - uint *iolock, - enum layout_break_reason reason) -{ - bool retry; - int error; - - xfs_assert_ilocked(XFS_I(inode), XFS_IOLOCK_SHARED | XFS_IOLOCK_EXCL); - - do { - retry = false; - switch (reason) { - case BREAK_UNMAP: - error = xfs_break_dax_layouts(inode, &retry); - if (error || retry) - break; - fallthrough; - case BREAK_WRITE: - error = xfs_break_leased_layouts(inode, iolock, &retry); - break; - default: - WARN_ON_ONCE(1); - error = -EINVAL; - } - } while (error == 0 && retry); - - return error; -} - /* Does this file, inode, or mount want synchronous writes? */ static inline bool xfs_file_sync_writes(struct file *filp) { diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 3e667a19b80ba..39e6f88e9691a 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -38,6 +38,7 @@ #include "xfs_ag.h" #include "xfs_log_priv.h" #include "xfs_health.h" +#include "xfs_pnfs.h" struct kmem_cache *xfs_inode_cache; @@ -3946,3 +3947,64 @@ xfs_inode_count_blocks( xfs_bmap_count_leaves(ifp, rblocks); *dblocks = ip->i_nblocks - *rblocks; } + +static void +xfs_wait_dax_page( + struct inode *inode) +{ + struct xfs_inode *ip = XFS_I(inode); + + xfs_iunlock(ip, XFS_MMAPLOCK_EXCL); + schedule(); + xfs_ilock(ip, XFS_MMAPLOCK_EXCL); +} + +int +xfs_break_dax_layouts( + struct inode *inode, + bool *retry) +{ + struct page *page; + + xfs_assert_ilocked(XFS_I(inode), XFS_MMAPLOCK_EXCL); + + page = dax_layout_busy_page(inode->i_mapping); + if (!page) + return 0; + + *retry = true; + return ___wait_var_event(&page->_refcount, + atomic_read(&page->_refcount) == 1, TASK_INTERRUPTIBLE, + 0, 0, xfs_wait_dax_page(inode)); +} + +int +xfs_break_layouts( + struct inode *inode, + uint *iolock, + enum layout_break_reason reason) +{ + bool retry; + int error; + + xfs_assert_ilocked(XFS_I(inode), XFS_IOLOCK_SHARED | XFS_IOLOCK_EXCL); + + do { + retry = false; + switch (reason) { + case BREAK_UNMAP: + error = xfs_break_dax_layouts(inode, &retry); + if (error || retry) + break; + fallthrough; + case BREAK_WRITE: + error = xfs_break_leased_layouts(inode, iolock, &retry); + break; + default: + WARN_ON_ONCE(1); + error = -EINVAL; + } + } while (error == 0 && retry); + + return error; +} diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index ab46ffb3ac19e..5164c5d3e549c 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -565,7 +565,6 @@ xfs_itruncate_extents( return xfs_itruncate_extents_flags(tpp, ip, whichfork, new_size, 0); } -/* from xfs_file.c */ int xfs_break_dax_layouts(struct inode *inode, bool *retry); int xfs_break_layouts(struct inode *inode, uint *iolock, enum layout_break_reason reason); From 3fc4844585c761e69654a237df41ffc5e51262e9 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:08 -0700 Subject: [PATCH 0196/1702] xfs: move xfs_iops.c declarations out of xfs_inode.h Similarly, move declarations of public symbols of xfs_iops.c from xfs_inode.h to xfs_iops.h. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_inode.h | 5 ----- fs/xfs/xfs_iops.h | 4 ++++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index 5164c5d3e549c..b2dde0e0f265a 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -569,11 +569,6 @@ int xfs_break_dax_layouts(struct inode *inode, bool *retry); int xfs_break_layouts(struct inode *inode, uint *iolock, enum layout_break_reason reason); -/* from xfs_iops.c */ -extern void xfs_setup_inode(struct xfs_inode *ip); -extern void xfs_setup_iops(struct xfs_inode *ip); -extern void xfs_diflags_to_iflags(struct xfs_inode *ip, bool init); - static inline void xfs_update_stable_writes(struct xfs_inode *ip) { if (bdev_stable_writes(xfs_inode_buftarg(ip)->bt_bdev)) diff --git a/fs/xfs/xfs_iops.h b/fs/xfs/xfs_iops.h index 7f84a0843b243..8a38c3e2ed0e8 100644 --- a/fs/xfs/xfs_iops.h +++ b/fs/xfs/xfs_iops.h @@ -19,4 +19,8 @@ int xfs_vn_setattr_size(struct mnt_idmap *idmap, int xfs_inode_init_security(struct inode *inode, struct inode *dir, const struct qstr *qstr); +extern void xfs_setup_inode(struct xfs_inode *ip); +extern void xfs_setup_iops(struct xfs_inode *ip); +extern void xfs_diflags_to_iflags(struct xfs_inode *ip, bool init); + #endif /* __XFS_IOPS_H__ */ From 00acb28d96746f78389f23a7b5309a917b45c12f Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:09 -0700 Subject: [PATCH 0197/1702] xfs: declare xfs_file.c symbols in xfs_file.h Move the two public symbols in xfs_file.c to xfs_file.h. We're about to add more public symbols in that source file, so let's finally create the header file. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_file.c | 1 + fs/xfs/xfs_file.h | 12 ++++++++++++ fs/xfs/xfs_ioctl.c | 1 + fs/xfs/xfs_iops.c | 1 + fs/xfs/xfs_iops.h | 3 --- 5 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 fs/xfs/xfs_file.h diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 40b778415f5fc..9961d4b5efbe6 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -24,6 +24,7 @@ #include "xfs_pnfs.h" #include "xfs_iomap.h" #include "xfs_reflink.h" +#include "xfs_file.h" #include #include diff --git a/fs/xfs/xfs_file.h b/fs/xfs/xfs_file.h new file mode 100644 index 0000000000000..7d39e3eca56dc --- /dev/null +++ b/fs/xfs/xfs_file.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_FILE_H__ +#define __XFS_FILE_H__ + +extern const struct file_operations xfs_file_operations; +extern const struct file_operations xfs_dir_file_operations; + +#endif /* __XFS_FILE_H__ */ diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index d0e2cec6210dd..1397edea20f19 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -39,6 +39,7 @@ #include "xfs_ioctl.h" #include "xfs_xattr.h" #include "xfs_rtbitmap.h" +#include "xfs_file.h" #include #include diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 66f8c47642e88..55ed2d1023d67 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -25,6 +25,7 @@ #include "xfs_error.h" #include "xfs_ioctl.h" #include "xfs_xattr.h" +#include "xfs_file.h" #include #include diff --git a/fs/xfs/xfs_iops.h b/fs/xfs/xfs_iops.h index 8a38c3e2ed0e8..3c1a2605ffd2b 100644 --- a/fs/xfs/xfs_iops.h +++ b/fs/xfs/xfs_iops.h @@ -8,9 +8,6 @@ struct xfs_inode; -extern const struct file_operations xfs_file_operations; -extern const struct file_operations xfs_dir_file_operations; - extern ssize_t xfs_vn_listxattr(struct dentry *, char *data, size_t size); int xfs_vn_setattr_size(struct mnt_idmap *idmap, From ee20808d848c87a51e176706d81b95a21747d6cf Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:10 -0700 Subject: [PATCH 0198/1702] xfs: create a new helper to return a file's allocation unit Create a new helper function to calculate the fundamental allocation unit (i.e. the smallest unit of space we can allocate) of a file. Things are going to get hairy with range-exchange on the realtime device, so prepare for this now. Remove the static attribute from xfs_is_falloc_aligned since the next patch will need it. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_file.c | 32 ++++++++++++-------------------- fs/xfs/xfs_file.h | 3 +++ fs/xfs/xfs_inode.c | 13 +++++++++++++ fs/xfs/xfs_inode.h | 1 + 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 9961d4b5efbe6..64278f8acaeee 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -39,33 +39,25 @@ static const struct vm_operations_struct xfs_file_vm_ops; * Decide if the given file range is aligned to the size of the fundamental * allocation unit for the file. */ -static bool +bool xfs_is_falloc_aligned( struct xfs_inode *ip, loff_t pos, long long int len) { - struct xfs_mount *mp = ip->i_mount; - uint64_t mask; - - if (XFS_IS_REALTIME_INODE(ip)) { - if (!is_power_of_2(mp->m_sb.sb_rextsize)) { - u64 rextbytes; - u32 mod; - - rextbytes = XFS_FSB_TO_B(mp, mp->m_sb.sb_rextsize); - div_u64_rem(pos, rextbytes, &mod); - if (mod) - return false; - div_u64_rem(len, rextbytes, &mod); - return mod == 0; - } - mask = XFS_FSB_TO_B(mp, mp->m_sb.sb_rextsize) - 1; - } else { - mask = mp->m_sb.sb_blocksize - 1; + unsigned int alloc_unit = xfs_inode_alloc_unitsize(ip); + + if (!is_power_of_2(alloc_unit)) { + u32 mod; + + div_u64_rem(pos, alloc_unit, &mod); + if (mod) + return false; + div_u64_rem(len, alloc_unit, &mod); + return mod == 0; } - return !((pos | len) & mask); + return !((pos | len) & (alloc_unit - 1)); } /* diff --git a/fs/xfs/xfs_file.h b/fs/xfs/xfs_file.h index 7d39e3eca56dc..2ad91f755caf3 100644 --- a/fs/xfs/xfs_file.h +++ b/fs/xfs/xfs_file.h @@ -9,4 +9,7 @@ extern const struct file_operations xfs_file_operations; extern const struct file_operations xfs_dir_file_operations; +bool xfs_is_falloc_aligned(struct xfs_inode *ip, loff_t pos, + long long int len); + #endif /* __XFS_FILE_H__ */ diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 39e6f88e9691a..492dae0efad24 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -4008,3 +4008,16 @@ xfs_break_layouts( return error; } + +/* Returns the size of fundamental allocation unit for a file, in bytes. */ +unsigned int +xfs_inode_alloc_unitsize( + struct xfs_inode *ip) +{ + unsigned int blocks = 1; + + if (XFS_IS_REALTIME_INODE(ip)) + blocks = ip->i_mount->m_sb.sb_rextsize; + + return XFS_FSB_TO_B(ip->i_mount, blocks); +} diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index b2dde0e0f265a..fa3e605901e24 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -625,6 +625,7 @@ int xfs_inode_reload_unlinked(struct xfs_inode *ip); bool xfs_ifork_zapped(const struct xfs_inode *ip, int whichfork); void xfs_inode_count_blocks(struct xfs_trans *tp, struct xfs_inode *ip, xfs_filblks_t *dblocks, xfs_filblks_t *rblocks); +unsigned int xfs_inode_alloc_unitsize(struct xfs_inode *ip); struct xfs_dir_update_params { const struct xfs_inode *dp; From 6b700a5be9b3b69419474622336c63fdc1cc3ca4 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:11 -0700 Subject: [PATCH 0199/1702] xfs: hoist multi-fsb allocation unit detection to a helper Replace the open-coded logic to decide if a file has a multi-fsb allocation unit to a helper to make the code easier to read. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_bmap_util.c | 4 ++-- fs/xfs/xfs_inode.h | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index 19e11d1da6607..53aa90a0ee3a8 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -542,7 +542,7 @@ xfs_can_free_eofblocks( * forever. */ end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)XFS_ISIZE(ip)); - if (XFS_IS_REALTIME_INODE(ip) && mp->m_sb.sb_rextsize > 1) + if (xfs_inode_has_bigrtalloc(ip)) end_fsb = xfs_rtb_roundup_rtx(mp, end_fsb); last_fsb = XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes); if (last_fsb <= end_fsb) @@ -843,7 +843,7 @@ xfs_free_file_space( endoffset_fsb = XFS_B_TO_FSBT(mp, offset + len); /* We can only free complete realtime extents. */ - if (XFS_IS_REALTIME_INODE(ip) && mp->m_sb.sb_rextsize > 1) { + if (xfs_inode_has_bigrtalloc(ip)) { startoffset_fsb = xfs_rtb_roundup_rtx(mp, startoffset_fsb); endoffset_fsb = xfs_rtb_rounddown_rtx(mp, endoffset_fsb); } diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index fa3e605901e24..f559e68ee7077 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -311,6 +311,15 @@ static inline bool xfs_inode_has_large_extent_counts(struct xfs_inode *ip) return ip->i_diflags2 & XFS_DIFLAG2_NREXT64; } +/* + * Decide if this file is a realtime file whose data allocation unit is larger + * than a single filesystem block. + */ +static inline bool xfs_inode_has_bigrtalloc(struct xfs_inode *ip) +{ + return XFS_IS_REALTIME_INODE(ip) && ip->i_mount->m_sb.sb_rextsize > 1; +} + /* * Return the buftarg used for data allocations on a given inode. */ From ac5cebeed61351c0a60c65bd20c70120469c46ff Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:12 -0700 Subject: [PATCH 0200/1702] xfs: refactor non-power-of-two alignment checks Create a helper function that can compute if a 64-bit number is an integer multiple of a 32-bit number, where the 32-bit number is not required to be an even power of two. This is needed for some new code for the realtime device, where we can set 37k allocation units and then have to remap them. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_file.c | 12 +++--------- fs/xfs/xfs_linux.h | 5 +++++ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 64278f8acaeee..d1d4158441bd9 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -47,15 +47,9 @@ xfs_is_falloc_aligned( { unsigned int alloc_unit = xfs_inode_alloc_unitsize(ip); - if (!is_power_of_2(alloc_unit)) { - u32 mod; - - div_u64_rem(pos, alloc_unit, &mod); - if (mod) - return false; - div_u64_rem(len, alloc_unit, &mod); - return mod == 0; - } + if (!is_power_of_2(alloc_unit)) + return isaligned_64(pos, alloc_unit) && + isaligned_64(len, alloc_unit); return !((pos | len) & (alloc_unit - 1)); } diff --git a/fs/xfs/xfs_linux.h b/fs/xfs/xfs_linux.h index 8f07c9f6157fb..ac355328121ac 100644 --- a/fs/xfs/xfs_linux.h +++ b/fs/xfs/xfs_linux.h @@ -198,6 +198,11 @@ static inline uint64_t howmany_64(uint64_t x, uint32_t y) return x; } +static inline bool isaligned_64(uint64_t x, uint32_t y) +{ + return do_div(x, y) == 0; +} + /* If @b is a power of 2, return log2(b). Else return -1. */ static inline int8_t log2_if_power2(unsigned long b) { From 15f78aa3eb07645e7bef15a53b4ae1c757907d2c Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:12 -0700 Subject: [PATCH 0201/1702] xfs: constify xfs_bmap_is_written_extent This predicate doesn't modify the structure that's being passed in, so we can mark it const. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_bmap.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_bmap.h b/fs/xfs/libxfs/xfs_bmap.h index f7662595309d8..b8bdbf1560e65 100644 --- a/fs/xfs/libxfs/xfs_bmap.h +++ b/fs/xfs/libxfs/xfs_bmap.h @@ -158,7 +158,7 @@ static inline bool xfs_bmap_is_real_extent(const struct xfs_bmbt_irec *irec) * Return true if the extent is a real, allocated extent, or false if it is a * delayed allocation, and unwritten extent or a hole. */ -static inline bool xfs_bmap_is_written_extent(struct xfs_bmbt_irec *irec) +static inline bool xfs_bmap_is_written_extent(const struct xfs_bmbt_irec *irec) { return xfs_bmap_is_real_extent(irec) && irec->br_state != XFS_EXT_UNWRITTEN; From 5b9932f6001c70b984e8c9c2fe09e443beb4baba Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:13 -0700 Subject: [PATCH 0202/1702] vfs: export remap and write check helpers Export these functions so that the next patch can use them to check the file ranges being passed to the XFS_IOC_EXCHANGE_RANGE operation. Cc: linux-fsdevel@vger.kernel.org Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/read_write.c | 1 + fs/remap_range.c | 4 ++-- include/linux/fs.h | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/read_write.c b/fs/read_write.c index d4c036e82b6c3..85c096f2c0d06 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -1667,6 +1667,7 @@ int generic_write_check_limits(struct file *file, loff_t pos, loff_t *count) return 0; } +EXPORT_SYMBOL_GPL(generic_write_check_limits); /* Like generic_write_checks(), but takes size of write instead of iter. */ int generic_write_checks_count(struct kiocb *iocb, loff_t *count) diff --git a/fs/remap_range.c b/fs/remap_range.c index de07f978ce3eb..28246dfc84851 100644 --- a/fs/remap_range.c +++ b/fs/remap_range.c @@ -99,8 +99,7 @@ static int generic_remap_checks(struct file *file_in, loff_t pos_in, return 0; } -static int remap_verify_area(struct file *file, loff_t pos, loff_t len, - bool write) +int remap_verify_area(struct file *file, loff_t pos, loff_t len, bool write) { int mask = write ? MAY_WRITE : MAY_READ; loff_t tmp; @@ -118,6 +117,7 @@ static int remap_verify_area(struct file *file, loff_t pos, loff_t len, return fsnotify_file_area_perm(file, mask, &pos, len); } +EXPORT_SYMBOL_GPL(remap_verify_area); /* * Ensure that we don't remap a partial EOF block in the middle of something diff --git a/include/linux/fs.h b/include/linux/fs.h index 8dfd53b52744a..0835faeebe7b3 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2119,6 +2119,7 @@ extern ssize_t vfs_read(struct file *, char __user *, size_t, loff_t *); extern ssize_t vfs_write(struct file *, const char __user *, size_t, loff_t *); extern ssize_t vfs_copy_file_range(struct file *, loff_t , struct file *, loff_t, size_t, unsigned int); +int remap_verify_area(struct file *file, loff_t pos, loff_t len, bool write); int __generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, struct file *file_out, loff_t pos_out, loff_t *len, unsigned int remap_flags, From 9a64d9b3109d01cca0b83c1d36538b7a37c5284e Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:14 -0700 Subject: [PATCH 0203/1702] xfs: introduce new file range exchange ioctl Introduce a new ioctl to handle exchanging ranges of bytes between files. The goal here is to perform the exchange atomically with respect to applications -- either they see the file contents before the exchange or they see that A-B is now B-A, even if the kernel crashes. My original goal with all this code was to make it so that online repair can build a replacement directory or xattr structure in a temporary file and commit the repair by atomically exchanging all the data blocks between the two files. However, I needed a way to test this mechanism thoroughly, so I've been evolving an ioctl interface since then. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/libxfs/xfs_fs.h | 41 +++++ fs/xfs/xfs_exchrange.c | 339 +++++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_exchrange.h | 30 ++++ fs/xfs/xfs_ioctl.c | 4 + 5 files changed, 415 insertions(+) create mode 100644 fs/xfs/xfs_exchrange.c create mode 100644 fs/xfs/xfs_exchrange.h diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index 76674ad5833e3..2474242f5a05f 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -67,6 +67,7 @@ xfs-y += xfs_aops.o \ xfs_dir2_readdir.o \ xfs_discard.o \ xfs_error.o \ + xfs_exchrange.o \ xfs_export.o \ xfs_extent_busy.o \ xfs_file.o \ diff --git a/fs/xfs/libxfs/xfs_fs.h b/fs/xfs/libxfs/xfs_fs.h index ca1b17d014377..8a1e30cf4dc88 100644 --- a/fs/xfs/libxfs/xfs_fs.h +++ b/fs/xfs/libxfs/xfs_fs.h @@ -772,6 +772,46 @@ struct xfs_scrub_metadata { # define XFS_XATTR_LIST_MAX 65536 #endif +/* + * Exchange part of file1 with part of the file that this ioctl that is being + * called against (which we'll call file2). Filesystems must be able to + * restart and complete the operation even after the system goes down. + */ +struct xfs_exchange_range { + __s32 file1_fd; + __u32 pad; /* must be zeroes */ + __u64 file1_offset; /* file1 offset, bytes */ + __u64 file2_offset; /* file2 offset, bytes */ + __u64 length; /* bytes to exchange */ + + __u64 flags; /* see XFS_EXCHANGE_RANGE_* below */ +}; + +/* + * Exchange file data all the way to the ends of both files, and then exchange + * the file sizes. This flag can be used to replace a file's contents with a + * different amount of data. length will be ignored. + */ +#define XFS_EXCHANGE_RANGE_TO_EOF (1ULL << 0) + +/* Flush all changes in file data and file metadata to disk before returning. */ +#define XFS_EXCHANGE_RANGE_DSYNC (1ULL << 1) + +/* Dry run; do all the parameter verification but do not change anything. */ +#define XFS_EXCHANGE_RANGE_DRY_RUN (1ULL << 2) + +/* + * Exchange only the parts of the two files where the file allocation units + * mapped to file1's range have been written to. This can accelerate + * scatter-gather atomic writes with a temp file if all writes are aligned to + * the file allocation unit. + */ +#define XFS_EXCHANGE_RANGE_FILE1_WRITTEN (1ULL << 3) + +#define XFS_EXCHANGE_RANGE_ALL_FLAGS (XFS_EXCHANGE_RANGE_TO_EOF | \ + XFS_EXCHANGE_RANGE_DSYNC | \ + XFS_EXCHANGE_RANGE_DRY_RUN | \ + XFS_EXCHANGE_RANGE_FILE1_WRITTEN) /* * ioctl commands that are used by Linux filesystems @@ -843,6 +883,7 @@ struct xfs_scrub_metadata { #define XFS_IOC_FSGEOMETRY _IOR ('X', 126, struct xfs_fsop_geom) #define XFS_IOC_BULKSTAT _IOR ('X', 127, struct xfs_bulkstat_req) #define XFS_IOC_INUMBERS _IOR ('X', 128, struct xfs_inumbers_req) +#define XFS_IOC_EXCHANGE_RANGE _IOWR('X', 129, struct xfs_exchange_range) /* XFS_IOC_GETFSUUID ---------- deprecated 140 */ diff --git a/fs/xfs/xfs_exchrange.c b/fs/xfs/xfs_exchrange.c new file mode 100644 index 0000000000000..4cd824e47f751 --- /dev/null +++ b/fs/xfs/xfs_exchrange.c @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_exchrange.h" +#include + +/* + * Generic code for exchanging ranges of two files via XFS_IOC_EXCHANGE_RANGE. + * This part deals with struct file objects and byte ranges and does not deal + * with XFS-specific data structures such as xfs_inodes and block ranges. This + * separation may some day facilitate porting to another filesystem. + * + * The goal is to exchange fxr.length bytes starting at fxr.file1_offset in + * file1 with the same number of bytes starting at fxr.file2_offset in file2. + * Implementations must call xfs_exchange_range_prep to prepare the two + * files prior to taking locks; and they must update the inode change and mod + * times of both files as part of the metadata update. The timestamp update + * and freshness checks must be done atomically as part of the data exchange + * operation to ensure correctness of the freshness check. + * xfs_exchange_range_finish must be called after the operation completes + * successfully but before locks are dropped. + */ + +/* Verify that we have security clearance to perform this operation. */ +static int +xfs_exchange_range_verify_area( + struct xfs_exchrange *fxr) +{ + int ret; + + ret = remap_verify_area(fxr->file1, fxr->file1_offset, fxr->length, + true); + if (ret) + return ret; + + return remap_verify_area(fxr->file2, fxr->file2_offset, fxr->length, + true); +} + +/* + * Performs necessary checks before doing a range exchange, having stabilized + * mutable inode attributes via i_rwsem. + */ +static inline int +xfs_exchange_range_checks( + struct xfs_exchrange *fxr, + unsigned int alloc_unit) +{ + struct inode *inode1 = file_inode(fxr->file1); + struct inode *inode2 = file_inode(fxr->file2); + uint64_t allocmask = alloc_unit - 1; + int64_t test_len; + uint64_t blen; + loff_t size1, size2, tmp; + int error; + + /* Don't touch certain kinds of inodes */ + if (IS_IMMUTABLE(inode1) || IS_IMMUTABLE(inode2)) + return -EPERM; + if (IS_SWAPFILE(inode1) || IS_SWAPFILE(inode2)) + return -ETXTBSY; + + size1 = i_size_read(inode1); + size2 = i_size_read(inode2); + + /* Ranges cannot start after EOF. */ + if (fxr->file1_offset > size1 || fxr->file2_offset > size2) + return -EINVAL; + + /* + * If the caller said to exchange to EOF, we set the length of the + * request large enough to cover everything to the end of both files. + */ + if (fxr->flags & XFS_EXCHANGE_RANGE_TO_EOF) { + fxr->length = max_t(int64_t, size1 - fxr->file1_offset, + size2 - fxr->file2_offset); + + error = xfs_exchange_range_verify_area(fxr); + if (error) + return error; + } + + /* + * The start of both ranges must be aligned to the file allocation + * unit. + */ + if (!IS_ALIGNED(fxr->file1_offset, alloc_unit) || + !IS_ALIGNED(fxr->file2_offset, alloc_unit)) + return -EINVAL; + + /* Ensure offsets don't wrap. */ + if (check_add_overflow(fxr->file1_offset, fxr->length, &tmp) || + check_add_overflow(fxr->file2_offset, fxr->length, &tmp)) + return -EINVAL; + + /* + * We require both ranges to end within EOF, unless we're exchanging + * to EOF. + */ + if (!(fxr->flags & XFS_EXCHANGE_RANGE_TO_EOF) && + (fxr->file1_offset + fxr->length > size1 || + fxr->file2_offset + fxr->length > size2)) + return -EINVAL; + + /* + * Make sure we don't hit any file size limits. If we hit any size + * limits such that test_length was adjusted, we abort the whole + * operation. + */ + test_len = fxr->length; + error = generic_write_check_limits(fxr->file2, fxr->file2_offset, + &test_len); + if (error) + return error; + error = generic_write_check_limits(fxr->file1, fxr->file1_offset, + &test_len); + if (error) + return error; + if (test_len != fxr->length) + return -EINVAL; + + /* + * If the user wanted us to exchange up to the infile's EOF, round up + * to the next allocation unit boundary for this check. Do the same + * for the outfile. + * + * Otherwise, reject the range length if it's not aligned to an + * allocation unit. + */ + if (fxr->file1_offset + fxr->length == size1) + blen = ALIGN(size1, alloc_unit) - fxr->file1_offset; + else if (fxr->file2_offset + fxr->length == size2) + blen = ALIGN(size2, alloc_unit) - fxr->file2_offset; + else if (!IS_ALIGNED(fxr->length, alloc_unit)) + return -EINVAL; + else + blen = fxr->length; + + /* Don't allow overlapped exchanges within the same file. */ + if (inode1 == inode2 && + fxr->file2_offset + blen > fxr->file1_offset && + fxr->file1_offset + blen > fxr->file2_offset) + return -EINVAL; + + /* + * Ensure that we don't exchange a partial EOF block into the middle of + * another file. + */ + if ((fxr->length & allocmask) == 0) + return 0; + + blen = fxr->length; + if (fxr->file2_offset + blen < size2) + blen &= ~allocmask; + + if (fxr->file1_offset + blen < size1) + blen &= ~allocmask; + + return blen == fxr->length ? 0 : -EINVAL; +} + +/* + * Check that the two inodes are eligible for range exchanges, the ranges make + * sense, and then flush all dirty data. Caller must ensure that the inodes + * have been locked against any other modifications. + */ +static inline int +xfs_exchange_range_prep( + struct xfs_exchrange *fxr, + unsigned int alloc_unit) +{ + struct inode *inode1 = file_inode(fxr->file1); + struct inode *inode2 = file_inode(fxr->file2); + bool same_inode = (inode1 == inode2); + int error; + + /* Check that we don't violate system file offset limits. */ + error = xfs_exchange_range_checks(fxr, alloc_unit); + if (error || fxr->length == 0) + return error; + + /* Wait for the completion of any pending IOs on both files */ + inode_dio_wait(inode1); + if (!same_inode) + inode_dio_wait(inode2); + + error = filemap_write_and_wait_range(inode1->i_mapping, + fxr->file1_offset, + fxr->file1_offset + fxr->length - 1); + if (error) + return error; + + error = filemap_write_and_wait_range(inode2->i_mapping, + fxr->file2_offset, + fxr->file2_offset + fxr->length - 1); + if (error) + return error; + + /* + * If the files or inodes involved require synchronous writes, amend + * the request to force the filesystem to flush all data and metadata + * to disk after the operation completes. + */ + if (((fxr->file1->f_flags | fxr->file2->f_flags) & O_SYNC) || + IS_SYNC(inode1) || IS_SYNC(inode2)) + fxr->flags |= XFS_EXCHANGE_RANGE_DSYNC; + + return 0; +} + +/* + * Finish a range exchange operation, if it was successful. Caller must ensure + * that the inodes are still locked against any other modifications. + */ +static inline int +xfs_exchange_range_finish( + struct xfs_exchrange *fxr) +{ + int error; + + error = file_remove_privs(fxr->file1); + if (error) + return error; + if (file_inode(fxr->file1) == file_inode(fxr->file2)) + return 0; + + return file_remove_privs(fxr->file2); +} + +/* Exchange parts of two files. */ +static int +xfs_exchange_range( + struct xfs_exchrange *fxr) +{ + struct inode *inode1 = file_inode(fxr->file1); + struct inode *inode2 = file_inode(fxr->file2); + int ret; + + BUILD_BUG_ON(XFS_EXCHANGE_RANGE_ALL_FLAGS & + XFS_EXCHANGE_RANGE_PRIV_FLAGS); + + /* Both files must be on the same mount/filesystem. */ + if (fxr->file1->f_path.mnt != fxr->file2->f_path.mnt) + return -EXDEV; + + if (fxr->flags & ~XFS_EXCHANGE_RANGE_ALL_FLAGS) + return -EINVAL; + + /* Userspace requests only honored for regular files. */ + if (S_ISDIR(inode1->i_mode) || S_ISDIR(inode2->i_mode)) + return -EISDIR; + if (!S_ISREG(inode1->i_mode) || !S_ISREG(inode2->i_mode)) + return -EINVAL; + + /* Both files must be opened for read and write. */ + if (!(fxr->file1->f_mode & FMODE_READ) || + !(fxr->file1->f_mode & FMODE_WRITE) || + !(fxr->file2->f_mode & FMODE_READ) || + !(fxr->file2->f_mode & FMODE_WRITE)) + return -EBADF; + + /* Neither file can be opened append-only. */ + if ((fxr->file1->f_flags & O_APPEND) || + (fxr->file2->f_flags & O_APPEND)) + return -EBADF; + + /* + * If we're not exchanging to EOF, we can check the areas before + * stabilizing both files' i_size. + */ + if (!(fxr->flags & XFS_EXCHANGE_RANGE_TO_EOF)) { + ret = xfs_exchange_range_verify_area(fxr); + if (ret) + return ret; + } + + /* Update cmtime if the fd/inode don't forbid it. */ + if (!(fxr->file1->f_mode & FMODE_NOCMTIME) && !IS_NOCMTIME(inode1)) + fxr->flags |= __XFS_EXCHANGE_RANGE_UPD_CMTIME1; + if (!(fxr->file2->f_mode & FMODE_NOCMTIME) && !IS_NOCMTIME(inode2)) + fxr->flags |= __XFS_EXCHANGE_RANGE_UPD_CMTIME2; + + file_start_write(fxr->file2); + ret = -EOPNOTSUPP; /* XXX call out to lower level code */ + file_end_write(fxr->file2); + if (ret) + return ret; + + fsnotify_modify(fxr->file1); + if (fxr->file2 != fxr->file1) + fsnotify_modify(fxr->file2); + return 0; +} + +/* Collect exchange-range arguments from userspace. */ +long +xfs_ioc_exchange_range( + struct file *file, + struct xfs_exchange_range __user *argp) +{ + struct xfs_exchrange fxr = { + .file2 = file, + }; + struct xfs_exchange_range args; + struct fd file1; + int error; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + if (memchr_inv(&args.pad, 0, sizeof(args.pad))) + return -EINVAL; + if (args.flags & ~XFS_EXCHANGE_RANGE_ALL_FLAGS) + return -EINVAL; + + fxr.file1_offset = args.file1_offset; + fxr.file2_offset = args.file2_offset; + fxr.length = args.length; + fxr.flags = args.flags; + + file1 = fdget(args.file1_fd); + if (!file1.file) + return -EBADF; + fxr.file1 = file1.file; + + error = xfs_exchange_range(&fxr); + fdput(file1); + return error; +} diff --git a/fs/xfs/xfs_exchrange.h b/fs/xfs/xfs_exchrange.h new file mode 100644 index 0000000000000..f80369c7df5d6 --- /dev/null +++ b/fs/xfs/xfs_exchrange.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_EXCHRANGE_H__ +#define __XFS_EXCHRANGE_H__ + +/* Update the mtime/cmtime of file1 and file2 */ +#define __XFS_EXCHANGE_RANGE_UPD_CMTIME1 (1ULL << 63) +#define __XFS_EXCHANGE_RANGE_UPD_CMTIME2 (1ULL << 62) + +#define XFS_EXCHANGE_RANGE_PRIV_FLAGS (__XFS_EXCHANGE_RANGE_UPD_CMTIME1 | \ + __XFS_EXCHANGE_RANGE_UPD_CMTIME2) + +struct xfs_exchrange { + struct file *file1; + struct file *file2; + + loff_t file1_offset; + loff_t file2_offset; + u64 length; + + u64 flags; /* XFS_EXCHANGE_RANGE flags */ +}; + +long xfs_ioc_exchange_range(struct file *file, + struct xfs_exchange_range __user *argp); + +#endif /* __XFS_EXCHRANGE_H__ */ diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 1397edea20f19..efa95892655d4 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -40,6 +40,7 @@ #include "xfs_xattr.h" #include "xfs_rtbitmap.h" #include "xfs_file.h" +#include "xfs_exchrange.h" #include #include @@ -2170,6 +2171,9 @@ xfs_file_ioctl( return error; } + case XFS_IOC_EXCHANGE_RANGE: + return xfs_ioc_exchange_range(filp, arg); + default: return -ENOTTY; } From 1518646eef26c220e9256906260ecaaa64503522 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:15 -0700 Subject: [PATCH 0204/1702] xfs: create a incompat flag for atomic file mapping exchanges Create a incompat flag so that we only attempt to process file mapping exchange log items if the filesystem supports it, and a geometry flag to advertise support if it's present. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_format.h | 23 ++++++++++++----------- fs/xfs/libxfs/xfs_fs.h | 1 + fs/xfs/libxfs/xfs_sb.c | 5 +++++ fs/xfs/xfs_mount.h | 2 ++ fs/xfs/xfs_super.c | 4 ++++ 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h index 2b2f9050fbfbb..ff1e28316e1bb 100644 --- a/fs/xfs/libxfs/xfs_format.h +++ b/fs/xfs/libxfs/xfs_format.h @@ -367,18 +367,19 @@ xfs_sb_has_ro_compat_feature( return (sbp->sb_features_ro_compat & feature) != 0; } -#define XFS_SB_FEAT_INCOMPAT_FTYPE (1 << 0) /* filetype in dirent */ -#define XFS_SB_FEAT_INCOMPAT_SPINODES (1 << 1) /* sparse inode chunks */ -#define XFS_SB_FEAT_INCOMPAT_META_UUID (1 << 2) /* metadata UUID */ -#define XFS_SB_FEAT_INCOMPAT_BIGTIME (1 << 3) /* large timestamps */ -#define XFS_SB_FEAT_INCOMPAT_NEEDSREPAIR (1 << 4) /* needs xfs_repair */ -#define XFS_SB_FEAT_INCOMPAT_NREXT64 (1 << 5) /* large extent counters */ +#define XFS_SB_FEAT_INCOMPAT_FTYPE (1 << 0) /* filetype in dirent */ +#define XFS_SB_FEAT_INCOMPAT_SPINODES (1 << 1) /* sparse inode chunks */ +#define XFS_SB_FEAT_INCOMPAT_META_UUID (1 << 2) /* metadata UUID */ +#define XFS_SB_FEAT_INCOMPAT_BIGTIME (1 << 3) /* large timestamps */ +#define XFS_SB_FEAT_INCOMPAT_NEEDSREPAIR (1 << 4) /* needs xfs_repair */ +#define XFS_SB_FEAT_INCOMPAT_NREXT64 (1 << 5) /* large extent counters */ +#define XFS_SB_FEAT_INCOMPAT_EXCHRANGE (1 << 6) /* exchangerange supported */ #define XFS_SB_FEAT_INCOMPAT_ALL \ - (XFS_SB_FEAT_INCOMPAT_FTYPE| \ - XFS_SB_FEAT_INCOMPAT_SPINODES| \ - XFS_SB_FEAT_INCOMPAT_META_UUID| \ - XFS_SB_FEAT_INCOMPAT_BIGTIME| \ - XFS_SB_FEAT_INCOMPAT_NEEDSREPAIR| \ + (XFS_SB_FEAT_INCOMPAT_FTYPE | \ + XFS_SB_FEAT_INCOMPAT_SPINODES | \ + XFS_SB_FEAT_INCOMPAT_META_UUID | \ + XFS_SB_FEAT_INCOMPAT_BIGTIME | \ + XFS_SB_FEAT_INCOMPAT_NEEDSREPAIR | \ XFS_SB_FEAT_INCOMPAT_NREXT64) #define XFS_SB_FEAT_INCOMPAT_UNKNOWN ~XFS_SB_FEAT_INCOMPAT_ALL diff --git a/fs/xfs/libxfs/xfs_fs.h b/fs/xfs/libxfs/xfs_fs.h index 8a1e30cf4dc88..53526fca7386c 100644 --- a/fs/xfs/libxfs/xfs_fs.h +++ b/fs/xfs/libxfs/xfs_fs.h @@ -239,6 +239,7 @@ typedef struct xfs_fsop_resblks { #define XFS_FSOP_GEOM_FLAGS_BIGTIME (1 << 21) /* 64-bit nsec timestamps */ #define XFS_FSOP_GEOM_FLAGS_INOBTCNT (1 << 22) /* inobt btree counter */ #define XFS_FSOP_GEOM_FLAGS_NREXT64 (1 << 23) /* large extent counters */ +#define XFS_FSOP_GEOM_FLAGS_EXCHANGE_RANGE (1 << 24) /* exchange range */ /* * Minimum and maximum sizes need for growth checks. diff --git a/fs/xfs/libxfs/xfs_sb.c b/fs/xfs/libxfs/xfs_sb.c index 73a4b895de670..c350e259b6855 100644 --- a/fs/xfs/libxfs/xfs_sb.c +++ b/fs/xfs/libxfs/xfs_sb.c @@ -26,6 +26,7 @@ #include "xfs_health.h" #include "xfs_ag.h" #include "xfs_rtbitmap.h" +#include "xfs_exchrange.h" /* * Physical superblock buffer manipulations. Shared with libxfs in userspace. @@ -175,6 +176,8 @@ xfs_sb_version_to_features( features |= XFS_FEAT_NEEDSREPAIR; if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_NREXT64) features |= XFS_FEAT_NREXT64; + if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_EXCHRANGE) + features |= XFS_FEAT_EXCHANGE_RANGE; return features; } @@ -1259,6 +1262,8 @@ xfs_fs_geometry( } if (xfs_has_large_extent_counts(mp)) geo->flags |= XFS_FSOP_GEOM_FLAGS_NREXT64; + if (xfs_has_exchange_range(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_EXCHANGE_RANGE; geo->rtsectsize = sbp->sb_blocksize; geo->dirblocksize = xfs_dir2_dirblock_bytes(sbp); diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index 6ec038b88454c..b022e5120dc42 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -292,6 +292,7 @@ typedef struct xfs_mount { #define XFS_FEAT_BIGTIME (1ULL << 24) /* large timestamps */ #define XFS_FEAT_NEEDSREPAIR (1ULL << 25) /* needs xfs_repair */ #define XFS_FEAT_NREXT64 (1ULL << 26) /* large extent counters */ +#define XFS_FEAT_EXCHANGE_RANGE (1ULL << 27) /* exchange range */ /* Mount features */ #define XFS_FEAT_NOATTR2 (1ULL << 48) /* disable attr2 creation */ @@ -355,6 +356,7 @@ __XFS_HAS_FEAT(inobtcounts, INOBTCNT) __XFS_HAS_FEAT(bigtime, BIGTIME) __XFS_HAS_FEAT(needsrepair, NEEDSREPAIR) __XFS_HAS_FEAT(large_extent_counts, NREXT64) +__XFS_HAS_FEAT(exchange_range, EXCHANGE_RANGE) /* * Mount features diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index bce020374c5eb..dbda72df3419b 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -1727,6 +1727,10 @@ xfs_fs_fill_super( goto out_filestream_unmount; } + if (xfs_has_exchange_range(mp)) + xfs_warn(mp, + "EXPERIMENTAL exchange-range feature enabled. Use at your own risk!"); + error = xfs_mountfs(mp); if (error) goto out_filestream_unmount; From 6c08f434bd33f88cf169e9e43c7a5e42fb3f2118 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:16 -0700 Subject: [PATCH 0205/1702] xfs: introduce a file mapping exchange log intent item Introduce a new intent log item to handle exchanging mappings between the forks of two files. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/libxfs/xfs_log_format.h | 42 +++++- fs/xfs/libxfs/xfs_log_recover.h | 2 + fs/xfs/xfs_exchmaps_item.c | 235 ++++++++++++++++++++++++++++++++ fs/xfs/xfs_exchmaps_item.h | 59 ++++++++ fs/xfs/xfs_log_recover.c | 2 + fs/xfs/xfs_super.c | 19 +++ 7 files changed, 357 insertions(+), 3 deletions(-) create mode 100644 fs/xfs/xfs_exchmaps_item.c create mode 100644 fs/xfs/xfs_exchmaps_item.h diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index 2474242f5a05f..68ca9726e7b7d 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -102,6 +102,7 @@ xfs-y += xfs_log.o \ xfs_buf_item.o \ xfs_buf_item_recover.o \ xfs_dquot_item_recover.o \ + xfs_exchmaps_item.o \ xfs_extfree_item.o \ xfs_attr_item.o \ xfs_icreate_item.o \ diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h index 16872972e1e97..09024431cae9a 100644 --- a/fs/xfs/libxfs/xfs_log_format.h +++ b/fs/xfs/libxfs/xfs_log_format.h @@ -117,8 +117,9 @@ struct xfs_unmount_log_format { #define XLOG_REG_TYPE_ATTRD_FORMAT 28 #define XLOG_REG_TYPE_ATTR_NAME 29 #define XLOG_REG_TYPE_ATTR_VALUE 30 -#define XLOG_REG_TYPE_MAX 30 - +#define XLOG_REG_TYPE_XMI_FORMAT 31 +#define XLOG_REG_TYPE_XMD_FORMAT 32 +#define XLOG_REG_TYPE_MAX 32 /* * Flags to log operation header @@ -243,6 +244,8 @@ typedef struct xfs_trans_header { #define XFS_LI_BUD 0x1245 #define XFS_LI_ATTRI 0x1246 /* attr set/remove intent*/ #define XFS_LI_ATTRD 0x1247 /* attr set/remove done */ +#define XFS_LI_XMI 0x1248 /* mapping exchange intent */ +#define XFS_LI_XMD 0x1249 /* mapping exchange done */ #define XFS_LI_TYPE_DESC \ { XFS_LI_EFI, "XFS_LI_EFI" }, \ @@ -260,7 +263,9 @@ typedef struct xfs_trans_header { { XFS_LI_BUI, "XFS_LI_BUI" }, \ { XFS_LI_BUD, "XFS_LI_BUD" }, \ { XFS_LI_ATTRI, "XFS_LI_ATTRI" }, \ - { XFS_LI_ATTRD, "XFS_LI_ATTRD" } + { XFS_LI_ATTRD, "XFS_LI_ATTRD" }, \ + { XFS_LI_XMI, "XFS_LI_XMI" }, \ + { XFS_LI_XMD, "XFS_LI_XMD" } /* * Inode Log Item Format definitions. @@ -878,6 +883,37 @@ struct xfs_bud_log_format { uint64_t bud_bui_id; /* id of corresponding bui */ }; +/* + * XMI/XMD (file mapping exchange) log format definitions + */ + +/* This is the structure used to lay out an mapping exchange log item. */ +struct xfs_xmi_log_format { + uint16_t xmi_type; /* xmi log item type */ + uint16_t xmi_size; /* size of this item */ + uint32_t __pad; /* must be zero */ + uint64_t xmi_id; /* xmi identifier */ + + uint64_t xmi_inode1; /* inumber of first file */ + uint64_t xmi_inode2; /* inumber of second file */ + uint64_t xmi_startoff1; /* block offset into file1 */ + uint64_t xmi_startoff2; /* block offset into file2 */ + uint64_t xmi_blockcount; /* number of blocks */ + uint64_t xmi_flags; /* XFS_EXCHMAPS_* */ + uint64_t xmi_isize1; /* intended file1 size */ + uint64_t xmi_isize2; /* intended file2 size */ +}; + +#define XFS_EXCHMAPS_LOGGED_FLAGS (0) + +/* This is the structure used to lay out an mapping exchange done log item. */ +struct xfs_xmd_log_format { + uint16_t xmd_type; /* xmd log item type */ + uint16_t xmd_size; /* size of this item */ + uint32_t __pad; + uint64_t xmd_xmi_id; /* id of corresponding xmi */ +}; + /* * Dquot Log format definitions. * diff --git a/fs/xfs/libxfs/xfs_log_recover.h b/fs/xfs/libxfs/xfs_log_recover.h index 9fe7a9564bca9..47b758b49cb35 100644 --- a/fs/xfs/libxfs/xfs_log_recover.h +++ b/fs/xfs/libxfs/xfs_log_recover.h @@ -75,6 +75,8 @@ extern const struct xlog_recover_item_ops xlog_cui_item_ops; extern const struct xlog_recover_item_ops xlog_cud_item_ops; extern const struct xlog_recover_item_ops xlog_attri_item_ops; extern const struct xlog_recover_item_ops xlog_attrd_item_ops; +extern const struct xlog_recover_item_ops xlog_xmi_item_ops; +extern const struct xlog_recover_item_ops xlog_xmd_item_ops; /* * Macros, structures, prototypes for internal log manager use. diff --git a/fs/xfs/xfs_exchmaps_item.c b/fs/xfs/xfs_exchmaps_item.c new file mode 100644 index 0000000000000..65b0ade41b3d6 --- /dev/null +++ b/fs/xfs/xfs_exchmaps_item.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_shared.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_exchmaps_item.h" +#include "xfs_log.h" +#include "xfs_bmap.h" +#include "xfs_icache.h" +#include "xfs_trans_space.h" +#include "xfs_error.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" + +struct kmem_cache *xfs_xmi_cache; +struct kmem_cache *xfs_xmd_cache; + +static const struct xfs_item_ops xfs_xmi_item_ops; + +static inline struct xfs_xmi_log_item *XMI_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_xmi_log_item, xmi_item); +} + +STATIC void +xfs_xmi_item_free( + struct xfs_xmi_log_item *xmi_lip) +{ + kvfree(xmi_lip->xmi_item.li_lv_shadow); + kmem_cache_free(xfs_xmi_cache, xmi_lip); +} + +/* + * Freeing the XMI requires that we remove it from the AIL if it has already + * been placed there. However, the XMI may not yet have been placed in the AIL + * when called by xfs_xmi_release() from XMD processing due to the ordering of + * committed vs unpin operations in bulk insert operations. Hence the reference + * count to ensure only the last caller frees the XMI. + */ +STATIC void +xfs_xmi_release( + struct xfs_xmi_log_item *xmi_lip) +{ + ASSERT(atomic_read(&xmi_lip->xmi_refcount) > 0); + if (atomic_dec_and_test(&xmi_lip->xmi_refcount)) { + xfs_trans_ail_delete(&xmi_lip->xmi_item, 0); + xfs_xmi_item_free(xmi_lip); + } +} + + +STATIC void +xfs_xmi_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + *nvecs += 1; + *nbytes += sizeof(struct xfs_xmi_log_format); +} + +/* + * This is called to fill in the vector of log iovecs for the given xmi log + * item. We use only 1 iovec, and we point that at the xmi_log_format structure + * embedded in the xmi item. + */ +STATIC void +xfs_xmi_item_format( + struct xfs_log_item *lip, + struct xfs_log_vec *lv) +{ + struct xfs_xmi_log_item *xmi_lip = XMI_ITEM(lip); + struct xfs_log_iovec *vecp = NULL; + + xmi_lip->xmi_format.xmi_type = XFS_LI_XMI; + xmi_lip->xmi_format.xmi_size = 1; + + xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_XMI_FORMAT, + &xmi_lip->xmi_format, + sizeof(struct xfs_xmi_log_format)); +} + +/* + * The unpin operation is the last place an XMI is manipulated in the log. It + * is either inserted in the AIL or aborted in the event of a log I/O error. In + * either case, the XMI transaction has been successfully committed to make it + * this far. Therefore, we expect whoever committed the XMI to either construct + * and commit the XMD or drop the XMD's reference in the event of error. Simply + * drop the log's XMI reference now that the log is done with it. + */ +STATIC void +xfs_xmi_item_unpin( + struct xfs_log_item *lip, + int remove) +{ + struct xfs_xmi_log_item *xmi_lip = XMI_ITEM(lip); + + xfs_xmi_release(xmi_lip); +} + +/* + * The XMI has been either committed or aborted if the transaction has been + * cancelled. If the transaction was cancelled, an XMD isn't going to be + * constructed and thus we free the XMI here directly. + */ +STATIC void +xfs_xmi_item_release( + struct xfs_log_item *lip) +{ + xfs_xmi_release(XMI_ITEM(lip)); +} + +/* Allocate and initialize an xmi item. */ +STATIC struct xfs_xmi_log_item * +xfs_xmi_init( + struct xfs_mount *mp) + +{ + struct xfs_xmi_log_item *xmi_lip; + + xmi_lip = kmem_cache_zalloc(xfs_xmi_cache, GFP_KERNEL | __GFP_NOFAIL); + + xfs_log_item_init(mp, &xmi_lip->xmi_item, XFS_LI_XMI, &xfs_xmi_item_ops); + xmi_lip->xmi_format.xmi_id = (uintptr_t)(void *)xmi_lip; + atomic_set(&xmi_lip->xmi_refcount, 2); + + return xmi_lip; +} + +static inline struct xfs_xmd_log_item *XMD_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_xmd_log_item, xmd_item); +} + +STATIC bool +xfs_xmi_item_match( + struct xfs_log_item *lip, + uint64_t intent_id) +{ + return XMI_ITEM(lip)->xmi_format.xmi_id == intent_id; +} + +static const struct xfs_item_ops xfs_xmi_item_ops = { + .flags = XFS_ITEM_INTENT, + .iop_size = xfs_xmi_item_size, + .iop_format = xfs_xmi_item_format, + .iop_unpin = xfs_xmi_item_unpin, + .iop_release = xfs_xmi_item_release, + .iop_match = xfs_xmi_item_match, +}; + +/* + * This routine is called to create an in-core file mapping exchange item from + * the xmi format structure which was logged on disk. It allocates an in-core + * xmi, copies the exchange information from the format structure into it, and + * adds the xmi to the AIL with the given LSN. + */ +STATIC int +xlog_recover_xmi_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_xmi_log_item *xmi_lip; + struct xfs_xmi_log_format *xmi_formatp; + size_t len; + + len = sizeof(struct xfs_xmi_log_format); + if (item->ri_buf[0].i_len != len) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; + } + + xmi_formatp = item->ri_buf[0].i_addr; + if (xmi_formatp->__pad != 0) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; + } + + xmi_lip = xfs_xmi_init(mp); + memcpy(&xmi_lip->xmi_format, xmi_formatp, len); + + /* not implemented yet */ + return -EIO; +} + +const struct xlog_recover_item_ops xlog_xmi_item_ops = { + .item_type = XFS_LI_XMI, + .commit_pass2 = xlog_recover_xmi_commit_pass2, +}; + +/* + * This routine is called when an XMD format structure is found in a committed + * transaction in the log. Its purpose is to cancel the corresponding XMI if it + * was still in the log. To do this it searches the AIL for the XMI with an id + * equal to that in the XMD format structure. If we find it we drop the XMD + * reference, which removes the XMI from the AIL and frees it. + */ +STATIC int +xlog_recover_xmd_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_xmd_log_format *xmd_formatp; + + xmd_formatp = item->ri_buf[0].i_addr; + if (item->ri_buf[0].i_len != sizeof(struct xfs_xmd_log_format)) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; + } + + xlog_recover_release_intent(log, XFS_LI_XMI, xmd_formatp->xmd_xmi_id); + return 0; +} + +const struct xlog_recover_item_ops xlog_xmd_item_ops = { + .item_type = XFS_LI_XMD, + .commit_pass2 = xlog_recover_xmd_commit_pass2, +}; diff --git a/fs/xfs/xfs_exchmaps_item.h b/fs/xfs/xfs_exchmaps_item.h new file mode 100644 index 0000000000000..ada1eb314e658 --- /dev/null +++ b/fs/xfs/xfs_exchmaps_item.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_EXCHMAPS_ITEM_H__ +#define __XFS_EXCHMAPS_ITEM_H__ + +/* + * The file mapping exchange intent item helps us exchange multiple file + * mappings between two inode forks. It does this by tracking the range of + * file block offsets that still need to be exchanged, and relogs as progress + * happens. + * + * *I items should be recorded in the *first* of a series of rolled + * transactions, and the *D items should be recorded in the same transaction + * that records the associated bmbt updates. + * + * Should the system crash after the commit of the first transaction but + * before the commit of the final transaction in a series, log recovery will + * use the redo information recorded by the intent items to replay the + * rest of the mapping exchanges. + */ + +/* kernel only XMI/XMD definitions */ + +struct xfs_mount; +struct kmem_cache; + +/* + * This is the incore file mapping exchange intent log item. It is used to log + * the fact that we are exchanging mappings between two files. It is used in + * conjunction with the incore file mapping exchange done log item described + * below. + * + * These log items follow the same rules as struct xfs_efi_log_item; see the + * comments about that structure (in xfs_extfree_item.h) for more details. + */ +struct xfs_xmi_log_item { + struct xfs_log_item xmi_item; + atomic_t xmi_refcount; + struct xfs_xmi_log_format xmi_format; +}; + +/* + * This is the incore file mapping exchange done log item. It is used to log + * the fact that an exchange mentioned in an earlier xmi item have been + * performed. + */ +struct xfs_xmd_log_item { + struct xfs_log_item xmd_item; + struct xfs_xmi_log_item *xmd_intent_log_item; + struct xfs_xmd_log_format xmd_format; +}; + +extern struct kmem_cache *xfs_xmi_cache; +extern struct kmem_cache *xfs_xmd_cache; + +#endif /* __XFS_EXCHMAPS_ITEM_H__ */ diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 41aec991433c5..1e5ba95adf2c7 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -1789,6 +1789,8 @@ static const struct xlog_recover_item_ops *xlog_recover_item_ops[] = { &xlog_bud_item_ops, &xlog_attri_item_ops, &xlog_attrd_item_ops, + &xlog_xmi_item_ops, + &xlog_xmd_item_ops, }; static const struct xlog_recover_item_ops * diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index dbda72df3419b..5c9ba974252d1 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -43,6 +43,7 @@ #include "xfs_iunlink_item.h" #include "xfs_dahash_test.h" #include "xfs_rtbitmap.h" +#include "xfs_exchmaps_item.h" #include "scrub/stats.h" #include "scrub/rcbag_btree.h" @@ -2189,8 +2190,24 @@ xfs_init_caches(void) if (!xfs_iunlink_cache) goto out_destroy_attri_cache; + xfs_xmd_cache = kmem_cache_create("xfs_xmd_item", + sizeof(struct xfs_xmd_log_item), + 0, 0, NULL); + if (!xfs_xmd_cache) + goto out_destroy_iul_cache; + + xfs_xmi_cache = kmem_cache_create("xfs_xmi_item", + sizeof(struct xfs_xmi_log_item), + 0, 0, NULL); + if (!xfs_xmi_cache) + goto out_destroy_xmd_cache; + return 0; + out_destroy_xmd_cache: + kmem_cache_destroy(xfs_xmd_cache); + out_destroy_iul_cache: + kmem_cache_destroy(xfs_iunlink_cache); out_destroy_attri_cache: kmem_cache_destroy(xfs_attri_cache); out_destroy_attrd_cache: @@ -2247,6 +2264,8 @@ xfs_destroy_caches(void) * destroy caches. */ rcu_barrier(); + kmem_cache_destroy(xfs_xmd_cache); + kmem_cache_destroy(xfs_xmi_cache); kmem_cache_destroy(xfs_iunlink_cache); kmem_cache_destroy(xfs_attri_cache); kmem_cache_destroy(xfs_attrd_cache); From 966ceafc7a437105ecfe1cadb3747b2965a260ca Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:17 -0700 Subject: [PATCH 0206/1702] xfs: create deferred log items for file mapping exchanges Now that we've created the skeleton of a log intent item to track and restart file mapping exchange operations, add the upper level logic to commit intent items and turn them into concrete work recorded in the log. This builds on the existing bmap update intent items that have been around for a while now. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/libxfs/xfs_defer.c | 6 + fs/xfs/libxfs/xfs_defer.h | 2 +- fs/xfs/libxfs/xfs_exchmaps.c | 1045 +++++++++++++++++++++++++++++++ fs/xfs/libxfs/xfs_exchmaps.h | 118 ++++ fs/xfs/libxfs/xfs_log_format.h | 24 +- fs/xfs/libxfs/xfs_trans_space.h | 4 + fs/xfs/xfs_exchmaps_item.c | 368 ++++++++++- fs/xfs/xfs_exchmaps_item.h | 5 + fs/xfs/xfs_exchrange.c | 49 ++ fs/xfs/xfs_exchrange.h | 8 + fs/xfs/xfs_trace.c | 1 + fs/xfs/xfs_trace.h | 217 +++++++ 13 files changed, 1844 insertions(+), 4 deletions(-) create mode 100644 fs/xfs/libxfs/xfs_exchmaps.c create mode 100644 fs/xfs/libxfs/xfs_exchmaps.h diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index 68ca9726e7b7d..b547a3dc03f87 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -34,6 +34,7 @@ xfs-y += $(addprefix libxfs/, \ xfs_dir2_node.o \ xfs_dir2_sf.o \ xfs_dquot_buf.o \ + xfs_exchmaps.o \ xfs_ialloc.o \ xfs_ialloc_btree.o \ xfs_iext_tree.o \ diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c index c13276095cc0e..061cc01245a91 100644 --- a/fs/xfs/libxfs/xfs_defer.c +++ b/fs/xfs/libxfs/xfs_defer.c @@ -27,6 +27,7 @@ #include "xfs_da_btree.h" #include "xfs_attr.h" #include "xfs_trans_priv.h" +#include "xfs_exchmaps.h" static struct kmem_cache *xfs_defer_pending_cache; @@ -1176,6 +1177,10 @@ xfs_defer_init_item_caches(void) error = xfs_attr_intent_init_cache(); if (error) goto err; + error = xfs_exchmaps_intent_init_cache(); + if (error) + goto err; + return 0; err: xfs_defer_destroy_item_caches(); @@ -1186,6 +1191,7 @@ xfs_defer_init_item_caches(void) void xfs_defer_destroy_item_caches(void) { + xfs_exchmaps_intent_destroy_cache(); xfs_attr_intent_destroy_cache(); xfs_extfree_intent_destroy_cache(); xfs_bmap_intent_destroy_cache(); diff --git a/fs/xfs/libxfs/xfs_defer.h b/fs/xfs/libxfs/xfs_defer.h index 18a9fb92dde8e..81cca60d70a3b 100644 --- a/fs/xfs/libxfs/xfs_defer.h +++ b/fs/xfs/libxfs/xfs_defer.h @@ -72,7 +72,7 @@ extern const struct xfs_defer_op_type xfs_rmap_update_defer_type; extern const struct xfs_defer_op_type xfs_extent_free_defer_type; extern const struct xfs_defer_op_type xfs_agfl_free_defer_type; extern const struct xfs_defer_op_type xfs_attr_defer_type; - +extern const struct xfs_defer_op_type xfs_exchmaps_defer_type; /* * Deferred operation item relogging limits. diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c new file mode 100644 index 0000000000000..b8e9450cc175f --- /dev/null +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -0,0 +1,1045 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_bmap.h" +#include "xfs_icache.h" +#include "xfs_quota.h" +#include "xfs_exchmaps.h" +#include "xfs_trace.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "xfs_error.h" +#include "xfs_errortag.h" +#include "xfs_health.h" +#include "xfs_exchmaps_item.h" + +struct kmem_cache *xfs_exchmaps_intent_cache; + +/* bmbt mappings adjacent to a pair of records. */ +struct xfs_exchmaps_adjacent { + struct xfs_bmbt_irec left1; + struct xfs_bmbt_irec right1; + struct xfs_bmbt_irec left2; + struct xfs_bmbt_irec right2; +}; + +#define ADJACENT_INIT { \ + .left1 = { .br_startblock = HOLESTARTBLOCK }, \ + .right1 = { .br_startblock = HOLESTARTBLOCK }, \ + .left2 = { .br_startblock = HOLESTARTBLOCK }, \ + .right2 = { .br_startblock = HOLESTARTBLOCK }, \ +} + +/* Information to reset reflink flag / CoW fork state after an exchange. */ + +/* + * If the reflink flag is set on either inode, make sure it has an incore CoW + * fork, since all reflink inodes must have them. If there's a CoW fork and it + * has mappings in it, make sure the inodes are tagged appropriately so that + * speculative preallocations can be GC'd if we run low of space. + */ +static inline void +xfs_exchmaps_ensure_cowfork( + struct xfs_inode *ip) +{ + struct xfs_ifork *cfork; + + if (xfs_is_reflink_inode(ip)) + xfs_ifork_init_cow(ip); + + cfork = xfs_ifork_ptr(ip, XFS_COW_FORK); + if (!cfork) + return; + if (cfork->if_bytes > 0) + xfs_inode_set_cowblocks_tag(ip); + else + xfs_inode_clear_cowblocks_tag(ip); +} + +/* + * Adjust the on-disk inode size upwards if needed so that we never add + * mappings into the file past EOF. This is crucial so that log recovery won't + * get confused by the sudden appearance of post-eof mappings. + */ +STATIC void +xfs_exchmaps_update_size( + struct xfs_trans *tp, + struct xfs_inode *ip, + struct xfs_bmbt_irec *imap, + xfs_fsize_t new_isize) +{ + struct xfs_mount *mp = tp->t_mountp; + xfs_fsize_t len; + + if (new_isize < 0) + return; + + len = min(XFS_FSB_TO_B(mp, imap->br_startoff + imap->br_blockcount), + new_isize); + + if (len <= ip->i_disk_size) + return; + + trace_xfs_exchmaps_update_inode_size(ip, len); + + ip->i_disk_size = len; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); +} + +/* Advance the incore state tracking after exchanging a mapping. */ +static inline void +xmi_advance( + struct xfs_exchmaps_intent *xmi, + const struct xfs_bmbt_irec *irec) +{ + xmi->xmi_startoff1 += irec->br_blockcount; + xmi->xmi_startoff2 += irec->br_blockcount; + xmi->xmi_blockcount -= irec->br_blockcount; +} + +/* Do we still have more mappings to exchange? */ +static inline bool +xmi_has_more_exchange_work(const struct xfs_exchmaps_intent *xmi) +{ + return xmi->xmi_blockcount > 0; +} + +/* Do we have post-operation cleanups to perform? */ +static inline bool +xmi_has_postop_work(const struct xfs_exchmaps_intent *xmi) +{ + return xmi->xmi_flags & (XFS_EXCHMAPS_CLEAR_INO1_REFLINK | + XFS_EXCHMAPS_CLEAR_INO2_REFLINK); +} + +/* Check all mappings to make sure we can actually exchange them. */ +int +xfs_exchmaps_check_forks( + struct xfs_mount *mp, + const struct xfs_exchmaps_req *req) +{ + struct xfs_ifork *ifp1, *ifp2; + int whichfork = xfs_exchmaps_reqfork(req); + + /* No fork? */ + ifp1 = xfs_ifork_ptr(req->ip1, whichfork); + ifp2 = xfs_ifork_ptr(req->ip2, whichfork); + if (!ifp1 || !ifp2) + return -EINVAL; + + /* We don't know how to exchange local format forks. */ + if (ifp1->if_format == XFS_DINODE_FMT_LOCAL || + ifp2->if_format == XFS_DINODE_FMT_LOCAL) + return -EINVAL; + + /* We don't support realtime data forks yet. */ + if (!XFS_IS_REALTIME_INODE(req->ip1)) + return 0; + if (whichfork == XFS_ATTR_FORK) + return 0; + return -EINVAL; +} + +#ifdef CONFIG_XFS_QUOTA +/* Log the actual updates to the quota accounting. */ +static inline void +xfs_exchmaps_update_quota( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi, + struct xfs_bmbt_irec *irec1, + struct xfs_bmbt_irec *irec2) +{ + int64_t ip1_delta = 0, ip2_delta = 0; + unsigned int qflag; + + qflag = XFS_IS_REALTIME_INODE(xmi->xmi_ip1) ? XFS_TRANS_DQ_RTBCOUNT : + XFS_TRANS_DQ_BCOUNT; + + if (xfs_bmap_is_real_extent(irec1)) { + ip1_delta -= irec1->br_blockcount; + ip2_delta += irec1->br_blockcount; + } + + if (xfs_bmap_is_real_extent(irec2)) { + ip1_delta += irec2->br_blockcount; + ip2_delta -= irec2->br_blockcount; + } + + xfs_trans_mod_dquot_byino(tp, xmi->xmi_ip1, qflag, ip1_delta); + xfs_trans_mod_dquot_byino(tp, xmi->xmi_ip2, qflag, ip2_delta); +} +#else +# define xfs_exchmaps_update_quota(tp, xmi, irec1, irec2) ((void)0) +#endif + +/* Decide if we want to skip this mapping from file1. */ +static inline bool +xfs_exchmaps_can_skip_mapping( + struct xfs_exchmaps_intent *xmi, + struct xfs_bmbt_irec *irec) +{ + /* Do not skip this mapping if the caller did not tell us to. */ + if (!(xmi->xmi_flags & XFS_EXCHMAPS_INO1_WRITTEN)) + return false; + + /* Do not skip mapped, written mappings. */ + if (xfs_bmap_is_written_extent(irec)) + return false; + + /* + * The mapping is unwritten or a hole. It cannot be a delalloc + * reservation because we already excluded those. It cannot be an + * unwritten mapping with dirty page cache because we flushed the page + * cache. We don't support realtime files yet, so we needn't (yet) + * deal with them. + */ + return true; +} + +/* + * Walk forward through the file ranges in @xmi until we find two different + * mappings to exchange. If there is work to do, return the mappings; + * otherwise we've reached the end of the range and xmi_blockcount will be + * zero. + * + * If the walk skips over a pair of mappings to the same storage, save them as + * the left records in @adj (if provided) so that the simulation phase can + * avoid an extra lookup. + */ +static int +xfs_exchmaps_find_mappings( + struct xfs_exchmaps_intent *xmi, + struct xfs_bmbt_irec *irec1, + struct xfs_bmbt_irec *irec2, + struct xfs_exchmaps_adjacent *adj) +{ + int nimaps; + int bmap_flags; + int error; + + bmap_flags = xfs_bmapi_aflag(xfs_exchmaps_whichfork(xmi)); + + for (; xmi_has_more_exchange_work(xmi); xmi_advance(xmi, irec1)) { + /* Read mapping from the first file */ + nimaps = 1; + error = xfs_bmapi_read(xmi->xmi_ip1, xmi->xmi_startoff1, + xmi->xmi_blockcount, irec1, &nimaps, + bmap_flags); + if (error) + return error; + if (nimaps != 1 || + irec1->br_startblock == DELAYSTARTBLOCK || + irec1->br_startoff != xmi->xmi_startoff1) { + /* + * We should never get no mapping or a delalloc mapping + * or something that doesn't match what we asked for, + * since the caller flushed both inodes and we hold the + * ILOCKs for both inodes. + */ + ASSERT(0); + return -EINVAL; + } + + if (xfs_exchmaps_can_skip_mapping(xmi, irec1)) { + trace_xfs_exchmaps_mapping1_skip(xmi->xmi_ip1, irec1); + continue; + } + + /* Read mapping from the second file */ + nimaps = 1; + error = xfs_bmapi_read(xmi->xmi_ip2, xmi->xmi_startoff2, + irec1->br_blockcount, irec2, &nimaps, + bmap_flags); + if (error) + return error; + if (nimaps != 1 || + irec2->br_startblock == DELAYSTARTBLOCK || + irec2->br_startoff != xmi->xmi_startoff2) { + /* + * We should never get no mapping or a delalloc mapping + * or something that doesn't match what we asked for, + * since the caller flushed both inodes and we hold the + * ILOCKs for both inodes. + */ + ASSERT(0); + return -EINVAL; + } + + /* + * We can only exchange as many blocks as the smaller of the + * two mapping maps. + */ + irec1->br_blockcount = min(irec1->br_blockcount, + irec2->br_blockcount); + + trace_xfs_exchmaps_mapping1(xmi->xmi_ip1, irec1); + trace_xfs_exchmaps_mapping2(xmi->xmi_ip2, irec2); + + /* We found something to exchange, so return it. */ + if (irec1->br_startblock != irec2->br_startblock) + return 0; + + /* + * Two mappings pointing to the same physical block must not + * have different states; that's filesystem corruption. Move + * on to the next mapping if they're both holes or both point + * to the same physical space extent. + */ + if (irec1->br_state != irec2->br_state) { + xfs_bmap_mark_sick(xmi->xmi_ip1, + xfs_exchmaps_whichfork(xmi)); + xfs_bmap_mark_sick(xmi->xmi_ip2, + xfs_exchmaps_whichfork(xmi)); + return -EFSCORRUPTED; + } + + /* + * Save the mappings if we're estimating work and skipping + * these identical mappings. + */ + if (adj) { + memcpy(&adj->left1, irec1, sizeof(*irec1)); + memcpy(&adj->left2, irec2, sizeof(*irec2)); + } + } + + return 0; +} + +/* Exchange these two mappings. */ +static void +xfs_exchmaps_one_step( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi, + struct xfs_bmbt_irec *irec1, + struct xfs_bmbt_irec *irec2) +{ + int whichfork = xfs_exchmaps_whichfork(xmi); + + xfs_exchmaps_update_quota(tp, xmi, irec1, irec2); + + /* Remove both mappings. */ + xfs_bmap_unmap_extent(tp, xmi->xmi_ip1, whichfork, irec1); + xfs_bmap_unmap_extent(tp, xmi->xmi_ip2, whichfork, irec2); + + /* + * Re-add both mappings. We exchange the file offsets between the two + * maps and add the opposite map, which has the effect of filling the + * logical offsets we just unmapped, but with with the physical mapping + * information exchanged. + */ + swap(irec1->br_startoff, irec2->br_startoff); + xfs_bmap_map_extent(tp, xmi->xmi_ip1, whichfork, irec2); + xfs_bmap_map_extent(tp, xmi->xmi_ip2, whichfork, irec1); + + /* Make sure we're not adding mappings past EOF. */ + if (whichfork == XFS_DATA_FORK) { + xfs_exchmaps_update_size(tp, xmi->xmi_ip1, irec2, + xmi->xmi_isize1); + xfs_exchmaps_update_size(tp, xmi->xmi_ip2, irec1, + xmi->xmi_isize2); + } + + /* + * Advance our cursor and exit. The caller (either defer ops or log + * recovery) will log the XMD item, and if *blockcount is nonzero, it + * will log a new XMI item for the remainder and call us back. + */ + xmi_advance(xmi, irec1); +} + +/* Clear the reflink flag after an exchange. */ +static inline void +xfs_exchmaps_clear_reflink( + struct xfs_trans *tp, + struct xfs_inode *ip) +{ + trace_xfs_reflink_unset_inode_flag(ip); + + ip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); +} + +/* Finish whatever work might come after an exchange operation. */ +static int +xfs_exchmaps_do_postop_work( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi) +{ + if (xmi->xmi_flags & XFS_EXCHMAPS_CLEAR_INO1_REFLINK) { + xfs_exchmaps_clear_reflink(tp, xmi->xmi_ip1); + xmi->xmi_flags &= ~XFS_EXCHMAPS_CLEAR_INO1_REFLINK; + } + + if (xmi->xmi_flags & XFS_EXCHMAPS_CLEAR_INO2_REFLINK) { + xfs_exchmaps_clear_reflink(tp, xmi->xmi_ip2); + xmi->xmi_flags &= ~XFS_EXCHMAPS_CLEAR_INO2_REFLINK; + } + + return 0; +} + +/* Finish one step in a mapping exchange operation, possibly relogging. */ +int +xfs_exchmaps_finish_one( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi) +{ + struct xfs_bmbt_irec irec1, irec2; + int error; + + if (xmi_has_more_exchange_work(xmi)) { + /* + * If the operation state says that some range of the files + * have not yet been exchanged, look for mappings in that range + * to exchange. If we find some mappings, exchange them. + */ + error = xfs_exchmaps_find_mappings(xmi, &irec1, &irec2, NULL); + if (error) + return error; + + if (xmi_has_more_exchange_work(xmi)) + xfs_exchmaps_one_step(tp, xmi, &irec1, &irec2); + + /* + * If the caller asked us to exchange the file sizes after the + * exchange and either we just exchanged the last mappings in + * the range or we didn't find anything to exchange, update the + * ondisk file sizes. + */ + if ((xmi->xmi_flags & XFS_EXCHMAPS_SET_SIZES) && + !xmi_has_more_exchange_work(xmi)) { + xmi->xmi_ip1->i_disk_size = xmi->xmi_isize1; + xmi->xmi_ip2->i_disk_size = xmi->xmi_isize2; + + xfs_trans_log_inode(tp, xmi->xmi_ip1, XFS_ILOG_CORE); + xfs_trans_log_inode(tp, xmi->xmi_ip2, XFS_ILOG_CORE); + } + } else if (xmi_has_postop_work(xmi)) { + /* + * Now that we're finished with the exchange operation, + * complete the post-op cleanup work. + */ + error = xfs_exchmaps_do_postop_work(tp, xmi); + if (error) + return error; + } + + /* If we still have work to do, ask for a new transaction. */ + if (xmi_has_more_exchange_work(xmi) || xmi_has_postop_work(xmi)) { + trace_xfs_exchmaps_defer(tp->t_mountp, xmi); + return -EAGAIN; + } + + /* + * If we reach here, we've finished all the exchange work and the post + * operation work. The last thing we need to do before returning to + * the caller is to make sure that COW forks are set up correctly. + */ + if (!(xmi->xmi_flags & XFS_EXCHMAPS_ATTR_FORK)) { + xfs_exchmaps_ensure_cowfork(xmi->xmi_ip1); + xfs_exchmaps_ensure_cowfork(xmi->xmi_ip2); + } + + return 0; +} + +/* + * Compute the amount of bmbt blocks we should reserve for each file. In the + * worst case, each exchange will fill a hole with a new mapping, which could + * result in a btree split every time we add a new leaf block. + */ +static inline uint64_t +xfs_exchmaps_bmbt_blocks( + struct xfs_mount *mp, + const struct xfs_exchmaps_req *req) +{ + return howmany_64(req->nr_exchanges, + XFS_MAX_CONTIG_BMAPS_PER_BLOCK(mp)) * + XFS_EXTENTADD_SPACE_RES(mp, xfs_exchmaps_reqfork(req)); +} + +/* Compute the space we should reserve for the rmap btree expansions. */ +static inline uint64_t +xfs_exchmaps_rmapbt_blocks( + struct xfs_mount *mp, + const struct xfs_exchmaps_req *req) +{ + if (!xfs_has_rmapbt(mp)) + return 0; + if (XFS_IS_REALTIME_INODE(req->ip1)) + return 0; + + return howmany_64(req->nr_exchanges, + XFS_MAX_CONTIG_RMAPS_PER_BLOCK(mp)) * + XFS_RMAPADD_SPACE_RES(mp); +} + +/* Estimate the bmbt and rmapbt overhead required to exchange mappings. */ +static int +xfs_exchmaps_estimate_overhead( + struct xfs_exchmaps_req *req) +{ + struct xfs_mount *mp = req->ip1->i_mount; + xfs_filblks_t bmbt_blocks; + xfs_filblks_t rmapbt_blocks; + xfs_filblks_t resblks = req->resblks; + + /* + * Compute the number of bmbt and rmapbt blocks we might need to handle + * the estimated number of exchanges. + */ + bmbt_blocks = xfs_exchmaps_bmbt_blocks(mp, req); + rmapbt_blocks = xfs_exchmaps_rmapbt_blocks(mp, req); + + trace_xfs_exchmaps_overhead(mp, bmbt_blocks, rmapbt_blocks); + + /* Make sure the change in file block count doesn't overflow. */ + if (check_add_overflow(req->ip1_bcount, bmbt_blocks, &req->ip1_bcount)) + return -EFBIG; + if (check_add_overflow(req->ip2_bcount, bmbt_blocks, &req->ip2_bcount)) + return -EFBIG; + + /* + * Add together the number of blocks we need to handle btree growth, + * then add it to the number of blocks we need to reserve to this + * transaction. + */ + if (check_add_overflow(resblks, bmbt_blocks, &resblks)) + return -ENOSPC; + if (check_add_overflow(resblks, bmbt_blocks, &resblks)) + return -ENOSPC; + if (check_add_overflow(resblks, rmapbt_blocks, &resblks)) + return -ENOSPC; + if (check_add_overflow(resblks, rmapbt_blocks, &resblks)) + return -ENOSPC; + + /* Can't actually reserve more than UINT_MAX blocks. */ + if (req->resblks > UINT_MAX) + return -ENOSPC; + + req->resblks = resblks; + trace_xfs_exchmaps_final_estimate(req); + return 0; +} + +/* Decide if we can merge two real mappings. */ +static inline bool +xmi_can_merge( + const struct xfs_bmbt_irec *b1, + const struct xfs_bmbt_irec *b2) +{ + /* Don't merge holes. */ + if (b1->br_startblock == HOLESTARTBLOCK || + b2->br_startblock == HOLESTARTBLOCK) + return false; + + /* We don't merge holes. */ + if (!xfs_bmap_is_real_extent(b1) || !xfs_bmap_is_real_extent(b2)) + return false; + + if (b1->br_startoff + b1->br_blockcount == b2->br_startoff && + b1->br_startblock + b1->br_blockcount == b2->br_startblock && + b1->br_state == b2->br_state && + b1->br_blockcount + b2->br_blockcount <= XFS_MAX_BMBT_EXTLEN) + return true; + + return false; +} + +/* + * Decide if we can merge three mappings. Caller must ensure all three + * mappings must not be holes or delalloc reservations. + */ +static inline bool +xmi_can_merge_all( + const struct xfs_bmbt_irec *l, + const struct xfs_bmbt_irec *m, + const struct xfs_bmbt_irec *r) +{ + xfs_filblks_t new_len; + + new_len = l->br_blockcount + m->br_blockcount + r->br_blockcount; + return new_len <= XFS_MAX_BMBT_EXTLEN; +} + +#define CLEFT_CONTIG 0x01 +#define CRIGHT_CONTIG 0x02 +#define CHOLE 0x04 +#define CBOTH_CONTIG (CLEFT_CONTIG | CRIGHT_CONTIG) + +#define NLEFT_CONTIG 0x10 +#define NRIGHT_CONTIG 0x20 +#define NHOLE 0x40 +#define NBOTH_CONTIG (NLEFT_CONTIG | NRIGHT_CONTIG) + +/* Estimate the effect of a single exchange on mapping count. */ +static inline int +xmi_delta_nextents_step( + struct xfs_mount *mp, + const struct xfs_bmbt_irec *left, + const struct xfs_bmbt_irec *curr, + const struct xfs_bmbt_irec *new, + const struct xfs_bmbt_irec *right) +{ + bool lhole, rhole, chole, nhole; + unsigned int state = 0; + int ret = 0; + + lhole = left->br_startblock == HOLESTARTBLOCK; + rhole = right->br_startblock == HOLESTARTBLOCK; + chole = curr->br_startblock == HOLESTARTBLOCK; + nhole = new->br_startblock == HOLESTARTBLOCK; + + if (chole) + state |= CHOLE; + if (!lhole && !chole && xmi_can_merge(left, curr)) + state |= CLEFT_CONTIG; + if (!rhole && !chole && xmi_can_merge(curr, right)) + state |= CRIGHT_CONTIG; + if ((state & CBOTH_CONTIG) == CBOTH_CONTIG && + !xmi_can_merge_all(left, curr, right)) + state &= ~CRIGHT_CONTIG; + + if (nhole) + state |= NHOLE; + if (!lhole && !nhole && xmi_can_merge(left, new)) + state |= NLEFT_CONTIG; + if (!rhole && !nhole && xmi_can_merge(new, right)) + state |= NRIGHT_CONTIG; + if ((state & NBOTH_CONTIG) == NBOTH_CONTIG && + !xmi_can_merge_all(left, new, right)) + state &= ~NRIGHT_CONTIG; + + switch (state & (CLEFT_CONTIG | CRIGHT_CONTIG | CHOLE)) { + case CLEFT_CONTIG | CRIGHT_CONTIG: + /* + * left/curr/right are the same mapping, so deleting curr + * causes 2 new mappings to be created. + */ + ret += 2; + break; + case 0: + /* + * curr is not contiguous with any mapping, so we remove curr + * completely + */ + ret--; + break; + case CHOLE: + /* hole, do nothing */ + break; + case CLEFT_CONTIG: + case CRIGHT_CONTIG: + /* trim either left or right, no change */ + break; + } + + switch (state & (NLEFT_CONTIG | NRIGHT_CONTIG | NHOLE)) { + case NLEFT_CONTIG | NRIGHT_CONTIG: + /* + * left/curr/right will become the same mapping, so adding + * curr causes the deletion of right. + */ + ret--; + break; + case 0: + /* new is not contiguous with any mapping */ + ret++; + break; + case NHOLE: + /* hole, do nothing. */ + break; + case NLEFT_CONTIG: + case NRIGHT_CONTIG: + /* new is absorbed into left or right, no change */ + break; + } + + trace_xfs_exchmaps_delta_nextents_step(mp, left, curr, new, right, ret, + state); + return ret; +} + +/* Make sure we don't overflow the extent (mapping) counters. */ +static inline int +xmi_ensure_delta_nextents( + struct xfs_exchmaps_req *req, + struct xfs_inode *ip, + int64_t delta) +{ + struct xfs_mount *mp = ip->i_mount; + int whichfork = xfs_exchmaps_reqfork(req); + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + uint64_t new_nextents; + xfs_extnum_t max_nextents; + + if (delta < 0) + return 0; + + /* + * It's always an error if the delta causes integer overflow. delta + * needs an explicit cast here to avoid warnings about implicit casts + * coded into the overflow check. + */ + if (check_add_overflow(ifp->if_nextents, (uint64_t)delta, + &new_nextents)) + return -EFBIG; + + if (XFS_TEST_ERROR(false, mp, XFS_ERRTAG_REDUCE_MAX_IEXTENTS) && + new_nextents > 10) + return -EFBIG; + + /* + * We always promote both inodes to have large extent counts if the + * superblock feature is enabled, so we only need to check against the + * theoretical maximum. + */ + max_nextents = xfs_iext_max_nextents(xfs_has_large_extent_counts(mp), + whichfork); + if (new_nextents > max_nextents) + return -EFBIG; + + return 0; +} + +/* Find the next mapping after irec. */ +static inline int +xmi_next( + struct xfs_inode *ip, + int bmap_flags, + const struct xfs_bmbt_irec *irec, + struct xfs_bmbt_irec *nrec) +{ + xfs_fileoff_t off; + xfs_filblks_t blockcount; + int nimaps = 1; + int error; + + off = irec->br_startoff + irec->br_blockcount; + blockcount = XFS_MAX_FILEOFF - off; + error = xfs_bmapi_read(ip, off, blockcount, nrec, &nimaps, bmap_flags); + if (error) + return error; + if (nrec->br_startblock == DELAYSTARTBLOCK || + nrec->br_startoff != off) { + /* + * If we don't get the mapping we want, return a zero-length + * mapping, which our estimator function will pretend is a hole. + * We shouldn't get delalloc reservations. + */ + nrec->br_startblock = HOLESTARTBLOCK; + } + + return 0; +} + +int __init +xfs_exchmaps_intent_init_cache(void) +{ + xfs_exchmaps_intent_cache = kmem_cache_create("xfs_exchmaps_intent", + sizeof(struct xfs_exchmaps_intent), + 0, 0, NULL); + + return xfs_exchmaps_intent_cache != NULL ? 0 : -ENOMEM; +} + +void +xfs_exchmaps_intent_destroy_cache(void) +{ + kmem_cache_destroy(xfs_exchmaps_intent_cache); + xfs_exchmaps_intent_cache = NULL; +} + +/* + * Decide if we will exchange the reflink flags between the two files after the + * exchange. The only time we want to do this is if we're exchanging all + * mappings under EOF and the inode reflink flags have different states. + */ +static inline bool +xmi_can_exchange_reflink_flags( + const struct xfs_exchmaps_req *req, + unsigned int reflink_state) +{ + struct xfs_mount *mp = req->ip1->i_mount; + + if (hweight32(reflink_state) != 1) + return false; + if (req->startoff1 != 0 || req->startoff2 != 0) + return false; + if (req->blockcount != XFS_B_TO_FSB(mp, req->ip1->i_disk_size)) + return false; + if (req->blockcount != XFS_B_TO_FSB(mp, req->ip2->i_disk_size)) + return false; + return true; +} + + +/* Allocate and initialize a new incore intent item from a request. */ +struct xfs_exchmaps_intent * +xfs_exchmaps_init_intent( + const struct xfs_exchmaps_req *req) +{ + struct xfs_exchmaps_intent *xmi; + unsigned int rs = 0; + + xmi = kmem_cache_zalloc(xfs_exchmaps_intent_cache, + GFP_NOFS | __GFP_NOFAIL); + INIT_LIST_HEAD(&xmi->xmi_list); + xmi->xmi_ip1 = req->ip1; + xmi->xmi_ip2 = req->ip2; + xmi->xmi_startoff1 = req->startoff1; + xmi->xmi_startoff2 = req->startoff2; + xmi->xmi_blockcount = req->blockcount; + xmi->xmi_isize1 = xmi->xmi_isize2 = -1; + xmi->xmi_flags = req->flags & XFS_EXCHMAPS_PARAMS; + + if (xfs_exchmaps_whichfork(xmi) == XFS_ATTR_FORK) + return xmi; + + if (req->flags & XFS_EXCHMAPS_SET_SIZES) { + xmi->xmi_flags |= XFS_EXCHMAPS_SET_SIZES; + xmi->xmi_isize1 = req->ip2->i_disk_size; + xmi->xmi_isize2 = req->ip1->i_disk_size; + } + + /* Record the state of each inode's reflink flag before the op. */ + if (xfs_is_reflink_inode(req->ip1)) + rs |= 1; + if (xfs_is_reflink_inode(req->ip2)) + rs |= 2; + + /* + * Figure out if we're clearing the reflink flags (which effectively + * exchanges them) after the operation. + */ + if (xmi_can_exchange_reflink_flags(req, rs)) { + if (rs & 1) + xmi->xmi_flags |= XFS_EXCHMAPS_CLEAR_INO1_REFLINK; + if (rs & 2) + xmi->xmi_flags |= XFS_EXCHMAPS_CLEAR_INO2_REFLINK; + } + + return xmi; +} + +/* + * Estimate the number of exchange operations and the number of file blocks + * in each file that will be affected by the exchange operation. + */ +int +xfs_exchmaps_estimate( + struct xfs_exchmaps_req *req) +{ + struct xfs_exchmaps_intent *xmi; + struct xfs_bmbt_irec irec1, irec2; + struct xfs_exchmaps_adjacent adj = ADJACENT_INIT; + xfs_filblks_t ip1_blocks = 0, ip2_blocks = 0; + int64_t d_nexts1, d_nexts2; + int bmap_flags; + int error; + + ASSERT(!(req->flags & ~XFS_EXCHMAPS_PARAMS)); + + bmap_flags = xfs_bmapi_aflag(xfs_exchmaps_reqfork(req)); + xmi = xfs_exchmaps_init_intent(req); + + /* + * To guard against the possibility of overflowing the extent counters, + * we have to estimate an upper bound on the potential increase in that + * counter. We can split the mapping at each end of the range, and for + * each step of the exchange we can split the mapping that we're + * working on if the mappings do not align. + */ + d_nexts1 = d_nexts2 = 3; + + while (xmi_has_more_exchange_work(xmi)) { + /* + * Walk through the file ranges until we find something to + * exchange. Because we're simulating the exchange, pass in + * adj to capture skipped mappings for correct estimation of + * bmbt record merges. + */ + error = xfs_exchmaps_find_mappings(xmi, &irec1, &irec2, &adj); + if (error) + goto out_free; + if (!xmi_has_more_exchange_work(xmi)) + break; + + /* Update accounting. */ + if (xfs_bmap_is_real_extent(&irec1)) + ip1_blocks += irec1.br_blockcount; + if (xfs_bmap_is_real_extent(&irec2)) + ip2_blocks += irec2.br_blockcount; + req->nr_exchanges++; + + /* Read the next mappings from both files. */ + error = xmi_next(req->ip1, bmap_flags, &irec1, &adj.right1); + if (error) + goto out_free; + + error = xmi_next(req->ip2, bmap_flags, &irec2, &adj.right2); + if (error) + goto out_free; + + /* Update extent count deltas. */ + d_nexts1 += xmi_delta_nextents_step(req->ip1->i_mount, + &adj.left1, &irec1, &irec2, &adj.right1); + + d_nexts2 += xmi_delta_nextents_step(req->ip1->i_mount, + &adj.left2, &irec2, &irec1, &adj.right2); + + /* Now pretend we exchanged the mappings. */ + if (xmi_can_merge(&adj.left2, &irec1)) + adj.left2.br_blockcount += irec1.br_blockcount; + else + memcpy(&adj.left2, &irec1, sizeof(irec1)); + + if (xmi_can_merge(&adj.left1, &irec2)) + adj.left1.br_blockcount += irec2.br_blockcount; + else + memcpy(&adj.left1, &irec2, sizeof(irec2)); + + xmi_advance(xmi, &irec1); + } + + /* Account for the blocks that are being exchanged. */ + if (XFS_IS_REALTIME_INODE(req->ip1) && + xfs_exchmaps_reqfork(req) == XFS_DATA_FORK) { + req->ip1_rtbcount = ip1_blocks; + req->ip2_rtbcount = ip2_blocks; + } else { + req->ip1_bcount = ip1_blocks; + req->ip2_bcount = ip2_blocks; + } + + /* + * Make sure that both forks have enough slack left in their extent + * counters that the exchange operation will not overflow. + */ + trace_xfs_exchmaps_delta_nextents(req, d_nexts1, d_nexts2); + if (req->ip1 == req->ip2) { + error = xmi_ensure_delta_nextents(req, req->ip1, + d_nexts1 + d_nexts2); + } else { + error = xmi_ensure_delta_nextents(req, req->ip1, d_nexts1); + if (error) + goto out_free; + error = xmi_ensure_delta_nextents(req, req->ip2, d_nexts2); + } + if (error) + goto out_free; + + trace_xfs_exchmaps_initial_estimate(req); + error = xfs_exchmaps_estimate_overhead(req); +out_free: + kmem_cache_free(xfs_exchmaps_intent_cache, xmi); + return error; +} + +/* Set the reflink flag before an operation. */ +static inline void +xfs_exchmaps_set_reflink( + struct xfs_trans *tp, + struct xfs_inode *ip) +{ + trace_xfs_reflink_set_inode_flag(ip); + + ip->i_diflags2 |= XFS_DIFLAG2_REFLINK; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); +} + +/* + * If either file has shared blocks and we're exchanging data forks, we must + * flag the other file as having shared blocks so that we get the shared-block + * rmap functions if we need to fix up the rmaps. + */ +void +xfs_exchmaps_ensure_reflink( + struct xfs_trans *tp, + const struct xfs_exchmaps_intent *xmi) +{ + unsigned int rs = 0; + + if (xfs_is_reflink_inode(xmi->xmi_ip1)) + rs |= 1; + if (xfs_is_reflink_inode(xmi->xmi_ip2)) + rs |= 2; + + if ((rs & 1) && !xfs_is_reflink_inode(xmi->xmi_ip2)) + xfs_exchmaps_set_reflink(tp, xmi->xmi_ip2); + + if ((rs & 2) && !xfs_is_reflink_inode(xmi->xmi_ip1)) + xfs_exchmaps_set_reflink(tp, xmi->xmi_ip1); +} + +/* Set the large extent count flag before an operation if needed. */ +static inline void +xfs_exchmaps_ensure_large_extent_counts( + struct xfs_trans *tp, + struct xfs_inode *ip) +{ + if (xfs_inode_has_large_extent_counts(ip)) + return; + + ip->i_diflags2 |= XFS_DIFLAG2_NREXT64; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); +} + +/* Widen the extent counter fields of both inodes if necessary. */ +void +xfs_exchmaps_upgrade_extent_counts( + struct xfs_trans *tp, + const struct xfs_exchmaps_intent *xmi) +{ + if (!xfs_has_large_extent_counts(tp->t_mountp)) + return; + + xfs_exchmaps_ensure_large_extent_counts(tp, xmi->xmi_ip1); + xfs_exchmaps_ensure_large_extent_counts(tp, xmi->xmi_ip2); +} + +/* + * Schedule an exchange a range of mappings from one inode to another. + * + * The use of file mapping exchange log intent items ensures the operation can + * be resumed even if the system goes down. The caller must commit the + * transaction to start the work. + * + * The caller must ensure the inodes must be joined to the transaction and + * ILOCKd; they will still be joined to the transaction at exit. + */ +void +xfs_exchange_mappings( + struct xfs_trans *tp, + const struct xfs_exchmaps_req *req) +{ + struct xfs_exchmaps_intent *xmi; + + xfs_assert_ilocked(req->ip1, XFS_ILOCK_EXCL); + xfs_assert_ilocked(req->ip2, XFS_ILOCK_EXCL); + ASSERT(!(req->flags & ~XFS_EXCHMAPS_LOGGED_FLAGS)); + if (req->flags & XFS_EXCHMAPS_SET_SIZES) + ASSERT(!(req->flags & XFS_EXCHMAPS_ATTR_FORK)); + ASSERT(xfs_has_exchange_range(tp->t_mountp)); + + if (req->blockcount == 0) + return; + + xmi = xfs_exchmaps_init_intent(req); + xfs_exchmaps_defer_add(tp, xmi); + xfs_exchmaps_ensure_reflink(tp, xmi); + xfs_exchmaps_upgrade_extent_counts(tp, xmi); +} diff --git a/fs/xfs/libxfs/xfs_exchmaps.h b/fs/xfs/libxfs/xfs_exchmaps.h new file mode 100644 index 0000000000000..e8fc3f80c68c2 --- /dev/null +++ b/fs/xfs/libxfs/xfs_exchmaps.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_EXCHMAPS_H__ +#define __XFS_EXCHMAPS_H__ + +/* In-core deferred operation info about a file mapping exchange request. */ +struct xfs_exchmaps_intent { + /* List of other incore deferred work. */ + struct list_head xmi_list; + + /* Inodes participating in the operation. */ + struct xfs_inode *xmi_ip1; + struct xfs_inode *xmi_ip2; + + /* File offset range information. */ + xfs_fileoff_t xmi_startoff1; + xfs_fileoff_t xmi_startoff2; + xfs_filblks_t xmi_blockcount; + + /* Set these file sizes after the operation, unless negative. */ + xfs_fsize_t xmi_isize1; + xfs_fsize_t xmi_isize2; + + uint64_t xmi_flags; /* XFS_EXCHMAPS_* flags */ +}; + +/* flags that can be passed to xfs_exchmaps_{estimate,mappings} */ +#define XFS_EXCHMAPS_PARAMS (XFS_EXCHMAPS_ATTR_FORK | \ + XFS_EXCHMAPS_SET_SIZES | \ + XFS_EXCHMAPS_INO1_WRITTEN) + +static inline int +xfs_exchmaps_whichfork(const struct xfs_exchmaps_intent *xmi) +{ + if (xmi->xmi_flags & XFS_EXCHMAPS_ATTR_FORK) + return XFS_ATTR_FORK; + return XFS_DATA_FORK; +} + +/* Parameters for a mapping exchange request. */ +struct xfs_exchmaps_req { + /* Inodes participating in the operation. */ + struct xfs_inode *ip1; + struct xfs_inode *ip2; + + /* File offset range information. */ + xfs_fileoff_t startoff1; + xfs_fileoff_t startoff2; + xfs_filblks_t blockcount; + + /* XFS_EXCHMAPS_* operation flags */ + uint64_t flags; + + /* + * Fields below this line are filled out by xfs_exchmaps_estimate; + * callers should initialize this part of the struct to zero. + */ + + /* + * Data device blocks to be moved out of ip1, and free space needed to + * handle the bmbt changes. + */ + xfs_filblks_t ip1_bcount; + + /* + * Data device blocks to be moved out of ip2, and free space needed to + * handle the bmbt changes. + */ + xfs_filblks_t ip2_bcount; + + /* rt blocks to be moved out of ip1. */ + xfs_filblks_t ip1_rtbcount; + + /* rt blocks to be moved out of ip2. */ + xfs_filblks_t ip2_rtbcount; + + /* Free space needed to handle the bmbt changes */ + unsigned long long resblks; + + /* Number of exchanges needed to complete the operation */ + unsigned long long nr_exchanges; +}; + +static inline int +xfs_exchmaps_reqfork(const struct xfs_exchmaps_req *req) +{ + if (req->flags & XFS_EXCHMAPS_ATTR_FORK) + return XFS_ATTR_FORK; + return XFS_DATA_FORK; +} + +int xfs_exchmaps_estimate(struct xfs_exchmaps_req *req); + +extern struct kmem_cache *xfs_exchmaps_intent_cache; + +int __init xfs_exchmaps_intent_init_cache(void); +void xfs_exchmaps_intent_destroy_cache(void); + +struct xfs_exchmaps_intent *xfs_exchmaps_init_intent( + const struct xfs_exchmaps_req *req); +void xfs_exchmaps_ensure_reflink(struct xfs_trans *tp, + const struct xfs_exchmaps_intent *xmi); +void xfs_exchmaps_upgrade_extent_counts(struct xfs_trans *tp, + const struct xfs_exchmaps_intent *xmi); + +int xfs_exchmaps_finish_one(struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi); + +int xfs_exchmaps_check_forks(struct xfs_mount *mp, + const struct xfs_exchmaps_req *req); + +void xfs_exchange_mappings(struct xfs_trans *tp, + const struct xfs_exchmaps_req *req); + +#endif /* __XFS_EXCHMAPS_H__ */ diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h index 09024431cae9a..8dbe1f997dfd5 100644 --- a/fs/xfs/libxfs/xfs_log_format.h +++ b/fs/xfs/libxfs/xfs_log_format.h @@ -904,7 +904,29 @@ struct xfs_xmi_log_format { uint64_t xmi_isize2; /* intended file2 size */ }; -#define XFS_EXCHMAPS_LOGGED_FLAGS (0) +/* Exchange mappings between extended attribute forks instead of data forks. */ +#define XFS_EXCHMAPS_ATTR_FORK (1ULL << 0) + +/* Set the file sizes when finished. */ +#define XFS_EXCHMAPS_SET_SIZES (1ULL << 1) + +/* + * Exchange the mappings of the two files only if the file allocation units + * mapped to file1's range have been written. + */ +#define XFS_EXCHMAPS_INO1_WRITTEN (1ULL << 2) + +/* Clear the reflink flag from inode1 after the operation. */ +#define XFS_EXCHMAPS_CLEAR_INO1_REFLINK (1ULL << 3) + +/* Clear the reflink flag from inode2 after the operation. */ +#define XFS_EXCHMAPS_CLEAR_INO2_REFLINK (1ULL << 4) + +#define XFS_EXCHMAPS_LOGGED_FLAGS (XFS_EXCHMAPS_ATTR_FORK | \ + XFS_EXCHMAPS_SET_SIZES | \ + XFS_EXCHMAPS_INO1_WRITTEN | \ + XFS_EXCHMAPS_CLEAR_INO1_REFLINK | \ + XFS_EXCHMAPS_CLEAR_INO2_REFLINK) /* This is the structure used to lay out an mapping exchange done log item. */ struct xfs_xmd_log_format { diff --git a/fs/xfs/libxfs/xfs_trans_space.h b/fs/xfs/libxfs/xfs_trans_space.h index 87b31c69a7732..9640fc232c147 100644 --- a/fs/xfs/libxfs/xfs_trans_space.h +++ b/fs/xfs/libxfs/xfs_trans_space.h @@ -10,6 +10,10 @@ * Components of space reservations. */ +/* Worst case number of bmaps that can be held in a block. */ +#define XFS_MAX_CONTIG_BMAPS_PER_BLOCK(mp) \ + (((mp)->m_bmap_dmxr[0]) - ((mp)->m_bmap_dmnr[0])) + /* Worst case number of rmaps that can be held in a block. */ #define XFS_MAX_CONTIG_RMAPS_PER_BLOCK(mp) \ (((mp)->m_rmap_mxr[0]) - ((mp)->m_rmap_mnr[0])) diff --git a/fs/xfs/xfs_exchmaps_item.c b/fs/xfs/xfs_exchmaps_item.c index 65b0ade41b3d6..a40216f33214c 100644 --- a/fs/xfs/xfs_exchmaps_item.c +++ b/fs/xfs/xfs_exchmaps_item.c @@ -16,13 +16,17 @@ #include "xfs_trans.h" #include "xfs_trans_priv.h" #include "xfs_exchmaps_item.h" +#include "xfs_exchmaps.h" #include "xfs_log.h" #include "xfs_bmap.h" #include "xfs_icache.h" +#include "xfs_bmap_btree.h" #include "xfs_trans_space.h" #include "xfs_error.h" #include "xfs_log_priv.h" #include "xfs_log_recover.h" +#include "xfs_exchrange.h" +#include "xfs_trace.h" struct kmem_cache *xfs_xmi_cache; struct kmem_cache *xfs_xmd_cache; @@ -144,6 +148,365 @@ static inline struct xfs_xmd_log_item *XMD_ITEM(struct xfs_log_item *lip) return container_of(lip, struct xfs_xmd_log_item, xmd_item); } +STATIC void +xfs_xmd_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + *nvecs += 1; + *nbytes += sizeof(struct xfs_xmd_log_format); +} + +/* + * This is called to fill in the vector of log iovecs for the given xmd log + * item. We use only 1 iovec, and we point that at the xmd_log_format structure + * embedded in the xmd item. + */ +STATIC void +xfs_xmd_item_format( + struct xfs_log_item *lip, + struct xfs_log_vec *lv) +{ + struct xfs_xmd_log_item *xmd_lip = XMD_ITEM(lip); + struct xfs_log_iovec *vecp = NULL; + + xmd_lip->xmd_format.xmd_type = XFS_LI_XMD; + xmd_lip->xmd_format.xmd_size = 1; + + xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_XMD_FORMAT, &xmd_lip->xmd_format, + sizeof(struct xfs_xmd_log_format)); +} + +/* + * The XMD is either committed or aborted if the transaction is cancelled. If + * the transaction is cancelled, drop our reference to the XMI and free the + * XMD. + */ +STATIC void +xfs_xmd_item_release( + struct xfs_log_item *lip) +{ + struct xfs_xmd_log_item *xmd_lip = XMD_ITEM(lip); + + xfs_xmi_release(xmd_lip->xmd_intent_log_item); + kvfree(xmd_lip->xmd_item.li_lv_shadow); + kmem_cache_free(xfs_xmd_cache, xmd_lip); +} + +static struct xfs_log_item * +xfs_xmd_item_intent( + struct xfs_log_item *lip) +{ + return &XMD_ITEM(lip)->xmd_intent_log_item->xmi_item; +} + +static const struct xfs_item_ops xfs_xmd_item_ops = { + .flags = XFS_ITEM_RELEASE_WHEN_COMMITTED | + XFS_ITEM_INTENT_DONE, + .iop_size = xfs_xmd_item_size, + .iop_format = xfs_xmd_item_format, + .iop_release = xfs_xmd_item_release, + .iop_intent = xfs_xmd_item_intent, +}; + +/* Log file mapping exchange information in the intent item. */ +STATIC struct xfs_log_item * +xfs_exchmaps_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count, + bool sort) +{ + struct xfs_xmi_log_item *xmi_lip; + struct xfs_exchmaps_intent *xmi; + struct xfs_xmi_log_format *xlf; + + ASSERT(count == 1); + + xmi = list_first_entry_or_null(items, struct xfs_exchmaps_intent, + xmi_list); + + xmi_lip = xfs_xmi_init(tp->t_mountp); + xlf = &xmi_lip->xmi_format; + + xlf->xmi_inode1 = xmi->xmi_ip1->i_ino; + xlf->xmi_inode2 = xmi->xmi_ip2->i_ino; + xlf->xmi_startoff1 = xmi->xmi_startoff1; + xlf->xmi_startoff2 = xmi->xmi_startoff2; + xlf->xmi_blockcount = xmi->xmi_blockcount; + xlf->xmi_isize1 = xmi->xmi_isize1; + xlf->xmi_isize2 = xmi->xmi_isize2; + xlf->xmi_flags = xmi->xmi_flags & XFS_EXCHMAPS_LOGGED_FLAGS; + + return &xmi_lip->xmi_item; +} + +STATIC struct xfs_log_item * +xfs_exchmaps_create_done( + struct xfs_trans *tp, + struct xfs_log_item *intent, + unsigned int count) +{ + struct xfs_xmi_log_item *xmi_lip = XMI_ITEM(intent); + struct xfs_xmd_log_item *xmd_lip; + + xmd_lip = kmem_cache_zalloc(xfs_xmd_cache, GFP_KERNEL | __GFP_NOFAIL); + xfs_log_item_init(tp->t_mountp, &xmd_lip->xmd_item, XFS_LI_XMD, + &xfs_xmd_item_ops); + xmd_lip->xmd_intent_log_item = xmi_lip; + xmd_lip->xmd_format.xmd_xmi_id = xmi_lip->xmi_format.xmi_id; + + return &xmd_lip->xmd_item; +} + +/* Add this deferred XMI to the transaction. */ +void +xfs_exchmaps_defer_add( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi) +{ + trace_xfs_exchmaps_defer(tp->t_mountp, xmi); + + xfs_defer_add(tp, &xmi->xmi_list, &xfs_exchmaps_defer_type); +} + +static inline struct xfs_exchmaps_intent *xmi_entry(const struct list_head *e) +{ + return list_entry(e, struct xfs_exchmaps_intent, xmi_list); +} + +/* Cancel a deferred file mapping exchange. */ +STATIC void +xfs_exchmaps_cancel_item( + struct list_head *item) +{ + struct xfs_exchmaps_intent *xmi = xmi_entry(item); + + kmem_cache_free(xfs_exchmaps_intent_cache, xmi); +} + +/* Process a deferred file mapping exchange. */ +STATIC int +xfs_exchmaps_finish_item( + struct xfs_trans *tp, + struct xfs_log_item *done, + struct list_head *item, + struct xfs_btree_cur **state) +{ + struct xfs_exchmaps_intent *xmi = xmi_entry(item); + int error; + + /* + * Exchange one more mappings between two files. If there's still more + * work to do, we want to requeue ourselves after all other pending + * deferred operations have finished. This includes all of the dfops + * that we queued directly as well as any new ones created in the + * process of finishing the others. Doing so prevents us from queuing + * a large number of XMI log items in kernel memory, which in turn + * prevents us from pinning the tail of the log (while logging those + * new XMI items) until the first XMI items can be processed. + */ + error = xfs_exchmaps_finish_one(tp, xmi); + if (error != -EAGAIN) + xfs_exchmaps_cancel_item(item); + return error; +} + +/* Abort all pending XMIs. */ +STATIC void +xfs_exchmaps_abort_intent( + struct xfs_log_item *intent) +{ + xfs_xmi_release(XMI_ITEM(intent)); +} + +/* Is this recovered XMI ok? */ +static inline bool +xfs_xmi_validate( + struct xfs_mount *mp, + struct xfs_xmi_log_item *xmi_lip) +{ + struct xfs_xmi_log_format *xlf = &xmi_lip->xmi_format; + + if (!xfs_has_exchange_range(mp)) + return false; + + if (xmi_lip->xmi_format.__pad != 0) + return false; + + if (xlf->xmi_flags & ~XFS_EXCHMAPS_LOGGED_FLAGS) + return false; + + if (!xfs_verify_ino(mp, xlf->xmi_inode1) || + !xfs_verify_ino(mp, xlf->xmi_inode2)) + return false; + + if (!xfs_verify_fileext(mp, xlf->xmi_startoff1, xlf->xmi_blockcount)) + return false; + + return xfs_verify_fileext(mp, xlf->xmi_startoff2, xlf->xmi_blockcount); +} + +/* + * Use the recovered log state to create a new request, estimate resource + * requirements, and create a new incore intent state. + */ +STATIC struct xfs_exchmaps_intent * +xfs_xmi_item_recover_intent( + struct xfs_mount *mp, + struct xfs_defer_pending *dfp, + const struct xfs_xmi_log_format *xlf, + struct xfs_exchmaps_req *req, + struct xfs_inode **ipp1, + struct xfs_inode **ipp2) +{ + struct xfs_inode *ip1, *ip2; + struct xfs_exchmaps_intent *xmi; + int error; + + /* + * Grab both inodes and set IRECOVERY to prevent trimming of post-eof + * mappings and freeing of unlinked inodes until we're totally done + * processing files. + */ + error = xlog_recover_iget(mp, xlf->xmi_inode1, &ip1); + if (error) + return ERR_PTR(error); + error = xlog_recover_iget(mp, xlf->xmi_inode2, &ip2); + if (error) + goto err_rele1; + + req->ip1 = ip1; + req->ip2 = ip2; + req->startoff1 = xlf->xmi_startoff1; + req->startoff2 = xlf->xmi_startoff2; + req->blockcount = xlf->xmi_blockcount; + req->flags = xlf->xmi_flags & XFS_EXCHMAPS_PARAMS; + + xfs_exchrange_ilock(NULL, ip1, ip2); + error = xfs_exchmaps_estimate(req); + xfs_exchrange_iunlock(ip1, ip2); + if (error) + goto err_rele2; + + *ipp1 = ip1; + *ipp2 = ip2; + xmi = xfs_exchmaps_init_intent(req); + xfs_defer_add_item(dfp, &xmi->xmi_list); + return xmi; + +err_rele2: + xfs_irele(ip2); +err_rele1: + xfs_irele(ip1); + req->ip2 = req->ip1 = NULL; + return ERR_PTR(error); +} + +/* Process a file mapping exchange item that was recovered from the log. */ +STATIC int +xfs_exchmaps_recover_work( + struct xfs_defer_pending *dfp, + struct list_head *capture_list) +{ + struct xfs_exchmaps_req req = { .flags = 0 }; + struct xfs_trans_res resv; + struct xfs_exchmaps_intent *xmi; + struct xfs_log_item *lip = dfp->dfp_intent; + struct xfs_xmi_log_item *xmi_lip = XMI_ITEM(lip); + struct xfs_mount *mp = lip->li_log->l_mp; + struct xfs_trans *tp; + struct xfs_inode *ip1, *ip2; + int error = 0; + + if (!xfs_xmi_validate(mp, xmi_lip)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + &xmi_lip->xmi_format, + sizeof(xmi_lip->xmi_format)); + return -EFSCORRUPTED; + } + + xmi = xfs_xmi_item_recover_intent(mp, dfp, &xmi_lip->xmi_format, &req, + &ip1, &ip2); + if (IS_ERR(xmi)) + return PTR_ERR(xmi); + + trace_xfs_exchmaps_recover(mp, xmi); + + resv = xlog_recover_resv(&M_RES(mp)->tr_write); + error = xfs_trans_alloc(mp, &resv, req.resblks, 0, 0, &tp); + if (error) + goto err_rele; + + xfs_exchrange_ilock(tp, ip1, ip2); + + xfs_exchmaps_ensure_reflink(tp, xmi); + xfs_exchmaps_upgrade_extent_counts(tp, xmi); + error = xlog_recover_finish_intent(tp, dfp); + if (error == -EFSCORRUPTED) + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + &xmi_lip->xmi_format, + sizeof(xmi_lip->xmi_format)); + if (error) + goto err_cancel; + + /* + * Commit transaction, which frees the transaction and saves the inodes + * for later replay activities. + */ + error = xfs_defer_ops_capture_and_commit(tp, capture_list); + goto err_unlock; + +err_cancel: + xfs_trans_cancel(tp); +err_unlock: + xfs_exchrange_iunlock(ip1, ip2); +err_rele: + xfs_irele(ip2); + xfs_irele(ip1); + return error; +} + +/* Relog an intent item to push the log tail forward. */ +static struct xfs_log_item * +xfs_exchmaps_relog_intent( + struct xfs_trans *tp, + struct xfs_log_item *intent, + struct xfs_log_item *done_item) +{ + struct xfs_xmi_log_item *xmi_lip; + struct xfs_xmi_log_format *old_xlf, *new_xlf; + + old_xlf = &XMI_ITEM(intent)->xmi_format; + + xmi_lip = xfs_xmi_init(tp->t_mountp); + new_xlf = &xmi_lip->xmi_format; + + new_xlf->xmi_inode1 = old_xlf->xmi_inode1; + new_xlf->xmi_inode2 = old_xlf->xmi_inode2; + new_xlf->xmi_startoff1 = old_xlf->xmi_startoff1; + new_xlf->xmi_startoff2 = old_xlf->xmi_startoff2; + new_xlf->xmi_blockcount = old_xlf->xmi_blockcount; + new_xlf->xmi_flags = old_xlf->xmi_flags; + new_xlf->xmi_isize1 = old_xlf->xmi_isize1; + new_xlf->xmi_isize2 = old_xlf->xmi_isize2; + + return &xmi_lip->xmi_item; +} + +const struct xfs_defer_op_type xfs_exchmaps_defer_type = { + .name = "exchmaps", + .max_items = 1, + .create_intent = xfs_exchmaps_create_intent, + .abort_intent = xfs_exchmaps_abort_intent, + .create_done = xfs_exchmaps_create_done, + .finish_item = xfs_exchmaps_finish_item, + .cancel_item = xfs_exchmaps_cancel_item, + .recover_work = xfs_exchmaps_recover_work, + .relog_intent = xfs_exchmaps_relog_intent, +}; + STATIC bool xfs_xmi_item_match( struct xfs_log_item *lip, @@ -194,8 +557,9 @@ xlog_recover_xmi_commit_pass2( xmi_lip = xfs_xmi_init(mp); memcpy(&xmi_lip->xmi_format, xmi_formatp, len); - /* not implemented yet */ - return -EIO; + xlog_recover_intent_item(log, &xmi_lip->xmi_item, lsn, + &xfs_exchmaps_defer_type); + return 0; } const struct xlog_recover_item_ops xlog_xmi_item_ops = { diff --git a/fs/xfs/xfs_exchmaps_item.h b/fs/xfs/xfs_exchmaps_item.h index ada1eb314e658..efa368d25d09c 100644 --- a/fs/xfs/xfs_exchmaps_item.h +++ b/fs/xfs/xfs_exchmaps_item.h @@ -56,4 +56,9 @@ struct xfs_xmd_log_item { extern struct kmem_cache *xfs_xmi_cache; extern struct kmem_cache *xfs_xmd_cache; +struct xfs_exchmaps_intent; + +void xfs_exchmaps_defer_add(struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi); + #endif /* __XFS_EXCHMAPS_ITEM_H__ */ diff --git a/fs/xfs/xfs_exchrange.c b/fs/xfs/xfs_exchrange.c index 4cd824e47f751..35351b9735210 100644 --- a/fs/xfs/xfs_exchrange.c +++ b/fs/xfs/xfs_exchrange.c @@ -13,8 +13,57 @@ #include "xfs_inode.h" #include "xfs_trans.h" #include "xfs_exchrange.h" +#include "xfs_exchmaps.h" #include +/* Lock (and optionally join) two inodes for a file range exchange. */ +void +xfs_exchrange_ilock( + struct xfs_trans *tp, + struct xfs_inode *ip1, + struct xfs_inode *ip2) +{ + if (ip1 != ip2) + xfs_lock_two_inodes(ip1, XFS_ILOCK_EXCL, + ip2, XFS_ILOCK_EXCL); + else + xfs_ilock(ip1, XFS_ILOCK_EXCL); + if (tp) { + xfs_trans_ijoin(tp, ip1, 0); + if (ip2 != ip1) + xfs_trans_ijoin(tp, ip2, 0); + } + +} + +/* Unlock two inodes after a file range exchange operation. */ +void +xfs_exchrange_iunlock( + struct xfs_inode *ip1, + struct xfs_inode *ip2) +{ + if (ip2 != ip1) + xfs_iunlock(ip2, XFS_ILOCK_EXCL); + xfs_iunlock(ip1, XFS_ILOCK_EXCL); +} + +/* + * Estimate the resource requirements to exchange file contents between the two + * files. The caller is required to hold the IOLOCK and the MMAPLOCK and to + * have flushed both inodes' pagecache and active direct-ios. + */ +int +xfs_exchrange_estimate( + struct xfs_exchmaps_req *req) +{ + int error; + + xfs_exchrange_ilock(NULL, req->ip1, req->ip2); + error = xfs_exchmaps_estimate(req); + xfs_exchrange_iunlock(req->ip1, req->ip2); + return error; +} + /* * Generic code for exchanging ranges of two files via XFS_IOC_EXCHANGE_RANGE. * This part deals with struct file objects and byte ranges and does not deal diff --git a/fs/xfs/xfs_exchrange.h b/fs/xfs/xfs_exchrange.h index f80369c7df5d6..039abcca546e8 100644 --- a/fs/xfs/xfs_exchrange.h +++ b/fs/xfs/xfs_exchrange.h @@ -27,4 +27,12 @@ struct xfs_exchrange { long xfs_ioc_exchange_range(struct file *file, struct xfs_exchange_range __user *argp); +struct xfs_exchmaps_req; + +void xfs_exchrange_ilock(struct xfs_trans *tp, struct xfs_inode *ip1, + struct xfs_inode *ip2); +void xfs_exchrange_iunlock(struct xfs_inode *ip1, struct xfs_inode *ip2); + +int xfs_exchrange_estimate(struct xfs_exchmaps_req *req); + #endif /* __XFS_EXCHRANGE_H__ */ diff --git a/fs/xfs/xfs_trace.c b/fs/xfs/xfs_trace.c index 1a963382e5e9e..9f38e69f1ce40 100644 --- a/fs/xfs/xfs_trace.c +++ b/fs/xfs/xfs_trace.c @@ -39,6 +39,7 @@ #include "xfs_buf_mem.h" #include "xfs_btree_mem.h" #include "xfs_bmap.h" +#include "xfs_exchmaps.h" /* * We include this last to have the helpers above available for the trace diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index aea97fc074f8d..7c17d1f80fec3 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -82,6 +82,8 @@ struct xfs_perag; struct xfbtree; struct xfs_btree_ops; struct xfs_bmap_intent; +struct xfs_exchmaps_intent; +struct xfs_exchmaps_req; #define XFS_ATTR_FILTER_FLAGS \ { XFS_ATTR_ROOT, "ROOT" }, \ @@ -4770,6 +4772,221 @@ DEFINE_XFBTREE_FREESP_EVENT(xfbtree_alloc_block); DEFINE_XFBTREE_FREESP_EVENT(xfbtree_free_block); #endif /* CONFIG_XFS_BTREE_IN_MEM */ +/* exchmaps tracepoints */ +#define XFS_EXCHMAPS_STRINGS \ + { XFS_EXCHMAPS_ATTR_FORK, "ATTRFORK" }, \ + { XFS_EXCHMAPS_SET_SIZES, "SETSIZES" }, \ + { XFS_EXCHMAPS_INO1_WRITTEN, "INO1_WRITTEN" }, \ + { XFS_EXCHMAPS_CLEAR_INO1_REFLINK, "CLEAR_INO1_REFLINK" }, \ + { XFS_EXCHMAPS_CLEAR_INO2_REFLINK, "CLEAR_INO2_REFLINK" } + +DEFINE_INODE_IREC_EVENT(xfs_exchmaps_mapping1_skip); +DEFINE_INODE_IREC_EVENT(xfs_exchmaps_mapping1); +DEFINE_INODE_IREC_EVENT(xfs_exchmaps_mapping2); +DEFINE_ITRUNC_EVENT(xfs_exchmaps_update_inode_size); + +TRACE_EVENT(xfs_exchmaps_overhead, + TP_PROTO(struct xfs_mount *mp, unsigned long long bmbt_blocks, + unsigned long long rmapbt_blocks), + TP_ARGS(mp, bmbt_blocks, rmapbt_blocks), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long long, bmbt_blocks) + __field(unsigned long long, rmapbt_blocks) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->bmbt_blocks = bmbt_blocks; + __entry->rmapbt_blocks = rmapbt_blocks; + ), + TP_printk("dev %d:%d bmbt_blocks 0x%llx rmapbt_blocks 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->bmbt_blocks, + __entry->rmapbt_blocks) +); + +DECLARE_EVENT_CLASS(xfs_exchmaps_estimate_class, + TP_PROTO(const struct xfs_exchmaps_req *req), + TP_ARGS(req), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino1) + __field(xfs_ino_t, ino2) + __field(xfs_fileoff_t, startoff1) + __field(xfs_fileoff_t, startoff2) + __field(xfs_filblks_t, blockcount) + __field(uint64_t, flags) + __field(xfs_filblks_t, ip1_bcount) + __field(xfs_filblks_t, ip2_bcount) + __field(xfs_filblks_t, ip1_rtbcount) + __field(xfs_filblks_t, ip2_rtbcount) + __field(unsigned long long, resblks) + __field(unsigned long long, nr_exchanges) + ), + TP_fast_assign( + __entry->dev = req->ip1->i_mount->m_super->s_dev; + __entry->ino1 = req->ip1->i_ino; + __entry->ino2 = req->ip2->i_ino; + __entry->startoff1 = req->startoff1; + __entry->startoff2 = req->startoff2; + __entry->blockcount = req->blockcount; + __entry->flags = req->flags; + __entry->ip1_bcount = req->ip1_bcount; + __entry->ip2_bcount = req->ip2_bcount; + __entry->ip1_rtbcount = req->ip1_rtbcount; + __entry->ip2_rtbcount = req->ip2_rtbcount; + __entry->resblks = req->resblks; + __entry->nr_exchanges = req->nr_exchanges; + ), + TP_printk("dev %d:%d ino1 0x%llx fileoff1 0x%llx ino2 0x%llx fileoff2 0x%llx fsbcount 0x%llx flags (%s) bcount1 0x%llx rtbcount1 0x%llx bcount2 0x%llx rtbcount2 0x%llx resblks 0x%llx nr_exchanges %llu", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino1, __entry->startoff1, + __entry->ino2, __entry->startoff2, + __entry->blockcount, + __print_flags_u64(__entry->flags, "|", XFS_EXCHMAPS_STRINGS), + __entry->ip1_bcount, + __entry->ip1_rtbcount, + __entry->ip2_bcount, + __entry->ip2_rtbcount, + __entry->resblks, + __entry->nr_exchanges) +); + +#define DEFINE_EXCHMAPS_ESTIMATE_EVENT(name) \ +DEFINE_EVENT(xfs_exchmaps_estimate_class, name, \ + TP_PROTO(const struct xfs_exchmaps_req *req), \ + TP_ARGS(req)) +DEFINE_EXCHMAPS_ESTIMATE_EVENT(xfs_exchmaps_initial_estimate); +DEFINE_EXCHMAPS_ESTIMATE_EVENT(xfs_exchmaps_final_estimate); + +DECLARE_EVENT_CLASS(xfs_exchmaps_intent_class, + TP_PROTO(struct xfs_mount *mp, const struct xfs_exchmaps_intent *xmi), + TP_ARGS(mp, xmi), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino1) + __field(xfs_ino_t, ino2) + __field(uint64_t, flags) + __field(xfs_fileoff_t, startoff1) + __field(xfs_fileoff_t, startoff2) + __field(xfs_filblks_t, blockcount) + __field(xfs_fsize_t, isize1) + __field(xfs_fsize_t, isize2) + __field(xfs_fsize_t, new_isize1) + __field(xfs_fsize_t, new_isize2) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->ino1 = xmi->xmi_ip1->i_ino; + __entry->ino2 = xmi->xmi_ip2->i_ino; + __entry->flags = xmi->xmi_flags; + __entry->startoff1 = xmi->xmi_startoff1; + __entry->startoff2 = xmi->xmi_startoff2; + __entry->blockcount = xmi->xmi_blockcount; + __entry->isize1 = xmi->xmi_ip1->i_disk_size; + __entry->isize2 = xmi->xmi_ip2->i_disk_size; + __entry->new_isize1 = xmi->xmi_isize1; + __entry->new_isize2 = xmi->xmi_isize2; + ), + TP_printk("dev %d:%d ino1 0x%llx fileoff1 0x%llx ino2 0x%llx fileoff2 0x%llx fsbcount 0x%llx flags (%s) isize1 0x%llx newisize1 0x%llx isize2 0x%llx newisize2 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino1, __entry->startoff1, + __entry->ino2, __entry->startoff2, + __entry->blockcount, + __print_flags_u64(__entry->flags, "|", XFS_EXCHMAPS_STRINGS), + __entry->isize1, __entry->new_isize1, + __entry->isize2, __entry->new_isize2) +); + +#define DEFINE_EXCHMAPS_INTENT_EVENT(name) \ +DEFINE_EVENT(xfs_exchmaps_intent_class, name, \ + TP_PROTO(struct xfs_mount *mp, const struct xfs_exchmaps_intent *xmi), \ + TP_ARGS(mp, xmi)) +DEFINE_EXCHMAPS_INTENT_EVENT(xfs_exchmaps_defer); +DEFINE_EXCHMAPS_INTENT_EVENT(xfs_exchmaps_recover); + +TRACE_EVENT(xfs_exchmaps_delta_nextents_step, + TP_PROTO(struct xfs_mount *mp, + const struct xfs_bmbt_irec *left, + const struct xfs_bmbt_irec *curr, + const struct xfs_bmbt_irec *new, + const struct xfs_bmbt_irec *right, + int delta, unsigned int state), + TP_ARGS(mp, left, curr, new, right, delta, state), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_fileoff_t, loff) + __field(xfs_fsblock_t, lstart) + __field(xfs_filblks_t, lcount) + __field(xfs_fileoff_t, coff) + __field(xfs_fsblock_t, cstart) + __field(xfs_filblks_t, ccount) + __field(xfs_fileoff_t, noff) + __field(xfs_fsblock_t, nstart) + __field(xfs_filblks_t, ncount) + __field(xfs_fileoff_t, roff) + __field(xfs_fsblock_t, rstart) + __field(xfs_filblks_t, rcount) + __field(int, delta) + __field(unsigned int, state) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->loff = left->br_startoff; + __entry->lstart = left->br_startblock; + __entry->lcount = left->br_blockcount; + __entry->coff = curr->br_startoff; + __entry->cstart = curr->br_startblock; + __entry->ccount = curr->br_blockcount; + __entry->noff = new->br_startoff; + __entry->nstart = new->br_startblock; + __entry->ncount = new->br_blockcount; + __entry->roff = right->br_startoff; + __entry->rstart = right->br_startblock; + __entry->rcount = right->br_blockcount; + __entry->delta = delta; + __entry->state = state; + ), + TP_printk("dev %d:%d left 0x%llx:0x%llx:0x%llx; curr 0x%llx:0x%llx:0x%llx <- new 0x%llx:0x%llx:0x%llx; right 0x%llx:0x%llx:0x%llx delta %d state 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->loff, __entry->lstart, __entry->lcount, + __entry->coff, __entry->cstart, __entry->ccount, + __entry->noff, __entry->nstart, __entry->ncount, + __entry->roff, __entry->rstart, __entry->rcount, + __entry->delta, __entry->state) +); + +TRACE_EVENT(xfs_exchmaps_delta_nextents, + TP_PROTO(const struct xfs_exchmaps_req *req, int64_t d_nexts1, + int64_t d_nexts2), + TP_ARGS(req, d_nexts1, d_nexts2), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino1) + __field(xfs_ino_t, ino2) + __field(xfs_extnum_t, nexts1) + __field(xfs_extnum_t, nexts2) + __field(int64_t, d_nexts1) + __field(int64_t, d_nexts2) + ), + TP_fast_assign( + int whichfork = xfs_exchmaps_reqfork(req); + + __entry->dev = req->ip1->i_mount->m_super->s_dev; + __entry->ino1 = req->ip1->i_ino; + __entry->ino2 = req->ip2->i_ino; + __entry->nexts1 = xfs_ifork_ptr(req->ip1, whichfork)->if_nextents; + __entry->nexts2 = xfs_ifork_ptr(req->ip2, whichfork)->if_nextents; + __entry->d_nexts1 = d_nexts1; + __entry->d_nexts2 = d_nexts2; + ), + TP_printk("dev %d:%d ino1 0x%llx nexts %llu ino2 0x%llx nexts %llu delta1 %lld delta2 %lld", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino1, __entry->nexts1, + __entry->ino2, __entry->nexts2, + __entry->d_nexts1, __entry->d_nexts2) +); + #endif /* _TRACE_XFS_H */ #undef TRACE_INCLUDE_PATH From 42672471f938cdab2573f32ce23915b78f0578f4 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:18 -0700 Subject: [PATCH 0207/1702] xfs: bind together the front and back ends of the file range exchange code So far, we've constructed the front end of the file range exchange code that does all the checking; and the back end of the file mapping exchange code that actually does the work. Glue these two pieces together so that we can turn on the functionality. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_exchrange.c | 334 ++++++++++++++++++++++++++++++++++++++++- fs/xfs/xfs_trace.c | 1 + fs/xfs/xfs_trace.h | 109 ++++++++++++++ 3 files changed, 443 insertions(+), 1 deletion(-) diff --git a/fs/xfs/xfs_exchrange.c b/fs/xfs/xfs_exchrange.c index 35351b9735210..0fc95e6471cb9 100644 --- a/fs/xfs/xfs_exchrange.c +++ b/fs/xfs/xfs_exchrange.c @@ -12,8 +12,15 @@ #include "xfs_defer.h" #include "xfs_inode.h" #include "xfs_trans.h" +#include "xfs_quota.h" +#include "xfs_bmap_util.h" +#include "xfs_reflink.h" +#include "xfs_trace.h" #include "xfs_exchrange.h" #include "xfs_exchmaps.h" +#include "xfs_sb.h" +#include "xfs_icache.h" +#include "xfs_log.h" #include /* Lock (and optionally join) two inodes for a file range exchange. */ @@ -64,6 +71,207 @@ xfs_exchrange_estimate( return error; } +#define QRETRY_IP1 (0x1) +#define QRETRY_IP2 (0x2) + +/* + * Obtain a quota reservation to make sure we don't hit EDQUOT. We can skip + * this if quota enforcement is disabled or if both inodes' dquots are the + * same. The qretry structure must be initialized to zeroes before the first + * call to this function. + */ +STATIC int +xfs_exchrange_reserve_quota( + struct xfs_trans *tp, + const struct xfs_exchmaps_req *req, + unsigned int *qretry) +{ + int64_t ddelta, rdelta; + int ip1_error = 0; + int error; + + /* + * Don't bother with a quota reservation if we're not enforcing them + * or the two inodes have the same dquots. + */ + if (!XFS_IS_QUOTA_ON(tp->t_mountp) || req->ip1 == req->ip2 || + (req->ip1->i_udquot == req->ip2->i_udquot && + req->ip1->i_gdquot == req->ip2->i_gdquot && + req->ip1->i_pdquot == req->ip2->i_pdquot)) + return 0; + + *qretry = 0; + + /* + * For each file, compute the net gain in the number of regular blocks + * that will be mapped into that file and reserve that much quota. The + * quota counts must be able to absorb at least that much space. + */ + ddelta = req->ip2_bcount - req->ip1_bcount; + rdelta = req->ip2_rtbcount - req->ip1_rtbcount; + if (ddelta > 0 || rdelta > 0) { + error = xfs_trans_reserve_quota_nblks(tp, req->ip1, + ddelta > 0 ? ddelta : 0, + rdelta > 0 ? rdelta : 0, + false); + if (error == -EDQUOT || error == -ENOSPC) { + /* + * Save this error and see what happens if we try to + * reserve quota for ip2. Then report both. + */ + *qretry |= QRETRY_IP1; + ip1_error = error; + error = 0; + } + if (error) + return error; + } + if (ddelta < 0 || rdelta < 0) { + error = xfs_trans_reserve_quota_nblks(tp, req->ip2, + ddelta < 0 ? -ddelta : 0, + rdelta < 0 ? -rdelta : 0, + false); + if (error == -EDQUOT || error == -ENOSPC) + *qretry |= QRETRY_IP2; + if (error) + return error; + } + if (ip1_error) + return ip1_error; + + /* + * For each file, forcibly reserve the gross gain in mapped blocks so + * that we don't trip over any quota block reservation assertions. + * We must reserve the gross gain because the quota code subtracts from + * bcount the number of blocks that we unmap; it does not add that + * quantity back to the quota block reservation. + */ + error = xfs_trans_reserve_quota_nblks(tp, req->ip1, req->ip1_bcount, + req->ip1_rtbcount, true); + if (error) + return error; + + return xfs_trans_reserve_quota_nblks(tp, req->ip2, req->ip2_bcount, + req->ip2_rtbcount, true); +} + +/* Exchange the mappings (and hence the contents) of two files' forks. */ +STATIC int +xfs_exchrange_mappings( + const struct xfs_exchrange *fxr, + struct xfs_inode *ip1, + struct xfs_inode *ip2) +{ + struct xfs_mount *mp = ip1->i_mount; + struct xfs_exchmaps_req req = { + .ip1 = ip1, + .ip2 = ip2, + .startoff1 = XFS_B_TO_FSBT(mp, fxr->file1_offset), + .startoff2 = XFS_B_TO_FSBT(mp, fxr->file2_offset), + .blockcount = XFS_B_TO_FSB(mp, fxr->length), + }; + struct xfs_trans *tp; + unsigned int qretry; + bool retried = false; + int error; + + trace_xfs_exchrange_mappings(fxr, ip1, ip2); + + if (fxr->flags & XFS_EXCHANGE_RANGE_TO_EOF) + req.flags |= XFS_EXCHMAPS_SET_SIZES; + if (fxr->flags & XFS_EXCHANGE_RANGE_FILE1_WRITTEN) + req.flags |= XFS_EXCHMAPS_INO1_WRITTEN; + + error = xfs_exchrange_estimate(&req); + if (error) + return error; + +retry: + /* Allocate the transaction, lock the inodes, and join them. */ + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, req.resblks, 0, + XFS_TRANS_RES_FDBLKS, &tp); + if (error) + return error; + + xfs_exchrange_ilock(tp, ip1, ip2); + + trace_xfs_exchrange_before(ip2, 2); + trace_xfs_exchrange_before(ip1, 1); + + error = xfs_exchmaps_check_forks(mp, &req); + if (error) + goto out_trans_cancel; + + /* + * Reserve ourselves some quota if any of them are in enforcing mode. + * In theory we only need enough to satisfy the change in the number + * of blocks between the two ranges being remapped. + */ + error = xfs_exchrange_reserve_quota(tp, &req, &qretry); + if ((error == -EDQUOT || error == -ENOSPC) && !retried) { + xfs_trans_cancel(tp); + xfs_exchrange_iunlock(ip1, ip2); + if (qretry & QRETRY_IP1) + xfs_blockgc_free_quota(ip1, 0); + if (qretry & QRETRY_IP2) + xfs_blockgc_free_quota(ip2, 0); + retried = true; + goto retry; + } + if (error) + goto out_trans_cancel; + + /* If we got this far on a dry run, all parameters are ok. */ + if (fxr->flags & XFS_EXCHANGE_RANGE_DRY_RUN) + goto out_trans_cancel; + + /* Update the mtime and ctime of both files. */ + if (fxr->flags & __XFS_EXCHANGE_RANGE_UPD_CMTIME1) + xfs_trans_ichgtime(tp, ip1, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); + if (fxr->flags & __XFS_EXCHANGE_RANGE_UPD_CMTIME2) + xfs_trans_ichgtime(tp, ip2, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); + + xfs_exchange_mappings(tp, &req); + + /* + * Force the log to persist metadata updates if the caller or the + * administrator requires this. The generic prep function already + * flushed the relevant parts of the page cache. + */ + if (xfs_has_wsync(mp) || (fxr->flags & XFS_EXCHANGE_RANGE_DSYNC)) + xfs_trans_set_sync(tp); + + error = xfs_trans_commit(tp); + + trace_xfs_exchrange_after(ip2, 2); + trace_xfs_exchrange_after(ip1, 1); + + if (error) + goto out_unlock; + + /* + * If the caller wanted us to exchange the contents of two complete + * files of unequal length, exchange the incore sizes now. This should + * be safe because we flushed both files' page caches, exchanged all + * the mappings, and updated the ondisk sizes. + */ + if (fxr->flags & XFS_EXCHANGE_RANGE_TO_EOF) { + loff_t temp; + + temp = i_size_read(VFS_I(ip2)); + i_size_write(VFS_I(ip2), i_size_read(VFS_I(ip1))); + i_size_write(VFS_I(ip1), temp); + } + +out_unlock: + xfs_exchrange_iunlock(ip1, ip2); + return error; + +out_trans_cancel: + xfs_trans_cancel(tp); + goto out_unlock; +} + /* * Generic code for exchanging ranges of two files via XFS_IOC_EXCHANGE_RANGE. * This part deals with struct file objects and byte ranges and does not deal @@ -287,6 +495,130 @@ xfs_exchange_range_finish( return file_remove_privs(fxr->file2); } +/* Prepare two files to have their data exchanged. */ +STATIC int +xfs_exchrange_prep( + struct xfs_exchrange *fxr, + struct xfs_inode *ip1, + struct xfs_inode *ip2) +{ + unsigned int alloc_unit = xfs_inode_alloc_unitsize(ip2); + int error; + + trace_xfs_exchrange_prep(fxr, ip1, ip2); + + /* Verify both files are either real-time or non-realtime */ + if (XFS_IS_REALTIME_INODE(ip1) != XFS_IS_REALTIME_INODE(ip2)) + return -EINVAL; + + /* + * The alignment checks in the generic helpers cannot deal with + * allocation units that are not powers of 2. This can happen with the + * realtime volume if the extent size is set. + */ + if (!is_power_of_2(alloc_unit)) + return -EOPNOTSUPP; + + error = xfs_exchange_range_prep(fxr, alloc_unit); + if (error || fxr->length == 0) + return error; + + /* Attach dquots to both inodes before changing block maps. */ + error = xfs_qm_dqattach(ip2); + if (error) + return error; + error = xfs_qm_dqattach(ip1); + if (error) + return error; + + trace_xfs_exchrange_flush(fxr, ip1, ip2); + + /* Flush the relevant ranges of both files. */ + error = xfs_flush_unmap_range(ip2, fxr->file2_offset, fxr->length); + if (error) + return error; + error = xfs_flush_unmap_range(ip1, fxr->file1_offset, fxr->length); + if (error) + return error; + + /* + * Cancel CoW fork preallocations for the ranges of both files. The + * prep function should have flushed all the dirty data, so the only + * CoW mappings remaining should be speculative. + */ + if (xfs_inode_has_cow_data(ip1)) { + error = xfs_reflink_cancel_cow_range(ip1, fxr->file1_offset, + fxr->length, true); + if (error) + return error; + } + + if (xfs_inode_has_cow_data(ip2)) { + error = xfs_reflink_cancel_cow_range(ip2, fxr->file2_offset, + fxr->length, true); + if (error) + return error; + } + + return 0; +} + +/* + * Exchange contents of files. This is the binding between the generic + * file-level concepts and the XFS inode-specific implementation. + */ +STATIC int +xfs_exchrange_contents( + struct xfs_exchrange *fxr) +{ + struct inode *inode1 = file_inode(fxr->file1); + struct inode *inode2 = file_inode(fxr->file2); + struct xfs_inode *ip1 = XFS_I(inode1); + struct xfs_inode *ip2 = XFS_I(inode2); + struct xfs_mount *mp = ip1->i_mount; + int error; + + if (!xfs_has_exchange_range(mp)) + return -EOPNOTSUPP; + + if (fxr->flags & ~(XFS_EXCHANGE_RANGE_ALL_FLAGS | + XFS_EXCHANGE_RANGE_PRIV_FLAGS)) + return -EINVAL; + + if (xfs_is_shutdown(mp)) + return -EIO; + + /* Lock both files against IO */ + error = xfs_ilock2_io_mmap(ip1, ip2); + if (error) + goto out_err; + + /* Prepare and then exchange file contents. */ + error = xfs_exchrange_prep(fxr, ip1, ip2); + if (error) + goto out_unlock; + + error = xfs_exchrange_mappings(fxr, ip1, ip2); + if (error) + goto out_unlock; + + /* + * Finish the exchange by removing special file privileges like any + * other file write would do. This may involve turning on support for + * logged xattrs if either file has security capabilities. + */ + error = xfs_exchange_range_finish(fxr); + if (error) + goto out_unlock; + +out_unlock: + xfs_iunlock2_io_mmap(ip1, ip2); +out_err: + if (error) + trace_xfs_exchrange_error(ip2, error, _RET_IP_); + return error; +} + /* Exchange parts of two files. */ static int xfs_exchange_range( @@ -341,7 +673,7 @@ xfs_exchange_range( fxr->flags |= __XFS_EXCHANGE_RANGE_UPD_CMTIME2; file_start_write(fxr->file2); - ret = -EOPNOTSUPP; /* XXX call out to lower level code */ + ret = xfs_exchrange_contents(fxr); file_end_write(fxr->file2); if (ret) return ret; diff --git a/fs/xfs/xfs_trace.c b/fs/xfs/xfs_trace.c index 9f38e69f1ce40..cf92a3bd56c79 100644 --- a/fs/xfs/xfs_trace.c +++ b/fs/xfs/xfs_trace.c @@ -40,6 +40,7 @@ #include "xfs_btree_mem.h" #include "xfs_bmap.h" #include "xfs_exchmaps.h" +#include "xfs_exchrange.h" /* * We include this last to have the helpers above available for the trace diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index 7c17d1f80fec3..729e728c2076f 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -84,6 +84,7 @@ struct xfs_btree_ops; struct xfs_bmap_intent; struct xfs_exchmaps_intent; struct xfs_exchmaps_req; +struct xfs_exchrange; #define XFS_ATTR_FILTER_FLAGS \ { XFS_ATTR_ROOT, "ROOT" }, \ @@ -4785,6 +4786,114 @@ DEFINE_INODE_IREC_EVENT(xfs_exchmaps_mapping1); DEFINE_INODE_IREC_EVENT(xfs_exchmaps_mapping2); DEFINE_ITRUNC_EVENT(xfs_exchmaps_update_inode_size); +#define XFS_EXCHRANGE_INODES \ + { 1, "file1" }, \ + { 2, "file2" } + +DECLARE_EVENT_CLASS(xfs_exchrange_inode_class, + TP_PROTO(struct xfs_inode *ip, int whichfile), + TP_ARGS(ip, whichfile), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(int, whichfile) + __field(xfs_ino_t, ino) + __field(int, format) + __field(xfs_extnum_t, nex) + __field(int, broot_size) + __field(int, fork_off) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->whichfile = whichfile; + __entry->ino = ip->i_ino; + __entry->format = ip->i_df.if_format; + __entry->nex = ip->i_df.if_nextents; + __entry->fork_off = xfs_inode_fork_boff(ip); + ), + TP_printk("dev %d:%d ino 0x%llx whichfile %s format %s num_extents %llu forkoff 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->whichfile, XFS_EXCHRANGE_INODES), + __print_symbolic(__entry->format, XFS_INODE_FORMAT_STR), + __entry->nex, + __entry->fork_off) +) + +#define DEFINE_EXCHRANGE_INODE_EVENT(name) \ +DEFINE_EVENT(xfs_exchrange_inode_class, name, \ + TP_PROTO(struct xfs_inode *ip, int whichfile), \ + TP_ARGS(ip, whichfile)) + +DEFINE_EXCHRANGE_INODE_EVENT(xfs_exchrange_before); +DEFINE_EXCHRANGE_INODE_EVENT(xfs_exchrange_after); +DEFINE_INODE_ERROR_EVENT(xfs_exchrange_error); + +#define XFS_EXCHANGE_RANGE_FLAGS_STRS \ + { XFS_EXCHANGE_RANGE_TO_EOF, "TO_EOF" }, \ + { XFS_EXCHANGE_RANGE_DSYNC , "DSYNC" }, \ + { XFS_EXCHANGE_RANGE_DRY_RUN, "DRY_RUN" }, \ + { XFS_EXCHANGE_RANGE_FILE1_WRITTEN, "F1_WRITTEN" }, \ + { __XFS_EXCHANGE_RANGE_UPD_CMTIME1, "CMTIME1" }, \ + { __XFS_EXCHANGE_RANGE_UPD_CMTIME2, "CMTIME2" } + +/* file exchange-range tracepoint class */ +DECLARE_EVENT_CLASS(xfs_exchrange_class, + TP_PROTO(const struct xfs_exchrange *fxr, struct xfs_inode *ip1, + struct xfs_inode *ip2), + TP_ARGS(fxr, ip1, ip2), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ip1_ino) + __field(loff_t, ip1_isize) + __field(loff_t, ip1_disize) + __field(xfs_ino_t, ip2_ino) + __field(loff_t, ip2_isize) + __field(loff_t, ip2_disize) + + __field(loff_t, file1_offset) + __field(loff_t, file2_offset) + __field(unsigned long long, length) + __field(unsigned long long, flags) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip1)->i_sb->s_dev; + __entry->ip1_ino = ip1->i_ino; + __entry->ip1_isize = VFS_I(ip1)->i_size; + __entry->ip1_disize = ip1->i_disk_size; + __entry->ip2_ino = ip2->i_ino; + __entry->ip2_isize = VFS_I(ip2)->i_size; + __entry->ip2_disize = ip2->i_disk_size; + + __entry->file1_offset = fxr->file1_offset; + __entry->file2_offset = fxr->file2_offset; + __entry->length = fxr->length; + __entry->flags = fxr->flags; + ), + TP_printk("dev %d:%d flags %s bytecount 0x%llx " + "ino1 0x%llx isize 0x%llx disize 0x%llx pos 0x%llx -> " + "ino2 0x%llx isize 0x%llx disize 0x%llx pos 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_flags_u64(__entry->flags, "|", XFS_EXCHANGE_RANGE_FLAGS_STRS), + __entry->length, + __entry->ip1_ino, + __entry->ip1_isize, + __entry->ip1_disize, + __entry->file1_offset, + __entry->ip2_ino, + __entry->ip2_isize, + __entry->ip2_disize, + __entry->file2_offset) +) + +#define DEFINE_EXCHRANGE_EVENT(name) \ +DEFINE_EVENT(xfs_exchrange_class, name, \ + TP_PROTO(const struct xfs_exchrange *fxr, struct xfs_inode *ip1, \ + struct xfs_inode *ip2), \ + TP_ARGS(fxr, ip1, ip2)) +DEFINE_EXCHRANGE_EVENT(xfs_exchrange_prep); +DEFINE_EXCHRANGE_EVENT(xfs_exchrange_flush); +DEFINE_EXCHRANGE_EVENT(xfs_exchrange_mappings); + TRACE_EVENT(xfs_exchmaps_overhead, TP_PROTO(struct xfs_mount *mp, unsigned long long bmbt_blocks, unsigned long long rmapbt_blocks), From 5fd022ec7d420dfca1eaaf997923a5d4dd0dcf62 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:19 -0700 Subject: [PATCH 0208/1702] xfs: add error injection to test file mapping exchange recovery Add an errortag so that we can test recovery of exchmaps log items. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_errortag.h | 4 +++- fs/xfs/libxfs/xfs_exchmaps.c | 3 +++ fs/xfs/xfs_error.c | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_errortag.h b/fs/xfs/libxfs/xfs_errortag.h index 01a9e86b30379..7002d7676a788 100644 --- a/fs/xfs/libxfs/xfs_errortag.h +++ b/fs/xfs/libxfs/xfs_errortag.h @@ -63,7 +63,8 @@ #define XFS_ERRTAG_ATTR_LEAF_TO_NODE 41 #define XFS_ERRTAG_WB_DELAY_MS 42 #define XFS_ERRTAG_WRITE_DELAY_MS 43 -#define XFS_ERRTAG_MAX 44 +#define XFS_ERRTAG_EXCHMAPS_FINISH_ONE 44 +#define XFS_ERRTAG_MAX 45 /* * Random factors for above tags, 1 means always, 2 means 1/2 time, etc. @@ -111,5 +112,6 @@ #define XFS_RANDOM_ATTR_LEAF_TO_NODE 1 #define XFS_RANDOM_WB_DELAY_MS 3000 #define XFS_RANDOM_WRITE_DELAY_MS 3000 +#define XFS_RANDOM_EXCHMAPS_FINISH_ONE 1 #endif /* __XFS_ERRORTAG_H_ */ diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index b8e9450cc175f..3b1f29e95fea6 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -437,6 +437,9 @@ xfs_exchmaps_finish_one( return error; } + if (XFS_TEST_ERROR(false, tp->t_mountp, XFS_ERRTAG_EXCHMAPS_FINISH_ONE)) + return -EIO; + /* If we still have work to do, ask for a new transaction. */ if (xmi_has_more_exchange_work(xmi) || xmi_has_postop_work(xmi)) { trace_xfs_exchmaps_defer(tp->t_mountp, xmi); diff --git a/fs/xfs/xfs_error.c b/fs/xfs/xfs_error.c index 7ad0e92c6b5b8..78cdc5064a8c4 100644 --- a/fs/xfs/xfs_error.c +++ b/fs/xfs/xfs_error.c @@ -62,6 +62,7 @@ static unsigned int xfs_errortag_random_default[] = { XFS_RANDOM_ATTR_LEAF_TO_NODE, XFS_RANDOM_WB_DELAY_MS, XFS_RANDOM_WRITE_DELAY_MS, + XFS_RANDOM_EXCHMAPS_FINISH_ONE, }; struct xfs_errortag_attr { @@ -179,6 +180,7 @@ XFS_ERRORTAG_ATTR_RW(da_leaf_split, XFS_ERRTAG_DA_LEAF_SPLIT); XFS_ERRORTAG_ATTR_RW(attr_leaf_to_node, XFS_ERRTAG_ATTR_LEAF_TO_NODE); XFS_ERRORTAG_ATTR_RW(wb_delay_ms, XFS_ERRTAG_WB_DELAY_MS); XFS_ERRORTAG_ATTR_RW(write_delay_ms, XFS_ERRTAG_WRITE_DELAY_MS); +XFS_ERRORTAG_ATTR_RW(exchmaps_finish_one, XFS_ERRTAG_EXCHMAPS_FINISH_ONE); static struct attribute *xfs_errortag_attrs[] = { XFS_ERRORTAG_ATTR_LIST(noerror), @@ -224,6 +226,7 @@ static struct attribute *xfs_errortag_attrs[] = { XFS_ERRORTAG_ATTR_LIST(attr_leaf_to_node), XFS_ERRORTAG_ATTR_LIST(wb_delay_ms), XFS_ERRORTAG_ATTR_LIST(write_delay_ms), + XFS_ERRORTAG_ATTR_LIST(exchmaps_finish_one), NULL, }; ATTRIBUTE_GROUPS(xfs_errortag); From 497d7a2608f8b7329e92bdaaf745ca127a582ad9 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:20 -0700 Subject: [PATCH 0209/1702] xfs: condense extended attributes after a mapping exchange operation Add a new file mapping exchange flag that enables us to perform post-exchange processing on file2 once we're done exchanging the extent mappings. If we were swapping mappings between extended attribute forks, we want to be able to convert file2's attr fork from block to inline format. (This implies that all fork contents are exchanged.) This isn't used anywhere right now, but we need to have the basic ondisk flags in place so that a future online xattr repair feature can create salvaged attrs in a temporary file and exchange the attr fork mappings when ready. If one file is in extents format and the other is inline, we will have to promote both to extents format to perform the exchange. After the exchange, we can try to condense the fixed file's attr fork back down to inline format if possible. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_exchmaps.c | 53 ++++++++++++++++++++++++++++++++++-- fs/xfs/libxfs/xfs_exchmaps.h | 5 ++++ fs/xfs/xfs_trace.h | 3 +- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index 3b1f29e95fea6..e46b314fa0cfd 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -24,6 +24,10 @@ #include "xfs_errortag.h" #include "xfs_health.h" #include "xfs_exchmaps_item.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr_leaf.h" +#include "xfs_attr.h" struct kmem_cache *xfs_exchmaps_intent_cache; @@ -121,7 +125,8 @@ static inline bool xmi_has_postop_work(const struct xfs_exchmaps_intent *xmi) { return xmi->xmi_flags & (XFS_EXCHMAPS_CLEAR_INO1_REFLINK | - XFS_EXCHMAPS_CLEAR_INO2_REFLINK); + XFS_EXCHMAPS_CLEAR_INO2_REFLINK | + __XFS_EXCHMAPS_INO2_SHORTFORM); } /* Check all mappings to make sure we can actually exchange them. */ @@ -360,6 +365,36 @@ xfs_exchmaps_one_step( xmi_advance(xmi, irec1); } +/* Convert inode2's leaf attr fork back to shortform, if possible.. */ +STATIC int +xfs_exchmaps_attr_to_sf( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi) +{ + struct xfs_da_args args = { + .dp = xmi->xmi_ip2, + .geo = tp->t_mountp->m_attr_geo, + .whichfork = XFS_ATTR_FORK, + .trans = tp, + }; + struct xfs_buf *bp; + int forkoff; + int error; + + if (!xfs_attr_is_leaf(xmi->xmi_ip2)) + return 0; + + error = xfs_attr3_leaf_read(tp, xmi->xmi_ip2, 0, &bp); + if (error) + return error; + + forkoff = xfs_attr_shortform_allfit(bp, xmi->xmi_ip2); + if (forkoff == 0) + return 0; + + return xfs_attr3_leaf_to_shortform(bp, &args, forkoff); +} + /* Clear the reflink flag after an exchange. */ static inline void xfs_exchmaps_clear_reflink( @@ -378,6 +413,16 @@ xfs_exchmaps_do_postop_work( struct xfs_trans *tp, struct xfs_exchmaps_intent *xmi) { + if (xmi->xmi_flags & __XFS_EXCHMAPS_INO2_SHORTFORM) { + int error = 0; + + if (xmi->xmi_flags & XFS_EXCHMAPS_ATTR_FORK) + error = xfs_exchmaps_attr_to_sf(tp, xmi); + xmi->xmi_flags &= ~__XFS_EXCHMAPS_INO2_SHORTFORM; + if (error) + return error; + } + if (xmi->xmi_flags & XFS_EXCHMAPS_CLEAR_INO1_REFLINK) { xfs_exchmaps_clear_reflink(tp, xmi->xmi_ip1); xmi->xmi_flags &= ~XFS_EXCHMAPS_CLEAR_INO1_REFLINK; @@ -809,8 +854,10 @@ xfs_exchmaps_init_intent( xmi->xmi_isize1 = xmi->xmi_isize2 = -1; xmi->xmi_flags = req->flags & XFS_EXCHMAPS_PARAMS; - if (xfs_exchmaps_whichfork(xmi) == XFS_ATTR_FORK) + if (xfs_exchmaps_whichfork(xmi) == XFS_ATTR_FORK) { + xmi->xmi_flags |= __XFS_EXCHMAPS_INO2_SHORTFORM; return xmi; + } if (req->flags & XFS_EXCHMAPS_SET_SIZES) { xmi->xmi_flags |= XFS_EXCHMAPS_SET_SIZES; @@ -1031,6 +1078,8 @@ xfs_exchange_mappings( { struct xfs_exchmaps_intent *xmi; + BUILD_BUG_ON(XFS_EXCHMAPS_INTERNAL_FLAGS & XFS_EXCHMAPS_LOGGED_FLAGS); + xfs_assert_ilocked(req->ip1, XFS_ILOCK_EXCL); xfs_assert_ilocked(req->ip2, XFS_ILOCK_EXCL); ASSERT(!(req->flags & ~XFS_EXCHMAPS_LOGGED_FLAGS)); diff --git a/fs/xfs/libxfs/xfs_exchmaps.h b/fs/xfs/libxfs/xfs_exchmaps.h index e8fc3f80c68c2..d8718fca606e5 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.h +++ b/fs/xfs/libxfs/xfs_exchmaps.h @@ -27,6 +27,11 @@ struct xfs_exchmaps_intent { uint64_t xmi_flags; /* XFS_EXCHMAPS_* flags */ }; +/* Try to convert inode2 from block to short format at the end, if possible. */ +#define __XFS_EXCHMAPS_INO2_SHORTFORM (1ULL << 63) + +#define XFS_EXCHMAPS_INTERNAL_FLAGS (__XFS_EXCHMAPS_INO2_SHORTFORM) + /* flags that can be passed to xfs_exchmaps_{estimate,mappings} */ #define XFS_EXCHMAPS_PARAMS (XFS_EXCHMAPS_ATTR_FORK | \ XFS_EXCHMAPS_SET_SIZES | \ diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index 729e728c2076f..caef95f2c87cb 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -4779,7 +4779,8 @@ DEFINE_XFBTREE_FREESP_EVENT(xfbtree_free_block); { XFS_EXCHMAPS_SET_SIZES, "SETSIZES" }, \ { XFS_EXCHMAPS_INO1_WRITTEN, "INO1_WRITTEN" }, \ { XFS_EXCHMAPS_CLEAR_INO1_REFLINK, "CLEAR_INO1_REFLINK" }, \ - { XFS_EXCHMAPS_CLEAR_INO2_REFLINK, "CLEAR_INO2_REFLINK" } + { XFS_EXCHMAPS_CLEAR_INO2_REFLINK, "CLEAR_INO2_REFLINK" }, \ + { __XFS_EXCHMAPS_INO2_SHORTFORM, "INO2_SF" } DEFINE_INODE_IREC_EVENT(xfs_exchmaps_mapping1_skip); DEFINE_INODE_IREC_EVENT(xfs_exchmaps_mapping1); From da165fbde23b84591b6ccdf6749277d2d767b770 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:20 -0700 Subject: [PATCH 0210/1702] xfs: condense directories after a mapping exchange operation The previous commit added a new file mapping exchange flag that enables us to perform post-swap processing on file2 once we're done exchanging extent mappings. Now add this ability for directories. This isn't used anywhere right now, but we need to have the basic ondisk flags in place so that a future online directory repair feature can create salvaged dirents in a temporary directory and exchange the data fork mappings when ready. If one file is in extents format and the other is inline, we will have to promote both to extents format to perform the exchange. After the exchange, we can try to condense the fixed directory down to inline format if possible. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_exchmaps.c | 43 ++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index e46b314fa0cfd..f199629adbf0a 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -28,6 +28,8 @@ #include "xfs_da_btree.h" #include "xfs_attr_leaf.h" #include "xfs_attr.h" +#include "xfs_dir2_priv.h" +#include "xfs_dir2.h" struct kmem_cache *xfs_exchmaps_intent_cache; @@ -395,6 +397,42 @@ xfs_exchmaps_attr_to_sf( return xfs_attr3_leaf_to_shortform(bp, &args, forkoff); } +/* Convert inode2's block dir fork back to shortform, if possible.. */ +STATIC int +xfs_exchmaps_dir_to_sf( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi) +{ + struct xfs_da_args args = { + .dp = xmi->xmi_ip2, + .geo = tp->t_mountp->m_dir_geo, + .whichfork = XFS_DATA_FORK, + .trans = tp, + }; + struct xfs_dir2_sf_hdr sfh; + struct xfs_buf *bp; + bool isblock; + int size; + int error; + + error = xfs_dir2_isblock(&args, &isblock); + if (error) + return error; + + if (!isblock) + return 0; + + error = xfs_dir3_block_read(tp, xmi->xmi_ip2, &bp); + if (error) + return error; + + size = xfs_dir2_block_sfsize(xmi->xmi_ip2, bp->b_addr, &sfh); + if (size > xfs_inode_data_fork_size(xmi->xmi_ip2)) + return 0; + + return xfs_dir2_block_to_sf(&args, bp, size, &sfh); +} + /* Clear the reflink flag after an exchange. */ static inline void xfs_exchmaps_clear_reflink( @@ -418,6 +456,8 @@ xfs_exchmaps_do_postop_work( if (xmi->xmi_flags & XFS_EXCHMAPS_ATTR_FORK) error = xfs_exchmaps_attr_to_sf(tp, xmi); + else if (S_ISDIR(VFS_I(xmi->xmi_ip2)->i_mode)) + error = xfs_exchmaps_dir_to_sf(tp, xmi); xmi->xmi_flags &= ~__XFS_EXCHMAPS_INO2_SHORTFORM; if (error) return error; @@ -882,6 +922,9 @@ xfs_exchmaps_init_intent( xmi->xmi_flags |= XFS_EXCHMAPS_CLEAR_INO2_REFLINK; } + if (S_ISDIR(VFS_I(xmi->xmi_ip2)->i_mode)) + xmi->xmi_flags |= __XFS_EXCHMAPS_INO2_SHORTFORM; + return xmi; } From 33a9be2b7016e79f47c4c1b523a0aa59d41893c0 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:21 -0700 Subject: [PATCH 0211/1702] xfs: condense symbolic links after a mapping exchange operation The previous commit added a new file mapping exchange flag that enables us to perform post-exchange processing on file2 once we're done exchanging the extent mappings. Now add this ability for symlinks. This isn't used anywhere right now, but we need to have the basic ondisk flags in place so that a future online symlink repair feature can salvage the remote target in a temporary link and exchange the data fork mappings when ready. If one file is in extents format and the other is inline, we will have to promote both to extents format to perform the exchange. After the exchange, we can try to condense the fixed symlink down to inline format if possible. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_exchmaps.c | 49 +++++++++++++++++++++++++++- fs/xfs/libxfs/xfs_symlink_remote.c | 47 +++++++++++++++++++++++++++ fs/xfs/libxfs/xfs_symlink_remote.h | 1 + fs/xfs/xfs_symlink.c | 51 ++++-------------------------- 4 files changed, 103 insertions(+), 45 deletions(-) diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index f199629adbf0a..f58240466b1c4 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -30,6 +30,7 @@ #include "xfs_attr.h" #include "xfs_dir2_priv.h" #include "xfs_dir2.h" +#include "xfs_symlink_remote.h" struct kmem_cache *xfs_exchmaps_intent_cache; @@ -433,6 +434,49 @@ xfs_exchmaps_dir_to_sf( return xfs_dir2_block_to_sf(&args, bp, size, &sfh); } +/* Convert inode2's remote symlink target back to shortform, if possible. */ +STATIC int +xfs_exchmaps_link_to_sf( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi) +{ + struct xfs_inode *ip = xmi->xmi_ip2; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + char *buf; + int error; + + if (ifp->if_format == XFS_DINODE_FMT_LOCAL || + ip->i_disk_size > xfs_inode_data_fork_size(ip)) + return 0; + + /* Read the current symlink target into a buffer. */ + buf = kmalloc(ip->i_disk_size + 1, + GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); + if (!buf) { + ASSERT(0); + return -ENOMEM; + } + + error = xfs_symlink_remote_read(ip, buf); + if (error) + goto free; + + /* Remove the blocks. */ + error = xfs_symlink_remote_truncate(tp, ip); + if (error) + goto free; + + /* Convert fork to local format and log our changes. */ + xfs_idestroy_fork(ifp); + ifp->if_bytes = 0; + ifp->if_format = XFS_DINODE_FMT_LOCAL; + xfs_init_local_fork(ip, XFS_DATA_FORK, buf, ip->i_disk_size); + xfs_trans_log_inode(tp, ip, XFS_ILOG_DDATA | XFS_ILOG_CORE); +free: + kfree(buf); + return error; +} + /* Clear the reflink flag after an exchange. */ static inline void xfs_exchmaps_clear_reflink( @@ -458,6 +502,8 @@ xfs_exchmaps_do_postop_work( error = xfs_exchmaps_attr_to_sf(tp, xmi); else if (S_ISDIR(VFS_I(xmi->xmi_ip2)->i_mode)) error = xfs_exchmaps_dir_to_sf(tp, xmi); + else if (S_ISLNK(VFS_I(xmi->xmi_ip2)->i_mode)) + error = xfs_exchmaps_link_to_sf(tp, xmi); xmi->xmi_flags &= ~__XFS_EXCHMAPS_INO2_SHORTFORM; if (error) return error; @@ -922,7 +968,8 @@ xfs_exchmaps_init_intent( xmi->xmi_flags |= XFS_EXCHMAPS_CLEAR_INO2_REFLINK; } - if (S_ISDIR(VFS_I(xmi->xmi_ip2)->i_mode)) + if (S_ISDIR(VFS_I(xmi->xmi_ip2)->i_mode) || + S_ISLNK(VFS_I(xmi->xmi_ip2)->i_mode)) xmi->xmi_flags |= __XFS_EXCHMAPS_INO2_SHORTFORM; return xmi; diff --git a/fs/xfs/libxfs/xfs_symlink_remote.c b/fs/xfs/libxfs/xfs_symlink_remote.c index ffb1317a92123..8f0d5c584f46f 100644 --- a/fs/xfs/libxfs/xfs_symlink_remote.c +++ b/fs/xfs/libxfs/xfs_symlink_remote.c @@ -380,3 +380,50 @@ xfs_symlink_write_target( ASSERT(pathlen == 0); return 0; } + +/* Remove all the blocks from a symlink and invalidate buffers. */ +int +xfs_symlink_remote_truncate( + struct xfs_trans *tp, + struct xfs_inode *ip) +{ + struct xfs_bmbt_irec mval[XFS_SYMLINK_MAPS]; + struct xfs_mount *mp = tp->t_mountp; + struct xfs_buf *bp; + int nmaps = XFS_SYMLINK_MAPS; + int done = 0; + int i; + int error; + + /* Read mappings and invalidate buffers. */ + error = xfs_bmapi_read(ip, 0, XFS_MAX_FILEOFF, mval, &nmaps, 0); + if (error) + return error; + + for (i = 0; i < nmaps; i++) { + if (!xfs_bmap_is_real_extent(&mval[i])) + break; + + error = xfs_trans_get_buf(tp, mp->m_ddev_targp, + XFS_FSB_TO_DADDR(mp, mval[i].br_startblock), + XFS_FSB_TO_BB(mp, mval[i].br_blockcount), 0, + &bp); + if (error) + return error; + + xfs_trans_binval(tp, bp); + } + + /* Unmap the remote blocks. */ + error = xfs_bunmapi(tp, ip, 0, XFS_MAX_FILEOFF, 0, nmaps, &done); + if (error) + return error; + if (!done) { + ASSERT(done); + xfs_inode_mark_sick(ip, XFS_SICK_INO_SYMLINK); + return -EFSCORRUPTED; + } + + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + return 0; +} diff --git a/fs/xfs/libxfs/xfs_symlink_remote.h b/fs/xfs/libxfs/xfs_symlink_remote.h index a63bd38ae4faf..ac3dac8f617ed 100644 --- a/fs/xfs/libxfs/xfs_symlink_remote.h +++ b/fs/xfs/libxfs/xfs_symlink_remote.h @@ -22,5 +22,6 @@ int xfs_symlink_remote_read(struct xfs_inode *ip, char *link); int xfs_symlink_write_target(struct xfs_trans *tp, struct xfs_inode *ip, const char *target_path, int pathlen, xfs_fsblock_t fs_blocks, uint resblks); +int xfs_symlink_remote_truncate(struct xfs_trans *tp, struct xfs_inode *ip); #endif /* __XFS_SYMLINK_REMOTE_H */ diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index 3e376d24c7c16..3daeebff4bb47 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -250,19 +250,12 @@ xfs_symlink( */ STATIC int xfs_inactive_symlink_rmt( - struct xfs_inode *ip) + struct xfs_inode *ip) { - struct xfs_buf *bp; - int done; - int error; - int i; - xfs_mount_t *mp; - xfs_bmbt_irec_t mval[XFS_SYMLINK_MAPS]; - int nmaps; - int size; - xfs_trans_t *tp; - - mp = ip->i_mount; + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + int error; + ASSERT(!xfs_need_iread_extents(&ip->i_df)); /* * We're freeing a symlink that has some @@ -286,44 +279,14 @@ xfs_inactive_symlink_rmt( * locked for the second transaction. In the error paths we need it * held so the cancel won't rele it, see below. */ - size = (int)ip->i_disk_size; ip->i_disk_size = 0; VFS_I(ip)->i_mode = (VFS_I(ip)->i_mode & ~S_IFMT) | S_IFREG; xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); - /* - * Find the block(s) so we can inval and unmap them. - */ - done = 0; - nmaps = ARRAY_SIZE(mval); - error = xfs_bmapi_read(ip, 0, xfs_symlink_blocks(mp, size), - mval, &nmaps, 0); - if (error) - goto error_trans_cancel; - /* - * Invalidate the block(s). No validation is done. - */ - for (i = 0; i < nmaps; i++) { - error = xfs_trans_get_buf(tp, mp->m_ddev_targp, - XFS_FSB_TO_DADDR(mp, mval[i].br_startblock), - XFS_FSB_TO_BB(mp, mval[i].br_blockcount), 0, - &bp); - if (error) - goto error_trans_cancel; - xfs_trans_binval(tp, bp); - } - /* - * Unmap the dead block(s) to the dfops. - */ - error = xfs_bunmapi(tp, ip, 0, size, 0, nmaps, &done); + + error = xfs_symlink_remote_truncate(tp, ip); if (error) goto error_trans_cancel; - ASSERT(done); - /* - * Commit the transaction. This first logs the EFI and the inode, then - * rolls and commits the transaction that frees the extents. - */ - xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); error = xfs_trans_commit(tp); if (error) { ASSERT(xfs_is_shutdown(mp)); From e62941103faa2eedba6a155316e059a490c743a6 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:22 -0700 Subject: [PATCH 0212/1702] xfs: make file range exchange support realtime files Now that bmap items support the realtime device, we can add the necessary pieces to the file range exchange code to support exchanging mappings. All we really need to do here is adjust the blockcount upwards to the end of the rt extent and remove the inode checks. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_exchmaps.c | 70 ++++++++++++++++++++++++++++++------ fs/xfs/xfs_exchrange.c | 9 +++++ 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index f58240466b1c4..7fa244228750a 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -152,12 +152,7 @@ xfs_exchmaps_check_forks( ifp2->if_format == XFS_DINODE_FMT_LOCAL) return -EINVAL; - /* We don't support realtime data forks yet. */ - if (!XFS_IS_REALTIME_INODE(req->ip1)) - return 0; - if (whichfork == XFS_ATTR_FORK) - return 0; - return -EINVAL; + return 0; } #ifdef CONFIG_XFS_QUOTA @@ -198,6 +193,8 @@ xfs_exchmaps_can_skip_mapping( struct xfs_exchmaps_intent *xmi, struct xfs_bmbt_irec *irec) { + struct xfs_mount *mp = xmi->xmi_ip1->i_mount; + /* Do not skip this mapping if the caller did not tell us to. */ if (!(xmi->xmi_flags & XFS_EXCHMAPS_INO1_WRITTEN)) return false; @@ -209,11 +206,64 @@ xfs_exchmaps_can_skip_mapping( /* * The mapping is unwritten or a hole. It cannot be a delalloc * reservation because we already excluded those. It cannot be an - * unwritten mapping with dirty page cache because we flushed the page - * cache. We don't support realtime files yet, so we needn't (yet) - * deal with them. + * unwritten extent with dirty page cache because we flushed the page + * cache. For files where the allocation unit is 1FSB (files on the + * data dev, rt files if the extent size is 1FSB), we can safely + * skip this mapping. */ - return true; + if (!xfs_inode_has_bigrtalloc(xmi->xmi_ip1)) + return true; + + /* + * For a realtime file with a multi-fsb allocation unit, the decision + * is trickier because we can only swap full allocation units. + * Unwritten mappings can appear in the middle of an rtx if the rtx is + * partially written, but they can also appear for preallocations. + * + * If the mapping is a hole, skip it entirely. Holes should align with + * rtx boundaries. + */ + if (!xfs_bmap_is_real_extent(irec)) + return true; + + /* + * All mappings below this point are unwritten. + * + * - If the beginning is not aligned to an rtx, trim the end of the + * mapping so that it does not cross an rtx boundary, and swap it. + * + * - If both ends are aligned to an rtx, skip the entire mapping. + */ + if (!isaligned_64(irec->br_startoff, mp->m_sb.sb_rextsize)) { + xfs_fileoff_t new_end; + + new_end = roundup_64(irec->br_startoff, mp->m_sb.sb_rextsize); + irec->br_blockcount = min(irec->br_blockcount, + new_end - irec->br_startoff); + return false; + } + if (isaligned_64(irec->br_blockcount, mp->m_sb.sb_rextsize)) + return true; + + /* + * All mappings below this point are unwritten, start on an rtx + * boundary, and do not end on an rtx boundary. + * + * - If the mapping is longer than one rtx, trim the end of the mapping + * down to an rtx boundary and skip it. + * + * - The mapping is shorter than one rtx. Swap it. + */ + if (irec->br_blockcount > mp->m_sb.sb_rextsize) { + xfs_fileoff_t new_end; + + new_end = rounddown_64(irec->br_startoff + irec->br_blockcount, + mp->m_sb.sb_rextsize); + irec->br_blockcount = new_end - irec->br_startoff; + return true; + } + + return false; } /* diff --git a/fs/xfs/xfs_exchrange.c b/fs/xfs/xfs_exchrange.c index 0fc95e6471cb9..90baf12bd97f9 100644 --- a/fs/xfs/xfs_exchrange.c +++ b/fs/xfs/xfs_exchrange.c @@ -21,6 +21,7 @@ #include "xfs_sb.h" #include "xfs_icache.h" #include "xfs_log.h" +#include "xfs_rtbitmap.h" #include /* Lock (and optionally join) two inodes for a file range exchange. */ @@ -182,6 +183,14 @@ xfs_exchrange_mappings( if (fxr->flags & XFS_EXCHANGE_RANGE_FILE1_WRITTEN) req.flags |= XFS_EXCHMAPS_INO1_WRITTEN; + /* + * Round the request length up to the nearest file allocation unit. + * The prep function already checked that the request offsets and + * length in @fxr are safe to round up. + */ + if (xfs_inode_has_bigrtalloc(ip2)) + req.blockcount = xfs_rtb_roundup_rtx(mp, req.blockcount); + error = xfs_exchrange_estimate(&req); if (error) return error; From b3e60f84838d5abc3a73d7ef0fc595dd1041c565 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:23 -0700 Subject: [PATCH 0213/1702] xfs: support non-power-of-two rtextsize with exchange-range The generic exchange-range alignment checks use (fast) bitmasking operations to perform block alignment checks on the exchange parameters. Unfortunately, bitmasks require that the alignment size be a power of two. This isn't true for realtime devices with a non-power-of-two extent size, so we have to copy-pasta the generic checks using long division for this to work properly. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_exchrange.c | 89 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 7 deletions(-) diff --git a/fs/xfs/xfs_exchrange.c b/fs/xfs/xfs_exchrange.c index 90baf12bd97f9..c8a655c92c92f 100644 --- a/fs/xfs/xfs_exchrange.c +++ b/fs/xfs/xfs_exchrange.c @@ -504,6 +504,75 @@ xfs_exchange_range_finish( return file_remove_privs(fxr->file2); } +/* + * Check the alignment of an exchange request when the allocation unit size + * isn't a power of two. The generic file-level helpers use (fast) + * bitmask-based alignment checks, but here we have to use slow long division. + */ +static int +xfs_exchrange_check_rtalign( + const struct xfs_exchrange *fxr, + struct xfs_inode *ip1, + struct xfs_inode *ip2, + unsigned int alloc_unit) +{ + uint64_t length = fxr->length; + uint64_t blen; + loff_t size1, size2; + + size1 = i_size_read(VFS_I(ip1)); + size2 = i_size_read(VFS_I(ip2)); + + /* The start of both ranges must be aligned to a rt extent. */ + if (!isaligned_64(fxr->file1_offset, alloc_unit) || + !isaligned_64(fxr->file2_offset, alloc_unit)) + return -EINVAL; + + if (fxr->flags & XFS_EXCHANGE_RANGE_TO_EOF) + length = max_t(int64_t, size1 - fxr->file1_offset, + size2 - fxr->file2_offset); + + /* + * If the user wanted us to exchange up to the infile's EOF, round up + * to the next rt extent boundary for this check. Do the same for the + * outfile. + * + * Otherwise, reject the range length if it's not rt extent aligned. + * We already confirmed the starting offsets' rt extent block + * alignment. + */ + if (fxr->file1_offset + length == size1) + blen = roundup_64(size1, alloc_unit) - fxr->file1_offset; + else if (fxr->file2_offset + length == size2) + blen = roundup_64(size2, alloc_unit) - fxr->file2_offset; + else if (!isaligned_64(length, alloc_unit)) + return -EINVAL; + else + blen = length; + + /* Don't allow overlapped exchanges within the same file. */ + if (ip1 == ip2 && + fxr->file2_offset + blen > fxr->file1_offset && + fxr->file1_offset + blen > fxr->file2_offset) + return -EINVAL; + + /* + * Ensure that we don't exchange a partial EOF rt extent into the + * middle of another file. + */ + if (isaligned_64(length, alloc_unit)) + return 0; + + blen = length; + if (fxr->file2_offset + length < size2) + blen = rounddown_64(blen, alloc_unit); + + if (fxr->file1_offset + blen < size1) + blen = rounddown_64(blen, alloc_unit); + + return blen == length ? 0 : -EINVAL; +} + /* Prepare two files to have their data exchanged. */ STATIC int xfs_exchrange_prep( @@ -511,6 +580,7 @@ xfs_exchrange_prep( struct xfs_inode *ip1, struct xfs_inode *ip2) { + struct xfs_mount *mp = ip2->i_mount; unsigned int alloc_unit = xfs_inode_alloc_unitsize(ip2); int error; @@ -520,13 +590,18 @@ xfs_exchrange_prep( if (XFS_IS_REALTIME_INODE(ip1) != XFS_IS_REALTIME_INODE(ip2)) return -EINVAL; - /* - * The alignment checks in the generic helpers cannot deal with - * allocation units that are not powers of 2. This can happen with the - * realtime volume if the extent size is set. - */ - if (!is_power_of_2(alloc_unit)) - return -EOPNOTSUPP; + /* Check non-power of two alignment issues, if necessary. */ + if (!is_power_of_2(alloc_unit)) { + error = xfs_exchrange_check_rtalign(fxr, ip1, ip2, alloc_unit); + if (error) + return error; + + /* + * Do the generic file-level checks with the regular block + * alignment. + */ + alloc_unit = mp->m_sb.sb_blocksize; + } error = xfs_exchange_range_prep(fxr, alloc_unit); if (error || fxr->length == 0) From 14f19991020b3c712d626727c17599f134cc6efa Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:24 -0700 Subject: [PATCH 0214/1702] xfs: capture inode generation numbers in the ondisk exchmaps log item Per some very late review comments, capture the generation numbers of both inodes involved in a file content exchange operation so that we don't accidentally target files with have been reallocated. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_log_format.h | 2 ++ fs/xfs/libxfs/xfs_log_recover.h | 2 ++ fs/xfs/xfs_exchmaps_item.c | 25 ++++++++++++++++++++----- fs/xfs/xfs_log_recover.c | 31 +++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 5 deletions(-) diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h index 8dbe1f997dfd5..accba2acd623d 100644 --- a/fs/xfs/libxfs/xfs_log_format.h +++ b/fs/xfs/libxfs/xfs_log_format.h @@ -896,6 +896,8 @@ struct xfs_xmi_log_format { uint64_t xmi_inode1; /* inumber of first file */ uint64_t xmi_inode2; /* inumber of second file */ + uint32_t xmi_igen1; /* generation of first file */ + uint32_t xmi_igen2; /* generation of second file */ uint64_t xmi_startoff1; /* block offset into file1 */ uint64_t xmi_startoff2; /* block offset into file2 */ uint64_t xmi_blockcount; /* number of blocks */ diff --git a/fs/xfs/libxfs/xfs_log_recover.h b/fs/xfs/libxfs/xfs_log_recover.h index 47b758b49cb35..521d327e4c89e 100644 --- a/fs/xfs/libxfs/xfs_log_recover.h +++ b/fs/xfs/libxfs/xfs_log_recover.h @@ -123,6 +123,8 @@ bool xlog_is_buffer_cancelled(struct xlog *log, xfs_daddr_t blkno, uint len); int xlog_recover_iget(struct xfs_mount *mp, xfs_ino_t ino, struct xfs_inode **ipp); +int xlog_recover_iget_handle(struct xfs_mount *mp, xfs_ino_t ino, uint32_t gen, + struct xfs_inode **ipp); void xlog_recover_release_intent(struct xlog *log, unsigned short intent_type, uint64_t intent_id); int xlog_alloc_buf_cancel_table(struct xlog *log); diff --git a/fs/xfs/xfs_exchmaps_item.c b/fs/xfs/xfs_exchmaps_item.c index a40216f33214c..264a121c5e16d 100644 --- a/fs/xfs/xfs_exchmaps_item.c +++ b/fs/xfs/xfs_exchmaps_item.c @@ -231,7 +231,9 @@ xfs_exchmaps_create_intent( xlf = &xmi_lip->xmi_format; xlf->xmi_inode1 = xmi->xmi_ip1->i_ino; + xlf->xmi_igen1 = VFS_I(xmi->xmi_ip1)->i_generation; xlf->xmi_inode2 = xmi->xmi_ip2->i_ino; + xlf->xmi_igen2 = VFS_I(xmi->xmi_ip2)->i_generation; xlf->xmi_startoff1 = xmi->xmi_startoff1; xlf->xmi_startoff2 = xmi->xmi_startoff2; xlf->xmi_blockcount = xmi->xmi_blockcount; @@ -368,14 +370,25 @@ xfs_xmi_item_recover_intent( /* * Grab both inodes and set IRECOVERY to prevent trimming of post-eof * mappings and freeing of unlinked inodes until we're totally done - * processing files. + * processing files. The ondisk format of this new log item contains + * file handle information, which is why recovery for other items do + * not check the inode generation number. */ - error = xlog_recover_iget(mp, xlf->xmi_inode1, &ip1); - if (error) + error = xlog_recover_iget_handle(mp, xlf->xmi_inode1, xlf->xmi_igen1, + &ip1); + if (error) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, xlf, + sizeof(*xlf)); return ERR_PTR(error); - error = xlog_recover_iget(mp, xlf->xmi_inode2, &ip2); - if (error) + } + + error = xlog_recover_iget_handle(mp, xlf->xmi_inode2, xlf->xmi_igen2, + &ip2); + if (error) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, xlf, + sizeof(*xlf)); goto err_rele1; + } req->ip1 = ip1; req->ip2 = ip2; @@ -485,6 +498,8 @@ xfs_exchmaps_relog_intent( new_xlf->xmi_inode1 = old_xlf->xmi_inode1; new_xlf->xmi_inode2 = old_xlf->xmi_inode2; + new_xlf->xmi_igen1 = old_xlf->xmi_igen1; + new_xlf->xmi_igen2 = old_xlf->xmi_igen2; new_xlf->xmi_startoff1 = old_xlf->xmi_startoff1; new_xlf->xmi_startoff2 = old_xlf->xmi_startoff2; new_xlf->xmi_blockcount = old_xlf->xmi_blockcount; diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 1e5ba95adf2c7..b445e8ce4a7d2 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -1767,6 +1767,37 @@ xlog_recover_iget( return 0; } +/* + * Get an inode so that we can recover a log operation. + * + * Log intent items that target inodes effectively contain a file handle. + * Check that the generation number matches the intent item like we do for + * other file handles. Log intent items defined after this validation weakness + * was identified must use this function. + */ +int +xlog_recover_iget_handle( + struct xfs_mount *mp, + xfs_ino_t ino, + uint32_t gen, + struct xfs_inode **ipp) +{ + struct xfs_inode *ip; + int error; + + error = xlog_recover_iget(mp, ino, &ip); + if (error) + return error; + + if (VFS_I(ip)->i_generation != gen) { + xfs_irele(ip); + return -EFSCORRUPTED; + } + + *ipp = ip; + return 0; +} + /****************************************************************************** * * Log recover routines From f783529bee39c3fa1451728007eb4890a94f2638 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:25 -0700 Subject: [PATCH 0215/1702] docs: update swapext -> exchmaps language Start reworking the atomic swapext design documentation to refer to its new file contents/mapping exchange name. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- .../xfs/xfs-online-fsck-design.rst | 263 +++++++++--------- 1 file changed, 138 insertions(+), 125 deletions(-) diff --git a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst index 1d161752f09ed..3afa1bc5f47ce 100644 --- a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst +++ b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst @@ -2167,7 +2167,7 @@ The ``xfblob_free`` function frees a specific blob, and the ``xfblob_truncate`` function frees them all because compaction is not needed. The details of repairing directories and extended attributes will be discussed -in a subsequent section about atomic extent swapping. +in a subsequent section about atomic file content exchanges. However, it should be noted that these repair functions only use blob storage to cache a small number of entries before adding them to a temporary ondisk file, which is why compaction is not required. @@ -2802,7 +2802,8 @@ follows this format: Repairs for file-based metadata such as extended attributes, directories, symbolic links, quota files and realtime bitmaps are performed by building a -new structure attached to a temporary file and swapping the forks. +new structure attached to a temporary file and exchanging all mappings in the +file forks. Afterward, the mappings in the old file fork are the candidate blocks for disposal. @@ -3851,8 +3852,8 @@ Because file forks can consume as much space as the entire filesystem, repairs cannot be staged in memory, even when a paging scheme is available. Therefore, online repair of file-based metadata createas a temporary file in the XFS filesystem, writes a new structure at the correct offsets into the -temporary file, and atomically swaps the fork mappings (and hence the fork -contents) to commit the repair. +temporary file, and atomically exchanges all file fork mappings (and hence the +fork contents) to commit the repair. Once the repair is complete, the old fork can be reaped as necessary; if the system goes down during the reap, the iunlink code will delete the blocks during log recovery. @@ -3862,10 +3863,11 @@ consistent to use a temporary file safely! This dependency is the reason why online repair can only use pageable kernel memory to stage ondisk space usage information. -Swapping metadata extents with a temporary file requires the owner field of the -block headers to match the file being repaired and not the temporary file. The -directory, extended attribute, and symbolic link functions were all modified to -allow callers to specify owner numbers explicitly. +Exchanging metadata file mappings with a temporary file requires the owner +field of the block headers to match the file being repaired and not the +temporary file. +The directory, extended attribute, and symbolic link functions were all +modified to allow callers to specify owner numbers explicitly. There is a downside to the reaping process -- if the system crashes during the reap phase and the fork extents are crosslinked, the iunlink processing will @@ -3974,8 +3976,8 @@ The proposed patches are in the `_ series. -Atomic Extent Swapping ----------------------- +Logged File Content Exchanges +----------------------------- Once repair builds a temporary file with a new data structure written into it, it must commit the new changes into the existing file. @@ -4010,17 +4012,21 @@ e. Old blocks in the file may be cross-linked with another structure and must These problems are overcome by creating a new deferred operation and a new type of log intent item to track the progress of an operation to exchange two file ranges. -The new deferred operation type chains together the same transactions used by -the reverse-mapping extent swap code. +The new exchange operation type chains together the same transactions used by +the reverse-mapping extent swap code, but records intermedia progress in the +log so that operations can be restarted after a crash. +This new functionality is called the file contents exchange (xfs_exchrange) +code. +The underlying implementation exchanges file fork mappings (xfs_exchmaps). The new log item records the progress of the exchange to ensure that once an exchange begins, it will always run to completion, even there are interruptions. -The new ``XFS_SB_FEAT_INCOMPAT_LOG_ATOMIC_SWAP`` log-incompatible feature flag +The new ``XFS_SB_FEAT_INCOMPAT_EXCHRANGE`` incompatible feature flag in the superblock protects these new log item records from being replayed on old kernels. The proposed patchset is the -`atomic extent swap +`file contents exchange `_ series. @@ -4061,72 +4067,73 @@ series. | The feature bit will not be cleared from the superblock until the log | | becomes clean. | | | -| Log-assisted extended attribute updates and atomic extent swaps both use | -| log incompat features and provide convenience wrappers around the | +| Log-assisted extended attribute updates and file content exchanges bothe | +| use log incompat features and provide convenience wrappers around the | | functionality. | +--------------------------------------------------------------------------+ -Mechanics of an Atomic Extent Swap -`````````````````````````````````` +Mechanics of a Logged File Content Exchange +``````````````````````````````````````````` -Swapping entire file forks is a complex task. +Exchanging contents between file forks is a complex task. The goal is to exchange all file fork mappings between two file fork offset ranges. There are likely to be many extent mappings in each fork, and the edges of the mappings aren't necessarily aligned. -Furthermore, there may be other updates that need to happen after the swap, +Furthermore, there may be other updates that need to happen after the exchange, such as exchanging file sizes, inode flags, or conversion of fork data to local format. -This is roughly the format of the new deferred extent swap work item: +This is roughly the format of the new deferred exchange-mapping work item: .. code-block:: c - struct xfs_swapext_intent { + struct xfs_exchmaps_intent { /* Inodes participating in the operation. */ - struct xfs_inode *sxi_ip1; - struct xfs_inode *sxi_ip2; + struct xfs_inode *xmi_ip1; + struct xfs_inode *xmi_ip2; /* File offset range information. */ - xfs_fileoff_t sxi_startoff1; - xfs_fileoff_t sxi_startoff2; - xfs_filblks_t sxi_blockcount; + xfs_fileoff_t xmi_startoff1; + xfs_fileoff_t xmi_startoff2; + xfs_filblks_t xmi_blockcount; /* Set these file sizes after the operation, unless negative. */ - xfs_fsize_t sxi_isize1; - xfs_fsize_t sxi_isize2; + xfs_fsize_t xmi_isize1; + xfs_fsize_t xmi_isize2; - /* XFS_SWAP_EXT_* log operation flags */ - uint64_t sxi_flags; + /* XFS_EXCHMAPS_* log operation flags */ + uint64_t xmi_flags; }; The new log intent item contains enough information to track two logical fork offset ranges: ``(inode1, startoff1, blockcount)`` and ``(inode2, startoff2, blockcount)``. -Each step of a swap operation exchanges the largest file range mapping possible -from one file to the other. -After each step in the swap operation, the two startoff fields are incremented -and the blockcount field is decremented to reflect the progress made. -The flags field captures behavioral parameters such as swapping the attr fork -instead of the data fork and other work to be done after the extent swap. -The two isize fields are used to swap the file size at the end of the operation -if the file data fork is the target of the swap operation. - -When the extent swap is initiated, the sequence of operations is as follows: - -1. Create a deferred work item for the extent swap. - At the start, it should contain the entirety of the file ranges to be - swapped. +Each step of an exchange operation exchanges the largest file range mapping +possible from one file to the other. +After each step in the exchange operation, the two startoff fields are +incremented and the blockcount field is decremented to reflect the progress +made. +The flags field captures behavioral parameters such as exchanging attr fork +mappings instead of the data fork and other work to be done after the exchange. +The two isize fields are used to exchange the file sizes at the end of the +operation if the file data fork is the target of the operation. + +When the exchange is initiated, the sequence of operations is as follows: + +1. Create a deferred work item for the file mapping exchange. + At the start, it should contain the entirety of the file block ranges to be + exchanged. 2. Call ``xfs_defer_finish`` to process the exchange. - This is encapsulated in ``xrep_tempswap_contents`` for scrub operations. + This is encapsulated in ``xrep_tempexch_contents`` for scrub operations. This will log an extent swap intent item to the transaction for the deferred - extent swap work item. + mapping exchange work item. -3. Until ``sxi_blockcount`` of the deferred extent swap work item is zero, +3. Until ``xmi_blockcount`` of the deferred mapping exchange work item is zero, - a. Read the block maps of both file ranges starting at ``sxi_startoff1`` and - ``sxi_startoff2``, respectively, and compute the longest extent that can - be swapped in a single step. + a. Read the block maps of both file ranges starting at ``xmi_startoff1`` and + ``xmi_startoff2``, respectively, and compute the longest extent that can + be exchanged in a single step. This is the minimum of the two ``br_blockcount`` s in the mappings. Keep advancing through the file forks until at least one of the mappings contains written blocks. @@ -4148,20 +4155,20 @@ When the extent swap is initiated, the sequence of operations is as follows: g. Extend the ondisk size of either file if necessary. - h. Log an extent swap done log item for the extent swap intent log item - that was read at the start of step 3. + h. Log a mapping exchange done log item for th mapping exchange intent log + item that was read at the start of step 3. i. Compute the amount of file range that has just been covered. This quantity is ``(map1.br_startoff + map1.br_blockcount - - sxi_startoff1)``, because step 3a could have skipped holes. + xmi_startoff1)``, because step 3a could have skipped holes. - j. Increase the starting offsets of ``sxi_startoff1`` and ``sxi_startoff2`` + j. Increase the starting offsets of ``xmi_startoff1`` and ``xmi_startoff2`` by the number of blocks computed in the previous step, and decrease - ``sxi_blockcount`` by the same quantity. + ``xmi_blockcount`` by the same quantity. This advances the cursor. - k. Log a new extent swap intent log item reflecting the advanced state of - the work item. + k. Log a new mapping exchange intent log item reflecting the advanced state + of the work item. l. Return the proper error code (EAGAIN) to the deferred operation manager to inform it that there is more work to be done. @@ -4172,22 +4179,23 @@ When the extent swap is initiated, the sequence of operations is as follows: This will be discussed in more detail in subsequent sections. If the filesystem goes down in the middle of an operation, log recovery will -find the most recent unfinished extent swap log intent item and restart from -there. -This is how extent swapping guarantees that an outside observer will either see -the old broken structure or the new one, and never a mismash of both. +find the most recent unfinished maping exchange log intent item and restart +from there. +This is how atomic file mapping exchanges guarantees that an outside observer +will either see the old broken structure or the new one, and never a mismash of +both. -Preparation for Extent Swapping -``````````````````````````````` +Preparation for File Content Exchanges +`````````````````````````````````````` There are a few things that need to be taken care of before initiating an -atomic extent swap operation. +atomic file mapping exchange operation. First, regular files require the page cache to be flushed to disk before the operation begins, and directio writes to be quiesced. -Like any filesystem operation, extent swapping must determine the maximum -amount of disk space and quota that can be consumed on behalf of both files in -the operation, and reserve that quantity of resources to avoid an unrecoverable -out of space failure once it starts dirtying metadata. +Like any filesystem operation, file mapping exchanges must determine the +maximum amount of disk space and quota that can be consumed on behalf of both +files in the operation, and reserve that quantity of resources to avoid an +unrecoverable out of space failure once it starts dirtying metadata. The preparation step scans the ranges of both files to estimate: - Data device blocks needed to handle the repeated updates to the fork @@ -4201,56 +4209,59 @@ The preparation step scans the ranges of both files to estimate: to different extents on the realtime volume, which could happen if the operation fails to run to completion. -The need for precise estimation increases the run time of the swap operation, -but it is very important to maintain correct accounting. -The filesystem must not run completely out of free space, nor can the extent -swap ever add more extent mappings to a fork than it can support. +The need for precise estimation increases the run time of the exchange +operation, but it is very important to maintain correct accounting. +The filesystem must not run completely out of free space, nor can the mapping +exchange ever add more extent mappings to a fork than it can support. Regular users are required to abide the quota limits, though metadata repairs may exceed quota to resolve inconsistent metadata elsewhere. -Special Features for Swapping Metadata File Extents -``````````````````````````````````````````````````` +Special Features for Exchanging Metadata File Contents +`````````````````````````````````````````````````````` Extended attributes, symbolic links, and directories can set the fork format to "local" and treat the fork as a literal area for data storage. Metadata repairs must take extra steps to support these cases: - If both forks are in local format and the fork areas are large enough, the - swap is performed by copying the incore fork contents, logging both forks, - and committing. - The atomic extent swap mechanism is not necessary, since this can be done - with a single transaction. + exchange is performed by copying the incore fork contents, logging both + forks, and committing. + The atomic file mapping exchange mechanism is not necessary, since this can + be done with a single transaction. -- If both forks map blocks, then the regular atomic extent swap is used. +- If both forks map blocks, then the regular atomic file mapping exchange is + used. - Otherwise, only one fork is in local format. The contents of the local format fork are converted to a block to perform the - swap. + exchange. The conversion to block format must be done in the same transaction that - logs the initial extent swap intent log item. - The regular atomic extent swap is used to exchange the mappings. - Special flags are set on the swap operation so that the transaction can be - rolled one more time to convert the second file's fork back to local format - so that the second file will be ready to go as soon as the ILOCK is dropped. + logs the initial mapping exchange intent log item. + The regular atomic mapping exchange is used to exchange the metadata file + mappings. + Special flags are set on the exchange operation so that the transaction can + be rolled one more time to convert the second file's fork back to local + format so that the second file will be ready to go as soon as the ILOCK is + dropped. Extended attributes and directories stamp the owning inode into every block, but the buffer verifiers do not actually check the inode number! Although there is no verification, it is still important to maintain -referential integrity, so prior to performing the extent swap, online repair -builds every block in the new data structure with the owner field of the file -being repaired. +referential integrity, so prior to performing the mapping exchange, online +repair builds every block in the new data structure with the owner field of the +file being repaired. -After a successful swap operation, the repair operation must reap the old fork -blocks by processing each fork mapping through the standard :ref:`file extent -reaping ` mechanism that is done post-repair. +After a successful exchange operation, the repair operation must reap the old +fork blocks by processing each fork mapping through the standard :ref:`file +extent reaping ` mechanism that is done post-repair. If the filesystem should go down during the reap part of the repair, the iunlink processing at the end of recovery will free both the temporary file and whatever blocks were not reaped. However, this iunlink processing omits the cross-link detection of online repair, and is not completely foolproof. -Swapping Temporary File Extents -``````````````````````````````` +Exchanging Temporary File Contents +`````````````````````````````````` To repair a metadata file, online repair proceeds as follows: @@ -4260,14 +4271,14 @@ To repair a metadata file, online repair proceeds as follows: file. The same fork must be written to as is being repaired. -3. Commit the scrub transaction, since the swap estimation step must be - completed before transaction reservations are made. +3. Commit the scrub transaction, since the exchange resource estimation step + must be completed before transaction reservations are made. -4. Call ``xrep_tempswap_trans_alloc`` to allocate a new scrub transaction with +4. Call ``xrep_tempexch_trans_alloc`` to allocate a new scrub transaction with the appropriate resource reservations, locks, and fill out a ``struct - xfs_swapext_req`` with the details of the swap operation. + xfs_exchmaps_req`` with the details of the exchange operation. -5. Call ``xrep_tempswap_contents`` to swap the contents. +5. Call ``xrep_tempexch_contents`` to exchange the contents. 6. Commit the transaction to complete the repair. @@ -4309,7 +4320,7 @@ To check the summary file against the bitmap: 3. Compare the contents of the xfile against the ondisk file. To repair the summary file, write the xfile contents into the temporary file -and use atomic extent swap to commit the new contents. +and use atomic mapping exchange to commit the new contents. The temporary file is then reaped. The proposed patchset is the @@ -4352,8 +4363,8 @@ Salvaging extended attributes is done as follows: memory or there are no more attr fork blocks to examine, unlock the file and add the staged extended attributes to the temporary file. -3. Use atomic extent swapping to exchange the new and old extended attribute - structures. +3. Use atomic file mapping exchange to exchange the new and old extended + attribute structures. The old attribute blocks are now attached to the temporary file. 4. Reap the temporary file. @@ -4410,7 +4421,8 @@ salvaging directories is straightforward: directory and add the staged dirents into the temporary directory. Truncate the staging files. -4. Use atomic extent swapping to exchange the new and old directory structures. +4. Use atomic file mapping exchange to exchange the new and old directory + structures. The old directory blocks are now attached to the temporary file. 5. Reap the temporary file. @@ -4542,7 +4554,7 @@ a :ref:`directory entry live update hook ` as follows: Instead, we stash updates in the xfarray and rely on the scanner thread to apply the stashed updates to the temporary directory. -5. When the scan is complete, atomically swap the contents of the temporary +5. When the scan is complete, atomically exchange the contents of the temporary directory and the directory being repaired. The temporary directory now contains the damaged directory structure. @@ -4629,8 +4641,8 @@ directory reconstruction: 5. Copy all non-parent pointer extended attributes to the temporary file. -6. When the scan is complete, atomically swap the attribute fork of the - temporary file and the file being repaired. +6. When the scan is complete, atomically exchange the mappings of the attribute + forks of the temporary file and the file being repaired. The temporary file now contains the damaged extended attribute structure. 7. Reap the temporary file. @@ -5105,18 +5117,18 @@ make it easier for code readers to understand what has been built, for whom it has been built, and why. Please feel free to contact the XFS mailing list with questions. -FIEXCHANGE_RANGE ----------------- +XFS_IOC_EXCHANGE_RANGE +---------------------- -As discussed earlier, a second frontend to the atomic extent swap mechanism is -a new ioctl call that userspace programs can use to commit updates to files -atomically. +As discussed earlier, a second frontend to the atomic file mapping exchange +mechanism is a new ioctl call that userspace programs can use to commit updates +to files atomically. This frontend has been out for review for several years now, though the necessary refinements to online repair and lack of customer demand mean that the proposal has not been pushed very hard. -Extent Swapping with Regular User Files -``````````````````````````````````````` +File Content Exchanges with Regular User Files +`````````````````````````````````````````````` As mentioned earlier, XFS has long had the ability to swap extents between files, which is used almost exclusively by ``xfs_fsr`` to defragment files. @@ -5131,12 +5143,12 @@ the consistency of the fork mappings with the reverse mapping index was to develop an iterative mechanism that used deferred bmap and rmap operations to swap mappings one at a time. This mechanism is identical to steps 2-3 from the procedure above except for -the new tracking items, because the atomic extent swap mechanism is an -iteration of an existing mechanism and not something totally novel. +the new tracking items, because the atomic file mapping exchange mechanism is +an iteration of an existing mechanism and not something totally novel. For the narrow case of file defragmentation, the file contents must be identical, so the recovery guarantees are not much of a gain. -Atomic extent swapping is much more flexible than the existing swapext +Atomic file content exchanges are much more flexible than the existing swapext implementations because it can guarantee that the caller never sees a mix of old and new contents even after a crash, and it can operate on two arbitrary file fork ranges. @@ -5147,11 +5159,11 @@ The extra flexibility enables several new use cases: Next, it opens a temporary file and calls the file clone operation to reflink the first file's contents into the temporary file. Writes to the original file should instead be written to the temporary file. - Finally, the process calls the atomic extent swap system call - (``FIEXCHANGE_RANGE``) to exchange the file contents, thereby committing all - of the updates to the original file, or none of them. + Finally, the process calls the atomic file mapping exchange system call + (``XFS_IOC_EXCHANGE_RANGE``) to exchange the file contents, thereby + committing all of the updates to the original file, or none of them. -.. _swapext_if_unchanged: +.. _exchrange_if_unchanged: - **Transactional file updates**: The same mechanism as above, but the caller only wants the commit to occur if the original file's contents have not @@ -5160,16 +5172,17 @@ The extra flexibility enables several new use cases: change timestamps of the original file before reflinking its data to the temporary file. When the program is ready to commit the changes, it passes the timestamps - into the kernel as arguments to the atomic extent swap system call. + into the kernel as arguments to the atomic file mapping exchange system call. The kernel only commits the changes if the provided timestamps match the original file. + A new ioctl (``XFS_IOC_COMMIT_RANGE``) is provided to perform this. - **Emulation of atomic block device writes**: Export a block device with a logical sector size matching the filesystem block size to force all writes to be aligned to the filesystem block size. Stage all writes to a temporary file, and when that is complete, call the - atomic extent swap system call with a flag to indicate that holes in the - temporary file should be ignored. + atomic file mapping exchange system call with a flag to indicate that holes + in the temporary file should be ignored. This emulates an atomic device write in software, and can support arbitrary scattered writes. @@ -5251,8 +5264,8 @@ of the file to try to share the physical space with a dummy file. Cloning the extent means that the original owners cannot overwrite the contents; any changes will be written somewhere else via copy-on-write. Clearspace makes its own copy of the frozen extent in an area that is not being -cleared, and uses ``FIEDEUPRANGE`` (or the :ref:`atomic extent swap -` feature) to change the target file's data extent +cleared, and uses ``FIEDEUPRANGE`` (or the :ref:`atomic file content exchanges +` feature) to change the target file's data extent mapping away from the area being cleared. When all other mappings have been moved, clearspace reflinks the space into the space collector file so that it becomes unavailable. From 0730e8d8ba1d1507f1d7fd719e1f835ce69961fe Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:26 -0700 Subject: [PATCH 0216/1702] xfs: enable logged file mapping exchange feature Add the XFS_SB_FEAT_INCOMPAT_EXCHRANGE feature to the set of features that we will permit when mounting a filesystem. This turns on support for the file range exchange feature. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_format.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h index ff1e28316e1bb..10153ce116d4c 100644 --- a/fs/xfs/libxfs/xfs_format.h +++ b/fs/xfs/libxfs/xfs_format.h @@ -380,7 +380,8 @@ xfs_sb_has_ro_compat_feature( XFS_SB_FEAT_INCOMPAT_META_UUID | \ XFS_SB_FEAT_INCOMPAT_BIGTIME | \ XFS_SB_FEAT_INCOMPAT_NEEDSREPAIR | \ - XFS_SB_FEAT_INCOMPAT_NREXT64) + XFS_SB_FEAT_INCOMPAT_NREXT64 | \ + XFS_SB_FEAT_INCOMPAT_EXCHRANGE) #define XFS_SB_FEAT_INCOMPAT_UNKNOWN ~XFS_SB_FEAT_INCOMPAT_ALL static inline bool From cab23a4233c601668da51dd53776f1f83d0dccee Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:27 -0700 Subject: [PATCH 0217/1702] xfs: hide private inodes from bulkstat and handle functions We're about to start adding functionality that uses internal inodes that are private to XFS. What this means is that userspace should never be able to access any information about these files, and should not be able to open these files by handle. To prevent users from ever finding the file or mis-interactions with the security apparatus, set S_PRIVATE on the inode. Don't allow bulkstat, open-by-handle, or linking of S_PRIVATE files into the directory tree. This should keep private inodes actually private. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_export.c | 2 +- fs/xfs/xfs_iops.c | 3 +++ fs/xfs/xfs_itable.c | 8 ++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/fs/xfs/xfs_export.c b/fs/xfs/xfs_export.c index 7cd09c3a82cb5..4b03221351c0f 100644 --- a/fs/xfs/xfs_export.c +++ b/fs/xfs/xfs_export.c @@ -160,7 +160,7 @@ xfs_nfs_get_inode( } } - if (VFS_I(ip)->i_generation != generation) { + if (VFS_I(ip)->i_generation != generation || IS_PRIVATE(VFS_I(ip))) { xfs_irele(ip); return ERR_PTR(-ESTALE); } diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 55ed2d1023d67..7f0c840f0fd2f 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -365,6 +365,9 @@ xfs_vn_link( if (unlikely(error)) return error; + if (IS_PRIVATE(inode)) + return -EPERM; + error = xfs_link(XFS_I(dir), XFS_I(inode), &name); if (unlikely(error)) return error; diff --git a/fs/xfs/xfs_itable.c b/fs/xfs/xfs_itable.c index 95fc31b9f87d6..c0757ab994957 100644 --- a/fs/xfs/xfs_itable.c +++ b/fs/xfs/xfs_itable.c @@ -97,6 +97,14 @@ xfs_bulkstat_one_int( vfsuid = i_uid_into_vfsuid(idmap, inode); vfsgid = i_gid_into_vfsgid(idmap, inode); + /* If this is a private inode, don't leak its details to userspace. */ + if (IS_PRIVATE(inode)) { + xfs_iunlock(ip, XFS_ILOCK_SHARED); + xfs_irele(ip); + error = -EINVAL; + goto out_advance; + } + /* xfs_iget returns the following without needing * further change. */ From 84c14ee39dd388d73054181c70b3f42af7ddc238 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:28 -0700 Subject: [PATCH 0218/1702] xfs: create temporary files and directories for online repair Teach the online repair code how to create temporary files or directories. These temporary files can be used to stage reconstructed information until we're ready to perform an atomic extent swap to commit the new metadata. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/scrub/parent.c | 2 +- fs/xfs/scrub/scrub.c | 3 + fs/xfs/scrub/scrub.h | 4 + fs/xfs/scrub/tempfile.c | 251 ++++++++++++++++++++++++++++++++++++++++ fs/xfs/scrub/tempfile.h | 28 +++++ fs/xfs/scrub/trace.h | 33 ++++++ fs/xfs/xfs_inode.c | 3 +- fs/xfs/xfs_inode.h | 2 + 9 files changed, 324 insertions(+), 3 deletions(-) create mode 100644 fs/xfs/scrub/tempfile.c create mode 100644 fs/xfs/scrub/tempfile.h diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index b547a3dc03f87..ae8488ab4d6b5 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -207,6 +207,7 @@ xfs-y += $(addprefix scrub/, \ refcount_repair.o \ repair.o \ rmap_repair.o \ + tempfile.o \ ) xfs-$(CONFIG_XFS_RT) += $(addprefix scrub/, \ diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c index 7db8736721461..5da10ed1fe8ce 100644 --- a/fs/xfs/scrub/parent.c +++ b/fs/xfs/scrub/parent.c @@ -143,7 +143,7 @@ xchk_parent_validate( } if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) return error; - if (dp == sc->ip || !S_ISDIR(VFS_I(dp)->i_mode)) { + if (dp == sc->ip || dp == sc->tempip || !S_ISDIR(VFS_I(dp)->i_mode)) { xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); goto out_rele; } diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 20fac9723c08c..d9012e9a6afd2 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -17,6 +17,7 @@ #include "xfs_scrub.h" #include "xfs_buf_mem.h" #include "xfs_rmap.h" +#include "xfs_exchrange.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/trace.h" @@ -24,6 +25,7 @@ #include "scrub/health.h" #include "scrub/stats.h" #include "scrub/xfile.h" +#include "scrub/tempfile.h" /* * Online Scrub and Repair @@ -211,6 +213,7 @@ xchk_teardown( sc->buf = NULL; } + xrep_tempfile_rele(sc); xchk_fsgates_disable(sc); return error; } diff --git a/fs/xfs/scrub/scrub.h b/fs/xfs/scrub/scrub.h index 9ad65b604fe12..e37d8599718e2 100644 --- a/fs/xfs/scrub/scrub.h +++ b/fs/xfs/scrub/scrub.h @@ -105,6 +105,10 @@ struct xfs_scrub { /* Lock flags for @ip. */ uint ilock_flags; + /* A temporary file on this filesystem, for staging new metadata. */ + struct xfs_inode *tempip; + uint temp_ilock_flags; + /* See the XCHK/XREP state flags below. */ unsigned int flags; diff --git a/fs/xfs/scrub/tempfile.c b/fs/xfs/scrub/tempfile.c new file mode 100644 index 0000000000000..68d245749bc1e --- /dev/null +++ b/fs/xfs/scrub/tempfile.c @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_ialloc.h" +#include "xfs_quota.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "xfs_dir2.h" +#include "xfs_exchrange.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/tempfile.h" + +/* + * Create a temporary file for reconstructing metadata, with the intention of + * atomically exchanging the temporary file's contents with the file that's + * being repaired. + */ +int +xrep_tempfile_create( + struct xfs_scrub *sc, + uint16_t mode) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_trans *tp = NULL; + struct xfs_dquot *udqp = NULL; + struct xfs_dquot *gdqp = NULL; + struct xfs_dquot *pdqp = NULL; + struct xfs_trans_res *tres; + struct xfs_inode *dp = mp->m_rootip; + xfs_ino_t ino; + unsigned int resblks; + bool is_dir = S_ISDIR(mode); + int error; + + if (xfs_is_shutdown(mp)) + return -EIO; + if (xfs_is_readonly(mp)) + return -EROFS; + + ASSERT(sc->tp == NULL); + ASSERT(sc->tempip == NULL); + + /* + * Make sure that we have allocated dquot(s) on disk. The temporary + * inode should be completely root owned so that we don't fail due to + * quota limits. + */ + error = xfs_qm_vop_dqalloc(dp, GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0, + XFS_QMOPT_QUOTALL, &udqp, &gdqp, &pdqp); + if (error) + return error; + + if (is_dir) { + resblks = XFS_MKDIR_SPACE_RES(mp, 0); + tres = &M_RES(mp)->tr_mkdir; + } else { + resblks = XFS_IALLOC_SPACE_RES(mp); + tres = &M_RES(mp)->tr_create_tmpfile; + } + + error = xfs_trans_alloc_icreate(mp, tres, udqp, gdqp, pdqp, resblks, + &tp); + if (error) + goto out_release_dquots; + + /* Allocate inode, set up directory. */ + error = xfs_dialloc(&tp, dp->i_ino, mode, &ino); + if (error) + goto out_trans_cancel; + error = xfs_init_new_inode(&nop_mnt_idmap, tp, dp, ino, mode, 0, 0, + 0, false, &sc->tempip); + if (error) + goto out_trans_cancel; + + /* Change the ownership of the inode to root. */ + VFS_I(sc->tempip)->i_uid = GLOBAL_ROOT_UID; + VFS_I(sc->tempip)->i_gid = GLOBAL_ROOT_GID; + sc->tempip->i_diflags &= ~(XFS_DIFLAG_REALTIME | XFS_DIFLAG_RTINHERIT); + xfs_trans_log_inode(tp, sc->tempip, XFS_ILOG_CORE); + + /* + * Mark our temporary file as private so that LSMs and the ACL code + * don't try to add their own metadata or reason about these files. + * The file should never be exposed to userspace. + */ + VFS_I(sc->tempip)->i_flags |= S_PRIVATE; + VFS_I(sc->tempip)->i_opflags &= ~IOP_XATTR; + + if (is_dir) { + error = xfs_dir_init(tp, sc->tempip, dp); + if (error) + goto out_trans_cancel; + } + + /* + * Attach the dquot(s) to the inodes and modify them incore. + * These ids of the inode couldn't have changed since the new + * inode has been locked ever since it was created. + */ + xfs_qm_vop_create_dqattach(tp, sc->tempip, udqp, gdqp, pdqp); + + /* + * Put our temp file on the unlinked list so it's purged automatically. + * All file-based metadata being reconstructed using this file must be + * atomically exchanged with the original file because the contents + * here will be purged when the inode is dropped or log recovery cleans + * out the unlinked list. + */ + error = xfs_iunlink(tp, sc->tempip); + if (error) + goto out_trans_cancel; + + error = xfs_trans_commit(tp); + if (error) + goto out_release_inode; + + trace_xrep_tempfile_create(sc); + + xfs_qm_dqrele(udqp); + xfs_qm_dqrele(gdqp); + xfs_qm_dqrele(pdqp); + + /* Finish setting up the incore / vfs context. */ + xfs_setup_iops(sc->tempip); + xfs_finish_inode_setup(sc->tempip); + + sc->temp_ilock_flags = 0; + return error; + +out_trans_cancel: + xfs_trans_cancel(tp); +out_release_inode: + /* + * Wait until after the current transaction is aborted to finish the + * setup of the inode and release the inode. This prevents recursive + * transactions and deadlocks from xfs_inactive. + */ + if (sc->tempip) { + xfs_finish_inode_setup(sc->tempip); + xchk_irele(sc, sc->tempip); + } +out_release_dquots: + xfs_qm_dqrele(udqp); + xfs_qm_dqrele(gdqp); + xfs_qm_dqrele(pdqp); + + return error; +} + +/* Take IOLOCK_EXCL on the temporary file, maybe. */ +bool +xrep_tempfile_iolock_nowait( + struct xfs_scrub *sc) +{ + if (xfs_ilock_nowait(sc->tempip, XFS_IOLOCK_EXCL)) { + sc->temp_ilock_flags |= XFS_IOLOCK_EXCL; + return true; + } + + return false; +} + +/* + * Take the temporary file's IOLOCK while holding a different inode's IOLOCK. + * In theory nobody else should hold the tempfile's IOLOCK, but we use trylock + * to avoid deadlocks and lockdep complaints. + */ +int +xrep_tempfile_iolock_polled( + struct xfs_scrub *sc) +{ + int error = 0; + + while (!xrep_tempfile_iolock_nowait(sc)) { + if (xchk_should_terminate(sc, &error)) + return error; + delay(1); + } + + return 0; +} + +/* Release IOLOCK_EXCL on the temporary file. */ +void +xrep_tempfile_iounlock( + struct xfs_scrub *sc) +{ + xfs_iunlock(sc->tempip, XFS_IOLOCK_EXCL); + sc->temp_ilock_flags &= ~XFS_IOLOCK_EXCL; +} + +/* Prepare the temporary file for metadata updates by grabbing ILOCK_EXCL. */ +void +xrep_tempfile_ilock( + struct xfs_scrub *sc) +{ + sc->temp_ilock_flags |= XFS_ILOCK_EXCL; + xfs_ilock(sc->tempip, XFS_ILOCK_EXCL); +} + +/* Try to grab ILOCK_EXCL on the temporary file. */ +bool +xrep_tempfile_ilock_nowait( + struct xfs_scrub *sc) +{ + if (xfs_ilock_nowait(sc->tempip, XFS_ILOCK_EXCL)) { + sc->temp_ilock_flags |= XFS_ILOCK_EXCL; + return true; + } + + return false; +} + +/* Unlock ILOCK_EXCL on the temporary file after an update. */ +void +xrep_tempfile_iunlock( + struct xfs_scrub *sc) +{ + xfs_iunlock(sc->tempip, XFS_ILOCK_EXCL); + sc->temp_ilock_flags &= ~XFS_ILOCK_EXCL; +} + +/* Release the temporary file. */ +void +xrep_tempfile_rele( + struct xfs_scrub *sc) +{ + if (!sc->tempip) + return; + + if (sc->temp_ilock_flags) { + xfs_iunlock(sc->tempip, sc->temp_ilock_flags); + sc->temp_ilock_flags = 0; + } + + xchk_irele(sc, sc->tempip); + sc->tempip = NULL; +} diff --git a/fs/xfs/scrub/tempfile.h b/fs/xfs/scrub/tempfile.h new file mode 100644 index 0000000000000..e165e0a3faf63 --- /dev/null +++ b/fs/xfs/scrub/tempfile.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_TEMPFILE_H__ +#define __XFS_SCRUB_TEMPFILE_H__ + +#ifdef CONFIG_XFS_ONLINE_REPAIR +int xrep_tempfile_create(struct xfs_scrub *sc, uint16_t mode); +void xrep_tempfile_rele(struct xfs_scrub *sc); + +bool xrep_tempfile_iolock_nowait(struct xfs_scrub *sc); +int xrep_tempfile_iolock_polled(struct xfs_scrub *sc); +void xrep_tempfile_iounlock(struct xfs_scrub *sc); + +void xrep_tempfile_ilock(struct xfs_scrub *sc); +bool xrep_tempfile_ilock_nowait(struct xfs_scrub *sc); +void xrep_tempfile_iunlock(struct xfs_scrub *sc); +#else +static inline void xrep_tempfile_iolock_both(struct xfs_scrub *sc) +{ + xchk_ilock(sc, XFS_IOLOCK_EXCL); +} +# define xrep_tempfile_rele(sc) +#endif /* CONFIG_XFS_ONLINE_REPAIR */ + +#endif /* __XFS_SCRUB_TEMPFILE_H__ */ diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index b1c7c79760d44..020b029b7988f 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2279,6 +2279,39 @@ TRACE_EVENT(xrep_rmap_live_update, __entry->flags) ); +TRACE_EVENT(xrep_tempfile_create, + TP_PROTO(struct xfs_scrub *sc), + TP_ARGS(sc), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned int, type) + __field(xfs_agnumber_t, agno) + __field(xfs_ino_t, inum) + __field(unsigned int, gen) + __field(unsigned int, flags) + __field(xfs_ino_t, temp_inum) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->ino = sc->file ? XFS_I(file_inode(sc->file))->i_ino : 0; + __entry->type = sc->sm->sm_type; + __entry->agno = sc->sm->sm_agno; + __entry->inum = sc->sm->sm_ino; + __entry->gen = sc->sm->sm_gen; + __entry->flags = sc->sm->sm_flags; + __entry->temp_inum = sc->tempip->i_ino; + ), + TP_printk("dev %d:%d ino 0x%llx type %s inum 0x%llx gen 0x%x flags 0x%x temp_inum 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __entry->inum, + __entry->gen, + __entry->flags, + __entry->temp_inum) +); + #endif /* IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) */ #endif /* _TRACE_XFS_SCRUB_TRACE_H */ diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 492dae0efad24..ac92c0525d9b6 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -42,7 +42,6 @@ struct kmem_cache *xfs_inode_cache; -STATIC int xfs_iunlink(struct xfs_trans *, struct xfs_inode *); STATIC int xfs_iunlink_remove(struct xfs_trans *tp, struct xfs_perag *pag, struct xfs_inode *); @@ -2151,7 +2150,7 @@ xfs_iunlink_insert_inode( * We place the on-disk inode on a list in the AGI. It will be pulled from this * list when the inode is freed. */ -STATIC int +int xfs_iunlink( struct xfs_trans *tp, struct xfs_inode *ip) diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index f559e68ee7077..596eec7156753 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -616,6 +616,8 @@ extern struct kmem_cache *xfs_inode_cache; bool xfs_inode_needs_inactive(struct xfs_inode *ip); +int xfs_iunlink(struct xfs_trans *tp, struct xfs_inode *ip); + void xfs_end_io(struct work_struct *work); int xfs_ilock2_io_mmap(struct xfs_inode *ip1, struct xfs_inode *ip2); From 20a3c1ecc35d2aa9ee1276e1cfd485689f29b662 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:29 -0700 Subject: [PATCH 0219/1702] xfs: refactor live buffer invalidation for repairs In an upcoming patch, we will need to be able to look for xfs_buf objects caching file-based metadata blocks without needing to walk the (possibly corrupt) structures to find all the buffers. Repair already has most of the code needed to scan the buffer cache, so hoist these utility functions. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/reap.c | 73 +++++++++++++++++++++++++++++++-------------- fs/xfs/scrub/reap.h | 20 +++++++++++++ 2 files changed, 71 insertions(+), 22 deletions(-) diff --git a/fs/xfs/scrub/reap.c b/fs/xfs/scrub/reap.c index 0252a3b5b65ac..7ae6253395e72 100644 --- a/fs/xfs/scrub/reap.c +++ b/fs/xfs/scrub/reap.c @@ -211,6 +211,48 @@ static inline void xreap_defer_finish_reset(struct xreap_state *rs) rs->force_roll = false; } +/* + * Compute the maximum length of a buffer cache scan (in units of sectors), + * given a quantity of fs blocks. + */ +xfs_daddr_t +xrep_bufscan_max_sectors( + struct xfs_mount *mp, + xfs_extlen_t fsblocks) +{ + int max_fsbs; + + /* Remote xattr values are the largest buffers that we support. */ + max_fsbs = xfs_attr3_rmt_blocks(mp, XFS_XATTR_SIZE_MAX); + + return XFS_FSB_TO_BB(mp, min_t(xfs_extlen_t, fsblocks, max_fsbs)); +} + +/* + * Return an incore buffer from a sector scan, or NULL if there are no buffers + * left to return. + */ +struct xfs_buf * +xrep_bufscan_advance( + struct xfs_mount *mp, + struct xrep_bufscan *scan) +{ + scan->__sector_count += scan->daddr_step; + while (scan->__sector_count <= scan->max_sectors) { + struct xfs_buf *bp = NULL; + int error; + + error = xfs_buf_incore(mp->m_ddev_targp, scan->daddr, + scan->__sector_count, XBF_LIVESCAN, &bp); + if (!error) + return bp; + + scan->__sector_count += scan->daddr_step; + } + + return NULL; +} + /* Try to invalidate the incore buffers for an extent that we're freeing. */ STATIC void xreap_agextent_binval( @@ -241,28 +283,15 @@ xreap_agextent_binval( * of any plausible size. */ while (bno < agbno_next) { - xfs_agblock_t fsbcount; - xfs_agblock_t max_fsbs; - - /* - * Max buffer size is the max remote xattr buffer size, which - * is one fs block larger than 64k. - */ - max_fsbs = min_t(xfs_agblock_t, agbno_next - bno, - xfs_attr3_rmt_blocks(mp, XFS_XATTR_SIZE_MAX)); - - for (fsbcount = 1; fsbcount <= max_fsbs; fsbcount++) { - struct xfs_buf *bp = NULL; - xfs_daddr_t daddr; - int error; - - daddr = XFS_AGB_TO_DADDR(mp, agno, bno); - error = xfs_buf_incore(mp->m_ddev_targp, daddr, - XFS_FSB_TO_BB(mp, fsbcount), - XBF_LIVESCAN, &bp); - if (error) - continue; - + struct xrep_bufscan scan = { + .daddr = XFS_AGB_TO_DADDR(mp, agno, bno), + .max_sectors = xrep_bufscan_max_sectors(mp, + agbno_next - bno), + .daddr_step = XFS_FSB_TO_BB(mp, 1), + }; + struct xfs_buf *bp; + + while ((bp = xrep_bufscan_advance(mp, &scan)) != NULL) { xfs_trans_bjoin(sc->tp, bp); xfs_trans_binval(sc->tp, bp); rs->invalidated++; diff --git a/fs/xfs/scrub/reap.h b/fs/xfs/scrub/reap.h index 0b69f16dd98f9..bb09e21fcb172 100644 --- a/fs/xfs/scrub/reap.h +++ b/fs/xfs/scrub/reap.h @@ -14,4 +14,24 @@ int xrep_reap_agblocks(struct xfs_scrub *sc, struct xagb_bitmap *bitmap, int xrep_reap_fsblocks(struct xfs_scrub *sc, struct xfsb_bitmap *bitmap, const struct xfs_owner_info *oinfo); +/* Buffer cache scan context. */ +struct xrep_bufscan { + /* Disk address for the buffers we want to scan. */ + xfs_daddr_t daddr; + + /* Maximum number of sectors to scan. */ + xfs_daddr_t max_sectors; + + /* Each round, increment the search length by this number of sectors. */ + xfs_daddr_t daddr_step; + + /* Internal scan state; initialize to zero. */ + xfs_daddr_t __sector_count; +}; + +xfs_daddr_t xrep_bufscan_max_sectors(struct xfs_mount *mp, + xfs_extlen_t fsblocks); +struct xfs_buf *xrep_bufscan_advance(struct xfs_mount *mp, + struct xrep_bufscan *scan); + #endif /* __XFS_SCRUB_REAP_H__ */ From e81ce4241318d5ec943b4fe205953498a66ca071 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:31 -0700 Subject: [PATCH 0220/1702] xfs: support preallocating and copying content into temporary files Create the routines we need to preallocate space in a temporary ondisk file and then copy the contents of an xfile into the tempfile. The upcoming rtsummary repair feature will construct the contents of a realtime summary file in memory, after which it will want to copy all that into the ondisk temporary file before atomically committing the new rtsummary contents. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/tempfile.c | 197 ++++++++++++++++++++++++++++++++++++++++ fs/xfs/scrub/tempfile.h | 15 +++ fs/xfs/scrub/trace.h | 39 ++++++++ 3 files changed, 251 insertions(+) diff --git a/fs/xfs/scrub/tempfile.c b/fs/xfs/scrub/tempfile.c index 68d245749bc1e..83e683e165618 100644 --- a/fs/xfs/scrub/tempfile.c +++ b/fs/xfs/scrub/tempfile.c @@ -14,14 +14,18 @@ #include "xfs_inode.h" #include "xfs_ialloc.h" #include "xfs_quota.h" +#include "xfs_bmap.h" #include "xfs_bmap_btree.h" #include "xfs_trans_space.h" #include "xfs_dir2.h" #include "xfs_exchrange.h" +#include "xfs_defer.h" #include "scrub/scrub.h" #include "scrub/common.h" +#include "scrub/repair.h" #include "scrub/trace.h" #include "scrub/tempfile.h" +#include "scrub/xfile.h" /* * Create a temporary file for reconstructing metadata, with the intention of @@ -249,3 +253,196 @@ xrep_tempfile_rele( xchk_irele(sc, sc->tempip); sc->tempip = NULL; } + +/* + * Make sure that the given range of the data fork of the temporary file is + * mapped to written blocks. The caller must ensure that both inodes are + * joined to the transaction. + */ +int +xrep_tempfile_prealloc( + struct xfs_scrub *sc, + xfs_fileoff_t off, + xfs_filblks_t len) +{ + struct xfs_bmbt_irec map; + xfs_fileoff_t end = off + len; + int error; + + ASSERT(sc->tempip != NULL); + ASSERT(!XFS_NOT_DQATTACHED(sc->mp, sc->tempip)); + + for (; off < end; off = map.br_startoff + map.br_blockcount) { + int nmaps = 1; + + /* + * If we have a real extent mapping this block then we're + * in ok shape. + */ + error = xfs_bmapi_read(sc->tempip, off, end - off, &map, &nmaps, + XFS_DATA_FORK); + if (error) + return error; + if (nmaps == 0) { + ASSERT(nmaps != 0); + return -EFSCORRUPTED; + } + + if (xfs_bmap_is_written_extent(&map)) + continue; + + /* + * If we find a delalloc reservation then something is very + * very wrong. Bail out. + */ + if (map.br_startblock == DELAYSTARTBLOCK) + return -EFSCORRUPTED; + + /* + * Make sure this block has a real zeroed extent allocated to + * it. + */ + nmaps = 1; + error = xfs_bmapi_write(sc->tp, sc->tempip, off, end - off, + XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO, 0, &map, + &nmaps); + if (error) + return error; + if (nmaps != 1) + return -EFSCORRUPTED; + + trace_xrep_tempfile_prealloc(sc, XFS_DATA_FORK, &map); + + /* Commit new extent and all deferred work. */ + error = xfs_defer_finish(&sc->tp); + if (error) + return error; + } + + return 0; +} + +/* + * Write data to each block of a file. The given range of the tempfile's data + * fork must already be populated with written extents. + */ +int +xrep_tempfile_copyin( + struct xfs_scrub *sc, + xfs_fileoff_t off, + xfs_filblks_t len, + xrep_tempfile_copyin_fn prep_fn, + void *data) +{ + LIST_HEAD(buffers_list); + struct xfs_mount *mp = sc->mp; + struct xfs_buf *bp; + xfs_fileoff_t flush_mask; + xfs_fileoff_t end = off + len; + loff_t pos = XFS_FSB_TO_B(mp, off); + int error = 0; + + ASSERT(S_ISREG(VFS_I(sc->tempip)->i_mode)); + + /* Flush buffers to disk every 512K */ + flush_mask = XFS_B_TO_FSBT(mp, (1U << 19)) - 1; + + for (; off < end; off++, pos += mp->m_sb.sb_blocksize) { + struct xfs_bmbt_irec map; + int nmaps = 1; + + /* Read block mapping for this file block. */ + error = xfs_bmapi_read(sc->tempip, off, 1, &map, &nmaps, 0); + if (error) + goto out_err; + if (nmaps == 0 || !xfs_bmap_is_written_extent(&map)) { + error = -EFSCORRUPTED; + goto out_err; + } + + /* Get the metadata buffer for this offset in the file. */ + error = xfs_trans_get_buf(sc->tp, mp->m_ddev_targp, + XFS_FSB_TO_DADDR(mp, map.br_startblock), + mp->m_bsize, 0, &bp); + if (error) + goto out_err; + + trace_xrep_tempfile_copyin(sc, XFS_DATA_FORK, &map); + + /* Read in a block's worth of data from the xfile. */ + error = prep_fn(sc, bp, data); + if (error) { + xfs_trans_brelse(sc->tp, bp); + goto out_err; + } + + /* Queue buffer, and flush if we have too much dirty data. */ + xfs_buf_delwri_queue_here(bp, &buffers_list); + xfs_trans_brelse(sc->tp, bp); + + if (!(off & flush_mask)) { + error = xfs_buf_delwri_submit(&buffers_list); + if (error) + goto out_err; + } + } + + /* + * Write the new blocks to disk. If the ordered list isn't empty after + * that, then something went wrong and we have to fail. This should + * never happen, but we'll check anyway. + */ + error = xfs_buf_delwri_submit(&buffers_list); + if (error) + goto out_err; + + if (!list_empty(&buffers_list)) { + ASSERT(list_empty(&buffers_list)); + error = -EIO; + goto out_err; + } + + return 0; + +out_err: + xfs_buf_delwri_cancel(&buffers_list); + return error; +} + +/* + * Set the temporary file's size. Caller must join the tempfile to the scrub + * transaction and is responsible for adjusting block mappings as needed. + */ +int +xrep_tempfile_set_isize( + struct xfs_scrub *sc, + unsigned long long isize) +{ + if (sc->tempip->i_disk_size == isize) + return 0; + + sc->tempip->i_disk_size = isize; + i_size_write(VFS_I(sc->tempip), isize); + return xrep_tempfile_roll_trans(sc); +} + +/* + * Roll a repair transaction involving the temporary file. Caller must join + * both the temporary file and the file being scrubbed to the transaction. + * This function return with both inodes joined to a new scrub transaction, + * or the usual negative errno. + */ +int +xrep_tempfile_roll_trans( + struct xfs_scrub *sc) +{ + int error; + + xfs_trans_log_inode(sc->tp, sc->tempip, XFS_ILOG_CORE); + error = xrep_roll_trans(sc); + if (error) + return error; + + xfs_trans_ijoin(sc->tp, sc->tempip, 0); + return 0; +} diff --git a/fs/xfs/scrub/tempfile.h b/fs/xfs/scrub/tempfile.h index e165e0a3faf63..7980f9c4de552 100644 --- a/fs/xfs/scrub/tempfile.h +++ b/fs/xfs/scrub/tempfile.h @@ -17,6 +17,21 @@ void xrep_tempfile_iounlock(struct xfs_scrub *sc); void xrep_tempfile_ilock(struct xfs_scrub *sc); bool xrep_tempfile_ilock_nowait(struct xfs_scrub *sc); void xrep_tempfile_iunlock(struct xfs_scrub *sc); + +int xrep_tempfile_prealloc(struct xfs_scrub *sc, xfs_fileoff_t off, + xfs_filblks_t len); + +enum xfs_blft; + +typedef int (*xrep_tempfile_copyin_fn)(struct xfs_scrub *sc, + struct xfs_buf *bp, void *data); + +int xrep_tempfile_copyin(struct xfs_scrub *sc, xfs_fileoff_t off, + xfs_filblks_t len, xrep_tempfile_copyin_fn fn, void *data); + +int xrep_tempfile_set_isize(struct xfs_scrub *sc, unsigned long long isize); + +int xrep_tempfile_roll_trans(struct xfs_scrub *sc); #else static inline void xrep_tempfile_iolock_both(struct xfs_scrub *sc) { diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index cbd70ecd30113..ae90731bf6ad7 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2314,6 +2314,45 @@ TRACE_EVENT(xrep_tempfile_create, __entry->temp_inum) ); +DECLARE_EVENT_CLASS(xrep_tempfile_class, + TP_PROTO(struct xfs_scrub *sc, int whichfork, + struct xfs_bmbt_irec *irec), + TP_ARGS(sc, whichfork, irec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(int, whichfork) + __field(xfs_fileoff_t, lblk) + __field(xfs_filblks_t, len) + __field(xfs_fsblock_t, pblk) + __field(int, state) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->ino = sc->tempip->i_ino; + __entry->whichfork = whichfork; + __entry->lblk = irec->br_startoff; + __entry->len = irec->br_blockcount; + __entry->pblk = irec->br_startblock; + __entry->state = irec->br_state; + ), + TP_printk("dev %d:%d ino 0x%llx whichfork %s fileoff 0x%llx fsbcount 0x%llx startblock 0x%llx state %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->whichfork, XFS_WHICHFORK_STRINGS), + __entry->lblk, + __entry->len, + __entry->pblk, + __entry->state) +); +#define DEFINE_XREP_TEMPFILE_EVENT(name) \ +DEFINE_EVENT(xrep_tempfile_class, name, \ + TP_PROTO(struct xfs_scrub *sc, int whichfork, \ + struct xfs_bmbt_irec *irec), \ + TP_ARGS(sc, whichfork, irec)) +DEFINE_XREP_TEMPFILE_EVENT(xrep_tempfile_prealloc); +DEFINE_XREP_TEMPFILE_EVENT(xrep_tempfile_copyin); + TRACE_EVENT(xreap_ifork_extent, TP_PROTO(struct xfs_scrub *sc, struct xfs_inode *ip, int whichfork, const struct xfs_bmbt_irec *irec), From 56596d8bffd2f3c80d9844835e6c118d89b67ca9 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:32 -0700 Subject: [PATCH 0221/1702] xfs: teach the tempfile to set up atomic file content exchanges Create some new routines to exchange the contents of a temporary file created to stage a repair with another ondisk file. This will be used by the realtime summary repair function to commit atomically the new rtsummary data, which will be staged in the tempfile. The rest of XFS coordinates access to the realtime metadata inodes solely through the ILOCK. For repair to hold its exclusive access to the realtime summary file, it has to allocate a single large transaction and roll it repeatedly throughout the repair while holding the ILOCK. In turn, this means that for now there's only a partial file mapping exchange implementation for the temporary file because we can only work within an existing transaction. For now, the only tempswap functions needed here are to estimate the resource requirements of the exchange, reserve more space/quota to an existing transaction, and kick off the actual exchange. The rest will be added in a later patch in preparation for repairing xattrs and directories. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/scrub.c | 8 +- fs/xfs/scrub/scrub.h | 7 ++ fs/xfs/scrub/tempexch.h | 21 +++++ fs/xfs/scrub/tempfile.c | 191 ++++++++++++++++++++++++++++++++++++++++ fs/xfs/scrub/trace.h | 1 + 5 files changed, 225 insertions(+), 3 deletions(-) create mode 100644 fs/xfs/scrub/tempexch.h diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index d9012e9a6afd2..ff156edf49a08 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -149,14 +149,15 @@ xchk_probe( /* Scrub setup and teardown */ +#define FSGATES_MASK (XCHK_FSGATES_ALL | XREP_FSGATES_ALL) static inline void xchk_fsgates_disable( struct xfs_scrub *sc) { - if (!(sc->flags & XCHK_FSGATES_ALL)) + if (!(sc->flags & FSGATES_MASK)) return; - trace_xchk_fsgates_disable(sc, sc->flags & XCHK_FSGATES_ALL); + trace_xchk_fsgates_disable(sc, sc->flags & FSGATES_MASK); if (sc->flags & XCHK_FSGATES_DRAIN) xfs_drain_wait_disable(); @@ -170,8 +171,9 @@ xchk_fsgates_disable( if (sc->flags & XCHK_FSGATES_RMAP) xfs_rmap_hook_disable(); - sc->flags &= ~XCHK_FSGATES_ALL; + sc->flags &= ~FSGATES_MASK; } +#undef FSGATES_MASK /* Free all the resources and finish the transactions. */ STATIC int diff --git a/fs/xfs/scrub/scrub.h b/fs/xfs/scrub/scrub.h index e37d8599718e2..d38f0b30416c0 100644 --- a/fs/xfs/scrub/scrub.h +++ b/fs/xfs/scrub/scrub.h @@ -131,6 +131,7 @@ struct xfs_scrub { #define XCHK_FSGATES_QUOTA (1U << 4) /* quota live update enabled */ #define XCHK_FSGATES_DIRENTS (1U << 5) /* directory live update enabled */ #define XCHK_FSGATES_RMAP (1U << 6) /* rmapbt live update enabled */ +#define XREP_FSGATES_EXCHANGE_RANGE (1U << 29) /* uses file content exchange */ #define XREP_RESET_PERAG_RESV (1U << 30) /* must reset AG space reservation */ #define XREP_ALREADY_FIXED (1U << 31) /* checking our repair work */ @@ -145,6 +146,12 @@ struct xfs_scrub { XCHK_FSGATES_DIRENTS | \ XCHK_FSGATES_RMAP) +/* + * The sole XREP_FSGATES* flag reflects a log intent item that is protected + * by a log-incompat feature flag. No code patching in use here. + */ +#define XREP_FSGATES_ALL (XREP_FSGATES_EXCHANGE_RANGE) + /* Metadata scrubbers */ int xchk_tester(struct xfs_scrub *sc); int xchk_superblock(struct xfs_scrub *sc); diff --git a/fs/xfs/scrub/tempexch.h b/fs/xfs/scrub/tempexch.h new file mode 100644 index 0000000000000..98222b684b6a0 --- /dev/null +++ b/fs/xfs/scrub/tempexch.h @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_TEMPEXCH_H__ +#define __XFS_SCRUB_TEMPEXCH_H__ + +#ifdef CONFIG_XFS_ONLINE_REPAIR +struct xrep_tempexch { + struct xfs_exchmaps_req req; +}; + +int xrep_tempexch_enable(struct xfs_scrub *sc); +int xrep_tempexch_trans_reserve(struct xfs_scrub *sc, int whichfork, + struct xrep_tempexch *ti); + +int xrep_tempexch_contents(struct xfs_scrub *sc, struct xrep_tempexch *ti); +#endif /* CONFIG_XFS_ONLINE_REPAIR */ + +#endif /* __XFS_SCRUB_TEMPEXCH_H__ */ diff --git a/fs/xfs/scrub/tempfile.c b/fs/xfs/scrub/tempfile.c index 83e683e165618..7791336ca8209 100644 --- a/fs/xfs/scrub/tempfile.c +++ b/fs/xfs/scrub/tempfile.c @@ -19,12 +19,14 @@ #include "xfs_trans_space.h" #include "xfs_dir2.h" #include "xfs_exchrange.h" +#include "xfs_exchmaps.h" #include "xfs_defer.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/repair.h" #include "scrub/trace.h" #include "scrub/tempfile.h" +#include "scrub/tempexch.h" #include "scrub/xfile.h" /* @@ -446,3 +448,192 @@ xrep_tempfile_roll_trans( xfs_trans_ijoin(sc->tp, sc->tempip, 0); return 0; } + +/* Enable file content exchanges. */ +int +xrep_tempexch_enable( + struct xfs_scrub *sc) +{ + if (sc->flags & XREP_FSGATES_EXCHANGE_RANGE) + return 0; + + if (!xfs_has_exchange_range(sc->mp)) + return -EOPNOTSUPP; + + trace_xchk_fsgates_enable(sc, XREP_FSGATES_EXCHANGE_RANGE); + + sc->flags |= XREP_FSGATES_EXCHANGE_RANGE; + return 0; +} + +/* + * Fill out the mapping exchange request in preparation for atomically + * committing the contents of a metadata file that we've rebuilt in the temp + * file. + */ +STATIC int +xrep_tempexch_prep_request( + struct xfs_scrub *sc, + int whichfork, + struct xrep_tempexch *tx) +{ + struct xfs_exchmaps_req *req = &tx->req; + + memset(tx, 0, sizeof(struct xrep_tempexch)); + + /* COW forks don't exist on disk. */ + if (whichfork == XFS_COW_FORK) { + ASSERT(0); + return -EINVAL; + } + + /* Both files should have the relevant forks. */ + if (!xfs_ifork_ptr(sc->ip, whichfork) || + !xfs_ifork_ptr(sc->tempip, whichfork)) { + ASSERT(xfs_ifork_ptr(sc->ip, whichfork) != NULL); + ASSERT(xfs_ifork_ptr(sc->tempip, whichfork) != NULL); + return -EINVAL; + } + + /* Exchange all mappings in both forks. */ + req->ip1 = sc->tempip; + req->ip2 = sc->ip; + req->startoff1 = 0; + req->startoff2 = 0; + switch (whichfork) { + case XFS_ATTR_FORK: + req->flags |= XFS_EXCHMAPS_ATTR_FORK; + break; + case XFS_DATA_FORK: + /* Always exchange sizes when exchanging data fork mappings. */ + req->flags |= XFS_EXCHMAPS_SET_SIZES; + break; + } + req->blockcount = XFS_MAX_FILEOFF; + + return 0; +} + +/* + * Obtain a quota reservation to make sure we don't hit EDQUOT. We can skip + * this if quota enforcement is disabled or if both inodes' dquots are the + * same. The qretry structure must be initialized to zeroes before the first + * call to this function. + */ +STATIC int +xrep_tempexch_reserve_quota( + struct xfs_scrub *sc, + const struct xrep_tempexch *tx) +{ + struct xfs_trans *tp = sc->tp; + const struct xfs_exchmaps_req *req = &tx->req; + int64_t ddelta, rdelta; + int error; + + /* + * Don't bother with a quota reservation if we're not enforcing them + * or the two inodes have the same dquots. + */ + if (!XFS_IS_QUOTA_ON(tp->t_mountp) || req->ip1 == req->ip2 || + (req->ip1->i_udquot == req->ip2->i_udquot && + req->ip1->i_gdquot == req->ip2->i_gdquot && + req->ip1->i_pdquot == req->ip2->i_pdquot)) + return 0; + + /* + * Quota reservation for each file comes from two sources. First, we + * need to account for any net gain in mapped blocks during the + * exchange. Second, we need reservation for the gross gain in mapped + * blocks so that we don't trip over any quota block reservation + * assertions. We must reserve the gross gain because the quota code + * subtracts from bcount the number of blocks that we unmap; it does + * not add that quantity back to the quota block reservation. + */ + ddelta = max_t(int64_t, 0, req->ip2_bcount - req->ip1_bcount); + rdelta = max_t(int64_t, 0, req->ip2_rtbcount - req->ip1_rtbcount); + error = xfs_trans_reserve_quota_nblks(tp, req->ip1, + ddelta + req->ip1_bcount, rdelta + req->ip1_rtbcount, + true); + if (error) + return error; + + ddelta = max_t(int64_t, 0, req->ip1_bcount - req->ip2_bcount); + rdelta = max_t(int64_t, 0, req->ip1_rtbcount - req->ip2_rtbcount); + return xfs_trans_reserve_quota_nblks(tp, req->ip2, + ddelta + req->ip2_bcount, rdelta + req->ip2_rtbcount, + true); +} + +/* + * Prepare an existing transaction for an atomic file contents exchange. + * + * This function fills out the mapping exchange request and resource estimation + * structures in preparation for exchanging the contents of a metadata file + * that has been rebuilt in the temp file. Next, it reserves space and quota + * for the transaction. + * + * The caller must hold ILOCK_EXCL of the scrub target file and the temporary + * file. The caller must join both inodes to the transaction with no unlock + * flags, and is responsible for dropping both ILOCKs when appropriate. Only + * use this when those ILOCKs cannot be dropped. + */ +int +xrep_tempexch_trans_reserve( + struct xfs_scrub *sc, + int whichfork, + struct xrep_tempexch *tx) +{ + int error; + + ASSERT(sc->tp != NULL); + xfs_assert_ilocked(sc->ip, XFS_ILOCK_EXCL); + xfs_assert_ilocked(sc->tempip, XFS_ILOCK_EXCL); + + error = xrep_tempexch_prep_request(sc, whichfork, tx); + if (error) + return error; + + error = xfs_exchmaps_estimate(&tx->req); + if (error) + return error; + + error = xfs_trans_reserve_more(sc->tp, tx->req.resblks, 0); + if (error) + return error; + + return xrep_tempexch_reserve_quota(sc, tx); +} + +/* + * Exchange file mappings (and hence file contents) between the file being + * repaired and the temporary file. Returns with both inodes locked and joined + * to a clean scrub transaction. + */ +int +xrep_tempexch_contents( + struct xfs_scrub *sc, + struct xrep_tempexch *tx) +{ + int error; + + ASSERT(sc->flags & XREP_FSGATES_EXCHANGE_RANGE); + + xfs_exchange_mappings(sc->tp, &tx->req); + error = xfs_defer_finish(&sc->tp); + if (error) + return error; + + /* + * If we exchanged the ondisk sizes of two metadata files, we must + * exchanged the incore sizes as well. + */ + if (tx->req.flags & XFS_EXCHMAPS_SET_SIZES) { + loff_t temp; + + temp = i_size_read(VFS_I(sc->ip)); + i_size_write(VFS_I(sc->ip), i_size_read(VFS_I(sc->tempip))); + i_size_write(VFS_I(sc->tempip), temp); + } + + return 0; +} diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index ae90731bf6ad7..8d05f2adae3dd 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -114,6 +114,7 @@ TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_HEALTHY); { XCHK_FSGATES_QUOTA, "fsgates_quota" }, \ { XCHK_FSGATES_DIRENTS, "fsgates_dirents" }, \ { XCHK_FSGATES_RMAP, "fsgates_rmap" }, \ + { XREP_FSGATES_EXCHANGE_RANGE, "fsgates_exchrange" }, \ { XREP_RESET_PERAG_RESV, "reset_perag_resv" }, \ { XREP_ALREADY_FIXED, "already_fixed" } From 5befb047b9f4de1747bf48c63cab997a97e0088b Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:30 -0700 Subject: [PATCH 0222/1702] xfs: add the ability to reap entire inode forks In preparation for supporting repair of indexed file-based metadata (such as realtime bitmaps, directories, and extended attribute data), add a function to reap the old blocks after a metadata repair finishes. IOWs, this is an elaborate bunmapi call that deals with crosslinked blocks by unmapping them without freeing them, and also scans for incore buffers to invalidate. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/reap.c | 372 +++++++++++++++++++++++++++++++++++++++++++ fs/xfs/scrub/reap.h | 1 + fs/xfs/scrub/trace.h | 63 ++++++++ 3 files changed, 436 insertions(+) diff --git a/fs/xfs/scrub/reap.c b/fs/xfs/scrub/reap.c index 7ae6253395e72..01ceaa4efa16b 100644 --- a/fs/xfs/scrub/reap.c +++ b/fs/xfs/scrub/reap.c @@ -675,3 +675,375 @@ xrep_reap_fsblocks( return 0; } + +/* + * Metadata files are not supposed to share blocks with anything else. + * If blocks are shared, we remove the reverse mapping (thus reducing the + * crosslink factor); if blocks are not shared, we also need to free them. + * + * This first step determines the longest subset of the passed-in imap + * (starting at its beginning) that is either crosslinked or not crosslinked. + * The blockcount will be adjust down as needed. + */ +STATIC int +xreap_bmapi_select( + struct xfs_scrub *sc, + struct xfs_inode *ip, + int whichfork, + struct xfs_bmbt_irec *imap, + bool *crosslinked) +{ + struct xfs_owner_info oinfo; + struct xfs_btree_cur *cur; + xfs_filblks_t len = 1; + xfs_agblock_t bno; + xfs_agblock_t agbno; + xfs_agblock_t agbno_next; + int error; + + agbno = XFS_FSB_TO_AGBNO(sc->mp, imap->br_startblock); + agbno_next = agbno + imap->br_blockcount; + + cur = xfs_rmapbt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp, + sc->sa.pag); + + xfs_rmap_ino_owner(&oinfo, ip->i_ino, whichfork, imap->br_startoff); + error = xfs_rmap_has_other_keys(cur, agbno, 1, &oinfo, crosslinked); + if (error) + goto out_cur; + + bno = agbno + 1; + while (bno < agbno_next) { + bool also_crosslinked; + + oinfo.oi_offset++; + error = xfs_rmap_has_other_keys(cur, bno, 1, &oinfo, + &also_crosslinked); + if (error) + goto out_cur; + + if (also_crosslinked != *crosslinked) + break; + + len++; + bno++; + } + + imap->br_blockcount = len; + trace_xreap_bmapi_select(sc->sa.pag, agbno, len, *crosslinked); +out_cur: + xfs_btree_del_cursor(cur, error); + return error; +} + +/* + * Decide if this buffer can be joined to a transaction. This is true for most + * buffers, but there are two cases that we want to catch: large remote xattr + * value buffers are not logged and can overflow the buffer log item dirty + * bitmap size; and oversized cached buffers if things have really gone + * haywire. + */ +static inline bool +xreap_buf_loggable( + const struct xfs_buf *bp) +{ + int i; + + for (i = 0; i < bp->b_map_count; i++) { + int chunks; + int map_size; + + chunks = DIV_ROUND_UP(BBTOB(bp->b_maps[i].bm_len), + XFS_BLF_CHUNK); + map_size = DIV_ROUND_UP(chunks, NBWORD); + if (map_size > XFS_BLF_DATAMAP_SIZE) + return false; + } + + return true; +} + +/* + * Invalidate any buffers for this file mapping. The @imap blockcount may be + * adjusted downward if we need to roll the transaction. + */ +STATIC int +xreap_bmapi_binval( + struct xfs_scrub *sc, + struct xfs_inode *ip, + int whichfork, + struct xfs_bmbt_irec *imap) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_perag *pag = sc->sa.pag; + int bmap_flags = xfs_bmapi_aflag(whichfork); + xfs_fileoff_t off; + xfs_fileoff_t max_off; + xfs_extlen_t scan_blocks; + xfs_agnumber_t agno = sc->sa.pag->pag_agno; + xfs_agblock_t bno; + xfs_agblock_t agbno; + xfs_agblock_t agbno_next; + unsigned int invalidated = 0; + int error; + + /* + * Avoid invalidating AG headers and post-EOFS blocks because we never + * own those. + */ + agbno = bno = XFS_FSB_TO_AGBNO(sc->mp, imap->br_startblock); + agbno_next = agbno + imap->br_blockcount; + if (!xfs_verify_agbno(pag, agbno) || + !xfs_verify_agbno(pag, agbno_next - 1)) + return 0; + + /* + * Buffers for file blocks can span multiple contiguous mappings. This + * means that for each block in the mapping, there could exist an + * xfs_buf indexed by that block with any length up to the maximum + * buffer size (remote xattr values) or to the next hole in the fork. + * To set up our binval scan, first we need to figure out the location + * of the next hole. + */ + off = imap->br_startoff + imap->br_blockcount; + max_off = off + xfs_attr3_rmt_blocks(mp, XFS_XATTR_SIZE_MAX); + while (off < max_off) { + struct xfs_bmbt_irec hmap; + int nhmaps = 1; + + error = xfs_bmapi_read(ip, off, max_off - off, &hmap, + &nhmaps, bmap_flags); + if (error) + return error; + if (nhmaps != 1 || hmap.br_startblock == DELAYSTARTBLOCK) { + ASSERT(0); + return -EFSCORRUPTED; + } + + if (!xfs_bmap_is_real_extent(&hmap)) + break; + + off = hmap.br_startoff + hmap.br_blockcount; + } + scan_blocks = off - imap->br_startoff; + + trace_xreap_bmapi_binval_scan(sc, imap, scan_blocks); + + /* + * If there are incore buffers for these blocks, invalidate them. If + * we can't (try)lock the buffer we assume it's owned by someone else + * and leave it alone. The buffer cache cannot detect aliasing, so + * employ nested loops to detect incore buffers of any plausible size. + */ + while (bno < agbno_next) { + struct xrep_bufscan scan = { + .daddr = XFS_AGB_TO_DADDR(mp, agno, bno), + .max_sectors = xrep_bufscan_max_sectors(mp, + scan_blocks), + .daddr_step = XFS_FSB_TO_BB(mp, 1), + }; + struct xfs_buf *bp; + + while ((bp = xrep_bufscan_advance(mp, &scan)) != NULL) { + if (xreap_buf_loggable(bp)) { + xfs_trans_bjoin(sc->tp, bp); + xfs_trans_binval(sc->tp, bp); + } else { + xfs_buf_stale(bp); + xfs_buf_relse(bp); + } + invalidated++; + + /* + * Stop invalidating if we've hit the limit; we should + * still have enough reservation left to free however + * much of the mapping we've seen so far. + */ + if (invalidated > XREAP_MAX_BINVAL) { + imap->br_blockcount = agbno_next - bno; + goto out; + } + } + + bno++; + scan_blocks--; + } + +out: + trace_xreap_bmapi_binval(sc->sa.pag, agbno, imap->br_blockcount); + return 0; +} + +/* + * Dispose of as much of the beginning of this file fork mapping as possible. + * The number of blocks disposed of is returned in @imap->br_blockcount. + */ +STATIC int +xrep_reap_bmapi_iter( + struct xfs_scrub *sc, + struct xfs_inode *ip, + int whichfork, + struct xfs_bmbt_irec *imap, + bool crosslinked) +{ + int error; + + if (crosslinked) { + /* + * If there are other rmappings, this block is cross linked and + * must not be freed. Remove the reverse mapping, leave the + * buffer cache in its possibly confused state, and move on. + * We don't want to risk discarding valid data buffers from + * anybody else who thinks they own the block, even though that + * runs the risk of stale buffer warnings in the future. + */ + trace_xreap_dispose_unmap_extent(sc->sa.pag, + XFS_FSB_TO_AGBNO(sc->mp, imap->br_startblock), + imap->br_blockcount); + + /* + * Schedule removal of the mapping from the fork. We use + * deferred log intents in this function to control the exact + * sequence of metadata updates. + */ + xfs_bmap_unmap_extent(sc->tp, ip, whichfork, imap); + xfs_trans_mod_dquot_byino(sc->tp, ip, XFS_TRANS_DQ_BCOUNT, + -(int64_t)imap->br_blockcount); + xfs_rmap_unmap_extent(sc->tp, ip, whichfork, imap); + return 0; + } + + /* + * If the block is not crosslinked, we can invalidate all the incore + * buffers for the extent, and then free the extent. This is a bit of + * a mess since we don't detect discontiguous buffers that are indexed + * by a block starting before the first block of the extent but overlap + * anyway. + */ + trace_xreap_dispose_free_extent(sc->sa.pag, + XFS_FSB_TO_AGBNO(sc->mp, imap->br_startblock), + imap->br_blockcount); + + /* + * Invalidate as many buffers as we can, starting at the beginning of + * this mapping. If this function sets blockcount to zero, the + * transaction is full of logged buffer invalidations, so we need to + * return early so that we can roll and retry. + */ + error = xreap_bmapi_binval(sc, ip, whichfork, imap); + if (error || imap->br_blockcount == 0) + return error; + + /* + * Schedule removal of the mapping from the fork. We use deferred log + * intents in this function to control the exact sequence of metadata + * updates. + */ + xfs_bmap_unmap_extent(sc->tp, ip, whichfork, imap); + xfs_trans_mod_dquot_byino(sc->tp, ip, XFS_TRANS_DQ_BCOUNT, + -(int64_t)imap->br_blockcount); + return xfs_free_extent_later(sc->tp, imap->br_startblock, + imap->br_blockcount, NULL, XFS_AG_RESV_NONE, true); +} + +/* + * Dispose of as much of this file extent as we can. Upon successful return, + * the imap will reflect the mapping that was removed from the fork. + */ +STATIC int +xreap_ifork_extent( + struct xfs_scrub *sc, + struct xfs_inode *ip, + int whichfork, + struct xfs_bmbt_irec *imap) +{ + xfs_agnumber_t agno; + bool crosslinked; + int error; + + ASSERT(sc->sa.pag == NULL); + + trace_xreap_ifork_extent(sc, ip, whichfork, imap); + + agno = XFS_FSB_TO_AGNO(sc->mp, imap->br_startblock); + sc->sa.pag = xfs_perag_get(sc->mp, agno); + if (!sc->sa.pag) + return -EFSCORRUPTED; + + error = xfs_alloc_read_agf(sc->sa.pag, sc->tp, 0, &sc->sa.agf_bp); + if (error) + goto out_pag; + + /* + * Decide the fate of the blocks at the beginning of the mapping, then + * update the mapping to use it with the unmap calls. + */ + error = xreap_bmapi_select(sc, ip, whichfork, imap, &crosslinked); + if (error) + goto out_agf; + + error = xrep_reap_bmapi_iter(sc, ip, whichfork, imap, crosslinked); + if (error) + goto out_agf; + +out_agf: + xfs_trans_brelse(sc->tp, sc->sa.agf_bp); + sc->sa.agf_bp = NULL; +out_pag: + xfs_perag_put(sc->sa.pag); + sc->sa.pag = NULL; + return error; +} + +/* + * Dispose of each block mapped to the given fork of the given file. Callers + * must hold ILOCK_EXCL, and ip can only be sc->ip or sc->tempip. The fork + * must not have any delalloc reservations. + */ +int +xrep_reap_ifork( + struct xfs_scrub *sc, + struct xfs_inode *ip, + int whichfork) +{ + xfs_fileoff_t off = 0; + int bmap_flags = xfs_bmapi_aflag(whichfork); + int error; + + ASSERT(xfs_has_rmapbt(sc->mp)); + ASSERT(ip == sc->ip || ip == sc->tempip); + ASSERT(whichfork == XFS_ATTR_FORK || !XFS_IS_REALTIME_INODE(ip)); + + while (off < XFS_MAX_FILEOFF) { + struct xfs_bmbt_irec imap; + int nimaps = 1; + + /* Read the next extent, skip past holes and delalloc. */ + error = xfs_bmapi_read(ip, off, XFS_MAX_FILEOFF - off, &imap, + &nimaps, bmap_flags); + if (error) + return error; + if (nimaps != 1 || imap.br_startblock == DELAYSTARTBLOCK) { + ASSERT(0); + return -EFSCORRUPTED; + } + + /* + * If this is a real space mapping, reap as much of it as we + * can in a single transaction. + */ + if (xfs_bmap_is_real_extent(&imap)) { + error = xreap_ifork_extent(sc, ip, whichfork, &imap); + if (error) + return error; + + error = xfs_defer_finish(&sc->tp); + if (error) + return error; + } + + off = imap.br_startoff + imap.br_blockcount; + } + + return 0; +} diff --git a/fs/xfs/scrub/reap.h b/fs/xfs/scrub/reap.h index bb09e21fcb172..3f2f1775e29db 100644 --- a/fs/xfs/scrub/reap.h +++ b/fs/xfs/scrub/reap.h @@ -13,6 +13,7 @@ int xrep_reap_agblocks(struct xfs_scrub *sc, struct xagb_bitmap *bitmap, const struct xfs_owner_info *oinfo, enum xfs_ag_resv_type type); int xrep_reap_fsblocks(struct xfs_scrub *sc, struct xfsb_bitmap *bitmap, const struct xfs_owner_info *oinfo); +int xrep_reap_ifork(struct xfs_scrub *sc, struct xfs_inode *ip, int whichfork); /* Buffer cache scan context. */ struct xrep_bufscan { diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 020b029b7988f..cbd70ecd30113 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -1539,6 +1539,7 @@ DEFINE_EVENT(xrep_extent_class, name, \ DEFINE_REPAIR_EXTENT_EVENT(xreap_dispose_unmap_extent); DEFINE_REPAIR_EXTENT_EVENT(xreap_dispose_free_extent); DEFINE_REPAIR_EXTENT_EVENT(xreap_agextent_binval); +DEFINE_REPAIR_EXTENT_EVENT(xreap_bmapi_binval); DEFINE_REPAIR_EXTENT_EVENT(xrep_agfl_insert); DECLARE_EVENT_CLASS(xrep_reap_find_class, @@ -1572,6 +1573,7 @@ DEFINE_EVENT(xrep_reap_find_class, name, \ bool crosslinked), \ TP_ARGS(pag, agbno, len, crosslinked)) DEFINE_REPAIR_REAP_FIND_EVENT(xreap_agextent_select); +DEFINE_REPAIR_REAP_FIND_EVENT(xreap_bmapi_select); DECLARE_EVENT_CLASS(xrep_rmap_class, TP_PROTO(struct xfs_mount *mp, xfs_agnumber_t agno, @@ -2312,6 +2314,67 @@ TRACE_EVENT(xrep_tempfile_create, __entry->temp_inum) ); +TRACE_EVENT(xreap_ifork_extent, + TP_PROTO(struct xfs_scrub *sc, struct xfs_inode *ip, int whichfork, + const struct xfs_bmbt_irec *irec), + TP_ARGS(sc, ip, whichfork, irec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(int, whichfork) + __field(xfs_fileoff_t, fileoff) + __field(xfs_filblks_t, len) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(int, state) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->ino = ip->i_ino; + __entry->whichfork = whichfork; + __entry->fileoff = irec->br_startoff; + __entry->len = irec->br_blockcount; + __entry->agno = XFS_FSB_TO_AGNO(sc->mp, irec->br_startblock); + __entry->agbno = XFS_FSB_TO_AGBNO(sc->mp, irec->br_startblock); + __entry->state = irec->br_state; + ), + TP_printk("dev %d:%d ip 0x%llx whichfork %s agno 0x%x agbno 0x%x fileoff 0x%llx fsbcount 0x%llx state 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->whichfork, XFS_WHICHFORK_STRINGS), + __entry->agno, + __entry->agbno, + __entry->fileoff, + __entry->len, + __entry->state) +); + +TRACE_EVENT(xreap_bmapi_binval_scan, + TP_PROTO(struct xfs_scrub *sc, const struct xfs_bmbt_irec *irec, + xfs_extlen_t scan_blocks), + TP_ARGS(sc, irec, scan_blocks), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_filblks_t, len) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(xfs_extlen_t, scan_blocks) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->len = irec->br_blockcount; + __entry->agno = XFS_FSB_TO_AGNO(sc->mp, irec->br_startblock); + __entry->agbno = XFS_FSB_TO_AGBNO(sc->mp, irec->br_startblock); + __entry->scan_blocks = scan_blocks; + ), + TP_printk("dev %d:%d agno 0x%x agbno 0x%x fsbcount 0x%llx scan_blocks 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agbno, + __entry->len, + __entry->scan_blocks) +); + #endif /* IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) */ #endif /* _TRACE_XFS_SCRUB_TRACE_H */ From abf039e2e4afde98e448253f9a7ecc784a87924d Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:33 -0700 Subject: [PATCH 0223/1702] xfs: online repair of realtime summaries Repair the realtime summary data by constructing a new rtsummary file in the scrub temporary file, then atomically swapping the contents. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/scrub/common.c | 1 + fs/xfs/scrub/repair.h | 3 + fs/xfs/scrub/rtsummary.c | 33 +++--- fs/xfs/scrub/rtsummary.h | 37 +++++++ fs/xfs/scrub/rtsummary_repair.c | 177 ++++++++++++++++++++++++++++++++ fs/xfs/scrub/scrub.c | 3 +- 7 files changed, 239 insertions(+), 16 deletions(-) create mode 100644 fs/xfs/scrub/rtsummary.h create mode 100644 fs/xfs/scrub/rtsummary_repair.c diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index ae8488ab4d6b5..5e3ac7ec8fa5b 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -212,6 +212,7 @@ xfs-y += $(addprefix scrub/, \ xfs-$(CONFIG_XFS_RT) += $(addprefix scrub/, \ rtbitmap_repair.o \ + rtsummary_repair.o \ ) xfs-$(CONFIG_XFS_QUOTA) += $(addprefix scrub/, \ diff --git a/fs/xfs/scrub/common.c b/fs/xfs/scrub/common.c index a27d33b6f4641..a2da2bef509aa 100644 --- a/fs/xfs/scrub/common.c +++ b/fs/xfs/scrub/common.c @@ -31,6 +31,7 @@ #include "xfs_ag.h" #include "xfs_error.h" #include "xfs_quota.h" +#include "xfs_exchmaps.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/trace.h" diff --git a/fs/xfs/scrub/repair.h b/fs/xfs/scrub/repair.h index ce082d941459f..0e2b695ab8f66 100644 --- a/fs/xfs/scrub/repair.h +++ b/fs/xfs/scrub/repair.h @@ -126,8 +126,10 @@ int xrep_fscounters(struct xfs_scrub *sc); #ifdef CONFIG_XFS_RT int xrep_rtbitmap(struct xfs_scrub *sc); +int xrep_rtsummary(struct xfs_scrub *sc); #else # define xrep_rtbitmap xrep_notsupported +# define xrep_rtsummary xrep_notsupported #endif /* CONFIG_XFS_RT */ #ifdef CONFIG_XFS_QUOTA @@ -212,6 +214,7 @@ xrep_setup_nothing( #define xrep_quotacheck xrep_notsupported #define xrep_nlinks xrep_notsupported #define xrep_fscounters xrep_notsupported +#define xrep_rtsummary xrep_notsupported #endif /* CONFIG_XFS_ONLINE_REPAIR */ diff --git a/fs/xfs/scrub/rtsummary.c b/fs/xfs/scrub/rtsummary.c index 5055092bd9e85..3fee603f52441 100644 --- a/fs/xfs/scrub/rtsummary.c +++ b/fs/xfs/scrub/rtsummary.c @@ -17,10 +17,14 @@ #include "xfs_bit.h" #include "xfs_bmap.h" #include "xfs_sb.h" +#include "xfs_exchmaps.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/trace.h" #include "scrub/xfile.h" +#include "scrub/repair.h" +#include "scrub/tempexch.h" +#include "scrub/rtsummary.h" /* * Realtime Summary @@ -32,18 +36,6 @@ * (potentially large) amount of data in pageable memory. */ -struct xchk_rtsummary { - struct xfs_rtalloc_args args; - - uint64_t rextents; - uint64_t rbmblocks; - uint64_t rsumsize; - unsigned int rsumlevels; - - /* Memory buffer for the summary comparison. */ - union xfs_suminfo_raw words[]; -}; - /* Set us up to check the rtsummary file. */ int xchk_setup_rtsummary( @@ -60,6 +52,12 @@ xchk_setup_rtsummary( return -ENOMEM; sc->buf = rts; + if (xchk_could_repair(sc)) { + error = xrep_setup_rtsummary(sc, rts); + if (error) + return error; + } + /* * Create an xfile to construct a new rtsummary file. The xfile allows * us to avoid pinning kernel memory for this purpose. @@ -70,7 +68,7 @@ xchk_setup_rtsummary( if (error) return error; - error = xchk_trans_alloc(sc, 0); + error = xchk_trans_alloc(sc, rts->resblks); if (error) return error; @@ -135,7 +133,7 @@ xfsum_store( sumoff << XFS_WORDLOG); } -static inline int +inline int xfsum_copyout( struct xfs_scrub *sc, xfs_rtsumoff_t sumoff, @@ -362,7 +360,12 @@ xchk_rtsummary( error = xchk_rtsum_compare(sc); out_rbm: - /* Unlock the rtbitmap since we're done with it. */ + /* + * Unlock the rtbitmap since we're done with it. All other writers of + * the rt free space metadata grab the bitmap and summary ILOCKs in + * that order, so we're still protected against allocation activities + * even if we continue on to the repair function. + */ xfs_iunlock(mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP); return error; } diff --git a/fs/xfs/scrub/rtsummary.h b/fs/xfs/scrub/rtsummary.h new file mode 100644 index 0000000000000..e1d50304d8d48 --- /dev/null +++ b/fs/xfs/scrub/rtsummary.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_RTSUMMARY_H__ +#define __XFS_SCRUB_RTSUMMARY_H__ + +struct xchk_rtsummary { +#ifdef CONFIG_XFS_ONLINE_REPAIR + struct xrep_tempexch tempexch; +#endif + struct xfs_rtalloc_args args; + + uint64_t rextents; + uint64_t rbmblocks; + uint64_t rsumsize; + unsigned int rsumlevels; + unsigned int resblks; + + /* suminfo position of xfile as we write buffers to disk. */ + xfs_rtsumoff_t prep_wordoff; + + /* Memory buffer for the summary comparison. */ + union xfs_suminfo_raw words[]; +}; + +int xfsum_copyout(struct xfs_scrub *sc, xfs_rtsumoff_t sumoff, + union xfs_suminfo_raw *rawinfo, unsigned int nr_words); + +#ifdef CONFIG_XFS_ONLINE_REPAIR +int xrep_setup_rtsummary(struct xfs_scrub *sc, struct xchk_rtsummary *rts); +#else +# define xrep_setup_rtsummary(sc, rts) (0) +#endif /* CONFIG_XFS_ONLINE_REPAIR */ + +#endif /* __XFS_SCRUB_RTSUMMARY_H__ */ diff --git a/fs/xfs/scrub/rtsummary_repair.c b/fs/xfs/scrub/rtsummary_repair.c new file mode 100644 index 0000000000000..c8bb6c4f15d05 --- /dev/null +++ b/fs/xfs/scrub/rtsummary_repair.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_rtalloc.h" +#include "xfs_inode.h" +#include "xfs_bit.h" +#include "xfs_bmap.h" +#include "xfs_bmap_btree.h" +#include "xfs_exchmaps.h" +#include "xfs_rtbitmap.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/tempfile.h" +#include "scrub/tempexch.h" +#include "scrub/reap.h" +#include "scrub/xfile.h" +#include "scrub/rtsummary.h" + +/* Set us up to repair the rtsummary file. */ +int +xrep_setup_rtsummary( + struct xfs_scrub *sc, + struct xchk_rtsummary *rts) +{ + struct xfs_mount *mp = sc->mp; + unsigned long long blocks; + int error; + + error = xrep_tempfile_create(sc, S_IFREG); + if (error) + return error; + + /* + * If we're doing a repair, we reserve enough blocks to write out a + * completely new summary file, plus twice as many blocks as we would + * need if we can only allocate one block per data fork mapping. This + * should cover the preallocation of the temporary file and exchanging + * the extent mappings. + * + * We cannot use xfs_exchmaps_estimate because we have not yet + * constructed the replacement rtsummary and therefore do not know how + * many extents it will use. By the time we do, we will have a dirty + * transaction (which we cannot drop because we cannot drop the + * rtsummary ILOCK) and cannot ask for more reservation. + */ + blocks = XFS_B_TO_FSB(mp, mp->m_rsumsize); + blocks += xfs_bmbt_calc_size(mp, blocks) * 2; + if (blocks > UINT_MAX) + return -EOPNOTSUPP; + + rts->resblks += blocks; + + /* + * Grab support for atomic file content exchanges before we allocate + * any transactions or grab ILOCKs. + */ + return xrep_tempexch_enable(sc); +} + +static int +xrep_rtsummary_prep_buf( + struct xfs_scrub *sc, + struct xfs_buf *bp, + void *data) +{ + struct xchk_rtsummary *rts = data; + struct xfs_mount *mp = sc->mp; + union xfs_suminfo_raw *ondisk; + int error; + + rts->args.mp = sc->mp; + rts->args.tp = sc->tp; + rts->args.sumbp = bp; + ondisk = xfs_rsumblock_infoptr(&rts->args, 0); + rts->args.sumbp = NULL; + + bp->b_ops = &xfs_rtbuf_ops; + + error = xfsum_copyout(sc, rts->prep_wordoff, ondisk, mp->m_blockwsize); + if (error) + return error; + + rts->prep_wordoff += mp->m_blockwsize; + xfs_trans_buf_set_type(sc->tp, bp, XFS_BLFT_RTSUMMARY_BUF); + return 0; +} + +/* Repair the realtime summary. */ +int +xrep_rtsummary( + struct xfs_scrub *sc) +{ + struct xchk_rtsummary *rts = sc->buf; + struct xfs_mount *mp = sc->mp; + xfs_filblks_t rsumblocks; + int error; + + /* We require the rmapbt to rebuild anything. */ + if (!xfs_has_rmapbt(mp)) + return -EOPNOTSUPP; + + /* Walk away if we disagree on the size of the rt bitmap. */ + if (rts->rbmblocks != mp->m_sb.sb_rbmblocks) + return 0; + + /* Make sure any problems with the fork are fixed. */ + error = xrep_metadata_inode_forks(sc); + if (error) + return error; + + /* + * Try to take ILOCK_EXCL of the temporary file. We had better be the + * only ones holding onto this inode, but we can't block while holding + * the rtsummary file's ILOCK_EXCL. + */ + while (!xrep_tempfile_ilock_nowait(sc)) { + if (xchk_should_terminate(sc, &error)) + return error; + delay(1); + } + + /* Make sure we have space allocated for the entire summary file. */ + rsumblocks = XFS_B_TO_FSB(mp, rts->rsumsize); + xfs_trans_ijoin(sc->tp, sc->ip, 0); + xfs_trans_ijoin(sc->tp, sc->tempip, 0); + error = xrep_tempfile_prealloc(sc, 0, rsumblocks); + if (error) + return error; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + return error; + + /* Copy the rtsummary file that we generated. */ + error = xrep_tempfile_copyin(sc, 0, rsumblocks, + xrep_rtsummary_prep_buf, rts); + if (error) + return error; + error = xrep_tempfile_set_isize(sc, rts->rsumsize); + if (error) + return error; + + /* + * Now exchange the contents. Nothing in repair uses the temporary + * buffer, so we can reuse it for the tempfile exchrange information. + */ + error = xrep_tempexch_trans_reserve(sc, XFS_DATA_FORK, &rts->tempexch); + if (error) + return error; + + error = xrep_tempexch_contents(sc, &rts->tempexch); + if (error) + return error; + + /* Reset incore state and blow out the summary cache. */ + if (mp->m_rsum_cache) + memset(mp->m_rsum_cache, 0xFF, mp->m_sb.sb_rbmblocks); + + mp->m_rsumlevels = rts->rsumlevels; + mp->m_rsumsize = rts->rsumsize; + + /* Free the old rtsummary blocks if they're not in use. */ + return xrep_reap_ifork(sc, sc->tempip, XFS_DATA_FORK); +} diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index ff156edf49a08..62a064c1a5d34 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -18,6 +18,7 @@ #include "xfs_buf_mem.h" #include "xfs_rmap.h" #include "xfs_exchrange.h" +#include "xfs_exchmaps.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/trace.h" @@ -354,7 +355,7 @@ static const struct xchk_meta_ops meta_scrub_ops[] = { .type = ST_FS, .setup = xchk_setup_rtsummary, .scrub = xchk_rtsummary, - .repair = xrep_notsupported, + .repair = xrep_rtsummary, }, [XFS_SCRUB_TYPE_UQUOTA] = { /* user quota */ .type = ST_FS, From 9eef772f3a194f6841850e45dacdf4207ec7da84 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:34 -0700 Subject: [PATCH 0224/1702] xfs: add an explicit owner field to xfs_da_args Add an explicit owner field to xfs_da_args, which will make it easier for online fsck to set the owner field of the temporary directory and xattr structures that it builds to repair damaged metadata. Note: I hopefully found all the xfs_da_args definitions by looking for automatic stack variable declarations and xfs_da_args.dp assignments: git grep -E '(args.*dp =|struct xfs_da_args[[:space:]]*[a-z0-9][a-z0-9]*)' Note that callers of xfs_attr_{get,set,change} can set the owner to zero (or leave it unset) to have the default set to args->dp. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 4 ++++ fs/xfs/libxfs/xfs_attr_leaf.c | 2 ++ fs/xfs/libxfs/xfs_bmap.c | 1 + fs/xfs/libxfs/xfs_da_btree.h | 1 + fs/xfs/libxfs/xfs_dir2.c | 5 +++++ fs/xfs/libxfs/xfs_exchmaps.c | 2 ++ fs/xfs/scrub/attr.c | 1 + fs/xfs/scrub/dabtree.c | 1 + fs/xfs/scrub/dir.c | 3 ++- fs/xfs/scrub/readdir.c | 2 ++ fs/xfs/xfs_attr_item.c | 1 + fs/xfs/xfs_dir2_readdir.c | 1 + fs/xfs/xfs_trace.h | 7 +++++-- 13 files changed, 28 insertions(+), 3 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 673a4b6d2e8d1..74d7694614434 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -264,6 +264,8 @@ xfs_attr_get( if (xfs_is_shutdown(args->dp->i_mount)) return -EIO; + if (!args->owner) + args->owner = args->dp->i_ino; args->geo = args->dp->i_mount->m_attr_geo; args->whichfork = XFS_ATTR_FORK; args->hashval = xfs_da_hashname(args->name, args->namelen); @@ -937,6 +939,8 @@ xfs_attr_set( if (error) return error; + if (!args->owner) + args->owner = args->dp->i_ino; args->geo = mp->m_attr_geo; args->whichfork = XFS_ATTR_FORK; args->hashval = xfs_da_hashname(args->name, args->namelen); diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index ac904cc1a97ba..e606eae8d3770 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -904,6 +904,7 @@ xfs_attr_shortform_to_leaf( nargs.whichfork = XFS_ATTR_FORK; nargs.trans = args->trans; nargs.op_flags = XFS_DA_OP_OKNOENT; + nargs.owner = args->owner; sfe = xfs_attr_sf_firstentry(sf); for (i = 0; i < sf->count; i++) { @@ -1106,6 +1107,7 @@ xfs_attr3_leaf_to_shortform( nargs.whichfork = XFS_ATTR_FORK; nargs.trans = args->trans; nargs.op_flags = XFS_DA_OP_OKNOENT; + nargs.owner = args->owner; for (i = 0; i < ichdr.count; entry++, i++) { if (entry->flags & XFS_ATTR_INCOMPLETE) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 656c95a22f2e6..46bbc9f0a1173 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -976,6 +976,7 @@ xfs_bmap_add_attrfork_local( dargs.total = dargs.geo->fsbcount; dargs.whichfork = XFS_DATA_FORK; dargs.trans = tp; + dargs.owner = ip->i_ino; return xfs_dir2_sf_to_block(&dargs); } diff --git a/fs/xfs/libxfs/xfs_da_btree.h b/fs/xfs/libxfs/xfs_da_btree.h index 706baf36e1751..7fb13f26edaa7 100644 --- a/fs/xfs/libxfs/xfs_da_btree.h +++ b/fs/xfs/libxfs/xfs_da_btree.h @@ -79,6 +79,7 @@ typedef struct xfs_da_args { int rmtvaluelen2; /* remote attr value length in bytes */ uint32_t op_flags; /* operation flags */ enum xfs_dacmp cmpresult; /* name compare result for lookups */ + xfs_ino_t owner; /* inode that owns the dir/attr data */ } xfs_da_args_t; /* diff --git a/fs/xfs/libxfs/xfs_dir2.c b/fs/xfs/libxfs/xfs_dir2.c index 4821519efad4b..9da99fa20c759 100644 --- a/fs/xfs/libxfs/xfs_dir2.c +++ b/fs/xfs/libxfs/xfs_dir2.c @@ -250,6 +250,7 @@ xfs_dir_init( args->geo = dp->i_mount->m_dir_geo; args->dp = dp; args->trans = tp; + args->owner = dp->i_ino; error = xfs_dir2_sf_create(args, pdp->i_ino); kfree(args); return error; @@ -295,6 +296,7 @@ xfs_dir_createname( args->whichfork = XFS_DATA_FORK; args->trans = tp; args->op_flags = XFS_DA_OP_ADDNAME | XFS_DA_OP_OKNOENT; + args->owner = dp->i_ino; if (!inum) args->op_flags |= XFS_DA_OP_JUSTCHECK; @@ -383,6 +385,7 @@ xfs_dir_lookup( args->whichfork = XFS_DATA_FORK; args->trans = tp; args->op_flags = XFS_DA_OP_OKNOENT; + args->owner = dp->i_ino; if (ci_name) args->op_flags |= XFS_DA_OP_CILOOKUP; @@ -456,6 +459,7 @@ xfs_dir_removename( args->total = total; args->whichfork = XFS_DATA_FORK; args->trans = tp; + args->owner = dp->i_ino; if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) { rval = xfs_dir2_sf_removename(args); @@ -517,6 +521,7 @@ xfs_dir_replace( args->total = total; args->whichfork = XFS_DATA_FORK; args->trans = tp; + args->owner = dp->i_ino; if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) { rval = xfs_dir2_sf_replace(args); diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index 7fa244228750a..8d28e8cce5e93 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -429,6 +429,7 @@ xfs_exchmaps_attr_to_sf( .geo = tp->t_mountp->m_attr_geo, .whichfork = XFS_ATTR_FORK, .trans = tp, + .owner = xmi->xmi_ip2->i_ino, }; struct xfs_buf *bp; int forkoff; @@ -459,6 +460,7 @@ xfs_exchmaps_dir_to_sf( .geo = tp->t_mountp->m_dir_geo, .whichfork = XFS_DATA_FORK, .trans = tp, + .owner = xmi->xmi_ip2->i_ino, }; struct xfs_dir2_sf_hdr sfh; struct xfs_buf *bp; diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index 83c7feb387147..0c467f4f8e778 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -169,6 +169,7 @@ xchk_xattr_listent( .hashval = xfs_da_hashname(name, namelen), .trans = context->tp, .valuelen = valuelen, + .owner = context->dp->i_ino, }; struct xchk_xattr_buf *ab; struct xchk_xattr *sx; diff --git a/fs/xfs/scrub/dabtree.c b/fs/xfs/scrub/dabtree.c index 82b150d3b8b70..fa6385a99ac4e 100644 --- a/fs/xfs/scrub/dabtree.c +++ b/fs/xfs/scrub/dabtree.c @@ -494,6 +494,7 @@ xchk_da_btree( ds->dargs.whichfork = whichfork; ds->dargs.trans = sc->tp; ds->dargs.op_flags = XFS_DA_OP_OKNOENT; + ds->dargs.owner = sc->ip->i_ino; ds->state = xfs_da_state_alloc(&ds->dargs); ds->sc = sc; ds->private = private; diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index 076a310b8eb00..042e28547e044 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -621,10 +621,11 @@ xchk_directory_blocks( { struct xfs_bmbt_irec got; struct xfs_da_args args = { - .dp = sc ->ip, + .dp = sc->ip, .whichfork = XFS_DATA_FORK, .geo = sc->mp->m_dir_geo, .trans = sc->tp, + .owner = sc->ip->i_ino, }; struct xfs_ifork *ifp = xfs_ifork_ptr(sc->ip, XFS_DATA_FORK); struct xfs_mount *mp = sc->mp; diff --git a/fs/xfs/scrub/readdir.c b/fs/xfs/scrub/readdir.c index dfdcb96b6c169..fb98b7624994a 100644 --- a/fs/xfs/scrub/readdir.c +++ b/fs/xfs/scrub/readdir.c @@ -273,6 +273,7 @@ xchk_dir_walk( .dp = dp, .geo = dp->i_mount->m_dir_geo, .trans = sc->tp, + .owner = dp->i_ino, }; bool isblock; int error; @@ -324,6 +325,7 @@ xchk_dir_lookup( .hashval = xfs_dir2_hashname(dp->i_mount, name), .whichfork = XFS_DATA_FORK, .op_flags = XFS_DA_OP_OKNOENT, + .owner = dp->i_ino, }; bool isblock, isleaf; int error; diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index 9b4c61e1c22e8..d460347056945 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -540,6 +540,7 @@ xfs_attri_recover_work( args->attr_filter = attrp->alfi_attr_filter & XFS_ATTRI_FILTER_MASK; args->op_flags = XFS_DA_OP_RECOVERY | XFS_DA_OP_OKNOENT | XFS_DA_OP_LOGGED; + args->owner = args->dp->i_ino; ASSERT(xfs_sb_version_haslogxattrs(&mp->m_sb)); diff --git a/fs/xfs/xfs_dir2_readdir.c b/fs/xfs/xfs_dir2_readdir.c index cf9296b7e06ff..4e811fa393add 100644 --- a/fs/xfs/xfs_dir2_readdir.c +++ b/fs/xfs/xfs_dir2_readdir.c @@ -532,6 +532,7 @@ xfs_readdir( args.dp = dp; args.geo = dp->i_mount->m_dir_geo; args.trans = tp; + args.owner = dp->i_ino; if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) return xfs_dir2_sf_getdents(&args, ctx); diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index caef95f2c87cb..939baf08331b5 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -1931,6 +1931,7 @@ DECLARE_EVENT_CLASS(xfs_da_class, __field(xfs_dahash_t, hashval) __field(xfs_ino_t, inumber) __field(uint32_t, op_flags) + __field(xfs_ino_t, owner) ), TP_fast_assign( __entry->dev = VFS_I(args->dp)->i_sb->s_dev; @@ -1941,9 +1942,10 @@ DECLARE_EVENT_CLASS(xfs_da_class, __entry->hashval = args->hashval; __entry->inumber = args->inumber; __entry->op_flags = args->op_flags; + __entry->owner = args->owner; ), TP_printk("dev %d:%d ino 0x%llx name %.*s namelen %d hashval 0x%x " - "inumber 0x%llx op_flags %s", + "inumber 0x%llx op_flags %s owner 0x%llx", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, __entry->namelen, @@ -1951,7 +1953,8 @@ DECLARE_EVENT_CLASS(xfs_da_class, __entry->namelen, __entry->hashval, __entry->inumber, - __print_flags(__entry->op_flags, "|", XFS_DA_OP_FLAGS)) + __print_flags(__entry->op_flags, "|", XFS_DA_OP_FLAGS), + __entry->owner) ) #define DEFINE_DIR2_EVENT(name) \ From 17a85dc64ae0804d33a2293686fc987a830a462d Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:34 -0700 Subject: [PATCH 0225/1702] xfs: use the xfs_da_args owner field to set new dir/attr block owner When we're creating leaf, data, freespace, or dabtree blocks for directories and xattrs, use the explicit owner field (instead of the xfs_inode) to set the owner field. This will enable online repair to construct replacement data structures in a temporary file without having to change the owner fields prior to swapping the new and old structures. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr_leaf.c | 2 +- fs/xfs/libxfs/xfs_attr_remote.c | 4 ++-- fs/xfs/libxfs/xfs_da_btree.c | 2 +- fs/xfs/libxfs/xfs_dir2_block.c | 19 ++++++++++--------- fs/xfs/libxfs/xfs_dir2_data.c | 2 +- fs/xfs/libxfs/xfs_dir2_leaf.c | 11 +++++------ fs/xfs/libxfs/xfs_dir2_node.c | 2 +- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index e606eae8d3770..8937c034b3309 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -1239,7 +1239,7 @@ xfs_attr3_leaf_create( ichdr.magic = XFS_ATTR3_LEAF_MAGIC; hdr3->blkno = cpu_to_be64(xfs_buf_daddr(bp)); - hdr3->owner = cpu_to_be64(dp->i_ino); + hdr3->owner = cpu_to_be64(args->owner); uuid_copy(&hdr3->uuid, &mp->m_sb.sb_meta_uuid); ichdr.freemap[0].base = sizeof(struct xfs_attr3_leaf_hdr); diff --git a/fs/xfs/libxfs/xfs_attr_remote.c b/fs/xfs/libxfs/xfs_attr_remote.c index ff04128287720..024895cc70299 100644 --- a/fs/xfs/libxfs/xfs_attr_remote.c +++ b/fs/xfs/libxfs/xfs_attr_remote.c @@ -522,8 +522,8 @@ xfs_attr_rmtval_set_value( return error; bp->b_ops = &xfs_attr3_rmt_buf_ops; - xfs_attr_rmtval_copyin(mp, bp, args->dp->i_ino, &offset, - &valuelen, &src); + xfs_attr_rmtval_copyin(mp, bp, args->owner, &offset, &valuelen, + &src); error = xfs_bwrite(bp); /* GROT: NOTE: synchronous write */ xfs_buf_relse(bp); diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c index 718d071bb21ae..743f6421cc04f 100644 --- a/fs/xfs/libxfs/xfs_da_btree.c +++ b/fs/xfs/libxfs/xfs_da_btree.c @@ -486,7 +486,7 @@ xfs_da3_node_create( memset(hdr3, 0, sizeof(struct xfs_da3_node_hdr)); ichdr.magic = XFS_DA3_NODE_MAGIC; hdr3->info.blkno = cpu_to_be64(xfs_buf_daddr(bp)); - hdr3->info.owner = cpu_to_be64(args->dp->i_ino); + hdr3->info.owner = cpu_to_be64(args->owner); uuid_copy(&hdr3->info.uuid, &mp->m_sb.sb_meta_uuid); } else { ichdr.magic = XFS_DA_NODE_MAGIC; diff --git a/fs/xfs/libxfs/xfs_dir2_block.c b/fs/xfs/libxfs/xfs_dir2_block.c index a2da007adb462..61cbc668f228a 100644 --- a/fs/xfs/libxfs/xfs_dir2_block.c +++ b/fs/xfs/libxfs/xfs_dir2_block.c @@ -163,12 +163,13 @@ xfs_dir3_block_read( static void xfs_dir3_block_init( - struct xfs_mount *mp, - struct xfs_trans *tp, - struct xfs_buf *bp, - struct xfs_inode *dp) + struct xfs_da_args *args, + struct xfs_buf *bp) { - struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr; + struct xfs_trans *tp = args->trans; + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr; bp->b_ops = &xfs_dir3_block_buf_ops; xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DIR_BLOCK_BUF); @@ -177,7 +178,7 @@ xfs_dir3_block_init( memset(hdr3, 0, sizeof(*hdr3)); hdr3->magic = cpu_to_be32(XFS_DIR3_BLOCK_MAGIC); hdr3->blkno = cpu_to_be64(xfs_buf_daddr(bp)); - hdr3->owner = cpu_to_be64(dp->i_ino); + hdr3->owner = cpu_to_be64(args->owner); uuid_copy(&hdr3->uuid, &mp->m_sb.sb_meta_uuid); return; @@ -1009,7 +1010,7 @@ xfs_dir2_leaf_to_block( /* * Start converting it to block form. */ - xfs_dir3_block_init(mp, tp, dbp, dp); + xfs_dir3_block_init(args, dbp); needlog = 1; needscan = 0; @@ -1129,7 +1130,7 @@ xfs_dir2_sf_to_block( error = xfs_dir3_data_init(args, blkno, &bp); if (error) goto out_free; - xfs_dir3_block_init(mp, tp, bp, dp); + xfs_dir3_block_init(args, bp); hdr = bp->b_addr; /* @@ -1169,7 +1170,7 @@ xfs_dir2_sf_to_block( * Create entry for . */ dep = bp->b_addr + offset; - dep->inumber = cpu_to_be64(dp->i_ino); + dep->inumber = cpu_to_be64(args->owner); dep->namelen = 1; dep->name[0] = '.'; xfs_dir2_data_put_ftype(mp, dep, XFS_DIR3_FT_DIR); diff --git a/fs/xfs/libxfs/xfs_dir2_data.c b/fs/xfs/libxfs/xfs_dir2_data.c index 7a6d965bea71b..c3ef720b5ff6e 100644 --- a/fs/xfs/libxfs/xfs_dir2_data.c +++ b/fs/xfs/libxfs/xfs_dir2_data.c @@ -725,7 +725,7 @@ xfs_dir3_data_init( memset(hdr3, 0, sizeof(*hdr3)); hdr3->magic = cpu_to_be32(XFS_DIR3_DATA_MAGIC); hdr3->blkno = cpu_to_be64(xfs_buf_daddr(bp)); - hdr3->owner = cpu_to_be64(dp->i_ino); + hdr3->owner = cpu_to_be64(args->owner); uuid_copy(&hdr3->uuid, &mp->m_sb.sb_meta_uuid); } else diff --git a/fs/xfs/libxfs/xfs_dir2_leaf.c b/fs/xfs/libxfs/xfs_dir2_leaf.c index 08dda5ce9d91c..20ce057d12e82 100644 --- a/fs/xfs/libxfs/xfs_dir2_leaf.c +++ b/fs/xfs/libxfs/xfs_dir2_leaf.c @@ -304,12 +304,12 @@ xfs_dir3_leafn_read( */ static void xfs_dir3_leaf_init( - struct xfs_mount *mp, - struct xfs_trans *tp, + struct xfs_da_args *args, struct xfs_buf *bp, - xfs_ino_t owner, uint16_t type) { + struct xfs_mount *mp = args->dp->i_mount; + struct xfs_trans *tp = args->trans; struct xfs_dir2_leaf *leaf = bp->b_addr; ASSERT(type == XFS_DIR2_LEAF1_MAGIC || type == XFS_DIR2_LEAFN_MAGIC); @@ -323,7 +323,7 @@ xfs_dir3_leaf_init( ? cpu_to_be16(XFS_DIR3_LEAF1_MAGIC) : cpu_to_be16(XFS_DIR3_LEAFN_MAGIC); leaf3->info.blkno = cpu_to_be64(xfs_buf_daddr(bp)); - leaf3->info.owner = cpu_to_be64(owner); + leaf3->info.owner = cpu_to_be64(args->owner); uuid_copy(&leaf3->info.uuid, &mp->m_sb.sb_meta_uuid); } else { memset(leaf, 0, sizeof(*leaf)); @@ -356,7 +356,6 @@ xfs_dir3_leaf_get_buf( { struct xfs_inode *dp = args->dp; struct xfs_trans *tp = args->trans; - struct xfs_mount *mp = dp->i_mount; struct xfs_buf *bp; int error; @@ -369,7 +368,7 @@ xfs_dir3_leaf_get_buf( if (error) return error; - xfs_dir3_leaf_init(mp, tp, bp, dp->i_ino, magic); + xfs_dir3_leaf_init(args, bp, magic); xfs_dir3_leaf_log_header(args, bp); if (magic == XFS_DIR2_LEAF1_MAGIC) xfs_dir3_leaf_log_tail(args, bp); diff --git a/fs/xfs/libxfs/xfs_dir2_node.c b/fs/xfs/libxfs/xfs_dir2_node.c index be0b8834028c0..1ad7405f9c389 100644 --- a/fs/xfs/libxfs/xfs_dir2_node.c +++ b/fs/xfs/libxfs/xfs_dir2_node.c @@ -349,7 +349,7 @@ xfs_dir3_free_get_buf( hdr.magic = XFS_DIR3_FREE_MAGIC; hdr3->hdr.blkno = cpu_to_be64(xfs_buf_daddr(bp)); - hdr3->hdr.owner = cpu_to_be64(dp->i_ino); + hdr3->hdr.owner = cpu_to_be64(args->owner); uuid_copy(&hdr3->hdr.uuid, &mp->m_sb.sb_meta_uuid); } else hdr.magic = XFS_DIR2_FREE_MAGIC; From 33c028ffe36ad7a91930acf0bd3d6ee7340022bf Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:35 -0700 Subject: [PATCH 0226/1702] xfs: reduce indenting in xfs_attr_node_list Reduce the indentation here so that we can add some things in the next patch without going over the column limits. Suggested-by: Christoph Hellwig Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_attr_list.c | 56 ++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/fs/xfs/xfs_attr_list.c b/fs/xfs/xfs_attr_list.c index a6819a642cc07..42a575db72678 100644 --- a/fs/xfs/xfs_attr_list.c +++ b/fs/xfs/xfs_attr_list.c @@ -310,46 +310,47 @@ xfs_attr_node_list( */ bp = NULL; if (cursor->blkno > 0) { + struct xfs_attr_leaf_entry *entries; + error = xfs_da3_node_read(context->tp, dp, cursor->blkno, &bp, XFS_ATTR_FORK); if (xfs_metadata_is_sick(error)) xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); - if ((error != 0) && (error != -EFSCORRUPTED)) + if (error != 0 && error != -EFSCORRUPTED) return error; - if (bp) { - struct xfs_attr_leaf_entry *entries; + if (!bp) + goto need_lookup; - node = bp->b_addr; - switch (be16_to_cpu(node->hdr.info.magic)) { - case XFS_DA_NODE_MAGIC: - case XFS_DA3_NODE_MAGIC: + node = bp->b_addr; + switch (be16_to_cpu(node->hdr.info.magic)) { + case XFS_DA_NODE_MAGIC: + case XFS_DA3_NODE_MAGIC: + trace_xfs_attr_list_wrong_blk(context); + xfs_trans_brelse(context->tp, bp); + bp = NULL; + break; + case XFS_ATTR_LEAF_MAGIC: + case XFS_ATTR3_LEAF_MAGIC: + leaf = bp->b_addr; + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, + &leafhdr, leaf); + entries = xfs_attr3_leaf_entryp(leaf); + if (cursor->hashval > be32_to_cpu( + entries[leafhdr.count - 1].hashval)) { trace_xfs_attr_list_wrong_blk(context); xfs_trans_brelse(context->tp, bp); bp = NULL; - break; - case XFS_ATTR_LEAF_MAGIC: - case XFS_ATTR3_LEAF_MAGIC: - leaf = bp->b_addr; - xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, - &leafhdr, leaf); - entries = xfs_attr3_leaf_entryp(leaf); - if (cursor->hashval > be32_to_cpu( - entries[leafhdr.count - 1].hashval)) { - trace_xfs_attr_list_wrong_blk(context); - xfs_trans_brelse(context->tp, bp); - bp = NULL; - } else if (cursor->hashval <= be32_to_cpu( - entries[0].hashval)) { - trace_xfs_attr_list_wrong_blk(context); - xfs_trans_brelse(context->tp, bp); - bp = NULL; - } - break; - default: + } else if (cursor->hashval <= be32_to_cpu( + entries[0].hashval)) { trace_xfs_attr_list_wrong_blk(context); xfs_trans_brelse(context->tp, bp); bp = NULL; } + break; + default: + trace_xfs_attr_list_wrong_blk(context); + xfs_trans_brelse(context->tp, bp); + bp = NULL; } } @@ -359,6 +360,7 @@ xfs_attr_node_list( * Note that start of node block is same as start of leaf block. */ if (bp == NULL) { +need_lookup: error = xfs_attr_node_list_lookup(context, cursor, &bp); if (error || !bp) return error; From f4887fbc41dcb1560ec5da982ac7c6ad04b71de5 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:36 -0700 Subject: [PATCH 0227/1702] xfs: validate attr leaf buffer owners Create a leaf block header checking function to validate the owner field of xattr leaf blocks. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 10 +++---- fs/xfs/libxfs/xfs_attr_leaf.c | 56 +++++++++++++++++++++++++++++------ fs/xfs/libxfs/xfs_attr_leaf.h | 4 ++- fs/xfs/libxfs/xfs_da_btree.c | 42 ++++++++++++++++++++++++++ fs/xfs/libxfs/xfs_da_btree.h | 1 + fs/xfs/libxfs/xfs_exchmaps.c | 3 +- fs/xfs/scrub/dabtree.c | 7 +++++ fs/xfs/xfs_attr_list.c | 24 +++++++++++++-- 8 files changed, 128 insertions(+), 19 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 74d7694614434..b3c9666cd0115 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -649,8 +649,8 @@ xfs_attr_leaf_remove_attr( int forkoff; int error; - error = xfs_attr3_leaf_read(args->trans, args->dp, args->blkno, - &bp); + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, + args->blkno, &bp); if (error) return error; @@ -681,7 +681,7 @@ xfs_attr_leaf_shrink( if (!xfs_attr_is_leaf(dp)) return 0; - error = xfs_attr3_leaf_read(args->trans, args->dp, 0, &bp); + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, &bp); if (error) return error; @@ -1158,7 +1158,7 @@ xfs_attr_leaf_try_add( struct xfs_buf *bp; int error; - error = xfs_attr3_leaf_read(args->trans, args->dp, 0, &bp); + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, &bp); if (error) return error; @@ -1206,7 +1206,7 @@ xfs_attr_leaf_hasname( { int error = 0; - error = xfs_attr3_leaf_read(args->trans, args->dp, 0, bp); + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, bp); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 8937c034b3309..17ec5ff5a4e3e 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -388,6 +388,27 @@ xfs_attr3_leaf_verify( return NULL; } +xfs_failaddr_t +xfs_attr3_leaf_header_check( + struct xfs_buf *bp, + xfs_ino_t owner) +{ + struct xfs_mount *mp = bp->b_mount; + + if (xfs_has_crc(mp)) { + struct xfs_attr3_leafblock *hdr3 = bp->b_addr; + + if (hdr3->hdr.info.hdr.magic != + cpu_to_be16(XFS_ATTR3_LEAF_MAGIC)) + return __this_address; + + if (be64_to_cpu(hdr3->hdr.info.owner) != owner) + return __this_address; + } + + return NULL; +} + static void xfs_attr3_leaf_write_verify( struct xfs_buf *bp) @@ -448,16 +469,30 @@ int xfs_attr3_leaf_read( struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, xfs_dablk_t bno, struct xfs_buf **bpp) { + xfs_failaddr_t fa; int err; err = xfs_da_read_buf(tp, dp, bno, 0, bpp, XFS_ATTR_FORK, &xfs_attr3_leaf_buf_ops); - if (!err && tp && *bpp) + if (err || !(*bpp)) + return err; + + fa = xfs_attr3_leaf_header_check(*bpp, owner); + if (fa) { + __xfs_buf_mark_corrupt(*bpp, fa); + xfs_trans_brelse(tp, *bpp); + *bpp = NULL; + xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); + return -EFSCORRUPTED; + } + + if (tp) xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_ATTR_LEAF_BUF); - return err; + return 0; } /*======================================================================== @@ -1160,7 +1195,7 @@ xfs_attr3_leaf_to_node( error = xfs_da_grow_inode(args, &blkno); if (error) goto out; - error = xfs_attr3_leaf_read(args->trans, dp, 0, &bp1); + error = xfs_attr3_leaf_read(args->trans, dp, args->owner, 0, &bp1); if (error) goto out; @@ -1995,7 +2030,7 @@ xfs_attr3_leaf_toosmall( if (blkno == 0) continue; error = xfs_attr3_leaf_read(state->args->trans, state->args->dp, - blkno, &bp); + state->args->owner, blkno, &bp); if (error) return error; @@ -2717,7 +2752,8 @@ xfs_attr3_leaf_clearflag( /* * Set up the operation. */ - error = xfs_attr3_leaf_read(args->trans, args->dp, args->blkno, &bp); + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, + args->blkno, &bp); if (error) return error; @@ -2781,7 +2817,8 @@ xfs_attr3_leaf_setflag( /* * Set up the operation. */ - error = xfs_attr3_leaf_read(args->trans, args->dp, args->blkno, &bp); + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, + args->blkno, &bp); if (error) return error; @@ -2840,7 +2877,8 @@ xfs_attr3_leaf_flipflags( /* * Read the block containing the "old" attr */ - error = xfs_attr3_leaf_read(args->trans, args->dp, args->blkno, &bp1); + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, + args->blkno, &bp1); if (error) return error; @@ -2848,8 +2886,8 @@ xfs_attr3_leaf_flipflags( * Read the block containing the "new" attr, if it is different */ if (args->blkno2 != args->blkno) { - error = xfs_attr3_leaf_read(args->trans, args->dp, args->blkno2, - &bp2); + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, + args->blkno2, &bp2); if (error) return error; } else { diff --git a/fs/xfs/libxfs/xfs_attr_leaf.h b/fs/xfs/libxfs/xfs_attr_leaf.h index 9b9948639c0fb..bac219589896a 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.h +++ b/fs/xfs/libxfs/xfs_attr_leaf.h @@ -98,12 +98,14 @@ int xfs_attr_leaf_order(struct xfs_buf *leaf1_bp, struct xfs_buf *leaf2_bp); int xfs_attr_leaf_newentsize(struct xfs_da_args *args, int *local); int xfs_attr3_leaf_read(struct xfs_trans *tp, struct xfs_inode *dp, - xfs_dablk_t bno, struct xfs_buf **bpp); + xfs_ino_t owner, xfs_dablk_t bno, struct xfs_buf **bpp); void xfs_attr3_leaf_hdr_from_disk(struct xfs_da_geometry *geo, struct xfs_attr3_icleaf_hdr *to, struct xfs_attr_leafblock *from); void xfs_attr3_leaf_hdr_to_disk(struct xfs_da_geometry *geo, struct xfs_attr_leafblock *to, struct xfs_attr3_icleaf_hdr *from); +xfs_failaddr_t xfs_attr3_leaf_header_check(struct xfs_buf *bp, + xfs_ino_t owner); #endif /* __XFS_ATTR_LEAF_H__ */ diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c index 743f6421cc04f..65eef87751871 100644 --- a/fs/xfs/libxfs/xfs_da_btree.c +++ b/fs/xfs/libxfs/xfs_da_btree.c @@ -252,6 +252,25 @@ xfs_da3_node_verify( return NULL; } +xfs_failaddr_t +xfs_da3_header_check( + struct xfs_buf *bp, + xfs_ino_t owner) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_da_blkinfo *hdr = bp->b_addr; + + if (!xfs_has_crc(mp)) + return NULL; + + switch (hdr->magic) { + case cpu_to_be16(XFS_ATTR3_LEAF_MAGIC): + return xfs_attr3_leaf_header_check(bp, owner); + } + + return NULL; +} + static void xfs_da3_node_write_verify( struct xfs_buf *bp) @@ -1591,6 +1610,7 @@ xfs_da3_node_lookup_int( struct xfs_da_node_entry *btree; struct xfs_da3_icnode_hdr nodehdr; struct xfs_da_args *args; + xfs_failaddr_t fa; xfs_dablk_t blkno; xfs_dahash_t hashval; xfs_dahash_t btreehashval; @@ -1629,6 +1649,12 @@ xfs_da3_node_lookup_int( if (magic == XFS_ATTR_LEAF_MAGIC || magic == XFS_ATTR3_LEAF_MAGIC) { + fa = xfs_attr3_leaf_header_check(blk->bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(blk->bp, fa); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } blk->magic = XFS_ATTR_LEAF_MAGIC; blk->hashval = xfs_attr_leaf_lasthash(blk->bp, NULL); break; @@ -1996,6 +2022,7 @@ xfs_da3_path_shift( struct xfs_da_node_entry *btree; struct xfs_da3_icnode_hdr nodehdr; struct xfs_buf *bp; + xfs_failaddr_t fa; xfs_dablk_t blkno = 0; int level; int error; @@ -2087,6 +2114,12 @@ xfs_da3_path_shift( break; case XFS_ATTR_LEAF_MAGIC: case XFS_ATTR3_LEAF_MAGIC: + fa = xfs_attr3_leaf_header_check(blk->bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(blk->bp, fa); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } blk->magic = XFS_ATTR_LEAF_MAGIC; ASSERT(level == path->active-1); blk->index = 0; @@ -2290,6 +2323,7 @@ xfs_da3_swap_lastblock( struct xfs_buf *last_buf; struct xfs_buf *sib_buf; struct xfs_buf *par_buf; + xfs_failaddr_t fa; xfs_dahash_t dead_hash; xfs_fileoff_t lastoff; xfs_dablk_t dead_blkno; @@ -2326,6 +2360,14 @@ xfs_da3_swap_lastblock( error = xfs_da3_node_read(tp, dp, last_blkno, &last_buf, w); if (error) return error; + fa = xfs_da3_header_check(last_buf, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(last_buf, fa); + xfs_trans_brelse(tp, last_buf); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + /* * Copy the last block into the dead buffer and log it. */ diff --git a/fs/xfs/libxfs/xfs_da_btree.h b/fs/xfs/libxfs/xfs_da_btree.h index 7fb13f26edaa7..99618e0c8a72b 100644 --- a/fs/xfs/libxfs/xfs_da_btree.h +++ b/fs/xfs/libxfs/xfs_da_btree.h @@ -236,6 +236,7 @@ void xfs_da3_node_hdr_from_disk(struct xfs_mount *mp, struct xfs_da3_icnode_hdr *to, struct xfs_da_intnode *from); void xfs_da3_node_hdr_to_disk(struct xfs_mount *mp, struct xfs_da_intnode *to, struct xfs_da3_icnode_hdr *from); +xfs_failaddr_t xfs_da3_header_check(struct xfs_buf *bp, xfs_ino_t owner); extern struct kmem_cache *xfs_da_state_cache; diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index 8d28e8cce5e93..9c9cf2e998b25 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -438,7 +438,8 @@ xfs_exchmaps_attr_to_sf( if (!xfs_attr_is_leaf(xmi->xmi_ip2)) return 0; - error = xfs_attr3_leaf_read(tp, xmi->xmi_ip2, 0, &bp); + error = xfs_attr3_leaf_read(tp, xmi->xmi_ip2, xmi->xmi_ip2->i_ino, 0, + &bp); if (error) return error; diff --git a/fs/xfs/scrub/dabtree.c b/fs/xfs/scrub/dabtree.c index fa6385a99ac4e..c71254088dffe 100644 --- a/fs/xfs/scrub/dabtree.c +++ b/fs/xfs/scrub/dabtree.c @@ -320,6 +320,7 @@ xchk_da_btree_block( struct xfs_da3_blkinfo *hdr3; struct xfs_da_args *dargs = &ds->dargs; struct xfs_inode *ip = ds->dargs.dp; + xfs_failaddr_t fa; xfs_ino_t owner; int *pmaxrecs; struct xfs_da3_icnode_hdr nodehdr; @@ -442,6 +443,12 @@ xchk_da_btree_block( goto out_freebp; } + fa = xfs_da3_header_check(blk->bp, dargs->owner); + if (fa) { + xchk_da_set_corrupt(ds, level); + goto out_freebp; + } + /* * If we've been handed a block that is below the dabtree root, does * its hashval match what the parent block expected to see? diff --git a/fs/xfs/xfs_attr_list.c b/fs/xfs/xfs_attr_list.c index 42a575db72678..f6496e33ff91b 100644 --- a/fs/xfs/xfs_attr_list.c +++ b/fs/xfs/xfs_attr_list.c @@ -214,6 +214,7 @@ xfs_attr_node_list_lookup( struct xfs_mount *mp = dp->i_mount; struct xfs_trans *tp = context->tp; struct xfs_buf *bp; + xfs_failaddr_t fa; int i; int error = 0; unsigned int expected_level = 0; @@ -273,6 +274,12 @@ xfs_attr_node_list_lookup( } } + fa = xfs_attr3_leaf_header_check(bp, dp->i_ino); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + goto out_releasebuf; + } + if (expected_level != 0) goto out_corruptbuf; @@ -281,6 +288,7 @@ xfs_attr_node_list_lookup( out_corruptbuf: xfs_buf_mark_corrupt(bp); +out_releasebuf: xfs_trans_brelse(tp, bp); xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); return -EFSCORRUPTED; @@ -297,6 +305,7 @@ xfs_attr_node_list( struct xfs_buf *bp; struct xfs_inode *dp = context->dp; struct xfs_mount *mp = dp->i_mount; + xfs_failaddr_t fa; int error = 0; trace_xfs_attr_node_list(context); @@ -332,6 +341,14 @@ xfs_attr_node_list( case XFS_ATTR_LEAF_MAGIC: case XFS_ATTR3_LEAF_MAGIC: leaf = bp->b_addr; + fa = xfs_attr3_leaf_header_check(bp, dp->i_ino); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_trans_brelse(context->tp, bp); + xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); + bp = NULL; + break; + } xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &leafhdr, leaf); entries = xfs_attr3_leaf_entryp(leaf); @@ -382,8 +399,8 @@ xfs_attr_node_list( break; cursor->blkno = leafhdr.forw; xfs_trans_brelse(context->tp, bp); - error = xfs_attr3_leaf_read(context->tp, dp, cursor->blkno, - &bp); + error = xfs_attr3_leaf_read(context->tp, dp, dp->i_ino, + cursor->blkno, &bp); if (error) return error; } @@ -503,7 +520,8 @@ xfs_attr_leaf_list( trace_xfs_attr_leaf_list(context); context->cursor.blkno = 0; - error = xfs_attr3_leaf_read(context->tp, context->dp, 0, &bp); + error = xfs_attr3_leaf_read(context->tp, context->dp, + context->dp->i_ino, 0, &bp); if (error) return error; From 8c25dc728bd1ca9344001aa6ef4556885572baa4 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:37 -0700 Subject: [PATCH 0228/1702] xfs: validate attr remote value buffer owners Check the owner field of xattr remote value blocks. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr_remote.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr_remote.c b/fs/xfs/libxfs/xfs_attr_remote.c index 024895cc70299..a8de9dc1e998a 100644 --- a/fs/xfs/libxfs/xfs_attr_remote.c +++ b/fs/xfs/libxfs/xfs_attr_remote.c @@ -280,12 +280,12 @@ xfs_attr_rmtval_copyout( struct xfs_mount *mp, struct xfs_buf *bp, struct xfs_inode *dp, + xfs_ino_t owner, int *offset, int *valuelen, uint8_t **dst) { char *src = bp->b_addr; - xfs_ino_t ino = dp->i_ino; xfs_daddr_t bno = xfs_buf_daddr(bp); int len = BBTOB(bp->b_length); int blksize = mp->m_attr_geo->blksize; @@ -299,11 +299,11 @@ xfs_attr_rmtval_copyout( byte_cnt = min(*valuelen, byte_cnt); if (xfs_has_crc(mp)) { - if (xfs_attr3_rmt_hdr_ok(src, ino, *offset, + if (xfs_attr3_rmt_hdr_ok(src, owner, *offset, byte_cnt, bno)) { xfs_alert(mp, "remote attribute header mismatch bno/off/len/owner (0x%llx/0x%x/Ox%x/0x%llx)", - bno, *offset, byte_cnt, ino); + bno, *offset, byte_cnt, owner); xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); return -EFSCORRUPTED; } @@ -427,8 +427,7 @@ xfs_attr_rmtval_get( return error; error = xfs_attr_rmtval_copyout(mp, bp, args->dp, - &offset, &valuelen, - &dst); + args->owner, &offset, &valuelen, &dst); xfs_buf_relse(bp); if (error) return error; From d44bea9b41ca25f91fd9f25ed2cc3bb2f6dab4bc Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:38 -0700 Subject: [PATCH 0229/1702] xfs: validate dabtree node buffer owners Check the owner field of dabtree node blocks. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_da_btree.c | 109 +++++++++++++++++++++++++++++++++++ fs/xfs/libxfs/xfs_da_btree.h | 1 + fs/xfs/xfs_attr_list.c | 9 +++ 3 files changed, 119 insertions(+) diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c index 65eef87751871..e6c28bccdbc07 100644 --- a/fs/xfs/libxfs/xfs_da_btree.c +++ b/fs/xfs/libxfs/xfs_da_btree.c @@ -252,6 +252,26 @@ xfs_da3_node_verify( return NULL; } +xfs_failaddr_t +xfs_da3_node_header_check( + struct xfs_buf *bp, + xfs_ino_t owner) +{ + struct xfs_mount *mp = bp->b_mount; + + if (xfs_has_crc(mp)) { + struct xfs_da3_blkinfo *hdr3 = bp->b_addr; + + if (hdr3->hdr.magic != cpu_to_be16(XFS_DA3_NODE_MAGIC)) + return __this_address; + + if (be64_to_cpu(hdr3->owner) != owner) + return __this_address; + } + + return NULL; +} + xfs_failaddr_t xfs_da3_header_check( struct xfs_buf *bp, @@ -266,6 +286,8 @@ xfs_da3_header_check( switch (hdr->magic) { case cpu_to_be16(XFS_ATTR3_LEAF_MAGIC): return xfs_attr3_leaf_header_check(bp, owner); + case cpu_to_be16(XFS_DA3_NODE_MAGIC): + return xfs_da3_node_header_check(bp, owner); } return NULL; @@ -1218,6 +1240,7 @@ xfs_da3_root_join( struct xfs_da3_icnode_hdr oldroothdr; int error; struct xfs_inode *dp = state->args->dp; + xfs_failaddr_t fa; trace_xfs_da_root_join(state->args); @@ -1244,6 +1267,13 @@ xfs_da3_root_join( error = xfs_da3_node_read(args->trans, dp, child, &bp, args->whichfork); if (error) return error; + fa = xfs_da3_header_check(bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_trans_brelse(args->trans, bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } xfs_da_blkinfo_onlychild_validate(bp->b_addr, oldroothdr.level); /* @@ -1278,6 +1308,7 @@ xfs_da3_node_toosmall( struct xfs_da_blkinfo *info; xfs_dablk_t blkno; struct xfs_buf *bp; + xfs_failaddr_t fa; struct xfs_da3_icnode_hdr nodehdr; int count; int forward; @@ -1352,6 +1383,13 @@ xfs_da3_node_toosmall( state->args->whichfork); if (error) return error; + fa = xfs_da3_node_header_check(bp, state->args->owner); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_trans_brelse(state->args->trans, bp); + xfs_da_mark_sick(state->args); + return -EFSCORRUPTED; + } node = bp->b_addr; xfs_da3_node_hdr_from_disk(dp->i_mount, &thdr, node); @@ -1674,6 +1712,13 @@ xfs_da3_node_lookup_int( return -EFSCORRUPTED; } + fa = xfs_da3_node_header_check(blk->bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(blk->bp, fa); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + blk->magic = XFS_DA_NODE_MAGIC; /* @@ -1846,6 +1891,7 @@ xfs_da3_blk_link( struct xfs_da_blkinfo *tmp_info; struct xfs_da_args *args; struct xfs_buf *bp; + xfs_failaddr_t fa; int before = 0; int error; struct xfs_inode *dp = state->args->dp; @@ -1889,6 +1935,13 @@ xfs_da3_blk_link( &bp, args->whichfork); if (error) return error; + fa = xfs_da3_header_check(bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_trans_brelse(args->trans, bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } ASSERT(bp != NULL); tmp_info = bp->b_addr; ASSERT(tmp_info->magic == old_info->magic); @@ -1910,6 +1963,13 @@ xfs_da3_blk_link( &bp, args->whichfork); if (error) return error; + fa = xfs_da3_header_check(bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_trans_brelse(args->trans, bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } ASSERT(bp != NULL); tmp_info = bp->b_addr; ASSERT(tmp_info->magic == old_info->magic); @@ -1939,6 +1999,7 @@ xfs_da3_blk_unlink( struct xfs_da_blkinfo *tmp_info; struct xfs_da_args *args; struct xfs_buf *bp; + xfs_failaddr_t fa; int error; /* @@ -1969,6 +2030,13 @@ xfs_da3_blk_unlink( &bp, args->whichfork); if (error) return error; + fa = xfs_da3_header_check(bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_trans_brelse(args->trans, bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } ASSERT(bp != NULL); tmp_info = bp->b_addr; ASSERT(tmp_info->magic == save_info->magic); @@ -1986,6 +2054,13 @@ xfs_da3_blk_unlink( &bp, args->whichfork); if (error) return error; + fa = xfs_da3_header_check(bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_trans_brelse(args->trans, bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } ASSERT(bp != NULL); tmp_info = bp->b_addr; ASSERT(tmp_info->magic == save_info->magic); @@ -2101,6 +2176,12 @@ xfs_da3_path_shift( switch (be16_to_cpu(info->magic)) { case XFS_DA_NODE_MAGIC: case XFS_DA3_NODE_MAGIC: + fa = xfs_da3_node_header_check(blk->bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(blk->bp, fa); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } blk->magic = XFS_DA_NODE_MAGIC; xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr, bp->b_addr); @@ -2406,6 +2487,13 @@ xfs_da3_swap_lastblock( error = xfs_da3_node_read(tp, dp, sib_blkno, &sib_buf, w); if (error) goto done; + fa = xfs_da3_header_check(sib_buf, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(sib_buf, fa); + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto done; + } sib_info = sib_buf->b_addr; if (XFS_IS_CORRUPT(mp, be32_to_cpu(sib_info->forw) != last_blkno || @@ -2427,6 +2515,13 @@ xfs_da3_swap_lastblock( error = xfs_da3_node_read(tp, dp, sib_blkno, &sib_buf, w); if (error) goto done; + fa = xfs_da3_header_check(sib_buf, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(sib_buf, fa); + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto done; + } sib_info = sib_buf->b_addr; if (XFS_IS_CORRUPT(mp, be32_to_cpu(sib_info->back) != last_blkno || @@ -2450,6 +2545,13 @@ xfs_da3_swap_lastblock( error = xfs_da3_node_read(tp, dp, par_blkno, &par_buf, w); if (error) goto done; + fa = xfs_da3_node_header_check(par_buf, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(par_buf, fa); + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto done; + } par_node = par_buf->b_addr; xfs_da3_node_hdr_from_disk(dp->i_mount, &par_hdr, par_node); if (XFS_IS_CORRUPT(mp, @@ -2499,6 +2601,13 @@ xfs_da3_swap_lastblock( error = xfs_da3_node_read(tp, dp, par_blkno, &par_buf, w); if (error) goto done; + fa = xfs_da3_node_header_check(par_buf, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(par_buf, fa); + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto done; + } par_node = par_buf->b_addr; xfs_da3_node_hdr_from_disk(dp->i_mount, &par_hdr, par_node); if (XFS_IS_CORRUPT(mp, par_hdr.level != level)) { diff --git a/fs/xfs/libxfs/xfs_da_btree.h b/fs/xfs/libxfs/xfs_da_btree.h index 99618e0c8a72b..7a004786ee0a2 100644 --- a/fs/xfs/libxfs/xfs_da_btree.h +++ b/fs/xfs/libxfs/xfs_da_btree.h @@ -237,6 +237,7 @@ void xfs_da3_node_hdr_from_disk(struct xfs_mount *mp, void xfs_da3_node_hdr_to_disk(struct xfs_mount *mp, struct xfs_da_intnode *to, struct xfs_da3_icnode_hdr *from); xfs_failaddr_t xfs_da3_header_check(struct xfs_buf *bp, xfs_ino_t owner); +xfs_failaddr_t xfs_da3_node_header_check(struct xfs_buf *bp, xfs_ino_t owner); extern struct kmem_cache *xfs_da_state_cache; diff --git a/fs/xfs/xfs_attr_list.c b/fs/xfs/xfs_attr_list.c index f6496e33ff91b..6a621f016f040 100644 --- a/fs/xfs/xfs_attr_list.c +++ b/fs/xfs/xfs_attr_list.c @@ -239,6 +239,10 @@ xfs_attr_node_list_lookup( goto out_corruptbuf; } + fa = xfs_da3_node_header_check(bp, dp->i_ino); + if (fa) + goto out_corruptbuf; + xfs_da3_node_hdr_from_disk(mp, &nodehdr, node); /* Tree taller than we can handle; bail out! */ @@ -335,6 +339,11 @@ xfs_attr_node_list( case XFS_DA_NODE_MAGIC: case XFS_DA3_NODE_MAGIC: trace_xfs_attr_list_wrong_blk(context); + fa = xfs_da3_node_header_check(bp, dp->i_ino); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); + } xfs_trans_brelse(context->tp, bp); bp = NULL; break; From 402eef10a1bab0b428c418cfbaaa0a62efc9c951 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:39 -0700 Subject: [PATCH 0230/1702] xfs: validate directory leaf buffer owners Check the owner field of directory leaf blocks. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_da_btree.c | 16 +++++++++ fs/xfs/libxfs/xfs_dir2.h | 2 ++ fs/xfs/libxfs/xfs_dir2_leaf.c | 65 +++++++++++++++++++++++++++++++---- fs/xfs/libxfs/xfs_dir2_node.c | 3 +- fs/xfs/libxfs/xfs_dir2_priv.h | 4 +-- fs/xfs/scrub/dir.c | 2 +- 6 files changed, 82 insertions(+), 10 deletions(-) diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c index e6c28bccdbc07..b13796629e221 100644 --- a/fs/xfs/libxfs/xfs_da_btree.c +++ b/fs/xfs/libxfs/xfs_da_btree.c @@ -288,8 +288,12 @@ xfs_da3_header_check( return xfs_attr3_leaf_header_check(bp, owner); case cpu_to_be16(XFS_DA3_NODE_MAGIC): return xfs_da3_node_header_check(bp, owner); + case cpu_to_be16(XFS_DIR3_LEAF1_MAGIC): + case cpu_to_be16(XFS_DIR3_LEAFN_MAGIC): + return xfs_dir3_leaf_header_check(bp, owner); } + ASSERT(0); return NULL; } @@ -1700,6 +1704,12 @@ xfs_da3_node_lookup_int( if (magic == XFS_DIR2_LEAFN_MAGIC || magic == XFS_DIR3_LEAFN_MAGIC) { + fa = xfs_dir3_leaf_header_check(blk->bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(blk->bp, fa); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } blk->magic = XFS_DIR2_LEAFN_MAGIC; blk->hashval = xfs_dir2_leaf_lasthash(args->dp, blk->bp, NULL); @@ -2208,6 +2218,12 @@ xfs_da3_path_shift( break; case XFS_DIR2_LEAFN_MAGIC: case XFS_DIR3_LEAFN_MAGIC: + fa = xfs_dir3_leaf_header_check(blk->bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(blk->bp, fa); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } blk->magic = XFS_DIR2_LEAFN_MAGIC; ASSERT(level == path->active-1); blk->index = 0; diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h index 8497d041f3163..2f728c26a4162 100644 --- a/fs/xfs/libxfs/xfs_dir2.h +++ b/fs/xfs/libxfs/xfs_dir2.h @@ -101,6 +101,8 @@ extern struct xfs_dir2_data_free *xfs_dir2_data_freefind( extern int xfs_dir_ino_validate(struct xfs_mount *mp, xfs_ino_t ino); +xfs_failaddr_t xfs_dir3_leaf_header_check(struct xfs_buf *bp, xfs_ino_t owner); + extern const struct xfs_buf_ops xfs_dir3_block_buf_ops; extern const struct xfs_buf_ops xfs_dir3_leafn_buf_ops; extern const struct xfs_buf_ops xfs_dir3_leaf1_buf_ops; diff --git a/fs/xfs/libxfs/xfs_dir2_leaf.c b/fs/xfs/libxfs/xfs_dir2_leaf.c index 20ce057d12e82..53b808e2a5f07 100644 --- a/fs/xfs/libxfs/xfs_dir2_leaf.c +++ b/fs/xfs/libxfs/xfs_dir2_leaf.c @@ -208,6 +208,29 @@ xfs_dir3_leaf_verify( return xfs_dir3_leaf_check_int(mp, &leafhdr, bp->b_addr, true); } +xfs_failaddr_t +xfs_dir3_leaf_header_check( + struct xfs_buf *bp, + xfs_ino_t owner) +{ + struct xfs_mount *mp = bp->b_mount; + + if (xfs_has_crc(mp)) { + struct xfs_dir3_leaf *hdr3 = bp->b_addr; + + if (hdr3->hdr.info.hdr.magic != + cpu_to_be16(XFS_DIR3_LEAF1_MAGIC) && + hdr3->hdr.info.hdr.magic != + cpu_to_be16(XFS_DIR3_LEAFN_MAGIC)) + return __this_address; + + if (be64_to_cpu(hdr3->hdr.info.owner) != owner) + return __this_address; + } + + return NULL; +} + static void xfs_dir3_leaf_read_verify( struct xfs_buf *bp) @@ -271,32 +294,60 @@ int xfs_dir3_leaf_read( struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, xfs_dablk_t fbno, struct xfs_buf **bpp) { + xfs_failaddr_t fa; int err; err = xfs_da_read_buf(tp, dp, fbno, 0, bpp, XFS_DATA_FORK, &xfs_dir3_leaf1_buf_ops); - if (!err && tp && *bpp) + if (err || !(*bpp)) + return err; + + fa = xfs_dir3_leaf_header_check(*bpp, owner); + if (fa) { + __xfs_buf_mark_corrupt(*bpp, fa); + xfs_trans_brelse(tp, *bpp); + *bpp = NULL; + xfs_dirattr_mark_sick(dp, XFS_DATA_FORK); + return -EFSCORRUPTED; + } + + if (tp) xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_DIR_LEAF1_BUF); - return err; + return 0; } int xfs_dir3_leafn_read( struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, xfs_dablk_t fbno, struct xfs_buf **bpp) { + xfs_failaddr_t fa; int err; err = xfs_da_read_buf(tp, dp, fbno, 0, bpp, XFS_DATA_FORK, &xfs_dir3_leafn_buf_ops); - if (!err && tp && *bpp) + if (err || !(*bpp)) + return err; + + fa = xfs_dir3_leaf_header_check(*bpp, owner); + if (fa) { + __xfs_buf_mark_corrupt(*bpp, fa); + xfs_trans_brelse(tp, *bpp); + *bpp = NULL; + xfs_dirattr_mark_sick(dp, XFS_DATA_FORK); + return -EFSCORRUPTED; + } + + if (tp) xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_DIR_LEAFN_BUF); - return err; + return 0; } /* @@ -646,7 +697,8 @@ xfs_dir2_leaf_addname( trace_xfs_dir2_leaf_addname(args); - error = xfs_dir3_leaf_read(tp, dp, args->geo->leafblk, &lbp); + error = xfs_dir3_leaf_read(tp, dp, args->owner, args->geo->leafblk, + &lbp); if (error) return error; @@ -1237,7 +1289,8 @@ xfs_dir2_leaf_lookup_int( tp = args->trans; mp = dp->i_mount; - error = xfs_dir3_leaf_read(tp, dp, args->geo->leafblk, &lbp); + error = xfs_dir3_leaf_read(tp, dp, args->owner, args->geo->leafblk, + &lbp); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_dir2_node.c b/fs/xfs/libxfs/xfs_dir2_node.c index 1ad7405f9c389..e21965788188b 100644 --- a/fs/xfs/libxfs/xfs_dir2_node.c +++ b/fs/xfs/libxfs/xfs_dir2_node.c @@ -1562,7 +1562,8 @@ xfs_dir2_leafn_toosmall( /* * Read the sibling leaf block. */ - error = xfs_dir3_leafn_read(state->args->trans, dp, blkno, &bp); + error = xfs_dir3_leafn_read(state->args->trans, dp, + state->args->owner, blkno, &bp); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_dir2_priv.h b/fs/xfs/libxfs/xfs_dir2_priv.h index 1db2e60ba827f..2f0e3ad47b371 100644 --- a/fs/xfs/libxfs/xfs_dir2_priv.h +++ b/fs/xfs/libxfs/xfs_dir2_priv.h @@ -95,9 +95,9 @@ void xfs_dir2_leaf_hdr_from_disk(struct xfs_mount *mp, void xfs_dir2_leaf_hdr_to_disk(struct xfs_mount *mp, struct xfs_dir2_leaf *to, struct xfs_dir3_icleaf_hdr *from); int xfs_dir3_leaf_read(struct xfs_trans *tp, struct xfs_inode *dp, - xfs_dablk_t fbno, struct xfs_buf **bpp); + xfs_ino_t owner, xfs_dablk_t fbno, struct xfs_buf **bpp); int xfs_dir3_leafn_read(struct xfs_trans *tp, struct xfs_inode *dp, - xfs_dablk_t fbno, struct xfs_buf **bpp); + xfs_ino_t owner, xfs_dablk_t fbno, struct xfs_buf **bpp); extern int xfs_dir2_block_to_leaf(struct xfs_da_args *args, struct xfs_buf *dbp); extern int xfs_dir2_leaf_addname(struct xfs_da_args *args); diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index 042e28547e044..d94e265a8e1f2 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -470,7 +470,7 @@ xchk_directory_leaf1_bestfree( int error; /* Read the free space block. */ - error = xfs_dir3_leaf_read(sc->tp, sc->ip, lblk, &bp); + error = xfs_dir3_leaf_read(sc->tp, sc->ip, sc->ip->i_ino, lblk, &bp); if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) return error; xchk_buffer_recheck(sc, bp); From cc6740ddb423db2066f7669eaaa377fdbf84ab1e Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:40 -0700 Subject: [PATCH 0231/1702] xfs: validate explicit directory data buffer owners Port the existing directory data header checking function to accept an owner number instead of an xfs_inode, then update the callsites to use xfs_da_args.owner when possible. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_dir2.h | 1 + fs/xfs/libxfs/xfs_dir2_block.c | 3 ++- fs/xfs/libxfs/xfs_dir2_data.c | 16 ++++++++++------ fs/xfs/libxfs/xfs_dir2_leaf.c | 21 +++++++++++---------- fs/xfs/libxfs/xfs_dir2_node.c | 7 +++---- fs/xfs/libxfs/xfs_dir2_priv.h | 3 ++- fs/xfs/scrub/dir.c | 14 +++++++------- fs/xfs/scrub/readdir.c | 2 +- fs/xfs/xfs_dir2_readdir.c | 3 ++- 9 files changed, 39 insertions(+), 31 deletions(-) diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h index 2f728c26a4162..d623bfdcd4218 100644 --- a/fs/xfs/libxfs/xfs_dir2.h +++ b/fs/xfs/libxfs/xfs_dir2.h @@ -102,6 +102,7 @@ extern struct xfs_dir2_data_free *xfs_dir2_data_freefind( extern int xfs_dir_ino_validate(struct xfs_mount *mp, xfs_ino_t ino); xfs_failaddr_t xfs_dir3_leaf_header_check(struct xfs_buf *bp, xfs_ino_t owner); +xfs_failaddr_t xfs_dir3_data_header_check(struct xfs_buf *bp, xfs_ino_t owner); extern const struct xfs_buf_ops xfs_dir3_block_buf_ops; extern const struct xfs_buf_ops xfs_dir3_leafn_buf_ops; diff --git a/fs/xfs/libxfs/xfs_dir2_block.c b/fs/xfs/libxfs/xfs_dir2_block.c index 61cbc668f228a..b20b08394aa06 100644 --- a/fs/xfs/libxfs/xfs_dir2_block.c +++ b/fs/xfs/libxfs/xfs_dir2_block.c @@ -982,7 +982,8 @@ xfs_dir2_leaf_to_block( * Read the data block if we don't already have it, give up if it fails. */ if (!dbp) { - error = xfs_dir3_data_read(tp, dp, args->geo->datablk, 0, &dbp); + error = xfs_dir3_data_read(tp, dp, args->owner, + args->geo->datablk, 0, &dbp); if (error) return error; } diff --git a/fs/xfs/libxfs/xfs_dir2_data.c b/fs/xfs/libxfs/xfs_dir2_data.c index c3ef720b5ff6e..ea0b9628df18b 100644 --- a/fs/xfs/libxfs/xfs_dir2_data.c +++ b/fs/xfs/libxfs/xfs_dir2_data.c @@ -395,17 +395,20 @@ static const struct xfs_buf_ops xfs_dir3_data_reada_buf_ops = { .verify_write = xfs_dir3_data_write_verify, }; -static xfs_failaddr_t +xfs_failaddr_t xfs_dir3_data_header_check( - struct xfs_inode *dp, - struct xfs_buf *bp) + struct xfs_buf *bp, + xfs_ino_t owner) { - struct xfs_mount *mp = dp->i_mount; + struct xfs_mount *mp = bp->b_mount; if (xfs_has_crc(mp)) { struct xfs_dir3_data_hdr *hdr3 = bp->b_addr; - if (be64_to_cpu(hdr3->hdr.owner) != dp->i_ino) + if (hdr3->hdr.magic != cpu_to_be32(XFS_DIR3_DATA_MAGIC)) + return __this_address; + + if (be64_to_cpu(hdr3->hdr.owner) != owner) return __this_address; } @@ -416,6 +419,7 @@ int xfs_dir3_data_read( struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, xfs_dablk_t bno, unsigned int flags, struct xfs_buf **bpp) @@ -429,7 +433,7 @@ xfs_dir3_data_read( return err; /* Check things that we can't do in the verifier. */ - fa = xfs_dir3_data_header_check(dp, *bpp); + fa = xfs_dir3_data_header_check(*bpp, owner); if (fa) { __xfs_buf_mark_corrupt(*bpp, fa); xfs_trans_brelse(tp, *bpp); diff --git a/fs/xfs/libxfs/xfs_dir2_leaf.c b/fs/xfs/libxfs/xfs_dir2_leaf.c index 53b808e2a5f07..0b1b852f6178b 100644 --- a/fs/xfs/libxfs/xfs_dir2_leaf.c +++ b/fs/xfs/libxfs/xfs_dir2_leaf.c @@ -885,9 +885,9 @@ xfs_dir2_leaf_addname( * Already had space in some data block. * Just read that one in. */ - error = xfs_dir3_data_read(tp, dp, - xfs_dir2_db_to_da(args->geo, use_block), - 0, &dbp); + error = xfs_dir3_data_read(tp, dp, args->owner, + xfs_dir2_db_to_da(args->geo, use_block), 0, + &dbp); if (error) { xfs_trans_brelse(tp, lbp); return error; @@ -1328,9 +1328,9 @@ xfs_dir2_leaf_lookup_int( if (newdb != curdb) { if (dbp) xfs_trans_brelse(tp, dbp); - error = xfs_dir3_data_read(tp, dp, - xfs_dir2_db_to_da(args->geo, newdb), - 0, &dbp); + error = xfs_dir3_data_read(tp, dp, args->owner, + xfs_dir2_db_to_da(args->geo, newdb), 0, + &dbp); if (error) { xfs_trans_brelse(tp, lbp); return error; @@ -1370,9 +1370,9 @@ xfs_dir2_leaf_lookup_int( ASSERT(cidb != -1); if (cidb != curdb) { xfs_trans_brelse(tp, dbp); - error = xfs_dir3_data_read(tp, dp, - xfs_dir2_db_to_da(args->geo, cidb), - 0, &dbp); + error = xfs_dir3_data_read(tp, dp, args->owner, + xfs_dir2_db_to_da(args->geo, cidb), 0, + &dbp); if (error) { xfs_trans_brelse(tp, lbp); return error; @@ -1666,7 +1666,8 @@ xfs_dir2_leaf_trim_data( /* * Read the offending data block. We need its buffer. */ - error = xfs_dir3_data_read(tp, dp, xfs_dir2_db_to_da(geo, db), 0, &dbp); + error = xfs_dir3_data_read(tp, dp, args->owner, + xfs_dir2_db_to_da(geo, db), 0, &dbp); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_dir2_node.c b/fs/xfs/libxfs/xfs_dir2_node.c index e21965788188b..dc85197b8448e 100644 --- a/fs/xfs/libxfs/xfs_dir2_node.c +++ b/fs/xfs/libxfs/xfs_dir2_node.c @@ -863,7 +863,7 @@ xfs_dir2_leafn_lookup_for_entry( ASSERT(state->extravalid); curbp = state->extrablk.bp; } else { - error = xfs_dir3_data_read(tp, dp, + error = xfs_dir3_data_read(tp, dp, args->owner, xfs_dir2_db_to_da(args->geo, newdb), 0, &curbp); @@ -1949,9 +1949,8 @@ xfs_dir2_node_addname_int( &freehdr, &findex); } else { /* Read the data block in. */ - error = xfs_dir3_data_read(tp, dp, - xfs_dir2_db_to_da(args->geo, dbno), - 0, &dbp); + error = xfs_dir3_data_read(tp, dp, args->owner, + xfs_dir2_db_to_da(args->geo, dbno), 0, &dbp); } if (error) return error; diff --git a/fs/xfs/libxfs/xfs_dir2_priv.h b/fs/xfs/libxfs/xfs_dir2_priv.h index 2f0e3ad47b371..879aa2e9fd730 100644 --- a/fs/xfs/libxfs/xfs_dir2_priv.h +++ b/fs/xfs/libxfs/xfs_dir2_priv.h @@ -78,7 +78,8 @@ extern void xfs_dir3_data_check(struct xfs_inode *dp, struct xfs_buf *bp); extern xfs_failaddr_t __xfs_dir3_data_check(struct xfs_inode *dp, struct xfs_buf *bp); int xfs_dir3_data_read(struct xfs_trans *tp, struct xfs_inode *dp, - xfs_dablk_t bno, unsigned int flags, struct xfs_buf **bpp); + xfs_ino_t owner, xfs_dablk_t bno, unsigned int flags, + struct xfs_buf **bpp); int xfs_dir3_data_readahead(struct xfs_inode *dp, xfs_dablk_t bno, unsigned int flags); diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index d94e265a8e1f2..6b572196bb43d 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -196,8 +196,8 @@ xchk_dir_rec( xchk_da_set_corrupt(ds, level); goto out; } - error = xfs_dir3_data_read(ds->dargs.trans, dp, rec_bno, - XFS_DABUF_MAP_HOLE_OK, &bp); + error = xfs_dir3_data_read(ds->dargs.trans, dp, ds->dargs.owner, + rec_bno, XFS_DABUF_MAP_HOLE_OK, &bp); if (!xchk_fblock_process_error(ds->sc, XFS_DATA_FORK, rec_bno, &error)) goto out; @@ -318,7 +318,8 @@ xchk_directory_data_bestfree( error = xfs_dir3_block_read(sc->tp, sc->ip, &bp); } else { /* dir data format */ - error = xfs_dir3_data_read(sc->tp, sc->ip, lblk, 0, &bp); + error = xfs_dir3_data_read(sc->tp, sc->ip, sc->ip->i_ino, lblk, + 0, &bp); } if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) goto out; @@ -531,10 +532,9 @@ xchk_directory_leaf1_bestfree( /* Check all the bestfree entries. */ for (i = 0; i < bestcount; i++, bestp++) { best = be16_to_cpu(*bestp); - error = xfs_dir3_data_read(sc->tp, sc->ip, + error = xfs_dir3_data_read(sc->tp, sc->ip, args->owner, xfs_dir2_db_to_da(args->geo, i), - XFS_DABUF_MAP_HOLE_OK, - &dbp); + XFS_DABUF_MAP_HOLE_OK, &dbp); if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) break; @@ -597,7 +597,7 @@ xchk_directory_free_bestfree( stale++; continue; } - error = xfs_dir3_data_read(sc->tp, sc->ip, + error = xfs_dir3_data_read(sc->tp, sc->ip, args->owner, (freehdr.firstdb + i) * args->geo->fsbcount, 0, &dbp); if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, diff --git a/fs/xfs/scrub/readdir.c b/fs/xfs/scrub/readdir.c index fb98b7624994a..bed15a9524a2f 100644 --- a/fs/xfs/scrub/readdir.c +++ b/fs/xfs/scrub/readdir.c @@ -175,7 +175,7 @@ xchk_read_leaf_dir_buf( if (new_off > *curoff) *curoff = new_off; - return xfs_dir3_data_read(tp, dp, map.br_startoff, 0, bpp); + return xfs_dir3_data_read(tp, dp, dp->i_ino, map.br_startoff, 0, bpp); } /* Call a function for every entry in a leaf directory. */ diff --git a/fs/xfs/xfs_dir2_readdir.c b/fs/xfs/xfs_dir2_readdir.c index 4e811fa393add..2c03371b542ae 100644 --- a/fs/xfs/xfs_dir2_readdir.c +++ b/fs/xfs/xfs_dir2_readdir.c @@ -282,7 +282,8 @@ xfs_dir2_leaf_readbuf( new_off = xfs_dir2_da_to_byte(geo, map.br_startoff); if (new_off > *cur_off) *cur_off = new_off; - error = xfs_dir3_data_read(args->trans, dp, map.br_startoff, 0, &bp); + error = xfs_dir3_data_read(args->trans, dp, args->owner, + map.br_startoff, 0, &bp); if (error) goto out; From 29b41ce919b7f0b0c2220e088e450d9b132bec36 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:41 -0700 Subject: [PATCH 0232/1702] xfs: validate explicit directory block buffer owners Port the existing directory block header checking function to accept an owner number instead of an xfs_inode, then update the callsites to use xfs_da_args.owner when possible. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_dir2.h | 1 + fs/xfs/libxfs/xfs_dir2_block.c | 20 ++++++++++++-------- fs/xfs/libxfs/xfs_dir2_priv.h | 4 ++-- fs/xfs/libxfs/xfs_exchmaps.c | 2 +- fs/xfs/scrub/dir.c | 2 +- fs/xfs/scrub/readdir.c | 2 +- fs/xfs/xfs_dir2_readdir.c | 2 +- 7 files changed, 19 insertions(+), 14 deletions(-) diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h index d623bfdcd4218..eb3a5c35025b5 100644 --- a/fs/xfs/libxfs/xfs_dir2.h +++ b/fs/xfs/libxfs/xfs_dir2.h @@ -103,6 +103,7 @@ extern int xfs_dir_ino_validate(struct xfs_mount *mp, xfs_ino_t ino); xfs_failaddr_t xfs_dir3_leaf_header_check(struct xfs_buf *bp, xfs_ino_t owner); xfs_failaddr_t xfs_dir3_data_header_check(struct xfs_buf *bp, xfs_ino_t owner); +xfs_failaddr_t xfs_dir3_block_header_check(struct xfs_buf *bp, xfs_ino_t owner); extern const struct xfs_buf_ops xfs_dir3_block_buf_ops; extern const struct xfs_buf_ops xfs_dir3_leafn_buf_ops; diff --git a/fs/xfs/libxfs/xfs_dir2_block.c b/fs/xfs/libxfs/xfs_dir2_block.c index b20b08394aa06..0f93ed1a4a74f 100644 --- a/fs/xfs/libxfs/xfs_dir2_block.c +++ b/fs/xfs/libxfs/xfs_dir2_block.c @@ -115,17 +115,20 @@ const struct xfs_buf_ops xfs_dir3_block_buf_ops = { .verify_struct = xfs_dir3_block_verify, }; -static xfs_failaddr_t +xfs_failaddr_t xfs_dir3_block_header_check( - struct xfs_inode *dp, - struct xfs_buf *bp) + struct xfs_buf *bp, + xfs_ino_t owner) { - struct xfs_mount *mp = dp->i_mount; + struct xfs_mount *mp = bp->b_mount; if (xfs_has_crc(mp)) { struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr; - if (be64_to_cpu(hdr3->owner) != dp->i_ino) + if (hdr3->magic != cpu_to_be32(XFS_DIR3_BLOCK_MAGIC)) + return __this_address; + + if (be64_to_cpu(hdr3->owner) != owner) return __this_address; } @@ -136,6 +139,7 @@ int xfs_dir3_block_read( struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, struct xfs_buf **bpp) { struct xfs_mount *mp = dp->i_mount; @@ -148,7 +152,7 @@ xfs_dir3_block_read( return err; /* Check things that we can't do in the verifier. */ - fa = xfs_dir3_block_header_check(dp, *bpp); + fa = xfs_dir3_block_header_check(*bpp, owner); if (fa) { __xfs_buf_mark_corrupt(*bpp, fa); xfs_trans_brelse(tp, *bpp); @@ -383,7 +387,7 @@ xfs_dir2_block_addname( tp = args->trans; /* Read the (one and only) directory block into bp. */ - error = xfs_dir3_block_read(tp, dp, &bp); + error = xfs_dir3_block_read(tp, dp, args->owner, &bp); if (error) return error; @@ -698,7 +702,7 @@ xfs_dir2_block_lookup_int( dp = args->dp; tp = args->trans; - error = xfs_dir3_block_read(tp, dp, &bp); + error = xfs_dir3_block_read(tp, dp, args->owner, &bp); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_dir2_priv.h b/fs/xfs/libxfs/xfs_dir2_priv.h index 879aa2e9fd730..adbc544c9befa 100644 --- a/fs/xfs/libxfs/xfs_dir2_priv.h +++ b/fs/xfs/libxfs/xfs_dir2_priv.h @@ -50,8 +50,8 @@ extern int xfs_dir_cilookup_result(struct xfs_da_args *args, /* xfs_dir2_block.c */ -extern int xfs_dir3_block_read(struct xfs_trans *tp, struct xfs_inode *dp, - struct xfs_buf **bpp); +int xfs_dir3_block_read(struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, struct xfs_buf **bpp); extern int xfs_dir2_block_addname(struct xfs_da_args *args); extern int xfs_dir2_block_lookup(struct xfs_da_args *args); extern int xfs_dir2_block_removename(struct xfs_da_args *args); diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index 9c9cf2e998b25..3880ae32eecf4 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -476,7 +476,7 @@ xfs_exchmaps_dir_to_sf( if (!isblock) return 0; - error = xfs_dir3_block_read(tp, xmi->xmi_ip2, &bp); + error = xfs_dir3_block_read(tp, xmi->xmi_ip2, xmi->xmi_ip2->i_ino, &bp); if (error) return error; diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index 6b572196bb43d..43f5bc8ce0d46 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -315,7 +315,7 @@ xchk_directory_data_bestfree( /* dir block format */ if (lblk != XFS_B_TO_FSBT(mp, XFS_DIR2_DATA_OFFSET)) xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); - error = xfs_dir3_block_read(sc->tp, sc->ip, &bp); + error = xfs_dir3_block_read(sc->tp, sc->ip, sc->ip->i_ino, &bp); } else { /* dir data format */ error = xfs_dir3_data_read(sc->tp, sc->ip, sc->ip->i_ino, lblk, diff --git a/fs/xfs/scrub/readdir.c b/fs/xfs/scrub/readdir.c index bed15a9524a2f..e940804693151 100644 --- a/fs/xfs/scrub/readdir.c +++ b/fs/xfs/scrub/readdir.c @@ -99,7 +99,7 @@ xchk_dir_walk_block( unsigned int off, next_off, end; int error; - error = xfs_dir3_block_read(sc->tp, dp, &bp); + error = xfs_dir3_block_read(sc->tp, dp, dp->i_ino, &bp); if (error) return error; diff --git a/fs/xfs/xfs_dir2_readdir.c b/fs/xfs/xfs_dir2_readdir.c index 2c03371b542ae..b3abad5a6cd80 100644 --- a/fs/xfs/xfs_dir2_readdir.c +++ b/fs/xfs/xfs_dir2_readdir.c @@ -157,7 +157,7 @@ xfs_dir2_block_getdents( if (xfs_dir2_dataptr_to_db(geo, ctx->pos) > geo->datablk) return 0; - error = xfs_dir3_block_read(args->trans, dp, &bp); + error = xfs_dir3_block_read(args->trans, dp, args->owner, &bp); if (error) return error; From fe6c9f8e48e0dcbfc3dba17edd88490c8579b34b Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:41 -0700 Subject: [PATCH 0233/1702] xfs: validate explicit directory free block owners Port the existing directory freespace block header checking function to accept an owner number instead of an xfs_inode, then update the callsites to use xfs_da_args.owner when possible. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_dir2_leaf.c | 3 ++- fs/xfs/libxfs/xfs_dir2_node.c | 32 ++++++++++++++++++-------------- fs/xfs/libxfs/xfs_dir2_priv.h | 4 ++-- fs/xfs/scrub/dir.c | 2 +- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/fs/xfs/libxfs/xfs_dir2_leaf.c b/fs/xfs/libxfs/xfs_dir2_leaf.c index 0b1b852f6178b..71c2f22a3f6e0 100644 --- a/fs/xfs/libxfs/xfs_dir2_leaf.c +++ b/fs/xfs/libxfs/xfs_dir2_leaf.c @@ -1806,7 +1806,8 @@ xfs_dir2_node_to_leaf( /* * Read the freespace block. */ - error = xfs_dir2_free_read(tp, dp, args->geo->freeblk, &fbp); + error = xfs_dir2_free_read(tp, dp, args->owner, args->geo->freeblk, + &fbp); if (error) return error; xfs_dir2_free_hdr_from_disk(mp, &freehdr, fbp->b_addr); diff --git a/fs/xfs/libxfs/xfs_dir2_node.c b/fs/xfs/libxfs/xfs_dir2_node.c index dc85197b8448e..fe8d4fa131289 100644 --- a/fs/xfs/libxfs/xfs_dir2_node.c +++ b/fs/xfs/libxfs/xfs_dir2_node.c @@ -175,11 +175,11 @@ const struct xfs_buf_ops xfs_dir3_free_buf_ops = { /* Everything ok in the free block header? */ static xfs_failaddr_t xfs_dir3_free_header_check( - struct xfs_inode *dp, - xfs_dablk_t fbno, - struct xfs_buf *bp) + struct xfs_buf *bp, + xfs_ino_t owner, + xfs_dablk_t fbno) { - struct xfs_mount *mp = dp->i_mount; + struct xfs_mount *mp = bp->b_mount; int maxbests = mp->m_dir_geo->free_max_bests; unsigned int firstdb; @@ -195,7 +195,7 @@ xfs_dir3_free_header_check( return __this_address; if (be32_to_cpu(hdr3->nvalid) < be32_to_cpu(hdr3->nused)) return __this_address; - if (be64_to_cpu(hdr3->hdr.owner) != dp->i_ino) + if (be64_to_cpu(hdr3->hdr.owner) != owner) return __this_address; } else { struct xfs_dir2_free_hdr *hdr = bp->b_addr; @@ -214,6 +214,7 @@ static int __xfs_dir3_free_read( struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, xfs_dablk_t fbno, unsigned int flags, struct xfs_buf **bpp) @@ -227,7 +228,7 @@ __xfs_dir3_free_read( return err; /* Check things that we can't do in the verifier. */ - fa = xfs_dir3_free_header_check(dp, fbno, *bpp); + fa = xfs_dir3_free_header_check(*bpp, owner, fbno); if (fa) { __xfs_buf_mark_corrupt(*bpp, fa); xfs_trans_brelse(tp, *bpp); @@ -299,20 +300,23 @@ int xfs_dir2_free_read( struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, xfs_dablk_t fbno, struct xfs_buf **bpp) { - return __xfs_dir3_free_read(tp, dp, fbno, 0, bpp); + return __xfs_dir3_free_read(tp, dp, owner, fbno, 0, bpp); } static int xfs_dir2_free_try_read( struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, xfs_dablk_t fbno, struct xfs_buf **bpp) { - return __xfs_dir3_free_read(tp, dp, fbno, XFS_DABUF_MAP_HOLE_OK, bpp); + return __xfs_dir3_free_read(tp, dp, owner, fbno, XFS_DABUF_MAP_HOLE_OK, + bpp); } static int @@ -717,7 +721,7 @@ xfs_dir2_leafn_lookup_for_addname( if (curbp) xfs_trans_brelse(tp, curbp); - error = xfs_dir2_free_read(tp, dp, + error = xfs_dir2_free_read(tp, dp, args->owner, xfs_dir2_db_to_da(args->geo, newfdb), &curbp); @@ -1356,8 +1360,8 @@ xfs_dir2_leafn_remove( * read in the free block. */ fdb = xfs_dir2_db_to_fdb(geo, db); - error = xfs_dir2_free_read(tp, dp, xfs_dir2_db_to_da(geo, fdb), - &fbp); + error = xfs_dir2_free_read(tp, dp, args->owner, + xfs_dir2_db_to_da(geo, fdb), &fbp); if (error) return error; free = fbp->b_addr; @@ -1716,7 +1720,7 @@ xfs_dir2_node_add_datablk( * that was just allocated. */ fbno = xfs_dir2_db_to_fdb(args->geo, *dbno); - error = xfs_dir2_free_try_read(tp, dp, + error = xfs_dir2_free_try_read(tp, dp, args->owner, xfs_dir2_db_to_da(args->geo, fbno), &fbp); if (error) return error; @@ -1863,7 +1867,7 @@ xfs_dir2_node_find_freeblk( * so this might not succeed. This should be really rare, so * there's no reason to avoid it. */ - error = xfs_dir2_free_try_read(tp, dp, + error = xfs_dir2_free_try_read(tp, dp, args->owner, xfs_dir2_db_to_da(args->geo, fbno), &fbp); if (error) @@ -2302,7 +2306,7 @@ xfs_dir2_node_trim_free( /* * Read the freespace block. */ - error = xfs_dir2_free_try_read(tp, dp, fo, &bp); + error = xfs_dir2_free_try_read(tp, dp, args->owner, fo, &bp); if (error) return error; /* diff --git a/fs/xfs/libxfs/xfs_dir2_priv.h b/fs/xfs/libxfs/xfs_dir2_priv.h index adbc544c9befa..3befb32509fa4 100644 --- a/fs/xfs/libxfs/xfs_dir2_priv.h +++ b/fs/xfs/libxfs/xfs_dir2_priv.h @@ -155,8 +155,8 @@ extern int xfs_dir2_node_removename(struct xfs_da_args *args); extern int xfs_dir2_node_replace(struct xfs_da_args *args); extern int xfs_dir2_node_trim_free(struct xfs_da_args *args, xfs_fileoff_t fo, int *rvalp); -extern int xfs_dir2_free_read(struct xfs_trans *tp, struct xfs_inode *dp, - xfs_dablk_t fbno, struct xfs_buf **bpp); +int xfs_dir2_free_read(struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, xfs_dablk_t fbno, struct xfs_buf **bpp); /* xfs_dir2_sf.c */ xfs_ino_t xfs_dir2_sf_get_ino(struct xfs_mount *mp, struct xfs_dir2_sf_hdr *hdr, diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index 43f5bc8ce0d46..7bac74621af77 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -577,7 +577,7 @@ xchk_directory_free_bestfree( int error; /* Read the free space block */ - error = xfs_dir2_free_read(sc->tp, sc->ip, lblk, &bp); + error = xfs_dir2_free_read(sc->tp, sc->ip, sc->ip->i_ino, lblk, &bp); if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) return error; xchk_buffer_recheck(sc, bp); From 98339edf0750e75e216521961c98c8105a830bf8 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:42 -0700 Subject: [PATCH 0234/1702] xfs: enable discarding of folios backing an xfile Create a new xfile function to discard the page cache that's backing part of an xfile. The next patch wil use this to drop parts of an xfile that aren't needed anymore. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/trace.h | 1 + fs/xfs/scrub/xfile.c | 12 ++++++++++++ fs/xfs/scrub/xfile.h | 1 + 3 files changed, 14 insertions(+) diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 8d05f2adae3dd..7d07912d8f758 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -948,6 +948,7 @@ DEFINE_XFILE_EVENT(xfile_store); DEFINE_XFILE_EVENT(xfile_seek_data); DEFINE_XFILE_EVENT(xfile_get_folio); DEFINE_XFILE_EVENT(xfile_put_folio); +DEFINE_XFILE_EVENT(xfile_discard); TRACE_EVENT(xfarray_create, TP_PROTO(struct xfarray *xfa, unsigned long long required_capacity), diff --git a/fs/xfs/scrub/xfile.c b/fs/xfs/scrub/xfile.c index 8cdd863db5850..4e254a0ba0036 100644 --- a/fs/xfs/scrub/xfile.c +++ b/fs/xfs/scrub/xfile.c @@ -310,3 +310,15 @@ xfile_put_folio( folio_unlock(folio); folio_put(folio); } + +/* Discard the page cache that's backing a range of the xfile. */ +void +xfile_discard( + struct xfile *xf, + loff_t pos, + u64 count) +{ + trace_xfile_discard(xf, pos, count); + + shmem_truncate_range(file_inode(xf->file), pos, pos + count - 1); +} diff --git a/fs/xfs/scrub/xfile.h b/fs/xfs/scrub/xfile.h index 76d78dba7e347..8dfbae1fe33a5 100644 --- a/fs/xfs/scrub/xfile.h +++ b/fs/xfs/scrub/xfile.h @@ -17,6 +17,7 @@ int xfile_load(struct xfile *xf, void *buf, size_t count, loff_t pos); int xfile_store(struct xfile *xf, const void *buf, size_t count, loff_t pos); +void xfile_discard(struct xfile *xf, loff_t pos, u64 count); loff_t xfile_seek_data(struct xfile *xf, loff_t pos); #define XFILE_MAX_FOLIO_SIZE (PAGE_SIZE << MAX_PAGECACHE_ORDER) From d2bd7eef4f217fc3e1ddf18b4f4eac848886bfa0 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:43 -0700 Subject: [PATCH 0235/1702] xfs: create a blob array data structure Create a simple 'blob array' data structure for storage of arbitrarily sized metadata objects that will be used to reconstruct metadata. For the intended usage (temporarily storing extended attribute names and values) we only have to support storing objects and retrieving them. Use the xfile abstraction to store the attribute information in memory that can be swapped out. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/scrub/xfblob.c | 151 ++++++++++++++++++++++++++++++++++++++++++ fs/xfs/scrub/xfblob.h | 24 +++++++ 3 files changed, 176 insertions(+) create mode 100644 fs/xfs/scrub/xfblob.c create mode 100644 fs/xfs/scrub/xfblob.h diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index 5e3ac7ec8fa5b..bc27757702fe8 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -208,6 +208,7 @@ xfs-y += $(addprefix scrub/, \ repair.o \ rmap_repair.o \ tempfile.o \ + xfblob.o \ ) xfs-$(CONFIG_XFS_RT) += $(addprefix scrub/, \ diff --git a/fs/xfs/scrub/xfblob.c b/fs/xfs/scrub/xfblob.c new file mode 100644 index 0000000000000..cec668debce5a --- /dev/null +++ b/fs/xfs/scrub/xfblob.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "scrub/scrub.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" + +/* + * XFS Blob Storage + * ================ + * Stores and retrieves blobs using an xfile. Objects are appended to the file + * and the offset is returned as a magic cookie for retrieval. + */ + +#define XB_KEY_MAGIC 0xABAADDAD +struct xb_key { + uint32_t xb_magic; /* XB_KEY_MAGIC */ + uint32_t xb_size; /* size of the blob, in bytes */ + loff_t xb_offset; /* byte offset of this key */ + /* blob comes after here */ +} __packed; + +/* Initialize a blob storage object. */ +int +xfblob_create( + const char *description, + struct xfblob **blobp) +{ + struct xfblob *blob; + struct xfile *xfile; + int error; + + error = xfile_create(description, 0, &xfile); + if (error) + return error; + + blob = kmalloc(sizeof(struct xfblob), XCHK_GFP_FLAGS); + if (!blob) { + error = -ENOMEM; + goto out_xfile; + } + + blob->xfile = xfile; + blob->last_offset = PAGE_SIZE; + + *blobp = blob; + return 0; + +out_xfile: + xfile_destroy(xfile); + return error; +} + +/* Destroy a blob storage object. */ +void +xfblob_destroy( + struct xfblob *blob) +{ + xfile_destroy(blob->xfile); + kfree(blob); +} + +/* Retrieve a blob. */ +int +xfblob_load( + struct xfblob *blob, + xfblob_cookie cookie, + void *ptr, + uint32_t size) +{ + struct xb_key key; + int error; + + error = xfile_load(blob->xfile, &key, sizeof(key), cookie); + if (error) + return error; + + if (key.xb_magic != XB_KEY_MAGIC || key.xb_offset != cookie) { + ASSERT(0); + return -ENODATA; + } + if (size < key.xb_size) { + ASSERT(0); + return -EFBIG; + } + + return xfile_load(blob->xfile, ptr, key.xb_size, + cookie + sizeof(key)); +} + +/* Store a blob. */ +int +xfblob_store( + struct xfblob *blob, + xfblob_cookie *cookie, + const void *ptr, + uint32_t size) +{ + struct xb_key key = { + .xb_offset = blob->last_offset, + .xb_magic = XB_KEY_MAGIC, + .xb_size = size, + }; + loff_t pos = blob->last_offset; + int error; + + error = xfile_store(blob->xfile, &key, sizeof(key), pos); + if (error) + return error; + + pos += sizeof(key); + error = xfile_store(blob->xfile, ptr, size, pos); + if (error) + goto out_err; + + *cookie = blob->last_offset; + blob->last_offset += sizeof(key) + size; + return 0; +out_err: + xfile_discard(blob->xfile, blob->last_offset, sizeof(key)); + return error; +} + +/* Free a blob. */ +int +xfblob_free( + struct xfblob *blob, + xfblob_cookie cookie) +{ + struct xb_key key; + int error; + + error = xfile_load(blob->xfile, &key, sizeof(key), cookie); + if (error) + return error; + + if (key.xb_magic != XB_KEY_MAGIC || key.xb_offset != cookie) { + ASSERT(0); + return -ENODATA; + } + + xfile_discard(blob->xfile, cookie, sizeof(key) + key.xb_size); + return 0; +} diff --git a/fs/xfs/scrub/xfblob.h b/fs/xfs/scrub/xfblob.h new file mode 100644 index 0000000000000..bd98647407f1d --- /dev/null +++ b/fs/xfs/scrub/xfblob.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_XFBLOB_H__ +#define __XFS_SCRUB_XFBLOB_H__ + +struct xfblob { + struct xfile *xfile; + loff_t last_offset; +}; + +typedef loff_t xfblob_cookie; + +int xfblob_create(const char *descr, struct xfblob **blobp); +void xfblob_destroy(struct xfblob *blob); +int xfblob_load(struct xfblob *blob, xfblob_cookie cookie, void *ptr, + uint32_t size); +int xfblob_store(struct xfblob *blob, xfblob_cookie *cookie, const void *ptr, + uint32_t size); +int xfblob_free(struct xfblob *blob, xfblob_cookie cookie); + +#endif /* __XFS_SCRUB_XFBLOB_H__ */ From 629fdaf5f5b1b7f7107ed4de04e0991a99501ced Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:44 -0700 Subject: [PATCH 0236/1702] xfs: use atomic extent swapping to fix user file fork data Build on the code that was recently added to the temporary repair file code so that we can atomically switch the contents of any file fork, even if the fork is in local format. The upcoming functions to repair xattrs, directories, and symlinks will need that capability. Repair can lock out access to these user files by holding IOLOCK_EXCL on these user files. Therefore, it is safe to drop the ILOCK of both the file being repaired and the tempfile being used for staging, and cancel the scrub transaction. We do this so that we can reuse the resource estimation and transaction allocation functions used by a regular file exchange operation. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_exchmaps.c | 2 +- fs/xfs/libxfs/xfs_exchmaps.h | 1 + fs/xfs/scrub/tempexch.h | 2 + fs/xfs/scrub/tempfile.c | 204 +++++++++++++++++++++++++++++++++++ fs/xfs/scrub/tempfile.h | 3 + 5 files changed, 211 insertions(+), 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index 3880ae32eecf4..44ab6a9235c0b 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -675,7 +675,7 @@ xfs_exchmaps_rmapbt_blocks( } /* Estimate the bmbt and rmapbt overhead required to exchange mappings. */ -static int +int xfs_exchmaps_estimate_overhead( struct xfs_exchmaps_req *req) { diff --git a/fs/xfs/libxfs/xfs_exchmaps.h b/fs/xfs/libxfs/xfs_exchmaps.h index d8718fca606e5..fa822dff202ad 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.h +++ b/fs/xfs/libxfs/xfs_exchmaps.h @@ -97,6 +97,7 @@ xfs_exchmaps_reqfork(const struct xfs_exchmaps_req *req) return XFS_DATA_FORK; } +int xfs_exchmaps_estimate_overhead(struct xfs_exchmaps_req *req); int xfs_exchmaps_estimate(struct xfs_exchmaps_req *req); extern struct kmem_cache *xfs_exchmaps_intent_cache; diff --git a/fs/xfs/scrub/tempexch.h b/fs/xfs/scrub/tempexch.h index 98222b684b6a0..c1dd4adec4f11 100644 --- a/fs/xfs/scrub/tempexch.h +++ b/fs/xfs/scrub/tempexch.h @@ -14,6 +14,8 @@ struct xrep_tempexch { int xrep_tempexch_enable(struct xfs_scrub *sc); int xrep_tempexch_trans_reserve(struct xfs_scrub *sc, int whichfork, struct xrep_tempexch *ti); +int xrep_tempexch_trans_alloc(struct xfs_scrub *sc, int whichfork, + struct xrep_tempexch *ti); int xrep_tempexch_contents(struct xfs_scrub *sc, struct xrep_tempexch *ti); #endif /* CONFIG_XFS_ONLINE_REPAIR */ diff --git a/fs/xfs/scrub/tempfile.c b/fs/xfs/scrub/tempfile.c index 7791336ca8209..0b3060be938fb 100644 --- a/fs/xfs/scrub/tempfile.c +++ b/fs/xfs/scrub/tempfile.c @@ -239,6 +239,28 @@ xrep_tempfile_iunlock( sc->temp_ilock_flags &= ~XFS_ILOCK_EXCL; } +/* + * Begin the process of making changes to both the file being scrubbed and + * the temporary file by taking ILOCK_EXCL on both. + */ +void +xrep_tempfile_ilock_both( + struct xfs_scrub *sc) +{ + xfs_lock_two_inodes(sc->ip, XFS_ILOCK_EXCL, sc->tempip, XFS_ILOCK_EXCL); + sc->ilock_flags |= XFS_ILOCK_EXCL; + sc->temp_ilock_flags |= XFS_ILOCK_EXCL; +} + +/* Unlock ILOCK_EXCL on both files. */ +void +xrep_tempfile_iunlock_both( + struct xfs_scrub *sc) +{ + xrep_tempfile_iunlock(sc); + xchk_iunlock(sc, XFS_ILOCK_EXCL); +} + /* Release the temporary file. */ void xrep_tempfile_rele( @@ -514,6 +536,89 @@ xrep_tempexch_prep_request( return 0; } +/* + * Fill out the mapping exchange resource estimation structures in preparation + * for exchanging the contents of a metadata file that we've rebuilt in the + * temp file. Caller must hold IOLOCK_EXCL but not ILOCK_EXCL on both files. + */ +STATIC int +xrep_tempexch_estimate( + struct xfs_scrub *sc, + struct xrep_tempexch *tx) +{ + struct xfs_exchmaps_req *req = &tx->req; + struct xfs_ifork *ifp; + struct xfs_ifork *tifp; + int whichfork = xfs_exchmaps_reqfork(req); + int state = 0; + + /* + * The exchmaps code only knows how to exchange file fork space + * mappings. Any fork data in local format must be promoted to a + * single block before the exchange can take place. + */ + ifp = xfs_ifork_ptr(sc->ip, whichfork); + if (ifp->if_format == XFS_DINODE_FMT_LOCAL) + state |= 1; + + tifp = xfs_ifork_ptr(sc->tempip, whichfork); + if (tifp->if_format == XFS_DINODE_FMT_LOCAL) + state |= 2; + + switch (state) { + case 0: + /* Both files have mapped extents; use the regular estimate. */ + return xfs_exchrange_estimate(req); + case 1: + /* + * The file being repaired is in local format, but the temp + * file has mapped extents. To perform the exchange, the file + * being repaired must have its shorform data converted to an + * ondisk block so that the forks will be in extents format. + * We need one resblk for the conversion; the number of + * exchanges is (worst case) the temporary file's extent count + * plus the block we converted. + */ + req->ip1_bcount = sc->tempip->i_nblocks; + req->ip2_bcount = 1; + req->nr_exchanges = 1 + tifp->if_nextents; + req->resblks = 1; + break; + case 2: + /* + * The temporary file is in local format, but the file being + * repaired has mapped extents. To perform the exchange, the + * temp file must have its shortform data converted to an + * ondisk block, and the fork changed to extents format. We + * need one resblk for the conversion; the number of exchanges + * is (worst case) the extent count of the file being repaired + * plus the block we converted. + */ + req->ip1_bcount = 1; + req->ip2_bcount = sc->ip->i_nblocks; + req->nr_exchanges = 1 + ifp->if_nextents; + req->resblks = 1; + break; + case 3: + /* + * Both forks are in local format. To perform the exchange, + * both files must have their shortform data converted to + * fsblocks, and both forks must be converted to extents + * format. We need two resblks for the two conversions, and + * the number of exchanges is 1 since there's only one block at + * fileoff 0. Presumably, the caller could not exchange the + * two inode fork areas directly. + */ + req->ip1_bcount = 1; + req->ip2_bcount = 1; + req->nr_exchanges = 1; + req->resblks = 2; + break; + } + + return xfs_exchmaps_estimate_overhead(req); +} + /* * Obtain a quota reservation to make sure we don't hit EDQUOT. We can skip * this if quota enforcement is disabled or if both inodes' dquots are the @@ -604,6 +709,55 @@ xrep_tempexch_trans_reserve( return xrep_tempexch_reserve_quota(sc, tx); } +/* + * Create a new transaction for a file contents exchange. + * + * This function fills out the mapping excahange request and resource + * estimation structures in preparation for exchanging the contents of a + * metadata file that has been rebuilt in the temp file. Next, it reserves + * space, takes ILOCK_EXCL of both inodes, joins them to the transaction and + * reserves quota for the transaction. + * + * The caller is responsible for dropping both ILOCKs when appropriate. + */ +int +xrep_tempexch_trans_alloc( + struct xfs_scrub *sc, + int whichfork, + struct xrep_tempexch *tx) +{ + unsigned int flags = 0; + int error; + + ASSERT(sc->tp == NULL); + + error = xrep_tempexch_prep_request(sc, whichfork, tx); + if (error) + return error; + + error = xrep_tempexch_estimate(sc, tx); + if (error) + return error; + + if (xfs_has_lazysbcount(sc->mp)) + flags |= XFS_TRANS_RES_FDBLKS; + + error = xrep_tempexch_enable(sc); + if (error) + return error; + + error = xfs_trans_alloc(sc->mp, &M_RES(sc->mp)->tr_itruncate, + tx->req.resblks, 0, flags, &sc->tp); + if (error) + return error; + + sc->temp_ilock_flags |= XFS_ILOCK_EXCL; + sc->ilock_flags |= XFS_ILOCK_EXCL; + xfs_exchrange_ilock(sc->tp, sc->ip, sc->tempip); + + return xrep_tempexch_reserve_quota(sc, tx); +} + /* * Exchange file mappings (and hence file contents) between the file being * repaired and the temporary file. Returns with both inodes locked and joined @@ -637,3 +791,53 @@ xrep_tempexch_contents( return 0; } + +/* + * Write local format data from one of the temporary file's forks into the same + * fork of file being repaired, and exchange the file sizes, if appropriate. + * Caller must ensure that the file being repaired has enough fork space to + * hold all the bytes. + */ +void +xrep_tempfile_copyout_local( + struct xfs_scrub *sc, + int whichfork) +{ + struct xfs_ifork *temp_ifp; + struct xfs_ifork *ifp; + unsigned int ilog_flags = XFS_ILOG_CORE; + + temp_ifp = xfs_ifork_ptr(sc->tempip, whichfork); + ifp = xfs_ifork_ptr(sc->ip, whichfork); + + ASSERT(temp_ifp != NULL); + ASSERT(ifp != NULL); + ASSERT(temp_ifp->if_format == XFS_DINODE_FMT_LOCAL); + ASSERT(ifp->if_format == XFS_DINODE_FMT_LOCAL); + + switch (whichfork) { + case XFS_DATA_FORK: + ASSERT(sc->tempip->i_disk_size <= + xfs_inode_data_fork_size(sc->ip)); + break; + case XFS_ATTR_FORK: + ASSERT(sc->tempip->i_forkoff >= sc->ip->i_forkoff); + break; + default: + ASSERT(0); + return; + } + + /* Recreate @sc->ip's incore fork (ifp) with data from temp_ifp. */ + xfs_idestroy_fork(ifp); + xfs_init_local_fork(sc->ip, whichfork, temp_ifp->if_data, + temp_ifp->if_bytes); + + if (whichfork == XFS_DATA_FORK) { + i_size_write(VFS_I(sc->ip), i_size_read(VFS_I(sc->tempip))); + sc->ip->i_disk_size = sc->tempip->i_disk_size; + } + + ilog_flags |= xfs_ilog_fdata(whichfork); + xfs_trans_log_inode(sc->tp, sc->ip, ilog_flags); +} diff --git a/fs/xfs/scrub/tempfile.h b/fs/xfs/scrub/tempfile.h index 7980f9c4de552..d57e4f145a7c8 100644 --- a/fs/xfs/scrub/tempfile.h +++ b/fs/xfs/scrub/tempfile.h @@ -17,6 +17,8 @@ void xrep_tempfile_iounlock(struct xfs_scrub *sc); void xrep_tempfile_ilock(struct xfs_scrub *sc); bool xrep_tempfile_ilock_nowait(struct xfs_scrub *sc); void xrep_tempfile_iunlock(struct xfs_scrub *sc); +void xrep_tempfile_iunlock_both(struct xfs_scrub *sc); +void xrep_tempfile_ilock_both(struct xfs_scrub *sc); int xrep_tempfile_prealloc(struct xfs_scrub *sc, xfs_fileoff_t off, xfs_filblks_t len); @@ -32,6 +34,7 @@ int xrep_tempfile_copyin(struct xfs_scrub *sc, xfs_fileoff_t off, int xrep_tempfile_set_isize(struct xfs_scrub *sc, unsigned long long isize); int xrep_tempfile_roll_trans(struct xfs_scrub *sc); +void xrep_tempfile_copyout_local(struct xfs_scrub *sc, int whichfork); #else static inline void xrep_tempfile_iolock_both(struct xfs_scrub *sc) { From e47dcf113ae348678143cc935a1183059c02c9ad Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:45 -0700 Subject: [PATCH 0237/1702] xfs: repair extended attributes If the extended attributes look bad, try to sift through the rubble to find whatever keys/values we can, stage a new attribute structure in a temporary file and use the atomic extent swapping mechanism to commit the results in bulk. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/libxfs/xfs_attr.c | 2 +- fs/xfs/libxfs/xfs_attr.h | 2 + fs/xfs/libxfs/xfs_da_format.h | 5 + fs/xfs/scrub/attr.c | 20 +- fs/xfs/scrub/attr.h | 7 + fs/xfs/scrub/attr_repair.c | 1207 +++++++++++++++++++++++++++++++++ fs/xfs/scrub/attr_repair.h | 11 + fs/xfs/scrub/repair.c | 46 ++ fs/xfs/scrub/repair.h | 6 + fs/xfs/scrub/scrub.c | 2 +- fs/xfs/scrub/trace.h | 83 +++ fs/xfs/scrub/xfarray.c | 17 + fs/xfs/scrub/xfarray.h | 2 + fs/xfs/scrub/xfblob.c | 17 + fs/xfs/scrub/xfblob.h | 2 + fs/xfs/scrub/xfile.h | 5 + fs/xfs/xfs_buf.c | 3 + fs/xfs/xfs_trace.h | 2 + 19 files changed, 1436 insertions(+), 4 deletions(-) create mode 100644 fs/xfs/scrub/attr_repair.c create mode 100644 fs/xfs/scrub/attr_repair.h diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index bc27757702fe8..8647629ac7bf9 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -194,6 +194,7 @@ ifeq ($(CONFIG_XFS_ONLINE_REPAIR),y) xfs-y += $(addprefix scrub/, \ agheader_repair.o \ alloc_repair.o \ + attr_repair.o \ bmap_repair.o \ cow_repair.o \ fscounters_repair.o \ diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index b3c9666cd0115..05d22c5e38855 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -1055,7 +1055,7 @@ xfs_attr_set( * External routines when attribute list is inside the inode *========================================================================*/ -static inline int xfs_attr_sf_totsize(struct xfs_inode *dp) +int xfs_attr_sf_totsize(struct xfs_inode *dp) { struct xfs_attr_sf_hdr *sf = dp->i_af.if_data; diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h index 81be9b3e40047..e4f55008552b4 100644 --- a/fs/xfs/libxfs/xfs_attr.h +++ b/fs/xfs/libxfs/xfs_attr.h @@ -618,4 +618,6 @@ extern struct kmem_cache *xfs_attr_intent_cache; int __init xfs_attr_intent_init_cache(void); void xfs_attr_intent_destroy_cache(void); +int xfs_attr_sf_totsize(struct xfs_inode *dp); + #endif /* __XFS_ATTR_H__ */ diff --git a/fs/xfs/libxfs/xfs_da_format.h b/fs/xfs/libxfs/xfs_da_format.h index 060e5c96b70f6..aac3fe0396140 100644 --- a/fs/xfs/libxfs/xfs_da_format.h +++ b/fs/xfs/libxfs/xfs_da_format.h @@ -721,6 +721,11 @@ struct xfs_attr3_leafblock { #define XFS_ATTR_INCOMPLETE (1u << XFS_ATTR_INCOMPLETE_BIT) #define XFS_ATTR_NSP_ONDISK_MASK (XFS_ATTR_ROOT | XFS_ATTR_SECURE) +#define XFS_ATTR_NAMESPACE_STR \ + { XFS_ATTR_LOCAL, "local" }, \ + { XFS_ATTR_ROOT, "root" }, \ + { XFS_ATTR_SECURE, "secure" } + /* * Alignment for namelist and valuelist entries (since they are mixed * there can be only one alignment value) diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index 0c467f4f8e778..7621e548d7301 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -10,6 +10,7 @@ #include "xfs_trans_resv.h" #include "xfs_mount.h" #include "xfs_log_format.h" +#include "xfs_trans.h" #include "xfs_inode.h" #include "xfs_da_format.h" #include "xfs_da_btree.h" @@ -20,6 +21,7 @@ #include "scrub/common.h" #include "scrub/dabtree.h" #include "scrub/attr.h" +#include "scrub/repair.h" /* Free the buffers linked from the xattr buffer. */ static void @@ -35,6 +37,8 @@ xchk_xattr_buf_cleanup( kvfree(ab->value); ab->value = NULL; ab->value_sz = 0; + kvfree(ab->name); + ab->name = NULL; } /* @@ -65,7 +69,7 @@ xchk_xattr_want_freemap( * reallocating the buffer if necessary. Buffer contents are not preserved * across a reallocation. */ -static int +int xchk_setup_xattr_buf( struct xfs_scrub *sc, size_t value_size) @@ -95,6 +99,12 @@ xchk_setup_xattr_buf( return -ENOMEM; } + if (xchk_could_repair(sc)) { + ab->name = kvmalloc(XATTR_NAME_MAX + 1, XCHK_GFP_FLAGS); + if (!ab->name) + return -ENOMEM; + } + resize_value: if (ab->value_sz >= value_size) return 0; @@ -121,6 +131,12 @@ xchk_setup_xattr( { int error; + if (xchk_could_repair(sc)) { + error = xrep_setup_xattr(sc); + if (error) + return error; + } + /* * We failed to get memory while checking attrs, so this time try to * get all the memory we're ever going to need. Allocate the buffer @@ -247,7 +263,7 @@ xchk_xattr_listent( * Within a char, the lowest bit of the char represents the byte with * the smallest address */ -STATIC bool +bool xchk_xattr_set_map( struct xfs_scrub *sc, unsigned long *map, diff --git a/fs/xfs/scrub/attr.h b/fs/xfs/scrub/attr.h index 48fd9402c4328..7db58af56646b 100644 --- a/fs/xfs/scrub/attr.h +++ b/fs/xfs/scrub/attr.h @@ -16,9 +16,16 @@ struct xchk_xattr_buf { /* Bitmap of free space in xattr leaf blocks. */ unsigned long *freemap; + /* Memory buffer used to hold salvaged xattr names. */ + unsigned char *name; + /* Memory buffer used to extract xattr values. */ void *value; size_t value_sz; }; +bool xchk_xattr_set_map(struct xfs_scrub *sc, unsigned long *map, + unsigned int start, unsigned int len); +int xchk_setup_xattr_buf(struct xfs_scrub *sc, size_t value_size); + #endif /* __XFS_SCRUB_ATTR_H__ */ diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c new file mode 100644 index 0000000000000..7b4318764d030 --- /dev/null +++ b/fs/xfs/scrub/attr_repair.c @@ -0,0 +1,1207 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_dir2.h" +#include "xfs_attr.h" +#include "xfs_attr_leaf.h" +#include "xfs_attr_sf.h" +#include "xfs_attr_remote.h" +#include "xfs_bmap.h" +#include "xfs_bmap_util.h" +#include "xfs_exchmaps.h" +#include "xfs_exchrange.h" +#include "xfs_acl.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/tempfile.h" +#include "scrub/tempexch.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" +#include "scrub/attr.h" +#include "scrub/reap.h" +#include "scrub/attr_repair.h" + +/* + * Extended Attribute Repair + * ========================= + * + * We repair extended attributes by reading the attr leaf blocks looking for + * attributes entries that look salvageable (name passes verifiers, value can + * be retrieved, etc). Each extended attribute worth salvaging is stashed in + * memory, and the stashed entries are periodically replayed into a temporary + * file to constrain memory use. Batching the construction of the temporary + * extended attribute structure in this fashion reduces lock cycling of the + * file being repaired and the temporary file. + * + * When salvaging completes, the remaining stashed attributes are replayed to + * the temporary file. An atomic file contents exchange is used to commit the + * new xattr blocks to the file being repaired. This will disrupt attrmulti + * cursors. + */ + +struct xrep_xattr_key { + /* Cookie for retrieval of the xattr name. */ + xfblob_cookie name_cookie; + + /* Cookie for retrieval of the xattr value. */ + xfblob_cookie value_cookie; + + /* XFS_ATTR_* flags */ + int flags; + + /* Length of the value and name. */ + uint32_t valuelen; + uint16_t namelen; +}; + +/* + * Stash up to 8 pages of attrs in xattr_records/xattr_blobs before we write + * them to the temp file. + */ +#define XREP_XATTR_MAX_STASH_BYTES (PAGE_SIZE * 8) + +struct xrep_xattr { + struct xfs_scrub *sc; + + /* Information for exchanging attr fork mappings at the end. */ + struct xrep_tempexch tx; + + /* xattr keys */ + struct xfarray *xattr_records; + + /* xattr values */ + struct xfblob *xattr_blobs; + + /* Number of attributes that we are salvaging. */ + unsigned long long attrs_found; +}; + +/* Set up to recreate the extended attributes. */ +int +xrep_setup_xattr( + struct xfs_scrub *sc) +{ + return xrep_tempfile_create(sc, S_IFREG); +} + +/* + * Decide if we want to salvage this attribute. We don't bother with + * incomplete or oversized keys or values. The @value parameter can be null + * for remote attrs. + */ +STATIC int +xrep_xattr_want_salvage( + struct xrep_xattr *rx, + unsigned int attr_flags, + const void *name, + int namelen, + const void *value, + int valuelen) +{ + if (attr_flags & XFS_ATTR_INCOMPLETE) + return false; + if (namelen > XATTR_NAME_MAX || namelen <= 0) + return false; + if (!xfs_attr_namecheck(name, namelen)) + return false; + if (valuelen > XATTR_SIZE_MAX || valuelen < 0) + return false; + if (hweight32(attr_flags & XFS_ATTR_NSP_ONDISK_MASK) > 1) + return false; + return true; +} + +/* Allocate an in-core record to hold xattrs while we rebuild the xattr data. */ +STATIC int +xrep_xattr_salvage_key( + struct xrep_xattr *rx, + int flags, + unsigned char *name, + int namelen, + unsigned char *value, + int valuelen) +{ + struct xrep_xattr_key key = { + .valuelen = valuelen, + .flags = flags & XFS_ATTR_NSP_ONDISK_MASK, + }; + unsigned int i = 0; + int error = 0; + + if (xchk_should_terminate(rx->sc, &error)) + return error; + + /* + * Truncate the name to the first character that would trip namecheck. + * If we no longer have a name after that, ignore this attribute. + */ + while (i < namelen && name[i] != 0) + i++; + if (i == 0) + return 0; + key.namelen = i; + + trace_xrep_xattr_salvage_rec(rx->sc->ip, flags, name, key.namelen, + valuelen); + + error = xfblob_store(rx->xattr_blobs, &key.name_cookie, name, + key.namelen); + if (error) + return error; + + error = xfblob_store(rx->xattr_blobs, &key.value_cookie, value, + key.valuelen); + if (error) + return error; + + error = xfarray_append(rx->xattr_records, &key); + if (error) + return error; + + rx->attrs_found++; + return 0; +} + +/* + * Record a shortform extended attribute key & value for later reinsertion + * into the inode. + */ +STATIC int +xrep_xattr_salvage_sf_attr( + struct xrep_xattr *rx, + struct xfs_attr_sf_hdr *hdr, + struct xfs_attr_sf_entry *sfe) +{ + struct xfs_scrub *sc = rx->sc; + struct xchk_xattr_buf *ab = sc->buf; + unsigned char *name = sfe->nameval; + unsigned char *value = &sfe->nameval[sfe->namelen]; + + if (!xchk_xattr_set_map(sc, ab->usedmap, (char *)name - (char *)hdr, + sfe->namelen)) + return 0; + + if (!xchk_xattr_set_map(sc, ab->usedmap, (char *)value - (char *)hdr, + sfe->valuelen)) + return 0; + + if (!xrep_xattr_want_salvage(rx, sfe->flags, sfe->nameval, + sfe->namelen, value, sfe->valuelen)) + return 0; + + return xrep_xattr_salvage_key(rx, sfe->flags, sfe->nameval, + sfe->namelen, value, sfe->valuelen); +} + +/* + * Record a local format extended attribute key & value for later reinsertion + * into the inode. + */ +STATIC int +xrep_xattr_salvage_local_attr( + struct xrep_xattr *rx, + struct xfs_attr_leaf_entry *ent, + unsigned int nameidx, + const char *buf_end, + struct xfs_attr_leaf_name_local *lentry) +{ + struct xchk_xattr_buf *ab = rx->sc->buf; + unsigned char *value; + unsigned int valuelen; + unsigned int namesize; + + /* + * Decode the leaf local entry format. If something seems wrong, we + * junk the attribute. + */ + value = &lentry->nameval[lentry->namelen]; + valuelen = be16_to_cpu(lentry->valuelen); + namesize = xfs_attr_leaf_entsize_local(lentry->namelen, valuelen); + if ((char *)lentry + namesize > buf_end) + return 0; + if (!xrep_xattr_want_salvage(rx, ent->flags, lentry->nameval, + lentry->namelen, value, valuelen)) + return 0; + if (!xchk_xattr_set_map(rx->sc, ab->usedmap, nameidx, namesize)) + return 0; + + /* Try to save this attribute. */ + return xrep_xattr_salvage_key(rx, ent->flags, lentry->nameval, + lentry->namelen, value, valuelen); +} + +/* + * Record a remote format extended attribute key & value for later reinsertion + * into the inode. + */ +STATIC int +xrep_xattr_salvage_remote_attr( + struct xrep_xattr *rx, + struct xfs_attr_leaf_entry *ent, + unsigned int nameidx, + const char *buf_end, + struct xfs_attr_leaf_name_remote *rentry, + unsigned int ent_idx, + struct xfs_buf *leaf_bp) +{ + struct xchk_xattr_buf *ab = rx->sc->buf; + struct xfs_da_args args = { + .trans = rx->sc->tp, + .dp = rx->sc->ip, + .index = ent_idx, + .geo = rx->sc->mp->m_attr_geo, + .owner = rx->sc->ip->i_ino, + .attr_filter = ent->flags & XFS_ATTR_NSP_ONDISK_MASK, + .namelen = rentry->namelen, + .name = rentry->name, + .value = ab->value, + .valuelen = be32_to_cpu(rentry->valuelen), + }; + unsigned int namesize; + int error; + + /* + * Decode the leaf remote entry format. If something seems wrong, we + * junk the attribute. Note that we should never find a zero-length + * remote attribute value. + */ + namesize = xfs_attr_leaf_entsize_remote(rentry->namelen); + if ((char *)rentry + namesize > buf_end) + return 0; + if (args.valuelen == 0 || + !xrep_xattr_want_salvage(rx, ent->flags, rentry->name, + rentry->namelen, NULL, args.valuelen)) + return 0; + if (!xchk_xattr_set_map(rx->sc, ab->usedmap, nameidx, namesize)) + return 0; + + /* + * Enlarge the buffer (if needed) to hold the value that we're trying + * to salvage from the old extended attribute data. + */ + error = xchk_setup_xattr_buf(rx->sc, args.valuelen); + if (error == -ENOMEM) + error = -EDEADLOCK; + if (error) + return error; + + /* Look up the remote value and stash it for reconstruction. */ + error = xfs_attr3_leaf_getvalue(leaf_bp, &args); + if (error || args.rmtblkno == 0) + goto err_free; + + error = xfs_attr_rmtval_get(&args); + if (error) + goto err_free; + + /* Try to save this attribute. */ + error = xrep_xattr_salvage_key(rx, ent->flags, rentry->name, + rentry->namelen, ab->value, args.valuelen); +err_free: + /* remote value was garbage, junk it */ + if (error == -EFSBADCRC || error == -EFSCORRUPTED) + error = 0; + return error; +} + +/* Extract every xattr key that we can from this attr fork block. */ +STATIC int +xrep_xattr_recover_leaf( + struct xrep_xattr *rx, + struct xfs_buf *bp) +{ + struct xfs_attr3_icleaf_hdr leafhdr; + struct xfs_scrub *sc = rx->sc; + struct xfs_mount *mp = sc->mp; + struct xfs_attr_leafblock *leaf; + struct xfs_attr_leaf_name_local *lentry; + struct xfs_attr_leaf_name_remote *rentry; + struct xfs_attr_leaf_entry *ent; + struct xfs_attr_leaf_entry *entries; + struct xchk_xattr_buf *ab = rx->sc->buf; + char *buf_end; + size_t off; + unsigned int nameidx; + unsigned int hdrsize; + int i; + int error = 0; + + bitmap_zero(ab->usedmap, mp->m_attr_geo->blksize); + + /* Check the leaf header */ + leaf = bp->b_addr; + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &leafhdr, leaf); + hdrsize = xfs_attr3_leaf_hdr_size(leaf); + xchk_xattr_set_map(sc, ab->usedmap, 0, hdrsize); + entries = xfs_attr3_leaf_entryp(leaf); + + buf_end = (char *)bp->b_addr + mp->m_attr_geo->blksize; + for (i = 0, ent = entries; i < leafhdr.count; ent++, i++) { + if (xchk_should_terminate(sc, &error)) + return error; + + /* Skip key if it conflicts with something else? */ + off = (char *)ent - (char *)leaf; + if (!xchk_xattr_set_map(sc, ab->usedmap, off, + sizeof(xfs_attr_leaf_entry_t))) + continue; + + /* Check the name information. */ + nameidx = be16_to_cpu(ent->nameidx); + if (nameidx < leafhdr.firstused || + nameidx >= mp->m_attr_geo->blksize) + continue; + + if (ent->flags & XFS_ATTR_LOCAL) { + lentry = xfs_attr3_leaf_name_local(leaf, i); + error = xrep_xattr_salvage_local_attr(rx, ent, nameidx, + buf_end, lentry); + } else { + rentry = xfs_attr3_leaf_name_remote(leaf, i); + error = xrep_xattr_salvage_remote_attr(rx, ent, nameidx, + buf_end, rentry, i, bp); + } + if (error) + return error; + } + + return 0; +} + +/* Try to recover shortform attrs. */ +STATIC int +xrep_xattr_recover_sf( + struct xrep_xattr *rx) +{ + struct xfs_scrub *sc = rx->sc; + struct xchk_xattr_buf *ab = sc->buf; + struct xfs_attr_sf_hdr *hdr; + struct xfs_attr_sf_entry *sfe; + struct xfs_attr_sf_entry *next; + struct xfs_ifork *ifp; + unsigned char *end; + int i; + int error = 0; + + ifp = xfs_ifork_ptr(rx->sc->ip, XFS_ATTR_FORK); + hdr = ifp->if_data; + + bitmap_zero(ab->usedmap, ifp->if_bytes); + end = (unsigned char *)ifp->if_data + ifp->if_bytes; + xchk_xattr_set_map(sc, ab->usedmap, 0, sizeof(*hdr)); + + sfe = xfs_attr_sf_firstentry(hdr); + if ((unsigned char *)sfe > end) + return 0; + + for (i = 0; i < hdr->count; i++) { + if (xchk_should_terminate(sc, &error)) + return error; + + next = xfs_attr_sf_nextentry(sfe); + if ((unsigned char *)next > end) + break; + + if (xchk_xattr_set_map(sc, ab->usedmap, + (char *)sfe - (char *)hdr, + sizeof(struct xfs_attr_sf_entry))) { + /* + * No conflicts with the sf entry; let's save this + * attribute. + */ + error = xrep_xattr_salvage_sf_attr(rx, hdr, sfe); + if (error) + return error; + } + + sfe = next; + } + + return 0; +} + +/* + * Try to return a buffer of xattr data for a given physical extent. + * + * Because the buffer cache get function complains if it finds a buffer + * matching the block number but not matching the length, we must be careful to + * look for incore buffers (up to the maximum length of a remote value) that + * could be hiding anywhere in the physical range. If we find an incore + * buffer, we can pass that to the caller. Optionally, read a single block and + * pass that back. + * + * Note the subtlety that remote attr value blocks for which there is no incore + * buffer will be passed to the callback one block at a time. These buffers + * will not have any ops attached and must be staled to prevent aliasing with + * multiblock buffers once we drop the ILOCK. + */ +STATIC int +xrep_xattr_find_buf( + struct xfs_mount *mp, + xfs_fsblock_t fsbno, + xfs_extlen_t max_len, + bool can_read, + struct xfs_buf **bpp) +{ + struct xrep_bufscan scan = { + .daddr = XFS_FSB_TO_DADDR(mp, fsbno), + .max_sectors = xrep_bufscan_max_sectors(mp, max_len), + .daddr_step = XFS_FSB_TO_BB(mp, 1), + }; + struct xfs_buf *bp; + + while ((bp = xrep_bufscan_advance(mp, &scan)) != NULL) { + *bpp = bp; + return 0; + } + + if (!can_read) { + *bpp = NULL; + return 0; + } + + return xfs_buf_read(mp->m_ddev_targp, scan.daddr, XFS_FSB_TO_BB(mp, 1), + XBF_TRYLOCK, bpp, NULL); +} + +/* + * Deal with a buffer that we found during our walk of the attr fork. + * + * Attribute leaf and node blocks are simple -- they're a single block, so we + * can walk them one at a time and we never have to worry about discontiguous + * multiblock buffers like we do for directories. + * + * Unfortunately, remote attr blocks add a lot of complexity here. Each disk + * block is totally self contained, in the sense that the v5 header provides no + * indication that there could be more data in the next block. The incore + * buffers can span multiple blocks, though they never cross extent records. + * However, they don't necessarily start or end on an extent record boundary. + * Therefore, we need a special buffer find function to walk the buffer cache + * for us. + * + * The caller must hold the ILOCK on the file being repaired. We use + * XBF_TRYLOCK here to skip any locked buffer on the assumption that we don't + * own the block and don't want to hang the system on a potentially garbage + * buffer. + */ +STATIC int +xrep_xattr_recover_block( + struct xrep_xattr *rx, + xfs_dablk_t dabno, + xfs_fsblock_t fsbno, + xfs_extlen_t max_len, + xfs_extlen_t *actual_len) +{ + struct xfs_da_blkinfo *info; + struct xfs_buf *bp; + int error; + + error = xrep_xattr_find_buf(rx->sc->mp, fsbno, max_len, true, &bp); + if (error) + return error; + info = bp->b_addr; + *actual_len = XFS_BB_TO_FSB(rx->sc->mp, bp->b_length); + + trace_xrep_xattr_recover_leafblock(rx->sc->ip, dabno, + be16_to_cpu(info->magic)); + + /* + * If the buffer has the right magic number for an attr leaf block and + * passes a structure check (we don't care about checksums), salvage + * as much as we can from the block. */ + if (info->magic == cpu_to_be16(XFS_ATTR3_LEAF_MAGIC) && + xrep_buf_verify_struct(bp, &xfs_attr3_leaf_buf_ops) && + xfs_attr3_leaf_header_check(bp, rx->sc->ip->i_ino) == NULL) + error = xrep_xattr_recover_leaf(rx, bp); + + /* + * If the buffer didn't already have buffer ops set, it was read in by + * the _find_buf function and could very well be /part/ of a multiblock + * remote block. Mark it stale so that it doesn't hang around in + * memory to cause problems. + */ + if (bp->b_ops == NULL) + xfs_buf_stale(bp); + + xfs_buf_relse(bp); + return error; +} + +/* Insert one xattr key/value. */ +STATIC int +xrep_xattr_insert_rec( + struct xrep_xattr *rx, + const struct xrep_xattr_key *key) +{ + struct xfs_da_args args = { + .dp = rx->sc->tempip, + .attr_filter = key->flags, + .attr_flags = XATTR_CREATE, + .namelen = key->namelen, + .valuelen = key->valuelen, + .owner = rx->sc->ip->i_ino, + }; + struct xchk_xattr_buf *ab = rx->sc->buf; + int error; + + /* + * Grab pointers to the scrub buffer so that we can use them to insert + * attrs into the temp file. + */ + args.name = ab->name; + args.value = ab->value; + + /* + * The attribute name is stored near the end of the in-core buffer, + * though we reserve one more byte to ensure null termination. + */ + ab->name[XATTR_NAME_MAX] = 0; + + error = xfblob_load(rx->xattr_blobs, key->name_cookie, ab->name, + key->namelen); + if (error) + return error; + + error = xfblob_free(rx->xattr_blobs, key->name_cookie); + if (error) + return error; + + error = xfblob_load(rx->xattr_blobs, key->value_cookie, args.value, + key->valuelen); + if (error) + return error; + + error = xfblob_free(rx->xattr_blobs, key->value_cookie); + if (error) + return error; + + ab->name[key->namelen] = 0; + + trace_xrep_xattr_insert_rec(rx->sc->tempip, key->flags, ab->name, + key->namelen, key->valuelen); + + /* + * xfs_attr_set creates and commits its own transaction. If the attr + * already exists, we'll just drop it during the rebuild. + */ + error = xfs_attr_set(&args); + if (error == -EEXIST) + error = 0; + + return error; +} + +/* + * Periodically flush salvaged attributes to the temporary file. This is done + * to reduce the memory requirements of the xattr rebuild because files can + * contain millions of attributes. + */ +STATIC int +xrep_xattr_flush_stashed( + struct xrep_xattr *rx) +{ + xfarray_idx_t array_cur; + int error; + + /* + * Entering this function, the scrub context has a reference to the + * inode being repaired, the temporary file, and a scrub transaction + * that we use during xattr salvaging to avoid livelocking if there + * are cycles in the xattr structures. We hold ILOCK_EXCL on both + * the inode being repaired, though it is not ijoined to the scrub + * transaction. + * + * To constrain kernel memory use, we occasionally flush salvaged + * xattrs from the xfarray and xfblob structures into the temporary + * file in preparation for exchanging the xattr structures at the end. + * Updating the temporary file requires a transaction, so we commit the + * scrub transaction and drop the two ILOCKs so that xfs_attr_set can + * allocate whatever transaction it wants. + * + * We still hold IOLOCK_EXCL on the inode being repaired, which + * prevents anyone from modifying the damaged xattr data while we + * repair it. + */ + error = xrep_trans_commit(rx->sc); + if (error) + return error; + xchk_iunlock(rx->sc, XFS_ILOCK_EXCL); + + /* + * Take the IOLOCK of the temporary file while we modify xattrs. This + * isn't strictly required because the temporary file is never revealed + * to userspace, but we follow the same locking rules. We still hold + * sc->ip's IOLOCK. + */ + error = xrep_tempfile_iolock_polled(rx->sc); + if (error) + return error; + + /* Add all the salvaged attrs to the temporary file. */ + foreach_xfarray_idx(rx->xattr_records, array_cur) { + struct xrep_xattr_key key; + + error = xfarray_load(rx->xattr_records, array_cur, &key); + if (error) + return error; + + error = xrep_xattr_insert_rec(rx, &key); + if (error) + return error; + } + + /* Empty out both arrays now that we've added the entries. */ + xfarray_truncate(rx->xattr_records); + xfblob_truncate(rx->xattr_blobs); + + xrep_tempfile_iounlock(rx->sc); + + /* Recreate the salvage transaction and relock the inode. */ + error = xchk_trans_alloc(rx->sc, 0); + if (error) + return error; + xchk_ilock(rx->sc, XFS_ILOCK_EXCL); + return 0; +} + +/* Decide if we've stashed too much xattr data in memory. */ +static inline bool +xrep_xattr_want_flush_stashed( + struct xrep_xattr *rx) +{ + unsigned long long bytes; + + bytes = xfarray_bytes(rx->xattr_records) + + xfblob_bytes(rx->xattr_blobs); + return bytes > XREP_XATTR_MAX_STASH_BYTES; +} + +/* Extract as many attribute keys and values as we can. */ +STATIC int +xrep_xattr_recover( + struct xrep_xattr *rx) +{ + struct xfs_bmbt_irec got; + struct xfs_scrub *sc = rx->sc; + struct xfs_da_geometry *geo = sc->mp->m_attr_geo; + xfs_fileoff_t offset; + xfs_extlen_t len; + xfs_dablk_t dabno; + int nmap; + int error; + + /* + * Iterate each xattr leaf block in the attr fork to scan them for any + * attributes that we might salvage. + */ + for (offset = 0; + offset < XFS_MAX_FILEOFF; + offset = got.br_startoff + got.br_blockcount) { + nmap = 1; + error = xfs_bmapi_read(sc->ip, offset, XFS_MAX_FILEOFF - offset, + &got, &nmap, XFS_BMAPI_ATTRFORK); + if (error) + return error; + if (nmap != 1) + return -EFSCORRUPTED; + if (!xfs_bmap_is_written_extent(&got)) + continue; + + for (dabno = round_up(got.br_startoff, geo->fsbcount); + dabno < got.br_startoff + got.br_blockcount; + dabno += len) { + xfs_fileoff_t curr_offset = dabno - got.br_startoff; + xfs_extlen_t maxlen; + + if (xchk_should_terminate(rx->sc, &error)) + return error; + + maxlen = min_t(xfs_filblks_t, INT_MAX, + got.br_blockcount - curr_offset); + error = xrep_xattr_recover_block(rx, dabno, + curr_offset + got.br_startblock, + maxlen, &len); + if (error) + return error; + + if (xrep_xattr_want_flush_stashed(rx)) { + error = xrep_xattr_flush_stashed(rx); + if (error) + return error; + } + } + } + + return 0; +} + +/* + * Reset the extended attribute fork to a state where we can start re-adding + * the salvaged attributes. + */ +STATIC int +xrep_xattr_fork_remove( + struct xfs_scrub *sc, + struct xfs_inode *ip) +{ + struct xfs_attr_sf_hdr *hdr; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_ATTR_FORK); + + /* + * If the data fork is in btree format, we can't change di_forkoff + * because we could run afoul of the rule that the data fork isn't + * supposed to be in btree format if there's enough space in the fork + * that it could have used extents format. Instead, reinitialize the + * attr fork to have a shortform structure with zero attributes. + */ + if (ip->i_df.if_format == XFS_DINODE_FMT_BTREE) { + ifp->if_format = XFS_DINODE_FMT_LOCAL; + hdr = xfs_idata_realloc(ip, (int)sizeof(*hdr) - ifp->if_bytes, + XFS_ATTR_FORK); + hdr->count = 0; + hdr->totsize = cpu_to_be16(sizeof(*hdr)); + xfs_trans_log_inode(sc->tp, ip, + XFS_ILOG_CORE | XFS_ILOG_ADATA); + return 0; + } + + /* If we still have attr fork extents, something's wrong. */ + if (ifp->if_nextents != 0) { + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec irec; + unsigned int i = 0; + + xfs_emerg(sc->mp, + "inode 0x%llx attr fork still has %llu attr extents, format %d?!", + ip->i_ino, ifp->if_nextents, ifp->if_format); + for_each_xfs_iext(ifp, &icur, &irec) { + xfs_err(sc->mp, + "[%u]: startoff %llu startblock %llu blockcount %llu state %u", + i++, irec.br_startoff, + irec.br_startblock, irec.br_blockcount, + irec.br_state); + } + ASSERT(0); + return -EFSCORRUPTED; + } + + xfs_attr_fork_remove(ip, sc->tp); + return 0; +} + +/* + * Free all the attribute fork blocks of the file being repaired and delete the + * fork. The caller must ILOCK the scrub file and join it to the transaction. + * This function returns with the inode joined to a clean transaction. + */ +int +xrep_xattr_reset_fork( + struct xfs_scrub *sc) +{ + int error; + + trace_xrep_xattr_reset_fork(sc->ip, sc->ip); + + /* Unmap all the attr blocks. */ + if (xfs_ifork_has_extents(&sc->ip->i_af)) { + error = xrep_reap_ifork(sc, sc->ip, XFS_ATTR_FORK); + if (error) + return error; + } + + error = xrep_xattr_fork_remove(sc, sc->ip); + if (error) + return error; + + return xfs_trans_roll_inode(&sc->tp, sc->ip); +} + +/* + * Free all the attribute fork blocks of the temporary file and delete the attr + * fork. The caller must ILOCK the tempfile and join it to the transaction. + * This function returns with the inode joined to a clean scrub transaction. + */ +STATIC int +xrep_xattr_reset_tempfile_fork( + struct xfs_scrub *sc) +{ + int error; + + trace_xrep_xattr_reset_fork(sc->ip, sc->tempip); + + /* + * Wipe out the attr fork of the temp file so that regular inode + * inactivation won't trip over the corrupt attr fork. + */ + if (xfs_ifork_has_extents(&sc->tempip->i_af)) { + error = xrep_reap_ifork(sc, sc->tempip, XFS_ATTR_FORK); + if (error) + return error; + } + + return xrep_xattr_fork_remove(sc, sc->tempip); +} + +/* + * Find all the extended attributes for this inode by scraping them out of the + * attribute key blocks by hand, and flushing them into the temp file. + * When we're done, free the staging memory before exchanging the xattr + * structures to reduce memory usage. + */ +STATIC int +xrep_xattr_salvage_attributes( + struct xrep_xattr *rx) +{ + struct xfs_inode *ip = rx->sc->ip; + int error; + + /* Short format xattrs are easy! */ + if (rx->sc->ip->i_af.if_format == XFS_DINODE_FMT_LOCAL) { + error = xrep_xattr_recover_sf(rx); + if (error) + return error; + + return xrep_xattr_flush_stashed(rx); + } + + /* + * For non-inline xattr structures, the salvage function scans the + * buffer cache looking for potential attr leaf blocks. The scan + * requires the ability to lock any buffer found and runs independently + * of any transaction <-> buffer item <-> buffer linkage. Therefore, + * roll the transaction to ensure there are no buffers joined. We hold + * the ILOCK independently of the transaction. + */ + error = xfs_trans_roll(&rx->sc->tp); + if (error) + return error; + + error = xfs_iread_extents(rx->sc->tp, ip, XFS_ATTR_FORK); + if (error) + return error; + + error = xrep_xattr_recover(rx); + if (error) + return error; + + return xrep_xattr_flush_stashed(rx); +} + +/* + * Prepare both inodes' attribute forks for an exchange. Promote the tempfile + * from short format to leaf format, and if the file being repaired has a short + * format attr fork, turn it into an empty extent list. + */ +STATIC int +xrep_xattr_swap_prep( + struct xfs_scrub *sc, + bool temp_local, + bool ip_local) +{ + int error; + + /* + * If the tempfile's attributes are in shortform format, convert that + * to a single leaf extent so that we can use the atomic mapping + * exchange. + */ + if (temp_local) { + struct xfs_da_args args = { + .dp = sc->tempip, + .geo = sc->mp->m_attr_geo, + .whichfork = XFS_ATTR_FORK, + .trans = sc->tp, + .total = 1, + .owner = sc->ip->i_ino, + }; + + error = xfs_attr_shortform_to_leaf(&args); + if (error) + return error; + + /* + * Roll the deferred log items to get us back to a clean + * transaction. + */ + error = xfs_defer_finish(&sc->tp); + if (error) + return error; + } + + /* + * If the file being repaired had a shortform attribute fork, convert + * that to an empty extent list in preparation for the atomic mapping + * exchange. + */ + if (ip_local) { + struct xfs_ifork *ifp; + + ifp = xfs_ifork_ptr(sc->ip, XFS_ATTR_FORK); + + xfs_idestroy_fork(ifp); + ifp->if_format = XFS_DINODE_FMT_EXTENTS; + ifp->if_nextents = 0; + ifp->if_bytes = 0; + ifp->if_data = NULL; + ifp->if_height = 0; + + xfs_trans_log_inode(sc->tp, sc->ip, + XFS_ILOG_CORE | XFS_ILOG_ADATA); + } + + return 0; +} + +/* Exchange the temporary file's attribute fork with the one being repaired. */ +STATIC int +xrep_xattr_swap( + struct xfs_scrub *sc, + struct xrep_tempexch *tx) +{ + bool ip_local, temp_local; + int error = 0; + + ip_local = sc->ip->i_af.if_format == XFS_DINODE_FMT_LOCAL; + temp_local = sc->tempip->i_af.if_format == XFS_DINODE_FMT_LOCAL; + + /* + * If the both files have a local format attr fork and the rebuilt + * xattr data would fit in the repaired file's attr fork, just copy + * the contents from the tempfile and declare ourselves done. + */ + if (ip_local && temp_local) { + int forkoff; + int newsize; + + newsize = xfs_attr_sf_totsize(sc->tempip); + forkoff = xfs_attr_shortform_bytesfit(sc->ip, newsize); + if (forkoff > 0) { + sc->ip->i_forkoff = forkoff; + xrep_tempfile_copyout_local(sc, XFS_ATTR_FORK); + return 0; + } + } + + /* Otherwise, make sure both attr forks are in block-mapping mode. */ + error = xrep_xattr_swap_prep(sc, temp_local, ip_local); + if (error) + return error; + + return xrep_tempexch_contents(sc, tx); +} + +/* + * Exchange the new extended attribute data (which we created in the tempfile) + * with the file being repaired. + */ +STATIC int +xrep_xattr_rebuild_tree( + struct xrep_xattr *rx) +{ + struct xfs_scrub *sc = rx->sc; + int error; + + /* + * If we didn't find any attributes to salvage, repair the file by + * zapping its attr fork. + */ + if (rx->attrs_found == 0) { + xfs_trans_ijoin(sc->tp, sc->ip, 0); + error = xrep_xattr_reset_fork(sc); + if (error) + return error; + + goto forget_acls; + } + + trace_xrep_xattr_rebuild_tree(sc->ip, sc->tempip); + + /* + * Commit the repair transaction and drop the ILOCKs so that we can use + * the atomic file content exchange helper functions to compute the + * correct resource reservations. + * + * We still hold IOLOCK_EXCL (aka i_rwsem) which will prevent xattr + * modifications, but there's nothing to prevent userspace from reading + * the attributes until we're ready for the exchange operation. Reads + * will return -EIO without shutting down the fs, so we're ok with + * that. + */ + error = xrep_trans_commit(sc); + if (error) + return error; + + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + /* + * Take the IOLOCK on the temporary file so that we can run xattr + * operations with the same locks held as we would for a normal file. + * We still hold sc->ip's IOLOCK. + */ + error = xrep_tempfile_iolock_polled(rx->sc); + if (error) + return error; + + /* Allocate exchange transaction and lock both inodes. */ + error = xrep_tempexch_trans_alloc(rx->sc, XFS_ATTR_FORK, &rx->tx); + if (error) + return error; + + /* + * Exchange the blocks mapped by the tempfile's attr fork with the file + * being repaired. The old attr blocks will then be attached to the + * tempfile, so reap its attr fork. + */ + error = xrep_xattr_swap(sc, &rx->tx); + if (error) + return error; + + error = xrep_xattr_reset_tempfile_fork(sc); + if (error) + return error; + + /* + * Roll to get a transaction without any inodes joined to it. Then we + * can drop the tempfile's ILOCK and IOLOCK before doing more work on + * the scrub target file. + */ + error = xfs_trans_roll(&sc->tp); + if (error) + return error; + + xrep_tempfile_iunlock(sc); + xrep_tempfile_iounlock(sc); + +forget_acls: + /* Invalidate cached ACLs now that we've reloaded all the xattrs. */ + xfs_forget_acl(VFS_I(sc->ip), SGI_ACL_FILE); + xfs_forget_acl(VFS_I(sc->ip), SGI_ACL_DEFAULT); + return 0; +} + +/* Tear down all the incore scan stuff we created. */ +STATIC void +xrep_xattr_teardown( + struct xrep_xattr *rx) +{ + xfblob_destroy(rx->xattr_blobs); + xfarray_destroy(rx->xattr_records); + kfree(rx); +} + +/* Set up the filesystem scan so we can regenerate extended attributes. */ +STATIC int +xrep_xattr_setup_scan( + struct xfs_scrub *sc, + struct xrep_xattr **rxp) +{ + struct xrep_xattr *rx; + char *descr; + int max_len; + int error; + + rx = kzalloc(sizeof(struct xrep_xattr), XCHK_GFP_FLAGS); + if (!rx) + return -ENOMEM; + rx->sc = sc; + + /* + * Allocate enough memory to handle loading local attr values from the + * xfblob data while flushing stashed attrs to the temporary file. + * We only realloc the buffer when salvaging remote attr values. + */ + max_len = xfs_attr_leaf_entsize_local_max(sc->mp->m_attr_geo->blksize); + error = xchk_setup_xattr_buf(rx->sc, max_len); + if (error == -ENOMEM) + error = -EDEADLOCK; + if (error) + goto out_rx; + + /* Set up some staging for salvaged attribute keys and values */ + descr = xchk_xfile_ino_descr(sc, "xattr keys"); + error = xfarray_create(descr, 0, sizeof(struct xrep_xattr_key), + &rx->xattr_records); + kfree(descr); + if (error) + goto out_rx; + + descr = xchk_xfile_ino_descr(sc, "xattr names"); + error = xfblob_create(descr, &rx->xattr_blobs); + kfree(descr); + if (error) + goto out_keys; + + *rxp = rx; + return 0; +out_keys: + xfarray_destroy(rx->xattr_records); +out_rx: + kfree(rx); + return error; +} + +/* + * Repair the extended attribute metadata. + * + * XXX: Remote attribute value buffers encompass the entire (up to 64k) buffer. + * The buffer cache in XFS can't handle aliased multiblock buffers, so this + * might misbehave if the attr fork is crosslinked with other filesystem + * metadata. + */ +int +xrep_xattr( + struct xfs_scrub *sc) +{ + struct xrep_xattr *rx = NULL; + int error; + + if (!xfs_inode_hasattr(sc->ip)) + return -ENOENT; + + /* The rmapbt is required to reap the old attr fork. */ + if (!xfs_has_rmapbt(sc->mp)) + return -EOPNOTSUPP; + + error = xrep_xattr_setup_scan(sc, &rx); + if (error) + return error; + + ASSERT(sc->ilock_flags & XFS_ILOCK_EXCL); + + error = xrep_xattr_salvage_attributes(rx); + if (error) + goto out_scan; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + goto out_scan; + + error = xrep_xattr_rebuild_tree(rx); + if (error) + goto out_scan; + +out_scan: + xrep_xattr_teardown(rx); + return error; +} diff --git a/fs/xfs/scrub/attr_repair.h b/fs/xfs/scrub/attr_repair.h new file mode 100644 index 0000000000000..0a9ffa7cfa906 --- /dev/null +++ b/fs/xfs/scrub/attr_repair.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_ATTR_REPAIR_H__ +#define __XFS_SCRUB_ATTR_REPAIR_H__ + +int xrep_xattr_reset_fork(struct xfs_scrub *sc); + +#endif /* __XFS_SCRUB_ATTR_REPAIR_H__ */ diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c index 443e62f724818..04aec0e9e4c3d 100644 --- a/fs/xfs/scrub/repair.c +++ b/fs/xfs/scrub/repair.c @@ -32,6 +32,9 @@ #include "xfs_reflink.h" #include "xfs_health.h" #include "xfs_buf_mem.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/trace.h" @@ -39,6 +42,7 @@ #include "scrub/bitmap.h" #include "scrub/stats.h" #include "scrub/xfile.h" +#include "scrub/attr_repair.h" /* * Attempt to repair some metadata, if the metadata is corrupt and userspace @@ -1136,6 +1140,17 @@ xrep_metadata_inode_forks( return error; } + /* Clear the attr forks since metadata shouldn't have that. */ + if (xfs_inode_hasattr(sc->ip)) { + if (!dirty) { + dirty = true; + xfs_trans_ijoin(sc->tp, sc->ip, 0); + } + error = xrep_xattr_reset_fork(sc); + if (error) + return error; + } + /* * If we modified the inode, roll the transaction but don't rejoin the * inode to the new transaction because xrep_bmap_data can do that. @@ -1201,3 +1216,34 @@ xrep_trans_cancel_hook_dummy( current->journal_info = *cookiep; *cookiep = NULL; } + +/* + * See if this buffer can pass the given ->verify_struct() function. + * + * If the buffer already has ops attached and they're not the ones that were + * passed in, we reject the buffer. Otherwise, we perform the structure test + * (note that we do not check CRCs) and return the outcome of the test. The + * buffer ops and error state are left unchanged. + */ +bool +xrep_buf_verify_struct( + struct xfs_buf *bp, + const struct xfs_buf_ops *ops) +{ + const struct xfs_buf_ops *old_ops = bp->b_ops; + xfs_failaddr_t fa; + int old_error; + + if (old_ops) { + if (old_ops != ops) + return false; + } + + old_error = bp->b_error; + bp->b_ops = ops; + fa = bp->b_ops->verify_struct(bp); + bp->b_ops = old_ops; + bp->b_error = old_error; + + return fa == NULL; +} diff --git a/fs/xfs/scrub/repair.h b/fs/xfs/scrub/repair.h index 0e2b695ab8f66..9cbfd8da5620b 100644 --- a/fs/xfs/scrub/repair.h +++ b/fs/xfs/scrub/repair.h @@ -90,6 +90,7 @@ int xrep_bmap(struct xfs_scrub *sc, int whichfork, bool allow_unwritten); int xrep_metadata_inode_forks(struct xfs_scrub *sc); int xrep_setup_ag_rmapbt(struct xfs_scrub *sc); int xrep_setup_ag_refcountbt(struct xfs_scrub *sc); +int xrep_setup_xattr(struct xfs_scrub *sc); /* Repair setup functions */ int xrep_setup_ag_allocbt(struct xfs_scrub *sc); @@ -123,6 +124,7 @@ int xrep_bmap_attr(struct xfs_scrub *sc); int xrep_bmap_cow(struct xfs_scrub *sc); int xrep_nlinks(struct xfs_scrub *sc); int xrep_fscounters(struct xfs_scrub *sc); +int xrep_xattr(struct xfs_scrub *sc); #ifdef CONFIG_XFS_RT int xrep_rtbitmap(struct xfs_scrub *sc); @@ -147,6 +149,8 @@ int xrep_trans_alloc_hook_dummy(struct xfs_mount *mp, void **cookiep, struct xfs_trans **tpp); void xrep_trans_cancel_hook_dummy(void **cookiep, struct xfs_trans *tp); +bool xrep_buf_verify_struct(struct xfs_buf *bp, const struct xfs_buf_ops *ops); + #else #define xrep_ino_dqattach(sc) (0) @@ -190,6 +194,7 @@ xrep_setup_nothing( #define xrep_setup_ag_allocbt xrep_setup_nothing #define xrep_setup_ag_rmapbt xrep_setup_nothing #define xrep_setup_ag_refcountbt xrep_setup_nothing +#define xrep_setup_xattr xrep_setup_nothing #define xrep_setup_inode(sc, imap) ((void)0) @@ -215,6 +220,7 @@ xrep_setup_nothing( #define xrep_nlinks xrep_notsupported #define xrep_fscounters xrep_notsupported #define xrep_rtsummary xrep_notsupported +#define xrep_xattr xrep_notsupported #endif /* CONFIG_XFS_ONLINE_REPAIR */ diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 62a064c1a5d34..547189a14b6bf 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -331,7 +331,7 @@ static const struct xchk_meta_ops meta_scrub_ops[] = { .type = ST_INODE, .setup = xchk_setup_xattr, .scrub = xchk_xattr, - .repair = xrep_notsupported, + .repair = xrep_xattr, }, [XFS_SCRUB_TYPE_SYMLINK] = { /* symbolic link */ .type = ST_INODE, diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 7d07912d8f758..026813205b470 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2416,6 +2416,89 @@ TRACE_EVENT(xreap_bmapi_binval_scan, __entry->scan_blocks) ); +TRACE_EVENT(xrep_xattr_recover_leafblock, + TP_PROTO(struct xfs_inode *ip, xfs_dablk_t dabno, uint16_t magic), + TP_ARGS(ip, dabno, magic), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_dablk_t, dabno) + __field(uint16_t, magic) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = ip->i_ino; + __entry->dabno = dabno; + __entry->magic = magic; + ), + TP_printk("dev %d:%d ino 0x%llx dablk 0x%x magic 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->dabno, + __entry->magic) +); + +DECLARE_EVENT_CLASS(xrep_xattr_salvage_class, + TP_PROTO(struct xfs_inode *ip, unsigned int flags, char *name, + unsigned int namelen, unsigned int valuelen), + TP_ARGS(ip, flags, name, namelen, valuelen), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned int, flags) + __field(unsigned int, namelen) + __dynamic_array(char, name, namelen) + __field(unsigned int, valuelen) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = ip->i_ino; + __entry->flags = flags; + __entry->namelen = namelen; + memcpy(__get_str(name), name, namelen); + __entry->valuelen = valuelen; + ), + TP_printk("dev %d:%d ino 0x%llx flags %s name '%.*s' valuelen 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_flags(__entry->flags, "|", XFS_ATTR_NAMESPACE_STR), + __entry->namelen, + __get_str(name), + __entry->valuelen) +); +#define DEFINE_XREP_XATTR_SALVAGE_EVENT(name) \ +DEFINE_EVENT(xrep_xattr_salvage_class, name, \ + TP_PROTO(struct xfs_inode *ip, unsigned int flags, char *name, \ + unsigned int namelen, unsigned int valuelen), \ + TP_ARGS(ip, flags, name, namelen, valuelen)) +DEFINE_XREP_XATTR_SALVAGE_EVENT(xrep_xattr_salvage_rec); +DEFINE_XREP_XATTR_SALVAGE_EVENT(xrep_xattr_insert_rec); + +TRACE_EVENT(xrep_xattr_class, + TP_PROTO(struct xfs_inode *ip, struct xfs_inode *arg_ip), + TP_ARGS(ip, arg_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_ino_t, src_ino) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = ip->i_ino; + __entry->src_ino = arg_ip->i_ino; + ), + TP_printk("dev %d:%d ino 0x%llx src 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->src_ino) +) +#define DEFINE_XREP_XATTR_EVENT(name) \ +DEFINE_EVENT(xrep_xattr_class, name, \ + TP_PROTO(struct xfs_inode *ip, struct xfs_inode *arg_ip), \ + TP_ARGS(ip, arg_ip)) +DEFINE_XREP_XATTR_EVENT(xrep_xattr_rebuild_tree); +DEFINE_XREP_XATTR_EVENT(xrep_xattr_reset_fork); + #endif /* IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) */ #endif /* _TRACE_XFS_SCRUB_TRACE_H */ diff --git a/fs/xfs/scrub/xfarray.c b/fs/xfs/scrub/xfarray.c index 17c982a4821d4..b65cd3fc5ac9b 100644 --- a/fs/xfs/scrub/xfarray.c +++ b/fs/xfs/scrub/xfarray.c @@ -1051,3 +1051,20 @@ xfarray_sort( kvfree(si); return error; } + +/* How many bytes is this array consuming? */ +unsigned long long +xfarray_bytes( + struct xfarray *array) +{ + return xfile_bytes(array->xfile); +} + +/* Empty the entire array. */ +void +xfarray_truncate( + struct xfarray *array) +{ + xfile_discard(array->xfile, 0, MAX_LFS_FILESIZE); + array->nr = 0; +} diff --git a/fs/xfs/scrub/xfarray.h b/fs/xfs/scrub/xfarray.h index acb2f94c56c13..3b10a58e9f146 100644 --- a/fs/xfs/scrub/xfarray.h +++ b/fs/xfs/scrub/xfarray.h @@ -44,6 +44,8 @@ int xfarray_unset(struct xfarray *array, xfarray_idx_t idx); int xfarray_store(struct xfarray *array, xfarray_idx_t idx, const void *ptr); int xfarray_store_anywhere(struct xfarray *array, const void *ptr); bool xfarray_element_is_null(struct xfarray *array, const void *ptr); +void xfarray_truncate(struct xfarray *array); +unsigned long long xfarray_bytes(struct xfarray *array); /* * Load an array element, but zero the buffer if there's no data because we diff --git a/fs/xfs/scrub/xfblob.c b/fs/xfs/scrub/xfblob.c index cec668debce5a..6ef2a9637f162 100644 --- a/fs/xfs/scrub/xfblob.c +++ b/fs/xfs/scrub/xfblob.c @@ -149,3 +149,20 @@ xfblob_free( xfile_discard(blob->xfile, cookie, sizeof(key) + key.xb_size); return 0; } + +/* How many bytes is this blob storage object consuming? */ +unsigned long long +xfblob_bytes( + struct xfblob *blob) +{ + return xfile_bytes(blob->xfile); +} + +/* Drop all the blobs. */ +void +xfblob_truncate( + struct xfblob *blob) +{ + xfile_discard(blob->xfile, PAGE_SIZE, MAX_LFS_FILESIZE - PAGE_SIZE); + blob->last_offset = PAGE_SIZE; +} diff --git a/fs/xfs/scrub/xfblob.h b/fs/xfs/scrub/xfblob.h index bd98647407f1d..78a67a06408f8 100644 --- a/fs/xfs/scrub/xfblob.h +++ b/fs/xfs/scrub/xfblob.h @@ -20,5 +20,7 @@ int xfblob_load(struct xfblob *blob, xfblob_cookie cookie, void *ptr, int xfblob_store(struct xfblob *blob, xfblob_cookie *cookie, const void *ptr, uint32_t size); int xfblob_free(struct xfblob *blob, xfblob_cookie cookie); +unsigned long long xfblob_bytes(struct xfblob *blob); +void xfblob_truncate(struct xfblob *blob); #endif /* __XFS_SCRUB_XFBLOB_H__ */ diff --git a/fs/xfs/scrub/xfile.h b/fs/xfs/scrub/xfile.h index 8dfbae1fe33a5..cc2cc1714cd4e 100644 --- a/fs/xfs/scrub/xfile.h +++ b/fs/xfs/scrub/xfile.h @@ -27,4 +27,9 @@ struct folio *xfile_get_folio(struct xfile *xf, loff_t offset, size_t len, unsigned int flags); void xfile_put_folio(struct xfile *xf, struct folio *folio); +static inline unsigned long long xfile_bytes(struct xfile *xf) +{ + return file_inode(xf->file)->i_blocks << SECTOR_SHIFT; +} + #endif /* __XFS_SCRUB_XFILE_H__ */ diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index f0fa02264edaa..8a0151e23f3d7 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -494,6 +494,9 @@ _xfs_buf_obj_cmp( * it stale has not yet committed. i.e. we are * reallocating a busy extent. Skip this buffer and * continue searching for an exact match. + * + * Note: If we're scanning for incore buffers to stale, don't + * complain if we find non-stale buffers. */ if (!(map->bm_flags & XBM_LIVESCAN)) ASSERT(bp->b_flags & XBF_STALE); diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index 939baf08331b5..a80c3063a13f1 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -31,6 +31,8 @@ * pos: file offset, in bytes * bytecount: number of bytes * + * dablk: directory or xattr block offset, in filesystem blocks + * * disize: ondisk file size, in bytes * isize: incore file size, in bytes * From 0ee230dec2626ef25dc96abd47b84494f9c251e3 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:46 -0700 Subject: [PATCH 0238/1702] xfs: scrub should set preen if attr leaf has holes If an attr block indicates that it could use compaction, set the preen flag to have the attr fork rebuilt, since the attr fork rebuilder can take care of that for us. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/attr.c | 2 ++ fs/xfs/scrub/dabtree.c | 16 ++++++++++++++++ fs/xfs/scrub/dabtree.h | 1 + fs/xfs/scrub/trace.h | 1 + 4 files changed, 20 insertions(+) diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index 7621e548d7301..ba06be86ac7d4 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -428,6 +428,8 @@ xchk_xattr_block( xchk_da_set_corrupt(ds, level); if (!xchk_xattr_set_map(ds->sc, ab->usedmap, 0, hdrsize)) xchk_da_set_corrupt(ds, level); + if (leafhdr.holes) + xchk_da_set_preen(ds, level); if (ds->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) goto out; diff --git a/fs/xfs/scrub/dabtree.c b/fs/xfs/scrub/dabtree.c index c71254088dffe..056de4819f866 100644 --- a/fs/xfs/scrub/dabtree.c +++ b/fs/xfs/scrub/dabtree.c @@ -78,6 +78,22 @@ xchk_da_set_corrupt( __return_address); } +/* Flag a da btree node in need of optimization. */ +void +xchk_da_set_preen( + struct xchk_da_btree *ds, + int level) +{ + struct xfs_scrub *sc = ds->sc; + + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_PREEN; + trace_xchk_fblock_preen(sc, ds->dargs.whichfork, + xfs_dir2_da_to_db(ds->dargs.geo, + ds->state->path.blk[level].blkno), + __return_address); +} + +/* Find an entry at a certain level in a da btree. */ static struct xfs_da_node_entry * xchk_da_btree_node_entry( struct xchk_da_btree *ds, diff --git a/fs/xfs/scrub/dabtree.h b/fs/xfs/scrub/dabtree.h index 4f8c2138a1ec6..d654c125feb4d 100644 --- a/fs/xfs/scrub/dabtree.h +++ b/fs/xfs/scrub/dabtree.h @@ -35,6 +35,7 @@ bool xchk_da_process_error(struct xchk_da_btree *ds, int level, int *error); /* Check for da btree corruption. */ void xchk_da_set_corrupt(struct xchk_da_btree *ds, int level); +void xchk_da_set_preen(struct xchk_da_btree *ds, int level); int xchk_da_btree_hash(struct xchk_da_btree *ds, int level, __be32 *hashp); int xchk_da_btree(struct xfs_scrub *sc, int whichfork, diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 026813205b470..ffaff7722bf23 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -365,6 +365,7 @@ DEFINE_EVENT(xchk_fblock_error_class, name, \ DEFINE_SCRUB_FBLOCK_ERROR_EVENT(xchk_fblock_error); DEFINE_SCRUB_FBLOCK_ERROR_EVENT(xchk_fblock_warning); +DEFINE_SCRUB_FBLOCK_ERROR_EVENT(xchk_fblock_preen); #ifdef CONFIG_XFS_QUOTA DECLARE_EVENT_CLASS(xchk_dqiter_class, From 40190f9f918aaf3355c8f777e6e658ea3bf77870 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:47 -0700 Subject: [PATCH 0239/1702] xfs: flag empty xattr leaf blocks for optimization Empty xattr leaf blocks at offset zero are a waste of space but otherwise harmless. If we encounter one, flag it as an opportunity for optimization. If we encounter empty attr leaf blocks anywhere else in the attr fork, that's corruption. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/attr.c | 11 +++++++++++ fs/xfs/scrub/dabtree.h | 2 ++ 2 files changed, 13 insertions(+) diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index ba06be86ac7d4..696971204b876 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -420,6 +420,17 @@ xchk_xattr_block( xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &leafhdr, leaf); hdrsize = xfs_attr3_leaf_hdr_size(leaf); + /* + * Empty xattr leaf blocks mapped at block 0 are probably a byproduct + * of a race between setxattr and a log shutdown. Anywhere else in the + * attr fork is a corruption. + */ + if (leafhdr.count == 0) { + if (blk->blkno == 0) + xchk_da_set_preen(ds, level); + else + xchk_da_set_corrupt(ds, level); + } if (leafhdr.usedbytes > mp->m_attr_geo->blksize) xchk_da_set_corrupt(ds, level); if (leafhdr.firstused > mp->m_attr_geo->blksize) diff --git a/fs/xfs/scrub/dabtree.h b/fs/xfs/scrub/dabtree.h index d654c125feb4d..de291e3b77dd8 100644 --- a/fs/xfs/scrub/dabtree.h +++ b/fs/xfs/scrub/dabtree.h @@ -37,6 +37,8 @@ bool xchk_da_process_error(struct xchk_da_btree *ds, int level, int *error); void xchk_da_set_corrupt(struct xchk_da_btree *ds, int level); void xchk_da_set_preen(struct xchk_da_btree *ds, int level); +void xchk_da_set_preen(struct xchk_da_btree *ds, int level); + int xchk_da_btree_hash(struct xchk_da_btree *ds, int level, __be32 *hashp); int xchk_da_btree(struct xfs_scrub *sc, int whichfork, xchk_da_btree_rec_fn scrub_fn, void *private); From e921533ef1a610ae92319269bd8e41ab09bf09c4 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:49 -0700 Subject: [PATCH 0240/1702] xfs: ensure unlinked list state is consistent with nlink during scrub Now that we have the means to tell if an inode is on an unlinked inode list or not, we can check that an inode with zero link count is on the unlinked list; and an inode that has nonzero link count is not on that list. Make repair clean things up too. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/inode.c | 19 ++++++++++++++++ fs/xfs/scrub/inode_repair.c | 45 +++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_inode.c | 5 +---- fs/xfs/xfs_inode.h | 2 ++ 4 files changed, 67 insertions(+), 4 deletions(-) diff --git a/fs/xfs/scrub/inode.c b/fs/xfs/scrub/inode.c index 6e2fe2d6250b3..d32716fb2fecf 100644 --- a/fs/xfs/scrub/inode.c +++ b/fs/xfs/scrub/inode.c @@ -739,6 +739,23 @@ xchk_inode_check_reflink_iflag( xchk_ino_set_corrupt(sc, ino); } +/* + * If this inode has zero link count, it must be on the unlinked list. If + * it has nonzero link count, it must not be on the unlinked list. + */ +STATIC void +xchk_inode_check_unlinked( + struct xfs_scrub *sc) +{ + if (VFS_I(sc->ip)->i_nlink == 0) { + if (!xfs_inode_on_unlinked_list(sc->ip)) + xchk_ino_set_corrupt(sc, sc->ip->i_ino); + } else { + if (xfs_inode_on_unlinked_list(sc->ip)) + xchk_ino_set_corrupt(sc, sc->ip->i_ino); + } +} + /* Scrub an inode. */ int xchk_inode( @@ -771,6 +788,8 @@ xchk_inode( if (S_ISREG(VFS_I(sc->ip)->i_mode)) xchk_inode_check_reflink_iflag(sc, sc->ip->i_ino); + xchk_inode_check_unlinked(sc); + xchk_inode_xref(sc, sc->ip->i_ino, &di); out: return error; diff --git a/fs/xfs/scrub/inode_repair.c b/fs/xfs/scrub/inode_repair.c index 097afba3043f8..c743772a523e9 100644 --- a/fs/xfs/scrub/inode_repair.c +++ b/fs/xfs/scrub/inode_repair.c @@ -1745,6 +1745,46 @@ xrep_inode_problems( return xrep_roll_trans(sc); } +/* + * Make sure this inode's unlinked list pointers are consistent with its + * link count. + */ +STATIC int +xrep_inode_unlinked( + struct xfs_scrub *sc) +{ + unsigned int nlink = VFS_I(sc->ip)->i_nlink; + int error; + + /* + * If this inode is linked from the directory tree and on the unlinked + * list, remove it from the unlinked list. + */ + if (nlink > 0 && xfs_inode_on_unlinked_list(sc->ip)) { + struct xfs_perag *pag; + int error; + + pag = xfs_perag_get(sc->mp, + XFS_INO_TO_AGNO(sc->mp, sc->ip->i_ino)); + error = xfs_iunlink_remove(sc->tp, pag, sc->ip); + xfs_perag_put(pag); + if (error) + return error; + } + + /* + * If this inode is not linked from the directory tree yet not on the + * unlinked list, put it on the unlinked list. + */ + if (nlink == 0 && !xfs_inode_on_unlinked_list(sc->ip)) { + error = xfs_iunlink(sc->tp, sc->ip); + if (error) + return error; + } + + return 0; +} + /* Repair an inode's fields. */ int xrep_inode( @@ -1794,5 +1834,10 @@ xrep_inode( return error; } + /* Reconnect incore unlinked list */ + error = xrep_inode_unlinked(sc); + if (error) + return error; + return xrep_defer_finish(sc); } diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index ac92c0525d9b6..b24c0e23d37d7 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -42,9 +42,6 @@ struct kmem_cache *xfs_inode_cache; -STATIC int xfs_iunlink_remove(struct xfs_trans *tp, struct xfs_perag *pag, - struct xfs_inode *); - /* * helper function to extract extent size hint from inode */ @@ -2252,7 +2249,7 @@ xfs_iunlink_remove_inode( /* * Pull the on-disk inode from the AGI unlinked list. */ -STATIC int +int xfs_iunlink_remove( struct xfs_trans *tp, struct xfs_perag *pag, diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index 596eec7156753..8157ae7f8e59f 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -617,6 +617,8 @@ extern struct kmem_cache *xfs_inode_cache; bool xfs_inode_needs_inactive(struct xfs_inode *ip); int xfs_iunlink(struct xfs_trans *tp, struct xfs_inode *ip); +int xfs_iunlink_remove(struct xfs_trans *tp, struct xfs_perag *pag, + struct xfs_inode *ip); void xfs_end_io(struct work_struct *work); From 6c631e79e73c7122c890ef943f8ca9aab9e1dec8 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:48 -0700 Subject: [PATCH 0241/1702] xfs: create an xattr iteration function for scrub Create a streamlined function to walk a file's xattrs, without all the cursor management stuff in the regular listxattr. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/scrub/attr.c | 125 ++++++--------- fs/xfs/scrub/dab_bitmap.h | 37 +++++ fs/xfs/scrub/listxattr.c | 312 ++++++++++++++++++++++++++++++++++++++ fs/xfs/scrub/listxattr.h | 17 +++ 5 files changed, 414 insertions(+), 78 deletions(-) create mode 100644 fs/xfs/scrub/dab_bitmap.h create mode 100644 fs/xfs/scrub/listxattr.c create mode 100644 fs/xfs/scrub/listxattr.h diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index 8647629ac7bf9..7dbe6b3befb33 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -165,6 +165,7 @@ xfs-y += $(addprefix scrub/, \ ialloc.o \ inode.o \ iscan.o \ + listxattr.o \ nlinks.o \ parent.o \ readdir.o \ diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index 696971204b876..8853e4d0eee3d 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -21,6 +21,7 @@ #include "scrub/common.h" #include "scrub/dabtree.h" #include "scrub/attr.h" +#include "scrub/listxattr.h" #include "scrub/repair.h" /* Free the buffers linked from the xattr buffer. */ @@ -153,90 +154,81 @@ xchk_setup_xattr( /* Extended Attributes */ -struct xchk_xattr { - struct xfs_attr_list_context context; - struct xfs_scrub *sc; -}; - /* * Check that an extended attribute key can be looked up by hash. * - * We use the XFS attribute list iterator (i.e. xfs_attr_list_ilocked) - * to call this function for every attribute key in an inode. Once - * we're here, we load the attribute value to see if any errors happen, - * or if we get more or less data than we expected. + * We use the extended attribute walk helper to call this function for every + * attribute key in an inode. Once we're here, we load the attribute value to + * see if any errors happen, or if we get more or less data than we expected. */ -static void -xchk_xattr_listent( - struct xfs_attr_list_context *context, - int flags, - unsigned char *name, - int namelen, - int valuelen) +static int +xchk_xattr_actor( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) { struct xfs_da_args args = { .op_flags = XFS_DA_OP_NOTIME, - .attr_filter = flags & XFS_ATTR_NSP_ONDISK_MASK, - .geo = context->dp->i_mount->m_attr_geo, + .attr_filter = attr_flags & XFS_ATTR_NSP_ONDISK_MASK, + .geo = sc->mp->m_attr_geo, .whichfork = XFS_ATTR_FORK, - .dp = context->dp, + .dp = ip, .name = name, .namelen = namelen, .hashval = xfs_da_hashname(name, namelen), - .trans = context->tp, + .trans = sc->tp, .valuelen = valuelen, - .owner = context->dp->i_ino, + .owner = ip->i_ino, }; struct xchk_xattr_buf *ab; - struct xchk_xattr *sx; int error = 0; - sx = container_of(context, struct xchk_xattr, context); - ab = sx->sc->buf; + ab = sc->buf; - if (xchk_should_terminate(sx->sc, &error)) { - context->seen_enough = error; - return; - } + if (xchk_should_terminate(sc, &error)) + return error; - if (flags & XFS_ATTR_INCOMPLETE) { + if (attr_flags & XFS_ATTR_INCOMPLETE) { /* Incomplete attr key, just mark the inode for preening. */ - xchk_ino_set_preen(sx->sc, context->dp->i_ino); - return; + xchk_ino_set_preen(sc, ip->i_ino); + return 0; } /* Only one namespace bit allowed. */ - if (hweight32(flags & XFS_ATTR_NSP_ONDISK_MASK) > 1) { - xchk_fblock_set_corrupt(sx->sc, XFS_ATTR_FORK, args.blkno); - goto fail_xref; + if (hweight32(attr_flags & XFS_ATTR_NSP_ONDISK_MASK) > 1) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, args.blkno); + return -ECANCELED; } /* Does this name make sense? */ if (!xfs_attr_namecheck(name, namelen)) { - xchk_fblock_set_corrupt(sx->sc, XFS_ATTR_FORK, args.blkno); - goto fail_xref; + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, args.blkno); + return -ECANCELED; } /* - * Local xattr values are stored in the attr leaf block, so we don't - * need to retrieve the value from a remote block to detect corruption - * problems. + * Local and shortform xattr values are stored in the attr leaf block, + * so we don't need to retrieve the value from a remote block to detect + * corruption problems. */ - if (flags & XFS_ATTR_LOCAL) - goto fail_xref; + if (value) + return 0; /* - * Try to allocate enough memory to extrat the attr value. If that - * doesn't work, we overload the seen_enough variable to convey - * the error message back to the main scrub function. + * Try to allocate enough memory to extract the attr value. If that + * doesn't work, return -EDEADLOCK as a signal to try again with a + * maximally sized buffer. */ - error = xchk_setup_xattr_buf(sx->sc, valuelen); + error = xchk_setup_xattr_buf(sc, valuelen); if (error == -ENOMEM) error = -EDEADLOCK; - if (error) { - context->seen_enough = error; - return; - } + if (error) + return error; args.value = ab->value; @@ -244,16 +236,13 @@ xchk_xattr_listent( /* ENODATA means the hash lookup failed and the attr is bad */ if (error == -ENODATA) error = -EFSCORRUPTED; - if (!xchk_fblock_process_error(sx->sc, XFS_ATTR_FORK, args.blkno, + if (!xchk_fblock_process_error(sc, XFS_ATTR_FORK, args.blkno, &error)) - goto fail_xref; + return error; if (args.valuelen != valuelen) - xchk_fblock_set_corrupt(sx->sc, XFS_ATTR_FORK, - args.blkno); -fail_xref: - if (sx->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) - context->seen_enough = 1; - return; + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, args.blkno); + + return 0; } /* @@ -618,16 +607,6 @@ int xchk_xattr( struct xfs_scrub *sc) { - struct xchk_xattr sx = { - .sc = sc, - .context = { - .dp = sc->ip, - .tp = sc->tp, - .resynch = 1, - .put_listent = xchk_xattr_listent, - .allow_incomplete = true, - }, - }; xfs_dablk_t last_checked = -1U; int error = 0; @@ -656,12 +635,6 @@ xchk_xattr( /* * Look up every xattr in this file by name and hash. * - * Use the backend implementation of xfs_attr_list to call - * xchk_xattr_listent on every attribute key in this inode. - * In other words, we use the same iterator/callback mechanism - * that listattr uses to scrub extended attributes, though in our - * _listent function, we check the value of the attribute. - * * The VFS only locks i_rwsem when modifying attrs, so keep all * three locks held because that's the only way to ensure we're * the only thread poking into the da btree. We traverse the da @@ -669,13 +642,9 @@ xchk_xattr( * iteration, which doesn't really follow the usual buffer * locking order. */ - error = xfs_attr_list_ilocked(&sx.context); + error = xchk_xattr_walk(sc, sc->ip, xchk_xattr_actor, NULL); if (!xchk_fblock_process_error(sc, XFS_ATTR_FORK, 0, &error)) return error; - /* Did our listent function try to return any errors? */ - if (sx.context.seen_enough < 0) - return sx.context.seen_enough; - return 0; } diff --git a/fs/xfs/scrub/dab_bitmap.h b/fs/xfs/scrub/dab_bitmap.h new file mode 100644 index 0000000000000..0c6e3aad43954 --- /dev/null +++ b/fs/xfs/scrub/dab_bitmap.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_DAB_BITMAP_H__ +#define __XFS_SCRUB_DAB_BITMAP_H__ + +/* Bitmaps, but for type-checked for xfs_dablk_t */ + +struct xdab_bitmap { + struct xbitmap32 dabitmap; +}; + +static inline void xdab_bitmap_init(struct xdab_bitmap *bitmap) +{ + xbitmap32_init(&bitmap->dabitmap); +} + +static inline void xdab_bitmap_destroy(struct xdab_bitmap *bitmap) +{ + xbitmap32_destroy(&bitmap->dabitmap); +} + +static inline int xdab_bitmap_set(struct xdab_bitmap *bitmap, + xfs_dablk_t dabno, xfs_extlen_t len) +{ + return xbitmap32_set(&bitmap->dabitmap, dabno, len); +} + +static inline bool xdab_bitmap_test(struct xdab_bitmap *bitmap, + xfs_dablk_t dabno, xfs_extlen_t *len) +{ + return xbitmap32_test(&bitmap->dabitmap, dabno, len); +} + +#endif /* __XFS_SCRUB_DAB_BITMAP_H__ */ diff --git a/fs/xfs/scrub/listxattr.c b/fs/xfs/scrub/listxattr.c new file mode 100644 index 0000000000000..cbe5911ecbbcf --- /dev/null +++ b/fs/xfs/scrub/listxattr.c @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_attr_leaf.h" +#include "xfs_attr_sf.h" +#include "xfs_trans.h" +#include "scrub/scrub.h" +#include "scrub/bitmap.h" +#include "scrub/dab_bitmap.h" +#include "scrub/listxattr.h" + +/* Call a function for every entry in a shortform xattr structure. */ +STATIC int +xchk_xattr_walk_sf( + struct xfs_scrub *sc, + struct xfs_inode *ip, + xchk_xattr_fn attr_fn, + void *priv) +{ + struct xfs_attr_sf_hdr *hdr = ip->i_af.if_data; + struct xfs_attr_sf_entry *sfe; + unsigned int i; + int error; + + sfe = xfs_attr_sf_firstentry(hdr); + for (i = 0; i < hdr->count; i++) { + error = attr_fn(sc, ip, sfe->flags, sfe->nameval, sfe->namelen, + &sfe->nameval[sfe->namelen], sfe->valuelen, + priv); + if (error) + return error; + + sfe = xfs_attr_sf_nextentry(sfe); + } + + return 0; +} + +/* Call a function for every entry in this xattr leaf block. */ +STATIC int +xchk_xattr_walk_leaf_entries( + struct xfs_scrub *sc, + struct xfs_inode *ip, + xchk_xattr_fn attr_fn, + struct xfs_buf *bp, + void *priv) +{ + struct xfs_attr3_icleaf_hdr ichdr; + struct xfs_mount *mp = sc->mp; + struct xfs_attr_leafblock *leaf = bp->b_addr; + struct xfs_attr_leaf_entry *entry; + unsigned int i; + int error; + + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &ichdr, leaf); + entry = xfs_attr3_leaf_entryp(leaf); + + for (i = 0; i < ichdr.count; entry++, i++) { + void *value; + unsigned char *name; + unsigned int namelen, valuelen; + + if (entry->flags & XFS_ATTR_LOCAL) { + struct xfs_attr_leaf_name_local *name_loc; + + name_loc = xfs_attr3_leaf_name_local(leaf, i); + name = name_loc->nameval; + namelen = name_loc->namelen; + value = &name_loc->nameval[name_loc->namelen]; + valuelen = be16_to_cpu(name_loc->valuelen); + } else { + struct xfs_attr_leaf_name_remote *name_rmt; + + name_rmt = xfs_attr3_leaf_name_remote(leaf, i); + name = name_rmt->name; + namelen = name_rmt->namelen; + value = NULL; + valuelen = be32_to_cpu(name_rmt->valuelen); + } + + error = attr_fn(sc, ip, entry->flags, name, namelen, value, + valuelen, priv); + if (error) + return error; + + } + + return 0; +} + +/* + * Call a function for every entry in a leaf-format xattr structure. Avoid + * memory allocations for the loop detector since there's only one block. + */ +STATIC int +xchk_xattr_walk_leaf( + struct xfs_scrub *sc, + struct xfs_inode *ip, + xchk_xattr_fn attr_fn, + void *priv) +{ + struct xfs_buf *leaf_bp; + int error; + + error = xfs_attr3_leaf_read(sc->tp, ip, ip->i_ino, 0, &leaf_bp); + if (error) + return error; + + error = xchk_xattr_walk_leaf_entries(sc, ip, attr_fn, leaf_bp, priv); + xfs_trans_brelse(sc->tp, leaf_bp); + return error; +} + +/* Find the leftmost leaf in the xattr dabtree. */ +STATIC int +xchk_xattr_find_leftmost_leaf( + struct xfs_scrub *sc, + struct xfs_inode *ip, + struct xdab_bitmap *seen_dablks, + struct xfs_buf **leaf_bpp) +{ + struct xfs_da3_icnode_hdr nodehdr; + struct xfs_mount *mp = sc->mp; + struct xfs_trans *tp = sc->tp; + struct xfs_da_intnode *node; + struct xfs_da_node_entry *btree; + struct xfs_buf *bp; + xfs_failaddr_t fa; + xfs_dablk_t blkno = 0; + unsigned int expected_level = 0; + int error; + + for (;;) { + xfs_extlen_t len = 1; + uint16_t magic; + + /* Make sure we haven't seen this new block already. */ + if (xdab_bitmap_test(seen_dablks, blkno, &len)) + return -EFSCORRUPTED; + + error = xfs_da3_node_read(tp, ip, blkno, &bp, XFS_ATTR_FORK); + if (error) + return error; + + node = bp->b_addr; + magic = be16_to_cpu(node->hdr.info.magic); + if (magic == XFS_ATTR_LEAF_MAGIC || + magic == XFS_ATTR3_LEAF_MAGIC) + break; + + error = -EFSCORRUPTED; + if (magic != XFS_DA_NODE_MAGIC && + magic != XFS_DA3_NODE_MAGIC) + goto out_buf; + + fa = xfs_da3_node_header_check(bp, ip->i_ino); + if (fa) + goto out_buf; + + xfs_da3_node_hdr_from_disk(mp, &nodehdr, node); + + if (nodehdr.count == 0 || nodehdr.level >= XFS_DA_NODE_MAXDEPTH) + goto out_buf; + + /* Check the level from the root node. */ + if (blkno == 0) + expected_level = nodehdr.level - 1; + else if (expected_level != nodehdr.level) + goto out_buf; + else + expected_level--; + + /* Remember that we've seen this node. */ + error = xdab_bitmap_set(seen_dablks, blkno, 1); + if (error) + goto out_buf; + + /* Find the next level towards the leaves of the dabtree. */ + btree = nodehdr.btree; + blkno = be32_to_cpu(btree->before); + xfs_trans_brelse(tp, bp); + } + + error = -EFSCORRUPTED; + fa = xfs_attr3_leaf_header_check(bp, ip->i_ino); + if (fa) + goto out_buf; + + if (expected_level != 0) + goto out_buf; + + /* Remember that we've seen this leaf. */ + error = xdab_bitmap_set(seen_dablks, blkno, 1); + if (error) + goto out_buf; + + *leaf_bpp = bp; + return 0; + +out_buf: + xfs_trans_brelse(tp, bp); + return error; +} + +/* Call a function for every entry in a node-format xattr structure. */ +STATIC int +xchk_xattr_walk_node( + struct xfs_scrub *sc, + struct xfs_inode *ip, + xchk_xattr_fn attr_fn, + void *priv) +{ + struct xfs_attr3_icleaf_hdr leafhdr; + struct xdab_bitmap seen_dablks; + struct xfs_mount *mp = sc->mp; + struct xfs_attr_leafblock *leaf; + struct xfs_buf *leaf_bp; + int error; + + xdab_bitmap_init(&seen_dablks); + + error = xchk_xattr_find_leftmost_leaf(sc, ip, &seen_dablks, &leaf_bp); + if (error) + goto out_bitmap; + + for (;;) { + xfs_extlen_t len; + + error = xchk_xattr_walk_leaf_entries(sc, ip, attr_fn, leaf_bp, + priv); + if (error) + goto out_leaf; + + /* Find the right sibling of this leaf block. */ + leaf = leaf_bp->b_addr; + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &leafhdr, leaf); + if (leafhdr.forw == 0) + goto out_leaf; + + xfs_trans_brelse(sc->tp, leaf_bp); + + /* Make sure we haven't seen this new leaf already. */ + len = 1; + if (xdab_bitmap_test(&seen_dablks, leafhdr.forw, &len)) { + error = -EFSCORRUPTED; + goto out_bitmap; + } + + error = xfs_attr3_leaf_read(sc->tp, ip, ip->i_ino, + leafhdr.forw, &leaf_bp); + if (error) + goto out_bitmap; + + /* Remember that we've seen this new leaf. */ + error = xdab_bitmap_set(&seen_dablks, leafhdr.forw, 1); + if (error) + goto out_leaf; + } + +out_leaf: + xfs_trans_brelse(sc->tp, leaf_bp); +out_bitmap: + xdab_bitmap_destroy(&seen_dablks); + return error; +} + +/* + * Call a function for every extended attribute in a file. + * + * Callers must hold the ILOCK. No validation or cursor restarts allowed. + * Returns -EFSCORRUPTED on any problem, including loops in the dabtree. + */ +int +xchk_xattr_walk( + struct xfs_scrub *sc, + struct xfs_inode *ip, + xchk_xattr_fn attr_fn, + void *priv) +{ + int error; + + xfs_assert_ilocked(ip, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); + + if (!xfs_inode_hasattr(ip)) + return 0; + + if (ip->i_af.if_format == XFS_DINODE_FMT_LOCAL) + return xchk_xattr_walk_sf(sc, ip, attr_fn, priv); + + /* attr functions require that the attr fork is loaded */ + error = xfs_iread_extents(sc->tp, ip, XFS_ATTR_FORK); + if (error) + return error; + + if (xfs_attr_is_leaf(ip)) + return xchk_xattr_walk_leaf(sc, ip, attr_fn, priv); + + return xchk_xattr_walk_node(sc, ip, attr_fn, priv); +} diff --git a/fs/xfs/scrub/listxattr.h b/fs/xfs/scrub/listxattr.h new file mode 100644 index 0000000000000..48fe89d05946b --- /dev/null +++ b/fs/xfs/scrub/listxattr.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_LISTXATTR_H__ +#define __XFS_SCRUB_LISTXATTR_H__ + +typedef int (*xchk_xattr_fn)(struct xfs_scrub *sc, struct xfs_inode *ip, + unsigned int attr_flags, const unsigned char *name, + unsigned int namelen, const void *value, unsigned int valuelen, + void *priv); + +int xchk_xattr_walk(struct xfs_scrub *sc, struct xfs_inode *ip, + xchk_xattr_fn attr_fn, void *priv); + +#endif /* __XFS_SCRUB_LISTXATTR_H__ */ From 8d81082a8c9541bdf6164c4639dc1936209fe1c4 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:50 -0700 Subject: [PATCH 0242/1702] xfs: inactivate directory data blocks Teach inode inactivation to delete all the incore buffers backing a directory. In normal runtime this should never happen because the VFS forbids rmdir on a non-empty directory. In the next patch, online directory repair stands up a new directory, exchanges it with the broken directory, and then drops the private temporary directory. If we cancel the repair just prior to exchanging the directory contents, the new directory will need to be torn down. Note: If we commit the repair, reaping will take care of all the ondisk space allocations and incore buffers for the old corrupt directory. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_inode.c | 51 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index b24c0e23d37d7..09d643a9e997b 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -16,6 +16,7 @@ #include "xfs_inode.h" #include "xfs_dir2.h" #include "xfs_attr.h" +#include "xfs_bit.h" #include "xfs_trans_space.h" #include "xfs_trans.h" #include "xfs_buf_item.h" @@ -1551,6 +1552,51 @@ xfs_release( return error; } +/* + * Mark all the buffers attached to this directory stale. In theory we should + * never be freeing a directory with any blocks at all, but this covers the + * case where we've recovered a directory swap with a "temporary" directory + * created by online repair and now need to dump it. + */ +STATIC void +xfs_inactive_dir( + struct xfs_inode *dp) +{ + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec got; + struct xfs_mount *mp = dp->i_mount; + struct xfs_da_geometry *geo = mp->m_dir_geo; + struct xfs_ifork *ifp = xfs_ifork_ptr(dp, XFS_DATA_FORK); + xfs_fileoff_t off; + + /* + * Invalidate each directory block. All directory blocks are of + * fsbcount length and alignment, so we only need to walk those same + * offsets. We hold the only reference to this inode, so we must wait + * for the buffer locks. + */ + for_each_xfs_iext(ifp, &icur, &got) { + for (off = round_up(got.br_startoff, geo->fsbcount); + off < got.br_startoff + got.br_blockcount; + off += geo->fsbcount) { + struct xfs_buf *bp = NULL; + xfs_fsblock_t fsbno; + int error; + + fsbno = (off - got.br_startoff) + got.br_startblock; + error = xfs_buf_incore(mp->m_ddev_targp, + XFS_FSB_TO_DADDR(mp, fsbno), + XFS_FSB_TO_BB(mp, geo->fsbcount), + XBF_LIVESCAN, &bp); + if (error) + continue; + + xfs_buf_stale(bp); + xfs_buf_relse(bp); + } + } +} + /* * xfs_inactive_truncate * @@ -1861,6 +1907,11 @@ xfs_inactive( goto out; } + if (S_ISDIR(VFS_I(ip)->i_mode) && ip->i_df.if_nextents > 0) { + xfs_inactive_dir(ip); + truncate = 1; + } + if (S_ISLNK(VFS_I(ip)->i_mode)) error = xfs_inactive_symlink(ip); else if (truncate) From b1991ee3e7cf852e95a3498801303cfbb4468681 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:51 -0700 Subject: [PATCH 0243/1702] xfs: online repair of directories If a directory looks like it's in bad shape, try to sift through the rubble to find whatever directory entries we can, scan the directory tree for the parent (if needed), stage the new directory contents in a temporary file and use the atomic extent swapping mechanism to commit the results in bulk. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/scrub/dir.c | 9 + fs/xfs/scrub/dir_repair.c | 1349 ++++++++++++++++++++++++++++++++++ fs/xfs/scrub/inode_repair.c | 5 + fs/xfs/scrub/nlinks.c | 23 + fs/xfs/scrub/nlinks_repair.c | 9 + fs/xfs/scrub/parent.c | 4 +- fs/xfs/scrub/readdir.c | 7 + fs/xfs/scrub/repair.c | 1 + fs/xfs/scrub/repair.h | 4 + fs/xfs/scrub/scrub.c | 2 +- fs/xfs/scrub/tempfile.c | 13 + fs/xfs/scrub/tempfile.h | 2 + fs/xfs/scrub/trace.h | 112 +++ fs/xfs/scrub/xfblob.h | 24 + 15 files changed, 1563 insertions(+), 2 deletions(-) create mode 100644 fs/xfs/scrub/dir_repair.c diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index 7dbe6b3befb33..5c9449e14f743 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -198,6 +198,7 @@ xfs-y += $(addprefix scrub/, \ attr_repair.o \ bmap_repair.o \ cow_repair.o \ + dir_repair.o \ fscounters_repair.o \ ialloc_repair.o \ inode_repair.o \ diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index 7bac74621af77..3fe6ffcf9c062 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -21,12 +21,21 @@ #include "scrub/dabtree.h" #include "scrub/readdir.h" #include "scrub/health.h" +#include "scrub/repair.h" /* Set us up to scrub directories. */ int xchk_setup_directory( struct xfs_scrub *sc) { + int error; + + if (xchk_could_repair(sc)) { + error = xrep_setup_directory(sc); + if (error) + return error; + } + return xchk_setup_inode_contents(sc, 0); } diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c new file mode 100644 index 0000000000000..48aa80d8c7dc4 --- /dev/null +++ b/fs/xfs/scrub/dir_repair.c @@ -0,0 +1,1349 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_bmap.h" +#include "xfs_quota.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "xfs_bmap_util.h" +#include "xfs_exchmaps.h" +#include "xfs_exchrange.h" +#include "xfs_ag.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/tempfile.h" +#include "scrub/tempexch.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" +#include "scrub/readdir.h" +#include "scrub/reap.h" + +/* + * Directory Repair + * ================ + * + * We repair directories by reading the directory data blocks looking for + * directory entries that look salvageable (name passes verifiers, entry points + * to a valid allocated inode, etc). Each entry worth salvaging is stashed in + * memory, and the stashed entries are periodically replayed into a temporary + * directory to constrain memory use. Batching the construction of the + * temporary directory in this fashion reduces lock cycling of the directory + * being repaired and the temporary directory, and will later become important + * for parent pointer scanning. + * + * Directory entries added to the temporary directory do not elevate the link + * counts of the inodes found. When salvaging completes, the remaining stashed + * entries are replayed to the temporary directory. An atomic mapping exchange + * is used to commit the new directory blocks to the directory being repaired. + * This will disrupt readdir cursors. + * + * Locking Issues + * -------------- + * + * If /a, /a/b, and /c are all directories, the VFS does not take i_rwsem on + * /a/b for a "mv /a/b /c/" operation. This means that only b's ILOCK protects + * b's dotdot update. This is in contrast to every other dotdot update (link, + * remove, mkdir). If the repair code drops the ILOCK, it must either + * revalidate the dotdot entry or use dirent hooks to capture updates from + * other threads. + */ + +/* Directory entry to be restored in the new directory. */ +struct xrep_dirent { + /* Cookie for retrieval of the dirent name. */ + xfblob_cookie name_cookie; + + /* Target inode number. */ + xfs_ino_t ino; + + /* Length of the dirent name. */ + uint8_t namelen; + + /* File type of the dirent. */ + uint8_t ftype; +}; + +/* + * Stash up to 8 pages of recovered dirent data in dir_entries and dir_names + * before we write them to the temp dir. + */ +#define XREP_DIR_MAX_STASH_BYTES (PAGE_SIZE * 8) + +struct xrep_dir { + struct xfs_scrub *sc; + + /* Fixed-size array of xrep_dirent structures. */ + struct xfarray *dir_entries; + + /* Blobs containing directory entry names. */ + struct xfblob *dir_names; + + /* Information for exchanging data forks at the end. */ + struct xrep_tempexch tx; + + /* Preallocated args struct for performing dir operations */ + struct xfs_da_args args; + + /* + * This is the parent that we're going to set on the reconstructed + * directory. + */ + xfs_ino_t parent_ino; + + /* How many subdirectories did we find? */ + uint64_t subdirs; + + /* How many dirents did we find? */ + unsigned int dirents; + + /* Directory entry name, plus the trailing null. */ + struct xfs_name xname; + unsigned char namebuf[MAXNAMELEN]; +}; + +/* Tear down all the incore stuff we created. */ +static void +xrep_dir_teardown( + struct xfs_scrub *sc) +{ + struct xrep_dir *rd = sc->buf; + + xfblob_destroy(rd->dir_names); + xfarray_destroy(rd->dir_entries); +} + +/* Set up for a directory repair. */ +int +xrep_setup_directory( + struct xfs_scrub *sc) +{ + struct xrep_dir *rd; + int error; + + error = xrep_tempfile_create(sc, S_IFDIR); + if (error) + return error; + + rd = kvzalloc(sizeof(struct xrep_dir), XCHK_GFP_FLAGS); + if (!rd) + return -ENOMEM; + rd->sc = sc; + rd->xname.name = rd->namebuf; + sc->buf = rd; + + return 0; +} + +/* + * If we're the root of a directory tree, we are our own parent. If we're an + * unlinked directory, the parent /won't/ have a link to us. Set the parent + * directory to the root for both cases. Returns NULLFSINO if we don't know + * what to do. + */ +static inline xfs_ino_t +xrep_dir_self_parent( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + + if (sc->ip->i_ino == sc->mp->m_sb.sb_rootino) + return sc->mp->m_sb.sb_rootino; + + if (VFS_I(sc->ip)->i_nlink == 0) + return sc->mp->m_sb.sb_rootino; + + return NULLFSINO; +} + +/* + * Look up the dotdot entry. Returns NULLFSINO if we don't know what to do. + * The next patch will check this more carefully. + */ +static inline xfs_ino_t +xrep_dir_lookup_parent( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + xfs_ino_t ino; + int error; + + error = xfs_dir_lookup(sc->tp, sc->ip, &xfs_name_dotdot, &ino, NULL); + if (error) + return NULLFSINO; + if (!xfs_verify_dir_ino(sc->mp, ino)) + return NULLFSINO; + + return ino; +} + +/* + * Try to find the parent of the directory being repaired. + * + * NOTE: This function will someday be augmented by the directory parent repair + * code, which will know how to check the parent and scan the filesystem if + * we cannot find anything. Inode scans will have to be done before we start + * salvaging directory entries, so we do this now. + */ +STATIC int +xrep_dir_find_parent( + struct xrep_dir *rd) +{ + xfs_ino_t ino; + + ino = xrep_dir_self_parent(rd); + if (ino != NULLFSINO) { + rd->parent_ino = ino; + return 0; + } + + ino = xrep_dir_lookup_parent(rd); + if (ino != NULLFSINO) { + rd->parent_ino = ino; + return 0; + } + + /* NOTE: A future patch will deal with moving orphans. */ + return -EFSCORRUPTED; +} + +/* + * Decide if we want to salvage this entry. We don't bother with oversized + * names or the dot entry. + */ +STATIC int +xrep_dir_want_salvage( + struct xrep_dir *rd, + const char *name, + int namelen, + xfs_ino_t ino) +{ + struct xfs_mount *mp = rd->sc->mp; + + /* No pointers to ourselves or to garbage. */ + if (ino == rd->sc->ip->i_ino) + return false; + if (!xfs_verify_dir_ino(mp, ino)) + return false; + + /* No weird looking names or dot entries. */ + if (namelen >= MAXNAMELEN || namelen <= 0) + return false; + if (namelen == 1 && name[0] == '.') + return false; + if (!xfs_dir2_namecheck(name, namelen)) + return false; + + return true; +} + +/* + * Remember that we want to create a dirent in the tempdir. These stashed + * actions will be replayed later. + */ +STATIC int +xrep_dir_stash_createname( + struct xrep_dir *rd, + const struct xfs_name *name, + xfs_ino_t ino) +{ + struct xrep_dirent dirent = { + .ino = ino, + .namelen = name->len, + .ftype = name->type, + }; + int error; + + trace_xrep_dir_stash_createname(rd->sc->tempip, name, ino); + + error = xfblob_storename(rd->dir_names, &dirent.name_cookie, name); + if (error) + return error; + + return xfarray_append(rd->dir_entries, &dirent); +} + +/* Allocate an in-core record to hold entries while we rebuild the dir data. */ +STATIC int +xrep_dir_salvage_entry( + struct xrep_dir *rd, + unsigned char *name, + unsigned int namelen, + xfs_ino_t ino) +{ + struct xfs_name xname = { + .name = name, + }; + struct xfs_scrub *sc = rd->sc; + struct xfs_inode *ip; + unsigned int i = 0; + int error = 0; + + if (xchk_should_terminate(sc, &error)) + return error; + + /* + * Truncate the name to the first character that would trip namecheck. + * If we no longer have a name after that, ignore this entry. + */ + while (i < namelen && name[i] != 0 && name[i] != '/') + i++; + if (i == 0) + return 0; + xname.len = i; + + /* Ignore '..' entries; we already picked the new parent. */ + if (xname.len == 2 && name[0] == '.' && name[1] == '.') { + trace_xrep_dir_salvaged_parent(sc->ip, ino); + return 0; + } + + trace_xrep_dir_salvage_entry(sc->ip, &xname, ino); + + /* + * Compute the ftype or dump the entry if we can't. We don't lock the + * inode because inodes can't change type while we have a reference. + */ + error = xchk_iget(sc, ino, &ip); + if (error) + return 0; + + xname.type = xfs_mode_to_ftype(VFS_I(ip)->i_mode); + xchk_irele(sc, ip); + + return xrep_dir_stash_createname(rd, &xname, ino); +} + +/* Record a shortform directory entry for later reinsertion. */ +STATIC int +xrep_dir_salvage_sf_entry( + struct xrep_dir *rd, + struct xfs_dir2_sf_hdr *sfp, + struct xfs_dir2_sf_entry *sfep) +{ + xfs_ino_t ino; + + ino = xfs_dir2_sf_get_ino(rd->sc->mp, sfp, sfep); + if (!xrep_dir_want_salvage(rd, sfep->name, sfep->namelen, ino)) + return 0; + + return xrep_dir_salvage_entry(rd, sfep->name, sfep->namelen, ino); +} + +/* Record a regular directory entry for later reinsertion. */ +STATIC int +xrep_dir_salvage_data_entry( + struct xrep_dir *rd, + struct xfs_dir2_data_entry *dep) +{ + xfs_ino_t ino; + + ino = be64_to_cpu(dep->inumber); + if (!xrep_dir_want_salvage(rd, dep->name, dep->namelen, ino)) + return 0; + + return xrep_dir_salvage_entry(rd, dep->name, dep->namelen, ino); +} + +/* Try to recover block/data format directory entries. */ +STATIC int +xrep_dir_recover_data( + struct xrep_dir *rd, + struct xfs_buf *bp) +{ + struct xfs_da_geometry *geo = rd->sc->mp->m_dir_geo; + unsigned int offset; + unsigned int end; + int error = 0; + + /* + * Loop over the data portion of the block. + * Each object is a real entry (dep) or an unused one (dup). + */ + offset = geo->data_entry_offset; + end = min_t(unsigned int, BBTOB(bp->b_length), + xfs_dir3_data_end_offset(geo, bp->b_addr)); + + while (offset < end) { + struct xfs_dir2_data_unused *dup = bp->b_addr + offset; + struct xfs_dir2_data_entry *dep = bp->b_addr + offset; + + if (xchk_should_terminate(rd->sc, &error)) + return error; + + /* Skip unused entries. */ + if (be16_to_cpu(dup->freetag) == XFS_DIR2_DATA_FREE_TAG) { + offset += be16_to_cpu(dup->length); + continue; + } + + /* Don't walk off the end of the block. */ + offset += xfs_dir2_data_entsize(rd->sc->mp, dep->namelen); + if (offset > end) + break; + + /* Ok, let's save this entry. */ + error = xrep_dir_salvage_data_entry(rd, dep); + if (error) + return error; + + } + + return 0; +} + +/* Try to recover shortform directory entries. */ +STATIC int +xrep_dir_recover_sf( + struct xrep_dir *rd) +{ + struct xfs_dir2_sf_hdr *hdr; + struct xfs_dir2_sf_entry *sfep; + struct xfs_dir2_sf_entry *next; + struct xfs_ifork *ifp; + xfs_ino_t ino; + unsigned char *end; + int error = 0; + + ifp = xfs_ifork_ptr(rd->sc->ip, XFS_DATA_FORK); + hdr = ifp->if_data; + end = (unsigned char *)ifp->if_data + ifp->if_bytes; + + ino = xfs_dir2_sf_get_parent_ino(hdr); + trace_xrep_dir_salvaged_parent(rd->sc->ip, ino); + + sfep = xfs_dir2_sf_firstentry(hdr); + while ((unsigned char *)sfep < end) { + if (xchk_should_terminate(rd->sc, &error)) + return error; + + next = xfs_dir2_sf_nextentry(rd->sc->mp, hdr, sfep); + if ((unsigned char *)next > end) + break; + + /* Ok, let's save this entry. */ + error = xrep_dir_salvage_sf_entry(rd, hdr, sfep); + if (error) + return error; + + sfep = next; + } + + return 0; +} + +/* + * Try to figure out the format of this directory from the data fork mappings + * and the directory size. If we can be reasonably sure of format, we can be + * more aggressive in salvaging directory entries. On return, @magic_guess + * will be set to DIR3_BLOCK_MAGIC if we think this is a "block format" + * directory; DIR3_DATA_MAGIC if we think this is a "data format" directory, + * and 0 if we can't tell. + */ +STATIC void +xrep_dir_guess_format( + struct xrep_dir *rd, + __be32 *magic_guess) +{ + struct xfs_inode *dp = rd->sc->ip; + struct xfs_mount *mp = rd->sc->mp; + struct xfs_da_geometry *geo = mp->m_dir_geo; + xfs_fileoff_t last; + int error; + + ASSERT(xfs_has_crc(mp)); + + *magic_guess = 0; + + /* + * If there's a single directory block and the directory size is + * exactly one block, this has to be a single block format directory. + */ + error = xfs_bmap_last_offset(dp, &last, XFS_DATA_FORK); + if (!error && XFS_FSB_TO_B(mp, last) == geo->blksize && + dp->i_disk_size == geo->blksize) { + *magic_guess = cpu_to_be32(XFS_DIR3_BLOCK_MAGIC); + return; + } + + /* + * If the last extent before the leaf offset matches the directory + * size and the directory size is larger than 1 block, this is a + * data format directory. + */ + last = geo->leafblk; + error = xfs_bmap_last_before(rd->sc->tp, dp, &last, XFS_DATA_FORK); + if (!error && + XFS_FSB_TO_B(mp, last) > geo->blksize && + XFS_FSB_TO_B(mp, last) == dp->i_disk_size) { + *magic_guess = cpu_to_be32(XFS_DIR3_DATA_MAGIC); + return; + } +} + +/* Recover directory entries from a specific directory block. */ +STATIC int +xrep_dir_recover_dirblock( + struct xrep_dir *rd, + __be32 magic_guess, + xfs_dablk_t dabno) +{ + struct xfs_dir2_data_hdr *hdr; + struct xfs_buf *bp; + __be32 oldmagic; + int error; + + /* + * Try to read buffer. We invalidate them in the next step so we don't + * bother to set a buffer type or ops. + */ + error = xfs_da_read_buf(rd->sc->tp, rd->sc->ip, dabno, + XFS_DABUF_MAP_HOLE_OK, &bp, XFS_DATA_FORK, NULL); + if (error || !bp) + return error; + + hdr = bp->b_addr; + oldmagic = hdr->magic; + + trace_xrep_dir_recover_dirblock(rd->sc->ip, dabno, + be32_to_cpu(hdr->magic), be32_to_cpu(magic_guess)); + + /* + * If we're sure of the block's format, proceed with the salvage + * operation using the specified magic number. + */ + if (magic_guess) { + hdr->magic = magic_guess; + goto recover; + } + + /* + * If we couldn't guess what type of directory this is, then we will + * only salvage entries from directory blocks that match the magic + * number and pass verifiers. + */ + switch (hdr->magic) { + case cpu_to_be32(XFS_DIR2_BLOCK_MAGIC): + case cpu_to_be32(XFS_DIR3_BLOCK_MAGIC): + if (!xrep_buf_verify_struct(bp, &xfs_dir3_block_buf_ops)) + goto out; + if (xfs_dir3_block_header_check(bp, rd->sc->ip->i_ino) != NULL) + goto out; + break; + case cpu_to_be32(XFS_DIR2_DATA_MAGIC): + case cpu_to_be32(XFS_DIR3_DATA_MAGIC): + if (!xrep_buf_verify_struct(bp, &xfs_dir3_data_buf_ops)) + goto out; + if (xfs_dir3_data_header_check(bp, rd->sc->ip->i_ino) != NULL) + goto out; + break; + default: + goto out; + } + +recover: + error = xrep_dir_recover_data(rd, bp); + +out: + hdr->magic = oldmagic; + xfs_trans_brelse(rd->sc->tp, bp); + return error; +} + +static inline void +xrep_dir_init_args( + struct xrep_dir *rd, + struct xfs_inode *dp, + const struct xfs_name *name) +{ + memset(&rd->args, 0, sizeof(struct xfs_da_args)); + rd->args.geo = rd->sc->mp->m_dir_geo; + rd->args.whichfork = XFS_DATA_FORK; + rd->args.owner = rd->sc->ip->i_ino; + rd->args.trans = rd->sc->tp; + rd->args.dp = dp; + if (!name) + return; + rd->args.name = name->name; + rd->args.namelen = name->len; + rd->args.filetype = name->type; + rd->args.hashval = xfs_dir2_hashname(rd->sc->mp, name); +} + +/* Replay a stashed createname into the temporary directory. */ +STATIC int +xrep_dir_replay_createname( + struct xrep_dir *rd, + const struct xfs_name *name, + xfs_ino_t inum, + xfs_extlen_t total) +{ + struct xfs_scrub *sc = rd->sc; + struct xfs_inode *dp = rd->sc->tempip; + bool is_block, is_leaf; + int error; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + + error = xfs_dir_ino_validate(sc->mp, inum); + if (error) + return error; + + trace_xrep_dir_replay_createname(dp, name, inum); + + xrep_dir_init_args(rd, dp, name); + rd->args.inumber = inum; + rd->args.total = total; + rd->args.op_flags = XFS_DA_OP_ADDNAME | XFS_DA_OP_OKNOENT; + + if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) + return xfs_dir2_sf_addname(&rd->args); + + error = xfs_dir2_isblock(&rd->args, &is_block); + if (error) + return error; + if (is_block) + return xfs_dir2_block_addname(&rd->args); + + error = xfs_dir2_isleaf(&rd->args, &is_leaf); + if (error) + return error; + if (is_leaf) + return xfs_dir2_leaf_addname(&rd->args); + + return xfs_dir2_node_addname(&rd->args); +} + +/* + * Add this stashed incore directory entry to the temporary directory. + * The caller must hold the tempdir's IOLOCK, must not hold any ILOCKs, and + * must not be in transaction context. + */ +STATIC int +xrep_dir_replay_update( + struct xrep_dir *rd, + const struct xfs_name *xname, + const struct xrep_dirent *dirent) +{ + struct xfs_mount *mp = rd->sc->mp; +#ifdef DEBUG + xfs_ino_t ino; +#endif + uint resblks; + int error; + + resblks = XFS_LINK_SPACE_RES(mp, xname->len); + error = xchk_trans_alloc(rd->sc, resblks); + if (error) + return error; + + /* Lock the temporary directory and join it to the transaction */ + xrep_tempfile_ilock(rd->sc); + xfs_trans_ijoin(rd->sc->tp, rd->sc->tempip, 0); + + /* + * Create a replacement dirent in the temporary directory. Note that + * _createname doesn't check for existing entries. There shouldn't be + * any in the temporary dir, but we'll verify this in debug mode. + */ +#ifdef DEBUG + error = xchk_dir_lookup(rd->sc, rd->sc->tempip, xname, &ino); + if (error != -ENOENT) { + ASSERT(error != -ENOENT); + goto out_cancel; + } +#endif + + error = xrep_dir_replay_createname(rd, xname, dirent->ino, resblks); + if (error) + goto out_cancel; + + if (xname->type == XFS_DIR3_FT_DIR) + rd->subdirs++; + rd->dirents++; + + /* Commit and unlock. */ + error = xrep_trans_commit(rd->sc); + if (error) + return error; + + xrep_tempfile_iunlock(rd->sc); + return 0; +out_cancel: + xchk_trans_cancel(rd->sc); + xrep_tempfile_iunlock(rd->sc); + return error; +} + +/* + * Flush stashed incore dirent updates that have been recorded by the scanner. + * This is done to reduce the memory requirements of the directory rebuild, + * since directories can contain up to 32GB of directory data. + * + * Caller must not hold transactions or ILOCKs. Caller must hold the tempdir + * IOLOCK. + */ +STATIC int +xrep_dir_replay_updates( + struct xrep_dir *rd) +{ + xfarray_idx_t array_cur; + int error; + + /* Add all the salvaged dirents to the temporary directory. */ + foreach_xfarray_idx(rd->dir_entries, array_cur) { + struct xrep_dirent dirent; + + error = xfarray_load(rd->dir_entries, array_cur, &dirent); + if (error) + return error; + + error = xfblob_loadname(rd->dir_names, dirent.name_cookie, + &rd->xname, dirent.namelen); + if (error) + return error; + rd->xname.type = dirent.ftype; + + error = xrep_dir_replay_update(rd, &rd->xname, &dirent); + if (error) + return error; + } + + /* Empty out both arrays now that we've added the entries. */ + xfarray_truncate(rd->dir_entries); + xfblob_truncate(rd->dir_names); + return 0; +} + +/* + * Periodically flush stashed directory entries to the temporary dir. This + * is done to reduce the memory requirements of the directory rebuild, since + * directories can contain up to 32GB of directory data. + */ +STATIC int +xrep_dir_flush_stashed( + struct xrep_dir *rd) +{ + int error; + + /* + * Entering this function, the scrub context has a reference to the + * inode being repaired, the temporary file, and a scrub transaction + * that we use during dirent salvaging to avoid livelocking if there + * are cycles in the directory structures. We hold ILOCK_EXCL on both + * the inode being repaired and the temporary file, though they are + * not ijoined to the scrub transaction. + * + * To constrain kernel memory use, we occasionally write salvaged + * dirents from the xfarray and xfblob structures into the temporary + * directory in preparation for exchanging the directory structures at + * the end. Updating the temporary file requires a transaction, so we + * commit the scrub transaction and drop the two ILOCKs so that + * we can allocate whatever transaction we want. + * + * We still hold IOLOCK_EXCL on the inode being repaired, which + * prevents anyone from accessing the damaged directory data while we + * repair it. + */ + error = xrep_trans_commit(rd->sc); + if (error) + return error; + xchk_iunlock(rd->sc, XFS_ILOCK_EXCL); + + /* + * Take the IOLOCK of the temporary file while we modify dirents. This + * isn't strictly required because the temporary file is never revealed + * to userspace, but we follow the same locking rules. We still hold + * sc->ip's IOLOCK. + */ + error = xrep_tempfile_iolock_polled(rd->sc); + if (error) + return error; + + /* Write to the tempdir all the updates that we've stashed. */ + error = xrep_dir_replay_updates(rd); + xrep_tempfile_iounlock(rd->sc); + if (error) + return error; + + /* + * Recreate the salvage transaction and relock the dir we're salvaging. + */ + error = xchk_trans_alloc(rd->sc, 0); + if (error) + return error; + xchk_ilock(rd->sc, XFS_ILOCK_EXCL); + return 0; +} + +/* Decide if we've stashed too much dirent data in memory. */ +static inline bool +xrep_dir_want_flush_stashed( + struct xrep_dir *rd) +{ + unsigned long long bytes; + + bytes = xfarray_bytes(rd->dir_entries) + xfblob_bytes(rd->dir_names); + return bytes > XREP_DIR_MAX_STASH_BYTES; +} + +/* Extract as many directory entries as we can. */ +STATIC int +xrep_dir_recover( + struct xrep_dir *rd) +{ + struct xfs_bmbt_irec got; + struct xfs_scrub *sc = rd->sc; + struct xfs_da_geometry *geo = sc->mp->m_dir_geo; + xfs_fileoff_t offset; + xfs_dablk_t dabno; + __be32 magic_guess; + int nmap; + int error; + + xrep_dir_guess_format(rd, &magic_guess); + + /* Iterate each directory data block in the data fork. */ + for (offset = 0; + offset < geo->leafblk; + offset = got.br_startoff + got.br_blockcount) { + nmap = 1; + error = xfs_bmapi_read(sc->ip, offset, geo->leafblk - offset, + &got, &nmap, 0); + if (error) + return error; + if (nmap != 1) + return -EFSCORRUPTED; + if (!xfs_bmap_is_written_extent(&got)) + continue; + + for (dabno = round_up(got.br_startoff, geo->fsbcount); + dabno < got.br_startoff + got.br_blockcount; + dabno += geo->fsbcount) { + if (xchk_should_terminate(rd->sc, &error)) + return error; + + error = xrep_dir_recover_dirblock(rd, + magic_guess, dabno); + if (error) + return error; + + /* Flush dirents to constrain memory usage. */ + if (xrep_dir_want_flush_stashed(rd)) { + error = xrep_dir_flush_stashed(rd); + if (error) + return error; + } + } + } + + return 0; +} + +/* + * Find all the directory entries for this inode by scraping them out of the + * directory leaf blocks by hand, and flushing them into the temp dir. + */ +STATIC int +xrep_dir_find_entries( + struct xrep_dir *rd) +{ + struct xfs_inode *dp = rd->sc->ip; + int error; + + /* + * Salvage directory entries from the old directory, and write them to + * the temporary directory. + */ + if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) { + error = xrep_dir_recover_sf(rd); + } else { + error = xfs_iread_extents(rd->sc->tp, dp, XFS_DATA_FORK); + if (error) + return error; + + error = xrep_dir_recover(rd); + } + if (error) + return error; + + return xrep_dir_flush_stashed(rd); +} + +/* Scan all files in the filesystem for dirents. */ +STATIC int +xrep_dir_salvage_entries( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + int error; + + /* + * Drop the ILOCK on this directory so that we can scan for this + * directory's parent. Figure out who is going to be the parent of + * this directory, then retake the ILOCK so that we can salvage + * directory entries. + */ + xchk_iunlock(sc, XFS_ILOCK_EXCL); + error = xrep_dir_find_parent(rd); + xchk_ilock(sc, XFS_ILOCK_EXCL); + if (error) + return error; + + /* + * Collect directory entries by parsing raw leaf blocks to salvage + * whatever we can. When we're done, free the staging memory before + * exchanging the directories to reduce memory usage. + */ + error = xrep_dir_find_entries(rd); + if (error) + return error; + + /* + * Cancel the repair transaction and drop the ILOCK so that we can + * (later) use the atomic mapping exchange functions to compute the + * correct block reservations and re-lock the inodes. + * + * We still hold IOLOCK_EXCL (aka i_rwsem) which will prevent directory + * modifications, but there's nothing to prevent userspace from reading + * the directory until we're ready for the exchange operation. Reads + * will return -EIO without shutting down the fs, so we're ok with + * that. + */ + error = xrep_trans_commit(sc); + if (error) + return error; + + xchk_iunlock(sc, XFS_ILOCK_EXCL); + return 0; +} + + +/* + * Free all the directory blocks and reset the data fork. The caller must + * join the inode to the transaction. This function returns with the inode + * joined to a clean scrub transaction. + */ +STATIC int +xrep_dir_reset_fork( + struct xrep_dir *rd, + xfs_ino_t parent_ino) +{ + struct xfs_scrub *sc = rd->sc; + struct xfs_ifork *ifp = xfs_ifork_ptr(sc->tempip, XFS_DATA_FORK); + int error; + + /* Unmap all the directory buffers. */ + if (xfs_ifork_has_extents(ifp)) { + error = xrep_reap_ifork(sc, sc->tempip, XFS_DATA_FORK); + if (error) + return error; + } + + trace_xrep_dir_reset_fork(sc->tempip, parent_ino); + + /* Reset the data fork to an empty data fork. */ + xfs_idestroy_fork(ifp); + ifp->if_bytes = 0; + sc->tempip->i_disk_size = 0; + + /* Reinitialize the short form directory. */ + xrep_dir_init_args(rd, sc->tempip, NULL); + return xfs_dir2_sf_create(&rd->args, parent_ino); +} + +/* + * Prepare both inodes' directory forks for exchanging mappings. Promote the + * tempfile from short format to leaf format, and if the file being repaired + * has a short format data fork, turn it into an empty extent list. + */ +STATIC int +xrep_dir_swap_prep( + struct xfs_scrub *sc, + bool temp_local, + bool ip_local) +{ + int error; + + /* + * If the tempfile's directory is in shortform format, convert that to + * a single leaf extent so that we can use the atomic mapping exchange. + */ + if (temp_local) { + struct xfs_da_args args = { + .dp = sc->tempip, + .geo = sc->mp->m_dir_geo, + .whichfork = XFS_DATA_FORK, + .trans = sc->tp, + .total = 1, + .owner = sc->ip->i_ino, + }; + + error = xfs_dir2_sf_to_block(&args); + if (error) + return error; + + /* + * Roll the deferred log items to get us back to a clean + * transaction. + */ + error = xfs_defer_finish(&sc->tp); + if (error) + return error; + } + + /* + * If the file being repaired had a shortform data fork, convert that + * to an empty extent list in preparation for the atomic mapping + * exchange. + */ + if (ip_local) { + struct xfs_ifork *ifp; + + ifp = xfs_ifork_ptr(sc->ip, XFS_DATA_FORK); + xfs_idestroy_fork(ifp); + ifp->if_format = XFS_DINODE_FMT_EXTENTS; + ifp->if_nextents = 0; + ifp->if_bytes = 0; + ifp->if_data = NULL; + ifp->if_height = 0; + + xfs_trans_log_inode(sc->tp, sc->ip, + XFS_ILOG_CORE | XFS_ILOG_DDATA); + } + + return 0; +} + +/* + * Replace the inode number of a directory entry. + */ +static int +xrep_dir_replace( + struct xrep_dir *rd, + struct xfs_inode *dp, + const struct xfs_name *name, + xfs_ino_t inum, + xfs_extlen_t total) +{ + struct xfs_scrub *sc = rd->sc; + bool is_block, is_leaf; + int error; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + + error = xfs_dir_ino_validate(sc->mp, inum); + if (error) + return error; + + xrep_dir_init_args(rd, dp, name); + rd->args.inumber = inum; + rd->args.total = total; + + if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) + return xfs_dir2_sf_replace(&rd->args); + + error = xfs_dir2_isblock(&rd->args, &is_block); + if (error) + return error; + if (is_block) + return xfs_dir2_block_replace(&rd->args); + + error = xfs_dir2_isleaf(&rd->args, &is_leaf); + if (error) + return error; + if (is_leaf) + return xfs_dir2_leaf_replace(&rd->args); + + return xfs_dir2_node_replace(&rd->args); +} + +/* + * Reset the link count of this directory and adjust the unlinked list pointers + * as needed. + */ +STATIC int +xrep_dir_set_nlink( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + struct xfs_inode *dp = sc->ip; + struct xfs_perag *pag; + unsigned int new_nlink = rd->subdirs + 2; + int error; + + /* + * The directory is not on the incore unlinked list, which means that + * it needs to be reachable via the directory tree. Update the nlink + * with our observed link count. + * + * XXX: A subsequent patch will handle parentless directories by moving + * them to the lost and found instead of aborting the repair. + */ + if (!xfs_inode_on_unlinked_list(dp)) + goto reset_nlink; + + /* + * The directory is on the unlinked list and we did not find any + * dirents. Set the link count to zero and let the directory + * inactivate when the last reference drops. + */ + if (rd->dirents == 0) { + new_nlink = 0; + goto reset_nlink; + } + + /* + * The directory is on the unlinked list and we found dirents. This + * directory needs to be reachable via the directory tree. Remove the + * dir from the unlinked list and update nlink with the observed link + * count. + */ + pag = xfs_perag_get(sc->mp, XFS_INO_TO_AGNO(sc->mp, dp->i_ino)); + if (!pag) { + ASSERT(0); + return -EFSCORRUPTED; + } + + error = xfs_iunlink_remove(sc->tp, pag, dp); + xfs_perag_put(pag); + if (error) + return error; + +reset_nlink: + if (VFS_I(dp)->i_nlink != new_nlink) + set_nlink(VFS_I(dp), new_nlink); + return 0; +} + +/* Exchange the temporary directory's data fork with the one being repaired. */ +STATIC int +xrep_dir_swap( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + bool ip_local, temp_local; + int error = 0; + + /* + * If we found enough subdirs to overflow this directory's link count, + * bail out to userspace before we modify anything. + */ + if (rd->subdirs + 2 > XFS_MAXLINK) + return -EFSCORRUPTED; + + /* + * Reset the temporary directory's '..' entry to point to the parent + * that we found. The temporary directory was created with the root + * directory as the parent, so we can skip this if repairing a + * subdirectory of the root. + * + * It's also possible that this replacement could also expand a sf + * tempdir into block format. + */ + if (rd->parent_ino != sc->mp->m_rootip->i_ino) { + error = xrep_dir_replace(rd, rd->sc->tempip, &xfs_name_dotdot, + rd->parent_ino, rd->tx.req.resblks); + if (error) + return error; + } + + /* + * Changing the dot and dotdot entries could have changed the shape of + * the directory, so we recompute these. + */ + ip_local = sc->ip->i_df.if_format == XFS_DINODE_FMT_LOCAL; + temp_local = sc->tempip->i_df.if_format == XFS_DINODE_FMT_LOCAL; + + /* + * If the both files have a local format data fork and the rebuilt + * directory data would fit in the repaired file's data fork, copy + * the contents from the tempfile and update the directory link count. + * We're done now. + */ + if (ip_local && temp_local && + sc->tempip->i_disk_size <= xfs_inode_data_fork_size(sc->ip)) { + xrep_tempfile_copyout_local(sc, XFS_DATA_FORK); + return xrep_dir_set_nlink(rd); + } + + /* + * Clean the transaction before we start working on exchanging + * directory contents. + */ + error = xrep_tempfile_roll_trans(rd->sc); + if (error) + return error; + + /* Otherwise, make sure both data forks are in block-mapping mode. */ + error = xrep_dir_swap_prep(sc, temp_local, ip_local); + if (error) + return error; + + /* + * Set nlink of the directory in the same transaction sequence that + * (atomically) commits the new directory data. + */ + error = xrep_dir_set_nlink(rd); + if (error) + return error; + + return xrep_tempexch_contents(sc, &rd->tx); +} + +/* + * Exchange the new directory contents (which we created in the tempfile) with + * the directory being repaired. + */ +STATIC int +xrep_dir_rebuild_tree( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + int error; + + trace_xrep_dir_rebuild_tree(sc->ip, rd->parent_ino); + + /* + * Take the IOLOCK on the temporary file so that we can run dir + * operations with the same locks held as we would for a normal file. + * We still hold sc->ip's IOLOCK. + */ + error = xrep_tempfile_iolock_polled(rd->sc); + if (error) + return error; + + /* Allocate transaction and ILOCK the scrub file and the temp file. */ + error = xrep_tempexch_trans_alloc(sc, XFS_DATA_FORK, &rd->tx); + if (error) + return error; + + /* + * Exchange the tempdir's data fork with the file being repaired. This + * recreates the transaction and re-takes the ILOCK in the scrub + * context. + */ + error = xrep_dir_swap(rd); + if (error) + return error; + + /* + * Release the old directory blocks and reset the data fork of the temp + * directory to an empty shortform directory because inactivation does + * nothing for directories. + */ + error = xrep_dir_reset_fork(rd, sc->mp->m_rootip->i_ino); + if (error) + return error; + + /* + * Roll to get a transaction without any inodes joined to it. Then we + * can drop the tempfile's ILOCK and IOLOCK before doing more work on + * the scrub target directory. + */ + error = xfs_trans_roll(&sc->tp); + if (error) + return error; + + xrep_tempfile_iunlock(sc); + xrep_tempfile_iounlock(sc); + return 0; +} + +/* Set up the filesystem scan so we can regenerate directory entries. */ +STATIC int +xrep_dir_setup_scan( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + char *descr; + int error; + + rd->parent_ino = NULLFSINO; + + /* Set up some staging memory for salvaging dirents. */ + descr = xchk_xfile_ino_descr(sc, "directory entries"); + error = xfarray_create(descr, 0, sizeof(struct xrep_dirent), + &rd->dir_entries); + kfree(descr); + if (error) + return error; + + descr = xchk_xfile_ino_descr(sc, "directory entry names"); + error = xfblob_create(descr, &rd->dir_names); + kfree(descr); + if (error) + goto out_xfarray; + + return 0; + +out_xfarray: + xfarray_destroy(rd->dir_entries); + rd->dir_entries = NULL; + return error; +} + +/* + * Repair the directory metadata. + * + * XXX: Directory entry buffers can be multiple fsblocks in size. The buffer + * cache in XFS can't handle aliased multiblock buffers, so this might + * misbehave if the directory blocks are crosslinked with other filesystem + * metadata. + * + * XXX: Is it necessary to check the dcache for this directory to make sure + * that we always recreate every cached entry? + */ +int +xrep_directory( + struct xfs_scrub *sc) +{ + struct xrep_dir *rd = sc->buf; + int error; + + /* The rmapbt is required to reap the old data fork. */ + if (!xfs_has_rmapbt(sc->mp)) + return -EOPNOTSUPP; + + error = xrep_dir_setup_scan(rd); + if (error) + return error; + + error = xrep_dir_salvage_entries(rd); + if (error) + goto out_teardown; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + goto out_teardown; + + error = xrep_dir_rebuild_tree(rd); + if (error) + goto out_teardown; + +out_teardown: + xrep_dir_teardown(sc); + return error; +} diff --git a/fs/xfs/scrub/inode_repair.c b/fs/xfs/scrub/inode_repair.c index c743772a523e9..0dde5df2f8d39 100644 --- a/fs/xfs/scrub/inode_repair.c +++ b/fs/xfs/scrub/inode_repair.c @@ -46,6 +46,7 @@ #include "scrub/repair.h" #include "scrub/iscan.h" #include "scrub/readdir.h" +#include "scrub/tempfile.h" /* * Inode Record Repair @@ -340,6 +341,10 @@ xrep_dinode_findmode_walk_directory( unsigned int lock_mode; int error = 0; + /* Ignore temporary repair directories. */ + if (xrep_is_tempfile(dp)) + return 0; + /* * Scan the directory to see if there it contains an entry pointing to * the directory that we are repairing. diff --git a/fs/xfs/scrub/nlinks.c b/fs/xfs/scrub/nlinks.c index 8a7d9557897cb..8b9aa73093d66 100644 --- a/fs/xfs/scrub/nlinks.c +++ b/fs/xfs/scrub/nlinks.c @@ -27,6 +27,7 @@ #include "scrub/nlinks.h" #include "scrub/trace.h" #include "scrub/readdir.h" +#include "scrub/tempfile.h" /* * Live Inode Link Count Checking @@ -152,6 +153,13 @@ xchk_nlinks_live_update( xnc = container_of(nb, struct xchk_nlink_ctrs, dhook.dirent_hook.nb); + /* + * Ignore temporary directories being used to stage dir repairs, since + * we don't bump the link counts of the children. + */ + if (xrep_is_tempfile(p->dp)) + return NOTIFY_DONE; + trace_xchk_nlinks_live_update(xnc->sc->mp, p->dp, action, p->ip->i_ino, p->delta, p->name->name, p->name->len); @@ -303,6 +311,13 @@ xchk_nlinks_collect_dir( unsigned int lock_mode; int error = 0; + /* + * Ignore temporary directories being used to stage dir repairs, since + * we don't bump the link counts of the children. + */ + if (xrep_is_tempfile(dp)) + return 0; + /* Prevent anyone from changing this directory while we walk it. */ xfs_ilock(dp, XFS_IOLOCK_SHARED); lock_mode = xfs_ilock_data_map_shared(dp); @@ -537,6 +552,14 @@ xchk_nlinks_compare_inode( unsigned int actual_nlink; int error; + /* + * Ignore temporary files being used to stage repairs, since we assume + * they're correct for non-directories, and the directory repair code + * doesn't bump the link counts for the children. + */ + if (xrep_is_tempfile(ip)) + return 0; + xfs_ilock(ip, XFS_ILOCK_SHARED); mutex_lock(&xnc->lock); diff --git a/fs/xfs/scrub/nlinks_repair.c b/fs/xfs/scrub/nlinks_repair.c index 58cacb8e94c1b..23eb08c4b5ad5 100644 --- a/fs/xfs/scrub/nlinks_repair.c +++ b/fs/xfs/scrub/nlinks_repair.c @@ -26,6 +26,7 @@ #include "scrub/iscan.h" #include "scrub/nlinks.h" #include "scrub/trace.h" +#include "scrub/tempfile.h" /* * Live Inode Link Count Repair @@ -68,6 +69,14 @@ xrep_nlinks_repair_inode( bool dirty = false; int error; + /* + * Ignore temporary files being used to stage repairs, since we assume + * they're correct for non-directories, and the directory repair code + * doesn't bump the link counts for the children. + */ + if (xrep_is_tempfile(ip)) + return 0; + xchk_ilock(sc, XFS_IOLOCK_EXCL); error = xfs_trans_alloc(mp, &M_RES(mp)->tr_link, 0, 0, 0, &sc->tp); diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c index 5da10ed1fe8ce..050a8e8914f6e 100644 --- a/fs/xfs/scrub/parent.c +++ b/fs/xfs/scrub/parent.c @@ -17,6 +17,7 @@ #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/readdir.h" +#include "scrub/tempfile.h" /* Set us up to scrub parents. */ int @@ -143,7 +144,8 @@ xchk_parent_validate( } if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) return error; - if (dp == sc->ip || dp == sc->tempip || !S_ISDIR(VFS_I(dp)->i_mode)) { + if (dp == sc->ip || xrep_is_tempfile(dp) || + !S_ISDIR(VFS_I(dp)->i_mode)) { xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); goto out_rele; } diff --git a/fs/xfs/scrub/readdir.c b/fs/xfs/scrub/readdir.c index e940804693151..028690761c629 100644 --- a/fs/xfs/scrub/readdir.c +++ b/fs/xfs/scrub/readdir.c @@ -333,6 +333,13 @@ xchk_dir_lookup( if (xfs_is_shutdown(dp->i_mount)) return -EIO; + /* + * A temporary directory's block headers are written with the owner + * set to sc->ip, so we must switch the owner here for the lookup. + */ + if (dp == sc->tempip) + args.owner = sc->ip->i_ino; + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); xfs_assert_ilocked(dp, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c index 04aec0e9e4c3d..369f0430e4ba1 100644 --- a/fs/xfs/scrub/repair.c +++ b/fs/xfs/scrub/repair.c @@ -35,6 +35,7 @@ #include "xfs_da_format.h" #include "xfs_da_btree.h" #include "xfs_attr.h" +#include "xfs_dir2.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/trace.h" diff --git a/fs/xfs/scrub/repair.h b/fs/xfs/scrub/repair.h index 9cbfd8da5620b..4e25aa95753a9 100644 --- a/fs/xfs/scrub/repair.h +++ b/fs/xfs/scrub/repair.h @@ -91,6 +91,7 @@ int xrep_metadata_inode_forks(struct xfs_scrub *sc); int xrep_setup_ag_rmapbt(struct xfs_scrub *sc); int xrep_setup_ag_refcountbt(struct xfs_scrub *sc); int xrep_setup_xattr(struct xfs_scrub *sc); +int xrep_setup_directory(struct xfs_scrub *sc); /* Repair setup functions */ int xrep_setup_ag_allocbt(struct xfs_scrub *sc); @@ -125,6 +126,7 @@ int xrep_bmap_cow(struct xfs_scrub *sc); int xrep_nlinks(struct xfs_scrub *sc); int xrep_fscounters(struct xfs_scrub *sc); int xrep_xattr(struct xfs_scrub *sc); +int xrep_directory(struct xfs_scrub *sc); #ifdef CONFIG_XFS_RT int xrep_rtbitmap(struct xfs_scrub *sc); @@ -195,6 +197,7 @@ xrep_setup_nothing( #define xrep_setup_ag_rmapbt xrep_setup_nothing #define xrep_setup_ag_refcountbt xrep_setup_nothing #define xrep_setup_xattr xrep_setup_nothing +#define xrep_setup_directory xrep_setup_nothing #define xrep_setup_inode(sc, imap) ((void)0) @@ -221,6 +224,7 @@ xrep_setup_nothing( #define xrep_fscounters xrep_notsupported #define xrep_rtsummary xrep_notsupported #define xrep_xattr xrep_notsupported +#define xrep_directory xrep_notsupported #endif /* CONFIG_XFS_ONLINE_REPAIR */ diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 547189a14b6bf..8e9e2bf121c2b 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -325,7 +325,7 @@ static const struct xchk_meta_ops meta_scrub_ops[] = { .type = ST_INODE, .setup = xchk_setup_directory, .scrub = xchk_directory, - .repair = xrep_notsupported, + .repair = xrep_directory, }, [XFS_SCRUB_TYPE_XATTR] = { /* extended attributes */ .type = ST_INODE, diff --git a/fs/xfs/scrub/tempfile.c b/fs/xfs/scrub/tempfile.c index 0b3060be938fb..4ca86a6a5be10 100644 --- a/fs/xfs/scrub/tempfile.c +++ b/fs/xfs/scrub/tempfile.c @@ -841,3 +841,16 @@ xrep_tempfile_copyout_local( ilog_flags |= xfs_ilog_fdata(whichfork); xfs_trans_log_inode(sc->tp, sc->ip, ilog_flags); } + +/* Decide if a given XFS inode is a temporary file for a repair. */ +bool +xrep_is_tempfile( + const struct xfs_inode *ip) +{ + const struct inode *inode = &ip->i_vnode; + + if (IS_PRIVATE(inode) && !(inode->i_opflags & IOP_XATTR)) + return true; + + return false; +} diff --git a/fs/xfs/scrub/tempfile.h b/fs/xfs/scrub/tempfile.h index d57e4f145a7c8..e51399f595fe9 100644 --- a/fs/xfs/scrub/tempfile.h +++ b/fs/xfs/scrub/tempfile.h @@ -35,11 +35,13 @@ int xrep_tempfile_set_isize(struct xfs_scrub *sc, unsigned long long isize); int xrep_tempfile_roll_trans(struct xfs_scrub *sc); void xrep_tempfile_copyout_local(struct xfs_scrub *sc, int whichfork); +bool xrep_is_tempfile(const struct xfs_inode *ip); #else static inline void xrep_tempfile_iolock_both(struct xfs_scrub *sc) { xchk_ilock(sc, XFS_IOLOCK_EXCL); } +# define xrep_is_tempfile(ip) (false) # define xrep_tempfile_rele(sc) #endif /* CONFIG_XFS_ONLINE_REPAIR */ diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index ffaff7722bf23..d6d9e8d6109cc 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2500,6 +2500,118 @@ DEFINE_EVENT(xrep_xattr_class, name, \ DEFINE_XREP_XATTR_EVENT(xrep_xattr_rebuild_tree); DEFINE_XREP_XATTR_EVENT(xrep_xattr_reset_fork); +TRACE_EVENT(xrep_dir_recover_dirblock, + TP_PROTO(struct xfs_inode *dp, xfs_dablk_t dabno, uint32_t magic, + uint32_t magic_guess), + TP_ARGS(dp, dabno, magic, magic_guess), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dir_ino) + __field(xfs_dablk_t, dabno) + __field(uint32_t, magic) + __field(uint32_t, magic_guess) + ), + TP_fast_assign( + __entry->dev = dp->i_mount->m_super->s_dev; + __entry->dir_ino = dp->i_ino; + __entry->dabno = dabno; + __entry->magic = magic; + __entry->magic_guess = magic_guess; + ), + TP_printk("dev %d:%d dir 0x%llx dablk 0x%x magic 0x%x magic_guess 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dir_ino, + __entry->dabno, + __entry->magic, + __entry->magic_guess) +); + +DECLARE_EVENT_CLASS(xrep_dir_class, + TP_PROTO(struct xfs_inode *dp, xfs_ino_t parent_ino), + TP_ARGS(dp, parent_ino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dir_ino) + __field(xfs_ino_t, parent_ino) + ), + TP_fast_assign( + __entry->dev = dp->i_mount->m_super->s_dev; + __entry->dir_ino = dp->i_ino; + __entry->parent_ino = parent_ino; + ), + TP_printk("dev %d:%d dir 0x%llx parent 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dir_ino, + __entry->parent_ino) +) +#define DEFINE_XREP_DIR_EVENT(name) \ +DEFINE_EVENT(xrep_dir_class, name, \ + TP_PROTO(struct xfs_inode *dp, xfs_ino_t parent_ino), \ + TP_ARGS(dp, parent_ino)) +DEFINE_XREP_DIR_EVENT(xrep_dir_rebuild_tree); +DEFINE_XREP_DIR_EVENT(xrep_dir_reset_fork); + +DECLARE_EVENT_CLASS(xrep_dirent_class, + TP_PROTO(struct xfs_inode *dp, const struct xfs_name *name, + xfs_ino_t ino), + TP_ARGS(dp, name, ino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dir_ino) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + __field(xfs_ino_t, ino) + __field(uint8_t, ftype) + ), + TP_fast_assign( + __entry->dev = dp->i_mount->m_super->s_dev; + __entry->dir_ino = dp->i_ino; + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + __entry->ino = ino; + __entry->ftype = name->type; + ), + TP_printk("dev %d:%d dir 0x%llx ftype %s name '%.*s' ino 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dir_ino, + __print_symbolic(__entry->ftype, XFS_DIR3_FTYPE_STR), + __entry->namelen, + __get_str(name), + __entry->ino) +) +#define DEFINE_XREP_DIRENT_EVENT(name) \ +DEFINE_EVENT(xrep_dirent_class, name, \ + TP_PROTO(struct xfs_inode *dp, const struct xfs_name *name, \ + xfs_ino_t ino), \ + TP_ARGS(dp, name, ino)) +DEFINE_XREP_DIRENT_EVENT(xrep_dir_salvage_entry); +DEFINE_XREP_DIRENT_EVENT(xrep_dir_stash_createname); +DEFINE_XREP_DIRENT_EVENT(xrep_dir_replay_createname); + +DECLARE_EVENT_CLASS(xrep_parent_salvage_class, + TP_PROTO(struct xfs_inode *dp, xfs_ino_t ino), + TP_ARGS(dp, ino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dir_ino) + __field(xfs_ino_t, ino) + ), + TP_fast_assign( + __entry->dev = dp->i_mount->m_super->s_dev; + __entry->dir_ino = dp->i_ino; + __entry->ino = ino; + ), + TP_printk("dev %d:%d dir 0x%llx parent 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dir_ino, + __entry->ino) +) +#define DEFINE_XREP_PARENT_SALVAGE_EVENT(name) \ +DEFINE_EVENT(xrep_parent_salvage_class, name, \ + TP_PROTO(struct xfs_inode *dp, xfs_ino_t ino), \ + TP_ARGS(dp, ino)) +DEFINE_XREP_PARENT_SALVAGE_EVENT(xrep_dir_salvaged_parent); + #endif /* IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) */ #endif /* _TRACE_XFS_SCRUB_TRACE_H */ diff --git a/fs/xfs/scrub/xfblob.h b/fs/xfs/scrub/xfblob.h index 78a67a06408f8..ae78322613cab 100644 --- a/fs/xfs/scrub/xfblob.h +++ b/fs/xfs/scrub/xfblob.h @@ -23,4 +23,28 @@ int xfblob_free(struct xfblob *blob, xfblob_cookie cookie); unsigned long long xfblob_bytes(struct xfblob *blob); void xfblob_truncate(struct xfblob *blob); +static inline int +xfblob_storename( + struct xfblob *blob, + xfblob_cookie *cookie, + const struct xfs_name *xname) +{ + return xfblob_store(blob, cookie, xname->name, xname->len); +} + +static inline int +xfblob_loadname( + struct xfblob *blob, + xfblob_cookie cookie, + struct xfs_name *xname, + uint32_t size) +{ + int ret = xfblob_load(blob, cookie, (void *)xname->name, size); + if (ret) + return ret; + + xname->len = size; + return 0; +} + #endif /* __XFS_SCRUB_XFBLOB_H__ */ From 669dfe883c8e20231495f80a28ec7cc0b8fdddc4 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:49 -0700 Subject: [PATCH 0244/1702] xfs: update the unlinked list when repairing link counts When we're repairing the link counts of a file, we must ensure either that the file has zero link count and is on the unlinked list; or that it has nonzero link count and is not on the unlinked list. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/nlinks_repair.c | 42 ++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/fs/xfs/scrub/nlinks_repair.c b/fs/xfs/scrub/nlinks_repair.c index b87618322f55b..58cacb8e94c1b 100644 --- a/fs/xfs/scrub/nlinks_repair.c +++ b/fs/xfs/scrub/nlinks_repair.c @@ -17,6 +17,7 @@ #include "xfs_iwalk.h" #include "xfs_ialloc.h" #include "xfs_sb.h" +#include "xfs_ag.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/repair.h" @@ -36,6 +37,20 @@ * inode is locked. */ +/* Remove an inode from the unlinked list. */ +STATIC int +xrep_nlinks_iunlink_remove( + struct xfs_scrub *sc) +{ + struct xfs_perag *pag; + int error; + + pag = xfs_perag_get(sc->mp, XFS_INO_TO_AGNO(sc->mp, sc->ip->i_ino)); + error = xfs_iunlink_remove(sc->tp, pag, sc->ip); + xfs_perag_put(pag); + return error; +} + /* * Correct the link count of the given inode. Because we have to grab locks * and resources in a certain order, it's possible that this will be a no-op. @@ -99,16 +114,25 @@ xrep_nlinks_repair_inode( } /* - * We did not find any links to this inode. If the inode agrees, we - * have nothing further to do. If not, the inode has a nonzero link - * count and we don't have anywhere to graft the child onto. Dropping - * a live inode's link count to zero can cause unexpected shutdowns in - * inactivation, so leave it alone. + * If this inode is linked from the directory tree and on the unlinked + * list, remove it from the unlinked list. */ - if (total_links == 0) { - if (actual_nlink != 0) - trace_xrep_nlinks_unfixable_inode(mp, ip, &obs); - goto out_trans; + if (total_links > 0 && xfs_inode_on_unlinked_list(ip)) { + error = xrep_nlinks_iunlink_remove(sc); + if (error) + goto out_trans; + dirty = true; + } + + /* + * If this inode is not linked from the directory tree yet not on the + * unlinked list, put it on the unlinked list. + */ + if (total_links == 0 && !xfs_inode_on_unlinked_list(ip)) { + error = xfs_iunlink(sc->tp, ip); + if (error) + goto out_trans; + dirty = true; } /* Commit the new link count if it changed. */ From a07b45576264e77ea1a781b552873e76b8b0dacc Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:52 -0700 Subject: [PATCH 0245/1702] xfs: scan the filesystem to repair a directory dotdot entry Teach the online directory repair code to scan the filesystem so that we can set the dotdot entry when we're rebuilding a directory. This involves dropping ILOCK on the directory that we're repairing, which means that the VFS can sneak in and tell us to update dotdot at any time. Deal with these races by using a dirent hook to absorb dotdot updates, and be careful not to check the scan results until after we've retaken the ILOCK. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/scrub/dir_repair.c | 70 ++++--- fs/xfs/scrub/findparent.c | 412 ++++++++++++++++++++++++++++++++++++++ fs/xfs/scrub/findparent.h | 49 +++++ fs/xfs/scrub/iscan.c | 18 +- fs/xfs/scrub/iscan.h | 1 + fs/xfs/scrub/trace.h | 1 + 7 files changed, 528 insertions(+), 24 deletions(-) create mode 100644 fs/xfs/scrub/findparent.c create mode 100644 fs/xfs/scrub/findparent.h diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index 5c9449e14f743..3c754777ec288 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -199,6 +199,7 @@ xfs-y += $(addprefix scrub/, \ bmap_repair.o \ cow_repair.o \ dir_repair.o \ + findparent.o \ fscounters_repair.o \ ialloc_repair.o \ inode_repair.o \ diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index 48aa80d8c7dc4..b17de79207dba 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -38,8 +38,10 @@ #include "scrub/xfile.h" #include "scrub/xfarray.h" #include "scrub/xfblob.h" +#include "scrub/iscan.h" #include "scrub/readdir.h" #include "scrub/reap.h" +#include "scrub/findparent.h" /* * Directory Repair @@ -108,10 +110,10 @@ struct xrep_dir { struct xfs_da_args args; /* - * This is the parent that we're going to set on the reconstructed - * directory. + * Information used to scan the filesystem to find the inumber of the + * dotdot entry for this directory. */ - xfs_ino_t parent_ino; + struct xrep_parent_scan_info pscan; /* How many subdirectories did we find? */ uint64_t subdirs; @@ -131,6 +133,7 @@ xrep_dir_teardown( { struct xrep_dir *rd = sc->buf; + xrep_findparent_scan_teardown(&rd->pscan); xfblob_destroy(rd->dir_names); xfarray_destroy(rd->dir_entries); } @@ -143,6 +146,8 @@ xrep_setup_directory( struct xrep_dir *rd; int error; + xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); + error = xrep_tempfile_create(sc, S_IFDIR); if (error) return error; @@ -179,8 +184,8 @@ xrep_dir_self_parent( } /* - * Look up the dotdot entry. Returns NULLFSINO if we don't know what to do. - * The next patch will check this more carefully. + * Look up the dotdot entry and confirm that it's really the parent. + * Returns NULLFSINO if we don't know what to do. */ static inline xfs_ino_t xrep_dir_lookup_parent( @@ -196,37 +201,39 @@ xrep_dir_lookup_parent( if (!xfs_verify_dir_ino(sc->mp, ino)) return NULLFSINO; + error = xrep_findparent_confirm(sc, &ino); + if (error) + return NULLFSINO; + return ino; } -/* - * Try to find the parent of the directory being repaired. - * - * NOTE: This function will someday be augmented by the directory parent repair - * code, which will know how to check the parent and scan the filesystem if - * we cannot find anything. Inode scans will have to be done before we start - * salvaging directory entries, so we do this now. - */ +/* Try to find the parent of the directory being repaired. */ STATIC int xrep_dir_find_parent( struct xrep_dir *rd) { xfs_ino_t ino; - ino = xrep_dir_self_parent(rd); + ino = xrep_findparent_self_reference(rd->sc); if (ino != NULLFSINO) { - rd->parent_ino = ino; + xrep_findparent_scan_finish_early(&rd->pscan, ino); return 0; } ino = xrep_dir_lookup_parent(rd); if (ino != NULLFSINO) { - rd->parent_ino = ino; + xrep_findparent_scan_finish_early(&rd->pscan, ino); return 0; } - /* NOTE: A future patch will deal with moving orphans. */ - return -EFSCORRUPTED; + /* + * A full filesystem scan is the last resort. On a busy filesystem, + * the scan can fail with -EBUSY if we cannot grab IOLOCKs. That means + * that we don't know what who the parent is, so we should return to + * userspace. + */ + return xrep_findparent_scan(&rd->pscan); } /* @@ -931,6 +938,10 @@ xrep_dir_salvage_entries( * the directory until we're ready for the exchange operation. Reads * will return -EIO without shutting down the fs, so we're ok with * that. + * + * The VFS can change dotdot on us, but the findparent scan will keep + * our incore parent inode up to date. See the note on locking issues + * for more details. */ error = xrep_trans_commit(sc); if (error) @@ -1154,6 +1165,14 @@ xrep_dir_swap( if (rd->subdirs + 2 > XFS_MAXLINK) return -EFSCORRUPTED; + /* + * If we never found the parent for this directory, we can't fix this + * directory. + */ + ASSERT(sc->ilock_flags & XFS_ILOCK_EXCL); + if (rd->pscan.parent_ino == NULLFSINO) + return -EFSCORRUPTED; + /* * Reset the temporary directory's '..' entry to point to the parent * that we found. The temporary directory was created with the root @@ -1163,9 +1182,9 @@ xrep_dir_swap( * It's also possible that this replacement could also expand a sf * tempdir into block format. */ - if (rd->parent_ino != sc->mp->m_rootip->i_ino) { + if (rd->pscan.parent_ino != sc->mp->m_rootip->i_ino) { error = xrep_dir_replace(rd, rd->sc->tempip, &xfs_name_dotdot, - rd->parent_ino, rd->tx.req.resblks); + rd->pscan.parent_ino, rd->tx.req.resblks); if (error) return error; } @@ -1224,7 +1243,7 @@ xrep_dir_rebuild_tree( struct xfs_scrub *sc = rd->sc; int error; - trace_xrep_dir_rebuild_tree(sc->ip, rd->parent_ino); + trace_xrep_dir_rebuild_tree(sc->ip, rd->pscan.parent_ino); /* * Take the IOLOCK on the temporary file so that we can run dir @@ -1281,8 +1300,6 @@ xrep_dir_setup_scan( char *descr; int error; - rd->parent_ino = NULLFSINO; - /* Set up some staging memory for salvaging dirents. */ descr = xchk_xfile_ino_descr(sc, "directory entries"); error = xfarray_create(descr, 0, sizeof(struct xrep_dirent), @@ -1297,8 +1314,15 @@ xrep_dir_setup_scan( if (error) goto out_xfarray; + error = xrep_findparent_scan_start(sc, &rd->pscan); + if (error) + goto out_xfblob; + return 0; +out_xfblob: + xfblob_destroy(rd->dir_names); + rd->dir_names = NULL; out_xfarray: xfarray_destroy(rd->dir_entries); rd->dir_entries = NULL; diff --git a/fs/xfs/scrub/findparent.c b/fs/xfs/scrub/findparent.c new file mode 100644 index 0000000000000..7b3ec8d7d6cc9 --- /dev/null +++ b/fs/xfs/scrub/findparent.c @@ -0,0 +1,412 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_dir2.h" +#include "xfs_bmap_btree.h" +#include "xfs_dir2_priv.h" +#include "xfs_trans_space.h" +#include "xfs_health.h" +#include "xfs_exchmaps.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/iscan.h" +#include "scrub/findparent.h" +#include "scrub/readdir.h" +#include "scrub/tempfile.h" + +/* + * Finding the Parent of a Directory + * ================================= + * + * Directories have parent pointers, in the sense that each directory contains + * a dotdot entry that points to the single allowed parent. The brute force + * way to find the parent of a given directory is to scan every directory in + * the filesystem looking for a child dirent that references this directory. + * + * This module wraps the process of scanning the directory tree. It requires + * that @sc->ip is the directory whose parent we want to find, and that the + * caller hold only the IOLOCK on that directory. The scan itself needs to + * take the ILOCK of each directory visited. + * + * Because we cannot hold @sc->ip's ILOCK during a scan of the whole fs, it is + * necessary to use dirent hook to update the parent scan results. Callers + * must not read the scan results without re-taking @sc->ip's ILOCK. + * + * There are a few shortcuts that we can take to avoid scanning the entire + * filesystem, such as noticing directory tree roots. + */ + +struct xrep_findparent_info { + /* The directory currently being scanned. */ + struct xfs_inode *dp; + + /* + * Scrub context. We're looking for a @dp containing a directory + * entry pointing to sc->ip->i_ino. + */ + struct xfs_scrub *sc; + + /* Optional scan information for a xrep_findparent_scan call. */ + struct xrep_parent_scan_info *parent_scan; + + /* + * Parent that we've found for sc->ip. If we're scanning the entire + * directory tree, we need this to ensure that we only find /one/ + * parent directory. + */ + xfs_ino_t found_parent; + + /* + * This is set to true if @found_parent was not observed directly from + * the directory scan but by noticing a change in dotdot entries after + * cycling the sc->ip IOLOCK. + */ + bool parent_tentative; +}; + +/* + * If this directory entry points to the scrub target inode, then the directory + * we're scanning is the parent of the scrub target inode. + */ +STATIC int +xrep_findparent_dirent( + struct xfs_scrub *sc, + struct xfs_inode *dp, + xfs_dir2_dataptr_t dapos, + const struct xfs_name *name, + xfs_ino_t ino, + void *priv) +{ + struct xrep_findparent_info *fpi = priv; + int error = 0; + + if (xchk_should_terminate(fpi->sc, &error)) + return error; + + if (ino != fpi->sc->ip->i_ino) + return 0; + + /* Ignore garbage directory entry names. */ + if (name->len == 0 || !xfs_dir2_namecheck(name->name, name->len)) + return -EFSCORRUPTED; + + /* + * Ignore dotdot and dot entries -- we're looking for parent -> child + * links only. + */ + if (name->name[0] == '.' && (name->len == 1 || + (name->len == 2 && name->name[1] == '.'))) + return 0; + + /* Uhoh, more than one parent for a dir? */ + if (fpi->found_parent != NULLFSINO && + !(fpi->parent_tentative && fpi->found_parent == fpi->dp->i_ino)) { + trace_xrep_findparent_dirent(fpi->sc->ip, 0); + return -EFSCORRUPTED; + } + + /* We found a potential parent; remember this. */ + trace_xrep_findparent_dirent(fpi->sc->ip, fpi->dp->i_ino); + fpi->found_parent = fpi->dp->i_ino; + fpi->parent_tentative = false; + + if (fpi->parent_scan) + xrep_findparent_scan_found(fpi->parent_scan, fpi->dp->i_ino); + + return 0; +} + +/* + * If this is a directory, walk the dirents looking for any that point to the + * scrub target inode. + */ +STATIC int +xrep_findparent_walk_directory( + struct xrep_findparent_info *fpi) +{ + struct xfs_scrub *sc = fpi->sc; + struct xfs_inode *dp = fpi->dp; + unsigned int lock_mode; + int error = 0; + + /* + * The inode being scanned cannot be its own parent, nor can any + * temporary directory we created to stage this repair. + */ + if (dp == sc->ip || dp == sc->tempip) + return 0; + + /* + * Similarly, temporary files created to stage a repair cannot be the + * parent of this inode. + */ + if (xrep_is_tempfile(dp)) + return 0; + + /* + * Scan the directory to see if there it contains an entry pointing to + * the directory that we are repairing. + */ + lock_mode = xfs_ilock_data_map_shared(dp); + + /* + * If this directory is known to be sick, we cannot scan it reliably + * and must abort. + */ + if (xfs_inode_has_sickness(dp, XFS_SICK_INO_CORE | + XFS_SICK_INO_BMBTD | + XFS_SICK_INO_DIR)) { + error = -EFSCORRUPTED; + goto out_unlock; + } + + /* + * We cannot complete our parent pointer scan if a directory looks as + * though it has been zapped by the inode record repair code. + */ + if (xchk_dir_looks_zapped(dp)) { + error = -EBUSY; + goto out_unlock; + } + + error = xchk_dir_walk(sc, dp, xrep_findparent_dirent, fpi); + if (error) + goto out_unlock; + +out_unlock: + xfs_iunlock(dp, lock_mode); + return error; +} + +/* + * Update this directory's dotdot pointer based on ongoing dirent updates. + */ +STATIC int +xrep_findparent_live_update( + struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct xfs_dir_update_params *p = data; + struct xrep_parent_scan_info *pscan; + struct xfs_scrub *sc; + + pscan = container_of(nb, struct xrep_parent_scan_info, + dhook.dirent_hook.nb); + sc = pscan->sc; + + /* + * If @p->ip is the subdirectory that we're interested in and we've + * already scanned @p->dp, update the dotdot target inumber to the + * parent inode. + */ + if (p->ip->i_ino == sc->ip->i_ino && + xchk_iscan_want_live_update(&pscan->iscan, p->dp->i_ino)) { + if (p->delta > 0) { + xrep_findparent_scan_found(pscan, p->dp->i_ino); + } else { + xrep_findparent_scan_found(pscan, NULLFSINO); + } + } + + return NOTIFY_DONE; +} + +/* + * Set up a scan to find the parent of a directory. The provided dirent hook + * will be called when there is a dotdot update for the inode being repaired. + */ +int +xrep_findparent_scan_start( + struct xfs_scrub *sc, + struct xrep_parent_scan_info *pscan) +{ + int error; + + if (!(sc->flags & XCHK_FSGATES_DIRENTS)) { + ASSERT(sc->flags & XCHK_FSGATES_DIRENTS); + return -EINVAL; + } + + pscan->sc = sc; + pscan->parent_ino = NULLFSINO; + + mutex_init(&pscan->lock); + + xchk_iscan_start(sc, 30000, 100, &pscan->iscan); + + /* + * Hook into the dirent update code. The hook only operates on inodes + * that were already scanned, and the scanner thread takes each inode's + * ILOCK, which means that any in-progress inode updates will finish + * before we can scan the inode. + */ + xfs_dir_hook_setup(&pscan->dhook, xrep_findparent_live_update); + error = xfs_dir_hook_add(sc->mp, &pscan->dhook); + if (error) + goto out_iscan; + + return 0; +out_iscan: + xchk_iscan_teardown(&pscan->iscan); + mutex_destroy(&pscan->lock); + return error; +} + +/* + * Scan the entire filesystem looking for a parent inode for the inode being + * scrubbed. @sc->ip must not be the root of a directory tree. Callers must + * not hold a dirty transaction or any lock that would interfere with taking + * an ILOCK. + * + * Returns 0 with @pscan->parent_ino set to the parent that we found. + * Returns 0 with @pscan->parent_ino set to NULLFSINO if we found no parents. + * Returns the usual negative errno if something else happened. + */ +int +xrep_findparent_scan( + struct xrep_parent_scan_info *pscan) +{ + struct xrep_findparent_info fpi = { + .sc = pscan->sc, + .found_parent = NULLFSINO, + .parent_scan = pscan, + }; + struct xfs_scrub *sc = pscan->sc; + int ret; + + ASSERT(S_ISDIR(VFS_IC(sc->ip)->i_mode)); + + while ((ret = xchk_iscan_iter(&pscan->iscan, &fpi.dp)) == 1) { + if (S_ISDIR(VFS_I(fpi.dp)->i_mode)) + ret = xrep_findparent_walk_directory(&fpi); + else + ret = 0; + xchk_iscan_mark_visited(&pscan->iscan, fpi.dp); + xchk_irele(sc, fpi.dp); + if (ret) + break; + + if (xchk_should_terminate(sc, &ret)) + break; + } + xchk_iscan_iter_finish(&pscan->iscan); + + return ret; +} + +/* Tear down a parent scan. */ +void +xrep_findparent_scan_teardown( + struct xrep_parent_scan_info *pscan) +{ + xfs_dir_hook_del(pscan->sc->mp, &pscan->dhook); + xchk_iscan_teardown(&pscan->iscan); + mutex_destroy(&pscan->lock); +} + +/* Finish a parent scan early. */ +void +xrep_findparent_scan_finish_early( + struct xrep_parent_scan_info *pscan, + xfs_ino_t ino) +{ + xrep_findparent_scan_found(pscan, ino); + xchk_iscan_finish_early(&pscan->iscan); +} + +/* + * Confirm that the directory @parent_ino actually contains a directory entry + * pointing to the child @sc->ip->ino. This function returns one of several + * ways: + * + * Returns 0 with @parent_ino unchanged if the parent was confirmed. + * Returns 0 with @parent_ino set to NULLFSINO if the parent was not valid. + * Returns the usual negative errno if something else happened. + */ +int +xrep_findparent_confirm( + struct xfs_scrub *sc, + xfs_ino_t *parent_ino) +{ + struct xrep_findparent_info fpi = { + .sc = sc, + .found_parent = NULLFSINO, + }; + int error; + + /* + * The root directory always points to itself. Unlinked dirs can point + * anywhere, so we point them at the root dir too. + */ + if (sc->ip == sc->mp->m_rootip || VFS_I(sc->ip)->i_nlink == 0) { + *parent_ino = sc->mp->m_sb.sb_rootino; + return 0; + } + + /* Reject garbage parent inode numbers and self-referential parents. */ + if (*parent_ino == NULLFSINO) + return 0; + if (!xfs_verify_dir_ino(sc->mp, *parent_ino) || + *parent_ino == sc->ip->i_ino) { + *parent_ino = NULLFSINO; + return 0; + } + + error = xchk_iget(sc, *parent_ino, &fpi.dp); + if (error) + return error; + + if (!S_ISDIR(VFS_I(fpi.dp)->i_mode)) { + *parent_ino = NULLFSINO; + goto out_rele; + } + + error = xrep_findparent_walk_directory(&fpi); + if (error) + goto out_rele; + + *parent_ino = fpi.found_parent; +out_rele: + xchk_irele(sc, fpi.dp); + return error; +} + +/* + * If we're the root of a directory tree, we are our own parent. If we're an + * unlinked directory, the parent /won't/ have a link to us. Set the parent + * directory to the root for both cases. Returns NULLFSINO if we don't know + * what to do. + */ +xfs_ino_t +xrep_findparent_self_reference( + struct xfs_scrub *sc) +{ + if (sc->ip->i_ino == sc->mp->m_sb.sb_rootino) + return sc->mp->m_sb.sb_rootino; + + if (VFS_I(sc->ip)->i_nlink == 0) + return sc->mp->m_sb.sb_rootino; + + return NULLFSINO; +} diff --git a/fs/xfs/scrub/findparent.h b/fs/xfs/scrub/findparent.h new file mode 100644 index 0000000000000..d946bc81f34e6 --- /dev/null +++ b/fs/xfs/scrub/findparent.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_FINDPARENT_H__ +#define __XFS_SCRUB_FINDPARENT_H__ + +struct xrep_parent_scan_info { + struct xfs_scrub *sc; + + /* Inode scan cursor. */ + struct xchk_iscan iscan; + + /* Hook to capture directory entry updates. */ + struct xfs_dir_hook dhook; + + /* Lock protecting parent_ino. */ + struct mutex lock; + + /* Parent inode that we've found. */ + xfs_ino_t parent_ino; + + bool lookup_parent; +}; + +int xrep_findparent_scan_start(struct xfs_scrub *sc, + struct xrep_parent_scan_info *pscan); +int xrep_findparent_scan(struct xrep_parent_scan_info *pscan); +void xrep_findparent_scan_teardown(struct xrep_parent_scan_info *pscan); + +static inline void +xrep_findparent_scan_found( + struct xrep_parent_scan_info *pscan, + xfs_ino_t ino) +{ + mutex_lock(&pscan->lock); + pscan->parent_ino = ino; + mutex_unlock(&pscan->lock); +} + +void xrep_findparent_scan_finish_early(struct xrep_parent_scan_info *pscan, + xfs_ino_t ino); + +int xrep_findparent_confirm(struct xfs_scrub *sc, xfs_ino_t *parent_ino); + +xfs_ino_t xrep_findparent_self_reference(struct xfs_scrub *sc); + +#endif /* __XFS_SCRUB_FINDPARENT_H__ */ diff --git a/fs/xfs/scrub/iscan.c b/fs/xfs/scrub/iscan.c index c643b7d79b607..c380207702e26 100644 --- a/fs/xfs/scrub/iscan.c +++ b/fs/xfs/scrub/iscan.c @@ -243,6 +243,17 @@ xchk_iscan_finish( mutex_unlock(&iscan->lock); } +/* Mark an inode scan finished before we actually scan anything. */ +void +xchk_iscan_finish_early( + struct xchk_iscan *iscan) +{ + ASSERT(iscan->cursor_ino == iscan->scan_start_ino); + ASSERT(iscan->__visited_ino == iscan->scan_start_ino); + + xchk_iscan_finish(iscan); +} + /* * Grab the AGI to advance the inode scan. Returns 0 if *agi_bpp is now set, * -ECANCELED if the live scan aborted, -EBUSY if the AGI could not be grabbed, @@ -436,8 +447,13 @@ xchk_iscan_iget( * It's possible that this inode has lost all of its links but * hasn't yet been inactivated. If we don't have a transaction * or it's not writable, flush the inodegc workers and wait. + * If we have a non-empty transaction, we must not block on + * inodegc, which allocates its own transactions. */ - xfs_inodegc_flush(mp); + if (sc->tp && !(sc->tp->t_flags & XFS_TRANS_NO_WRITECOUNT)) + xfs_inodegc_push(mp); + else + xfs_inodegc_flush(mp); return xchk_iscan_iget_retry(iscan, true); } diff --git a/fs/xfs/scrub/iscan.h b/fs/xfs/scrub/iscan.h index 5e0e4ed9dea6d..f9f47fa01a9ea 100644 --- a/fs/xfs/scrub/iscan.h +++ b/fs/xfs/scrub/iscan.h @@ -88,6 +88,7 @@ xchk_iscan_set_agi_trylock(struct xchk_iscan *iscan) void xchk_iscan_start(struct xfs_scrub *sc, unsigned int iget_timeout, unsigned int iget_retry_delay, struct xchk_iscan *iscan); +void xchk_iscan_finish_early(struct xchk_iscan *iscan); void xchk_iscan_teardown(struct xchk_iscan *iscan); int xchk_iscan_iter(struct xchk_iscan *iscan, struct xfs_inode **ipp); diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index d6d9e8d6109cc..85537a87516e9 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2611,6 +2611,7 @@ DEFINE_EVENT(xrep_parent_salvage_class, name, \ TP_PROTO(struct xfs_inode *dp, xfs_ino_t ino), \ TP_ARGS(dp, ino)) DEFINE_XREP_PARENT_SALVAGE_EVENT(xrep_dir_salvaged_parent); +DEFINE_XREP_PARENT_SALVAGE_EVENT(xrep_findparent_dirent); #endif /* IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) */ From cc22edab9ea7f3ebcb61d41a417d4397e9b7b128 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:53 -0700 Subject: [PATCH 0246/1702] xfs: online repair of parent pointers Teach the online repair code to fix parent pointers for directories. For now, this means correcting the dotdot entry of an existing directory that is otherwise consistent. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/scrub/parent.c | 10 ++ fs/xfs/scrub/parent_repair.c | 221 +++++++++++++++++++++++++++++++++++ fs/xfs/scrub/repair.h | 4 + fs/xfs/scrub/scrub.c | 2 +- fs/xfs/scrub/trace.h | 1 + 6 files changed, 238 insertions(+), 1 deletion(-) create mode 100644 fs/xfs/scrub/parent_repair.c diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index 3c754777ec288..d48646f86563f 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -205,6 +205,7 @@ xfs-y += $(addprefix scrub/, \ inode_repair.o \ newbt.o \ nlinks_repair.o \ + parent_repair.o \ rcbag_btree.o \ rcbag.o \ reap.o \ diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c index 050a8e8914f6e..acb6282c3d148 100644 --- a/fs/xfs/scrub/parent.c +++ b/fs/xfs/scrub/parent.c @@ -10,6 +10,7 @@ #include "xfs_trans_resv.h" #include "xfs_mount.h" #include "xfs_log_format.h" +#include "xfs_trans.h" #include "xfs_inode.h" #include "xfs_icache.h" #include "xfs_dir2.h" @@ -18,12 +19,21 @@ #include "scrub/common.h" #include "scrub/readdir.h" #include "scrub/tempfile.h" +#include "scrub/repair.h" /* Set us up to scrub parents. */ int xchk_setup_parent( struct xfs_scrub *sc) { + int error; + + if (xchk_could_repair(sc)) { + error = xrep_setup_parent(sc); + if (error) + return error; + } + return xchk_setup_inode_contents(sc, 0); } diff --git a/fs/xfs/scrub/parent_repair.c b/fs/xfs/scrub/parent_repair.c new file mode 100644 index 0000000000000..0a9651bb0b057 --- /dev/null +++ b/fs/xfs/scrub/parent_repair.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_dir2.h" +#include "xfs_bmap_btree.h" +#include "xfs_dir2_priv.h" +#include "xfs_trans_space.h" +#include "xfs_health.h" +#include "xfs_exchmaps.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/iscan.h" +#include "scrub/findparent.h" +#include "scrub/readdir.h" + +/* + * Repairing The Directory Parent Pointer + * ====================================== + * + * Currently, only directories support parent pointers (in the form of '..' + * entries), so we simply scan the filesystem and update the '..' entry. + * + * Note that because the only parent pointer is the dotdot entry, we won't + * touch an unhealthy directory, since the directory repair code is perfectly + * capable of rebuilding a directory with the proper parent inode. + * + * See the section on locking issues in dir_repair.c for more information about + * conflicts with the VFS. The findparent code wll keep our incore parent + * inode up to date. + */ + +struct xrep_parent { + struct xfs_scrub *sc; + + /* + * Information used to scan the filesystem to find the inumber of the + * dotdot entry for this directory. + */ + struct xrep_parent_scan_info pscan; +}; + +/* Tear down all the incore stuff we created. */ +static void +xrep_parent_teardown( + struct xrep_parent *rp) +{ + xrep_findparent_scan_teardown(&rp->pscan); +} + +/* Set up for a parent repair. */ +int +xrep_setup_parent( + struct xfs_scrub *sc) +{ + struct xrep_parent *rp; + + xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); + + rp = kvzalloc(sizeof(struct xrep_parent), XCHK_GFP_FLAGS); + if (!rp) + return -ENOMEM; + rp->sc = sc; + sc->buf = rp; + + return 0; +} + +/* + * Scan all files in the filesystem for a child dirent that we can turn into + * the dotdot entry for this directory. + */ +STATIC int +xrep_parent_find_dotdot( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + xfs_ino_t ino; + unsigned int sick, checked; + int error; + + /* + * Avoid sick directories. There shouldn't be anyone else clearing the + * directory's sick status. + */ + xfs_inode_measure_sickness(sc->ip, &sick, &checked); + if (sick & XFS_SICK_INO_DIR) + return -EFSCORRUPTED; + + ino = xrep_findparent_self_reference(sc); + if (ino != NULLFSINO) { + xrep_findparent_scan_finish_early(&rp->pscan, ino); + return 0; + } + + /* + * Drop the ILOCK on this directory so that we can scan for the dotdot + * entry. Figure out who is going to be the parent of this directory, + * then retake the ILOCK so that we can salvage directory entries. + */ + xchk_iunlock(sc, XFS_ILOCK_EXCL); + error = xrep_findparent_scan(&rp->pscan); + xchk_ilock(sc, XFS_ILOCK_EXCL); + + return error; +} + +/* Reset a directory's dotdot entry, if needed. */ +STATIC int +xrep_parent_reset_dotdot( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + xfs_ino_t ino; + unsigned int spaceres; + int error = 0; + + ASSERT(sc->ilock_flags & XFS_ILOCK_EXCL); + + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, &ino); + if (error || ino == rp->pscan.parent_ino) + return error; + + xfs_trans_ijoin(sc->tp, sc->ip, 0); + + trace_xrep_parent_reset_dotdot(sc->ip, rp->pscan.parent_ino); + + /* + * Reserve more space just in case we have to expand the dir. We're + * allowed to exceed quota to repair inconsistent metadata. + */ + spaceres = XFS_RENAME_SPACE_RES(sc->mp, xfs_name_dotdot.len); + error = xfs_trans_reserve_more_inode(sc->tp, sc->ip, spaceres, 0, + true); + if (error) + return error; + + error = xfs_dir_replace(sc->tp, sc->ip, &xfs_name_dotdot, + rp->pscan.parent_ino, spaceres); + if (error) + return error; + + /* + * Roll transaction to detach the inode from the transaction but retain + * ILOCK_EXCL. + */ + return xfs_trans_roll(&sc->tp); +} + +/* + * Commit the new parent pointer structure (currently only the dotdot entry) to + * the file that we're repairing. + */ +STATIC int +xrep_parent_rebuild_tree( + struct xrep_parent *rp) +{ + if (rp->pscan.parent_ino == NULLFSINO) { + /* Cannot fix orphaned directories yet. */ + return -EFSCORRUPTED; + } + + return xrep_parent_reset_dotdot(rp); +} + +/* Set up the filesystem scan so we can look for parents. */ +STATIC int +xrep_parent_setup_scan( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + + return xrep_findparent_scan_start(sc, &rp->pscan); +} + +int +xrep_parent( + struct xfs_scrub *sc) +{ + struct xrep_parent *rp = sc->buf; + int error; + + error = xrep_parent_setup_scan(rp); + if (error) + return error; + + error = xrep_parent_find_dotdot(rp); + if (error) + goto out_teardown; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + goto out_teardown; + + error = xrep_parent_rebuild_tree(rp); + if (error) + goto out_teardown; + +out_teardown: + xrep_parent_teardown(rp); + return error; +} diff --git a/fs/xfs/scrub/repair.h b/fs/xfs/scrub/repair.h index 4e25aa95753a9..e53374fa54308 100644 --- a/fs/xfs/scrub/repair.h +++ b/fs/xfs/scrub/repair.h @@ -92,6 +92,7 @@ int xrep_setup_ag_rmapbt(struct xfs_scrub *sc); int xrep_setup_ag_refcountbt(struct xfs_scrub *sc); int xrep_setup_xattr(struct xfs_scrub *sc); int xrep_setup_directory(struct xfs_scrub *sc); +int xrep_setup_parent(struct xfs_scrub *sc); /* Repair setup functions */ int xrep_setup_ag_allocbt(struct xfs_scrub *sc); @@ -127,6 +128,7 @@ int xrep_nlinks(struct xfs_scrub *sc); int xrep_fscounters(struct xfs_scrub *sc); int xrep_xattr(struct xfs_scrub *sc); int xrep_directory(struct xfs_scrub *sc); +int xrep_parent(struct xfs_scrub *sc); #ifdef CONFIG_XFS_RT int xrep_rtbitmap(struct xfs_scrub *sc); @@ -198,6 +200,7 @@ xrep_setup_nothing( #define xrep_setup_ag_refcountbt xrep_setup_nothing #define xrep_setup_xattr xrep_setup_nothing #define xrep_setup_directory xrep_setup_nothing +#define xrep_setup_parent xrep_setup_nothing #define xrep_setup_inode(sc, imap) ((void)0) @@ -225,6 +228,7 @@ xrep_setup_nothing( #define xrep_rtsummary xrep_notsupported #define xrep_xattr xrep_notsupported #define xrep_directory xrep_notsupported +#define xrep_parent xrep_notsupported #endif /* CONFIG_XFS_ONLINE_REPAIR */ diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 8e9e2bf121c2b..520d83db193c3 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -343,7 +343,7 @@ static const struct xchk_meta_ops meta_scrub_ops[] = { .type = ST_INODE, .setup = xchk_setup_parent, .scrub = xchk_parent, - .repair = xrep_notsupported, + .repair = xrep_parent, }, [XFS_SCRUB_TYPE_RTBITMAP] = { /* realtime bitmap */ .type = ST_FS, diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 85537a87516e9..e1755fe63e67f 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2550,6 +2550,7 @@ DEFINE_EVENT(xrep_dir_class, name, \ TP_ARGS(dp, parent_ino)) DEFINE_XREP_DIR_EVENT(xrep_dir_rebuild_tree); DEFINE_XREP_DIR_EVENT(xrep_dir_reset_fork); +DEFINE_XREP_DIR_EVENT(xrep_parent_reset_dotdot); DECLARE_EVENT_CLASS(xrep_dirent_class, TP_PROTO(struct xfs_inode *dp, const struct xfs_name *name, From 1e58a8ccf2597c9259a8e71a2bffac5e11e12ea0 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:55 -0700 Subject: [PATCH 0247/1702] xfs: move orphan files to the orphanage When we're repairing a directory structure or fixing the dotdot entry of a subdirectory, it's possible that we won't ever find a parent for the subdirectory. When this is the case, move it to the orphanage, aka /lost+found. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- .../xfs/xfs-online-fsck-design.rst | 19 +- fs/xfs/Makefile | 1 + fs/xfs/scrub/dir_repair.c | 130 ++++- fs/xfs/scrub/orphanage.c | 498 ++++++++++++++++++ fs/xfs/scrub/orphanage.h | 75 +++ fs/xfs/scrub/parent_repair.c | 100 +++- fs/xfs/scrub/scrub.c | 2 + fs/xfs/scrub/scrub.h | 4 + fs/xfs/scrub/trace.h | 28 + fs/xfs/xfs_inode.c | 6 +- fs/xfs/xfs_inode.h | 1 + 11 files changed, 844 insertions(+), 20 deletions(-) create mode 100644 fs/xfs/scrub/orphanage.c create mode 100644 fs/xfs/scrub/orphanage.h diff --git a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst index 3afa1bc5f47ce..37dddaaeda50c 100644 --- a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst +++ b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst @@ -4778,14 +4778,21 @@ Orphaned files are adopted by the orphanage as follows: The ``xrep_orphanage_iolock_two`` function follows the inode locking strategy discussed earlier. -3. Call ``xrep_orphanage_compute_blkres`` and ``xrep_orphanage_compute_name`` - to compute the new name in the orphanage and the block reservation required. - -4. Use ``xrep_orphanage_adoption_prep`` to reserve resources to the repair +3. Use ``xrep_adoption_trans_alloc`` to reserve resources to the repair transaction. -5. Call ``xrep_orphanage_adopt`` to reparent the orphaned file into the lost - and found, and update the kernel dentry cache. +4. Call ``xrep_orphanage_compute_name`` to compute the new name in the + orphanage. + +5. If the adoption is going to happen, call ``xrep_adoption_reparent`` to + reparent the orphaned file into the lost and found and invalidate the dentry + cache. + +6. Call ``xrep_adoption_finish`` to commit any filesystem updates, release the + orphanage ILOCK, and clean the scrub transaction. + +7. If a runtime error happens, call ``xrep_adoption_cancel`` to release all + resources. The proposed patches are in the `orphanage adoption diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index d48646f86563f..1e23d1b3cd7b1 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -205,6 +205,7 @@ xfs-y += $(addprefix scrub/, \ inode_repair.o \ newbt.o \ nlinks_repair.o \ + orphanage.o \ parent_repair.o \ rcbag_btree.o \ rcbag.o \ diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index 34fe720fde0eb..c150b2efa2c21 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -42,6 +42,7 @@ #include "scrub/readdir.h" #include "scrub/reap.h" #include "scrub/findparent.h" +#include "scrub/orphanage.h" /* * Directory Repair @@ -115,12 +116,21 @@ struct xrep_dir { */ struct xrep_parent_scan_info pscan; + /* + * Context information for attaching this directory to the lost+found + * if this directory does not have a parent. + */ + struct xrep_adoption adoption; + /* How many subdirectories did we find? */ uint64_t subdirs; /* How many dirents did we find? */ unsigned int dirents; + /* Should we move this directory to the orphanage? */ + bool needs_adoption; + /* Directory entry name, plus the trailing null. */ struct xfs_name xname; unsigned char namebuf[MAXNAMELEN]; @@ -148,6 +158,10 @@ xrep_setup_directory( xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); + error = xrep_orphanage_try_create(sc); + if (error) + return error; + error = xrep_tempfile_create(sc, S_IFDIR); if (error) return error; @@ -1137,10 +1151,8 @@ xrep_dir_set_nlink( /* * The directory is not on the incore unlinked list, which means that * it needs to be reachable via the directory tree. Update the nlink - * with our observed link count. - * - * XXX: A subsequent patch will handle parentless directories by moving - * them to the lost and found instead of aborting the repair. + * with our observed link count. If the directory has no parent, it + * will be moved to the orphanage. */ if (!xfs_inode_on_unlinked_list(dp)) goto reset_nlink; @@ -1151,6 +1163,7 @@ xrep_dir_set_nlink( * inactivate when the last reference drops. */ if (rd->dirents == 0) { + rd->needs_adoption = false; new_nlink = 0; goto reset_nlink; } @@ -1159,7 +1172,8 @@ xrep_dir_set_nlink( * The directory is on the unlinked list and we found dirents. This * directory needs to be reachable via the directory tree. Remove the * dir from the unlinked list and update nlink with the observed link - * count. + * count. If the directory has no parent, it will be moved to the + * orphanage. */ pag = xfs_perag_get(sc->mp, XFS_INO_TO_AGNO(sc->mp, dp->i_ino)); if (!pag) { @@ -1195,12 +1209,16 @@ xrep_dir_swap( return -EFSCORRUPTED; /* - * If we never found the parent for this directory, we can't fix this - * directory. + * If we never found the parent for this directory, temporarily assign + * the root dir as the parent; we'll move this to the orphanage after + * exchanging the dir contents. We hold the ILOCK of the dir being + * repaired, so we're not worried about racy updates of dotdot. */ ASSERT(sc->ilock_flags & XFS_ILOCK_EXCL); - if (rd->pscan.parent_ino == NULLFSINO) - return -EFSCORRUPTED; + if (rd->pscan.parent_ino == NULLFSINO) { + rd->needs_adoption = true; + rd->pscan.parent_ino = rd->sc->mp->m_sb.sb_rootino; + } /* * Reset the temporary directory's '..' entry to point to the parent @@ -1358,6 +1376,91 @@ xrep_dir_setup_scan( return error; } +/* + * Move the current file to the orphanage. + * + * Caller must hold IOLOCK_EXCL on @sc->ip, and no other inode locks. Upon + * successful return, the scrub transaction will have enough extra reservation + * to make the move; it will hold IOLOCK_EXCL and ILOCK_EXCL of @sc->ip and the + * orphanage; and both inodes will be ijoined. + */ +STATIC int +xrep_dir_move_to_orphanage( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + xfs_ino_t orig_parent, new_parent; + int error; + + /* + * We are about to drop the ILOCK on sc->ip to lock the orphanage and + * prepare for the adoption. Therefore, look up the old dotdot entry + * for sc->ip so that we can compare it after we re-lock sc->ip. + */ + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, &orig_parent); + if (error) + return error; + + /* + * Drop the ILOCK on the scrub target and commit the transaction. + * Adoption computes its own resource requirements and gathers the + * necessary components. + */ + error = xrep_trans_commit(sc); + if (error) + return error; + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + /* If we can take the orphanage's iolock then we're ready to move. */ + if (!xrep_orphanage_ilock_nowait(sc, XFS_IOLOCK_EXCL)) { + xchk_iunlock(sc, sc->ilock_flags); + error = xrep_orphanage_iolock_two(sc); + if (error) + return error; + } + + /* Grab transaction and ILOCK the two files. */ + error = xrep_adoption_trans_alloc(sc, &rd->adoption); + if (error) + return error; + + error = xrep_adoption_compute_name(&rd->adoption, &rd->xname); + if (error) + return error; + + /* + * Now that we've reacquired the ILOCK on sc->ip, look up the dotdot + * entry again. If the parent changed or the child was unlinked while + * the child directory was unlocked, we don't need to move the child to + * the orphanage after all. + */ + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, &new_parent); + if (error) + return error; + + /* + * Attach to the orphanage if we still have a linked directory and it + * hasn't been moved. + */ + if (orig_parent == new_parent && VFS_I(sc->ip)->i_nlink > 0) { + error = xrep_adoption_move(&rd->adoption); + if (error) + return error; + } + + /* + * Launder the scrub transaction so we can drop the orphanage ILOCK + * and IOLOCK. Return holding the scrub target's ILOCK and IOLOCK. + */ + error = xrep_adoption_trans_roll(&rd->adoption); + if (error) + return error; + + xrep_orphanage_iunlock(sc, XFS_ILOCK_EXCL); + xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + return 0; +} + /* * Repair the directory metadata. * @@ -1396,6 +1499,15 @@ xrep_directory( if (error) goto out_teardown; + if (rd->needs_adoption) { + if (!xrep_orphanage_can_adopt(rd->sc)) + error = -EFSCORRUPTED; + else + error = xrep_dir_move_to_orphanage(rd); + if (error) + goto out_teardown; + } + out_teardown: xrep_dir_teardown(sc); return error; diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c new file mode 100644 index 0000000000000..41733be3ef45f --- /dev/null +++ b/fs/xfs/scrub/orphanage.c @@ -0,0 +1,498 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_ialloc.h" +#include "xfs_quota.h" +#include "xfs_trans_space.h" +#include "xfs_dir2.h" +#include "xfs_icache.h" +#include "xfs_bmap.h" +#include "xfs_bmap_btree.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/repair.h" +#include "scrub/trace.h" +#include "scrub/orphanage.h" +#include "scrub/readdir.h" + +#include + +/* + * The Orphanage + * ============= + * + * If the directory tree is damaged, children of that directory become + * inaccessible via that file path. If a child has no other parents, the file + * is said to be orphaned. xfs_repair fixes this situation by creating a + * orphanage directory (specifically, /lost+found) and creating a directory + * entry pointing to the orphaned file. + * + * Online repair follows this tactic by creating a root-owned /lost+found + * directory if one does not exist. If an orphan is found, it will move that + * files into orphanage. + */ + +/* Make the orphanage owned by root. */ +STATIC int +xrep_chown_orphanage( + struct xfs_scrub *sc, + struct xfs_inode *dp) +{ + struct xfs_trans *tp; + struct xfs_mount *mp = sc->mp; + struct xfs_dquot *udqp = NULL, *gdqp = NULL, *pdqp = NULL; + struct xfs_dquot *oldu = NULL, *oldg = NULL, *oldp = NULL; + struct inode *inode = VFS_I(dp); + int error; + + error = xfs_qm_vop_dqalloc(dp, GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0, + XFS_QMOPT_QUOTALL, &udqp, &gdqp, &pdqp); + if (error) + return error; + + error = xfs_trans_alloc_ichange(dp, udqp, gdqp, pdqp, true, &tp); + if (error) + goto out_dqrele; + + /* + * Always clear setuid/setgid/sticky on the orphanage since we don't + * normally want that functionality on this directory and xfs_repair + * doesn't create it this way either. Leave the other access bits + * unchanged. + */ + inode->i_mode &= ~(S_ISUID | S_ISGID | S_ISVTX); + + /* + * Change the ownerships and register quota modifications + * in the transaction. + */ + if (!uid_eq(inode->i_uid, GLOBAL_ROOT_UID)) { + if (XFS_IS_UQUOTA_ON(mp)) + oldu = xfs_qm_vop_chown(tp, dp, &dp->i_udquot, udqp); + inode->i_uid = GLOBAL_ROOT_UID; + } + if (!gid_eq(inode->i_gid, GLOBAL_ROOT_GID)) { + if (XFS_IS_GQUOTA_ON(mp)) + oldg = xfs_qm_vop_chown(tp, dp, &dp->i_gdquot, gdqp); + inode->i_gid = GLOBAL_ROOT_GID; + } + if (dp->i_projid != 0) { + if (XFS_IS_PQUOTA_ON(mp)) + oldp = xfs_qm_vop_chown(tp, dp, &dp->i_pdquot, pdqp); + dp->i_projid = 0; + } + + dp->i_diflags &= ~(XFS_DIFLAG_REALTIME | XFS_DIFLAG_RTINHERIT); + xfs_trans_log_inode(tp, dp, XFS_ILOG_CORE); + + XFS_STATS_INC(mp, xs_ig_attrchg); + + if (xfs_has_wsync(mp)) + xfs_trans_set_sync(tp); + error = xfs_trans_commit(tp); + + xfs_qm_dqrele(oldu); + xfs_qm_dqrele(oldg); + xfs_qm_dqrele(oldp); + +out_dqrele: + xfs_qm_dqrele(udqp); + xfs_qm_dqrele(gdqp); + xfs_qm_dqrele(pdqp); + return error; +} + +#define ORPHANAGE "lost+found" + +/* Create the orphanage directory, and set sc->orphanage to it. */ +int +xrep_orphanage_create( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct dentry *root_dentry, *orphanage_dentry; + struct inode *root_inode = VFS_I(sc->mp->m_rootip); + struct inode *orphanage_inode; + int error; + + if (xfs_is_shutdown(mp)) + return -EIO; + if (xfs_is_readonly(mp)) { + sc->orphanage = NULL; + return 0; + } + + ASSERT(sc->tp == NULL); + ASSERT(sc->orphanage == NULL); + + /* Find the dentry for the root directory... */ + root_dentry = d_find_alias(root_inode); + if (!root_dentry) { + error = -EFSCORRUPTED; + goto out; + } + + /* ...which is a directory, right? */ + if (!d_is_dir(root_dentry)) { + error = -EFSCORRUPTED; + goto out_dput_root; + } + + /* Try to find the orphanage directory. */ + inode_lock_nested(root_inode, I_MUTEX_PARENT); + orphanage_dentry = lookup_one_len(ORPHANAGE, root_dentry, + strlen(ORPHANAGE)); + if (IS_ERR(orphanage_dentry)) { + error = PTR_ERR(orphanage_dentry); + goto out_unlock_root; + } + + /* + * Nothing found? Call mkdir to create the orphanage. Create the + * directory without other-user access because we're live and someone + * could have been relying partly on minimal access to a parent + * directory to control access to a file we put in here. + */ + if (d_really_is_negative(orphanage_dentry)) { + error = vfs_mkdir(&nop_mnt_idmap, root_inode, orphanage_dentry, + 0750); + if (error) + goto out_dput_orphanage; + } + + /* Not a directory? Bail out. */ + if (!d_is_dir(orphanage_dentry)) { + error = -ENOTDIR; + goto out_dput_orphanage; + } + + /* + * Grab a reference to the orphanage. This /should/ succeed since + * we hold the root directory locked and therefore nobody can delete + * the orphanage. + */ + orphanage_inode = igrab(d_inode(orphanage_dentry)); + if (!orphanage_inode) { + error = -ENOENT; + goto out_dput_orphanage; + } + + /* Make sure the orphanage is owned by root. */ + error = xrep_chown_orphanage(sc, XFS_I(orphanage_inode)); + if (error) + goto out_dput_orphanage; + + /* Stash the reference for later and bail out. */ + sc->orphanage = XFS_I(orphanage_inode); + sc->orphanage_ilock_flags = 0; + +out_dput_orphanage: + dput(orphanage_dentry); +out_unlock_root: + inode_unlock(VFS_I(sc->mp->m_rootip)); +out_dput_root: + dput(root_dentry); +out: + return error; +} + +void +xrep_orphanage_ilock( + struct xfs_scrub *sc, + unsigned int ilock_flags) +{ + sc->orphanage_ilock_flags |= ilock_flags; + xfs_ilock(sc->orphanage, ilock_flags); +} + +bool +xrep_orphanage_ilock_nowait( + struct xfs_scrub *sc, + unsigned int ilock_flags) +{ + if (xfs_ilock_nowait(sc->orphanage, ilock_flags)) { + sc->orphanage_ilock_flags |= ilock_flags; + return true; + } + + return false; +} + +void +xrep_orphanage_iunlock( + struct xfs_scrub *sc, + unsigned int ilock_flags) +{ + xfs_iunlock(sc->orphanage, ilock_flags); + sc->orphanage_ilock_flags &= ~ilock_flags; +} + +/* Grab the IOLOCK of the orphanage and sc->ip. */ +int +xrep_orphanage_iolock_two( + struct xfs_scrub *sc) +{ + int error = 0; + + while (true) { + if (xchk_should_terminate(sc, &error)) + return error; + + /* + * Normal XFS takes the IOLOCK before grabbing a transaction. + * Scrub holds a transaction, which means that we can't block + * on either IOLOCK. + */ + if (xrep_orphanage_ilock_nowait(sc, XFS_IOLOCK_EXCL)) { + if (xchk_ilock_nowait(sc, XFS_IOLOCK_EXCL)) + break; + xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + } + delay(1); + } + + return 0; +} + +/* Release the orphanage. */ +void +xrep_orphanage_rele( + struct xfs_scrub *sc) +{ + if (!sc->orphanage) + return; + + if (sc->orphanage_ilock_flags) + xfs_iunlock(sc->orphanage, sc->orphanage_ilock_flags); + + xchk_irele(sc, sc->orphanage); + sc->orphanage = NULL; +} + +/* Adoption moves a file into /lost+found */ + +/* Can the orphanage adopt @sc->ip? */ +bool +xrep_orphanage_can_adopt( + struct xfs_scrub *sc) +{ + ASSERT(sc->ip != NULL); + + if (!sc->orphanage) + return false; + if (sc->ip == sc->orphanage) + return false; + if (xfs_internal_inum(sc->mp, sc->ip->i_ino)) + return false; + return true; +} + +/* + * Create a new transaction to send a child to the orphanage. + * + * Allocate a new transaction with sufficient disk space to handle the + * adoption, take ILOCK_EXCL of the orphanage and sc->ip, joins them to the + * transaction, and reserve quota to reparent the latter. Caller must hold the + * IOLOCK of the orphanage and sc->ip. + */ +int +xrep_adoption_trans_alloc( + struct xfs_scrub *sc, + struct xrep_adoption *adopt) +{ + struct xfs_mount *mp = sc->mp; + unsigned int child_blkres = 0; + int error; + + ASSERT(sc->tp == NULL); + ASSERT(sc->ip != NULL); + ASSERT(sc->orphanage != NULL); + ASSERT(sc->ilock_flags & XFS_IOLOCK_EXCL); + ASSERT(sc->orphanage_ilock_flags & XFS_IOLOCK_EXCL); + ASSERT(!(sc->ilock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL))); + ASSERT(!(sc->orphanage_ilock_flags & + (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL))); + + /* Compute the worst case space reservation that we need. */ + adopt->sc = sc; + adopt->orphanage_blkres = XFS_LINK_SPACE_RES(mp, MAXNAMELEN); + if (S_ISDIR(VFS_I(sc->ip)->i_mode)) + child_blkres = XFS_RENAME_SPACE_RES(mp, xfs_name_dotdot.len); + adopt->child_blkres = child_blkres; + + /* + * Allocate a transaction to link the child into the parent, along with + * enough disk space to handle expansion of both the orphanage and the + * dotdot entry of a child directory. + */ + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_link, + adopt->orphanage_blkres + adopt->child_blkres, 0, 0, + &sc->tp); + if (error) + return error; + + xfs_lock_two_inodes(sc->orphanage, XFS_ILOCK_EXCL, + sc->ip, XFS_ILOCK_EXCL); + sc->ilock_flags |= XFS_ILOCK_EXCL; + sc->orphanage_ilock_flags |= XFS_ILOCK_EXCL; + + xfs_trans_ijoin(sc->tp, sc->orphanage, 0); + xfs_trans_ijoin(sc->tp, sc->ip, 0); + + /* + * Reserve enough quota in the orphan directory to add the new name. + * Normally the orphanage should have user/group/project ids of zero + * and hence is not subject to quota enforcement, but we're allowed to + * exceed quota to reattach disconnected parts of the directory tree. + */ + error = xfs_trans_reserve_quota_nblks(sc->tp, sc->orphanage, + adopt->orphanage_blkres, 0, true); + if (error) + goto out_cancel; + + /* + * Reserve enough quota in the child directory to change dotdot. + * Here we're also allowed to exceed file quota to repair inconsistent + * metadata. + */ + if (adopt->child_blkres) { + error = xfs_trans_reserve_quota_nblks(sc->tp, sc->ip, + adopt->child_blkres, 0, true); + if (error) + goto out_cancel; + } + + return 0; +out_cancel: + xchk_trans_cancel(sc); + xrep_orphanage_iunlock(sc, XFS_ILOCK_EXCL); + xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + return error; +} + +/* + * Compute the xfs_name for the directory entry that we're adding to the + * orphanage. Caller must hold ILOCKs of sc->ip and the orphanage and must not + * reuse namebuf until the adoption completes or is dissolved. + */ +int +xrep_adoption_compute_name( + struct xrep_adoption *adopt, + struct xfs_name *xname) +{ + struct xfs_scrub *sc = adopt->sc; + char *namebuf = (void *)xname->name; + xfs_ino_t ino; + unsigned int incr = 0; + int error = 0; + + adopt->xname = xname; + xname->len = snprintf(namebuf, MAXNAMELEN, "%llu", sc->ip->i_ino); + xname->type = xfs_mode_to_ftype(VFS_I(sc->ip)->i_mode); + + /* Make sure the filename is unique in the lost+found. */ + error = xchk_dir_lookup(sc, sc->orphanage, xname, &ino); + while (error == 0 && incr < 10000) { + xname->len = snprintf(namebuf, MAXNAMELEN, "%llu.%u", + sc->ip->i_ino, ++incr); + error = xchk_dir_lookup(sc, sc->orphanage, xname, &ino); + } + if (error == 0) { + /* We already have 10,000 entries in the orphanage? */ + return -EFSCORRUPTED; + } + + if (error != -ENOENT) + return error; + return 0; +} + +/* + * Move the current file to the orphanage under the computed name. + * + * Returns with a dirty transaction so that the caller can handle any other + * work, such as fixing up unlinked lists or resetting link counts. + */ +int +xrep_adoption_move( + struct xrep_adoption *adopt) +{ + struct xfs_scrub *sc = adopt->sc; + bool isdir = S_ISDIR(VFS_I(sc->ip)->i_mode); + int error; + + trace_xrep_adoption_reparent(sc->orphanage, adopt->xname, + sc->ip->i_ino); + + /* Create the new name in the orphanage. */ + error = xfs_dir_createname(sc->tp, sc->orphanage, adopt->xname, + sc->ip->i_ino, adopt->orphanage_blkres); + if (error) + return error; + + /* + * Bump the link count of the orphanage if we just added a + * subdirectory, and update its timestamps. + */ + xfs_trans_ichgtime(sc->tp, sc->orphanage, + XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); + if (isdir) + xfs_bumplink(sc->tp, sc->orphanage); + xfs_trans_log_inode(sc->tp, sc->orphanage, XFS_ILOG_CORE); + + /* Replace the dotdot entry if the child is a subdirectory. */ + if (isdir) { + error = xfs_dir_replace(sc->tp, sc->ip, &xfs_name_dotdot, + sc->orphanage->i_ino, adopt->child_blkres); + if (error) + return error; + } + + /* + * Notify dirent hooks that we moved the file to /lost+found, and + * finish all the deferred work so that we know the adoption is fully + * recorded in the log. + */ + xfs_dir_update_hook(sc->orphanage, sc->ip, 1, adopt->xname); + return 0; +} + +/* + * Roll to a clean scrub transaction so that we can release the orphanage, + * even if xrep_adoption_move was not called. + * + * Commits all the work and deferred ops attached to an adoption request and + * rolls to a clean scrub transaction. On success, returns 0 with the scrub + * context holding a clean transaction with no inodes joined. On failure, + * returns negative errno with no scrub transaction. All inode locks are + * still held after this function returns. + */ +int +xrep_adoption_trans_roll( + struct xrep_adoption *adopt) +{ + struct xfs_scrub *sc = adopt->sc; + int error; + + trace_xrep_adoption_trans_roll(sc->orphanage, sc->ip, + !!(sc->tp->t_flags & XFS_TRANS_DIRTY)); + + /* Finish all the deferred ops to commit all repairs. */ + error = xrep_defer_finish(sc); + if (error) + return error; + + /* Roll the transaction once more to detach the inodes. */ + return xfs_trans_roll(&sc->tp); +} diff --git a/fs/xfs/scrub/orphanage.h b/fs/xfs/scrub/orphanage.h new file mode 100644 index 0000000000000..319179ab788d3 --- /dev/null +++ b/fs/xfs/scrub/orphanage.h @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_ORPHANAGE_H__ +#define __XFS_SCRUB_ORPHANAGE_H__ + +#ifdef CONFIG_XFS_ONLINE_REPAIR +int xrep_orphanage_create(struct xfs_scrub *sc); + +/* + * If we're doing a repair, ensure that the orphanage exists and attach it to + * the scrub context. + */ +static inline int +xrep_orphanage_try_create( + struct xfs_scrub *sc) +{ + int error; + + ASSERT(sc->sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR); + + error = xrep_orphanage_create(sc); + switch (error) { + case 0: + case -ENOENT: + case -ENOTDIR: + case -ENOSPC: + /* + * If the orphanage can't be found or isn't a directory, we'll + * keep going, but we won't be able to attach the file to the + * orphanage if we can't find the parent. + */ + return 0; + } + + return error; +} + +int xrep_orphanage_iolock_two(struct xfs_scrub *sc); + +void xrep_orphanage_ilock(struct xfs_scrub *sc, unsigned int ilock_flags); +bool xrep_orphanage_ilock_nowait(struct xfs_scrub *sc, + unsigned int ilock_flags); +void xrep_orphanage_iunlock(struct xfs_scrub *sc, unsigned int ilock_flags); + +void xrep_orphanage_rele(struct xfs_scrub *sc); + +/* Information about a request to add a file to the orphanage. */ +struct xrep_adoption { + struct xfs_scrub *sc; + + /* Name used for the adoption. */ + struct xfs_name *xname; + + /* Block reservations for orphanage and child (if directory). */ + unsigned int orphanage_blkres; + unsigned int child_blkres; +}; + +bool xrep_orphanage_can_adopt(struct xfs_scrub *sc); + +int xrep_adoption_trans_alloc(struct xfs_scrub *sc, + struct xrep_adoption *adopt); +int xrep_adoption_compute_name(struct xrep_adoption *adopt, + struct xfs_name *xname); +int xrep_adoption_move(struct xrep_adoption *adopt); +int xrep_adoption_trans_roll(struct xrep_adoption *adopt); +#else +struct xrep_adoption { /* empty */ }; +# define xrep_orphanage_rele(sc) ((void)0) +#endif /* CONFIG_XFS_ONLINE_REPAIR */ + +#endif /* __XFS_SCRUB_ORPHANAGE_H__ */ diff --git a/fs/xfs/scrub/parent_repair.c b/fs/xfs/scrub/parent_repair.c index 826926c2bb0dd..ebb5791bf839e 100644 --- a/fs/xfs/scrub/parent_repair.c +++ b/fs/xfs/scrub/parent_repair.c @@ -32,6 +32,8 @@ #include "scrub/iscan.h" #include "scrub/findparent.h" #include "scrub/readdir.h" +#include "scrub/tempfile.h" +#include "scrub/orphanage.h" /* * Repairing The Directory Parent Pointer @@ -57,6 +59,13 @@ struct xrep_parent { * dotdot entry for this directory. */ struct xrep_parent_scan_info pscan; + + /* Orphanage reparenting request. */ + struct xrep_adoption adoption; + + /* Directory entry name, plus the trailing null. */ + struct xfs_name xname; + unsigned char namebuf[MAXNAMELEN]; }; /* Tear down all the incore stuff we created. */ @@ -80,9 +89,10 @@ xrep_setup_parent( if (!rp) return -ENOMEM; rp->sc = sc; + rp->xname.name = rp->namebuf; sc->buf = rp; - return 0; + return xrep_orphanage_try_create(sc); } /* @@ -179,6 +189,91 @@ xrep_parent_reset_dotdot( return xfs_trans_roll(&sc->tp); } +/* + * Move the current file to the orphanage. + * + * Caller must hold IOLOCK_EXCL on @sc->ip, and no other inode locks. Upon + * successful return, the scrub transaction will have enough extra reservation + * to make the move; it will hold IOLOCK_EXCL and ILOCK_EXCL of @sc->ip and the + * orphanage; and both inodes will be ijoined. + */ +STATIC int +xrep_parent_move_to_orphanage( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + xfs_ino_t orig_parent, new_parent; + int error; + + /* + * We are about to drop the ILOCK on sc->ip to lock the orphanage and + * prepare for the adoption. Therefore, look up the old dotdot entry + * for sc->ip so that we can compare it after we re-lock sc->ip. + */ + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, &orig_parent); + if (error) + return error; + + /* + * Drop the ILOCK on the scrub target and commit the transaction. + * Adoption computes its own resource requirements and gathers the + * necessary components. + */ + error = xrep_trans_commit(sc); + if (error) + return error; + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + /* If we can take the orphanage's iolock then we're ready to move. */ + if (!xrep_orphanage_ilock_nowait(sc, XFS_IOLOCK_EXCL)) { + xchk_iunlock(sc, sc->ilock_flags); + error = xrep_orphanage_iolock_two(sc); + if (error) + return error; + } + + /* Grab transaction and ILOCK the two files. */ + error = xrep_adoption_trans_alloc(sc, &rp->adoption); + if (error) + return error; + + error = xrep_adoption_compute_name(&rp->adoption, &rp->xname); + if (error) + return error; + + /* + * Now that we've reacquired the ILOCK on sc->ip, look up the dotdot + * entry again. If the parent changed or the child was unlinked while + * the child directory was unlocked, we don't need to move the child to + * the orphanage after all. + */ + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, &new_parent); + if (error) + return error; + + /* + * Attach to the orphanage if we still have a linked directory and it + * hasn't been moved. + */ + if (orig_parent == new_parent && VFS_I(sc->ip)->i_nlink > 0) { + error = xrep_adoption_move(&rp->adoption); + if (error) + return error; + } + + /* + * Launder the scrub transaction so we can drop the orphanage ILOCK + * and IOLOCK. Return holding the scrub target's ILOCK and IOLOCK. + */ + error = xrep_adoption_trans_roll(&rp->adoption); + if (error) + return error; + + xrep_orphanage_iunlock(sc, XFS_ILOCK_EXCL); + xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + return 0; +} + /* * Commit the new parent pointer structure (currently only the dotdot entry) to * the file that we're repairing. @@ -188,7 +283,8 @@ xrep_parent_rebuild_tree( struct xrep_parent *rp) { if (rp->pscan.parent_ino == NULLFSINO) { - /* Cannot fix orphaned directories yet. */ + if (xrep_orphanage_can_adopt(rp->sc)) + return xrep_parent_move_to_orphanage(rp); return -EFSCORRUPTED; } diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 520d83db193c3..6417628ce26be 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -27,6 +27,7 @@ #include "scrub/stats.h" #include "scrub/xfile.h" #include "scrub/tempfile.h" +#include "scrub/orphanage.h" /* * Online Scrub and Repair @@ -217,6 +218,7 @@ xchk_teardown( } xrep_tempfile_rele(sc); + xrep_orphanage_rele(sc); xchk_fsgates_disable(sc); return error; } diff --git a/fs/xfs/scrub/scrub.h b/fs/xfs/scrub/scrub.h index d38f0b30416c0..7abe498f7a461 100644 --- a/fs/xfs/scrub/scrub.h +++ b/fs/xfs/scrub/scrub.h @@ -105,6 +105,10 @@ struct xfs_scrub { /* Lock flags for @ip. */ uint ilock_flags; + /* The orphanage, for stashing files that have lost their parent. */ + uint orphanage_ilock_flags; + struct xfs_inode *orphanage; + /* A temporary file on this filesystem, for staging new metadata. */ struct xfs_inode *tempip; uint temp_ilock_flags; diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index d68ec8e2781e5..7c49aa6f6b8da 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2588,6 +2588,34 @@ DEFINE_EVENT(xrep_dirent_class, name, \ DEFINE_XREP_DIRENT_EVENT(xrep_dir_salvage_entry); DEFINE_XREP_DIRENT_EVENT(xrep_dir_stash_createname); DEFINE_XREP_DIRENT_EVENT(xrep_dir_replay_createname); +DEFINE_XREP_DIRENT_EVENT(xrep_adoption_reparent); + +DECLARE_EVENT_CLASS(xrep_adoption_class, + TP_PROTO(struct xfs_inode *dp, struct xfs_inode *ip, bool moved), + TP_ARGS(dp, ip, moved), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dir_ino) + __field(xfs_ino_t, child_ino) + __field(bool, moved) + ), + TP_fast_assign( + __entry->dev = dp->i_mount->m_super->s_dev; + __entry->dir_ino = dp->i_ino; + __entry->child_ino = ip->i_ino; + __entry->moved = moved; + ), + TP_printk("dev %d:%d dir 0x%llx child 0x%llx moved? %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dir_ino, + __entry->child_ino, + __entry->moved) +); +#define DEFINE_XREP_ADOPTION_EVENT(name) \ +DEFINE_EVENT(xrep_adoption_class, name, \ + TP_PROTO(struct xfs_inode *dp, struct xfs_inode *ip, bool moved), \ + TP_ARGS(dp, ip, moved)) +DEFINE_XREP_ADOPTION_EVENT(xrep_adoption_trans_roll); DECLARE_EVENT_CLASS(xrep_parent_salvage_class, TP_PROTO(struct xfs_inode *dp, xfs_ino_t ino), diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 09d643a9e997b..803a646870145 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -914,10 +914,10 @@ xfs_droplink( /* * Increment the link count on an inode & log the change. */ -static void +void xfs_bumplink( - xfs_trans_t *tp, - xfs_inode_t *ip) + struct xfs_trans *tp, + struct xfs_inode *ip) { xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_CHG); diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index 8157ae7f8e59f..18bc3d7750a04 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -625,6 +625,7 @@ void xfs_end_io(struct work_struct *work); int xfs_ilock2_io_mmap(struct xfs_inode *ip1, struct xfs_inode *ip2); void xfs_iunlock2_io_mmap(struct xfs_inode *ip1, struct xfs_inode *ip2); void xfs_iunlock2_remapping(struct xfs_inode *ip1, struct xfs_inode *ip2); +void xfs_bumplink(struct xfs_trans *tp, struct xfs_inode *ip); static inline bool xfs_inode_unlinked_incomplete( From 34c9382c128270d0f4c8b36783b30f3c8085b2dd Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:54 -0700 Subject: [PATCH 0248/1702] xfs: ask the dentry cache if it knows the parent of a directory It's possible that the dentry cache can tell us the parent of a directory. Therefore, when repairing directory dot dot entries, query the dcache as a last resort before scanning the entire filesystem. A reviewer asks: "How high is the chance that we actually have a valid dcache entry for a file in a corrupted directory?" There's a decent chance of this actually working. Say you have a 1000-block directory foo, and block 980 gets corrupted. Let's further suppose that block 0 has a correct entry for ".." and "bar". If someone accesses /mnt/foo/bar, that will cause the dcache to create a dentry from /mnt to /mnt/foo whose d_parent points back to /mnt. If you then want to rebuild the directory, XFS can obtain the parent from the dcache without needing to wander into parent pointers or scan the filesystem to find /mnt's connection to foo. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/dir_repair.c | 29 +++++++++++++++++++++++++++ fs/xfs/scrub/findparent.c | 38 +++++++++++++++++++++++++++++++++++- fs/xfs/scrub/findparent.h | 1 + fs/xfs/scrub/parent_repair.c | 13 ++++++++++++ fs/xfs/scrub/trace.h | 1 + 5 files changed, 81 insertions(+), 1 deletion(-) diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index b17de79207dba..34fe720fde0eb 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -208,6 +208,29 @@ xrep_dir_lookup_parent( return ino; } +/* + * Look up '..' in the dentry cache and confirm that it's really the parent. + * Returns NULLFSINO if the dcache misses or if the hit is implausible. + */ +static inline xfs_ino_t +xrep_dir_dcache_parent( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + xfs_ino_t parent_ino; + int error; + + parent_ino = xrep_findparent_from_dcache(sc); + if (parent_ino == NULLFSINO) + return parent_ino; + + error = xrep_findparent_confirm(sc, &parent_ino); + if (error) + return NULLFSINO; + + return parent_ino; +} + /* Try to find the parent of the directory being repaired. */ STATIC int xrep_dir_find_parent( @@ -221,6 +244,12 @@ xrep_dir_find_parent( return 0; } + ino = xrep_dir_dcache_parent(rd); + if (ino != NULLFSINO) { + xrep_findparent_scan_finish_early(&rd->pscan, ino); + return 0; + } + ino = xrep_dir_lookup_parent(rd); if (ino != NULLFSINO) { xrep_findparent_scan_finish_early(&rd->pscan, ino); diff --git a/fs/xfs/scrub/findparent.c b/fs/xfs/scrub/findparent.c index 7b3ec8d7d6cc9..712dd73e4789f 100644 --- a/fs/xfs/scrub/findparent.c +++ b/fs/xfs/scrub/findparent.c @@ -53,7 +53,8 @@ * must not read the scan results without re-taking @sc->ip's ILOCK. * * There are a few shortcuts that we can take to avoid scanning the entire - * filesystem, such as noticing directory tree roots. + * filesystem, such as noticing directory tree roots and querying the dentry + * cache for parent information. */ struct xrep_findparent_info { @@ -410,3 +411,38 @@ xrep_findparent_self_reference( return NULLFSINO; } + +/* Check the dentry cache to see if knows of a parent for the scrub target. */ +xfs_ino_t +xrep_findparent_from_dcache( + struct xfs_scrub *sc) +{ + struct inode *pip = NULL; + struct dentry *dentry, *parent; + xfs_ino_t ret = NULLFSINO; + + dentry = d_find_alias(VFS_I(sc->ip)); + if (!dentry) + goto out; + + parent = dget_parent(dentry); + if (!parent) + goto out_dput; + + ASSERT(parent->d_sb == sc->ip->i_mount->m_super); + + pip = igrab(d_inode(parent)); + dput(parent); + + if (S_ISDIR(pip->i_mode)) { + trace_xrep_findparent_from_dcache(sc->ip, XFS_I(pip)->i_ino); + ret = XFS_I(pip)->i_ino; + } + + xchk_irele(sc, XFS_I(pip)); + +out_dput: + dput(dentry); +out: + return ret; +} diff --git a/fs/xfs/scrub/findparent.h b/fs/xfs/scrub/findparent.h index d946bc81f34e6..501f99d3164ed 100644 --- a/fs/xfs/scrub/findparent.h +++ b/fs/xfs/scrub/findparent.h @@ -45,5 +45,6 @@ void xrep_findparent_scan_finish_early(struct xrep_parent_scan_info *pscan, int xrep_findparent_confirm(struct xfs_scrub *sc, xfs_ino_t *parent_ino); xfs_ino_t xrep_findparent_self_reference(struct xfs_scrub *sc); +xfs_ino_t xrep_findparent_from_dcache(struct xfs_scrub *sc); #endif /* __XFS_SCRUB_FINDPARENT_H__ */ diff --git a/fs/xfs/scrub/parent_repair.c b/fs/xfs/scrub/parent_repair.c index 0a9651bb0b057..826926c2bb0dd 100644 --- a/fs/xfs/scrub/parent_repair.c +++ b/fs/xfs/scrub/parent_repair.c @@ -118,7 +118,20 @@ xrep_parent_find_dotdot( * then retake the ILOCK so that we can salvage directory entries. */ xchk_iunlock(sc, XFS_ILOCK_EXCL); + + /* Does the VFS dcache have an answer for us? */ + ino = xrep_findparent_from_dcache(sc); + if (ino != NULLFSINO) { + error = xrep_findparent_confirm(sc, &ino); + if (!error && ino != NULLFSINO) { + xrep_findparent_scan_finish_early(&rp->pscan, ino); + goto out_relock; + } + } + + /* Scan the entire filesystem for a parent. */ error = xrep_findparent_scan(&rp->pscan); +out_relock: xchk_ilock(sc, XFS_ILOCK_EXCL); return error; diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index e1755fe63e67f..d68ec8e2781e5 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2613,6 +2613,7 @@ DEFINE_EVENT(xrep_parent_salvage_class, name, \ TP_ARGS(dp, ino)) DEFINE_XREP_PARENT_SALVAGE_EVENT(xrep_dir_salvaged_parent); DEFINE_XREP_PARENT_SALVAGE_EVENT(xrep_findparent_dirent); +DEFINE_XREP_PARENT_SALVAGE_EVENT(xrep_findparent_from_dcache); #endif /* IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) */ From ef744be416b5c649d287604730400dfa728779fe Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:58 -0700 Subject: [PATCH 0249/1702] xfs: expose xfs_bmap_local_to_extents for online repair Allow online repair to call xfs_bmap_local_to_extents and add a void * argument at the end so that online repair can pass its own context. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_bmap.c | 11 ++++++----- fs/xfs/libxfs/xfs_bmap.h | 6 ++++++ fs/xfs/libxfs/xfs_symlink_remote.c | 3 ++- fs/xfs/libxfs/xfs_symlink_remote.h | 3 ++- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 46bbc9f0a1173..59b8b9dc29ccf 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -779,7 +779,7 @@ xfs_bmap_local_to_extents_empty( } -STATIC int /* error */ +int /* error */ xfs_bmap_local_to_extents( xfs_trans_t *tp, /* transaction pointer */ xfs_inode_t *ip, /* incore inode pointer */ @@ -789,7 +789,8 @@ xfs_bmap_local_to_extents( void (*init_fn)(struct xfs_trans *tp, struct xfs_buf *bp, struct xfs_inode *ip, - struct xfs_ifork *ifp)) + struct xfs_ifork *ifp, void *priv), + void *priv) { int error = 0; int flags; /* logging flags returned */ @@ -850,7 +851,7 @@ xfs_bmap_local_to_extents( * log here. Note that init_fn must also set the buffer log item type * correctly. */ - init_fn(tp, bp, ip, ifp); + init_fn(tp, bp, ip, ifp, priv); /* account for the change in fork size */ xfs_idata_realloc(ip, -ifp->if_bytes, whichfork); @@ -982,8 +983,8 @@ xfs_bmap_add_attrfork_local( if (S_ISLNK(VFS_I(ip)->i_mode)) return xfs_bmap_local_to_extents(tp, ip, 1, flags, - XFS_DATA_FORK, - xfs_symlink_local_to_remote); + XFS_DATA_FORK, xfs_symlink_local_to_remote, + NULL); /* should only be called for types that support local format data */ ASSERT(0); diff --git a/fs/xfs/libxfs/xfs_bmap.h b/fs/xfs/libxfs/xfs_bmap.h index b8bdbf1560e65..32fb2a455c294 100644 --- a/fs/xfs/libxfs/xfs_bmap.h +++ b/fs/xfs/libxfs/xfs_bmap.h @@ -179,6 +179,12 @@ unsigned int xfs_bmap_compute_attr_offset(struct xfs_mount *mp); int xfs_bmap_add_attrfork(struct xfs_inode *ip, int size, int rsvd); void xfs_bmap_local_to_extents_empty(struct xfs_trans *tp, struct xfs_inode *ip, int whichfork); +int xfs_bmap_local_to_extents(struct xfs_trans *tp, struct xfs_inode *ip, + xfs_extlen_t total, int *logflagsp, int whichfork, + void (*init_fn)(struct xfs_trans *tp, struct xfs_buf *bp, + struct xfs_inode *ip, struct xfs_ifork *ifp, + void *priv), + void *priv); void xfs_bmap_compute_maxlevels(struct xfs_mount *mp, int whichfork); int xfs_bmap_first_unused(struct xfs_trans *tp, struct xfs_inode *ip, xfs_extlen_t len, xfs_fileoff_t *unused, int whichfork); diff --git a/fs/xfs/libxfs/xfs_symlink_remote.c b/fs/xfs/libxfs/xfs_symlink_remote.c index 8f0d5c584f46f..d150576ddd0af 100644 --- a/fs/xfs/libxfs/xfs_symlink_remote.c +++ b/fs/xfs/libxfs/xfs_symlink_remote.c @@ -169,7 +169,8 @@ xfs_symlink_local_to_remote( struct xfs_trans *tp, struct xfs_buf *bp, struct xfs_inode *ip, - struct xfs_ifork *ifp) + struct xfs_ifork *ifp, + void *priv) { struct xfs_mount *mp = ip->i_mount; char *buf; diff --git a/fs/xfs/libxfs/xfs_symlink_remote.h b/fs/xfs/libxfs/xfs_symlink_remote.h index ac3dac8f617ed..83b89a1deb9f2 100644 --- a/fs/xfs/libxfs/xfs_symlink_remote.h +++ b/fs/xfs/libxfs/xfs_symlink_remote.h @@ -16,7 +16,8 @@ int xfs_symlink_hdr_set(struct xfs_mount *mp, xfs_ino_t ino, uint32_t offset, bool xfs_symlink_hdr_ok(xfs_ino_t ino, uint32_t offset, uint32_t size, struct xfs_buf *bp); void xfs_symlink_local_to_remote(struct xfs_trans *tp, struct xfs_buf *bp, - struct xfs_inode *ip, struct xfs_ifork *ifp); + struct xfs_inode *ip, struct xfs_ifork *ifp, + void *priv); xfs_failaddr_t xfs_symlink_shortform_verify(void *sfp, int64_t size); int xfs_symlink_remote_read(struct xfs_inode *ip, char *link); int xfs_symlink_write_target(struct xfs_trans *tp, struct xfs_inode *ip, From e6c9e75fbe792e1fb3bc7e7efce5c6bb015023c5 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:56 -0700 Subject: [PATCH 0250/1702] xfs: move files to orphanage instead of letting nlinks drop to zero If we encounter an inode with a nonzero link count but zero observed links, move it to the orphanage. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- .../xfs/xfs-online-fsck-design.rst | 3 +- fs/xfs/scrub/nlinks.c | 20 ++- fs/xfs/scrub/nlinks.h | 7 + fs/xfs/scrub/nlinks_repair.c | 123 ++++++++++++++++-- fs/xfs/scrub/repair.h | 2 + fs/xfs/scrub/trace.c | 1 + fs/xfs/scrub/trace.h | 26 ++++ 7 files changed, 163 insertions(+), 19 deletions(-) diff --git a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst index 37dddaaeda50c..74a8e42c74bd0 100644 --- a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst +++ b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst @@ -4789,7 +4789,8 @@ Orphaned files are adopted by the orphanage as follows: cache. 6. Call ``xrep_adoption_finish`` to commit any filesystem updates, release the - orphanage ILOCK, and clean the scrub transaction. + orphanage ILOCK, and clean the scrub transaction. Call + ``xrep_adoption_commit`` to commit the updates and the scrub transaction. 7. If a runtime error happens, call ``xrep_adoption_cancel`` to release all resources. diff --git a/fs/xfs/scrub/nlinks.c b/fs/xfs/scrub/nlinks.c index 8b9aa73093d66..c456523fac9c6 100644 --- a/fs/xfs/scrub/nlinks.c +++ b/fs/xfs/scrub/nlinks.c @@ -24,6 +24,7 @@ #include "scrub/xfile.h" #include "scrub/xfarray.h" #include "scrub/iscan.h" +#include "scrub/orphanage.h" #include "scrub/nlinks.h" #include "scrub/trace.h" #include "scrub/readdir.h" @@ -44,11 +45,23 @@ int xchk_setup_nlinks( struct xfs_scrub *sc) { + struct xchk_nlink_ctrs *xnc; + int error; + xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); - sc->buf = kzalloc(sizeof(struct xchk_nlink_ctrs), XCHK_GFP_FLAGS); - if (!sc->buf) + if (xchk_could_repair(sc)) { + error = xrep_setup_nlinks(sc); + if (error) + return error; + } + + xnc = kvzalloc(sizeof(struct xchk_nlink_ctrs), XCHK_GFP_FLAGS); + if (!xnc) return -ENOMEM; + xnc->xname.name = xnc->namebuf; + xnc->sc = sc; + sc->buf = xnc; return xchk_setup_fs(sc); } @@ -873,9 +886,6 @@ xchk_nlinks_setup_scan( xfs_agino_t first_agino, last_agino; int error; - ASSERT(xnc->sc == NULL); - xnc->sc = sc; - mutex_init(&xnc->lock); /* Retry iget every tenth of a second for up to 30 seconds. */ diff --git a/fs/xfs/scrub/nlinks.h b/fs/xfs/scrub/nlinks.h index a950f3daf204c..b820712bfd87c 100644 --- a/fs/xfs/scrub/nlinks.h +++ b/fs/xfs/scrub/nlinks.h @@ -28,6 +28,13 @@ struct xchk_nlink_ctrs { * from other writer threads. */ struct xfs_dir_hook dhook; + + /* Orphanage reparenting request. */ + struct xrep_adoption adoption; + + /* Directory entry name, plus the trailing null. */ + struct xfs_name xname; + char namebuf[MAXNAMELEN]; }; /* diff --git a/fs/xfs/scrub/nlinks_repair.c b/fs/xfs/scrub/nlinks_repair.c index 23eb08c4b5ad5..0cb67339eac89 100644 --- a/fs/xfs/scrub/nlinks_repair.c +++ b/fs/xfs/scrub/nlinks_repair.c @@ -24,6 +24,7 @@ #include "scrub/xfile.h" #include "scrub/xfarray.h" #include "scrub/iscan.h" +#include "scrub/orphanage.h" #include "scrub/nlinks.h" #include "scrub/trace.h" #include "scrub/tempfile.h" @@ -38,6 +39,34 @@ * inode is locked. */ +/* Set up to repair inode link counts. */ +int +xrep_setup_nlinks( + struct xfs_scrub *sc) +{ + return xrep_orphanage_try_create(sc); +} + +/* + * Inodes that aren't the root directory or the orphanage, have a nonzero link + * count, and no observed parents should be moved to the orphanage. + */ +static inline bool +xrep_nlinks_is_orphaned( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int actual_nlink, + const struct xchk_nlink *obs) +{ + struct xfs_mount *mp = ip->i_mount; + + if (obs->parents != 0) + return false; + if (ip == mp->m_rootip || ip == sc->orphanage) + return false; + return actual_nlink != 0; +} + /* Remove an inode from the unlinked list. */ STATIC int xrep_nlinks_iunlink_remove( @@ -66,6 +95,7 @@ xrep_nlinks_repair_inode( struct xfs_inode *ip = sc->ip; uint64_t total_links; uint64_t actual_nlink; + bool orphanage_available = false; bool dirty = false; int error; @@ -77,14 +107,41 @@ xrep_nlinks_repair_inode( if (xrep_is_tempfile(ip)) return 0; - xchk_ilock(sc, XFS_IOLOCK_EXCL); + /* + * If the filesystem has an orphanage attached to the scrub context, + * prepare for a link count repair that could involve @ip being adopted + * by the lost+found. + */ + if (xrep_orphanage_can_adopt(sc)) { + error = xrep_orphanage_iolock_two(sc); + if (error) + return error; + + error = xrep_adoption_trans_alloc(sc, &xnc->adoption); + if (error) { + xchk_iunlock(sc, XFS_IOLOCK_EXCL); + xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + } else { + orphanage_available = true; + } + } - error = xfs_trans_alloc(mp, &M_RES(mp)->tr_link, 0, 0, 0, &sc->tp); - if (error) - return error; + /* + * Either there is no orphanage or we couldn't allocate resources for + * that kind of update. Let's try again with only the resources we + * need for a simple link count update, since that's much more common. + */ + if (!orphanage_available) { + xchk_ilock(sc, XFS_IOLOCK_EXCL); + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_link, 0, 0, 0, + &sc->tp); + if (error) + return error; - xchk_ilock(sc, XFS_ILOCK_EXCL); - xfs_trans_ijoin(sc->tp, ip, 0); + xchk_ilock(sc, XFS_ILOCK_EXCL); + xfs_trans_ijoin(sc->tp, ip, 0); + } mutex_lock(&xnc->lock); @@ -122,6 +179,41 @@ xrep_nlinks_repair_inode( goto out_trans; } + /* + * Decide if we're going to move this file to the orphanage, and fix + * up the incore link counts if we are. + */ + if (orphanage_available && + xrep_nlinks_is_orphaned(sc, ip, actual_nlink, &obs)) { + /* Figure out what name we're going to use here. */ + error = xrep_adoption_compute_name(&xnc->adoption, &xnc->xname); + if (error) + goto out_trans; + + /* + * Reattach this file to the directory tree by moving it to + * the orphanage per the adoption parameters that we already + * computed. + */ + error = xrep_adoption_move(&xnc->adoption); + if (error) + goto out_trans; + + /* + * Re-read the link counts since the reparenting will have + * updated our scan info. + */ + mutex_lock(&xnc->lock); + error = xfarray_load_sparse(xnc->nlinks, ip->i_ino, &obs); + mutex_unlock(&xnc->lock); + if (error) + goto out_trans; + + total_links = xchk_nlink_total(ip, &obs); + actual_nlink = VFS_I(ip)->i_nlink; + dirty = true; + } + /* * If this inode is linked from the directory tree and on the unlinked * list, remove it from the unlinked list. @@ -165,14 +257,19 @@ xrep_nlinks_repair_inode( xfs_trans_log_inode(sc->tp, ip, XFS_ILOG_CORE); error = xrep_trans_commit(sc); - xchk_iunlock(sc, XFS_ILOCK_EXCL | XFS_IOLOCK_EXCL); - return error; + goto out_unlock; out_scanlock: mutex_unlock(&xnc->lock); out_trans: xchk_trans_cancel(sc); - xchk_iunlock(sc, XFS_ILOCK_EXCL | XFS_IOLOCK_EXCL); +out_unlock: + xchk_iunlock(sc, XFS_ILOCK_EXCL); + if (orphanage_available) { + xrep_orphanage_iunlock(sc, XFS_ILOCK_EXCL); + xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + } + xchk_iunlock(sc, XFS_IOLOCK_EXCL); return error; } @@ -205,10 +302,10 @@ xrep_nlinks( /* * We need ftype for an accurate count of the number of child * subdirectory links. Child subdirectories with a back link (dotdot - * entry) but no forward link are unfixable, so we cannot repair the - * link count of the parent directory based on the back link count - * alone. Filesystems without ftype support are rare (old V4) so we - * just skip out here. + * entry) but no forward link are moved to the orphanage, so we cannot + * repair the link count of the parent directory based on the back link + * count alone. Filesystems without ftype support are rare (old V4) so + * we just skip out here. */ if (!xfs_has_ftype(sc->mp)) return -EOPNOTSUPP; diff --git a/fs/xfs/scrub/repair.h b/fs/xfs/scrub/repair.h index e53374fa54308..7e6aba7fe5586 100644 --- a/fs/xfs/scrub/repair.h +++ b/fs/xfs/scrub/repair.h @@ -93,6 +93,7 @@ int xrep_setup_ag_refcountbt(struct xfs_scrub *sc); int xrep_setup_xattr(struct xfs_scrub *sc); int xrep_setup_directory(struct xfs_scrub *sc); int xrep_setup_parent(struct xfs_scrub *sc); +int xrep_setup_nlinks(struct xfs_scrub *sc); /* Repair setup functions */ int xrep_setup_ag_allocbt(struct xfs_scrub *sc); @@ -201,6 +202,7 @@ xrep_setup_nothing( #define xrep_setup_xattr xrep_setup_nothing #define xrep_setup_directory xrep_setup_nothing #define xrep_setup_parent xrep_setup_nothing +#define xrep_setup_nlinks xrep_setup_nothing #define xrep_setup_inode(sc, imap) ((void)0) diff --git a/fs/xfs/scrub/trace.c b/fs/xfs/scrub/trace.c index 3dd281d6d1854..b2ce7b22cad34 100644 --- a/fs/xfs/scrub/trace.c +++ b/fs/xfs/scrub/trace.c @@ -24,6 +24,7 @@ #include "scrub/xfarray.h" #include "scrub/quota.h" #include "scrub/iscan.h" +#include "scrub/orphanage.h" #include "scrub/nlinks.h" #include "scrub/fscounters.h" diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 7c49aa6f6b8da..2a4c54f7992a2 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2643,6 +2643,32 @@ DEFINE_XREP_PARENT_SALVAGE_EVENT(xrep_dir_salvaged_parent); DEFINE_XREP_PARENT_SALVAGE_EVENT(xrep_findparent_dirent); DEFINE_XREP_PARENT_SALVAGE_EVENT(xrep_findparent_from_dcache); +TRACE_EVENT(xrep_nlinks_set_record, + TP_PROTO(struct xfs_mount *mp, xfs_ino_t ino, + const struct xchk_nlink *obs), + TP_ARGS(mp, ino, obs), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_nlink_t, parents) + __field(xfs_nlink_t, backrefs) + __field(xfs_nlink_t, children) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->ino = ino; + __entry->parents = obs->parents; + __entry->backrefs = obs->backrefs; + __entry->children = obs->children; + ), + TP_printk("dev %d:%d ino 0x%llx parents %u backrefs %u children %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->parents, + __entry->backrefs, + __entry->children) +); + #endif /* IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) */ #endif /* _TRACE_XFS_SCRUB_TRACE_H */ From ea8214c3195c2ed3a205dea42bbe7746712fc461 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:59 -0700 Subject: [PATCH 0251/1702] xfs: pass the owner to xfs_symlink_write_target Require callers of xfs_symlink_write_target to pass the owner number explicitly. This sets us up for online repair to be able to write a remote symlink target to sc->tempip with sc->ip's inumber in the block heaader. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_symlink_remote.c | 4 ++-- fs/xfs/libxfs/xfs_symlink_remote.h | 4 ++-- fs/xfs/xfs_symlink.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fs/xfs/libxfs/xfs_symlink_remote.c b/fs/xfs/libxfs/xfs_symlink_remote.c index d150576ddd0af..f228127a88ff2 100644 --- a/fs/xfs/libxfs/xfs_symlink_remote.c +++ b/fs/xfs/libxfs/xfs_symlink_remote.c @@ -311,6 +311,7 @@ int xfs_symlink_write_target( struct xfs_trans *tp, struct xfs_inode *ip, + xfs_ino_t owner, const char *target_path, int pathlen, xfs_fsblock_t fs_blocks, @@ -365,8 +366,7 @@ xfs_symlink_write_target( byte_cnt = min(byte_cnt, pathlen); buf = bp->b_addr; - buf += xfs_symlink_hdr_set(mp, ip->i_ino, offset, byte_cnt, - bp); + buf += xfs_symlink_hdr_set(mp, owner, offset, byte_cnt, bp); memcpy(buf, cur_chunk, byte_cnt); diff --git a/fs/xfs/libxfs/xfs_symlink_remote.h b/fs/xfs/libxfs/xfs_symlink_remote.h index 83b89a1deb9f2..c1672fe1f17bb 100644 --- a/fs/xfs/libxfs/xfs_symlink_remote.h +++ b/fs/xfs/libxfs/xfs_symlink_remote.h @@ -21,8 +21,8 @@ void xfs_symlink_local_to_remote(struct xfs_trans *tp, struct xfs_buf *bp, xfs_failaddr_t xfs_symlink_shortform_verify(void *sfp, int64_t size); int xfs_symlink_remote_read(struct xfs_inode *ip, char *link); int xfs_symlink_write_target(struct xfs_trans *tp, struct xfs_inode *ip, - const char *target_path, int pathlen, xfs_fsblock_t fs_blocks, - uint resblks); + xfs_ino_t owner, const char *target_path, int pathlen, + xfs_fsblock_t fs_blocks, uint resblks); int xfs_symlink_remote_truncate(struct xfs_trans *tp, struct xfs_inode *ip); #endif /* __XFS_SYMLINK_REMOTE_H */ diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index 3daeebff4bb47..fb060aaf6d40f 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -181,8 +181,8 @@ xfs_symlink( xfs_qm_vop_create_dqattach(tp, ip, udqp, gdqp, pdqp); resblks -= XFS_IALLOC_SPACE_RES(mp); - error = xfs_symlink_write_target(tp, ip, target_path, pathlen, - fs_blocks, resblks); + error = xfs_symlink_write_target(tp, ip, ip->i_ino, target_path, + pathlen, fs_blocks, resblks); if (error) goto out_trans_cancel; resblks -= fs_blocks; From 73597e3e42b4a15030e6f93b71b53a04377ea419 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:57 -0700 Subject: [PATCH 0252/1702] xfs: ensure dentry consistency when the orphanage adopts a file When the orphanage adopts a file, that file becomes a child of the orphanage. The dentry cache may have entries for the orphanage directory and the name we've chosen, so (1) make sure we abort if the dcache has a positive entry because something's not right; and (2) invalidate and purge negative dentries if the adoption goes through. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/orphanage.c | 91 ++++++++++++++++++++++++++++++++++++++++ fs/xfs/scrub/trace.h | 42 +++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c index 41733be3ef45f..885b7d478a0ab 100644 --- a/fs/xfs/scrub/orphanage.c +++ b/fs/xfs/scrub/orphanage.c @@ -418,6 +418,90 @@ xrep_adoption_compute_name( return 0; } +/* + * Make sure the dcache does not have a positive dentry for the name we've + * chosen. The caller should have checked with the ondisk directory, so any + * discrepancy is a sign that something is seriously wrong. + */ +static int +xrep_adoption_check_dcache( + struct xrep_adoption *adopt) +{ + struct qstr qname = QSTR_INIT(adopt->xname->name, + adopt->xname->len); + struct dentry *d_orphanage, *d_child; + int error = 0; + + d_orphanage = d_find_alias(VFS_I(adopt->sc->orphanage)); + if (!d_orphanage) + return 0; + + d_child = d_hash_and_lookup(d_orphanage, &qname); + if (d_child) { + trace_xrep_adoption_check_child(adopt->sc->mp, d_child); + + if (d_is_positive(d_child)) { + ASSERT(d_is_negative(d_child)); + error = -EFSCORRUPTED; + } + + dput(d_child); + } + + dput(d_orphanage); + if (error) + return error; + + /* + * Do we need to update d_parent of the dentry for the file being + * repaired? There shouldn't be a hashed dentry with a parent since + * the file had nonzero nlink but wasn't connected to any parent dir. + */ + d_child = d_find_alias(VFS_I(adopt->sc->ip)); + if (!d_child) + return 0; + + trace_xrep_adoption_check_alias(adopt->sc->mp, d_child); + + if (d_child->d_parent && !d_unhashed(d_child)) { + ASSERT(d_child->d_parent == NULL || d_unhashed(d_child)); + error = -EFSCORRUPTED; + } + + dput(d_child); + return error; +} + +/* + * Remove all negative dentries from the dcache. There should not be any + * positive entries, since we've maintained our lock on the orphanage + * directory. + */ +static void +xrep_adoption_zap_dcache( + struct xrep_adoption *adopt) +{ + struct qstr qname = QSTR_INIT(adopt->xname->name, + adopt->xname->len); + struct dentry *d_orphanage, *d_child; + + d_orphanage = d_find_alias(VFS_I(adopt->sc->orphanage)); + if (!d_orphanage) + return; + + d_child = d_hash_and_lookup(d_orphanage, &qname); + while (d_child != NULL) { + trace_xrep_adoption_invalidate_child(adopt->sc->mp, d_child); + + ASSERT(d_is_negative(d_child)); + d_invalidate(d_child); + dput(d_child); + d_child = d_lookup(d_orphanage, &qname); + } + + dput(d_orphanage); +} + /* * Move the current file to the orphanage under the computed name. * @@ -435,6 +519,10 @@ xrep_adoption_move( trace_xrep_adoption_reparent(sc->orphanage, adopt->xname, sc->ip->i_ino); + error = xrep_adoption_check_dcache(adopt); + if (error) + return error; + /* Create the new name in the orphanage. */ error = xfs_dir_createname(sc->tp, sc->orphanage, adopt->xname, sc->ip->i_ino, adopt->orphanage_blkres); @@ -465,6 +553,9 @@ xrep_adoption_move( * recorded in the log. */ xfs_dir_update_hook(sc->orphanage, sc->ip, 1, adopt->xname); + + /* Remove negative dentries from the lost+found's dcache */ + xrep_adoption_zap_dcache(adopt); return 0; } diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 2a4c54f7992a2..668da6ff2ca26 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2669,6 +2669,48 @@ TRACE_EVENT(xrep_nlinks_set_record, __entry->children) ); +DECLARE_EVENT_CLASS(xrep_dentry_class, + TP_PROTO(struct xfs_mount *mp, const struct dentry *dentry), + TP_ARGS(mp, dentry), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, flags) + __field(unsigned long, ino) + __field(bool, positive) + __field(unsigned long, parent_ino) + __field(unsigned int, namelen) + __dynamic_array(char, name, dentry->d_name.len) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->flags = dentry->d_flags; + __entry->positive = d_is_positive(dentry); + if (dentry->d_parent && d_inode(dentry->d_parent)) + __entry->parent_ino = d_inode(dentry->d_parent)->i_ino; + else + __entry->parent_ino = -1UL; + __entry->ino = d_inode(dentry) ? d_inode(dentry)->i_ino : 0; + __entry->namelen = dentry->d_name.len; + memcpy(__get_str(name), dentry->d_name.name, dentry->d_name.len); + ), + TP_printk("dev %d:%d flags 0x%x positive? %d parent_ino 0x%lx ino 0x%lx name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->flags, + __entry->positive, + __entry->parent_ino, + __entry->ino, + __entry->namelen, + __get_str(name)) +); +#define DEFINE_REPAIR_DENTRY_EVENT(name) \ +DEFINE_EVENT(xrep_dentry_class, name, \ + TP_PROTO(struct xfs_mount *mp, const struct dentry *dentry), \ + TP_ARGS(mp, dentry)) +DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_check_child); +DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_check_alias); +DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_check_dentry); +DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_invalidate_child); + #endif /* IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) */ #endif /* _TRACE_XFS_SCRUB_TRACE_H */ From 10d587ecb77f33d966cfa2e8a57d51296df986db Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:55:00 -0700 Subject: [PATCH 0253/1702] xfs: check AGI unlinked inode buckets Look for corruptions in the AGI unlinked bucket chains. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/agheader.c | 40 ++++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_inode.c | 2 +- fs/xfs/xfs_inode.h | 1 + 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/fs/xfs/scrub/agheader.c b/fs/xfs/scrub/agheader.c index e954f07679dd7..1528f14bd9251 100644 --- a/fs/xfs/scrub/agheader.c +++ b/fs/xfs/scrub/agheader.c @@ -15,6 +15,7 @@ #include "xfs_ialloc.h" #include "xfs_rmap.h" #include "xfs_ag.h" +#include "xfs_inode.h" #include "scrub/scrub.h" #include "scrub/common.h" @@ -865,6 +866,43 @@ xchk_agi_xref( /* scrub teardown will take care of sc->sa for us */ } +/* + * Check the unlinked buckets for links to bad inodes. We hold the AGI, so + * there cannot be any threads updating unlinked list pointers in this AG. + */ +STATIC void +xchk_iunlink( + struct xfs_scrub *sc, + struct xfs_agi *agi) +{ + unsigned int i; + struct xfs_inode *ip; + + for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) { + xfs_agino_t agino = be32_to_cpu(agi->agi_unlinked[i]); + + while (agino != NULLAGINO) { + if (agino % XFS_AGI_UNLINKED_BUCKETS != i) { + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + return; + } + + ip = xfs_iunlink_lookup(sc->sa.pag, agino); + if (!ip) { + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + return; + } + + if (!xfs_inode_on_unlinked_list(ip)) { + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + return; + } + + agino = ip->i_next_unlinked; + } + } +} + /* Scrub the AGI. */ int xchk_agi( @@ -949,6 +987,8 @@ xchk_agi( if (pag->pagi_freecount != be32_to_cpu(agi->agi_freecount)) xchk_block_set_corrupt(sc, sc->sa.agi_bp); + xchk_iunlink(sc, agi); + xchk_agi_xref(sc); out: return error; diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 803a646870145..fed0cd6bffdf0 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -1985,7 +1985,7 @@ xfs_inactive( * only unlinked, referenced inodes can be on the unlinked inode list. If we * don't find the inode in cache, then let the caller handle the situation. */ -static struct xfs_inode * +struct xfs_inode * xfs_iunlink_lookup( struct xfs_perag *pag, xfs_agino_t agino) diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index 18bc3d7750a04..c74c48bc09453 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -619,6 +619,7 @@ bool xfs_inode_needs_inactive(struct xfs_inode *ip); int xfs_iunlink(struct xfs_trans *tp, struct xfs_inode *ip); int xfs_iunlink_remove(struct xfs_trans *tp, struct xfs_perag *pag, struct xfs_inode *ip); +struct xfs_inode *xfs_iunlink_lookup(struct xfs_perag *pag, xfs_agino_t agino); void xfs_end_io(struct work_struct *work); From 5b57257025f97c643d502253055651c81b0ffc66 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:55:01 -0700 Subject: [PATCH 0254/1702] xfs: hoist AGI repair context to a heap object Save ~460 bytes of stack space by moving all the repair context to a heap object. We're going to add even more context data in the next patch, which is why we really need to do this now. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/agheader_repair.c | 105 ++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 42 deletions(-) diff --git a/fs/xfs/scrub/agheader_repair.c b/fs/xfs/scrub/agheader_repair.c index 427054b65b238..d210bd7d5eb13 100644 --- a/fs/xfs/scrub/agheader_repair.c +++ b/fs/xfs/scrub/agheader_repair.c @@ -796,15 +796,29 @@ enum { XREP_AGI_MAX }; +struct xrep_agi { + struct xfs_scrub *sc; + + /* AGI buffer, tracked separately */ + struct xfs_buf *agi_bp; + + /* context for finding btree roots */ + struct xrep_find_ag_btree fab[XREP_AGI_MAX]; + + /* old AGI contents in case we have to revert */ + struct xfs_agi old_agi; +}; + /* * Given the inode btree roots described by *fab, find the roots, check them * for sanity, and pass the root data back out via *fab. */ STATIC int xrep_agi_find_btrees( - struct xfs_scrub *sc, - struct xrep_find_ag_btree *fab) + struct xrep_agi *ragi) { + struct xfs_scrub *sc = ragi->sc; + struct xrep_find_ag_btree *fab = ragi->fab; struct xfs_buf *agf_bp; struct xfs_mount *mp = sc->mp; int error; @@ -837,10 +851,11 @@ xrep_agi_find_btrees( */ STATIC void xrep_agi_init_header( - struct xfs_scrub *sc, - struct xfs_buf *agi_bp, - struct xfs_agi *old_agi) + struct xrep_agi *ragi) { + struct xfs_scrub *sc = ragi->sc; + struct xfs_buf *agi_bp = ragi->agi_bp; + struct xfs_agi *old_agi = &ragi->old_agi; struct xfs_agi *agi = agi_bp->b_addr; struct xfs_perag *pag = sc->sa.pag; struct xfs_mount *mp = sc->mp; @@ -868,10 +883,12 @@ xrep_agi_init_header( /* Set btree root information in an AGI. */ STATIC void xrep_agi_set_roots( - struct xfs_scrub *sc, - struct xfs_agi *agi, - struct xrep_find_ag_btree *fab) + struct xrep_agi *ragi) { + struct xfs_scrub *sc = ragi->sc; + struct xfs_agi *agi = ragi->agi_bp->b_addr; + struct xrep_find_ag_btree *fab = ragi->fab; + agi->agi_root = cpu_to_be32(fab[XREP_AGI_INOBT].root); agi->agi_level = cpu_to_be32(fab[XREP_AGI_INOBT].height); @@ -884,9 +901,10 @@ xrep_agi_set_roots( /* Update the AGI counters. */ STATIC int xrep_agi_calc_from_btrees( - struct xfs_scrub *sc, - struct xfs_buf *agi_bp) + struct xrep_agi *ragi) { + struct xfs_scrub *sc = ragi->sc; + struct xfs_buf *agi_bp = ragi->agi_bp; struct xfs_btree_cur *cur; struct xfs_agi *agi = agi_bp->b_addr; struct xfs_mount *mp = sc->mp; @@ -931,9 +949,10 @@ xrep_agi_calc_from_btrees( /* Trigger reinitialization of the in-core data. */ STATIC int xrep_agi_commit_new( - struct xfs_scrub *sc, - struct xfs_buf *agi_bp) + struct xrep_agi *ragi) { + struct xfs_scrub *sc = ragi->sc; + struct xfs_buf *agi_bp = ragi->agi_bp; struct xfs_perag *pag; struct xfs_agi *agi = agi_bp->b_addr; @@ -956,33 +975,36 @@ xrep_agi_commit_new( /* Repair the AGI. */ int xrep_agi( - struct xfs_scrub *sc) + struct xfs_scrub *sc) { - struct xrep_find_ag_btree fab[XREP_AGI_MAX] = { - [XREP_AGI_INOBT] = { - .rmap_owner = XFS_RMAP_OWN_INOBT, - .buf_ops = &xfs_inobt_buf_ops, - .maxlevels = M_IGEO(sc->mp)->inobt_maxlevels, - }, - [XREP_AGI_FINOBT] = { - .rmap_owner = XFS_RMAP_OWN_INOBT, - .buf_ops = &xfs_finobt_buf_ops, - .maxlevels = M_IGEO(sc->mp)->inobt_maxlevels, - }, - [XREP_AGI_END] = { - .buf_ops = NULL - }, - }; - struct xfs_agi old_agi; - struct xfs_mount *mp = sc->mp; - struct xfs_buf *agi_bp; - struct xfs_agi *agi; - int error; + struct xrep_agi *ragi; + struct xfs_mount *mp = sc->mp; + int error; /* We require the rmapbt to rebuild anything. */ if (!xfs_has_rmapbt(mp)) return -EOPNOTSUPP; + sc->buf = kzalloc(sizeof(struct xrep_agi), XCHK_GFP_FLAGS); + if (!sc->buf) + return -ENOMEM; + ragi = sc->buf; + ragi->sc = sc; + + ragi->fab[XREP_AGI_INOBT] = (struct xrep_find_ag_btree){ + .rmap_owner = XFS_RMAP_OWN_INOBT, + .buf_ops = &xfs_inobt_buf_ops, + .maxlevels = M_IGEO(sc->mp)->inobt_maxlevels, + }; + ragi->fab[XREP_AGI_FINOBT] = (struct xrep_find_ag_btree){ + .rmap_owner = XFS_RMAP_OWN_INOBT, + .buf_ops = &xfs_finobt_buf_ops, + .maxlevels = M_IGEO(sc->mp)->inobt_maxlevels, + }; + ragi->fab[XREP_AGI_END] = (struct xrep_find_ag_btree){ + .buf_ops = NULL, + }; + /* * Make sure we have the AGI buffer, as scrub might have decided it * was corrupt after xfs_ialloc_read_agi failed with -EFSCORRUPTED. @@ -990,14 +1012,13 @@ xrep_agi( error = xfs_trans_read_buf(mp, sc->tp, mp->m_ddev_targp, XFS_AG_DADDR(mp, sc->sa.pag->pag_agno, XFS_AGI_DADDR(mp)), - XFS_FSS_TO_BB(mp, 1), 0, &agi_bp, NULL); + XFS_FSS_TO_BB(mp, 1), 0, &ragi->agi_bp, NULL); if (error) return error; - agi_bp->b_ops = &xfs_agi_buf_ops; - agi = agi_bp->b_addr; + ragi->agi_bp->b_ops = &xfs_agi_buf_ops; /* Find the AGI btree roots. */ - error = xrep_agi_find_btrees(sc, fab); + error = xrep_agi_find_btrees(ragi); if (error) return error; @@ -1006,18 +1027,18 @@ xrep_agi( return error; /* Start rewriting the header and implant the btrees we found. */ - xrep_agi_init_header(sc, agi_bp, &old_agi); - xrep_agi_set_roots(sc, agi, fab); - error = xrep_agi_calc_from_btrees(sc, agi_bp); + xrep_agi_init_header(ragi); + xrep_agi_set_roots(ragi); + error = xrep_agi_calc_from_btrees(ragi); if (error) goto out_revert; /* Reinitialize in-core state. */ - return xrep_agi_commit_new(sc, agi_bp); + return xrep_agi_commit_new(ragi); out_revert: /* Mark the incore AGI state stale and revert the AGI. */ clear_bit(XFS_AGSTATE_AGI_INIT, &sc->sa.pag->pag_opstate); - memcpy(agi, &old_agi, sizeof(old_agi)); + memcpy(ragi->agi_bp->b_addr, &ragi->old_agi, sizeof(struct xfs_agi)); return error; } From 2651923d8d8db00a57665822f017fa7c76758044 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:54:59 -0700 Subject: [PATCH 0255/1702] xfs: online repair of symbolic links If a symbolic link target looks bad, try to sift through the rubble to find as much of the target buffer that we can, and stage a new target (short or remote format as needed) in a temporary file and use the atomic extent swapping mechanism to commit the results. In the worst case, we replace the target with an overly long filename that cannot possibly resolve. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/scrub/repair.h | 8 + fs/xfs/scrub/scrub.c | 2 +- fs/xfs/scrub/symlink.c | 13 +- fs/xfs/scrub/symlink_repair.c | 506 ++++++++++++++++++++++++++++++++++ fs/xfs/scrub/tempfile.c | 13 + fs/xfs/scrub/trace.h | 46 ++++ 7 files changed, 587 insertions(+), 2 deletions(-) create mode 100644 fs/xfs/scrub/symlink_repair.c diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index 1e23d1b3cd7b1..4e1eb3b6dbc45 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -213,6 +213,7 @@ xfs-y += $(addprefix scrub/, \ refcount_repair.o \ repair.o \ rmap_repair.o \ + symlink_repair.o \ tempfile.o \ xfblob.o \ ) diff --git a/fs/xfs/scrub/repair.h b/fs/xfs/scrub/repair.h index 7e6aba7fe5586..622eb486a16fb 100644 --- a/fs/xfs/scrub/repair.h +++ b/fs/xfs/scrub/repair.h @@ -94,6 +94,7 @@ int xrep_setup_xattr(struct xfs_scrub *sc); int xrep_setup_directory(struct xfs_scrub *sc); int xrep_setup_parent(struct xfs_scrub *sc); int xrep_setup_nlinks(struct xfs_scrub *sc); +int xrep_setup_symlink(struct xfs_scrub *sc, unsigned int *resblks); /* Repair setup functions */ int xrep_setup_ag_allocbt(struct xfs_scrub *sc); @@ -130,6 +131,7 @@ int xrep_fscounters(struct xfs_scrub *sc); int xrep_xattr(struct xfs_scrub *sc); int xrep_directory(struct xfs_scrub *sc); int xrep_parent(struct xfs_scrub *sc); +int xrep_symlink(struct xfs_scrub *sc); #ifdef CONFIG_XFS_RT int xrep_rtbitmap(struct xfs_scrub *sc); @@ -206,6 +208,11 @@ xrep_setup_nothing( #define xrep_setup_inode(sc, imap) ((void)0) +static inline int xrep_setup_symlink(struct xfs_scrub *sc, unsigned int *x) +{ + return 0; +} + #define xrep_revalidate_allocbt (NULL) #define xrep_revalidate_iallocbt (NULL) @@ -231,6 +238,7 @@ xrep_setup_nothing( #define xrep_xattr xrep_notsupported #define xrep_directory xrep_notsupported #define xrep_parent xrep_notsupported +#define xrep_symlink xrep_notsupported #endif /* CONFIG_XFS_ONLINE_REPAIR */ diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 6417628ce26be..301d5b753fdd5 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -339,7 +339,7 @@ static const struct xchk_meta_ops meta_scrub_ops[] = { .type = ST_INODE, .setup = xchk_setup_symlink, .scrub = xchk_symlink, - .repair = xrep_notsupported, + .repair = xrep_symlink, }, [XFS_SCRUB_TYPE_PARENT] = { /* parent pointers */ .type = ST_INODE, diff --git a/fs/xfs/scrub/symlink.c b/fs/xfs/scrub/symlink.c index d77d8a9598f63..c848bcc07cd5b 100644 --- a/fs/xfs/scrub/symlink.c +++ b/fs/xfs/scrub/symlink.c @@ -10,6 +10,7 @@ #include "xfs_trans_resv.h" #include "xfs_mount.h" #include "xfs_log_format.h" +#include "xfs_trans.h" #include "xfs_inode.h" #include "xfs_symlink.h" #include "xfs_health.h" @@ -17,18 +18,28 @@ #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/health.h" +#include "scrub/repair.h" /* Set us up to scrub a symbolic link. */ int xchk_setup_symlink( struct xfs_scrub *sc) { + unsigned int resblks = 0; + int error; + /* Allocate the buffer without the inode lock held. */ sc->buf = kvzalloc(XFS_SYMLINK_MAXLEN + 1, XCHK_GFP_FLAGS); if (!sc->buf) return -ENOMEM; - return xchk_setup_inode_contents(sc, 0); + if (xchk_could_repair(sc)) { + error = xrep_setup_symlink(sc, &resblks); + if (error) + return error; + } + + return xchk_setup_inode_contents(sc, resblks); } /* Symbolic links. */ diff --git a/fs/xfs/scrub/symlink_repair.c b/fs/xfs/scrub/symlink_repair.c new file mode 100644 index 0000000000000..178304959535a --- /dev/null +++ b/fs/xfs/scrub/symlink_repair.c @@ -0,0 +1,506 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_inode_fork.h" +#include "xfs_symlink.h" +#include "xfs_bmap.h" +#include "xfs_quota.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "xfs_symlink_remote.h" +#include "xfs_exchmaps.h" +#include "xfs_exchrange.h" +#include "xfs_health.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/tempfile.h" +#include "scrub/tempexch.h" +#include "scrub/reap.h" + +/* + * Symbolic Link Repair + * ==================== + * + * We repair symbolic links by reading whatever target data we can find, up to + * the first NULL byte. If the recovered target strlen matches i_size, then + * we rewrite the target. In all other cases, we replace the target with an + * overly long string that cannot possibly resolve. The new target is written + * into a private hidden temporary file, and then a file contents exchange + * commits the new symlink target to the file being repaired. + */ + +/* Set us up to repair the symlink file. */ +int +xrep_setup_symlink( + struct xfs_scrub *sc, + unsigned int *resblks) +{ + struct xfs_mount *mp = sc->mp; + unsigned long long blocks; + int error; + + error = xrep_tempfile_create(sc, S_IFLNK); + if (error) + return error; + + /* + * If we're doing a repair, we reserve enough blocks to write out a + * completely new symlink file, plus twice as many blocks as we would + * need if we can only allocate one block per data fork mapping. This + * should cover the preallocation of the temporary file and exchanging + * the extent mappings. + * + * We cannot use xfs_exchmaps_estimate because we have not yet + * constructed the replacement symlink and therefore do not know how + * many extents it will use. By the time we do, we will have a dirty + * transaction (which we cannot drop because we cannot drop the + * symlink ILOCK) and cannot ask for more reservation. + */ + blocks = xfs_symlink_blocks(sc->mp, XFS_SYMLINK_MAXLEN); + blocks += xfs_bmbt_calc_size(mp, blocks) * 2; + if (blocks > UINT_MAX) + return -EOPNOTSUPP; + + *resblks += blocks; + return 0; +} + +/* + * Try to salvage the pathname from remote blocks. Returns the number of bytes + * salvaged or a negative errno. + */ +STATIC ssize_t +xrep_symlink_salvage_remote( + struct xfs_scrub *sc) +{ + struct xfs_bmbt_irec mval[XFS_SYMLINK_MAPS]; + struct xfs_inode *ip = sc->ip; + struct xfs_buf *bp; + char *target_buf = sc->buf; + xfs_failaddr_t fa; + xfs_filblks_t fsblocks; + xfs_daddr_t d; + loff_t len; + loff_t offset = 0; + unsigned int byte_cnt; + bool magic_ok; + bool hdr_ok; + int n; + int nmaps = XFS_SYMLINK_MAPS; + int error; + + /* We'll only read until the buffer is full. */ + len = min_t(loff_t, ip->i_disk_size, XFS_SYMLINK_MAXLEN); + fsblocks = xfs_symlink_blocks(sc->mp, len); + error = xfs_bmapi_read(ip, 0, fsblocks, mval, &nmaps, 0); + if (error) + return error; + + for (n = 0; n < nmaps; n++) { + struct xfs_dsymlink_hdr *dsl; + + d = XFS_FSB_TO_DADDR(sc->mp, mval[n].br_startblock); + + /* Read the rmt block. We'll run the verifiers manually. */ + error = xfs_trans_read_buf(sc->mp, sc->tp, sc->mp->m_ddev_targp, + d, XFS_FSB_TO_BB(sc->mp, mval[n].br_blockcount), + 0, &bp, NULL); + if (error) + return error; + bp->b_ops = &xfs_symlink_buf_ops; + + /* How many bytes do we expect to get out of this buffer? */ + byte_cnt = XFS_FSB_TO_B(sc->mp, mval[n].br_blockcount); + byte_cnt = XFS_SYMLINK_BUF_SPACE(sc->mp, byte_cnt); + byte_cnt = min_t(unsigned int, byte_cnt, len); + + /* + * See if the verifiers accept this block. We're willing to + * salvage if the if the offset/byte/ino are ok and either the + * verifier passed or the magic is ok. Anything else and we + * stop dead in our tracks. + */ + fa = bp->b_ops->verify_struct(bp); + dsl = bp->b_addr; + magic_ok = dsl->sl_magic == cpu_to_be32(XFS_SYMLINK_MAGIC); + hdr_ok = xfs_symlink_hdr_ok(ip->i_ino, offset, byte_cnt, bp); + if (!hdr_ok || (fa != NULL && !magic_ok)) + break; + + memcpy(target_buf + offset, dsl + 1, byte_cnt); + + len -= byte_cnt; + offset += byte_cnt; + } + return offset; +} + +/* + * Try to salvage an inline symlink's contents. Returns the number of bytes + * salvaged or a negative errno. + */ +STATIC ssize_t +xrep_symlink_salvage_inline( + struct xfs_scrub *sc) +{ + struct xfs_inode *ip = sc->ip; + char *target_buf = sc->buf; + char *old_target; + struct xfs_ifork *ifp; + unsigned int nr; + + ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + if (!ifp->if_data) + return 0; + + /* + * If inode repair zapped the link target, pretend that we didn't find + * any bytes at all so that we can replace the (now totally lost) link + * target with a warning message. + */ + old_target = ifp->if_data; + if (xfs_inode_has_sickness(sc->ip, XFS_SICK_INO_SYMLINK_ZAPPED) && + sc->ip->i_disk_size == 1 && old_target[0] == '?') + return 0; + + nr = min(XFS_SYMLINK_MAXLEN, xfs_inode_data_fork_size(ip)); + strncpy(target_buf, ifp->if_data, nr); + return nr; +} + +#define DUMMY_TARGET \ + "The target of this symbolic link could not be recovered at all and " \ + "has been replaced with this explanatory message. To avoid " \ + "accidentally pointing to an existing file path, this message is " \ + "longer than the maximum supported file name length. That is an " \ + "acceptable length for a symlink target on XFS but will produce " \ + "File Name Too Long errors if resolved." + +/* Salvage whatever we can of the target. */ +STATIC int +xrep_symlink_salvage( + struct xfs_scrub *sc) +{ + char *target_buf = sc->buf; + ssize_t buflen = 0; + + BUILD_BUG_ON(sizeof(DUMMY_TARGET) - 1 <= NAME_MAX); + + /* + * Salvage the target if there weren't any corruption problems observed + * while scanning it. + */ + if (!(sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) { + if (sc->ip->i_df.if_format == XFS_DINODE_FMT_LOCAL) + buflen = xrep_symlink_salvage_inline(sc); + else + buflen = xrep_symlink_salvage_remote(sc); + if (buflen < 0) + return buflen; + + /* + * NULL-terminate the buffer because the ondisk target does not + * do that for us. If salvage didn't find the exact amount of + * data that we expected to find, don't salvage anything. + */ + target_buf[buflen] = 0; + if (strlen(target_buf) != sc->ip->i_disk_size) + buflen = 0; + } + + /* + * Change an empty target into a dummy target and clear the symlink + * target zapped flag. + */ + if (buflen == 0) { + sc->sick_mask |= XFS_SICK_INO_SYMLINK_ZAPPED; + sprintf(target_buf, DUMMY_TARGET); + } + + trace_xrep_symlink_salvage_target(sc->ip, target_buf, + strlen(target_buf)); + return 0; +} + +STATIC void +xrep_symlink_local_to_remote( + struct xfs_trans *tp, + struct xfs_buf *bp, + struct xfs_inode *ip, + struct xfs_ifork *ifp, + void *priv) +{ + struct xfs_scrub *sc = priv; + struct xfs_dsymlink_hdr *dsl = bp->b_addr; + + xfs_symlink_local_to_remote(tp, bp, ip, ifp, NULL); + + if (!xfs_has_crc(sc->mp)) + return; + + dsl->sl_owner = cpu_to_be64(sc->ip->i_ino); + xfs_trans_log_buf(tp, bp, 0, + sizeof(struct xfs_dsymlink_hdr) + ifp->if_bytes - 1); +} + +/* + * Prepare both links' data forks for an exchange. Promote the tempfile from + * local format to extents format, and if the file being repaired has a short + * format data fork, turn it into an empty extent list. + */ +STATIC int +xrep_symlink_swap_prep( + struct xfs_scrub *sc, + bool temp_local, + bool ip_local) +{ + int error; + + /* + * If the temp link is in shortform format, convert that to a remote + * target so that we can use the atomic mapping exchange. + */ + if (temp_local) { + int logflags = XFS_ILOG_CORE; + + error = xfs_bmap_local_to_extents(sc->tp, sc->tempip, 1, + &logflags, XFS_DATA_FORK, + xrep_symlink_local_to_remote, + sc); + if (error) + return error; + + xfs_trans_log_inode(sc->tp, sc->ip, 0); + + error = xfs_defer_finish(&sc->tp); + if (error) + return error; + } + + /* + * If the file being repaired had a shortform data fork, convert that + * to an empty extent list in preparation for the atomic mapping + * exchange. + */ + if (ip_local) { + struct xfs_ifork *ifp; + + ifp = xfs_ifork_ptr(sc->ip, XFS_DATA_FORK); + xfs_idestroy_fork(ifp); + ifp->if_format = XFS_DINODE_FMT_EXTENTS; + ifp->if_nextents = 0; + ifp->if_bytes = 0; + ifp->if_data = NULL; + ifp->if_height = 0; + + xfs_trans_log_inode(sc->tp, sc->ip, + XFS_ILOG_CORE | XFS_ILOG_DDATA); + } + + return 0; +} + +/* Exchange the temporary symlink's data fork with the one being repaired. */ +STATIC int +xrep_symlink_swap( + struct xfs_scrub *sc) +{ + struct xrep_tempexch *tx = sc->buf; + bool ip_local, temp_local; + int error; + + ip_local = sc->ip->i_df.if_format == XFS_DINODE_FMT_LOCAL; + temp_local = sc->tempip->i_df.if_format == XFS_DINODE_FMT_LOCAL; + + /* + * If the both links have a local format data fork and the rebuilt + * remote data would fit in the repaired file's data fork, copy the + * contents from the tempfile and declare ourselves done. + */ + if (ip_local && temp_local && + sc->tempip->i_disk_size <= xfs_inode_data_fork_size(sc->ip)) { + xrep_tempfile_copyout_local(sc, XFS_DATA_FORK); + return 0; + } + + /* Otherwise, make sure both data forks are in block-mapping mode. */ + error = xrep_symlink_swap_prep(sc, temp_local, ip_local); + if (error) + return error; + + return xrep_tempexch_contents(sc, tx); +} + +/* + * Free all the remote blocks and reset the data fork. The caller must join + * the inode to the transaction. This function returns with the inode joined + * to a clean scrub transaction. + */ +STATIC int +xrep_symlink_reset_fork( + struct xfs_scrub *sc) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(sc->tempip, XFS_DATA_FORK); + int error; + + /* Unmap all the remote target buffers. */ + if (xfs_ifork_has_extents(ifp)) { + error = xrep_reap_ifork(sc, sc->tempip, XFS_DATA_FORK); + if (error) + return error; + } + + trace_xrep_symlink_reset_fork(sc->tempip); + + /* Reset the temp symlink target to dummy content. */ + xfs_idestroy_fork(ifp); + return xfs_symlink_write_target(sc->tp, sc->tempip, sc->tempip->i_ino, + "?", 1, 0, 0); +} + +/* + * Reinitialize a link target. Caller must ensure the inode is joined to + * the transaction. + */ +STATIC int +xrep_symlink_rebuild( + struct xfs_scrub *sc) +{ + struct xrep_tempexch *tx; + char *target_buf = sc->buf; + xfs_fsblock_t fs_blocks; + unsigned int target_len; + unsigned int resblks; + int error; + + /* How many blocks do we need? */ + target_len = strlen(target_buf); + ASSERT(target_len != 0); + if (target_len == 0 || target_len > XFS_SYMLINK_MAXLEN) + return -EFSCORRUPTED; + + trace_xrep_symlink_rebuild(sc->ip); + + /* + * In preparation to write the new symlink target to the temporary + * file, drop the ILOCK of the file being repaired (it shouldn't be + * joined) and take the ILOCK of the temporary file. + * + * The VFS does not take the IOLOCK while reading a symlink (and new + * symlinks are hidden with INEW until they've been written) so it's + * possible that a readlink() could see the old corrupted contents + * while we're doing this. + */ + xchk_iunlock(sc, XFS_ILOCK_EXCL); + xrep_tempfile_ilock(sc); + xfs_trans_ijoin(sc->tp, sc->tempip, 0); + + /* + * Reserve resources to reinitialize the target. We're allowed to + * exceed file quota to repair inconsistent metadata, though this is + * unlikely. + */ + fs_blocks = xfs_symlink_blocks(sc->mp, target_len); + resblks = XFS_SYMLINK_SPACE_RES(sc->mp, target_len, fs_blocks); + error = xfs_trans_reserve_quota_nblks(sc->tp, sc->tempip, resblks, 0, + true); + if (error) + return error; + + /* Erase the dummy target set up by the tempfile initialization. */ + xfs_idestroy_fork(&sc->tempip->i_df); + sc->tempip->i_df.if_bytes = 0; + sc->tempip->i_df.if_format = XFS_DINODE_FMT_EXTENTS; + + /* Write the salvaged target to the temporary link. */ + error = xfs_symlink_write_target(sc->tp, sc->tempip, sc->ip->i_ino, + target_buf, target_len, fs_blocks, resblks); + if (error) + return error; + + /* + * Commit the repair transaction so that we can use the atomic mapping + * exchange functions to compute the correct block reservations and + * re-lock the inodes. + */ + target_buf = NULL; + error = xrep_trans_commit(sc); + if (error) + return error; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + return error; + + xrep_tempfile_iunlock(sc); + + /* + * We're done with the temporary buffer, so we can reuse it for the + * tempfile contents exchange information. + */ + tx = sc->buf; + error = xrep_tempexch_trans_alloc(sc, XFS_DATA_FORK, tx); + if (error) + return error; + + /* + * Exchange the temp link's data fork with the file being repaired. + * This recreates the transaction and takes the ILOCKs of the file + * being repaired and the temporary file. + */ + error = xrep_symlink_swap(sc); + if (error) + return error; + + /* + * Release the old symlink blocks and reset the data fork of the temp + * link to an empty shortform link. This is the last repair action we + * perform on the symlink, so we don't need to clean the transaction. + */ + return xrep_symlink_reset_fork(sc); +} + +/* Repair a symbolic link. */ +int +xrep_symlink( + struct xfs_scrub *sc) +{ + int error; + + /* The rmapbt is required to reap the old data fork. */ + if (!xfs_has_rmapbt(sc->mp)) + return -EOPNOTSUPP; + + ASSERT(sc->ilock_flags & XFS_ILOCK_EXCL); + + error = xrep_symlink_salvage(sc); + if (error) + return error; + + /* Now reset the target. */ + error = xrep_symlink_rebuild(sc); + if (error) + return error; + + return xrep_trans_commit(sc); +} diff --git a/fs/xfs/scrub/tempfile.c b/fs/xfs/scrub/tempfile.c index 4ca86a6a5be10..c72e447eb8ec3 100644 --- a/fs/xfs/scrub/tempfile.c +++ b/fs/xfs/scrub/tempfile.c @@ -21,6 +21,7 @@ #include "xfs_exchrange.h" #include "xfs_exchmaps.h" #include "xfs_defer.h" +#include "xfs_symlink_remote.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/repair.h" @@ -109,6 +110,18 @@ xrep_tempfile_create( error = xfs_dir_init(tp, sc->tempip, dp); if (error) goto out_trans_cancel; + } else if (S_ISLNK(VFS_I(sc->tempip)->i_mode)) { + /* + * Initialize the temporary symlink with a meaningless target + * that won't trip the verifiers. Repair must rewrite the + * target with meaningful content before swapping with the file + * being repaired. A single-byte target will not write a + * remote target block, so the owner is irrelevant. + */ + error = xfs_symlink_write_target(tp, sc->tempip, + sc->tempip->i_ino, ".", 1, 0, 0); + if (error) + goto out_trans_cancel; } /* diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 668da6ff2ca26..03cb095fc1a12 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2711,6 +2711,52 @@ DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_check_alias); DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_check_dentry); DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_invalidate_child); +TRACE_EVENT(xrep_symlink_salvage_target, + TP_PROTO(struct xfs_inode *ip, char *target, unsigned int targetlen), + TP_ARGS(ip, target, targetlen), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned int, targetlen) + __dynamic_array(char, target, targetlen + 1) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = ip->i_ino; + __entry->targetlen = targetlen; + memcpy(__get_str(target), target, targetlen); + __get_str(target)[targetlen] = 0; + ), + TP_printk("dev %d:%d ip 0x%llx target '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->targetlen, + __get_str(target)) +); + +DECLARE_EVENT_CLASS(xrep_symlink_class, + TP_PROTO(struct xfs_inode *ip), + TP_ARGS(ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = ip->i_ino; + ), + TP_printk("dev %d:%d ip 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino) +); + +#define DEFINE_XREP_SYMLINK_EVENT(name) \ +DEFINE_EVENT(xrep_symlink_class, name, \ + TP_PROTO(struct xfs_inode *ip), \ + TP_ARGS(ip)) +DEFINE_XREP_SYMLINK_EVENT(xrep_symlink_rebuild); +DEFINE_XREP_SYMLINK_EVENT(xrep_symlink_reset_fork); + #endif /* IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) */ #endif /* _TRACE_XFS_SCRUB_TRACE_H */ From ab97f4b1c030750f2475bf4da8a9554d02206640 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:55:02 -0700 Subject: [PATCH 0256/1702] xfs: repair AGI unlinked inode bucket lists Teach the AGI repair code to rebuild the unlinked buckets and lists. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/agheader_repair.c | 774 ++++++++++++++++++++++++++++++++- fs/xfs/scrub/agino_bitmap.h | 49 +++ fs/xfs/scrub/trace.h | 255 +++++++++++ 3 files changed, 1074 insertions(+), 4 deletions(-) create mode 100644 fs/xfs/scrub/agino_bitmap.h diff --git a/fs/xfs/scrub/agheader_repair.c b/fs/xfs/scrub/agheader_repair.c index d210bd7d5eb13..0dbc484b182f9 100644 --- a/fs/xfs/scrub/agheader_repair.c +++ b/fs/xfs/scrub/agheader_repair.c @@ -21,13 +21,18 @@ #include "xfs_rmap_btree.h" #include "xfs_refcount_btree.h" #include "xfs_ag.h" +#include "xfs_inode.h" +#include "xfs_iunlink_item.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/trace.h" #include "scrub/repair.h" #include "scrub/bitmap.h" #include "scrub/agb_bitmap.h" +#include "scrub/agino_bitmap.h" #include "scrub/reap.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" /* Superblock */ @@ -796,6 +801,8 @@ enum { XREP_AGI_MAX }; +#define XREP_AGI_LOOKUP_BATCH 32 + struct xrep_agi { struct xfs_scrub *sc; @@ -807,8 +814,34 @@ struct xrep_agi { /* old AGI contents in case we have to revert */ struct xfs_agi old_agi; + + /* bitmap of which inodes are unlinked */ + struct xagino_bitmap iunlink_bmp; + + /* heads of the unlinked inode bucket lists */ + xfs_agino_t iunlink_heads[XFS_AGI_UNLINKED_BUCKETS]; + + /* scratchpad for batched lookups of the radix tree */ + struct xfs_inode *lookup_batch[XREP_AGI_LOOKUP_BATCH]; + + /* Map of ino -> next_ino for unlinked inode processing. */ + struct xfarray *iunlink_next; + + /* Map of ino -> prev_ino for unlinked inode processing. */ + struct xfarray *iunlink_prev; }; +static void +xrep_agi_buf_cleanup( + void *buf) +{ + struct xrep_agi *ragi = buf; + + xfarray_destroy(ragi->iunlink_prev); + xfarray_destroy(ragi->iunlink_next); + xagino_bitmap_destroy(&ragi->iunlink_bmp); +} + /* * Given the inode btree roots described by *fab, find the roots, check them * for sanity, and pass the root data back out via *fab. @@ -871,10 +904,6 @@ xrep_agi_init_header( if (xfs_has_crc(mp)) uuid_copy(&agi->agi_uuid, &mp->m_sb.sb_meta_uuid); - /* We don't know how to fix the unlinked list yet. */ - memcpy(&agi->agi_unlinked, &old_agi->agi_unlinked, - sizeof(agi->agi_unlinked)); - /* Mark the incore AGF data stale until we're done fixing things. */ ASSERT(xfs_perag_initialised_agi(pag)); clear_bit(XFS_AGSTATE_AGI_INIT, &pag->pag_opstate); @@ -946,6 +975,714 @@ xrep_agi_calc_from_btrees( return error; } +/* + * Record a forwards unlinked chain pointer from agino -> next_agino in our + * staging information. + */ +static inline int +xrep_iunlink_store_next( + struct xrep_agi *ragi, + xfs_agino_t agino, + xfs_agino_t next_agino) +{ + ASSERT(next_agino != 0); + + return xfarray_store(ragi->iunlink_next, agino, &next_agino); +} + +/* + * Record a backwards unlinked chain pointer from prev_ino <- agino in our + * staging information. + */ +static inline int +xrep_iunlink_store_prev( + struct xrep_agi *ragi, + xfs_agino_t agino, + xfs_agino_t prev_agino) +{ + ASSERT(prev_agino != 0); + + return xfarray_store(ragi->iunlink_prev, agino, &prev_agino); +} + +/* + * Given an @agino, look up the next inode in the iunlink bucket. Returns + * NULLAGINO if we're at the end of the chain, 0 if @agino is not in memory + * like it should be, or a per-AG inode number. + */ +static inline xfs_agino_t +xrep_iunlink_next( + struct xfs_scrub *sc, + xfs_agino_t agino) +{ + struct xfs_inode *ip; + + ip = xfs_iunlink_lookup(sc->sa.pag, agino); + if (!ip) + return 0; + + return ip->i_next_unlinked; +} + +/* + * Load the inode @agino into memory, set its i_prev_unlinked, and drop the + * inode so it can be inactivated. Returns NULLAGINO if we're at the end of + * the chain or if we should stop walking the chain due to corruption; or a + * per-AG inode number. + */ +STATIC xfs_agino_t +xrep_iunlink_reload_next( + struct xrep_agi *ragi, + xfs_agino_t prev_agino, + xfs_agino_t agino) +{ + struct xfs_scrub *sc = ragi->sc; + struct xfs_inode *ip; + xfs_ino_t ino; + xfs_agino_t ret = NULLAGINO; + int error; + + ino = XFS_AGINO_TO_INO(sc->mp, sc->sa.pag->pag_agno, agino); + error = xchk_iget(ragi->sc, ino, &ip); + if (error) + return ret; + + trace_xrep_iunlink_reload_next(ip, prev_agino); + + /* If this is a linked inode, stop processing the chain. */ + if (VFS_I(ip)->i_nlink != 0) { + xrep_iunlink_store_next(ragi, agino, NULLAGINO); + goto rele; + } + + ip->i_prev_unlinked = prev_agino; + ret = ip->i_next_unlinked; + + /* + * Drop the inode reference that we just took. We hold the AGI, so + * this inode cannot move off the unlinked list and hence cannot be + * reclaimed. + */ +rele: + xchk_irele(sc, ip); + return ret; +} + +/* + * Walk an AGI unlinked bucket's list to load incore any unlinked inodes that + * still existed at mount time. This can happen if iunlink processing fails + * during log recovery. + */ +STATIC int +xrep_iunlink_walk_ondisk_bucket( + struct xrep_agi *ragi, + unsigned int bucket) +{ + struct xfs_scrub *sc = ragi->sc; + struct xfs_agi *agi = sc->sa.agi_bp->b_addr; + xfs_agino_t prev_agino = NULLAGINO; + xfs_agino_t next_agino; + int error = 0; + + next_agino = be32_to_cpu(agi->agi_unlinked[bucket]); + while (next_agino != NULLAGINO) { + xfs_agino_t agino = next_agino; + + if (xchk_should_terminate(ragi->sc, &error)) + return error; + + trace_xrep_iunlink_walk_ondisk_bucket(sc->sa.pag, bucket, + prev_agino, agino); + + if (bucket != agino % XFS_AGI_UNLINKED_BUCKETS) + break; + + next_agino = xrep_iunlink_next(sc, agino); + if (!next_agino) + next_agino = xrep_iunlink_reload_next(ragi, prev_agino, + agino); + + prev_agino = agino; + } + + return 0; +} + +/* Decide if this is an unlinked inode in this AG. */ +STATIC bool +xrep_iunlink_igrab( + struct xfs_perag *pag, + struct xfs_inode *ip) +{ + struct xfs_mount *mp = pag->pag_mount; + + if (XFS_INO_TO_AGNO(mp, ip->i_ino) != pag->pag_agno) + return false; + + if (!xfs_inode_on_unlinked_list(ip)) + return false; + + return true; +} + +/* + * Mark the given inode in the lookup batch in our unlinked inode bitmap, and + * remember if this inode is the start of the unlinked chain. + */ +STATIC int +xrep_iunlink_visit( + struct xrep_agi *ragi, + unsigned int batch_idx) +{ + struct xfs_mount *mp = ragi->sc->mp; + struct xfs_inode *ip = ragi->lookup_batch[batch_idx]; + xfs_agino_t agino; + unsigned int bucket; + int error; + + ASSERT(XFS_INO_TO_AGNO(mp, ip->i_ino) == ragi->sc->sa.pag->pag_agno); + ASSERT(xfs_inode_on_unlinked_list(ip)); + + agino = XFS_INO_TO_AGINO(mp, ip->i_ino); + bucket = agino % XFS_AGI_UNLINKED_BUCKETS; + + trace_xrep_iunlink_visit(ragi->sc->sa.pag, bucket, + ragi->iunlink_heads[bucket], ip); + + error = xagino_bitmap_set(&ragi->iunlink_bmp, agino, 1); + if (error) + return error; + + if (ip->i_prev_unlinked == NULLAGINO) { + if (ragi->iunlink_heads[bucket] == NULLAGINO) + ragi->iunlink_heads[bucket] = agino; + } + + return 0; +} + +/* + * Find all incore unlinked inodes so that we can rebuild the unlinked buckets. + * We hold the AGI so there should not be any modifications to the unlinked + * list. + */ +STATIC int +xrep_iunlink_mark_incore( + struct xrep_agi *ragi) +{ + struct xfs_perag *pag = ragi->sc->sa.pag; + struct xfs_mount *mp = pag->pag_mount; + uint32_t first_index = 0; + bool done = false; + unsigned int nr_found = 0; + + do { + unsigned int i; + int error = 0; + + if (xchk_should_terminate(ragi->sc, &error)) + return error; + + rcu_read_lock(); + + nr_found = radix_tree_gang_lookup(&pag->pag_ici_root, + (void **)&ragi->lookup_batch, first_index, + XREP_AGI_LOOKUP_BATCH); + if (!nr_found) { + rcu_read_unlock(); + return 0; + } + + for (i = 0; i < nr_found; i++) { + struct xfs_inode *ip = ragi->lookup_batch[i]; + + if (done || !xrep_iunlink_igrab(pag, ip)) + ragi->lookup_batch[i] = NULL; + + /* + * Update the index for the next lookup. Catch + * overflows into the next AG range which can occur if + * we have inodes in the last block of the AG and we + * are currently pointing to the last inode. + * + * Because we may see inodes that are from the wrong AG + * due to RCU freeing and reallocation, only update the + * index if it lies in this AG. It was a race that lead + * us to see this inode, so another lookup from the + * same index will not find it again. + */ + if (XFS_INO_TO_AGNO(mp, ip->i_ino) != pag->pag_agno) + continue; + first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1); + if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino)) + done = true; + } + + /* unlock now we've grabbed the inodes. */ + rcu_read_unlock(); + + for (i = 0; i < nr_found; i++) { + if (!ragi->lookup_batch[i]) + continue; + error = xrep_iunlink_visit(ragi, i); + if (error) + return error; + } + } while (!done); + + return 0; +} + +/* Mark all the unlinked ondisk inodes in this inobt record in iunlink_bmp. */ +STATIC int +xrep_iunlink_mark_ondisk_rec( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *rec, + void *priv) +{ + struct xfs_inobt_rec_incore irec; + struct xrep_agi *ragi = priv; + struct xfs_scrub *sc = ragi->sc; + struct xfs_mount *mp = cur->bc_mp; + xfs_agino_t agino; + unsigned int i; + int error = 0; + + xfs_inobt_btrec_to_irec(mp, rec, &irec); + + for (i = 0, agino = irec.ir_startino; + i < XFS_INODES_PER_CHUNK; + i++, agino++) { + struct xfs_inode *ip; + unsigned int len = 1; + + /* Skip free inodes */ + if (XFS_INOBT_MASK(i) & irec.ir_free) + continue; + /* Skip inodes we've seen before */ + if (xagino_bitmap_test(&ragi->iunlink_bmp, agino, &len)) + continue; + + /* + * Skip incore inodes; these were already picked up by + * the _mark_incore step. + */ + rcu_read_lock(); + ip = radix_tree_lookup(&sc->sa.pag->pag_ici_root, agino); + rcu_read_unlock(); + if (ip) + continue; + + /* + * Try to look up this inode. If we can't get it, just move + * on because we haven't actually scrubbed the inobt or the + * inodes yet. + */ + error = xchk_iget(ragi->sc, + XFS_AGINO_TO_INO(mp, sc->sa.pag->pag_agno, + agino), + &ip); + if (error) + continue; + + trace_xrep_iunlink_reload_ondisk(ip); + + if (VFS_I(ip)->i_nlink == 0) + error = xagino_bitmap_set(&ragi->iunlink_bmp, agino, 1); + xchk_irele(sc, ip); + if (error) + break; + } + + return error; +} + +/* + * Find ondisk inodes that are unlinked and not in cache, and mark them in + * iunlink_bmp. We haven't checked the inobt yet, so we don't error out if + * the btree is corrupt. + */ +STATIC void +xrep_iunlink_mark_ondisk( + struct xrep_agi *ragi) +{ + struct xfs_scrub *sc = ragi->sc; + struct xfs_buf *agi_bp = ragi->agi_bp; + struct xfs_btree_cur *cur; + int error; + + cur = xfs_inobt_init_cursor(sc->sa.pag, sc->tp, agi_bp); + error = xfs_btree_query_all(cur, xrep_iunlink_mark_ondisk_rec, ragi); + xfs_btree_del_cursor(cur, error); +} + +/* + * Walk an iunlink bucket's inode list. For each inode that should be on this + * chain, clear its entry in in iunlink_bmp because it's ok and we don't need + * to touch it further. + */ +STATIC int +xrep_iunlink_resolve_bucket( + struct xrep_agi *ragi, + unsigned int bucket) +{ + struct xfs_scrub *sc = ragi->sc; + struct xfs_inode *ip; + xfs_agino_t prev_agino = NULLAGINO; + xfs_agino_t next_agino = ragi->iunlink_heads[bucket]; + int error = 0; + + while (next_agino != NULLAGINO) { + if (xchk_should_terminate(ragi->sc, &error)) + return error; + + /* Find the next inode in the chain. */ + ip = xfs_iunlink_lookup(sc->sa.pag, next_agino); + if (!ip) { + /* Inode not incore? Terminate the chain. */ + trace_xrep_iunlink_resolve_uncached(sc->sa.pag, + bucket, prev_agino, next_agino); + + next_agino = NULLAGINO; + break; + } + + if (next_agino % XFS_AGI_UNLINKED_BUCKETS != bucket) { + /* + * Inode is in the wrong bucket. Advance the list, + * but pretend we didn't see this inode. + */ + trace_xrep_iunlink_resolve_wronglist(sc->sa.pag, + bucket, prev_agino, next_agino); + + next_agino = ip->i_next_unlinked; + continue; + } + + if (!xfs_inode_on_unlinked_list(ip)) { + /* + * Incore inode doesn't think this inode is on an + * unlinked list. This is probably because we reloaded + * it from disk. Advance the list, but pretend we + * didn't see this inode; we'll fix that later. + */ + trace_xrep_iunlink_resolve_nolist(sc->sa.pag, + bucket, prev_agino, next_agino); + next_agino = ip->i_next_unlinked; + continue; + } + + trace_xrep_iunlink_resolve_ok(sc->sa.pag, bucket, prev_agino, + next_agino); + + /* + * Otherwise, this inode's unlinked pointers are ok. Clear it + * from the unlinked bitmap since we're done with it, and make + * sure the chain is still correct. + */ + error = xagino_bitmap_clear(&ragi->iunlink_bmp, next_agino, 1); + if (error) + return error; + + /* Remember the previous inode's next pointer. */ + if (prev_agino != NULLAGINO) { + error = xrep_iunlink_store_next(ragi, prev_agino, + next_agino); + if (error) + return error; + } + + /* Remember this inode's previous pointer. */ + error = xrep_iunlink_store_prev(ragi, next_agino, prev_agino); + if (error) + return error; + + /* Advance the list and remember this inode. */ + prev_agino = next_agino; + next_agino = ip->i_next_unlinked; + } + + /* Update the previous inode's next pointer. */ + if (prev_agino != NULLAGINO) { + error = xrep_iunlink_store_next(ragi, prev_agino, next_agino); + if (error) + return error; + } + + return 0; +} + +/* Reinsert this unlinked inode into the head of the staged bucket list. */ +STATIC int +xrep_iunlink_add_to_bucket( + struct xrep_agi *ragi, + xfs_agino_t agino) +{ + xfs_agino_t current_head; + unsigned int bucket; + int error; + + bucket = agino % XFS_AGI_UNLINKED_BUCKETS; + + /* Point this inode at the current head of the bucket list. */ + current_head = ragi->iunlink_heads[bucket]; + + trace_xrep_iunlink_add_to_bucket(ragi->sc->sa.pag, bucket, agino, + current_head); + + error = xrep_iunlink_store_next(ragi, agino, current_head); + if (error) + return error; + + /* Remember the head inode's previous pointer. */ + if (current_head != NULLAGINO) { + error = xrep_iunlink_store_prev(ragi, current_head, agino); + if (error) + return error; + } + + ragi->iunlink_heads[bucket] = agino; + return 0; +} + +/* Reinsert unlinked inodes into the staged iunlink buckets. */ +STATIC int +xrep_iunlink_add_lost_inodes( + uint32_t start, + uint32_t len, + void *priv) +{ + struct xrep_agi *ragi = priv; + int error; + + for (; len > 0; start++, len--) { + error = xrep_iunlink_add_to_bucket(ragi, start); + if (error) + return error; + } + + return 0; +} + +/* + * Figure out the iunlink bucket values and find inodes that need to be + * reinserted into the list. + */ +STATIC int +xrep_iunlink_rebuild_buckets( + struct xrep_agi *ragi) +{ + unsigned int i; + int error; + + /* + * Walk the ondisk AGI unlinked list to find inodes that are on the + * list but aren't in memory. This can happen if a past log recovery + * tried to clear the iunlinked list but failed. Our scan rebuilds the + * unlinked list using incore inodes, so we must load and link them + * properly. + */ + for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) { + error = xrep_iunlink_walk_ondisk_bucket(ragi, i); + if (error) + return error; + } + + /* + * Record all the incore unlinked inodes in iunlink_bmp that we didn't + * find by walking the ondisk iunlink buckets. This shouldn't happen, + * but we can't risk forgetting an inode somewhere. + */ + error = xrep_iunlink_mark_incore(ragi); + if (error) + return error; + + /* + * If there are ondisk inodes that are unlinked and are not been loaded + * into cache, record them in iunlink_bmp. + */ + xrep_iunlink_mark_ondisk(ragi); + + /* + * Walk each iunlink bucket to (re)construct as much of the incore list + * as would be correct. For each inode that survives this step, mark + * it clear in iunlink_bmp; we're done with those inodes. + */ + for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) { + error = xrep_iunlink_resolve_bucket(ragi, i); + if (error) + return error; + } + + /* + * Any unlinked inodes that we didn't find through the bucket list + * walk (or was ignored by the walk) must be inserted into the bucket + * list. Stage this in memory for now. + */ + return xagino_bitmap_walk(&ragi->iunlink_bmp, + xrep_iunlink_add_lost_inodes, ragi); +} + +/* Update i_next_iunlinked for the inode @agino. */ +STATIC int +xrep_iunlink_relink_next( + struct xrep_agi *ragi, + xfarray_idx_t idx, + xfs_agino_t next_agino) +{ + struct xfs_scrub *sc = ragi->sc; + struct xfs_perag *pag = sc->sa.pag; + struct xfs_inode *ip; + xfarray_idx_t agino = idx - 1; + bool want_rele = false; + int error = 0; + + ip = xfs_iunlink_lookup(pag, agino); + if (!ip) { + xfs_ino_t ino; + xfs_agino_t prev_agino; + + /* + * No inode exists in cache. Load it off the disk so that we + * can reinsert it into the incore unlinked list. + */ + ino = XFS_AGINO_TO_INO(sc->mp, pag->pag_agno, agino); + error = xchk_iget(sc, ino, &ip); + if (error) + return -EFSCORRUPTED; + + want_rele = true; + + /* Set the backward pointer since this just came off disk. */ + error = xfarray_load(ragi->iunlink_prev, agino, &prev_agino); + if (error) + goto out_rele; + + trace_xrep_iunlink_relink_prev(ip, prev_agino); + ip->i_prev_unlinked = prev_agino; + } + + /* Update the forward pointer. */ + if (ip->i_next_unlinked != next_agino) { + error = xfs_iunlink_log_inode(sc->tp, ip, pag, next_agino); + if (error) + goto out_rele; + + trace_xrep_iunlink_relink_next(ip, next_agino); + ip->i_next_unlinked = next_agino; + } + +out_rele: + /* + * The iunlink lookup doesn't igrab because we hold the AGI buffer lock + * and the inode cannot be reclaimed. However, if we used iget to load + * a missing inode, we must irele it here. + */ + if (want_rele) + xchk_irele(sc, ip); + return error; +} + +/* Update i_prev_iunlinked for the inode @agino. */ +STATIC int +xrep_iunlink_relink_prev( + struct xrep_agi *ragi, + xfarray_idx_t idx, + xfs_agino_t prev_agino) +{ + struct xfs_scrub *sc = ragi->sc; + struct xfs_perag *pag = sc->sa.pag; + struct xfs_inode *ip; + xfarray_idx_t agino = idx - 1; + bool want_rele = false; + int error = 0; + + ASSERT(prev_agino != 0); + + ip = xfs_iunlink_lookup(pag, agino); + if (!ip) { + xfs_ino_t ino; + xfs_agino_t next_agino; + + /* + * No inode exists in cache. Load it off the disk so that we + * can reinsert it into the incore unlinked list. + */ + ino = XFS_AGINO_TO_INO(sc->mp, pag->pag_agno, agino); + error = xchk_iget(sc, ino, &ip); + if (error) + return -EFSCORRUPTED; + + want_rele = true; + + /* Set the forward pointer since this just came off disk. */ + error = xfarray_load(ragi->iunlink_prev, agino, &next_agino); + if (error) + goto out_rele; + + error = xfs_iunlink_log_inode(sc->tp, ip, pag, next_agino); + if (error) + goto out_rele; + + trace_xrep_iunlink_relink_next(ip, next_agino); + ip->i_next_unlinked = next_agino; + } + + /* Update the backward pointer. */ + if (ip->i_prev_unlinked != prev_agino) { + trace_xrep_iunlink_relink_prev(ip, prev_agino); + ip->i_prev_unlinked = prev_agino; + } + +out_rele: + /* + * The iunlink lookup doesn't igrab because we hold the AGI buffer lock + * and the inode cannot be reclaimed. However, if we used iget to load + * a missing inode, we must irele it here. + */ + if (want_rele) + xchk_irele(sc, ip); + return error; +} + +/* Log all the iunlink updates we need to finish regenerating the AGI. */ +STATIC int +xrep_iunlink_commit( + struct xrep_agi *ragi) +{ + struct xfs_agi *agi = ragi->agi_bp->b_addr; + xfarray_idx_t idx = XFARRAY_CURSOR_INIT; + xfs_agino_t agino; + unsigned int i; + int error; + + /* Fix all the forward links */ + while ((error = xfarray_iter(ragi->iunlink_next, &idx, &agino)) == 1) { + error = xrep_iunlink_relink_next(ragi, idx, agino); + if (error) + return error; + } + + /* Fix all the back links */ + idx = XFARRAY_CURSOR_INIT; + while ((error = xfarray_iter(ragi->iunlink_prev, &idx, &agino)) == 1) { + error = xrep_iunlink_relink_prev(ragi, idx, agino); + if (error) + return error; + } + + /* Copy the staged iunlink buckets to the new AGI. */ + for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) { + trace_xrep_iunlink_commit_bucket(ragi->sc->sa.pag, i, + be32_to_cpu(ragi->old_agi.agi_unlinked[i]), + ragi->iunlink_heads[i]); + + agi->agi_unlinked[i] = cpu_to_be32(ragi->iunlink_heads[i]); + } + + return 0; +} + /* Trigger reinitialization of the in-core data. */ STATIC int xrep_agi_commit_new( @@ -979,6 +1716,8 @@ xrep_agi( { struct xrep_agi *ragi; struct xfs_mount *mp = sc->mp; + char *descr; + unsigned int i; int error; /* We require the rmapbt to rebuild anything. */ @@ -1005,6 +1744,26 @@ xrep_agi( .buf_ops = NULL, }; + for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) + ragi->iunlink_heads[i] = NULLAGINO; + + xagino_bitmap_init(&ragi->iunlink_bmp); + sc->buf_cleanup = xrep_agi_buf_cleanup; + + descr = xchk_xfile_ag_descr(sc, "iunlinked next pointers"); + error = xfarray_create(descr, 0, sizeof(xfs_agino_t), + &ragi->iunlink_next); + kfree(descr); + if (error) + return error; + + descr = xchk_xfile_ag_descr(sc, "iunlinked prev pointers"); + error = xfarray_create(descr, 0, sizeof(xfs_agino_t), + &ragi->iunlink_prev); + kfree(descr); + if (error) + return error; + /* * Make sure we have the AGI buffer, as scrub might have decided it * was corrupt after xfs_ialloc_read_agi failed with -EFSCORRUPTED. @@ -1022,6 +1781,10 @@ xrep_agi( if (error) return error; + error = xrep_iunlink_rebuild_buckets(ragi); + if (error) + return error; + /* Last chance to abort before we start committing fixes. */ if (xchk_should_terminate(sc, &error)) return error; @@ -1030,6 +1793,9 @@ xrep_agi( xrep_agi_init_header(ragi); xrep_agi_set_roots(ragi); error = xrep_agi_calc_from_btrees(ragi); + if (error) + goto out_revert; + error = xrep_iunlink_commit(ragi); if (error) goto out_revert; diff --git a/fs/xfs/scrub/agino_bitmap.h b/fs/xfs/scrub/agino_bitmap.h new file mode 100644 index 0000000000000..56d7db5f16999 --- /dev/null +++ b/fs/xfs/scrub/agino_bitmap.h @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_AGINO_BITMAP_H__ +#define __XFS_SCRUB_AGINO_BITMAP_H__ + +/* Bitmaps, but for type-checked for xfs_agino_t */ + +struct xagino_bitmap { + struct xbitmap32 aginobitmap; +}; + +static inline void xagino_bitmap_init(struct xagino_bitmap *bitmap) +{ + xbitmap32_init(&bitmap->aginobitmap); +} + +static inline void xagino_bitmap_destroy(struct xagino_bitmap *bitmap) +{ + xbitmap32_destroy(&bitmap->aginobitmap); +} + +static inline int xagino_bitmap_clear(struct xagino_bitmap *bitmap, + xfs_agino_t agino, unsigned int len) +{ + return xbitmap32_clear(&bitmap->aginobitmap, agino, len); +} + +static inline int xagino_bitmap_set(struct xagino_bitmap *bitmap, + xfs_agino_t agino, unsigned int len) +{ + return xbitmap32_set(&bitmap->aginobitmap, agino, len); +} + +static inline bool xagino_bitmap_test(struct xagino_bitmap *bitmap, + xfs_agino_t agino, unsigned int *len) +{ + return xbitmap32_test(&bitmap->aginobitmap, agino, len); +} + +static inline int xagino_bitmap_walk(struct xagino_bitmap *bitmap, + xbitmap32_walk_fn fn, void *priv) +{ + return xbitmap32_walk(&bitmap->aginobitmap, fn, priv); +} + +#endif /* __XFS_SCRUB_AGINO_BITMAP_H__ */ diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 03cb095fc1a12..814db1d1747a0 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2757,6 +2757,261 @@ DEFINE_EVENT(xrep_symlink_class, name, \ DEFINE_XREP_SYMLINK_EVENT(xrep_symlink_rebuild); DEFINE_XREP_SYMLINK_EVENT(xrep_symlink_reset_fork); +TRACE_EVENT(xrep_iunlink_visit, + TP_PROTO(struct xfs_perag *pag, unsigned int bucket, + xfs_agino_t bucket_agino, struct xfs_inode *ip), + TP_ARGS(pag, bucket, bucket_agino, ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(unsigned int, bucket) + __field(xfs_agino_t, bucket_agino) + __field(xfs_agino_t, prev_agino) + __field(xfs_agino_t, next_agino) + ), + TP_fast_assign( + __entry->dev = pag->pag_mount->m_super->s_dev; + __entry->agno = pag->pag_agno; + __entry->agino = XFS_INO_TO_AGINO(pag->pag_mount, ip->i_ino); + __entry->bucket = bucket; + __entry->bucket_agino = bucket_agino; + __entry->prev_agino = ip->i_prev_unlinked; + __entry->next_agino = ip->i_next_unlinked; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u agino 0x%x bucket_agino 0x%x prev_agino 0x%x next_agino 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->bucket, + __entry->agino, + __entry->bucket_agino, + __entry->prev_agino, + __entry->next_agino) +); + +TRACE_EVENT(xrep_iunlink_reload_next, + TP_PROTO(struct xfs_inode *ip, xfs_agino_t prev_agino), + TP_ARGS(ip, prev_agino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(xfs_agino_t, old_prev_agino) + __field(xfs_agino_t, prev_agino) + __field(xfs_agino_t, next_agino) + __field(unsigned int, nlink) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); + __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->old_prev_agino = ip->i_prev_unlinked; + __entry->prev_agino = prev_agino; + __entry->next_agino = ip->i_next_unlinked; + __entry->nlink = VFS_I(ip)->i_nlink; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u agino 0x%x nlink %u old_prev_agino %u prev_agino 0x%x next_agino 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agino % XFS_AGI_UNLINKED_BUCKETS, + __entry->agino, + __entry->nlink, + __entry->old_prev_agino, + __entry->prev_agino, + __entry->next_agino) +); + +TRACE_EVENT(xrep_iunlink_reload_ondisk, + TP_PROTO(struct xfs_inode *ip), + TP_ARGS(ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(unsigned int, nlink) + __field(xfs_agino_t, next_agino) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); + __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->nlink = VFS_I(ip)->i_nlink; + __entry->next_agino = ip->i_next_unlinked; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u agino 0x%x nlink %u next_agino 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agino % XFS_AGI_UNLINKED_BUCKETS, + __entry->agino, + __entry->nlink, + __entry->next_agino) +); + +TRACE_EVENT(xrep_iunlink_walk_ondisk_bucket, + TP_PROTO(struct xfs_perag *pag, unsigned int bucket, + xfs_agino_t prev_agino, xfs_agino_t next_agino), + TP_ARGS(pag, bucket, prev_agino, next_agino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(unsigned int, bucket) + __field(xfs_agino_t, prev_agino) + __field(xfs_agino_t, next_agino) + ), + TP_fast_assign( + __entry->dev = pag->pag_mount->m_super->s_dev; + __entry->agno = pag->pag_agno; + __entry->bucket = bucket; + __entry->prev_agino = prev_agino; + __entry->next_agino = next_agino; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u prev_agino 0x%x next_agino 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->bucket, + __entry->prev_agino, + __entry->next_agino) +); + +DECLARE_EVENT_CLASS(xrep_iunlink_resolve_class, + TP_PROTO(struct xfs_perag *pag, unsigned int bucket, + xfs_agino_t prev_agino, xfs_agino_t next_agino), + TP_ARGS(pag, bucket, prev_agino, next_agino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(unsigned int, bucket) + __field(xfs_agino_t, prev_agino) + __field(xfs_agino_t, next_agino) + ), + TP_fast_assign( + __entry->dev = pag->pag_mount->m_super->s_dev; + __entry->agno = pag->pag_agno; + __entry->bucket = bucket; + __entry->prev_agino = prev_agino; + __entry->next_agino = next_agino; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u prev_agino 0x%x next_agino 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->bucket, + __entry->prev_agino, + __entry->next_agino) +); +#define DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(name) \ +DEFINE_EVENT(xrep_iunlink_resolve_class, name, \ + TP_PROTO(struct xfs_perag *pag, unsigned int bucket, \ + xfs_agino_t prev_agino, xfs_agino_t next_agino), \ + TP_ARGS(pag, bucket, prev_agino, next_agino)) +DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_uncached); +DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_wronglist); +DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_nolist); +DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_ok); + +TRACE_EVENT(xrep_iunlink_relink_next, + TP_PROTO(struct xfs_inode *ip, xfs_agino_t next_agino), + TP_ARGS(ip, next_agino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(xfs_agino_t, next_agino) + __field(xfs_agino_t, new_next_agino) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); + __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->next_agino = ip->i_next_unlinked; + __entry->new_next_agino = next_agino; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u agino 0x%x next_agino 0x%x -> 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agino % XFS_AGI_UNLINKED_BUCKETS, + __entry->agino, + __entry->next_agino, + __entry->new_next_agino) +); + +TRACE_EVENT(xrep_iunlink_relink_prev, + TP_PROTO(struct xfs_inode *ip, xfs_agino_t prev_agino), + TP_ARGS(ip, prev_agino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(xfs_agino_t, prev_agino) + __field(xfs_agino_t, new_prev_agino) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->agno = XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino); + __entry->agino = XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino); + __entry->prev_agino = ip->i_prev_unlinked; + __entry->new_prev_agino = prev_agino; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u agino 0x%x prev_agino 0x%x -> 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agino % XFS_AGI_UNLINKED_BUCKETS, + __entry->agino, + __entry->prev_agino, + __entry->new_prev_agino) +); + +TRACE_EVENT(xrep_iunlink_add_to_bucket, + TP_PROTO(struct xfs_perag *pag, unsigned int bucket, + xfs_agino_t agino, xfs_agino_t curr_head), + TP_ARGS(pag, bucket, agino, curr_head), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(unsigned int, bucket) + __field(xfs_agino_t, agino) + __field(xfs_agino_t, next_agino) + ), + TP_fast_assign( + __entry->dev = pag->pag_mount->m_super->s_dev; + __entry->agno = pag->pag_agno; + __entry->bucket = bucket; + __entry->agino = agino; + __entry->next_agino = curr_head; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u agino 0x%x next_agino 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->bucket, + __entry->agino, + __entry->next_agino) +); + +TRACE_EVENT(xrep_iunlink_commit_bucket, + TP_PROTO(struct xfs_perag *pag, unsigned int bucket, + xfs_agino_t old_agino, xfs_agino_t agino), + TP_ARGS(pag, bucket, old_agino, agino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(unsigned int, bucket) + __field(xfs_agino_t, old_agino) + __field(xfs_agino_t, agino) + ), + TP_fast_assign( + __entry->dev = pag->pag_mount->m_super->s_dev; + __entry->agno = pag->pag_agno; + __entry->bucket = bucket; + __entry->old_agino = old_agino; + __entry->agino = agino; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u agino 0x%x -> 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->bucket, + __entry->old_agino, + __entry->agino) +); + #endif /* IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) */ #endif /* _TRACE_XFS_SCRUB_TRACE_H */ From 40cb8613d6122ca80a9e42e4cecc4d308c3b80fb Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:55:03 -0700 Subject: [PATCH 0257/1702] xfs: check unused nlink fields in the ondisk inode v2/v3 inodes use di_nlink and not di_onlink; and v1 inodes use di_onlink and not di_nlink. Whichever field is not in use, make sure its contents are zero, and teach xfs_scrub to fix that if it is. This clears a bunch of missing scrub failure errors in xfs/385 for core.onlink. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_inode_buf.c | 8 ++++++++ fs/xfs/scrub/inode_repair.c | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index d0dcce462bf42..d79002343d0b6 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -491,6 +491,14 @@ xfs_dinode_verify( return __this_address; } + if (dip->di_version > 1) { + if (dip->di_onlink) + return __this_address; + } else { + if (dip->di_nlink) + return __this_address; + } + /* don't allow invalid i_size */ di_size = be64_to_cpu(dip->di_size); if (di_size & (1ULL << 63)) diff --git a/fs/xfs/scrub/inode_repair.c b/fs/xfs/scrub/inode_repair.c index 0dde5df2f8d39..e3b74ea50fdef 100644 --- a/fs/xfs/scrub/inode_repair.c +++ b/fs/xfs/scrub/inode_repair.c @@ -516,6 +516,17 @@ xrep_dinode_mode( return 0; } +/* Fix unused link count fields having nonzero values. */ +STATIC void +xrep_dinode_nlinks( + struct xfs_dinode *dip) +{ + if (dip->di_version > 1) + dip->di_onlink = 0; + else + dip->di_nlink = 0; +} + /* Fix any conflicting flags that the verifiers complain about. */ STATIC void xrep_dinode_flags( @@ -1377,6 +1388,7 @@ xrep_dinode_core( iget_error = xrep_dinode_mode(ri, dip); if (iget_error) goto write; + xrep_dinode_nlinks(dip); xrep_dinode_flags(sc, dip, ri->rt_extents > 0); xrep_dinode_size(ri, dip); xrep_dinode_extsize_hints(sc, dip); From 2935213a6831a0087442d406301c2cdcc87b0f61 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:55:04 -0700 Subject: [PATCH 0258/1702] xfs: try to avoid allocating from sick inode clusters I noticed that xfs/413 and xfs/375 occasionally failed while fuzzing core.mode of an inode. The root cause of these problems is that the field we fuzzed (core.mode or core.magic, typically) causes the entire inode cluster buffer verification to fail, which affects several inodes at once. The repair process tries to create either a /lost+found or a temporary repair file, but regrettably it picks the same inode cluster that we just corrupted, with the result that repair triggers the demise of the filesystem. Try avoid this by making the inode allocation path detect when the perag health status indicates that someone has found bad inode cluster buffers, and try to read the inode cluster buffer. If the cluster buffer fails the verifiers, try another AG. This isn't foolproof and can result in premature ENOSPC, but that might be better than shutting down. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_ialloc.c | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/fs/xfs/libxfs/xfs_ialloc.c b/fs/xfs/libxfs/xfs_ialloc.c index cb37f0007731f..14c81f227c5bb 100644 --- a/fs/xfs/libxfs/xfs_ialloc.c +++ b/fs/xfs/libxfs/xfs_ialloc.c @@ -1057,6 +1057,33 @@ xfs_inobt_first_free_inode( return xfs_lowbit64(realfree); } +/* + * If this AG has corrupt inodes, check if allocating this inode would fail + * with corruption errors. Returns 0 if we're clear, or EAGAIN to try again + * somewhere else. + */ +static int +xfs_dialloc_check_ino( + struct xfs_perag *pag, + struct xfs_trans *tp, + xfs_ino_t ino) +{ + struct xfs_imap imap; + struct xfs_buf *bp; + int error; + + error = xfs_imap(pag, tp, ino, &imap, 0); + if (error) + return -EAGAIN; + + error = xfs_imap_to_bp(pag->pag_mount, tp, &imap, &bp); + if (error) + return -EAGAIN; + + xfs_trans_brelse(tp, bp); + return 0; +} + /* * Allocate an inode using the inobt-only algorithm. */ @@ -1309,6 +1336,13 @@ xfs_dialloc_ag_inobt( ASSERT((XFS_AGINO_TO_OFFSET(mp, rec.ir_startino) % XFS_INODES_PER_CHUNK) == 0); ino = XFS_AGINO_TO_INO(mp, pag->pag_agno, rec.ir_startino + offset); + + if (xfs_ag_has_sickness(pag, XFS_SICK_AG_INODES)) { + error = xfs_dialloc_check_ino(pag, tp, ino); + if (error) + goto error0; + } + rec.ir_free &= ~XFS_INOBT_MASK(offset); rec.ir_freecount--; error = xfs_inobt_update(cur, &rec); @@ -1584,6 +1618,12 @@ xfs_dialloc_ag( XFS_INODES_PER_CHUNK) == 0); ino = XFS_AGINO_TO_INO(mp, pag->pag_agno, rec.ir_startino + offset); + if (xfs_ag_has_sickness(pag, XFS_SICK_AG_INODES)) { + error = xfs_dialloc_check_ino(pag, tp, ino); + if (error) + goto error_cur; + } + /* * Modify or remove the finobt record. */ From 5f204051d998ec3d7306db0d749bddcbf4c97693 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:55:05 -0700 Subject: [PATCH 0259/1702] xfs: pin inodes that would otherwise overflow link count The VFS inc_nlink function does not explicitly check for integer overflows in the i_nlink field. Instead, it checks the link count against s_max_links in the vfs_{link,create,rename} functions. XFS sets the maximum link count to 2.1 billion, so integer overflows should not be a problem. However. It's possible that online repair could find that a file has more than four billion links, particularly if the link count got corrupted while creating hardlinks to the file. The di_nlinkv2 field is not large enough to store a value larger than 2^32, so we ought to define a magic pin value of ~0U which means that the inode never gets deleted. This will prevent a UAF error if the repair finds this situation and users begin deleting links to the file. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_format.h | 6 ++++++ fs/xfs/scrub/dir_repair.c | 11 +++-------- fs/xfs/scrub/nlinks.c | 4 +++- fs/xfs/scrub/nlinks_repair.c | 8 ++------ fs/xfs/xfs_inode.c | 33 ++++++++++++++++++++++----------- 5 files changed, 36 insertions(+), 26 deletions(-) diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h index 10153ce116d4c..f1818c54af6f8 100644 --- a/fs/xfs/libxfs/xfs_format.h +++ b/fs/xfs/libxfs/xfs_format.h @@ -899,6 +899,12 @@ static inline uint xfs_dinode_size(int version) */ #define XFS_MAXLINK ((1U << 31) - 1U) +/* + * Any file that hits the maximum ondisk link count should be pinned to avoid + * a use-after-free situation. + */ +#define XFS_NLINK_PINNED (~0U) + /* * Values for di_format * diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index c150b2efa2c21..38957da26b94a 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -1145,7 +1145,9 @@ xrep_dir_set_nlink( struct xfs_scrub *sc = rd->sc; struct xfs_inode *dp = sc->ip; struct xfs_perag *pag; - unsigned int new_nlink = rd->subdirs + 2; + unsigned int new_nlink = min_t(unsigned long long, + rd->subdirs + 2, + XFS_NLINK_PINNED); int error; /* @@ -1201,13 +1203,6 @@ xrep_dir_swap( bool ip_local, temp_local; int error = 0; - /* - * If we found enough subdirs to overflow this directory's link count, - * bail out to userspace before we modify anything. - */ - if (rd->subdirs + 2 > XFS_MAXLINK) - return -EFSCORRUPTED; - /* * If we never found the parent for this directory, temporarily assign * the root dir as the parent; we'll move this to the orphanage after diff --git a/fs/xfs/scrub/nlinks.c b/fs/xfs/scrub/nlinks.c index c456523fac9c6..fcb9c473f372e 100644 --- a/fs/xfs/scrub/nlinks.c +++ b/fs/xfs/scrub/nlinks.c @@ -607,9 +607,11 @@ xchk_nlinks_compare_inode( * this as a corruption. The VFS won't let users increase the link * count, but it will let them decrease it. */ - if (total_links > XFS_MAXLINK) { + if (total_links > XFS_NLINK_PINNED) { xchk_ino_set_corrupt(sc, ip->i_ino); goto out_corrupt; + } else if (total_links > XFS_MAXLINK) { + xchk_ino_set_warning(sc, ip->i_ino); } /* Link counts should match. */ diff --git a/fs/xfs/scrub/nlinks_repair.c b/fs/xfs/scrub/nlinks_repair.c index 0cb67339eac89..83f8637bb08fd 100644 --- a/fs/xfs/scrub/nlinks_repair.c +++ b/fs/xfs/scrub/nlinks_repair.c @@ -238,14 +238,10 @@ xrep_nlinks_repair_inode( /* Commit the new link count if it changed. */ if (total_links != actual_nlink) { - if (total_links > XFS_MAXLINK) { - trace_xrep_nlinks_unfixable_inode(mp, ip, &obs); - goto out_trans; - } - trace_xrep_nlinks_update_inode(mp, ip, &obs); - set_nlink(VFS_I(ip), total_links); + set_nlink(VFS_I(ip), min_t(unsigned long long, total_links, + XFS_NLINK_PINNED)); dirty = true; } diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index fed0cd6bffdf0..03dcb4ac04312 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -890,22 +890,25 @@ xfs_init_new_inode( */ static int /* error */ xfs_droplink( - xfs_trans_t *tp, - xfs_inode_t *ip) + struct xfs_trans *tp, + struct xfs_inode *ip) { - if (VFS_I(ip)->i_nlink == 0) { - xfs_alert(ip->i_mount, - "%s: Attempt to drop inode (%llu) with nlink zero.", - __func__, ip->i_ino); - return -EFSCORRUPTED; - } + struct inode *inode = VFS_I(ip); xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_CHG); - drop_nlink(VFS_I(ip)); + if (inode->i_nlink == 0) { + xfs_info_ratelimited(tp->t_mountp, + "Inode 0x%llx link count dropped below zero. Pinning link count.", + ip->i_ino); + set_nlink(inode, XFS_NLINK_PINNED); + } + if (inode->i_nlink != XFS_NLINK_PINNED) + drop_nlink(inode); + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); - if (VFS_I(ip)->i_nlink) + if (inode->i_nlink) return 0; return xfs_iunlink(tp, ip); @@ -919,9 +922,17 @@ xfs_bumplink( struct xfs_trans *tp, struct xfs_inode *ip) { + struct inode *inode = VFS_I(ip); + xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_CHG); - inc_nlink(VFS_I(ip)); + if (inode->i_nlink == XFS_NLINK_PINNED - 1) + xfs_info_ratelimited(tp->t_mountp, + "Inode 0x%llx link count exceeded maximum. Pinning link count.", + ip->i_ino); + if (inode->i_nlink != XFS_NLINK_PINNED) + inc_nlink(inode); + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); } From d85fe250f2eb61e19029e9e0d30095c5f646e2f2 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:55:07 -0700 Subject: [PATCH 0260/1702] docs: update the parent pointers documentation to the final version Now that we've decided on the ondisk format of parent pointers, update the documentation to reflect that. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- .../xfs/xfs-online-fsck-design.rst | 94 +++++++++++-------- 1 file changed, 53 insertions(+), 41 deletions(-) diff --git a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst index 74a8e42c74bd0..1e3211d12247d 100644 --- a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst +++ b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst @@ -4465,10 +4465,10 @@ reconstruction of filesystem space metadata. The parent pointer feature, however, makes total directory reconstruction possible. -XFS parent pointers include the dirent name and location of the entry within -the parent directory. +XFS parent pointers contain the information needed to identify the +corresponding directory entry in the parent directory. In other words, child files use extended attributes to store pointers to -parents in the form ``(parent_inum, parent_gen, dirent_pos) → (dirent_name)``. +parents in the form ``(dirent_name) → (parent_inum, parent_gen)``. The directory checking process can be strengthened to ensure that the target of each dirent also contains a parent pointer pointing back to the dirent. Likewise, each parent pointer can be checked by ensuring that the target of @@ -4476,8 +4476,6 @@ each parent pointer is a directory and that it contains a dirent matching the parent pointer. Both online and offline repair can use this strategy. -**Note**: The ondisk format of parent pointers is not yet finalized. - +--------------------------------------------------------------------------+ | **Historical Sidebar**: | +--------------------------------------------------------------------------+ @@ -4519,8 +4517,58 @@ Both online and offline repair can use this strategy. | Chandan increased the maximum extent counts of both data and attribute | | forks, thereby ensuring that the extended attribute structure can grow | | to handle the maximum hardlink count of any file. | +| | +| For this second effort, the ondisk parent pointer format as originally | +| proposed was ``(parent_inum, parent_gen, dirent_pos) → (dirent_name)``. | +| The format was changed during development to eliminate the requirement | +| of repair tools needing to to ensure that the ``dirent_pos`` field | +| always matched when reconstructing a directory. | +| | +| There were a few other ways to have solved that problem: | +| | +| 1. The field could be designated advisory, since the other three values | +| are sufficient to find the entry in the parent. | +| However, this makes indexed key lookup impossible while repairs are | +| ongoing. | +| | +| 2. We could allow creating directory entries at specified offsets, which | +| solves the referential integrity problem but runs the risk that | +| dirent creation will fail due to conflicts with the free space in the | +| directory. | +| | +| These conflicts could be resolved by appending the directory entry | +| and amending the xattr code to support updating an xattr key and | +| reindexing the dabtree, though this would have to be performed with | +| the parent directory still locked. | +| | +| 3. Same as above, but remove the old parent pointer entry and add a new | +| one atomically. | +| | +| 4. Change the ondisk xattr format to | +| ``(parent_inum, name) → (parent_gen)``, which would provide the attr | +| name uniqueness that we require, without forcing repair code to | +| update the dirent position. | +| Unfortunately, this requires changes to the xattr code to support | +| attr names as long as 263 bytes. | +| | +| 5. Change the ondisk xattr format to ``(parent_inum, hash(name)) → | +| (name, parent_gen)``. | +| If the hash is sufficiently resistant to collisions (e.g. sha256) | +| then this should provide the attr name uniqueness that we require. | +| Names shorter than 247 bytes could be stored directly. | +| | +| 6. Change the ondisk xattr format to ``(dirent_name) → (parent_ino, | +| parent_gen)``. This format doesn't require any of the complicated | +| nested name hashing of the previous suggestions. However, it was | +| discovered that multiple hardlinks to the same inode with the same | +| filename caused performance problems with hashed xattr lookups, so | +| the parent inumber is now xor'd into the hash index. | +| | +| In the end, it was decided that solution #6 was the most compact and the | +| most performant. A new hash function was designed for parent pointers. | +--------------------------------------------------------------------------+ + Case Study: Repairing Directories with Parent Pointers ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -4569,42 +4617,6 @@ The proposed patchset is the `_ series. -**Unresolved Question**: How will repair ensure that the ``dirent_pos`` fields -match in the reconstructed directory? - -*Answer*: There are a few ways to solve this problem: - -1. The field could be designated advisory, since the other three values are - sufficient to find the entry in the parent. - However, this makes indexed key lookup impossible while repairs are ongoing. - -2. We could allow creating directory entries at specified offsets, which solves - the referential integrity problem but runs the risk that dirent creation - will fail due to conflicts with the free space in the directory. - - These conflicts could be resolved by appending the directory entry and - amending the xattr code to support updating an xattr key and reindexing the - dabtree, though this would have to be performed with the parent directory - still locked. - -3. Same as above, but remove the old parent pointer entry and add a new one - atomically. - -4. Change the ondisk xattr format to ``(parent_inum, name) → (parent_gen)``, - which would provide the attr name uniqueness that we require, without - forcing repair code to update the dirent position. - Unfortunately, this requires changes to the xattr code to support attr - names as long as 263 bytes. - -5. Change the ondisk xattr format to ``(parent_inum, hash(name)) → - (name, parent_gen)``. - If the hash is sufficiently resistant to collisions (e.g. sha256) then - this should provide the attr name uniqueness that we require. - Names shorter than 247 bytes could be stored directly. - -Discussion is ongoing under the `parent pointers patch deluge -`_. - Case Study: Repairing Parent Pointers ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 5220727ce8ee779076283ae8f6ff26187a16241c Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:55:08 -0700 Subject: [PATCH 0261/1702] docs: update online directory and parent pointer repair sections Update the case studies of online directory and parent pointer reconstruction to reflect what they actually do in the final version. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- .../xfs/xfs-online-fsck-design.rst | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst index 1e3211d12247d..1ea4e59c9cdbd 100644 --- a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst +++ b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst @@ -4576,8 +4576,9 @@ Directory rebuilding uses a :ref:`coordinated inode scan ` and a :ref:`directory entry live update hook ` as follows: 1. Set up a temporary directory for generating the new directory structure, - an xfblob for storing entry names, and an xfarray for stashing directory - updates. + an xfblob for storing entry names, and an xfarray for stashing the fixed + size fields involved in a directory update: ``(child inumber, add vs. + remove, name cookie, ftype)``. 2. Set up an inode scanner and hook into the directory entry code to receive updates on directory operations. @@ -4586,35 +4587,34 @@ a :ref:`directory entry live update hook ` as follows: pointer references the directory of interest. If so: - a. Stash an addname entry for this dirent in the xfarray for later. + a. Stash the parent pointer name and an addname entry for this dirent in the + xfblob and xfarray, respectively. - b. When finished scanning that file, flush the stashed updates to the - temporary directory. + b. When finished scanning that file or the kernel memory consumption exceeds + a threshold, flush the stashed updates to the temporary directory. 4. For each live directory update received via the hook, decide if the child has already been scanned. If so: - a. Stash an addname or removename entry for this dirent update in the - xfarray for later. + a. Stash the parent pointer name an addname or removename entry for this + dirent update in the xfblob and xfarray for later. We cannot write directly to the temporary directory because hook functions are not allowed to modify filesystem metadata. Instead, we stash updates in the xfarray and rely on the scanner thread to apply the stashed updates to the temporary directory. -5. When the scan is complete, atomically exchange the contents of the temporary +5. When the scan is complete, replay any stashed entries in the xfarray. + +6. When the scan is complete, atomically exchange the contents of the temporary directory and the directory being repaired. The temporary directory now contains the damaged directory structure. -6. Reap the temporary directory. - -7. Update the dirent position field of parent pointers as necessary. - This may require the queuing of a substantial number of xattr log intent - items. +7. Reap the temporary directory. The proposed patchset is the `parent pointers directory repair -`_ +`_ series. Case Study: Repairing Parent Pointers @@ -4624,8 +4624,9 @@ Online reconstruction of a file's parent pointer information works similarly to directory reconstruction: 1. Set up a temporary file for generating a new extended attribute structure, - an `xfblob` for storing parent pointer names, and an xfarray for - stashing parent pointer updates. + an xfblob for storing parent pointer names, and an xfarray for stashing the + fixed size fields involved in a parent pointer update: ``(parent inumber, + parent generation, add vs. remove, name cookie)``. 2. Set up an inode scanner and hook into the directory entry code to receive updates on directory operations. @@ -4634,34 +4635,36 @@ directory reconstruction: dirent references the file of interest. If so: - a. Stash an addpptr entry for this parent pointer in the xfblob and xfarray - for later. + a. Stash the dirent name and an addpptr entry for this parent pointer in the + xfblob and xfarray, respectively. - b. When finished scanning the directory, flush the stashed updates to the - temporary directory. + b. When finished scanning the directory or the kernel memory consumption + exceeds a threshold, flush the stashed updates to the temporary file. 4. For each live directory update received via the hook, decide if the parent has already been scanned. If so: - a. Stash an addpptr or removepptr entry for this dirent update in the - xfarray for later. + a. Stash the dirent name and an addpptr or removepptr entry for this dirent + update in the xfblob and xfarray for later. We cannot write parent pointers directly to the temporary file because hook functions are not allowed to modify filesystem metadata. Instead, we stash updates in the xfarray and rely on the scanner thread to apply the stashed parent pointer updates to the temporary file. -5. Copy all non-parent pointer extended attributes to the temporary file. +5. When the scan is complete, replay any stashed entries in the xfarray. + +6. Copy all non-parent pointer extended attributes to the temporary file. -6. When the scan is complete, atomically exchange the mappings of the attribute +7. When the scan is complete, atomically exchange the mappings of the attribute forks of the temporary file and the file being repaired. The temporary file now contains the damaged extended attribute structure. -7. Reap the temporary file. +8. Reap the temporary file. The proposed patchset is the `parent pointers repair -`_ +`_ series. Digression: Offline Checking of Parent Pointers From 1a5f6e08d4e379a23da5be974aee50b26a20c5b0 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:55:06 -0700 Subject: [PATCH 0262/1702] xfs: create subordinate scrub contexts for xchk_metadata_inode_subtype When a file-based metadata structure is being scrubbed in xchk_metadata_inode_subtype, we should create an entirely new scrub context so that each scrubber doesn't trip over another's buffers. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/common.c | 23 +++------------ fs/xfs/scrub/repair.c | 67 +++++++++---------------------------------- fs/xfs/scrub/scrub.c | 63 ++++++++++++++++++++++++++++++++++++++++ fs/xfs/scrub/scrub.h | 11 +++++++ 4 files changed, 91 insertions(+), 73 deletions(-) diff --git a/fs/xfs/scrub/common.c b/fs/xfs/scrub/common.c index a2da2bef509aa..48302532d10d1 100644 --- a/fs/xfs/scrub/common.c +++ b/fs/xfs/scrub/common.c @@ -1203,27 +1203,12 @@ xchk_metadata_inode_subtype( struct xfs_scrub *sc, unsigned int scrub_type) { - __u32 smtype = sc->sm->sm_type; - unsigned int sick_mask = sc->sick_mask; + struct xfs_scrub_subord *sub; int error; - sc->sm->sm_type = scrub_type; - - switch (scrub_type) { - case XFS_SCRUB_TYPE_INODE: - error = xchk_inode(sc); - break; - case XFS_SCRUB_TYPE_BMBTD: - error = xchk_bmap_data(sc); - break; - default: - ASSERT(0); - error = -EFSCORRUPTED; - break; - } - - sc->sick_mask = sick_mask; - sc->sm->sm_type = smtype; + sub = xchk_scrub_create_subord(sc, scrub_type); + error = sub->sc.ops->scrub(&sub->sc); + xchk_scrub_free_subord(sub); return error; } diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c index 369f0430e4ba1..b6aff89679d58 100644 --- a/fs/xfs/scrub/repair.c +++ b/fs/xfs/scrub/repair.c @@ -1009,55 +1009,27 @@ xrep_metadata_inode_subtype( struct xfs_scrub *sc, unsigned int scrub_type) { - __u32 smtype = sc->sm->sm_type; - __u32 smflags = sc->sm->sm_flags; - unsigned int sick_mask = sc->sick_mask; + struct xfs_scrub_subord *sub; int error; /* - * Let's see if the inode needs repair. We're going to open-code calls - * to the scrub and repair functions so that we can hang on to the + * Let's see if the inode needs repair. Use a subordinate scrub context + * to call the scrub and repair functions so that we can hang on to the * resources that we already acquired instead of using the standard * setup/teardown routines. */ - sc->sm->sm_flags &= ~XFS_SCRUB_FLAGS_OUT; - sc->sm->sm_type = scrub_type; - - switch (scrub_type) { - case XFS_SCRUB_TYPE_INODE: - error = xchk_inode(sc); - break; - case XFS_SCRUB_TYPE_BMBTD: - error = xchk_bmap_data(sc); - break; - case XFS_SCRUB_TYPE_BMBTA: - error = xchk_bmap_attr(sc); - break; - default: - ASSERT(0); - error = -EFSCORRUPTED; - } + sub = xchk_scrub_create_subord(sc, scrub_type); + error = sub->sc.ops->scrub(&sub->sc); if (error) goto out; - - if (!xrep_will_attempt(sc)) + if (!xrep_will_attempt(&sub->sc)) goto out; /* * Repair some part of the inode. This will potentially join the inode * to the transaction. */ - switch (scrub_type) { - case XFS_SCRUB_TYPE_INODE: - error = xrep_inode(sc); - break; - case XFS_SCRUB_TYPE_BMBTD: - error = xrep_bmap(sc, XFS_DATA_FORK, false); - break; - case XFS_SCRUB_TYPE_BMBTA: - error = xrep_bmap(sc, XFS_ATTR_FORK, false); - break; - } + error = sub->sc.ops->repair(&sub->sc); if (error) goto out; @@ -1066,10 +1038,10 @@ xrep_metadata_inode_subtype( * that the inode will not be joined to the transaction when we exit * the function. */ - error = xfs_defer_finish(&sc->tp); + error = xfs_defer_finish(&sub->sc.tp); if (error) goto out; - error = xfs_trans_roll(&sc->tp); + error = xfs_trans_roll(&sub->sc.tp); if (error) goto out; @@ -1077,31 +1049,18 @@ xrep_metadata_inode_subtype( * Clear the corruption flags and re-check the metadata that we just * repaired. */ - sc->sm->sm_flags &= ~XFS_SCRUB_FLAGS_OUT; - - switch (scrub_type) { - case XFS_SCRUB_TYPE_INODE: - error = xchk_inode(sc); - break; - case XFS_SCRUB_TYPE_BMBTD: - error = xchk_bmap_data(sc); - break; - case XFS_SCRUB_TYPE_BMBTA: - error = xchk_bmap_attr(sc); - break; - } + sub->sc.sm->sm_flags &= ~XFS_SCRUB_FLAGS_OUT; + error = sub->sc.ops->scrub(&sub->sc); if (error) goto out; /* If corruption persists, the repair has failed. */ - if (xchk_needs_repair(sc->sm)) { + if (xchk_needs_repair(sub->sc.sm)) { error = -EFSCORRUPTED; goto out; } out: - sc->sick_mask = sick_mask; - sc->sm->sm_type = smtype; - sc->sm->sm_flags = smflags; + xchk_scrub_free_subord(sub); return error; } diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 301d5b753fdd5..ebb06838c31be 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -177,6 +177,39 @@ xchk_fsgates_disable( } #undef FSGATES_MASK +/* Free the resources associated with a scrub subtype. */ +void +xchk_scrub_free_subord( + struct xfs_scrub_subord *sub) +{ + struct xfs_scrub *sc = sub->parent_sc; + + ASSERT(sc->ip == sub->sc.ip); + ASSERT(sc->orphanage == sub->sc.orphanage); + ASSERT(sc->tempip == sub->sc.tempip); + + sc->sm->sm_type = sub->old_smtype; + sc->sm->sm_flags = sub->old_smflags | + (sc->sm->sm_flags & XFS_SCRUB_FLAGS_OUT); + sc->tp = sub->sc.tp; + + if (sub->sc.buf) { + if (sub->sc.buf_cleanup) + sub->sc.buf_cleanup(sub->sc.buf); + kvfree(sub->sc.buf); + } + if (sub->sc.xmbtp) + xmbuf_free(sub->sc.xmbtp); + if (sub->sc.xfile) + xfile_destroy(sub->sc.xfile); + + sc->ilock_flags = sub->sc.ilock_flags; + sc->orphanage_ilock_flags = sub->sc.orphanage_ilock_flags; + sc->temp_ilock_flags = sub->sc.temp_ilock_flags; + + kfree(sub); +} + /* Free all the resources and finish the transactions. */ STATIC int xchk_teardown( @@ -505,6 +538,36 @@ static inline void xchk_postmortem(struct xfs_scrub *sc) } #endif /* CONFIG_XFS_ONLINE_REPAIR */ +/* + * Create a new scrub context from an existing one, but with a different scrub + * type. + */ +struct xfs_scrub_subord * +xchk_scrub_create_subord( + struct xfs_scrub *sc, + unsigned int subtype) +{ + struct xfs_scrub_subord *sub; + + sub = kzalloc(sizeof(*sub), XCHK_GFP_FLAGS); + if (!sub) + return ERR_PTR(-ENOMEM); + + sub->old_smtype = sc->sm->sm_type; + sub->old_smflags = sc->sm->sm_flags; + sub->parent_sc = sc; + memcpy(&sub->sc, sc, sizeof(struct xfs_scrub)); + sub->sc.ops = &meta_scrub_ops[subtype]; + sub->sc.sm->sm_type = subtype; + sub->sc.sm->sm_flags &= ~XFS_SCRUB_FLAGS_OUT; + sub->sc.buf = NULL; + sub->sc.buf_cleanup = NULL; + sub->sc.xfile = NULL; + sub->sc.xmbtp = NULL; + + return sub; +} + /* Dispatch metadata scrubbing. */ int xfs_scrub_metadata( diff --git a/fs/xfs/scrub/scrub.h b/fs/xfs/scrub/scrub.h index 7abe498f7a461..54a4242bc79cf 100644 --- a/fs/xfs/scrub/scrub.h +++ b/fs/xfs/scrub/scrub.h @@ -156,6 +156,17 @@ struct xfs_scrub { */ #define XREP_FSGATES_ALL (XREP_FSGATES_EXCHANGE_RANGE) +struct xfs_scrub_subord { + struct xfs_scrub sc; + struct xfs_scrub *parent_sc; + unsigned int old_smtype; + unsigned int old_smflags; +}; + +struct xfs_scrub_subord *xchk_scrub_create_subord(struct xfs_scrub *sc, + unsigned int subtype); +void xchk_scrub_free_subord(struct xfs_scrub_subord *sub); + /* Metadata scrubbers */ int xchk_tester(struct xfs_scrub *sc); int xchk_superblock(struct xfs_scrub *sc); From b0ffe661fab4b939e4472ef96b8dac3c74e0e03e Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:55:06 -0700 Subject: [PATCH 0263/1702] xfs: fix performance problems when fstrimming a subset of a fragmented AG On a 10TB filesystem where the free space in each AG is heavily fragmented, I noticed some very high runtimes on a FITRIM call for the entire filesystem. xfs_scrub likes to report progress information on each phase of the scrub, which means that a strace for the entire filesystem: ioctl(3, FITRIM, {start=0x0, len=10995116277760, minlen=0}) = 0 <686.209839> shows that scrub is uncommunicative for the entire duration. Reducing the size of the FITRIM requests to a single AG at a time produces lower times for each individual call, but even this isn't quite acceptable, because the time between progress reports are still very high: Strace for the first 4x 1TB AGs looks like (2): ioctl(3, FITRIM, {start=0x0, len=1099511627776, minlen=0}) = 0 <68.352033> ioctl(3, FITRIM, {start=0x10000000000, len=1099511627776, minlen=0}) = 0 <68.760323> ioctl(3, FITRIM, {start=0x20000000000, len=1099511627776, minlen=0}) = 0 <67.235226> ioctl(3, FITRIM, {start=0x30000000000, len=1099511627776, minlen=0}) = 0 <69.465744> I then had the idea to limit the length parameter of each call to a smallish amount (~11GB) so that we could report progress relatively quickly, but much to my surprise, each FITRIM call still took ~68 seconds! Unfortunately, the by-length fstrim implementation handles this poorly because it walks the entire free space by length index (cntbt), which is a very inefficient way to walk a subset of the blocks of an AG. Therefore, create a second implementation that will walk the bnobt and perform the trims in block number order. This implementation avoids the worst problems of the original code, though it lacks the desirable attribute of freeing the biggest chunks first. On the other hand, this second implementation will be much easier to constrain the system call latency, and makes it much easier to report fstrim progress to anyone who's running xfs_scrub. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner ar_startblock == NULLAGBLOCK) - error = xfs_alloc_lookup_ge(cur, 0, tcur->ar_blockcount, &i); - else - error = xfs_alloc_lookup_le(cur, tcur->ar_startblock, - tcur->ar_blockcount, &i); + if (tcur->by_bno) { + /* sub-AG discard request always starts at tcur->start */ + cur = xfs_bnobt_init_cursor(mp, tp, agbp, pag); + error = xfs_alloc_lookup_le(cur, tcur->start, 0, &i); + if (!error && !i) + error = xfs_alloc_lookup_ge(cur, tcur->start, 0, &i); + } else if (tcur->start == 0) { + /* first time through a by-len starts with max length */ + cur = xfs_cntbt_init_cursor(mp, tp, agbp, pag); + error = xfs_alloc_lookup_ge(cur, 0, tcur->count, &i); + } else { + /* nth time through a by-len starts where we left off */ + cur = xfs_cntbt_init_cursor(mp, tp, agbp, pag); + error = xfs_alloc_lookup_le(cur, tcur->start, tcur->count, &i); + } if (error) goto out_del_cursor; if (i == 0) { /* nothing of that length left in the AG, we are done */ - tcur->ar_blockcount = 0; + tcur->count = 0; goto out_del_cursor; } @@ -204,8 +213,6 @@ xfs_trim_gather_extents( while (i) { xfs_agblock_t fbno; xfs_extlen_t flen; - xfs_daddr_t dbno; - xfs_extlen_t dlen; error = xfs_alloc_get_rec(cur, &fbno, &flen, &i); if (error) @@ -221,38 +228,46 @@ xfs_trim_gather_extents( * Update the cursor to point at this extent so we * restart the next batch from this extent. */ - tcur->ar_startblock = fbno; - tcur->ar_blockcount = flen; - break; - } - - /* - * use daddr format for all range/len calculations as that is - * the format the range/len variables are supplied in by - * userspace. - */ - dbno = XFS_AGB_TO_DADDR(mp, pag->pag_agno, fbno); - dlen = XFS_FSB_TO_BB(mp, flen); - - /* - * Too small? Give up. - */ - if (dlen < minlen) { - trace_xfs_discard_toosmall(mp, pag->pag_agno, fbno, flen); - tcur->ar_blockcount = 0; + tcur->start = fbno; + tcur->count = flen; break; } /* * If the extent is entirely outside of the range we are - * supposed to discard skip it. Do not bother to trim - * down partially overlapping ranges for now. + * supposed to skip it. Do not bother to trim down partially + * overlapping ranges for now. */ - if (dbno + dlen < start || dbno > end) { + if (fbno + flen < tcur->start) { + trace_xfs_discard_exclude(mp, pag->pag_agno, fbno, flen); + goto next_extent; + } + if (fbno > tcur->end) { trace_xfs_discard_exclude(mp, pag->pag_agno, fbno, flen); + if (tcur->by_bno) { + tcur->count = 0; + break; + } goto next_extent; } + /* Trim the extent returned to the range we want. */ + if (fbno < tcur->start) { + flen -= tcur->start - fbno; + fbno = tcur->start; + } + if (fbno + flen > tcur->end + 1) + flen = tcur->end - fbno + 1; + + /* Too small? Give up. */ + if (flen < tcur->minlen) { + trace_xfs_discard_toosmall(mp, pag->pag_agno, fbno, flen); + if (tcur->by_bno) + goto next_extent; + tcur->count = 0; + break; + } + /* * If any blocks in the range are still busy, skip the * discard and try again the next time. @@ -266,7 +281,10 @@ xfs_trim_gather_extents( &extents->extent_list); *blocks_trimmed += flen; next_extent: - error = xfs_btree_decrement(cur, 0, &i); + if (tcur->by_bno) + error = xfs_btree_increment(cur, 0, &i); + else + error = xfs_btree_decrement(cur, 0, &i); if (error) break; @@ -276,7 +294,7 @@ xfs_trim_gather_extents( * is no more extents to search. */ if (i == 0) - tcur->ar_blockcount = 0; + tcur->count = 0; } /* @@ -306,17 +324,22 @@ xfs_trim_should_stop(void) static int xfs_trim_extents( struct xfs_perag *pag, - xfs_daddr_t start, - xfs_daddr_t end, - xfs_daddr_t minlen, + xfs_agblock_t start, + xfs_agblock_t end, + xfs_extlen_t minlen, uint64_t *blocks_trimmed) { - struct xfs_alloc_rec_incore tcur = { - .ar_blockcount = pag->pagf_longest, - .ar_startblock = NULLAGBLOCK, + struct xfs_trim_cur tcur = { + .start = start, + .count = pag->pagf_longest, + .end = end, + .minlen = minlen, }; int error = 0; + if (start != 0 || end != pag->block_count) + tcur.by_bno = true; + do { struct xfs_busy_extents *extents; @@ -330,8 +353,8 @@ xfs_trim_extents( extents->owner = extents; INIT_LIST_HEAD(&extents->extent_list); - error = xfs_trim_gather_extents(pag, start, end, minlen, - &tcur, extents, blocks_trimmed); + error = xfs_trim_gather_extents(pag, &tcur, extents, + blocks_trimmed); if (error) { kfree(extents); break; @@ -354,7 +377,7 @@ xfs_trim_extents( if (xfs_trim_should_stop()) break; - } while (tcur.ar_blockcount != 0); + } while (tcur.count != 0); return error; @@ -378,8 +401,10 @@ xfs_ioc_trim( unsigned int granularity = bdev_discard_granularity(mp->m_ddev_targp->bt_bdev); struct fstrim_range range; - xfs_daddr_t start, end, minlen; - xfs_agnumber_t agno; + xfs_daddr_t start, end; + xfs_extlen_t minlen; + xfs_agnumber_t start_agno, end_agno; + xfs_agblock_t start_agbno, end_agbno; uint64_t blocks_trimmed = 0; int error, last_error = 0; @@ -399,7 +424,8 @@ xfs_ioc_trim( return -EFAULT; range.minlen = max_t(u64, granularity, range.minlen); - minlen = BTOBB(range.minlen); + minlen = XFS_B_TO_FSB(mp, range.minlen); + /* * Truncating down the len isn't actually quite correct, but using * BBTOB would mean we trivially get overflows for values @@ -413,15 +439,21 @@ xfs_ioc_trim( return -EINVAL; start = BTOBB(range.start); - end = start + BTOBBT(range.len) - 1; + end = min_t(xfs_daddr_t, start + BTOBBT(range.len), + XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks)) - 1; + + start_agno = xfs_daddr_to_agno(mp, start); + start_agbno = xfs_daddr_to_agbno(mp, start); + end_agno = xfs_daddr_to_agno(mp, end); + end_agbno = xfs_daddr_to_agbno(mp, end); - if (end > XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks) - 1) - end = XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks) - 1; + for_each_perag_range(mp, start_agno, end_agno, pag) { + xfs_agblock_t agend = pag->block_count; - agno = xfs_daddr_to_agno(mp, start); - for_each_perag_range(mp, agno, xfs_daddr_to_agno(mp, end), pag) { - error = xfs_trim_extents(pag, start, end, minlen, - &blocks_trimmed); + if (start_agno == end_agno) + agend = end_agbno; + error = xfs_trim_extents(pag, start_agbno, agend, minlen, + &blocks_trimmed); if (error) last_error = error; @@ -429,6 +461,7 @@ xfs_ioc_trim( xfs_perag_rele(pag); break; } + start_agbno = 0; } if (last_error) From 7560c937b4b5a3c671053be86ff00156a6fdd399 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 15 Apr 2024 14:55:11 -0700 Subject: [PATCH 0264/1702] xfs: Increase XFS_DEFER_OPS_NR_INODES to 5 Renames that generate parent pointer updates can join up to 5 inodes locked in sorted order. So we need to increase the number of defer ops inodes and relock them in the same way. Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong Reviewed-by: Catherine Hoang [djwong: have one sorting function] Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_defer.c | 6 +++++- fs/xfs/libxfs/xfs_defer.h | 8 +++++++- fs/xfs/xfs_inode.c | 27 ++++++++++++++++++--------- fs/xfs/xfs_inode.h | 2 ++ 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c index 061cc01245a91..4a078e07e1a0a 100644 --- a/fs/xfs/libxfs/xfs_defer.c +++ b/fs/xfs/libxfs/xfs_defer.c @@ -1092,7 +1092,11 @@ xfs_defer_ops_continue( ASSERT(!(tp->t_flags & XFS_TRANS_DIRTY)); /* Lock the captured resources to the new transaction. */ - if (dfc->dfc_held.dr_inos == 2) + if (dfc->dfc_held.dr_inos > 2) { + xfs_sort_inodes(dfc->dfc_held.dr_ip, dfc->dfc_held.dr_inos); + xfs_lock_inodes(dfc->dfc_held.dr_ip, dfc->dfc_held.dr_inos, + XFS_ILOCK_EXCL); + } else if (dfc->dfc_held.dr_inos == 2) xfs_lock_two_inodes(dfc->dfc_held.dr_ip[0], XFS_ILOCK_EXCL, dfc->dfc_held.dr_ip[1], XFS_ILOCK_EXCL); else if (dfc->dfc_held.dr_inos == 1) diff --git a/fs/xfs/libxfs/xfs_defer.h b/fs/xfs/libxfs/xfs_defer.h index 81cca60d70a3b..8b338031e487c 100644 --- a/fs/xfs/libxfs/xfs_defer.h +++ b/fs/xfs/libxfs/xfs_defer.h @@ -77,7 +77,13 @@ extern const struct xfs_defer_op_type xfs_exchmaps_defer_type; /* * Deferred operation item relogging limits. */ -#define XFS_DEFER_OPS_NR_INODES 2 /* join up to two inodes */ + +/* + * Rename w/ parent pointers can require up to 5 inodes with deferred ops to + * be joined to the transaction: src_dp, target_dp, src_ip, target_ip, and wip. + * These inodes are locked in sorted order by their inode numbers + */ +#define XFS_DEFER_OPS_NR_INODES 5 #define XFS_DEFER_OPS_NR_BUFS 2 /* join up to two buffers */ /* Resources that must be held across a transaction roll. */ diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 03dcb4ac04312..efd040094753f 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -418,7 +418,7 @@ xfs_lock_inumorder( * lock more than one at a time, lockdep will report false positives saying we * have violated locking orders. */ -static void +void xfs_lock_inodes( struct xfs_inode **ips, int inodes, @@ -2802,7 +2802,7 @@ xfs_sort_for_rename( struct xfs_inode **i_tab,/* out: sorted array of inodes */ int *num_inodes) /* in/out: inodes in array */ { - int i, j; + int i; ASSERT(*num_inodes == __XFS_SORT_INODES); memset(i_tab, 0, *num_inodes * sizeof(struct xfs_inode *)); @@ -2824,17 +2824,26 @@ xfs_sort_for_rename( i_tab[i++] = wip; *num_inodes = i; + xfs_sort_inodes(i_tab, *num_inodes); +} + +void +xfs_sort_inodes( + struct xfs_inode **i_tab, + unsigned int num_inodes) +{ + int i, j; + + ASSERT(num_inodes <= __XFS_SORT_INODES); + /* * Sort the elements via bubble sort. (Remember, there are at * most 5 elements to sort, so this is adequate.) */ - for (i = 0; i < *num_inodes; i++) { - for (j = 1; j < *num_inodes; j++) { - if (i_tab[j]->i_ino < i_tab[j-1]->i_ino) { - struct xfs_inode *temp = i_tab[j]; - i_tab[j] = i_tab[j-1]; - i_tab[j-1] = temp; - } + for (i = 0; i < num_inodes; i++) { + for (j = 1; j < num_inodes; j++) { + if (i_tab[j]->i_ino < i_tab[j-1]->i_ino) + swap(i_tab[j], i_tab[j - 1]); } } } diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index c74c48bc09453..a6da1ab8ab136 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -627,6 +627,8 @@ int xfs_ilock2_io_mmap(struct xfs_inode *ip1, struct xfs_inode *ip2); void xfs_iunlock2_io_mmap(struct xfs_inode *ip1, struct xfs_inode *ip2); void xfs_iunlock2_remapping(struct xfs_inode *ip1, struct xfs_inode *ip2); void xfs_bumplink(struct xfs_trans *tp, struct xfs_inode *ip); +void xfs_lock_inodes(struct xfs_inode **ips, int inodes, uint lock_mode); +void xfs_sort_inodes(struct xfs_inode **i_tab, unsigned int num_inodes); static inline bool xfs_inode_unlinked_incomplete( From c91fe20e5ae78484859278c4f55e1c7f89760537 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:55:09 -0700 Subject: [PATCH 0265/1702] docs: update offline parent pointer repair strategy Now update how xfs_repair checks and repairs parent pointer info. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- .../xfs/xfs-online-fsck-design.rst | 81 ++++++++++++++----- 1 file changed, 60 insertions(+), 21 deletions(-) diff --git a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst index 1ea4e59c9cdbd..70e3e629d8b3f 100644 --- a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst +++ b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst @@ -4675,26 +4675,56 @@ files are erased long before directory tree connectivity checks are performed. Parent pointer checks are therefore a second pass to be added to the existing connectivity checks: -1. After the set of surviving files has been established (i.e. phase 6), +1. After the set of surviving files has been established (phase 6), walk the surviving directories of each AG in the filesystem. This is already performed as part of the connectivity checks. -2. For each directory entry found, record the name in an xfblob, and store - ``(child_ag_inum, parent_inum, parent_gen, dirent_pos)`` tuples in a - per-AG in-memory slab. +2. For each directory entry found, + + a. If the name has already been stored in the xfblob, then use that cookie + and skip the next step. + + b. Otherwise, record the name in an xfblob, and remember the xfblob cookie. + Unique mappings are critical for + + 1. Deduplicating names to reduce memory usage, and + + 2. Creating a stable sort key for the parent pointer indexes so that the + parent pointer validation described below will work. + + c. Store ``(child_ag_inum, parent_inum, parent_gen, name_hash, name_len, + name_cookie)`` tuples in a per-AG in-memory slab. The ``name_hash`` + referenced in this section is the regular directory entry name hash, not + the specialized one used for parent pointer xattrs. 3. For each AG in the filesystem, - a. Sort the per-AG tuples in order of child_ag_inum, parent_inum, and - dirent_pos. + a. Sort the per-AG tuple set in order of ``child_ag_inum``, ``parent_inum``, + ``name_hash``, and ``name_cookie``. + Having a single ``name_cookie`` for each ``name`` is critical for + handling the uncommon case of a directory containing multiple hardlinks + to the same file where all the names hash to the same value. b. For each inode in the AG, 1. Scan the inode for parent pointers. - Record the names in a per-file xfblob, and store ``(parent_inum, - parent_gen, dirent_pos)`` tuples in a per-file slab. + For each parent pointer found, + + a. Validate the ondisk parent pointer. + If validation fails, move on to the next parent pointer in the + file. + + b. If the name has already been stored in the xfblob, then use that + cookie and skip the next step. + + c. Record the name in a per-file xfblob, and remember the xfblob + cookie. + + d. Store ``(parent_inum, parent_gen, name_hash, name_len, + name_cookie)`` tuples in a per-file slab. - 2. Sort the per-file tuples in order of parent_inum, and dirent_pos. + 2. Sort the per-file tuples in order of ``parent_inum``, ``name_hash``, + and ``name_cookie``. 3. Position one slab cursor at the start of the inode's records in the per-AG tuple slab. @@ -4703,28 +4733,37 @@ connectivity checks: 4. Position a second slab cursor at the start of the per-file tuple slab. - 5. Iterate the two cursors in lockstep, comparing the parent_ino and - dirent_pos fields of the records under each cursor. + 5. Iterate the two cursors in lockstep, comparing the ``parent_ino``, + ``name_hash``, and ``name_cookie`` fields of the records under each + cursor: - a. Tuples in the per-AG list but not the per-file list are missing and - need to be written to the inode. + a. If the per-AG cursor is at a lower point in the keyspace than the + per-file cursor, then the per-AG cursor points to a missing parent + pointer. + Add the parent pointer to the inode and advance the per-AG + cursor. - b. Tuples in the per-file list but not the per-AG list are dangling - and need to be removed from the inode. + b. If the per-file cursor is at a lower point in the keyspace than + the per-AG cursor, then the per-file cursor points to a dangling + parent pointer. + Remove the parent pointer from the inode and advance the per-file + cursor. - c. For tuples in both lists, update the parent_gen and name components - of the parent pointer if necessary. + c. Otherwise, both cursors point at the same parent pointer. + Update the parent_gen component if necessary. + Advance both cursors. 4. Move on to examining link counts, as we do today. The proposed patchset is the `offline parent pointers repair -`_ +`_ series. -Rebuilding directories from parent pointers in offline repair is very -challenging because it currently uses a single-pass scan of the filesystem -during phase 3 to decide which files are corrupt enough to be zapped. +Rebuilding directories from parent pointers in offline repair would be very +challenging because xfs_repair currently uses two single-pass scans of the +filesystem during phases 3 and 4 to decide which files are corrupt enough to be +zapped. This scan would have to be converted into a multi-pass scan: 1. The first pass of the scan zaps corrupt inodes, forks, and attributes From f103df763563ad6849307ed5985d1513acc586dd Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 15 Apr 2024 14:55:12 -0700 Subject: [PATCH 0266/1702] xfs: Increase XFS_QM_TRANS_MAXDQS to 5 With parent pointers enabled, a rename operation can update up to 5 inodes: src_dp, target_dp, src_ip, target_ip and wip. This causes their dquots to a be attached to the transaction chain, so we need to increase XFS_QM_TRANS_MAXDQS. This patch also add a helper function xfs_dqlockn to lock an arbitrary number of dquots. Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_dquot.c | 41 ++++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_dquot.h | 1 + fs/xfs/xfs_qm.h | 2 +- fs/xfs/xfs_trans_dquot.c | 15 ++++++++++----- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index c98cb468c3578..13aba84bd64af 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -1371,6 +1371,47 @@ xfs_dqlock2( } } +static int +xfs_dqtrx_cmp( + const void *a, + const void *b) +{ + const struct xfs_dqtrx *qa = a; + const struct xfs_dqtrx *qb = b; + + if (qa->qt_dquot->q_id > qb->qt_dquot->q_id) + return 1; + if (qa->qt_dquot->q_id < qb->qt_dquot->q_id) + return -1; + return 0; +} + +void +xfs_dqlockn( + struct xfs_dqtrx *q) +{ + unsigned int i; + + BUILD_BUG_ON(XFS_QM_TRANS_MAXDQS > MAX_LOCKDEP_SUBCLASSES); + + /* Sort in order of dquot id, do not allow duplicates */ + for (i = 0; i < XFS_QM_TRANS_MAXDQS && q[i].qt_dquot != NULL; i++) { + unsigned int j; + + for (j = 0; j < i; j++) + ASSERT(q[i].qt_dquot != q[j].qt_dquot); + } + if (i == 0) + return; + + sort(q, i, sizeof(struct xfs_dqtrx), xfs_dqtrx_cmp, NULL); + + mutex_lock(&q[0].qt_dquot->q_qlock); + for (i = 1; i < XFS_QM_TRANS_MAXDQS && q[i].qt_dquot != NULL; i++) + mutex_lock_nested(&q[i].qt_dquot->q_qlock, + XFS_QLOCK_NESTED + i - 1); +} + int __init xfs_qm_init(void) { diff --git a/fs/xfs/xfs_dquot.h b/fs/xfs/xfs_dquot.h index 956272d9b302f..677bb2dc9ac91 100644 --- a/fs/xfs/xfs_dquot.h +++ b/fs/xfs/xfs_dquot.h @@ -223,6 +223,7 @@ int xfs_qm_dqget_uncached(struct xfs_mount *mp, void xfs_qm_dqput(struct xfs_dquot *dqp); void xfs_dqlock2(struct xfs_dquot *, struct xfs_dquot *); +void xfs_dqlockn(struct xfs_dqtrx *q); void xfs_dquot_set_prealloc_limits(struct xfs_dquot *); diff --git a/fs/xfs/xfs_qm.h b/fs/xfs/xfs_qm.h index f5993012bf98f..6e09dfcd13e25 100644 --- a/fs/xfs/xfs_qm.h +++ b/fs/xfs/xfs_qm.h @@ -136,7 +136,7 @@ enum { XFS_QM_TRANS_PRJ, XFS_QM_TRANS_DQTYPES }; -#define XFS_QM_TRANS_MAXDQS 2 +#define XFS_QM_TRANS_MAXDQS 5 struct xfs_dquot_acct { struct xfs_dqtrx dqs[XFS_QM_TRANS_DQTYPES][XFS_QM_TRANS_MAXDQS]; }; diff --git a/fs/xfs/xfs_trans_dquot.c b/fs/xfs/xfs_trans_dquot.c index 577b535a595cb..b368e13424c4f 100644 --- a/fs/xfs/xfs_trans_dquot.c +++ b/fs/xfs/xfs_trans_dquot.c @@ -379,24 +379,29 @@ xfs_trans_mod_dquot( /* * Given an array of dqtrx structures, lock all the dquots associated and join - * them to the transaction, provided they have been modified. We know that the - * highest number of dquots of one type - usr, grp and prj - involved in a - * transaction is 3 so we don't need to make this very generic. + * them to the transaction, provided they have been modified. */ STATIC void xfs_trans_dqlockedjoin( struct xfs_trans *tp, struct xfs_dqtrx *q) { + unsigned int i; ASSERT(q[0].qt_dquot != NULL); if (q[1].qt_dquot == NULL) { xfs_dqlock(q[0].qt_dquot); xfs_trans_dqjoin(tp, q[0].qt_dquot); - } else { - ASSERT(XFS_QM_TRANS_MAXDQS == 2); + } else if (q[2].qt_dquot == NULL) { xfs_dqlock2(q[0].qt_dquot, q[1].qt_dquot); xfs_trans_dqjoin(tp, q[0].qt_dquot); xfs_trans_dqjoin(tp, q[1].qt_dquot); + } else { + xfs_dqlockn(q); + for (i = 0; i < XFS_QM_TRANS_MAXDQS; i++) { + if (q[i].qt_dquot == NULL) + break; + xfs_trans_dqjoin(tp, q[i].qt_dquot); + } } } From 67bdcd499909708195b9408c106b94250955c5ff Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:55:10 -0700 Subject: [PATCH 0267/1702] docs: describe xfs directory tree online fsck I've added a scrubber that checks the directory tree structure and fixes them; describe this in the design documentation. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- .../xfs/xfs-online-fsck-design.rst | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst index 70e3e629d8b3f..12aa638408304 100644 --- a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst +++ b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst @@ -4785,6 +4785,130 @@ This scan would have to be converted into a multi-pass scan: This code has not yet been constructed. +.. _dirtree: + +Case Study: Directory Tree Structure +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As mentioned earlier, the filesystem directory tree is supposed to be a +directed acylic graph structure. +However, each node in this graph is a separate ``xfs_inode`` object with its +own locks, which makes validating the tree qualities difficult. +Fortunately, non-directories are allowed to have multiple parents and cannot +have children, so only directories need to be scanned. +Directories typically constitute 5-10% of the files in a filesystem, which +reduces the amount of work dramatically. + +If the directory tree could be frozen, it would be easy to discover cycles and +disconnected regions by running a depth (or breadth) first search downwards +from the root directory and marking a bitmap for each directory found. +At any point in the walk, trying to set an already set bit means there is a +cycle. +After the scan completes, XORing the marked inode bitmap with the inode +allocation bitmap reveals disconnected inodes. +However, one of online repair's design goals is to avoid locking the entire +filesystem unless it's absolutely necessary. +Directory tree updates can move subtrees across the scanner wavefront on a live +filesystem, so the bitmap algorithm cannot be applied. + +Directory parent pointers enable an incremental approach to validation of the +tree structure. +Instead of using one thread to scan the entire filesystem, multiple threads can +walk from individual subdirectories upwards towards the root. +For this to work, all directory entries and parent pointers must be internally +consistent, each directory entry must have a parent pointer, and the link +counts of all directories must be correct. +Each scanner thread must be able to take the IOLOCK of an alleged parent +directory while holding the IOLOCK of the child directory to prevent either +directory from being moved within the tree. +This is not possible since the VFS does not take the IOLOCK of a child +subdirectory when moving that subdirectory, so instead the scanner stabilizes +the parent -> child relationship by taking the ILOCKs and installing a dirent +update hook to detect changes. + +The scanning process uses a dirent hook to detect changes to the directories +mentioned in the scan data. +The scan works as follows: + +1. For each subdirectory in the filesystem, + + a. For each parent pointer of that subdirectory, + + 1. Create a path object for that parent pointer, and mark the + subdirectory inode number in the path object's bitmap. + + 2. Record the parent pointer name and inode number in a path structure. + + 3. If the alleged parent is the subdirectory being scrubbed, the path is + a cycle. + Mark the path for deletion and repeat step 1a with the next + subdirectory parent pointer. + + 4. Try to mark the alleged parent inode number in a bitmap in the path + object. + If the bit is already set, then there is a cycle in the directory + tree. + Mark the path as a cycle and repeat step 1a with the next subdirectory + parent pointer. + + 5. Load the alleged parent. + If the alleged parent is not a linked directory, abort the scan + because the parent pointer information is inconsistent. + + 6. For each parent pointer of this alleged ancestor directory, + + a. Record the parent pointer name and inode number in the path object + if no parent has been set for that level. + + b. If an ancestor has more than one parent, mark the path as corrupt. + Repeat step 1a with the next subdirectory parent pointer. + + c. Repeat steps 1a3-1a6 for the ancestor identified in step 1a6a. + This repeats until the directory tree root is reached or no parents + are found. + + 7. If the walk terminates at the root directory, mark the path as ok. + + 8. If the walk terminates without reaching the root, mark the path as + disconnected. + +2. If the directory entry update hook triggers, check all paths already found + by the scan. + If the entry matches part of a path, mark that path and the scan stale. + When the scanner thread sees that the scan has been marked stale, it deletes + all scan data and starts over. + +Repairing the directory tree works as follows: + +1. Walk each path of the target subdirectory. + + a. Corrupt paths and cycle paths are counted as suspect. + + b. Paths already marked for deletion are counted as bad. + + c. Paths that reached the root are counted as good. + +2. If the subdirectory is either the root directory or has zero link count, + delete all incoming directory entries in the immediate parents. + Repairs are complete. + +3. If the subdirectory has exactly one path, set the dotdot entry to the + parent and exit. + +4. If the subdirectory has at least one good path, delete all the other + incoming directory entries in the immediate parents. + +5. If the subdirectory has no good paths and more than one suspect path, delete + all the other incoming directory entries in the immediate parents. + +6. If the subdirectory has zero paths, attach it to the lost and found. + +The proposed patches are in the +`directory tree repair +`_ +series. + + .. _orphanage: The Orphanage From 267979b4ce75767a9caf69072e1b92416e286a29 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 15 Apr 2024 14:55:12 -0700 Subject: [PATCH 0268/1702] xfs: Hold inode locks in xfs_ialloc Modify xfs_ialloc to hold locks after return. Caller will be responsible for manual unlock. We will need this later to hold locks across parent pointer operations Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong Reviewed-by: Catherine Hoang [djwong: hold the parent ilocked across transaction rolls too] Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_inode.c | 12 +++++++++--- fs/xfs/xfs_qm.c | 4 +++- fs/xfs/xfs_symlink.c | 6 ++++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index efd040094753f..2ec005e6c1dab 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -747,6 +747,8 @@ xfs_inode_inherit_flags2( /* * Initialise a newly allocated inode and return the in-core inode to the * caller locked exclusively. + * + * Caller is responsible for unlocking the inode manually upon return */ int xfs_init_new_inode( @@ -873,7 +875,7 @@ xfs_init_new_inode( /* * Log the new values stuffed into the inode. */ - xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); xfs_trans_log_inode(tp, ip, flags); /* now that we have an i_mode we can setup the inode structure */ @@ -1101,8 +1103,7 @@ xfs_create( * the transaction cancel unlocking dp so don't do it explicitly in the * error path. */ - xfs_trans_ijoin(tp, dp, XFS_ILOCK_EXCL); - unlock_dp_on_error = false; + xfs_trans_ijoin(tp, dp, 0); error = xfs_dir_createname(tp, dp, name, ip->i_ino, resblks - XFS_IALLOC_SPACE_RES(mp)); @@ -1151,6 +1152,8 @@ xfs_create( xfs_qm_dqrele(pdqp); *ipp = ip; + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_iunlock(dp, XFS_ILOCK_EXCL); return 0; out_trans_cancel: @@ -1162,6 +1165,7 @@ xfs_create( * transactions and deadlocks from xfs_inactive. */ if (ip) { + xfs_iunlock(ip, XFS_ILOCK_EXCL); xfs_finish_inode_setup(ip); xfs_irele(ip); } @@ -1247,6 +1251,7 @@ xfs_create_tmpfile( xfs_qm_dqrele(pdqp); *ipp = ip; + xfs_iunlock(ip, XFS_ILOCK_EXCL); return 0; out_trans_cancel: @@ -1258,6 +1263,7 @@ xfs_create_tmpfile( * transactions and deadlocks from xfs_inactive. */ if (ip) { + xfs_iunlock(ip, XFS_ILOCK_EXCL); xfs_finish_inode_setup(ip); xfs_irele(ip); } diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index 0f4cf4170c357..47120b745c47f 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -836,8 +836,10 @@ xfs_qm_qino_alloc( ASSERT(xfs_is_shutdown(mp)); xfs_alert(mp, "%s failed (error %d)!", __func__, error); } - if (need_alloc) + if (need_alloc) { + xfs_iunlock(*ipp, XFS_ILOCK_EXCL); xfs_finish_inode_setup(*ipp); + } return error; } diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index fb060aaf6d40f..85ef56fdd7dfe 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -172,8 +172,7 @@ xfs_symlink( * the transaction cancel unlocking dp so don't do it explicitly in the * error path. */ - xfs_trans_ijoin(tp, dp, XFS_ILOCK_EXCL); - unlock_dp_on_error = false; + xfs_trans_ijoin(tp, dp, 0); /* * Also attach the dquot(s) to it, if applicable. @@ -215,6 +214,8 @@ xfs_symlink( xfs_qm_dqrele(pdqp); *ipp = ip; + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_iunlock(dp, XFS_ILOCK_EXCL); return 0; out_trans_cancel: @@ -226,6 +227,7 @@ xfs_symlink( * transactions and deadlocks from xfs_inactive. */ if (ip) { + xfs_iunlock(ip, XFS_ILOCK_EXCL); xfs_finish_inode_setup(ip); xfs_irele(ip); } From bd5562111d58392298a3c3b93caad71dff681b4b Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 15 Apr 2024 14:55:13 -0700 Subject: [PATCH 0269/1702] xfs: Hold inode locks in xfs_trans_alloc_dir Modify xfs_trans_alloc_dir to hold locks after return. Caller will be responsible for manual unlock. We will need this later to hold locks across parent pointer operations Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong Reviewed-by: Catherine Hoang Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_inode.c | 14 ++++++++++++-- fs/xfs/xfs_trans.c | 9 +++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 2ec005e6c1dab..36e1012e156a1 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -1368,10 +1368,15 @@ xfs_link( if (xfs_has_wsync(mp) || xfs_has_dirsync(mp)) xfs_trans_set_sync(tp); - return xfs_trans_commit(tp); + error = xfs_trans_commit(tp); + xfs_iunlock(tdp, XFS_ILOCK_EXCL); + xfs_iunlock(sip, XFS_ILOCK_EXCL); + return error; error_return: xfs_trans_cancel(tp); + xfs_iunlock(tdp, XFS_ILOCK_EXCL); + xfs_iunlock(sip, XFS_ILOCK_EXCL); std_return: if (error == -ENOSPC && nospace_error) error = nospace_error; @@ -2781,15 +2786,20 @@ xfs_remove( error = xfs_trans_commit(tp); if (error) - goto std_return; + goto out_unlock; if (is_dir && xfs_inode_is_filestream(ip)) xfs_filestream_deassociate(ip); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_iunlock(dp, XFS_ILOCK_EXCL); return 0; out_trans_cancel: xfs_trans_cancel(tp); + out_unlock: + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_iunlock(dp, XFS_ILOCK_EXCL); std_return: return error; } diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 7350640059cc6..50d878d78a5e1 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -1430,6 +1430,8 @@ xfs_trans_alloc_ichange( * The caller must ensure that the on-disk dquots attached to this inode have * already been allocated and initialized. The ILOCKs will be dropped when the * transaction is committed or cancelled. + * + * Caller is responsible for unlocking the inodes manually upon return */ int xfs_trans_alloc_dir( @@ -1460,8 +1462,8 @@ xfs_trans_alloc_dir( xfs_lock_two_inodes(dp, XFS_ILOCK_EXCL, ip, XFS_ILOCK_EXCL); - xfs_trans_ijoin(tp, dp, XFS_ILOCK_EXCL); - xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, dp, 0); + xfs_trans_ijoin(tp, ip, 0); error = xfs_qm_dqattach_locked(dp, false); if (error) { @@ -1484,6 +1486,9 @@ xfs_trans_alloc_dir( if (error == -EDQUOT || error == -ENOSPC) { if (!retried) { xfs_trans_cancel(tp); + xfs_iunlock(dp, XFS_ILOCK_EXCL); + if (dp != ip) + xfs_iunlock(ip, XFS_ILOCK_EXCL); xfs_blockgc_free_quota(dp, 0); retried = true; goto retry; From 69291726caf12b28e1854cbffb73bd8354579fe7 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 15 Apr 2024 14:55:14 -0700 Subject: [PATCH 0270/1702] xfs: Hold inode locks in xfs_rename Modify xfs_rename to hold all inode locks across a rename operation We will need this later when we add parent pointers Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong Reviewed-by: Catherine Hoang Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_inode.c | 45 +++++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 36e1012e156a1..2aec7ab59aeb7 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -2804,6 +2804,21 @@ xfs_remove( return error; } +static inline void +xfs_iunlock_rename( + struct xfs_inode **i_tab, + int num_inodes) +{ + int i; + + for (i = num_inodes - 1; i >= 0; i--) { + /* Skip duplicate inodes if src and target dps are the same */ + if (!i_tab[i] || (i > 0 && i_tab[i] == i_tab[i - 1])) + continue; + xfs_iunlock(i_tab[i], XFS_ILOCK_EXCL); + } +} + /* * Enter all inodes for a rename transaction into a sorted array. */ @@ -3113,8 +3128,10 @@ xfs_rename( * Attach the dquots to the inodes */ error = xfs_qm_vop_rename_dqattach(inodes); - if (error) - goto out_trans_cancel; + if (error) { + xfs_trans_cancel(tp); + goto out_release_wip; + } /* * Lock all the participating inodes. Depending upon whether @@ -3125,18 +3142,16 @@ xfs_rename( xfs_lock_inodes(inodes, num_inodes, XFS_ILOCK_EXCL); /* - * Join all the inodes to the transaction. From this point on, - * we can rely on either trans_commit or trans_cancel to unlock - * them. + * Join all the inodes to the transaction. */ - xfs_trans_ijoin(tp, src_dp, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, src_dp, 0); if (new_parent) - xfs_trans_ijoin(tp, target_dp, XFS_ILOCK_EXCL); - xfs_trans_ijoin(tp, src_ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, target_dp, 0); + xfs_trans_ijoin(tp, src_ip, 0); if (target_ip) - xfs_trans_ijoin(tp, target_ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, target_ip, 0); if (wip) - xfs_trans_ijoin(tp, wip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, wip, 0); /* * If we are using project inheritance, we only allow renames @@ -3150,10 +3165,13 @@ xfs_rename( } /* RENAME_EXCHANGE is unique from here on. */ - if (flags & RENAME_EXCHANGE) - return xfs_cross_rename(tp, src_dp, src_name, src_ip, + if (flags & RENAME_EXCHANGE) { + error = xfs_cross_rename(tp, src_dp, src_name, src_ip, target_dp, target_name, target_ip, spaceres); + xfs_iunlock_rename(inodes, num_inodes); + return error; + } /* * Try to reserve quota to handle an expansion of the target directory. @@ -3167,6 +3185,7 @@ xfs_rename( if (error == -EDQUOT || error == -ENOSPC) { if (!retried) { xfs_trans_cancel(tp); + xfs_iunlock_rename(inodes, num_inodes); xfs_blockgc_free_quota(target_dp, 0); retried = true; goto retry; @@ -3393,12 +3412,14 @@ xfs_rename( xfs_dir_update_hook(src_dp, wip, 1, src_name); error = xfs_finish_rename(tp); + xfs_iunlock_rename(inodes, num_inodes); if (wip) xfs_irele(wip); return error; out_trans_cancel: xfs_trans_cancel(tp); + xfs_iunlock_rename(inodes, num_inodes); out_release_wip: if (wip) xfs_irele(wip); From 34ef5e17d5fd62a49bd9790834560261ebdec607 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:55:15 -0700 Subject: [PATCH 0271/1702] xfs: don't pick up IOLOCK during rmapbt repair scan Now that we've fixed the directory operations to hold the ILOCK until they're finished with rmapbt updates for directory shape changes, we no longer need to take this lock when scanning directories for rmapbt records. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/rmap_repair.c | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/fs/xfs/scrub/rmap_repair.c b/fs/xfs/scrub/rmap_repair.c index e8e07b683eab6..25acd69614c2c 100644 --- a/fs/xfs/scrub/rmap_repair.c +++ b/fs/xfs/scrub/rmap_repair.c @@ -578,23 +578,9 @@ xrep_rmap_scan_inode( struct xrep_rmap *rr, struct xfs_inode *ip) { - unsigned int lock_mode = 0; + unsigned int lock_mode = xrep_rmap_scan_ilock(ip); int error; - /* - * Directory updates (create/link/unlink/rename) drop the directory's - * ILOCK before finishing any rmapbt updates associated with directory - * shape changes. For this scan to coordinate correctly with the live - * update hook, we must take the only lock (i_rwsem) that is held all - * the way to dir op completion. This will get fixed by the parent - * pointer patchset. - */ - if (S_ISDIR(VFS_I(ip)->i_mode)) { - lock_mode = XFS_IOLOCK_SHARED; - xfs_ilock(ip, lock_mode); - } - lock_mode |= xrep_rmap_scan_ilock(ip); - /* Check the data fork. */ error = xrep_rmap_scan_ifork(rr, ip, XFS_DATA_FORK); if (error) From df760471477400ccd3ddcea85d2d6d92f4dad28c Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 15 Apr 2024 14:55:16 -0700 Subject: [PATCH 0272/1702] xfs: unlock new repair tempfiles after creation After creation, drop the ILOCK on temporary files that have been created to stage a repair. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/tempfile.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/xfs/scrub/tempfile.c b/fs/xfs/scrub/tempfile.c index c72e447eb8ec3..6f39504a216ea 100644 --- a/fs/xfs/scrub/tempfile.c +++ b/fs/xfs/scrub/tempfile.c @@ -153,6 +153,7 @@ xrep_tempfile_create( xfs_qm_dqrele(pdqp); /* Finish setting up the incore / vfs context. */ + xfs_iunlock(sc->tempip, XFS_ILOCK_EXCL); xfs_setup_iops(sc->tempip); xfs_finish_inode_setup(sc->tempip); @@ -168,6 +169,7 @@ xrep_tempfile_create( * transactions and deadlocks from xfs_inactive. */ if (sc->tempip) { + xfs_iunlock(sc->tempip, XFS_ILOCK_EXCL); xfs_finish_inode_setup(sc->tempip); xchk_irele(sc, sc->tempip); } From 94eabddc24b3ec2d9e0ff77e17722a2afb092155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 10 Apr 2024 15:41:10 +0200 Subject: [PATCH 0273/1702] HSI: omap_ssi_core: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). Trivially convert this driver from always returning zero in the remove callback to the void returning variant. Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/bc6b1caafa977346b33c1040d0f8e616bc0457bf.1712756364.git.u.kleine-koenig@pengutronix.de Signed-off-by: Sebastian Reichel --- drivers/hsi/controllers/omap_ssi_core.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/hsi/controllers/omap_ssi_core.c b/drivers/hsi/controllers/omap_ssi_core.c index 6802efb4d6cdc..3140990a61644 100644 --- a/drivers/hsi/controllers/omap_ssi_core.c +++ b/drivers/hsi/controllers/omap_ssi_core.c @@ -547,7 +547,7 @@ static int ssi_probe(struct platform_device *pd) return err; } -static int ssi_remove(struct platform_device *pd) +static void ssi_remove(struct platform_device *pd) { struct hsi_controller *ssi = platform_get_drvdata(pd); @@ -561,8 +561,6 @@ static int ssi_remove(struct platform_device *pd) platform_set_drvdata(pd, NULL); pm_runtime_disable(&pd->dev); - - return 0; } #ifdef CONFIG_PM @@ -618,7 +616,7 @@ MODULE_DEVICE_TABLE(of, omap_ssi_of_match); static struct platform_driver ssi_pdriver = { .probe = ssi_probe, - .remove = ssi_remove, + .remove_new = ssi_remove, .driver = { .name = "omap_ssi", .pm = DEV_PM_OPS, From c076486b6a28aa584b3e86312442bac09279a015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 10 Apr 2024 15:41:11 +0200 Subject: [PATCH 0274/1702] HSI: omap_ssi_port: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). Trivially convert this driver from always returning zero in the remove callback to the void returning variant. Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/11e06f4cce041436bd63fb24361f3cee06bd2d59.1712756364.git.u.kleine-koenig@pengutronix.de Signed-off-by: Sebastian Reichel --- drivers/hsi/controllers/omap_ssi_port.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/hsi/controllers/omap_ssi_port.c b/drivers/hsi/controllers/omap_ssi_port.c index c78d9c9f73712..f0b3eca7376e4 100644 --- a/drivers/hsi/controllers/omap_ssi_port.c +++ b/drivers/hsi/controllers/omap_ssi_port.c @@ -1224,7 +1224,7 @@ static int ssi_port_probe(struct platform_device *pd) return err; } -static int ssi_port_remove(struct platform_device *pd) +static void ssi_port_remove(struct platform_device *pd) { struct hsi_port *port = platform_get_drvdata(pd); struct omap_ssi_port *omap_port = hsi_port_drvdata(port); @@ -1251,8 +1251,6 @@ static int ssi_port_remove(struct platform_device *pd) pm_runtime_dont_use_autosuspend(&pd->dev); pm_runtime_disable(&pd->dev); - - return 0; } static int ssi_restore_divisor(struct omap_ssi_port *omap_port) @@ -1387,7 +1385,7 @@ MODULE_DEVICE_TABLE(of, omap_ssi_port_of_match); struct platform_driver ssi_port_pdriver = { .probe = ssi_port_probe, - .remove = ssi_port_remove, + .remove_new = ssi_port_remove, .driver = { .name = "omap_ssi_port", .of_match_table = omap_ssi_port_of_match, From 23f59f4e837bba9db8d25ae85b8455d53b23665b Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Fri, 12 Apr 2024 01:47:36 -0700 Subject: [PATCH 0275/1702] RDMA/mana_ib: Use num_comp_vectors of ib_device Use num_comp_vectors of struct ib_device instead of max_num_queues from gdma_context. Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1712911656-17352-1-git-send-email-kotaranov@linux.microsoft.com Reviewed-by: Long Li Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/cq.c | 7 +------ drivers/infiniband/hw/mana/device.c | 2 +- drivers/infiniband/hw/mana/qp.c | 4 ++-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index c9129218f1be1..dc931b9c34919 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -12,19 +12,14 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_device *ibdev = ibcq->device; struct mana_ib_create_cq ucmd = {}; struct mana_ib_dev *mdev; - struct gdma_context *gc; int err; mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); - gc = mdev_to_gc(mdev); if (udata->inlen < sizeof(ucmd)) return -EINVAL; - if (attr->comp_vector > gc->max_num_queues) - return -EINVAL; - - cq->comp_vector = attr->comp_vector; + cq->comp_vector = attr->comp_vector % ibdev->num_comp_vectors; err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)); if (err) { diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c index 6fa902ee80a62..07e97de318863 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -74,7 +74,7 @@ static int mana_ib_probe(struct auxiliary_device *adev, * num_comp_vectors needs to set to the max MSIX index * when interrupts and event queues are implemented */ - dev->ib_dev.num_comp_vectors = 1; + dev->ib_dev.num_comp_vectors = mdev->gdma_context->max_num_queues; dev->ib_dev.dev.parent = mdev->gdma_context->dev; ret = mana_gd_register_device(&mdev->gdma_context->mana_ib); diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 8fedf6e019251..280e85a83f7eb 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -198,7 +198,7 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, cq_spec.gdma_region = cq->queue.gdma_region; cq_spec.queue_size = cq->cqe * COMP_ENTRY_SIZE; cq_spec.modr_ctx_id = 0; - eq = &mpc->ac->eqs[cq->comp_vector % gc->max_num_queues]; + eq = &mpc->ac->eqs[cq->comp_vector]; cq_spec.attached_eq = eq->eq->id; ret = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_RQ, @@ -357,7 +357,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, cq_spec.gdma_region = send_cq->queue.gdma_region; cq_spec.queue_size = send_cq->cqe * COMP_ENTRY_SIZE; cq_spec.modr_ctx_id = 0; - eq_vec = send_cq->comp_vector % gc->max_num_queues; + eq_vec = send_cq->comp_vector; eq = &mpc->ac->eqs[eq_vec]; cq_spec.attached_eq = eq->eq->id; From 98b889c43935c43ad15783dbfb1e59b4ee7f4a56 Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Wed, 10 Apr 2024 01:42:26 -0700 Subject: [PATCH 0276/1702] RDMA/mana_ib: Add EQ creation for rnic adapter Create an error EQ for the RNIC adapter. Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1712738551-22075-2-git-send-email-kotaranov@linux.microsoft.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/device.c | 13 ++++++++++--- drivers/infiniband/hw/mana/main.c | 26 ++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 5 +++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c index 07e97de318863..08fdd917d0752 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -92,15 +92,23 @@ static int mana_ib_probe(struct auxiliary_device *adev, goto deregister_device; } + ret = mana_ib_create_eqs(dev); + if (ret) { + ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret); + goto deregister_device; + } + ret = ib_register_device(&dev->ib_dev, "mana_%d", mdev->gdma_context->dev); if (ret) - goto deregister_device; + goto destroy_eqs; dev_set_drvdata(&adev->dev, dev); return 0; +destroy_eqs: + mana_ib_destroy_eqs(dev); deregister_device: mana_gd_deregister_device(dev->gdma_dev); free_ib_device: @@ -113,9 +121,8 @@ static void mana_ib_remove(struct auxiliary_device *adev) struct mana_ib_dev *dev = dev_get_drvdata(&adev->dev); ib_unregister_device(&dev->ib_dev); - + mana_ib_destroy_eqs(dev); mana_gd_deregister_device(dev->gdma_dev); - ib_dealloc_device(&dev->ib_dev); } diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index b31dcff326994..bcf6b282f9d7a 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -611,3 +611,29 @@ int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev) return 0; } + +int mana_ib_create_eqs(struct mana_ib_dev *mdev) +{ + struct gdma_context *gc = mdev_to_gc(mdev); + struct gdma_queue_spec spec = {}; + int err; + + spec.type = GDMA_EQ; + spec.monitor_avl_buf = false; + spec.queue_size = EQ_SIZE; + spec.eq.callback = NULL; + spec.eq.context = mdev; + spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE; + spec.eq.msix_index = 0; + + err = mana_gd_create_mana_eq(&gc->mana_ib, &spec, &mdev->fatal_err_eq); + if (err) + return err; + + return 0; +} + +void mana_ib_destroy_eqs(struct mana_ib_dev *mdev) +{ + mana_gd_destroy_queue(mdev_to_gc(mdev), mdev->fatal_err_eq); +} diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index ceca21cef72ab..7c55204125de3 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -54,6 +54,7 @@ struct mana_ib_queue { struct mana_ib_dev { struct ib_device ib_dev; struct gdma_dev *gdma_dev; + struct gdma_queue *fatal_err_eq; struct mana_ib_adapter_caps adapter_caps; }; @@ -233,4 +234,8 @@ int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index, void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext); int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *mdev); + +int mana_ib_create_eqs(struct mana_ib_dev *mdev); + +void mana_ib_destroy_eqs(struct mana_ib_dev *mdev); #endif From 1a79c2b9d4a08788cf1554981f10d23fbad77d11 Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Wed, 10 Apr 2024 01:42:27 -0700 Subject: [PATCH 0277/1702] RDMA/mana_ib: Create and destroy rnic adapter Add functions for RNIC creation and destruction. If creation fails, the ib_probe fails as well. Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1712738551-22075-3-git-send-email-kotaranov@linux.microsoft.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/device.c | 9 +++++- drivers/infiniband/hw/mana/main.c | 43 ++++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 28 ++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c index 08fdd917d0752..721e2ab8388fc 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -98,15 +98,21 @@ static int mana_ib_probe(struct auxiliary_device *adev, goto deregister_device; } + ret = mana_ib_gd_create_rnic_adapter(dev); + if (ret) + goto destroy_eqs; + ret = ib_register_device(&dev->ib_dev, "mana_%d", mdev->gdma_context->dev); if (ret) - goto destroy_eqs; + goto destroy_rnic; dev_set_drvdata(&adev->dev, dev); return 0; +destroy_rnic: + mana_ib_gd_destroy_rnic_adapter(dev); destroy_eqs: mana_ib_destroy_eqs(dev); deregister_device: @@ -121,6 +127,7 @@ static void mana_ib_remove(struct auxiliary_device *adev) struct mana_ib_dev *dev = dev_get_drvdata(&adev->dev); ib_unregister_device(&dev->ib_dev); + mana_ib_gd_destroy_rnic_adapter(dev); mana_ib_destroy_eqs(dev); mana_gd_deregister_device(dev->gdma_dev); ib_dealloc_device(&dev->ib_dev); diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index bcf6b282f9d7a..344e85f4940d8 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -637,3 +637,46 @@ void mana_ib_destroy_eqs(struct mana_ib_dev *mdev) { mana_gd_destroy_queue(mdev_to_gc(mdev), mdev->fatal_err_eq); } + +int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev) +{ + struct mana_rnic_create_adapter_resp resp = {}; + struct mana_rnic_create_adapter_req req = {}; + struct gdma_context *gc = mdev_to_gc(mdev); + int err; + + mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_ADAPTER, sizeof(req), sizeof(resp)); + req.hdr.req.msg_version = GDMA_MESSAGE_V2; + req.hdr.dev_id = gc->mana_ib.dev_id; + req.notify_eq_id = mdev->fatal_err_eq->id; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err) { + ibdev_err(&mdev->ib_dev, "Failed to create RNIC adapter err %d", err); + return err; + } + mdev->adapter_handle = resp.adapter; + + return 0; +} + +int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev) +{ + struct mana_rnic_destroy_adapter_resp resp = {}; + struct mana_rnic_destroy_adapter_req req = {}; + struct gdma_context *gc; + int err; + + gc = mdev_to_gc(mdev); + mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_ADAPTER, sizeof(req), sizeof(resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err) { + ibdev_err(&mdev->ib_dev, "Failed to destroy RNIC adapter err %d", err); + return err; + } + + return 0; +} diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index 7c55204125de3..842f9c63a495b 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -54,6 +54,7 @@ struct mana_ib_queue { struct mana_ib_dev { struct ib_device ib_dev; struct gdma_dev *gdma_dev; + mana_handle_t adapter_handle; struct gdma_queue *fatal_err_eq; struct mana_ib_adapter_caps adapter_caps; }; @@ -113,6 +114,8 @@ struct mana_ib_rwq_ind_table { enum mana_ib_command_code { MANA_IB_GET_ADAPTER_CAP = 0x30001, + MANA_IB_CREATE_ADAPTER = 0x30002, + MANA_IB_DESTROY_ADAPTER = 0x30003, }; struct mana_ib_query_adapter_caps_req { @@ -141,6 +144,27 @@ struct mana_ib_query_adapter_caps_resp { u32 max_inline_data_size; }; /* HW Data */ +struct mana_rnic_create_adapter_req { + struct gdma_req_hdr hdr; + u32 notify_eq_id; + u32 reserved; + u64 feature_flags; +}; /*HW Data */ + +struct mana_rnic_create_adapter_resp { + struct gdma_resp_hdr hdr; + mana_handle_t adapter; +}; /* HW Data */ + +struct mana_rnic_destroy_adapter_req { + struct gdma_req_hdr hdr; + mana_handle_t adapter; +}; /*HW Data */ + +struct mana_rnic_destroy_adapter_resp { + struct gdma_resp_hdr hdr; +}; /* HW Data */ + static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev) { return mdev->gdma_dev->gdma_context; @@ -238,4 +262,8 @@ int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *mdev); int mana_ib_create_eqs(struct mana_ib_dev *mdev); void mana_ib_destroy_eqs(struct mana_ib_dev *mdev); + +int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev); + +int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev); #endif From 4bda1d5332ec1b00262ad53f6a4cfa88190a048d Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Wed, 10 Apr 2024 01:42:28 -0700 Subject: [PATCH 0278/1702] RDMA/mana_ib: Implement port parameters Implement port parameters for RNIC: 1) extend query_port() method 2) implement get_link_layer() 3) implement query_pkey() Only port 1 can store GIDs. Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1712738551-22075-4-git-send-email-kotaranov@linux.microsoft.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/device.c | 2 ++ drivers/infiniband/hw/mana/main.c | 37 +++++++++++++++++++++++++++- drivers/infiniband/hw/mana/mana_ib.h | 4 +++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c index 721e2ab8388fc..ef04cc48264cd 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -29,12 +29,14 @@ static const struct ib_device_ops mana_ib_dev_ops = { .destroy_rwq_ind_table = mana_ib_destroy_rwq_ind_table, .destroy_wq = mana_ib_destroy_wq, .disassociate_ucontext = mana_ib_disassociate_ucontext, + .get_link_layer = mana_ib_get_link_layer, .get_port_immutable = mana_ib_get_port_immutable, .mmap = mana_ib_mmap, .modify_qp = mana_ib_modify_qp, .modify_wq = mana_ib_modify_wq, .query_device = mana_ib_query_device, .query_gid = mana_ib_query_gid, + .query_pkey = mana_ib_query_pkey, .query_port = mana_ib_query_port, .reg_user_mr = mana_ib_reg_user_mr, diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index 344e85f4940d8..b2817c92f1c0e 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -555,7 +555,42 @@ int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, int mana_ib_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { - /* This version doesn't return port properties */ + struct net_device *ndev = mana_ib_get_netdev(ibdev, port); + + if (!ndev) + return -EINVAL; + + memset(props, 0, sizeof(*props)); + props->max_mtu = IB_MTU_4096; + props->active_mtu = ib_mtu_int_to_enum(ndev->mtu); + + if (netif_carrier_ok(ndev) && netif_running(ndev)) { + props->state = IB_PORT_ACTIVE; + props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; + } else { + props->state = IB_PORT_DOWN; + props->phys_state = IB_PORT_PHYS_STATE_DISABLED; + } + + props->active_width = IB_WIDTH_4X; + props->active_speed = IB_SPEED_EDR; + props->pkey_tbl_len = 1; + if (port == 1) + props->gid_tbl_len = 16; + + return 0; +} + +enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_num) +{ + return IB_LINK_LAYER_ETHERNET; +} + +int mana_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) +{ + if (index != 0) + return -EINVAL; + *pkey = IB_DEFAULT_PKEY_FULL; return 0; } diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index 842f9c63a495b..b9117cbc76297 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -266,4 +266,8 @@ void mana_ib_destroy_eqs(struct mana_ib_dev *mdev); int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev); int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev); + +int mana_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey); + +enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_num); #endif From 8b184e4f1c328d9b37994f66224550befdefe49b Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Wed, 10 Apr 2024 01:42:29 -0700 Subject: [PATCH 0279/1702] RDMA/mana_ib: Enable RoCE on port 1 Set netdev and RoCEv2 flag to enable GID population on port 1. Use GIDs of the master netdev. As mc->ports[] stores slave devices, use a helper to get the master netdev. Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1712738551-22075-5-git-send-email-kotaranov@linux.microsoft.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/device.c | 15 +++++++++++++++ drivers/infiniband/hw/mana/main.c | 15 +++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c index ef04cc48264cd..77994d32f87b4 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -53,6 +53,7 @@ static int mana_ib_probe(struct auxiliary_device *adev, { struct mana_adev *madev = container_of(adev, struct mana_adev, adev); struct gdma_dev *mdev = madev->mdev; + struct net_device *upper_ndev; struct mana_context *mc; struct mana_ib_dev *dev; int ret; @@ -79,6 +80,20 @@ static int mana_ib_probe(struct auxiliary_device *adev, dev->ib_dev.num_comp_vectors = mdev->gdma_context->max_num_queues; dev->ib_dev.dev.parent = mdev->gdma_context->dev; + rcu_read_lock(); /* required to get upper dev */ + upper_ndev = netdev_master_upper_dev_get_rcu(mc->ports[0]); + if (!upper_ndev) { + rcu_read_unlock(); + ibdev_err(&dev->ib_dev, "Failed to get master netdev"); + goto free_ib_device; + } + ret = ib_device_set_netdev(&dev->ib_dev, upper_ndev, 1); + rcu_read_unlock(); + if (ret) { + ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret); + goto free_ib_device; + } + ret = mana_gd_register_device(&mdev->gdma_context->mana_ib); if (ret) { ibdev_err(&dev->ib_dev, "Failed to register device, ret %d", diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index b2817c92f1c0e..c020183385d4b 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -525,11 +525,18 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma) int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { - /* - * This version only support RAW_PACKET - * other values need to be filled for other types - */ + struct ib_port_attr attr; + int err; + + err = ib_query_port(ibdev, port_num, &attr); + if (err) + return err; + + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET; + if (port_num == 1) + immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; return 0; } From faafb8b126ad6043663a77e6b234bca932f60694 Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Wed, 10 Apr 2024 01:42:30 -0700 Subject: [PATCH 0280/1702] RDMA/mana_ib: Adding and deleting GIDs Implement add_gid and del_gid for RNIC. IPv4 and IPv6 addresses are supported. Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1712738551-22075-6-git-send-email-kotaranov@linux.microsoft.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/device.c | 2 + drivers/infiniband/hw/mana/main.c | 60 ++++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 35 ++++++++++++++++ 3 files changed, 97 insertions(+) diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c index 77994d32f87b4..5d9fd59b1ff2e 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -15,6 +15,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { .driver_id = RDMA_DRIVER_MANA, .uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION, + .add_gid = mana_ib_gd_add_gid, .alloc_pd = mana_ib_alloc_pd, .alloc_ucontext = mana_ib_alloc_ucontext, .create_cq = mana_ib_create_cq, @@ -23,6 +24,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { .create_wq = mana_ib_create_wq, .dealloc_pd = mana_ib_dealloc_pd, .dealloc_ucontext = mana_ib_dealloc_ucontext, + .del_gid = mana_ib_gd_del_gid, .dereg_mr = mana_ib_dereg_mr, .destroy_cq = mana_ib_destroy_cq, .destroy_qp = mana_ib_destroy_qp, diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index c020183385d4b..e404762ba0296 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -722,3 +722,63 @@ int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev) return 0; } + +int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context) +{ + struct mana_ib_dev *mdev = container_of(attr->device, struct mana_ib_dev, ib_dev); + enum rdma_network_type ntype = rdma_gid_attr_network_type(attr); + struct mana_rnic_config_addr_resp resp = {}; + struct gdma_context *gc = mdev_to_gc(mdev); + struct mana_rnic_config_addr_req req = {}; + int err; + + if (ntype != RDMA_NETWORK_IPV4 && ntype != RDMA_NETWORK_IPV6) { + ibdev_dbg(&mdev->ib_dev, "Unsupported rdma network type %d", ntype); + return -EINVAL; + } + + mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_IP_ADDR, sizeof(req), sizeof(resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + req.op = ADDR_OP_ADD; + req.sgid_type = (ntype == RDMA_NETWORK_IPV6) ? SGID_TYPE_IPV6 : SGID_TYPE_IPV4; + copy_in_reverse(req.ip_addr, attr->gid.raw, sizeof(union ib_gid)); + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err) { + ibdev_err(&mdev->ib_dev, "Failed to config IP addr err %d\n", err); + return err; + } + + return 0; +} + +int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context) +{ + struct mana_ib_dev *mdev = container_of(attr->device, struct mana_ib_dev, ib_dev); + enum rdma_network_type ntype = rdma_gid_attr_network_type(attr); + struct mana_rnic_config_addr_resp resp = {}; + struct gdma_context *gc = mdev_to_gc(mdev); + struct mana_rnic_config_addr_req req = {}; + int err; + + if (ntype != RDMA_NETWORK_IPV4 && ntype != RDMA_NETWORK_IPV6) { + ibdev_dbg(&mdev->ib_dev, "Unsupported rdma network type %d", ntype); + return -EINVAL; + } + + mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_IP_ADDR, sizeof(req), sizeof(resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + req.op = ADDR_OP_REMOVE; + req.sgid_type = (ntype == RDMA_NETWORK_IPV6) ? SGID_TYPE_IPV6 : SGID_TYPE_IPV4; + copy_in_reverse(req.ip_addr, attr->gid.raw, sizeof(union ib_gid)); + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err) { + ibdev_err(&mdev->ib_dev, "Failed to config IP addr err %d\n", err); + return err; + } + + return 0; +} diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index b9117cbc76297..89ac5b39dbce4 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -116,6 +116,7 @@ enum mana_ib_command_code { MANA_IB_GET_ADAPTER_CAP = 0x30001, MANA_IB_CREATE_ADAPTER = 0x30002, MANA_IB_DESTROY_ADAPTER = 0x30003, + MANA_IB_CONFIG_IP_ADDR = 0x30004, }; struct mana_ib_query_adapter_caps_req { @@ -165,6 +166,28 @@ struct mana_rnic_destroy_adapter_resp { struct gdma_resp_hdr hdr; }; /* HW Data */ +enum mana_ib_addr_op { + ADDR_OP_ADD = 1, + ADDR_OP_REMOVE = 2, +}; + +enum sgid_entry_type { + SGID_TYPE_IPV4 = 1, + SGID_TYPE_IPV6 = 2, +}; + +struct mana_rnic_config_addr_req { + struct gdma_req_hdr hdr; + mana_handle_t adapter; + enum mana_ib_addr_op op; + enum sgid_entry_type sgid_type; + u8 ip_addr[16]; +}; /* HW Data */ + +struct mana_rnic_config_addr_resp { + struct gdma_resp_hdr hdr; +}; /* HW Data */ + static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev) { return mdev->gdma_dev->gdma_context; @@ -181,6 +204,14 @@ static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32 return mc->ports[port - 1]; } +static inline void copy_in_reverse(u8 *dst, const u8 *src, u32 size) +{ + u32 i; + + for (i = 0; i < size; i++) + dst[size - 1 - i] = src[i]; +} + int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq); int mana_ib_create_zero_offset_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem, @@ -270,4 +301,8 @@ int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev); int mana_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey); enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_num); + +int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context); + +int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context); #endif From 8859f009ace237ffc165c95edcc113d3824b9bf3 Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Wed, 10 Apr 2024 01:42:31 -0700 Subject: [PATCH 0281/1702] RDMA/mana_ib: Configure mac address in RNIC Set local mac address in RNIC, which is required by the HW. Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1712738551-22075-7-git-send-email-kotaranov@linux.microsoft.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/device.c | 9 +++++++++ drivers/infiniband/hw/mana/main.c | 22 ++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 15 +++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c index 5d9fd59b1ff2e..fca4d0d85c645 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -58,6 +58,7 @@ static int mana_ib_probe(struct auxiliary_device *adev, struct net_device *upper_ndev; struct mana_context *mc; struct mana_ib_dev *dev; + u8 mac_addr[ETH_ALEN]; int ret; mc = mdev->driver_data; @@ -89,6 +90,7 @@ static int mana_ib_probe(struct auxiliary_device *adev, ibdev_err(&dev->ib_dev, "Failed to get master netdev"); goto free_ib_device; } + ether_addr_copy(mac_addr, upper_ndev->dev_addr); ret = ib_device_set_netdev(&dev->ib_dev, upper_ndev, 1); rcu_read_unlock(); if (ret) { @@ -121,6 +123,13 @@ static int mana_ib_probe(struct auxiliary_device *adev, if (ret) goto destroy_eqs; + ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr); + if (ret) { + ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", + ret); + goto destroy_rnic; + } + ret = ib_register_device(&dev->ib_dev, "mana_%d", mdev->gdma_context->dev); if (ret) diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index e404762ba0296..f5401471bffe2 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -782,3 +782,25 @@ int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context) return 0; } + +int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 *mac) +{ + struct mana_rnic_config_mac_addr_resp resp = {}; + struct mana_rnic_config_mac_addr_req req = {}; + struct gdma_context *gc = mdev_to_gc(mdev); + int err; + + mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_MAC_ADDR, sizeof(req), sizeof(resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + req.op = op; + copy_in_reverse(req.mac_addr, mac, ETH_ALEN); + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err) { + ibdev_err(&mdev->ib_dev, "Failed to config Mac addr err %d", err); + return err; + } + + return 0; +} diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index 89ac5b39dbce4..4c1240da0c5fc 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -117,6 +117,7 @@ enum mana_ib_command_code { MANA_IB_CREATE_ADAPTER = 0x30002, MANA_IB_DESTROY_ADAPTER = 0x30003, MANA_IB_CONFIG_IP_ADDR = 0x30004, + MANA_IB_CONFIG_MAC_ADDR = 0x30005, }; struct mana_ib_query_adapter_caps_req { @@ -188,6 +189,18 @@ struct mana_rnic_config_addr_resp { struct gdma_resp_hdr hdr; }; /* HW Data */ +struct mana_rnic_config_mac_addr_req { + struct gdma_req_hdr hdr; + mana_handle_t adapter; + enum mana_ib_addr_op op; + u8 mac_addr[ETH_ALEN]; + u8 reserved[6]; +}; /* HW Data */ + +struct mana_rnic_config_mac_addr_resp { + struct gdma_resp_hdr hdr; +}; /* HW Data */ + static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev) { return mdev->gdma_dev->gdma_context; @@ -305,4 +318,6 @@ enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_n int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context); int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context); + +int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 *mac); #endif From 203b70fda63425a4eb29f03f9074859afe821a39 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Thu, 11 Apr 2024 11:38:51 +0800 Subject: [PATCH 0282/1702] RDMA/hns: Fix return value in hns_roce_map_mr_sg As described in the ib_map_mr_sg function comment, it returns the number of sg elements that were mapped to the memory region. However, hns_roce_map_mr_sg returns the number of pages required for mapping the DMA area. Fix it. Fixes: 9b2cf76c9f05 ("RDMA/hns: Optimize PBL buffer allocation process") Signed-off-by: Zhengchao Shao Link: https://lore.kernel.org/r/20240411033851.2884771-1-shaozhengchao@huawei.com Reviewed-by: Junxian Huang Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_mr.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 9e05b57a2d67d..80c050d7d0ea6 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -441,18 +441,18 @@ int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, struct ib_device *ibdev = &hr_dev->ib_dev; struct hns_roce_mr *mr = to_hr_mr(ibmr); struct hns_roce_mtr *mtr = &mr->pbl_mtr; - int ret = 0; + int ret, sg_num = 0; mr->npages = 0; mr->page_list = kvcalloc(mr->pbl_mtr.hem_cfg.buf_pg_count, sizeof(dma_addr_t), GFP_KERNEL); if (!mr->page_list) - return ret; + return sg_num; - ret = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, hns_roce_set_page); - if (ret < 1) { + sg_num = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, hns_roce_set_page); + if (sg_num < 1) { ibdev_err(ibdev, "failed to store sg pages %u %u, cnt = %d.\n", - mr->npages, mr->pbl_mtr.hem_cfg.buf_pg_count, ret); + mr->npages, mr->pbl_mtr.hem_cfg.buf_pg_count, sg_num); goto err_page_list; } @@ -463,17 +463,16 @@ int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, ret = hns_roce_mtr_map(hr_dev, mtr, mr->page_list, mr->npages); if (ret) { ibdev_err(ibdev, "failed to map sg mtr, ret = %d.\n", ret); - ret = 0; + sg_num = 0; } else { mr->pbl_mtr.hem_cfg.buf_pg_shift = (u32)ilog2(ibmr->page_size); - ret = mr->npages; } err_page_list: kvfree(mr->page_list); mr->page_list = NULL; - return ret; + return sg_num; } static void hns_roce_mw_free(struct hns_roce_dev *hr_dev, From bfb6be401470206ac02cbfdaf7b76ee040c1ae3d Mon Sep 17 00:00:00 2001 From: Yangyang Li Date: Fri, 12 Apr 2024 17:16:07 +0800 Subject: [PATCH 0283/1702] RDMA/hns: Use macro instead of magic number Use macro instead of magic number. Signed-off-by: Yangyang Li Signed-off-by: Chengchang Tang Signed-off-by: Junxian Huang Link: https://lore.kernel.org/r/20240412091616.370789-2-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 34 ++++++++++++---------- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 13 +++++++++ drivers/infiniband/hw/hns/hns_roce_qp.c | 3 +- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 423ab66c5856d..30ac5fb5ab16f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -3204,13 +3204,14 @@ static int set_mtpt_pbl(struct hns_roce_dev *hr_dev, /* Aligned to the hardware address access unit */ for (i = 0; i < ARRAY_SIZE(pages); i++) - pages[i] >>= 6; + pages[i] >>= MPT_PBL_BUF_ADDR_S; pbl_ba = hns_roce_get_mtr_ba(&mr->pbl_mtr); mpt_entry->pbl_size = cpu_to_le32(mr->npages); - mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> 3); - hr_reg_write(mpt_entry, MPT_PBL_BA_H, upper_32_bits(pbl_ba >> 3)); + mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> MPT_PBL_BA_ADDR_S); + hr_reg_write(mpt_entry, MPT_PBL_BA_H, + upper_32_bits(pbl_ba >> MPT_PBL_BA_ADDR_S)); mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0])); hr_reg_write(mpt_entry, MPT_PA0_H, upper_32_bits(pages[0])); @@ -3331,8 +3332,10 @@ static int hns_roce_v2_frmr_write_mtpt(struct hns_roce_dev *hr_dev, mpt_entry->pbl_size = cpu_to_le32(mr->npages); - mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >> 3)); - hr_reg_write(mpt_entry, MPT_PBL_BA_H, upper_32_bits(pbl_ba >> 3)); + mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >> + MPT_PBL_BA_ADDR_S)); + hr_reg_write(mpt_entry, MPT_PBL_BA_H, + upper_32_bits(pbl_ba >> MPT_PBL_BA_ADDR_S)); return 0; } @@ -3578,14 +3581,14 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.ba_pg_shift)); hr_reg_write(cq_context, CQC_CQE_BUF_PG_SZ, to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.buf_pg_shift)); - hr_reg_write(cq_context, CQC_CQE_BA_L, dma_handle >> 3); - hr_reg_write(cq_context, CQC_CQE_BA_H, (dma_handle >> (32 + 3))); + hr_reg_write(cq_context, CQC_CQE_BA_L, dma_handle >> CQC_CQE_BA_L_S); + hr_reg_write(cq_context, CQC_CQE_BA_H, dma_handle >> CQC_CQE_BA_H_S); hr_reg_write_bool(cq_context, CQC_DB_RECORD_EN, hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB); hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_L, ((u32)hr_cq->db.dma) >> 1); hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_H, - hr_cq->db.dma >> 32); + hr_cq->db.dma >> CQC_CQE_DB_RECORD_ADDR_H_S); hr_reg_write(cq_context, CQC_CQ_MAX_CNT, HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM); hr_reg_write(cq_context, CQC_CQ_PERIOD, @@ -4517,16 +4520,16 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, return -EINVAL; } - hr_reg_write(context, QPC_TRRL_BA_L, trrl_ba >> 4); + hr_reg_write(context, QPC_TRRL_BA_L, trrl_ba >> QPC_TRRL_BA_L_S); hr_reg_clear(qpc_mask, QPC_TRRL_BA_L); - context->trrl_ba = cpu_to_le32(trrl_ba >> (16 + 4)); + context->trrl_ba = cpu_to_le32(trrl_ba >> QPC_TRRL_BA_M_S); qpc_mask->trrl_ba = 0; - hr_reg_write(context, QPC_TRRL_BA_H, trrl_ba >> (32 + 16 + 4)); + hr_reg_write(context, QPC_TRRL_BA_H, trrl_ba >> QPC_TRRL_BA_H_S); hr_reg_clear(qpc_mask, QPC_TRRL_BA_H); - context->irrl_ba = cpu_to_le32(irrl_ba >> 6); + context->irrl_ba = cpu_to_le32(irrl_ba >> QPC_IRRL_BA_L_S); qpc_mask->irrl_ba = 0; - hr_reg_write(context, QPC_IRRL_BA_H, irrl_ba >> (32 + 6)); + hr_reg_write(context, QPC_IRRL_BA_H, irrl_ba >> QPC_IRRL_BA_H_S); hr_reg_clear(qpc_mask, QPC_IRRL_BA_H); hr_reg_enable(context, QPC_RMT_E2E); @@ -4588,8 +4591,9 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, hr_reg_clear(qpc_mask, QPC_TRRL_HEAD_MAX); hr_reg_clear(qpc_mask, QPC_TRRL_TAIL_MAX); +#define MAX_LP_SGEN 3 /* rocee send 2^lp_sgen_ini segs every time */ - hr_reg_write(context, QPC_LP_SGEN_INI, 3); + hr_reg_write(context, QPC_LP_SGEN_INI, MAX_LP_SGEN); hr_reg_clear(qpc_mask, QPC_LP_SGEN_INI); if (udata && ibqp->qp_type == IB_QPT_RC && @@ -4681,7 +4685,7 @@ static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr, *tail = (*tail == hr_dev->caps.num_qps - 1) ? 0 : (*tail + 1); list_for_each_entry(hr_dip, &hr_dev->dip_list, node) { - if (!memcmp(grh->dgid.raw, hr_dip->dgid, 16)) { + if (!memcmp(grh->dgid.raw, hr_dip->dgid, GID_LEN_V2)) { *dip_idx = hr_dip->dip_idx; goto out; } diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index df04bc8ede57b..4bac34f6bbe8a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -276,6 +276,10 @@ struct hns_roce_v2_cq_context { __le32 byte_64_se_cqe_idx; }; +#define CQC_CQE_BA_L_S 3 +#define CQC_CQE_BA_H_S (32 + CQC_CQE_BA_L_S) +#define CQC_CQE_DB_RECORD_ADDR_H_S 32 + #define HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM 0x0 #define HNS_ROCE_V2_CQ_DEFAULT_INTERVAL 0x0 @@ -447,6 +451,12 @@ struct hns_roce_v2_qp_context { struct hns_roce_v2_qp_context_ex ext; }; +#define QPC_TRRL_BA_L_S 4 +#define QPC_TRRL_BA_M_S (16 + QPC_TRRL_BA_L_S) +#define QPC_TRRL_BA_H_S (32 + QPC_TRRL_BA_M_S) +#define QPC_IRRL_BA_L_S 6 +#define QPC_IRRL_BA_H_S (32 + QPC_IRRL_BA_L_S) + #define QPC_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_qp_context, h, l) #define QPC_TST QPC_FIELD_LOC(2, 0) @@ -716,6 +726,9 @@ struct hns_roce_v2_mpt_entry { __le32 byte_64_buf_pa1; }; +#define MPT_PBL_BUF_ADDR_S 6 +#define MPT_PBL_BA_ADDR_S 3 + #define MPT_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_mpt_entry, h, l) #define MPT_ST MPT_FIELD_LOC(1, 0) diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 697230f964b10..cac3fe588672c 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -410,7 +410,8 @@ static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) bankid = get_qp_bankid(hr_qp->qpn); - ida_free(&hr_dev->qp_table.bank[bankid].ida, hr_qp->qpn >> 3); + ida_free(&hr_dev->qp_table.bank[bankid].ida, + hr_qp->qpn / HNS_ROCE_QP_BANK_NUM); mutex_lock(&hr_dev->qp_table.bank_mutex); hr_dev->qp_table.bank[bankid].inuse--; From f4caa864af84f801a5821ea2ba6c1cc46f8252c1 Mon Sep 17 00:00:00 2001 From: Chengchang Tang Date: Fri, 12 Apr 2024 17:16:08 +0800 Subject: [PATCH 0284/1702] RDMA/hns: Remove unused parameters and variables Remove unused parameters and variables. Signed-off-by: Chengchang Tang Signed-off-by: Junxian Huang Link: https://lore.kernel.org/r/20240412091616.370789-3-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_alloc.c | 3 +-- drivers/infiniband/hw/hns/hns_roce_device.h | 5 ++--- drivers/infiniband/hw/hns/hns_roce_hem.c | 13 +++++-------- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 20 +++++++------------- drivers/infiniband/hw/hns/hns_roce_mr.c | 4 ++-- drivers/infiniband/hw/hns/hns_roce_qp.c | 4 +--- drivers/infiniband/hw/hns/hns_roce_srq.c | 4 ++-- 7 files changed, 20 insertions(+), 33 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c index 11a78ceae5689..950c133d4220e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_alloc.c +++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c @@ -153,8 +153,7 @@ int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, return total; } -int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, - int buf_cnt, struct ib_umem *umem, +int hns_roce_get_umem_bufs(dma_addr_t *bufs, int buf_cnt, struct ib_umem *umem, unsigned int page_shift) { struct ib_block_iter biter; diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 78b4d19ff8486..37888f78849d4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -925,8 +925,7 @@ struct hns_roce_hw { int (*rereg_write_mtpt)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, int flags, void *mb_buf); - int (*frmr_write_mtpt)(struct hns_roce_dev *hr_dev, void *mb_buf, - struct hns_roce_mr *mr); + int (*frmr_write_mtpt)(void *mb_buf, struct hns_roce_mr *mr); int (*mw_write_mtpt)(void *mb_buf, struct hns_roce_mw *mw); void (*write_cqc)(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts, @@ -1232,7 +1231,7 @@ struct hns_roce_buf *hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, int buf_cnt, struct hns_roce_buf *buf, unsigned int page_shift); -int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, +int hns_roce_get_umem_bufs(dma_addr_t *bufs, int buf_cnt, struct ib_umem *umem, unsigned int page_shift); diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index a4b3f19161dc1..a9ea55506779d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -986,15 +986,13 @@ static void hem_list_free_all(struct hns_roce_dev *hr_dev, } } -static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr, - u64 table_addr) +static void hem_list_link_bt(void *base_addr, u64 table_addr) { *(u64 *)(base_addr) = table_addr; } /* assign L0 table address to hem from root bt */ -static void hem_list_assign_bt(struct hns_roce_dev *hr_dev, - struct hns_roce_hem_item *hem, void *cpu_addr, +static void hem_list_assign_bt(struct hns_roce_hem_item *hem, void *cpu_addr, u64 phy_addr) { hem->addr = cpu_addr; @@ -1163,8 +1161,7 @@ static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, if (level > 1) { pre = hem_ptrs[level - 1]; step = (cur->start - pre->start) / step * BA_BYTE_LEN; - hem_list_link_bt(hr_dev, pre->addr + step, - cur->dma_addr); + hem_list_link_bt(pre->addr + step, cur->dma_addr); } } @@ -1222,7 +1219,7 @@ static int alloc_fake_root_bt(struct hns_roce_dev *hr_dev, void *cpu_base, if (!hem) return -ENOMEM; - hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); + hem_list_assign_bt(hem, cpu_base, phy_base); list_add(&hem->list, branch_head); list_add(&hem->sibling, leaf_head); @@ -1245,7 +1242,7 @@ static int setup_middle_bt(struct hns_roce_dev *hr_dev, void *cpu_base, /* if exist mid bt, link L1 to L0 */ list_for_each_entry_safe(hem, temp_hem, branch_head, list) { offset = (hem->start - r->offset) / step * BA_BYTE_LEN; - hem_list_link_bt(hr_dev, cpu_base + offset, hem->dma_addr); + hem_list_link_bt(cpu_base + offset, hem->dma_addr); total++; } diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 30ac5fb5ab16f..e3f87090bad09 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -3304,8 +3304,7 @@ static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev, return ret; } -static int hns_roce_v2_frmr_write_mtpt(struct hns_roce_dev *hr_dev, - void *mb_buf, struct hns_roce_mr *mr) +static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr) { dma_addr_t pbl_ba = hns_roce_get_mtr_ba(&mr->pbl_mtr); struct hns_roce_v2_mpt_entry *mpt_entry; @@ -4216,8 +4215,7 @@ static void set_access_flags(struct hns_roce_qp *hr_qp, } static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp, - struct hns_roce_v2_qp_context *context, - struct hns_roce_v2_qp_context *qpc_mask) + struct hns_roce_v2_qp_context *context) { hr_reg_write(context, QPC_SGE_SHIFT, to_hr_hem_entries_shift(hr_qp->sge.sge_cnt, @@ -4239,7 +4237,6 @@ static inline int get_pdn(struct ib_pd *ib_pd) } static void modify_qp_reset_to_init(struct ib_qp *ibqp, - const struct ib_qp_attr *attr, struct hns_roce_v2_qp_context *context, struct hns_roce_v2_qp_context *qpc_mask) { @@ -4258,7 +4255,7 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp, hr_reg_write(context, QPC_RQWS, ilog2(hr_qp->rq.max_gs)); - set_qpc_wqe_cnt(hr_qp, context, qpc_mask); + set_qpc_wqe_cnt(hr_qp, context); /* No VLAN need to set 0xFFF */ hr_reg_write(context, QPC_VLAN_ID, 0xfff); @@ -4299,7 +4296,6 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp, } static void modify_qp_init_to_init(struct ib_qp *ibqp, - const struct ib_qp_attr *attr, struct hns_roce_v2_qp_context *context, struct hns_roce_v2_qp_context *qpc_mask) { @@ -4619,8 +4615,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, return 0; } -static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, - const struct ib_qp_attr *attr, int attr_mask, +static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, int attr_mask, struct hns_roce_v2_qp_context *context, struct hns_roce_v2_qp_context *qpc_mask) { @@ -5034,15 +5029,14 @@ static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp, if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { memset(qpc_mask, 0, hr_dev->caps.qpc_sz); - modify_qp_reset_to_init(ibqp, attr, context, qpc_mask); + modify_qp_reset_to_init(ibqp, context, qpc_mask); } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) { - modify_qp_init_to_init(ibqp, attr, context, qpc_mask); + modify_qp_init_to_init(ibqp, context, qpc_mask); } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context, qpc_mask, udata); } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) { - ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context, - qpc_mask); + ret = modify_qp_rtr_to_rts(ibqp, attr_mask, context, qpc_mask); } return ret; diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 80c050d7d0ea6..1a61dceb33197 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -162,7 +162,7 @@ static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev, if (mr->type != MR_TYPE_FRMR) ret = hr_dev->hw->write_mtpt(hr_dev, mailbox->buf, mr); else - ret = hr_dev->hw->frmr_write_mtpt(hr_dev, mailbox->buf, mr); + ret = hr_dev->hw->frmr_write_mtpt(mailbox->buf, mr); if (ret) { dev_err(dev, "failed to write mtpt, ret = %d.\n", ret); goto err_page; @@ -755,7 +755,7 @@ static int mtr_map_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr) return -ENOMEM; if (mtr->umem) - npage = hns_roce_get_umem_bufs(hr_dev, pages, page_count, + npage = hns_roce_get_umem_bufs(pages, page_count, mtr->umem, page_shift); else npage = hns_roce_get_kmem_bufs(hr_dev, pages, page_count, diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index cac3fe588672c..dc3cb26f434ec 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -1118,7 +1118,6 @@ static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, } static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, - struct ib_pd *ib_pd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata, struct hns_roce_qp *hr_qp) @@ -1272,7 +1271,6 @@ int hns_roce_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr, struct ib_device *ibdev = qp->device; struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); struct hns_roce_qp *hr_qp = to_hr_qp(qp); - struct ib_pd *pd = qp->pd; int ret; ret = check_qp_type(hr_dev, init_attr->qp_type, !!udata); @@ -1287,7 +1285,7 @@ int hns_roce_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr, hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; } - ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp); + ret = hns_roce_create_qp_common(hr_dev, init_attr, udata, hr_qp); if (ret) ibdev_err(ibdev, "create QP type 0x%x failed(%d)\n", init_attr->qp_type, ret); diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index 4abae94778544..e4705ccdfa651 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -250,7 +250,7 @@ static void free_srq_wqe_buf(struct hns_roce_dev *hr_dev, hns_roce_mtr_destroy(hr_dev, &srq->buf_mtr); } -static int alloc_srq_wrid(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) +static int alloc_srq_wrid(struct hns_roce_srq *srq) { srq->wrid = kvmalloc_array(srq->wqe_cnt, sizeof(u64), GFP_KERNEL); if (!srq->wrid) @@ -366,7 +366,7 @@ static int alloc_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, goto err_idx; if (!udata) { - ret = alloc_srq_wrid(hr_dev, srq); + ret = alloc_srq_wrid(srq); if (ret) goto err_wqe_buf; } From 2ce384307f2ddf39dc662878e151722199afc9ae Mon Sep 17 00:00:00 2001 From: Chengchang Tang Date: Fri, 12 Apr 2024 17:16:09 +0800 Subject: [PATCH 0285/1702] RDMA/hns: Add max_ah and cq moderation capacities in query_device() Add max_ah and cq moderation capacities to hns_roce_query_device(). Fixes: 9a4435375cd1 ("IB/hns: Add driver files for hns RoCE driver") Signed-off-by: Chengchang Tang Signed-off-by: Junxian Huang Link: https://lore.kernel.org/r/20240412091616.370789-4-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_device.h | 3 +++ drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 2 +- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 2 +- drivers/infiniband/hw/hns/hns_roce_main.c | 7 +++++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 37888f78849d4..ff0b3f68ee3a4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -100,6 +100,9 @@ #define CQ_BANKID_SHIFT 2 #define CQ_BANKID_MASK GENMASK(1, 0) +#define HNS_ROCE_MAX_CQ_COUNT 0xFFFF +#define HNS_ROCE_MAX_CQ_PERIOD 0xFFFF + enum { SERV_TYPE_RC, SERV_TYPE_UC, diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index e3f87090bad09..2a97a81ae19fc 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -5848,7 +5848,7 @@ static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) dev_info(hr_dev->dev, "cq_period(%u) reached the upper limit, adjusted to 65.\n", cq_period); - cq_period = HNS_ROCE_MAX_CQ_PERIOD; + cq_period = HNS_ROCE_MAX_CQ_PERIOD_HIP08; } cq_period *= HNS_ROCE_CLOCK_ADJUST; } diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index 4bac34f6bbe8a..def1d15a03c7e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -1347,7 +1347,7 @@ struct fmea_ram_ecc { /* only for RNR timeout issue of HIP08 */ #define HNS_ROCE_CLOCK_ADJUST 1000 -#define HNS_ROCE_MAX_CQ_PERIOD 65 +#define HNS_ROCE_MAX_CQ_PERIOD_HIP08 65 #define HNS_ROCE_MAX_EQ_PERIOD 65 #define HNS_ROCE_RNR_TIMER_10NS 1 #define HNS_ROCE_1US_CFG 999 diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 1dc60c2b2b7ab..4d94fcb8685ab 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -40,6 +40,7 @@ #include "hns_roce_common.h" #include "hns_roce_device.h" #include "hns_roce_hem.h" +#include "hns_roce_hw_v2.h" static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u32 port, const u8 *addr) @@ -192,6 +193,12 @@ static int hns_roce_query_device(struct ib_device *ib_dev, IB_ATOMIC_HCA : IB_ATOMIC_NONE; props->max_pkeys = 1; props->local_ca_ack_delay = hr_dev->caps.local_ca_ack_delay; + props->max_ah = INT_MAX; + props->cq_caps.max_cq_moderation_period = HNS_ROCE_MAX_CQ_PERIOD; + props->cq_caps.max_cq_moderation_count = HNS_ROCE_MAX_CQ_COUNT; + if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) + props->cq_caps.max_cq_moderation_period = HNS_ROCE_MAX_CQ_PERIOD_HIP08; + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) { props->max_srq = hr_dev->caps.num_srqs; props->max_srq_wr = hr_dev->caps.max_srq_wrs; From b46494b6f9c19f141114a57729e198698f40af37 Mon Sep 17 00:00:00 2001 From: Chengchang Tang Date: Fri, 12 Apr 2024 17:16:10 +0800 Subject: [PATCH 0286/1702] RDMA/hns: Fix deadlock on SRQ async events. xa_lock for SRQ table may be required in AEQ. Use xa_store_irq()/ xa_erase_irq() to avoid deadlock. Fixes: 81fce6291d99 ("RDMA/hns: Add SRQ asynchronous event support") Signed-off-by: Chengchang Tang Signed-off-by: Junxian Huang Link: https://lore.kernel.org/r/20240412091616.370789-5-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_main.c | 1 + drivers/infiniband/hw/hns/hns_roce_srq.c | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 4d94fcb8685ab..d202258368ed9 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -37,6 +37,7 @@ #include #include #include +#include "hnae3.h" #include "hns_roce_common.h" #include "hns_roce_device.h" #include "hns_roce_hem.h" diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index e4705ccdfa651..7210e53a82f33 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -123,7 +123,7 @@ static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) return ret; } - ret = xa_err(xa_store(&srq_table->xa, srq->srqn, srq, GFP_KERNEL)); + ret = xa_err(xa_store_irq(&srq_table->xa, srq->srqn, srq, GFP_KERNEL)); if (ret) { ibdev_err(ibdev, "failed to store SRQC, ret = %d.\n", ret); goto err_put; @@ -136,7 +136,7 @@ static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) return 0; err_xa: - xa_erase(&srq_table->xa, srq->srqn); + xa_erase_irq(&srq_table->xa, srq->srqn); err_put: hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn); @@ -154,7 +154,7 @@ static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) dev_err(hr_dev->dev, "DESTROY_SRQ failed (%d) for SRQN %06lx\n", ret, srq->srqn); - xa_erase(&srq_table->xa, srq->srqn); + xa_erase_irq(&srq_table->xa, srq->srqn); if (refcount_dec_and_test(&srq->refcount)) complete(&srq->free); From a942ec2745ca864cd8512142100e4027dc306a42 Mon Sep 17 00:00:00 2001 From: Chengchang Tang Date: Fri, 12 Apr 2024 17:16:11 +0800 Subject: [PATCH 0287/1702] RDMA/hns: Fix UAF for cq async event The refcount of CQ is not protected by locks. When CQ asynchronous events and CQ destruction are concurrent, CQ may have been released, which will cause UAF. Use the xa_lock() to protect the CQ refcount. Fixes: 9a4435375cd1 ("IB/hns: Add driver files for hns RoCE driver") Signed-off-by: Chengchang Tang Signed-off-by: Junxian Huang Link: https://lore.kernel.org/r/20240412091616.370789-6-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_cq.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 7250d0643b5c5..68e22f368d43a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -149,7 +149,7 @@ static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) return ret; } - ret = xa_err(xa_store(&cq_table->array, hr_cq->cqn, hr_cq, GFP_KERNEL)); + ret = xa_err(xa_store_irq(&cq_table->array, hr_cq->cqn, hr_cq, GFP_KERNEL)); if (ret) { ibdev_err(ibdev, "failed to xa_store CQ, ret = %d.\n", ret); goto err_put; @@ -163,7 +163,7 @@ static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) return 0; err_xa: - xa_erase(&cq_table->array, hr_cq->cqn); + xa_erase_irq(&cq_table->array, hr_cq->cqn); err_put: hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); @@ -182,7 +182,7 @@ static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) dev_err(dev, "DESTROY_CQ failed (%d) for CQN %06lx\n", ret, hr_cq->cqn); - xa_erase(&cq_table->array, hr_cq->cqn); + xa_erase_irq(&cq_table->array, hr_cq->cqn); /* Waiting interrupt process procedure carried out */ synchronize_irq(hr_dev->eq_table.eq[hr_cq->vector].irq); @@ -476,13 +476,6 @@ void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) struct ib_event event; struct ib_cq *ibcq; - hr_cq = xa_load(&hr_dev->cq_table.array, - cqn & (hr_dev->caps.num_cqs - 1)); - if (!hr_cq) { - dev_warn(dev, "async event for bogus CQ 0x%06x\n", cqn); - return; - } - if (event_type != HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID && event_type != HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR && event_type != HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW) { @@ -491,7 +484,16 @@ void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) return; } - refcount_inc(&hr_cq->refcount); + xa_lock(&hr_dev->cq_table.array); + hr_cq = xa_load(&hr_dev->cq_table.array, + cqn & (hr_dev->caps.num_cqs - 1)); + if (hr_cq) + refcount_inc(&hr_cq->refcount); + xa_unlock(&hr_dev->cq_table.array); + if (!hr_cq) { + dev_warn(dev, "async event for bogus CQ 0x%06x\n", cqn); + return; + } ibcq = &hr_cq->ib_cq; if (ibcq->event_handler) { From dc3bda6e568e9310b7cd07769dd70a3f0cd696ca Mon Sep 17 00:00:00 2001 From: wenglianfa Date: Fri, 12 Apr 2024 17:16:12 +0800 Subject: [PATCH 0288/1702] RDMA/hns: Fix mismatch exception rollback When dma_alloc_coherent() fails in hns_roce_alloc_hem(), just call kfree() to release hem instead of hns_roce_free_hem(). Fixes: c00743cbf2b8 ("RDMA/hns: Simplify 'struct hns_roce_hem' allocation") Signed-off-by: wenglianfa Signed-off-by: Junxian Huang Link: https://lore.kernel.org/r/20240412091616.370789-7-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_hem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index a9ea55506779d..1c2ec803e0303 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -281,7 +281,7 @@ static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, return hem; fail: - hns_roce_free_hem(hr_dev, hem); + kfree(hem); return NULL; } From ee045493283403969591087bd405fa280103282a Mon Sep 17 00:00:00 2001 From: Chengchang Tang Date: Fri, 12 Apr 2024 17:16:13 +0800 Subject: [PATCH 0289/1702] RDMA/hns: Fix GMV table pagesize GMV's BA table only supports 4K pages. Currently, PAGESIZE is used to calculate gmv_bt_num, which will cause an abnormal number of gmv_bt_num in a 64K OS. Fixes: d6d91e46210f ("RDMA/hns: Add support for configuring GMV table") Signed-off-by: Chengchang Tang Signed-off-by: Junxian Huang Link: https://lore.kernel.org/r/20240412091616.370789-8-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 2a97a81ae19fc..89d0f5b8be758 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -2101,7 +2101,7 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev) caps->gmv_bt_num * (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz)); - caps->gmv_entry_num = caps->gmv_bt_num * (PAGE_SIZE / + caps->gmv_entry_num = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz); } else { u32 func_num = max_t(u32, 1, hr_dev->func_num); From 9a84848dcee289966e8a2c21223bb0d7bc44f201 Mon Sep 17 00:00:00 2001 From: wenglianfa Date: Fri, 12 Apr 2024 17:16:14 +0800 Subject: [PATCH 0290/1702] RDMA/hns: Add mutex_destroy() Add mutex_destroy(). Signed-off-by: wenglianfa Signed-off-by: Junxian Huang Link: https://lore.kernel.org/r/20240412091616.370789-9-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_cq.c | 1 + drivers/infiniband/hw/hns/hns_roce_hem.c | 2 ++ drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 6 +++++- drivers/infiniband/hw/hns/hns_roce_main.c | 24 ++++++++++++++++++++-- drivers/infiniband/hw/hns/hns_roce_qp.c | 9 ++++++-- drivers/infiniband/hw/hns/hns_roce_srq.c | 2 ++ 6 files changed, 39 insertions(+), 5 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 68e22f368d43a..56dc3908da2f4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -536,4 +536,5 @@ void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev) for (i = 0; i < HNS_ROCE_CQ_BANK_NUM; i++) ida_destroy(&hr_dev->cq_table.bank[i].ida); + mutex_destroy(&hr_dev->cq_table.bank_mutex); } diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index 1c2ec803e0303..02baa853a76c9 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -877,6 +877,7 @@ void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, if (hns_roce_check_whether_mhop(hr_dev, table->type)) { hns_roce_cleanup_mhop_hem_table(hr_dev, table); + mutex_destroy(&table->mutex); return; } @@ -891,6 +892,7 @@ void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, hns_roce_free_hem(hr_dev, table->hem[i]); } + mutex_destroy(&table->mutex); kfree(table->hem); } diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 89d0f5b8be758..5d526b5c4b818 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -2667,6 +2667,8 @@ static void free_mr_exit(struct hns_roce_dev *hr_dev) kfree(free_mr->rsv_pd); free_mr->rsv_pd = NULL; } + + mutex_destroy(&free_mr->mutex); } static int free_mr_alloc_res(struct hns_roce_dev *hr_dev) @@ -2817,8 +2819,10 @@ static int free_mr_init(struct hns_roce_dev *hr_dev) mutex_init(&free_mr->mutex); ret = free_mr_alloc_res(hr_dev); - if (ret) + if (ret) { + mutex_destroy(&free_mr->mutex); return ret; + } ret = free_mr_modify_qp(hr_dev); if (ret) diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index d202258368ed9..4cb0af7335870 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -429,6 +429,9 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, return 0; error_fail_copy_to_udata: + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || + hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) + mutex_destroy(&context->page_mutex); hns_roce_dealloc_uar_entry(context); error_fail_uar_entry: @@ -445,6 +448,10 @@ static void hns_roce_dealloc_ucontext(struct ib_ucontext *ibcontext) struct hns_roce_ucontext *context = to_hr_ucontext(ibcontext); struct hns_roce_dev *hr_dev = to_hr_dev(ibcontext->device); + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || + hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) + mutex_destroy(&context->page_mutex); + hns_roce_dealloc_uar_entry(context); ida_free(&hr_dev->uar_ida.ida, (int)context->uar.logic_idx); @@ -933,6 +940,15 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) return ret; } +static void hns_roce_teardown_hca(struct hns_roce_dev *hr_dev) +{ + hns_roce_cleanup_bitmap(hr_dev); + + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || + hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) + mutex_destroy(&hr_dev->pgdir_mutex); +} + /** * hns_roce_setup_hca - setup host channel adapter * @hr_dev: pointer to hns roce device @@ -981,6 +997,10 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) err_uar_table_free: ida_destroy(&hr_dev->uar_ida.ida); + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || + hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) + mutex_destroy(&hr_dev->pgdir_mutex); + return ret; } @@ -1126,7 +1146,7 @@ int hns_roce_init(struct hns_roce_dev *hr_dev) hr_dev->hw->hw_exit(hr_dev); error_failed_engine_init: - hns_roce_cleanup_bitmap(hr_dev); + hns_roce_teardown_hca(hr_dev); error_failed_setup_hca: hns_roce_cleanup_hem(hr_dev); @@ -1156,7 +1176,7 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev) if (hr_dev->hw->hw_exit) hr_dev->hw->hw_exit(hr_dev); - hns_roce_cleanup_bitmap(hr_dev); + hns_roce_teardown_hca(hr_dev); hns_roce_cleanup_hem(hr_dev); if (hr_dev->cmd_mod) diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index dc3cb26f434ec..db34665d1dfbf 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -1140,7 +1140,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &ucmd); if (ret) { ibdev_err(ibdev, "failed to set QP param, ret = %d.\n", ret); - return ret; + goto err_out; } if (!udata) { @@ -1148,7 +1148,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, if (ret) { ibdev_err(ibdev, "failed to alloc wrid, ret = %d.\n", ret); - return ret; + goto err_out; } } @@ -1219,6 +1219,8 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, free_qp_buf(hr_dev, hr_qp); err_buf: free_kernel_wrid(hr_qp); +err_out: + mutex_destroy(&hr_qp->mutex); return ret; } @@ -1234,6 +1236,7 @@ void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, free_qp_buf(hr_dev, hr_qp); free_kernel_wrid(hr_qp); free_qp_db(hr_dev, hr_qp, udata); + mutex_destroy(&hr_qp->mutex); } static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type, @@ -1573,5 +1576,7 @@ void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev) for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) ida_destroy(&hr_dev->qp_table.bank[i].ida); + mutex_destroy(&hr_dev->qp_table.bank_mutex); + mutex_destroy(&hr_dev->qp_table.scc_mutex); kfree(hr_dev->qp_table.idx_table.spare_idx); } diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index 7210e53a82f33..f1997abc97cac 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -518,6 +518,7 @@ int hns_roce_create_srq(struct ib_srq *ib_srq, err_srq_buf: free_srq_buf(hr_dev, srq); err_out: + mutex_destroy(&srq->mutex); atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_SRQ_CREATE_ERR_CNT]); return ret; @@ -532,6 +533,7 @@ int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) free_srqn(hr_dev, srq); free_srq_db(hr_dev, srq, udata); free_srq_buf(hr_dev, srq); + mutex_destroy(&srq->mutex); return 0; } From 4125269bb9b22e1d8cdf4412c81be8074dbc61ca Mon Sep 17 00:00:00 2001 From: Chengchang Tang Date: Fri, 12 Apr 2024 17:16:15 +0800 Subject: [PATCH 0291/1702] RDMA/hns: Use complete parentheses in macros Use complete parentheses to ensure that macro expansion does not produce unexpected results. Fixes: a25d13cbe816 ("RDMA/hns: Add the interfaces to support multi hop addressing for the contexts in hip08") Signed-off-by: Chengchang Tang Signed-off-by: Junxian Huang Link: https://lore.kernel.org/r/20240412091616.370789-10-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_hem.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.h b/drivers/infiniband/hw/hns/hns_roce_hem.h index 6fb51db9682b8..9c415b2541af8 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.h +++ b/drivers/infiniband/hw/hns/hns_roce_hem.h @@ -57,16 +57,16 @@ enum { }; #define check_whether_bt_num_3(type, hop_num) \ - (type < HEM_TYPE_MTT && hop_num == 2) + ((type) < HEM_TYPE_MTT && (hop_num) == 2) #define check_whether_bt_num_2(type, hop_num) \ - ((type < HEM_TYPE_MTT && hop_num == 1) || \ - (type >= HEM_TYPE_MTT && hop_num == 2)) + (((type) < HEM_TYPE_MTT && (hop_num) == 1) || \ + ((type) >= HEM_TYPE_MTT && (hop_num) == 2)) #define check_whether_bt_num_1(type, hop_num) \ - ((type < HEM_TYPE_MTT && hop_num == HNS_ROCE_HOP_NUM_0) || \ - (type >= HEM_TYPE_MTT && hop_num == 1) || \ - (type >= HEM_TYPE_MTT && hop_num == HNS_ROCE_HOP_NUM_0)) + (((type) < HEM_TYPE_MTT && (hop_num) == HNS_ROCE_HOP_NUM_0) || \ + ((type) >= HEM_TYPE_MTT && (hop_num) == 1) || \ + ((type) >= HEM_TYPE_MTT && (hop_num) == HNS_ROCE_HOP_NUM_0)) struct hns_roce_hem { void *buf; From 349e859952285ab9689779fb46de163f13f18f43 Mon Sep 17 00:00:00 2001 From: Chengchang Tang Date: Fri, 12 Apr 2024 17:16:16 +0800 Subject: [PATCH 0292/1702] RDMA/hns: Modify the print level of CQE error Too much print may lead to a panic in kernel. Change ibdev_err() to ibdev_err_ratelimited(), and change the printing level of cqe dump to debug level. Fixes: 7c044adca272 ("RDMA/hns: Simplify the cqe code of poll cq") Signed-off-by: Chengchang Tang Signed-off-by: Junxian Huang Link: https://lore.kernel.org/r/20240412091616.370789-11-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 5d526b5c4b818..4287818a737f9 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -3713,8 +3713,9 @@ static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, wc->status == IB_WC_WR_FLUSH_ERR)) return; - ibdev_err(&hr_dev->ib_dev, "error cqe status 0x%x:\n", cqe_status); - print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 4, cqe, + ibdev_err_ratelimited(&hr_dev->ib_dev, "error cqe status 0x%x:\n", + cqe_status); + print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_NONE, 16, 4, cqe, cq->cqe_size, false); wc->vendor_err = hr_reg_read(cqe, CQE_SUB_STATUS); From ed53d391b6401d624172d1c97458d115893c6e0b Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Tue, 2 Apr 2024 16:36:05 +0200 Subject: [PATCH 0293/1702] dt-bindings: timer: renesas,cmt: Add R-Car V4M support Document support for the Compare Match Timer Type0 (CMT0) and Type1 (CMT1) in the Renesas R-Car V4M (R8A779H0) SoC. Signed-off-by: Geert Uytterhoeven Reviewed-by: Wolfram Sang Acked-by: Conor Dooley Link: https://lore.kernel.org/r/3e8a7a93261d8ad264dec2fa2784fe1bbfc4a23c.1712068536.git.geert+renesas@glider.be Signed-off-by: Rob Herring (Arm) --- Documentation/devicetree/bindings/timer/renesas,cmt.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/timer/renesas,cmt.yaml b/Documentation/devicetree/bindings/timer/renesas,cmt.yaml index a0be1755ea28b..5e09c04da30e4 100644 --- a/Documentation/devicetree/bindings/timer/renesas,cmt.yaml +++ b/Documentation/devicetree/bindings/timer/renesas,cmt.yaml @@ -103,6 +103,7 @@ properties: - renesas,r8a779a0-cmt0 # 32-bit CMT0 on R-Car V3U - renesas,r8a779f0-cmt0 # 32-bit CMT0 on R-Car S4-8 - renesas,r8a779g0-cmt0 # 32-bit CMT0 on R-Car V4H + - renesas,r8a779h0-cmt0 # 32-bit CMT0 on R-Car V4M - const: renesas,rcar-gen4-cmt0 # 32-bit CMT0 on R-Car Gen4 - items: @@ -110,6 +111,7 @@ properties: - renesas,r8a779a0-cmt1 # 48-bit CMT on R-Car V3U - renesas,r8a779f0-cmt1 # 48-bit CMT on R-Car S4-8 - renesas,r8a779g0-cmt1 # 48-bit CMT on R-Car V4H + - renesas,r8a779h0-cmt1 # 48-bit CMT on R-Car V4M - const: renesas,rcar-gen4-cmt1 # 48-bit CMT on R-Car Gen4 reg: From 58d4b25c883192377bf6255a46d46d0589e6f4ca Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Tue, 2 Apr 2024 16:37:02 +0200 Subject: [PATCH 0294/1702] dt-bindings: timer: renesas,tmu: Add R-Car V4M support Document support for the Timer Unit (TMU) in the Renesas R-Car V4M (R8A779H0) SoC. Signed-off-by: Geert Uytterhoeven Reviewed-by: Wolfram Sang Acked-by: Conor Dooley Link: https://lore.kernel.org/r/8a39386b1a33db6e83e852b3b365bc1adeb25242.1712068574.git.geert+renesas@glider.be Signed-off-by: Rob Herring (Arm) --- Documentation/devicetree/bindings/timer/renesas,tmu.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/timer/renesas,tmu.yaml b/Documentation/devicetree/bindings/timer/renesas,tmu.yaml index 84bbe15028a1d..360a5cf1ae9c7 100644 --- a/Documentation/devicetree/bindings/timer/renesas,tmu.yaml +++ b/Documentation/devicetree/bindings/timer/renesas,tmu.yaml @@ -39,6 +39,7 @@ properties: - renesas,tmu-r8a779a0 # R-Car V3U - renesas,tmu-r8a779f0 # R-Car S4-8 - renesas,tmu-r8a779g0 # R-Car V4H + - renesas,tmu-r8a779h0 # R-Car V4M - const: renesas,tmu reg: From a310822fe731ba22928c40bdd3b4da69483b0b81 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 11 Apr 2024 08:46:10 +0200 Subject: [PATCH 0295/1702] pinctrl: freescale: imx8ulp: fix module autoloading Add MODULE_DEVICE_TABLE(), so the module could be properly autoloaded based on the alias from of_device_id table. Pin controllers are considered core components, so usually they are built-in, however these can be built and used as modules on some generic kernel. Signed-off-by: Krzysztof Kozlowski Message-ID: <20240411064614.7409-1-krzk@kernel.org> Signed-off-by: Linus Walleij --- drivers/pinctrl/freescale/pinctrl-imx8ulp.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/pinctrl/freescale/pinctrl-imx8ulp.c b/drivers/pinctrl/freescale/pinctrl-imx8ulp.c index 2e86ca9fc7ac1..5632c72851479 100644 --- a/drivers/pinctrl/freescale/pinctrl-imx8ulp.c +++ b/drivers/pinctrl/freescale/pinctrl-imx8ulp.c @@ -252,6 +252,7 @@ static const struct of_device_id imx8ulp_pinctrl_of_match[] = { { .compatible = "fsl,imx8ulp-iomuxc1", }, { /* sentinel */ } }; +MODULE_DEVICE_TABLE(of, imx8ulp_pinctrl_of_match); static int imx8ulp_pinctrl_probe(struct platform_device *pdev) { From d42005c03a48fbb49d7a5c808992618f398963ca Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 11 Apr 2024 08:46:11 +0200 Subject: [PATCH 0296/1702] pinctrl: mediatek: fix module autoloading Add MODULE_DEVICE_TABLE(), so the module could be properly autoloaded based on the alias from of_device_id table. Pin controllers are considered core components, so usually they are built-in, however these can be built and used as modules on some generic kernel. Signed-off-by: Krzysztof Kozlowski Reviewed-by: AngeloGioacchino Del Regno Message-ID: <20240411064614.7409-2-krzk@kernel.org> Signed-off-by: Linus Walleij --- drivers/pinctrl/mediatek/pinctrl-mt6765.c | 1 + drivers/pinctrl/mediatek/pinctrl-mt6779.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/pinctrl/mediatek/pinctrl-mt6765.c b/drivers/pinctrl/mediatek/pinctrl-mt6765.c index f6ec41eb6e0c9..72609cf747607 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mt6765.c +++ b/drivers/pinctrl/mediatek/pinctrl-mt6765.c @@ -1086,6 +1086,7 @@ static const struct of_device_id mt6765_pinctrl_of_match[] = { { .compatible = "mediatek,mt6765-pinctrl", .data = &mt6765_data }, { } }; +MODULE_DEVICE_TABLE(of, mt6765_pinctrl_of_match); static struct platform_driver mt6765_pinctrl_driver = { .driver = { diff --git a/drivers/pinctrl/mediatek/pinctrl-mt6779.c b/drivers/pinctrl/mediatek/pinctrl-mt6779.c index 62d4f5ad6737d..591905e4132ab 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mt6779.c +++ b/drivers/pinctrl/mediatek/pinctrl-mt6779.c @@ -762,6 +762,7 @@ static const struct of_device_id mt6779_pinctrl_of_match[] = { { .compatible = "mediatek,mt6779-pinctrl", .data = &mt6779_data }, { } }; +MODULE_DEVICE_TABLE(of, mt6779_pinctrl_of_match); static struct platform_driver mt6779_pinctrl_driver = { .driver = { From 75589d6b11000605cebcbafd92e620dafc44694f Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 11 Apr 2024 08:46:12 +0200 Subject: [PATCH 0297/1702] pinctrl: loongson2: fix module autoloading Add MODULE_DEVICE_TABLE(), so the module could be properly autoloaded based on the alias from of_device_id table. Pin controllers are considered core components, so usually they are built-in, however these can be built and used as modules on some generic kernel. Signed-off-by: Krzysztof Kozlowski Message-ID: <20240411064614.7409-3-krzk@kernel.org> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-loongson2.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/pinctrl/pinctrl-loongson2.c b/drivers/pinctrl/pinctrl-loongson2.c index a72ffeca26fbf..4d4fbeadafb7c 100644 --- a/drivers/pinctrl/pinctrl-loongson2.c +++ b/drivers/pinctrl/pinctrl-loongson2.c @@ -286,6 +286,7 @@ static const struct of_device_id loongson2_pinctrl_dt_match[] = { }, { } }; +MODULE_DEVICE_TABLE(of, loongson2_pinctrl_dt_match); static struct platform_driver loongson2_pinctrl_driver = { .probe = loongson2_pinctrl_probe, From abda4619f41cd52a90b068ce728027c529d7b759 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 11 Apr 2024 08:46:13 +0200 Subject: [PATCH 0298/1702] pinctrl: qcom: sm7150: fix module autoloading Add MODULE_DEVICE_TABLE(), so the module could be properly autoloaded based on the alias from of_device_id table. Pin controllers are considered core components, so usually they are built-in, however these can be built and used as modules on some generic kernel. Reviewed-by: Konrad Dybcio Signed-off-by: Krzysztof Kozlowski Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Bjorn Andersson Message-ID: <20240411064614.7409-4-krzk@kernel.org> Signed-off-by: Linus Walleij --- drivers/pinctrl/qcom/pinctrl-sm7150.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/pinctrl/qcom/pinctrl-sm7150.c b/drivers/pinctrl/qcom/pinctrl-sm7150.c index c25357ca1963e..c542f9bc6bcd8 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm7150.c +++ b/drivers/pinctrl/qcom/pinctrl-sm7150.c @@ -1246,6 +1246,7 @@ static const struct of_device_id sm7150_tlmm_of_match[] = { { .compatible = "qcom,sm7150-tlmm", }, { }, }; +MODULE_DEVICE_TABLE(of, sm7150_tlmm_of_match); static struct platform_driver sm7150_tlmm_driver = { .driver = { From 9d2df36538d5ad7338007dc5726dabca4aa16813 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 11 Apr 2024 08:46:14 +0200 Subject: [PATCH 0299/1702] pinctrl: realtek: fix module autoloading Add MODULE_DEVICE_TABLE(), so the module could be properly autoloaded based on the alias from of_device_id table. Pin controllers are considered core components, so usually they are built-in, however these can be built and used as modules on some generic kernel. Signed-off-by: Krzysztof Kozlowski Message-ID: <20240411064614.7409-5-krzk@kernel.org> Signed-off-by: Linus Walleij --- drivers/pinctrl/realtek/pinctrl-rtd1315e.c | 1 + drivers/pinctrl/realtek/pinctrl-rtd1319d.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/pinctrl/realtek/pinctrl-rtd1315e.c b/drivers/pinctrl/realtek/pinctrl-rtd1315e.c index 10afc736a52bb..86f919122bed6 100644 --- a/drivers/pinctrl/realtek/pinctrl-rtd1315e.c +++ b/drivers/pinctrl/realtek/pinctrl-rtd1315e.c @@ -1414,6 +1414,7 @@ static const struct of_device_id rtd1315e_pinctrl_of_match[] = { { .compatible = "realtek,rtd1315e-pinctrl", }, {}, }; +MODULE_DEVICE_TABLE(of, rtd1315e_pinctrl_of_match); static struct platform_driver rtd1315e_pinctrl_driver = { .driver = { diff --git a/drivers/pinctrl/realtek/pinctrl-rtd1319d.c b/drivers/pinctrl/realtek/pinctrl-rtd1319d.c index b1a654ac30dcf..474c337d2d052 100644 --- a/drivers/pinctrl/realtek/pinctrl-rtd1319d.c +++ b/drivers/pinctrl/realtek/pinctrl-rtd1319d.c @@ -1584,6 +1584,7 @@ static const struct of_device_id rtd1319d_pinctrl_of_match[] = { { .compatible = "realtek,rtd1319d-pinctrl", }, {}, }; +MODULE_DEVICE_TABLE(of, rtd1319d_pinctrl_of_match); static struct platform_driver rtd1319d_pinctrl_driver = { .driver = { From 3ba11e684d163f4ab17719a8bbe1382276312e27 Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Fri, 12 Apr 2024 08:51:28 +0800 Subject: [PATCH 0300/1702] pinctrl: pinconf-generic: print hex value Hex value will be easier to match hardware register bits layout, so same as pinconf_generic_dump_config, print hex value. Signed-off-by: Peng Fan Message-ID: <20240412005128.2937486-1-peng.fan@oss.nxp.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/pinconf-generic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pinctrl/pinconf-generic.c b/drivers/pinctrl/pinconf-generic.c index cada5d18ffae1..80de389199bd4 100644 --- a/drivers/pinctrl/pinconf-generic.c +++ b/drivers/pinctrl/pinconf-generic.c @@ -88,7 +88,7 @@ static void pinconf_generic_dump_one(struct pinctrl_dev *pctldev, seq_puts(s, items[i].display); /* Print unit if available */ if (items[i].has_arg) { - seq_printf(s, " (%u", + seq_printf(s, " (0x%x", pinconf_to_config_argument(config)); if (items[i].format) seq_printf(s, " %s)", items[i].format); From 782b72a222a5f444f78606697e8f88893b9d0531 Mon Sep 17 00:00:00 2001 From: Herman van Hazendonk Date: Wed, 17 Apr 2024 09:35:32 +0200 Subject: [PATCH 0301/1702] dt-bindings: pinctrl: qcom,pmic-mpp: add support for PM8901 The PM8901 is used alongside the APQ8060/MSM8660 on the APQ8060 Dragonboard and HP TouchPad. It works the same as all others, so just add the compatible string for this variant. Signed-off-by: Herman van Hazendonk Message-ID: <20240417073532.3718510-1-github.com@herrie.org> Signed-off-by: Linus Walleij --- Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.yaml index fe717d8d47982..43146709e2044 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.yaml @@ -35,6 +35,7 @@ properties: - qcom,pm8038-mpp - qcom,pm8058-mpp - qcom,pm8821-mpp + - qcom,pm8901-mpp - qcom,pm8917-mpp - qcom,pm8921-mpp - const: qcom,ssbi-mpp From 795bb82d12a16a4cee42845b0e4c7e3276574e5d Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Tue, 16 Apr 2024 21:14:52 +0300 Subject: [PATCH 0302/1702] fsnotify: fix UAF from FS_ERROR event on a shutting down filesystem Protect against use after free when filesystem calls fsnotify_sb_error() during fs shutdown. Move freeing of sb->s_fsnotify_info to destroy_super_work(), because it may be accessed from fs shutdown context. Reported-by: syzbot+5e3f9b2a67b45f16d4e6@syzkaller.appspotmail.com Suggested-by: Jan Kara Link: https://lore.kernel.org/linux-fsdevel/20240416173211.4lnmgctyo4jn5fha@quack3/ Fixes: 07a3b8d0bf72 ("fsnotify: lazy attach fsnotify_sb_info state to sb") Reviewed-by: Christian Brauner Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara Message-Id: <20240416181452.567070-1-amir73il@gmail.com> --- fs/notify/fsnotify.c | 6 +++++- fs/super.c | 1 + include/linux/fsnotify_backend.h | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c index 2ae965ef37e85..ff69ae24c4e89 100644 --- a/fs/notify/fsnotify.c +++ b/fs/notify/fsnotify.c @@ -103,7 +103,11 @@ void fsnotify_sb_delete(struct super_block *sb) WARN_ON(fsnotify_sb_has_priority_watchers(sb, FSNOTIFY_PRIO_CONTENT)); WARN_ON(fsnotify_sb_has_priority_watchers(sb, FSNOTIFY_PRIO_PRE_CONTENT)); - kfree(sbinfo); +} + +void fsnotify_sb_free(struct super_block *sb) +{ + kfree(sb->s_fsnotify_info); } /* diff --git a/fs/super.c b/fs/super.c index 71d9779c42b10..81656c7807db8 100644 --- a/fs/super.c +++ b/fs/super.c @@ -274,6 +274,7 @@ static void destroy_super_work(struct work_struct *work) { struct super_block *s = container_of(work, struct super_block, destroy_work); + fsnotify_sb_free(s); security_sb_free(s); put_user_ns(s->s_user_ns); kfree(s->s_subtype); diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index 7f1ab8264e41d..4dd6143db2716 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -576,6 +576,7 @@ extern int __fsnotify_parent(struct dentry *dentry, __u32 mask, const void *data extern void __fsnotify_inode_delete(struct inode *inode); extern void __fsnotify_vfsmount_delete(struct vfsmount *mnt); extern void fsnotify_sb_delete(struct super_block *sb); +extern void fsnotify_sb_free(struct super_block *sb); extern u32 fsnotify_get_cookie(void); static inline __u32 fsnotify_parent_needed_mask(__u32 mask) @@ -880,6 +881,9 @@ static inline void __fsnotify_vfsmount_delete(struct vfsmount *mnt) static inline void fsnotify_sb_delete(struct super_block *sb) {} +static inline void fsnotify_sb_free(struct super_block *sb) +{} + static inline void fsnotify_update_flags(struct dentry *dentry) {} From d75d7dc26f29141fe31167c5414605d4087f0abb Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 11 Apr 2024 14:12:11 -0300 Subject: [PATCH 0303/1702] iommu/arm-smmu: Convert to domain_alloc_paging() Now that the BLOCKED and IDENTITY behaviors are managed with their own domains change to the domain_alloc_paging() op. The check for using_legacy_binding is now redundant, arm_smmu_def_domain_type() always returns IOMMU_DOMAIN_IDENTITY for this mode, so the core code will never attempt to create a DMA domain in the first place. Since commit a4fdd9762272 ("iommu: Use flush queue capability") the core code only passes in IDENTITY/BLOCKED/UNMANAGED/DMA domain types. It will not pass in IDENTITY or BLOCKED if the global statics exist, so the test for DMA is also redundant now too. Cc: Dmitry Baryshkov Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/0-v1-3632c65678e0+2f1-smmu_alloc_paging_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index c572d877b0e10..5935f44e1d1d6 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -859,14 +859,10 @@ static void arm_smmu_destroy_domain_context(struct arm_smmu_domain *smmu_domain) arm_smmu_rpm_put(smmu); } -static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) +static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev) { struct arm_smmu_domain *smmu_domain; - if (type != IOMMU_DOMAIN_UNMANAGED) { - if (using_legacy_binding || type != IOMMU_DOMAIN_DMA) - return NULL; - } /* * Allocate the domain and initialise some of its data structures. * We can't really do anything meaningful until we've added a @@ -1596,7 +1592,7 @@ static struct iommu_ops arm_smmu_ops = { .identity_domain = &arm_smmu_identity_domain, .blocked_domain = &arm_smmu_blocked_domain, .capable = arm_smmu_capable, - .domain_alloc = arm_smmu_domain_alloc, + .domain_alloc_paging = arm_smmu_domain_alloc_paging, .probe_device = arm_smmu_probe_device, .release_device = arm_smmu_release_device, .probe_finalize = arm_smmu_probe_finalize, From 80fea979dd9d48d67c5b48d2f690c5da3e543ebd Mon Sep 17 00:00:00 2001 From: Aleksandr Aprelkov Date: Wed, 3 Apr 2024 12:37:59 +0700 Subject: [PATCH 0304/1702] iommu/arm-smmu-v3: Free MSIs in case of ENOMEM If devm_add_action() returns -ENOMEM, then MSIs are allocated but not not freed on teardown. Use devm_add_action_or_reset() instead to keep the static analyser happy. Found by Linux Verification Center (linuxtesting.org) with SVACE. Signed-off-by: Aleksandr Aprelkov Link: https://lore.kernel.org/r/20240403053759.643164-1-aaprelkov@usergate.com [will: Tweak commit message, remove warning message] Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 79c18e95dd293..ac7164baa68c2 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3398,7 +3398,7 @@ static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX); /* Add callback to free MSIs on teardown */ - devm_add_action(dev, arm_smmu_free_msis, dev); + devm_add_action_or_reset(dev, arm_smmu_free_msis, dev); } static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) From 54a75d8f14c5ecb6636942765f976fb4633e010c Mon Sep 17 00:00:00 2001 From: Georgi Djakov Date: Wed, 17 Apr 2024 06:37:25 -0700 Subject: [PATCH 0305/1702] dt-bindings: iommu: Add Qualcomm TBU The "apps_smmu" on the Qualcomm sdm845 platform is an implementation of the SMMU-500, that consists of a single TCU (Translation Control Unit) and multiple TBUs (Translation Buffer Units). These TBUs have hardware debugging features that are specific and only present on Qualcomm hardware. Represent them as independent DT nodes. List all the resources that are needed to operate them (such as registers, clocks, power domains and interconnects). Reviewed-by: Rob Herring Signed-off-by: Georgi Djakov Link: https://lore.kernel.org/r/20240417133731.2055383-2-quic_c_gdjako@quicinc.com Signed-off-by: Will Deacon --- .../devicetree/bindings/iommu/qcom,tbu.yaml | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 Documentation/devicetree/bindings/iommu/qcom,tbu.yaml diff --git a/Documentation/devicetree/bindings/iommu/qcom,tbu.yaml b/Documentation/devicetree/bindings/iommu/qcom,tbu.yaml new file mode 100644 index 0000000000000..82dfe935573ec --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/qcom,tbu.yaml @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iommu/qcom,tbu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm TBU (Translation Buffer Unit) + +maintainers: + - Georgi Djakov + +description: + The Qualcomm SMMU500 implementation consists of TCU and TBU. The TBU contains + a Translation Lookaside Buffer (TLB) that caches page tables. TBUs provides + debug features to trace and trigger debug transactions. There are multiple TBU + instances with each client core. + +properties: + compatible: + enum: + - qcom,sc7280-tbu + - qcom,sdm845-tbu + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + interconnects: + maxItems: 1 + + power-domains: + maxItems: 1 + + qcom,stream-id-range: + description: | + Phandle of a SMMU device and Stream ID range (address and size) that + is assigned by the TBU + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + - items: + - description: phandle of a smmu node + - description: stream id base address + - description: stream id size + +required: + - compatible + - reg + - qcom,stream-id-range + +additionalProperties: false + +examples: + - | + #include + #include + #include + + tbu@150e1000 { + compatible = "qcom,sdm845-tbu"; + reg = <0x150e1000 0x1000>; + clocks = <&gcc GCC_AGGRE_NOC_PCIE_TBU_CLK>; + interconnects = <&system_noc MASTER_GNOC_SNOC QCOM_ICC_TAG_ACTIVE_ONLY + &config_noc SLAVE_IMEM_CFG QCOM_ICC_TAG_ACTIVE_ONLY>; + power-domains = <&gcc HLOS1_VOTE_AGGRE_NOC_MMU_PCIE_TBU_GDSC>; + qcom,stream-id-range = <&apps_smmu 0x1c00 0x400>; + }; +... From 414ecb030870a31262e5fd29fd372ee73b32a6be Mon Sep 17 00:00:00 2001 From: Georgi Djakov Date: Wed, 17 Apr 2024 06:37:26 -0700 Subject: [PATCH 0306/1702] iommu/arm-smmu-qcom-debug: Add support for TBUs Operating the TBUs (Translation Buffer Units) from Linux on Qualcomm platforms can help with debugging context faults. To help with that, the TBUs can run ATOS (Address Translation Operations) to manually trigger address translation of IOVA to physical address in hardware and provide more details when a context fault happens. The driver will control the resources needed by the TBU to allow running the debug operations such as ATOS, check for outstanding transactions, do snapshot capture etc. Signed-off-by: Georgi Djakov Link: https://lore.kernel.org/r/20240417133731.2055383-3-quic_c_gdjako@quicinc.com Signed-off-by: Will Deacon --- drivers/iommu/Kconfig | 12 +- .../iommu/arm/arm-smmu/arm-smmu-qcom-debug.c | 353 ++++++++++++++++++ drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h | 2 + drivers/iommu/arm/arm-smmu/arm-smmu.h | 2 + 4 files changed, 365 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 0af39bbbe3a30..0ac80e1094f3a 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -379,10 +379,14 @@ config ARM_SMMU_QCOM_DEBUG depends on ARM_SMMU_QCOM help Support for implementation specific debug features in ARM SMMU - hardware found in QTI platforms. - - Say Y here to enable debug for issues such as TLB sync timeouts - which requires implementation defined register dumps. + hardware found in QTI platforms. This include support for + the Translation Buffer Units (TBU) that can be used to obtain + additional information when debugging memory management issues + like context faults. + + Say Y here to enable debug for issues such as context faults + or TLB sync timeouts which requires implementation defined + register dumps. config ARM_SMMU_V3 tristate "ARM Ltd. System MMU Version 3 (SMMUv3) Support" diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c index bb89d49adf8d2..eff7ca94ec8d0 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c @@ -1,15 +1,66 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. */ +#include #include +#include #include +#include +#include +#include +#include +#include #include +#include #include "arm-smmu.h" #include "arm-smmu-qcom.h" +#define TBU_DBG_TIMEOUT_US 100 +#define DEBUG_AXUSER_REG 0x30 +#define DEBUG_AXUSER_CDMID GENMASK_ULL(43, 36) +#define DEBUG_AXUSER_CDMID_VAL 0xff +#define DEBUG_PAR_REG 0x28 +#define DEBUG_PAR_FAULT_VAL BIT(0) +#define DEBUG_PAR_PA GENMASK_ULL(47, 12) +#define DEBUG_SID_HALT_REG 0x0 +#define DEBUG_SID_HALT_VAL BIT(16) +#define DEBUG_SID_HALT_SID GENMASK(9, 0) +#define DEBUG_SR_HALT_ACK_REG 0x20 +#define DEBUG_SR_HALT_ACK_VAL BIT(1) +#define DEBUG_SR_ECATS_RUNNING_VAL BIT(0) +#define DEBUG_TXN_AXCACHE GENMASK(5, 2) +#define DEBUG_TXN_AXPROT GENMASK(8, 6) +#define DEBUG_TXN_AXPROT_PRIV 0x1 +#define DEBUG_TXN_AXPROT_NSEC 0x2 +#define DEBUG_TXN_TRIGG_REG 0x18 +#define DEBUG_TXN_TRIGGER BIT(0) +#define DEBUG_VA_ADDR_REG 0x8 + +static LIST_HEAD(tbu_list); +static DEFINE_MUTEX(tbu_list_lock); +static DEFINE_SPINLOCK(atos_lock); + +struct qcom_tbu { + struct device *dev; + struct device_node *smmu_np; + u32 sid_range[2]; + struct list_head list; + struct clk *clk; + struct icc_path *path; + void __iomem *base; + spinlock_t halt_lock; /* multiple halt or resume can't execute concurrently */ + int halt_count; +}; + +static struct qcom_smmu *to_qcom_smmu(struct arm_smmu_device *smmu) +{ + return container_of(smmu, struct qcom_smmu, smmu); +} + void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu) { int ret; @@ -49,3 +100,305 @@ void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu) tbu_pwr_status, sync_inv_ack, sync_inv_progress); } } + +static struct qcom_tbu *qcom_find_tbu(struct qcom_smmu *qsmmu, u32 sid) +{ + struct qcom_tbu *tbu; + u32 start, end; + + guard(mutex)(&tbu_list_lock); + + if (list_empty(&tbu_list)) + return NULL; + + list_for_each_entry(tbu, &tbu_list, list) { + start = tbu->sid_range[0]; + end = start + tbu->sid_range[1]; + + if (qsmmu->smmu.dev->of_node == tbu->smmu_np && + start <= sid && sid < end) + return tbu; + } + dev_err(qsmmu->smmu.dev, "Unable to find TBU for sid 0x%x\n", sid); + + return NULL; +} + +static int qcom_tbu_halt(struct qcom_tbu *tbu, struct arm_smmu_domain *smmu_domain) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + int ret = 0, idx = smmu_domain->cfg.cbndx; + u32 val, fsr, status; + + guard(spinlock_irqsave)(&tbu->halt_lock); + if (tbu->halt_count) { + tbu->halt_count++; + return ret; + } + + val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val |= DEBUG_SID_HALT_VAL; + writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if ((fsr & ARM_SMMU_FSR_FAULT) && (fsr & ARM_SMMU_FSR_SS)) { + u32 sctlr_orig, sctlr; + + /* + * We are in a fault. Our request to halt the bus will not + * complete until transactions in front of us (such as the fault + * itself) have completed. Disable iommu faults and terminate + * any existing transactions. + */ + sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); + sctlr = sctlr_orig & ~(ARM_SMMU_SCTLR_CFCFG | ARM_SMMU_SCTLR_CFIE); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, ARM_SMMU_RESUME_TERMINATE); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); + } + + if (readl_poll_timeout_atomic(tbu->base + DEBUG_SR_HALT_ACK_REG, status, + (status & DEBUG_SR_HALT_ACK_VAL), + 0, TBU_DBG_TIMEOUT_US)) { + dev_err(tbu->dev, "Timeout while trying to halt TBU!\n"); + ret = -ETIMEDOUT; + + val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_VAL; + writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + + return ret; + } + + tbu->halt_count = 1; + + return ret; +} + +static void qcom_tbu_resume(struct qcom_tbu *tbu) +{ + u32 val; + + guard(spinlock_irqsave)(&tbu->halt_lock); + if (!tbu->halt_count) { + WARN(1, "%s: halt_count is 0", dev_name(tbu->dev)); + return; + } + + if (tbu->halt_count > 1) { + tbu->halt_count--; + return; + } + + val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_VAL; + writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + + tbu->halt_count = 0; +} + +static phys_addr_t qcom_tbu_trigger_atos(struct arm_smmu_domain *smmu_domain, + struct qcom_tbu *tbu, dma_addr_t iova, u32 sid) +{ + bool atos_timedout = false; + phys_addr_t phys = 0; + ktime_t timeout; + u64 val; + + /* Set address and stream-id */ + val = readq_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_SID; + val |= FIELD_PREP(DEBUG_SID_HALT_SID, sid); + writeq_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + writeq_relaxed(iova, tbu->base + DEBUG_VA_ADDR_REG); + val = FIELD_PREP(DEBUG_AXUSER_CDMID, DEBUG_AXUSER_CDMID_VAL); + writeq_relaxed(val, tbu->base + DEBUG_AXUSER_REG); + + /* Write-back read and write-allocate */ + val = FIELD_PREP(DEBUG_TXN_AXCACHE, 0xf); + + /* Non-secure access */ + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_NSEC); + + /* Privileged access */ + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_PRIV); + + val |= DEBUG_TXN_TRIGGER; + writeq_relaxed(val, tbu->base + DEBUG_TXN_TRIGG_REG); + + timeout = ktime_add_us(ktime_get(), TBU_DBG_TIMEOUT_US); + for (;;) { + val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); + if (!(val & DEBUG_SR_ECATS_RUNNING_VAL)) + break; + val = readl_relaxed(tbu->base + DEBUG_PAR_REG); + if (val & DEBUG_PAR_FAULT_VAL) + break; + if (ktime_compare(ktime_get(), timeout) > 0) { + atos_timedout = true; + break; + } + } + + val = readq_relaxed(tbu->base + DEBUG_PAR_REG); + if (val & DEBUG_PAR_FAULT_VAL) + dev_err(tbu->dev, "ATOS generated a fault interrupt! PAR = %llx, SID=0x%x\n", + val, sid); + else if (atos_timedout) + dev_err_ratelimited(tbu->dev, "ATOS translation timed out!\n"); + else + phys = FIELD_GET(DEBUG_PAR_PA, val); + + /* Reset hardware */ + writeq_relaxed(0, tbu->base + DEBUG_TXN_TRIGG_REG); + writeq_relaxed(0, tbu->base + DEBUG_VA_ADDR_REG); + val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_SID; + writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + + return phys; +} + +static phys_addr_t qcom_iova_to_phys(struct arm_smmu_domain *smmu_domain, + dma_addr_t iova, u32 sid) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct qcom_smmu *qsmmu = to_qcom_smmu(smmu); + int idx = smmu_domain->cfg.cbndx; + struct qcom_tbu *tbu; + u32 sctlr_orig, sctlr; + phys_addr_t phys = 0; + int attempt = 0; + int ret; + u64 fsr; + + tbu = qcom_find_tbu(qsmmu, sid); + if (!tbu) + return 0; + + ret = icc_set_bw(tbu->path, 0, UINT_MAX); + if (ret) + return ret; + + ret = clk_prepare_enable(tbu->clk); + if (ret) + goto disable_icc; + + ret = qcom_tbu_halt(tbu, smmu_domain); + if (ret) + goto disable_clk; + + /* + * ATOS/ECATS can trigger the fault interrupt, so disable it temporarily + * and check for an interrupt manually. + */ + sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); + sctlr = sctlr_orig & ~(ARM_SMMU_SCTLR_CFCFG | ARM_SMMU_SCTLR_CFIE); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (fsr & ARM_SMMU_FSR_FAULT) { + /* Clear pending interrupts */ + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + + /* + * TBU halt takes care of resuming any stalled transcation. + * Kept it here for completeness sake. + */ + if (fsr & ARM_SMMU_FSR_SS) + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, + ARM_SMMU_RESUME_TERMINATE); + } + + /* Only one concurrent atos operation */ + scoped_guard(spinlock_irqsave, &atos_lock) { + /* + * If the translation fails, attempt the lookup more time." + */ + do { + phys = qcom_tbu_trigger_atos(smmu_domain, tbu, iova, sid); + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (fsr & ARM_SMMU_FSR_FAULT) { + /* Clear pending interrupts */ + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + + if (fsr & ARM_SMMU_FSR_SS) + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, + ARM_SMMU_RESUME_TERMINATE); + } + } while (!phys && attempt++ < 2); + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); + } + qcom_tbu_resume(tbu); + + /* Read to complete prior write transcations */ + readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); + +disable_clk: + clk_disable_unprepare(tbu->clk); +disable_icc: + icc_set_bw(tbu->path, 0, 0); + + return phys; +} + +static int qcom_tbu_probe(struct platform_device *pdev) +{ + struct of_phandle_args args = { .args_count = 2 }; + struct device_node *np = pdev->dev.of_node; + struct device *dev = &pdev->dev; + struct qcom_tbu *tbu; + + tbu = devm_kzalloc(dev, sizeof(*tbu), GFP_KERNEL); + if (!tbu) + return -ENOMEM; + + tbu->dev = dev; + INIT_LIST_HEAD(&tbu->list); + spin_lock_init(&tbu->halt_lock); + + if (of_parse_phandle_with_args(np, "qcom,stream-id-range", "#iommu-cells", 0, &args)) { + dev_err(dev, "Cannot parse the 'qcom,stream-id-range' DT property\n"); + return -EINVAL; + } + + tbu->smmu_np = args.np; + tbu->sid_range[0] = args.args[0]; + tbu->sid_range[1] = args.args[1]; + of_node_put(args.np); + + tbu->base = devm_of_iomap(dev, np, 0, NULL); + if (IS_ERR(tbu->base)) + return PTR_ERR(tbu->base); + + tbu->clk = devm_clk_get_optional(dev, NULL); + if (IS_ERR(tbu->clk)) + return PTR_ERR(tbu->clk); + + tbu->path = devm_of_icc_get(dev, NULL); + if (IS_ERR(tbu->path)) + return PTR_ERR(tbu->path); + + guard(mutex)(&tbu_list_lock); + list_add_tail(&tbu->list, &tbu_list); + + return 0; +} + +static const struct of_device_id qcom_tbu_of_match[] = { + { .compatible = "qcom,sc7280-tbu" }, + { .compatible = "qcom,sdm845-tbu" }, + { } +}; + +static struct platform_driver qcom_tbu_driver = { + .driver = { + .name = "qcom_tbu", + .of_match_table = qcom_tbu_of_match, + }, + .probe = qcom_tbu_probe, +}; +builtin_platform_driver(qcom_tbu_driver); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h index 593910567b884..9bb3ae7d62da6 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h @@ -30,6 +30,8 @@ struct qcom_smmu_match_data { const struct arm_smmu_impl *adreno_impl; }; +irqreturn_t qcom_smmu_context_fault(int irq, void *dev); + #ifdef CONFIG_ARM_SMMU_QCOM_DEBUG void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu); #else diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h index 836ed6799a801..1670e95c4637e 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.h +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h @@ -136,6 +136,7 @@ enum arm_smmu_cbar_type { #define ARM_SMMU_CBAR_VMID GENMASK(7, 0) #define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2)) +#define ARM_SMMU_CBFRSYNRA_SID GENMASK(15, 0) #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2)) #define ARM_SMMU_CBA2R_VMID16 GENMASK(31, 16) @@ -238,6 +239,7 @@ enum arm_smmu_cbar_type { #define ARM_SMMU_CB_ATSR 0x8f0 #define ARM_SMMU_ATSR_ACTIVE BIT(0) +#define ARM_SMMU_RESUME_TERMINATE BIT(0) /* Maximum number of context banks per SMMU */ #define ARM_SMMU_MAX_CBS 128 From 960be6e10d4fcb18cfe863e0ceb3213a75eecb81 Mon Sep 17 00:00:00 2001 From: Georgi Djakov Date: Wed, 17 Apr 2024 06:37:27 -0700 Subject: [PATCH 0307/1702] iommu/arm-smmu: Allow using a threaded handler for context interrupts Threaded IRQ handlers run in a less critical context compared to normal IRQs, so they can perform more complex and time-consuming operations without causing significant delays in other parts of the kernel. During a context fault, it might be needed to do more processing and gather debug information from TBUs in the handler. These operations may sleep, so add an option to use a threaded IRQ handler in these cases. Signed-off-by: Georgi Djakov Link: https://lore.kernel.org/r/20240417133731.2055383-4-quic_c_gdjako@quicinc.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu.c | 12 ++++++++++-- drivers/iommu/arm/arm-smmu/arm-smmu.h | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index 5935f44e1d1d6..87c81f75cf844 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -806,8 +806,16 @@ static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain, else context_fault = arm_smmu_context_fault; - ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED, - "arm-smmu-context-fault", smmu_domain); + if (smmu->impl && smmu->impl->context_fault_needs_threaded_irq) + ret = devm_request_threaded_irq(smmu->dev, irq, NULL, + context_fault, + IRQF_ONESHOT | IRQF_SHARED, + "arm-smmu-context-fault", + smmu_domain); + else + ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED, + "arm-smmu-context-fault", smmu_domain); + if (ret < 0) { dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", cfg->irptndx, irq); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h index 1670e95c4637e..4765c6945c344 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.h +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h @@ -438,6 +438,7 @@ struct arm_smmu_impl { int (*def_domain_type)(struct device *dev); irqreturn_t (*global_fault)(int irq, void *dev); irqreturn_t (*context_fault)(int irq, void *dev); + bool context_fault_needs_threaded_irq; int (*alloc_context_bank)(struct arm_smmu_domain *smmu_domain, struct arm_smmu_device *smmu, struct device *dev, int start); From d374555ef993433f4d2e08b700dbd27788427d61 Mon Sep 17 00:00:00 2001 From: Georgi Djakov Date: Wed, 17 Apr 2024 06:37:28 -0700 Subject: [PATCH 0308/1702] iommu/arm-smmu-qcom: Use a custom context fault handler for sdm845 The sdm845 platform now supports TBUs, so let's get additional debug info from the TBUs when a context fault occurs. Implement a custom context fault handler that does both software + hardware page table walks and TLB Invalidate All. Signed-off-by: Georgi Djakov Link: https://lore.kernel.org/r/20240417133731.2055383-5-quic_c_gdjako@quicinc.com Signed-off-by: Will Deacon --- .../iommu/arm/arm-smmu/arm-smmu-qcom-debug.c | 143 ++++++++++++++++++ drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 4 + 2 files changed, 147 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c index eff7ca94ec8d0..552199cbd9e25 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c @@ -345,6 +345,149 @@ static phys_addr_t qcom_iova_to_phys(struct arm_smmu_domain *smmu_domain, return phys; } +static phys_addr_t qcom_smmu_iova_to_phys_hard(struct arm_smmu_domain *smmu_domain, dma_addr_t iova) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + int idx = smmu_domain->cfg.cbndx; + u32 frsynra; + u16 sid; + + frsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); + sid = FIELD_GET(ARM_SMMU_CBFRSYNRA_SID, frsynra); + + return qcom_iova_to_phys(smmu_domain, iova, sid); +} + +static phys_addr_t qcom_smmu_verify_fault(struct arm_smmu_domain *smmu_domain, dma_addr_t iova, u32 fsr) +{ + struct io_pgtable *iop = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops); + struct arm_smmu_device *smmu = smmu_domain->smmu; + phys_addr_t phys_post_tlbiall; + phys_addr_t phys; + + phys = qcom_smmu_iova_to_phys_hard(smmu_domain, iova); + io_pgtable_tlb_flush_all(iop); + phys_post_tlbiall = qcom_smmu_iova_to_phys_hard(smmu_domain, iova); + + if (phys != phys_post_tlbiall) { + dev_err(smmu->dev, + "ATOS results differed across TLBIALL... (before: %pa after: %pa)\n", + &phys, &phys_post_tlbiall); + } + + return (phys == 0 ? phys_post_tlbiall : phys); +} + +irqreturn_t qcom_smmu_context_fault(int irq, void *dev) +{ + struct arm_smmu_domain *smmu_domain = dev; + struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; + struct arm_smmu_device *smmu = smmu_domain->smmu; + u32 fsr, fsynr, cbfrsynra, resume = 0; + int idx = smmu_domain->cfg.cbndx; + phys_addr_t phys_soft; + unsigned long iova; + int ret, tmp; + + static DEFINE_RATELIMIT_STATE(_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (!(fsr & ARM_SMMU_FSR_FAULT)) + return IRQ_NONE; + + fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); + iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); + cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); + + if (list_empty(&tbu_list)) { + ret = report_iommu_fault(&smmu_domain->domain, NULL, iova, + fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); + + if (ret == -ENOSYS) + dev_err_ratelimited(smmu->dev, + "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", + fsr, iova, fsynr, cbfrsynra, idx); + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + return IRQ_HANDLED; + } + + phys_soft = ops->iova_to_phys(ops, iova); + + tmp = report_iommu_fault(&smmu_domain->domain, NULL, iova, + fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); + if (!tmp || tmp == -EBUSY) { + dev_dbg(smmu->dev, + "Context fault handled by client: iova=0x%08lx, fsr=0x%x, fsynr=0x%x, cb=%d\n", + iova, fsr, fsynr, idx); + dev_dbg(smmu->dev, "soft iova-to-phys=%pa\n", &phys_soft); + ret = IRQ_HANDLED; + resume = ARM_SMMU_RESUME_TERMINATE; + } else { + phys_addr_t phys_atos = qcom_smmu_verify_fault(smmu_domain, iova, fsr); + + if (__ratelimit(&_rs)) { + dev_err(smmu->dev, + "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", + fsr, iova, fsynr, cbfrsynra, idx); + dev_err(smmu->dev, + "FSR = %08x [%s%s%s%s%s%s%s%s%s], SID=0x%x\n", + fsr, + (fsr & 0x02) ? "TF " : "", + (fsr & 0x04) ? "AFF " : "", + (fsr & 0x08) ? "PF " : "", + (fsr & 0x10) ? "EF " : "", + (fsr & 0x20) ? "TLBMCF " : "", + (fsr & 0x40) ? "TLBLKF " : "", + (fsr & 0x80) ? "MHF " : "", + (fsr & 0x40000000) ? "SS " : "", + (fsr & 0x80000000) ? "MULTI " : "", + cbfrsynra); + + dev_err(smmu->dev, + "soft iova-to-phys=%pa\n", &phys_soft); + if (!phys_soft) + dev_err(smmu->dev, + "SOFTWARE TABLE WALK FAILED! Looks like %s accessed an unmapped address!\n", + dev_name(smmu->dev)); + if (phys_atos) + dev_err(smmu->dev, "hard iova-to-phys (ATOS)=%pa\n", + &phys_atos); + else + dev_err(smmu->dev, "hard iova-to-phys (ATOS) failed\n"); + } + ret = IRQ_NONE; + resume = ARM_SMMU_RESUME_TERMINATE; + } + + /* + * If the client returns -EBUSY, do not clear FSR and do not RESUME + * if stalled. This is required to keep the IOMMU client stalled on + * the outstanding fault. This gives the client a chance to take any + * debug action and then terminate the stalled transaction. + * So, the sequence in case of stall on fault should be: + * 1) Do not clear FSR or write to RESUME here + * 2) Client takes any debug action + * 3) Client terminates the stalled transaction and resumes the IOMMU + * 4) Client clears FSR. The FSR should only be cleared after 3) and + * not before so that the fault remains outstanding. This ensures + * SCTLR.HUPCF has the desired effect if subsequent transactions also + * need to be terminated. + */ + if (tmp != -EBUSY) { + /* Clear the faulting FSR */ + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + + /* Retry or terminate any stalled transactions */ + if (fsr & ARM_SMMU_FSR_SS) + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, resume); + } + + return ret; +} + static int qcom_tbu_probe(struct platform_device *pdev) { struct of_phandle_args args = { .args_count = 2 }; diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 5c7cfc51b57c0..a901230dbabd1 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -422,6 +422,10 @@ static const struct arm_smmu_impl sdm845_smmu_500_impl = { .reset = qcom_sdm845_smmu500_reset, .write_s2cr = qcom_smmu_write_s2cr, .tlb_sync = qcom_smmu_tlb_sync, +#ifdef CONFIG_ARM_SMMU_QCOM_DEBUG + .context_fault = qcom_smmu_context_fault, + .context_fault_needs_threaded_irq = true, +#endif }; static const struct arm_smmu_impl qcom_adreno_smmu_v2_impl = { From b8ca7ce709f8210c13eec022e87a12111db5d745 Mon Sep 17 00:00:00 2001 From: Georgi Djakov Date: Wed, 17 Apr 2024 06:37:30 -0700 Subject: [PATCH 0309/1702] iommu/arm-smmu-qcom: Use the custom fault handler on more platforms The TBU support is now available, so let's allow it to be used on other platforms that have the Qualcomm SMMU-500 implementation with TBUs. This will allow the context fault handler to query the TBUs when a context fault occurs. Signed-off-by: Georgi Djakov Link: https://lore.kernel.org/r/20240417133731.2055383-7-quic_c_gdjako@quicinc.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index a901230dbabd1..25f034677f568 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -413,6 +413,10 @@ static const struct arm_smmu_impl qcom_smmu_500_impl = { .reset = arm_mmu500_reset, .write_s2cr = qcom_smmu_write_s2cr, .tlb_sync = qcom_smmu_tlb_sync, +#ifdef CONFIG_ARM_SMMU_QCOM_DEBUG + .context_fault = qcom_smmu_context_fault, + .context_fault_needs_threaded_irq = true, +#endif }; static const struct arm_smmu_impl sdm845_smmu_500_impl = { From 7643f3fe27729b20e6fcf6b314b00b8b93504356 Mon Sep 17 00:00:00 2001 From: Jaegeuk Kim Date: Wed, 17 Apr 2024 20:01:55 +0000 Subject: [PATCH 0310/1702] f2fs: assign the write hint per stream by default This reverts commit 930e2607638d ("f2fs: remove obsolete whint_mode"), as we decide to pass write hints to the disk. Cc: Hyunchul Lee Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- Documentation/filesystems/f2fs.rst | 29 ++++++++++++ fs/f2fs/data.c | 2 + fs/f2fs/f2fs.h | 3 ++ fs/f2fs/file.c | 15 +++++- fs/f2fs/segment.c | 76 +++++++++++++++++++++++++++--- 5 files changed, 118 insertions(+), 7 deletions(-) diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst index efc3493fd6f84..68a0885fb5e69 100644 --- a/Documentation/filesystems/f2fs.rst +++ b/Documentation/filesystems/f2fs.rst @@ -774,6 +774,35 @@ In order to identify whether the data in the victim segment are valid or not, F2FS manages a bitmap. Each bit represents the validity of a block, and the bitmap is composed of a bit stream covering whole blocks in main area. +Write-hint Policy +----------------- + +F2FS sets the whint all the time with the below policy. + +===================== ======================== =================== +User F2FS Block +===================== ======================== =================== +N/A META WRITE_LIFE_NONE|REQ_META +N/A HOT_NODE WRITE_LIFE_NONE +N/A WARM_NODE WRITE_LIFE_MEDIUM +N/A COLD_NODE WRITE_LIFE_LONG +ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME +extension list " " + +-- buffered io +N/A COLD_DATA WRITE_LIFE_EXTREME +N/A HOT_DATA WRITE_LIFE_SHORT +N/A WARM_DATA WRITE_LIFE_NOT_SET + +-- direct io +WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME +WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT +WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET +WRITE_LIFE_NONE " WRITE_LIFE_NONE +WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM +WRITE_LIFE_LONG " WRITE_LIFE_LONG +===================== ======================== =================== + Fallocate(2) Policy ------------------- diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 5d641fac02ba7..ed7d08785fcff 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -465,6 +465,8 @@ static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) } else { bio->bi_end_io = f2fs_write_end_io; bio->bi_private = sbi; + bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, + fio->type, fio->temp); } iostat_alloc_and_bind_ctx(sbi, bio, NULL); diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index dd530dc700052..3f7196122574b 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -3722,6 +3722,7 @@ void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, block_t old_addr, block_t new_addr, unsigned char version, bool recover_curseg, bool recover_newaddr); +int f2fs_get_segment_temp(int seg_type); int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, block_t old_blkaddr, block_t *new_blkaddr, struct f2fs_summary *sum, int type, @@ -3745,6 +3746,8 @@ void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi); int __init f2fs_create_segment_manager_caches(void); void f2fs_destroy_segment_manager_caches(void); int f2fs_rw_hint_to_seg_type(enum rw_hint hint); +enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi, + enum page_type type, enum temp_type temp); unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi, unsigned int segno); unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi, diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index ac1ae85f3cc3d..d382f8bc2fbe6 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -4685,8 +4685,21 @@ static int f2fs_dio_write_end_io(struct kiocb *iocb, ssize_t size, int error, return 0; } +static void f2fs_dio_write_submit_io(const struct iomap_iter *iter, + struct bio *bio, loff_t file_offset) +{ + struct inode *inode = iter->inode; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int seg_type = f2fs_rw_hint_to_seg_type(inode->i_write_hint); + enum temp_type temp = f2fs_get_segment_temp(seg_type); + + bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, DATA, temp); + submit_bio(bio); +} + static const struct iomap_dio_ops f2fs_iomap_dio_write_ops = { - .end_io = f2fs_dio_write_end_io, + .end_io = f2fs_dio_write_end_io, + .submit_io = f2fs_dio_write_submit_io, }; static void f2fs_flush_buffered_write(struct address_space *mapping, diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index f0da516ba8dc3..2206199e80998 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -3364,6 +3364,65 @@ int f2fs_rw_hint_to_seg_type(enum rw_hint hint) } } +/* + * This returns write hints for each segment type. This hints will be + * passed down to block layer as below by default. + * + * User F2FS Block + * ---- ---- ----- + * META WRITE_LIFE_NONE|REQ_META + * HOT_NODE WRITE_LIFE_NONE + * WARM_NODE WRITE_LIFE_MEDIUM + * COLD_NODE WRITE_LIFE_LONG + * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME + * extension list " " + * + * -- buffered io + * COLD_DATA WRITE_LIFE_EXTREME + * HOT_DATA WRITE_LIFE_SHORT + * WARM_DATA WRITE_LIFE_NOT_SET + * + * -- direct io + * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME + * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT + * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET + * WRITE_LIFE_NONE " WRITE_LIFE_NONE + * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM + * WRITE_LIFE_LONG " WRITE_LIFE_LONG + */ +enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi, + enum page_type type, enum temp_type temp) +{ + switch (type) { + case DATA: + switch (temp) { + case WARM: + return WRITE_LIFE_NOT_SET; + case HOT: + return WRITE_LIFE_SHORT; + case COLD: + return WRITE_LIFE_EXTREME; + default: + return WRITE_LIFE_NONE; + } + case NODE: + switch (temp) { + case WARM: + return WRITE_LIFE_MEDIUM; + case HOT: + return WRITE_LIFE_NONE; + case COLD: + return WRITE_LIFE_LONG; + default: + return WRITE_LIFE_NONE; + } + case META: + return WRITE_LIFE_NONE; + default: + return WRITE_LIFE_NONE; + } +} + static int __get_segment_type_2(struct f2fs_io_info *fio) { if (fio->type == DATA) @@ -3443,6 +3502,15 @@ static int __get_segment_type_6(struct f2fs_io_info *fio) } } +int f2fs_get_segment_temp(int seg_type) +{ + if (IS_HOT(seg_type)) + return HOT; + else if (IS_WARM(seg_type)) + return WARM; + return COLD; +} + static int __get_segment_type(struct f2fs_io_info *fio) { int type = 0; @@ -3461,12 +3529,8 @@ static int __get_segment_type(struct f2fs_io_info *fio) f2fs_bug_on(fio->sbi, true); } - if (IS_HOT(type)) - fio->temp = HOT; - else if (IS_WARM(type)) - fio->temp = WARM; - else - fio->temp = COLD; + fio->temp = f2fs_get_segment_temp(type); + return type; } From db92e6c729d87e6f5b0f467f01a96dbf1e452106 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Tue, 16 Apr 2024 15:23:15 +0800 Subject: [PATCH 0311/1702] f2fs: convert f2fs_mpage_readpages() to use folio Convert f2fs_mpage_readpages() to use folio and related functionality. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/data.c | 80 +++++++++++++++++++++++++------------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index ed7d08785fcff..2fd86cff81e2b 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -2345,7 +2345,7 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, * Major change was from block_size == page_size in f2fs by default. */ static int f2fs_mpage_readpages(struct inode *inode, - struct readahead_control *rac, struct page *page) + struct readahead_control *rac, struct folio *folio) { struct bio *bio = NULL; sector_t last_block_in_bio = 0; @@ -2365,6 +2365,7 @@ static int f2fs_mpage_readpages(struct inode *inode, #endif unsigned nr_pages = rac ? readahead_count(rac) : 1; unsigned max_nr_pages = nr_pages; + pgoff_t index; int ret = 0; map.m_pblk = 0; @@ -2378,64 +2379,63 @@ static int f2fs_mpage_readpages(struct inode *inode, for (; nr_pages; nr_pages--) { if (rac) { - page = readahead_page(rac); - prefetchw(&page->flags); + folio = readahead_folio(rac); + prefetchw(&folio->flags); } -#ifdef CONFIG_F2FS_FS_COMPRESSION - if (f2fs_compressed_file(inode)) { - /* there are remained compressed pages, submit them */ - if (!f2fs_cluster_can_merge_page(&cc, page->index)) { - ret = f2fs_read_multi_pages(&cc, &bio, - max_nr_pages, - &last_block_in_bio, - rac != NULL, false); - f2fs_destroy_compress_ctx(&cc, false); - if (ret) - goto set_error_page; - } - if (cc.cluster_idx == NULL_CLUSTER) { - if (nc_cluster_idx == - page->index >> cc.log_cluster_size) { - goto read_single_page; - } - - ret = f2fs_is_compressed_cluster(inode, page->index); - if (ret < 0) - goto set_error_page; - else if (!ret) { - nc_cluster_idx = - page->index >> cc.log_cluster_size; - goto read_single_page; - } + index = folio_index(folio); - nc_cluster_idx = NULL_CLUSTER; - } - ret = f2fs_init_compress_ctx(&cc); +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (!f2fs_compressed_file(inode)) + goto read_single_page; + + /* there are remained compressed pages, submit them */ + if (!f2fs_cluster_can_merge_page(&cc, index)) { + ret = f2fs_read_multi_pages(&cc, &bio, + max_nr_pages, + &last_block_in_bio, + rac != NULL, false); + f2fs_destroy_compress_ctx(&cc, false); if (ret) goto set_error_page; + } + if (cc.cluster_idx == NULL_CLUSTER) { + if (nc_cluster_idx == index >> cc.log_cluster_size) + goto read_single_page; - f2fs_compress_ctx_add_page(&cc, page); + ret = f2fs_is_compressed_cluster(inode, index); + if (ret < 0) + goto set_error_page; + else if (!ret) { + nc_cluster_idx = + index >> cc.log_cluster_size; + goto read_single_page; + } - goto next_page; + nc_cluster_idx = NULL_CLUSTER; } + ret = f2fs_init_compress_ctx(&cc); + if (ret) + goto set_error_page; + + f2fs_compress_ctx_add_page(&cc, &folio->page); + + goto next_page; read_single_page: #endif - ret = f2fs_read_single_page(inode, page, max_nr_pages, &map, + ret = f2fs_read_single_page(inode, &folio->page, max_nr_pages, &map, &bio, &last_block_in_bio, rac); if (ret) { #ifdef CONFIG_F2FS_FS_COMPRESSION set_error_page: #endif - zero_user_segment(page, 0, PAGE_SIZE); - unlock_page(page); + folio_zero_segment(folio, 0, folio_size(folio)); + folio_unlock(folio); } #ifdef CONFIG_F2FS_FS_COMPRESSION next_page: #endif - if (rac) - put_page(page); #ifdef CONFIG_F2FS_FS_COMPRESSION if (f2fs_compressed_file(inode)) { @@ -2472,7 +2472,7 @@ static int f2fs_read_data_folio(struct file *file, struct folio *folio) if (f2fs_has_inline_data(inode)) ret = f2fs_read_inline_data(inode, page); if (ret == -EAGAIN) - ret = f2fs_mpage_readpages(inode, NULL, page); + ret = f2fs_mpage_readpages(inode, NULL, folio); return ret; } From ed54eed355675f79d9322c07d92f38037fc6b514 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Tue, 16 Apr 2024 15:23:16 +0800 Subject: [PATCH 0312/1702] f2fs: convert f2fs_read_single_page() to use folio Convert f2fs_read_single_page() to use folio and related functionality. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/data.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 2fd86cff81e2b..8874ca3a32e21 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -2063,7 +2063,7 @@ static inline loff_t f2fs_readpage_limit(struct inode *inode) return i_size_read(inode); } -static int f2fs_read_single_page(struct inode *inode, struct page *page, +static int f2fs_read_single_page(struct inode *inode, struct folio *folio, unsigned nr_pages, struct f2fs_map_blocks *map, struct bio **bio_ret, @@ -2076,9 +2076,10 @@ static int f2fs_read_single_page(struct inode *inode, struct page *page, sector_t last_block; sector_t last_block_in_file; sector_t block_nr; + pgoff_t index = folio_index(folio); int ret = 0; - block_in_file = (sector_t)page_index(page); + block_in_file = (sector_t)index; last_block = block_in_file + nr_pages; last_block_in_file = bytes_to_blks(inode, f2fs_readpage_limit(inode) + blocksize - 1); @@ -2109,7 +2110,7 @@ static int f2fs_read_single_page(struct inode *inode, struct page *page, got_it: if ((map->m_flags & F2FS_MAP_MAPPED)) { block_nr = map->m_pblk + block_in_file - map->m_lblk; - SetPageMappedToDisk(page); + folio_set_mappedtodisk(folio); if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr, DATA_GENERIC_ENHANCE_READ)) { @@ -2118,15 +2119,15 @@ static int f2fs_read_single_page(struct inode *inode, struct page *page, } } else { zero_out: - zero_user_segment(page, 0, PAGE_SIZE); - if (f2fs_need_verity(inode, page->index) && - !fsverity_verify_page(page)) { + folio_zero_segment(folio, 0, folio_size(folio)); + if (f2fs_need_verity(inode, index) && + !fsverity_verify_folio(folio)) { ret = -EIO; goto out; } - if (!PageUptodate(page)) - SetPageUptodate(page); - unlock_page(page); + if (!folio_test_uptodate(folio)) + folio_mark_uptodate(folio); + folio_unlock(folio); goto out; } @@ -2136,14 +2137,14 @@ static int f2fs_read_single_page(struct inode *inode, struct page *page, */ if (bio && (!page_is_mergeable(F2FS_I_SB(inode), bio, *last_block_in_bio, block_nr) || - !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) { + !f2fs_crypt_mergeable_bio(bio, inode, index, NULL))) { submit_and_realloc: f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); bio = NULL; } if (bio == NULL) { bio = f2fs_grab_read_bio(inode, block_nr, nr_pages, - is_readahead ? REQ_RAHEAD : 0, page->index, + is_readahead ? REQ_RAHEAD : 0, index, false); if (IS_ERR(bio)) { ret = PTR_ERR(bio); @@ -2158,7 +2159,7 @@ static int f2fs_read_single_page(struct inode *inode, struct page *page, */ f2fs_wait_on_block_writeback(inode, block_nr); - if (bio_add_page(bio, page, blocksize, 0) < blocksize) + if (!bio_add_folio(bio, folio, blocksize, 0)) goto submit_and_realloc; inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA); @@ -2424,7 +2425,7 @@ static int f2fs_mpage_readpages(struct inode *inode, read_single_page: #endif - ret = f2fs_read_single_page(inode, &folio->page, max_nr_pages, &map, + ret = f2fs_read_single_page(inode, folio, max_nr_pages, &map, &bio, &last_block_in_bio, rac); if (ret) { #ifdef CONFIG_F2FS_FS_COMPRESSION From 96ea46f30b2657375fc7695f78ddb2cb50413509 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Tue, 16 Apr 2024 15:23:17 +0800 Subject: [PATCH 0313/1702] f2fs: convert f2fs_read_inline_data() to use folio Convert f2fs_read_inline_data() to use folio and related functionality, and also convert its caller to use folio. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/data.c | 11 +++++------ fs/f2fs/f2fs.h | 4 ++-- fs/f2fs/inline.c | 34 +++++++++++++++++----------------- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 8874ca3a32e21..763f0952efdda 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -2458,20 +2458,19 @@ static int f2fs_mpage_readpages(struct inode *inode, static int f2fs_read_data_folio(struct file *file, struct folio *folio) { - struct page *page = &folio->page; - struct inode *inode = page_file_mapping(page)->host; + struct inode *inode = folio_file_mapping(folio)->host; int ret = -EAGAIN; - trace_f2fs_readpage(page, DATA); + trace_f2fs_readpage(&folio->page, DATA); if (!f2fs_is_compress_backend_ready(inode)) { - unlock_page(page); + folio_unlock(folio); return -EOPNOTSUPP; } /* If the file has inline data, try to read it directly */ if (f2fs_has_inline_data(inode)) - ret = f2fs_read_inline_data(inode, page); + ret = f2fs_read_inline_data(inode, folio); if (ret == -EAGAIN) ret = f2fs_mpage_readpages(inode, NULL, folio); return ret; @@ -3400,7 +3399,7 @@ static int prepare_write_begin(struct f2fs_sb_info *sbi, if (f2fs_has_inline_data(inode)) { if (pos + len <= MAX_INLINE_DATA(inode)) { - f2fs_do_read_inline_data(page, ipage); + f2fs_do_read_inline_data(page_folio(page), ipage); set_inode_flag(inode, FI_DATA_EXIST); if (inode->i_nlink) set_page_private_inline(ipage); diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 3f7196122574b..a0ae99bcca39b 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -4154,10 +4154,10 @@ extern struct kmem_cache *f2fs_inode_entry_slab; bool f2fs_may_inline_data(struct inode *inode); bool f2fs_sanity_check_inline_data(struct inode *inode); bool f2fs_may_inline_dentry(struct inode *inode); -void f2fs_do_read_inline_data(struct page *page, struct page *ipage); +void f2fs_do_read_inline_data(struct folio *folio, struct page *ipage); void f2fs_truncate_inline_inode(struct inode *inode, struct page *ipage, u64 from); -int f2fs_read_inline_data(struct inode *inode, struct page *page); +int f2fs_read_inline_data(struct inode *inode, struct folio *folio); int f2fs_convert_inline_page(struct dnode_of_data *dn, struct page *page); int f2fs_convert_inline_inode(struct inode *inode); int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry); diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index 3d3218a4b29df..7638d0d7b7eed 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -61,22 +61,22 @@ bool f2fs_may_inline_dentry(struct inode *inode) return true; } -void f2fs_do_read_inline_data(struct page *page, struct page *ipage) +void f2fs_do_read_inline_data(struct folio *folio, struct page *ipage) { - struct inode *inode = page->mapping->host; + struct inode *inode = folio_file_mapping(folio)->host; - if (PageUptodate(page)) + if (folio_test_uptodate(folio)) return; - f2fs_bug_on(F2FS_P_SB(page), page->index); + f2fs_bug_on(F2FS_I_SB(inode), folio_index(folio)); - zero_user_segment(page, MAX_INLINE_DATA(inode), PAGE_SIZE); + folio_zero_segment(folio, MAX_INLINE_DATA(inode), folio_size(folio)); /* Copy the whole inline data block */ - memcpy_to_page(page, 0, inline_data_addr(inode, ipage), + memcpy_to_folio(folio, 0, inline_data_addr(inode, ipage), MAX_INLINE_DATA(inode)); - if (!PageUptodate(page)) - SetPageUptodate(page); + if (!folio_test_uptodate(folio)) + folio_mark_uptodate(folio); } void f2fs_truncate_inline_inode(struct inode *inode, @@ -97,13 +97,13 @@ void f2fs_truncate_inline_inode(struct inode *inode, clear_inode_flag(inode, FI_DATA_EXIST); } -int f2fs_read_inline_data(struct inode *inode, struct page *page) +int f2fs_read_inline_data(struct inode *inode, struct folio *folio) { struct page *ipage; ipage = f2fs_get_node_page(F2FS_I_SB(inode), inode->i_ino); if (IS_ERR(ipage)) { - unlock_page(page); + folio_unlock(folio); return PTR_ERR(ipage); } @@ -112,15 +112,15 @@ int f2fs_read_inline_data(struct inode *inode, struct page *page) return -EAGAIN; } - if (page->index) - zero_user_segment(page, 0, PAGE_SIZE); + if (folio_index(folio)) + folio_zero_segment(folio, 0, folio_size(folio)); else - f2fs_do_read_inline_data(page, ipage); + f2fs_do_read_inline_data(folio, ipage); - if (!PageUptodate(page)) - SetPageUptodate(page); + if (!folio_test_uptodate(folio)) + folio_mark_uptodate(folio); f2fs_put_page(ipage, 1); - unlock_page(page); + folio_unlock(folio); return 0; } @@ -166,7 +166,7 @@ int f2fs_convert_inline_page(struct dnode_of_data *dn, struct page *page) f2fs_bug_on(F2FS_P_SB(page), folio_test_writeback(page_folio(page))); - f2fs_do_read_inline_data(page, dn->inode_page); + f2fs_do_read_inline_data(page_folio(page), dn->inode_page); set_page_dirty(page); /* clear dirty state */ From 92f750d847c997ff4b0f04d83443af00fb729ba3 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Tue, 16 Apr 2024 15:23:18 +0800 Subject: [PATCH 0314/1702] f2fs: convert f2fs__page tracepoint class to use folio Convert f2fs__page tracepoint class() and its instances to use folio and related functionality, and rename it to f2fs__folio(). Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/checkpoint.c | 4 ++-- fs/f2fs/data.c | 10 ++++----- fs/f2fs/node.c | 4 ++-- include/trace/events/f2fs.h | 42 ++++++++++++++++++------------------- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index eac698b8dd387..5d05a413f4510 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -345,7 +345,7 @@ static int __f2fs_write_meta_page(struct page *page, { struct f2fs_sb_info *sbi = F2FS_P_SB(page); - trace_f2fs_writepage(page, META); + trace_f2fs_writepage(page_folio(page), META); if (unlikely(f2fs_cp_error(sbi))) { if (is_sbi_flag_set(sbi, SBI_IS_CLOSE)) { @@ -492,7 +492,7 @@ long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type, static bool f2fs_dirty_meta_folio(struct address_space *mapping, struct folio *folio) { - trace_f2fs_set_page_dirty(&folio->page, META); + trace_f2fs_set_page_dirty(folio, META); if (!folio_test_uptodate(folio)) folio_mark_uptodate(folio); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 763f0952efdda..bee1e45f76b8a 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -2461,7 +2461,7 @@ static int f2fs_read_data_folio(struct file *file, struct folio *folio) struct inode *inode = folio_file_mapping(folio)->host; int ret = -EAGAIN; - trace_f2fs_readpage(&folio->page, DATA); + trace_f2fs_readpage(folio, DATA); if (!f2fs_is_compress_backend_ready(inode)) { folio_unlock(folio); @@ -2710,7 +2710,7 @@ int f2fs_do_write_data_page(struct f2fs_io_info *fio) } else { set_inode_flag(inode, FI_UPDATE_WRITE); } - trace_f2fs_do_write_data_page(fio->page, IPU); + trace_f2fs_do_write_data_page(page_folio(page), IPU); return err; } @@ -2739,7 +2739,7 @@ int f2fs_do_write_data_page(struct f2fs_io_info *fio) /* LFS mode write path */ f2fs_outplace_write_data(&dn, fio); - trace_f2fs_do_write_data_page(page, OPU); + trace_f2fs_do_write_data_page(page_folio(page), OPU); set_inode_flag(inode, FI_APPEND_WRITE); out_writepage: f2fs_put_dnode(&dn); @@ -2786,7 +2786,7 @@ int f2fs_write_single_data_page(struct page *page, int *submitted, .last_block = last_block, }; - trace_f2fs_writepage(page, DATA); + trace_f2fs_writepage(page_folio(page), DATA); /* we should bypass data pages to proceed the kworker jobs */ if (unlikely(f2fs_cp_error(sbi))) { @@ -3760,7 +3760,7 @@ static bool f2fs_dirty_data_folio(struct address_space *mapping, { struct inode *inode = mapping->host; - trace_f2fs_set_page_dirty(&folio->page, DATA); + trace_f2fs_set_page_dirty(folio, DATA); if (!folio_test_uptodate(folio)) folio_mark_uptodate(folio); diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 3b9eb56936832..95cecf08cb377 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -1624,7 +1624,7 @@ static int __write_node_page(struct page *page, bool atomic, bool *submitted, }; unsigned int seq; - trace_f2fs_writepage(page, NODE); + trace_f2fs_writepage(page_folio(page), NODE); if (unlikely(f2fs_cp_error(sbi))) { /* keep node pages in remount-ro mode */ @@ -2171,7 +2171,7 @@ static int f2fs_write_node_pages(struct address_space *mapping, static bool f2fs_dirty_node_folio(struct address_space *mapping, struct folio *folio) { - trace_f2fs_set_page_dirty(&folio->page, NODE); + trace_f2fs_set_page_dirty(folio, NODE); if (!folio_test_uptodate(folio)) folio_mark_uptodate(folio); diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h index 7ed0fc430dc61..371ba28415f5c 100644 --- a/include/trace/events/f2fs.h +++ b/include/trace/events/f2fs.h @@ -1304,11 +1304,11 @@ TRACE_EVENT(f2fs_write_end, __entry->copied) ); -DECLARE_EVENT_CLASS(f2fs__page, +DECLARE_EVENT_CLASS(f2fs__folio, - TP_PROTO(struct page *page, int type), + TP_PROTO(struct folio *folio, int type), - TP_ARGS(page, type), + TP_ARGS(folio, type), TP_STRUCT__entry( __field(dev_t, dev) @@ -1321,14 +1321,14 @@ DECLARE_EVENT_CLASS(f2fs__page, ), TP_fast_assign( - __entry->dev = page_file_mapping(page)->host->i_sb->s_dev; - __entry->ino = page_file_mapping(page)->host->i_ino; + __entry->dev = folio_file_mapping(folio)->host->i_sb->s_dev; + __entry->ino = folio_file_mapping(folio)->host->i_ino; __entry->type = type; __entry->dir = - S_ISDIR(page_file_mapping(page)->host->i_mode); - __entry->index = page->index; - __entry->dirty = PageDirty(page); - __entry->uptodate = PageUptodate(page); + S_ISDIR(folio_file_mapping(folio)->host->i_mode); + __entry->index = folio_index(folio); + __entry->dirty = folio_test_dirty(folio); + __entry->uptodate = folio_test_uptodate(folio); ), TP_printk("dev = (%d,%d), ino = %lu, %s, %s, index = %lu, " @@ -1341,32 +1341,32 @@ DECLARE_EVENT_CLASS(f2fs__page, __entry->uptodate) ); -DEFINE_EVENT(f2fs__page, f2fs_writepage, +DEFINE_EVENT(f2fs__folio, f2fs_writepage, - TP_PROTO(struct page *page, int type), + TP_PROTO(struct folio *folio, int type), - TP_ARGS(page, type) + TP_ARGS(folio, type) ); -DEFINE_EVENT(f2fs__page, f2fs_do_write_data_page, +DEFINE_EVENT(f2fs__folio, f2fs_do_write_data_page, - TP_PROTO(struct page *page, int type), + TP_PROTO(struct folio *folio, int type), - TP_ARGS(page, type) + TP_ARGS(folio, type) ); -DEFINE_EVENT(f2fs__page, f2fs_readpage, +DEFINE_EVENT(f2fs__folio, f2fs_readpage, - TP_PROTO(struct page *page, int type), + TP_PROTO(struct folio *folio, int type), - TP_ARGS(page, type) + TP_ARGS(folio, type) ); -DEFINE_EVENT(f2fs__page, f2fs_set_page_dirty, +DEFINE_EVENT(f2fs__folio, f2fs_set_page_dirty, - TP_PROTO(struct page *page, int type), + TP_PROTO(struct folio *folio, int type), - TP_ARGS(page, type) + TP_ARGS(folio, type) ); TRACE_EVENT(f2fs_replace_atomic_write_block, From 5bf624c0122960ebeeb544e2bc1a2e531ac11392 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Tue, 16 Apr 2024 15:21:07 +0800 Subject: [PATCH 0315/1702] f2fs: fix comment in sanity_check_raw_super() Commit d7e9a9037de2 ("f2fs: Support Block Size == Page Size") missed to adjust comment in sanity_check_raw_super(), fix it. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/super.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index df32573d1f620..5f4262f063024 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -3456,7 +3456,7 @@ static int sanity_check_raw_super(struct f2fs_sb_info *sbi, } } - /* Currently, support only 4KB block size */ + /* only support block_size equals to PAGE_SIZE */ if (le32_to_cpu(raw_super->log_blocksize) != F2FS_BLKSIZE_BITS) { f2fs_info(sbi, "Invalid log_blocksize (%u), supports only %u", le32_to_cpu(raw_super->log_blocksize), From 06b206d9e2b4c3e142d60d21912a7b46988dea29 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Tue, 16 Apr 2024 15:21:08 +0800 Subject: [PATCH 0316/1702] f2fs: remove unnecessary block size check in init_f2fs_fs() After commit d7e9a9037de2 ("f2fs: Support Block Size == Page Size"), F2FS_BLKSIZE equals to PAGE_SIZE, remove unnecessary check condition. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/super.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 5f4262f063024..ef673f853366b 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -4931,12 +4931,6 @@ static int __init init_f2fs_fs(void) { int err; - if (PAGE_SIZE != F2FS_BLKSIZE) { - printk("F2FS not supported on PAGE_SIZE(%lu) != BLOCK_SIZE(%lu)\n", - PAGE_SIZE, F2FS_BLKSIZE); - return -EINVAL; - } - err = init_inodecache(); if (err) goto fail; From 0a7c2fda3448b8f11a32c3e2fe94e70bd468be33 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 15 Apr 2024 15:45:20 +0200 Subject: [PATCH 0317/1702] clk: sophgo: avoid open-coded 64-bit division On 32-bit architectures, the 64-bit division leads to a link failure: arm-linux-gnueabi-ld: drivers/clk/sophgo/clk-cv18xx-pll.o: in function `fpll_calc_rate': clk-cv18xx-pll.c:(.text.fpll_calc_rate+0x26): undefined reference to `__aeabi_uldivmod' This one is not called in a fast path, and there is already another div_u64() variant used in the same function, so convert it to div64_u64_rem(). Fixes: 80fd61ec4612 ("clk: sophgo: Add clock support for CV1800 SoC") Signed-off-by: Arnd Bergmann Link: https://lore.kernel.org/r/20240415134532.3467817-1-arnd@kernel.org Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202404122344.d5pb2N1I-lkp@intel.com/ Closes: https://lore.kernel.org/oe-kbuild-all/202404140310.QEjZKtTN-lkp@intel.com/ Reviewed-by: Inochi Amaoto Signed-off-by: Stephen Boyd --- drivers/clk/sophgo/clk-cv18xx-pll.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/clk/sophgo/clk-cv18xx-pll.c b/drivers/clk/sophgo/clk-cv18xx-pll.c index c546dad1791c3..29e24098bf5f9 100644 --- a/drivers/clk/sophgo/clk-cv18xx-pll.c +++ b/drivers/clk/sophgo/clk-cv18xx-pll.c @@ -205,8 +205,7 @@ static unsigned long fpll_calc_rate(unsigned long parent_rate, unsigned long rate; dividend <<= PLL_SYN_FACTOR_DOT_POS - 1; - rate = dividend / factor; - dividend %= factor; + rate = div64_u64_rem(dividend, factor, ÷nd); if (is_full_parent) { dividend <<= 1; From 4c0c087772d7e29bc2489ddb068d5167140bfc38 Mon Sep 17 00:00:00 2001 From: Alexandre Mergnat Date: Thu, 18 Apr 2024 16:17:00 +0200 Subject: [PATCH 0318/1702] clk: mediatek: mt8365-mm: fix DPI0 parent To have a working display through DPI, a workaround has been implemented downstream to add "mm_dpi0_dpi0" and "dpi0_sel" to the DPI node. Shortly, that add an extra clock. It seems consistent to have the "dpi0_sel" as parent. Additionnaly, "vpll_dpix" isn't used/managed. Then, set the "mm_dpi0_dpi0" parent clock to "dpi0_sel". The new clock tree is: clk26m lvdspll lvdspll_X (2, 4, 8, 16) dpi0_sel mm_dpi0_dpi0 Fixes: d46adccb7966 ("clk: mediatek: add driver for MT8365 SoC") Signed-off-by: Alexandre Mergnat Link: https://lore.kernel.org/r/20231023-display-support-v3-12-53388f3ed34b@baylibre.com Reviewed-by: AngeloGioacchino Del Regno Signed-off-by: Stephen Boyd --- drivers/clk/mediatek/clk-mt8365-mm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clk/mediatek/clk-mt8365-mm.c b/drivers/clk/mediatek/clk-mt8365-mm.c index 01a2ef8f594ef..3f62ec7507336 100644 --- a/drivers/clk/mediatek/clk-mt8365-mm.c +++ b/drivers/clk/mediatek/clk-mt8365-mm.c @@ -53,7 +53,7 @@ static const struct mtk_gate mm_clks[] = { GATE_MM0(CLK_MM_MM_DSI0, "mm_dsi0", "mm_sel", 17), GATE_MM0(CLK_MM_MM_DISP_RDMA1, "mm_disp_rdma1", "mm_sel", 18), GATE_MM0(CLK_MM_MM_MDP_RDMA1, "mm_mdp_rdma1", "mm_sel", 19), - GATE_MM0(CLK_MM_DPI0_DPI0, "mm_dpi0_dpi0", "vpll_dpix", 20), + GATE_MM0(CLK_MM_DPI0_DPI0, "mm_dpi0_dpi0", "dpi0_sel", 20), GATE_MM0(CLK_MM_MM_FAKE, "mm_fake", "mm_sel", 21), GATE_MM0(CLK_MM_MM_SMI_COMMON, "mm_smi_common", "mm_sel", 22), GATE_MM0(CLK_MM_MM_SMI_LARB0, "mm_smi_larb0", "mm_sel", 23), From 5677925ed74759be0c46bb7b6a9cfc5ebf957523 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 13 Apr 2024 15:54:35 +0200 Subject: [PATCH 0319/1702] clk: highbank: Remove an unused field in struct hb_clk In "struct hb_clk", the 'parent_name' field is unused. Remove it. Found with cppcheck, unusedStructMember. Signed-off-by: Christophe JAILLET Link: https://lore.kernel.org/r/90b19f2af3077075d4254e01d5ae919c423d067e.1713016457.git.christophe.jaillet@wanadoo.fr Signed-off-by: Stephen Boyd --- drivers/clk/clk-highbank.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/clk/clk-highbank.c b/drivers/clk/clk-highbank.c index 2a0cea2946f98..6e68a41a70a1b 100644 --- a/drivers/clk/clk-highbank.c +++ b/drivers/clk/clk-highbank.c @@ -37,7 +37,6 @@ struct hb_clk { struct clk_hw hw; void __iomem *reg; - char *parent_name; }; #define to_hb_clk(p) container_of(p, struct hb_clk, hw) From 8e931ef1bdc5d60c530d182eeccad8b7e1ad6ed1 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 13 Apr 2024 15:46:09 +0200 Subject: [PATCH 0320/1702] clk: gemini: Remove an unused field in struct clk_gemini_pci In "struct clk_gemini_pci", the 'rate' field is unused. Remove it. Found with cppcheck, unusedStructMember. Signed-off-by: Christophe JAILLET Link: https://lore.kernel.org/r/556770c7701868f9f1c0569674903bee3eff30cb.1713015940.git.christophe.jaillet@wanadoo.fr Signed-off-by: Stephen Boyd --- drivers/clk/clk-gemini.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/clk/clk-gemini.c b/drivers/clk/clk-gemini.c index ba0ff01bf4dc4..856b008e07c6b 100644 --- a/drivers/clk/clk-gemini.c +++ b/drivers/clk/clk-gemini.c @@ -67,12 +67,10 @@ struct gemini_gate_data { * struct clk_gemini_pci - Gemini PCI clock * @hw: corresponding clock hardware entry * @map: regmap to access the registers - * @rate: current rate */ struct clk_gemini_pci { struct clk_hw hw; struct regmap *map; - unsigned long rate; }; /** From 1758c68c81b8b881818fcebaaeb91055362a82f8 Mon Sep 17 00:00:00 2001 From: Catalin Popescu Date: Mon, 15 Apr 2024 16:03:48 +0200 Subject: [PATCH 0321/1702] clk: rs9: fix wrong default value for clock amplitude According to 9FGV0241, 9FGV0441 & 9FGV0841 datasheets, the default value for the clock amplitude is 0.8V, while the driver assumes 0.7V. Additionally, define constants for default values for both clock amplitude and spread spectrum and use them. Fixes: 892e0ddea1aa ("clk: rs9: Add Renesas 9-series PCIe clock generator driver") Signed-off-by: Catalin Popescu Reviewed-by: Marek Vasut Link: https://lore.kernel.org/r/20240415140348.2887619-1-catalin.popescu@leica-geosystems.com Signed-off-by: Stephen Boyd --- drivers/clk/clk-renesas-pcie.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/clk/clk-renesas-pcie.c b/drivers/clk/clk-renesas-pcie.c index 53e21ac302e6d..4c3a5e4eb77ac 100644 --- a/drivers/clk/clk-renesas-pcie.c +++ b/drivers/clk/clk-renesas-pcie.c @@ -25,10 +25,12 @@ #define RS9_REG_SS_AMP_0V7 0x1 #define RS9_REG_SS_AMP_0V8 0x2 #define RS9_REG_SS_AMP_0V9 0x3 +#define RS9_REG_SS_AMP_DEFAULT RS9_REG_SS_AMP_0V8 #define RS9_REG_SS_AMP_MASK 0x3 #define RS9_REG_SS_SSC_100 0 #define RS9_REG_SS_SSC_M025 (1 << 3) #define RS9_REG_SS_SSC_M050 (3 << 3) +#define RS9_REG_SS_SSC_DEFAULT RS9_REG_SS_SSC_100 #define RS9_REG_SS_SSC_MASK (3 << 3) #define RS9_REG_SS_SSC_LOCK BIT(5) #define RS9_REG_SR 0x2 @@ -205,8 +207,8 @@ static int rs9_get_common_config(struct rs9_driver_data *rs9) int ret; /* Set defaults */ - rs9->pll_amplitude = RS9_REG_SS_AMP_0V7; - rs9->pll_ssc = RS9_REG_SS_SSC_100; + rs9->pll_amplitude = RS9_REG_SS_AMP_DEFAULT; + rs9->pll_ssc = RS9_REG_SS_SSC_DEFAULT; /* Output clock amplitude */ ret = of_property_read_u32(np, "renesas,out-amplitude-microvolt", @@ -247,13 +249,13 @@ static void rs9_update_config(struct rs9_driver_data *rs9) int i; /* If amplitude is non-default, update it. */ - if (rs9->pll_amplitude != RS9_REG_SS_AMP_0V7) { + if (rs9->pll_amplitude != RS9_REG_SS_AMP_DEFAULT) { regmap_update_bits(rs9->regmap, RS9_REG_SS, RS9_REG_SS_AMP_MASK, rs9->pll_amplitude); } /* If SSC is non-default, update it. */ - if (rs9->pll_ssc != RS9_REG_SS_SSC_100) { + if (rs9->pll_ssc != RS9_REG_SS_SSC_DEFAULT) { regmap_update_bits(rs9->regmap, RS9_REG_SS, RS9_REG_SS_SSC_MASK, rs9->pll_ssc); } From 2c03d9560ecebf2865b963e457ba89299c5f1966 Mon Sep 17 00:00:00 2001 From: Dave Chinner Date: Wed, 3 Apr 2024 08:28:28 +1100 Subject: [PATCH 0322/1702] xfs: fix CIL sparse lock context warnings Sparse reports: fs/xfs/xfs_log_cil.c:1127:1: warning: context imbalance in 'xlog_cil_push_work' - different lock contexts for basic block fs/xfs/xfs_log_cil.c:1380:1: warning: context imbalance in 'xlog_cil_push_background' - wrong count at exit fs/xfs/xfs_log_cil.c:1623:9: warning: context imbalance in 'xlog_cil_commit' - unexpected unlock xlog_cil_push_background() has a locking annotations for an rw_sem. Sparse does not track lock contexts for rw_sems, so the annotation generates false warnings. Remove the annotation. xlog_wait_on_iclog() drops the log->l_ic_loglock. The function has a sparse annotation, but the prototype in xfs_log_priv.h does not. Hence the warning from xlog_cil_push_work() which calls xlog_wait_on_iclog(). Add the missing annotation. Signed-off-by: Dave Chinner Reviewed-by: Christoph Hellwig Signed-off-by: Chandan Babu R --- fs/xfs/xfs_log_cil.c | 2 +- fs/xfs/xfs_log_priv.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/xfs/xfs_log_cil.c b/fs/xfs/xfs_log_cil.c index 73f5b7f628f4f..f51cbc6405c15 100644 --- a/fs/xfs/xfs_log_cil.c +++ b/fs/xfs/xfs_log_cil.c @@ -1378,7 +1378,7 @@ xlog_cil_push_work( */ static void xlog_cil_push_background( - struct xlog *log) __releases(cil->xc_ctx_lock) + struct xlog *log) { struct xfs_cil *cil = log->l_cilp; int space_used = atomic_read(&cil->xc_ctx->space_used); diff --git a/fs/xfs/xfs_log_priv.h b/fs/xfs/xfs_log_priv.h index 43881575cd498..a78c4bfaad447 100644 --- a/fs/xfs/xfs_log_priv.h +++ b/fs/xfs/xfs_log_priv.h @@ -620,7 +620,8 @@ xlog_wait( remove_wait_queue(wq, &wait); } -int xlog_wait_on_iclog(struct xlog_in_core *iclog); +int xlog_wait_on_iclog(struct xlog_in_core *iclog) + __releases(iclog->ic_log->l_icloglock); /* * The LSN is valid so long as it is behind the current LSN. If it isn't, this From 810e6d2fac01ab1e189f3002a364f19bd5d0f444 Mon Sep 17 00:00:00 2001 From: Abel Vesa Date: Thu, 18 Apr 2024 16:41:32 +0300 Subject: [PATCH 0323/1702] clk: qcom: clk-alpha-pll: Skip reconfiguring the running Lucid Evo The PLL0 is configured by the bootloader and is the parent of the mdp_clk_src. The Trion implementation of the configure function is already skipping this step if the PLL is enabled, so lets extend the same behavior to Lucid Evo variant. Signed-off-by: Abel Vesa Reviewed-by: Bjorn Andersson Link: https://lore.kernel.org/r/20240418-clk-qcom-lucid-evo-skip-configuring-enabled-v1-1-caede5f1c7a3@linaro.org Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/clk-alpha-pll.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index 8a412ef47e163..4c5aeccff0ef5 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -2114,6 +2114,15 @@ void clk_lucid_evo_pll_configure(struct clk_alpha_pll *pll, struct regmap *regma { u32 lval = config->l; + /* + * If the bootloader left the PLL enabled it's likely that there are + * RCGs that will lock up if we disable the PLL below. + */ + if (trion_pll_is_enabled(pll, regmap)) { + pr_debug("Lucid Evo PLL is already enabled, skipping configuration\n"); + return; + } + lval |= TRION_PLL_CAL_VAL << LUCID_EVO_PLL_CAL_L_VAL_SHIFT; clk_alpha_pll_write_config(regmap, PLL_L_VAL(pll), lval); clk_alpha_pll_write_config(regmap, PLL_ALPHA_VAL(pll), config->alpha); From 3cb55215479f04940240792479c60fad53c2d202 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 13 Apr 2024 16:04:04 +0200 Subject: [PATCH 0324/1702] clk: qcom: rpm: Remove an unused field in struct rpm_cc In "struct rpm_cc", the 'rpm' field is unused. Remove it. Found with cppcheck, unusedStructMember. Signed-off-by: Christophe JAILLET Reviewed-by: Stephen Boyd Link: https://lore.kernel.org/r/9f92330c717e6f2dab27b1307565ffb108c304a7.1713017032.git.christophe.jaillet@wanadoo.fr Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/clk-rpm.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/clk/qcom/clk-rpm.c b/drivers/clk/qcom/clk-rpm.c index 745026ef4d9cf..9da034f8f2ff5 100644 --- a/drivers/clk/qcom/clk-rpm.c +++ b/drivers/clk/qcom/clk-rpm.c @@ -98,7 +98,6 @@ struct clk_rpm { }; struct rpm_cc { - struct qcom_rpm *rpm; struct clk_rpm **clks; size_t num_clks; u32 xo_buffer_value; From ca0b44e20a6f3032224599f02e7c8fb49525c894 Mon Sep 17 00:00:00 2001 From: Michael Guralnik Date: Tue, 16 Apr 2024 15:01:44 +0300 Subject: [PATCH 0325/1702] IB/core: Implement a limit on UMAD receive List The existing behavior of ib_umad, which maintains received MAD packets in an unbounded list, poses a risk of uncontrolled growth. As user-space applications extract packets from this list, the rate of extraction may not match the rate of incoming packets, leading to potential list overflow. To address this, we introduce a limit to the size of the list. After considering typical scenarios, such as OpenSM processing, which can handle approximately 100k packets per second, and the 1-second retry timeout for most packets, we set the list size limit to 200k. Packets received beyond this limit are dropped, assuming they are likely timed out by the time they are handled by user-space. Notably, packets queued on the receive list due to reasons like timed-out sends are preserved even when the list is full. Signed-off-by: Michael Guralnik Reviewed-by: Mark Zhang Link: https://lore.kernel.org/r/7197cb58a7d9e78399008f25036205ceab07fbd5.1713268818.git.leon@kernel.org Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/user_mad.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c index f5feca7fa9b9c..2ed749f50a29f 100644 --- a/drivers/infiniband/core/user_mad.c +++ b/drivers/infiniband/core/user_mad.c @@ -63,6 +63,8 @@ MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("InfiniBand userspace MAD packet access"); MODULE_LICENSE("Dual BSD/GPL"); +#define MAX_UMAD_RECV_LIST_SIZE 200000 + enum { IB_UMAD_MAX_PORTS = RDMA_MAX_PORTS, IB_UMAD_MAX_AGENTS = 32, @@ -113,6 +115,7 @@ struct ib_umad_file { struct mutex mutex; struct ib_umad_port *port; struct list_head recv_list; + atomic_t recv_list_size; struct list_head send_list; struct list_head port_list; spinlock_t send_lock; @@ -180,24 +183,28 @@ static struct ib_mad_agent *__get_agent(struct ib_umad_file *file, int id) return file->agents_dead ? NULL : file->agent[id]; } -static int queue_packet(struct ib_umad_file *file, - struct ib_mad_agent *agent, - struct ib_umad_packet *packet) +static int queue_packet(struct ib_umad_file *file, struct ib_mad_agent *agent, + struct ib_umad_packet *packet, bool is_recv_mad) { int ret = 1; mutex_lock(&file->mutex); + if (is_recv_mad && + atomic_read(&file->recv_list_size) > MAX_UMAD_RECV_LIST_SIZE) + goto unlock; + for (packet->mad.hdr.id = 0; packet->mad.hdr.id < IB_UMAD_MAX_AGENTS; packet->mad.hdr.id++) if (agent == __get_agent(file, packet->mad.hdr.id)) { list_add_tail(&packet->list, &file->recv_list); + atomic_inc(&file->recv_list_size); wake_up_interruptible(&file->recv_wait); ret = 0; break; } - +unlock: mutex_unlock(&file->mutex); return ret; @@ -224,7 +231,7 @@ static void send_handler(struct ib_mad_agent *agent, if (send_wc->status == IB_WC_RESP_TIMEOUT_ERR) { packet->length = IB_MGMT_MAD_HDR; packet->mad.hdr.status = ETIMEDOUT; - if (!queue_packet(file, agent, packet)) + if (!queue_packet(file, agent, packet, false)) return; } kfree(packet); @@ -284,7 +291,7 @@ static void recv_handler(struct ib_mad_agent *agent, rdma_destroy_ah_attr(&ah_attr); } - if (queue_packet(file, agent, packet)) + if (queue_packet(file, agent, packet, true)) goto err2; return; @@ -409,6 +416,7 @@ static ssize_t ib_umad_read(struct file *filp, char __user *buf, packet = list_entry(file->recv_list.next, struct ib_umad_packet, list); list_del(&packet->list); + atomic_dec(&file->recv_list_size); mutex_unlock(&file->mutex); @@ -421,6 +429,7 @@ static ssize_t ib_umad_read(struct file *filp, char __user *buf, /* Requeue packet */ mutex_lock(&file->mutex); list_add(&packet->list, &file->recv_list); + atomic_inc(&file->recv_list_size); mutex_unlock(&file->mutex); } else { if (packet->recv_wc) From 7ca07a174f3b4021ae0a2d60fed1f5c993cf4b49 Mon Sep 17 00:00:00 2001 From: Satya Priya Kakitapalli Date: Tue, 13 Feb 2024 12:17:24 +0530 Subject: [PATCH 0326/1702] clk: qcom: gcc-sm8150: De-register gcc_cpuss_ahb_clk_src De-register the gcc_cpuss_ahb_clk_src and its branch clocks as there is no rate setting happening on them. Signed-off-by: Satya Priya Kakitapalli Link: https://lore.kernel.org/r/20240213-gcc-ao-support-v2-1-fd2127e8d8f4@quicinc.com Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/gcc-sm8150.c | 61 ----------------------------------- 1 file changed, 61 deletions(-) diff --git a/drivers/clk/qcom/gcc-sm8150.c b/drivers/clk/qcom/gcc-sm8150.c index a47ef9dfa8080..1f748141d12c6 100644 --- a/drivers/clk/qcom/gcc-sm8150.c +++ b/drivers/clk/qcom/gcc-sm8150.c @@ -207,28 +207,6 @@ static const struct clk_parent_data gcc_parents_7[] = { { .hw = &gpll0_out_even.clkr.hw }, }; -static const struct freq_tbl ftbl_gcc_cpuss_ahb_clk_src[] = { - F(19200000, P_BI_TCXO, 1, 0, 0), - F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0), - F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), - { } -}; - -static struct clk_rcg2 gcc_cpuss_ahb_clk_src = { - .cmd_rcgr = 0x48014, - .mnd_width = 0, - .hid_width = 5, - .parent_map = gcc_parent_map_0, - .freq_tbl = ftbl_gcc_cpuss_ahb_clk_src, - .clkr.hw.init = &(struct clk_init_data){ - .name = "gcc_cpuss_ahb_clk_src", - .parent_data = gcc_parents_0, - .num_parents = ARRAY_SIZE(gcc_parents_0), - .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, - }, -}; - static const struct freq_tbl ftbl_gcc_emac_ptp_clk_src[] = { F(19200000, P_BI_TCXO, 1, 0, 0), F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), @@ -1361,24 +1339,6 @@ static struct clk_branch gcc_cfg_noc_usb3_sec_axi_clk = { }, }; -static struct clk_branch gcc_cpuss_ahb_clk = { - .halt_reg = 0x48000, - .halt_check = BRANCH_HALT_VOTED, - .clkr = { - .enable_reg = 0x52004, - .enable_mask = BIT(21), - .hw.init = &(struct clk_init_data){ - .name = "gcc_cpuss_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ - &gcc_cpuss_ahb_clk_src.clkr.hw }, - .num_parents = 1, - /* required for cpuss */ - .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_cpuss_dvm_bus_clk = { .halt_reg = 0x48190, .halt_check = BRANCH_HALT, @@ -2685,24 +2645,6 @@ static struct clk_branch gcc_sdcc4_apps_clk = { }, }; -static struct clk_branch gcc_sys_noc_cpuss_ahb_clk = { - .halt_reg = 0x4819c, - .halt_check = BRANCH_HALT_VOTED, - .clkr = { - .enable_reg = 0x52004, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "gcc_sys_noc_cpuss_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ - &gcc_cpuss_ahb_clk_src.clkr.hw }, - .num_parents = 1, - /* required for cpuss */ - .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_tsif_ahb_clk = { .halt_reg = 0x36004, .halt_check = BRANCH_HALT, @@ -3550,8 +3492,6 @@ static struct clk_regmap *gcc_sm8150_clocks[] = { [GCC_CAMERA_XO_CLK] = &gcc_camera_xo_clk.clkr, [GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr, [GCC_CFG_NOC_USB3_SEC_AXI_CLK] = &gcc_cfg_noc_usb3_sec_axi_clk.clkr, - [GCC_CPUSS_AHB_CLK] = &gcc_cpuss_ahb_clk.clkr, - [GCC_CPUSS_AHB_CLK_SRC] = &gcc_cpuss_ahb_clk_src.clkr, [GCC_CPUSS_DVM_BUS_CLK] = &gcc_cpuss_dvm_bus_clk.clkr, [GCC_CPUSS_GNOC_CLK] = &gcc_cpuss_gnoc_clk.clkr, [GCC_CPUSS_RBCPR_CLK] = &gcc_cpuss_rbcpr_clk.clkr, @@ -3669,7 +3609,6 @@ static struct clk_regmap *gcc_sm8150_clocks[] = { [GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr, [GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr, [GCC_SDCC4_APPS_CLK_SRC] = &gcc_sdcc4_apps_clk_src.clkr, - [GCC_SYS_NOC_CPUSS_AHB_CLK] = &gcc_sys_noc_cpuss_ahb_clk.clkr, [GCC_TSIF_AHB_CLK] = &gcc_tsif_ahb_clk.clkr, [GCC_TSIF_INACTIVITY_TIMERS_CLK] = &gcc_tsif_inactivity_timers_clk.clkr, [GCC_TSIF_REF_CLK] = &gcc_tsif_ref_clk.clkr, From 4f2bc4acbb1916b8cd2ce4bb3ba7b1cd7cb705fa Mon Sep 17 00:00:00 2001 From: Gabor Juhos Date: Mon, 11 Mar 2024 19:45:19 +0100 Subject: [PATCH 0327/1702] clk: qcom: clk-alpha-pll: remove invalid Stromer register offset The offset of the CONFIG_CTL_U register defined for the Stromer PLL is wrong. It is not aligned on a 4 bytes boundary which might causes errors in regmap operations. Maybe the intention behind of using the 0xff value was to indicate that the register is not implemented in the PLL, but this is not verified anywhere in the code. Moreover, this value is not used even in other register offset arrays despite that those PLLs also have unimplemented registers. Additionally, on the Stromer PLLs the current code only touches the CONFIG_CTL_U register if the result of pll_has_64bit_config() is true which condition is not affected by the change. Due to the reasons above, simply remove the CONFIG_CTL_U entry from the Stromer specific array. Fixes: e47a4f55f240 ("clk: qcom: clk-alpha-pll: Add support for Stromer PLLs") Signed-off-by: Gabor Juhos Reviewed-by: Konrad Dybcio Link: https://lore.kernel.org/r/20240311-alpha-pll-stromer-cleanup-v1-1-f7c0c5607cca@gmail.com Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/clk-alpha-pll.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index 4c5aeccff0ef5..acb68ec0c579e 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -213,7 +213,6 @@ const u8 clk_alpha_pll_regs[][PLL_OFF_MAX_REGS] = { [PLL_OFF_USER_CTL] = 0x18, [PLL_OFF_USER_CTL_U] = 0x1c, [PLL_OFF_CONFIG_CTL] = 0x20, - [PLL_OFF_CONFIG_CTL_U] = 0xff, [PLL_OFF_TEST_CTL] = 0x30, [PLL_OFF_TEST_CTL_U] = 0x34, [PLL_OFF_STATUS] = 0x28, From 8c48466cd7eda9afb37f26c8c9a68f39fae5ef32 Mon Sep 17 00:00:00 2001 From: Gabor Juhos Date: Mon, 11 Mar 2024 19:45:20 +0100 Subject: [PATCH 0328/1702] clk: qcom: clk-alpha-pll: reorder Stromer register offsets The register offset arrays are ordered based on the register offsets for all PLLs but the Stromer. For consistency, reorder the Stromer specific array as well. No functional changes. Signed-off-by: Gabor Juhos Reviewed-by: Konrad Dybcio Link: https://lore.kernel.org/r/20240311-alpha-pll-stromer-cleanup-v1-2-f7c0c5607cca@gmail.com Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/clk-alpha-pll.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index acb68ec0c579e..7f0ed5bd51e3d 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -213,9 +213,9 @@ const u8 clk_alpha_pll_regs[][PLL_OFF_MAX_REGS] = { [PLL_OFF_USER_CTL] = 0x18, [PLL_OFF_USER_CTL_U] = 0x1c, [PLL_OFF_CONFIG_CTL] = 0x20, + [PLL_OFF_STATUS] = 0x28, [PLL_OFF_TEST_CTL] = 0x30, [PLL_OFF_TEST_CTL_U] = 0x34, - [PLL_OFF_STATUS] = 0x28, }, [CLK_ALPHA_PLL_TYPE_STROMER_PLUS] = { [PLL_OFF_L_VAL] = 0x04, From 3b985489178cfb545118bd4dbefb6311f4bbfcf2 Mon Sep 17 00:00:00 2001 From: Gabor Juhos Date: Thu, 21 Mar 2024 09:59:04 +0100 Subject: [PATCH 0329/1702] clk: qcom: clk-alpha-pll: fix kerneldoc of struct clk_alpha_pll Add missing descriptions of the 'num_vco' and 'flags' members to clk_alpha_pll structure's documentation. Also reorder the member description entries to match the order of the declarations. Eliminates the following warnings: drivers/clk/qcom/clk-alpha-pll.h:72: info: Scanning doc for struct clk_alpha_pll drivers/clk/qcom/clk-alpha-pll.h:91: warning: Function parameter or struct member 'num_vco' not described in 'clk_alpha_pll' drivers/clk/qcom/clk-alpha-pll.h:91: warning: Function parameter or struct member 'flags' not described in 'clk_alpha_pll' No functional changes. Signed-off-by: Gabor Juhos Reviewed-by: Dmitry Baryshkov Link: https://lore.kernel.org/r/20240321-alpha-pll-kerneldoc-v1-1-0d76926b72c3@gmail.com Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/clk-alpha-pll.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/clk/qcom/clk-alpha-pll.h b/drivers/clk/qcom/clk-alpha-pll.h index fb6d50263bb9d..1bb2d031dc9fa 100644 --- a/drivers/clk/qcom/clk-alpha-pll.h +++ b/drivers/clk/qcom/clk-alpha-pll.h @@ -73,8 +73,10 @@ struct pll_vco { /** * struct clk_alpha_pll - phase locked loop (PLL) * @offset: base address of registers - * @vco_table: array of VCO settings * @regs: alpha pll register map (see @clk_alpha_pll_regs) + * @vco_table: array of VCO settings + * @num_vco: number of VCO settings in @vco_table + * @flags: bitmask to indicate features supported by the hardware * @clkr: regmap clock handle */ struct clk_alpha_pll { From 231ce08b662a58d4392da998699b3d4a7e2e87cf Mon Sep 17 00:00:00 2001 From: Len Brown Date: Sun, 21 Apr 2024 12:53:53 -0400 Subject: [PATCH 0330/1702] tools/power turbostat: Add "snapshot:" Makefile target Kernel developers often need to diagnose remote customer systems with the latest turbostat, yet customers are running binary distros with out-dated turbostat and the customer has no experience cloning linux kernel trees. Add a turbostat "snapshot" makefile target to create a standalone source snapshot from the developer's git tree, appropriately hacked so that the customer can build turbostat without a kernel tree. Include the turbostat binary in the snapshot, for convenience in those situations where the source and destination are trusted, (and have new enough glibc to execute). The snapshot is named with the date it was taken rather than the turbostat VERSION, as it could occur between VERSIONS... Signed-off-by: Len Brown --- tools/power/x86/turbostat/Makefile | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tools/power/x86/turbostat/Makefile b/tools/power/x86/turbostat/Makefile index 92e139b9c792d..2d6dce2c8f77f 100644 --- a/tools/power/x86/turbostat/Makefile +++ b/tools/power/x86/turbostat/Makefile @@ -3,6 +3,8 @@ CC = $(CROSS_COMPILE)gcc BUILD_OUTPUT := $(CURDIR) PREFIX ?= /usr DESTDIR ?= +DAY := $(shell date +%Y.%m.%d) +SNAPSHOT = turbostat-$(DAY) ifeq ("$(origin O)", "command line") BUILD_OUTPUT := $(O) @@ -22,9 +24,30 @@ override CFLAGS += -D_FORTIFY_SOURCE=2 .PHONY : clean clean : @rm -f $(BUILD_OUTPUT)/turbostat + @rm -f $(SNAPSHOT).tar.gz install : turbostat - install -d $(DESTDIR)$(PREFIX)/bin + install -d $(DESTDIR)$(PREFIX)/bin install $(BUILD_OUTPUT)/turbostat $(DESTDIR)$(PREFIX)/bin/turbostat - install -d $(DESTDIR)$(PREFIX)/share/man/man8 + install -d $(DESTDIR)$(PREFIX)/share/man/man8 install -m 644 turbostat.8 $(DESTDIR)$(PREFIX)/share/man/man8 + +snapshot: turbostat + @rm -rf $(SNAPSHOT) + @mkdir $(SNAPSHOT) + @cp turbostat Makefile turbostat.c turbostat.8 ../../../../arch/x86/include/asm/intel-family.h $(SNAPSHOT) + + @sed -e 's/^#include /#include "bits.h"/' ../../../../arch/x86/include/asm/msr-index.h > $(SNAPSHOT)/msr-index.h + @echo '#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))' >> $(SNAPSHOT)/msr-index.h + @echo "#define BIT(x) (1 << (x))" > $(SNAPSHOT)/bits.h + @echo "#define BIT_ULL(nr) (1ULL << (nr))" >> $(SNAPSHOT)/bits.h + @echo "#define GENMASK(h, l) (((~0UL) << (l)) & (~0UL >> (sizeof(long) * 8 - 1 - (h))))" >> $(SNAPSHOT)/bits.h + @echo "#define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (sizeof(long long) * 8 - 1 - (h))))" >> $(SNAPSHOT)/bits.h + + @echo PWD=. > $(SNAPSHOT)/Makefile + @echo "CFLAGS += -DMSRHEADER='\"msr-index.h\"'" >> $(SNAPSHOT)/Makefile + @echo "CFLAGS += -DINTEL_FAMILY_HEADER='\"intel-family.h\"'" >> $(SNAPSHOT)/Makefile + @sed -e's/.*MSRHEADER.*//' -e's/.*INTEL_FAMILY_HEADER.*//' Makefile >> $(SNAPSHOT)/Makefile + + @rm -f $(SNAPSHOT).tar.gz + tar cvzf $(SNAPSHOT).tar.gz $(SNAPSHOT) From ae3326ac5742506409a03ce5d69716a8dba4eabc Mon Sep 17 00:00:00 2001 From: Len Brown Date: Sun, 21 Apr 2024 14:45:10 -0400 Subject: [PATCH 0331/1702] tools/power turbostat: Harden probe_intel_uncore_frequency() If sysfs directory "intel_uncore_frequency/cluster00/" exists, then use uncore cluster code (now its own routine). The previous check for "intel_uncore_frequency/package_00_die_00/current_freq_khz", could be unreliable in the face of sparse die id's. Signed-off-by: Len Brown --- tools/power/x86/turbostat/turbostat.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index 98256468e2480..ca33fb057d1fd 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -5294,16 +5294,13 @@ static void dump_sysfs_file(char *path) fprintf(outf, "%s: %s", strrchr(path, '/') + 1, cpuidle_buf); } -static void probe_intel_uncore_frequency(void) +static void probe_intel_uncore_frequency_legacy(void) { int i, j; char path[256]; - if (!genuine_intel) - return; - if (access("/sys/devices/system/cpu/intel_uncore_frequency/package_00_die_00/current_freq_khz", R_OK)) - goto probe_cluster; + return; BIC_PRESENT(BIC_UNCORE_MHZ); @@ -5335,9 +5332,13 @@ static void probe_intel_uncore_frequency(void) fprintf(outf, " %d MHz\n", k / 1000); } } - return; +} + +static void probe_intel_uncore_frequency_cluster(void) +{ + int i; + char path[256]; -probe_cluster: if (access("/sys/devices/system/cpu/intel_uncore_frequency/uncore00/current_freq_khz", R_OK)) return; @@ -5351,6 +5352,7 @@ static void probe_intel_uncore_frequency(void) sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", i); + /* uncore## start at 00 and skip no numbers, so stop upon first missing */ if (access(path_base, R_OK)) break; @@ -5382,6 +5384,17 @@ static void probe_intel_uncore_frequency(void) } } +static void probe_intel_uncore_frequency(void) +{ + if (!genuine_intel) + return; + + if (access("/sys/devices/system/cpu/intel_uncore_frequency/uncore00", R_OK) == 0) + probe_intel_uncore_frequency_cluster(); + else + probe_intel_uncore_frequency_legacy(); +} + static void probe_graphics(void) { /* Xe graphics sysfs knobs */ From cda203388687aa075db6f8996c3c4549fa518ea8 Mon Sep 17 00:00:00 2001 From: Len Brown Date: Sun, 21 Apr 2024 11:56:48 -0400 Subject: [PATCH 0332/1702] tools/power turbostat: Remember global max_die_id This is necessary to gracefully handle sparse die_id's. no functional change Signed-off-by: Len Brown --- tools/power/x86/turbostat/turbostat.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index ca33fb057d1fd..e6d643a58cd89 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -1415,6 +1415,7 @@ struct topo_params { int allowed_cpus; int allowed_cores; int max_cpu_num; + int max_die_id; int max_node_num; int nodes_per_pkg; int cores_per_node; @@ -6980,7 +6981,6 @@ void topology_probe(bool startup) int i; int max_core_id = 0; int max_package_id = 0; - int max_die_id = 0; int max_siblings = 0; /* Initialize num_cpus, max_cpu_num */ @@ -7097,8 +7097,8 @@ void topology_probe(bool startup) /* get die information */ cpus[i].die_id = get_die_id(i); - if (cpus[i].die_id > max_die_id) - max_die_id = cpus[i].die_id; + if (cpus[i].die_id > topo.max_die_id) + topo.max_die_id = cpus[i].die_id; /* get numa node information */ cpus[i].physical_node_id = get_physical_node_id(&cpus[i]); @@ -7124,9 +7124,9 @@ void topology_probe(bool startup) if (!summary_only && topo.cores_per_node > 1) BIC_PRESENT(BIC_Core); - topo.num_die = max_die_id + 1; + topo.num_die = topo.max_die_id + 1; if (debug > 1) - fprintf(outf, "max_die_id %d, sizing for %d die\n", max_die_id, topo.num_die); + fprintf(outf, "max_die_id %d, sizing for %d die\n", topo.max_die_id, topo.num_die); if (!summary_only && topo.num_die > 1) BIC_PRESENT(BIC_Die); From c8b246ea2ea5771f2a0ca6f6a9a520406e6b6eb7 Mon Sep 17 00:00:00 2001 From: Len Brown Date: Sun, 21 Apr 2024 12:02:24 -0400 Subject: [PATCH 0333/1702] tools/power turbostat: Survive sparse die_id Turbostat assumed that every package had a die_id = 0. When this assumption was violated, it exited when looking for the package uncore frequency: turbostat: /sys/.../intel_uncore_frequency/package_01_die_00/current_freq_khz: open failed: No such file or directory Signed-off-by: Len Brown --- tools/power/x86/turbostat/turbostat.c | 45 +++++++++++++++++---------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index e6d643a58cd89..4b95fd90e16c3 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -2992,14 +2992,29 @@ int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp) return 0; } -unsigned long long get_uncore_mhz(int package, int die) +unsigned long long get_legacy_uncore_mhz(int package) { char path[128]; + int die; + static int warn_once; - sprintf(path, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d/current_freq_khz", package, - die); + /* + * for this package, use the first die_id that exists + */ + for (die = 0; die <= topo.max_die_id; ++die) { + + sprintf(path, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d/current_freq_khz", + package, die); - return (snapshot_sysfs_counter(path) / 1000); + if (access(path, R_OK) == 0) + return (snapshot_sysfs_counter(path) / 1000); + } + if (!warn_once) { + warnx("BUG: %s: No %s", __func__, path); + warn_once = 1; + } + + return 0; } int get_epb(int cpu) @@ -3631,9 +3646,8 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) p->pkg_temp_c = tj_max - ((msr >> 16) & 0x7F); } - /* n.b. assume die0 uncore frequency applies to whole package */ if (DO_BIC(BIC_UNCORE_MHZ)) - p->uncore_mhz = get_uncore_mhz(p->package_id, 0); + p->uncore_mhz = get_legacy_uncore_mhz(p->package_id); if (DO_BIC(BIC_GFX_rc6)) p->gfx_rc6_ms = gfx_info[GFX_rc6].val_ull; @@ -5300,22 +5314,22 @@ static void probe_intel_uncore_frequency_legacy(void) int i, j; char path[256]; - if (access("/sys/devices/system/cpu/intel_uncore_frequency/package_00_die_00/current_freq_khz", R_OK)) - return; - - BIC_PRESENT(BIC_UNCORE_MHZ); - - if (quiet) - return; - for (i = 0; i < topo.num_packages; ++i) { - for (j = 0; j < topo.num_die; ++j) { + for (j = 0; j <= topo.max_die_id; ++j) { int k, l; char path_base[128]; sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d", i, j); + if (access(path_base, R_OK)) + continue; + + BIC_PRESENT(BIC_UNCORE_MHZ); + + if (quiet) + return; + sprintf(path, "%s/min_freq_khz", path_base); k = read_sysfs_int(path); sprintf(path, "%s/max_freq_khz", path_base); @@ -5480,7 +5494,6 @@ static void probe_graphics(void) else if (!access("/sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz", R_OK)) gfx_info[GFX_MHz].path = "/sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz"; - if (!access("/sys/class/drm/card0/gt_act_freq_mhz", R_OK)) gfx_info[GFX_ACTMHz].path = "/sys/class/drm/card0/gt_act_freq_mhz"; else if (!access("/sys/class/graphics/fb0/device/drm/card0/gt_act_freq_mhz", R_OK)) From 1496dd413b2e0974a040fa93a2ddc51cc9847fd8 Mon Sep 17 00:00:00 2001 From: Shengjiu Wang Date: Thu, 21 Mar 2024 21:14:02 +0800 Subject: [PATCH 0334/1702] clk: imx: imx8mp: Add pm_runtime support for power saving Add pm_runtime support for power saving. In pm runtime suspend state the registers will be reseted, so add registers save in pm runtime suspend and restore them in pm runtime resume. Signed-off-by: Shengjiu Wang Reviewed-by: Peng Fan Reviewed-by: Marc Kleine-Budde Link: https://lore.kernel.org/r/1711026842-7268-1-git-send-email-shengjiu.wang@nxp.com Signed-off-by: Abel Vesa --- drivers/clk/imx/clk-imx8mp-audiomix.c | 157 ++++++++++++++++++++++---- 1 file changed, 136 insertions(+), 21 deletions(-) diff --git a/drivers/clk/imx/clk-imx8mp-audiomix.c b/drivers/clk/imx/clk-imx8mp-audiomix.c index 55ed211a5e0b1..574a032309c1b 100644 --- a/drivers/clk/imx/clk-imx8mp-audiomix.c +++ b/drivers/clk/imx/clk-imx8mp-audiomix.c @@ -7,10 +7,12 @@ #include #include +#include #include #include #include #include +#include #include @@ -18,6 +20,7 @@ #define CLKEN0 0x000 #define CLKEN1 0x004 +#define EARC 0x200 #define SAI1_MCLK_SEL 0x300 #define SAI2_MCLK_SEL 0x304 #define SAI3_MCLK_SEL 0x308 @@ -26,6 +29,11 @@ #define SAI7_MCLK_SEL 0x314 #define PDM_SEL 0x318 #define SAI_PLL_GNRL_CTL 0x400 +#define SAI_PLL_FDIVL_CTL0 0x404 +#define SAI_PLL_FDIVL_CTL1 0x408 +#define SAI_PLL_SSCG_CTL 0x40C +#define SAI_PLL_MNIT_CTL 0x410 +#define IPG_LP_CTRL 0x504 #define SAIn_MCLK1_PARENT(n) \ static const struct clk_parent_data \ @@ -182,26 +190,82 @@ static struct clk_imx8mp_audiomix_sel sels[] = { CLK_SAIn(7) }; +static const u16 audiomix_regs[] = { + CLKEN0, + CLKEN1, + EARC, + SAI1_MCLK_SEL, + SAI2_MCLK_SEL, + SAI3_MCLK_SEL, + SAI5_MCLK_SEL, + SAI6_MCLK_SEL, + SAI7_MCLK_SEL, + PDM_SEL, + SAI_PLL_GNRL_CTL, + SAI_PLL_FDIVL_CTL0, + SAI_PLL_FDIVL_CTL1, + SAI_PLL_SSCG_CTL, + SAI_PLL_MNIT_CTL, + IPG_LP_CTRL, +}; + +struct clk_imx8mp_audiomix_priv { + void __iomem *base; + u32 regs_save[ARRAY_SIZE(audiomix_regs)]; + + /* Must be last */ + struct clk_hw_onecell_data clk_data; +}; + +static void clk_imx8mp_audiomix_save_restore(struct device *dev, bool save) +{ + struct clk_imx8mp_audiomix_priv *priv = dev_get_drvdata(dev); + void __iomem *base = priv->base; + int i; + + if (save) { + for (i = 0; i < ARRAY_SIZE(audiomix_regs); i++) + priv->regs_save[i] = readl(base + audiomix_regs[i]); + } else { + for (i = 0; i < ARRAY_SIZE(audiomix_regs); i++) + writel(priv->regs_save[i], base + audiomix_regs[i]); + } +} + static int clk_imx8mp_audiomix_probe(struct platform_device *pdev) { - struct clk_hw_onecell_data *priv; + struct clk_imx8mp_audiomix_priv *priv; + struct clk_hw_onecell_data *clk_hw_data; struct device *dev = &pdev->dev; void __iomem *base; struct clk_hw *hw; - int i; + int i, ret; priv = devm_kzalloc(dev, - struct_size(priv, hws, IMX8MP_CLK_AUDIOMIX_END), + struct_size(priv, clk_data.hws, IMX8MP_CLK_AUDIOMIX_END), GFP_KERNEL); if (!priv) return -ENOMEM; - priv->num = IMX8MP_CLK_AUDIOMIX_END; + clk_hw_data = &priv->clk_data; + clk_hw_data->num = IMX8MP_CLK_AUDIOMIX_END; base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) return PTR_ERR(base); + priv->base = base; + dev_set_drvdata(dev, priv); + + /* + * pm_runtime_enable needs to be called before clk register. + * That is to make core->rpm_enabled to be true for clock + * usage. + */ + pm_runtime_get_noresume(dev); + pm_runtime_set_active(dev); + pm_runtime_enable(dev); + for (i = 0; i < ARRAY_SIZE(sels); i++) { if (sels[i].num_parents == 1) { hw = devm_clk_hw_register_gate_parent_data(dev, @@ -216,10 +280,12 @@ static int clk_imx8mp_audiomix_probe(struct platform_device *pdev) 0, NULL, NULL); } - if (IS_ERR(hw)) - return PTR_ERR(hw); + if (IS_ERR(hw)) { + ret = PTR_ERR(hw); + goto err_clk_register; + } - priv->hws[sels[i].clkid] = hw; + clk_hw_data->hws[sels[i].clkid] = hw; } /* SAI PLL */ @@ -228,39 +294,86 @@ static int clk_imx8mp_audiomix_probe(struct platform_device *pdev) ARRAY_SIZE(clk_imx8mp_audiomix_pll_parents), CLK_SET_RATE_NO_REPARENT, base + SAI_PLL_GNRL_CTL, 0, 2, 0, NULL, NULL); - priv->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL_REF_SEL] = hw; + clk_hw_data->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL_REF_SEL] = hw; hw = imx_dev_clk_hw_pll14xx(dev, "sai_pll", "sai_pll_ref_sel", base + 0x400, &imx_1443x_pll); - if (IS_ERR(hw)) - return PTR_ERR(hw); - priv->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL] = hw; + if (IS_ERR(hw)) { + ret = PTR_ERR(hw); + goto err_clk_register; + } + clk_hw_data->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL] = hw; hw = devm_clk_hw_register_mux_parent_data_table(dev, "sai_pll_bypass", clk_imx8mp_audiomix_pll_bypass_sels, ARRAY_SIZE(clk_imx8mp_audiomix_pll_bypass_sels), CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT, base + SAI_PLL_GNRL_CTL, 16, 1, 0, NULL, NULL); - if (IS_ERR(hw)) - return PTR_ERR(hw); - priv->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL_BYPASS] = hw; + if (IS_ERR(hw)) { + ret = PTR_ERR(hw); + goto err_clk_register; + } + + clk_hw_data->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL_BYPASS] = hw; hw = devm_clk_hw_register_gate(dev, "sai_pll_out", "sai_pll_bypass", 0, base + SAI_PLL_GNRL_CTL, 13, 0, NULL); - if (IS_ERR(hw)) - return PTR_ERR(hw); - priv->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL_OUT] = hw; + if (IS_ERR(hw)) { + ret = PTR_ERR(hw); + goto err_clk_register; + } + clk_hw_data->hws[IMX8MP_CLK_AUDIOMIX_SAI_PLL_OUT] = hw; hw = devm_clk_hw_register_fixed_factor(dev, "sai_pll_out_div2", "sai_pll_out", 0, 1, 2); - if (IS_ERR(hw)) - return PTR_ERR(hw); + if (IS_ERR(hw)) { + ret = PTR_ERR(hw); + goto err_clk_register; + } + + ret = devm_of_clk_add_hw_provider(&pdev->dev, of_clk_hw_onecell_get, + clk_hw_data); + if (ret) + goto err_clk_register; + + pm_runtime_put_sync(dev); + return 0; + +err_clk_register: + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); + return ret; +} + +static int clk_imx8mp_audiomix_remove(struct platform_device *pdev) +{ + pm_runtime_disable(&pdev->dev); + + return 0; +} + +static int clk_imx8mp_audiomix_runtime_suspend(struct device *dev) +{ + clk_imx8mp_audiomix_save_restore(dev, true); - return devm_of_clk_add_hw_provider(&pdev->dev, of_clk_hw_onecell_get, - priv); + return 0; } +static int clk_imx8mp_audiomix_runtime_resume(struct device *dev) +{ + clk_imx8mp_audiomix_save_restore(dev, false); + + return 0; +} + +static const struct dev_pm_ops clk_imx8mp_audiomix_pm_ops = { + SET_RUNTIME_PM_OPS(clk_imx8mp_audiomix_runtime_suspend, + clk_imx8mp_audiomix_runtime_resume, NULL) + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) +}; + static const struct of_device_id clk_imx8mp_audiomix_of_match[] = { { .compatible = "fsl,imx8mp-audio-blk-ctrl" }, { /* sentinel */ } @@ -269,9 +382,11 @@ MODULE_DEVICE_TABLE(of, clk_imx8mp_audiomix_of_match); static struct platform_driver clk_imx8mp_audiomix_driver = { .probe = clk_imx8mp_audiomix_probe, + .remove = clk_imx8mp_audiomix_remove, .driver = { .name = "imx8mp-audio-blk-ctrl", .of_match_table = clk_imx8mp_audiomix_of_match, + .pm = &clk_imx8mp_audiomix_pm_ops, }, }; From 7b54d9113cd4923432c0b2441c5e2663873b4e5b Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 19 Apr 2024 10:09:14 +0000 Subject: [PATCH 0335/1702] clk: samsung: gs101: propagate PERIC0 USI SPI clock rate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce nMUX() for MUX clocks that can be reparented on clock rate change. "nMUX" comes from "n-to-1 selector", hopefully emphasising that the selector can change on clock rate changes. Ideally MUX/MUX_F() should change to not have the CLK_SET_RATE_NO_REPARENT flag set by default, and all their users to be updated to add the flag back (like in the case of DIV and GATE). But this is a very intrusive change and because for now only GS101 allows MUX reparenting on clock rate change, stick with nMUX(). GS101 defines MUX clocks that are dedicated for each instance of the IP. One example is USI IP (SPI, I2C, serial). The reparenting of these MUX clocks will not affect other instances of the same IP or different IPs altogether. When SPI transfer is being prepared, the spi-s3c64xx driver will call clk_set_rate() to change the rate of SPI source clock (IPCLK). But IPCLK is a gate (leaf) clock, so it must propagate the rate change up the clock tree, so that corresponding MUX/DIV clocks can actually change their values. Add CLK_SET_RATE_PARENT flag to corresponding clocks for all USI instances in GS101 PERIC0: USI{1-8, 14}. This change involves the following clocks: PERIC0 USI*: Clock Div range MUX Selection ------------------------------------------------------------------- gout_peric0_peric0_top0_ipclk_* - - dout_peric0_usi*_usi /1..16 - mout_peric0_usi*_usi_user - {24.5 MHz, 400 MHz} With input clock of 400 MHz this scheme provides the following IPCLK rate range, for each USI block: PERIC0 USI*: 1.5 MHz ... 400 MHz Accounting for internal /4 divider in SPI blocks, and because the max SPI frequency is limited at 50 MHz, it gives us next SPI SCK rates: PERIC0 USI_SPI*: 384 KHz ... 49.9 MHz Fixes: 893f133a040b ("clk: samsung: gs101: add support for cmu_peric0") Reviewed-by: Peter Griffin Acked-by: André Draszik Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240419100915.2168573-2-tudor.ambarus@linaro.org Signed-off-by: Krzysztof Kozlowski --- drivers/clk/samsung/clk-gs101.c | 135 +++++++++++++++++--------------- drivers/clk/samsung/clk.h | 11 ++- 2 files changed, 81 insertions(+), 65 deletions(-) diff --git a/drivers/clk/samsung/clk-gs101.c b/drivers/clk/samsung/clk-gs101.c index d065e343a85dd..76d4d0fa1168a 100644 --- a/drivers/clk/samsung/clk-gs101.c +++ b/drivers/clk/samsung/clk-gs101.c @@ -2763,33 +2763,33 @@ static const struct samsung_mux_clock peric0_mux_clks[] __initconst = { MUX(CLK_MOUT_PERIC0_USI0_UART_USER, "mout_peric0_usi0_uart_user", mout_peric0_usi0_uart_user_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI0_UART_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI14_USI_USER, - "mout_peric0_usi14_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI14_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI1_USI_USER, - "mout_peric0_usi1_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI1_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI2_USI_USER, - "mout_peric0_usi2_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI2_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI3_USI_USER, - "mout_peric0_usi3_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI3_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI4_USI_USER, - "mout_peric0_usi4_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI4_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI5_USI_USER, - "mout_peric0_usi5_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI5_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI6_USI_USER, - "mout_peric0_usi6_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI6_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI7_USI_USER, - "mout_peric0_usi7_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI7_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_USI8_USI_USER, - "mout_peric0_usi8_usi_user", mout_peric0_usi_usi_user_p, - PLL_CON0_MUX_CLKCMU_PERIC0_USI8_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI14_USI_USER, + "mout_peric0_usi14_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI14_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI1_USI_USER, + "mout_peric0_usi1_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI1_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI2_USI_USER, + "mout_peric0_usi2_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI2_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI3_USI_USER, + "mout_peric0_usi3_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI3_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI4_USI_USER, + "mout_peric0_usi4_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI4_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI5_USI_USER, + "mout_peric0_usi5_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI5_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI6_USI_USER, + "mout_peric0_usi6_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI6_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI7_USI_USER, + "mout_peric0_usi7_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI7_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC0_USI8_USI_USER, + "mout_peric0_usi8_usi_user", mout_peric0_usi_usi_user_p, + PLL_CON0_MUX_CLKCMU_PERIC0_USI8_USI_USER, 4, 1), }; static const struct samsung_div_clock peric0_div_clks[] __initconst = { @@ -2798,33 +2798,42 @@ static const struct samsung_div_clock peric0_div_clks[] __initconst = { DIV(CLK_DOUT_PERIC0_USI0_UART, "dout_peric0_usi0_uart", "mout_peric0_usi0_uart_user", CLK_CON_DIV_DIV_CLK_PERIC0_USI0_UART, 0, 4), - DIV(CLK_DOUT_PERIC0_USI14_USI, - "dout_peric0_usi14_usi", "mout_peric0_usi14_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI14_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI1_USI, - "dout_peric0_usi1_usi", "mout_peric0_usi1_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI1_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI2_USI, - "dout_peric0_usi2_usi", "mout_peric0_usi2_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI2_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI3_USI, - "dout_peric0_usi3_usi", "mout_peric0_usi3_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI3_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI4_USI, - "dout_peric0_usi4_usi", "mout_peric0_usi4_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI4_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI5_USI, - "dout_peric0_usi5_usi", "mout_peric0_usi5_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI5_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI6_USI, - "dout_peric0_usi6_usi", "mout_peric0_usi6_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI6_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI7_USI, - "dout_peric0_usi7_usi", "mout_peric0_usi7_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI7_USI, 0, 4), - DIV(CLK_DOUT_PERIC0_USI8_USI, - "dout_peric0_usi8_usi", "mout_peric0_usi8_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI8_USI, 0, 4), + DIV_F(CLK_DOUT_PERIC0_USI14_USI, + "dout_peric0_usi14_usi", "mout_peric0_usi14_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI14_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI1_USI, + "dout_peric0_usi1_usi", "mout_peric0_usi1_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI1_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI2_USI, + "dout_peric0_usi2_usi", "mout_peric0_usi2_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI2_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI3_USI, + "dout_peric0_usi3_usi", "mout_peric0_usi3_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI3_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI4_USI, + "dout_peric0_usi4_usi", "mout_peric0_usi4_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI4_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI5_USI, + "dout_peric0_usi5_usi", "mout_peric0_usi5_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI5_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI6_USI, + "dout_peric0_usi6_usi", "mout_peric0_usi6_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI6_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI7_USI, + "dout_peric0_usi7_usi", "mout_peric0_usi7_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI7_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI8_USI, + "dout_peric0_usi8_usi", "mout_peric0_usi8_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI8_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), }; static const struct samsung_gate_clock peric0_gate_clks[] __initconst = { @@ -2857,11 +2866,11 @@ static const struct samsung_gate_clock peric0_gate_clks[] __initconst = { GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_0, "gout_peric0_peric0_top0_ipclk_0", "dout_peric0_usi1_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_0, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_1, "gout_peric0_peric0_top0_ipclk_1", "dout_peric0_usi2_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_1, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_10, "gout_peric0_peric0_top0_ipclk_10", "dout_peric0_i3c", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_10, @@ -2889,27 +2898,27 @@ static const struct samsung_gate_clock peric0_gate_clks[] __initconst = { GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_2, "gout_peric0_peric0_top0_ipclk_2", "dout_peric0_usi3_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_2, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_3, "gout_peric0_peric0_top0_ipclk_3", "dout_peric0_usi4_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_3, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_4, "gout_peric0_peric0_top0_ipclk_4", "dout_peric0_usi5_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_4, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_5, "gout_peric0_peric0_top0_ipclk_5", "dout_peric0_usi6_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_5, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_6, "gout_peric0_peric0_top0_ipclk_6", "dout_peric0_usi7_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_6, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_7, "gout_peric0_peric0_top0_ipclk_7", "dout_peric0_usi8_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_7, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC0_PERIC0_TOP0_IPCLK_8, "gout_peric0_peric0_top0_ipclk_8", "dout_peric0_i3c", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_IPCLK_8, @@ -2990,7 +2999,7 @@ static const struct samsung_gate_clock peric0_gate_clks[] __initconst = { GATE(CLK_GOUT_PERIC0_PERIC0_TOP1_IPCLK_2, "gout_peric0_peric0_top1_ipclk_2", "dout_peric0_usi14_usi", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP1_IPCLKPORT_IPCLK_2, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), /* Disabling this clock makes the system hang. Mark the clock as critical. */ GATE(CLK_GOUT_PERIC0_PERIC0_TOP1_PCLK_0, "gout_peric0_peric0_top1_pclk_0", "mout_peric0_bus_user", diff --git a/drivers/clk/samsung/clk.h b/drivers/clk/samsung/clk.h index a70bd7cce39fd..fb06caa71f0ae 100644 --- a/drivers/clk/samsung/clk.h +++ b/drivers/clk/samsung/clk.h @@ -133,7 +133,7 @@ struct samsung_mux_clock { .name = cname, \ .parent_names = pnames, \ .num_parents = ARRAY_SIZE(pnames), \ - .flags = (f) | CLK_SET_RATE_NO_REPARENT, \ + .flags = f, \ .offset = o, \ .shift = s, \ .width = w, \ @@ -141,9 +141,16 @@ struct samsung_mux_clock { } #define MUX(_id, cname, pnames, o, s, w) \ - __MUX(_id, cname, pnames, o, s, w, 0, 0) + __MUX(_id, cname, pnames, o, s, w, CLK_SET_RATE_NO_REPARENT, 0) #define MUX_F(_id, cname, pnames, o, s, w, f, mf) \ + __MUX(_id, cname, pnames, o, s, w, (f) | CLK_SET_RATE_NO_REPARENT, mf) + +/* Used by MUX clocks where reparenting on clock rate change is allowed. */ +#define nMUX(_id, cname, pnames, o, s, w) \ + __MUX(_id, cname, pnames, o, s, w, 0, 0) + +#define nMUX_F(_id, cname, pnames, o, s, w, f, mf) \ __MUX(_id, cname, pnames, o, s, w, f, mf) /** From 7cf0324ba0bc61a8c360d23d284e06d2994b1fef Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 19 Apr 2024 10:09:15 +0000 Subject: [PATCH 0336/1702] clk: samsung: gs101: propagate PERIC1 USI SPI clock rate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When SPI transfer is being prepared, the spi-s3c64xx driver will call clk_set_rate() to change the rate of SPI source clock (IPCLK). But IPCLK is a gate (leaf) clock, so it must propagate the rate change up the clock tree, so that corresponding MUX/DIV clocks can actually change their values. Add CLK_SET_RATE_PARENT flag to corresponding clocks for all USI instances in GS101 PERIC1: USI{0, 9, 10, 11, 12, 13}. This change involves the following clocks: PERIC1 USI*: Clock Div range MUX Selection ------------------------------------------------------------------- gout_peric1_peric1_top0_ipclk_* - - dout_peric1_usi*_usi /1..16 - mout_peric1_usi*_usi_user - {24.5 MHz, 400 MHz} With input clock of 400 MHz this scheme provides the following IPCLK rate range, for each USI block: PERIC1 USI*: 1.5 MHz ... 400 MHz Accounting for internal /4 divider in SPI blocks, and because the max SPI frequency is limited at 50 MHz, it gives us next SPI SCK rates: PERIC1 USI_SPI*: 384 KHz ... 49.9 MHz Which shall be fine for the applications of the SPI bus. Note that with this we allow the reparenting of the MUX_USIx clocks to OSCCLK. Each instance of the USI IP has its own MUX_USI clock, thus the reparenting of a MUX_USI clock corresponds to a single instance of the USI IP. The datasheet mentions OSCCLK just in the low-power mode context, but the downstream driver reparents too the MUX_USI clocks to OSCCLK. Follow the downstream driver and do the same. Fixes: 2999e786d7e9 ("clk: samsung: gs101: add support for cmu_peric1") Reviewed-by: Peter Griffin Acked-by: André Draszik Signed-off-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240419100915.2168573-3-tudor.ambarus@linaro.org Signed-off-by: Krzysztof Kozlowski --- drivers/clk/samsung/clk-gs101.c | 90 ++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/drivers/clk/samsung/clk-gs101.c b/drivers/clk/samsung/clk-gs101.c index 76d4d0fa1168a..bd3c1b02715b5 100644 --- a/drivers/clk/samsung/clk-gs101.c +++ b/drivers/clk/samsung/clk-gs101.c @@ -3239,47 +3239,53 @@ static const struct samsung_mux_clock peric1_mux_clks[] __initconst = { MUX(CLK_MOUT_PERIC1_I3C_USER, "mout_peric1_i3c_user", mout_peric1_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC1_I3C_USER, 4, 1), - MUX(CLK_MOUT_PERIC1_USI0_USI_USER, - "mout_peric1_usi0_usi_user", mout_peric1_nonbususer_p, - PLL_CON0_MUX_CLKCMU_PERIC1_USI0_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC1_USI10_USI_USER, - "mout_peric1_usi10_usi_user", mout_peric1_nonbususer_p, - PLL_CON0_MUX_CLKCMU_PERIC1_USI10_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC1_USI11_USI_USER, - "mout_peric1_usi11_usi_user", mout_peric1_nonbususer_p, - PLL_CON0_MUX_CLKCMU_PERIC1_USI11_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC1_USI12_USI_USER, - "mout_peric1_usi12_usi_user", mout_peric1_nonbususer_p, - PLL_CON0_MUX_CLKCMU_PERIC1_USI12_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC1_USI13_USI_USER, - "mout_peric1_usi13_usi_user", mout_peric1_nonbususer_p, - PLL_CON0_MUX_CLKCMU_PERIC1_USI13_USI_USER, 4, 1), - MUX(CLK_MOUT_PERIC1_USI9_USI_USER, - "mout_peric1_usi9_usi_user", mout_peric1_nonbususer_p, - PLL_CON0_MUX_CLKCMU_PERIC1_USI9_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC1_USI0_USI_USER, + "mout_peric1_usi0_usi_user", mout_peric1_nonbususer_p, + PLL_CON0_MUX_CLKCMU_PERIC1_USI0_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC1_USI10_USI_USER, + "mout_peric1_usi10_usi_user", mout_peric1_nonbususer_p, + PLL_CON0_MUX_CLKCMU_PERIC1_USI10_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC1_USI11_USI_USER, + "mout_peric1_usi11_usi_user", mout_peric1_nonbususer_p, + PLL_CON0_MUX_CLKCMU_PERIC1_USI11_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC1_USI12_USI_USER, + "mout_peric1_usi12_usi_user", mout_peric1_nonbususer_p, + PLL_CON0_MUX_CLKCMU_PERIC1_USI12_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC1_USI13_USI_USER, + "mout_peric1_usi13_usi_user", mout_peric1_nonbususer_p, + PLL_CON0_MUX_CLKCMU_PERIC1_USI13_USI_USER, 4, 1), + nMUX(CLK_MOUT_PERIC1_USI9_USI_USER, + "mout_peric1_usi9_usi_user", mout_peric1_nonbususer_p, + PLL_CON0_MUX_CLKCMU_PERIC1_USI9_USI_USER, 4, 1), }; static const struct samsung_div_clock peric1_div_clks[] __initconst = { DIV(CLK_DOUT_PERIC1_I3C, "dout_peric1_i3c", "mout_peric1_i3c_user", CLK_CON_DIV_DIV_CLK_PERIC1_I3C, 0, 4), - DIV(CLK_DOUT_PERIC1_USI0_USI, - "dout_peric1_usi0_usi", "mout_peric1_usi0_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI0_USI, 0, 4), - DIV(CLK_DOUT_PERIC1_USI10_USI, - "dout_peric1_usi10_usi", "mout_peric1_usi10_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI10_USI, 0, 4), - DIV(CLK_DOUT_PERIC1_USI11_USI, - "dout_peric1_usi11_usi", "mout_peric1_usi11_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI11_USI, 0, 4), - DIV(CLK_DOUT_PERIC1_USI12_USI, - "dout_peric1_usi12_usi", "mout_peric1_usi12_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI12_USI, 0, 4), - DIV(CLK_DOUT_PERIC1_USI13_USI, - "dout_peric1_usi13_usi", "mout_peric1_usi13_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI13_USI, 0, 4), - DIV(CLK_DOUT_PERIC1_USI9_USI, - "dout_peric1_usi9_usi", "mout_peric1_usi9_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI9_USI, 0, 4), + DIV_F(CLK_DOUT_PERIC1_USI0_USI, + "dout_peric1_usi0_usi", "mout_peric1_usi0_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI0_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI10_USI, + "dout_peric1_usi10_usi", "mout_peric1_usi10_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI10_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI11_USI, + "dout_peric1_usi11_usi", "mout_peric1_usi11_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI11_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI12_USI, + "dout_peric1_usi12_usi", "mout_peric1_usi12_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI12_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI13_USI, + "dout_peric1_usi13_usi", "mout_peric1_usi13_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI13_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI9_USI, + "dout_peric1_usi9_usi", "mout_peric1_usi9_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI9_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), }; static const struct samsung_gate_clock peric1_gate_clks[] __initconst = { @@ -3314,27 +3320,27 @@ static const struct samsung_gate_clock peric1_gate_clks[] __initconst = { GATE(CLK_GOUT_PERIC1_PERIC1_TOP0_IPCLK_1, "gout_peric1_peric1_top0_ipclk_1", "dout_peric1_usi0_usi", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_IPCLK_1, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC1_PERIC1_TOP0_IPCLK_2, "gout_peric1_peric1_top0_ipclk_2", "dout_peric1_usi9_usi", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_IPCLK_2, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC1_PERIC1_TOP0_IPCLK_3, "gout_peric1_peric1_top0_ipclk_3", "dout_peric1_usi10_usi", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_IPCLK_3, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC1_PERIC1_TOP0_IPCLK_4, "gout_peric1_peric1_top0_ipclk_4", "dout_peric1_usi11_usi", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_IPCLK_4, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC1_PERIC1_TOP0_IPCLK_5, "gout_peric1_peric1_top0_ipclk_5", "dout_peric1_usi12_usi", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_IPCLK_5, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC1_PERIC1_TOP0_IPCLK_6, "gout_peric1_peric1_top0_ipclk_6", "dout_peric1_usi13_usi", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_IPCLK_6, - 21, 0, 0), + 21, CLK_SET_RATE_PARENT, 0), GATE(CLK_GOUT_PERIC1_PERIC1_TOP0_IPCLK_8, "gout_peric1_peric1_top0_ipclk_8", "dout_peric1_i3c", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_IPCLK_8, From 27a7a9d903a0473cb763afa662c38debf6f65bfa Mon Sep 17 00:00:00 2001 From: Dave Chinner Date: Wed, 3 Apr 2024 08:28:30 +1100 Subject: [PATCH 0337/1702] xfs: silence sparse warning when checking version number Scrub checks the superblock version number against the known good feature bits that can be set in the version mask. It calculates the version mask to compare like so: vernum_mask = cpu_to_be16(~XFS_SB_VERSION_OKBITS | XFS_SB_VERSION_NUMBITS | XFS_SB_VERSION_ALIGNBIT | XFS_SB_VERSION_DALIGNBIT | XFS_SB_VERSION_SHAREDBIT | XFS_SB_VERSION_LOGV2BIT | XFS_SB_VERSION_SECTORBIT | XFS_SB_VERSION_EXTFLGBIT | XFS_SB_VERSION_DIRV2BIT); This generates a sparse warning: fs/xfs/scrub/agheader.c:168:23: warning: cast truncates bits from constant value (ffff3f8f becomes 3f8f) This is because '~XFS_SB_VERSION_OKBITS' is considered a 32 bit constant, even though it's value is always under 16 bits. This is a kinda silly thing to do, because: /* * Supported feature bit list is just all bits in the versionnum field because * we've used them all up and understand them all. Except, of course, for the * shared superblock bit, which nobody knows what it does and so is unsupported. */ #define XFS_SB_VERSION_OKBITS \ ((XFS_SB_VERSION_NUMBITS | XFS_SB_VERSION_ALLFBITS) & \ ~XFS_SB_VERSION_SHAREDBIT) #define XFS_SB_VERSION_NUMBITS 0x000f #define XFS_SB_VERSION_ALLFBITS 0xfff0 #define XFS_SB_VERSION_SHAREDBIT 0x0200 XFS_SB_VERSION_OKBITS has a value of 0xfdff, and so ~XFS_SB_VERSION_OKBITS == XFS_SB_VERSION_SHAREDBIT. The calculated mask already sets XFS_SB_VERSION_SHAREDBIT, so starting with ~XFS_SB_VERSION_OKBITS is completely redundant.... Signed-off-by: Dave Chinner Reviewed-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/scrub/agheader.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/xfs/scrub/agheader.c b/fs/xfs/scrub/agheader.c index 1528f14bd9251..f8e5b67128d25 100644 --- a/fs/xfs/scrub/agheader.c +++ b/fs/xfs/scrub/agheader.c @@ -166,8 +166,7 @@ xchk_superblock( xchk_block_set_corrupt(sc, bp); /* Check sb_versionnum bits that are set at mkfs time. */ - vernum_mask = cpu_to_be16(~XFS_SB_VERSION_OKBITS | - XFS_SB_VERSION_NUMBITS | + vernum_mask = cpu_to_be16(XFS_SB_VERSION_NUMBITS | XFS_SB_VERSION_ALIGNBIT | XFS_SB_VERSION_DALIGNBIT | XFS_SB_VERSION_SHAREDBIT | From 76f011f7e659305daf8beba0986fd4a03b3e56f6 Mon Sep 17 00:00:00 2001 From: Dave Chinner Date: Wed, 3 Apr 2024 08:28:32 +1100 Subject: [PATCH 0338/1702] xfs: fix sparse warnings about unused interval tree functions Sparse throws warnings about the interval tree functions that are defined and then not used in the scrub bitmap code: fs/xfs/scrub/bitmap.c:57:1: warning: unused function 'xbitmap64_tree_iter_next' [-Wunused-function] INTERVAL_TREE_DEFINE(struct xbitmap64_node, bn_rbnode, uint64_t, ^ ./include/linux/interval_tree_generic.h:151:33: note: expanded from macro 'INTERVAL_TREE_DEFINE' ITSTATIC ITSTRUCT * \ ^ :3:1: note: expanded from here xbitmap64_tree_iter_next ^ fs/xfs/scrub/bitmap.c:331:1: warning: unused function 'xbitmap32_tree_iter_next' [-Wunused-function] INTERVAL_TREE_DEFINE(struct xbitmap32_node, bn_rbnode, uint32_t, ^ ./include/linux/interval_tree_generic.h:151:33: note: expanded from macro 'INTERVAL_TREE_DEFINE' ITSTATIC ITSTRUCT * \ ^ :59:1: note: expanded from here xbitmap32_tree_iter_next Fix these by marking the functions created by the interval tree creation macro as __maybe_unused to suppress this warning. Signed-off-by: Dave Chinner Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/scrub/bitmap.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/fs/xfs/scrub/bitmap.c b/fs/xfs/scrub/bitmap.c index 0cb8d43912a84..7ba35a7a7920e 100644 --- a/fs/xfs/scrub/bitmap.c +++ b/fs/xfs/scrub/bitmap.c @@ -40,22 +40,23 @@ struct xbitmap64_node { * These functions are defined by the INTERVAL_TREE_DEFINE macro, but we'll * forward-declare them anyway for clarity. */ -static inline void +static inline __maybe_unused void xbitmap64_tree_insert(struct xbitmap64_node *node, struct rb_root_cached *root); -static inline void +static inline __maybe_unused void xbitmap64_tree_remove(struct xbitmap64_node *node, struct rb_root_cached *root); -static inline struct xbitmap64_node * +static inline __maybe_unused struct xbitmap64_node * xbitmap64_tree_iter_first(struct rb_root_cached *root, uint64_t start, uint64_t last); -static inline struct xbitmap64_node * +static inline __maybe_unused struct xbitmap64_node * xbitmap64_tree_iter_next(struct xbitmap64_node *node, uint64_t start, uint64_t last); INTERVAL_TREE_DEFINE(struct xbitmap64_node, bn_rbnode, uint64_t, - __bn_subtree_last, START, LAST, static inline, xbitmap64_tree) + __bn_subtree_last, START, LAST, static inline __maybe_unused, + xbitmap64_tree) /* Iterate each interval of a bitmap. Do not change the bitmap. */ #define for_each_xbitmap64_extent(bn, bitmap) \ @@ -314,22 +315,23 @@ struct xbitmap32_node { * These functions are defined by the INTERVAL_TREE_DEFINE macro, but we'll * forward-declare them anyway for clarity. */ -static inline void +static inline __maybe_unused void xbitmap32_tree_insert(struct xbitmap32_node *node, struct rb_root_cached *root); -static inline void +static inline __maybe_unused void xbitmap32_tree_remove(struct xbitmap32_node *node, struct rb_root_cached *root); -static inline struct xbitmap32_node * +static inline __maybe_unused struct xbitmap32_node * xbitmap32_tree_iter_first(struct rb_root_cached *root, uint32_t start, uint32_t last); -static inline struct xbitmap32_node * +static inline __maybe_unused struct xbitmap32_node * xbitmap32_tree_iter_next(struct xbitmap32_node *node, uint32_t start, uint32_t last); INTERVAL_TREE_DEFINE(struct xbitmap32_node, bn_rbnode, uint32_t, - __bn_subtree_last, START, LAST, static inline, xbitmap32_tree) + __bn_subtree_last, START, LAST, static inline __maybe_unused, + xbitmap32_tree) /* Iterate each interval of a bitmap. Do not change the bitmap. */ #define for_each_xbitmap32_extent(bn, bitmap) \ From 7d7c82a04d3d93c8b6f8ed71aa5bc39cdc96377c Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Sat, 13 Apr 2024 00:59:27 +0200 Subject: [PATCH 0339/1702] xfs: Fix typo in comment s/somethign/something/ Signed-off-by: Thorsten Blum Reviewed-by: "Darrick J. Wong" Reviewed-by: Christoph Hellwig Signed-off-by: Chandan Babu R --- fs/xfs/xfs_log_priv.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/xfs/xfs_log_priv.h b/fs/xfs/xfs_log_priv.h index a78c4bfaad447..40e22ec0fbe69 100644 --- a/fs/xfs/xfs_log_priv.h +++ b/fs/xfs/xfs_log_priv.h @@ -681,7 +681,7 @@ xlog_valid_lsn( * flags to control the kmalloc() behaviour within kvmalloc(). Hence kmalloc() * will do direct reclaim and compaction in the slow path, both of which are * horrendously expensive. We just want kmalloc to fail fast and fall back to - * vmalloc if it can't get somethign straight away from the free lists or + * vmalloc if it can't get something straight away from the free lists or * buddy allocator. Hence we have to open code kvmalloc outselves here. * * This assumes that the caller uses memalloc_nofs_save task context here, so From d983ff63af6068a6773283660222fff10f66e184 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Thu, 7 Mar 2024 11:39:18 +0300 Subject: [PATCH 0340/1702] xfs: small cleanup in xrep_update_qflags() The "mp" pointer is the same as "sc->mp" so this change doesn't affect runtime at all. However, it's nicer to use same name for both the lock and the unlock. Signed-off-by: Dan Carpenter Reviewed-by: Christoph Hellwig Signed-off-by: Chandan Babu R --- fs/xfs/scrub/repair.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c index b6aff89679d58..92f81e374bc08 100644 --- a/fs/xfs/scrub/repair.c +++ b/fs/xfs/scrub/repair.c @@ -729,7 +729,7 @@ xrep_update_qflags( xfs_trans_log_buf(sc->tp, bp, 0, sizeof(struct xfs_dsb) - 1); no_update: - mutex_unlock(&sc->mp->m_quotainfo->qi_quotaofflock); + mutex_unlock(&mp->m_quotainfo->qi_quotaofflock); } /* Force a quotacheck the next time we mount. */ From 05150d46a33fdb567d9849b831aa139e1693e39c Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Mon, 4 Mar 2024 10:24:08 +0800 Subject: [PATCH 0341/1702] xfs: Remove unused function is_rt_data_fork The function are defined in the rmap_repair.c file, but not called elsewhere, so delete the unused function. fs/xfs/scrub/rmap_repair.c:436:1: warning: unused function 'is_rt_data_fork'. Reported-by: Abaci Robot Closes: https://bugzilla.openanolis.cn/show_bug.cgi?id=8425 Signed-off-by: Jiapeng Chong Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/scrub/rmap_repair.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/fs/xfs/scrub/rmap_repair.c b/fs/xfs/scrub/rmap_repair.c index 25acd69614c2c..e8080eba37d29 100644 --- a/fs/xfs/scrub/rmap_repair.c +++ b/fs/xfs/scrub/rmap_repair.c @@ -432,14 +432,6 @@ xrep_rmap_scan_iroot_btree( return error; } -static inline bool -is_rt_data_fork( - struct xfs_inode *ip, - int whichfork) -{ - return XFS_IS_REALTIME_INODE(ip) && whichfork == XFS_DATA_FORK; -} - /* * Iterate the block mapping btree to collect rmap records for anything in this * fork that matches the AG. Sets @mappings_done to true if we've scanned the From e78a3ce28331583cc0b57b0602528c4d7628fc19 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 5 Apr 2024 08:07:08 +0200 Subject: [PATCH 0342/1702] xfs: move more logic into xfs_extent_busy_clear_one Move the handling of discarded entries into xfs_extent_busy_clear_one to reuse the length check and tidy up the logic in the caller. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/xfs_extent_busy.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/fs/xfs/xfs_extent_busy.c b/fs/xfs/xfs_extent_busy.c index 56cfa1498571e..6fbffa46e5e94 100644 --- a/fs/xfs/xfs_extent_busy.c +++ b/fs/xfs/xfs_extent_busy.c @@ -518,20 +518,26 @@ xfs_extent_busy_trim( goto out; } -STATIC void +static bool xfs_extent_busy_clear_one( - struct xfs_mount *mp, struct xfs_perag *pag, - struct xfs_extent_busy *busyp) + struct xfs_extent_busy *busyp, + bool do_discard) { if (busyp->length) { - trace_xfs_extent_busy_clear(mp, busyp->agno, busyp->bno, - busyp->length); + if (do_discard && + !(busyp->flags & XFS_EXTENT_BUSY_SKIP_DISCARD)) { + busyp->flags = XFS_EXTENT_BUSY_DISCARDED; + return false; + } + trace_xfs_extent_busy_clear(pag->pag_mount, busyp->agno, + busyp->bno, busyp->length); rb_erase(&busyp->rb_node, &pag->pagb_tree); } list_del_init(&busyp->list); kfree(busyp); + return true; } static void @@ -575,13 +581,8 @@ xfs_extent_busy_clear( wakeup = false; } - if (do_discard && busyp->length && - !(busyp->flags & XFS_EXTENT_BUSY_SKIP_DISCARD)) { - busyp->flags = XFS_EXTENT_BUSY_DISCARDED; - } else { - xfs_extent_busy_clear_one(mp, pag, busyp); + if (xfs_extent_busy_clear_one(pag, busyp, do_discard)) wakeup = true; - } } if (pag) From c37d6ed87462751bc979f133a570716089fe40de Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 5 Apr 2024 08:07:09 +0200 Subject: [PATCH 0343/1702] xfs: unwind xfs_extent_busy_clear The current structure of xfs_extent_busy_clear that locks the first busy extent in each AG and unlocks when switching to a new AG makes sparse unhappy as the lock critical section tracking can't cope with taking the lock conditionally and inside a loop. Rewrite xfs_extent_busy_clear so that it has an outer loop only advancing when moving to a new AG, and an inner loop that consumes busy extents for the given AG to make life easier for sparse and to also make this logic more obvious to humans. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/xfs_extent_busy.c | 59 +++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/fs/xfs/xfs_extent_busy.c b/fs/xfs/xfs_extent_busy.c index 6fbffa46e5e94..a73e7c73b664c 100644 --- a/fs/xfs/xfs_extent_busy.c +++ b/fs/xfs/xfs_extent_busy.c @@ -540,21 +540,6 @@ xfs_extent_busy_clear_one( return true; } -static void -xfs_extent_busy_put_pag( - struct xfs_perag *pag, - bool wakeup) - __releases(pag->pagb_lock) -{ - if (wakeup) { - pag->pagb_gen++; - wake_up_all(&pag->pagb_wait); - } - - spin_unlock(&pag->pagb_lock); - xfs_perag_put(pag); -} - /* * Remove all extents on the passed in list from the busy extents tree. * If do_discard is set skip extents that need to be discarded, and mark @@ -566,27 +551,33 @@ xfs_extent_busy_clear( struct list_head *list, bool do_discard) { - struct xfs_extent_busy *busyp, *n; - struct xfs_perag *pag = NULL; - xfs_agnumber_t agno = NULLAGNUMBER; - bool wakeup = false; - - list_for_each_entry_safe(busyp, n, list, list) { - if (busyp->agno != agno) { - if (pag) - xfs_extent_busy_put_pag(pag, wakeup); - agno = busyp->agno; - pag = xfs_perag_get(mp, agno); - spin_lock(&pag->pagb_lock); - wakeup = false; - } + struct xfs_extent_busy *busyp, *next; - if (xfs_extent_busy_clear_one(pag, busyp, do_discard)) - wakeup = true; - } + busyp = list_first_entry_or_null(list, typeof(*busyp), list); + if (!busyp) + return; + + do { + bool wakeup = false; + struct xfs_perag *pag; - if (pag) - xfs_extent_busy_put_pag(pag, wakeup); + pag = xfs_perag_get(mp, busyp->agno); + spin_lock(&pag->pagb_lock); + do { + next = list_next_entry(busyp, list); + if (xfs_extent_busy_clear_one(pag, busyp, do_discard)) + wakeup = true; + busyp = next; + } while (!list_entry_is_head(busyp, list, list) && + busyp->agno == pag->pag_agno); + + if (wakeup) { + pag->pagb_gen++; + wake_up_all(&pag->pagb_wait); + } + spin_unlock(&pag->pagb_lock); + xfs_perag_put(pag); + } while (!list_entry_is_head(busyp, list, list)); } /* From c0ac6cb251bdc3bcc04b2070296788e0b93652d7 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 5 Apr 2024 08:07:10 +0200 Subject: [PATCH 0344/1702] xfs: remove the unused xfs_extent_busy_enomem trace event Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/xfs_trace.h | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index a80c3063a13f1..0f844396754d7 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -1659,7 +1659,6 @@ DEFINE_EVENT(xfs_extent_busy_class, name, \ xfs_agblock_t agbno, xfs_extlen_t len), \ TP_ARGS(mp, agno, agbno, len)) DEFINE_BUSY_EVENT(xfs_extent_busy); -DEFINE_BUSY_EVENT(xfs_extent_busy_enomem); DEFINE_BUSY_EVENT(xfs_extent_busy_force); DEFINE_BUSY_EVENT(xfs_extent_busy_reuse); DEFINE_BUSY_EVENT(xfs_extent_busy_clear); From 4887e53163825189c70a0db4f13b42eae8798625 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Apr 2024 08:13:16 +0200 Subject: [PATCH 0345/1702] xfs: compile out v4 support if disabled Add a few strategic IS_ENABLED statements to let the compiler eliminate unused code when CONFIG_XFS_SUPPORT_V4 is disabled. This saves multiple kilobytes of .text in my .config: $ size xfs.o.* text data bss dec hex filename 1363633 294836 592 1659061 1950b5 xfs.o.new 1371453 294868 592 1666913 196f61 xfs.o.old Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/xfs_mount.h | 40 +++++++++++++++++++++++++++++++--------- fs/xfs/xfs_super.c | 22 +++++++++++++--------- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index b022e5120dc42..283a5a26db317 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -332,19 +332,10 @@ static inline void xfs_add_ ## name (struct xfs_mount *mp) \ __XFS_ADD_FEAT(attr, ATTR) __XFS_HAS_FEAT(nlink, NLINK) __XFS_ADD_FEAT(quota, QUOTA) -__XFS_HAS_FEAT(align, ALIGN) __XFS_HAS_FEAT(dalign, DALIGN) -__XFS_HAS_FEAT(logv2, LOGV2) __XFS_HAS_FEAT(sector, SECTOR) -__XFS_HAS_FEAT(extflg, EXTFLG) __XFS_HAS_FEAT(asciici, ASCIICI) -__XFS_HAS_FEAT(lazysbcount, LAZYSBCOUNT) -__XFS_ADD_FEAT(attr2, ATTR2) __XFS_HAS_FEAT(parent, PARENT) -__XFS_ADD_FEAT(projid32, PROJID32) -__XFS_HAS_FEAT(crc, CRC) -__XFS_HAS_FEAT(v3inodes, V3INODES) -__XFS_HAS_FEAT(pquotino, PQUOTINO) __XFS_HAS_FEAT(ftype, FTYPE) __XFS_HAS_FEAT(finobt, FINOBT) __XFS_HAS_FEAT(rmapbt, RMAPBT) @@ -358,6 +349,37 @@ __XFS_HAS_FEAT(needsrepair, NEEDSREPAIR) __XFS_HAS_FEAT(large_extent_counts, NREXT64) __XFS_HAS_FEAT(exchange_range, EXCHANGE_RANGE) +/* + * Some features are always on for v5 file systems, allow the compiler to + * eliminiate dead code when building without v4 support. + */ +#define __XFS_HAS_V4_FEAT(name, NAME) \ +static inline bool xfs_has_ ## name (struct xfs_mount *mp) \ +{ \ + return !IS_ENABLED(CONFIG_XFS_SUPPORT_V4) || \ + (mp->m_features & XFS_FEAT_ ## NAME); \ +} + +#define __XFS_ADD_V4_FEAT(name, NAME) \ + __XFS_HAS_V4_FEAT(name, NAME); \ +static inline void xfs_add_ ## name (struct xfs_mount *mp) \ +{ \ + if (IS_ENABLED(CONFIG_XFS_SUPPORT_V4)) { \ + mp->m_features |= XFS_FEAT_ ## NAME; \ + xfs_sb_version_add ## name(&mp->m_sb); \ + } \ +} + +__XFS_HAS_V4_FEAT(align, ALIGN) +__XFS_HAS_V4_FEAT(logv2, LOGV2) +__XFS_HAS_V4_FEAT(extflg, EXTFLG) +__XFS_HAS_V4_FEAT(lazysbcount, LAZYSBCOUNT) +__XFS_ADD_V4_FEAT(attr2, ATTR2) +__XFS_ADD_V4_FEAT(projid32, PROJID32) +__XFS_HAS_V4_FEAT(v3inodes, V3INODES) +__XFS_HAS_V4_FEAT(crc, CRC) +__XFS_HAS_V4_FEAT(pquotino, PQUOTINO) + /* * Mount features * diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 5c9ba974252d1..02c1a4aeaa4c3 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -1580,17 +1580,21 @@ xfs_fs_fill_super( if (error) goto out_free_sb; - /* V4 support is undergoing deprecation. */ - if (!xfs_has_crc(mp)) { -#ifdef CONFIG_XFS_SUPPORT_V4 + /* + * V4 support is undergoing deprecation. + * + * Note: this has to use an open coded m_features check as xfs_has_crc + * always returns false for !CONFIG_XFS_SUPPORT_V4. + */ + if (!(mp->m_features & XFS_FEAT_CRC)) { + if (!IS_ENABLED(CONFIG_XFS_SUPPORT_V4)) { + xfs_warn(mp, + "Deprecated V4 format (crc=0) not supported by kernel."); + error = -EINVAL; + goto out_free_sb; + } xfs_warn_once(mp, "Deprecated V4 format (crc=0) will not be supported after September 2030."); -#else - xfs_warn(mp, - "Deprecated V4 format (crc=0) not supported by kernel."); - error = -EINVAL; - goto out_free_sb; -#endif } /* ASCII case insensitivity is undergoing deprecation. */ From 977b07f769970aec97b907cfc93fb681ecffc9fe Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Mon, 1 Apr 2024 21:28:15 +0800 Subject: [PATCH 0346/1702] dt-bindings: clock: add i.MX95 clock header Add clock header for i.MX95 BLK CTL modules Signed-off-by: Peng Fan Acked-by: Rob Herring Link: https://lore.kernel.org/r/20240401-imx95-blk-ctl-v6-1-84d4eca1e759@nxp.com Signed-off-by: Abel Vesa --- include/dt-bindings/clock/nxp,imx95-clock.h | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 include/dt-bindings/clock/nxp,imx95-clock.h diff --git a/include/dt-bindings/clock/nxp,imx95-clock.h b/include/dt-bindings/clock/nxp,imx95-clock.h new file mode 100644 index 0000000000000..782662c3e740e --- /dev/null +++ b/include/dt-bindings/clock/nxp,imx95-clock.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Copyright 2024 NXP + */ + +#ifndef __DT_BINDINGS_CLOCK_IMX95_H +#define __DT_BINDINGS_CLOCK_IMX95_H + +#define IMX95_CLK_VPUBLK_WAVE 0 +#define IMX95_CLK_VPUBLK_JPEG_ENC 1 +#define IMX95_CLK_VPUBLK_JPEG_DEC 2 + +#define IMX95_CLK_CAMBLK_CSI2_FOR0 0 +#define IMX95_CLK_CAMBLK_CSI2_FOR1 1 +#define IMX95_CLK_CAMBLK_ISP_AXI 2 +#define IMX95_CLK_CAMBLK_ISP_PIXEL 3 +#define IMX95_CLK_CAMBLK_ISP 4 + +#define IMX95_CLK_DISPMIX_LVDS_PHY_DIV 0 +#define IMX95_CLK_DISPMIX_LVDS_CH0_GATE 1 +#define IMX95_CLK_DISPMIX_LVDS_CH1_GATE 2 +#define IMX95_CLK_DISPMIX_PIX_DI0_GATE 3 +#define IMX95_CLK_DISPMIX_PIX_DI1_GATE 4 + +#define IMX95_CLK_DISPMIX_ENG0_SEL 0 +#define IMX95_CLK_DISPMIX_ENG1_SEL 1 + +#endif /* __DT_BINDINGS_CLOCK_IMX95_H */ From b773f5ad2bfd2d00bd2c5ea7022bc5b86d23a1b7 Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Mon, 1 Apr 2024 21:28:16 +0800 Subject: [PATCH 0347/1702] dt-bindings: clock: support i.MX95 BLK CTL module i.MX95 includes BLK CTL module in several MIXes, such as VPU_CSR in VPUMIX, CAMERA_CSR in CAMERAMIX and etc. The BLK CTL module is used for various settings of a specific MIX, such as clock, QoS and etc. This patch is to add some BLK CTL modules that has clock features. Reviewed-by: Krzysztof Kozlowski Signed-off-by: Peng Fan Link: https://lore.kernel.org/r/20240401-imx95-blk-ctl-v6-2-84d4eca1e759@nxp.com Signed-off-by: Abel Vesa --- .../bindings/clock/nxp,imx95-blk-ctl.yaml | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 Documentation/devicetree/bindings/clock/nxp,imx95-blk-ctl.yaml diff --git a/Documentation/devicetree/bindings/clock/nxp,imx95-blk-ctl.yaml b/Documentation/devicetree/bindings/clock/nxp,imx95-blk-ctl.yaml new file mode 100644 index 0000000000000..2dffc02dcd8b5 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/nxp,imx95-blk-ctl.yaml @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/nxp,imx95-blk-ctl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: NXP i.MX95 Block Control + +maintainers: + - Peng Fan + +properties: + compatible: + items: + - enum: + - nxp,imx95-lvds-csr + - nxp,imx95-display-csr + - nxp,imx95-camera-csr + - nxp,imx95-vpu-csr + - const: syscon + + reg: + maxItems: 1 + + power-domains: + maxItems: 1 + + clocks: + maxItems: 1 + + '#clock-cells': + const: 1 + description: + The clock consumer should specify the desired clock by having the clock + ID in its "clocks" phandle cell. See + include/dt-bindings/clock/nxp,imx95-clock.h + +required: + - compatible + - reg + - '#clock-cells' + - power-domains + - clocks + +additionalProperties: false + +examples: + - | + syscon@4c410000 { + compatible = "nxp,imx95-vpu-csr", "syscon"; + reg = <0x4c410000 0x10000>; + #clock-cells = <1>; + clocks = <&scmi_clk 114>; + power-domains = <&scmi_devpd 21>; + }; +... From c6e87b066756ec4b3f5f9061b508f3bd724ec652 Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Mon, 1 Apr 2024 21:28:17 +0800 Subject: [PATCH 0348/1702] dt-bindings: clock: support i.MX95 Display Master CSR module i.MX95 DISPLAY_MASTER_CSR includes registers to control DSI clock settings, clock gating, and pixel link select. Add dt-schema for it. Reviewed-by: Krzysztof Kozlowski Signed-off-by: Peng Fan Link: https://lore.kernel.org/r/20240401-imx95-blk-ctl-v6-3-84d4eca1e759@nxp.com Signed-off-by: Abel Vesa --- .../clock/nxp,imx95-display-master-csr.yaml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 Documentation/devicetree/bindings/clock/nxp,imx95-display-master-csr.yaml diff --git a/Documentation/devicetree/bindings/clock/nxp,imx95-display-master-csr.yaml b/Documentation/devicetree/bindings/clock/nxp,imx95-display-master-csr.yaml new file mode 100644 index 0000000000000..07f7412e76583 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/nxp,imx95-display-master-csr.yaml @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/nxp,imx95-display-master-csr.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: NXP i.MX95 Display Master Block Control + +maintainers: + - Peng Fan + +properties: + compatible: + items: + - const: nxp,imx95-display-master-csr + - const: syscon + + reg: + maxItems: 1 + + power-domains: + maxItems: 1 + + clocks: + maxItems: 1 + + '#clock-cells': + const: 1 + description: + The clock consumer should specify the desired clock by having the clock + ID in its "clocks" phandle cell. See + include/dt-bindings/clock/nxp,imx95-clock.h + + mux-controller: + type: object + $ref: /schemas/mux/reg-mux.yaml + +required: + - compatible + - reg + - '#clock-cells' + - mux-controller + - power-domains + - clocks + +additionalProperties: false + +examples: + - | + syscon@4c410000 { + compatible = "nxp,imx95-display-master-csr", "syscon"; + reg = <0x4c410000 0x10000>; + #clock-cells = <1>; + clocks = <&scmi_clk 62>; + power-domains = <&scmi_devpd 3>; + + mux: mux-controller { + compatible = "mmio-mux"; + #mux-control-cells = <1>; + mux-reg-masks = <0x4 0x00000001>; /* Pixel_link_sel */ + idle-states = <0>; + }; + }; +... From 5224b189462ff70df328f173b71acfd925092c3c Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Mon, 1 Apr 2024 21:28:18 +0800 Subject: [PATCH 0349/1702] clk: imx: add i.MX95 BLK CTL clk driver i.MX95 has BLK CTL modules in various MIXes, the BLK CTL modules support clock features such as mux/gate/div. This patch is to add the clock feature of BLK CTL modules Signed-off-by: Peng Fan Reviewed-by: Abel Vesa Link: https://lore.kernel.org/r/20240401-imx95-blk-ctl-v6-4-84d4eca1e759@nxp.com Signed-off-by: Abel Vesa --- drivers/clk/imx/Kconfig | 7 + drivers/clk/imx/Makefile | 1 + drivers/clk/imx/clk-imx95-blk-ctl.c | 438 ++++++++++++++++++++++++++++ 3 files changed, 446 insertions(+) create mode 100644 drivers/clk/imx/clk-imx95-blk-ctl.c diff --git a/drivers/clk/imx/Kconfig b/drivers/clk/imx/Kconfig index db3bca5f4ec9c..6da0fba68225e 100644 --- a/drivers/clk/imx/Kconfig +++ b/drivers/clk/imx/Kconfig @@ -114,6 +114,13 @@ config CLK_IMX93 help Build the driver for i.MX93 CCM Clock Driver +config CLK_IMX95_BLK_CTL + tristate "IMX95 Clock Driver for BLK CTL" + depends on ARCH_MXC || COMPILE_TEST + select MXC_CLK + help + Build the clock driver for i.MX95 BLK CTL + config CLK_IMXRT1050 tristate "IMXRT1050 CCM Clock Driver" depends on SOC_IMXRT || COMPILE_TEST diff --git a/drivers/clk/imx/Makefile b/drivers/clk/imx/Makefile index d4b8e10b1970d..03f2b2a1ab631 100644 --- a/drivers/clk/imx/Makefile +++ b/drivers/clk/imx/Makefile @@ -31,6 +31,7 @@ obj-$(CONFIG_CLK_IMX8MP) += clk-imx8mp.o clk-imx8mp-audiomix.o obj-$(CONFIG_CLK_IMX8MQ) += clk-imx8mq.o obj-$(CONFIG_CLK_IMX93) += clk-imx93.o +obj-$(CONFIG_CLK_IMX95_BLK_CTL) += clk-imx95-blk-ctl.o obj-$(CONFIG_MXC_CLK_SCU) += clk-imx-scu.o clk-imx-lpcg-scu.o clk-imx-acm.o clk-imx-scu-$(CONFIG_CLK_IMX8QXP) += clk-scu.o clk-imx8qxp.o \ diff --git a/drivers/clk/imx/clk-imx95-blk-ctl.c b/drivers/clk/imx/clk-imx95-blk-ctl.c new file mode 100644 index 0000000000000..74f595f9e5e34 --- /dev/null +++ b/drivers/clk/imx/clk-imx95-blk-ctl.c @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2024 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + CLK_GATE, + CLK_DIVIDER, + CLK_MUX, +}; + +struct imx95_blk_ctl { + struct device *dev; + spinlock_t lock; + struct clk *clk_apb; + + void __iomem *base; + /* clock gate register */ + u32 clk_reg_restore; +}; + +struct imx95_blk_ctl_clk_dev_data { + const char *name; + const char * const *parent_names; + u32 num_parents; + u32 reg; + u32 bit_idx; + u32 bit_width; + u32 clk_type; + u32 flags; + u32 flags2; + u32 type; +}; + +struct imx95_blk_ctl_dev_data { + const struct imx95_blk_ctl_clk_dev_data *clk_dev_data; + u32 num_clks; + bool rpm_enabled; + u32 clk_reg_offset; +}; + +static const struct imx95_blk_ctl_clk_dev_data vpublk_clk_dev_data[] = { + [IMX95_CLK_VPUBLK_WAVE] = { + .name = "vpublk_wave_vpu", + .parent_names = (const char *[]){ "vpu", }, + .num_parents = 1, + .reg = 8, + .bit_idx = 0, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_VPUBLK_JPEG_ENC] = { + .name = "vpublk_jpeg_enc", + .parent_names = (const char *[]){ "vpujpeg", }, + .num_parents = 1, + .reg = 8, + .bit_idx = 1, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_VPUBLK_JPEG_DEC] = { + .name = "vpublk_jpeg_dec", + .parent_names = (const char *[]){ "vpujpeg", }, + .num_parents = 1, + .reg = 8, + .bit_idx = 2, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + } +}; + +static const struct imx95_blk_ctl_dev_data vpublk_dev_data = { + .num_clks = ARRAY_SIZE(vpublk_clk_dev_data), + .clk_dev_data = vpublk_clk_dev_data, + .rpm_enabled = true, + .clk_reg_offset = 8, +}; + +static const struct imx95_blk_ctl_clk_dev_data camblk_clk_dev_data[] = { + [IMX95_CLK_CAMBLK_CSI2_FOR0] = { + .name = "camblk_csi2_for0", + .parent_names = (const char *[]){ "camisi", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 0, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_CAMBLK_CSI2_FOR1] = { + .name = "camblk_csi2_for1", + .parent_names = (const char *[]){ "camisi", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 1, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_CAMBLK_ISP_AXI] = { + .name = "camblk_isp_axi", + .parent_names = (const char *[]){ "camaxi", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 4, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_CAMBLK_ISP_PIXEL] = { + .name = "camblk_isp_pixel", + .parent_names = (const char *[]){ "camisi", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 5, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_CAMBLK_ISP] = { + .name = "camblk_isp", + .parent_names = (const char *[]){ "camisi", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 6, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + } +}; + +static const struct imx95_blk_ctl_dev_data camblk_dev_data = { + .num_clks = ARRAY_SIZE(camblk_clk_dev_data), + .clk_dev_data = camblk_clk_dev_data, + .clk_reg_offset = 0, +}; + +static const struct imx95_blk_ctl_clk_dev_data lvds_clk_dev_data[] = { + [IMX95_CLK_DISPMIX_LVDS_PHY_DIV] = { + .name = "ldb_phy_div", + .parent_names = (const char *[]){ "ldbpll", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 0, + .bit_width = 1, + .type = CLK_DIVIDER, + .flags2 = CLK_DIVIDER_POWER_OF_TWO, + }, + [IMX95_CLK_DISPMIX_LVDS_CH0_GATE] = { + .name = "lvds_ch0_gate", + .parent_names = (const char *[]){ "ldb_phy_div", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 1, + .bit_width = 1, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_DISPMIX_LVDS_CH1_GATE] = { + .name = "lvds_ch1_gate", + .parent_names = (const char *[]){ "ldb_phy_div", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 2, + .bit_width = 1, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_DISPMIX_PIX_DI0_GATE] = { + .name = "lvds_di0_gate", + .parent_names = (const char *[]){ "ldb_pll_div7", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 3, + .bit_width = 1, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, + [IMX95_CLK_DISPMIX_PIX_DI1_GATE] = { + .name = "lvds_di1_gate", + .parent_names = (const char *[]){ "ldb_pll_div7", }, + .num_parents = 1, + .reg = 0, + .bit_idx = 4, + .bit_width = 1, + .type = CLK_GATE, + .flags = CLK_SET_RATE_PARENT, + .flags2 = CLK_GATE_SET_TO_DISABLE, + }, +}; + +static const struct imx95_blk_ctl_dev_data lvds_csr_dev_data = { + .num_clks = ARRAY_SIZE(lvds_clk_dev_data), + .clk_dev_data = lvds_clk_dev_data, + .clk_reg_offset = 0, +}; + +static const struct imx95_blk_ctl_clk_dev_data dispmix_csr_clk_dev_data[] = { + [IMX95_CLK_DISPMIX_ENG0_SEL] = { + .name = "disp_engine0_sel", + .parent_names = (const char *[]){"videopll1", "dsi_pll", "ldb_pll_div7", }, + .num_parents = 4, + .reg = 0, + .bit_idx = 0, + .bit_width = 2, + .type = CLK_MUX, + .flags = CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT, + }, + [IMX95_CLK_DISPMIX_ENG1_SEL] = { + .name = "disp_engine1_sel", + .parent_names = (const char *[]){"videopll1", "dsi_pll", "ldb_pll_div7", }, + .num_parents = 4, + .reg = 0, + .bit_idx = 2, + .bit_width = 2, + .type = CLK_MUX, + .flags = CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT, + } +}; + +static const struct imx95_blk_ctl_dev_data dispmix_csr_dev_data = { + .num_clks = ARRAY_SIZE(dispmix_csr_clk_dev_data), + .clk_dev_data = dispmix_csr_clk_dev_data, + .clk_reg_offset = 0, +}; + +static int imx95_bc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + const struct imx95_blk_ctl_dev_data *bc_data; + struct imx95_blk_ctl *bc; + struct clk_hw_onecell_data *clk_hw_data; + struct clk_hw **hws; + void __iomem *base; + int i, ret; + + bc = devm_kzalloc(dev, sizeof(*bc), GFP_KERNEL); + if (!bc) + return -ENOMEM; + bc->dev = dev; + dev_set_drvdata(&pdev->dev, bc); + + spin_lock_init(&bc->lock); + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + bc->base = base; + bc->clk_apb = devm_clk_get(dev, NULL); + if (IS_ERR(bc->clk_apb)) + return dev_err_probe(dev, PTR_ERR(bc->clk_apb), "failed to get APB clock\n"); + + ret = clk_prepare_enable(bc->clk_apb); + if (ret) { + dev_err(dev, "failed to enable apb clock: %d\n", ret); + return ret; + } + + bc_data = of_device_get_match_data(dev); + if (!bc_data) + return devm_of_platform_populate(dev); + + clk_hw_data = devm_kzalloc(dev, struct_size(clk_hw_data, hws, bc_data->num_clks), + GFP_KERNEL); + if (!clk_hw_data) + return -ENOMEM; + + if (bc_data->rpm_enabled) + pm_runtime_enable(&pdev->dev); + + clk_hw_data->num = bc_data->num_clks; + hws = clk_hw_data->hws; + + for (i = 0; i < bc_data->num_clks; i++) { + const struct imx95_blk_ctl_clk_dev_data *data = &bc_data->clk_dev_data[i]; + void __iomem *reg = base + data->reg; + + if (data->type == CLK_MUX) { + hws[i] = clk_hw_register_mux(dev, data->name, data->parent_names, + data->num_parents, data->flags, reg, + data->bit_idx, data->bit_width, + data->flags2, &bc->lock); + } else if (data->type == CLK_DIVIDER) { + hws[i] = clk_hw_register_divider(dev, data->name, data->parent_names[0], + data->flags, reg, data->bit_idx, + data->bit_width, data->flags2, &bc->lock); + } else { + hws[i] = clk_hw_register_gate(dev, data->name, data->parent_names[0], + data->flags, reg, data->bit_idx, + data->flags2, &bc->lock); + } + if (IS_ERR(hws[i])) { + ret = PTR_ERR(hws[i]); + dev_err(dev, "failed to register: %s:%d\n", data->name, ret); + goto cleanup; + } + } + + ret = of_clk_add_hw_provider(dev->of_node, of_clk_hw_onecell_get, clk_hw_data); + if (ret) + goto cleanup; + + ret = devm_of_platform_populate(dev); + if (ret) { + of_clk_del_provider(dev->of_node); + goto cleanup; + } + + if (pm_runtime_enabled(bc->dev)) + clk_disable_unprepare(bc->clk_apb); + + return 0; + +cleanup: + for (i = 0; i < bc_data->num_clks; i++) { + if (IS_ERR_OR_NULL(hws[i])) + continue; + clk_hw_unregister(hws[i]); + } + + if (bc_data->rpm_enabled) + pm_runtime_disable(&pdev->dev); + + return ret; +} + +#ifdef CONFIG_PM +static int imx95_bc_runtime_suspend(struct device *dev) +{ + struct imx95_blk_ctl *bc = dev_get_drvdata(dev); + + clk_disable_unprepare(bc->clk_apb); + return 0; +} + +static int imx95_bc_runtime_resume(struct device *dev) +{ + struct imx95_blk_ctl *bc = dev_get_drvdata(dev); + + return clk_prepare_enable(bc->clk_apb); +} +#endif + +#ifdef CONFIG_PM_SLEEP +static int imx95_bc_suspend(struct device *dev) +{ + struct imx95_blk_ctl *bc = dev_get_drvdata(dev); + const struct imx95_blk_ctl_dev_data *bc_data; + int ret; + + bc_data = of_device_get_match_data(dev); + if (!bc_data) + return 0; + + if (bc_data->rpm_enabled) { + ret = pm_runtime_get_sync(bc->dev); + if (ret < 0) { + pm_runtime_put_noidle(bc->dev); + return ret; + } + } + + bc->clk_reg_restore = readl(bc->base + bc_data->clk_reg_offset); + + return 0; +} + +static int imx95_bc_resume(struct device *dev) +{ + struct imx95_blk_ctl *bc = dev_get_drvdata(dev); + const struct imx95_blk_ctl_dev_data *bc_data; + + bc_data = of_device_get_match_data(dev); + if (!bc_data) + return 0; + + writel(bc->clk_reg_restore, bc->base + bc_data->clk_reg_offset); + + if (bc_data->rpm_enabled) + pm_runtime_put(bc->dev); + + return 0; +} +#endif + +static const struct dev_pm_ops imx95_bc_pm_ops = { + SET_RUNTIME_PM_OPS(imx95_bc_runtime_suspend, imx95_bc_runtime_resume, NULL) + SET_SYSTEM_SLEEP_PM_OPS(imx95_bc_suspend, imx95_bc_resume) +}; + +static const struct of_device_id imx95_bc_of_match[] = { + { .compatible = "nxp,imx95-camera-csr", .data = &camblk_dev_data }, + { .compatible = "nxp,imx95-display-master-csr", }, + { .compatible = "nxp,imx95-lvds-csr", .data = &lvds_csr_dev_data }, + { .compatible = "nxp,imx95-display-csr", .data = &dispmix_csr_dev_data }, + { .compatible = "nxp,imx95-vpu-csr", .data = &vpublk_dev_data }, + { /* Sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, imx95_bc_of_match); + +static struct platform_driver imx95_bc_driver = { + .probe = imx95_bc_probe, + .driver = { + .name = "imx95-blk-ctl", + .of_match_table = imx95_bc_of_match, + .pm = &imx95_bc_pm_ops, + }, +}; +module_platform_driver(imx95_bc_driver); + +MODULE_DESCRIPTION("NXP i.MX95 blk ctl driver"); +MODULE_AUTHOR("Peng Fan "); +MODULE_LICENSE("GPL"); From 330c4f94b0d73e440de3f738a625e38defe1bc15 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Apr 2024 13:20:07 +0200 Subject: [PATCH 0350/1702] xfs: make XFS_TRANS_LOWMODE match the other XFS_TRANS_ definitions Commit bb7b1c9c5dd3 ("xfs: tag transactions that contain intent done items") switched the XFS_TRANS_ definitions to be bit based, and using comments above the definitions. As XFS_TRANS_LOWMODE was last and has a big fat comment it was missed. Switch it to the same style. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_shared.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/xfs/libxfs/xfs_shared.h b/fs/xfs/libxfs/xfs_shared.h index dfd61fa8332e1..f35640ad3e7fe 100644 --- a/fs/xfs/libxfs/xfs_shared.h +++ b/fs/xfs/libxfs/xfs_shared.h @@ -124,7 +124,6 @@ void xfs_log_get_max_trans_res(struct xfs_mount *mp, #define XFS_TRANS_RES_FDBLKS (1u << 6) /* Transaction contains an intent done log item */ #define XFS_TRANS_HAS_INTENT_DONE (1u << 7) - /* * LOWMODE is used by the allocator to activate the lowspace algorithm - when * free space is running low the extent allocator may choose to allocate an @@ -136,7 +135,7 @@ void xfs_log_get_max_trans_res(struct xfs_mount *mp, * for free space from AG 0. If the correct transaction reservations have been * made then this algorithm will eventually find all the space it needs. */ -#define XFS_TRANS_LOWMODE 0x100 /* allocate in low space mode */ +#define XFS_TRANS_LOWMODE (1u << 8) /* * Field values for xfs_trans_mod_sb. From b7e23c0e2e3b1c520a3370f058870b914071a470 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Apr 2024 13:20:08 +0200 Subject: [PATCH 0351/1702] xfs: refactor realtime inode locking Create helper functions to deal with locking realtime metadata inodes. This enables us to maintain correct locking order once we start adding the realtime rmap and refcount btree inodes. Signed-off-by: "Darrick J. Wong" Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 7 ++--- fs/xfs/libxfs/xfs_rtbitmap.c | 57 ++++++++++++++++++++++++++++++++++++ fs/xfs/libxfs/xfs_rtbitmap.h | 17 +++++++++++ fs/xfs/scrub/common.c | 1 + fs/xfs/scrub/fscounters.c | 4 +-- fs/xfs/xfs_fsmap.c | 4 +-- fs/xfs/xfs_rtalloc.c | 20 ++++--------- 7 files changed, 87 insertions(+), 23 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 59b8b9dc29ccf..3e56173a8fe4d 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -5418,12 +5418,9 @@ __xfs_bunmapi( if (isrt) { /* - * Synchronize by locking the bitmap inode. + * Synchronize by locking the realtime bitmap. */ - xfs_ilock(mp->m_rbmip, XFS_ILOCK_EXCL|XFS_ILOCK_RTBITMAP); - xfs_trans_ijoin(tp, mp->m_rbmip, XFS_ILOCK_EXCL); - xfs_ilock(mp->m_rsumip, XFS_ILOCK_EXCL|XFS_ILOCK_RTSUM); - xfs_trans_ijoin(tp, mp->m_rsumip, XFS_ILOCK_EXCL); + xfs_rtbitmap_lock(tp, mp); } extno = 0; diff --git a/fs/xfs/libxfs/xfs_rtbitmap.c b/fs/xfs/libxfs/xfs_rtbitmap.c index f246d6dbf4eca..386b672c50589 100644 --- a/fs/xfs/libxfs/xfs_rtbitmap.c +++ b/fs/xfs/libxfs/xfs_rtbitmap.c @@ -1168,3 +1168,60 @@ xfs_rtsummary_wordcount( blocks = xfs_rtsummary_blockcount(mp, rsumlevels, rbmblocks); return XFS_FSB_TO_B(mp, blocks) >> XFS_WORDLOG; } + +/* + * Lock both realtime free space metadata inodes for a freespace update. If a + * transaction is given, the inodes will be joined to the transaction and the + * ILOCKs will be released on transaction commit. + */ +void +xfs_rtbitmap_lock( + struct xfs_trans *tp, + struct xfs_mount *mp) +{ + xfs_ilock(mp->m_rbmip, XFS_ILOCK_EXCL | XFS_ILOCK_RTBITMAP); + if (tp) + xfs_trans_ijoin(tp, mp->m_rbmip, XFS_ILOCK_EXCL); + + xfs_ilock(mp->m_rsumip, XFS_ILOCK_EXCL | XFS_ILOCK_RTSUM); + if (tp) + xfs_trans_ijoin(tp, mp->m_rsumip, XFS_ILOCK_EXCL); +} + +/* Unlock both realtime free space metadata inodes after a freespace update. */ +void +xfs_rtbitmap_unlock( + struct xfs_mount *mp) +{ + xfs_iunlock(mp->m_rsumip, XFS_ILOCK_EXCL | XFS_ILOCK_RTSUM); + xfs_iunlock(mp->m_rbmip, XFS_ILOCK_EXCL | XFS_ILOCK_RTBITMAP); +} + +/* + * Lock the realtime free space metadata inodes for a freespace scan. Callers + * must walk metadata blocks in order of increasing file offset. + */ +void +xfs_rtbitmap_lock_shared( + struct xfs_mount *mp, + unsigned int rbmlock_flags) +{ + if (rbmlock_flags & XFS_RBMLOCK_BITMAP) + xfs_ilock(mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP); + + if (rbmlock_flags & XFS_RBMLOCK_SUMMARY) + xfs_ilock(mp->m_rsumip, XFS_ILOCK_SHARED | XFS_ILOCK_RTSUM); +} + +/* Unlock the realtime free space metadata inodes after a freespace scan. */ +void +xfs_rtbitmap_unlock_shared( + struct xfs_mount *mp, + unsigned int rbmlock_flags) +{ + if (rbmlock_flags & XFS_RBMLOCK_SUMMARY) + xfs_iunlock(mp->m_rsumip, XFS_ILOCK_SHARED | XFS_ILOCK_RTSUM); + + if (rbmlock_flags & XFS_RBMLOCK_BITMAP) + xfs_iunlock(mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP); +} diff --git a/fs/xfs/libxfs/xfs_rtbitmap.h b/fs/xfs/libxfs/xfs_rtbitmap.h index 152a66750af55..6186585f2c376 100644 --- a/fs/xfs/libxfs/xfs_rtbitmap.h +++ b/fs/xfs/libxfs/xfs_rtbitmap.h @@ -360,6 +360,19 @@ xfs_filblks_t xfs_rtsummary_blockcount(struct xfs_mount *mp, unsigned int rsumlevels, xfs_extlen_t rbmblocks); unsigned long long xfs_rtsummary_wordcount(struct xfs_mount *mp, unsigned int rsumlevels, xfs_extlen_t rbmblocks); + +void xfs_rtbitmap_lock(struct xfs_trans *tp, struct xfs_mount *mp); +void xfs_rtbitmap_unlock(struct xfs_mount *mp); + +/* Lock the rt bitmap inode in shared mode */ +#define XFS_RBMLOCK_BITMAP (1U << 0) +/* Lock the rt summary inode in shared mode */ +#define XFS_RBMLOCK_SUMMARY (1U << 1) + +void xfs_rtbitmap_lock_shared(struct xfs_mount *mp, + unsigned int rbmlock_flags); +void xfs_rtbitmap_unlock_shared(struct xfs_mount *mp, + unsigned int rbmlock_flags); #else /* CONFIG_XFS_RT */ # define xfs_rtfree_extent(t,b,l) (-ENOSYS) # define xfs_rtfree_blocks(t,rb,rl) (-ENOSYS) @@ -378,6 +391,10 @@ xfs_rtbitmap_blockcount(struct xfs_mount *mp, xfs_rtbxlen_t rtextents) # define xfs_rtbitmap_wordcount(mp, r) (0) # define xfs_rtsummary_blockcount(mp, l, b) (0) # define xfs_rtsummary_wordcount(mp, l, b) (0) +# define xfs_rtbitmap_lock(tp, mp) do { } while (0) +# define xfs_rtbitmap_unlock(mp) do { } while (0) +# define xfs_rtbitmap_lock_shared(mp, lf) do { } while (0) +# define xfs_rtbitmap_unlock_shared(mp, lf) do { } while (0) #endif /* CONFIG_XFS_RT */ #endif /* __XFS_RTBITMAP_H__ */ diff --git a/fs/xfs/scrub/common.c b/fs/xfs/scrub/common.c index 48302532d10d1..a7d3a22796620 100644 --- a/fs/xfs/scrub/common.c +++ b/fs/xfs/scrub/common.c @@ -32,6 +32,7 @@ #include "xfs_error.h" #include "xfs_quota.h" #include "xfs_exchmaps.h" +#include "xfs_rtbitmap.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/trace.h" diff --git a/fs/xfs/scrub/fscounters.c b/fs/xfs/scrub/fscounters.c index da2f6729699dd..67ac870052b16 100644 --- a/fs/xfs/scrub/fscounters.c +++ b/fs/xfs/scrub/fscounters.c @@ -415,7 +415,7 @@ xchk_fscount_count_frextents( if (!xfs_has_realtime(mp)) return 0; - xfs_ilock(sc->mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP); + xfs_rtbitmap_lock_shared(sc->mp, XFS_RBMLOCK_BITMAP); error = xfs_rtalloc_query_all(sc->mp, sc->tp, xchk_fscount_add_frextent, fsc); if (error) { @@ -424,7 +424,7 @@ xchk_fscount_count_frextents( } out_unlock: - xfs_iunlock(sc->mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP); + xfs_rtbitmap_unlock_shared(sc->mp, XFS_RBMLOCK_BITMAP); return error; } #else diff --git a/fs/xfs/xfs_fsmap.c b/fs/xfs/xfs_fsmap.c index de59eec747657..85dbb46452ca0 100644 --- a/fs/xfs/xfs_fsmap.c +++ b/fs/xfs/xfs_fsmap.c @@ -533,7 +533,7 @@ xfs_getfsmap_rtdev_rtbitmap( trace_xfs_fsmap_low_key_linear(mp, info->dev, start_rtb); trace_xfs_fsmap_high_key_linear(mp, info->dev, end_rtb); - xfs_ilock(mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP); + xfs_rtbitmap_lock_shared(mp, XFS_RBMLOCK_BITMAP); /* * Set up query parameters to return free rtextents covering the range @@ -557,7 +557,7 @@ xfs_getfsmap_rtdev_rtbitmap( if (error) goto err; err: - xfs_iunlock(mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP); + xfs_rtbitmap_unlock_shared(mp, XFS_RBMLOCK_BITMAP); return error; } #endif /* CONFIG_XFS_RT */ diff --git a/fs/xfs/xfs_rtalloc.c b/fs/xfs/xfs_rtalloc.c index e66f9bd5de5cf..86f928d30feda 100644 --- a/fs/xfs/xfs_rtalloc.c +++ b/fs/xfs/xfs_rtalloc.c @@ -957,10 +957,10 @@ xfs_growfs_rt( nargs.tp = tp; /* - * Lock out other callers by grabbing the bitmap inode lock. + * Lock out other callers by grabbing the bitmap and summary + * inode locks and joining them to the transaction. */ - xfs_ilock(mp->m_rbmip, XFS_ILOCK_EXCL | XFS_ILOCK_RTBITMAP); - xfs_trans_ijoin(tp, mp->m_rbmip, XFS_ILOCK_EXCL); + xfs_rtbitmap_lock(tp, mp); /* * Update the bitmap inode's size ondisk and incore. We need * to update the incore size so that inode inactivation won't @@ -970,11 +970,6 @@ xfs_growfs_rt( nsbp->sb_rbmblocks * nsbp->sb_blocksize; i_size_write(VFS_I(mp->m_rbmip), mp->m_rbmip->i_disk_size); xfs_trans_log_inode(tp, mp->m_rbmip, XFS_ILOG_CORE); - /* - * Get the summary inode into the transaction. - */ - xfs_ilock(mp->m_rsumip, XFS_ILOCK_EXCL | XFS_ILOCK_RTSUM); - xfs_trans_ijoin(tp, mp->m_rsumip, XFS_ILOCK_EXCL); /* * Update the summary inode's size. We need to update the * incore size so that inode inactivation won't punch what it @@ -1142,10 +1137,10 @@ xfs_rtalloc_reinit_frextents( uint64_t val = 0; int error; - xfs_ilock(mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP); + xfs_rtbitmap_lock_shared(mp, XFS_RBMLOCK_BITMAP); error = xfs_rtalloc_query_all(mp, NULL, xfs_rtalloc_count_frextent, &val); - xfs_iunlock(mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP); + xfs_rtbitmap_unlock_shared(mp, XFS_RBMLOCK_BITMAP); if (error) return error; @@ -1382,10 +1377,7 @@ xfs_bmap_rtalloc( * Lock out modifications to both the RT bitmap and summary inodes */ if (!rtlocked) { - xfs_ilock(mp->m_rbmip, XFS_ILOCK_EXCL|XFS_ILOCK_RTBITMAP); - xfs_trans_ijoin(ap->tp, mp->m_rbmip, XFS_ILOCK_EXCL); - xfs_ilock(mp->m_rsumip, XFS_ILOCK_EXCL|XFS_ILOCK_RTSUM); - xfs_trans_ijoin(ap->tp, mp->m_rsumip, XFS_ILOCK_EXCL); + xfs_rtbitmap_lock(ap->tp, mp); rtlocked = true; } From 9871d0963751293bf0587759a9b6b8f808e35c7c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Apr 2024 13:20:09 +0200 Subject: [PATCH 0352/1702] xfs: free RT extents after updating the bmap btree Currently xfs_bmap_del_extent_real frees RT extents before updating the bmap btree, while it frees regular blocks after performing the bmap btree update for convoluted historic reasons. Switch to free the RT blocks in the same place as the regular data blocks instead to simply the code and fix a very theoretical bug. A short history of this code researched by Dave Chiner below: The truncate for data device extents was originally a two-phase operation. First it removed the bmapbt record, but because this can free BMBT extents, it can use up all the free space tree reservation space. So the transaction gets rolled to commit the BMBT change and the xfs_bmap_finish() call that frees the data extent runs with a new transaction reservation that allows different free space btrees to be logged without overrun. However, on crash, this could lose the free space because there was nothing to tell recovery about the extents removed from the BMBT, hence EFIs were introduced. They tie the extent free operation to the bmapbt record removal commit for recovery of the second phase of the extent removal process. Then RT extents came along. RT extent freeing does not require a free space btree reservation because the free space metadata is static and transaction size is bound. Hence we don't need to care if the BMBT record removal modifies the per-ag free space trees and we don't need a two-phase extent remove transaction. The only thing we have to care about is not losing space on crash. Hence instead of recording the extent for freeing in the bmap list for xfs_bmap_finish() to process in a new transaction, it simply freed the rtextent directly. So the original code (from 1994) simply replaced the "free AG extent later" queueing with a direct free. This code was originally at the start of xfs_dmap_del_extent(), but the xfs_bmap_add_free() got moved to the end of the function via the "do_fx" flag (the current code logic) in 1997 (commit c4fac74eaa58 in the historic xfs-import tree) because there was a shutdown occurring because of a case where splitting the extent record failed because the BMBT split and the filesystem didn't have enough space for the split to be done. (FWIW, I'm not sure this can happen anymore.) The commit backed out the BMBT change on ENOSPC error, and in doing so I think this actually breaks RT free space tracking. However, it then returns an ENOSPC error, and we have a dirty transaction in the RT case so this will shut down the filesysetm when the transaction is cancelled. Hence the corrupted "bmbt now points at freed rt dev space" condition never make it to disk, but it's still the wrong way to handle the issue. IOWs, this proposed change fixes that "shutdown at ENOSPC on rt devices" situation that was introduced by the above commit back in 1997. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 3e56173a8fe4d..f529aa4071092 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -5109,8 +5109,7 @@ xfs_bmap_del_extent_real( { xfs_fsblock_t del_endblock=0; /* first block past del */ xfs_fileoff_t del_endoff; /* first offset past del */ - int do_fx; /* free extent at end of routine */ - int error; /* error return value */ + int error = 0; /* error return value */ struct xfs_bmbt_irec got; /* current extent entry */ xfs_fileoff_t got_endoff; /* first offset past got */ int i; /* temp state */ @@ -5153,20 +5152,10 @@ xfs_bmap_del_extent_real( return -ENOSPC; *logflagsp = XFS_ILOG_CORE; - if (xfs_ifork_is_realtime(ip, whichfork)) { - if (!(bflags & XFS_BMAPI_REMAP)) { - error = xfs_rtfree_blocks(tp, del->br_startblock, - del->br_blockcount); - if (error) - return error; - } - - do_fx = 0; + if (xfs_ifork_is_realtime(ip, whichfork)) qfield = XFS_TRANS_DQ_RTBCOUNT; - } else { - do_fx = 1; + else qfield = XFS_TRANS_DQ_BCOUNT; - } nblks = del->br_blockcount; del_endblock = del->br_startblock + del->br_blockcount; @@ -5314,18 +5303,21 @@ xfs_bmap_del_extent_real( /* * If we need to, add to list of extents to delete. */ - if (do_fx && !(bflags & XFS_BMAPI_REMAP)) { + if (!(bflags & XFS_BMAPI_REMAP)) { if (xfs_is_reflink_inode(ip) && whichfork == XFS_DATA_FORK) { xfs_refcount_decrease_extent(tp, del); + } else if (xfs_ifork_is_realtime(ip, whichfork)) { + error = xfs_rtfree_blocks(tp, del->br_startblock, + del->br_blockcount); } else { error = xfs_free_extent_later(tp, del->br_startblock, del->br_blockcount, NULL, XFS_AG_RESV_NONE, ((bflags & XFS_BMAPI_NODISCARD) || del->br_state == XFS_EXT_UNWRITTEN)); - if (error) - return error; } + if (error) + return error; } /* From de37dbd0ccc6933fbf4bd7b3ccbc5ac640e80b28 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Apr 2024 13:20:10 +0200 Subject: [PATCH 0353/1702] xfs: move RT inode locking out of __xfs_bunmapi __xfs_bunmapi is a bit of an odd place to lock the rtbitmap and rtsummary inodes given that it is very high level code. While this only looks ugly right now, it will become a problem when supporting delayed allocations for RT inodes as __xfs_bunmapi might end up deleting only delalloc extents and thus never unlock the rt inodes. Move the locking into xfs_bmap_del_extent_real just before the call to xfs_rtfree_blocks instead and use a new flag in the transaction to ensure that the locking happens only once. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 15 ++++++++------- fs/xfs/libxfs/xfs_shared.h | 3 +++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index f529aa4071092..22d44627ec599 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -5307,6 +5307,14 @@ xfs_bmap_del_extent_real( if (xfs_is_reflink_inode(ip) && whichfork == XFS_DATA_FORK) { xfs_refcount_decrease_extent(tp, del); } else if (xfs_ifork_is_realtime(ip, whichfork)) { + /* + * Ensure the bitmap and summary inodes are locked + * and joined to the transaction before modifying them. + */ + if (!(tp->t_flags & XFS_TRANS_RTBITMAP_LOCKED)) { + tp->t_flags |= XFS_TRANS_RTBITMAP_LOCKED; + xfs_rtbitmap_lock(tp, mp); + } error = xfs_rtfree_blocks(tp, del->br_startblock, del->br_blockcount); } else { @@ -5408,13 +5416,6 @@ __xfs_bunmapi( } else cur = NULL; - if (isrt) { - /* - * Synchronize by locking the realtime bitmap. - */ - xfs_rtbitmap_lock(tp, mp); - } - extno = 0; while (end != (xfs_fileoff_t)-1 && end >= start && (nexts == 0 || extno < nexts)) { diff --git a/fs/xfs/libxfs/xfs_shared.h b/fs/xfs/libxfs/xfs_shared.h index f35640ad3e7fe..34f104ed372c0 100644 --- a/fs/xfs/libxfs/xfs_shared.h +++ b/fs/xfs/libxfs/xfs_shared.h @@ -137,6 +137,9 @@ void xfs_log_get_max_trans_res(struct xfs_mount *mp, */ #define XFS_TRANS_LOWMODE (1u << 8) +/* Transaction has locked the rtbitmap and rtsum inodes */ +#define XFS_TRANS_RTBITMAP_LOCKED (1u << 9) + /* * Field values for xfs_trans_mod_sb. */ From 5e1e4d4fc79c6e5f80864860208ceae27dead8a2 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Apr 2024 13:20:11 +0200 Subject: [PATCH 0354/1702] xfs: block deltas in xfs_trans_unreserve_and_mod_sb must be positive And to make that more clear, rearrange the code a bit and add asserts and a comment. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/xfs_trans.c | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 50d878d78a5e1..95921ad935477 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -594,28 +594,38 @@ xfs_trans_unreserve_and_mod_sb( { struct xfs_mount *mp = tp->t_mountp; bool rsvd = (tp->t_flags & XFS_TRANS_RESERVE) != 0; - int64_t blkdelta = 0; - int64_t rtxdelta = 0; + int64_t blkdelta = tp->t_blk_res; + int64_t rtxdelta = tp->t_rtx_res; int64_t idelta = 0; int64_t ifreedelta = 0; int error; - /* calculate deltas */ - if (tp->t_blk_res > 0) - blkdelta = tp->t_blk_res; - if ((tp->t_fdblocks_delta != 0) && - (xfs_has_lazysbcount(mp) || - (tp->t_flags & XFS_TRANS_SB_DIRTY))) + /* + * Calculate the deltas. + * + * t_fdblocks_delta and t_frextents_delta can be positive or negative: + * + * - positive values indicate blocks freed in the transaction. + * - negative values indicate blocks allocated in the transaction + * + * Negative values can only happen if the transaction has a block + * reservation that covers the allocated block. The end result is + * that the calculated delta values must always be positive and we + * can only put back previous allocated or reserved blocks here. + */ + ASSERT(tp->t_blk_res || tp->t_fdblocks_delta >= 0); + if (xfs_has_lazysbcount(mp) || (tp->t_flags & XFS_TRANS_SB_DIRTY)) { blkdelta += tp->t_fdblocks_delta; + ASSERT(blkdelta >= 0); + } - if (tp->t_rtx_res > 0) - rtxdelta = tp->t_rtx_res; - if ((tp->t_frextents_delta != 0) && - (tp->t_flags & XFS_TRANS_SB_DIRTY)) + ASSERT(tp->t_rtx_res || tp->t_frextents_delta >= 0); + if (tp->t_flags & XFS_TRANS_SB_DIRTY) { rtxdelta += tp->t_frextents_delta; + ASSERT(rtxdelta >= 0); + } - if (xfs_has_lazysbcount(mp) || - (tp->t_flags & XFS_TRANS_SB_DIRTY)) { + if (xfs_has_lazysbcount(mp) || (tp->t_flags & XFS_TRANS_SB_DIRTY)) { idelta = tp->t_icount_delta; ifreedelta = tp->t_ifree_delta; } From f30f656e25eb72c4309e76b16fa45062e183a2ee Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Apr 2024 13:20:12 +0200 Subject: [PATCH 0355/1702] xfs: split xfs_mod_freecounter xfs_mod_freecounter has two entirely separate code paths for adding or subtracting from the free counters. Only the subtract case looks at the rsvd flag and can return an error. Split xfs_mod_freecounter into separate helpers for subtracting or adding the freecounter, and remove all the impossible to reach error handling for the addition case. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_ag.c | 4 +-- fs/xfs/libxfs/xfs_ag_resv.c | 24 ++++--------- fs/xfs/libxfs/xfs_ag_resv.h | 2 +- fs/xfs/libxfs/xfs_alloc.c | 4 +-- fs/xfs/libxfs/xfs_bmap.c | 23 +++++++------ fs/xfs/scrub/fscounters.c | 2 +- fs/xfs/scrub/repair.c | 5 +-- fs/xfs/xfs_fsops.c | 29 +++++----------- fs/xfs/xfs_fsops.h | 2 +- fs/xfs/xfs_mount.c | 67 +++++++++++++++++++------------------ fs/xfs/xfs_mount.h | 27 ++++++++++----- fs/xfs/xfs_super.c | 6 +--- fs/xfs/xfs_trace.h | 1 - fs/xfs/xfs_trans.c | 25 +++++--------- 14 files changed, 97 insertions(+), 124 deletions(-) diff --git a/fs/xfs/libxfs/xfs_ag.c b/fs/xfs/libxfs/xfs_ag.c index 09fe9412eab49..240e079cb3fbb 100644 --- a/fs/xfs/libxfs/xfs_ag.c +++ b/fs/xfs/libxfs/xfs_ag.c @@ -963,9 +963,7 @@ xfs_ag_shrink_space( * Disable perag reservations so it doesn't cause the allocation request * to fail. We'll reestablish reservation before we return. */ - error = xfs_ag_resv_free(pag); - if (error) - return error; + xfs_ag_resv_free(pag); /* internal log shouldn't also show up in the free space btrees */ error = xfs_alloc_vextent_exact_bno(&args, diff --git a/fs/xfs/libxfs/xfs_ag_resv.c b/fs/xfs/libxfs/xfs_ag_resv.c index da1057bd0e606..216423df939e5 100644 --- a/fs/xfs/libxfs/xfs_ag_resv.c +++ b/fs/xfs/libxfs/xfs_ag_resv.c @@ -126,14 +126,13 @@ xfs_ag_resv_needed( } /* Clean out a reservation */ -static int +static void __xfs_ag_resv_free( struct xfs_perag *pag, enum xfs_ag_resv_type type) { struct xfs_ag_resv *resv; xfs_extlen_t oldresv; - int error; trace_xfs_ag_resv_free(pag, type, 0); @@ -149,30 +148,19 @@ __xfs_ag_resv_free( oldresv = resv->ar_orig_reserved; else oldresv = resv->ar_reserved; - error = xfs_mod_fdblocks(pag->pag_mount, oldresv, true); + xfs_add_fdblocks(pag->pag_mount, oldresv); resv->ar_reserved = 0; resv->ar_asked = 0; resv->ar_orig_reserved = 0; - - if (error) - trace_xfs_ag_resv_free_error(pag->pag_mount, pag->pag_agno, - error, _RET_IP_); - return error; } /* Free a per-AG reservation. */ -int +void xfs_ag_resv_free( struct xfs_perag *pag) { - int error; - int err2; - - error = __xfs_ag_resv_free(pag, XFS_AG_RESV_RMAPBT); - err2 = __xfs_ag_resv_free(pag, XFS_AG_RESV_METADATA); - if (err2 && !error) - error = err2; - return error; + __xfs_ag_resv_free(pag, XFS_AG_RESV_RMAPBT); + __xfs_ag_resv_free(pag, XFS_AG_RESV_METADATA); } static int @@ -216,7 +204,7 @@ __xfs_ag_resv_init( if (XFS_TEST_ERROR(false, mp, XFS_ERRTAG_AG_RESV_FAIL)) error = -ENOSPC; else - error = xfs_mod_fdblocks(mp, -(int64_t)hidden_space, true); + error = xfs_dec_fdblocks(mp, hidden_space, true); if (error) { trace_xfs_ag_resv_init_error(pag->pag_mount, pag->pag_agno, error, _RET_IP_); diff --git a/fs/xfs/libxfs/xfs_ag_resv.h b/fs/xfs/libxfs/xfs_ag_resv.h index b74b210008ea7..ff20ed93de772 100644 --- a/fs/xfs/libxfs/xfs_ag_resv.h +++ b/fs/xfs/libxfs/xfs_ag_resv.h @@ -6,7 +6,7 @@ #ifndef __XFS_AG_RESV_H__ #define __XFS_AG_RESV_H__ -int xfs_ag_resv_free(struct xfs_perag *pag); +void xfs_ag_resv_free(struct xfs_perag *pag); int xfs_ag_resv_init(struct xfs_perag *pag, struct xfs_trans *tp); bool xfs_ag_resv_critical(struct xfs_perag *pag, enum xfs_ag_resv_type type); diff --git a/fs/xfs/libxfs/xfs_alloc.c b/fs/xfs/libxfs/xfs_alloc.c index 9da52e92172ab..6cb8b2ddc541b 100644 --- a/fs/xfs/libxfs/xfs_alloc.c +++ b/fs/xfs/libxfs/xfs_alloc.c @@ -79,7 +79,7 @@ xfs_prealloc_blocks( } /* - * The number of blocks per AG that we withhold from xfs_mod_fdblocks to + * The number of blocks per AG that we withhold from xfs_dec_fdblocks to * guarantee that we can refill the AGFL prior to allocating space in a nearly * full AG. Although the space described by the free space btrees, the * blocks used by the freesp btrees themselves, and the blocks owned by the @@ -89,7 +89,7 @@ xfs_prealloc_blocks( * until the fs goes down, we subtract this many AG blocks from the incore * fdblocks to ensure user allocation does not overcommit the space the * filesystem needs for the AGFLs. The rmap btree uses a per-AG reservation to - * withhold space from xfs_mod_fdblocks, so we do not account for that here. + * withhold space from xfs_dec_fdblocks, so we do not account for that here. */ #define XFS_ALLOCBT_AGFL_RESERVE 4 diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 22d44627ec599..0311a98df1c59 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -1985,10 +1985,11 @@ xfs_bmap_add_extent_delay_real( } /* adjust for changes in reserved delayed indirect blocks */ - if (da_new != da_old) { - ASSERT(state == 0 || da_new < da_old); - error = xfs_mod_fdblocks(mp, (int64_t)(da_old - da_new), - false); + if (da_new < da_old) { + xfs_add_fdblocks(mp, da_old - da_new); + } else if (da_new > da_old) { + ASSERT(state == 0); + error = xfs_dec_fdblocks(mp, da_new - da_old, false); } xfs_bmap_check_leaf_extents(bma->cur, bma->ip, whichfork); @@ -2690,8 +2691,8 @@ xfs_bmap_add_extent_hole_delay( } if (oldlen != newlen) { ASSERT(oldlen > newlen); - xfs_mod_fdblocks(ip->i_mount, (int64_t)(oldlen - newlen), - false); + xfs_add_fdblocks(ip->i_mount, oldlen - newlen); + /* * Nothing to do for disk quota accounting here. */ @@ -4110,11 +4111,11 @@ xfs_bmapi_reserve_delalloc( indlen = (xfs_extlen_t)xfs_bmap_worst_indlen(ip, alen); ASSERT(indlen > 0); - error = xfs_mod_fdblocks(mp, -((int64_t)alen), false); + error = xfs_dec_fdblocks(mp, alen, false); if (error) goto out_unreserve_quota; - error = xfs_mod_fdblocks(mp, -((int64_t)indlen), false); + error = xfs_dec_fdblocks(mp, indlen, false); if (error) goto out_unreserve_blocks; @@ -4142,7 +4143,7 @@ xfs_bmapi_reserve_delalloc( return 0; out_unreserve_blocks: - xfs_mod_fdblocks(mp, alen, false); + xfs_add_fdblocks(mp, alen); out_unreserve_quota: if (XFS_IS_QUOTA_ON(mp)) xfs_quota_unreserve_blkres(ip, alen); @@ -4928,7 +4929,7 @@ xfs_bmap_del_extent_delay( ASSERT(got_endoff >= del_endoff); if (isrt) - xfs_mod_frextents(mp, xfs_rtb_to_rtx(mp, del->br_blockcount)); + xfs_add_frextents(mp, xfs_rtb_to_rtx(mp, del->br_blockcount)); /* * Update the inode delalloc counter now and wait to update the @@ -5015,7 +5016,7 @@ xfs_bmap_del_extent_delay( if (!isrt) da_diff += del->br_blockcount; if (da_diff) { - xfs_mod_fdblocks(mp, da_diff, false); + xfs_add_fdblocks(mp, da_diff); xfs_mod_delalloc(mp, -da_diff); } return error; diff --git a/fs/xfs/scrub/fscounters.c b/fs/xfs/scrub/fscounters.c index 67ac870052b16..b662e52fe6956 100644 --- a/fs/xfs/scrub/fscounters.c +++ b/fs/xfs/scrub/fscounters.c @@ -517,7 +517,7 @@ xchk_fscounters( /* * If the filesystem is not frozen, the counter summation calls above - * can race with xfs_mod_freecounter, which subtracts a requested space + * can race with xfs_dec_freecounter, which subtracts a requested space * reservation from the counter and undoes the subtraction if that made * the counter go negative. Therefore, it's possible to see negative * values here, and we should only flag that as a corruption if we diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c index 92f81e374bc08..67478294f11ae 100644 --- a/fs/xfs/scrub/repair.c +++ b/fs/xfs/scrub/repair.c @@ -968,9 +968,7 @@ xrep_reset_perag_resv( ASSERT(sc->tp); sc->flags &= ~XREP_RESET_PERAG_RESV; - error = xfs_ag_resv_free(sc->sa.pag); - if (error) - goto out; + xfs_ag_resv_free(sc->sa.pag); error = xfs_ag_resv_init(sc->sa.pag, sc->tp); if (error == -ENOSPC) { xfs_err(sc->mp, @@ -979,7 +977,6 @@ xrep_reset_perag_resv( error = 0; } -out: return error; } diff --git a/fs/xfs/xfs_fsops.c b/fs/xfs/xfs_fsops.c index 83f708f62ed9f..c211ea2b63c4d 100644 --- a/fs/xfs/xfs_fsops.c +++ b/fs/xfs/xfs_fsops.c @@ -213,10 +213,8 @@ xfs_growfs_data_private( struct xfs_perag *pag; pag = xfs_perag_get(mp, id.agno); - error = xfs_ag_resv_free(pag); + xfs_ag_resv_free(pag); xfs_perag_put(pag); - if (error) - return error; } /* * Reserve AG metadata blocks. ENOSPC here does not mean there @@ -385,14 +383,14 @@ xfs_reserve_blocks( */ if (mp->m_resblks > request) { lcounter = mp->m_resblks_avail - request; - if (lcounter > 0) { /* release unused blocks */ + if (lcounter > 0) { /* release unused blocks */ fdblks_delta = lcounter; mp->m_resblks_avail -= lcounter; } mp->m_resblks = request; if (fdblks_delta) { spin_unlock(&mp->m_sb_lock); - error = xfs_mod_fdblocks(mp, fdblks_delta, 0); + xfs_add_fdblocks(mp, fdblks_delta); spin_lock(&mp->m_sb_lock); } @@ -428,9 +426,9 @@ xfs_reserve_blocks( */ fdblks_delta = min(free, delta); spin_unlock(&mp->m_sb_lock); - error = xfs_mod_fdblocks(mp, -fdblks_delta, 0); + error = xfs_dec_fdblocks(mp, fdblks_delta, 0); if (!error) - xfs_mod_fdblocks(mp, fdblks_delta, 0); + xfs_add_fdblocks(mp, fdblks_delta); spin_lock(&mp->m_sb_lock); } out: @@ -556,24 +554,13 @@ xfs_fs_reserve_ag_blocks( /* * Free space reserved for per-AG metadata. */ -int +void xfs_fs_unreserve_ag_blocks( struct xfs_mount *mp) { xfs_agnumber_t agno; struct xfs_perag *pag; - int error = 0; - int err2; - for_each_perag(mp, agno, pag) { - err2 = xfs_ag_resv_free(pag); - if (err2 && !error) - error = err2; - } - - if (error) - xfs_warn(mp, - "Error %d freeing per-AG metadata reserve pool.", error); - - return error; + for_each_perag(mp, agno, pag) + xfs_ag_resv_free(pag); } diff --git a/fs/xfs/xfs_fsops.h b/fs/xfs/xfs_fsops.h index 44457b0a05937..3e2f73bcf8314 100644 --- a/fs/xfs/xfs_fsops.h +++ b/fs/xfs/xfs_fsops.h @@ -12,6 +12,6 @@ int xfs_reserve_blocks(struct xfs_mount *mp, uint64_t request); int xfs_fs_goingdown(struct xfs_mount *mp, uint32_t inflags); int xfs_fs_reserve_ag_blocks(struct xfs_mount *mp); -int xfs_fs_unreserve_ag_blocks(struct xfs_mount *mp); +void xfs_fs_unreserve_ag_blocks(struct xfs_mount *mp); #endif /* __XFS_FSOPS_H__ */ diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index d37ba10f5fa33..b9df7cc9c473d 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -1136,16 +1136,44 @@ xfs_fs_writable( return true; } -/* Adjust m_fdblocks or m_frextents. */ +void +xfs_add_freecounter( + struct xfs_mount *mp, + struct percpu_counter *counter, + uint64_t delta) +{ + bool has_resv_pool = (counter == &mp->m_fdblocks); + uint64_t res_used; + + /* + * If the reserve pool is depleted, put blocks back into it first. + * Most of the time the pool is full. + */ + if (!has_resv_pool || mp->m_resblks == mp->m_resblks_avail) { + percpu_counter_add(counter, delta); + return; + } + + spin_lock(&mp->m_sb_lock); + res_used = mp->m_resblks - mp->m_resblks_avail; + if (res_used > delta) { + mp->m_resblks_avail += delta; + } else { + delta -= res_used; + mp->m_resblks_avail = mp->m_resblks; + percpu_counter_add(counter, delta); + } + spin_unlock(&mp->m_sb_lock); +} + int -xfs_mod_freecounter( +xfs_dec_freecounter( struct xfs_mount *mp, struct percpu_counter *counter, - int64_t delta, + uint64_t delta, bool rsvd) { int64_t lcounter; - long long res_used; uint64_t set_aside = 0; s32 batch; bool has_resv_pool; @@ -1155,31 +1183,6 @@ xfs_mod_freecounter( if (rsvd) ASSERT(has_resv_pool); - if (delta > 0) { - /* - * If the reserve pool is depleted, put blocks back into it - * first. Most of the time the pool is full. - */ - if (likely(!has_resv_pool || - mp->m_resblks == mp->m_resblks_avail)) { - percpu_counter_add(counter, delta); - return 0; - } - - spin_lock(&mp->m_sb_lock); - res_used = (long long)(mp->m_resblks - mp->m_resblks_avail); - - if (res_used > delta) { - mp->m_resblks_avail += delta; - } else { - delta -= res_used; - mp->m_resblks_avail = mp->m_resblks; - percpu_counter_add(counter, delta); - } - spin_unlock(&mp->m_sb_lock); - return 0; - } - /* * Taking blocks away, need to be more accurate the closer we * are to zero. @@ -1207,7 +1210,7 @@ xfs_mod_freecounter( */ if (has_resv_pool) set_aside = xfs_fdblocks_unavailable(mp); - percpu_counter_add_batch(counter, delta, batch); + percpu_counter_add_batch(counter, -((int64_t)delta), batch); if (__percpu_counter_compare(counter, set_aside, XFS_FDBLOCKS_BATCH) >= 0) { /* we had space! */ @@ -1219,11 +1222,11 @@ xfs_mod_freecounter( * that took us to ENOSPC. */ spin_lock(&mp->m_sb_lock); - percpu_counter_add(counter, -delta); + percpu_counter_add(counter, delta); if (!has_resv_pool || !rsvd) goto fdblocks_enospc; - lcounter = (long long)mp->m_resblks_avail + delta; + lcounter = (long long)mp->m_resblks_avail - delta; if (lcounter >= 0) { mp->m_resblks_avail = lcounter; spin_unlock(&mp->m_sb_lock); diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index 283a5a26db317..3a985c2ff06fe 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -562,19 +562,30 @@ xfs_fdblocks_unavailable( return mp->m_alloc_set_aside + atomic64_read(&mp->m_allocbt_blks); } -int xfs_mod_freecounter(struct xfs_mount *mp, struct percpu_counter *counter, - int64_t delta, bool rsvd); +int xfs_dec_freecounter(struct xfs_mount *mp, struct percpu_counter *counter, + uint64_t delta, bool rsvd); +void xfs_add_freecounter(struct xfs_mount *mp, struct percpu_counter *counter, + uint64_t delta); -static inline int -xfs_mod_fdblocks(struct xfs_mount *mp, int64_t delta, bool reserved) +static inline int xfs_dec_fdblocks(struct xfs_mount *mp, uint64_t delta, + bool reserved) { - return xfs_mod_freecounter(mp, &mp->m_fdblocks, delta, reserved); + return xfs_dec_freecounter(mp, &mp->m_fdblocks, delta, reserved); } -static inline int -xfs_mod_frextents(struct xfs_mount *mp, int64_t delta) +static inline void xfs_add_fdblocks(struct xfs_mount *mp, uint64_t delta) { - return xfs_mod_freecounter(mp, &mp->m_frextents, delta, false); + xfs_add_freecounter(mp, &mp->m_fdblocks, delta); +} + +static inline int xfs_dec_frextents(struct xfs_mount *mp, uint64_t delta) +{ + return xfs_dec_freecounter(mp, &mp->m_frextents, delta, false); +} + +static inline void xfs_add_frextents(struct xfs_mount *mp, uint64_t delta) +{ + xfs_add_freecounter(mp, &mp->m_frextents, delta); } extern int xfs_readsb(xfs_mount_t *, int); diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 02c1a4aeaa4c3..ed4a2f8c1936f 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -1882,11 +1882,7 @@ xfs_remount_ro( xfs_inodegc_stop(mp); /* Free the per-AG metadata reservation pool. */ - error = xfs_fs_unreserve_ag_blocks(mp); - if (error) { - xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); - return error; - } + xfs_fs_unreserve_ag_blocks(mp); /* * Before we sync the metadata, we need to free up the reserve block diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index 0f844396754d7..08df5bb70a6c7 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -3069,7 +3069,6 @@ DEFINE_AG_RESV_EVENT(xfs_ag_resv_free_extent); DEFINE_AG_RESV_EVENT(xfs_ag_resv_critical); DEFINE_AG_RESV_EVENT(xfs_ag_resv_needed); -DEFINE_AG_ERROR_EVENT(xfs_ag_resv_free_error); DEFINE_AG_ERROR_EVENT(xfs_ag_resv_init_error); /* refcount tracepoint classes */ diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 95921ad935477..828da4ac4316d 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -163,7 +163,7 @@ xfs_trans_reserve( * fail if the count would go below zero. */ if (blocks > 0) { - error = xfs_mod_fdblocks(mp, -((int64_t)blocks), rsvd); + error = xfs_dec_fdblocks(mp, blocks, rsvd); if (error != 0) return -ENOSPC; tp->t_blk_res += blocks; @@ -210,7 +210,7 @@ xfs_trans_reserve( * fail if the count would go below zero. */ if (rtextents > 0) { - error = xfs_mod_frextents(mp, -((int64_t)rtextents)); + error = xfs_dec_frextents(mp, rtextents); if (error) { error = -ENOSPC; goto undo_log; @@ -234,7 +234,7 @@ xfs_trans_reserve( undo_blocks: if (blocks > 0) { - xfs_mod_fdblocks(mp, (int64_t)blocks, rsvd); + xfs_add_fdblocks(mp, blocks); tp->t_blk_res = 0; } return error; @@ -593,12 +593,10 @@ xfs_trans_unreserve_and_mod_sb( struct xfs_trans *tp) { struct xfs_mount *mp = tp->t_mountp; - bool rsvd = (tp->t_flags & XFS_TRANS_RESERVE) != 0; int64_t blkdelta = tp->t_blk_res; int64_t rtxdelta = tp->t_rtx_res; int64_t idelta = 0; int64_t ifreedelta = 0; - int error; /* * Calculate the deltas. @@ -631,10 +629,8 @@ xfs_trans_unreserve_and_mod_sb( } /* apply the per-cpu counters */ - if (blkdelta) { - error = xfs_mod_fdblocks(mp, blkdelta, rsvd); - ASSERT(!error); - } + if (blkdelta) + xfs_add_fdblocks(mp, blkdelta); if (idelta) percpu_counter_add_batch(&mp->m_icount, idelta, @@ -643,10 +639,8 @@ xfs_trans_unreserve_and_mod_sb( if (ifreedelta) percpu_counter_add(&mp->m_ifree, ifreedelta); - if (rtxdelta) { - error = xfs_mod_frextents(mp, rtxdelta); - ASSERT(!error); - } + if (rtxdelta) + xfs_add_frextents(mp, rtxdelta); if (!(tp->t_flags & XFS_TRANS_SB_DIRTY)) return; @@ -682,7 +676,6 @@ xfs_trans_unreserve_and_mod_sb( */ ASSERT(mp->m_sb.sb_imax_pct >= 0); ASSERT(mp->m_sb.sb_rextslog >= 0); - return; } /* Add the given log item to the transaction's list of log items. */ @@ -1301,9 +1294,9 @@ xfs_trans_reserve_more_inode( return 0; /* Quota failed, give back the new reservation. */ - xfs_mod_fdblocks(mp, dblocks, tp->t_flags & XFS_TRANS_RESERVE); + xfs_add_fdblocks(mp, dblocks); tp->t_blk_res -= dblocks; - xfs_mod_frextents(mp, rtx); + xfs_add_frextents(mp, rtx); tp->t_rtx_res -= rtx; return error; } From dc1b17a25c321c8f1b4f90f9d6f8afb1d132b69c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Apr 2024 13:20:13 +0200 Subject: [PATCH 0356/1702] xfs: reinstate RT support in xfs_bmapi_reserve_delalloc Allocate data blocks for RT inodes using xfs_dec_frextents. While at it optimize the data device case by doing only a single xfs_dec_fdblocks call for the extent itself and the indirect blocks. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 0311a98df1c59..f14224ff4d561 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4069,6 +4069,7 @@ xfs_bmapi_reserve_delalloc( struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); xfs_extlen_t alen; xfs_extlen_t indlen; + uint64_t fdblocks; int error; xfs_fileoff_t aoff = off; @@ -4111,14 +4112,18 @@ xfs_bmapi_reserve_delalloc( indlen = (xfs_extlen_t)xfs_bmap_worst_indlen(ip, alen); ASSERT(indlen > 0); - error = xfs_dec_fdblocks(mp, alen, false); - if (error) - goto out_unreserve_quota; + fdblocks = indlen; + if (XFS_IS_REALTIME_INODE(ip)) { + error = xfs_dec_frextents(mp, xfs_rtb_to_rtx(mp, alen)); + if (error) + goto out_unreserve_quota; + } else { + fdblocks += alen; + } - error = xfs_dec_fdblocks(mp, indlen, false); + error = xfs_dec_fdblocks(mp, fdblocks, false); if (error) - goto out_unreserve_blocks; - + goto out_unreserve_frextents; ip->i_delayed_blks += alen; xfs_mod_delalloc(ip->i_mount, alen + indlen); @@ -4142,8 +4147,9 @@ xfs_bmapi_reserve_delalloc( return 0; -out_unreserve_blocks: - xfs_add_fdblocks(mp, alen); +out_unreserve_frextents: + if (XFS_IS_REALTIME_INODE(ip)) + xfs_add_frextents(mp, xfs_rtb_to_rtx(mp, alen)); out_unreserve_quota: if (XFS_IS_QUOTA_ON(mp)) xfs_quota_unreserve_blkres(ip, alen); From 7e77d57a1fea5f6bfe166210385ba9f227a606d1 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Apr 2024 13:20:14 +0200 Subject: [PATCH 0357/1702] xfs: cleanup fdblock/frextent accounting in xfs_bmap_del_extent_delay The code to account fdblocks and frextents in xfs_bmap_del_extent_delay is a bit weird in that it accounts frextents before the iext tree manipulations and fdblocks after it. Given that the iext tree manipulations cannot fail currently that's not really a problem, but still odd. Move the frextent manipulation to the end, and use a fdblocks variable to account of the unconditional indirect blocks and the data blocks only freed for !RT. This prepares for following updates in the area and already makes the code more readable. Also remove the !isrt assert given that this code clearly handles rt extents correctly, and we'll soon reinstate delalloc support for RT inodes. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index f14224ff4d561..d7a5ce2e4305a 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4919,6 +4919,7 @@ xfs_bmap_del_extent_delay( xfs_fileoff_t del_endoff, got_endoff; xfs_filblks_t got_indlen, new_indlen, stolen; uint32_t state = xfs_bmap_fork_to_state(whichfork); + uint64_t fdblocks; int error = 0; bool isrt; @@ -4934,15 +4935,11 @@ xfs_bmap_del_extent_delay( ASSERT(got->br_startoff <= del->br_startoff); ASSERT(got_endoff >= del_endoff); - if (isrt) - xfs_add_frextents(mp, xfs_rtb_to_rtx(mp, del->br_blockcount)); - /* * Update the inode delalloc counter now and wait to update the * sb counters as we might have to borrow some blocks for the * indirect block accounting. */ - ASSERT(!isrt); error = xfs_quota_unreserve_blkres(ip, del->br_blockcount); if (error) return error; @@ -5019,12 +5016,15 @@ xfs_bmap_del_extent_delay( ASSERT(da_old >= da_new); da_diff = da_old - da_new; - if (!isrt) - da_diff += del->br_blockcount; - if (da_diff) { - xfs_add_fdblocks(mp, da_diff); - xfs_mod_delalloc(mp, -da_diff); - } + fdblocks = da_diff; + + if (isrt) + xfs_add_frextents(mp, xfs_rtb_to_rtx(mp, del->br_blockcount)); + else + fdblocks += del->br_blockcount; + + xfs_add_fdblocks(mp, fdblocks); + xfs_mod_delalloc(mp, -(int64_t)fdblocks); return error; } From 7099bd0f243fa7511de6e95b0b8807ba7d3e5204 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Apr 2024 13:20:15 +0200 Subject: [PATCH 0358/1702] xfs: support RT inodes in xfs_mod_delalloc To prepare for re-enabling delalloc on RT devices, track the data blocks (which use the RT device when the inode sits on it) and the indirect blocks (which don't) separately to xfs_mod_delalloc, and add a new percpu counter to also track the RT delalloc blocks. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 12 ++++++------ fs/xfs/scrub/fscounters.c | 6 +++++- fs/xfs/scrub/fscounters.h | 1 + fs/xfs/scrub/fscounters_repair.c | 12 +++++++++++- fs/xfs/xfs_mount.c | 18 +++++++++++++++--- fs/xfs/xfs_mount.h | 9 ++++++++- fs/xfs/xfs_super.c | 11 ++++++++++- 7 files changed, 56 insertions(+), 13 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index d7a5ce2e4305a..a2b715450ec23 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -1977,7 +1977,7 @@ xfs_bmap_add_extent_delay_real( } if (da_new != da_old) - xfs_mod_delalloc(mp, (int64_t)da_new - da_old); + xfs_mod_delalloc(bma->ip, 0, (int64_t)da_new - da_old); if (bma->cur) { da_new += bma->cur->bc_bmap.allocated; @@ -2696,7 +2696,7 @@ xfs_bmap_add_extent_hole_delay( /* * Nothing to do for disk quota accounting here. */ - xfs_mod_delalloc(ip->i_mount, (int64_t)newlen - oldlen); + xfs_mod_delalloc(ip, 0, (int64_t)newlen - oldlen); } } @@ -3373,7 +3373,7 @@ xfs_bmap_alloc_account( * yet. */ if (ap->wasdel) { - xfs_mod_delalloc(ap->ip->i_mount, -(int64_t)ap->length); + xfs_mod_delalloc(ap->ip, -(int64_t)ap->length, 0); return; } @@ -3397,7 +3397,7 @@ xfs_bmap_alloc_account( xfs_trans_log_inode(ap->tp, ap->ip, XFS_ILOG_CORE); if (ap->wasdel) { ap->ip->i_delayed_blks -= ap->length; - xfs_mod_delalloc(ap->ip->i_mount, -(int64_t)ap->length); + xfs_mod_delalloc(ap->ip, -(int64_t)ap->length, 0); fld = isrt ? XFS_TRANS_DQ_DELRTBCOUNT : XFS_TRANS_DQ_DELBCOUNT; } else { fld = isrt ? XFS_TRANS_DQ_RTBCOUNT : XFS_TRANS_DQ_BCOUNT; @@ -4126,7 +4126,7 @@ xfs_bmapi_reserve_delalloc( goto out_unreserve_frextents; ip->i_delayed_blks += alen; - xfs_mod_delalloc(ip->i_mount, alen + indlen); + xfs_mod_delalloc(ip, alen, indlen); got->br_startoff = aoff; got->br_startblock = nullstartblock(indlen); @@ -5024,7 +5024,7 @@ xfs_bmap_del_extent_delay( fdblocks += del->br_blockcount; xfs_add_fdblocks(mp, fdblocks); - xfs_mod_delalloc(mp, -(int64_t)fdblocks); + xfs_mod_delalloc(ip, -(int64_t)del->br_blockcount, -da_diff); return error; } diff --git a/fs/xfs/scrub/fscounters.c b/fs/xfs/scrub/fscounters.c index b662e52fe6956..1d3e98346933e 100644 --- a/fs/xfs/scrub/fscounters.c +++ b/fs/xfs/scrub/fscounters.c @@ -412,6 +412,7 @@ xchk_fscount_count_frextents( int error; fsc->frextents = 0; + fsc->frextents_delayed = 0; if (!xfs_has_realtime(mp)) return 0; @@ -423,6 +424,8 @@ xchk_fscount_count_frextents( goto out_unlock; } + fsc->frextents_delayed = percpu_counter_sum(&mp->m_delalloc_rtextents); + out_unlock: xfs_rtbitmap_unlock_shared(sc->mp, XFS_RBMLOCK_BITMAP); return error; @@ -434,6 +437,7 @@ xchk_fscount_count_frextents( struct xchk_fscounters *fsc) { fsc->frextents = 0; + fsc->frextents_delayed = 0; return 0; } #endif /* CONFIG_XFS_RT */ @@ -593,7 +597,7 @@ xchk_fscounters( } if (!xchk_fscount_within_range(sc, frextents, &mp->m_frextents, - fsc->frextents)) { + fsc->frextents - fsc->frextents_delayed)) { if (fsc->frozen) xchk_set_corrupt(sc); else diff --git a/fs/xfs/scrub/fscounters.h b/fs/xfs/scrub/fscounters.h index 461a13d25f4b3..bcf56e1c36f91 100644 --- a/fs/xfs/scrub/fscounters.h +++ b/fs/xfs/scrub/fscounters.h @@ -12,6 +12,7 @@ struct xchk_fscounters { uint64_t ifree; uint64_t fdblocks; uint64_t frextents; + uint64_t frextents_delayed; unsigned long long icount_min; unsigned long long icount_max; bool frozen; diff --git a/fs/xfs/scrub/fscounters_repair.c b/fs/xfs/scrub/fscounters_repair.c index 94cdb852bee46..469bf645dbea5 100644 --- a/fs/xfs/scrub/fscounters_repair.c +++ b/fs/xfs/scrub/fscounters_repair.c @@ -65,7 +65,17 @@ xrep_fscounters( percpu_counter_set(&mp->m_icount, fsc->icount); percpu_counter_set(&mp->m_ifree, fsc->ifree); percpu_counter_set(&mp->m_fdblocks, fsc->fdblocks); - percpu_counter_set(&mp->m_frextents, fsc->frextents); + + /* + * Online repair is only supported on v5 file systems, which require + * lazy sb counters and thus no update of sb_fdblocks here. But as of + * now we don't support lazy counting sb_frextents yet, and thus need + * to also update it directly here. And for that we need to keep + * track of the delalloc reservations separately, as they are are + * subtracted from m_frextents, but not included in sb_frextents. + */ + percpu_counter_set(&mp->m_frextents, + fsc->frextents - fsc->frextents_delayed); mp->m_sb.sb_frextents = fsc->frextents; return 0; diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index b9df7cc9c473d..b2e5653b5200c 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -34,6 +34,7 @@ #include "xfs_health.h" #include "xfs_trace.h" #include "xfs_ag.h" +#include "xfs_rtbitmap.h" #include "scrub/stats.h" static DEFINE_MUTEX(xfs_uuid_table_mutex); @@ -1408,9 +1409,20 @@ xfs_clear_incompat_log_features( #define XFS_DELALLOC_BATCH (4096) void xfs_mod_delalloc( - struct xfs_mount *mp, - int64_t delta) + struct xfs_inode *ip, + int64_t data_delta, + int64_t ind_delta) { - percpu_counter_add_batch(&mp->m_delalloc_blks, delta, + struct xfs_mount *mp = ip->i_mount; + + if (XFS_IS_REALTIME_INODE(ip)) { + percpu_counter_add_batch(&mp->m_delalloc_rtextents, + xfs_rtb_to_rtx(mp, data_delta), + XFS_DELALLOC_BATCH); + if (!ind_delta) + return; + data_delta = 0; + } + percpu_counter_add_batch(&mp->m_delalloc_blks, data_delta + ind_delta, XFS_DELALLOC_BATCH); } diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index 3a985c2ff06fe..ca6f105990a2c 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -195,6 +195,12 @@ typedef struct xfs_mount { * extents or anything related to the rt device. */ struct percpu_counter m_delalloc_blks; + + /* + * RT version of the above. + */ + struct percpu_counter m_delalloc_rtextents; + /* * Global count of allocation btree blocks in use across all AGs. Only * used when perag reservation is enabled. Helps prevent block @@ -605,6 +611,7 @@ struct xfs_error_cfg * xfs_error_get_cfg(struct xfs_mount *mp, void xfs_force_summary_recalc(struct xfs_mount *mp); int xfs_add_incompat_log_feature(struct xfs_mount *mp, uint32_t feature); bool xfs_clear_incompat_log_features(struct xfs_mount *mp); -void xfs_mod_delalloc(struct xfs_mount *mp, int64_t delta); +void xfs_mod_delalloc(struct xfs_inode *ip, int64_t data_delta, + int64_t ind_delta); #endif /* __XFS_MOUNT_H__ */ diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index ed4a2f8c1936f..e525a6c477ff9 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -1052,12 +1052,18 @@ xfs_init_percpu_counters( if (error) goto free_fdblocks; - error = percpu_counter_init(&mp->m_frextents, 0, GFP_KERNEL); + error = percpu_counter_init(&mp->m_delalloc_rtextents, 0, GFP_KERNEL); if (error) goto free_delalloc; + error = percpu_counter_init(&mp->m_frextents, 0, GFP_KERNEL); + if (error) + goto free_delalloc_rt; + return 0; +free_delalloc_rt: + percpu_counter_destroy(&mp->m_delalloc_rtextents); free_delalloc: percpu_counter_destroy(&mp->m_delalloc_blks); free_fdblocks: @@ -1086,6 +1092,9 @@ xfs_destroy_percpu_counters( percpu_counter_destroy(&mp->m_icount); percpu_counter_destroy(&mp->m_ifree); percpu_counter_destroy(&mp->m_fdblocks); + ASSERT(xfs_is_shutdown(mp) || + percpu_counter_sum(&mp->m_delalloc_rtextents) == 0); + percpu_counter_destroy(&mp->m_delalloc_rtextents); ASSERT(xfs_is_shutdown(mp) || percpu_counter_sum(&mp->m_delalloc_blks) == 0); percpu_counter_destroy(&mp->m_delalloc_blks); From 727f8431638fdb5dc9ce9c81bdcc33fb416d45ee Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Apr 2024 13:20:16 +0200 Subject: [PATCH 0359/1702] xfs: look at m_frextents in xfs_iomap_prealloc_size for RT allocations Add a check for files on the RT subvolume and use m_frextents instead of m_fdblocks to adjust the preallocation size. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/xfs_iomap.c | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 4087af7f3c9f3..bba5a0d87d038 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -28,6 +28,7 @@ #include "xfs_dquot.h" #include "xfs_reflink.h" #include "xfs_health.h" +#include "xfs_rtbitmap.h" #define XFS_ALLOC_ALIGN(mp, off) \ (((off) >> mp->m_allocsize_log) << mp->m_allocsize_log) @@ -404,6 +405,29 @@ xfs_quota_calc_throttle( } } +static int64_t +xfs_iomap_freesp( + struct percpu_counter *counter, + uint64_t low_space[XFS_LOWSP_MAX], + int *shift) +{ + int64_t freesp; + + freesp = percpu_counter_read_positive(counter); + if (freesp < low_space[XFS_LOWSP_5_PCNT]) { + *shift = 2; + if (freesp < low_space[XFS_LOWSP_4_PCNT]) + (*shift)++; + if (freesp < low_space[XFS_LOWSP_3_PCNT]) + (*shift)++; + if (freesp < low_space[XFS_LOWSP_2_PCNT]) + (*shift)++; + if (freesp < low_space[XFS_LOWSP_1_PCNT]) + (*shift)++; + } + return freesp; +} + /* * If we don't have a user specified preallocation size, dynamically increase * the preallocation size as the size of the file grows. Cap the maximum size @@ -486,18 +510,13 @@ xfs_iomap_prealloc_size( alloc_blocks = XFS_FILEOFF_MIN(roundup_pow_of_two(XFS_MAX_BMBT_EXTLEN), alloc_blocks); - freesp = percpu_counter_read_positive(&mp->m_fdblocks); - if (freesp < mp->m_low_space[XFS_LOWSP_5_PCNT]) { - shift = 2; - if (freesp < mp->m_low_space[XFS_LOWSP_4_PCNT]) - shift++; - if (freesp < mp->m_low_space[XFS_LOWSP_3_PCNT]) - shift++; - if (freesp < mp->m_low_space[XFS_LOWSP_2_PCNT]) - shift++; - if (freesp < mp->m_low_space[XFS_LOWSP_1_PCNT]) - shift++; - } + if (unlikely(XFS_IS_REALTIME_INODE(ip))) + freesp = xfs_rtx_to_rtb(mp, + xfs_iomap_freesp(&mp->m_frextents, + mp->m_low_rtexts, &shift)); + else + freesp = xfs_iomap_freesp(&mp->m_fdblocks, mp->m_low_space, + &shift); /* * Check each quota to cap the prealloc size, provide a shift value to From da2b9c3a8d2cbdeec3f13cebf4c6c86c13e1077e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Apr 2024 13:20:17 +0200 Subject: [PATCH 0360/1702] xfs: rework splitting of indirect block reservations Move the check if we have enough indirect blocks and the stealing of the deleted extent blocks out of xfs_bmap_split_indlen and into the caller to prepare for handling delayed allocation of RT extents that can't easily be stolen. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index a2b715450ec23..fe33254ca3907 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4831,31 +4831,17 @@ xfs_bmapi_remap( * ores == 1). The number of stolen blocks is returned. The availability and * subsequent accounting of stolen blocks is the responsibility of the caller. */ -static xfs_filblks_t +static void xfs_bmap_split_indlen( xfs_filblks_t ores, /* original res. */ xfs_filblks_t *indlen1, /* ext1 worst indlen */ - xfs_filblks_t *indlen2, /* ext2 worst indlen */ - xfs_filblks_t avail) /* stealable blocks */ + xfs_filblks_t *indlen2) /* ext2 worst indlen */ { xfs_filblks_t len1 = *indlen1; xfs_filblks_t len2 = *indlen2; xfs_filblks_t nres = len1 + len2; /* new total res. */ - xfs_filblks_t stolen = 0; xfs_filblks_t resfactor; - /* - * Steal as many blocks as we can to try and satisfy the worst case - * indlen for both new extents. - */ - if (ores < nres && avail) - stolen = XFS_FILBLKS_MIN(nres - ores, avail); - ores += stolen; - - /* nothing else to do if we've satisfied the new reservation */ - if (ores >= nres) - return stolen; - /* * We can't meet the total required reservation for the two extents. * Calculate the percent of the overall shortage between both extents @@ -4900,8 +4886,6 @@ xfs_bmap_split_indlen( *indlen1 = len1; *indlen2 = len2; - - return stolen; } int @@ -4917,7 +4901,7 @@ xfs_bmap_del_extent_delay( struct xfs_bmbt_irec new; int64_t da_old, da_new, da_diff = 0; xfs_fileoff_t del_endoff, got_endoff; - xfs_filblks_t got_indlen, new_indlen, stolen; + xfs_filblks_t got_indlen, new_indlen, stolen = 0; uint32_t state = xfs_bmap_fork_to_state(whichfork); uint64_t fdblocks; int error = 0; @@ -4996,8 +4980,19 @@ xfs_bmap_del_extent_delay( new_indlen = xfs_bmap_worst_indlen(ip, new.br_blockcount); WARN_ON_ONCE(!got_indlen || !new_indlen); - stolen = xfs_bmap_split_indlen(da_old, &got_indlen, &new_indlen, - del->br_blockcount); + /* + * Steal as many blocks as we can to try and satisfy the worst + * case indlen for both new extents. + */ + da_new = got_indlen + new_indlen; + if (da_new > da_old) { + stolen = XFS_FILBLKS_MIN(da_new - da_old, + del->br_blockcount); + da_old += stolen; + } + if (da_new > da_old) + xfs_bmap_split_indlen(da_old, &got_indlen, &new_indlen); + da_new = got_indlen + new_indlen; got->br_startblock = nullstartblock((int)got_indlen); @@ -5009,7 +5004,6 @@ xfs_bmap_del_extent_delay( xfs_iext_next(ifp, icur); xfs_iext_insert(ip, icur, &new, state); - da_new = got_indlen + new_indlen - stolen; del->br_blockcount -= stolen; break; } From bd1753d8c42b6bd5d9a81c81d1ce6e3affe3a59f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Apr 2024 13:20:18 +0200 Subject: [PATCH 0361/1702] xfs: stop the steal (of data blocks for RT indirect blocks) When xfs_bmap_del_extent_delay has to split an indirect block it tries to steal blocks from the the part that gets unmapped to increase the indirect block reservation that now needs to cover for two extents instead of one. This works perfectly fine on the data device, where the data and indirect blocks come from the same pool. It has no chance of working when the inode sits on the RT device. To support re-enabling delalloc for inodes on the RT device, make this behavior conditional on not being for rt extents. Note that split of delalloc extents should only happen on writeback failure, as for other kinds of hole punching we first write back all data and thus convert the delalloc reservations covering the hole to a real allocation. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index fe33254ca3907..8a1446e025e07 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4983,9 +4983,14 @@ xfs_bmap_del_extent_delay( /* * Steal as many blocks as we can to try and satisfy the worst * case indlen for both new extents. + * + * However, we can't just steal reservations from the data + * blocks if this is an RT inodes as the data and metadata + * blocks come from different pools. We'll have to live with + * under-filled indirect reservation in this case. */ da_new = got_indlen + new_indlen; - if (da_new > da_old) { + if (da_new > da_old && !isrt) { stolen = XFS_FILBLKS_MIN(da_new - da_old, del->br_blockcount); da_old += stolen; From 6a94b1acda7e7262418e23f906c12a2b08b69d12 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Apr 2024 13:20:19 +0200 Subject: [PATCH 0362/1702] xfs: reinstate delalloc for RT inodes (if sb_rextsize == 1) Commit aff3a9edb708 ("xfs: Use preallocation for inodes with extsz hints") disabled delayed allocation for all inodes with extent size hints due a data exposure problem. It turns out we fixed this data exposure problem since by always creating unwritten extents for delalloc conversions due to more data exposure problems, but the writeback path doesn't actually support extent size hints when converting delalloc these days, which probably isn't a problem given that people using the hints know what they get. However due to the way how xfs_get_extsz_hint is implemented, it always claims an extent size hint for RT inodes even if the RT extent size is a single FSB. Due to that the above commit effectively disabled delalloc support for RT inodes. Switch xfs_get_extsz_hint to return 0 for this case and work around that in a few places to reinstate delalloc support for RT inodes on file systems with an sb_rextsize of 1. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/xfs_inode.c | 3 ++- fs/xfs/xfs_iomap.c | 2 -- fs/xfs/xfs_iops.c | 2 +- fs/xfs/xfs_rtalloc.c | 2 ++ 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 2aec7ab59aeb7..3c843223b4edd 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -58,7 +58,8 @@ xfs_get_extsz_hint( return 0; if ((ip->i_diflags & XFS_DIFLAG_EXTSIZE) && ip->i_extsize) return ip->i_extsize; - if (XFS_IS_REALTIME_INODE(ip)) + if (XFS_IS_REALTIME_INODE(ip) && + ip->i_mount->m_sb.sb_rextsize > 1) return ip->i_mount->m_sb.sb_rextsize; return 0; } diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index bba5a0d87d038..9ce0f6b9df93e 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -1001,8 +1001,6 @@ xfs_buffered_write_iomap_begin( return xfs_direct_write_iomap_begin(inode, offset, count, flags, iomap, srcmap); - ASSERT(!XFS_IS_REALTIME_INODE(ip)); - error = xfs_qm_dqattach(ip); if (error) return error; diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 7f0c840f0fd2f..ad76704ab1332 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -525,7 +525,7 @@ xfs_stat_blksize( * always return the realtime extent size. */ if (XFS_IS_REALTIME_INODE(ip)) - return XFS_FSB_TO_B(mp, xfs_get_extsz_hint(ip)); + return XFS_FSB_TO_B(mp, xfs_get_extsz_hint(ip) ? : 1); /* * Allow large block sizes to be reported to userspace programs if the diff --git a/fs/xfs/xfs_rtalloc.c b/fs/xfs/xfs_rtalloc.c index 86f928d30feda..b476a876478d9 100644 --- a/fs/xfs/xfs_rtalloc.c +++ b/fs/xfs/xfs_rtalloc.c @@ -1341,6 +1341,8 @@ xfs_bmap_rtalloc( int error; align = xfs_get_extsz_hint(ap->ip); + if (!align) + align = 1; retry: error = xfs_bmap_extsize_align(mp, &ap->got, &ap->prev, align, 1, ap->eof, 0, From 72a9913a8554f5c05b20005980d3dfbf905af9f8 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 30 Mar 2024 19:08:48 +0900 Subject: [PATCH 0363/1702] parisc: vdso: remove unused C build rule in vdso32/Makefile The build rule for C is unused because 'obj-cvdso32' is not defined in this Makefile. If you add a C file into arch/parisc/kernel/vdso32/ in the future, you can revert this commit. The kernel does not keep unused code. Signed-off-by: Masahiro Yamada Acked-by: Helge Deller --- arch/parisc/kernel/vdso32/Makefile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/arch/parisc/kernel/vdso32/Makefile b/arch/parisc/kernel/vdso32/Makefile index 4459a48d23033..e45d46bf46a23 100644 --- a/arch/parisc/kernel/vdso32/Makefile +++ b/arch/parisc/kernel/vdso32/Makefile @@ -26,23 +26,18 @@ $(obj)/vdso32_wrapper.o : $(obj)/vdso32.so FORCE # Force dependency (incbin is bad) # link rule for the .so file, .lds has to be first -$(obj)/vdso32.so: $(src)/vdso32.lds $(obj-vdso32) $(obj-cvdso32) $(VDSO_LIBGCC) FORCE +$(obj)/vdso32.so: $(src)/vdso32.lds $(obj-vdso32) $(VDSO_LIBGCC) FORCE $(call if_changed,vdso32ld) # assembly rules for the .S files $(obj-vdso32): %.o: %.S FORCE $(call if_changed_dep,vdso32as) -$(obj-cvdso32): %.o: %.c FORCE - $(call if_changed_dep,vdso32cc) - # actual build commands quiet_cmd_vdso32ld = VDSO32L $@ cmd_vdso32ld = $(CROSS32CC) $(c_flags) -Wl,-T $(filter-out FORCE, $^) -o $@ quiet_cmd_vdso32as = VDSO32A $@ cmd_vdso32as = $(CROSS32CC) $(a_flags) -c -o $@ $< -quiet_cmd_vdso32cc = VDSO32C $@ - cmd_vdso32cc = $(CROSS32CC) $(c_flags) -c -fPIC -mno-fast-indirect-calls -o $@ $< # Generate VDSO offsets using helper script gen-vdsosym := $(srctree)/$(src)/gen_vdso_offsets.sh From 10f94d8fcc0880c93d7697184fe199022792a61c Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 8 Apr 2024 09:17:52 +0200 Subject: [PATCH 0364/1702] scripts/unifdef: avoid constexpr keyword Starting with c23, 'constexpr' is a keyword in C like in C++ and cannot be used as an identifier: scripts/unifdef.c:206:25: error: 'constexpr' can only be used in variable declarations 206 | static bool constexpr; /* constant #if expression */ | ^ scripts/unifdef.c:880:13: error: expected identifier or '(' 880 | constexpr = false; | ^ Rename this instance to allow changing to C23 at some point in the future. Signed-off-by: Arnd Bergmann Reviewed-By: Tony Finch Reviewed-by: Nicolas Schier Signed-off-by: Masahiro Yamada --- scripts/unifdef.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/unifdef.c b/scripts/unifdef.c index db00e3e30a59d..ff15efd6e7d74 100644 --- a/scripts/unifdef.c +++ b/scripts/unifdef.c @@ -203,7 +203,7 @@ static int depth; /* current #if nesting */ static int delcount; /* count of deleted lines */ static unsigned blankcount; /* count of blank lines */ static unsigned blankmax; /* maximum recent blankcount */ -static bool constexpr; /* constant #if expression */ +static bool constexpression; /* constant #if expression */ static bool zerosyms = true; /* to format symdepth output */ static bool firstsym; /* ditto */ @@ -819,7 +819,7 @@ static const struct ops { /* * Function for evaluating the innermost parts of expressions, * viz. !expr (expr) number defined(symbol) symbol - * We reset the constexpr flag in the last two cases. + * We reset the constexpression flag in the last two cases. */ static Linetype eval_unary(const struct ops *ops, int *valp, const char **cpp) @@ -877,7 +877,7 @@ eval_unary(const struct ops *ops, int *valp, const char **cpp) cp = skipcomment(cp); if (defparen && *cp++ != ')') return (LT_ERROR); - constexpr = false; + constexpression = false; } else if (!endsym(*cp)) { debug("eval%d symbol", ops - eval_ops); sym = findsym(cp); @@ -895,7 +895,7 @@ eval_unary(const struct ops *ops, int *valp, const char **cpp) lt = *valp ? LT_TRUE : LT_FALSE; cp = skipargs(cp); } - constexpr = false; + constexpression = false; } else { debug("eval%d bad expr", ops - eval_ops); return (LT_ERROR); @@ -955,10 +955,10 @@ ifeval(const char **cpp) int val = 0; debug("eval %s", *cpp); - constexpr = killconsts ? false : true; + constexpression = killconsts ? false : true; ret = eval_table(eval_ops, &val, cpp); debug("eval = %d", val); - return (constexpr ? LT_IF : ret == LT_ERROR ? LT_IF : ret); + return (constexpression ? LT_IF : ret == LT_ERROR ? LT_IF : ret); } /* From 5b6d8ef6f056d8130168746c5459d7a3fb494859 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Thu, 28 Mar 2024 15:00:17 +0100 Subject: [PATCH 0365/1702] kdb: Use str_plural() to fix Coccinelle warning Fixes the following Coccinelle/coccicheck warning reported by string_choices.cocci: opportunity for str_plural(days) Signed-off-by: Thorsten Blum Reviewed-by: Douglas Anderson Link: https://lore.kernel.org/r/20240328140015.388654-3-thorsten.blum@toblux.com Signed-off-by: Daniel Thompson --- kernel/debug/kdb/kdb_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/debug/kdb/kdb_main.c b/kernel/debug/kdb/kdb_main.c index d05066cb40b2e..664bae55f2c9d 100644 --- a/kernel/debug/kdb/kdb_main.c +++ b/kernel/debug/kdb/kdb_main.c @@ -2517,7 +2517,7 @@ static int kdb_summary(int argc, const char **argv) if (val.uptime > (24*60*60)) { int days = val.uptime / (24*60*60); val.uptime %= (24*60*60); - kdb_printf("%d day%s ", days, days == 1 ? "" : "s"); + kdb_printf("%d day%s ", days, str_plural(days)); } kdb_printf("%02ld:%02ld\n", val.uptime/(60*60), (val.uptime/60)%60); From 2b23b6097303ed0ba5f4bc036a1c07b6027af5c6 Mon Sep 17 00:00:00 2001 From: Bob Pearson Date: Fri, 29 Mar 2024 09:55:04 -0500 Subject: [PATCH 0366/1702] RDMA/rxe: Fix seg fault in rxe_comp_queue_pkt In rxe_comp_queue_pkt() an incoming response packet skb is enqueued to the resp_pkts queue and then a decision is made whether to run the completer task inline or schedule it. Finally the skb is dereferenced to bump a 'hw' performance counter. This is wrong because if the completer task is already running in a separate thread it may have already processed the skb and freed it which can cause a seg fault. This has been observed infrequently in testing at high scale. This patch fixes this by changing the order of enqueuing the packet until after the counter is accessed. Link: https://lore.kernel.org/r/20240329145513.35381-4-rpearsonhpe@gmail.com Signed-off-by: Bob Pearson Fixes: 0b1e5b99a48b ("IB/rxe: Add port protocol stats") Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe_comp.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c index b78b8c0856abd..c997b7cbf2a9e 100644 --- a/drivers/infiniband/sw/rxe/rxe_comp.c +++ b/drivers/infiniband/sw/rxe/rxe_comp.c @@ -131,12 +131,12 @@ void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) { int must_sched; - skb_queue_tail(&qp->resp_pkts, skb); - - must_sched = skb_queue_len(&qp->resp_pkts) > 1; + must_sched = skb_queue_len(&qp->resp_pkts) > 0; if (must_sched != 0) rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_COMPLETER_SCHED); + skb_queue_tail(&qp->resp_pkts, skb); + if (must_sched) rxe_sched_task(&qp->comp.task); else From b703374837a8f8422fa3f1edcf65505421a65a6a Mon Sep 17 00:00:00 2001 From: Bob Pearson Date: Fri, 29 Mar 2024 09:55:05 -0500 Subject: [PATCH 0367/1702] RDMA/rxe: Allow good work requests to be executed A previous commit incorrectly added an 'if(!err)' before scheduling the requester task in rxe_post_send_kernel(). But if there were send wrs successfully added to the send queue before a bad wr they might never get executed. This commit fixes this by scheduling the requester task if any wqes were successfully posted in rxe_post_send_kernel() in rxe_verbs.c. Link: https://lore.kernel.org/r/20240329145513.35381-5-rpearsonhpe@gmail.com Signed-off-by: Bob Pearson Fixes: 5bf944f24129 ("RDMA/rxe: Add error messages") Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe_verbs.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index 614581989b381..a49784e5156c5 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -888,6 +888,7 @@ static int rxe_post_send_kernel(struct rxe_qp *qp, { int err = 0; unsigned long flags; + int good = 0; spin_lock_irqsave(&qp->sq.sq_lock, flags); while (ibwr) { @@ -895,12 +896,15 @@ static int rxe_post_send_kernel(struct rxe_qp *qp, if (err) { *bad_wr = ibwr; break; + } else { + good++; } ibwr = ibwr->next; } spin_unlock_irqrestore(&qp->sq.sq_lock, flags); - if (!err) + /* kickoff processing of any posted wqes */ + if (good) rxe_sched_task(&qp->req.task); spin_lock_irqsave(&qp->state_lock, flags); From ff30e45376d2ea68e032e6430babc0df15c4fc39 Mon Sep 17 00:00:00 2001 From: Bob Pearson Date: Fri, 29 Mar 2024 09:55:06 -0500 Subject: [PATCH 0368/1702] RDMA/rxe: Remove redundant scheduling of rxe_completer In rxe_post_send_kernel() if the qp is in the error state after posting the work requests the rxe_completer() task is scheduled. But, the only way to move the qp into the error state is to call rxe_qp_error() which also schedules the rxe_completer() task to drain the queues. Calling it a second time has no effect. This commit removes the redundant call. Link: https://lore.kernel.org/r/20240329145513.35381-6-rpearsonhpe@gmail.com Signed-off-by: Bob Pearson Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe_verbs.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index a49784e5156c5..71b0f834030f3 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -907,11 +907,6 @@ static int rxe_post_send_kernel(struct rxe_qp *qp, if (good) rxe_sched_task(&qp->req.task); - spin_lock_irqsave(&qp->state_lock, flags); - if (qp_state(qp) == IB_QPS_ERR) - rxe_sched_task(&qp->comp.task); - spin_unlock_irqrestore(&qp->state_lock, flags); - return err; } From 67f57892f9b2c93a3a020109d2285232fbde8b81 Mon Sep 17 00:00:00 2001 From: Bob Pearson Date: Fri, 29 Mar 2024 09:55:07 -0500 Subject: [PATCH 0369/1702] RDMA/rxe: Merge request and complete tasks Currently the rxe driver has three work queue tasks per qp. These are the req.task, comp.task and resp.task which call rxe_requester(), rxe_completer() and rxe_responder() respectively directly or on work queues. Each of these subroutines checks to see if there is work to be performed on the send queue or on the response packet queue or the request packet queue and will run until there is no work remaining or yield the cpu and reschedule itself until there is no work remaining. This commit combines the req.task and comp.task into a single send.task and renames the resp.task to the recv.task. The combined send.task calls rxe_requester() and rxe_completer() serially and continues until all work on both the send queue and the response packet queue are done. In various benchmarks the performance is either improved or left the same. At high scale there is a significant reduction in the load on the cpu. This is the first step in combining these two tasks. Once they are serialized cross rescheduling of req.task and comp.task can be more efficiently handled by just letting the send.task continue to run. This will be done in the next several patches. Link: https://lore.kernel.org/r/20240329145513.35381-7-rpearsonhpe@gmail.com Signed-off-by: Bob Pearson Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe_comp.c | 20 +++++----- drivers/infiniband/sw/rxe/rxe_hw_counters.c | 2 +- drivers/infiniband/sw/rxe/rxe_hw_counters.h | 2 +- drivers/infiniband/sw/rxe/rxe_loc.h | 3 +- drivers/infiniband/sw/rxe/rxe_net.c | 4 +- drivers/infiniband/sw/rxe/rxe_qp.c | 44 ++++++++------------- drivers/infiniband/sw/rxe/rxe_req.c | 25 ++++++++++-- drivers/infiniband/sw/rxe/rxe_resp.c | 6 +-- drivers/infiniband/sw/rxe/rxe_verbs.c | 6 +-- drivers/infiniband/sw/rxe/rxe_verbs.h | 6 +-- 10 files changed, 63 insertions(+), 55 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c index c997b7cbf2a9e..ea64a25fe8763 100644 --- a/drivers/infiniband/sw/rxe/rxe_comp.c +++ b/drivers/infiniband/sw/rxe/rxe_comp.c @@ -122,7 +122,7 @@ void retransmit_timer(struct timer_list *t) spin_lock_irqsave(&qp->state_lock, flags); if (qp->valid) { qp->comp.timeout = 1; - rxe_sched_task(&qp->comp.task); + rxe_sched_task(&qp->send_task); } spin_unlock_irqrestore(&qp->state_lock, flags); } @@ -133,14 +133,14 @@ void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) must_sched = skb_queue_len(&qp->resp_pkts) > 0; if (must_sched != 0) - rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_COMPLETER_SCHED); + rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_SENDER_SCHED); skb_queue_tail(&qp->resp_pkts, skb); if (must_sched) - rxe_sched_task(&qp->comp.task); + rxe_sched_task(&qp->send_task); else - rxe_run_task(&qp->comp.task); + rxe_run_task(&qp->send_task); } static inline enum comp_state get_wqe(struct rxe_qp *qp, @@ -325,7 +325,7 @@ static inline enum comp_state check_ack(struct rxe_qp *qp, qp->comp.psn = pkt->psn; if (qp->req.wait_psn) { qp->req.wait_psn = 0; - rxe_sched_task(&qp->req.task); + rxe_sched_task(&qp->send_task); } } return COMPST_ERROR_RETRY; @@ -476,7 +476,7 @@ static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe) */ if (qp->req.wait_fence) { qp->req.wait_fence = 0; - rxe_sched_task(&qp->req.task); + rxe_sched_task(&qp->send_task); } } @@ -515,7 +515,7 @@ static inline enum comp_state complete_ack(struct rxe_qp *qp, if (qp->req.need_rd_atomic) { qp->comp.timeout_retry = 0; qp->req.need_rd_atomic = 0; - rxe_sched_task(&qp->req.task); + rxe_sched_task(&qp->send_task); } } @@ -541,7 +541,7 @@ static inline enum comp_state complete_wqe(struct rxe_qp *qp, if (qp->req.wait_psn) { qp->req.wait_psn = 0; - rxe_sched_task(&qp->req.task); + rxe_sched_task(&qp->send_task); } } @@ -737,7 +737,7 @@ int rxe_completer(struct rxe_qp *qp) if (qp->req.wait_psn) { qp->req.wait_psn = 0; - rxe_sched_task(&qp->req.task); + rxe_sched_task(&qp->send_task); } state = COMPST_DONE; @@ -792,7 +792,7 @@ int rxe_completer(struct rxe_qp *qp) RXE_CNT_COMP_RETRY); qp->req.need_retry = 1; qp->comp.started_retry = 1; - rxe_sched_task(&qp->req.task); + rxe_sched_task(&qp->send_task); } goto done; diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.c b/drivers/infiniband/sw/rxe/rxe_hw_counters.c index a012522b577a0..437917a7d8f2c 100644 --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.c +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.c @@ -14,7 +14,7 @@ static const struct rdma_stat_desc rxe_counter_descs[] = { [RXE_CNT_RCV_RNR].name = "rcvd_rnr_err", [RXE_CNT_SND_RNR].name = "send_rnr_err", [RXE_CNT_RCV_SEQ_ERR].name = "rcvd_seq_err", - [RXE_CNT_COMPLETER_SCHED].name = "ack_deferred", + [RXE_CNT_SENDER_SCHED].name = "ack_deferred", [RXE_CNT_RETRY_EXCEEDED].name = "retry_exceeded_err", [RXE_CNT_RNR_RETRY_EXCEEDED].name = "retry_rnr_exceeded_err", [RXE_CNT_COMP_RETRY].name = "completer_retry_err", diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.h b/drivers/infiniband/sw/rxe/rxe_hw_counters.h index 71f4d4fa9dc8b..051f9e1c3852c 100644 --- a/drivers/infiniband/sw/rxe/rxe_hw_counters.h +++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.h @@ -18,7 +18,7 @@ enum rxe_counters { RXE_CNT_RCV_RNR, RXE_CNT_SND_RNR, RXE_CNT_RCV_SEQ_ERR, - RXE_CNT_COMPLETER_SCHED, + RXE_CNT_SENDER_SCHED, RXE_CNT_RETRY_EXCEEDED, RXE_CNT_RNR_RETRY_EXCEEDED, RXE_CNT_COMP_RETRY, diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h index 746110898a0e6..ded46119151bb 100644 --- a/drivers/infiniband/sw/rxe/rxe_loc.h +++ b/drivers/infiniband/sw/rxe/rxe_loc.h @@ -164,7 +164,8 @@ void rxe_dealloc(struct ib_device *ib_dev); int rxe_completer(struct rxe_qp *qp); int rxe_requester(struct rxe_qp *qp); -int rxe_responder(struct rxe_qp *qp); +int rxe_sender(struct rxe_qp *qp); +int rxe_receiver(struct rxe_qp *qp); /* rxe_icrc.c */ int rxe_icrc_init(struct rxe_dev *rxe); diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index cd59666158b18..928508558df46 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -351,7 +351,7 @@ static void rxe_skb_tx_dtor(struct sk_buff *skb) if (unlikely(qp->need_req_skb && skb_out < RXE_INFLIGHT_SKBS_PER_QP_LOW)) - rxe_sched_task(&qp->req.task); + rxe_sched_task(&qp->send_task); rxe_put(qp); } @@ -443,7 +443,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, if ((qp_type(qp) != IB_QPT_RC) && (pkt->mask & RXE_END_MASK)) { pkt->wqe->state = wqe_state_done; - rxe_sched_task(&qp->comp.task); + rxe_sched_task(&qp->send_task); } rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS); diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index e3589c02013ec..c7d99063594bc 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -265,8 +265,7 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, qp->req.opcode = -1; qp->comp.opcode = -1; - rxe_init_task(&qp->req.task, qp, rxe_requester); - rxe_init_task(&qp->comp.task, qp, rxe_completer); + rxe_init_task(&qp->send_task, qp, rxe_sender); qp->qp_timeout_jiffies = 0; /* Can't be set for UD/UC in modify_qp */ if (init->qp_type == IB_QPT_RC) { @@ -337,7 +336,7 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp, return err; } - rxe_init_task(&qp->resp.task, qp, rxe_responder); + rxe_init_task(&qp->recv_task, qp, rxe_receiver); qp->resp.opcode = OPCODE_NONE; qp->resp.msn = 0; @@ -514,14 +513,12 @@ int rxe_qp_chk_attr(struct rxe_dev *rxe, struct rxe_qp *qp, static void rxe_qp_reset(struct rxe_qp *qp) { /* stop tasks from running */ - rxe_disable_task(&qp->resp.task); - rxe_disable_task(&qp->comp.task); - rxe_disable_task(&qp->req.task); + rxe_disable_task(&qp->recv_task); + rxe_disable_task(&qp->send_task); /* drain work and packet queuesc */ - rxe_requester(qp); - rxe_completer(qp); - rxe_responder(qp); + rxe_sender(qp); + rxe_receiver(qp); if (qp->rq.queue) rxe_queue_reset(qp->rq.queue); @@ -548,9 +545,8 @@ static void rxe_qp_reset(struct rxe_qp *qp) cleanup_rd_atomic_resources(qp); /* reenable tasks */ - rxe_enable_task(&qp->resp.task); - rxe_enable_task(&qp->comp.task); - rxe_enable_task(&qp->req.task); + rxe_enable_task(&qp->recv_task); + rxe_enable_task(&qp->send_task); } /* move the qp to the error state */ @@ -562,9 +558,8 @@ void rxe_qp_error(struct rxe_qp *qp) qp->attr.qp_state = IB_QPS_ERR; /* drain work and packet queues */ - rxe_sched_task(&qp->resp.task); - rxe_sched_task(&qp->comp.task); - rxe_sched_task(&qp->req.task); + rxe_sched_task(&qp->recv_task); + rxe_sched_task(&qp->send_task); spin_unlock_irqrestore(&qp->state_lock, flags); } @@ -575,8 +570,7 @@ static void rxe_qp_sqd(struct rxe_qp *qp, struct ib_qp_attr *attr, spin_lock_irqsave(&qp->state_lock, flags); qp->attr.sq_draining = 1; - rxe_sched_task(&qp->comp.task); - rxe_sched_task(&qp->req.task); + rxe_sched_task(&qp->send_task); spin_unlock_irqrestore(&qp->state_lock, flags); } @@ -821,19 +815,15 @@ static void rxe_qp_do_cleanup(struct work_struct *work) del_timer_sync(&qp->rnr_nak_timer); } - if (qp->resp.task.func) - rxe_cleanup_task(&qp->resp.task); + if (qp->recv_task.func) + rxe_cleanup_task(&qp->recv_task); - if (qp->req.task.func) - rxe_cleanup_task(&qp->req.task); - - if (qp->comp.task.func) - rxe_cleanup_task(&qp->comp.task); + if (qp->send_task.func) + rxe_cleanup_task(&qp->send_task); /* flush out any receive wr's or pending requests */ - rxe_requester(qp); - rxe_completer(qp); - rxe_responder(qp); + rxe_sender(qp); + rxe_receiver(qp); if (qp->sq.queue) rxe_queue_cleanup(qp->sq.queue); diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index d8c41fd626a94..31a611ced3c5e 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -108,7 +108,7 @@ void rnr_nak_timer(struct timer_list *t) /* request a send queue retry */ qp->req.need_retry = 1; qp->req.wait_for_rnr_timer = 0; - rxe_sched_task(&qp->req.task); + rxe_sched_task(&qp->send_task); } spin_unlock_irqrestore(&qp->state_lock, flags); } @@ -659,7 +659,7 @@ static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe) * which can lead to a deadlock. So go ahead and complete * it now. */ - rxe_sched_task(&qp->comp.task); + rxe_sched_task(&qp->send_task); return 0; } @@ -786,7 +786,7 @@ int rxe_requester(struct rxe_qp *qp) qp->req.wqe_index); wqe->state = wqe_state_done; wqe->status = IB_WC_SUCCESS; - rxe_sched_task(&qp->comp.task); + rxe_sched_task(&qp->send_task); goto done; } payload = mtu; @@ -855,7 +855,7 @@ int rxe_requester(struct rxe_qp *qp) */ qp->need_req_skb = 1; - rxe_sched_task(&qp->req.task); + rxe_sched_task(&qp->send_task); goto exit; } @@ -878,3 +878,20 @@ int rxe_requester(struct rxe_qp *qp) out: return ret; } + +int rxe_sender(struct rxe_qp *qp) +{ + int req_ret; + int comp_ret; + + /* process the send queue */ + req_ret = rxe_requester(qp); + + /* process the response queue */ + comp_ret = rxe_completer(qp); + + /* exit the task loop if both requester and completer + * are ready + */ + return (req_ret && comp_ret) ? -EAGAIN : 0; +} diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index 963382f625d71..3ce7a32b5dcf8 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -58,9 +58,9 @@ void rxe_resp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) (skb_queue_len(&qp->req_pkts) > 1); if (must_sched) - rxe_sched_task(&qp->resp.task); + rxe_sched_task(&qp->recv_task); else - rxe_run_task(&qp->resp.task); + rxe_run_task(&qp->recv_task); } static inline enum resp_states get_req(struct rxe_qp *qp, @@ -1485,7 +1485,7 @@ static void flush_recv_queue(struct rxe_qp *qp, bool notify) qp->resp.wqe = NULL; } -int rxe_responder(struct rxe_qp *qp) +int rxe_receiver(struct rxe_qp *qp) { struct rxe_dev *rxe = to_rdev(qp->ibqp.device); enum resp_states state; diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index 71b0f834030f3..d07f7bd3b2ae2 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -905,7 +905,7 @@ static int rxe_post_send_kernel(struct rxe_qp *qp, /* kickoff processing of any posted wqes */ if (good) - rxe_sched_task(&qp->req.task); + rxe_sched_task(&qp->send_task); return err; } @@ -935,7 +935,7 @@ static int rxe_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, if (qp->is_user) { /* Utilize process context to do protocol processing */ - rxe_run_task(&qp->req.task); + rxe_run_task(&qp->send_task); } else { err = rxe_post_send_kernel(qp, wr, bad_wr); if (err) @@ -1045,7 +1045,7 @@ static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, spin_lock_irqsave(&qp->state_lock, flags); if (qp_state(qp) == IB_QPS_ERR) - rxe_sched_task(&qp->resp.task); + rxe_sched_task(&qp->recv_task); spin_unlock_irqrestore(&qp->state_lock, flags); return err; diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index ccb9d19ffe8ab..af8939b8c7a11 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -113,7 +113,6 @@ struct rxe_req_info { int need_retry; int wait_for_rnr_timer; int noack_pkts; - struct rxe_task task; }; struct rxe_comp_info { @@ -124,7 +123,6 @@ struct rxe_comp_info { int started_retry; u32 retry_cnt; u32 rnr_retry; - struct rxe_task task; }; enum rdatm_res_state { @@ -196,7 +194,6 @@ struct rxe_resp_info { unsigned int res_head; unsigned int res_tail; struct resp_res *res; - struct rxe_task task; }; struct rxe_qp { @@ -229,6 +226,9 @@ struct rxe_qp { struct sk_buff_head req_pkts; struct sk_buff_head resp_pkts; + struct rxe_task send_task; + struct rxe_task recv_task; + struct rxe_req_info req; struct rxe_comp_info comp; struct rxe_resp_info resp; From cd8aaddf0d6dbd4798d2de2f4e1cd62a91ac62f0 Mon Sep 17 00:00:00 2001 From: Bob Pearson Date: Fri, 29 Mar 2024 09:55:08 -0500 Subject: [PATCH 0370/1702] RDMA/rxe: Remove save/rollback_state in rxe_requester Now that req.task and comp.task are merged it is no longer necessary to call save_state() before calling rxe_xmit_pkt() and rollback_state() if rxe_xmit_pkt() fails. This was done originally to prevent races between rxe_completer() and rxe_requester() which now cannot happen. Link: https://lore.kernel.org/r/20240329145513.35381-8-rpearsonhpe@gmail.com Signed-off-by: Bob Pearson Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe_req.c | 40 ++--------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 31a611ced3c5e..e20462c3040d1 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -573,30 +573,6 @@ static void update_wqe_psn(struct rxe_qp *qp, qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK; } -static void save_state(struct rxe_send_wqe *wqe, - struct rxe_qp *qp, - struct rxe_send_wqe *rollback_wqe, - u32 *rollback_psn) -{ - rollback_wqe->state = wqe->state; - rollback_wqe->first_psn = wqe->first_psn; - rollback_wqe->last_psn = wqe->last_psn; - rollback_wqe->dma = wqe->dma; - *rollback_psn = qp->req.psn; -} - -static void rollback_state(struct rxe_send_wqe *wqe, - struct rxe_qp *qp, - struct rxe_send_wqe *rollback_wqe, - u32 rollback_psn) -{ - wqe->state = rollback_wqe->state; - wqe->first_psn = rollback_wqe->first_psn; - wqe->last_psn = rollback_wqe->last_psn; - wqe->dma = rollback_wqe->dma; - qp->req.psn = rollback_psn; -} - static void update_state(struct rxe_qp *qp, struct rxe_pkt_info *pkt) { qp->req.opcode = pkt->opcode; @@ -676,8 +652,6 @@ int rxe_requester(struct rxe_qp *qp) int opcode; int err; int ret; - struct rxe_send_wqe rollback_wqe; - u32 rollback_psn; struct rxe_queue *q = qp->sq.queue; struct rxe_ah *ah; struct rxe_av *av; @@ -799,9 +773,6 @@ int rxe_requester(struct rxe_qp *qp) pkt.mask = rxe_opcode[opcode].mask; pkt.wqe = wqe; - /* save wqe state before we build and send packet */ - save_state(wqe, qp, &rollback_wqe, &rollback_psn); - av = rxe_get_av(&pkt, &ah); if (unlikely(!av)) { rxe_dbg_qp(qp, "Failed no address vector\n"); @@ -834,10 +805,6 @@ int rxe_requester(struct rxe_qp *qp) if (ah) rxe_put(ah); - /* update wqe state as though we had sent it */ - update_wqe_state(qp, wqe, &pkt); - update_wqe_psn(qp, wqe, &pkt, payload); - err = rxe_xmit_packet(qp, &pkt, skb); if (err) { if (err != -EAGAIN) { @@ -845,11 +812,6 @@ int rxe_requester(struct rxe_qp *qp) goto err; } - /* the packet was dropped so reset wqe to the state - * before we sent it so we can try to resend - */ - rollback_state(wqe, qp, &rollback_wqe, rollback_psn); - /* force a delay until the dropped packet is freed and * the send queue is drained below the low water mark */ @@ -859,6 +821,8 @@ int rxe_requester(struct rxe_qp *qp) goto exit; } + update_wqe_state(qp, wqe, &pkt); + update_wqe_psn(qp, wqe, &pkt, payload); update_state(qp, &pkt); /* A non-zero return value will cause rxe_do_task to From 4891f4fed04718a642ff4a4563128699c47d8918 Mon Sep 17 00:00:00 2001 From: Bob Pearson Date: Fri, 29 Mar 2024 09:55:09 -0500 Subject: [PATCH 0371/1702] RDMA/rxe: Don't schedule rxe_completer from rxe_requester Now that rxe_completer() is always called serially after rxe_requester() there is no reason to schedule rxe_completer() from rxe_requester(). Link: https://lore.kernel.org/r/20240329145513.35381-9-rpearsonhpe@gmail.com Signed-off-by: Bob Pearson Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe_net.c | 6 ------ drivers/infiniband/sw/rxe/rxe_req.c | 9 ++------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index 928508558df46..a2fc118e7ec1f 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -440,12 +440,6 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, return err; } - if ((qp_type(qp) != IB_QPT_RC) && - (pkt->mask & RXE_END_MASK)) { - pkt->wqe->state = wqe_state_done; - rxe_sched_task(&qp->send_task); - } - rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS); goto done; diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index e20462c3040d1..34c55dee07741 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -545,6 +545,8 @@ static void update_wqe_state(struct rxe_qp *qp, if (pkt->mask & RXE_END_MASK) { if (qp_type(qp) == IB_QPT_RC) wqe->state = wqe_state_pending; + else + wqe->state = wqe_state_done; } else { wqe->state = wqe_state_processing; } @@ -631,12 +633,6 @@ static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe) wqe->status = IB_WC_SUCCESS; qp->req.wqe_index = queue_next_index(qp->sq.queue, qp->req.wqe_index); - /* There is no ack coming for local work requests - * which can lead to a deadlock. So go ahead and complete - * it now. - */ - rxe_sched_task(&qp->send_task); - return 0; } @@ -760,7 +756,6 @@ int rxe_requester(struct rxe_qp *qp) qp->req.wqe_index); wqe->state = wqe_state_done; wqe->status = IB_WC_SUCCESS; - rxe_sched_task(&qp->send_task); goto done; } payload = mtu; From 3d807a3ebc48a2e1685ebfa9d26ea2c9ceb9c53e Mon Sep 17 00:00:00 2001 From: Bob Pearson Date: Fri, 29 Mar 2024 09:55:10 -0500 Subject: [PATCH 0372/1702] RDMA/rxe: Don't call rxe_requester from rxe_completer Instead of rescheduling rxe_requester from rxe_completer() just extend the duration of rxe_sender() by one pass. Setting run_requester_again forces rxe_completer() to return 0 which will cause rxe_sender() to be called at least one more time. Link: https://lore.kernel.org/r/20240329145513.35381-10-rpearsonhpe@gmail.com Signed-off-by: Bob Pearson Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe_comp.c | 17 ++++++++++------- drivers/infiniband/sw/rxe/rxe_verbs.h | 1 + 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c index ea64a25fe8763..357c1d516efbf 100644 --- a/drivers/infiniband/sw/rxe/rxe_comp.c +++ b/drivers/infiniband/sw/rxe/rxe_comp.c @@ -325,7 +325,7 @@ static inline enum comp_state check_ack(struct rxe_qp *qp, qp->comp.psn = pkt->psn; if (qp->req.wait_psn) { qp->req.wait_psn = 0; - rxe_sched_task(&qp->send_task); + qp->req.again = 1; } } return COMPST_ERROR_RETRY; @@ -476,7 +476,7 @@ static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe) */ if (qp->req.wait_fence) { qp->req.wait_fence = 0; - rxe_sched_task(&qp->send_task); + qp->req.again = 1; } } @@ -515,7 +515,7 @@ static inline enum comp_state complete_ack(struct rxe_qp *qp, if (qp->req.need_rd_atomic) { qp->comp.timeout_retry = 0; qp->req.need_rd_atomic = 0; - rxe_sched_task(&qp->send_task); + qp->req.again = 1; } } @@ -541,7 +541,7 @@ static inline enum comp_state complete_wqe(struct rxe_qp *qp, if (qp->req.wait_psn) { qp->req.wait_psn = 0; - rxe_sched_task(&qp->send_task); + qp->req.again = 1; } } @@ -654,6 +654,8 @@ int rxe_completer(struct rxe_qp *qp) int ret; unsigned long flags; + qp->req.again = 0; + spin_lock_irqsave(&qp->state_lock, flags); if (!qp->valid || qp_state(qp) == IB_QPS_ERR || qp_state(qp) == IB_QPS_RESET) { @@ -737,7 +739,7 @@ int rxe_completer(struct rxe_qp *qp) if (qp->req.wait_psn) { qp->req.wait_psn = 0; - rxe_sched_task(&qp->send_task); + qp->req.again = 1; } state = COMPST_DONE; @@ -792,7 +794,7 @@ int rxe_completer(struct rxe_qp *qp) RXE_CNT_COMP_RETRY); qp->req.need_retry = 1; qp->comp.started_retry = 1; - rxe_sched_task(&qp->send_task); + qp->req.again = 1; } goto done; @@ -843,8 +845,9 @@ int rxe_completer(struct rxe_qp *qp) ret = 0; goto out; exit: - ret = -EAGAIN; + ret = (qp->req.again) ? 0 : -EAGAIN; out: + qp->req.again = 0; if (pkt) free_pkt(pkt); return ret; diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index af8939b8c7a11..3c1354f82283e 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -113,6 +113,7 @@ struct rxe_req_info { int need_retry; int wait_for_rnr_timer; int noack_pkts; + int again; }; struct rxe_comp_info { From 23bc06af547f2ca3b7d345e09fd8d04575406274 Mon Sep 17 00:00:00 2001 From: Bob Pearson Date: Fri, 29 Mar 2024 09:55:11 -0500 Subject: [PATCH 0373/1702] RDMA/rxe: Don't call direct between tasks Replace calls to rxe_run_task() with rxe_sched_task(). This prevents the tasks from all running on the same cpu. This change slightly reduces performance for single qp send and write benchmarks in loopback mode but greatly improves the performance with multiple qps because if run task is used all the work tends to be performed on one cpu. For actual on the wire benchmarks there is no noticeable performance change. Link: https://lore.kernel.org/r/20240329145513.35381-11-rpearsonhpe@gmail.com Signed-off-by: Bob Pearson Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe_comp.c | 13 ++----------- drivers/infiniband/sw/rxe/rxe_resp.c | 12 +----------- drivers/infiniband/sw/rxe/rxe_verbs.c | 2 +- 3 files changed, 4 insertions(+), 23 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c index 357c1d516efbf..d48af21807458 100644 --- a/drivers/infiniband/sw/rxe/rxe_comp.c +++ b/drivers/infiniband/sw/rxe/rxe_comp.c @@ -129,18 +129,9 @@ void retransmit_timer(struct timer_list *t) void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) { - int must_sched; - - must_sched = skb_queue_len(&qp->resp_pkts) > 0; - if (must_sched != 0) - rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_SENDER_SCHED); - + rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_SENDER_SCHED); skb_queue_tail(&qp->resp_pkts, skb); - - if (must_sched) - rxe_sched_task(&qp->send_task); - else - rxe_run_task(&qp->send_task); + rxe_sched_task(&qp->send_task); } static inline enum comp_state get_wqe(struct rxe_qp *qp, diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index 3ce7a32b5dcf8..c6a7fa3054fad 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -49,18 +49,8 @@ static char *resp_state_name[] = { /* rxe_recv calls here to add a request packet to the input queue */ void rxe_resp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) { - int must_sched; - struct rxe_pkt_info *pkt = SKB_TO_PKT(skb); - skb_queue_tail(&qp->req_pkts, skb); - - must_sched = (pkt->opcode == IB_OPCODE_RC_RDMA_READ_REQUEST) || - (skb_queue_len(&qp->req_pkts) > 1); - - if (must_sched) - rxe_sched_task(&qp->recv_task); - else - rxe_run_task(&qp->recv_task); + rxe_sched_task(&qp->recv_task); } static inline enum resp_states get_req(struct rxe_qp *qp, diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index d07f7bd3b2ae2..c7d4d8ab5a094 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -935,7 +935,7 @@ static int rxe_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, if (qp->is_user) { /* Utilize process context to do protocol processing */ - rxe_run_task(&qp->send_task); + rxe_sched_task(&qp->send_task); } else { err = rxe_post_send_kernel(qp, wr, bad_wr); if (err) From 8776618dbbd1b6f210b31509507e1aad461d6435 Mon Sep 17 00:00:00 2001 From: Bob Pearson Date: Fri, 29 Mar 2024 09:55:12 -0500 Subject: [PATCH 0374/1702] RDMA/rxe: Fix incorrect rxe_put in error path In rxe_send() a ref is taken on the qp to keep it alive until the kfree_skb() has a chance to call the skb destructor rxe_skb_tx_dtor() which drops the reference. If the packet has an incorrect protocol the error path just calls kfree_skb() which will call the destructor which will drop the ref. Currently the driver also calls rxe_put() which is incorrect. Additionally since the packets sent to rxe_send() are under the control of the driver and it only ever produces IPV4 or IPV6 packets the simplest fix is to remove all the code in this block. Link: https://lore.kernel.org/r/20240329145513.35381-12-rpearsonhpe@gmail.com Signed-off-by: Bob Pearson Fixes: 9eb7f8e44d13 ("IB/rxe: Move refcounting earlier in rxe_send()") Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe_net.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index a2fc118e7ec1f..d81440038f916 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -366,18 +366,10 @@ static int rxe_send(struct sk_buff *skb, struct rxe_pkt_info *pkt) rxe_get(pkt->qp); atomic_inc(&pkt->qp->skb_out); - if (skb->protocol == htons(ETH_P_IP)) { + if (skb->protocol == htons(ETH_P_IP)) err = ip_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb); - } else if (skb->protocol == htons(ETH_P_IPV6)) { + else err = ip6_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb); - } else { - rxe_dbg_qp(pkt->qp, "Unknown layer 3 protocol: %d\n", - skb->protocol); - atomic_dec(&pkt->qp->skb_out); - rxe_put(pkt->qp); - kfree_skb(skb); - return -EINVAL; - } if (unlikely(net_xmit_eval(err))) { rxe_dbg_qp(pkt->qp, "error sending packet: %d\n", err); From 55bec1c440e6852e907c47cd33fbbf63fcc5f1ba Mon Sep 17 00:00:00 2001 From: Bob Pearson Date: Fri, 29 Mar 2024 09:55:13 -0500 Subject: [PATCH 0375/1702] RDMA/rxe: Make rxe_loopback match rxe_send behavior The rxe send path currently counts the number of skbs outstanding between the rxe driver and the ethernet driver to prevent too many packets to accumulate waiting to send. This patch makes the local loopback path behave the same way. The loopback path forwards the packets to the receive path which will eventually call kfree_skb on all packets and drop the qp references. This makes the loopback path more useful for software testing. Link: https://lore.kernel.org/r/20240329145513.35381-13-rpearsonhpe@gmail.com Signed-off-by: Bob Pearson Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe_net.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index d81440038f916..d081409450a41 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -386,6 +386,12 @@ static int rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt) { memcpy(SKB_TO_PKT(skb), pkt, sizeof(*pkt)); + skb->destructor = rxe_skb_tx_dtor; + skb->sk = pkt->qp->sk->sk; + + rxe_get(pkt->qp); + atomic_inc(&pkt->qp->skb_out); + if (skb->protocol == htons(ETH_P_IP)) skb_pull(skb, sizeof(struct iphdr)); else From 9cc6290991e6cfc9a6447823275fa4ba4d902103 Mon Sep 17 00:00:00 2001 From: Bob Pearson Date: Fri, 29 Mar 2024 09:55:14 -0500 Subject: [PATCH 0376/1702] RDMA/rxe: Get rid of pkt resend on err Currently the rxe_driver detects packet drops by ip_local_out() which occur before the packet is sent on the wire and attempts to resend them. This is redundant with the usual retry mechanism which covers packets that get dropped in transit to or from the remote node. The way this is implemented is not robust since it sets need_req_skb and waits for the number of local skbs outstanding for this qp to drop below a low water mark. This is racy since the skb may be sent to the destructor before the requester can set the need_req_skb flag. This will cause a deadlock in the send path for that qp. This patch removes this mechanism since the normal retry path will correct the error and resend the packet and it makes no difference if the packet is dropped locally or later. Link: https://lore.kernel.org/r/20240329145513.35381-14-rpearsonhpe@gmail.com Signed-off-by: Bob Pearson Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe_net.c | 7 +------ drivers/infiniband/sw/rxe/rxe_req.c | 14 ++------------ 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index d081409450a41..b58eab75df978 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -371,12 +371,7 @@ static int rxe_send(struct sk_buff *skb, struct rxe_pkt_info *pkt) else err = ip6_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb); - if (unlikely(net_xmit_eval(err))) { - rxe_dbg_qp(pkt->qp, "error sending packet: %d\n", err); - return -EAGAIN; - } - - return 0; + return err; } /* fix up a send packet to match the packets diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 34c55dee07741..cd14c4c2dff9d 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -802,18 +802,8 @@ int rxe_requester(struct rxe_qp *qp) err = rxe_xmit_packet(qp, &pkt, skb); if (err) { - if (err != -EAGAIN) { - wqe->status = IB_WC_LOC_QP_OP_ERR; - goto err; - } - - /* force a delay until the dropped packet is freed and - * the send queue is drained below the low water mark - */ - qp->need_req_skb = 1; - - rxe_sched_task(&qp->send_task); - goto exit; + wqe->status = IB_WC_LOC_QP_OP_ERR; + goto err; } update_wqe_state(qp, wqe, &pkt); From 1a633bdc8fd9e9e4a9f9a668ae122edfc5aacc86 Mon Sep 17 00:00:00 2001 From: Bob Pearson Date: Fri, 29 Mar 2024 09:55:15 -0500 Subject: [PATCH 0377/1702] RDMA/rxe: Let destroy qp succeed with stuck packet In some situations a sent packet may get queued in the NIC longer than than timeout of a ULP. Currently if this happens the ULP may try to reset the link by destroying the qp and setting up an alternate connection but will fail because the rxe driver is waiting for the packet to finish getting sent and be returned to the skb destructor function where the qp reference holding things up will be dropped. This patch modifies the way that the qp is passed to the destructor to pass the qp index and not a qp pointer. Then the destructor will attempt to lookup the qp from its index and if it fails exit early. This requires taking a reference on the struct sock rather than the qp allowing the qp to be destroyed while the sk is still around waiting for the packet to finish. Link: https://lore.kernel.org/r/20240329145513.35381-15-rpearsonhpe@gmail.com Signed-off-by: Bob Pearson Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/rxe/rxe_net.c | 42 +++++++++++++++++++++-------- drivers/infiniband/sw/rxe/rxe_qp.c | 2 +- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index b58eab75df978..ca9a82e1c4c7e 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -345,25 +345,44 @@ int rxe_prepare(struct rxe_av *av, struct rxe_pkt_info *pkt, static void rxe_skb_tx_dtor(struct sk_buff *skb) { - struct sock *sk = skb->sk; - struct rxe_qp *qp = sk->sk_user_data; - int skb_out = atomic_dec_return(&qp->skb_out); + struct net_device *ndev = skb->dev; + struct rxe_dev *rxe; + unsigned int qp_index; + struct rxe_qp *qp; + int skb_out; + + rxe = rxe_get_dev_from_net(ndev); + if (!rxe && is_vlan_dev(ndev)) + rxe = rxe_get_dev_from_net(vlan_dev_real_dev(ndev)); + if (WARN_ON(!rxe)) + return; - if (unlikely(qp->need_req_skb && - skb_out < RXE_INFLIGHT_SKBS_PER_QP_LOW)) + qp_index = (int)(uintptr_t)skb->sk->sk_user_data; + if (!qp_index) + return; + + qp = rxe_pool_get_index(&rxe->qp_pool, qp_index); + if (!qp) + goto put_dev; + + skb_out = atomic_dec_return(&qp->skb_out); + if (qp->need_req_skb && skb_out < RXE_INFLIGHT_SKBS_PER_QP_LOW) rxe_sched_task(&qp->send_task); rxe_put(qp); +put_dev: + ib_device_put(&rxe->ib_dev); + sock_put(skb->sk); } static int rxe_send(struct sk_buff *skb, struct rxe_pkt_info *pkt) { int err; + struct sock *sk = pkt->qp->sk->sk; + sock_hold(sk); + skb->sk = sk; skb->destructor = rxe_skb_tx_dtor; - skb->sk = pkt->qp->sk->sk; - - rxe_get(pkt->qp); atomic_inc(&pkt->qp->skb_out); if (skb->protocol == htons(ETH_P_IP)) @@ -379,12 +398,13 @@ static int rxe_send(struct sk_buff *skb, struct rxe_pkt_info *pkt) */ static int rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt) { + struct sock *sk = pkt->qp->sk->sk; + memcpy(SKB_TO_PKT(skb), pkt, sizeof(*pkt)); + sock_hold(sk); + skb->sk = sk; skb->destructor = rxe_skb_tx_dtor; - skb->sk = pkt->qp->sk->sk; - - rxe_get(pkt->qp); atomic_inc(&pkt->qp->skb_out); if (skb->protocol == htons(ETH_P_IP)) diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index c7d99063594bc..d2f7b5195c19d 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -244,7 +244,7 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, err = sock_create_kern(&init_net, AF_INET, SOCK_DGRAM, 0, &qp->sk); if (err < 0) return err; - qp->sk->sk->sk_user_data = qp; + qp->sk->sk->sk_user_data = (void *)(uintptr_t)qp->elem.index; /* pick a source UDP port number for this QP based on * the source QPN. this spreads traffic for different QPs From 20516d6e51dd9994afda8d556507cfbe7853384b Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 11 Apr 2024 13:46:14 -0300 Subject: [PATCH 0378/1702] x86: Stop using weak symbols for __iowrite32_copy() Start switching iomap_copy routines over to use #define and arch provided inline/macro functions instead of weak symbols. Inline functions allow more compiler optimization and this is often a driver hot path. x86 has the only weak implementation for __iowrite32_copy(), so replace it with a static inline containing the same single instruction inline assembly. The compiler will generate the "mov edx,ecx" in a more optimal way. Remove iomap_copy_64.S Link: https://lore.kernel.org/r/1-v3-1893cd8b9369+1925-mlx5_arm_wc_jgg@nvidia.com Acked-by: Arnd Bergmann Signed-off-by: Jason Gunthorpe --- arch/x86/include/asm/io.h | 17 +++++++++++++++++ arch/x86/lib/Makefile | 1 - arch/x86/lib/iomap_copy_64.S | 15 --------------- include/linux/io.h | 5 ++++- lib/iomap_copy.c | 6 +++--- 5 files changed, 24 insertions(+), 20 deletions(-) delete mode 100644 arch/x86/lib/iomap_copy_64.S diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h index 294cd2a408181..4b99ed326b174 100644 --- a/arch/x86/include/asm/io.h +++ b/arch/x86/include/asm/io.h @@ -209,6 +209,23 @@ void memset_io(volatile void __iomem *, int, size_t); #define memcpy_toio memcpy_toio #define memset_io memset_io +#ifdef CONFIG_X86_64 +/* + * Commit 0f07496144c2 ("[PATCH] Add faster __iowrite32_copy routine for + * x86_64") says that circa 2006 rep movsl is noticeably faster than a copy + * loop. + */ +static inline void __iowrite32_copy(void __iomem *to, const void *from, + size_t count) +{ + asm volatile("rep ; movsl" + : "=&c"(count), "=&D"(to), "=&S"(from) + : "0"(count), "1"(to), "2"(from) + : "memory"); +} +#define __iowrite32_copy __iowrite32_copy +#endif + /* * ISA space is 'always mapped' on a typical x86 system, no need to * explicitly ioremap() it. The fact that the ISA IO space is mapped diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index 6da73513f0266..98583a9dbab33 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -53,7 +53,6 @@ ifneq ($(CONFIG_X86_CMPXCHG64),y) lib-y += atomic64_386_32.o endif else - obj-y += iomap_copy_64.o ifneq ($(CONFIG_GENERIC_CSUM),y) lib-y += csum-partial_64.o csum-copy_64.o csum-wrappers_64.o endif diff --git a/arch/x86/lib/iomap_copy_64.S b/arch/x86/lib/iomap_copy_64.S deleted file mode 100644 index 6ff2f56cb0f71..0000000000000 --- a/arch/x86/lib/iomap_copy_64.S +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright 2006 PathScale, Inc. All Rights Reserved. - */ - -#include - -/* - * override generic version in lib/iomap_copy.c - */ -SYM_FUNC_START(__iowrite32_copy) - movl %edx,%ecx - rep movsl - RET -SYM_FUNC_END(__iowrite32_copy) diff --git a/include/linux/io.h b/include/linux/io.h index 235ba7d80a8f0..ce86120ce9d52 100644 --- a/include/linux/io.h +++ b/include/linux/io.h @@ -16,7 +16,10 @@ struct device; struct resource; -__visible void __iowrite32_copy(void __iomem *to, const void *from, size_t count); +#ifndef __iowrite32_copy +void __iowrite32_copy(void __iomem *to, const void *from, size_t count); +#endif + void __ioread32_copy(void *to, const void __iomem *from, size_t count); void __iowrite64_copy(void __iomem *to, const void *from, size_t count); diff --git a/lib/iomap_copy.c b/lib/iomap_copy.c index 5de7c04e05ef5..8ddcbb53507df 100644 --- a/lib/iomap_copy.c +++ b/lib/iomap_copy.c @@ -16,9 +16,8 @@ * time. Order of access is not guaranteed, nor is a memory barrier * performed afterwards. */ -void __attribute__((weak)) __iowrite32_copy(void __iomem *to, - const void *from, - size_t count) +#ifndef __iowrite32_copy +void __iowrite32_copy(void __iomem *to, const void *from, size_t count) { u32 __iomem *dst = to; const u32 *src = from; @@ -28,6 +27,7 @@ void __attribute__((weak)) __iowrite32_copy(void __iomem *to, __raw_writel(*src++, dst++); } EXPORT_SYMBOL_GPL(__iowrite32_copy); +#endif /** * __ioread32_copy - copy data from MMIO space, in 32-bit units From 6ae798cbef4ba1f180aa1a590e33a2d89f7cc34f Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 11 Apr 2024 13:46:15 -0300 Subject: [PATCH 0379/1702] s390: Implement __iowrite32_copy() It is trivial to implement an inline to do this, so provide it in the s390 headers. Like the 64 bit version it should just invoke zpci_memcpy_toio() with the correct size. Link: https://lore.kernel.org/r/2-v3-1893cd8b9369+1925-mlx5_arm_wc_jgg@nvidia.com Acked-by: Niklas Schnelle Signed-off-by: Jason Gunthorpe --- arch/s390/include/asm/io.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/s390/include/asm/io.h b/arch/s390/include/asm/io.h index 4453ad7c11ace..00704fc8a54b3 100644 --- a/arch/s390/include/asm/io.h +++ b/arch/s390/include/asm/io.h @@ -73,6 +73,14 @@ static inline void ioport_unmap(void __iomem *p) #define __raw_writel zpci_write_u32 #define __raw_writeq zpci_write_u64 +/* combine single writes by using store-block insn */ +static inline void __iowrite32_copy(void __iomem *to, const void *from, + size_t count) +{ + zpci_memcpy_toio(to, from, count * 4); +} +#define __iowrite32_copy __iowrite32_copy + #endif /* CONFIG_PCI */ #include From e7bc47b16622d1016b3b77bbdb20fb9e213045f2 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 11 Apr 2024 13:46:16 -0300 Subject: [PATCH 0380/1702] s390: Stop using weak symbols for __iowrite64_copy() Complete switching the __iowriteXX_copy() routines over to use #define and arch provided inline/macro functions instead of weak symbols. S390 has an implementation that simply calls another memcpy function. Inline this so the callers don't have to do two jumps. Link: https://lore.kernel.org/r/3-v3-1893cd8b9369+1925-mlx5_arm_wc_jgg@nvidia.com Acked-by: Niklas Schnelle Acked-by: Arnd Bergmann Signed-off-by: Jason Gunthorpe --- arch/s390/include/asm/io.h | 7 +++++++ arch/s390/pci/pci.c | 6 ------ include/linux/io.h | 3 +++ lib/iomap_copy.c | 7 +++---- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/arch/s390/include/asm/io.h b/arch/s390/include/asm/io.h index 00704fc8a54b3..0fbc992d7a5ea 100644 --- a/arch/s390/include/asm/io.h +++ b/arch/s390/include/asm/io.h @@ -81,6 +81,13 @@ static inline void __iowrite32_copy(void __iomem *to, const void *from, } #define __iowrite32_copy __iowrite32_copy +static inline void __iowrite64_copy(void __iomem *to, const void *from, + size_t count) +{ + zpci_memcpy_toio(to, from, count * 8); +} +#define __iowrite64_copy __iowrite64_copy + #endif /* CONFIG_PCI */ #include diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 26afde0d1ed34..0de0f6e405b51 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -250,12 +250,6 @@ resource_size_t pcibios_align_resource(void *data, const struct resource *res, return 0; } -/* combine single writes by using store-block insn */ -void __iowrite64_copy(void __iomem *to, const void *from, size_t count) -{ - zpci_memcpy_toio(to, from, count * 8); -} - void __iomem *ioremap_prot(phys_addr_t phys_addr, size_t size, unsigned long prot) { diff --git a/include/linux/io.h b/include/linux/io.h index ce86120ce9d52..42e132808f003 100644 --- a/include/linux/io.h +++ b/include/linux/io.h @@ -21,7 +21,10 @@ void __iowrite32_copy(void __iomem *to, const void *from, size_t count); #endif void __ioread32_copy(void *to, const void __iomem *from, size_t count); + +#ifndef __iowrite64_copy void __iowrite64_copy(void __iomem *to, const void *from, size_t count); +#endif #ifdef CONFIG_MMU int ioremap_page_range(unsigned long addr, unsigned long end, diff --git a/lib/iomap_copy.c b/lib/iomap_copy.c index 8ddcbb53507df..2fd5712fb7c02 100644 --- a/lib/iomap_copy.c +++ b/lib/iomap_copy.c @@ -60,9 +60,8 @@ EXPORT_SYMBOL_GPL(__ioread32_copy); * time. Order of access is not guaranteed, nor is a memory barrier * performed afterwards. */ -void __attribute__((weak)) __iowrite64_copy(void __iomem *to, - const void *from, - size_t count) +#ifndef __iowrite64_copy +void __iowrite64_copy(void __iomem *to, const void *from, size_t count) { #ifdef CONFIG_64BIT u64 __iomem *dst = to; @@ -75,5 +74,5 @@ void __attribute__((weak)) __iowrite64_copy(void __iomem *to, __iowrite32_copy(to, from, count * 2); #endif } - EXPORT_SYMBOL_GPL(__iowrite64_copy); +#endif From ead79118dae6f9f982532002e82c2fb291ae0480 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 11 Apr 2024 13:46:17 -0300 Subject: [PATCH 0381/1702] arm64/io: Provide a WC friendly __iowriteXX_copy() The kernel provides driver support for using write combining IO memory through the __iowriteXX_copy() API which is commonly used as an optional optimization to generate 16/32/64 byte MemWr TLPs in a PCIe environment. iomap_copy.c provides a generic implementation as a simple 4/8 byte at a time copy loop that has worked well with past ARM64 CPUs, giving a high frequency of large TLPs being successfully formed. However modern ARM64 CPUs are quite sensitive to how the write combining CPU HW is operated and a compiler generated loop with intermixed load/store is not sufficient to frequently generate a large TLP. The CPUs would like to see the entire TLP generated by consecutive store instructions from registers. Compilers like gcc tend to intermix loads and stores and have poor code generation, in part, due to the ARM64 situation that writeq() does not codegen anything other than "[xN]". However even with that resolved compilers like clang still do not have good code generation. This means on modern ARM64 CPUs the rate at which __iowriteXX_copy() successfully generates large TLPs is very small (less than 1 in 10,000) tries), to the point that the use of WC is pointless. Implement __iowrite32/64_copy() specifically for ARM64 and use inline assembly to build consecutive blocks of STR instructions. Provide direct support for 64/32/16 large TLP generation in this manner. Optimize for common constant lengths so that the compiler can directly inline the store blocks. This brings the frequency of large TLP generation up to a high level that is comparable with older CPU generations. As the __iowriteXX_copy() family of APIs is intended for use with WC incorporate the DGH hint directly into the function. Link: https://lore.kernel.org/r/4-v3-1893cd8b9369+1925-mlx5_arm_wc_jgg@nvidia.com Cc: Arnd Bergmann Cc: Catalin Marinas Cc: Will Deacon Cc: Mark Rutland Cc: linux-arch@vger.kernel.org Cc: linux-arm-kernel@lists.infradead.org Reviewed-by: Catalin Marinas Signed-off-by: Jason Gunthorpe --- arch/arm64/include/asm/io.h | 132 ++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/io.c | 42 ++++++++++++ 2 files changed, 174 insertions(+) diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 8d825522c55c8..4ff0ae3f6d669 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -139,6 +139,138 @@ extern void __memset_io(volatile void __iomem *, int, size_t); #define memcpy_fromio(a,c,l) __memcpy_fromio((a),(c),(l)) #define memcpy_toio(c,a,l) __memcpy_toio((c),(a),(l)) +/* + * The ARM64 iowrite implementation is intended to support drivers that want to + * use write combining. For instance PCI drivers using write combining with a 64 + * byte __iowrite64_copy() expect to get a 64 byte MemWr TLP on the PCIe bus. + * + * Newer ARM core have sensitive write combining buffers, it is important that + * the stores be contiguous blocks of store instructions. Normal memcpy + * approaches have a very low chance to generate write combining. + * + * Since this is the only API on ARM64 that should be used with write combining + * it also integrates the DGH hint which is supposed to lower the latency to + * emit the large TLP from the CPU. + */ + +static inline void __const_memcpy_toio_aligned32(volatile u32 __iomem *to, + const u32 *from, size_t count) +{ + switch (count) { + case 8: + asm volatile("str %w0, [%8, #4 * 0]\n" + "str %w1, [%8, #4 * 1]\n" + "str %w2, [%8, #4 * 2]\n" + "str %w3, [%8, #4 * 3]\n" + "str %w4, [%8, #4 * 4]\n" + "str %w5, [%8, #4 * 5]\n" + "str %w6, [%8, #4 * 6]\n" + "str %w7, [%8, #4 * 7]\n" + : + : "rZ"(from[0]), "rZ"(from[1]), "rZ"(from[2]), + "rZ"(from[3]), "rZ"(from[4]), "rZ"(from[5]), + "rZ"(from[6]), "rZ"(from[7]), "r"(to)); + break; + case 4: + asm volatile("str %w0, [%4, #4 * 0]\n" + "str %w1, [%4, #4 * 1]\n" + "str %w2, [%4, #4 * 2]\n" + "str %w3, [%4, #4 * 3]\n" + : + : "rZ"(from[0]), "rZ"(from[1]), "rZ"(from[2]), + "rZ"(from[3]), "r"(to)); + break; + case 2: + asm volatile("str %w0, [%2, #4 * 0]\n" + "str %w1, [%2, #4 * 1]\n" + : + : "rZ"(from[0]), "rZ"(from[1]), "r"(to)); + break; + case 1: + __raw_writel(*from, to); + break; + default: + BUILD_BUG(); + } +} + +void __iowrite32_copy_full(void __iomem *to, const void *from, size_t count); + +static inline void __const_iowrite32_copy(void __iomem *to, const void *from, + size_t count) +{ + if (count == 8 || count == 4 || count == 2 || count == 1) { + __const_memcpy_toio_aligned32(to, from, count); + dgh(); + } else { + __iowrite32_copy_full(to, from, count); + } +} + +#define __iowrite32_copy(to, from, count) \ + (__builtin_constant_p(count) ? \ + __const_iowrite32_copy(to, from, count) : \ + __iowrite32_copy_full(to, from, count)) + +static inline void __const_memcpy_toio_aligned64(volatile u64 __iomem *to, + const u64 *from, size_t count) +{ + switch (count) { + case 8: + asm volatile("str %x0, [%8, #8 * 0]\n" + "str %x1, [%8, #8 * 1]\n" + "str %x2, [%8, #8 * 2]\n" + "str %x3, [%8, #8 * 3]\n" + "str %x4, [%8, #8 * 4]\n" + "str %x5, [%8, #8 * 5]\n" + "str %x6, [%8, #8 * 6]\n" + "str %x7, [%8, #8 * 7]\n" + : + : "rZ"(from[0]), "rZ"(from[1]), "rZ"(from[2]), + "rZ"(from[3]), "rZ"(from[4]), "rZ"(from[5]), + "rZ"(from[6]), "rZ"(from[7]), "r"(to)); + break; + case 4: + asm volatile("str %x0, [%4, #8 * 0]\n" + "str %x1, [%4, #8 * 1]\n" + "str %x2, [%4, #8 * 2]\n" + "str %x3, [%4, #8 * 3]\n" + : + : "rZ"(from[0]), "rZ"(from[1]), "rZ"(from[2]), + "rZ"(from[3]), "r"(to)); + break; + case 2: + asm volatile("str %x0, [%2, #8 * 0]\n" + "str %x1, [%2, #8 * 1]\n" + : + : "rZ"(from[0]), "rZ"(from[1]), "r"(to)); + break; + case 1: + __raw_writeq(*from, to); + break; + default: + BUILD_BUG(); + } +} + +void __iowrite64_copy_full(void __iomem *to, const void *from, size_t count); + +static inline void __const_iowrite64_copy(void __iomem *to, const void *from, + size_t count) +{ + if (count == 8 || count == 4 || count == 2 || count == 1) { + __const_memcpy_toio_aligned64(to, from, count); + dgh(); + } else { + __iowrite64_copy_full(to, from, count); + } +} + +#define __iowrite64_copy(to, from, count) \ + (__builtin_constant_p(count) ? \ + __const_iowrite64_copy(to, from, count) : \ + __iowrite64_copy_full(to, from, count)) + /* * I/O memory mapping functions. */ diff --git a/arch/arm64/kernel/io.c b/arch/arm64/kernel/io.c index aa7a4ec6a3ae6..ef48089fbfe1a 100644 --- a/arch/arm64/kernel/io.c +++ b/arch/arm64/kernel/io.c @@ -37,6 +37,48 @@ void __memcpy_fromio(void *to, const volatile void __iomem *from, size_t count) } EXPORT_SYMBOL(__memcpy_fromio); +/* + * This generates a memcpy that works on a from/to address which is aligned to + * bits. Count is in terms of the number of bits sized quantities to copy. It + * optimizes to use the STR groupings when possible so that it is WC friendly. + */ +#define memcpy_toio_aligned(to, from, count, bits) \ + ({ \ + volatile u##bits __iomem *_to = to; \ + const u##bits *_from = from; \ + size_t _count = count; \ + const u##bits *_end_from = _from + ALIGN_DOWN(_count, 8); \ + \ + for (; _from < _end_from; _from += 8, _to += 8) \ + __const_memcpy_toio_aligned##bits(_to, _from, 8); \ + if ((_count % 8) >= 4) { \ + __const_memcpy_toio_aligned##bits(_to, _from, 4); \ + _from += 4; \ + _to += 4; \ + } \ + if ((_count % 4) >= 2) { \ + __const_memcpy_toio_aligned##bits(_to, _from, 2); \ + _from += 2; \ + _to += 2; \ + } \ + if (_count % 2) \ + __const_memcpy_toio_aligned##bits(_to, _from, 1); \ + }) + +void __iowrite64_copy_full(void __iomem *to, const void *from, size_t count) +{ + memcpy_toio_aligned(to, from, count, 64); + dgh(); +} +EXPORT_SYMBOL(__iowrite64_copy_full); + +void __iowrite32_copy_full(void __iomem *to, const void *from, size_t count) +{ + memcpy_toio_aligned(to, from, count, 32); + dgh(); +} +EXPORT_SYMBOL(__iowrite32_copy_full); + /* * Copy data from "real" memory space to IO memory space. */ From 2b7a5e1fe02231acc5d50339b2f10833565ef559 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 11 Apr 2024 13:46:18 -0300 Subject: [PATCH 0382/1702] net: hns3: Remove io_stop_wc() calls after __iowrite64_copy() Now that the ARM64 arch implementation does the DGH as part of __iowrite64_copy() there is no reason to open code this in drivers. Link: https://lore.kernel.org/r/5-v3-1893cd8b9369+1925-mlx5_arm_wc_jgg@nvidia.com Reviewed-by: Jijie Shao Signed-off-by: Jason Gunthorpe --- drivers/net/ethernet/hisilicon/hns3/hns3_enet.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 19668a8d22f76..04b9e86363f8f 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -2068,8 +2068,6 @@ static void hns3_tx_push_bd(struct hns3_enet_ring *ring, int num) __iowrite64_copy(ring->tqp->mem_base, desc, (sizeof(struct hns3_desc) * HNS3_MAX_PUSH_BD_NUM) / HNS3_BYTES_PER_64BIT); - - io_stop_wc(); } static void hns3_tx_mem_doorbell(struct hns3_enet_ring *ring) @@ -2088,8 +2086,6 @@ static void hns3_tx_mem_doorbell(struct hns3_enet_ring *ring) u64_stats_update_begin(&ring->syncp); ring->stats.tx_mem_doorbell += ring->pending_buf; u64_stats_update_end(&ring->syncp); - - io_stop_wc(); } static void hns3_tx_doorbell(struct hns3_enet_ring *ring, int num, From ef302283ddfceaba2657923af3f90fd58e6dff06 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 11 Apr 2024 13:46:19 -0300 Subject: [PATCH 0383/1702] IB/mlx5: Use __iowrite64_copy() for write combining stores mlx5 has a built in self-test at driver startup to evaluate if the platform supports write combining to generate a 64 byte PCIe TLP or not. This has proven necessary because a lot of common scenarios end up with broken write combining (especially inside virtual machines) and there is other way to learn this information. This self test has been consistently failing on new ARM64 CPU designs (specifically with NVIDIA Grace's implementation of Neoverse V2). The C loop around writeq() generates some pretty terrible ARM64 assembly, but historically this has worked on a lot of existing ARM64 CPUs till now. We see it succeed about 1 time in 10,000 on the worst effected systems. The CPU architects speculate that the load instructions interspersed with the stores makes the WC buffers statistically flush too often and thus the generation of large TLPs becomes infrequent. This makes the boot up test unreliable in that it indicates no write-combining, however userspace would be fine since it uses a ST4 instruction. Further, S390 has similar issues where only the special zpci_memcpy_toio() will actually generate large TLPs, and the open coded loop does not trigger it at all. Fix both ARM64 and S390 by switching to __iowrite64_copy() which now provides architecture specific variants that have a high change of generating a large TLP with write combining. x86 continues to use a similar writeq loop in the generate __iowrite64_copy(). Fixes: 11f552e21755 ("IB/mlx5: Test write combining support") Link: https://lore.kernel.org/r/6-v3-1893cd8b9369+1925-mlx5_arm_wc_jgg@nvidia.com Tested-by: Niklas Schnelle Acked-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mem.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/mem.c b/drivers/infiniband/hw/mlx5/mem.c index 96ffbbaf0a73d..5a22be14d958f 100644 --- a/drivers/infiniband/hw/mlx5/mem.c +++ b/drivers/infiniband/hw/mlx5/mem.c @@ -30,6 +30,7 @@ * SOFTWARE. */ +#include #include #include "mlx5_ib.h" #include @@ -108,7 +109,6 @@ static int post_send_nop(struct mlx5_ib_dev *dev, struct ib_qp *ibqp, u64 wr_id, __be32 mmio_wqe[16] = {}; unsigned long flags; unsigned int idx; - int i; if (unlikely(dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)) return -EIO; @@ -148,10 +148,8 @@ static int post_send_nop(struct mlx5_ib_dev *dev, struct ib_qp *ibqp, u64 wr_id, * we hit doorbell */ wmb(); - for (i = 0; i < 8; i++) - mlx5_write64(&mmio_wqe[i * 2], - bf->bfreg->map + bf->offset + i * 8); - io_stop_wc(); + __iowrite64_copy(bf->bfreg->map + bf->offset, mmio_wqe, + sizeof(mmio_wqe) / 8); bf->offset ^= bf->buf_size; From 2641ee13c449ef76b0c369ae94dd8dd5dc8c2008 Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Mon, 15 Apr 2024 17:36:45 +0100 Subject: [PATCH 0384/1702] clk: scmi: Allocate CLK operations dynamically SCMI Clocks descriptors expose an increasing number of properties, thing which, in turn, leads to a varying set of supported CLK operations to be associated with each clock. Providing statically pre-defined CLK operations structs for all the possible combinations of allowed clock features is becoming cumbersome and error-prone. Allocate the per-clock operations descriptors dynamically and populate it with the strictly needed set of operations depending on the advertised clock properties: one descriptor is created for each distinct combination of clock operations, so minimizing the number of clk_ops structures to the strictly minimum needed. CC: Michael Turquette CC: Stephen Boyd CC: linux-clk@vger.kernel.org Signed-off-by: Cristian Marussi Link: https://lore.kernel.org/r/20240415163649.895268-2-cristian.marussi@arm.com Reviewed-by: Sudeep Holla Signed-off-by: Stephen Boyd --- drivers/clk/clk-scmi.c | 186 ++++++++++++++++++++++++++++++----------- 1 file changed, 136 insertions(+), 50 deletions(-) diff --git a/drivers/clk/clk-scmi.c b/drivers/clk/clk-scmi.c index 8cbe24789c24b..4b6c43d7f5f3b 100644 --- a/drivers/clk/clk-scmi.c +++ b/drivers/clk/clk-scmi.c @@ -2,9 +2,10 @@ /* * System Control and Power Interface (SCMI) Protocol based clock driver * - * Copyright (C) 2018-2022 ARM Ltd. + * Copyright (C) 2018-2024 ARM Ltd. */ +#include #include #include #include @@ -16,6 +17,13 @@ #define NOT_ATOMIC false #define ATOMIC true +enum scmi_clk_feats { + SCMI_CLK_ATOMIC_SUPPORTED, + SCMI_CLK_FEATS_COUNT +}; + +#define SCMI_MAX_CLK_OPS BIT(SCMI_CLK_FEATS_COUNT) + static const struct scmi_clk_proto_ops *scmi_proto_clk_ops; struct scmi_clk { @@ -158,42 +166,6 @@ static int scmi_clk_atomic_is_enabled(struct clk_hw *hw) return !!enabled; } -/* - * We can provide enable/disable/is_enabled atomic callbacks only if the - * underlying SCMI transport for an SCMI instance is configured to handle - * SCMI commands in an atomic manner. - * - * When no SCMI atomic transport support is available we instead provide only - * the prepare/unprepare API, as allowed by the clock framework when atomic - * calls are not available. - * - * Two distinct sets of clk_ops are provided since we could have multiple SCMI - * instances with different underlying transport quality, so they cannot be - * shared. - */ -static const struct clk_ops scmi_clk_ops = { - .recalc_rate = scmi_clk_recalc_rate, - .round_rate = scmi_clk_round_rate, - .set_rate = scmi_clk_set_rate, - .prepare = scmi_clk_enable, - .unprepare = scmi_clk_disable, - .set_parent = scmi_clk_set_parent, - .get_parent = scmi_clk_get_parent, - .determine_rate = scmi_clk_determine_rate, -}; - -static const struct clk_ops scmi_atomic_clk_ops = { - .recalc_rate = scmi_clk_recalc_rate, - .round_rate = scmi_clk_round_rate, - .set_rate = scmi_clk_set_rate, - .enable = scmi_clk_atomic_enable, - .disable = scmi_clk_atomic_disable, - .is_enabled = scmi_clk_atomic_is_enabled, - .set_parent = scmi_clk_set_parent, - .get_parent = scmi_clk_get_parent, - .determine_rate = scmi_clk_determine_rate, -}; - static int scmi_clk_ops_init(struct device *dev, struct scmi_clk *sclk, const struct clk_ops *scmi_ops) { @@ -230,17 +202,129 @@ static int scmi_clk_ops_init(struct device *dev, struct scmi_clk *sclk, return ret; } +/** + * scmi_clk_ops_alloc() - Alloc and configure clock operations + * @dev: A device reference for devres + * @feats_key: A bitmap representing the desired clk_ops capabilities + * + * Allocate and configure a proper set of clock operations depending on the + * specifically required SCMI clock features. + * + * Return: A pointer to the allocated and configured clk_ops on success, + * or NULL on allocation failure. + */ +static const struct clk_ops * +scmi_clk_ops_alloc(struct device *dev, unsigned long feats_key) +{ + struct clk_ops *ops; + + ops = devm_kzalloc(dev, sizeof(*ops), GFP_KERNEL); + if (!ops) + return NULL; + /* + * We can provide enable/disable/is_enabled atomic callbacks only if the + * underlying SCMI transport for an SCMI instance is configured to + * handle SCMI commands in an atomic manner. + * + * When no SCMI atomic transport support is available we instead provide + * only the prepare/unprepare API, as allowed by the clock framework + * when atomic calls are not available. + */ + if (feats_key & BIT(SCMI_CLK_ATOMIC_SUPPORTED)) { + ops->enable = scmi_clk_atomic_enable; + ops->disable = scmi_clk_atomic_disable; + ops->is_enabled = scmi_clk_atomic_is_enabled; + } else { + ops->prepare = scmi_clk_enable; + ops->unprepare = scmi_clk_disable; + } + + /* Rate ops */ + ops->recalc_rate = scmi_clk_recalc_rate; + ops->round_rate = scmi_clk_round_rate; + ops->determine_rate = scmi_clk_determine_rate; + ops->set_rate = scmi_clk_set_rate; + + /* Parent ops */ + ops->get_parent = scmi_clk_get_parent; + ops->set_parent = scmi_clk_set_parent; + + return ops; +} + +/** + * scmi_clk_ops_select() - Select a proper set of clock operations + * @sclk: A reference to an SCMI clock descriptor + * @atomic_capable: A flag to indicate if atomic mode is supported by the + * transport + * @atomic_threshold_us: Platform atomic threshold value in microseconds: + * clk_ops are atomic when clock enable latency is less + * than this threshold + * @clk_ops_db: A reference to the array used as a database to store all the + * created clock operations combinations. + * @db_size: Maximum number of entries held by @clk_ops_db + * + * After having built a bitmap descriptor to represent the set of features + * needed by this SCMI clock, at first use it to lookup into the set of + * previously allocated clk_ops to check if a suitable combination of clock + * operations was already created; when no match is found allocate a brand new + * set of clk_ops satisfying the required combination of features and save it + * for future references. + * + * In this way only one set of clk_ops is ever created for each different + * combination that is effectively needed by a driver instance. + * + * Return: A pointer to the allocated and configured clk_ops on success, or + * NULL otherwise. + */ +static const struct clk_ops * +scmi_clk_ops_select(struct scmi_clk *sclk, bool atomic_capable, + unsigned int atomic_threshold_us, + const struct clk_ops **clk_ops_db, size_t db_size) +{ + const struct scmi_clock_info *ci = sclk->info; + unsigned int feats_key = 0; + const struct clk_ops *ops; + + /* + * Note that when transport is atomic but SCMI protocol did not + * specify (or support) an enable_latency associated with a + * clock, we default to use atomic operations mode. + */ + if (atomic_capable && ci->enable_latency <= atomic_threshold_us) + feats_key |= BIT(SCMI_CLK_ATOMIC_SUPPORTED); + + if (WARN_ON(feats_key >= db_size)) + return NULL; + + /* Lookup previously allocated ops */ + ops = clk_ops_db[feats_key]; + if (ops) + return ops; + + /* Did not find a pre-allocated clock_ops */ + ops = scmi_clk_ops_alloc(sclk->dev, feats_key); + if (!ops) + return NULL; + + /* Store new ops combinations */ + clk_ops_db[feats_key] = ops; + + return ops; +} + static int scmi_clocks_probe(struct scmi_device *sdev) { int idx, count, err; - unsigned int atomic_threshold; - bool is_atomic; + unsigned int atomic_threshold_us; + bool transport_is_atomic; struct clk_hw **hws; struct clk_hw_onecell_data *clk_data; struct device *dev = &sdev->dev; struct device_node *np = dev->of_node; const struct scmi_handle *handle = sdev->handle; struct scmi_protocol_handle *ph; + const struct clk_ops *scmi_clk_ops_db[SCMI_MAX_CLK_OPS] = {}; if (!handle) return -ENODEV; @@ -264,7 +348,8 @@ static int scmi_clocks_probe(struct scmi_device *sdev) clk_data->num = count; hws = clk_data->hws; - is_atomic = handle->is_transport_atomic(handle, &atomic_threshold); + transport_is_atomic = handle->is_transport_atomic(handle, + &atomic_threshold_us); for (idx = 0; idx < count; idx++) { struct scmi_clk *sclk; @@ -286,15 +371,17 @@ static int scmi_clocks_probe(struct scmi_device *sdev) sclk->dev = dev; /* - * Note that when transport is atomic but SCMI protocol did not - * specify (or support) an enable_latency associated with a - * clock, we default to use atomic operations mode. + * Note that the scmi_clk_ops_db is on the stack, not global, + * because it cannot be shared between mulitple probe-sequences + * to avoid sharing the devm_ allocated clk_ops between multiple + * SCMI clk driver instances. */ - if (is_atomic && - sclk->info->enable_latency <= atomic_threshold) - scmi_ops = &scmi_atomic_clk_ops; - else - scmi_ops = &scmi_clk_ops; + scmi_ops = scmi_clk_ops_select(sclk, transport_is_atomic, + atomic_threshold_us, + scmi_clk_ops_db, + ARRAY_SIZE(scmi_clk_ops_db)); + if (!scmi_ops) + return -ENOMEM; /* Initialize clock parent data. */ if (sclk->info->num_parents > 0) { @@ -318,8 +405,7 @@ static int scmi_clocks_probe(struct scmi_device *sdev) } else { dev_dbg(dev, "Registered clock:%s%s\n", sclk->info->name, - scmi_ops == &scmi_atomic_clk_ops ? - " (atomic ops)" : ""); + scmi_ops->enable ? " (atomic ops)" : ""); hws[idx] = &sclk->hw; } } From a1b8faf8784c434444432008b57274c8935cca5c Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Mon, 15 Apr 2024 17:36:46 +0100 Subject: [PATCH 0385/1702] clk: scmi: Add support for state control restricted clocks Some exposed SCMI Clocks could be marked as non-supporting state changes. Configure a clk_ops descriptor which does not provide the state change callbacks for such clocks when registering with CLK framework. CC: Michael Turquette CC: Stephen Boyd CC: linux-clk@vger.kernel.org Signed-off-by: Cristian Marussi Link: https://lore.kernel.org/r/20240415163649.895268-3-cristian.marussi@arm.com Reviewed-by: Sudeep Holla Signed-off-by: Stephen Boyd --- drivers/clk/clk-scmi.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/drivers/clk/clk-scmi.c b/drivers/clk/clk-scmi.c index 4b6c43d7f5f3b..e707085739656 100644 --- a/drivers/clk/clk-scmi.c +++ b/drivers/clk/clk-scmi.c @@ -19,6 +19,7 @@ enum scmi_clk_feats { SCMI_CLK_ATOMIC_SUPPORTED, + SCMI_CLK_STATE_CTRL_SUPPORTED, SCMI_CLK_FEATS_COUNT }; @@ -230,15 +231,19 @@ scmi_clk_ops_alloc(struct device *dev, unsigned long feats_key) * only the prepare/unprepare API, as allowed by the clock framework * when atomic calls are not available. */ - if (feats_key & BIT(SCMI_CLK_ATOMIC_SUPPORTED)) { - ops->enable = scmi_clk_atomic_enable; - ops->disable = scmi_clk_atomic_disable; - ops->is_enabled = scmi_clk_atomic_is_enabled; - } else { - ops->prepare = scmi_clk_enable; - ops->unprepare = scmi_clk_disable; + if (feats_key & BIT(SCMI_CLK_STATE_CTRL_SUPPORTED)) { + if (feats_key & BIT(SCMI_CLK_ATOMIC_SUPPORTED)) { + ops->enable = scmi_clk_atomic_enable; + ops->disable = scmi_clk_atomic_disable; + } else { + ops->prepare = scmi_clk_enable; + ops->unprepare = scmi_clk_disable; + } } + if (feats_key & BIT(SCMI_CLK_ATOMIC_SUPPORTED)) + ops->is_enabled = scmi_clk_atomic_is_enabled; + /* Rate ops */ ops->recalc_rate = scmi_clk_recalc_rate; ops->round_rate = scmi_clk_round_rate; @@ -294,6 +299,9 @@ scmi_clk_ops_select(struct scmi_clk *sclk, bool atomic_capable, if (atomic_capable && ci->enable_latency <= atomic_threshold_us) feats_key |= BIT(SCMI_CLK_ATOMIC_SUPPORTED); + if (!ci->state_ctrl_forbidden) + feats_key |= BIT(SCMI_CLK_STATE_CTRL_SUPPORTED); + if (WARN_ON(feats_key >= db_size)) return NULL; From c3ad1d0a7ef28bc503caf3d5242f2dda55df5d3f Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Mon, 15 Apr 2024 17:36:47 +0100 Subject: [PATCH 0386/1702] clk: scmi: Add support for rate change restricted clocks Some exposed SCMI Clocks could be marked as non-supporting rate changes. Configure a clk_ops descriptors which does not provide the rate change callbacks for such clocks when registering with CLK framework. CC: Michael Turquette CC: Stephen Boyd CC: linux-clk@vger.kernel.org Signed-off-by: Cristian Marussi Link: https://lore.kernel.org/r/20240415163649.895268-4-cristian.marussi@arm.com Reviewed-by: Sudeep Holla Signed-off-by: Stephen Boyd --- drivers/clk/clk-scmi.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/clk/clk-scmi.c b/drivers/clk/clk-scmi.c index e707085739656..ba234b56f7428 100644 --- a/drivers/clk/clk-scmi.c +++ b/drivers/clk/clk-scmi.c @@ -20,6 +20,7 @@ enum scmi_clk_feats { SCMI_CLK_ATOMIC_SUPPORTED, SCMI_CLK_STATE_CTRL_SUPPORTED, + SCMI_CLK_RATE_CTRL_SUPPORTED, SCMI_CLK_FEATS_COUNT }; @@ -248,7 +249,8 @@ scmi_clk_ops_alloc(struct device *dev, unsigned long feats_key) ops->recalc_rate = scmi_clk_recalc_rate; ops->round_rate = scmi_clk_round_rate; ops->determine_rate = scmi_clk_determine_rate; - ops->set_rate = scmi_clk_set_rate; + if (feats_key & BIT(SCMI_CLK_RATE_CTRL_SUPPORTED)) + ops->set_rate = scmi_clk_set_rate; /* Parent ops */ ops->get_parent = scmi_clk_get_parent; @@ -302,6 +304,9 @@ scmi_clk_ops_select(struct scmi_clk *sclk, bool atomic_capable, if (!ci->state_ctrl_forbidden) feats_key |= BIT(SCMI_CLK_STATE_CTRL_SUPPORTED); + if (!ci->rate_ctrl_forbidden) + feats_key |= BIT(SCMI_CLK_RATE_CTRL_SUPPORTED); + if (WARN_ON(feats_key >= db_size)) return NULL; From fa23e091236b812a6143cf0f49821d8f5a6b84fa Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Mon, 15 Apr 2024 17:36:48 +0100 Subject: [PATCH 0387/1702] clk: scmi: Add support for re-parenting restricted clocks Some exposed SCMI Clocks could be marked as non-supporting re-parenting changes. Configure a clk_ops descriptor which does not provide the re-parenting callbacks for such clocks when registering with CLK framework. CC: Michael Turquette CC: Stephen Boyd CC: linux-clk@vger.kernel.org Signed-off-by: Cristian Marussi Link: https://lore.kernel.org/r/20240415163649.895268-5-cristian.marussi@arm.com Reviewed-by: Sudeep Holla Signed-off-by: Stephen Boyd --- drivers/clk/clk-scmi.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/clk/clk-scmi.c b/drivers/clk/clk-scmi.c index ba234b56f7428..ce0f26ee632fb 100644 --- a/drivers/clk/clk-scmi.c +++ b/drivers/clk/clk-scmi.c @@ -21,6 +21,7 @@ enum scmi_clk_feats { SCMI_CLK_ATOMIC_SUPPORTED, SCMI_CLK_STATE_CTRL_SUPPORTED, SCMI_CLK_RATE_CTRL_SUPPORTED, + SCMI_CLK_PARENT_CTRL_SUPPORTED, SCMI_CLK_FEATS_COUNT }; @@ -254,7 +255,8 @@ scmi_clk_ops_alloc(struct device *dev, unsigned long feats_key) /* Parent ops */ ops->get_parent = scmi_clk_get_parent; - ops->set_parent = scmi_clk_set_parent; + if (feats_key & BIT(SCMI_CLK_PARENT_CTRL_SUPPORTED)) + ops->set_parent = scmi_clk_set_parent; return ops; } @@ -307,6 +309,9 @@ scmi_clk_ops_select(struct scmi_clk *sclk, bool atomic_capable, if (!ci->rate_ctrl_forbidden) feats_key |= BIT(SCMI_CLK_RATE_CTRL_SUPPORTED); + if (!ci->parent_ctrl_forbidden) + feats_key |= BIT(SCMI_CLK_PARENT_CTRL_SUPPORTED); + if (WARN_ON(feats_key >= db_size)) return NULL; From 87af9481af53c041358be0c46adb103c62218b7f Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Mon, 15 Apr 2024 17:36:49 +0100 Subject: [PATCH 0388/1702] clk: scmi: Add support for get/set duty_cycle operations Provide the CLK framework callbacks related to get/set clock duty cycle if the related SCMI clock supports OEM extended configurations. CC: Michael Turquette CC: Stephen Boyd CC: linux-clk@vger.kernel.org Signed-off-by: Cristian Marussi Link: https://lore.kernel.org/r/20240415163649.895268-6-cristian.marussi@arm.com Reviewed-by: Sudeep Holla Signed-off-by: Stephen Boyd --- drivers/clk/clk-scmi.c | 49 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/drivers/clk/clk-scmi.c b/drivers/clk/clk-scmi.c index ce0f26ee632fb..d86a02563f6c8 100644 --- a/drivers/clk/clk-scmi.c +++ b/drivers/clk/clk-scmi.c @@ -22,6 +22,7 @@ enum scmi_clk_feats { SCMI_CLK_STATE_CTRL_SUPPORTED, SCMI_CLK_RATE_CTRL_SUPPORTED, SCMI_CLK_PARENT_CTRL_SUPPORTED, + SCMI_CLK_DUTY_CYCLE_SUPPORTED, SCMI_CLK_FEATS_COUNT }; @@ -169,6 +170,45 @@ static int scmi_clk_atomic_is_enabled(struct clk_hw *hw) return !!enabled; } +static int scmi_clk_get_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) +{ + int ret; + u32 val; + struct scmi_clk *clk = to_scmi_clk(hw); + + ret = scmi_proto_clk_ops->config_oem_get(clk->ph, clk->id, + SCMI_CLOCK_CFG_DUTY_CYCLE, + &val, NULL, false); + if (!ret) { + duty->num = val; + duty->den = 100; + } else { + dev_warn(clk->dev, + "Failed to get duty cycle for clock ID %d\n", clk->id); + } + + return ret; +} + +static int scmi_clk_set_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) +{ + int ret; + u32 val; + struct scmi_clk *clk = to_scmi_clk(hw); + + /* SCMI OEM Duty Cycle is expressed as a percentage */ + val = (duty->num * 100) / duty->den; + ret = scmi_proto_clk_ops->config_oem_set(clk->ph, clk->id, + SCMI_CLOCK_CFG_DUTY_CYCLE, + val, false); + if (ret) + dev_warn(clk->dev, + "Failed to set duty cycle(%u/%u) for clock ID %d\n", + duty->num, duty->den, clk->id); + + return ret; +} + static int scmi_clk_ops_init(struct device *dev, struct scmi_clk *sclk, const struct clk_ops *scmi_ops) { @@ -258,6 +298,12 @@ scmi_clk_ops_alloc(struct device *dev, unsigned long feats_key) if (feats_key & BIT(SCMI_CLK_PARENT_CTRL_SUPPORTED)) ops->set_parent = scmi_clk_set_parent; + /* Duty cycle */ + if (feats_key & BIT(SCMI_CLK_DUTY_CYCLE_SUPPORTED)) { + ops->get_duty_cycle = scmi_clk_get_duty_cycle; + ops->set_duty_cycle = scmi_clk_set_duty_cycle; + } + return ops; } @@ -312,6 +358,9 @@ scmi_clk_ops_select(struct scmi_clk *sclk, bool atomic_capable, if (!ci->parent_ctrl_forbidden) feats_key |= BIT(SCMI_CLK_PARENT_CTRL_SUPPORTED); + if (ci->extended_config) + feats_key |= BIT(SCMI_CLK_DUTY_CYCLE_SUPPORTED); + if (WARN_ON(feats_key >= db_size)) return NULL; From da6011a648687b5318b3705374e6120da0ec9b9a Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Fri, 5 Apr 2024 17:56:01 -0500 Subject: [PATCH 0389/1702] dt-bindings: kbuild: Simplify examples target patsubst Instead of stripping off the $(srctree) multiple times do it once up front, but keep the src/obj path as it is going to be needed in subsequent commit. Rename the variable to CHK_DT_EXAMPLES to better reflect what it contains. Signed-off-by: Rob Herring Signed-off-by: Masahiro Yamada --- Documentation/devicetree/bindings/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/Makefile b/Documentation/devicetree/bindings/Makefile index 5e08e3a6a97b1..95f1436ebcd01 100644 --- a/Documentation/devicetree/bindings/Makefile +++ b/Documentation/devicetree/bindings/Makefile @@ -32,7 +32,7 @@ find_cmd = $(find_all_cmd) | \ sed 's|^$(srctree)/||' | \ grep -F -e "$(subst :," -e ",$(DT_SCHEMA_FILES))" | \ sed 's|^|$(srctree)/|' -CHK_DT_DOCS := $(shell $(find_cmd)) +CHK_DT_EXAMPLES := $(patsubst $(srctree)/%.yaml,%.example.dtb, $(shell $(find_cmd))) quiet_cmd_yamllint = LINT $(src) cmd_yamllint = ($(find_cmd) | \ @@ -68,8 +68,8 @@ $(obj)/processed-schema.json: $(DT_DOCS) $(src)/.yamllint check_dtschema_version $(call if_changed_rule,chkdt) always-y += processed-schema.json -always-$(CHECK_DT_BINDING) += $(patsubst $(srctree)/$(src)/%.yaml,%.example.dts, $(CHK_DT_DOCS)) -always-$(CHECK_DT_BINDING) += $(patsubst $(srctree)/$(src)/%.yaml,%.example.dtb, $(CHK_DT_DOCS)) +always-$(CHECK_DT_BINDING) += $(patsubst $(obj)/%,%, $(CHK_DT_EXAMPLES)) +always-$(CHECK_DT_BINDING) += $(patsubst $(obj)/%.dtb,%.dts, $(CHK_DT_EXAMPLES)) # Hack: avoid 'Argument list too long' error for 'make clean'. Remove most of # build artifacts here before they are processed by scripts/Makefile.clean From 6552b72c3a4e8567c8f5fb2a4f6f792e2ed5dde1 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Fri, 5 Apr 2024 17:56:02 -0500 Subject: [PATCH 0390/1702] dt-bindings: kbuild: Split targets out to separate rules Masahiro pointed out the use of if_changed_rule is incorrect and command line changes are not correctly accounted for. To fix this, split up the DT binding validation target, dt_binding_check, into multiple rules for each step: yamllint, schema validtion with meta-schema, and building the processed schema. One change in behavior is the yamllint or schema validation will be re-run again when there are warnings present. Reported-by: Masahiro Yamada Link: https://lore.kernel.org/all/20220817152027.16928-1-masahiroy@kernel.org/ Signed-off-by: Rob Herring Signed-off-by: Masahiro Yamada --- Documentation/devicetree/bindings/Makefile | 25 ++++++++++++---------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/Documentation/devicetree/bindings/Makefile b/Documentation/devicetree/bindings/Makefile index 95f1436ebcd01..3779405269ab9 100644 --- a/Documentation/devicetree/bindings/Makefile +++ b/Documentation/devicetree/bindings/Makefile @@ -37,11 +37,13 @@ CHK_DT_EXAMPLES := $(patsubst $(srctree)/%.yaml,%.example.dtb, $(shell $(find_cm quiet_cmd_yamllint = LINT $(src) cmd_yamllint = ($(find_cmd) | \ xargs -n200 -P$$(nproc) \ - $(DT_SCHEMA_LINT) -f parsable -c $(srctree)/$(src)/.yamllint >&2) || true + $(DT_SCHEMA_LINT) -f parsable -c $(srctree)/$(src)/.yamllint >&2) \ + && touch $@ || true -quiet_cmd_chk_bindings = CHKDT $@ +quiet_cmd_chk_bindings = CHKDT $(src) cmd_chk_bindings = ($(find_cmd) | \ - xargs -n200 -P$$(nproc) $(DT_DOC_CHECKER) -u $(srctree)/$(src)) || true + xargs -n200 -P$$(nproc) $(DT_DOC_CHECKER) -u $(srctree)/$(src)) \ + && touch $@ || true quiet_cmd_mk_schema = SCHEMA $@ cmd_mk_schema = f=$$(mktemp) ; \ @@ -49,12 +51,6 @@ quiet_cmd_mk_schema = SCHEMA $@ $(DT_MK_SCHEMA) -j $(DT_MK_SCHEMA_FLAGS) @$$f > $@ ; \ rm -f $$f -define rule_chkdt - $(if $(DT_SCHEMA_LINT),$(call cmd,yamllint),) - $(call cmd,chk_bindings) - $(call cmd,mk_schema) -endef - DT_DOCS = $(patsubst $(srctree)/%,%,$(shell $(find_all_cmd))) override DTC_FLAGS := \ @@ -64,8 +60,15 @@ override DTC_FLAGS := \ -Wno-unique_unit_address \ -Wunique_unit_address_if_enabled -$(obj)/processed-schema.json: $(DT_DOCS) $(src)/.yamllint check_dtschema_version FORCE - $(call if_changed_rule,chkdt) +$(obj)/processed-schema.json: $(DT_DOCS) check_dtschema_version FORCE + $(call if_changed,mk_schema) + +always-$(CHECK_DT_BINDING) += .dt-binding.checked .yamllint.checked +$(obj)/.yamllint.checked: $(DT_DOCS) $(src)/.yamllint FORCE + $(if $(DT_SCHEMA_LINT),$(call if_changed,yamllint),) + +$(obj)/.dt-binding.checked: $(DT_DOCS) FORCE + $(call if_changed,chk_bindings) always-y += processed-schema.json always-$(CHECK_DT_BINDING) += $(patsubst $(obj)/%,%, $(CHK_DT_EXAMPLES)) From 3b23118bdbd898dc2f4de8f549d598d492c42ba8 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Tue, 16 Apr 2024 17:00:51 +0200 Subject: [PATCH 0391/1702] clk: renesas: r8a779a0: Fix CANFD parent clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit According to Figure 52A.1 ("RS-CANFD Module Block Diagram (in classical CAN mode)") in the R-Car V3U Series User’s Manual Rev. 0.5, the parent clock for the CANFD peripheral module clock is the S3D2 clock. Fixes: 9b621b6adff53346 ("clk: renesas: r8a779a0: Add CANFD module clock") Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/aef9300f44c9141b1465343f91c5cc7303249b6e.1713279523.git.geert+renesas@glider.be --- drivers/clk/renesas/r8a779a0-cpg-mssr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clk/renesas/r8a779a0-cpg-mssr.c b/drivers/clk/renesas/r8a779a0-cpg-mssr.c index 4c2872f45387f..ff3f85e906fe1 100644 --- a/drivers/clk/renesas/r8a779a0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a779a0-cpg-mssr.c @@ -139,7 +139,7 @@ static const struct mssr_mod_clk r8a779a0_mod_clks[] __initconst = { DEF_MOD("avb3", 214, R8A779A0_CLK_S3D2), DEF_MOD("avb4", 215, R8A779A0_CLK_S3D2), DEF_MOD("avb5", 216, R8A779A0_CLK_S3D2), - DEF_MOD("canfd0", 328, R8A779A0_CLK_CANFD), + DEF_MOD("canfd0", 328, R8A779A0_CLK_S3D2), DEF_MOD("csi40", 331, R8A779A0_CLK_CSI0), DEF_MOD("csi41", 400, R8A779A0_CLK_CSI0), DEF_MOD("csi42", 401, R8A779A0_CLK_CSI0), From 50f0cbd5cc433bf7b22633b4f37b08de64dbfba5 Mon Sep 17 00:00:00 2001 From: Cong Dang Date: Tue, 16 Apr 2024 16:56:27 +0200 Subject: [PATCH 0392/1702] clk: renesas: r8a779h0: Add MSIOF clocks Add the module clocks used by the Clock-Synchronized Serial Interfaces with FIFO (MSIOF) on the Renesas R-Car V4M (R8A779H0) SoC. Signed-off-by: Cong Dang Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/86ce05ae274d384c5221bd136415a7b0a1579592.1713279332.git.geert+renesas@glider.be --- drivers/clk/renesas/r8a779h0-cpg-mssr.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/clk/renesas/r8a779h0-cpg-mssr.c b/drivers/clk/renesas/r8a779h0-cpg-mssr.c index a7d272285db04..b9ecf90912092 100644 --- a/drivers/clk/renesas/r8a779h0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a779h0-cpg-mssr.c @@ -184,6 +184,12 @@ static const struct mssr_mod_clk r8a779h0_mod_clks[] = { DEF_MOD("i2c1", 519, R8A779H0_CLK_S0D6_PER), DEF_MOD("i2c2", 520, R8A779H0_CLK_S0D6_PER), DEF_MOD("i2c3", 521, R8A779H0_CLK_S0D6_PER), + DEF_MOD("msi0", 618, R8A779H0_CLK_MSO), + DEF_MOD("msi1", 619, R8A779H0_CLK_MSO), + DEF_MOD("msi2", 620, R8A779H0_CLK_MSO), + DEF_MOD("msi3", 621, R8A779H0_CLK_MSO), + DEF_MOD("msi4", 622, R8A779H0_CLK_MSO), + DEF_MOD("msi5", 623, R8A779H0_CLK_MSO), DEF_MOD("rpc-if", 629, R8A779H0_CLK_RPCD2), DEF_MOD("scif0", 702, R8A779H0_CLK_SASYNCPERD4), DEF_MOD("scif1", 703, R8A779H0_CLK_SASYNCPERD4), From ef9916d0e28297410583f89c329a8ba3940dd8fa Mon Sep 17 00:00:00 2001 From: Cong Dang Date: Tue, 16 Apr 2024 16:58:19 +0200 Subject: [PATCH 0393/1702] clk: renesas: r8a779h0: Add INTC-EX clock Add the module clock used by the Interrupt Controller for External Devices (INTC-EX) aka IRQC on the Renesas R-Car V4M (R8A779H0) SoC. Signed-off-by: Cong Dang Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/e260fd8eac0187c690ac6c62673b29f97e2ad5a4.1713279470.git.geert+renesas@glider.be --- drivers/clk/renesas/r8a779h0-cpg-mssr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/clk/renesas/r8a779h0-cpg-mssr.c b/drivers/clk/renesas/r8a779h0-cpg-mssr.c index b9ecf90912092..079b55b30b23b 100644 --- a/drivers/clk/renesas/r8a779h0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a779h0-cpg-mssr.c @@ -184,6 +184,7 @@ static const struct mssr_mod_clk r8a779h0_mod_clks[] = { DEF_MOD("i2c1", 519, R8A779H0_CLK_S0D6_PER), DEF_MOD("i2c2", 520, R8A779H0_CLK_S0D6_PER), DEF_MOD("i2c3", 521, R8A779H0_CLK_S0D6_PER), + DEF_MOD("irqc", 611, R8A779H0_CLK_CL16M), DEF_MOD("msi0", 618, R8A779H0_CLK_MSO), DEF_MOD("msi1", 619, R8A779H0_CLK_MSO), DEF_MOD("msi2", 620, R8A779H0_CLK_MSO), From 44019387fce230beda35b83da3a2c9fc5787704e Mon Sep 17 00:00:00 2001 From: Lad Prabhakar Date: Wed, 3 Apr 2024 21:09:52 +0100 Subject: [PATCH 0394/1702] clk: renesas: r9a07g043: Add clock and reset entry for PLIC Add the missing clock and reset entry for PLIC. Also add R9A07G043_NCEPLIC_ACLK to the critical clocks list. Fixes: 95d48d270305ad2c ("clk: renesas: r9a07g043: Add support for RZ/Five SoC") Signed-off-by: Lad Prabhakar Reviewed-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20240403200952.633084-1-prabhakar.mahadev-lad.rj@bp.renesas.com Signed-off-by: Geert Uytterhoeven --- drivers/clk/renesas/r9a07g043-cpg.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/clk/renesas/r9a07g043-cpg.c b/drivers/clk/renesas/r9a07g043-cpg.c index e36d2ec2c0f54..16acc95f3c623 100644 --- a/drivers/clk/renesas/r9a07g043-cpg.c +++ b/drivers/clk/renesas/r9a07g043-cpg.c @@ -280,6 +280,10 @@ static const struct rzg2l_mod_clk r9a07g043_mod_clks[] = { 0x5a8, 1), DEF_MOD("tsu_pclk", R9A07G043_TSU_PCLK, R9A07G043_CLK_TSU, 0x5ac, 0), +#ifdef CONFIG_RISCV + DEF_MOD("nceplic_aclk", R9A07G043_NCEPLIC_ACLK, R9A07G043_CLK_P1, + 0x608, 0), +#endif }; static const struct rzg2l_reset r9a07g043_resets[] = { @@ -338,6 +342,10 @@ static const struct rzg2l_reset r9a07g043_resets[] = { DEF_RST(R9A07G043_ADC_PRESETN, 0x8a8, 0), DEF_RST(R9A07G043_ADC_ADRST_N, 0x8a8, 1), DEF_RST(R9A07G043_TSU_PRESETN, 0x8ac, 0), +#ifdef CONFIG_RISCV + DEF_RST(R9A07G043_NCEPLIC_ARESETN, 0x908, 0), +#endif + }; static const unsigned int r9a07g043_crit_mod_clks[] __initconst = { @@ -347,6 +355,7 @@ static const unsigned int r9a07g043_crit_mod_clks[] __initconst = { #endif #ifdef CONFIG_RISCV MOD_CLK_BASE + R9A07G043_IAX45_CLK, + MOD_CLK_BASE + R9A07G043_NCEPLIC_ACLK, #endif MOD_CLK_BASE + R9A07G043_DMAC_ACLK, }; From 2487dc87aeeb58a07406f1da0ee36e069c3f42e3 Mon Sep 17 00:00:00 2001 From: Lad Prabhakar Date: Mon, 26 Feb 2024 19:25:30 +0000 Subject: [PATCH 0395/1702] pinctrl: renesas: rzg2l: Remove extra space in function parameter Remove unnecessary space in rzg2l_pinctrl_pm_setup_pfc() function parameter. Signed-off-by: Lad Prabhakar Reviewed-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20240226192530.141945-1-prabhakar.mahadev-lad.rj@bp.renesas.com Signed-off-by: Geert Uytterhoeven --- drivers/pinctrl/renesas/pinctrl-rzg2l.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pinctrl/renesas/pinctrl-rzg2l.c b/drivers/pinctrl/renesas/pinctrl-rzg2l.c index 20425afc6b331..dbcf009837ef8 100644 --- a/drivers/pinctrl/renesas/pinctrl-rzg2l.c +++ b/drivers/pinctrl/renesas/pinctrl-rzg2l.c @@ -2514,7 +2514,7 @@ static void rzg2l_pinctrl_pm_setup_dedicated_regs(struct rzg2l_pinctrl *pctrl, b } } -static void rzg2l_pinctrl_pm_setup_pfc(struct rzg2l_pinctrl *pctrl) +static void rzg2l_pinctrl_pm_setup_pfc(struct rzg2l_pinctrl *pctrl) { u32 nports = pctrl->data->n_port_pins / RZG2L_PINS_PER_PORT; const struct rzg2l_hwcfg *hwcfg = pctrl->data->hwcfg; From c3bec9547c1be0cce3060368dd92abf610c65f24 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Tue, 16 Apr 2024 17:47:22 +0200 Subject: [PATCH 0396/1702] pinctrl: renesas: r8a779h0: Fix IRQ suffixes The suffixes of the IRQ identifiers, as used for pins related to the Interrupt Controller for External Devices (INTC-EX), are inconsistent. Correct them to match the Pin Multiplex attachment in Rev.0.51 of the R-Car V4M Series Hardware User's Manual. Fixes: 291f7856fc451cbe ("pinctrl: renesas: Initial R8A779H0 (R-Car V4M) PFC support") Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/7d3c7498d9e8eda5583b15f9163eb25bb797ed24.1713282028.git.geert+renesas@glider.be --- drivers/pinctrl/renesas/pfc-r8a779h0.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/pinctrl/renesas/pfc-r8a779h0.c b/drivers/pinctrl/renesas/pfc-r8a779h0.c index afa8f06c85cf5..0cbfe7637fc97 100644 --- a/drivers/pinctrl/renesas/pfc-r8a779h0.c +++ b/drivers/pinctrl/renesas/pfc-r8a779h0.c @@ -75,10 +75,10 @@ #define GPSR0_9 F_(MSIOF5_SYNC, IP1SR0_7_4) #define GPSR0_8 F_(MSIOF5_SS1, IP1SR0_3_0) #define GPSR0_7 F_(MSIOF5_SS2, IP0SR0_31_28) -#define GPSR0_6 F_(IRQ0, IP0SR0_27_24) -#define GPSR0_5 F_(IRQ1, IP0SR0_23_20) -#define GPSR0_4 F_(IRQ2, IP0SR0_19_16) -#define GPSR0_3 F_(IRQ3, IP0SR0_15_12) +#define GPSR0_6 F_(IRQ0_A, IP0SR0_27_24) +#define GPSR0_5 F_(IRQ1_A, IP0SR0_23_20) +#define GPSR0_4 F_(IRQ2_A, IP0SR0_19_16) +#define GPSR0_3 F_(IRQ3_A, IP0SR0_15_12) #define GPSR0_2 F_(GP0_02, IP0SR0_11_8) #define GPSR0_1 F_(GP0_01, IP0SR0_7_4) #define GPSR0_0 F_(GP0_00, IP0SR0_3_0) @@ -265,10 +265,10 @@ #define IP0SR0_3_0 F_(0, 0) FM(ERROROUTC_N_B) FM(TCLK2_B) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) #define IP0SR0_7_4 F_(0, 0) FM(MSIOF3_SS1) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) #define IP0SR0_11_8 F_(0, 0) FM(MSIOF3_SS2) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) -#define IP0SR0_15_12 FM(IRQ3) FM(MSIOF3_SCK) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) -#define IP0SR0_19_16 FM(IRQ2) FM(MSIOF3_TXD) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) -#define IP0SR0_23_20 FM(IRQ1) FM(MSIOF3_RXD) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) -#define IP0SR0_27_24 FM(IRQ0) FM(MSIOF3_SYNC) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) +#define IP0SR0_15_12 FM(IRQ3_A) FM(MSIOF3_SCK) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) +#define IP0SR0_19_16 FM(IRQ2_A) FM(MSIOF3_TXD) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) +#define IP0SR0_23_20 FM(IRQ1_A) FM(MSIOF3_RXD) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) +#define IP0SR0_27_24 FM(IRQ0_A) FM(MSIOF3_SYNC) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) #define IP0SR0_31_28 FM(MSIOF5_SS2) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) /* IP1SR0 */ /* 0 */ /* 1 */ /* 2 */ /* 3 4 5 6 7 8 9 A B C D E F */ @@ -672,16 +672,16 @@ static const u16 pinmux_data[] = { PINMUX_IPSR_GPSR(IP0SR0_11_8, MSIOF3_SS2), - PINMUX_IPSR_GPSR(IP0SR0_15_12, IRQ3), + PINMUX_IPSR_GPSR(IP0SR0_15_12, IRQ3_A), PINMUX_IPSR_GPSR(IP0SR0_15_12, MSIOF3_SCK), - PINMUX_IPSR_GPSR(IP0SR0_19_16, IRQ2), + PINMUX_IPSR_GPSR(IP0SR0_19_16, IRQ2_A), PINMUX_IPSR_GPSR(IP0SR0_19_16, MSIOF3_TXD), - PINMUX_IPSR_GPSR(IP0SR0_23_20, IRQ1), + PINMUX_IPSR_GPSR(IP0SR0_23_20, IRQ1_A), PINMUX_IPSR_GPSR(IP0SR0_23_20, MSIOF3_RXD), - PINMUX_IPSR_GPSR(IP0SR0_27_24, IRQ0), + PINMUX_IPSR_GPSR(IP0SR0_27_24, IRQ0_A), PINMUX_IPSR_GPSR(IP0SR0_27_24, MSIOF3_SYNC), PINMUX_IPSR_GPSR(IP0SR0_31_28, MSIOF5_SS2), From 21fc4d195922f6b29233d2d22e9631cada7db259 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Tue, 16 Apr 2024 17:47:23 +0200 Subject: [PATCH 0397/1702] pinctrl: renesas: r8a779h0: Add INTC-EX pins, groups, and function Add pins, groups, and function for the Interrupt Controller for External Devices (INTC-EX) on the Renesas R-Car V4M (R8A779H0) SoC. Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/258d03b27b77f60cc03fc3257bb4c6715b612a61.1713282028.git.geert+renesas@glider.be --- drivers/pinctrl/renesas/pfc-r8a779h0.c | 112 +++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/drivers/pinctrl/renesas/pfc-r8a779h0.c b/drivers/pinctrl/renesas/pfc-r8a779h0.c index 0cbfe7637fc97..438d1f2739dd4 100644 --- a/drivers/pinctrl/renesas/pfc-r8a779h0.c +++ b/drivers/pinctrl/renesas/pfc-r8a779h0.c @@ -1660,6 +1660,90 @@ static const unsigned int i2c3_mux[] = { SDA3_MARK, SCL3_MARK, }; +/* - INTC-EX ---------------------------------------------------------------- */ +static const unsigned int intc_ex_irq0_a_pins[] = { + /* IRQ0_A */ + RCAR_GP_PIN(0, 6), +}; +static const unsigned int intc_ex_irq0_a_mux[] = { + IRQ0_A_MARK, +}; +static const unsigned int intc_ex_irq0_b_pins[] = { + /* IRQ0_B */ + RCAR_GP_PIN(1, 20), +}; +static const unsigned int intc_ex_irq0_b_mux[] = { + IRQ0_B_MARK, +}; + +static const unsigned int intc_ex_irq1_a_pins[] = { + /* IRQ1_A */ + RCAR_GP_PIN(0, 5), +}; +static const unsigned int intc_ex_irq1_a_mux[] = { + IRQ1_A_MARK, +}; +static const unsigned int intc_ex_irq1_b_pins[] = { + /* IRQ1_B */ + RCAR_GP_PIN(1, 21), +}; +static const unsigned int intc_ex_irq1_b_mux[] = { + IRQ1_B_MARK, +}; + +static const unsigned int intc_ex_irq2_a_pins[] = { + /* IRQ2_A */ + RCAR_GP_PIN(0, 4), +}; +static const unsigned int intc_ex_irq2_a_mux[] = { + IRQ2_A_MARK, +}; +static const unsigned int intc_ex_irq2_b_pins[] = { + /* IRQ2_B */ + RCAR_GP_PIN(0, 13), +}; +static const unsigned int intc_ex_irq2_b_mux[] = { + IRQ2_B_MARK, +}; + +static const unsigned int intc_ex_irq3_a_pins[] = { + /* IRQ3_A */ + RCAR_GP_PIN(0, 3), +}; +static const unsigned int intc_ex_irq3_a_mux[] = { + IRQ3_A_MARK, +}; +static const unsigned int intc_ex_irq3_b_pins[] = { + /* IRQ3_B */ + RCAR_GP_PIN(1, 23), +}; +static const unsigned int intc_ex_irq3_b_mux[] = { + IRQ3_B_MARK, +}; + +static const unsigned int intc_ex_irq4_a_pins[] = { + /* IRQ4_A */ + RCAR_GP_PIN(1, 17), +}; +static const unsigned int intc_ex_irq4_a_mux[] = { + IRQ4_A_MARK, +}; +static const unsigned int intc_ex_irq4_b_pins[] = { + /* IRQ4_B */ + RCAR_GP_PIN(2, 3), +}; +static const unsigned int intc_ex_irq4_b_mux[] = { + IRQ4_B_MARK, +}; + +static const unsigned int intc_ex_irq5_pins[] = { + /* IRQ5 */ + RCAR_GP_PIN(2, 2), +}; +static const unsigned int intc_ex_irq5_mux[] = { + IRQ5_MARK, +}; + /* - MMC -------------------------------------------------------------------- */ static const unsigned int mmc_data_pins[] = { /* MMC_SD_D[0:3], MMC_D[4:7] */ @@ -2416,6 +2500,18 @@ static const struct sh_pfc_pin_group pinmux_groups[] = { SH_PFC_PIN_GROUP(i2c2), SH_PFC_PIN_GROUP(i2c3), + SH_PFC_PIN_GROUP(intc_ex_irq0_a), + SH_PFC_PIN_GROUP(intc_ex_irq0_b), + SH_PFC_PIN_GROUP(intc_ex_irq1_a), + SH_PFC_PIN_GROUP(intc_ex_irq1_b), + SH_PFC_PIN_GROUP(intc_ex_irq2_a), + SH_PFC_PIN_GROUP(intc_ex_irq2_b), + SH_PFC_PIN_GROUP(intc_ex_irq3_a), + SH_PFC_PIN_GROUP(intc_ex_irq3_b), + SH_PFC_PIN_GROUP(intc_ex_irq4_a), + SH_PFC_PIN_GROUP(intc_ex_irq4_b), + SH_PFC_PIN_GROUP(intc_ex_irq5), + BUS_DATA_PIN_GROUP(mmc_data, 1), BUS_DATA_PIN_GROUP(mmc_data, 4), BUS_DATA_PIN_GROUP(mmc_data, 8), @@ -2629,6 +2725,20 @@ static const char * const i2c3_groups[] = { "i2c3", }; +static const char * const intc_ex_groups[] = { + "intc_ex_irq0_a", + "intc_ex_irq0_b", + "intc_ex_irq1_a", + "intc_ex_irq1_b", + "intc_ex_irq2_a", + "intc_ex_irq2_b", + "intc_ex_irq3_a", + "intc_ex_irq3_b", + "intc_ex_irq4_a", + "intc_ex_irq4_b", + "intc_ex_irq5", +}; + static const char * const mmc_groups[] = { "mmc_data1", "mmc_data4", @@ -2813,6 +2923,8 @@ static const struct sh_pfc_function pinmux_functions[] = { SH_PFC_FUNCTION(i2c2), SH_PFC_FUNCTION(i2c3), + SH_PFC_FUNCTION(intc_ex), + SH_PFC_FUNCTION(mmc), SH_PFC_FUNCTION(msiof0), From b591dfb8330e30da0dcc8f5466c6f18173cdab32 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 17 Apr 2024 16:04:07 +0100 Subject: [PATCH 0398/1702] udf: Convert udf_symlink_filler() to use a folio Remove the conversion to struct page and use folio APIs throughout. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Jan Kara Message-Id: <20240417150416.752929-2-willy@infradead.org> --- fs/udf/symlink.c | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/fs/udf/symlink.c b/fs/udf/symlink.c index f7eaf7b145940..0105e7e2ba3d9 100644 --- a/fs/udf/symlink.c +++ b/fs/udf/symlink.c @@ -99,18 +99,17 @@ static int udf_pc_to_char(struct super_block *sb, unsigned char *from, static int udf_symlink_filler(struct file *file, struct folio *folio) { - struct page *page = &folio->page; - struct inode *inode = page->mapping->host; + struct inode *inode = folio->mapping->host; struct buffer_head *bh = NULL; unsigned char *symlink; int err = 0; - unsigned char *p = page_address(page); + unsigned char *p = folio_address(folio); struct udf_inode_info *iinfo = UDF_I(inode); /* We don't support symlinks longer than one block */ if (inode->i_size > inode->i_sb->s_blocksize) { err = -ENAMETOOLONG; - goto out_unlock; + goto out; } if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { @@ -120,24 +119,15 @@ static int udf_symlink_filler(struct file *file, struct folio *folio) if (!bh) { if (!err) err = -EFSCORRUPTED; - goto out_err; + goto out; } symlink = bh->b_data; } err = udf_pc_to_char(inode->i_sb, symlink, inode->i_size, p, PAGE_SIZE); brelse(bh); - if (err) - goto out_err; - - SetPageUptodate(page); - unlock_page(page); - return 0; - -out_err: - SetPageError(page); -out_unlock: - unlock_page(page); +out: + folio_end_read(folio, err == 0); return err; } From d08f069cc2dafddf81b23db2ead45539b12cd0c2 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 17 Apr 2024 16:04:08 +0100 Subject: [PATCH 0399/1702] udf: Convert udf_write_begin() to use a folio Use the folio APIs throughout instead of the deprecated page APIs. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Jan Kara Message-Id: <20240417150416.752929-3-willy@infradead.org> --- fs/udf/inode.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 2f831a3a91afe..5146b9d7aba36 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -254,7 +254,7 @@ static int udf_write_begin(struct file *file, struct address_space *mapping, struct page **pagep, void **fsdata) { struct udf_inode_info *iinfo = UDF_I(file_inode(file)); - struct page *page; + struct folio *folio; int ret; if (iinfo->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) { @@ -266,12 +266,13 @@ static int udf_write_begin(struct file *file, struct address_space *mapping, } if (WARN_ON_ONCE(pos >= PAGE_SIZE)) return -EIO; - page = grab_cache_page_write_begin(mapping, 0); - if (!page) - return -ENOMEM; - *pagep = page; - if (!PageUptodate(page)) - udf_adinicb_readpage(page); + folio = __filemap_get_folio(mapping, 0, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) + return PTR_ERR(folio); + *pagep = &folio->page; + if (!folio_test_uptodate(folio)) + udf_adinicb_readpage(&folio->page); return 0; } From db6754090a4f99c67e05ae6b87343ba6e013531f Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 17 Apr 2024 16:04:09 +0100 Subject: [PATCH 0400/1702] udf: Convert udf_expand_file_adinicb() to use a folio Use the folio APIs throughout this function. Signed-off-by: Matthew Wilcox (Oracle) Fixes: 1eeceaec794e ("udf: Convert udf_expand_file_adinicb() to avoid kmap_atomic()") Signed-off-by: Jan Kara Message-Id: <20240417150416.752929-4-willy@infradead.org> --- fs/udf/inode.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 5146b9d7aba36..59215494e6f6f 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -342,7 +342,7 @@ const struct address_space_operations udf_aops = { */ int udf_expand_file_adinicb(struct inode *inode) { - struct page *page; + struct folio *folio; struct udf_inode_info *iinfo = UDF_I(inode); int err; @@ -358,12 +358,13 @@ int udf_expand_file_adinicb(struct inode *inode) return 0; } - page = find_or_create_page(inode->i_mapping, 0, GFP_KERNEL); - if (!page) - return -ENOMEM; + folio = __filemap_get_folio(inode->i_mapping, 0, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_KERNEL); + if (IS_ERR(folio)) + return PTR_ERR(folio); - if (!PageUptodate(page)) - udf_adinicb_readpage(page); + if (!folio_test_uptodate(folio)) + udf_adinicb_readpage(&folio->page); down_write(&iinfo->i_data_sem); memset(iinfo->i_data + iinfo->i_lenEAttr, 0x00, iinfo->i_lenAlloc); @@ -372,22 +373,22 @@ int udf_expand_file_adinicb(struct inode *inode) iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; else iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; - set_page_dirty(page); - unlock_page(page); + folio_mark_dirty(folio); + folio_unlock(folio); up_write(&iinfo->i_data_sem); err = filemap_fdatawrite(inode->i_mapping); if (err) { /* Restore everything back so that we don't lose data... */ - lock_page(page); + folio_lock(folio); down_write(&iinfo->i_data_sem); - memcpy_to_page(page, 0, iinfo->i_data + iinfo->i_lenEAttr, - inode->i_size); - unlock_page(page); + memcpy_from_folio(iinfo->i_data + iinfo->i_lenEAttr, + folio, 0, inode->i_size); + folio_unlock(folio); iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB; iinfo->i_lenAlloc = inode->i_size; up_write(&iinfo->i_data_sem); } - put_page(page); + folio_put(folio); mark_inode_dirty(inode); return err; From d257d924a3fc6fe26594fba221c8be62f043b8d0 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 17 Apr 2024 16:04:10 +0100 Subject: [PATCH 0401/1702] udf: Convert udf_adinicb_readpage() to udf_adinicb_read_folio() Now that all three callers have a folio, convert this function to take a folio, and use the folio APIs. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Jan Kara Message-Id: <20240417150416.752929-5-willy@infradead.org> --- fs/udf/inode.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 59215494e6f6f..d345621565225 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -208,19 +208,14 @@ static int udf_writepages(struct address_space *mapping, return write_cache_pages(mapping, wbc, udf_adinicb_writepage, NULL); } -static void udf_adinicb_readpage(struct page *page) +static void udf_adinicb_read_folio(struct folio *folio) { - struct inode *inode = page->mapping->host; - char *kaddr; + struct inode *inode = folio->mapping->host; struct udf_inode_info *iinfo = UDF_I(inode); loff_t isize = i_size_read(inode); - kaddr = kmap_local_page(page); - memcpy(kaddr, iinfo->i_data + iinfo->i_lenEAttr, isize); - memset(kaddr + isize, 0, PAGE_SIZE - isize); - flush_dcache_page(page); - SetPageUptodate(page); - kunmap_local(kaddr); + folio_fill_tail(folio, 0, iinfo->i_data + iinfo->i_lenEAttr, isize); + folio_mark_uptodate(folio); } static int udf_read_folio(struct file *file, struct folio *folio) @@ -228,7 +223,7 @@ static int udf_read_folio(struct file *file, struct folio *folio) struct udf_inode_info *iinfo = UDF_I(file_inode(file)); if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { - udf_adinicb_readpage(&folio->page); + udf_adinicb_read_folio(folio); folio_unlock(folio); return 0; } @@ -272,7 +267,7 @@ static int udf_write_begin(struct file *file, struct address_space *mapping, return PTR_ERR(folio); *pagep = &folio->page; if (!folio_test_uptodate(folio)) - udf_adinicb_readpage(&folio->page); + udf_adinicb_read_folio(folio); return 0; } @@ -364,7 +359,7 @@ int udf_expand_file_adinicb(struct inode *inode) return PTR_ERR(folio); if (!folio_test_uptodate(folio)) - udf_adinicb_readpage(&folio->page); + udf_adinicb_read_folio(folio); down_write(&iinfo->i_data_sem); memset(iinfo->i_data + iinfo->i_lenEAttr, 0x00, iinfo->i_lenAlloc); From 2f1c1bd7b18768377ff4a84974d77e6e8b371a67 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 17 Apr 2024 16:04:11 +0100 Subject: [PATCH 0402/1702] udf: Convert udf_symlink_getattr() to use a folio We're getting this from the page cache, so it's definitely a folio. Saves a call to compound_head() hidden in put_page(). Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Jan Kara Message-Id: <20240417150416.752929-6-willy@infradead.org> --- fs/udf/symlink.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fs/udf/symlink.c b/fs/udf/symlink.c index 0105e7e2ba3d9..fe03745d09b18 100644 --- a/fs/udf/symlink.c +++ b/fs/udf/symlink.c @@ -137,12 +137,12 @@ static int udf_symlink_getattr(struct mnt_idmap *idmap, { struct dentry *dentry = path->dentry; struct inode *inode = d_backing_inode(dentry); - struct page *page; + struct folio *folio; generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); - page = read_mapping_page(inode->i_mapping, 0, NULL); - if (IS_ERR(page)) - return PTR_ERR(page); + folio = read_mapping_folio(inode->i_mapping, 0, NULL); + if (IS_ERR(folio)) + return PTR_ERR(folio); /* * UDF uses non-trivial encoding of symlinks so i_size does not match * number of characters reported by readlink(2) which apparently some @@ -152,8 +152,8 @@ static int udf_symlink_getattr(struct mnt_idmap *idmap, * let's report the length of string returned by readlink(2) for * st_size. */ - stat->size = strlen(page_address(page)); - put_page(page); + stat->size = strlen(folio_address(folio)); + folio_put(folio); return 0; } From f5985ef281f9efff3e8291123fdb8a748b506951 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 17 Apr 2024 16:04:12 +0100 Subject: [PATCH 0403/1702] udf: Convert udf_page_mkwrite() to use a folio Convert the vm_fault page to a folio, then use it throughout. Replaces five calls to compound_head() with one. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Jan Kara Message-Id: <20240417150416.752929-7-willy@infradead.org> --- fs/udf/file.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/fs/udf/file.c b/fs/udf/file.c index 0ceac4b5937c7..97c59585208ca 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -39,7 +39,7 @@ static vm_fault_t udf_page_mkwrite(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; struct inode *inode = file_inode(vma->vm_file); struct address_space *mapping = inode->i_mapping; - struct page *page = vmf->page; + struct folio *folio = page_folio(vmf->page); loff_t size; unsigned int end; vm_fault_t ret = VM_FAULT_LOCKED; @@ -48,31 +48,31 @@ static vm_fault_t udf_page_mkwrite(struct vm_fault *vmf) sb_start_pagefault(inode->i_sb); file_update_time(vma->vm_file); filemap_invalidate_lock_shared(mapping); - lock_page(page); + folio_lock(folio); size = i_size_read(inode); - if (page->mapping != inode->i_mapping || page_offset(page) >= size) { - unlock_page(page); + if (folio->mapping != inode->i_mapping || folio_pos(folio) >= size) { + folio_unlock(folio); ret = VM_FAULT_NOPAGE; goto out_unlock; } /* Space is already allocated for in-ICB file */ if (UDF_I(inode)->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) goto out_dirty; - if (page->index == size >> PAGE_SHIFT) + if (folio->index == size >> PAGE_SHIFT) end = size & ~PAGE_MASK; else end = PAGE_SIZE; - err = __block_write_begin(page, 0, end, udf_get_block); + err = __block_write_begin(&folio->page, 0, end, udf_get_block); if (err) { - unlock_page(page); + folio_unlock(folio); ret = vmf_fs_error(err); goto out_unlock; } - block_commit_write(page, 0, end); + block_commit_write(&folio->page, 0, end); out_dirty: - set_page_dirty(page); - wait_for_stable_page(page); + folio_mark_dirty(folio); + folio_wait_stable(folio); out_unlock: filemap_invalidate_unlock_shared(mapping); sb_end_pagefault(inode->i_sb); From e29741676fac5d4430b3d2b799a5ff671ea9f023 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 17 Apr 2024 16:04:13 +0100 Subject: [PATCH 0404/1702] udf: Use a folio in udf_write_end() Convert the page to a folio and use the folio APIs. Replaces three calls to compound_head() with one. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Jan Kara Message-Id: <20240417150416.752929-8-willy@infradead.org> --- fs/udf/inode.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index d345621565225..2fb21c5ffccfe 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -276,17 +276,19 @@ static int udf_write_end(struct file *file, struct address_space *mapping, struct page *page, void *fsdata) { struct inode *inode = file_inode(file); + struct folio *folio; loff_t last_pos; if (UDF_I(inode)->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) return generic_write_end(file, mapping, pos, len, copied, page, fsdata); + folio = page_folio(page); last_pos = pos + copied; if (last_pos > inode->i_size) i_size_write(inode, last_pos); - set_page_dirty(page); - unlock_page(page); - put_page(page); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); return copied; } From f566d5b9fb7136d39d4e9c54d84c82835b539b4e Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:19 -0700 Subject: [PATCH 0405/1702] xfs: remove XFS_DA_OP_REMOVE Nobody checks this flag, so get rid of it. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.h | 1 - fs/xfs/libxfs/xfs_da_btree.h | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h index e4f55008552b4..670ab2a613fc6 100644 --- a/fs/xfs/libxfs/xfs_attr.h +++ b/fs/xfs/libxfs/xfs_attr.h @@ -590,7 +590,6 @@ xfs_attr_init_add_state(struct xfs_da_args *args) static inline enum xfs_delattr_state xfs_attr_init_remove_state(struct xfs_da_args *args) { - args->op_flags |= XFS_DA_OP_REMOVE; if (xfs_attr_is_shortform(args->dp)) return XFS_DAS_SF_REMOVE; if (xfs_attr_is_leaf(args->dp)) diff --git a/fs/xfs/libxfs/xfs_da_btree.h b/fs/xfs/libxfs/xfs_da_btree.h index 7a004786ee0a2..76e764080d994 100644 --- a/fs/xfs/libxfs/xfs_da_btree.h +++ b/fs/xfs/libxfs/xfs_da_btree.h @@ -91,9 +91,8 @@ typedef struct xfs_da_args { #define XFS_DA_OP_OKNOENT (1u << 3) /* lookup op, ENOENT ok, else die */ #define XFS_DA_OP_CILOOKUP (1u << 4) /* lookup returns CI name if found */ #define XFS_DA_OP_NOTIME (1u << 5) /* don't update inode timestamps */ -#define XFS_DA_OP_REMOVE (1u << 6) /* this is a remove operation */ -#define XFS_DA_OP_RECOVERY (1u << 7) /* Log recovery operation */ -#define XFS_DA_OP_LOGGED (1u << 8) /* Use intent items to track op */ +#define XFS_DA_OP_RECOVERY (1u << 6) /* Log recovery operation */ +#define XFS_DA_OP_LOGGED (1u << 7) /* Use intent items to track op */ #define XFS_DA_OP_FLAGS \ { XFS_DA_OP_JUSTCHECK, "JUSTCHECK" }, \ @@ -102,7 +101,6 @@ typedef struct xfs_da_args { { XFS_DA_OP_OKNOENT, "OKNOENT" }, \ { XFS_DA_OP_CILOOKUP, "CILOOKUP" }, \ { XFS_DA_OP_NOTIME, "NOTIME" }, \ - { XFS_DA_OP_REMOVE, "REMOVE" }, \ { XFS_DA_OP_RECOVERY, "RECOVERY" }, \ { XFS_DA_OP_LOGGED, "LOGGED" } From 779a4b606c7678343e3603398fe07de38b817ef4 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:20 -0700 Subject: [PATCH 0406/1702] xfs: remove XFS_DA_OP_NOTIME The only user of this flag sets it prior to an xfs_attr_get_ilocked call, which doesn't update anything. Get rid of the flag. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 5 ++--- fs/xfs/libxfs/xfs_da_btree.h | 6 ++---- fs/xfs/scrub/attr.c | 1 - 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 05d22c5e38855..30e6084122d8b 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -365,7 +365,7 @@ xfs_attr_try_sf_addname( * Commit the shortform mods, and we're done. * NOTE: this is also the error path (EEXIST, etc). */ - if (!error && !(args->op_flags & XFS_DA_OP_NOTIME)) + if (!error) xfs_trans_ichgtime(args->trans, dp, XFS_ICHGTIME_CHG); if (xfs_has_wsync(dp->i_mount)) @@ -1033,8 +1033,7 @@ xfs_attr_set( if (xfs_has_wsync(mp)) xfs_trans_set_sync(args->trans); - if (!(args->op_flags & XFS_DA_OP_NOTIME)) - xfs_trans_ichgtime(args->trans, dp, XFS_ICHGTIME_CHG); + xfs_trans_ichgtime(args->trans, dp, XFS_ICHGTIME_CHG); /* * Commit the last in the sequence of transactions. diff --git a/fs/xfs/libxfs/xfs_da_btree.h b/fs/xfs/libxfs/xfs_da_btree.h index 76e764080d994..b04a3290ffacc 100644 --- a/fs/xfs/libxfs/xfs_da_btree.h +++ b/fs/xfs/libxfs/xfs_da_btree.h @@ -90,9 +90,8 @@ typedef struct xfs_da_args { #define XFS_DA_OP_ADDNAME (1u << 2) /* this is an add operation */ #define XFS_DA_OP_OKNOENT (1u << 3) /* lookup op, ENOENT ok, else die */ #define XFS_DA_OP_CILOOKUP (1u << 4) /* lookup returns CI name if found */ -#define XFS_DA_OP_NOTIME (1u << 5) /* don't update inode timestamps */ -#define XFS_DA_OP_RECOVERY (1u << 6) /* Log recovery operation */ -#define XFS_DA_OP_LOGGED (1u << 7) /* Use intent items to track op */ +#define XFS_DA_OP_RECOVERY (1u << 5) /* Log recovery operation */ +#define XFS_DA_OP_LOGGED (1u << 6) /* Use intent items to track op */ #define XFS_DA_OP_FLAGS \ { XFS_DA_OP_JUSTCHECK, "JUSTCHECK" }, \ @@ -100,7 +99,6 @@ typedef struct xfs_da_args { { XFS_DA_OP_ADDNAME, "ADDNAME" }, \ { XFS_DA_OP_OKNOENT, "OKNOENT" }, \ { XFS_DA_OP_CILOOKUP, "CILOOKUP" }, \ - { XFS_DA_OP_NOTIME, "NOTIME" }, \ { XFS_DA_OP_RECOVERY, "RECOVERY" }, \ { XFS_DA_OP_LOGGED, "LOGGED" } diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index 8853e4d0eee3d..5b855d7c98211 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -173,7 +173,6 @@ xchk_xattr_actor( void *priv) { struct xfs_da_args args = { - .op_flags = XFS_DA_OP_NOTIME, .attr_filter = attr_flags & XFS_ATTR_NSP_ONDISK_MASK, .geo = sc->mp->m_attr_geo, .whichfork = XFS_ATTR_FORK, From 54275d8496f3e4764302cebc0e9517d950ba6589 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:21 -0700 Subject: [PATCH 0407/1702] xfs: remove xfs_da_args.attr_flags This field only ever contains XATTR_{CREATE,REPLACE}, and it only goes as deep as xfs_attr_set. Remove the field from the structure and replace it with an enum specifying exactly what kind of change we want to make to the xattr structure. Upsert is the name that we'll give to the flags==0 operation, because we're either updating an existing value or inserting it, and the caller doesn't care. Note: The "UPSERTR" name created here is to make userspace porting easier. It will be removed in the next patch. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 7 ++++--- fs/xfs/libxfs/xfs_attr.h | 9 ++++++++- fs/xfs/libxfs/xfs_da_btree.h | 1 - fs/xfs/scrub/attr_repair.c | 3 +-- fs/xfs/xfs_acl.c | 2 +- fs/xfs/xfs_ioctl.c | 14 ++++++-------- fs/xfs/xfs_iops.c | 2 +- fs/xfs/xfs_trace.h | 7 +------ fs/xfs/xfs_xattr.c | 19 +++++++++++++++---- fs/xfs/xfs_xattr.h | 3 ++- 10 files changed, 39 insertions(+), 28 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 30e6084122d8b..b04e09143869d 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -922,7 +922,8 @@ xfs_attr_defer_add( */ int xfs_attr_set( - struct xfs_da_args *args) + struct xfs_da_args *args, + enum xfs_attr_update op) { struct xfs_inode *dp = args->dp; struct xfs_mount *mp = dp->i_mount; @@ -1008,7 +1009,7 @@ xfs_attr_set( } /* Pure create fails if the attr already exists */ - if (args->attr_flags & XATTR_CREATE) + if (op == XFS_ATTRUPDATE_CREATE) goto out_trans_cancel; xfs_attr_defer_add(args, XFS_ATTRI_OP_FLAGS_REPLACE); break; @@ -1018,7 +1019,7 @@ xfs_attr_set( goto out_trans_cancel; /* Pure replace fails if no existing attr to replace. */ - if (args->attr_flags & XATTR_REPLACE) + if (op == XFS_ATTRUPDATE_REPLACE) goto out_trans_cancel; xfs_attr_defer_add(args, XFS_ATTRI_OP_FLAGS_SET); break; diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h index 670ab2a613fc6..228360f7c85ce 100644 --- a/fs/xfs/libxfs/xfs_attr.h +++ b/fs/xfs/libxfs/xfs_attr.h @@ -544,7 +544,14 @@ int xfs_inode_hasattr(struct xfs_inode *ip); bool xfs_attr_is_leaf(struct xfs_inode *ip); int xfs_attr_get_ilocked(struct xfs_da_args *args); int xfs_attr_get(struct xfs_da_args *args); -int xfs_attr_set(struct xfs_da_args *args); + +enum xfs_attr_update { + XFS_ATTRUPDATE_UPSERTR, /* set/remove value, replace any existing attr */ + XFS_ATTRUPDATE_CREATE, /* set value, fail if attr already exists */ + XFS_ATTRUPDATE_REPLACE, /* set value, fail if attr does not exist */ +}; + +int xfs_attr_set(struct xfs_da_args *args, enum xfs_attr_update op); int xfs_attr_set_iter(struct xfs_attr_intent *attr); int xfs_attr_remove_iter(struct xfs_attr_intent *attr); bool xfs_attr_namecheck(const void *name, size_t length); diff --git a/fs/xfs/libxfs/xfs_da_btree.h b/fs/xfs/libxfs/xfs_da_btree.h index b04a3290ffacc..706b529a81feb 100644 --- a/fs/xfs/libxfs/xfs_da_btree.h +++ b/fs/xfs/libxfs/xfs_da_btree.h @@ -60,7 +60,6 @@ typedef struct xfs_da_args { void *value; /* set of bytes (maybe contain NULLs) */ int valuelen; /* length of value */ unsigned int attr_filter; /* XFS_ATTR_{ROOT,SECURE,INCOMPLETE} */ - unsigned int attr_flags; /* XATTR_{CREATE,REPLACE} */ xfs_dahash_t hashval; /* hash value of name */ xfs_ino_t inumber; /* input/output inode number */ struct xfs_inode *dp; /* directory inode to manipulate */ diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c index 7b4318764d030..3066d662ea13f 100644 --- a/fs/xfs/scrub/attr_repair.c +++ b/fs/xfs/scrub/attr_repair.c @@ -557,7 +557,6 @@ xrep_xattr_insert_rec( struct xfs_da_args args = { .dp = rx->sc->tempip, .attr_filter = key->flags, - .attr_flags = XATTR_CREATE, .namelen = key->namelen, .valuelen = key->valuelen, .owner = rx->sc->ip->i_ino, @@ -605,7 +604,7 @@ xrep_xattr_insert_rec( * xfs_attr_set creates and commits its own transaction. If the attr * already exists, we'll just drop it during the rebuild. */ - error = xfs_attr_set(&args); + error = xfs_attr_set(&args, XFS_ATTRUPDATE_CREATE); if (error == -EEXIST) error = 0; diff --git a/fs/xfs/xfs_acl.c b/fs/xfs/xfs_acl.c index 4bf69c9c088e2..12f98af9e7092 100644 --- a/fs/xfs/xfs_acl.c +++ b/fs/xfs/xfs_acl.c @@ -203,7 +203,7 @@ __xfs_set_acl(struct inode *inode, struct posix_acl *acl, int type) xfs_acl_to_disk(args.value, acl); } - error = xfs_attr_change(&args); + error = xfs_attr_change(&args, XFS_ATTRUPDATE_UPSERTR); kvfree(args.value); /* diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index efa95892655d4..0eca7a9096e23 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -361,15 +361,15 @@ xfs_attr_filter( return 0; } -static unsigned int -xfs_attr_flags( +static inline enum xfs_attr_update +xfs_xattr_flags( u32 ioc_flags) { if (ioc_flags & XFS_IOC_ATTR_CREATE) - return XATTR_CREATE; + return XFS_ATTRUPDATE_CREATE; if (ioc_flags & XFS_IOC_ATTR_REPLACE) - return XATTR_REPLACE; - return 0; + return XFS_ATTRUPDATE_REPLACE; + return XFS_ATTRUPDATE_UPSERTR; } int @@ -476,7 +476,6 @@ xfs_attrmulti_attr_get( struct xfs_da_args args = { .dp = XFS_I(inode), .attr_filter = xfs_attr_filter(flags), - .attr_flags = xfs_attr_flags(flags), .name = name, .namelen = strlen(name), .valuelen = *len, @@ -510,7 +509,6 @@ xfs_attrmulti_attr_set( struct xfs_da_args args = { .dp = XFS_I(inode), .attr_filter = xfs_attr_filter(flags), - .attr_flags = xfs_attr_flags(flags), .name = name, .namelen = strlen(name), }; @@ -528,7 +526,7 @@ xfs_attrmulti_attr_set( args.valuelen = len; } - error = xfs_attr_change(&args); + error = xfs_attr_change(&args, xfs_xattr_flags(flags)); if (!error && (flags & XFS_IOC_ATTR_ROOT)) xfs_forget_acl(inode, name); kfree(args.value); diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index ad76704ab1332..d02ca2248bbcf 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -63,7 +63,7 @@ xfs_initxattrs( .value = xattr->value, .valuelen = xattr->value_len, }; - error = xfs_attr_change(&args); + error = xfs_attr_change(&args, XFS_ATTRUPDATE_UPSERTR); if (error < 0) break; } diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index 08df5bb70a6c7..57f225ba7e8ab 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -1999,7 +1999,6 @@ DECLARE_EVENT_CLASS(xfs_attr_class, __field(int, valuelen) __field(xfs_dahash_t, hashval) __field(unsigned int, attr_filter) - __field(unsigned int, attr_flags) __field(uint32_t, op_flags) ), TP_fast_assign( @@ -2011,11 +2010,10 @@ DECLARE_EVENT_CLASS(xfs_attr_class, __entry->valuelen = args->valuelen; __entry->hashval = args->hashval; __entry->attr_filter = args->attr_filter; - __entry->attr_flags = args->attr_flags; __entry->op_flags = args->op_flags; ), TP_printk("dev %d:%d ino 0x%llx name %.*s namelen %d valuelen %d " - "hashval 0x%x filter %s flags %s op_flags %s", + "hashval 0x%x filter %s op_flags %s", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, __entry->namelen, @@ -2025,9 +2023,6 @@ DECLARE_EVENT_CLASS(xfs_attr_class, __entry->hashval, __print_flags(__entry->attr_filter, "|", XFS_ATTR_FILTER_FLAGS), - __print_flags(__entry->attr_flags, "|", - { XATTR_CREATE, "CREATE" }, - { XATTR_REPLACE, "REPLACE" }), __print_flags(__entry->op_flags, "|", XFS_DA_OP_FLAGS)) ) diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c index 4ebf7052eb673..6ce1e06f650a5 100644 --- a/fs/xfs/xfs_xattr.c +++ b/fs/xfs/xfs_xattr.c @@ -73,7 +73,8 @@ xfs_attr_want_log_assist( */ int xfs_attr_change( - struct xfs_da_args *args) + struct xfs_da_args *args, + enum xfs_attr_update op) { struct xfs_mount *mp = args->dp->i_mount; int error; @@ -88,7 +89,7 @@ xfs_attr_change( args->op_flags |= XFS_DA_OP_LOGGED; } - return xfs_attr_set(args); + return xfs_attr_set(args, op); } @@ -115,6 +116,17 @@ xfs_xattr_get(const struct xattr_handler *handler, struct dentry *unused, return args.valuelen; } +static inline enum xfs_attr_update +xfs_xattr_flags_to_op( + int flags) +{ + if (flags & XATTR_CREATE) + return XFS_ATTRUPDATE_CREATE; + if (flags & XATTR_REPLACE) + return XFS_ATTRUPDATE_REPLACE; + return XFS_ATTRUPDATE_UPSERTR; +} + static int xfs_xattr_set(const struct xattr_handler *handler, struct mnt_idmap *idmap, struct dentry *unused, @@ -124,7 +136,6 @@ xfs_xattr_set(const struct xattr_handler *handler, struct xfs_da_args args = { .dp = XFS_I(inode), .attr_filter = handler->flags, - .attr_flags = flags, .name = name, .namelen = strlen(name), .value = (void *)value, @@ -132,7 +143,7 @@ xfs_xattr_set(const struct xattr_handler *handler, }; int error; - error = xfs_attr_change(&args); + error = xfs_attr_change(&args, xfs_xattr_flags_to_op(flags)); if (!error && (handler->flags & XFS_ATTR_ROOT)) xfs_forget_acl(inode, name); return error; diff --git a/fs/xfs/xfs_xattr.h b/fs/xfs/xfs_xattr.h index cec766cad26cd..c3eb858fb59e7 100644 --- a/fs/xfs/xfs_xattr.h +++ b/fs/xfs/xfs_xattr.h @@ -6,7 +6,8 @@ #ifndef __XFS_XATTR_H__ #define __XFS_XATTR_H__ -int xfs_attr_change(struct xfs_da_args *args); +enum xfs_attr_update; +int xfs_attr_change(struct xfs_da_args *args, enum xfs_attr_update op); extern const struct xattr_handler * const xfs_xattr_handlers[]; From ef80de940a6344da1d4f12c948a0ad4d6ff6e841 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:24 -0700 Subject: [PATCH 0408/1702] xfs: attr fork iext must be loaded before calling xfs_attr_is_leaf Christoph noticed that the xfs_attr_is_leaf in xfs_attr_get_ilocked can access the incore extent tree of the attr fork, but nothing in the xfs_attr_get path guarantees that the incore tree is actually loaded. Most of the time it is, but seeing as xfs_attr_is_leaf ignores the return value of xfs_iext_get_extent I guess we've been making choices based on random stack contents and nobody's complained? Reported-by: Christoph Hellwig Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 17 ++++++++++++++++ fs/xfs/xfs_attr_item.c | 42 ++++++++++++++++++++++++++++++++++------ fs/xfs/xfs_attr_list.c | 7 +++++++ 3 files changed, 60 insertions(+), 6 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index f8f7445b063c0..54edc690ac1e1 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -87,6 +87,8 @@ xfs_attr_is_leaf( struct xfs_iext_cursor icur; struct xfs_bmbt_irec imap; + ASSERT(!xfs_need_iread_extents(ifp)); + if (ifp->if_nextents != 1 || ifp->if_format != XFS_DINODE_FMT_EXTENTS) return false; @@ -224,11 +226,21 @@ int xfs_attr_get_ilocked( struct xfs_da_args *args) { + int error; + xfs_assert_ilocked(args->dp, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); if (!xfs_inode_hasattr(args->dp)) return -ENOATTR; + /* + * The incore attr fork iext tree must be loaded for xfs_attr_is_leaf + * to work correctly. + */ + error = xfs_iread_extents(args->trans, args->dp, XFS_ATTR_FORK); + if (error) + return error; + if (args->dp->i_af.if_format == XFS_DINODE_FMT_LOCAL) return xfs_attr_shortform_getvalue(args); if (xfs_attr_is_leaf(args->dp)) @@ -870,6 +882,11 @@ xfs_attr_lookup( return -ENOATTR; } + /* Prerequisite for xfs_attr_is_leaf */ + error = xfs_iread_extents(args->trans, args->dp, XFS_ATTR_FORK); + if (error) + return error; + if (xfs_attr_is_leaf(dp)) { error = xfs_attr_leaf_hasname(args, &bp); diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index d460347056945..541455731618b 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -498,6 +498,25 @@ xfs_attri_validate( return xfs_verify_ino(mp, attrp->alfi_ino); } +static int +xfs_attri_iread_extents( + struct xfs_inode *ip) +{ + struct xfs_trans *tp; + int error; + + error = xfs_trans_alloc_empty(ip->i_mount, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + error = xfs_iread_extents(tp, ip, XFS_ATTR_FORK); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_trans_cancel(tp); + + return error; +} + static inline struct xfs_attr_intent * xfs_attri_recover_work( struct xfs_mount *mp, @@ -508,13 +527,22 @@ xfs_attri_recover_work( { struct xfs_attr_intent *attr; struct xfs_da_args *args; + struct xfs_inode *ip; int local; int error; - error = xlog_recover_iget(mp, attrp->alfi_ino, ipp); + error = xlog_recover_iget(mp, attrp->alfi_ino, &ip); if (error) return ERR_PTR(error); + if (xfs_inode_has_attr_fork(ip)) { + error = xfs_attri_iread_extents(ip); + if (error) { + xfs_irele(ip); + return ERR_PTR(error); + } + } + attr = kzalloc(sizeof(struct xfs_attr_intent) + sizeof(struct xfs_da_args), GFP_KERNEL | __GFP_NOFAIL); args = (struct xfs_da_args *)(attr + 1); @@ -531,7 +559,7 @@ xfs_attri_recover_work( attr->xattri_nameval = xfs_attri_log_nameval_get(nv); ASSERT(attr->xattri_nameval); - args->dp = *ipp; + args->dp = ip; args->geo = mp->m_attr_geo; args->whichfork = XFS_ATTR_FORK; args->name = nv->name.i_addr; @@ -561,6 +589,7 @@ xfs_attri_recover_work( } xfs_defer_add_item(dfp, &attr->xattri_list); + *ipp = ip; return attr; } @@ -615,16 +644,17 @@ xfs_attr_recover_work( XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, &attrip->attri_format, sizeof(attrip->attri_format)); - if (error) { - xfs_trans_cancel(tp); - goto out_unlock; - } + if (error) + goto out_cancel; error = xfs_defer_ops_capture_and_commit(tp, capture_list); out_unlock: xfs_iunlock(ip, XFS_ILOCK_EXCL); xfs_irele(ip); return error; +out_cancel: + xfs_trans_cancel(tp); + goto out_unlock; } /* Re-log an intent item to push the log tail forward. */ diff --git a/fs/xfs/xfs_attr_list.c b/fs/xfs/xfs_attr_list.c index 6a621f016f040..97c8f3dcfb89d 100644 --- a/fs/xfs/xfs_attr_list.c +++ b/fs/xfs/xfs_attr_list.c @@ -544,6 +544,7 @@ xfs_attr_list_ilocked( struct xfs_attr_list_context *context) { struct xfs_inode *dp = context->dp; + int error; xfs_assert_ilocked(dp, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); @@ -554,6 +555,12 @@ xfs_attr_list_ilocked( return 0; if (dp->i_af.if_format == XFS_DINODE_FMT_LOCAL) return xfs_attr_shortform_list(context); + + /* Prerequisite for xfs_attr_is_leaf */ + error = xfs_iread_extents(NULL, dp, XFS_ATTR_FORK); + if (error) + return error; + if (xfs_attr_is_leaf(dp)) return xfs_attr_leaf_list(context); return xfs_attr_node_list(context); From 8ef1d96a985e4dc07ffbd71bd7fc5604a80cc644 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:24 -0700 Subject: [PATCH 0409/1702] xfs: require XFS_SB_FEAT_INCOMPAT_LOG_XATTRS for attr log intent item recovery The XFS_SB_FEAT_INCOMPAT_LOG_XATTRS feature bit protects a filesystem from old kernels that do not know how to recover extended attribute log intent items. Make this check mandatory instead of a debugging assert. Fixes: fd920008784ea ("xfs: Set up infrastructure for log attribute replay") Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_attr_item.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index 541455731618b..dfe7039dac989 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -469,6 +469,9 @@ xfs_attri_validate( unsigned int op = attrp->alfi_op_flags & XFS_ATTRI_OP_FLAGS_TYPE_MASK; + if (!xfs_sb_version_haslogxattrs(&mp->m_sb)) + return false; + if (attrp->__pad != 0) return false; @@ -570,8 +573,6 @@ xfs_attri_recover_work( XFS_DA_OP_LOGGED; args->owner = args->dp->i_ino; - ASSERT(xfs_sb_version_haslogxattrs(&mp->m_sb)); - switch (attr->xattri_op_flags) { case XFS_ATTRI_OP_FLAGS_SET: case XFS_ATTRI_OP_FLAGS_REPLACE: From c27411d4c640037d70e2fa5751616e175e52ca5e Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:22 -0700 Subject: [PATCH 0410/1702] xfs: make attr removal an explicit operation Parent pointers match attrs on name+value, unlike everything else which matches on only the name. Therefore, we cannot keep using the heuristic that !value means remove. Make this an explicit operation code. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 19 ++++++++++--------- fs/xfs/libxfs/xfs_attr.h | 3 ++- fs/xfs/xfs_acl.c | 17 +++++++++-------- fs/xfs/xfs_ioctl.c | 9 ++++++--- fs/xfs/xfs_iops.c | 2 +- fs/xfs/xfs_xattr.c | 9 ++++++--- 6 files changed, 34 insertions(+), 25 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index b04e09143869d..f8f7445b063c0 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -916,10 +916,6 @@ xfs_attr_defer_add( trace_xfs_attr_defer_add(new->xattri_dela_state, args->dp); } -/* - * Note: If args->value is NULL the attribute will be removed, just like the - * Linux ->setattr API. - */ int xfs_attr_set( struct xfs_da_args *args, @@ -955,7 +951,10 @@ xfs_attr_set( args->op_flags = XFS_DA_OP_OKNOENT | (args->op_flags & XFS_DA_OP_LOGGED); - if (args->value) { + switch (op) { + case XFS_ATTRUPDATE_UPSERT: + case XFS_ATTRUPDATE_CREATE: + case XFS_ATTRUPDATE_REPLACE: XFS_STATS_INC(mp, xs_attr_set); args->total = xfs_attr_calc_size(args, &local); @@ -975,9 +974,11 @@ xfs_attr_set( if (!local) rmt_blks = xfs_attr3_rmt_blocks(mp, args->valuelen); - } else { + break; + case XFS_ATTRUPDATE_REMOVE: XFS_STATS_INC(mp, xs_attr_remove); rmt_blks = xfs_attr3_rmt_blocks(mp, XFS_XATTR_SIZE_MAX); + break; } /* @@ -989,7 +990,7 @@ xfs_attr_set( if (error) return error; - if (args->value || xfs_inode_hasattr(dp)) { + if (op != XFS_ATTRUPDATE_REMOVE || xfs_inode_hasattr(dp)) { error = xfs_iext_count_may_overflow(dp, XFS_ATTR_FORK, XFS_IEXT_ATTR_MANIP_CNT(rmt_blks)); if (error == -EFBIG) @@ -1002,7 +1003,7 @@ xfs_attr_set( error = xfs_attr_lookup(args); switch (error) { case -EEXIST: - if (!args->value) { + if (op == XFS_ATTRUPDATE_REMOVE) { /* if no value, we are performing a remove operation */ xfs_attr_defer_add(args, XFS_ATTRI_OP_FLAGS_REMOVE); break; @@ -1015,7 +1016,7 @@ xfs_attr_set( break; case -ENOATTR: /* Can't remove what isn't there. */ - if (!args->value) + if (op == XFS_ATTRUPDATE_REMOVE) goto out_trans_cancel; /* Pure replace fails if no existing attr to replace. */ diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h index 228360f7c85ce..c8005f52102ad 100644 --- a/fs/xfs/libxfs/xfs_attr.h +++ b/fs/xfs/libxfs/xfs_attr.h @@ -546,7 +546,8 @@ int xfs_attr_get_ilocked(struct xfs_da_args *args); int xfs_attr_get(struct xfs_da_args *args); enum xfs_attr_update { - XFS_ATTRUPDATE_UPSERTR, /* set/remove value, replace any existing attr */ + XFS_ATTRUPDATE_REMOVE, /* remove attr */ + XFS_ATTRUPDATE_UPSERT, /* set value, replace any existing attr */ XFS_ATTRUPDATE_CREATE, /* set value, fail if attr already exists */ XFS_ATTRUPDATE_REPLACE, /* set value, fail if attr does not exist */ }; diff --git a/fs/xfs/xfs_acl.c b/fs/xfs/xfs_acl.c index 12f98af9e7092..c7c3dcfa2718f 100644 --- a/fs/xfs/xfs_acl.c +++ b/fs/xfs/xfs_acl.c @@ -201,16 +201,17 @@ __xfs_set_acl(struct inode *inode, struct posix_acl *acl, int type) if (!args.value) return -ENOMEM; xfs_acl_to_disk(args.value, acl); + error = xfs_attr_change(&args, XFS_ATTRUPDATE_UPSERT); + kvfree(args.value); + } else { + error = xfs_attr_change(&args, XFS_ATTRUPDATE_REMOVE); + /* + * If the attribute didn't exist to start with that's fine. + */ + if (error == -ENOATTR) + error = 0; } - error = xfs_attr_change(&args, XFS_ATTRUPDATE_UPSERTR); - kvfree(args.value); - - /* - * If the attribute didn't exist to start with that's fine. - */ - if (!acl && error == -ENOATTR) - error = 0; if (!error) set_cached_acl(inode, type, acl); return error; diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 0eca7a9096e23..e30f9f40f086d 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -363,13 +363,16 @@ xfs_attr_filter( static inline enum xfs_attr_update xfs_xattr_flags( - u32 ioc_flags) + u32 ioc_flags, + void *value) { + if (!value) + return XFS_ATTRUPDATE_REMOVE; if (ioc_flags & XFS_IOC_ATTR_CREATE) return XFS_ATTRUPDATE_CREATE; if (ioc_flags & XFS_IOC_ATTR_REPLACE) return XFS_ATTRUPDATE_REPLACE; - return XFS_ATTRUPDATE_UPSERTR; + return XFS_ATTRUPDATE_UPSERT; } int @@ -526,7 +529,7 @@ xfs_attrmulti_attr_set( args.valuelen = len; } - error = xfs_attr_change(&args, xfs_xattr_flags(flags)); + error = xfs_attr_change(&args, xfs_xattr_flags(flags, args.value)); if (!error && (flags & XFS_IOC_ATTR_ROOT)) xfs_forget_acl(inode, name); kfree(args.value); diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index d02ca2248bbcf..659fd10c0cda6 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -63,7 +63,7 @@ xfs_initxattrs( .value = xattr->value, .valuelen = xattr->value_len, }; - error = xfs_attr_change(&args, XFS_ATTRUPDATE_UPSERTR); + error = xfs_attr_change(&args, XFS_ATTRUPDATE_UPSERT); if (error < 0) break; } diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c index 6ce1e06f650a5..0cbb93cf2869c 100644 --- a/fs/xfs/xfs_xattr.c +++ b/fs/xfs/xfs_xattr.c @@ -118,13 +118,16 @@ xfs_xattr_get(const struct xattr_handler *handler, struct dentry *unused, static inline enum xfs_attr_update xfs_xattr_flags_to_op( - int flags) + int flags, + const void *value) { + if (!value) + return XFS_ATTRUPDATE_REMOVE; if (flags & XATTR_CREATE) return XFS_ATTRUPDATE_CREATE; if (flags & XATTR_REPLACE) return XFS_ATTRUPDATE_REPLACE; - return XFS_ATTRUPDATE_UPSERTR; + return XFS_ATTRUPDATE_UPSERT; } static int @@ -143,7 +146,7 @@ xfs_xattr_set(const struct xattr_handler *handler, }; int error; - error = xfs_attr_change(&args, xfs_xattr_flags_to_op(flags)); + error = xfs_attr_change(&args, xfs_xattr_flags_to_op(flags, value)); if (!error && (handler->flags & XFS_ATTR_ROOT)) xfs_forget_acl(inode, name); return error; From f759784cb61ceb77604326cd53cc2da88d24842f Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:25 -0700 Subject: [PATCH 0411/1702] xfs: use an XFS_OPSTATE_ flag for detecting if logged xattrs are available Per reviewer request, use an OPSTATE flag (+ helpers) to decide if logged xattrs are enabled, instead of querying the xfs_sb. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_attr_item.c | 2 +- fs/xfs/xfs_mount.c | 16 ++++++++++++++++ fs/xfs/xfs_mount.h | 6 +++++- fs/xfs/xfs_xattr.c | 3 ++- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index dfe7039dac989..e5e7ddbc594b9 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -469,7 +469,7 @@ xfs_attri_validate( unsigned int op = attrp->alfi_op_flags & XFS_ATTRI_OP_FLAGS_TYPE_MASK; - if (!xfs_sb_version_haslogxattrs(&mp->m_sb)) + if (!xfs_is_using_logged_xattrs(mp)) return false; if (attrp->__pad != 0) diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index b2e5653b5200c..09eef1721ef4f 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -231,6 +231,13 @@ xfs_readsb( mp->m_features |= xfs_sb_version_to_features(sbp); xfs_reinit_percpu_counters(mp); + /* + * If logged xattrs are enabled after log recovery finishes, then set + * the opstate so that log recovery will work properly. + */ + if (xfs_sb_version_haslogxattrs(&mp->m_sb)) + xfs_set_using_logged_xattrs(mp); + /* no need to be quiet anymore, so reset the buf ops */ bp->b_ops = &xfs_sb_buf_ops; @@ -829,6 +836,15 @@ xfs_mountfs( goto out_inodegc_shrinker; } + /* + * If logged xattrs are still enabled after log recovery finishes, then + * they'll be available until unmount. Otherwise, turn them off. + */ + if (xfs_sb_version_haslogxattrs(&mp->m_sb)) + xfs_set_using_logged_xattrs(mp); + else + xfs_clear_using_logged_xattrs(mp); + /* Enable background inode inactivation workers. */ xfs_inodegc_start(mp); xfs_blockgc_start(mp); diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index ca6f105990a2c..d0567dfbc0368 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -444,6 +444,8 @@ __XFS_HAS_FEAT(nouuid, NOUUID) #define XFS_OPSTATE_QUOTACHECK_RUNNING 10 /* Do we want to clear log incompat flags? */ #define XFS_OPSTATE_UNSET_LOG_INCOMPAT 11 +/* Filesystem can use logged extended attributes */ +#define XFS_OPSTATE_USE_LARP 12 #define __XFS_IS_OPSTATE(name, NAME) \ static inline bool xfs_is_ ## name (struct xfs_mount *mp) \ @@ -472,6 +474,7 @@ __XFS_IS_OPSTATE(quotacheck_running, QUOTACHECK_RUNNING) # define xfs_is_quotacheck_running(mp) (false) #endif __XFS_IS_OPSTATE(done_with_log_incompat, UNSET_LOG_INCOMPAT) +__XFS_IS_OPSTATE(using_logged_xattrs, USE_LARP) static inline bool xfs_should_warn(struct xfs_mount *mp, long nr) @@ -491,7 +494,8 @@ xfs_should_warn(struct xfs_mount *mp, long nr) { (1UL << XFS_OPSTATE_WARNED_SHRINK), "wshrink" }, \ { (1UL << XFS_OPSTATE_WARNED_LARP), "wlarp" }, \ { (1UL << XFS_OPSTATE_QUOTACHECK_RUNNING), "quotacheck" }, \ - { (1UL << XFS_OPSTATE_UNSET_LOG_INCOMPAT), "unset_log_incompat" } + { (1UL << XFS_OPSTATE_UNSET_LOG_INCOMPAT), "unset_log_incompat" }, \ + { (1UL << XFS_OPSTATE_USE_LARP), "logged_xattrs" } /* * Max and min values for mount-option defined I/O diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c index 0cbb93cf2869c..ba56a9e73144b 100644 --- a/fs/xfs/xfs_xattr.c +++ b/fs/xfs/xfs_xattr.c @@ -31,7 +31,7 @@ xfs_attr_grab_log_assist( int error = 0; /* xattr update log intent items are already enabled */ - if (xfs_sb_version_haslogxattrs(&mp->m_sb)) + if (xfs_is_using_logged_xattrs(mp)) return 0; /* @@ -48,6 +48,7 @@ xfs_attr_grab_log_assist( XFS_SB_FEAT_INCOMPAT_LOG_XATTRS); if (error) return error; + xfs_set_using_logged_xattrs(mp); xfs_warn_mount(mp, XFS_OPSTATE_WARNED_LARP, "EXPERIMENTAL logged extended attributes feature in use. Use at your own risk!"); From cda60317ac57add7a0a2865aa29afbc6caad3e9a Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:23 -0700 Subject: [PATCH 0412/1702] xfs: rearrange xfs_da_args a bit to use less space A few notes about struct xfs_da_args: The XFS_ATTR_* flags only go up as far as XFS_ATTR_INCOMPLETE, which means that attr_filter could be a u8 field. I've reduced the number of XFS_DA_OP_* flags down to the point where op_flags would also fit into a u8. filetype has 7 bytes of slack after it, which is wasteful. namelen will never be greater than MAXNAMELEN, which is 256. This field could be reduced to a short. Rearrange the fields in xfs_da_args to waste less space. This reduces the structure size from 136 bytes to 128. Later when we add extra fields to support parent pointer replacement, this will only bloat the structure to 144 bytes, instead of 168. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_da_btree.h | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/fs/xfs/libxfs/xfs_da_btree.h b/fs/xfs/libxfs/xfs_da_btree.h index 706b529a81feb..17cef594b5bbb 100644 --- a/fs/xfs/libxfs/xfs_da_btree.h +++ b/fs/xfs/libxfs/xfs_da_btree.h @@ -54,16 +54,20 @@ enum xfs_dacmp { */ typedef struct xfs_da_args { struct xfs_da_geometry *geo; /* da block geometry */ - const uint8_t *name; /* string (maybe not NULL terminated) */ - int namelen; /* length of string (maybe no NULL) */ - uint8_t filetype; /* filetype of inode for directories */ + const uint8_t *name; /* string (maybe not NULL terminated) */ void *value; /* set of bytes (maybe contain NULLs) */ - int valuelen; /* length of value */ - unsigned int attr_filter; /* XFS_ATTR_{ROOT,SECURE,INCOMPLETE} */ - xfs_dahash_t hashval; /* hash value of name */ - xfs_ino_t inumber; /* input/output inode number */ struct xfs_inode *dp; /* directory inode to manipulate */ struct xfs_trans *trans; /* current trans (changes over time) */ + + xfs_ino_t inumber; /* input/output inode number */ + xfs_ino_t owner; /* inode that owns the dir/attr data */ + + int valuelen; /* length of value */ + uint8_t filetype; /* filetype of inode for directories */ + uint8_t op_flags; /* operation flags */ + uint8_t attr_filter; /* XFS_ATTR_{ROOT,SECURE,INCOMPLETE} */ + short namelen; /* length of string (maybe no NULL) */ + xfs_dahash_t hashval; /* hash value of name */ xfs_extlen_t total; /* total blocks needed, for 1st bmap */ int whichfork; /* data or attribute fork */ xfs_dablk_t blkno; /* blkno of attr leaf of interest */ @@ -76,9 +80,7 @@ typedef struct xfs_da_args { xfs_dablk_t rmtblkno2; /* remote attr value starting blkno */ int rmtblkcnt2; /* remote attr value block count */ int rmtvaluelen2; /* remote attr value length in bytes */ - uint32_t op_flags; /* operation flags */ enum xfs_dacmp cmpresult; /* name compare result for lookups */ - xfs_ino_t owner; /* inode that owns the dir/attr data */ } xfs_da_args_t; /* From ad206ae50eca62836c5460ab5bbf2a6c59a268e7 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:26 -0700 Subject: [PATCH 0413/1702] xfs: check opcode and iovec count match in xlog_recover_attri_commit_pass2 Check that the number of recovered log iovecs is what is expected for the xattri opcode is expecting. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_attr_item.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index e5e7ddbc594b9..d3559e6b24b7d 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -737,6 +737,7 @@ xlog_recover_attri_commit_pass2( const void *attr_value = NULL; const void *attr_name; size_t len; + unsigned int op; attri_formatp = item->ri_buf[0].i_addr; attr_name = item->ri_buf[1].i_addr; @@ -755,6 +756,32 @@ xlog_recover_attri_commit_pass2( return -EFSCORRUPTED; } + /* Check the number of log iovecs makes sense for the op code. */ + op = attri_formatp->alfi_op_flags & XFS_ATTRI_OP_FLAGS_TYPE_MASK; + switch (op) { + case XFS_ATTRI_OP_FLAGS_SET: + case XFS_ATTRI_OP_FLAGS_REPLACE: + /* Log item, attr name, attr value */ + if (item->ri_total != 3) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + break; + case XFS_ATTRI_OP_FLAGS_REMOVE: + /* Log item, attr name */ + if (item->ri_total != 2) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + break; + default: + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + /* Validate the attr name */ if (item->ri_buf[1].i_len != xlog_calc_iovec_len(attri_formatp->alfi_name_len)) { From f660ec8eaeb50d0317c29601aacabdb15e5f2203 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:27 -0700 Subject: [PATCH 0414/1702] xfs: fix missing check for invalid attr flags The xattr scrubber doesn't check for undefined flags in shortform attr entries. Therefore, define a mask XFS_ATTR_ONDISK_MASK that has all possible XFS_ATTR_* flags in it, and use that to check for unknown bits in xchk_xattr_actor. Refactor the check in the dabtree scanner function to use the new mask as well. The redundant checks need to be in place because the dabtree check examines the hash mappings and therefore needs to decode the attr leaf entries to compute the namehash. This happens before the walk of the xattr entries themselves. Fixes: ae0506eba78fd ("xfs: check used space of shortform xattr structures") Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_da_format.h | 5 +++++ fs/xfs/scrub/attr.c | 13 +++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/fs/xfs/libxfs/xfs_da_format.h b/fs/xfs/libxfs/xfs_da_format.h index aac3fe0396140..ecd0616f5776a 100644 --- a/fs/xfs/libxfs/xfs_da_format.h +++ b/fs/xfs/libxfs/xfs_da_format.h @@ -719,8 +719,13 @@ struct xfs_attr3_leafblock { #define XFS_ATTR_ROOT (1u << XFS_ATTR_ROOT_BIT) #define XFS_ATTR_SECURE (1u << XFS_ATTR_SECURE_BIT) #define XFS_ATTR_INCOMPLETE (1u << XFS_ATTR_INCOMPLETE_BIT) + #define XFS_ATTR_NSP_ONDISK_MASK (XFS_ATTR_ROOT | XFS_ATTR_SECURE) +#define XFS_ATTR_ONDISK_MASK (XFS_ATTR_NSP_ONDISK_MASK | \ + XFS_ATTR_LOCAL | \ + XFS_ATTR_INCOMPLETE) + #define XFS_ATTR_NAMESPACE_STR \ { XFS_ATTR_LOCAL, "local" }, \ { XFS_ATTR_ROOT, "root" }, \ diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index 5b855d7c98211..5ca79af47e81e 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -192,6 +192,11 @@ xchk_xattr_actor( if (xchk_should_terminate(sc, &error)) return error; + if (attr_flags & ~XFS_ATTR_ONDISK_MASK) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, args.blkno); + return -ECANCELED; + } + if (attr_flags & XFS_ATTR_INCOMPLETE) { /* Incomplete attr key, just mark the inode for preening. */ xchk_ino_set_preen(sc, ip->i_ino); @@ -481,7 +486,6 @@ xchk_xattr_rec( xfs_dahash_t hash; int nameidx; int hdrsize; - unsigned int badflags; int error; ASSERT(blk->magic == XFS_ATTR_LEAF_MAGIC); @@ -511,10 +515,11 @@ xchk_xattr_rec( /* Retrieve the entry and check it. */ hash = be32_to_cpu(ent->hashval); - badflags = ~(XFS_ATTR_LOCAL | XFS_ATTR_ROOT | XFS_ATTR_SECURE | - XFS_ATTR_INCOMPLETE); - if ((ent->flags & badflags) != 0) + if (ent->flags & ~XFS_ATTR_ONDISK_MASK) { xchk_da_set_corrupt(ds, level); + return 0; + } + if (ent->flags & XFS_ATTR_LOCAL) { lentry = (struct xfs_attr_leaf_name_local *) (((char *)bp->b_addr) + nameidx); From 309dc9cbbb4379241bcc9b5a6a42c04279a0e5a7 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:28 -0700 Subject: [PATCH 0415/1702] xfs: check shortform attr entry flags specifically While reviewing flag checking in the attr scrub functions, we noticed that the shortform attr scanner didn't catch entries that have the LOCAL or INCOMPLETE bits set. Neither of these flags can ever be set on a shortform attr, so we need to check this narrower set of valid flags. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/attr.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index 5ca79af47e81e..fd22d652a63a1 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -579,6 +579,15 @@ xchk_xattr_check_sf( break; } + /* + * Shortform entries do not set LOCAL or INCOMPLETE, so the + * only valid flag bits here are for namespaces. + */ + if (sfe->flags & ~XFS_ATTR_NSP_ONDISK_MASK) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + break; + } + if (!xchk_xattr_set_map(sc, ab->usedmap, (char *)sfe - (char *)sf, sizeof(struct xfs_attr_sf_entry))) { From 992c3b5c3fe6f42778436649ddae2b7a2984b7aa Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:29 -0700 Subject: [PATCH 0416/1702] xfs: restructure xfs_attr_complete_op a bit Eliminate the local variable from this function so that we can streamline things a bit later when we add the PPTR_REPLACE op code. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 54edc690ac1e1..ba59dab6c56db 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -432,14 +432,13 @@ xfs_attr_complete_op( enum xfs_delattr_state replace_state) { struct xfs_da_args *args = attr->xattri_da_args; - bool do_replace = args->op_flags & XFS_DA_OP_REPLACE; + + if (!(args->op_flags & XFS_DA_OP_REPLACE)) + replace_state = XFS_DAS_DONE; args->op_flags &= ~XFS_DA_OP_REPLACE; args->attr_filter &= ~XFS_ATTR_INCOMPLETE; - if (do_replace) - return replace_state; - - return XFS_DAS_DONE; + return replace_state; } static int From 2a2c05d013d0562076ec475a6deb0991ce1942ca Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:29 -0700 Subject: [PATCH 0417/1702] xfs: use helpers to extract xattr op from opflags Create helper functions to extract the xattr op from the ondisk xattri log item and the incore attr intent item. These will get more use in the patches that follow. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.h | 5 +++++ fs/xfs/xfs_attr_item.c | 16 ++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h index c8005f52102ad..79b457adb7bda 100644 --- a/fs/xfs/libxfs/xfs_attr.h +++ b/fs/xfs/libxfs/xfs_attr.h @@ -529,6 +529,11 @@ struct xfs_attr_intent { struct xfs_bmbt_irec xattri_map; }; +static inline unsigned int +xfs_attr_intent_op(const struct xfs_attr_intent *attr) +{ + return attr->xattri_op_flags & XFS_ATTRI_OP_FLAGS_TYPE_MASK; +} /*======================================================================== * Function prototypes for the kernel. diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index d3559e6b24b7d..b4c2dcb4581bc 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -308,6 +308,12 @@ xfs_attrd_item_intent( return &ATTRD_ITEM(lip)->attrd_attrip->attri_item; } +static inline unsigned int +xfs_attr_log_item_op(const struct xfs_attri_log_format *attrp) +{ + return attrp->alfi_op_flags & XFS_ATTRI_OP_FLAGS_TYPE_MASK; +} + /* Log an attr to the intent item. */ STATIC void xfs_attr_log_item( @@ -466,8 +472,7 @@ xfs_attri_validate( struct xfs_mount *mp, struct xfs_attri_log_format *attrp) { - unsigned int op = attrp->alfi_op_flags & - XFS_ATTRI_OP_FLAGS_TYPE_MASK; + unsigned int op = xfs_attr_log_item_op(attrp); if (!xfs_is_using_logged_xattrs(mp)) return false; @@ -551,8 +556,7 @@ xfs_attri_recover_work( args = (struct xfs_da_args *)(attr + 1); attr->xattri_da_args = args; - attr->xattri_op_flags = attrp->alfi_op_flags & - XFS_ATTRI_OP_FLAGS_TYPE_MASK; + attr->xattri_op_flags = xfs_attr_log_item_op(attrp); /* * We're reconstructing the deferred work state structure from the @@ -573,7 +577,7 @@ xfs_attri_recover_work( XFS_DA_OP_LOGGED; args->owner = args->dp->i_ino; - switch (attr->xattri_op_flags) { + switch (xfs_attr_intent_op(attr)) { case XFS_ATTRI_OP_FLAGS_SET: case XFS_ATTRI_OP_FLAGS_REPLACE: args->value = nv->value.i_addr; @@ -757,7 +761,7 @@ xlog_recover_attri_commit_pass2( } /* Check the number of log iovecs makes sense for the op code. */ - op = attri_formatp->alfi_op_flags & XFS_ATTRI_OP_FLAGS_TYPE_MASK; + op = xfs_attr_log_item_op(attri_formatp); switch (op) { case XFS_ATTRI_OP_FLAGS_SET: case XFS_ATTRI_OP_FLAGS_REPLACE: From 1c7f09d210aba2f2bb206e2e8c97c9f11a3fd880 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:30 -0700 Subject: [PATCH 0418/1702] xfs: validate recovered name buffers when recovering xattr items Strengthen the xattri log item recovery code by checking that we actually have the required name and newname buffers for whatever operation we're replaying. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_attr_item.c | 58 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index b4c2dcb4581bc..ebd6e98d9c661 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -741,22 +741,20 @@ xlog_recover_attri_commit_pass2( const void *attr_value = NULL; const void *attr_name; size_t len; - unsigned int op; - - attri_formatp = item->ri_buf[0].i_addr; - attr_name = item->ri_buf[1].i_addr; + unsigned int op, i = 0; /* Validate xfs_attri_log_format before the large memory allocation */ len = sizeof(struct xfs_attri_log_format); - if (item->ri_buf[0].i_len != len) { + if (item->ri_buf[i].i_len != len) { XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } + attri_formatp = item->ri_buf[i].i_addr; if (!xfs_attri_validate(mp, attri_formatp)) { XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, - item->ri_buf[0].i_addr, item->ri_buf[0].i_len); + attri_formatp, len); return -EFSCORRUPTED; } @@ -785,31 +783,69 @@ xlog_recover_attri_commit_pass2( attri_formatp, len); return -EFSCORRUPTED; } + i++; /* Validate the attr name */ - if (item->ri_buf[1].i_len != + if (item->ri_buf[i].i_len != xlog_calc_iovec_len(attri_formatp->alfi_name_len)) { XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, - item->ri_buf[0].i_addr, item->ri_buf[0].i_len); + attri_formatp, len); return -EFSCORRUPTED; } + attr_name = item->ri_buf[i].i_addr; if (!xfs_attr_namecheck(attr_name, attri_formatp->alfi_name_len)) { XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, - item->ri_buf[1].i_addr, item->ri_buf[1].i_len); + attri_formatp, len); return -EFSCORRUPTED; } + i++; /* Validate the attr value, if present */ if (attri_formatp->alfi_value_len != 0) { - if (item->ri_buf[2].i_len != xlog_calc_iovec_len(attri_formatp->alfi_value_len)) { + if (item->ri_buf[i].i_len != xlog_calc_iovec_len(attri_formatp->alfi_value_len)) { XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } - attr_value = item->ri_buf[2].i_addr; + attr_value = item->ri_buf[i].i_addr; + i++; + } + + /* + * Make sure we got the correct number of buffers for the operation + * that we just loaded. + */ + if (i != item->ri_total) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + + switch (op) { + case XFS_ATTRI_OP_FLAGS_REMOVE: + /* Regular remove operations operate only on names. */ + if (attr_value != NULL || attri_formatp->alfi_value_len != 0) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + fallthrough; + case XFS_ATTRI_OP_FLAGS_SET: + case XFS_ATTRI_OP_FLAGS_REPLACE: + /* + * Regular xattr set/remove/replace operations require a name + * and do not take a newname. Values are optional for set and + * replace. + */ + if (attr_name == NULL || attri_formatp->alfi_name_len == 0) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + break; } /* From 0aeeeb796980f74bf87ef175335ee1a9a1229767 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:31 -0700 Subject: [PATCH 0419/1702] xfs: always set args->value in xfs_attri_item_recover Always set args->value to the recovered value buffer. This reduces the amount of code in the switch statement, and hence the amount of thinking that I have to do. We validated the recovered buffers, supposedly. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_attr_item.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index ebd6e98d9c661..8a13e2840692c 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -572,6 +572,8 @@ xfs_attri_recover_work( args->name = nv->name.i_addr; args->namelen = nv->name.i_len; args->hashval = xfs_da_hashname(args->name, args->namelen); + args->value = nv->value.i_addr; + args->valuelen = nv->value.i_len; args->attr_filter = attrp->alfi_attr_filter & XFS_ATTRI_FILTER_MASK; args->op_flags = XFS_DA_OP_RECOVERY | XFS_DA_OP_OKNOENT | XFS_DA_OP_LOGGED; @@ -580,8 +582,6 @@ xfs_attri_recover_work( switch (xfs_attr_intent_op(attr)) { case XFS_ATTRI_OP_FLAGS_SET: case XFS_ATTRI_OP_FLAGS_REPLACE: - args->value = nv->value.i_addr; - args->valuelen = nv->value.i_len; args->total = xfs_attr_calc_size(args, &local); if (xfs_inode_hasattr(args->dp)) attr->xattri_dela_state = xfs_attr_init_replace_state(args); From c07f018bc094c5f30cb827ec9f11a23ace3435ec Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:32 -0700 Subject: [PATCH 0420/1702] xfs: use local variables for name and value length in _attri_commit_pass2 We're about to start using tagged unions in the xattr log format, so create a bunch of local variables in the recovery function so we only have to decode the log item fields once. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_attr_item.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index 8a13e2840692c..59723e5f483e2 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -738,9 +738,11 @@ xlog_recover_attri_commit_pass2( struct xfs_attri_log_item *attrip; struct xfs_attri_log_format *attri_formatp; struct xfs_attri_log_nameval *nv; - const void *attr_value = NULL; const void *attr_name; + const void *attr_value = NULL; size_t len; + unsigned int name_len = 0; + unsigned int value_len = 0; unsigned int op, i = 0; /* Validate xfs_attri_log_format before the large memory allocation */ @@ -769,6 +771,8 @@ xlog_recover_attri_commit_pass2( attri_formatp, len); return -EFSCORRUPTED; } + name_len = attri_formatp->alfi_name_len; + value_len = attri_formatp->alfi_value_len; break; case XFS_ATTRI_OP_FLAGS_REMOVE: /* Log item, attr name */ @@ -777,6 +781,7 @@ xlog_recover_attri_commit_pass2( attri_formatp, len); return -EFSCORRUPTED; } + name_len = attri_formatp->alfi_name_len; break; default: XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, @@ -786,15 +791,14 @@ xlog_recover_attri_commit_pass2( i++; /* Validate the attr name */ - if (item->ri_buf[i].i_len != - xlog_calc_iovec_len(attri_formatp->alfi_name_len)) { + if (item->ri_buf[i].i_len != xlog_calc_iovec_len(name_len)) { XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, attri_formatp, len); return -EFSCORRUPTED; } attr_name = item->ri_buf[i].i_addr; - if (!xfs_attr_namecheck(attr_name, attri_formatp->alfi_name_len)) { + if (!xfs_attr_namecheck(attr_name, name_len)) { XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, attri_formatp, len); return -EFSCORRUPTED; @@ -802,8 +806,8 @@ xlog_recover_attri_commit_pass2( i++; /* Validate the attr value, if present */ - if (attri_formatp->alfi_value_len != 0) { - if (item->ri_buf[i].i_len != xlog_calc_iovec_len(attri_formatp->alfi_value_len)) { + if (value_len != 0) { + if (item->ri_buf[i].i_len != xlog_calc_iovec_len(value_len)) { XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, item->ri_buf[0].i_addr, item->ri_buf[0].i_len); @@ -827,7 +831,7 @@ xlog_recover_attri_commit_pass2( switch (op) { case XFS_ATTRI_OP_FLAGS_REMOVE: /* Regular remove operations operate only on names. */ - if (attr_value != NULL || attri_formatp->alfi_value_len != 0) { + if (attr_value != NULL || value_len != 0) { XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, attri_formatp, len); return -EFSCORRUPTED; @@ -840,7 +844,7 @@ xlog_recover_attri_commit_pass2( * and do not take a newname. Values are optional for set and * replace. */ - if (attr_name == NULL || attri_formatp->alfi_name_len == 0) { + if (attr_name == NULL || name_len == 0) { XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, attri_formatp, len); return -EFSCORRUPTED; @@ -853,9 +857,8 @@ xlog_recover_attri_commit_pass2( * name/value buffer to the recovered incore log item and drop our * reference. */ - nv = xfs_attri_log_nameval_alloc(attr_name, - attri_formatp->alfi_name_len, attr_value, - attri_formatp->alfi_value_len); + nv = xfs_attri_log_nameval_alloc(attr_name, name_len, + attr_value, value_len); attrip = xfs_attri_init(mp, nv); memcpy(&attrip->attri_format, attri_formatp, len); From 50855427c25426afbd98e9b4b00cb4a383614d88 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:33 -0700 Subject: [PATCH 0421/1702] xfs: refactor name/length checks in xfs_attri_validate Move the name and length checks into the attr op switch statement so that we can perform more specific checks of the value length. Over the next few patches we're going to add new attr op flags with different validation requirements. While we're at it, remove the incorrect comment. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_attr_item.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index 59723e5f483e2..c8f92166b9ad6 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -466,6 +466,12 @@ xfs_attri_item_match( return ATTRI_ITEM(lip)->attri_format.alfi_id == intent_id; } +static inline bool +xfs_attri_validate_namelen(unsigned int namelen) +{ + return namelen > 0 && namelen <= XATTR_NAME_MAX; +} + /* Is this recovered ATTRI format ok? */ static inline bool xfs_attri_validate( @@ -486,23 +492,24 @@ xfs_attri_validate( if (attrp->alfi_attr_filter & ~XFS_ATTRI_FILTER_MASK) return false; - /* alfi_op_flags should be either a set or remove */ switch (op) { case XFS_ATTRI_OP_FLAGS_SET: case XFS_ATTRI_OP_FLAGS_REPLACE: + if (attrp->alfi_value_len > XATTR_SIZE_MAX) + return false; + if (!xfs_attri_validate_namelen(attrp->alfi_name_len)) + return false; + break; case XFS_ATTRI_OP_FLAGS_REMOVE: + if (attrp->alfi_value_len != 0) + return false; + if (!xfs_attri_validate_namelen(attrp->alfi_name_len)) + return false; break; default: return false; } - if (attrp->alfi_value_len > XATTR_SIZE_MAX) - return false; - - if ((attrp->alfi_name_len > XATTR_NAME_MAX) || - (attrp->alfi_name_len == 0)) - return false; - return xfs_verify_ino(mp, attrp->alfi_ino); } From ffdcc3b8eb4d5ab263d04b9c4b2c6072c7b3c1e9 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:33 -0700 Subject: [PATCH 0422/1702] xfs: refactor name/value iovec validation in xlog_recover_attri_commit_pass2 Hoist the code that checks the attr name and value iovecs into separate helpers so that we can add more callsites for the new parent pointer attr intent items. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_attr_item.c | 64 ++++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index c8f92166b9ad6..39536303a7b64 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -734,6 +734,46 @@ const struct xfs_defer_op_type xfs_attr_defer_type = { .relog_intent = xfs_attr_relog_intent, }; +static inline void * +xfs_attri_validate_name_iovec( + struct xfs_mount *mp, + struct xfs_attri_log_format *attri_formatp, + const struct xfs_log_iovec *iovec, + unsigned int name_len) +{ + if (iovec->i_len != xlog_calc_iovec_len(name_len)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, sizeof(*attri_formatp)); + return NULL; + } + + if (!xfs_attr_namecheck(iovec->i_addr, name_len)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, sizeof(*attri_formatp)); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + iovec->i_addr, iovec->i_len); + return NULL; + } + + return iovec->i_addr; +} + +static inline void * +xfs_attri_validate_value_iovec( + struct xfs_mount *mp, + struct xfs_attri_log_format *attri_formatp, + const struct xfs_log_iovec *iovec, + unsigned int value_len) +{ + if (iovec->i_len != xlog_calc_iovec_len(value_len)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, sizeof(*attri_formatp)); + return NULL; + } + + return iovec->i_addr; +} + STATIC int xlog_recover_attri_commit_pass2( struct xlog *log, @@ -798,30 +838,18 @@ xlog_recover_attri_commit_pass2( i++; /* Validate the attr name */ - if (item->ri_buf[i].i_len != xlog_calc_iovec_len(name_len)) { - XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, - attri_formatp, len); - return -EFSCORRUPTED; - } - - attr_name = item->ri_buf[i].i_addr; - if (!xfs_attr_namecheck(attr_name, name_len)) { - XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, - attri_formatp, len); + attr_name = xfs_attri_validate_name_iovec(mp, attri_formatp, + &item->ri_buf[i], name_len); + if (!attr_name) return -EFSCORRUPTED; - } i++; /* Validate the attr value, if present */ if (value_len != 0) { - if (item->ri_buf[i].i_len != xlog_calc_iovec_len(value_len)) { - XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, - item->ri_buf[0].i_addr, - item->ri_buf[0].i_len); + attr_value = xfs_attri_validate_value_iovec(mp, attri_formatp, + &item->ri_buf[i], value_len); + if (!attr_value) return -EFSCORRUPTED; - } - - attr_value = item->ri_buf[i].i_addr; i++; } From ea0b3e814741fb64e7785b564ea619578058e0b0 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:34 -0700 Subject: [PATCH 0423/1702] xfs: enforce one namespace per attribute Create a standardized helper function to enforce one namespace bit per extended attribute, and refactor all the open-coded hweight logic. This function is not a static inline to avoid porting hassles in userspace. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 11 +++++++++++ fs/xfs/libxfs/xfs_attr.h | 4 +++- fs/xfs/libxfs/xfs_attr_leaf.c | 7 ++++++- fs/xfs/scrub/attr.c | 12 +++++------- fs/xfs/scrub/attr_repair.c | 4 +--- fs/xfs/xfs_attr_item.c | 10 ++++++++-- fs/xfs/xfs_attr_list.c | 11 +++++++---- 7 files changed, 41 insertions(+), 18 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index ba59dab6c56db..629fb25d149cf 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -1532,12 +1532,23 @@ xfs_attr_node_get( return error; } +/* Enforce that there is at most one namespace bit per attr. */ +inline bool xfs_attr_check_namespace(unsigned int attr_flags) +{ + return hweight32(attr_flags & XFS_ATTR_NSP_ONDISK_MASK) < 2; +} + /* Returns true if the attribute entry name is valid. */ bool xfs_attr_namecheck( + unsigned int attr_flags, const void *name, size_t length) { + /* Only one namespace bit allowed. */ + if (!xfs_attr_check_namespace(attr_flags)) + return false; + /* * MAXNAMELEN includes the trailing null, but (name/length) leave it * out, so use >= for the length check. diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h index 79b457adb7bda..cd106b0a424fa 100644 --- a/fs/xfs/libxfs/xfs_attr.h +++ b/fs/xfs/libxfs/xfs_attr.h @@ -560,7 +560,9 @@ enum xfs_attr_update { int xfs_attr_set(struct xfs_da_args *args, enum xfs_attr_update op); int xfs_attr_set_iter(struct xfs_attr_intent *attr); int xfs_attr_remove_iter(struct xfs_attr_intent *attr); -bool xfs_attr_namecheck(const void *name, size_t length); +bool xfs_attr_check_namespace(unsigned int attr_flags); +bool xfs_attr_namecheck(unsigned int attr_flags, const void *name, + size_t length); int xfs_attr_calc_size(struct xfs_da_args *args, int *local); void xfs_init_attr_trans(struct xfs_da_args *args, struct xfs_trans_res *tres, unsigned int *total); diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 17ec5ff5a4e3e..3b024ab892e68 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -950,6 +950,11 @@ xfs_attr_shortform_to_leaf( nargs.hashval = xfs_da_hashname(sfe->nameval, sfe->namelen); nargs.attr_filter = sfe->flags & XFS_ATTR_NSP_ONDISK_MASK; + if (!xfs_attr_check_namespace(sfe->flags)) { + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto out; + } error = xfs_attr3_leaf_lookup_int(bp, &nargs); /* set a->index */ ASSERT(error == -ENOATTR); error = xfs_attr3_leaf_add(bp, &nargs); @@ -1063,7 +1068,7 @@ xfs_attr_shortform_verify( * one namespace flag per xattr, so we can just count the * bits (i.e. hweight) here. */ - if (hweight8(sfep->flags & XFS_ATTR_NSP_ONDISK_MASK) > 1) + if (!xfs_attr_check_namespace(sfep->flags)) return __this_address; sfep = next_sfep; diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index fd22d652a63a1..7789bd2f09507 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -203,14 +203,8 @@ xchk_xattr_actor( return 0; } - /* Only one namespace bit allowed. */ - if (hweight32(attr_flags & XFS_ATTR_NSP_ONDISK_MASK) > 1) { - xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, args.blkno); - return -ECANCELED; - } - /* Does this name make sense? */ - if (!xfs_attr_namecheck(name, namelen)) { + if (!xfs_attr_namecheck(attr_flags, name, namelen)) { xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, args.blkno); return -ECANCELED; } @@ -519,6 +513,10 @@ xchk_xattr_rec( xchk_da_set_corrupt(ds, level); return 0; } + if (!xfs_attr_check_namespace(ent->flags)) { + xchk_da_set_corrupt(ds, level); + return 0; + } if (ent->flags & XFS_ATTR_LOCAL) { lentry = (struct xfs_attr_leaf_name_local *) diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c index 3066d662ea13f..8b89c112c492f 100644 --- a/fs/xfs/scrub/attr_repair.c +++ b/fs/xfs/scrub/attr_repair.c @@ -123,12 +123,10 @@ xrep_xattr_want_salvage( return false; if (namelen > XATTR_NAME_MAX || namelen <= 0) return false; - if (!xfs_attr_namecheck(name, namelen)) + if (!xfs_attr_namecheck(attr_flags, name, namelen)) return false; if (valuelen > XATTR_SIZE_MAX || valuelen < 0) return false; - if (hweight32(attr_flags & XFS_ATTR_NSP_ONDISK_MASK) > 1) - return false; return true; } diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index 39536303a7b64..a65ac74797680 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -492,6 +492,10 @@ xfs_attri_validate( if (attrp->alfi_attr_filter & ~XFS_ATTRI_FILTER_MASK) return false; + if (!xfs_attr_check_namespace(attrp->alfi_attr_filter & + XFS_ATTR_NSP_ONDISK_MASK)) + return false; + switch (op) { case XFS_ATTRI_OP_FLAGS_SET: case XFS_ATTRI_OP_FLAGS_REPLACE: @@ -633,7 +637,8 @@ xfs_attr_recover_work( */ attrp = &attrip->attri_format; if (!xfs_attri_validate(mp, attrp) || - !xfs_attr_namecheck(nv->name.i_addr, nv->name.i_len)) + !xfs_attr_namecheck(attrp->alfi_attr_filter, nv->name.i_addr, + nv->name.i_len)) return -EFSCORRUPTED; attr = xfs_attri_recover_work(mp, dfp, attrp, &ip, nv); @@ -747,7 +752,8 @@ xfs_attri_validate_name_iovec( return NULL; } - if (!xfs_attr_namecheck(iovec->i_addr, name_len)) { + if (!xfs_attr_namecheck(attri_formatp->alfi_attr_filter, iovec->i_addr, + name_len)) { XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, attri_formatp, sizeof(*attri_formatp)); XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, diff --git a/fs/xfs/xfs_attr_list.c b/fs/xfs/xfs_attr_list.c index 97c8f3dcfb89d..903ed46c68872 100644 --- a/fs/xfs/xfs_attr_list.c +++ b/fs/xfs/xfs_attr_list.c @@ -82,7 +82,8 @@ xfs_attr_shortform_list( (dp->i_af.if_bytes + sf->count * 16) < context->bufsize)) { for (i = 0, sfe = xfs_attr_sf_firstentry(sf); i < sf->count; i++) { if (XFS_IS_CORRUPT(context->dp->i_mount, - !xfs_attr_namecheck(sfe->nameval, + !xfs_attr_namecheck(sfe->flags, + sfe->nameval, sfe->namelen))) { xfs_dirattr_mark_sick(context->dp, XFS_ATTR_FORK); return -EFSCORRUPTED; @@ -122,7 +123,8 @@ xfs_attr_shortform_list( for (i = 0, sfe = xfs_attr_sf_firstentry(sf); i < sf->count; i++) { if (unlikely( ((char *)sfe < (char *)sf) || - ((char *)sfe >= ((char *)sf + dp->i_af.if_bytes)))) { + ((char *)sfe >= ((char *)sf + dp->i_af.if_bytes)) || + !xfs_attr_check_namespace(sfe->flags))) { XFS_CORRUPTION_ERROR("xfs_attr_shortform_list", XFS_ERRLEVEL_LOW, context->dp->i_mount, sfe, @@ -177,7 +179,7 @@ xfs_attr_shortform_list( cursor->offset = 0; } if (XFS_IS_CORRUPT(context->dp->i_mount, - !xfs_attr_namecheck(sbp->name, + !xfs_attr_namecheck(sbp->flags, sbp->name, sbp->namelen))) { xfs_dirattr_mark_sick(context->dp, XFS_ATTR_FORK); error = -EFSCORRUPTED; @@ -502,7 +504,8 @@ xfs_attr3_leaf_list_int( } if (XFS_IS_CORRUPT(context->dp->i_mount, - !xfs_attr_namecheck(name, namelen))) { + !xfs_attr_namecheck(entry->flags, name, + namelen))) { xfs_dirattr_mark_sick(context->dp, XFS_ATTR_FORK); return -EFSCORRUPTED; } From 63211876ced33fbb730f515e8d830de53533fc82 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:35 -0700 Subject: [PATCH 0424/1702] xfs: rearrange xfs_attr_match parameters Rearrange the parameters to this function so that they match the order of attr listent: attr_flags -> name -> namelen -> value -> valuelen. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr_leaf.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 3b024ab892e68..bb00183d13492 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -510,9 +510,9 @@ xfs_attr3_leaf_read( static bool xfs_attr_match( struct xfs_da_args *args, - uint8_t namelen, - unsigned char *name, - int flags) + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen) { if (args->namelen != namelen) @@ -522,12 +522,12 @@ xfs_attr_match( /* Recovery ignores the INCOMPLETE flag. */ if ((args->op_flags & XFS_DA_OP_RECOVERY) && - args->attr_filter == (flags & XFS_ATTR_NSP_ONDISK_MASK)) + args->attr_filter == (attr_flags & XFS_ATTR_NSP_ONDISK_MASK)) return true; /* All remaining matches need to be filtered by INCOMPLETE state. */ if (args->attr_filter != - (flags & (XFS_ATTR_NSP_ONDISK_MASK | XFS_ATTR_INCOMPLETE))) + (attr_flags & (XFS_ATTR_NSP_ONDISK_MASK | XFS_ATTR_INCOMPLETE))) return false; return true; } @@ -746,8 +746,8 @@ xfs_attr_sf_findname( for (sfe = xfs_attr_sf_firstentry(sf); sfe < xfs_attr_sf_endptr(sf); sfe = xfs_attr_sf_nextentry(sfe)) { - if (xfs_attr_match(args, sfe->namelen, sfe->nameval, - sfe->flags)) + if (xfs_attr_match(args, sfe->flags, sfe->nameval, + sfe->namelen)) return sfe; } @@ -2443,15 +2443,16 @@ xfs_attr3_leaf_lookup_int( */ if (entry->flags & XFS_ATTR_LOCAL) { name_loc = xfs_attr3_leaf_name_local(leaf, probe); - if (!xfs_attr_match(args, name_loc->namelen, - name_loc->nameval, entry->flags)) + if (!xfs_attr_match(args, entry->flags, + name_loc->nameval, + name_loc->namelen)) continue; args->index = probe; return -EEXIST; } else { name_rmt = xfs_attr3_leaf_name_remote(leaf, probe); - if (!xfs_attr_match(args, name_rmt->namelen, - name_rmt->name, entry->flags)) + if (!xfs_attr_match(args, entry->flags, name_rmt->name, + name_rmt->namelen)) continue; args->index = probe; args->rmtvaluelen = be32_to_cpu(name_rmt->valuelen); From f49af061f49c004fb6df7f791f39f9ed370f767b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Apr 2024 09:47:36 -0700 Subject: [PATCH 0425/1702] xfs: check the flags earlier in xfs_attr_match Checking the flags match is much cheaper than a memcmp, so do it early on in xfs_attr_match, and also add a little helper to calculate the match mask right under the comment explaining the logic for it. Signed-off-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_attr_leaf.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index bb00183d13492..c47fad39744ee 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -507,6 +507,13 @@ xfs_attr3_leaf_read( * INCOMPLETE flag will not be set in attr->attr_filter, but rather * XFS_DA_OP_RECOVERY will be set in args->op_flags. */ +static inline unsigned int xfs_attr_match_mask(const struct xfs_da_args *args) +{ + if (args->op_flags & XFS_DA_OP_RECOVERY) + return XFS_ATTR_NSP_ONDISK_MASK; + return XFS_ATTR_NSP_ONDISK_MASK | XFS_ATTR_INCOMPLETE; +} + static bool xfs_attr_match( struct xfs_da_args *args, @@ -514,21 +521,15 @@ xfs_attr_match( const unsigned char *name, unsigned int namelen) { + unsigned int mask = xfs_attr_match_mask(args); if (args->namelen != namelen) return false; + if ((args->attr_filter & mask) != (attr_flags & mask)) + return false; if (memcmp(args->name, name, namelen) != 0) return false; - /* Recovery ignores the INCOMPLETE flag. */ - if ((args->op_flags & XFS_DA_OP_RECOVERY) && - args->attr_filter == (attr_flags & XFS_ATTR_NSP_ONDISK_MASK)) - return true; - - /* All remaining matches need to be filtered by INCOMPLETE state. */ - if (args->attr_filter != - (attr_flags & (XFS_ATTR_NSP_ONDISK_MASK | XFS_ATTR_INCOMPLETE))) - return false; return true; } From 9713dc88773d066413ae23aa474b13241507a89e Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:37 -0700 Subject: [PATCH 0426/1702] xfs: move xfs_attr_defer_add to xfs_attr_item.c Move the code that adds the incore xfs_attr_item deferred work data to a transaction live with the ATTRI log item code. This means that the upper level extended attribute code no longer has to know about the inner workings of the ATTRI log items. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 37 +++---------------------------------- fs/xfs/xfs_attr_item.c | 30 ++++++++++++++++++++++++++++++ fs/xfs/xfs_attr_item.h | 8 ++++++++ 3 files changed, 41 insertions(+), 34 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 629fb25d149cf..50eab63ff3be1 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -901,37 +901,6 @@ xfs_attr_lookup( return error; } -static void -xfs_attr_defer_add( - struct xfs_da_args *args, - unsigned int op_flags) -{ - - struct xfs_attr_intent *new; - - new = kmem_cache_zalloc(xfs_attr_intent_cache, - GFP_KERNEL | __GFP_NOFAIL); - new->xattri_op_flags = op_flags; - new->xattri_da_args = args; - - switch (op_flags) { - case XFS_ATTRI_OP_FLAGS_SET: - new->xattri_dela_state = xfs_attr_init_add_state(args); - break; - case XFS_ATTRI_OP_FLAGS_REPLACE: - new->xattri_dela_state = xfs_attr_init_replace_state(args); - break; - case XFS_ATTRI_OP_FLAGS_REMOVE: - new->xattri_dela_state = xfs_attr_init_remove_state(args); - break; - default: - ASSERT(0); - } - - xfs_defer_add(args->trans, &new->xattri_list, &xfs_attr_defer_type); - trace_xfs_attr_defer_add(new->xattri_dela_state, args->dp); -} - int xfs_attr_set( struct xfs_da_args *args, @@ -1021,14 +990,14 @@ xfs_attr_set( case -EEXIST: if (op == XFS_ATTRUPDATE_REMOVE) { /* if no value, we are performing a remove operation */ - xfs_attr_defer_add(args, XFS_ATTRI_OP_FLAGS_REMOVE); + xfs_attr_defer_add(args, XFS_ATTR_DEFER_REMOVE); break; } /* Pure create fails if the attr already exists */ if (op == XFS_ATTRUPDATE_CREATE) goto out_trans_cancel; - xfs_attr_defer_add(args, XFS_ATTRI_OP_FLAGS_REPLACE); + xfs_attr_defer_add(args, XFS_ATTR_DEFER_REPLACE); break; case -ENOATTR: /* Can't remove what isn't there. */ @@ -1038,7 +1007,7 @@ xfs_attr_set( /* Pure replace fails if no existing attr to replace. */ if (op == XFS_ATTRUPDATE_REPLACE) goto out_trans_cancel; - xfs_attr_defer_add(args, XFS_ATTRI_OP_FLAGS_SET); + xfs_attr_defer_add(args, XFS_ATTR_DEFER_SET); break; default: goto out_trans_cancel; diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index a65ac74797680..a7d6c9af47e88 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -727,6 +727,36 @@ xfs_attr_create_done( return &attrdp->attrd_item; } +void +xfs_attr_defer_add( + struct xfs_da_args *args, + enum xfs_attr_defer_op op) +{ + struct xfs_attr_intent *new; + + new = kmem_cache_zalloc(xfs_attr_intent_cache, + GFP_NOFS | __GFP_NOFAIL); + new->xattri_da_args = args; + + switch (op) { + case XFS_ATTR_DEFER_SET: + new->xattri_op_flags = XFS_ATTRI_OP_FLAGS_SET; + new->xattri_dela_state = xfs_attr_init_add_state(args); + break; + case XFS_ATTR_DEFER_REPLACE: + new->xattri_op_flags = XFS_ATTRI_OP_FLAGS_REPLACE; + new->xattri_dela_state = xfs_attr_init_replace_state(args); + break; + case XFS_ATTR_DEFER_REMOVE: + new->xattri_op_flags = XFS_ATTRI_OP_FLAGS_REMOVE; + new->xattri_dela_state = xfs_attr_init_remove_state(args); + break; + } + + xfs_defer_add(args->trans, &new->xattri_list, &xfs_attr_defer_type); + trace_xfs_attr_defer_add(new->xattri_dela_state, args->dp); +} + const struct xfs_defer_op_type xfs_attr_defer_type = { .name = "attr", .max_items = 1, diff --git a/fs/xfs/xfs_attr_item.h b/fs/xfs/xfs_attr_item.h index 3280a79302876..c32b669b0e16a 100644 --- a/fs/xfs/xfs_attr_item.h +++ b/fs/xfs/xfs_attr_item.h @@ -51,4 +51,12 @@ struct xfs_attrd_log_item { extern struct kmem_cache *xfs_attri_cache; extern struct kmem_cache *xfs_attrd_cache; +enum xfs_attr_defer_op { + XFS_ATTR_DEFER_SET, + XFS_ATTR_DEFER_REMOVE, + XFS_ATTR_DEFER_REPLACE, +}; + +void xfs_attr_defer_add(struct xfs_da_args *args, enum xfs_attr_defer_op op); + #endif /* __XFS_ATTR_ITEM_H__ */ From a64e0134754bf88021e937aa34f1fbb5b524e585 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:38 -0700 Subject: [PATCH 0427/1702] xfs: create a separate hashname function for extended attributes Create a separate function to compute name hashvalues for extended attributes. When we get to parent pointers we'll be altering the rules so that metadump obfuscation doesn't turn heinous. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 28 ++++++++++++++++++++++++++-- fs/xfs/libxfs/xfs_attr.h | 14 ++++++++++++++ fs/xfs/libxfs/xfs_attr_leaf.c | 3 +-- fs/xfs/scrub/attr.c | 11 ++++++++--- fs/xfs/xfs_attr_item.c | 2 +- fs/xfs/xfs_attr_list.c | 5 ++++- 6 files changed, 54 insertions(+), 9 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 50eab63ff3be1..8262c263be9d1 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -280,7 +280,7 @@ xfs_attr_get( args->owner = args->dp->i_ino; args->geo = args->dp->i_mount->m_attr_geo; args->whichfork = XFS_ATTR_FORK; - args->hashval = xfs_da_hashname(args->name, args->namelen); + xfs_attr_sethash(args); /* Entirely possible to look up a name which doesn't exist */ args->op_flags = XFS_DA_OP_OKNOENT; @@ -415,6 +415,30 @@ xfs_attr_sf_addname( return error; } +/* Compute the hash value for a user/root/secure extended attribute */ +xfs_dahash_t +xfs_attr_hashname( + const uint8_t *name, + int namelen) +{ + return xfs_da_hashname(name, namelen); +} + +/* Compute the hash value for any extended attribute from any namespace. */ +xfs_dahash_t +xfs_attr_hashval( + struct xfs_mount *mp, + unsigned int attr_flags, + const uint8_t *name, + int namelen, + const void *value, + int valuelen) +{ + ASSERT(xfs_attr_check_namespace(attr_flags)); + + return xfs_attr_hashname(name, namelen); +} + /* * Handle the state change on completion of a multi-state attr operation. * @@ -925,7 +949,7 @@ xfs_attr_set( args->owner = args->dp->i_ino; args->geo = mp->m_attr_geo; args->whichfork = XFS_ATTR_FORK; - args->hashval = xfs_da_hashname(args->name, args->namelen); + xfs_attr_sethash(args); /* * We have no control over the attribute names that userspace passes us diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h index cd106b0a424fa..c63b1d610e53b 100644 --- a/fs/xfs/libxfs/xfs_attr.h +++ b/fs/xfs/libxfs/xfs_attr.h @@ -628,6 +628,20 @@ xfs_attr_init_replace_state(struct xfs_da_args *args) return xfs_attr_init_add_state(args); } +xfs_dahash_t xfs_attr_hashname(const uint8_t *name, int namelen); + +xfs_dahash_t xfs_attr_hashval(struct xfs_mount *mp, unsigned int attr_flags, + const uint8_t *name, int namelen, const void *value, + int valuelen); + +/* Set the hash value for any extended attribute from any namespace. */ +static inline void xfs_attr_sethash(struct xfs_da_args *args) +{ + args->hashval = xfs_attr_hashval(args->dp->i_mount, args->attr_filter, + args->name, args->namelen, + args->value, args->valuelen); +} + extern struct kmem_cache *xfs_attr_intent_cache; int __init xfs_attr_intent_init_cache(void); void xfs_attr_intent_destroy_cache(void); diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index c47fad39744ee..e54a8372a30a4 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -948,14 +948,13 @@ xfs_attr_shortform_to_leaf( nargs.namelen = sfe->namelen; nargs.value = &sfe->nameval[nargs.namelen]; nargs.valuelen = sfe->valuelen; - nargs.hashval = xfs_da_hashname(sfe->nameval, - sfe->namelen); nargs.attr_filter = sfe->flags & XFS_ATTR_NSP_ONDISK_MASK; if (!xfs_attr_check_namespace(sfe->flags)) { xfs_da_mark_sick(args); error = -EFSCORRUPTED; goto out; } + xfs_attr_sethash(&nargs); error = xfs_attr3_leaf_lookup_int(bp, &nargs); /* set a->index */ ASSERT(error == -ENOATTR); error = xfs_attr3_leaf_add(bp, &nargs); diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index 7789bd2f09507..22d7ef4df1699 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -179,7 +179,6 @@ xchk_xattr_actor( .dp = ip, .name = name, .namelen = namelen, - .hashval = xfs_da_hashname(name, namelen), .trans = sc->tp, .valuelen = valuelen, .owner = ip->i_ino, @@ -230,6 +229,7 @@ xchk_xattr_actor( args.value = ab->value; + xfs_attr_sethash(&args); error = xfs_attr_get_ilocked(&args); /* ENODATA means the hash lookup failed and the attr is bad */ if (error == -ENODATA) @@ -525,7 +525,10 @@ xchk_xattr_rec( xchk_da_set_corrupt(ds, level); goto out; } - calc_hash = xfs_da_hashname(lentry->nameval, lentry->namelen); + calc_hash = xfs_attr_hashval(mp, ent->flags, lentry->nameval, + lentry->namelen, + lentry->nameval + lentry->namelen, + be16_to_cpu(lentry->valuelen)); } else { rentry = (struct xfs_attr_leaf_name_remote *) (((char *)bp->b_addr) + nameidx); @@ -533,7 +536,9 @@ xchk_xattr_rec( xchk_da_set_corrupt(ds, level); goto out; } - calc_hash = xfs_da_hashname(rentry->name, rentry->namelen); + calc_hash = xfs_attr_hashval(mp, ent->flags, rentry->name, + rentry->namelen, NULL, + be32_to_cpu(rentry->valuelen)); } if (calc_hash != hash) xchk_da_set_corrupt(ds, level); diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index a7d6c9af47e88..4a57bcff49ebd 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -582,13 +582,13 @@ xfs_attri_recover_work( args->whichfork = XFS_ATTR_FORK; args->name = nv->name.i_addr; args->namelen = nv->name.i_len; - args->hashval = xfs_da_hashname(args->name, args->namelen); args->value = nv->value.i_addr; args->valuelen = nv->value.i_len; args->attr_filter = attrp->alfi_attr_filter & XFS_ATTRI_FILTER_MASK; args->op_flags = XFS_DA_OP_RECOVERY | XFS_DA_OP_OKNOENT | XFS_DA_OP_LOGGED; args->owner = args->dp->i_ino; + xfs_attr_sethash(args); switch (xfs_attr_intent_op(attr)) { case XFS_ATTRI_OP_FLAGS_SET: diff --git a/fs/xfs/xfs_attr_list.c b/fs/xfs/xfs_attr_list.c index 903ed46c68872..9bc4b5322539a 100644 --- a/fs/xfs/xfs_attr_list.c +++ b/fs/xfs/xfs_attr_list.c @@ -135,12 +135,15 @@ xfs_attr_shortform_list( } sbp->entno = i; - sbp->hash = xfs_da_hashname(sfe->nameval, sfe->namelen); sbp->name = sfe->nameval; sbp->namelen = sfe->namelen; /* These are bytes, and both on-disk, don't endian-flip */ sbp->valuelen = sfe->valuelen; sbp->flags = sfe->flags; + sbp->hash = xfs_attr_hashval(dp->i_mount, sfe->flags, + sfe->nameval, sfe->namelen, + sfe->nameval + sfe->namelen, + sfe->valuelen); sfe = xfs_attr_sf_nextentry(sfe); sbp++; nsbuf++; From 98493ff878859eb0adefbc57a49ad47a92dfd252 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 22 Apr 2024 09:47:38 -0700 Subject: [PATCH 0428/1702] xfs: add parent pointer support to attribute code Add the new parent attribute type. XFS_ATTR_PARENT is used only for parent pointer entries; it uses reserved blocks like XFS_ATTR_ROOT. Signed-off-by: Mark Tinguely Signed-off-by: Dave Chinner Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_da_format.h | 9 +++++++-- fs/xfs/libxfs/xfs_log_format.h | 1 + fs/xfs/xfs_trace.h | 3 ++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/fs/xfs/libxfs/xfs_da_format.h b/fs/xfs/libxfs/xfs_da_format.h index ecd0616f5776a..0c80f7ab9475a 100644 --- a/fs/xfs/libxfs/xfs_da_format.h +++ b/fs/xfs/libxfs/xfs_da_format.h @@ -714,13 +714,17 @@ struct xfs_attr3_leafblock { #define XFS_ATTR_LOCAL_BIT 0 /* attr is stored locally */ #define XFS_ATTR_ROOT_BIT 1 /* limit access to trusted attrs */ #define XFS_ATTR_SECURE_BIT 2 /* limit access to secure attrs */ +#define XFS_ATTR_PARENT_BIT 3 /* parent pointer attrs */ #define XFS_ATTR_INCOMPLETE_BIT 7 /* attr in middle of create/delete */ #define XFS_ATTR_LOCAL (1u << XFS_ATTR_LOCAL_BIT) #define XFS_ATTR_ROOT (1u << XFS_ATTR_ROOT_BIT) #define XFS_ATTR_SECURE (1u << XFS_ATTR_SECURE_BIT) +#define XFS_ATTR_PARENT (1u << XFS_ATTR_PARENT_BIT) #define XFS_ATTR_INCOMPLETE (1u << XFS_ATTR_INCOMPLETE_BIT) -#define XFS_ATTR_NSP_ONDISK_MASK (XFS_ATTR_ROOT | XFS_ATTR_SECURE) +#define XFS_ATTR_NSP_ONDISK_MASK (XFS_ATTR_ROOT | \ + XFS_ATTR_SECURE | \ + XFS_ATTR_PARENT) #define XFS_ATTR_ONDISK_MASK (XFS_ATTR_NSP_ONDISK_MASK | \ XFS_ATTR_LOCAL | \ @@ -729,7 +733,8 @@ struct xfs_attr3_leafblock { #define XFS_ATTR_NAMESPACE_STR \ { XFS_ATTR_LOCAL, "local" }, \ { XFS_ATTR_ROOT, "root" }, \ - { XFS_ATTR_SECURE, "secure" } + { XFS_ATTR_SECURE, "secure" }, \ + { XFS_ATTR_PARENT, "parent" } /* * Alignment for namelist and valuelist entries (since they are mixed diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h index accba2acd623d..020aebd101432 100644 --- a/fs/xfs/libxfs/xfs_log_format.h +++ b/fs/xfs/libxfs/xfs_log_format.h @@ -1034,6 +1034,7 @@ struct xfs_icreate_log { */ #define XFS_ATTRI_FILTER_MASK (XFS_ATTR_ROOT | \ XFS_ATTR_SECURE | \ + XFS_ATTR_PARENT | \ XFS_ATTR_INCOMPLETE) /* diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index 57f225ba7e8ab..5621db48e7634 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -91,7 +91,8 @@ struct xfs_exchrange; #define XFS_ATTR_FILTER_FLAGS \ { XFS_ATTR_ROOT, "ROOT" }, \ { XFS_ATTR_SECURE, "SECURE" }, \ - { XFS_ATTR_INCOMPLETE, "INCOMPLETE" } + { XFS_ATTR_INCOMPLETE, "INCOMPLETE" }, \ + { XFS_ATTR_PARENT, "PARENT" } DECLARE_EVENT_CLASS(xfs_attr_list_class, TP_PROTO(struct xfs_attr_list_context *ctx), From 8337d58ab2868f231a29824cd86d2e309bd36fa9 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 22 Apr 2024 09:47:39 -0700 Subject: [PATCH 0429/1702] xfs: define parent pointer ondisk extended attribute format We need to define the parent pointer attribute format before we start adding support for it into all the code that needs to use it. The EA format we will use encodes the following information: name={dirent name} value={parent inumber, parent inode generation} hash=xfs_dir2_hashname(dirent name) ^ (parent_inumber) The inode/gen gives all the information we need to reliably identify the parent without requiring child->parent lock ordering, and allows userspace to do pathname component level reconstruction without the kernel ever needing to verify the parent itself as part of ioctl calls. By using the name-value lookup mode in the extended attribute code to match parent pointers using both the xattr name and value, we can identify the exact parent pointer EA we need to modify/remove in rename/unlink operations without searching the entire EA space. By storing the dirent name, we have enough information to be able to validate and reconstruct damaged directory trees. Earlier iterations of this patchset encoded the directory offset in the parent pointer key, but this format required repair to keep that in sync across directory rebuilds, which is unnecessary complexity. Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_da_format.h | 13 +++++++++++++ fs/xfs/libxfs/xfs_ondisk.h | 1 + 2 files changed, 14 insertions(+) diff --git a/fs/xfs/libxfs/xfs_da_format.h b/fs/xfs/libxfs/xfs_da_format.h index 0c80f7ab9475a..1395ad1937c53 100644 --- a/fs/xfs/libxfs/xfs_da_format.h +++ b/fs/xfs/libxfs/xfs_da_format.h @@ -890,4 +890,17 @@ static inline unsigned int xfs_dir2_dirblock_bytes(struct xfs_sb *sbp) xfs_failaddr_t xfs_da3_blkinfo_verify(struct xfs_buf *bp, struct xfs_da3_blkinfo *hdr3); +/* + * Parent pointer attribute format definition + * + * The xattr name contains the dirent name. + * The xattr value encodes the parent inode number and generation to ease + * opening parents by handle. + * The xattr hashval is xfs_dir2_namehash() ^ p_ino + */ +struct xfs_parent_rec { + __be64 p_ino; + __be32 p_gen; +} __packed; + #endif /* __XFS_DA_FORMAT_H__ */ diff --git a/fs/xfs/libxfs/xfs_ondisk.h b/fs/xfs/libxfs/xfs_ondisk.h index 81885a6a028ed..25952ef584eee 100644 --- a/fs/xfs/libxfs/xfs_ondisk.h +++ b/fs/xfs/libxfs/xfs_ondisk.h @@ -119,6 +119,7 @@ xfs_check_ondisk_structs(void) XFS_CHECK_OFFSET(xfs_dir2_sf_entry_t, offset, 1); XFS_CHECK_OFFSET(xfs_dir2_sf_entry_t, name, 3); XFS_CHECK_STRUCT_SIZE(xfs_dir2_sf_hdr_t, 10); + XFS_CHECK_STRUCT_SIZE(struct xfs_parent_rec, 12); /* log structures */ XFS_CHECK_STRUCT_SIZE(struct xfs_buf_log_format, 88); From f041455eb5773eda3291903ad6d1f33d4798e9a2 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:40 -0700 Subject: [PATCH 0430/1702] xfs: allow xattr matching on name and value for parent pointers If a file is hardlinked with the same name but from multiple parents, the parent pointers will all have the same dirent name (== attr name) but with different parent_ino/parent_gen values. To disambiguate, we need to be able to match on both the attr name and the attr value. This is in contrast to regular xattrs, which are matchtg edit d only on name. Therefore, plumb in the ability to match shortform and local attrs on name and value in the XFS_ATTR_PARENT namespace. Parent pointer attr values are never large enough to be stored in a remote attr, so we need can reject these cases as corruption. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr_leaf.c | 52 +++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index e54a8372a30a4..1a374c6885d7f 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -514,12 +514,37 @@ static inline unsigned int xfs_attr_match_mask(const struct xfs_da_args *args) return XFS_ATTR_NSP_ONDISK_MASK | XFS_ATTR_INCOMPLETE; } +static inline bool +xfs_attr_parent_match( + const struct xfs_da_args *args, + const void *value, + unsigned int valuelen) +{ + ASSERT(args->value != NULL); + + /* Parent pointers do not use remote values */ + if (!value) + return false; + + /* + * The only value we support is a parent rec. However, we'll accept + * any valuelen so that offline repair can delete ATTR_PARENT values + * that are not parent pointers. + */ + if (valuelen != args->valuelen) + return false; + + return memcmp(args->value, value, valuelen) == 0; +} + static bool xfs_attr_match( struct xfs_da_args *args, unsigned int attr_flags, const unsigned char *name, - unsigned int namelen) + unsigned int namelen, + const void *value, + unsigned int valuelen) { unsigned int mask = xfs_attr_match_mask(args); @@ -530,6 +555,9 @@ xfs_attr_match( if (memcmp(args->name, name, namelen) != 0) return false; + if (attr_flags & XFS_ATTR_PARENT) + return xfs_attr_parent_match(args, value, valuelen); + return true; } @@ -539,6 +567,13 @@ xfs_attr_copy_value( unsigned char *value, int valuelen) { + /* + * Parent pointer lookups require the caller to specify the name and + * value, so don't copy anything. + */ + if (args->attr_filter & XFS_ATTR_PARENT) + return 0; + /* * No copy if all we have to do is get the length */ @@ -748,7 +783,8 @@ xfs_attr_sf_findname( sfe < xfs_attr_sf_endptr(sf); sfe = xfs_attr_sf_nextentry(sfe)) { if (xfs_attr_match(args, sfe->flags, sfe->nameval, - sfe->namelen)) + sfe->namelen, &sfe->nameval[sfe->namelen], + sfe->valuelen)) return sfe; } @@ -2444,18 +2480,22 @@ xfs_attr3_leaf_lookup_int( if (entry->flags & XFS_ATTR_LOCAL) { name_loc = xfs_attr3_leaf_name_local(leaf, probe); if (!xfs_attr_match(args, entry->flags, - name_loc->nameval, - name_loc->namelen)) + name_loc->nameval, name_loc->namelen, + &name_loc->nameval[name_loc->namelen], + be16_to_cpu(name_loc->valuelen))) continue; args->index = probe; return -EEXIST; } else { + unsigned int valuelen; + name_rmt = xfs_attr3_leaf_name_remote(leaf, probe); + valuelen = be32_to_cpu(name_rmt->valuelen); if (!xfs_attr_match(args, entry->flags, name_rmt->name, - name_rmt->namelen)) + name_rmt->namelen, NULL, valuelen)) continue; args->index = probe; - args->rmtvaluelen = be32_to_cpu(name_rmt->valuelen); + args->rmtvaluelen = valuelen; args->rmtblkno = be32_to_cpu(name_rmt->valueblk); args->rmtblkcnt = xfs_attr3_rmt_blocks( args->dp->i_mount, From a918f5f2cd2c9d2bf94f485c5cebbf47fb0627df Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:41 -0700 Subject: [PATCH 0431/1702] xfs: refactor xfs_is_using_logged_xattrs checks in attr item recovery Move this feature check down to the per-op checks so that we can ensure that we never see parent pointer attr items on non-pptr filesystems, and that logged xattrs are turned on for non-pptr attr items. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_attr_item.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index 4a57bcff49ebd..413e3d3959a5d 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -480,9 +480,6 @@ xfs_attri_validate( { unsigned int op = xfs_attr_log_item_op(attrp); - if (!xfs_is_using_logged_xattrs(mp)) - return false; - if (attrp->__pad != 0) return false; @@ -499,12 +496,16 @@ xfs_attri_validate( switch (op) { case XFS_ATTRI_OP_FLAGS_SET: case XFS_ATTRI_OP_FLAGS_REPLACE: + if (!xfs_is_using_logged_xattrs(mp)) + return false; if (attrp->alfi_value_len > XATTR_SIZE_MAX) return false; if (!xfs_attri_validate_namelen(attrp->alfi_name_len)) return false; break; case XFS_ATTRI_OP_FLAGS_REMOVE: + if (!xfs_is_using_logged_xattrs(mp)) + return false; if (attrp->alfi_value_len != 0) return false; if (!xfs_attri_validate_namelen(attrp->alfi_name_len)) From 5773f7f82be5aa98e4883566072d33342814cebe Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:42 -0700 Subject: [PATCH 0432/1702] xfs: create attr log item opcodes and formats for parent pointers Make the necessary alterations to the extended attribute log intent item ondisk format so that we can log parent pointer operations. This requires the creation of new opcodes specific to parent pointers, and a new four-argument replace operation to handle renames. At this point this part of the patchset has changed so much from what Allison original wrote that I no longer think her SoB applies. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 19 +++ fs/xfs/libxfs/xfs_attr.h | 4 +- fs/xfs/libxfs/xfs_da_btree.h | 4 + fs/xfs/libxfs/xfs_log_format.h | 22 ++- fs/xfs/xfs_attr_item.c | 259 ++++++++++++++++++++++++++++++--- fs/xfs/xfs_attr_item.h | 2 + 6 files changed, 284 insertions(+), 26 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 8262c263be9d1..78c87c405e33c 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -439,6 +439,23 @@ xfs_attr_hashval( return xfs_attr_hashname(name, namelen); } +/* + * PPTR_REPLACE operations require the caller to set the old and new names and + * values explicitly. Update the canonical fields to the new name and value + * here now that the removal phase has finished. + */ +static void +xfs_attr_update_pptr_replace_args( + struct xfs_da_args *args) +{ + ASSERT(args->new_namelen > 0); + args->name = args->new_name; + args->namelen = args->new_namelen; + args->value = args->new_value; + args->valuelen = args->new_valuelen; + xfs_attr_sethash(args); +} + /* * Handle the state change on completion of a multi-state attr operation. * @@ -459,6 +476,8 @@ xfs_attr_complete_op( if (!(args->op_flags & XFS_DA_OP_REPLACE)) replace_state = XFS_DAS_DONE; + else if (xfs_attr_intent_op(attr) == XFS_ATTRI_OP_FLAGS_PPTR_REPLACE) + xfs_attr_update_pptr_replace_args(args); args->op_flags &= ~XFS_DA_OP_REPLACE; args->attr_filter &= ~XFS_ATTR_INCOMPLETE; diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h index c63b1d610e53b..d0ed7ea58ab0f 100644 --- a/fs/xfs/libxfs/xfs_attr.h +++ b/fs/xfs/libxfs/xfs_attr.h @@ -510,8 +510,8 @@ struct xfs_attr_intent { struct xfs_da_args *xattri_da_args; /* - * Shared buffer containing the attr name and value so that the logging - * code can share large memory buffers between log items. + * Shared buffer containing the attr name, new name, and value so that + * the logging code can share large memory buffers between log items. */ struct xfs_attri_log_nameval *xattri_nameval; diff --git a/fs/xfs/libxfs/xfs_da_btree.h b/fs/xfs/libxfs/xfs_da_btree.h index 17cef594b5bbb..354d5d65043e4 100644 --- a/fs/xfs/libxfs/xfs_da_btree.h +++ b/fs/xfs/libxfs/xfs_da_btree.h @@ -55,7 +55,9 @@ enum xfs_dacmp { typedef struct xfs_da_args { struct xfs_da_geometry *geo; /* da block geometry */ const uint8_t *name; /* string (maybe not NULL terminated) */ + const uint8_t *new_name; /* new attr name */ void *value; /* set of bytes (maybe contain NULLs) */ + void *new_value; /* new xattr value (may contain NULLs) */ struct xfs_inode *dp; /* directory inode to manipulate */ struct xfs_trans *trans; /* current trans (changes over time) */ @@ -63,10 +65,12 @@ typedef struct xfs_da_args { xfs_ino_t owner; /* inode that owns the dir/attr data */ int valuelen; /* length of value */ + int new_valuelen; /* length of new_value */ uint8_t filetype; /* filetype of inode for directories */ uint8_t op_flags; /* operation flags */ uint8_t attr_filter; /* XFS_ATTR_{ROOT,SECURE,INCOMPLETE} */ short namelen; /* length of string (maybe no NULL) */ + short new_namelen; /* length of new attr name */ xfs_dahash_t hashval; /* hash value of name */ xfs_extlen_t total; /* total blocks needed, for 1st bmap */ int whichfork; /* data or attribute fork */ diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h index 020aebd101432..632dd97324557 100644 --- a/fs/xfs/libxfs/xfs_log_format.h +++ b/fs/xfs/libxfs/xfs_log_format.h @@ -115,11 +115,13 @@ struct xfs_unmount_log_format { #define XLOG_REG_TYPE_BUD_FORMAT 26 #define XLOG_REG_TYPE_ATTRI_FORMAT 27 #define XLOG_REG_TYPE_ATTRD_FORMAT 28 -#define XLOG_REG_TYPE_ATTR_NAME 29 +#define XLOG_REG_TYPE_ATTR_NAME 29 #define XLOG_REG_TYPE_ATTR_VALUE 30 #define XLOG_REG_TYPE_XMI_FORMAT 31 #define XLOG_REG_TYPE_XMD_FORMAT 32 -#define XLOG_REG_TYPE_MAX 32 +#define XLOG_REG_TYPE_ATTR_NEWNAME 33 +#define XLOG_REG_TYPE_ATTR_NEWVALUE 34 +#define XLOG_REG_TYPE_MAX 34 /* * Flags to log operation header @@ -1026,6 +1028,9 @@ struct xfs_icreate_log { #define XFS_ATTRI_OP_FLAGS_SET 1 /* Set the attribute */ #define XFS_ATTRI_OP_FLAGS_REMOVE 2 /* Remove the attribute */ #define XFS_ATTRI_OP_FLAGS_REPLACE 3 /* Replace the attribute */ +#define XFS_ATTRI_OP_FLAGS_PPTR_SET 4 /* Set parent pointer */ +#define XFS_ATTRI_OP_FLAGS_PPTR_REMOVE 5 /* Remove parent pointer */ +#define XFS_ATTRI_OP_FLAGS_PPTR_REPLACE 6 /* Replace parent pointer */ #define XFS_ATTRI_OP_FLAGS_TYPE_MASK 0xFF /* Flags type mask */ /* @@ -1048,7 +1053,18 @@ struct xfs_attri_log_format { uint64_t alfi_id; /* attri identifier */ uint64_t alfi_ino; /* the inode for this attr operation */ uint32_t alfi_op_flags; /* marks the op as a set or remove */ - uint32_t alfi_name_len; /* attr name length */ + union { + uint32_t alfi_name_len; /* attr name length */ + struct { + /* + * For PPTR_REPLACE, these are the lengths of the old + * and new attr names. The new and old values must + * have the same length. + */ + uint16_t alfi_old_name_len; + uint16_t alfi_new_name_len; + }; + }; uint32_t alfi_value_len; /* attr value length */ uint32_t alfi_attr_filter;/* attr filter flags */ }; diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index 413e3d3959a5d..be5064f5a5315 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -73,8 +73,12 @@ static inline struct xfs_attri_log_nameval * xfs_attri_log_nameval_alloc( const void *name, unsigned int name_len, + const void *new_name, + unsigned int new_name_len, const void *value, - unsigned int value_len) + unsigned int value_len, + const void *new_value, + unsigned int new_value_len) { struct xfs_attri_log_nameval *nv; @@ -83,15 +87,26 @@ xfs_attri_log_nameval_alloc( * this. But kvmalloc() utterly sucks, so we use our own version. */ nv = xlog_kvmalloc(sizeof(struct xfs_attri_log_nameval) + - name_len + value_len); + name_len + new_name_len + value_len + + new_value_len); nv->name.i_addr = nv + 1; nv->name.i_len = name_len; nv->name.i_type = XLOG_REG_TYPE_ATTR_NAME; memcpy(nv->name.i_addr, name, name_len); + if (new_name_len) { + nv->new_name.i_addr = nv->name.i_addr + name_len; + nv->new_name.i_len = new_name_len; + memcpy(nv->new_name.i_addr, new_name, new_name_len); + } else { + nv->new_name.i_addr = NULL; + nv->new_name.i_len = 0; + } + nv->new_name.i_type = XLOG_REG_TYPE_ATTR_NEWNAME; + if (value_len) { - nv->value.i_addr = nv->name.i_addr + name_len; + nv->value.i_addr = nv->name.i_addr + name_len + new_name_len; nv->value.i_len = value_len; memcpy(nv->value.i_addr, value, value_len); } else { @@ -100,6 +115,17 @@ xfs_attri_log_nameval_alloc( } nv->value.i_type = XLOG_REG_TYPE_ATTR_VALUE; + if (new_value_len) { + nv->new_value.i_addr = nv->name.i_addr + name_len + + new_name_len + value_len; + nv->new_value.i_len = new_value_len; + memcpy(nv->new_value.i_addr, new_value, new_value_len); + } else { + nv->new_value.i_addr = NULL; + nv->new_value.i_len = 0; + } + nv->new_value.i_type = XLOG_REG_TYPE_ATTR_NEWVALUE; + refcount_set(&nv->refcount, 1); return nv; } @@ -145,11 +171,20 @@ xfs_attri_item_size( *nbytes += sizeof(struct xfs_attri_log_format) + xlog_calc_iovec_len(nv->name.i_len); - if (!nv->value.i_len) - return; + if (nv->new_name.i_len) { + *nvecs += 1; + *nbytes += xlog_calc_iovec_len(nv->new_name.i_len); + } - *nvecs += 1; - *nbytes += xlog_calc_iovec_len(nv->value.i_len); + if (nv->value.i_len) { + *nvecs += 1; + *nbytes += xlog_calc_iovec_len(nv->value.i_len); + } + + if (nv->new_value.i_len) { + *nvecs += 1; + *nbytes += xlog_calc_iovec_len(nv->new_value.i_len); + } } /* @@ -179,15 +214,28 @@ xfs_attri_item_format( ASSERT(nv->name.i_len > 0); attrip->attri_format.alfi_size++; + if (nv->new_name.i_len > 0) + attrip->attri_format.alfi_size++; + if (nv->value.i_len > 0) attrip->attri_format.alfi_size++; + if (nv->new_value.i_len > 0) + attrip->attri_format.alfi_size++; + xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_ATTRI_FORMAT, &attrip->attri_format, sizeof(struct xfs_attri_log_format)); xlog_copy_from_iovec(lv, &vecp, &nv->name); + + if (nv->new_name.i_len > 0) + xlog_copy_from_iovec(lv, &vecp, &nv->new_name); + if (nv->value.i_len > 0) xlog_copy_from_iovec(lv, &vecp, &nv->value); + + if (nv->new_value.i_len > 0) + xlog_copy_from_iovec(lv, &vecp, &nv->new_value); } /* @@ -322,6 +370,8 @@ xfs_attr_log_item( const struct xfs_attr_intent *attr) { struct xfs_attri_log_format *attrp; + struct xfs_attri_log_nameval *nv = attr->xattri_nameval; + struct xfs_da_args *args = attr->xattri_da_args; /* * At this point the xfs_attr_intent has been constructed, and we've @@ -329,13 +379,25 @@ xfs_attr_log_item( * structure with fields from this xfs_attr_intent */ attrp = &attrip->attri_format; - attrp->alfi_ino = attr->xattri_da_args->dp->i_ino; + attrp->alfi_ino = args->dp->i_ino; ASSERT(!(attr->xattri_op_flags & ~XFS_ATTRI_OP_FLAGS_TYPE_MASK)); attrp->alfi_op_flags = attr->xattri_op_flags; - attrp->alfi_value_len = attr->xattri_nameval->value.i_len; - attrp->alfi_name_len = attr->xattri_nameval->name.i_len; - ASSERT(!(attr->xattri_da_args->attr_filter & ~XFS_ATTRI_FILTER_MASK)); - attrp->alfi_attr_filter = attr->xattri_da_args->attr_filter; + attrp->alfi_value_len = nv->value.i_len; + + switch (xfs_attr_log_item_op(attrp)) { + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: + ASSERT(nv->value.i_len == nv->new_value.i_len); + + attrp->alfi_old_name_len = nv->name.i_len; + attrp->alfi_new_name_len = nv->new_name.i_len; + break; + default: + attrp->alfi_name_len = nv->name.i_len; + break; + } + + ASSERT(!(args->attr_filter & ~XFS_ATTRI_FILTER_MASK)); + attrp->alfi_attr_filter = args->attr_filter; } /* Get an ATTRI. */ @@ -374,8 +436,11 @@ xfs_attr_create_intent( * Transfer our reference to the name/value buffer to the * deferred work state structure. */ - attr->xattri_nameval = xfs_attri_log_nameval_alloc(args->name, - args->namelen, args->value, args->valuelen); + attr->xattri_nameval = xfs_attri_log_nameval_alloc( + args->name, args->namelen, + args->new_name, args->new_namelen, + args->value, args->valuelen, + args->new_value, args->new_valuelen); } attrip = xfs_attri_init(mp, attr->xattri_nameval); @@ -494,6 +559,17 @@ xfs_attri_validate( return false; switch (op) { + case XFS_ATTRI_OP_FLAGS_PPTR_SET: + case XFS_ATTRI_OP_FLAGS_PPTR_REMOVE: + if (!xfs_has_parent(mp)) + return false; + if (attrp->alfi_value_len != sizeof(struct xfs_parent_rec)) + return false; + if (!xfs_attri_validate_namelen(attrp->alfi_name_len)) + return false; + if (!(attrp->alfi_attr_filter & XFS_ATTR_PARENT)) + return false; + break; case XFS_ATTRI_OP_FLAGS_SET: case XFS_ATTRI_OP_FLAGS_REPLACE: if (!xfs_is_using_logged_xattrs(mp)) @@ -511,6 +587,18 @@ xfs_attri_validate( if (!xfs_attri_validate_namelen(attrp->alfi_name_len)) return false; break; + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: + if (!xfs_has_parent(mp)) + return false; + if (!xfs_attri_validate_namelen(attrp->alfi_old_name_len)) + return false; + if (!xfs_attri_validate_namelen(attrp->alfi_new_name_len)) + return false; + if (attrp->alfi_value_len != sizeof(struct xfs_parent_rec)) + return false; + if (!(attrp->alfi_attr_filter & XFS_ATTR_PARENT)) + return false; + break; default: return false; } @@ -583,8 +671,12 @@ xfs_attri_recover_work( args->whichfork = XFS_ATTR_FORK; args->name = nv->name.i_addr; args->namelen = nv->name.i_len; + args->new_name = nv->new_name.i_addr; + args->new_namelen = nv->new_name.i_len; args->value = nv->value.i_addr; args->valuelen = nv->value.i_len; + args->new_value = nv->new_value.i_addr; + args->new_valuelen = nv->new_value.i_len; args->attr_filter = attrp->alfi_attr_filter & XFS_ATTRI_FILTER_MASK; args->op_flags = XFS_DA_OP_RECOVERY | XFS_DA_OP_OKNOENT | XFS_DA_OP_LOGGED; @@ -592,6 +684,8 @@ xfs_attri_recover_work( xfs_attr_sethash(args); switch (xfs_attr_intent_op(attr)) { + case XFS_ATTRI_OP_FLAGS_PPTR_SET: + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: case XFS_ATTRI_OP_FLAGS_SET: case XFS_ATTRI_OP_FLAGS_REPLACE: args->total = xfs_attr_calc_size(args, &local); @@ -600,6 +694,7 @@ xfs_attri_recover_work( else attr->xattri_dela_state = xfs_attr_init_add_state(args); break; + case XFS_ATTRI_OP_FLAGS_PPTR_REMOVE: case XFS_ATTRI_OP_FLAGS_REMOVE: attr->xattri_dela_state = xfs_attr_init_remove_state(args); break; @@ -700,7 +795,17 @@ xfs_attr_relog_intent( new_attrp->alfi_ino = old_attrp->alfi_ino; new_attrp->alfi_op_flags = old_attrp->alfi_op_flags; new_attrp->alfi_value_len = old_attrp->alfi_value_len; - new_attrp->alfi_name_len = old_attrp->alfi_name_len; + + switch (xfs_attr_log_item_op(old_attrp)) { + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: + new_attrp->alfi_new_name_len = old_attrp->alfi_new_name_len; + new_attrp->alfi_old_name_len = old_attrp->alfi_old_name_len; + break; + default: + new_attrp->alfi_name_len = old_attrp->alfi_name_len; + break; + } + new_attrp->alfi_attr_filter = old_attrp->alfi_attr_filter; return &new_attrip->attri_item; @@ -734,22 +839,61 @@ xfs_attr_defer_add( enum xfs_attr_defer_op op) { struct xfs_attr_intent *new; + unsigned int log_op = 0; + bool is_pptr = args->attr_filter & XFS_ATTR_PARENT; + + if (is_pptr) { + ASSERT(xfs_has_parent(args->dp->i_mount)); + ASSERT((args->attr_filter & ~XFS_ATTR_PARENT) == 0); + ASSERT(args->op_flags & XFS_DA_OP_LOGGED); + ASSERT(args->valuelen == sizeof(struct xfs_parent_rec)); + } new = kmem_cache_zalloc(xfs_attr_intent_cache, GFP_NOFS | __GFP_NOFAIL); new->xattri_da_args = args; + /* Compute log operation from the higher level op and namespace. */ switch (op) { case XFS_ATTR_DEFER_SET: - new->xattri_op_flags = XFS_ATTRI_OP_FLAGS_SET; - new->xattri_dela_state = xfs_attr_init_add_state(args); + if (is_pptr) + log_op = XFS_ATTRI_OP_FLAGS_PPTR_SET; + else + log_op = XFS_ATTRI_OP_FLAGS_SET; break; case XFS_ATTR_DEFER_REPLACE: - new->xattri_op_flags = XFS_ATTRI_OP_FLAGS_REPLACE; - new->xattri_dela_state = xfs_attr_init_replace_state(args); + if (is_pptr) + log_op = XFS_ATTRI_OP_FLAGS_PPTR_REPLACE; + else + log_op = XFS_ATTRI_OP_FLAGS_REPLACE; break; case XFS_ATTR_DEFER_REMOVE: - new->xattri_op_flags = XFS_ATTRI_OP_FLAGS_REMOVE; + if (is_pptr) + log_op = XFS_ATTRI_OP_FLAGS_PPTR_REMOVE; + else + log_op = XFS_ATTRI_OP_FLAGS_REMOVE; + break; + default: + ASSERT(0); + break; + } + new->xattri_op_flags = log_op; + + /* Set up initial attr operation state. */ + switch (log_op) { + case XFS_ATTRI_OP_FLAGS_PPTR_SET: + case XFS_ATTRI_OP_FLAGS_SET: + new->xattri_dela_state = xfs_attr_init_add_state(args); + break; + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: + ASSERT(args->new_valuelen == args->valuelen); + new->xattri_dela_state = xfs_attr_init_replace_state(args); + break; + case XFS_ATTRI_OP_FLAGS_REPLACE: + new->xattri_dela_state = xfs_attr_init_replace_state(args); + break; + case XFS_ATTRI_OP_FLAGS_PPTR_REMOVE: + case XFS_ATTRI_OP_FLAGS_REMOVE: new->xattri_dela_state = xfs_attr_init_remove_state(args); break; } @@ -824,9 +968,13 @@ xlog_recover_attri_commit_pass2( struct xfs_attri_log_nameval *nv; const void *attr_name; const void *attr_value = NULL; + const void *attr_new_name = NULL; + const void *attr_new_value = NULL; size_t len; unsigned int name_len = 0; unsigned int value_len = 0; + unsigned int new_name_len = 0; + unsigned int new_value_len = 0; unsigned int op, i = 0; /* Validate xfs_attri_log_format before the large memory allocation */ @@ -847,6 +995,17 @@ xlog_recover_attri_commit_pass2( /* Check the number of log iovecs makes sense for the op code. */ op = xfs_attr_log_item_op(attri_formatp); switch (op) { + case XFS_ATTRI_OP_FLAGS_PPTR_REMOVE: + case XFS_ATTRI_OP_FLAGS_PPTR_SET: + /* Log item, attr name, attr value */ + if (item->ri_total != 3) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + name_len = attri_formatp->alfi_name_len; + value_len = attri_formatp->alfi_value_len; + break; case XFS_ATTRI_OP_FLAGS_SET: case XFS_ATTRI_OP_FLAGS_REPLACE: /* Log item, attr name, attr value */ @@ -867,6 +1026,20 @@ xlog_recover_attri_commit_pass2( } name_len = attri_formatp->alfi_name_len; break; + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: + /* + * Log item, attr name, new attr name, attr value, new attr + * value + */ + if (item->ri_total != 5) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + name_len = attri_formatp->alfi_old_name_len; + new_name_len = attri_formatp->alfi_new_name_len; + new_value_len = value_len = attri_formatp->alfi_value_len; + break; default: XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, attri_formatp, len); @@ -881,6 +1054,16 @@ xlog_recover_attri_commit_pass2( return -EFSCORRUPTED; i++; + /* Validate the new attr name */ + if (new_name_len > 0) { + attr_new_name = xfs_attri_validate_name_iovec(mp, + attri_formatp, &item->ri_buf[i], + new_name_len); + if (!attr_new_name) + return -EFSCORRUPTED; + i++; + } + /* Validate the attr value, if present */ if (value_len != 0) { attr_value = xfs_attri_validate_value_iovec(mp, attri_formatp, @@ -890,6 +1073,16 @@ xlog_recover_attri_commit_pass2( i++; } + /* Validate the new attr value, if present */ + if (new_value_len != 0) { + attr_new_value = xfs_attri_validate_value_iovec(mp, + attri_formatp, &item->ri_buf[i], + new_value_len); + if (!attr_new_value) + return -EFSCORRUPTED; + i++; + } + /* * Make sure we got the correct number of buffers for the operation * that we just loaded. @@ -909,12 +1102,17 @@ xlog_recover_attri_commit_pass2( return -EFSCORRUPTED; } fallthrough; + case XFS_ATTRI_OP_FLAGS_PPTR_REMOVE: + case XFS_ATTRI_OP_FLAGS_PPTR_SET: case XFS_ATTRI_OP_FLAGS_SET: case XFS_ATTRI_OP_FLAGS_REPLACE: /* * Regular xattr set/remove/replace operations require a name * and do not take a newname. Values are optional for set and * replace. + * + * Name-value set/remove operations must have a name, do not + * take a newname, and can take a value. */ if (attr_name == NULL || name_len == 0) { XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, @@ -922,6 +1120,23 @@ xlog_recover_attri_commit_pass2( return -EFSCORRUPTED; } break; + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: + /* + * Name-value replace operations require the caller to + * specify the old and new names and values explicitly. + * Values are optional. + */ + if (attr_name == NULL || name_len == 0) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + if (attr_new_name == NULL || new_name_len == 0) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + break; } /* @@ -930,7 +1145,9 @@ xlog_recover_attri_commit_pass2( * reference. */ nv = xfs_attri_log_nameval_alloc(attr_name, name_len, - attr_value, value_len); + attr_new_name, new_name_len, + attr_value, value_len, + attr_new_value, new_value_len); attrip = xfs_attri_init(mp, nv); memcpy(&attrip->attri_format, attri_formatp, len); diff --git a/fs/xfs/xfs_attr_item.h b/fs/xfs/xfs_attr_item.h index c32b669b0e16a..e74128cbb7225 100644 --- a/fs/xfs/xfs_attr_item.h +++ b/fs/xfs/xfs_attr_item.h @@ -13,7 +13,9 @@ struct kmem_zone; struct xfs_attri_log_nameval { struct xfs_log_iovec name; + struct xfs_log_iovec new_name; /* PPTR_REPLACE only */ struct xfs_log_iovec value; + struct xfs_log_iovec new_value; /* PPTR_REPLACE only */ refcount_t refcount; /* name and value follow the end of this struct */ From ae673f534a30976ce5e709c4535a59c12b786ef3 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:43 -0700 Subject: [PATCH 0433/1702] xfs: record inode generation in xattr update log intent items For parent pointer updates, record the i_generation of the file that is being updated so that we don't accidentally jump generations. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_log_format.h | 2 +- fs/xfs/xfs_attr_item.c | 33 +++++++++++++++++++++++++++------ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h index 632dd97324557..3e6682ed656b3 100644 --- a/fs/xfs/libxfs/xfs_log_format.h +++ b/fs/xfs/libxfs/xfs_log_format.h @@ -1049,7 +1049,7 @@ struct xfs_icreate_log { struct xfs_attri_log_format { uint16_t alfi_type; /* attri log item type */ uint16_t alfi_size; /* size of this item */ - uint32_t __pad; /* pad to 64 bit aligned */ + uint32_t alfi_igen; /* generation of alfi_ino for pptr ops */ uint64_t alfi_id; /* attri identifier */ uint64_t alfi_ino; /* the inode for this attr operation */ uint32_t alfi_op_flags; /* marks the op as a set or remove */ diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index be5064f5a5315..2898eeb16366e 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -388,9 +388,14 @@ xfs_attr_log_item( case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: ASSERT(nv->value.i_len == nv->new_value.i_len); + attrp->alfi_igen = VFS_I(args->dp)->i_generation; attrp->alfi_old_name_len = nv->name.i_len; attrp->alfi_new_name_len = nv->new_name.i_len; break; + case XFS_ATTRI_OP_FLAGS_PPTR_REMOVE: + case XFS_ATTRI_OP_FLAGS_PPTR_SET: + attrp->alfi_igen = VFS_I(args->dp)->i_generation; + fallthrough; default: attrp->alfi_name_len = nv->name.i_len; break; @@ -545,9 +550,6 @@ xfs_attri_validate( { unsigned int op = xfs_attr_log_item_op(attrp); - if (attrp->__pad != 0) - return false; - if (attrp->alfi_op_flags & ~XFS_ATTRI_OP_FLAGS_TYPE_MASK) return false; @@ -639,9 +641,27 @@ xfs_attri_recover_work( int local; int error; - error = xlog_recover_iget(mp, attrp->alfi_ino, &ip); - if (error) - return ERR_PTR(error); + /* + * Parent pointer attr items record the generation but regular logged + * xattrs do not; select the right iget function. + */ + switch (xfs_attr_log_item_op(attrp)) { + case XFS_ATTRI_OP_FLAGS_PPTR_SET: + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: + case XFS_ATTRI_OP_FLAGS_PPTR_REMOVE: + error = xlog_recover_iget_handle(mp, attrp->alfi_ino, + attrp->alfi_igen, &ip); + break; + default: + error = xlog_recover_iget(mp, attrp->alfi_ino, &ip); + break; + } + if (error) { + xfs_irele(ip); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, attrp, + sizeof(*attrp)); + return ERR_PTR(-EFSCORRUPTED); + } if (xfs_inode_has_attr_fork(ip)) { error = xfs_attri_iread_extents(ip); @@ -793,6 +813,7 @@ xfs_attr_relog_intent( new_attrp = &new_attrip->attri_format; new_attrp->alfi_ino = old_attrp->alfi_ino; + new_attrp->alfi_igen = old_attrp->alfi_igen; new_attrp->alfi_op_flags = old_attrp->alfi_op_flags; new_attrp->alfi_value_len = old_attrp->alfi_value_len; From 297da63379c6cba504a33aa7c526f36b148d4610 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 22 Apr 2024 09:47:43 -0700 Subject: [PATCH 0434/1702] xfs: Expose init_xattrs in xfs_create_tmpfile Tmp files are used as part of rename operations and will need attr forks initialized for parent pointers. Expose the init_xattrs parameter to the calling function to initialize the fork. Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_inode.c | 5 +++-- fs/xfs/xfs_inode.h | 2 +- fs/xfs/xfs_iops.c | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 3c843223b4edd..060e4e767b51b 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -1185,6 +1185,7 @@ xfs_create_tmpfile( struct mnt_idmap *idmap, struct xfs_inode *dp, umode_t mode, + bool init_xattrs, struct xfs_inode **ipp) { struct xfs_mount *mp = dp->i_mount; @@ -1225,7 +1226,7 @@ xfs_create_tmpfile( error = xfs_dialloc(&tp, dp->i_ino, mode, &ino); if (!error) error = xfs_init_new_inode(idmap, tp, dp, ino, mode, - 0, 0, prid, false, &ip); + 0, 0, prid, init_xattrs, &ip); if (error) goto out_trans_cancel; @@ -3037,7 +3038,7 @@ xfs_rename_alloc_whiteout( int error; error = xfs_create_tmpfile(idmap, dp, S_IFCHR | WHITEOUT_MODE, - &tmpfile); + false, &tmpfile); if (error) return error; diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index a6da1ab8ab136..04a91e312993b 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -522,7 +522,7 @@ int xfs_create(struct mnt_idmap *idmap, umode_t mode, dev_t rdev, bool need_xattr, struct xfs_inode **ipp); int xfs_create_tmpfile(struct mnt_idmap *idmap, - struct xfs_inode *dp, umode_t mode, + struct xfs_inode *dp, umode_t mode, bool init_xattrs, struct xfs_inode **ipp); int xfs_remove(struct xfs_inode *dp, struct xfs_name *name, struct xfs_inode *ip); diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 659fd10c0cda6..d32322f9ecde3 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -201,7 +201,7 @@ xfs_generic_create( xfs_create_need_xattr(dir, default_acl, acl), &ip); } else { - error = xfs_create_tmpfile(idmap, XFS_I(dir), mode, &ip); + error = xfs_create_tmpfile(idmap, XFS_I(dir), mode, false, &ip); } if (unlikely(error)) goto out_free_acl; From a08d6729637428b6ef8c6a5a94d8c6db7b805a44 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 22 Apr 2024 09:47:44 -0700 Subject: [PATCH 0435/1702] xfs: add parent pointer validator functions The attr name of a parent pointer is a string, and the attr value of a parent pointer is (more or less) a file handle. So we need to modify attr_namecheck to verify the parent pointer name, and add a xfs_parent_valuecheck function to sanitize the handle. At the same time, we need to validate attr values during log recovery if the xattr is really a parent pointer. Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong [djwong: move functions to xfs_parent.c, adjust for new disk format] Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/libxfs/xfs_attr.c | 5 +++ fs/xfs/libxfs/xfs_parent.c | 92 ++++++++++++++++++++++++++++++++++++++ fs/xfs/libxfs/xfs_parent.h | 15 +++++++ fs/xfs/xfs_attr_item.c | 10 +++++ 5 files changed, 123 insertions(+) create mode 100644 fs/xfs/libxfs/xfs_parent.c create mode 100644 fs/xfs/libxfs/xfs_parent.h diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index 4e1eb3b6dbc45..4956ea9a307b8 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -42,6 +42,7 @@ xfs-y += $(addprefix libxfs/, \ xfs_inode_buf.o \ xfs_log_rlimit.o \ xfs_ag_resv.o \ + xfs_parent.o \ xfs_rmap.o \ xfs_rmap_btree.o \ xfs_refcount.o \ diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 78c87c405e33c..93524efa6e56c 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -26,6 +26,7 @@ #include "xfs_trace.h" #include "xfs_attr_item.h" #include "xfs_xattr.h" +#include "xfs_parent.h" struct kmem_cache *xfs_attr_intent_cache; @@ -1568,6 +1569,10 @@ xfs_attr_namecheck( if (length >= MAXNAMELEN) return false; + /* Parent pointers have their own validation. */ + if (attr_flags & XFS_ATTR_PARENT) + return xfs_parent_namecheck(attr_flags, name, length); + /* There shouldn't be any nulls here */ return !memchr(name, 0, length); } diff --git a/fs/xfs/libxfs/xfs_parent.c b/fs/xfs/libxfs/xfs_parent.c new file mode 100644 index 0000000000000..5961fa8c85615 --- /dev/null +++ b/fs/xfs/libxfs/xfs_parent.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2022-2024 Oracle. + * All rights reserved. + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_da_format.h" +#include "xfs_log_format.h" +#include "xfs_shared.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_bmap_btree.h" +#include "xfs_inode.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_trans.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_attr_sf.h" +#include "xfs_bmap.h" +#include "xfs_defer.h" +#include "xfs_log.h" +#include "xfs_xattr.h" +#include "xfs_parent.h" +#include "xfs_trans_space.h" + +/* + * Parent pointer attribute handling. + * + * Because the attribute name is a filename component, it will never be longer + * than 255 bytes and must not contain nulls or slashes. These are roughly the + * same constraints that apply to attribute names. + * + * The attribute value must always be a struct xfs_parent_rec. This means the + * attribute will never be in remote format because 12 bytes is nowhere near + * xfs_attr_leaf_entsize_local_max() (~75% of block size). + * + * Creating a new parent attribute will always create a new attribute - there + * should never, ever be an existing attribute in the tree for a new inode. + * ENOSPC behavior is problematic - creating the inode without the parent + * pointer is effectively a corruption, so we allow parent attribute creation + * to dip into the reserve block pool to avoid unexpected ENOSPC errors from + * occurring. + */ + +/* Return true if parent pointer attr name is valid. */ +bool +xfs_parent_namecheck( + unsigned int attr_flags, + const void *name, + size_t length) +{ + /* + * Parent pointers always use logged operations, so there should never + * be incomplete xattrs. + */ + if (attr_flags & XFS_ATTR_INCOMPLETE) + return false; + + return xfs_dir2_namecheck(name, length); +} + +/* Return true if parent pointer attr value is valid. */ +bool +xfs_parent_valuecheck( + struct xfs_mount *mp, + const void *value, + size_t valuelen) +{ + const struct xfs_parent_rec *rec = value; + + if (!xfs_has_parent(mp)) + return false; + + /* The xattr value must be a parent record. */ + if (valuelen != sizeof(struct xfs_parent_rec)) + return false; + + /* The parent record must be local. */ + if (value == NULL) + return false; + + /* The parent inumber must be valid. */ + if (!xfs_verify_dir_ino(mp, be64_to_cpu(rec->p_ino))) + return false; + + return true; +} diff --git a/fs/xfs/libxfs/xfs_parent.h b/fs/xfs/libxfs/xfs_parent.h new file mode 100644 index 0000000000000..ef8aff8607801 --- /dev/null +++ b/fs/xfs/libxfs/xfs_parent.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2022-2024 Oracle. + * All Rights Reserved. + */ +#ifndef __XFS_PARENT_H__ +#define __XFS_PARENT_H__ + +/* Metadata validators */ +bool xfs_parent_namecheck(unsigned int attr_flags, const void *name, + size_t length); +bool xfs_parent_valuecheck(struct xfs_mount *mp, const void *value, + size_t valuelen); + +#endif /* __XFS_PARENT_H__ */ diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index 2898eeb16366e..2b10ac4c5fce2 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -27,6 +27,7 @@ #include "xfs_error.h" #include "xfs_log_priv.h" #include "xfs_log_recover.h" +#include "xfs_parent.h" struct kmem_cache *xfs_attri_cache; struct kmem_cache *xfs_attrd_cache; @@ -973,6 +974,15 @@ xfs_attri_validate_value_iovec( return NULL; } + if ((attri_formatp->alfi_attr_filter & XFS_ATTR_PARENT) && + !xfs_parent_valuecheck(mp, iovec->i_addr, value_len)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, sizeof(*attri_formatp)); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + iovec->i_addr, iovec->i_len); + return NULL; + } + return iovec->i_addr; } From 7dba4a5fe1c5cdf0859830380c52f29295cbf345 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 22 Apr 2024 09:47:45 -0700 Subject: [PATCH 0436/1702] xfs: extend transaction reservations for parent attributes We need to add, remove or modify parent pointer attributes during create/link/unlink/rename operations atomically with the dirents in the parent directories being modified. This means they need to be modified in the same transaction as the parent directories, and so we need to add the required space for the attribute modifications to the transaction reservations. Signed-off-by: Dave Chinner Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong [djwong: fix indenting errors, adjust for new log format] Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_trans_resv.c | 326 +++++++++++++++++++++++++++------ 1 file changed, 274 insertions(+), 52 deletions(-) diff --git a/fs/xfs/libxfs/xfs_trans_resv.c b/fs/xfs/libxfs/xfs_trans_resv.c index 6cd45e8c118da..6dbe6e7251e7c 100644 --- a/fs/xfs/libxfs/xfs_trans_resv.c +++ b/fs/xfs/libxfs/xfs_trans_resv.c @@ -20,6 +20,9 @@ #include "xfs_qm.h" #include "xfs_trans_space.h" #include "xfs_rtbitmap.h" +#include "xfs_attr_item.h" +#include "xfs_log.h" +#include "xfs_da_format.h" #define _ALLOC true #define _FREE false @@ -422,29 +425,110 @@ xfs_calc_itruncate_reservation_minlogsize( return xfs_calc_itruncate_reservation(mp, true); } +static inline unsigned int xfs_calc_pptr_link_overhead(void) +{ + return sizeof(struct xfs_attri_log_format) + + xlog_calc_iovec_len(sizeof(struct xfs_parent_rec)) + + xlog_calc_iovec_len(MAXNAMELEN - 1); +} +static inline unsigned int xfs_calc_pptr_unlink_overhead(void) +{ + return sizeof(struct xfs_attri_log_format) + + xlog_calc_iovec_len(sizeof(struct xfs_parent_rec)) + + xlog_calc_iovec_len(MAXNAMELEN - 1); +} +static inline unsigned int xfs_calc_pptr_replace_overhead(void) +{ + return sizeof(struct xfs_attri_log_format) + + xlog_calc_iovec_len(sizeof(struct xfs_parent_rec)) + + xlog_calc_iovec_len(MAXNAMELEN - 1) + + xlog_calc_iovec_len(sizeof(struct xfs_parent_rec)) + + xlog_calc_iovec_len(MAXNAMELEN - 1); +} + /* * In renaming a files we can modify: * the five inodes involved: 5 * inode size * the two directory btrees: 2 * (max depth + v2) * dir block size * the two directory bmap btrees: 2 * max depth * block size * And the bmap_finish transaction can free dir and bmap blocks (two sets - * of bmap blocks) giving: + * of bmap blocks) giving (t2): * the agf for the ags in which the blocks live: 3 * sector size * the agfl for the ags in which the blocks live: 3 * sector size * the superblock for the free block count: sector size * the allocation btrees: 3 exts * 2 trees * (2 * max depth - 1) * block size + * If parent pointers are enabled (t3), then each transaction in the chain + * must be capable of setting or removing the extended attribute + * containing the parent information. It must also be able to handle + * the three xattr intent items that track the progress of the parent + * pointer update. */ STATIC uint xfs_calc_rename_reservation( struct xfs_mount *mp) { - return XFS_DQUOT_LOGRES(mp) + - max((xfs_calc_inode_res(mp, 5) + - xfs_calc_buf_res(2 * XFS_DIROP_LOG_COUNT(mp), - XFS_FSB_TO_B(mp, 1))), - (xfs_calc_buf_res(7, mp->m_sb.sb_sectsize) + - xfs_calc_buf_res(xfs_allocfree_block_count(mp, 3), - XFS_FSB_TO_B(mp, 1)))); + unsigned int overhead = XFS_DQUOT_LOGRES(mp); + struct xfs_trans_resv *resp = M_RES(mp); + unsigned int t1, t2, t3 = 0; + + t1 = xfs_calc_inode_res(mp, 5) + + xfs_calc_buf_res(2 * XFS_DIROP_LOG_COUNT(mp), + XFS_FSB_TO_B(mp, 1)); + + t2 = xfs_calc_buf_res(7, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(xfs_allocfree_block_count(mp, 3), + XFS_FSB_TO_B(mp, 1)); + + if (xfs_has_parent(mp)) { + unsigned int rename_overhead, exchange_overhead; + + t3 = max(resp->tr_attrsetm.tr_logres, + resp->tr_attrrm.tr_logres); + + /* + * For a standard rename, the three xattr intent log items + * are (1) replacing the pptr for the source file; (2) + * removing the pptr on the dest file; and (3) adding a + * pptr for the whiteout file in the src dir. + * + * For an RENAME_EXCHANGE, there are two xattr intent + * items to replace the pptr for both src and dest + * files. Link counts don't change and there is no + * whiteout. + * + * In the worst case we can end up relogging all log + * intent items to allow the log tail to move ahead, so + * they become overhead added to each transaction in a + * processing chain. + */ + rename_overhead = xfs_calc_pptr_replace_overhead() + + xfs_calc_pptr_unlink_overhead() + + xfs_calc_pptr_link_overhead(); + exchange_overhead = 2 * xfs_calc_pptr_replace_overhead(); + + overhead += max(rename_overhead, exchange_overhead); + } + + return overhead + max3(t1, t2, t3); +} + +static inline unsigned int +xfs_rename_log_count( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + /* One for the rename, one more for freeing blocks */ + unsigned int ret = XFS_RENAME_LOG_COUNT; + + /* + * Pre-reserve enough log reservation to handle the transaction + * rolling needed to remove or add one parent pointer. + */ + if (xfs_has_parent(mp)) + ret += max(resp->tr_attrsetm.tr_logcount, + resp->tr_attrrm.tr_logcount); + + return ret; } /* @@ -461,6 +545,23 @@ xfs_calc_iunlink_remove_reservation( 2 * M_IGEO(mp)->inode_cluster_size; } +static inline unsigned int +xfs_link_log_count( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + unsigned int ret = XFS_LINK_LOG_COUNT; + + /* + * Pre-reserve enough log reservation to handle the transaction + * rolling needed to add one parent pointer. + */ + if (xfs_has_parent(mp)) + ret += resp->tr_attrsetm.tr_logcount; + + return ret; +} + /* * For creating a link to an inode: * the parent directory inode: inode size @@ -477,14 +578,23 @@ STATIC uint xfs_calc_link_reservation( struct xfs_mount *mp) { - return XFS_DQUOT_LOGRES(mp) + - xfs_calc_iunlink_remove_reservation(mp) + - max((xfs_calc_inode_res(mp, 2) + - xfs_calc_buf_res(XFS_DIROP_LOG_COUNT(mp), - XFS_FSB_TO_B(mp, 1))), - (xfs_calc_buf_res(3, mp->m_sb.sb_sectsize) + - xfs_calc_buf_res(xfs_allocfree_block_count(mp, 1), - XFS_FSB_TO_B(mp, 1)))); + unsigned int overhead = XFS_DQUOT_LOGRES(mp); + struct xfs_trans_resv *resp = M_RES(mp); + unsigned int t1, t2, t3 = 0; + + overhead += xfs_calc_iunlink_remove_reservation(mp); + t1 = xfs_calc_inode_res(mp, 2) + + xfs_calc_buf_res(XFS_DIROP_LOG_COUNT(mp), XFS_FSB_TO_B(mp, 1)); + t2 = xfs_calc_buf_res(3, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(xfs_allocfree_block_count(mp, 1), + XFS_FSB_TO_B(mp, 1)); + + if (xfs_has_parent(mp)) { + t3 = resp->tr_attrsetm.tr_logres; + overhead += xfs_calc_pptr_link_overhead(); + } + + return overhead + max3(t1, t2, t3); } /* @@ -499,6 +609,23 @@ xfs_calc_iunlink_add_reservation(xfs_mount_t *mp) M_IGEO(mp)->inode_cluster_size; } +static inline unsigned int +xfs_remove_log_count( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + unsigned int ret = XFS_REMOVE_LOG_COUNT; + + /* + * Pre-reserve enough log reservation to handle the transaction + * rolling needed to add one parent pointer. + */ + if (xfs_has_parent(mp)) + ret += resp->tr_attrrm.tr_logcount; + + return ret; +} + /* * For removing a directory entry we can modify: * the parent directory inode: inode size @@ -515,14 +642,24 @@ STATIC uint xfs_calc_remove_reservation( struct xfs_mount *mp) { - return XFS_DQUOT_LOGRES(mp) + - xfs_calc_iunlink_add_reservation(mp) + - max((xfs_calc_inode_res(mp, 2) + - xfs_calc_buf_res(XFS_DIROP_LOG_COUNT(mp), - XFS_FSB_TO_B(mp, 1))), - (xfs_calc_buf_res(4, mp->m_sb.sb_sectsize) + - xfs_calc_buf_res(xfs_allocfree_block_count(mp, 2), - XFS_FSB_TO_B(mp, 1)))); + unsigned int overhead = XFS_DQUOT_LOGRES(mp); + struct xfs_trans_resv *resp = M_RES(mp); + unsigned int t1, t2, t3 = 0; + + overhead += xfs_calc_iunlink_add_reservation(mp); + + t1 = xfs_calc_inode_res(mp, 2) + + xfs_calc_buf_res(XFS_DIROP_LOG_COUNT(mp), XFS_FSB_TO_B(mp, 1)); + t2 = xfs_calc_buf_res(4, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(xfs_allocfree_block_count(mp, 2), + XFS_FSB_TO_B(mp, 1)); + + if (xfs_has_parent(mp)) { + t3 = resp->tr_attrrm.tr_logres; + overhead += xfs_calc_pptr_unlink_overhead(); + } + + return overhead + max3(t1, t2, t3); } /* @@ -571,12 +708,40 @@ xfs_calc_icreate_resv_alloc( xfs_calc_finobt_res(mp); } +static inline unsigned int +xfs_icreate_log_count( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + unsigned int ret = XFS_CREATE_LOG_COUNT; + + /* + * Pre-reserve enough log reservation to handle the transaction + * rolling needed to add one parent pointer. + */ + if (xfs_has_parent(mp)) + ret += resp->tr_attrsetm.tr_logcount; + + return ret; +} + STATIC uint -xfs_calc_icreate_reservation(xfs_mount_t *mp) +xfs_calc_icreate_reservation( + struct xfs_mount *mp) { - return XFS_DQUOT_LOGRES(mp) + - max(xfs_calc_icreate_resv_alloc(mp), - xfs_calc_create_resv_modify(mp)); + struct xfs_trans_resv *resp = M_RES(mp); + unsigned int overhead = XFS_DQUOT_LOGRES(mp); + unsigned int t1, t2, t3 = 0; + + t1 = xfs_calc_icreate_resv_alloc(mp); + t2 = xfs_calc_create_resv_modify(mp); + + if (xfs_has_parent(mp)) { + t3 = resp->tr_attrsetm.tr_logres; + overhead += xfs_calc_pptr_link_overhead(); + } + + return overhead + max3(t1, t2, t3); } STATIC uint @@ -589,6 +754,23 @@ xfs_calc_create_tmpfile_reservation( return res + xfs_calc_iunlink_add_reservation(mp); } +static inline unsigned int +xfs_mkdir_log_count( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + unsigned int ret = XFS_MKDIR_LOG_COUNT; + + /* + * Pre-reserve enough log reservation to handle the transaction + * rolling needed to add one parent pointer. + */ + if (xfs_has_parent(mp)) + ret += resp->tr_attrsetm.tr_logcount; + + return ret; +} + /* * Making a new directory is the same as creating a new file. */ @@ -599,6 +781,22 @@ xfs_calc_mkdir_reservation( return xfs_calc_icreate_reservation(mp); } +static inline unsigned int +xfs_symlink_log_count( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + unsigned int ret = XFS_SYMLINK_LOG_COUNT; + + /* + * Pre-reserve enough log reservation to handle the transaction + * rolling needed to add one parent pointer. + */ + if (xfs_has_parent(mp)) + ret += resp->tr_attrsetm.tr_logcount; + + return ret; +} /* * Making a new symplink is the same as creating a new file, but @@ -911,54 +1109,76 @@ xfs_calc_sb_reservation( return xfs_calc_buf_res(1, mp->m_sb.sb_sectsize); } -void -xfs_trans_resv_calc( +/* + * Namespace reservations. + * + * These get tricky when parent pointers are enabled as we have attribute + * modifications occurring from within these transactions. Rather than confuse + * each of these reservation calculations with the conditional attribute + * reservations, add them here in a clear and concise manner. This requires that + * the attribute reservations have already been calculated. + * + * Note that we only include the static attribute reservation here; the runtime + * reservation will have to be modified by the size of the attributes being + * added/removed/modified. See the comments on the attribute reservation + * calculations for more details. + */ +STATIC void +xfs_calc_namespace_reservations( struct xfs_mount *mp, struct xfs_trans_resv *resp) { - int logcount_adj = 0; - - /* - * The following transactions are logged in physical format and - * require a permanent reservation on space. - */ - resp->tr_write.tr_logres = xfs_calc_write_reservation(mp, false); - resp->tr_write.tr_logcount = XFS_WRITE_LOG_COUNT; - resp->tr_write.tr_logflags |= XFS_TRANS_PERM_LOG_RES; - - resp->tr_itruncate.tr_logres = xfs_calc_itruncate_reservation(mp, false); - resp->tr_itruncate.tr_logcount = XFS_ITRUNCATE_LOG_COUNT; - resp->tr_itruncate.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + ASSERT(resp->tr_attrsetm.tr_logres > 0); resp->tr_rename.tr_logres = xfs_calc_rename_reservation(mp); - resp->tr_rename.tr_logcount = XFS_RENAME_LOG_COUNT; + resp->tr_rename.tr_logcount = xfs_rename_log_count(mp, resp); resp->tr_rename.tr_logflags |= XFS_TRANS_PERM_LOG_RES; resp->tr_link.tr_logres = xfs_calc_link_reservation(mp); - resp->tr_link.tr_logcount = XFS_LINK_LOG_COUNT; + resp->tr_link.tr_logcount = xfs_link_log_count(mp, resp); resp->tr_link.tr_logflags |= XFS_TRANS_PERM_LOG_RES; resp->tr_remove.tr_logres = xfs_calc_remove_reservation(mp); - resp->tr_remove.tr_logcount = XFS_REMOVE_LOG_COUNT; + resp->tr_remove.tr_logcount = xfs_remove_log_count(mp, resp); resp->tr_remove.tr_logflags |= XFS_TRANS_PERM_LOG_RES; resp->tr_symlink.tr_logres = xfs_calc_symlink_reservation(mp); - resp->tr_symlink.tr_logcount = XFS_SYMLINK_LOG_COUNT; + resp->tr_symlink.tr_logcount = xfs_symlink_log_count(mp, resp); resp->tr_symlink.tr_logflags |= XFS_TRANS_PERM_LOG_RES; resp->tr_create.tr_logres = xfs_calc_icreate_reservation(mp); - resp->tr_create.tr_logcount = XFS_CREATE_LOG_COUNT; + resp->tr_create.tr_logcount = xfs_icreate_log_count(mp, resp); resp->tr_create.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + resp->tr_mkdir.tr_logres = xfs_calc_mkdir_reservation(mp); + resp->tr_mkdir.tr_logcount = xfs_mkdir_log_count(mp, resp); + resp->tr_mkdir.tr_logflags |= XFS_TRANS_PERM_LOG_RES; +} + +void +xfs_trans_resv_calc( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + int logcount_adj = 0; + + /* + * The following transactions are logged in physical format and + * require a permanent reservation on space. + */ + resp->tr_write.tr_logres = xfs_calc_write_reservation(mp, false); + resp->tr_write.tr_logcount = XFS_WRITE_LOG_COUNT; + resp->tr_write.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + resp->tr_itruncate.tr_logres = xfs_calc_itruncate_reservation(mp, false); + resp->tr_itruncate.tr_logcount = XFS_ITRUNCATE_LOG_COUNT; + resp->tr_itruncate.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + resp->tr_create_tmpfile.tr_logres = xfs_calc_create_tmpfile_reservation(mp); resp->tr_create_tmpfile.tr_logcount = XFS_CREATE_TMPFILE_LOG_COUNT; resp->tr_create_tmpfile.tr_logflags |= XFS_TRANS_PERM_LOG_RES; - resp->tr_mkdir.tr_logres = xfs_calc_mkdir_reservation(mp); - resp->tr_mkdir.tr_logcount = XFS_MKDIR_LOG_COUNT; - resp->tr_mkdir.tr_logflags |= XFS_TRANS_PERM_LOG_RES; - resp->tr_ifree.tr_logres = xfs_calc_ifree_reservation(mp); resp->tr_ifree.tr_logcount = XFS_INACTIVE_LOG_COUNT; resp->tr_ifree.tr_logflags |= XFS_TRANS_PERM_LOG_RES; @@ -988,6 +1208,8 @@ xfs_trans_resv_calc( resp->tr_qm_dqalloc.tr_logcount = XFS_WRITE_LOG_COUNT; resp->tr_qm_dqalloc.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + xfs_calc_namespace_reservations(mp, resp); + /* * The following transactions are logged in logical format with * a default log count. From fb102fe7fe02e70f8a49cc7f74bc0769cdab2912 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:46 -0700 Subject: [PATCH 0437/1702] xfs: create a hashname function for parent pointers Although directory entry and parent pointer recordsets look very similar (name -> ino), there's one major difference between them: a file can be hardlinked from multiple parent directories with the same filename. This is common in shared container environments where a base directory tree might be hardlink-copied multiple times. IOWs the same 'ls' program might be hardlinked to multiple /srv/*/bin/ls paths. We don't want parent pointer operations to bog down on hash collisions between the same dirent name, so create a special hash function that mixes in the parent directory inode number. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 3 +++ fs/xfs/libxfs/xfs_parent.c | 47 ++++++++++++++++++++++++++++++++++++++ fs/xfs/libxfs/xfs_parent.h | 5 ++++ fs/xfs/scrub/attr.c | 4 ++++ 4 files changed, 59 insertions(+) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 93524efa6e56c..8c283e5c24702 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -437,6 +437,9 @@ xfs_attr_hashval( { ASSERT(xfs_attr_check_namespace(attr_flags)); + if (attr_flags & XFS_ATTR_PARENT) + return xfs_parent_hashattr(mp, name, namelen, value, valuelen); + return xfs_attr_hashname(name, namelen); } diff --git a/fs/xfs/libxfs/xfs_parent.c b/fs/xfs/libxfs/xfs_parent.c index 5961fa8c85615..d564baf2549c5 100644 --- a/fs/xfs/libxfs/xfs_parent.c +++ b/fs/xfs/libxfs/xfs_parent.c @@ -90,3 +90,50 @@ xfs_parent_valuecheck( return true; } + +/* Compute the attribute name hash for a parent pointer. */ +xfs_dahash_t +xfs_parent_hashval( + struct xfs_mount *mp, + const uint8_t *name, + int namelen, + xfs_ino_t parent_ino) +{ + struct xfs_name xname = { + .name = name, + .len = namelen, + }; + + /* + * Use the same dirent name hash as would be used on the directory, but + * mix in the parent inode number to avoid collisions on hardlinked + * files with identical names but different parents. + */ + return xfs_dir2_hashname(mp, &xname) ^ + upper_32_bits(parent_ino) ^ lower_32_bits(parent_ino); +} + +/* Compute the attribute name hash from the xattr components. */ +xfs_dahash_t +xfs_parent_hashattr( + struct xfs_mount *mp, + const uint8_t *name, + int namelen, + const void *value, + int valuelen) +{ + const struct xfs_parent_rec *rec = value; + + /* Requires a local attr value in xfs_parent_rec format */ + if (valuelen != sizeof(struct xfs_parent_rec)) { + ASSERT(valuelen == sizeof(struct xfs_parent_rec)); + return 0; + } + + if (!value) { + ASSERT(value != NULL); + return 0; + } + + return xfs_parent_hashval(mp, name, namelen, be64_to_cpu(rec->p_ino)); +} diff --git a/fs/xfs/libxfs/xfs_parent.h b/fs/xfs/libxfs/xfs_parent.h index ef8aff8607801..6a4028871b72a 100644 --- a/fs/xfs/libxfs/xfs_parent.h +++ b/fs/xfs/libxfs/xfs_parent.h @@ -12,4 +12,9 @@ bool xfs_parent_namecheck(unsigned int attr_flags, const void *name, bool xfs_parent_valuecheck(struct xfs_mount *mp, const void *value, size_t valuelen); +xfs_dahash_t xfs_parent_hashval(struct xfs_mount *mp, const uint8_t *name, + int namelen, xfs_ino_t parent_ino); +xfs_dahash_t xfs_parent_hashattr(struct xfs_mount *mp, const uint8_t *name, + int namelen, const void *value, int valuelen); + #endif /* __XFS_PARENT_H__ */ diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index 22d7ef4df1699..c07d050b39b2e 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -536,6 +536,10 @@ xchk_xattr_rec( xchk_da_set_corrupt(ds, level); goto out; } + if (ent->flags & XFS_ATTR_PARENT) { + xchk_da_set_corrupt(ds, level); + goto out; + } calc_hash = xfs_attr_hashval(mp, ent->flags, rentry->name, rentry->namelen, NULL, be32_to_cpu(rentry->valuelen)); From b7c62d90c12c6cc86f10b8a62cefe0029374b6ff Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 22 Apr 2024 09:47:47 -0700 Subject: [PATCH 0438/1702] xfs: parent pointer attribute creation Add parent pointer attribute during xfs_create, and subroutines to initialize attributes. Note that the xfs_attr_intent object contains a pointer to the caller's xfs_da_args object, so the latter must persist until transaction commit. Signed-off-by: Dave Chinner Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong [djwong: shorten names, adjust to new format, set init_xattrs for parent pointers] Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/libxfs/xfs_parent.c | 68 +++++++++++++++++++++++++++++++++ fs/xfs/libxfs/xfs_parent.h | 65 +++++++++++++++++++++++++++++++ fs/xfs/libxfs/xfs_trans_space.c | 52 +++++++++++++++++++++++++ fs/xfs/libxfs/xfs_trans_space.h | 9 +++-- fs/xfs/scrub/tempfile.c | 2 +- fs/xfs/xfs_inode.c | 32 +++++++++++++--- fs/xfs/xfs_iops.c | 15 +++++++- fs/xfs/xfs_super.c | 10 +++++ 9 files changed, 242 insertions(+), 12 deletions(-) create mode 100644 fs/xfs/libxfs/xfs_trans_space.c diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index 4956ea9a307b8..0c1a0b67af93c 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -51,6 +51,7 @@ xfs-y += $(addprefix libxfs/, \ xfs_symlink_remote.o \ xfs_trans_inode.o \ xfs_trans_resv.o \ + xfs_trans_space.o \ xfs_types.o \ ) # xfs_rtbitmap is shared with libxfs diff --git a/fs/xfs/libxfs/xfs_parent.c b/fs/xfs/libxfs/xfs_parent.c index d564baf2549c5..65616cfc1a2bf 100644 --- a/fs/xfs/libxfs/xfs_parent.c +++ b/fs/xfs/libxfs/xfs_parent.c @@ -27,6 +27,10 @@ #include "xfs_xattr.h" #include "xfs_parent.h" #include "xfs_trans_space.h" +#include "xfs_attr_item.h" +#include "xfs_health.h" + +struct kmem_cache *xfs_parent_args_cache; /* * Parent pointer attribute handling. @@ -137,3 +141,67 @@ xfs_parent_hashattr( return xfs_parent_hashval(mp, name, namelen, be64_to_cpu(rec->p_ino)); } + +/* + * Initialize the parent pointer arguments structure. Caller must have zeroed + * the contents of @args. @tp is only required for updates. + */ +static void +xfs_parent_da_args_init( + struct xfs_da_args *args, + struct xfs_trans *tp, + struct xfs_parent_rec *rec, + struct xfs_inode *child, + xfs_ino_t owner, + const struct xfs_name *parent_name) +{ + args->geo = child->i_mount->m_attr_geo; + args->whichfork = XFS_ATTR_FORK; + args->attr_filter = XFS_ATTR_PARENT; + args->op_flags = XFS_DA_OP_LOGGED | XFS_DA_OP_OKNOENT; + args->trans = tp; + args->dp = child; + args->owner = owner; + args->name = parent_name->name; + args->namelen = parent_name->len; + args->value = rec; + args->valuelen = sizeof(struct xfs_parent_rec); + xfs_attr_sethash(args); +} + +/* Make sure the incore state is ready for a parent pointer query/update. */ +static inline int +xfs_parent_iread_extents( + struct xfs_trans *tp, + struct xfs_inode *child) +{ + /* Parent pointers require that the attr fork must exist. */ + if (XFS_IS_CORRUPT(child->i_mount, !xfs_inode_has_attr_fork(child))) { + xfs_inode_mark_sick(child, XFS_SICK_INO_PARENT); + return -EFSCORRUPTED; + } + + return xfs_iread_extents(tp, child, XFS_ATTR_FORK); +} + +/* Add a parent pointer to reflect a dirent addition. */ +int +xfs_parent_addname( + struct xfs_trans *tp, + struct xfs_parent_args *ppargs, + struct xfs_inode *dp, + const struct xfs_name *parent_name, + struct xfs_inode *child) +{ + int error; + + error = xfs_parent_iread_extents(tp, child); + if (error) + return error; + + xfs_inode_to_parent_rec(&ppargs->rec, dp); + xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, + child->i_ino, parent_name); + xfs_attr_defer_add(&ppargs->args, XFS_ATTR_DEFER_SET); + return 0; +} diff --git a/fs/xfs/libxfs/xfs_parent.h b/fs/xfs/libxfs/xfs_parent.h index 6a4028871b72a..6de24e3ef318c 100644 --- a/fs/xfs/libxfs/xfs_parent.h +++ b/fs/xfs/libxfs/xfs_parent.h @@ -17,4 +17,69 @@ xfs_dahash_t xfs_parent_hashval(struct xfs_mount *mp, const uint8_t *name, xfs_dahash_t xfs_parent_hashattr(struct xfs_mount *mp, const uint8_t *name, int namelen, const void *value, int valuelen); +/* Initializes a xfs_parent_rec to be stored as an attribute name. */ +static inline void +xfs_parent_rec_init( + struct xfs_parent_rec *rec, + xfs_ino_t ino, + uint32_t gen) +{ + rec->p_ino = cpu_to_be64(ino); + rec->p_gen = cpu_to_be32(gen); +} + +/* Initializes a xfs_parent_rec to be stored as an attribute name. */ +static inline void +xfs_inode_to_parent_rec( + struct xfs_parent_rec *rec, + const struct xfs_inode *dp) +{ + xfs_parent_rec_init(rec, dp->i_ino, VFS_IC(dp)->i_generation); +} + +extern struct kmem_cache *xfs_parent_args_cache; + +/* + * Parent pointer information needed to pass around the deferred xattr update + * machinery. + */ +struct xfs_parent_args { + struct xfs_parent_rec rec; + struct xfs_da_args args; +}; + +/* + * Start a parent pointer update by allocating the context object we need to + * perform a parent pointer update. + */ +static inline int +xfs_parent_start( + struct xfs_mount *mp, + struct xfs_parent_args **ppargsp) +{ + if (!xfs_has_parent(mp)) { + *ppargsp = NULL; + return 0; + } + + *ppargsp = kmem_cache_zalloc(xfs_parent_args_cache, GFP_KERNEL); + if (!*ppargsp) + return -ENOMEM; + return 0; +} + +/* Finish a parent pointer update by freeing the context object. */ +static inline void +xfs_parent_finish( + struct xfs_mount *mp, + struct xfs_parent_args *ppargs) +{ + if (ppargs) + kmem_cache_free(xfs_parent_args_cache, ppargs); +} + +int xfs_parent_addname(struct xfs_trans *tp, struct xfs_parent_args *ppargs, + struct xfs_inode *dp, const struct xfs_name *parent_name, + struct xfs_inode *child); + #endif /* __XFS_PARENT_H__ */ diff --git a/fs/xfs/libxfs/xfs_trans_space.c b/fs/xfs/libxfs/xfs_trans_space.c new file mode 100644 index 0000000000000..90532c3fa2053 --- /dev/null +++ b/fs/xfs/libxfs/xfs_trans_space.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_da_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_da_btree.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" + +/* Calculate the disk space required to add a parent pointer. */ +unsigned int +xfs_parent_calc_space_res( + struct xfs_mount *mp, + unsigned int namelen) +{ + /* + * Parent pointers are always the first attr in an attr tree, and never + * larger than a block + */ + return XFS_DAENTER_SPACE_RES(mp, XFS_ATTR_FORK) + + XFS_NEXTENTADD_SPACE_RES(mp, namelen, XFS_ATTR_FORK); +} + +unsigned int +xfs_create_space_res( + struct xfs_mount *mp, + unsigned int namelen) +{ + unsigned int ret; + + ret = XFS_IALLOC_SPACE_RES(mp) + XFS_DIRENTER_SPACE_RES(mp, namelen); + if (xfs_has_parent(mp)) + ret += xfs_parent_calc_space_res(mp, namelen); + + return ret; +} + +unsigned int +xfs_mkdir_space_res( + struct xfs_mount *mp, + unsigned int namelen) +{ + return xfs_create_space_res(mp, namelen); +} diff --git a/fs/xfs/libxfs/xfs_trans_space.h b/fs/xfs/libxfs/xfs_trans_space.h index 9640fc232c147..6cda87153b38c 100644 --- a/fs/xfs/libxfs/xfs_trans_space.h +++ b/fs/xfs/libxfs/xfs_trans_space.h @@ -80,8 +80,6 @@ /* This macro is not used - see inline code in xfs_attr_set */ #define XFS_ATTRSET_SPACE_RES(mp, v) \ (XFS_DAENTER_SPACE_RES(mp, XFS_ATTR_FORK) + XFS_B_TO_FSB(mp, v)) -#define XFS_CREATE_SPACE_RES(mp,nl) \ - (XFS_IALLOC_SPACE_RES(mp) + XFS_DIRENTER_SPACE_RES(mp,nl)) #define XFS_DIOSTRAT_SPACE_RES(mp, v) \ (XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK) + (v)) #define XFS_GROWFS_SPACE_RES(mp) \ @@ -90,8 +88,6 @@ ((b) + XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK)) #define XFS_LINK_SPACE_RES(mp,nl) \ XFS_DIRENTER_SPACE_RES(mp,nl) -#define XFS_MKDIR_SPACE_RES(mp,nl) \ - (XFS_IALLOC_SPACE_RES(mp) + XFS_DIRENTER_SPACE_RES(mp,nl)) #define XFS_QM_DQALLOC_SPACE_RES(mp) \ (XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK) + \ XFS_DQUOT_CLUSTER_SIZE_FSB) @@ -106,5 +102,10 @@ #define XFS_IFREE_SPACE_RES(mp) \ (xfs_has_finobt(mp) ? M_IGEO(mp)->inobt_maxlevels : 0) +unsigned int xfs_parent_calc_space_res(struct xfs_mount *mp, + unsigned int namelen); + +unsigned int xfs_create_space_res(struct xfs_mount *mp, unsigned int namelen); +unsigned int xfs_mkdir_space_res(struct xfs_mount *mp, unsigned int namelen); #endif /* __XFS_TRANS_SPACE_H__ */ diff --git a/fs/xfs/scrub/tempfile.c b/fs/xfs/scrub/tempfile.c index 6f39504a216ea..ddbcccb3dba13 100644 --- a/fs/xfs/scrub/tempfile.c +++ b/fs/xfs/scrub/tempfile.c @@ -71,7 +71,7 @@ xrep_tempfile_create( return error; if (is_dir) { - resblks = XFS_MKDIR_SPACE_RES(mp, 0); + resblks = xfs_mkdir_space_res(mp, 0); tres = &M_RES(mp)->tr_mkdir; } else { resblks = XFS_IALLOC_SPACE_RES(mp); diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 060e4e767b51b..0dd4111a6773e 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -40,6 +40,8 @@ #include "xfs_log_priv.h" #include "xfs_health.h" #include "xfs_pnfs.h" +#include "xfs_parent.h" +#include "xfs_xattr.h" struct kmem_cache *xfs_inode_cache; @@ -1017,7 +1019,7 @@ xfs_dir_hook_setup( int xfs_create( struct mnt_idmap *idmap, - xfs_inode_t *dp, + struct xfs_inode *dp, struct xfs_name *name, umode_t mode, dev_t rdev, @@ -1029,7 +1031,7 @@ xfs_create( struct xfs_inode *ip = NULL; struct xfs_trans *tp = NULL; int error; - bool unlock_dp_on_error = false; + bool unlock_dp_on_error = false; prid_t prid; struct xfs_dquot *udqp = NULL; struct xfs_dquot *gdqp = NULL; @@ -1037,6 +1039,7 @@ xfs_create( struct xfs_trans_res *tres; uint resblks; xfs_ino_t ino; + struct xfs_parent_args *ppargs; trace_xfs_create(dp, name); @@ -1058,13 +1061,17 @@ xfs_create( return error; if (is_dir) { - resblks = XFS_MKDIR_SPACE_RES(mp, name->len); + resblks = xfs_mkdir_space_res(mp, name->len); tres = &M_RES(mp)->tr_mkdir; } else { - resblks = XFS_CREATE_SPACE_RES(mp, name->len); + resblks = xfs_create_space_res(mp, name->len); tres = &M_RES(mp)->tr_create; } + error = xfs_parent_start(mp, &ppargs); + if (error) + goto out_release_dquots; + /* * Initially assume that the file does not exist and * reserve the resources for that case. If that is not @@ -1080,7 +1087,7 @@ xfs_create( resblks, &tp); } if (error) - goto out_release_dquots; + goto out_parent; xfs_ilock(dp, XFS_ILOCK_EXCL | XFS_ILOCK_PARENT); unlock_dp_on_error = true; @@ -1123,6 +1130,16 @@ xfs_create( xfs_bumplink(tp, dp); } + /* + * If we have parent pointers, we need to add the attribute containing + * the parent information now. + */ + if (ppargs) { + error = xfs_parent_addname(tp, ppargs, dp, name, ip); + if (error) + goto out_trans_cancel; + } + /* * Create ip with a reference from dp, and add '.' and '..' references * if it's a directory. @@ -1155,6 +1172,7 @@ xfs_create( *ipp = ip; xfs_iunlock(ip, XFS_ILOCK_EXCL); xfs_iunlock(dp, XFS_ILOCK_EXCL); + xfs_parent_finish(mp, ppargs); return 0; out_trans_cancel: @@ -1170,6 +1188,8 @@ xfs_create( xfs_finish_inode_setup(ip); xfs_irele(ip); } + out_parent: + xfs_parent_finish(mp, ppargs); out_release_dquots: xfs_qm_dqrele(udqp); xfs_qm_dqrele(gdqp); @@ -3038,7 +3058,7 @@ xfs_rename_alloc_whiteout( int error; error = xfs_create_tmpfile(idmap, dp, S_IFCHR | WHITEOUT_MODE, - false, &tmpfile); + xfs_has_parent(dp->i_mount), &tmpfile); if (error) return error; diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index d32322f9ecde3..ff222827e5508 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -157,6 +157,8 @@ xfs_create_need_xattr( if (dir->i_sb->s_security) return true; #endif + if (xfs_has_parent(XFS_I(dir)->i_mount)) + return true; return false; } @@ -201,7 +203,18 @@ xfs_generic_create( xfs_create_need_xattr(dir, default_acl, acl), &ip); } else { - error = xfs_create_tmpfile(idmap, XFS_I(dir), mode, false, &ip); + bool init_xattrs = false; + + /* + * If this temporary file will be linkable, set up the file + * with an attr fork to receive a parent pointer. + */ + if (!(tmpfile->f_flags & O_EXCL) && + xfs_has_parent(XFS_I(dir)->i_mount)) + init_xattrs = true; + + error = xfs_create_tmpfile(idmap, XFS_I(dir), mode, + init_xattrs, &ip); } if (unlikely(error)) goto out_free_acl; diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index e525a6c477ff9..c303d7ff95977 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -44,6 +44,7 @@ #include "xfs_dahash_test.h" #include "xfs_rtbitmap.h" #include "xfs_exchmaps_item.h" +#include "xfs_parent.h" #include "scrub/stats.h" #include "scrub/rcbag_btree.h" @@ -2211,8 +2212,16 @@ xfs_init_caches(void) if (!xfs_xmi_cache) goto out_destroy_xmd_cache; + xfs_parent_args_cache = kmem_cache_create("xfs_parent_args", + sizeof(struct xfs_parent_args), + 0, 0, NULL); + if (!xfs_parent_args_cache) + goto out_destroy_xmi_cache; + return 0; + out_destroy_xmi_cache: + kmem_cache_destroy(xfs_xmi_cache); out_destroy_xmd_cache: kmem_cache_destroy(xfs_xmd_cache); out_destroy_iul_cache: @@ -2273,6 +2282,7 @@ xfs_destroy_caches(void) * destroy caches. */ rcu_barrier(); + kmem_cache_destroy(xfs_parent_args_cache); kmem_cache_destroy(xfs_xmd_cache); kmem_cache_destroy(xfs_xmi_cache); kmem_cache_destroy(xfs_iunlink_cache); From f1097be220fa938de5114db57a1ddb5de2bf6046 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 22 Apr 2024 09:47:48 -0700 Subject: [PATCH 0439/1702] xfs: add parent attributes to link This patch modifies xfs_link to add a parent pointer to the inode. Signed-off-by: Dave Chinner Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong [djwong: minor rebase fixes] Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_trans_space.c | 14 +++++++++++ fs/xfs/libxfs/xfs_trans_space.h | 3 +-- fs/xfs/scrub/dir_repair.c | 2 +- fs/xfs/scrub/orphanage.c | 2 +- fs/xfs/xfs_inode.c | 43 ++++++++++++++++++++++++++++----- 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/fs/xfs/libxfs/xfs_trans_space.c b/fs/xfs/libxfs/xfs_trans_space.c index 90532c3fa2053..cf775750120e8 100644 --- a/fs/xfs/libxfs/xfs_trans_space.c +++ b/fs/xfs/libxfs/xfs_trans_space.c @@ -50,3 +50,17 @@ xfs_mkdir_space_res( { return xfs_create_space_res(mp, namelen); } + +unsigned int +xfs_link_space_res( + struct xfs_mount *mp, + unsigned int namelen) +{ + unsigned int ret; + + ret = XFS_DIRENTER_SPACE_RES(mp, namelen); + if (xfs_has_parent(mp)) + ret += xfs_parent_calc_space_res(mp, namelen); + + return ret; +} diff --git a/fs/xfs/libxfs/xfs_trans_space.h b/fs/xfs/libxfs/xfs_trans_space.h index 6cda87153b38c..5539634009fb2 100644 --- a/fs/xfs/libxfs/xfs_trans_space.h +++ b/fs/xfs/libxfs/xfs_trans_space.h @@ -86,8 +86,6 @@ (2 * (mp)->m_alloc_maxlevels) #define XFS_GROWFSRT_SPACE_RES(mp,b) \ ((b) + XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK)) -#define XFS_LINK_SPACE_RES(mp,nl) \ - XFS_DIRENTER_SPACE_RES(mp,nl) #define XFS_QM_DQALLOC_SPACE_RES(mp) \ (XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK) + \ XFS_DQUOT_CLUSTER_SIZE_FSB) @@ -107,5 +105,6 @@ unsigned int xfs_parent_calc_space_res(struct xfs_mount *mp, unsigned int xfs_create_space_res(struct xfs_mount *mp, unsigned int namelen); unsigned int xfs_mkdir_space_res(struct xfs_mount *mp, unsigned int namelen); +unsigned int xfs_link_space_res(struct xfs_mount *mp, unsigned int namelen); #endif /* __XFS_TRANS_SPACE_H__ */ diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index 38957da26b94a..575397aef1f7a 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -704,7 +704,7 @@ xrep_dir_replay_update( uint resblks; int error; - resblks = XFS_LINK_SPACE_RES(mp, xname->len); + resblks = xfs_link_space_res(mp, xname->len); error = xchk_trans_alloc(rd->sc, resblks); if (error) return error; diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c index 885b7d478a0ab..5e2c3546f2e95 100644 --- a/fs/xfs/scrub/orphanage.c +++ b/fs/xfs/scrub/orphanage.c @@ -326,7 +326,7 @@ xrep_adoption_trans_alloc( /* Compute the worst case space reservation that we need. */ adopt->sc = sc; - adopt->orphanage_blkres = XFS_LINK_SPACE_RES(mp, MAXNAMELEN); + adopt->orphanage_blkres = xfs_link_space_res(mp, MAXNAMELEN); if (S_ISDIR(VFS_I(sc->ip)->i_mode)) child_blkres = XFS_RENAME_SPACE_RES(mp, xfs_name_dotdot.len); adopt->child_blkres = child_blkres; diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 0dd4111a6773e..61a390f5e2ae6 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -1299,14 +1299,15 @@ xfs_create_tmpfile( int xfs_link( - xfs_inode_t *tdp, - xfs_inode_t *sip, + struct xfs_inode *tdp, + struct xfs_inode *sip, struct xfs_name *target_name) { - xfs_mount_t *mp = tdp->i_mount; - xfs_trans_t *tp; + struct xfs_mount *mp = tdp->i_mount; + struct xfs_trans *tp; int error, nospace_error = 0; int resblks; + struct xfs_parent_args *ppargs; trace_xfs_link(tdp, target_name); @@ -1325,11 +1326,25 @@ xfs_link( if (error) goto std_return; - resblks = XFS_LINK_SPACE_RES(mp, target_name->len); + error = xfs_parent_start(mp, &ppargs); + if (error) + goto std_return; + + resblks = xfs_link_space_res(mp, target_name->len); error = xfs_trans_alloc_dir(tdp, &M_RES(mp)->tr_link, sip, &resblks, &tp, &nospace_error); if (error) - goto std_return; + goto out_parent; + + /* + * We don't allow reservationless or quotaless hardlinking when parent + * pointers are enabled because we can't back out if the xattrs must + * grow. + */ + if (ppargs && nospace_error) { + error = nospace_error; + goto error_return; + } /* * If we are using project inheritance, we only allow hard link @@ -1380,6 +1395,19 @@ xfs_link( xfs_trans_log_inode(tp, tdp, XFS_ILOG_CORE); xfs_bumplink(tp, sip); + + /* + * If we have parent pointers, we now need to add the parent record to + * the attribute fork of the inode. If this is the initial parent + * attribute, we need to create it correctly, otherwise we can just add + * the parent to the inode. + */ + if (ppargs) { + error = xfs_parent_addname(tp, ppargs, tdp, target_name, sip); + if (error) + goto error_return; + } + xfs_dir_update_hook(tdp, sip, 1, target_name); /* @@ -1393,12 +1421,15 @@ xfs_link( error = xfs_trans_commit(tp); xfs_iunlock(tdp, XFS_ILOCK_EXCL); xfs_iunlock(sip, XFS_ILOCK_EXCL); + xfs_parent_finish(mp, ppargs); return error; error_return: xfs_trans_cancel(tp); xfs_iunlock(tdp, XFS_ILOCK_EXCL); xfs_iunlock(sip, XFS_ILOCK_EXCL); + out_parent: + xfs_parent_finish(mp, ppargs); std_return: if (error == -ENOSPC && nospace_error) error = nospace_error; From 5d31a85dcc1fa4c5d4a925c6da67751653a700ba Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 22 Apr 2024 09:47:49 -0700 Subject: [PATCH 0440/1702] xfs: add parent attributes to symlink This patch modifies xfs_symlink to add a parent pointer to the inode. Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong [djwong: minor rebase fixups] Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_trans_space.c | 17 +++++++++++++++++ fs/xfs/libxfs/xfs_trans_space.h | 4 ++-- fs/xfs/scrub/symlink_repair.c | 2 +- fs/xfs/xfs_symlink.c | 30 +++++++++++++++++++++++++----- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/fs/xfs/libxfs/xfs_trans_space.c b/fs/xfs/libxfs/xfs_trans_space.c index cf775750120e8..c8adda82debe0 100644 --- a/fs/xfs/libxfs/xfs_trans_space.c +++ b/fs/xfs/libxfs/xfs_trans_space.c @@ -64,3 +64,20 @@ xfs_link_space_res( return ret; } + +unsigned int +xfs_symlink_space_res( + struct xfs_mount *mp, + unsigned int namelen, + unsigned int fsblocks) +{ + unsigned int ret; + + ret = XFS_IALLOC_SPACE_RES(mp) + XFS_DIRENTER_SPACE_RES(mp, namelen) + + fsblocks; + + if (xfs_has_parent(mp)) + ret += xfs_parent_calc_space_res(mp, namelen); + + return ret; +} diff --git a/fs/xfs/libxfs/xfs_trans_space.h b/fs/xfs/libxfs/xfs_trans_space.h index 5539634009fb2..354ad1d6e18d6 100644 --- a/fs/xfs/libxfs/xfs_trans_space.h +++ b/fs/xfs/libxfs/xfs_trans_space.h @@ -95,8 +95,6 @@ XFS_DIRREMOVE_SPACE_RES(mp) #define XFS_RENAME_SPACE_RES(mp,nl) \ (XFS_DIRREMOVE_SPACE_RES(mp) + XFS_DIRENTER_SPACE_RES(mp,nl)) -#define XFS_SYMLINK_SPACE_RES(mp,nl,b) \ - (XFS_IALLOC_SPACE_RES(mp) + XFS_DIRENTER_SPACE_RES(mp,nl) + (b)) #define XFS_IFREE_SPACE_RES(mp) \ (xfs_has_finobt(mp) ? M_IGEO(mp)->inobt_maxlevels : 0) @@ -106,5 +104,7 @@ unsigned int xfs_parent_calc_space_res(struct xfs_mount *mp, unsigned int xfs_create_space_res(struct xfs_mount *mp, unsigned int namelen); unsigned int xfs_mkdir_space_res(struct xfs_mount *mp, unsigned int namelen); unsigned int xfs_link_space_res(struct xfs_mount *mp, unsigned int namelen); +unsigned int xfs_symlink_space_res(struct xfs_mount *mp, unsigned int namelen, + unsigned int fsblocks); #endif /* __XFS_TRANS_SPACE_H__ */ diff --git a/fs/xfs/scrub/symlink_repair.c b/fs/xfs/scrub/symlink_repair.c index 178304959535a..c8b5a5b878ac9 100644 --- a/fs/xfs/scrub/symlink_repair.c +++ b/fs/xfs/scrub/symlink_repair.c @@ -421,7 +421,7 @@ xrep_symlink_rebuild( * unlikely. */ fs_blocks = xfs_symlink_blocks(sc->mp, target_len); - resblks = XFS_SYMLINK_SPACE_RES(sc->mp, target_len, fs_blocks); + resblks = xfs_symlink_space_res(sc->mp, target_len, fs_blocks); error = xfs_trans_reserve_quota_nblks(sc->tp, sc->tempip, resblks, 0, true); if (error) diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index 85ef56fdd7dfe..17aee806ec2e1 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -25,6 +25,8 @@ #include "xfs_error.h" #include "xfs_health.h" #include "xfs_symlink_remote.h" +#include "xfs_parent.h" +#include "xfs_defer.h" int xfs_readlink( @@ -100,6 +102,7 @@ xfs_symlink( struct xfs_dquot *pdqp = NULL; uint resblks; xfs_ino_t ino; + struct xfs_parent_args *ppargs; *ipp = NULL; @@ -130,18 +133,24 @@ xfs_symlink( /* * The symlink will fit into the inode data fork? - * There can't be any attributes so we get the whole variable part. + * If there are no parent pointers, then there wont't be any attributes. + * So we get the whole variable part, and do not need to reserve extra + * blocks. Otherwise, we need to reserve the blocks. */ - if (pathlen <= XFS_LITINO(mp)) + if (pathlen <= XFS_LITINO(mp) && !xfs_has_parent(mp)) fs_blocks = 0; else fs_blocks = xfs_symlink_blocks(mp, pathlen); - resblks = XFS_SYMLINK_SPACE_RES(mp, link_name->len, fs_blocks); + resblks = xfs_symlink_space_res(mp, link_name->len, fs_blocks); + + error = xfs_parent_start(mp, &ppargs); + if (error) + goto out_release_dquots; error = xfs_trans_alloc_icreate(mp, &M_RES(mp)->tr_symlink, udqp, gdqp, pdqp, resblks, &tp); if (error) - goto out_release_dquots; + goto out_parent; xfs_ilock(dp, XFS_ILOCK_EXCL | XFS_ILOCK_PARENT); unlock_dp_on_error = true; @@ -161,7 +170,7 @@ xfs_symlink( if (!error) error = xfs_init_new_inode(idmap, tp, dp, ino, S_IFLNK | (mode & ~S_IFMT), 1, 0, prid, - false, &ip); + xfs_has_parent(mp), &ip); if (error) goto out_trans_cancel; @@ -195,6 +204,14 @@ xfs_symlink( goto out_trans_cancel; xfs_trans_ichgtime(tp, dp, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); xfs_trans_log_inode(tp, dp, XFS_ILOG_CORE); + + /* Add parent pointer for the new symlink. */ + if (ppargs) { + error = xfs_parent_addname(tp, ppargs, dp, link_name, ip); + if (error) + goto out_trans_cancel; + } + xfs_dir_update_hook(dp, ip, 1, link_name); /* @@ -216,6 +233,7 @@ xfs_symlink( *ipp = ip; xfs_iunlock(ip, XFS_ILOCK_EXCL); xfs_iunlock(dp, XFS_ILOCK_EXCL); + xfs_parent_finish(mp, ppargs); return 0; out_trans_cancel: @@ -231,6 +249,8 @@ xfs_symlink( xfs_finish_inode_setup(ip); xfs_irele(ip); } +out_parent: + xfs_parent_finish(mp, ppargs); out_release_dquots: xfs_qm_dqrele(udqp); xfs_qm_dqrele(gdqp); From d2d18330f63cd70b50eddac76de7c59a36f2faa7 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 22 Apr 2024 09:47:49 -0700 Subject: [PATCH 0441/1702] xfs: remove parent pointers in unlink This patch removes the parent pointer attribute during unlink Signed-off-by: Dave Chinner Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong [djwong: adjust to new ondisk format, minor rebase fixes] Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_parent.c | 22 ++++++++++++++++++++++ fs/xfs/libxfs/xfs_parent.h | 3 +++ fs/xfs/libxfs/xfs_trans_space.c | 13 +++++++++++++ fs/xfs/libxfs/xfs_trans_space.h | 3 +-- fs/xfs/xfs_inode.c | 27 +++++++++++++++++++++------ 5 files changed, 60 insertions(+), 8 deletions(-) diff --git a/fs/xfs/libxfs/xfs_parent.c b/fs/xfs/libxfs/xfs_parent.c index 65616cfc1a2bf..6142e68f2338c 100644 --- a/fs/xfs/libxfs/xfs_parent.c +++ b/fs/xfs/libxfs/xfs_parent.c @@ -205,3 +205,25 @@ xfs_parent_addname( xfs_attr_defer_add(&ppargs->args, XFS_ATTR_DEFER_SET); return 0; } + +/* Remove a parent pointer to reflect a dirent removal. */ +int +xfs_parent_removename( + struct xfs_trans *tp, + struct xfs_parent_args *ppargs, + struct xfs_inode *dp, + const struct xfs_name *parent_name, + struct xfs_inode *child) +{ + int error; + + error = xfs_parent_iread_extents(tp, child); + if (error) + return error; + + xfs_inode_to_parent_rec(&ppargs->rec, dp); + xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, + child->i_ino, parent_name); + xfs_attr_defer_add(&ppargs->args, XFS_ATTR_DEFER_REMOVE); + return 0; +} diff --git a/fs/xfs/libxfs/xfs_parent.h b/fs/xfs/libxfs/xfs_parent.h index 6de24e3ef318c..4a7fd48c226a4 100644 --- a/fs/xfs/libxfs/xfs_parent.h +++ b/fs/xfs/libxfs/xfs_parent.h @@ -81,5 +81,8 @@ xfs_parent_finish( int xfs_parent_addname(struct xfs_trans *tp, struct xfs_parent_args *ppargs, struct xfs_inode *dp, const struct xfs_name *parent_name, struct xfs_inode *child); +int xfs_parent_removename(struct xfs_trans *tp, struct xfs_parent_args *ppargs, + struct xfs_inode *dp, const struct xfs_name *parent_name, + struct xfs_inode *child); #endif /* __XFS_PARENT_H__ */ diff --git a/fs/xfs/libxfs/xfs_trans_space.c b/fs/xfs/libxfs/xfs_trans_space.c index c8adda82debe0..df729e4f1a4c9 100644 --- a/fs/xfs/libxfs/xfs_trans_space.c +++ b/fs/xfs/libxfs/xfs_trans_space.c @@ -81,3 +81,16 @@ xfs_symlink_space_res( return ret; } + +unsigned int +xfs_remove_space_res( + struct xfs_mount *mp, + unsigned int namelen) +{ + unsigned int ret = XFS_DIRREMOVE_SPACE_RES(mp); + + if (xfs_has_parent(mp)) + ret += xfs_parent_calc_space_res(mp, namelen); + + return ret; +} diff --git a/fs/xfs/libxfs/xfs_trans_space.h b/fs/xfs/libxfs/xfs_trans_space.h index 354ad1d6e18d6..a4490813c56f1 100644 --- a/fs/xfs/libxfs/xfs_trans_space.h +++ b/fs/xfs/libxfs/xfs_trans_space.h @@ -91,8 +91,6 @@ XFS_DQUOT_CLUSTER_SIZE_FSB) #define XFS_QM_QINOCREATE_SPACE_RES(mp) \ XFS_IALLOC_SPACE_RES(mp) -#define XFS_REMOVE_SPACE_RES(mp) \ - XFS_DIRREMOVE_SPACE_RES(mp) #define XFS_RENAME_SPACE_RES(mp,nl) \ (XFS_DIRREMOVE_SPACE_RES(mp) + XFS_DIRENTER_SPACE_RES(mp,nl)) #define XFS_IFREE_SPACE_RES(mp) \ @@ -106,5 +104,6 @@ unsigned int xfs_mkdir_space_res(struct xfs_mount *mp, unsigned int namelen); unsigned int xfs_link_space_res(struct xfs_mount *mp, unsigned int namelen); unsigned int xfs_symlink_space_res(struct xfs_mount *mp, unsigned int namelen, unsigned int fsblocks); +unsigned int xfs_remove_space_res(struct xfs_mount *mp, unsigned int namelen); #endif /* __XFS_TRANS_SPACE_H__ */ diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 61a390f5e2ae6..c4a1c2dd5261d 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -2721,16 +2721,17 @@ xfs_iunpin_wait( */ int xfs_remove( - xfs_inode_t *dp, + struct xfs_inode *dp, struct xfs_name *name, - xfs_inode_t *ip) + struct xfs_inode *ip) { - xfs_mount_t *mp = dp->i_mount; - xfs_trans_t *tp = NULL; + struct xfs_mount *mp = dp->i_mount; + struct xfs_trans *tp = NULL; int is_dir = S_ISDIR(VFS_I(ip)->i_mode); int dontcare; int error = 0; uint resblks; + struct xfs_parent_args *ppargs; trace_xfs_remove(dp, name); @@ -2747,6 +2748,10 @@ xfs_remove( if (error) goto std_return; + error = xfs_parent_start(mp, &ppargs); + if (error) + goto std_return; + /* * We try to get the real space reservation first, allowing for * directory btree deletion(s) implying possible bmap insert(s). If we @@ -2758,12 +2763,12 @@ xfs_remove( * the directory code can handle a reservationless update and we don't * want to prevent a user from trying to free space by deleting things. */ - resblks = XFS_REMOVE_SPACE_RES(mp); + resblks = xfs_remove_space_res(mp, name->len); error = xfs_trans_alloc_dir(dp, &M_RES(mp)->tr_remove, ip, &resblks, &tp, &dontcare); if (error) { ASSERT(error != -ENOSPC); - goto std_return; + goto out_parent; } /* @@ -2823,6 +2828,13 @@ xfs_remove( goto out_trans_cancel; } + /* Remove parent pointer. */ + if (ppargs) { + error = xfs_parent_removename(tp, ppargs, dp, name, ip); + if (error) + goto out_trans_cancel; + } + /* * Drop the link from dp to ip, and if ip was a directory, remove the * '.' and '..' references since we freed the directory. @@ -2846,6 +2858,7 @@ xfs_remove( xfs_iunlock(ip, XFS_ILOCK_EXCL); xfs_iunlock(dp, XFS_ILOCK_EXCL); + xfs_parent_finish(mp, ppargs); return 0; out_trans_cancel: @@ -2853,6 +2866,8 @@ xfs_remove( out_unlock: xfs_iunlock(ip, XFS_ILOCK_EXCL); xfs_iunlock(dp, XFS_ILOCK_EXCL); + out_parent: + xfs_parent_finish(mp, ppargs); std_return: return error; } From 5a8338c88284df4e9e697225aa65f2709333a659 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 22 Apr 2024 09:47:50 -0700 Subject: [PATCH 0442/1702] xfs: Add parent pointers to rename This patch removes the old parent pointer attribute during the rename operation, and re-adds the updated parent pointer. Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong [djwong: adjust to new ondisk format] Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_parent.c | 30 +++++++++++++ fs/xfs/libxfs/xfs_parent.h | 6 +++ fs/xfs/libxfs/xfs_trans_space.c | 25 +++++++++++ fs/xfs/libxfs/xfs_trans_space.h | 6 ++- fs/xfs/scrub/orphanage.c | 3 +- fs/xfs/scrub/parent_repair.c | 3 +- fs/xfs/xfs_inode.c | 80 ++++++++++++++++++++++++++++++--- 7 files changed, 142 insertions(+), 11 deletions(-) diff --git a/fs/xfs/libxfs/xfs_parent.c b/fs/xfs/libxfs/xfs_parent.c index 6142e68f2338c..fdf643bfde4df 100644 --- a/fs/xfs/libxfs/xfs_parent.c +++ b/fs/xfs/libxfs/xfs_parent.c @@ -227,3 +227,33 @@ xfs_parent_removename( xfs_attr_defer_add(&ppargs->args, XFS_ATTR_DEFER_REMOVE); return 0; } + +/* Replace one parent pointer with another to reflect a rename. */ +int +xfs_parent_replacename( + struct xfs_trans *tp, + struct xfs_parent_args *ppargs, + struct xfs_inode *old_dp, + const struct xfs_name *old_name, + struct xfs_inode *new_dp, + const struct xfs_name *new_name, + struct xfs_inode *child) +{ + int error; + + error = xfs_parent_iread_extents(tp, child); + if (error) + return error; + + xfs_inode_to_parent_rec(&ppargs->rec, old_dp); + xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, + child->i_ino, old_name); + + xfs_inode_to_parent_rec(&ppargs->new_rec, new_dp); + ppargs->args.new_name = new_name->name; + ppargs->args.new_namelen = new_name->len; + ppargs->args.new_value = &ppargs->new_rec; + ppargs->args.new_valuelen = sizeof(struct xfs_parent_rec); + xfs_attr_defer_add(&ppargs->args, XFS_ATTR_DEFER_REPLACE); + return 0; +} diff --git a/fs/xfs/libxfs/xfs_parent.h b/fs/xfs/libxfs/xfs_parent.h index 4a7fd48c226a4..768633b313671 100644 --- a/fs/xfs/libxfs/xfs_parent.h +++ b/fs/xfs/libxfs/xfs_parent.h @@ -45,6 +45,7 @@ extern struct kmem_cache *xfs_parent_args_cache; */ struct xfs_parent_args { struct xfs_parent_rec rec; + struct xfs_parent_rec new_rec; struct xfs_da_args args; }; @@ -84,5 +85,10 @@ int xfs_parent_addname(struct xfs_trans *tp, struct xfs_parent_args *ppargs, int xfs_parent_removename(struct xfs_trans *tp, struct xfs_parent_args *ppargs, struct xfs_inode *dp, const struct xfs_name *parent_name, struct xfs_inode *child); +int xfs_parent_replacename(struct xfs_trans *tp, + struct xfs_parent_args *ppargs, + struct xfs_inode *old_dp, const struct xfs_name *old_name, + struct xfs_inode *new_dp, const struct xfs_name *new_name, + struct xfs_inode *child); #endif /* __XFS_PARENT_H__ */ diff --git a/fs/xfs/libxfs/xfs_trans_space.c b/fs/xfs/libxfs/xfs_trans_space.c index df729e4f1a4c9..b9dc3752f702c 100644 --- a/fs/xfs/libxfs/xfs_trans_space.c +++ b/fs/xfs/libxfs/xfs_trans_space.c @@ -94,3 +94,28 @@ xfs_remove_space_res( return ret; } + +unsigned int +xfs_rename_space_res( + struct xfs_mount *mp, + unsigned int src_namelen, + bool target_exists, + unsigned int target_namelen, + bool has_whiteout) +{ + unsigned int ret; + + ret = XFS_DIRREMOVE_SPACE_RES(mp) + + XFS_DIRENTER_SPACE_RES(mp, target_namelen); + + if (xfs_has_parent(mp)) { + if (has_whiteout) + ret += xfs_parent_calc_space_res(mp, src_namelen); + ret += 2 * xfs_parent_calc_space_res(mp, target_namelen); + } + + if (target_exists) + ret += xfs_parent_calc_space_res(mp, target_namelen); + + return ret; +} diff --git a/fs/xfs/libxfs/xfs_trans_space.h b/fs/xfs/libxfs/xfs_trans_space.h index a4490813c56f1..1155ff2d37e29 100644 --- a/fs/xfs/libxfs/xfs_trans_space.h +++ b/fs/xfs/libxfs/xfs_trans_space.h @@ -91,8 +91,6 @@ XFS_DQUOT_CLUSTER_SIZE_FSB) #define XFS_QM_QINOCREATE_SPACE_RES(mp) \ XFS_IALLOC_SPACE_RES(mp) -#define XFS_RENAME_SPACE_RES(mp,nl) \ - (XFS_DIRREMOVE_SPACE_RES(mp) + XFS_DIRENTER_SPACE_RES(mp,nl)) #define XFS_IFREE_SPACE_RES(mp) \ (xfs_has_finobt(mp) ? M_IGEO(mp)->inobt_maxlevels : 0) @@ -106,4 +104,8 @@ unsigned int xfs_symlink_space_res(struct xfs_mount *mp, unsigned int namelen, unsigned int fsblocks); unsigned int xfs_remove_space_res(struct xfs_mount *mp, unsigned int namelen); +unsigned int xfs_rename_space_res(struct xfs_mount *mp, + unsigned int src_namelen, bool target_exists, + unsigned int target_namelen, bool has_whiteout); + #endif /* __XFS_TRANS_SPACE_H__ */ diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c index 5e2c3546f2e95..94bcc2799188f 100644 --- a/fs/xfs/scrub/orphanage.c +++ b/fs/xfs/scrub/orphanage.c @@ -328,7 +328,8 @@ xrep_adoption_trans_alloc( adopt->sc = sc; adopt->orphanage_blkres = xfs_link_space_res(mp, MAXNAMELEN); if (S_ISDIR(VFS_I(sc->ip)->i_mode)) - child_blkres = XFS_RENAME_SPACE_RES(mp, xfs_name_dotdot.len); + child_blkres = xfs_rename_space_res(mp, 0, false, + xfs_name_dotdot.len, false); adopt->child_blkres = child_blkres; /* diff --git a/fs/xfs/scrub/parent_repair.c b/fs/xfs/scrub/parent_repair.c index ebb5791bf839e..63590e1b35060 100644 --- a/fs/xfs/scrub/parent_repair.c +++ b/fs/xfs/scrub/parent_repair.c @@ -171,7 +171,8 @@ xrep_parent_reset_dotdot( * Reserve more space just in case we have to expand the dir. We're * allowed to exceed quota to repair inconsistent metadata. */ - spaceres = XFS_RENAME_SPACE_RES(sc->mp, xfs_name_dotdot.len); + spaceres = xfs_rename_space_res(sc->mp, 0, false, xfs_name_dotdot.len, + false); error = xfs_trans_reserve_more_inode(sc->tp, sc->ip, spaceres, 0, true); if (error) diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index c4a1c2dd5261d..59488c17e1c1c 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -3148,6 +3148,9 @@ xfs_rename( struct xfs_trans *tp; struct xfs_inode *wip = NULL; /* whiteout inode */ struct xfs_inode *inodes[__XFS_SORT_INODES]; + struct xfs_parent_args *src_ppargs = NULL; + struct xfs_parent_args *tgt_ppargs = NULL; + struct xfs_parent_args *wip_ppargs = NULL; int i; int num_inodes = __XFS_SORT_INODES; bool new_parent = (src_dp != target_dp); @@ -3179,9 +3182,26 @@ xfs_rename( xfs_sort_for_rename(src_dp, target_dp, src_ip, target_ip, wip, inodes, &num_inodes); + error = xfs_parent_start(mp, &src_ppargs); + if (error) + goto out_release_wip; + + if (wip) { + error = xfs_parent_start(mp, &wip_ppargs); + if (error) + goto out_src_ppargs; + } + + if (target_ip) { + error = xfs_parent_start(mp, &tgt_ppargs); + if (error) + goto out_wip_ppargs; + } + retry: nospace_error = 0; - spaceres = XFS_RENAME_SPACE_RES(mp, target_name->len); + spaceres = xfs_rename_space_res(mp, src_name->len, target_ip != NULL, + target_name->len, wip != NULL); error = xfs_trans_alloc(mp, &M_RES(mp)->tr_rename, spaceres, 0, 0, &tp); if (error == -ENOSPC) { nospace_error = error; @@ -3190,7 +3210,17 @@ xfs_rename( &tp); } if (error) - goto out_release_wip; + goto out_tgt_ppargs; + + /* + * We don't allow reservationless renaming when parent pointers are + * enabled because we can't back out if the xattrs must grow. + */ + if (src_ppargs && nospace_error) { + error = nospace_error; + xfs_trans_cancel(tp); + goto out_tgt_ppargs; + } /* * Attach the dquots to the inodes @@ -3198,7 +3228,7 @@ xfs_rename( error = xfs_qm_vop_rename_dqattach(inodes); if (error) { xfs_trans_cancel(tp); - goto out_release_wip; + goto out_tgt_ppargs; } /* @@ -3267,6 +3297,15 @@ xfs_rename( goto out_trans_cancel; } + /* + * We don't allow quotaless renaming when parent pointers are enabled + * because we can't back out if the xattrs must grow. + */ + if (src_ppargs && nospace_error) { + error = nospace_error; + goto out_trans_cancel; + } + /* * Check for expected errors before we dirty the transaction * so we can return an error without a transaction abort. @@ -3459,6 +3498,28 @@ xfs_rename( if (error) goto out_trans_cancel; + /* Schedule parent pointer updates. */ + if (wip_ppargs) { + error = xfs_parent_addname(tp, wip_ppargs, src_dp, src_name, + wip); + if (error) + goto out_trans_cancel; + } + + if (src_ppargs) { + error = xfs_parent_replacename(tp, src_ppargs, src_dp, + src_name, target_dp, target_name, src_ip); + if (error) + goto out_trans_cancel; + } + + if (tgt_ppargs) { + error = xfs_parent_removename(tp, tgt_ppargs, target_dp, + target_name, target_ip); + if (error) + goto out_trans_cancel; + } + xfs_trans_ichgtime(tp, src_dp, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); xfs_trans_log_inode(tp, src_dp, XFS_ILOG_CORE); if (new_parent) @@ -3480,14 +3541,19 @@ xfs_rename( xfs_dir_update_hook(src_dp, wip, 1, src_name); error = xfs_finish_rename(tp); - xfs_iunlock_rename(inodes, num_inodes); - if (wip) - xfs_irele(wip); - return error; + nospace_error = 0; + goto out_unlock; out_trans_cancel: xfs_trans_cancel(tp); +out_unlock: xfs_iunlock_rename(inodes, num_inodes); +out_tgt_ppargs: + xfs_parent_finish(mp, tgt_ppargs); +out_wip_ppargs: + xfs_parent_finish(mp, wip_ppargs); +out_src_ppargs: + xfs_parent_finish(mp, src_ppargs); out_release_wip: if (wip) xfs_irele(wip); From 1c12949e50e191933c08758ae53e31b852e730d6 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 22 Apr 2024 09:47:51 -0700 Subject: [PATCH 0443/1702] xfs: Add parent pointers to xfs_cross_rename Cross renames are handled separately from standard renames, and need different handling to update the parent attributes correctly. Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_inode.c | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 59488c17e1c1c..ebe2ce9bd9eee 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -2972,15 +2972,17 @@ xfs_cross_rename( struct xfs_inode *dp1, struct xfs_name *name1, struct xfs_inode *ip1, + struct xfs_parent_args *ip1_ppargs, struct xfs_inode *dp2, struct xfs_name *name2, struct xfs_inode *ip2, + struct xfs_parent_args *ip2_ppargs, int spaceres) { - int error = 0; - int ip1_flags = 0; - int ip2_flags = 0; - int dp2_flags = 0; + int error = 0; + int ip1_flags = 0; + int ip2_flags = 0; + int dp2_flags = 0; /* Swap inode number for dirent in first parent */ error = xfs_dir_replace(tp, dp1, name1, ip2->i_ino, spaceres); @@ -3049,6 +3051,21 @@ xfs_cross_rename( } } + /* Schedule parent pointer replacements */ + if (ip1_ppargs) { + error = xfs_parent_replacename(tp, ip1_ppargs, dp1, name1, dp2, + name2, ip1); + if (error) + goto out_trans_abort; + } + + if (ip2_ppargs) { + error = xfs_parent_replacename(tp, ip2_ppargs, dp2, name2, dp1, + name1, ip2); + if (error) + goto out_trans_abort; + } + if (ip1_flags) { xfs_trans_ichgtime(tp, ip1, ip1_flags); xfs_trans_log_inode(tp, ip1, XFS_ILOG_CORE); @@ -3265,10 +3282,10 @@ xfs_rename( /* RENAME_EXCHANGE is unique from here on. */ if (flags & RENAME_EXCHANGE) { error = xfs_cross_rename(tp, src_dp, src_name, src_ip, - target_dp, target_name, target_ip, - spaceres); - xfs_iunlock_rename(inodes, num_inodes); - return error; + src_ppargs, target_dp, target_name, target_ip, + tgt_ppargs, spaceres); + nospace_error = 0; + goto out_unlock; } /* From daf9f884906bcfcffe26967aee9ece893fba019b Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 22 Apr 2024 09:47:52 -0700 Subject: [PATCH 0444/1702] xfs: don't return XFS_ATTR_PARENT attributes via listxattr Parent pointers are internal filesystem metadata. They're not intended to be directly visible to userspace, so filter them out of xfs_xattr_put_listent so that they don't appear in listxattr. Signed-off-by: Allison Henderson Inspired-by: Andrey Albershteyn Reviewed-by: Darrick J. Wong [djwong: change this to XFS_ATTR_PRIVATE_NSP_MASK per fsverity patchset] Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_da_format.h | 3 +++ fs/xfs/xfs_xattr.c | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/fs/xfs/libxfs/xfs_da_format.h b/fs/xfs/libxfs/xfs_da_format.h index 1395ad1937c53..ebde6eb1da65d 100644 --- a/fs/xfs/libxfs/xfs_da_format.h +++ b/fs/xfs/libxfs/xfs_da_format.h @@ -726,6 +726,9 @@ struct xfs_attr3_leafblock { XFS_ATTR_SECURE | \ XFS_ATTR_PARENT) +/* Private attr namespaces not exposed to userspace */ +#define XFS_ATTR_PRIVATE_NSP_MASK (XFS_ATTR_PARENT) + #define XFS_ATTR_ONDISK_MASK (XFS_ATTR_NSP_ONDISK_MASK | \ XFS_ATTR_LOCAL | \ XFS_ATTR_INCOMPLETE) diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c index ba56a9e73144b..1e82d11d980f2 100644 --- a/fs/xfs/xfs_xattr.c +++ b/fs/xfs/xfs_xattr.c @@ -229,6 +229,10 @@ xfs_xattr_put_listent( ASSERT(context->count >= 0); + /* Don't expose private xattr namespaces. */ + if (flags & XFS_ATTR_PRIVATE_NSP_MASK) + return; + if (flags & XFS_ATTR_ROOT) { #ifdef CONFIG_XFS_POSIX_ACL if (namelen == SGI_ACL_FILE_SIZE && From 8f4b980ee67fe53a77b70b1fdd8e15f2fe37180c Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 22 Apr 2024 09:47:53 -0700 Subject: [PATCH 0445/1702] xfs: pass the attr value to put_listent when possible Pass the attr value to put_listent when we have local xattrs or shortform xattrs. This will enable the GETPARENTS ioctl to use xfs_attr_list as its backend. Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.h | 5 +++-- fs/xfs/libxfs/xfs_attr_sf.h | 1 + fs/xfs/xfs_attr_list.c | 8 +++++++- fs/xfs/xfs_ioctl.c | 1 + fs/xfs/xfs_xattr.c | 1 + 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h index d0ed7ea58ab0f..d12583dd7eecb 100644 --- a/fs/xfs/libxfs/xfs_attr.h +++ b/fs/xfs/libxfs/xfs_attr.h @@ -47,8 +47,9 @@ struct xfs_attrlist_cursor_kern { /* void; state communicated via *context */ -typedef void (*put_listent_func_t)(struct xfs_attr_list_context *, int, - unsigned char *, int, int); +typedef void (*put_listent_func_t)(struct xfs_attr_list_context *context, + int flags, unsigned char *name, int namelen, void *value, + int valuelen); struct xfs_attr_list_context { struct xfs_trans *tp; diff --git a/fs/xfs/libxfs/xfs_attr_sf.h b/fs/xfs/libxfs/xfs_attr_sf.h index bc44222230248..73bdc0e556825 100644 --- a/fs/xfs/libxfs/xfs_attr_sf.h +++ b/fs/xfs/libxfs/xfs_attr_sf.h @@ -16,6 +16,7 @@ typedef struct xfs_attr_sf_sort { uint8_t flags; /* flags bits (see xfs_attr_leaf.h) */ xfs_dahash_t hash; /* this entry's hash value */ unsigned char *name; /* name value, pointer into buffer */ + void *value; } xfs_attr_sf_sort_t; #define XFS_ATTR_SF_ENTSIZE_MAX /* max space for name&value */ \ diff --git a/fs/xfs/xfs_attr_list.c b/fs/xfs/xfs_attr_list.c index 9bc4b5322539a..5c947e5ce8b88 100644 --- a/fs/xfs/xfs_attr_list.c +++ b/fs/xfs/xfs_attr_list.c @@ -92,6 +92,7 @@ xfs_attr_shortform_list( sfe->flags, sfe->nameval, (int)sfe->namelen, + &sfe->nameval[sfe->namelen], (int)sfe->valuelen); /* * Either search callback finished early or @@ -138,6 +139,7 @@ xfs_attr_shortform_list( sbp->name = sfe->nameval; sbp->namelen = sfe->namelen; /* These are bytes, and both on-disk, don't endian-flip */ + sbp->value = &sfe->nameval[sfe->namelen], sbp->valuelen = sfe->valuelen; sbp->flags = sfe->flags; sbp->hash = xfs_attr_hashval(dp->i_mount, sfe->flags, @@ -192,6 +194,7 @@ xfs_attr_shortform_list( sbp->flags, sbp->name, sbp->namelen, + sbp->value, sbp->valuelen); if (context->seen_enough) break; @@ -479,6 +482,7 @@ xfs_attr3_leaf_list_int( */ for (; i < ichdr.count; entry++, i++) { char *name; + void *value; int namelen, valuelen; if (be32_to_cpu(entry->hashval) != cursor->hashval) { @@ -496,6 +500,7 @@ xfs_attr3_leaf_list_int( name_loc = xfs_attr3_leaf_name_local(leaf, i); name = name_loc->nameval; namelen = name_loc->namelen; + value = &name_loc->nameval[name_loc->namelen]; valuelen = be16_to_cpu(name_loc->valuelen); } else { xfs_attr_leaf_name_remote_t *name_rmt; @@ -503,6 +508,7 @@ xfs_attr3_leaf_list_int( name_rmt = xfs_attr3_leaf_name_remote(leaf, i); name = name_rmt->name; namelen = name_rmt->namelen; + value = NULL; valuelen = be32_to_cpu(name_rmt->valuelen); } @@ -513,7 +519,7 @@ xfs_attr3_leaf_list_int( return -EFSCORRUPTED; } context->put_listent(context, entry->flags, - name, namelen, valuelen); + name, namelen, value, valuelen); if (context->seen_enough) break; cursor->offset++; diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index e30f9f40f086d..7a2a5cf06a5cb 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -310,6 +310,7 @@ xfs_ioc_attr_put_listent( int flags, unsigned char *name, int namelen, + void *value, int valuelen) { struct xfs_attrlist *alist = context->buffer; diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c index 1e82d11d980f2..b43f7081b0f46 100644 --- a/fs/xfs/xfs_xattr.c +++ b/fs/xfs/xfs_xattr.c @@ -222,6 +222,7 @@ xfs_xattr_put_listent( int flags, unsigned char *name, int namelen, + void *value, int valuelen) { char *prefix; From af69d852dfe62b925d0df401eafad40698c889c6 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:54 -0700 Subject: [PATCH 0446/1702] xfs: move handle ioctl code to xfs_handle.c Move the handle managemnet code (and the attrmulti code that uses it) to xfs_handle.c. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/xfs_handle.c | 618 +++++++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_handle.h | 28 ++ fs/xfs/xfs_ioctl.c | 592 +---------------------------------------- fs/xfs/xfs_ioctl.h | 28 -- fs/xfs/xfs_ioctl32.c | 1 + 6 files changed, 649 insertions(+), 619 deletions(-) create mode 100644 fs/xfs/xfs_handle.c create mode 100644 fs/xfs/xfs_handle.h diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index 0c1a0b67af93c..c969b11ce0f47 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -78,6 +78,7 @@ xfs-y += xfs_aops.o \ xfs_fsmap.o \ xfs_fsops.o \ xfs_globals.o \ + xfs_handle.o \ xfs_health.o \ xfs_icache.o \ xfs_ioctl.o \ diff --git a/fs/xfs/xfs_handle.c b/fs/xfs/xfs_handle.c new file mode 100644 index 0000000000000..13c2479a3053f --- /dev/null +++ b/fs/xfs/xfs_handle.c @@ -0,0 +1,618 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All rights reserved. + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_shared.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_bmap_btree.h" +#include "xfs_inode.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_trans.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_ioctl.h" +#include "xfs_parent.h" +#include "xfs_da_btree.h" +#include "xfs_handle.h" +#include "xfs_health.h" +#include "xfs_icache.h" +#include "xfs_export.h" +#include "xfs_xattr.h" +#include "xfs_acl.h" + +#include + +/* + * xfs_find_handle maps from userspace xfs_fsop_handlereq structure to + * a file or fs handle. + * + * XFS_IOC_PATH_TO_FSHANDLE + * returns fs handle for a mount point or path within that mount point + * XFS_IOC_FD_TO_HANDLE + * returns full handle for a FD opened in user space + * XFS_IOC_PATH_TO_HANDLE + * returns full handle for a path + */ +int +xfs_find_handle( + unsigned int cmd, + xfs_fsop_handlereq_t *hreq) +{ + int hsize; + xfs_handle_t handle; + struct inode *inode; + struct fd f = {NULL}; + struct path path; + int error; + struct xfs_inode *ip; + + if (cmd == XFS_IOC_FD_TO_HANDLE) { + f = fdget(hreq->fd); + if (!f.file) + return -EBADF; + inode = file_inode(f.file); + } else { + error = user_path_at(AT_FDCWD, hreq->path, 0, &path); + if (error) + return error; + inode = d_inode(path.dentry); + } + ip = XFS_I(inode); + + /* + * We can only generate handles for inodes residing on a XFS filesystem, + * and only for regular files, directories or symbolic links. + */ + error = -EINVAL; + if (inode->i_sb->s_magic != XFS_SB_MAGIC) + goto out_put; + + error = -EBADF; + if (!S_ISREG(inode->i_mode) && + !S_ISDIR(inode->i_mode) && + !S_ISLNK(inode->i_mode)) + goto out_put; + + + memcpy(&handle.ha_fsid, ip->i_mount->m_fixedfsid, sizeof(xfs_fsid_t)); + + if (cmd == XFS_IOC_PATH_TO_FSHANDLE) { + /* + * This handle only contains an fsid, zero the rest. + */ + memset(&handle.ha_fid, 0, sizeof(handle.ha_fid)); + hsize = sizeof(xfs_fsid_t); + } else { + handle.ha_fid.fid_len = sizeof(xfs_fid_t) - + sizeof(handle.ha_fid.fid_len); + handle.ha_fid.fid_pad = 0; + handle.ha_fid.fid_gen = inode->i_generation; + handle.ha_fid.fid_ino = ip->i_ino; + hsize = sizeof(xfs_handle_t); + } + + error = -EFAULT; + if (copy_to_user(hreq->ohandle, &handle, hsize) || + copy_to_user(hreq->ohandlen, &hsize, sizeof(__s32))) + goto out_put; + + error = 0; + + out_put: + if (cmd == XFS_IOC_FD_TO_HANDLE) + fdput(f); + else + path_put(&path); + return error; +} + +/* + * No need to do permission checks on the various pathname components + * as the handle operations are privileged. + */ +STATIC int +xfs_handle_acceptable( + void *context, + struct dentry *dentry) +{ + return 1; +} + +/* + * Convert userspace handle data into a dentry. + */ +struct dentry * +xfs_handle_to_dentry( + struct file *parfilp, + void __user *uhandle, + u32 hlen) +{ + xfs_handle_t handle; + struct xfs_fid64 fid; + + /* + * Only allow handle opens under a directory. + */ + if (!S_ISDIR(file_inode(parfilp)->i_mode)) + return ERR_PTR(-ENOTDIR); + + if (hlen != sizeof(xfs_handle_t)) + return ERR_PTR(-EINVAL); + if (copy_from_user(&handle, uhandle, hlen)) + return ERR_PTR(-EFAULT); + if (handle.ha_fid.fid_len != + sizeof(handle.ha_fid) - sizeof(handle.ha_fid.fid_len)) + return ERR_PTR(-EINVAL); + + memset(&fid, 0, sizeof(struct fid)); + fid.ino = handle.ha_fid.fid_ino; + fid.gen = handle.ha_fid.fid_gen; + + return exportfs_decode_fh(parfilp->f_path.mnt, (struct fid *)&fid, 3, + FILEID_INO32_GEN | XFS_FILEID_TYPE_64FLAG, + xfs_handle_acceptable, NULL); +} + +STATIC struct dentry * +xfs_handlereq_to_dentry( + struct file *parfilp, + xfs_fsop_handlereq_t *hreq) +{ + return xfs_handle_to_dentry(parfilp, hreq->ihandle, hreq->ihandlen); +} + +int +xfs_open_by_handle( + struct file *parfilp, + xfs_fsop_handlereq_t *hreq) +{ + const struct cred *cred = current_cred(); + int error; + int fd; + int permflag; + struct file *filp; + struct inode *inode; + struct dentry *dentry; + fmode_t fmode; + struct path path; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + dentry = xfs_handlereq_to_dentry(parfilp, hreq); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + inode = d_inode(dentry); + + /* Restrict xfs_open_by_handle to directories & regular files. */ + if (!(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode))) { + error = -EPERM; + goto out_dput; + } + +#if BITS_PER_LONG != 32 + hreq->oflags |= O_LARGEFILE; +#endif + + permflag = hreq->oflags; + fmode = OPEN_FMODE(permflag); + if ((!(permflag & O_APPEND) || (permflag & O_TRUNC)) && + (fmode & FMODE_WRITE) && IS_APPEND(inode)) { + error = -EPERM; + goto out_dput; + } + + if ((fmode & FMODE_WRITE) && IS_IMMUTABLE(inode)) { + error = -EPERM; + goto out_dput; + } + + /* Can't write directories. */ + if (S_ISDIR(inode->i_mode) && (fmode & FMODE_WRITE)) { + error = -EISDIR; + goto out_dput; + } + + fd = get_unused_fd_flags(0); + if (fd < 0) { + error = fd; + goto out_dput; + } + + path.mnt = parfilp->f_path.mnt; + path.dentry = dentry; + filp = dentry_open(&path, hreq->oflags, cred); + dput(dentry); + if (IS_ERR(filp)) { + put_unused_fd(fd); + return PTR_ERR(filp); + } + + if (S_ISREG(inode->i_mode)) { + filp->f_flags |= O_NOATIME; + filp->f_mode |= FMODE_NOCMTIME; + } + + fd_install(fd, filp); + return fd; + + out_dput: + dput(dentry); + return error; +} + +int +xfs_readlink_by_handle( + struct file *parfilp, + xfs_fsop_handlereq_t *hreq) +{ + struct dentry *dentry; + __u32 olen; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + dentry = xfs_handlereq_to_dentry(parfilp, hreq); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + /* Restrict this handle operation to symlinks only. */ + if (!d_is_symlink(dentry)) { + error = -EINVAL; + goto out_dput; + } + + if (copy_from_user(&olen, hreq->ohandlen, sizeof(__u32))) { + error = -EFAULT; + goto out_dput; + } + + error = vfs_readlink(dentry, hreq->ohandle, olen); + + out_dput: + dput(dentry); + return error; +} + +/* + * Format an attribute and copy it out to the user's buffer. + * Take care to check values and protect against them changing later, + * we may be reading them directly out of a user buffer. + */ +static void +xfs_ioc_attr_put_listent( + struct xfs_attr_list_context *context, + int flags, + unsigned char *name, + int namelen, + void *value, + int valuelen) +{ + struct xfs_attrlist *alist = context->buffer; + struct xfs_attrlist_ent *aep; + int arraytop; + + ASSERT(!context->seen_enough); + ASSERT(context->count >= 0); + ASSERT(context->count < (ATTR_MAX_VALUELEN/8)); + ASSERT(context->firstu >= sizeof(*alist)); + ASSERT(context->firstu <= context->bufsize); + + /* + * Only list entries in the right namespace. + */ + if (context->attr_filter != (flags & XFS_ATTR_NSP_ONDISK_MASK)) + return; + + arraytop = sizeof(*alist) + + context->count * sizeof(alist->al_offset[0]); + + /* decrement by the actual bytes used by the attr */ + context->firstu -= round_up(offsetof(struct xfs_attrlist_ent, a_name) + + namelen + 1, sizeof(uint32_t)); + if (context->firstu < arraytop) { + trace_xfs_attr_list_full(context); + alist->al_more = 1; + context->seen_enough = 1; + return; + } + + aep = context->buffer + context->firstu; + aep->a_valuelen = valuelen; + memcpy(aep->a_name, name, namelen); + aep->a_name[namelen] = 0; + alist->al_offset[context->count++] = context->firstu; + alist->al_count = context->count; + trace_xfs_attr_list_add(context); +} + +static unsigned int +xfs_attr_filter( + u32 ioc_flags) +{ + if (ioc_flags & XFS_IOC_ATTR_ROOT) + return XFS_ATTR_ROOT; + if (ioc_flags & XFS_IOC_ATTR_SECURE) + return XFS_ATTR_SECURE; + return 0; +} + +static inline enum xfs_attr_update +xfs_xattr_flags( + u32 ioc_flags, + void *value) +{ + if (!value) + return XFS_ATTRUPDATE_REMOVE; + if (ioc_flags & XFS_IOC_ATTR_CREATE) + return XFS_ATTRUPDATE_CREATE; + if (ioc_flags & XFS_IOC_ATTR_REPLACE) + return XFS_ATTRUPDATE_REPLACE; + return XFS_ATTRUPDATE_UPSERT; +} + +int +xfs_ioc_attr_list( + struct xfs_inode *dp, + void __user *ubuf, + size_t bufsize, + int flags, + struct xfs_attrlist_cursor __user *ucursor) +{ + struct xfs_attr_list_context context = { }; + struct xfs_attrlist *alist; + void *buffer; + int error; + + if (bufsize < sizeof(struct xfs_attrlist) || + bufsize > XFS_XATTR_LIST_MAX) + return -EINVAL; + + /* + * Reject flags, only allow namespaces. + */ + if (flags & ~(XFS_IOC_ATTR_ROOT | XFS_IOC_ATTR_SECURE)) + return -EINVAL; + if (flags == (XFS_IOC_ATTR_ROOT | XFS_IOC_ATTR_SECURE)) + return -EINVAL; + + /* + * Validate the cursor. + */ + if (copy_from_user(&context.cursor, ucursor, sizeof(context.cursor))) + return -EFAULT; + if (context.cursor.pad1 || context.cursor.pad2) + return -EINVAL; + if (!context.cursor.initted && + (context.cursor.hashval || context.cursor.blkno || + context.cursor.offset)) + return -EINVAL; + + buffer = kvzalloc(bufsize, GFP_KERNEL); + if (!buffer) + return -ENOMEM; + + /* + * Initialize the output buffer. + */ + context.dp = dp; + context.resynch = 1; + context.attr_filter = xfs_attr_filter(flags); + context.buffer = buffer; + context.bufsize = round_down(bufsize, sizeof(uint32_t)); + context.firstu = context.bufsize; + context.put_listent = xfs_ioc_attr_put_listent; + + alist = context.buffer; + alist->al_count = 0; + alist->al_more = 0; + alist->al_offset[0] = context.bufsize; + + error = xfs_attr_list(&context); + if (error) + goto out_free; + + if (copy_to_user(ubuf, buffer, bufsize) || + copy_to_user(ucursor, &context.cursor, sizeof(context.cursor))) + error = -EFAULT; +out_free: + kvfree(buffer); + return error; +} + +int +xfs_attrlist_by_handle( + struct file *parfilp, + struct xfs_fsop_attrlist_handlereq __user *p) +{ + struct xfs_fsop_attrlist_handlereq al_hreq; + struct dentry *dentry; + int error = -ENOMEM; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (copy_from_user(&al_hreq, p, sizeof(al_hreq))) + return -EFAULT; + + dentry = xfs_handlereq_to_dentry(parfilp, &al_hreq.hreq); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + error = xfs_ioc_attr_list(XFS_I(d_inode(dentry)), al_hreq.buffer, + al_hreq.buflen, al_hreq.flags, &p->pos); + dput(dentry); + return error; +} + +static int +xfs_attrmulti_attr_get( + struct inode *inode, + unsigned char *name, + unsigned char __user *ubuf, + uint32_t *len, + uint32_t flags) +{ + struct xfs_da_args args = { + .dp = XFS_I(inode), + .attr_filter = xfs_attr_filter(flags), + .name = name, + .namelen = strlen(name), + .valuelen = *len, + }; + int error; + + if (*len > XFS_XATTR_SIZE_MAX) + return -EINVAL; + + error = xfs_attr_get(&args); + if (error) + goto out_kfree; + + *len = args.valuelen; + if (copy_to_user(ubuf, args.value, args.valuelen)) + error = -EFAULT; + +out_kfree: + kvfree(args.value); + return error; +} + +static int +xfs_attrmulti_attr_set( + struct inode *inode, + unsigned char *name, + const unsigned char __user *ubuf, + uint32_t len, + uint32_t flags) +{ + struct xfs_da_args args = { + .dp = XFS_I(inode), + .attr_filter = xfs_attr_filter(flags), + .name = name, + .namelen = strlen(name), + }; + int error; + + if (IS_IMMUTABLE(inode) || IS_APPEND(inode)) + return -EPERM; + + if (ubuf) { + if (len > XFS_XATTR_SIZE_MAX) + return -EINVAL; + args.value = memdup_user(ubuf, len); + if (IS_ERR(args.value)) + return PTR_ERR(args.value); + args.valuelen = len; + } + + error = xfs_attr_change(&args, xfs_xattr_flags(flags, args.value)); + if (!error && (flags & XFS_IOC_ATTR_ROOT)) + xfs_forget_acl(inode, name); + kfree(args.value); + return error; +} + +int +xfs_ioc_attrmulti_one( + struct file *parfilp, + struct inode *inode, + uint32_t opcode, + void __user *uname, + void __user *value, + uint32_t *len, + uint32_t flags) +{ + unsigned char *name; + int error; + + if ((flags & XFS_IOC_ATTR_ROOT) && (flags & XFS_IOC_ATTR_SECURE)) + return -EINVAL; + + name = strndup_user(uname, MAXNAMELEN); + if (IS_ERR(name)) + return PTR_ERR(name); + + switch (opcode) { + case ATTR_OP_GET: + error = xfs_attrmulti_attr_get(inode, name, value, len, flags); + break; + case ATTR_OP_REMOVE: + value = NULL; + *len = 0; + fallthrough; + case ATTR_OP_SET: + error = mnt_want_write_file(parfilp); + if (error) + break; + error = xfs_attrmulti_attr_set(inode, name, value, *len, flags); + mnt_drop_write_file(parfilp); + break; + default: + error = -EINVAL; + break; + } + + kfree(name); + return error; +} + +int +xfs_attrmulti_by_handle( + struct file *parfilp, + void __user *arg) +{ + int error; + xfs_attr_multiop_t *ops; + xfs_fsop_attrmulti_handlereq_t am_hreq; + struct dentry *dentry; + unsigned int i, size; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (copy_from_user(&am_hreq, arg, sizeof(xfs_fsop_attrmulti_handlereq_t))) + return -EFAULT; + + /* overflow check */ + if (am_hreq.opcount >= INT_MAX / sizeof(xfs_attr_multiop_t)) + return -E2BIG; + + dentry = xfs_handlereq_to_dentry(parfilp, &am_hreq.hreq); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + error = -E2BIG; + size = am_hreq.opcount * sizeof(xfs_attr_multiop_t); + if (!size || size > 16 * PAGE_SIZE) + goto out_dput; + + ops = memdup_user(am_hreq.ops, size); + if (IS_ERR(ops)) { + error = PTR_ERR(ops); + goto out_dput; + } + + error = 0; + for (i = 0; i < am_hreq.opcount; i++) { + ops[i].am_error = xfs_ioc_attrmulti_one(parfilp, + d_inode(dentry), ops[i].am_opcode, + ops[i].am_attrname, ops[i].am_attrvalue, + &ops[i].am_length, ops[i].am_flags); + } + + if (copy_to_user(am_hreq.ops, ops, size)) + error = -EFAULT; + + kfree(ops); + out_dput: + dput(dentry); + return error; +} diff --git a/fs/xfs/xfs_handle.h b/fs/xfs/xfs_handle.h new file mode 100644 index 0000000000000..e39eaf4689da9 --- /dev/null +++ b/fs/xfs/xfs_handle.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All rights reserved. + */ +#ifndef __XFS_HANDLE_H__ +#define __XFS_HANDLE_H__ + +int xfs_attrlist_by_handle(struct file *parfilp, + struct xfs_fsop_attrlist_handlereq __user *p); +int xfs_attrmulti_by_handle(struct file *parfilp, void __user *arg); + +int xfs_find_handle(unsigned int cmd, struct xfs_fsop_handlereq *hreq); +int xfs_open_by_handle(struct file *parfilp, struct xfs_fsop_handlereq *hreq); +int xfs_readlink_by_handle(struct file *parfilp, + struct xfs_fsop_handlereq *hreq); + +int xfs_ioc_attrmulti_one(struct file *parfilp, struct inode *inode, + uint32_t opcode, void __user *uname, void __user *value, + uint32_t *len, uint32_t flags); +int xfs_ioc_attr_list(struct xfs_inode *dp, void __user *ubuf, + size_t bufsize, int flags, + struct xfs_attrlist_cursor __user *ucursor); + +struct dentry *xfs_handle_to_dentry(struct file *parfilp, void __user *uhandle, + u32 hlen); + +#endif /* __XFS_HANDLE_H__ */ diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 7a2a5cf06a5cb..ed05fcd6261d8 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -23,11 +23,9 @@ #include "xfs_fsops.h" #include "xfs_discard.h" #include "xfs_quota.h" -#include "xfs_export.h" #include "xfs_trace.h" #include "xfs_icache.h" #include "xfs_trans.h" -#include "xfs_acl.h" #include "xfs_btree.h" #include #include "xfs_fsmap.h" @@ -37,602 +35,14 @@ #include "xfs_health.h" #include "xfs_reflink.h" #include "xfs_ioctl.h" -#include "xfs_xattr.h" #include "xfs_rtbitmap.h" #include "xfs_file.h" #include "xfs_exchrange.h" +#include "xfs_handle.h" #include -#include #include -/* - * xfs_find_handle maps from userspace xfs_fsop_handlereq structure to - * a file or fs handle. - * - * XFS_IOC_PATH_TO_FSHANDLE - * returns fs handle for a mount point or path within that mount point - * XFS_IOC_FD_TO_HANDLE - * returns full handle for a FD opened in user space - * XFS_IOC_PATH_TO_HANDLE - * returns full handle for a path - */ -int -xfs_find_handle( - unsigned int cmd, - xfs_fsop_handlereq_t *hreq) -{ - int hsize; - xfs_handle_t handle; - struct inode *inode; - struct fd f = {NULL}; - struct path path; - int error; - struct xfs_inode *ip; - - if (cmd == XFS_IOC_FD_TO_HANDLE) { - f = fdget(hreq->fd); - if (!f.file) - return -EBADF; - inode = file_inode(f.file); - } else { - error = user_path_at(AT_FDCWD, hreq->path, 0, &path); - if (error) - return error; - inode = d_inode(path.dentry); - } - ip = XFS_I(inode); - - /* - * We can only generate handles for inodes residing on a XFS filesystem, - * and only for regular files, directories or symbolic links. - */ - error = -EINVAL; - if (inode->i_sb->s_magic != XFS_SB_MAGIC) - goto out_put; - - error = -EBADF; - if (!S_ISREG(inode->i_mode) && - !S_ISDIR(inode->i_mode) && - !S_ISLNK(inode->i_mode)) - goto out_put; - - - memcpy(&handle.ha_fsid, ip->i_mount->m_fixedfsid, sizeof(xfs_fsid_t)); - - if (cmd == XFS_IOC_PATH_TO_FSHANDLE) { - /* - * This handle only contains an fsid, zero the rest. - */ - memset(&handle.ha_fid, 0, sizeof(handle.ha_fid)); - hsize = sizeof(xfs_fsid_t); - } else { - handle.ha_fid.fid_len = sizeof(xfs_fid_t) - - sizeof(handle.ha_fid.fid_len); - handle.ha_fid.fid_pad = 0; - handle.ha_fid.fid_gen = inode->i_generation; - handle.ha_fid.fid_ino = ip->i_ino; - hsize = sizeof(xfs_handle_t); - } - - error = -EFAULT; - if (copy_to_user(hreq->ohandle, &handle, hsize) || - copy_to_user(hreq->ohandlen, &hsize, sizeof(__s32))) - goto out_put; - - error = 0; - - out_put: - if (cmd == XFS_IOC_FD_TO_HANDLE) - fdput(f); - else - path_put(&path); - return error; -} - -/* - * No need to do permission checks on the various pathname components - * as the handle operations are privileged. - */ -STATIC int -xfs_handle_acceptable( - void *context, - struct dentry *dentry) -{ - return 1; -} - -/* - * Convert userspace handle data into a dentry. - */ -struct dentry * -xfs_handle_to_dentry( - struct file *parfilp, - void __user *uhandle, - u32 hlen) -{ - xfs_handle_t handle; - struct xfs_fid64 fid; - - /* - * Only allow handle opens under a directory. - */ - if (!S_ISDIR(file_inode(parfilp)->i_mode)) - return ERR_PTR(-ENOTDIR); - - if (hlen != sizeof(xfs_handle_t)) - return ERR_PTR(-EINVAL); - if (copy_from_user(&handle, uhandle, hlen)) - return ERR_PTR(-EFAULT); - if (handle.ha_fid.fid_len != - sizeof(handle.ha_fid) - sizeof(handle.ha_fid.fid_len)) - return ERR_PTR(-EINVAL); - - memset(&fid, 0, sizeof(struct fid)); - fid.ino = handle.ha_fid.fid_ino; - fid.gen = handle.ha_fid.fid_gen; - - return exportfs_decode_fh(parfilp->f_path.mnt, (struct fid *)&fid, 3, - FILEID_INO32_GEN | XFS_FILEID_TYPE_64FLAG, - xfs_handle_acceptable, NULL); -} - -STATIC struct dentry * -xfs_handlereq_to_dentry( - struct file *parfilp, - xfs_fsop_handlereq_t *hreq) -{ - return xfs_handle_to_dentry(parfilp, hreq->ihandle, hreq->ihandlen); -} - -int -xfs_open_by_handle( - struct file *parfilp, - xfs_fsop_handlereq_t *hreq) -{ - const struct cred *cred = current_cred(); - int error; - int fd; - int permflag; - struct file *filp; - struct inode *inode; - struct dentry *dentry; - fmode_t fmode; - struct path path; - - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; - - dentry = xfs_handlereq_to_dentry(parfilp, hreq); - if (IS_ERR(dentry)) - return PTR_ERR(dentry); - inode = d_inode(dentry); - - /* Restrict xfs_open_by_handle to directories & regular files. */ - if (!(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode))) { - error = -EPERM; - goto out_dput; - } - -#if BITS_PER_LONG != 32 - hreq->oflags |= O_LARGEFILE; -#endif - - permflag = hreq->oflags; - fmode = OPEN_FMODE(permflag); - if ((!(permflag & O_APPEND) || (permflag & O_TRUNC)) && - (fmode & FMODE_WRITE) && IS_APPEND(inode)) { - error = -EPERM; - goto out_dput; - } - - if ((fmode & FMODE_WRITE) && IS_IMMUTABLE(inode)) { - error = -EPERM; - goto out_dput; - } - - /* Can't write directories. */ - if (S_ISDIR(inode->i_mode) && (fmode & FMODE_WRITE)) { - error = -EISDIR; - goto out_dput; - } - - fd = get_unused_fd_flags(0); - if (fd < 0) { - error = fd; - goto out_dput; - } - - path.mnt = parfilp->f_path.mnt; - path.dentry = dentry; - filp = dentry_open(&path, hreq->oflags, cred); - dput(dentry); - if (IS_ERR(filp)) { - put_unused_fd(fd); - return PTR_ERR(filp); - } - - if (S_ISREG(inode->i_mode)) { - filp->f_flags |= O_NOATIME; - filp->f_mode |= FMODE_NOCMTIME; - } - - fd_install(fd, filp); - return fd; - - out_dput: - dput(dentry); - return error; -} - -int -xfs_readlink_by_handle( - struct file *parfilp, - xfs_fsop_handlereq_t *hreq) -{ - struct dentry *dentry; - __u32 olen; - int error; - - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; - - dentry = xfs_handlereq_to_dentry(parfilp, hreq); - if (IS_ERR(dentry)) - return PTR_ERR(dentry); - - /* Restrict this handle operation to symlinks only. */ - if (!d_is_symlink(dentry)) { - error = -EINVAL; - goto out_dput; - } - - if (copy_from_user(&olen, hreq->ohandlen, sizeof(__u32))) { - error = -EFAULT; - goto out_dput; - } - - error = vfs_readlink(dentry, hreq->ohandle, olen); - - out_dput: - dput(dentry); - return error; -} - -/* - * Format an attribute and copy it out to the user's buffer. - * Take care to check values and protect against them changing later, - * we may be reading them directly out of a user buffer. - */ -static void -xfs_ioc_attr_put_listent( - struct xfs_attr_list_context *context, - int flags, - unsigned char *name, - int namelen, - void *value, - int valuelen) -{ - struct xfs_attrlist *alist = context->buffer; - struct xfs_attrlist_ent *aep; - int arraytop; - - ASSERT(!context->seen_enough); - ASSERT(context->count >= 0); - ASSERT(context->count < (ATTR_MAX_VALUELEN/8)); - ASSERT(context->firstu >= sizeof(*alist)); - ASSERT(context->firstu <= context->bufsize); - - /* - * Only list entries in the right namespace. - */ - if (context->attr_filter != (flags & XFS_ATTR_NSP_ONDISK_MASK)) - return; - - arraytop = sizeof(*alist) + - context->count * sizeof(alist->al_offset[0]); - - /* decrement by the actual bytes used by the attr */ - context->firstu -= round_up(offsetof(struct xfs_attrlist_ent, a_name) + - namelen + 1, sizeof(uint32_t)); - if (context->firstu < arraytop) { - trace_xfs_attr_list_full(context); - alist->al_more = 1; - context->seen_enough = 1; - return; - } - - aep = context->buffer + context->firstu; - aep->a_valuelen = valuelen; - memcpy(aep->a_name, name, namelen); - aep->a_name[namelen] = 0; - alist->al_offset[context->count++] = context->firstu; - alist->al_count = context->count; - trace_xfs_attr_list_add(context); -} - -static unsigned int -xfs_attr_filter( - u32 ioc_flags) -{ - if (ioc_flags & XFS_IOC_ATTR_ROOT) - return XFS_ATTR_ROOT; - if (ioc_flags & XFS_IOC_ATTR_SECURE) - return XFS_ATTR_SECURE; - return 0; -} - -static inline enum xfs_attr_update -xfs_xattr_flags( - u32 ioc_flags, - void *value) -{ - if (!value) - return XFS_ATTRUPDATE_REMOVE; - if (ioc_flags & XFS_IOC_ATTR_CREATE) - return XFS_ATTRUPDATE_CREATE; - if (ioc_flags & XFS_IOC_ATTR_REPLACE) - return XFS_ATTRUPDATE_REPLACE; - return XFS_ATTRUPDATE_UPSERT; -} - -int -xfs_ioc_attr_list( - struct xfs_inode *dp, - void __user *ubuf, - size_t bufsize, - int flags, - struct xfs_attrlist_cursor __user *ucursor) -{ - struct xfs_attr_list_context context = { }; - struct xfs_attrlist *alist; - void *buffer; - int error; - - if (bufsize < sizeof(struct xfs_attrlist) || - bufsize > XFS_XATTR_LIST_MAX) - return -EINVAL; - - /* - * Reject flags, only allow namespaces. - */ - if (flags & ~(XFS_IOC_ATTR_ROOT | XFS_IOC_ATTR_SECURE)) - return -EINVAL; - if (flags == (XFS_IOC_ATTR_ROOT | XFS_IOC_ATTR_SECURE)) - return -EINVAL; - - /* - * Validate the cursor. - */ - if (copy_from_user(&context.cursor, ucursor, sizeof(context.cursor))) - return -EFAULT; - if (context.cursor.pad1 || context.cursor.pad2) - return -EINVAL; - if (!context.cursor.initted && - (context.cursor.hashval || context.cursor.blkno || - context.cursor.offset)) - return -EINVAL; - - buffer = kvzalloc(bufsize, GFP_KERNEL); - if (!buffer) - return -ENOMEM; - - /* - * Initialize the output buffer. - */ - context.dp = dp; - context.resynch = 1; - context.attr_filter = xfs_attr_filter(flags); - context.buffer = buffer; - context.bufsize = round_down(bufsize, sizeof(uint32_t)); - context.firstu = context.bufsize; - context.put_listent = xfs_ioc_attr_put_listent; - - alist = context.buffer; - alist->al_count = 0; - alist->al_more = 0; - alist->al_offset[0] = context.bufsize; - - error = xfs_attr_list(&context); - if (error) - goto out_free; - - if (copy_to_user(ubuf, buffer, bufsize) || - copy_to_user(ucursor, &context.cursor, sizeof(context.cursor))) - error = -EFAULT; -out_free: - kvfree(buffer); - return error; -} - -STATIC int -xfs_attrlist_by_handle( - struct file *parfilp, - struct xfs_fsop_attrlist_handlereq __user *p) -{ - struct xfs_fsop_attrlist_handlereq al_hreq; - struct dentry *dentry; - int error = -ENOMEM; - - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; - if (copy_from_user(&al_hreq, p, sizeof(al_hreq))) - return -EFAULT; - - dentry = xfs_handlereq_to_dentry(parfilp, &al_hreq.hreq); - if (IS_ERR(dentry)) - return PTR_ERR(dentry); - - error = xfs_ioc_attr_list(XFS_I(d_inode(dentry)), al_hreq.buffer, - al_hreq.buflen, al_hreq.flags, &p->pos); - dput(dentry); - return error; -} - -static int -xfs_attrmulti_attr_get( - struct inode *inode, - unsigned char *name, - unsigned char __user *ubuf, - uint32_t *len, - uint32_t flags) -{ - struct xfs_da_args args = { - .dp = XFS_I(inode), - .attr_filter = xfs_attr_filter(flags), - .name = name, - .namelen = strlen(name), - .valuelen = *len, - }; - int error; - - if (*len > XFS_XATTR_SIZE_MAX) - return -EINVAL; - - error = xfs_attr_get(&args); - if (error) - goto out_kfree; - - *len = args.valuelen; - if (copy_to_user(ubuf, args.value, args.valuelen)) - error = -EFAULT; - -out_kfree: - kvfree(args.value); - return error; -} - -static int -xfs_attrmulti_attr_set( - struct inode *inode, - unsigned char *name, - const unsigned char __user *ubuf, - uint32_t len, - uint32_t flags) -{ - struct xfs_da_args args = { - .dp = XFS_I(inode), - .attr_filter = xfs_attr_filter(flags), - .name = name, - .namelen = strlen(name), - }; - int error; - - if (IS_IMMUTABLE(inode) || IS_APPEND(inode)) - return -EPERM; - - if (ubuf) { - if (len > XFS_XATTR_SIZE_MAX) - return -EINVAL; - args.value = memdup_user(ubuf, len); - if (IS_ERR(args.value)) - return PTR_ERR(args.value); - args.valuelen = len; - } - - error = xfs_attr_change(&args, xfs_xattr_flags(flags, args.value)); - if (!error && (flags & XFS_IOC_ATTR_ROOT)) - xfs_forget_acl(inode, name); - kfree(args.value); - return error; -} - -int -xfs_ioc_attrmulti_one( - struct file *parfilp, - struct inode *inode, - uint32_t opcode, - void __user *uname, - void __user *value, - uint32_t *len, - uint32_t flags) -{ - unsigned char *name; - int error; - - if ((flags & XFS_IOC_ATTR_ROOT) && (flags & XFS_IOC_ATTR_SECURE)) - return -EINVAL; - - name = strndup_user(uname, MAXNAMELEN); - if (IS_ERR(name)) - return PTR_ERR(name); - - switch (opcode) { - case ATTR_OP_GET: - error = xfs_attrmulti_attr_get(inode, name, value, len, flags); - break; - case ATTR_OP_REMOVE: - value = NULL; - *len = 0; - fallthrough; - case ATTR_OP_SET: - error = mnt_want_write_file(parfilp); - if (error) - break; - error = xfs_attrmulti_attr_set(inode, name, value, *len, flags); - mnt_drop_write_file(parfilp); - break; - default: - error = -EINVAL; - break; - } - - kfree(name); - return error; -} - -STATIC int -xfs_attrmulti_by_handle( - struct file *parfilp, - void __user *arg) -{ - int error; - xfs_attr_multiop_t *ops; - xfs_fsop_attrmulti_handlereq_t am_hreq; - struct dentry *dentry; - unsigned int i, size; - - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; - if (copy_from_user(&am_hreq, arg, sizeof(xfs_fsop_attrmulti_handlereq_t))) - return -EFAULT; - - /* overflow check */ - if (am_hreq.opcount >= INT_MAX / sizeof(xfs_attr_multiop_t)) - return -E2BIG; - - dentry = xfs_handlereq_to_dentry(parfilp, &am_hreq.hreq); - if (IS_ERR(dentry)) - return PTR_ERR(dentry); - - error = -E2BIG; - size = am_hreq.opcount * sizeof(xfs_attr_multiop_t); - if (!size || size > 16 * PAGE_SIZE) - goto out_dput; - - ops = memdup_user(am_hreq.ops, size); - if (IS_ERR(ops)) { - error = PTR_ERR(ops); - goto out_dput; - } - - error = 0; - for (i = 0; i < am_hreq.opcount; i++) { - ops[i].am_error = xfs_ioc_attrmulti_one(parfilp, - d_inode(dentry), ops[i].am_opcode, - ops[i].am_attrname, ops[i].am_attrvalue, - &ops[i].am_length, ops[i].am_flags); - } - - if (copy_to_user(am_hreq.ops, ops, size)) - error = -EFAULT; - - kfree(ops); - out_dput: - dput(dentry); - return error; -} - /* Return 0 on success or positive error */ int xfs_fsbulkstat_one_fmt( diff --git a/fs/xfs/xfs_ioctl.h b/fs/xfs/xfs_ioctl.h index 38be600b5e1e8..12124946f347e 100644 --- a/fs/xfs/xfs_ioctl.h +++ b/fs/xfs/xfs_ioctl.h @@ -14,34 +14,6 @@ int xfs_ioc_swapext( xfs_swapext_t *sxp); -extern int -xfs_find_handle( - unsigned int cmd, - xfs_fsop_handlereq_t *hreq); - -extern int -xfs_open_by_handle( - struct file *parfilp, - xfs_fsop_handlereq_t *hreq); - -extern int -xfs_readlink_by_handle( - struct file *parfilp, - xfs_fsop_handlereq_t *hreq); - -int xfs_ioc_attrmulti_one(struct file *parfilp, struct inode *inode, - uint32_t opcode, void __user *uname, void __user *value, - uint32_t *len, uint32_t flags); -int xfs_ioc_attr_list(struct xfs_inode *dp, void __user *ubuf, - size_t bufsize, int flags, - struct xfs_attrlist_cursor __user *ucursor); - -extern struct dentry * -xfs_handle_to_dentry( - struct file *parfilp, - void __user *uhandle, - u32 hlen); - extern int xfs_fileattr_get( struct dentry *dentry, diff --git a/fs/xfs/xfs_ioctl32.c b/fs/xfs/xfs_ioctl32.c index ee35eea1ecce6..b64785dc4354e 100644 --- a/fs/xfs/xfs_ioctl32.c +++ b/fs/xfs/xfs_ioctl32.c @@ -24,6 +24,7 @@ #include "xfs_ioctl32.h" #include "xfs_trace.h" #include "xfs_sb.h" +#include "xfs_handle.h" #define _NATIVE_IOC(cmd, type) \ _IOC(_IOC_DIR(cmd), _IOC_TYPE(cmd), _IOC_NR(cmd), sizeof(type)) From b8c9d4253da43c02b287831f7e576568f24fbe58 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:55 -0700 Subject: [PATCH 0447/1702] xfs: split out handle management helpers a bit Split out the functions that generate file/fs handles and map them back into dentries in preparation for the GETPARENTS ioctl next. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_fs.h | 4 +- fs/xfs/xfs_handle.c | 98 +++++++++++++++++++++++++++++------------- 2 files changed, 70 insertions(+), 32 deletions(-) diff --git a/fs/xfs/libxfs/xfs_fs.h b/fs/xfs/libxfs/xfs_fs.h index 53526fca7386c..97384ab95de43 100644 --- a/fs/xfs/libxfs/xfs_fs.h +++ b/fs/xfs/libxfs/xfs_fs.h @@ -633,7 +633,9 @@ typedef struct xfs_fsop_attrmulti_handlereq { /* * per machine unique filesystem identifier types. */ -typedef struct { __u32 val[2]; } xfs_fsid_t; /* file system id type */ +typedef struct xfs_fsid { + __u32 val[2]; /* file system id type */ +} xfs_fsid_t; typedef struct xfs_fid { __u16 fid_len; /* length of remainder */ diff --git a/fs/xfs/xfs_handle.c b/fs/xfs/xfs_handle.c index 13c2479a3053f..b9f4d9860682a 100644 --- a/fs/xfs/xfs_handle.c +++ b/fs/xfs/xfs_handle.c @@ -30,6 +30,42 @@ #include +static inline size_t +xfs_filehandle_fid_len(void) +{ + struct xfs_handle *handle = NULL; + + return sizeof(struct xfs_fid) - sizeof(handle->ha_fid.fid_len); +} + +static inline size_t +xfs_filehandle_init( + struct xfs_mount *mp, + xfs_ino_t ino, + uint32_t gen, + struct xfs_handle *handle) +{ + memcpy(&handle->ha_fsid, mp->m_fixedfsid, sizeof(struct xfs_fsid)); + + handle->ha_fid.fid_len = xfs_filehandle_fid_len(); + handle->ha_fid.fid_pad = 0; + handle->ha_fid.fid_gen = gen; + handle->ha_fid.fid_ino = ino; + + return sizeof(struct xfs_handle); +} + +static inline size_t +xfs_fshandle_init( + struct xfs_mount *mp, + struct xfs_handle *handle) +{ + memcpy(&handle->ha_fsid, mp->m_fixedfsid, sizeof(struct xfs_fsid)); + memset(&handle->ha_fid, 0, sizeof(handle->ha_fid)); + + return sizeof(struct xfs_fsid); +} + /* * xfs_find_handle maps from userspace xfs_fsop_handlereq structure to * a file or fs handle. @@ -84,20 +120,11 @@ xfs_find_handle( memcpy(&handle.ha_fsid, ip->i_mount->m_fixedfsid, sizeof(xfs_fsid_t)); - if (cmd == XFS_IOC_PATH_TO_FSHANDLE) { - /* - * This handle only contains an fsid, zero the rest. - */ - memset(&handle.ha_fid, 0, sizeof(handle.ha_fid)); - hsize = sizeof(xfs_fsid_t); - } else { - handle.ha_fid.fid_len = sizeof(xfs_fid_t) - - sizeof(handle.ha_fid.fid_len); - handle.ha_fid.fid_pad = 0; - handle.ha_fid.fid_gen = inode->i_generation; - handle.ha_fid.fid_ino = ip->i_ino; - hsize = sizeof(xfs_handle_t); - } + if (cmd == XFS_IOC_PATH_TO_FSHANDLE) + hsize = xfs_fshandle_init(ip->i_mount, &handle); + else + hsize = xfs_filehandle_init(ip->i_mount, ip->i_ino, + inode->i_generation, &handle); error = -EFAULT; if (copy_to_user(hreq->ohandle, &handle, hsize) || @@ -126,6 +153,31 @@ xfs_handle_acceptable( return 1; } +/* Convert handle already copied to kernel space into a dentry. */ +static struct dentry * +xfs_khandle_to_dentry( + struct file *file, + struct xfs_handle *handle) +{ + struct xfs_fid64 fid = { + .ino = handle->ha_fid.fid_ino, + .gen = handle->ha_fid.fid_gen, + }; + + /* + * Only allow handle opens under a directory. + */ + if (!S_ISDIR(file_inode(file)->i_mode)) + return ERR_PTR(-ENOTDIR); + + if (handle->ha_fid.fid_len != xfs_filehandle_fid_len()) + return ERR_PTR(-EINVAL); + + return exportfs_decode_fh(file->f_path.mnt, (struct fid *)&fid, 3, + FILEID_INO32_GEN | XFS_FILEID_TYPE_64FLAG, + xfs_handle_acceptable, NULL); +} + /* * Convert userspace handle data into a dentry. */ @@ -136,29 +188,13 @@ xfs_handle_to_dentry( u32 hlen) { xfs_handle_t handle; - struct xfs_fid64 fid; - - /* - * Only allow handle opens under a directory. - */ - if (!S_ISDIR(file_inode(parfilp)->i_mode)) - return ERR_PTR(-ENOTDIR); if (hlen != sizeof(xfs_handle_t)) return ERR_PTR(-EINVAL); if (copy_from_user(&handle, uhandle, hlen)) return ERR_PTR(-EFAULT); - if (handle.ha_fid.fid_len != - sizeof(handle.ha_fid) - sizeof(handle.ha_fid.fid_len)) - return ERR_PTR(-EINVAL); - - memset(&fid, 0, sizeof(struct fid)); - fid.ino = handle.ha_fid.fid_ino; - fid.gen = handle.ha_fid.fid_gen; - return exportfs_decode_fh(parfilp->f_path.mnt, (struct fid *)&fid, 3, - FILEID_INO32_GEN | XFS_FILEID_TYPE_64FLAG, - xfs_handle_acceptable, NULL); + return xfs_khandle_to_dentry(parfilp, &handle); } STATIC struct dentry * From 233f4e12bbb2c5fb1588b857336a26e8bb6942af Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:55 -0700 Subject: [PATCH 0448/1702] xfs: add parent pointer ioctls This patch adds a pair of new file ioctls to retrieve the parent pointer of a given inode. They both return the same results, but one operates on the file descriptor passed to ioctl() whereas the other allows the caller to specify a file handle for which the caller wants results. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_fs.h | 74 +++++++++ fs/xfs/libxfs/xfs_ondisk.h | 5 + fs/xfs/libxfs/xfs_parent.c | 34 +++++ fs/xfs/libxfs/xfs_parent.h | 5 + fs/xfs/xfs_export.c | 2 +- fs/xfs/xfs_export.h | 2 + fs/xfs/xfs_handle.c | 298 +++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_handle.h | 5 + fs/xfs/xfs_ioctl.c | 6 +- fs/xfs/xfs_trace.c | 1 + fs/xfs/xfs_trace.h | 92 ++++++++++++ 11 files changed, 522 insertions(+), 2 deletions(-) diff --git a/fs/xfs/libxfs/xfs_fs.h b/fs/xfs/libxfs/xfs_fs.h index 97384ab95de43..ea654df0505f2 100644 --- a/fs/xfs/libxfs/xfs_fs.h +++ b/fs/xfs/libxfs/xfs_fs.h @@ -816,6 +816,78 @@ struct xfs_exchange_range { XFS_EXCHANGE_RANGE_DRY_RUN | \ XFS_EXCHANGE_RANGE_FILE1_WRITTEN) +/* Iterating parent pointers of files. */ + +/* target was the root directory */ +#define XFS_GETPARENTS_OFLAG_ROOT (1U << 0) + +/* Cursor is done iterating pptrs */ +#define XFS_GETPARENTS_OFLAG_DONE (1U << 1) + +#define XFS_GETPARENTS_OFLAGS_ALL (XFS_GETPARENTS_OFLAG_ROOT | \ + XFS_GETPARENTS_OFLAG_DONE) + +#define XFS_GETPARENTS_IFLAGS_ALL (0) + +struct xfs_getparents_rec { + struct xfs_handle gpr_parent; /* Handle to parent */ + __u32 gpr_reclen; /* Length of entire record */ + __u32 gpr_reserved; /* zero */ + char gpr_name[]; /* Null-terminated filename */ +}; + +/* Iterate through this file's directory parent pointers */ +struct xfs_getparents { + /* + * Structure to track progress in iterating the parent pointers. + * Must be initialized to zeroes before the first ioctl call, and + * not touched by callers after that. + */ + struct xfs_attrlist_cursor gp_cursor; + + /* Input flags: XFS_GETPARENTS_IFLAG* */ + __u16 gp_iflags; + + /* Output flags: XFS_GETPARENTS_OFLAG* */ + __u16 gp_oflags; + + /* Size of the gp_buffer in bytes */ + __u32 gp_bufsize; + + /* Must be set to zero */ + __u64 gp_reserved; + + /* Pointer to a buffer in which to place xfs_getparents_rec */ + __u64 gp_buffer; +}; + +static inline struct xfs_getparents_rec * +xfs_getparents_first_rec(struct xfs_getparents *gp) +{ + return (struct xfs_getparents_rec *)(uintptr_t)gp->gp_buffer; +} + +static inline struct xfs_getparents_rec * +xfs_getparents_next_rec(struct xfs_getparents *gp, + struct xfs_getparents_rec *gpr) +{ + void *next = ((void *)gpr + gpr->gpr_reclen); + void *end = (void *)(uintptr_t)(gp->gp_buffer + gp->gp_bufsize); + + if (next >= end) + return NULL; + + return next; +} + +/* Iterate through this file handle's directory parent pointers. */ +struct xfs_getparents_by_handle { + /* Handle to file whose parents we want. */ + struct xfs_handle gph_handle; + + struct xfs_getparents gph_request; +}; + /* * ioctl commands that are used by Linux filesystems */ @@ -851,6 +923,8 @@ struct xfs_exchange_range { /* XFS_IOC_GETFSMAP ------ hoisted 59 */ #define XFS_IOC_SCRUB_METADATA _IOWR('X', 60, struct xfs_scrub_metadata) #define XFS_IOC_AG_GEOMETRY _IOWR('X', 61, struct xfs_ag_geometry) +#define XFS_IOC_GETPARENTS _IOWR('X', 62, struct xfs_getparents) +#define XFS_IOC_GETPARENTS_BY_HANDLE _IOWR('X', 63, struct xfs_getparents_by_handle) /* * ioctl commands that replace IRIX syssgi()'s diff --git a/fs/xfs/libxfs/xfs_ondisk.h b/fs/xfs/libxfs/xfs_ondisk.h index 25952ef584eee..e8cdd77d03fa8 100644 --- a/fs/xfs/libxfs/xfs_ondisk.h +++ b/fs/xfs/libxfs/xfs_ondisk.h @@ -156,6 +156,11 @@ xfs_check_ondisk_structs(void) XFS_CHECK_OFFSET(struct xfs_efi_log_format_32, efi_extents, 16); XFS_CHECK_OFFSET(struct xfs_efi_log_format_64, efi_extents, 16); + /* parent pointer ioctls */ + XFS_CHECK_STRUCT_SIZE(struct xfs_getparents_rec, 32); + XFS_CHECK_STRUCT_SIZE(struct xfs_getparents, 40); + XFS_CHECK_STRUCT_SIZE(struct xfs_getparents_by_handle, 64); + /* * The v5 superblock format extended several v4 header structures with * additional data. While new fields are only accessible on v5 diff --git a/fs/xfs/libxfs/xfs_parent.c b/fs/xfs/libxfs/xfs_parent.c index fdf643bfde4df..504de1ef33876 100644 --- a/fs/xfs/libxfs/xfs_parent.c +++ b/fs/xfs/libxfs/xfs_parent.c @@ -257,3 +257,37 @@ xfs_parent_replacename( xfs_attr_defer_add(&ppargs->args, XFS_ATTR_DEFER_REPLACE); return 0; } + +/* + * Extract parent pointer information from any parent pointer xattr into + * @parent_ino/gen. The last two parameters can be NULL pointers. + * + * Returns 0 if this is not a parent pointer xattr at all; or -EFSCORRUPTED for + * garbage. + */ +int +xfs_parent_from_attr( + struct xfs_mount *mp, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + xfs_ino_t *parent_ino, + uint32_t *parent_gen) +{ + const struct xfs_parent_rec *rec = value; + + ASSERT(attr_flags & XFS_ATTR_PARENT); + + if (!xfs_parent_namecheck(attr_flags, name, namelen)) + return -EFSCORRUPTED; + if (!xfs_parent_valuecheck(mp, value, valuelen)) + return -EFSCORRUPTED; + + if (parent_ino) + *parent_ino = be64_to_cpu(rec->p_ino); + if (parent_gen) + *parent_gen = be32_to_cpu(rec->p_gen); + return 0; +} diff --git a/fs/xfs/libxfs/xfs_parent.h b/fs/xfs/libxfs/xfs_parent.h index 768633b313671..d7ab09e738ad4 100644 --- a/fs/xfs/libxfs/xfs_parent.h +++ b/fs/xfs/libxfs/xfs_parent.h @@ -91,4 +91,9 @@ int xfs_parent_replacename(struct xfs_trans *tp, struct xfs_inode *new_dp, const struct xfs_name *new_name, struct xfs_inode *child); +int xfs_parent_from_attr(struct xfs_mount *mp, unsigned int attr_flags, + const unsigned char *name, unsigned int namelen, + const void *value, unsigned int valuelen, + xfs_ino_t *parent_ino, uint32_t *parent_gen); + #endif /* __XFS_PARENT_H__ */ diff --git a/fs/xfs/xfs_export.c b/fs/xfs/xfs_export.c index 4b03221351c0f..201489d3de089 100644 --- a/fs/xfs/xfs_export.c +++ b/fs/xfs/xfs_export.c @@ -102,7 +102,7 @@ xfs_fs_encode_fh( return fileid_type; } -STATIC struct inode * +struct inode * xfs_nfs_get_inode( struct super_block *sb, u64 ino, diff --git a/fs/xfs/xfs_export.h b/fs/xfs/xfs_export.h index 64471a3ddb04d..3cd85e8901a5f 100644 --- a/fs/xfs/xfs_export.h +++ b/fs/xfs/xfs_export.h @@ -57,4 +57,6 @@ struct xfs_fid64 { /* This flag goes on the wire. Don't play with it. */ #define XFS_FILEID_TYPE_64FLAG 0x80 /* NFS fileid has 64bit inodes */ +struct inode *xfs_nfs_get_inode(struct super_block *sb, u64 ino, u32 gen); + #endif /* __XFS_EXPORT_H__ */ diff --git a/fs/xfs/xfs_handle.c b/fs/xfs/xfs_handle.c index b9f4d9860682a..c8785ed595434 100644 --- a/fs/xfs/xfs_handle.c +++ b/fs/xfs/xfs_handle.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * Copyright (c) 2022-2024 Oracle. * All rights reserved. */ #include "xfs.h" @@ -178,6 +179,30 @@ xfs_khandle_to_dentry( xfs_handle_acceptable, NULL); } +/* Convert handle already copied to kernel space into an xfs_inode. */ +static struct xfs_inode * +xfs_khandle_to_inode( + struct file *file, + struct xfs_handle *handle) +{ + struct xfs_inode *ip = XFS_I(file_inode(file)); + struct xfs_mount *mp = ip->i_mount; + struct inode *inode; + + if (!S_ISDIR(VFS_I(ip)->i_mode)) + return ERR_PTR(-ENOTDIR); + + if (handle->ha_fid.fid_len != xfs_filehandle_fid_len()) + return ERR_PTR(-EINVAL); + + inode = xfs_nfs_get_inode(mp->m_super, handle->ha_fid.fid_ino, + handle->ha_fid.fid_gen); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + return XFS_I(inode); +} + /* * Convert userspace handle data into a dentry. */ @@ -652,3 +677,276 @@ xfs_attrmulti_by_handle( dput(dentry); return error; } + +struct xfs_getparents_ctx { + struct xfs_attr_list_context context; + struct xfs_getparents_by_handle gph; + + /* File to target */ + struct xfs_inode *ip; + + /* Internal buffer where we format records */ + void *krecords; + + /* Last record filled out */ + struct xfs_getparents_rec *lastrec; + + unsigned int count; +}; + +static inline unsigned int +xfs_getparents_rec_sizeof( + unsigned int namelen) +{ + return round_up(sizeof(struct xfs_getparents_rec) + namelen + 1, + sizeof(uint64_t)); +} + +static void +xfs_getparents_put_listent( + struct xfs_attr_list_context *context, + int flags, + unsigned char *name, + int namelen, + void *value, + int valuelen) +{ + struct xfs_getparents_ctx *gpx = + container_of(context, struct xfs_getparents_ctx, context); + struct xfs_inode *ip = context->dp; + struct xfs_mount *mp = ip->i_mount; + struct xfs_getparents *gp = &gpx->gph.gph_request; + struct xfs_getparents_rec *gpr = gpx->krecords + context->firstu; + unsigned short reclen = + xfs_getparents_rec_sizeof(namelen); + xfs_ino_t ino; + uint32_t gen; + int error; + + if (!(flags & XFS_ATTR_PARENT)) + return; + + error = xfs_parent_from_attr(mp, flags, name, namelen, value, valuelen, + &ino, &gen); + if (error) { + xfs_inode_mark_sick(ip, XFS_SICK_INO_PARENT); + context->seen_enough = -EFSCORRUPTED; + return; + } + + /* + * We found a parent pointer, but we've filled up the buffer. Signal + * to the caller that we did /not/ reach the end of the parent pointer + * recordset. + */ + if (context->firstu > context->bufsize - reclen) { + context->seen_enough = 1; + return; + } + + /* Format the parent pointer directly into the caller buffer. */ + gpr->gpr_reclen = reclen; + xfs_filehandle_init(mp, ino, gen, &gpr->gpr_parent); + memcpy(gpr->gpr_name, name, namelen); + gpr->gpr_name[namelen] = 0; + + trace_xfs_getparents_put_listent(ip, gp, context, gpr); + + context->firstu += reclen; + gpx->count++; + gpx->lastrec = gpr; +} + +/* Expand the last record to fill the rest of the caller's buffer. */ +static inline void +xfs_getparents_expand_lastrec( + struct xfs_getparents_ctx *gpx) +{ + struct xfs_getparents *gp = &gpx->gph.gph_request; + struct xfs_getparents_rec *gpr = gpx->lastrec; + + if (!gpx->lastrec) + gpr = gpx->krecords; + + gpr->gpr_reclen = gp->gp_bufsize - ((void *)gpr - gpx->krecords); + + trace_xfs_getparents_expand_lastrec(gpx->ip, gp, &gpx->context, gpr); +} + +static inline void __user *u64_to_uptr(u64 val) +{ + return (void __user *)(uintptr_t)val; +} + +/* Retrieve the parent pointers for a given inode. */ +STATIC int +xfs_getparents( + struct xfs_getparents_ctx *gpx) +{ + struct xfs_getparents *gp = &gpx->gph.gph_request; + struct xfs_inode *ip = gpx->ip; + struct xfs_mount *mp = ip->i_mount; + size_t bufsize; + int error; + + /* Check size of buffer requested by user */ + if (gp->gp_bufsize > XFS_XATTR_LIST_MAX) + return -ENOMEM; + if (gp->gp_bufsize < xfs_getparents_rec_sizeof(1)) + return -EINVAL; + + if (gp->gp_iflags & ~XFS_GETPARENTS_IFLAGS_ALL) + return -EINVAL; + if (gp->gp_reserved) + return -EINVAL; + + bufsize = round_down(gp->gp_bufsize, sizeof(uint64_t)); + gpx->krecords = kvzalloc(bufsize, GFP_KERNEL); + if (!gpx->krecords) { + bufsize = min(bufsize, PAGE_SIZE); + gpx->krecords = kvzalloc(bufsize, GFP_KERNEL); + if (!gpx->krecords) + return -ENOMEM; + } + + gpx->context.dp = ip; + gpx->context.resynch = 1; + gpx->context.put_listent = xfs_getparents_put_listent; + gpx->context.bufsize = bufsize; + /* firstu is used to track the bytes filled in the buffer */ + gpx->context.firstu = 0; + + /* Copy the cursor provided by caller */ + memcpy(&gpx->context.cursor, &gp->gp_cursor, + sizeof(struct xfs_attrlist_cursor)); + gpx->count = 0; + gp->gp_oflags = 0; + + trace_xfs_getparents_begin(ip, gp, &gpx->context.cursor); + + error = xfs_attr_list(&gpx->context); + if (error) + goto out_free_buf; + if (gpx->context.seen_enough < 0) { + error = gpx->context.seen_enough; + goto out_free_buf; + } + xfs_getparents_expand_lastrec(gpx); + + /* Update the caller with the current cursor position */ + memcpy(&gp->gp_cursor, &gpx->context.cursor, + sizeof(struct xfs_attrlist_cursor)); + + /* Is this the root directory? */ + if (ip->i_ino == mp->m_sb.sb_rootino) + gp->gp_oflags |= XFS_GETPARENTS_OFLAG_ROOT; + + if (gpx->context.seen_enough == 0) { + /* + * If we did not run out of buffer space, then we reached the + * end of the pptr recordset, so set the DONE flag. + */ + gp->gp_oflags |= XFS_GETPARENTS_OFLAG_DONE; + } else if (gpx->count == 0) { + /* + * If we ran out of buffer space before copying any parent + * pointers at all, the caller's buffer was too short. Tell + * userspace that, erm, the message is too long. + */ + error = -EMSGSIZE; + goto out_free_buf; + } + + trace_xfs_getparents_end(ip, gp, &gpx->context.cursor); + + ASSERT(gpx->context.firstu <= gpx->gph.gph_request.gp_bufsize); + + /* Copy the records to userspace. */ + if (copy_to_user(u64_to_uptr(gpx->gph.gph_request.gp_buffer), + gpx->krecords, gpx->context.firstu)) + error = -EFAULT; + +out_free_buf: + kvfree(gpx->krecords); + gpx->krecords = NULL; + return error; +} + +/* Retrieve the parents of this file and pass them back to userspace. */ +int +xfs_ioc_getparents( + struct file *file, + struct xfs_getparents __user *ureq) +{ + struct xfs_getparents_ctx gpx = { + .ip = XFS_I(file_inode(file)), + }; + struct xfs_getparents *kreq = &gpx.gph.gph_request; + struct xfs_mount *mp = gpx.ip->i_mount; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (!xfs_has_parent(mp)) + return -EOPNOTSUPP; + if (copy_from_user(kreq, ureq, sizeof(*kreq))) + return -EFAULT; + + error = xfs_getparents(&gpx); + if (error) + return error; + + if (copy_to_user(ureq, kreq, sizeof(*kreq))) + return -EFAULT; + + return 0; +} + +/* Retrieve the parents of this file handle and pass them back to userspace. */ +int +xfs_ioc_getparents_by_handle( + struct file *file, + struct xfs_getparents_by_handle __user *ureq) +{ + struct xfs_getparents_ctx gpx = { }; + struct xfs_inode *ip = XFS_I(file_inode(file)); + struct xfs_mount *mp = ip->i_mount; + struct xfs_getparents_by_handle *kreq = &gpx.gph; + struct xfs_handle *handle = &kreq->gph_handle; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (!xfs_has_parent(mp)) + return -EOPNOTSUPP; + if (copy_from_user(kreq, ureq, sizeof(*kreq))) + return -EFAULT; + + /* + * We don't use exportfs_decode_fh because it does too much work here. + * If the handle refers to a directory, the exportfs code will walk + * upwards through the directory tree to connect the dentries to the + * root directory dentry. For GETPARENTS we don't care about that + * because we're not actually going to open a file descriptor; we only + * want to open an inode and read its parent pointers. + * + * Note that xfs_scrub uses GETPARENTS to log that it will try to fix a + * corrupted file's metadata. For this usecase we would really rather + * userspace single-step the path reconstruction to avoid loops or + * other strange things if the directory tree is corrupt. + */ + gpx.ip = xfs_khandle_to_inode(file, handle); + if (IS_ERR(gpx.ip)) + return PTR_ERR(gpx.ip); + + error = xfs_getparents(&gpx); + if (error) + goto out_rele; + + if (copy_to_user(ureq, kreq, sizeof(*kreq))) + error = -EFAULT; + +out_rele: + xfs_irele(gpx.ip); + return error; +} diff --git a/fs/xfs/xfs_handle.h b/fs/xfs/xfs_handle.h index e39eaf4689da9..6799a86d8565c 100644 --- a/fs/xfs/xfs_handle.h +++ b/fs/xfs/xfs_handle.h @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * Copyright (c) 2022-2024 Oracle. * All rights reserved. */ #ifndef __XFS_HANDLE_H__ @@ -25,4 +26,8 @@ int xfs_ioc_attr_list(struct xfs_inode *dp, void __user *ubuf, struct dentry *xfs_handle_to_dentry(struct file *parfilp, void __user *uhandle, u32 hlen); +int xfs_ioc_getparents(struct file *file, struct xfs_getparents __user *arg); +int xfs_ioc_getparents_by_handle(struct file *file, + struct xfs_getparents_by_handle __user *arg); + #endif /* __XFS_HANDLE_H__ */ diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index ed05fcd6261d8..0e97070abe802 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -35,6 +35,7 @@ #include "xfs_health.h" #include "xfs_reflink.h" #include "xfs_ioctl.h" +#include "xfs_xattr.h" #include "xfs_rtbitmap.h" #include "xfs_file.h" #include "xfs_exchrange.h" @@ -1424,7 +1425,10 @@ xfs_file_ioctl( case XFS_IOC_FSGETXATTRA: return xfs_ioc_fsgetxattra(ip, arg); - + case XFS_IOC_GETPARENTS: + return xfs_ioc_getparents(filp, arg); + case XFS_IOC_GETPARENTS_BY_HANDLE: + return xfs_ioc_getparents_by_handle(filp, arg); case XFS_IOC_GETBMAP: case XFS_IOC_GETBMAPA: case XFS_IOC_GETBMAPX: diff --git a/fs/xfs/xfs_trace.c b/fs/xfs/xfs_trace.c index cf92a3bd56c79..9c7fbaae2717d 100644 --- a/fs/xfs/xfs_trace.c +++ b/fs/xfs/xfs_trace.c @@ -41,6 +41,7 @@ #include "xfs_bmap.h" #include "xfs_exchmaps.h" #include "xfs_exchrange.h" +#include "xfs_parent.h" /* * We include this last to have the helpers above available for the trace diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index 5621db48e7634..05cb59bd0b80e 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -87,6 +87,9 @@ struct xfs_bmap_intent; struct xfs_exchmaps_intent; struct xfs_exchmaps_req; struct xfs_exchrange; +struct xfs_getparents; +struct xfs_parent_irec; +struct xfs_attrlist_cursor_kern; #define XFS_ATTR_FILTER_FLAGS \ { XFS_ATTR_ROOT, "ROOT" }, \ @@ -5096,6 +5099,95 @@ TRACE_EVENT(xfs_exchmaps_delta_nextents, __entry->d_nexts1, __entry->d_nexts2) ); +DECLARE_EVENT_CLASS(xfs_getparents_rec_class, + TP_PROTO(struct xfs_inode *ip, const struct xfs_getparents *ppi, + const struct xfs_attr_list_context *context, + const struct xfs_getparents_rec *pptr), + TP_ARGS(ip, ppi, context, pptr), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned int, firstu) + __field(unsigned short, reclen) + __field(unsigned int, bufsize) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __string(name, pptr->gpr_name) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = ip->i_ino; + __entry->firstu = context->firstu; + __entry->reclen = pptr->gpr_reclen; + __entry->bufsize = ppi->gp_bufsize; + __entry->parent_ino = pptr->gpr_parent.ha_fid.fid_ino; + __entry->parent_gen = pptr->gpr_parent.ha_fid.fid_gen; + __assign_str(name, pptr->gpr_name); + ), + TP_printk("dev %d:%d ino 0x%llx firstu %u reclen %u bufsize %u parent_ino 0x%llx parent_gen 0x%x name '%s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->firstu, + __entry->reclen, + __entry->bufsize, + __entry->parent_ino, + __entry->parent_gen, + __get_str(name)) +) +#define DEFINE_XFS_GETPARENTS_REC_EVENT(name) \ +DEFINE_EVENT(xfs_getparents_rec_class, name, \ + TP_PROTO(struct xfs_inode *ip, const struct xfs_getparents *ppi, \ + const struct xfs_attr_list_context *context, \ + const struct xfs_getparents_rec *pptr), \ + TP_ARGS(ip, ppi, context, pptr)) +DEFINE_XFS_GETPARENTS_REC_EVENT(xfs_getparents_put_listent); +DEFINE_XFS_GETPARENTS_REC_EVENT(xfs_getparents_expand_lastrec); + +DECLARE_EVENT_CLASS(xfs_getparents_class, + TP_PROTO(struct xfs_inode *ip, const struct xfs_getparents *ppi, + const struct xfs_attrlist_cursor_kern *cur), + TP_ARGS(ip, ppi, cur), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned short, iflags) + __field(unsigned short, oflags) + __field(unsigned int, bufsize) + __field(unsigned int, hashval) + __field(unsigned int, blkno) + __field(unsigned int, offset) + __field(int, initted) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = ip->i_ino; + __entry->iflags = ppi->gp_iflags; + __entry->oflags = ppi->gp_oflags; + __entry->bufsize = ppi->gp_bufsize; + __entry->hashval = cur->hashval; + __entry->blkno = cur->blkno; + __entry->offset = cur->offset; + __entry->initted = cur->initted; + ), + TP_printk("dev %d:%d ino 0x%llx iflags 0x%x oflags 0x%x bufsize %u cur_init? %d hashval 0x%x blkno %u offset %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->iflags, + __entry->oflags, + __entry->bufsize, + __entry->initted, + __entry->hashval, + __entry->blkno, + __entry->offset) +) +#define DEFINE_XFS_GETPARENTS_EVENT(name) \ +DEFINE_EVENT(xfs_getparents_class, name, \ + TP_PROTO(struct xfs_inode *ip, const struct xfs_getparents *ppi, \ + const struct xfs_attrlist_cursor_kern *cur), \ + TP_ARGS(ip, ppi, cur)) +DEFINE_XFS_GETPARENTS_EVENT(xfs_getparents_begin); +DEFINE_XFS_GETPARENTS_EVENT(xfs_getparents_end); + #endif /* _TRACE_XFS_H */ #undef TRACE_INCLUDE_PATH From 7dafb449b7922c1eec6fee3ed85b679d51f0f431 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 22 Apr 2024 09:47:56 -0700 Subject: [PATCH 0449/1702] xfs: don't remove the attr fork when parent pointers are enabled When an inode is removed, it may also cause the attribute fork to be removed if it is the last attribute. This transaction gets flushed to the log, but if the system goes down before we could inactivate the symlink, the log recovery tries to inactivate this inode (since it is on the unlinked list) but the verifier trips over the remote value and leaks it. Hence we ended up with a file in this odd state on a "clean" mount. The "obvious" fix is to prohibit erasure of the attr fork to avoid tripping over the verifiers when pptrs are enabled. Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr_leaf.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 1a374c6885d7f..b9e98950eb3d8 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -891,7 +891,8 @@ xfs_attr_sf_removename( */ if (totsize == sizeof(struct xfs_attr_sf_hdr) && xfs_has_attr2(mp) && (dp->i_df.if_format != XFS_DINODE_FMT_BTREE) && - !(args->op_flags & (XFS_DA_OP_ADDNAME | XFS_DA_OP_REPLACE))) { + !(args->op_flags & (XFS_DA_OP_ADDNAME | XFS_DA_OP_REPLACE)) && + !xfs_has_parent(mp)) { xfs_attr_fork_remove(dp, args->trans); } else { xfs_idata_realloc(dp, -size, XFS_ATTR_FORK); @@ -900,7 +901,8 @@ xfs_attr_sf_removename( ASSERT(totsize > sizeof(struct xfs_attr_sf_hdr) || (args->op_flags & XFS_DA_OP_ADDNAME) || !xfs_has_attr2(mp) || - dp->i_df.if_format == XFS_DINODE_FMT_BTREE); + dp->i_df.if_format == XFS_DINODE_FMT_BTREE || + xfs_has_parent(mp)); xfs_trans_log_inode(args->trans, dp, XFS_ILOG_CORE | XFS_ILOG_ADATA); } From 5f98ec1cb5c264e4815e21d632ee0b3d6e700e3d Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Mon, 22 Apr 2024 09:47:57 -0700 Subject: [PATCH 0450/1702] xfs: add a incompat feature bit for parent pointers Create an incompat feature bit and a fs geometry flag so that we can enable the feature in the ondisk superblock and advertise its existence to userspace. Signed-off-by: Mark Tinguely Signed-off-by: Dave Chinner Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_format.h | 1 + fs/xfs/libxfs/xfs_fs.h | 1 + fs/xfs/libxfs/xfs_sb.c | 4 ++++ fs/xfs/xfs_super.c | 4 ++++ 4 files changed, 10 insertions(+) diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h index f1818c54af6f8..b457e457e1f71 100644 --- a/fs/xfs/libxfs/xfs_format.h +++ b/fs/xfs/libxfs/xfs_format.h @@ -374,6 +374,7 @@ xfs_sb_has_ro_compat_feature( #define XFS_SB_FEAT_INCOMPAT_NEEDSREPAIR (1 << 4) /* needs xfs_repair */ #define XFS_SB_FEAT_INCOMPAT_NREXT64 (1 << 5) /* large extent counters */ #define XFS_SB_FEAT_INCOMPAT_EXCHRANGE (1 << 6) /* exchangerange supported */ +#define XFS_SB_FEAT_INCOMPAT_PARENT (1 << 7) /* parent pointers */ #define XFS_SB_FEAT_INCOMPAT_ALL \ (XFS_SB_FEAT_INCOMPAT_FTYPE | \ XFS_SB_FEAT_INCOMPAT_SPINODES | \ diff --git a/fs/xfs/libxfs/xfs_fs.h b/fs/xfs/libxfs/xfs_fs.h index ea654df0505f2..dd13bfa500f2e 100644 --- a/fs/xfs/libxfs/xfs_fs.h +++ b/fs/xfs/libxfs/xfs_fs.h @@ -240,6 +240,7 @@ typedef struct xfs_fsop_resblks { #define XFS_FSOP_GEOM_FLAGS_INOBTCNT (1 << 22) /* inobt btree counter */ #define XFS_FSOP_GEOM_FLAGS_NREXT64 (1 << 23) /* large extent counters */ #define XFS_FSOP_GEOM_FLAGS_EXCHANGE_RANGE (1 << 24) /* exchange range */ +#define XFS_FSOP_GEOM_FLAGS_PARENT (1 << 25) /* linux parent pointers */ /* * Minimum and maximum sizes need for growth checks. diff --git a/fs/xfs/libxfs/xfs_sb.c b/fs/xfs/libxfs/xfs_sb.c index c350e259b6855..09e4bf949bf88 100644 --- a/fs/xfs/libxfs/xfs_sb.c +++ b/fs/xfs/libxfs/xfs_sb.c @@ -178,6 +178,8 @@ xfs_sb_version_to_features( features |= XFS_FEAT_NREXT64; if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_EXCHRANGE) features |= XFS_FEAT_EXCHANGE_RANGE; + if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_PARENT) + features |= XFS_FEAT_PARENT; return features; } @@ -1254,6 +1256,8 @@ xfs_fs_geometry( geo->flags |= XFS_FSOP_GEOM_FLAGS_BIGTIME; if (xfs_has_inobtcounts(mp)) geo->flags |= XFS_FSOP_GEOM_FLAGS_INOBTCNT; + if (xfs_has_parent(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_PARENT; if (xfs_has_sector(mp)) { geo->flags |= XFS_FSOP_GEOM_FLAGS_SECTOR; geo->logsectsize = sbp->sb_logsectsize; diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index c303d7ff95977..27e9f749c4c7f 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -1746,6 +1746,10 @@ xfs_fs_fill_super( xfs_warn(mp, "EXPERIMENTAL exchange-range feature enabled. Use at your own risk!"); + if (xfs_has_parent(mp)) + xfs_warn(mp, + "EXPERIMENTAL parent pointer feature enabled. Use at your own risk!"); + error = xfs_mountfs(mp); if (error) goto out_filestream_unmount; From 7ea816ca4043c2bc6052f696b6aebe2c22980a03 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:58 -0700 Subject: [PATCH 0451/1702] xfs: fix unit conversion error in xfs_log_calc_max_attrsetm_res Dave and I were discussing some recent test regressions as a result of me turning on nrext64=1 on realtime filesystems, when we noticed that the minimum log size of a 32M filesystem jumped from 954 blocks to 4287 blocks. Digging through xfs_log_calc_max_attrsetm_res, Dave noticed that @size contains the maximum estimated amount of space needed for a local format xattr, in bytes, but we feed this quantity to XFS_NEXTENTADD_SPACE_RES, which requires units of blocks. This has resulted in an overestimation of the minimum log size over the years. We should nominally correct this, but there's a backwards compatibility problem -- if we enable it now, the minimum log size will decrease. If a corrected mkfs formats a filesystem with this new smaller log size, a user will encounter mount failures on an uncorrected kernel due to the larger minimum log size computations there. Therefore, turn this on for parent pointers because it wasn't merged at all upstream when this issue was discovered. Signed-off-by: Darrick J. Wong Reviewed-by: Allison Henderson Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_log_rlimit.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/fs/xfs/libxfs/xfs_log_rlimit.c b/fs/xfs/libxfs/xfs_log_rlimit.c index 9975b93a7412d..3518d5e21df03 100644 --- a/fs/xfs/libxfs/xfs_log_rlimit.c +++ b/fs/xfs/libxfs/xfs_log_rlimit.c @@ -16,6 +16,29 @@ #include "xfs_bmap_btree.h" #include "xfs_trace.h" +/* + * Shortly after enabling the large extents count feature in 2023, longstanding + * bugs were found in the code that computes the minimum log size. Luckily, + * the bugs resulted in over-estimates of that size, so there's no impact to + * existing users. However, we don't want to reduce the minimum log size + * because that can create the situation where a newer mkfs writes a new + * filesystem that an older kernel won't mount. + * + * Therefore, we only may correct the computation starting with filesystem + * features that didn't exist in 2023. In other words, only turn this on if + * the filesystem has parent pointers. + * + * This function can be called before the XFS_HAS_* flags have been set up, + * (e.g. mkfs) so we must check the ondisk superblock. + */ +static inline bool +xfs_want_minlogsize_fixes( + struct xfs_sb *sb) +{ + return xfs_sb_is_v5(sb) && + xfs_sb_has_incompat_feature(sb, XFS_SB_FEAT_INCOMPAT_PARENT); +} + /* * Calculate the maximum length in bytes that would be required for a local * attribute value as large attributes out of line are not logged. @@ -31,6 +54,15 @@ xfs_log_calc_max_attrsetm_res( MAXNAMELEN - 1; nblks = XFS_DAENTER_SPACE_RES(mp, XFS_ATTR_FORK); nblks += XFS_B_TO_FSB(mp, size); + + /* + * If the feature set is new enough, correct a unit conversion error in + * the xattr transaction reservation code that resulted in oversized + * minimum log size computations. + */ + if (xfs_want_minlogsize_fixes(&mp->m_sb)) + size = XFS_B_TO_FSB(mp, size); + nblks += XFS_NEXTENTADD_SPACE_RES(mp, size, XFS_ATTR_FORK); return M_RES(mp)->tr_attrsetm.tr_logres + From 6ed858c7c678218aa8df9d9e75d5e9955c105415 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:47:59 -0700 Subject: [PATCH 0452/1702] xfs: drop compatibility minimum log size computations for reflink Let's also drop the oversized minimum log computations for reflink and rmap that were the result of bugs introduced many years ago. Signed-off-by: Darrick J. Wong Reviewed-by: Allison Henderson Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_log_rlimit.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/fs/xfs/libxfs/xfs_log_rlimit.c b/fs/xfs/libxfs/xfs_log_rlimit.c index 3518d5e21df03..d3bd6a86c8fe9 100644 --- a/fs/xfs/libxfs/xfs_log_rlimit.c +++ b/fs/xfs/libxfs/xfs_log_rlimit.c @@ -24,6 +24,11 @@ * because that can create the situation where a newer mkfs writes a new * filesystem that an older kernel won't mount. * + * Several years prior, we also discovered that the transaction reservations + * for rmap and reflink operations were unnecessarily large. That was fixed, + * but the minimum log size computation was left alone to avoid the + * compatibility problems noted above. Fix that too. + * * Therefore, we only may correct the computation starting with filesystem * features that didn't exist in 2023. In other words, only turn this on if * the filesystem has parent pointers. @@ -80,6 +85,15 @@ xfs_log_calc_trans_resv_for_minlogblocks( { unsigned int rmap_maxlevels = mp->m_rmap_maxlevels; + /* + * If the feature set is new enough, drop the oversized minimum log + * size computation introduced by the original reflink code. + */ + if (xfs_want_minlogsize_fixes(&mp->m_sb)) { + xfs_trans_resv_calc(mp, resv); + return; + } + /* * In the early days of rmap+reflink, we always set the rmap maxlevels * to 9 even if the AG was small enough that it would never grow to From 2a009397eb5ae178670cbd7101e9635cf6412b35 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:00 -0700 Subject: [PATCH 0453/1702] xfs: revert commit 44af6c7e59b12 In my haste to fix what I thought was a performance problem in the attr scrub code, I neglected to notice that the xfs_attr_get_ilocked also had the effect of checking that attributes can actually be looked up through the attr dabtree. Fix this. Fixes: 44af6c7e59b12 ("xfs: don't load local xattr values during scrub") Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/attr.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index c07d050b39b2e..393ed36709b3a 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -208,14 +208,6 @@ xchk_xattr_actor( return -ECANCELED; } - /* - * Local and shortform xattr values are stored in the attr leaf block, - * so we don't need to retrieve the value from a remote block to detect - * corruption problems. - */ - if (value) - return 0; - /* * Try to allocate enough memory to extract the attr value. If that * doesn't work, return -EDEADLOCK as a signal to try again with a @@ -229,6 +221,11 @@ xchk_xattr_actor( args.value = ab->value; + /* + * Get the attr value to ensure that lookup can find this attribute + * through the dabtree indexing and that remote value retrieval also + * works correctly. + */ xfs_attr_sethash(&args); error = xfs_attr_get_ilocked(&args); /* ENODATA means the hash lookup failed and the attr is bad */ From 67ac7091e35bd34b75c0ec77331b53ca052e0cb3 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:00 -0700 Subject: [PATCH 0454/1702] xfs: enable parent pointers Add parent pointers to the list of supported features. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_format.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h index b457e457e1f71..61f51becff4f7 100644 --- a/fs/xfs/libxfs/xfs_format.h +++ b/fs/xfs/libxfs/xfs_format.h @@ -382,7 +382,8 @@ xfs_sb_has_ro_compat_feature( XFS_SB_FEAT_INCOMPAT_BIGTIME | \ XFS_SB_FEAT_INCOMPAT_NEEDSREPAIR | \ XFS_SB_FEAT_INCOMPAT_NREXT64 | \ - XFS_SB_FEAT_INCOMPAT_EXCHRANGE) + XFS_SB_FEAT_INCOMPAT_EXCHRANGE | \ + XFS_SB_FEAT_INCOMPAT_PARENT) #define XFS_SB_FEAT_INCOMPAT_UNKNOWN ~XFS_SB_FEAT_INCOMPAT_ALL static inline bool From 61b3f0df5c235806d372aaf696ce9aee7746d18f Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:01 -0700 Subject: [PATCH 0455/1702] xfs: check dirents have parent pointers If the fs has parent pointers, we need to check that each child dirent points to a file that has a parent pointer pointing back at us. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_parent.c | 22 ++++++++ fs/xfs/libxfs/xfs_parent.h | 5 ++ fs/xfs/scrub/dir.c | 112 ++++++++++++++++++++++++++++++++++++- 3 files changed, 138 insertions(+), 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_parent.c b/fs/xfs/libxfs/xfs_parent.c index 504de1ef33876..1e53df5f83323 100644 --- a/fs/xfs/libxfs/xfs_parent.c +++ b/fs/xfs/libxfs/xfs_parent.c @@ -291,3 +291,25 @@ xfs_parent_from_attr( *parent_gen = be32_to_cpu(rec->p_gen); return 0; } + +/* + * Look up a parent pointer record (@parent_name -> @pptr) of @ip. + * + * Caller must hold at least ILOCK_SHARED. The scratchpad need not be + * initialized. + * + * Returns 0 if the pointer is found, -ENOATTR if there is no match, or a + * negative errno. + */ +int +xfs_parent_lookup( + struct xfs_trans *tp, + struct xfs_inode *ip, + const struct xfs_name *parent_name, + struct xfs_parent_rec *pptr, + struct xfs_da_args *scratch) +{ + memset(scratch, 0, sizeof(struct xfs_da_args)); + xfs_parent_da_args_init(scratch, tp, pptr, ip, ip->i_ino, parent_name); + return xfs_attr_get_ilocked(scratch); +} diff --git a/fs/xfs/libxfs/xfs_parent.h b/fs/xfs/libxfs/xfs_parent.h index d7ab09e738ad4..97788582321a6 100644 --- a/fs/xfs/libxfs/xfs_parent.h +++ b/fs/xfs/libxfs/xfs_parent.h @@ -96,4 +96,9 @@ int xfs_parent_from_attr(struct xfs_mount *mp, unsigned int attr_flags, const void *value, unsigned int valuelen, xfs_ino_t *parent_ino, uint32_t *parent_gen); +/* Repair functions */ +int xfs_parent_lookup(struct xfs_trans *tp, struct xfs_inode *ip, + const struct xfs_name *name, struct xfs_parent_rec *pptr, + struct xfs_da_args *scratch); + #endif /* __XFS_PARENT_H__ */ diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index 3fe6ffcf9c062..e11d73eb89352 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -16,6 +16,8 @@ #include "xfs_dir2.h" #include "xfs_dir2_priv.h" #include "xfs_health.h" +#include "xfs_attr.h" +#include "xfs_parent.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/dabtree.h" @@ -41,6 +43,14 @@ xchk_setup_directory( /* Directories */ +struct xchk_dir { + struct xfs_scrub *sc; + + /* information for parent pointer validation. */ + struct xfs_parent_rec pptr_rec; + struct xfs_da_args pptr_args; +}; + /* Scrub a directory entry. */ /* Check that an inode's mode matches a given XFS_DIR3_FT_* type. */ @@ -63,6 +73,90 @@ xchk_dir_check_ftype( xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); } +/* + * Try to lock a child file for checking parent pointers. Returns the inode + * flags for the locks we now hold, or zero if we failed. + */ +STATIC unsigned int +xchk_dir_lock_child( + struct xfs_scrub *sc, + struct xfs_inode *ip) +{ + if (!xfs_ilock_nowait(ip, XFS_IOLOCK_SHARED)) + return 0; + + if (!xfs_ilock_nowait(ip, XFS_ILOCK_SHARED)) { + xfs_iunlock(ip, XFS_IOLOCK_SHARED); + return 0; + } + + if (!xfs_inode_has_attr_fork(ip) || !xfs_need_iread_extents(&ip->i_af)) + return XFS_IOLOCK_SHARED | XFS_ILOCK_SHARED; + + xfs_iunlock(ip, XFS_ILOCK_SHARED); + + if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL)) { + xfs_iunlock(ip, XFS_IOLOCK_SHARED); + return 0; + } + + return XFS_IOLOCK_SHARED | XFS_ILOCK_EXCL; +} + +/* Check the backwards link (parent pointer) associated with this dirent. */ +STATIC int +xchk_dir_parent_pointer( + struct xchk_dir *sd, + const struct xfs_name *name, + struct xfs_inode *ip) +{ + struct xfs_scrub *sc = sd->sc; + int error; + + xfs_inode_to_parent_rec(&sd->pptr_rec, sc->ip); + error = xfs_parent_lookup(sc->tp, ip, name, &sd->pptr_rec, + &sd->pptr_args); + if (error == -ENOATTR) + xchk_fblock_xref_set_corrupt(sc, XFS_DATA_FORK, 0); + + return 0; +} + +/* Look for a parent pointer matching this dirent, if the child isn't busy. */ +STATIC int +xchk_dir_check_pptr_fast( + struct xchk_dir *sd, + xfs_dir2_dataptr_t dapos, + const struct xfs_name *name, + struct xfs_inode *ip) +{ + struct xfs_scrub *sc = sd->sc; + unsigned int lockmode; + int error; + + /* dot and dotdot entries do not have parent pointers */ + if (xfs_dir2_samename(name, &xfs_name_dot) || + xfs_dir2_samename(name, &xfs_name_dotdot)) + return 0; + + /* No self-referential non-dot or dotdot dirents. */ + if (ip == sc->ip) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return -ECANCELED; + } + + /* Try to lock the inode. */ + lockmode = xchk_dir_lock_child(sc, ip); + if (!lockmode) { + xchk_set_incomplete(sc); + return -ECANCELED; + } + + error = xchk_dir_parent_pointer(sd, name, ip); + xfs_iunlock(ip, lockmode); + return error; +} + /* * Scrub a single directory entry. * @@ -80,6 +174,7 @@ xchk_dir_actor( { struct xfs_mount *mp = dp->i_mount; struct xfs_inode *ip; + struct xchk_dir *sd = priv; xfs_ino_t lookup_ino; xfs_dablk_t offset; int error = 0; @@ -146,6 +241,14 @@ xchk_dir_actor( goto out; xchk_dir_check_ftype(sc, offset, ip, name->type); + + if (xfs_has_parent(mp)) { + error = xchk_dir_check_pptr_fast(sd, dapos, name, ip); + if (error) + goto out_rele; + } + +out_rele: xchk_irele(sc, ip); out: if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) @@ -767,6 +870,7 @@ int xchk_directory( struct xfs_scrub *sc) { + struct xchk_dir *sd; int error; if (!S_ISDIR(VFS_I(sc->ip)->i_mode)) @@ -799,8 +903,14 @@ xchk_directory( if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) return 0; + sd = kvzalloc(sizeof(struct xchk_dir), XCHK_GFP_FLAGS); + if (!sd) + return -ENOMEM; + sd->sc = sc; + /* Look up every name in this directory by hash. */ - error = xchk_dir_walk(sc, sc->ip, xchk_dir_actor, NULL); + error = xchk_dir_walk(sc, sc->ip, xchk_dir_actor, sd); + kvfree(sd); if (error && error != -ECANCELED) return error; From b961c8bf1fc3d0232209a49f47d1cd7a55b7a861 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:02 -0700 Subject: [PATCH 0456/1702] xfs: deferred scrub of dirents If the trylock-based parent pointer check fails, retain those dirents and check them at the end. This may involve dropping the locks on the file being scanned, so yay. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/dir.c | 234 ++++++++++++++++++++++++++++++++++++++++- fs/xfs/scrub/readdir.c | 78 ++++++++++++++ fs/xfs/scrub/readdir.h | 3 + fs/xfs/scrub/trace.h | 34 ++++++ 4 files changed, 346 insertions(+), 3 deletions(-) diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index e11d73eb89352..62474d0557c41 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -24,6 +24,10 @@ #include "scrub/readdir.h" #include "scrub/health.h" #include "scrub/repair.h" +#include "scrub/trace.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" /* Set us up to scrub directories. */ int @@ -43,12 +47,37 @@ xchk_setup_directory( /* Directories */ +/* Deferred directory entry that we saved for later. */ +struct xchk_dirent { + /* Cookie for retrieval of the dirent name. */ + xfblob_cookie name_cookie; + + /* Child inode number. */ + xfs_ino_t ino; + + /* Length of the pptr name. */ + uint8_t namelen; +}; + struct xchk_dir { struct xfs_scrub *sc; /* information for parent pointer validation. */ struct xfs_parent_rec pptr_rec; struct xfs_da_args pptr_args; + + /* Fixed-size array of xchk_dirent structures. */ + struct xfarray *dir_entries; + + /* Blobs containing dirent names. */ + struct xfblob *dir_names; + + /* If we've cycled the ILOCK, we must revalidate deferred dirents. */ + bool need_revalidate; + + /* Name buffer for dirent revalidation. */ + struct xfs_name xname; + uint8_t namebuf[MAXNAMELEN]; }; /* Scrub a directory entry. */ @@ -148,8 +177,26 @@ xchk_dir_check_pptr_fast( /* Try to lock the inode. */ lockmode = xchk_dir_lock_child(sc, ip); if (!lockmode) { - xchk_set_incomplete(sc); - return -ECANCELED; + struct xchk_dirent save_de = { + .namelen = name->len, + .ino = ip->i_ino, + }; + + /* Couldn't lock the inode, so save the dirent for later. */ + trace_xchk_dir_defer(sc->ip, name, ip->i_ino); + + error = xfblob_storename(sd->dir_names, &save_de.name_cookie, + name); + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, + &error)) + return error; + + error = xfarray_append(sd->dir_entries, &save_de); + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, + &error)) + return error; + + return 0; } error = xchk_dir_parent_pointer(sd, name, ip); @@ -865,6 +912,142 @@ xchk_directory_blocks( return error; } +/* + * Revalidate a dirent that we collected in the past but couldn't check because + * of lock contention. Returns 0 if the dirent is still valid, -ENOENT if it + * has gone away on us, or a negative errno. + */ +STATIC int +xchk_dir_revalidate_dirent( + struct xchk_dir *sd, + const struct xfs_name *xname, + xfs_ino_t ino) +{ + struct xfs_scrub *sc = sd->sc; + xfs_ino_t child_ino; + int error; + + /* + * Look up the directory entry. If we get -ENOENT, the directory entry + * went away and there's nothing to revalidate. Return any other + * error. + */ + error = xchk_dir_lookup(sc, sc->ip, xname, &child_ino); + if (error) + return error; + + /* The inode number changed, nothing to revalidate. */ + if (ino != child_ino) + return -ENOENT; + + return 0; +} + +/* + * Check a directory entry's parent pointers the slow way, which means we cycle + * locks a bunch and put up with revalidation until we get it done. + */ +STATIC int +xchk_dir_slow_dirent( + struct xchk_dir *sd, + struct xchk_dirent *dirent, + const struct xfs_name *xname) +{ + struct xfs_scrub *sc = sd->sc; + struct xfs_inode *ip; + unsigned int lockmode; + int error; + + /* Check that the deferred dirent still exists. */ + if (sd->need_revalidate) { + error = xchk_dir_revalidate_dirent(sd, xname, dirent->ino); + if (error == -ENOENT) + return 0; + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, + &error)) + return error; + } + + error = xchk_iget(sc, dirent->ino, &ip); + if (error == -EINVAL || error == -ENOENT) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return 0; + } + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) + return error; + + /* + * If we can grab both IOLOCK and ILOCK of the alleged child, we can + * proceed with the validation. + */ + lockmode = xchk_dir_lock_child(sc, ip); + if (lockmode) { + trace_xchk_dir_slowpath(sc->ip, xname, ip->i_ino); + goto check_pptr; + } + + /* + * We couldn't lock the child file. Drop all the locks and try to + * get them again, one at a time. + */ + xchk_iunlock(sc, sc->ilock_flags); + sd->need_revalidate = true; + + trace_xchk_dir_ultraslowpath(sc->ip, xname, ip->i_ino); + + error = xchk_dir_trylock_for_pptrs(sc, ip, &lockmode); + if (error) + goto out_rele; + + /* Revalidate, since we just cycled the locks. */ + error = xchk_dir_revalidate_dirent(sd, xname, dirent->ino); + if (error == -ENOENT) { + error = 0; + goto out_unlock; + } + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) + goto out_unlock; + +check_pptr: + error = xchk_dir_parent_pointer(sd, xname, ip); +out_unlock: + xfs_iunlock(ip, lockmode); +out_rele: + xchk_irele(sc, ip); + return error; +} + +/* Check all the dirents that we deferred the first time around. */ +STATIC int +xchk_dir_finish_slow_dirents( + struct xchk_dir *sd) +{ + xfarray_idx_t array_cur; + int error; + + foreach_xfarray_idx(sd->dir_entries, array_cur) { + struct xchk_dirent dirent; + + if (sd->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + error = xfarray_load(sd->dir_entries, array_cur, &dirent); + if (error) + return error; + + error = xfblob_loadname(sd->dir_names, dirent.name_cookie, + &sd->xname, dirent.namelen); + if (error) + return error; + + error = xchk_dir_slow_dirent(sd, &dirent, &sd->xname); + if (error) + return error; + } + + return 0; +} + /* Scrub a whole directory. */ int xchk_directory( @@ -907,11 +1090,56 @@ xchk_directory( if (!sd) return -ENOMEM; sd->sc = sc; + sd->xname.name = sd->namebuf; + + if (xfs_has_parent(sc->mp)) { + char *descr; + + /* + * Set up some staging memory for dirents that we can't check + * due to locking contention. + */ + descr = xchk_xfile_ino_descr(sc, "slow directory entries"); + error = xfarray_create(descr, 0, sizeof(struct xchk_dirent), + &sd->dir_entries); + kfree(descr); + if (error) + goto out_sd; + + descr = xchk_xfile_ino_descr(sc, "slow directory entry names"); + error = xfblob_create(descr, &sd->dir_names); + kfree(descr); + if (error) + goto out_entries; + } /* Look up every name in this directory by hash. */ error = xchk_dir_walk(sc, sc->ip, xchk_dir_actor, sd); + if (error == -ECANCELED) + error = 0; + if (error) + goto out_names; + + if (xfs_has_parent(sc->mp)) { + error = xchk_dir_finish_slow_dirents(sd); + if (error == -ETIMEDOUT) { + /* Couldn't grab a lock, scrub was marked incomplete */ + error = 0; + goto out_names; + } + if (error) + goto out_names; + } + +out_names: + if (sd->dir_names) + xfblob_destroy(sd->dir_names); +out_entries: + if (sd->dir_entries) + xfarray_destroy(sd->dir_entries); +out_sd: kvfree(sd); - if (error && error != -ECANCELED) + if (error) return error; /* If the dir is clean, it is clearly not zapped. */ diff --git a/fs/xfs/scrub/readdir.c b/fs/xfs/scrub/readdir.c index 028690761c629..28a94c78b0b19 100644 --- a/fs/xfs/scrub/readdir.c +++ b/fs/xfs/scrub/readdir.c @@ -18,6 +18,7 @@ #include "xfs_trans.h" #include "xfs_error.h" #include "scrub/scrub.h" +#include "scrub/common.h" #include "scrub/readdir.h" /* Call a function for every entry in a shortform directory. */ @@ -380,3 +381,80 @@ xchk_dir_lookup( *ino = args.inumber; return error; } + +/* + * Try to grab the IOLOCK and ILOCK of sc->ip and ip, returning @ip's lock + * state. The caller may have a transaction, so we must use trylock for both + * IOLOCKs. + */ +static inline unsigned int +xchk_dir_trylock_both( + struct xfs_scrub *sc, + struct xfs_inode *ip) +{ + if (!xchk_ilock_nowait(sc, XFS_IOLOCK_EXCL)) + return 0; + + if (!xfs_ilock_nowait(ip, XFS_IOLOCK_SHARED)) + goto parent_iolock; + + xchk_ilock(sc, XFS_ILOCK_EXCL); + if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL)) + goto parent_ilock; + + return XFS_IOLOCK_SHARED | XFS_ILOCK_EXCL; + +parent_ilock: + xchk_iunlock(sc, XFS_ILOCK_EXCL); + xfs_iunlock(ip, XFS_IOLOCK_SHARED); +parent_iolock: + xchk_iunlock(sc, XFS_IOLOCK_EXCL); + return 0; +} + +/* + * Try for a limited time to grab the IOLOCK and ILOCK of both the scrub target + * (@sc->ip) and the inode at the other end (@ip) of a directory or parent + * pointer link so that we can check that link. + * + * We do not know ahead of time that the directory tree is /not/ corrupt, so we + * cannot use the "lock two inode" functions because we do not know that there + * is not a racing thread trying to take the locks in opposite order. First + * take IOLOCK_EXCL of the scrub target, and then try to take IOLOCK_SHARED + * of @ip to synchronize with the VFS. Next, take ILOCK_EXCL of the scrub + * target and @ip to synchronize with XFS. + * + * If the trylocks succeed, *lockmode will be set to the locks held for @ip; + * @sc->ilock_flags will be set for the locks held for @sc->ip; and zero will + * be returned. If not, returns -EDEADLOCK to try again; or -ETIMEDOUT if + * XCHK_TRY_HARDER was set. Returns -EINTR if the process has been killed. + */ +int +xchk_dir_trylock_for_pptrs( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int *lockmode) +{ + unsigned int nr; + int error = 0; + + ASSERT(sc->ilock_flags == 0); + + for (nr = 0; nr < HZ; nr++) { + *lockmode = xchk_dir_trylock_both(sc, ip); + if (*lockmode) + return 0; + + if (xchk_should_terminate(sc, &error)) + return error; + + delay(1); + } + + if (sc->flags & XCHK_TRY_HARDER) { + xchk_set_incomplete(sc); + return -ETIMEDOUT; + } + + return -EDEADLOCK; +} diff --git a/fs/xfs/scrub/readdir.h b/fs/xfs/scrub/readdir.h index 55787f4df123f..da501877a64dd 100644 --- a/fs/xfs/scrub/readdir.h +++ b/fs/xfs/scrub/readdir.h @@ -16,4 +16,7 @@ int xchk_dir_walk(struct xfs_scrub *sc, struct xfs_inode *dp, int xchk_dir_lookup(struct xfs_scrub *sc, struct xfs_inode *dp, const struct xfs_name *name, xfs_ino_t *ino); +int xchk_dir_trylock_for_pptrs(struct xfs_scrub *sc, struct xfs_inode *ip, + unsigned int *lockmode); + #endif /* __XFS_SCRUB_READDIR_H__ */ diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 814db1d1747a0..4db762480b8d4 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -1511,6 +1511,40 @@ DEFINE_EVENT(xchk_nlinks_diff_class, name, \ TP_ARGS(mp, ip, live)) DEFINE_SCRUB_NLINKS_DIFF_EVENT(xchk_nlinks_compare_inode); +DECLARE_EVENT_CLASS(xchk_pptr_class, + TP_PROTO(struct xfs_inode *ip, const struct xfs_name *name, + xfs_ino_t far_ino), + TP_ARGS(ip, name, far_ino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + __field(xfs_ino_t, far_ino) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = ip->i_ino; + __entry->namelen = name->len; + memcpy(__get_str(name), name, name->len); + __entry->far_ino = far_ino; + ), + TP_printk("dev %d:%d ino 0x%llx name '%.*s' far_ino 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->namelen, + __get_str(name), + __entry->far_ino) +) +#define DEFINE_XCHK_PPTR_EVENT(name) \ +DEFINE_EVENT(xchk_pptr_class, name, \ + TP_PROTO(struct xfs_inode *ip, const struct xfs_name *name, \ + xfs_ino_t far_ino), \ + TP_ARGS(ip, name, far_ino)) +DEFINE_XCHK_PPTR_EVENT(xchk_dir_defer); +DEFINE_XCHK_PPTR_EVENT(xchk_dir_slowpath); +DEFINE_XCHK_PPTR_EVENT(xchk_dir_ultraslowpath); + /* repair tracepoints */ #if IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) From 0d29a20fbdba89501bd2ac003faeee666c9a5008 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:03 -0700 Subject: [PATCH 0457/1702] xfs: scrub parent pointers Actually check parent pointers now. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/parent.c | 371 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 371 insertions(+) diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c index acb6282c3d148..6ebbb71041269 100644 --- a/fs/xfs/scrub/parent.c +++ b/fs/xfs/scrub/parent.c @@ -15,11 +15,15 @@ #include "xfs_icache.h" #include "xfs_dir2.h" #include "xfs_dir2_priv.h" +#include "xfs_attr.h" +#include "xfs_parent.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/readdir.h" #include "scrub/tempfile.h" #include "scrub/repair.h" +#include "scrub/listxattr.h" +#include "scrub/trace.h" /* Set us up to scrub parents. */ int @@ -197,6 +201,370 @@ xchk_parent_validate( return error; } +/* + * Checking of Parent Pointers + * =========================== + * + * On filesystems with directory parent pointers, we check the referential + * integrity by visiting each parent pointer of a child file and checking that + * the directory referenced by the pointer actually has a dirent pointing + * forward to the child file. + */ + +struct xchk_pptrs { + struct xfs_scrub *sc; + + /* How many parent pointers did we find at the end? */ + unsigned long long pptrs_found; + + /* Parent of this directory. */ + xfs_ino_t parent_ino; +}; + +/* Does this parent pointer match the dotdot entry? */ +STATIC int +xchk_parent_scan_dotdot( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xchk_pptrs *pp = priv; + xfs_ino_t parent_ino; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, &parent_ino, NULL); + if (error) + return error; + + if (pp->parent_ino == parent_ino) + return -ECANCELED; + + return 0; +} + +/* Look up the dotdot entry so that we can check it as we walk the pptrs. */ +STATIC int +xchk_parent_pptr_and_dotdot( + struct xchk_pptrs *pp) +{ + struct xfs_scrub *sc = pp->sc; + int error; + + /* Look up '..' */ + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, &pp->parent_ino); + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, 0, &error)) + return error; + if (!xfs_verify_dir_ino(sc->mp, pp->parent_ino)) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return 0; + } + + /* Is this the root dir? Then '..' must point to itself. */ + if (sc->ip == sc->mp->m_rootip) { + if (sc->ip->i_ino != pp->parent_ino) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return 0; + } + + /* + * If this is now an unlinked directory, the dotdot value is + * meaningless as long as it points to a valid inode. + */ + if (VFS_I(sc->ip)->i_nlink == 0) + return 0; + + if (pp->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + /* Otherwise, walk the pptrs again, and check. */ + error = xchk_xattr_walk(sc, sc->ip, xchk_parent_scan_dotdot, pp); + if (error == -ECANCELED) { + /* Found a parent pointer that matches dotdot. */ + return 0; + } + if (!error || error == -EFSCORRUPTED) { + /* Found a broken parent pointer or no match. */ + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + return 0; + } + return error; +} + +/* + * Try to lock a parent directory for checking dirents. Returns the inode + * flags for the locks we now hold, or zero if we failed. + */ +STATIC unsigned int +xchk_parent_lock_dir( + struct xfs_scrub *sc, + struct xfs_inode *dp) +{ + if (!xfs_ilock_nowait(dp, XFS_IOLOCK_SHARED)) + return 0; + + if (!xfs_ilock_nowait(dp, XFS_ILOCK_SHARED)) { + xfs_iunlock(dp, XFS_IOLOCK_SHARED); + return 0; + } + + if (!xfs_need_iread_extents(&dp->i_df)) + return XFS_IOLOCK_SHARED | XFS_ILOCK_SHARED; + + xfs_iunlock(dp, XFS_ILOCK_SHARED); + + if (!xfs_ilock_nowait(dp, XFS_ILOCK_EXCL)) { + xfs_iunlock(dp, XFS_IOLOCK_SHARED); + return 0; + } + + return XFS_IOLOCK_SHARED | XFS_ILOCK_EXCL; +} + +/* Check the forward link (dirent) associated with this parent pointer. */ +STATIC int +xchk_parent_dirent( + struct xchk_pptrs *pp, + const struct xfs_name *xname, + struct xfs_inode *dp) +{ + struct xfs_scrub *sc = pp->sc; + xfs_ino_t child_ino; + int error; + + /* + * Use the name attached to this parent pointer to look up the + * directory entry in the alleged parent. + */ + error = xchk_dir_lookup(sc, dp, xname, &child_ino); + if (error == -ENOENT) { + xchk_fblock_xref_set_corrupt(sc, XFS_ATTR_FORK, 0); + return 0; + } + if (!xchk_fblock_xref_process_error(sc, XFS_ATTR_FORK, 0, &error)) + return error; + + /* Does the inode number match? */ + if (child_ino != sc->ip->i_ino) { + xchk_fblock_xref_set_corrupt(sc, XFS_ATTR_FORK, 0); + return 0; + } + + return 0; +} + +/* Try to grab a parent directory. */ +STATIC int +xchk_parent_iget( + struct xchk_pptrs *pp, + const struct xfs_parent_rec *pptr, + struct xfs_inode **dpp) +{ + struct xfs_scrub *sc = pp->sc; + struct xfs_inode *ip; + xfs_ino_t parent_ino = be64_to_cpu(pptr->p_ino); + int error; + + /* Validate inode number. */ + error = xfs_dir_ino_validate(sc->mp, parent_ino); + if (error) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + return -ECANCELED; + } + + error = xchk_iget(sc, parent_ino, &ip); + if (error == -EINVAL || error == -ENOENT) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + return -ECANCELED; + } + if (!xchk_fblock_xref_process_error(sc, XFS_ATTR_FORK, 0, &error)) + return error; + + /* The parent must be a directory. */ + if (!S_ISDIR(VFS_I(ip)->i_mode)) { + xchk_fblock_xref_set_corrupt(sc, XFS_ATTR_FORK, 0); + goto out_rele; + } + + /* Validate generation number. */ + if (VFS_I(ip)->i_generation != be32_to_cpu(pptr->p_gen)) { + xchk_fblock_xref_set_corrupt(sc, XFS_ATTR_FORK, 0); + goto out_rele; + } + + *dpp = ip; + return 0; +out_rele: + xchk_irele(sc, ip); + return 0; +} + +/* + * Walk an xattr of a file. If this xattr is a parent pointer, follow it up + * to a parent directory and check that the parent has a dirent pointing back + * to us. + */ +STATIC int +xchk_parent_scan_attr( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xfs_name xname = { + .name = name, + .len = namelen, + }; + struct xchk_pptrs *pp = priv; + struct xfs_inode *dp = NULL; + const struct xfs_parent_rec *pptr_rec = value; + xfs_ino_t parent_ino; + unsigned int lockmode; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, &parent_ino, NULL); + if (error) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + return error; + } + + /* No self-referential parent pointers. */ + if (parent_ino == sc->ip->i_ino) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + return -ECANCELED; + } + + pp->pptrs_found++; + + error = xchk_parent_iget(pp, pptr_rec, &dp); + if (error) + return error; + if (!dp) + return 0; + + /* Try to lock the inode. */ + lockmode = xchk_parent_lock_dir(sc, dp); + if (!lockmode) { + xchk_set_incomplete(sc); + error = -ECANCELED; + goto out_rele; + } + + error = xchk_parent_dirent(pp, &xname, dp); + if (error) + goto out_unlock; + +out_unlock: + xfs_iunlock(dp, lockmode); +out_rele: + xchk_irele(sc, dp); + return error; +} + +/* + * Compare the number of parent pointers to the link count. For + * non-directories these should be the same. For unlinked directories the + * count should be zero; for linked directories, it should be nonzero. + */ +STATIC int +xchk_parent_count_pptrs( + struct xchk_pptrs *pp) +{ + struct xfs_scrub *sc = pp->sc; + + if (S_ISDIR(VFS_I(sc->ip)->i_mode)) { + if (sc->ip == sc->mp->m_rootip) + pp->pptrs_found++; + + if (VFS_I(sc->ip)->i_nlink == 0 && pp->pptrs_found > 0) + xchk_ino_set_corrupt(sc, sc->ip->i_ino); + else if (VFS_I(sc->ip)->i_nlink > 0 && + pp->pptrs_found == 0) + xchk_ino_set_corrupt(sc, sc->ip->i_ino); + } else { + if (VFS_I(sc->ip)->i_nlink != pp->pptrs_found) + xchk_ino_set_corrupt(sc, sc->ip->i_ino); + } + + return 0; +} + +/* Check parent pointers of a file. */ +STATIC int +xchk_parent_pptr( + struct xfs_scrub *sc) +{ + struct xchk_pptrs *pp; + int error; + + pp = kvzalloc(sizeof(struct xchk_pptrs), XCHK_GFP_FLAGS); + if (!pp) + return -ENOMEM; + pp->sc = sc; + + error = xchk_xattr_walk(sc, sc->ip, xchk_parent_scan_attr, pp); + if (error == -ECANCELED) { + error = 0; + goto out_pp; + } + if (error) + goto out_pp; + + if (pp->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out_pp; + + /* + * For subdirectories, make sure the dotdot entry references the same + * inode as the parent pointers. + * + * If we're scanning a /consistent/ directory, there should only be + * one parent pointer, and it should point to the same directory as + * the dotdot entry. + * + * However, a corrupt directory tree might feature a subdirectory with + * multiple parents. The directory loop scanner is responsible for + * correcting that kind of problem, so for now we only validate that + * the dotdot entry matches /one/ of the parents. + */ + if (S_ISDIR(VFS_I(sc->ip)->i_mode)) { + error = xchk_parent_pptr_and_dotdot(pp); + if (error) + goto out_pp; + } + + if (pp->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out_pp; + + /* + * Complain if the number of parent pointers doesn't match the link + * count. This could be a sign of missing parent pointers (or an + * incorrect link count). + */ + error = xchk_parent_count_pptrs(pp); + if (error) + goto out_pp; + +out_pp: + kvfree(pp); + return error; +} + /* Scrub a parent pointer. */ int xchk_parent( @@ -206,6 +574,9 @@ xchk_parent( xfs_ino_t parent_ino; int error = 0; + if (xfs_has_parent(mp)) + return xchk_parent_pptr(sc); + /* * If we're a directory, check that the '..' link points up to * a directory that has one entry pointing to us. From 8ad345306d1eba337a3b88f9acbe762f9e770f76 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:04 -0700 Subject: [PATCH 0458/1702] xfs: deferred scrub of parent pointers If the trylock-based dirent check fails, retain those parent pointers and check them at the end. This may involve dropping the locks on the file being scanned, so yay. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 2 +- fs/xfs/scrub/parent.c | 267 ++++++++++++++++++++++++++++++++++++++++-- fs/xfs/scrub/trace.h | 3 + 3 files changed, 264 insertions(+), 8 deletions(-) diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index c969b11ce0f47..af99a455ce4db 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -177,6 +177,7 @@ xfs-y += $(addprefix scrub/, \ scrub.o \ symlink.o \ xfarray.o \ + xfblob.o \ xfile.o \ ) @@ -218,7 +219,6 @@ xfs-y += $(addprefix scrub/, \ rmap_repair.o \ symlink_repair.o \ tempfile.o \ - xfblob.o \ ) xfs-$(CONFIG_XFS_RT) += $(addprefix scrub/, \ diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c index 6ebbb71041269..f14e6f643fb60 100644 --- a/fs/xfs/scrub/parent.c +++ b/fs/xfs/scrub/parent.c @@ -23,6 +23,9 @@ #include "scrub/tempfile.h" #include "scrub/repair.h" #include "scrub/listxattr.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" #include "scrub/trace.h" /* Set us up to scrub parents. */ @@ -211,6 +214,18 @@ xchk_parent_validate( * forward to the child file. */ +/* Deferred parent pointer entry that we saved for later. */ +struct xchk_pptr { + /* Cookie for retrieval of the pptr name. */ + xfblob_cookie name_cookie; + + /* Parent pointer record. */ + struct xfs_parent_rec pptr_rec; + + /* Length of the pptr name. */ + uint8_t namelen; +}; + struct xchk_pptrs { struct xfs_scrub *sc; @@ -219,6 +234,22 @@ struct xchk_pptrs { /* Parent of this directory. */ xfs_ino_t parent_ino; + + /* Fixed-size array of xchk_pptr structures. */ + struct xfarray *pptr_entries; + + /* Blobs containing parent pointer names. */ + struct xfblob *pptr_names; + + /* Scratch buffer for scanning pptr xattrs */ + struct xfs_da_args pptr_args; + + /* If we've cycled the ILOCK, we must revalidate all deferred pptrs. */ + bool need_revalidate; + + /* Name buffer */ + struct xfs_name xname; + char namebuf[MAXNAMELEN]; }; /* Does this parent pointer match the dotdot entry? */ @@ -461,8 +492,25 @@ xchk_parent_scan_attr( /* Try to lock the inode. */ lockmode = xchk_parent_lock_dir(sc, dp); if (!lockmode) { - xchk_set_incomplete(sc); - error = -ECANCELED; + struct xchk_pptr save_pp = { + .pptr_rec = *pptr_rec, /* struct copy */ + .namelen = namelen, + }; + + /* Couldn't lock the inode, so save the pptr for later. */ + trace_xchk_parent_defer(sc->ip, &xname, dp->i_ino); + + error = xfblob_storename(pp->pptr_names, &save_pp.name_cookie, + &xname); + if (!xchk_fblock_xref_process_error(sc, XFS_ATTR_FORK, 0, + &error)) + goto out_rele; + + error = xfarray_append(pp->pptr_entries, &save_pp); + if (!xchk_fblock_xref_process_error(sc, XFS_ATTR_FORK, 0, + &error)) + goto out_rele; + goto out_rele; } @@ -477,6 +525,162 @@ xchk_parent_scan_attr( return error; } +/* + * Revalidate a parent pointer that we collected in the past but couldn't check + * because of lock contention. Returns 0 if the parent pointer is still valid, + * -ENOENT if it has gone away on us, or a negative errno. + */ +STATIC int +xchk_parent_revalidate_pptr( + struct xchk_pptrs *pp, + const struct xfs_name *xname, + struct xfs_parent_rec *pptr) +{ + struct xfs_scrub *sc = pp->sc; + int error; + + error = xfs_parent_lookup(sc->tp, sc->ip, xname, pptr, &pp->pptr_args); + if (error == -ENOATTR) { + /* Parent pointer went away, nothing to revalidate. */ + return -ENOENT; + } + + return error; +} + +/* + * Check a parent pointer the slow way, which means we cycle locks a bunch + * and put up with revalidation until we get it done. + */ +STATIC int +xchk_parent_slow_pptr( + struct xchk_pptrs *pp, + const struct xfs_name *xname, + struct xfs_parent_rec *pptr) +{ + struct xfs_scrub *sc = pp->sc; + struct xfs_inode *dp = NULL; + unsigned int lockmode; + int error; + + /* Check that the deferred parent pointer still exists. */ + if (pp->need_revalidate) { + error = xchk_parent_revalidate_pptr(pp, xname, pptr); + if (error == -ENOENT) + return 0; + if (!xchk_fblock_xref_process_error(sc, XFS_ATTR_FORK, 0, + &error)) + return error; + } + + error = xchk_parent_iget(pp, pptr, &dp); + if (error) + return error; + if (!dp) + return 0; + + /* + * If we can grab both IOLOCK and ILOCK of the alleged parent, we + * can proceed with the validation. + */ + lockmode = xchk_parent_lock_dir(sc, dp); + if (lockmode) { + trace_xchk_parent_slowpath(sc->ip, xname, dp->i_ino); + goto check_dirent; + } + + /* + * We couldn't lock the parent dir. Drop all the locks and try to + * get them again, one at a time. + */ + xchk_iunlock(sc, sc->ilock_flags); + pp->need_revalidate = true; + + trace_xchk_parent_ultraslowpath(sc->ip, xname, dp->i_ino); + + error = xchk_dir_trylock_for_pptrs(sc, dp, &lockmode); + if (error) + goto out_rele; + + /* Revalidate the parent pointer now that we cycled locks. */ + error = xchk_parent_revalidate_pptr(pp, xname, pptr); + if (error == -ENOENT) { + error = 0; + goto out_unlock; + } + if (!xchk_fblock_xref_process_error(sc, XFS_ATTR_FORK, 0, &error)) + goto out_unlock; + +check_dirent: + error = xchk_parent_dirent(pp, xname, dp); +out_unlock: + xfs_iunlock(dp, lockmode); +out_rele: + xchk_irele(sc, dp); + return error; +} + +/* Check all the parent pointers that we deferred the first time around. */ +STATIC int +xchk_parent_finish_slow_pptrs( + struct xchk_pptrs *pp) +{ + xfarray_idx_t array_cur; + int error; + + foreach_xfarray_idx(pp->pptr_entries, array_cur) { + struct xchk_pptr pptr; + + if (pp->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + error = xfarray_load(pp->pptr_entries, array_cur, &pptr); + if (error) + return error; + + error = xfblob_loadname(pp->pptr_names, pptr.name_cookie, + &pp->xname, pptr.namelen); + if (error) + return error; + + error = xchk_parent_slow_pptr(pp, &pp->xname, &pptr.pptr_rec); + if (error) + return error; + } + + /* Empty out both xfiles now that we've checked everything. */ + xfarray_truncate(pp->pptr_entries); + xfblob_truncate(pp->pptr_names); + return 0; +} + +/* Count the number of parent pointers. */ +STATIC int +xchk_parent_count_pptr( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xchk_pptrs *pp = priv; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, NULL, NULL); + if (error) + return error; + + pp->pptrs_found++; + return 0; +} + /* * Compare the number of parent pointers to the link count. For * non-directories these should be the same. For unlinked directories the @@ -487,6 +691,23 @@ xchk_parent_count_pptrs( struct xchk_pptrs *pp) { struct xfs_scrub *sc = pp->sc; + int error; + + /* + * If we cycled the ILOCK while cross-checking parent pointers with + * dirents, then we need to recalculate the number of parent pointers. + */ + if (pp->need_revalidate) { + pp->pptrs_found = 0; + error = xchk_xattr_walk(sc, sc->ip, xchk_parent_count_pptr, pp); + if (error == -EFSCORRUPTED) { + /* Found a bad parent pointer */ + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + return 0; + } + if (error) + return error; + } if (S_ISDIR(VFS_I(sc->ip)->i_mode)) { if (sc->ip == sc->mp->m_rootip) @@ -511,23 +732,51 @@ xchk_parent_pptr( struct xfs_scrub *sc) { struct xchk_pptrs *pp; + char *descr; int error; pp = kvzalloc(sizeof(struct xchk_pptrs), XCHK_GFP_FLAGS); if (!pp) return -ENOMEM; pp->sc = sc; + pp->xname.name = pp->namebuf; + + /* + * Set up some staging memory for parent pointers that we can't check + * due to locking contention. + */ + descr = xchk_xfile_ino_descr(sc, "slow parent pointer entries"); + error = xfarray_create(descr, 0, sizeof(struct xchk_pptr), + &pp->pptr_entries); + kfree(descr); + if (error) + goto out_pp; + + descr = xchk_xfile_ino_descr(sc, "slow parent pointer names"); + error = xfblob_create(descr, &pp->pptr_names); + kfree(descr); + if (error) + goto out_entries; error = xchk_xattr_walk(sc, sc->ip, xchk_parent_scan_attr, pp); if (error == -ECANCELED) { error = 0; - goto out_pp; + goto out_names; } if (error) - goto out_pp; + goto out_names; + + error = xchk_parent_finish_slow_pptrs(pp); + if (error == -ETIMEDOUT) { + /* Couldn't grab a lock, scrub was marked incomplete */ + error = 0; + goto out_names; + } + if (error) + goto out_names; if (pp->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) - goto out_pp; + goto out_names; /* * For subdirectories, make sure the dotdot entry references the same @@ -545,7 +794,7 @@ xchk_parent_pptr( if (S_ISDIR(VFS_I(sc->ip)->i_mode)) { error = xchk_parent_pptr_and_dotdot(pp); if (error) - goto out_pp; + goto out_names; } if (pp->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) @@ -558,8 +807,12 @@ xchk_parent_pptr( */ error = xchk_parent_count_pptrs(pp); if (error) - goto out_pp; + goto out_names; +out_names: + xfblob_destroy(pp->pptr_names); +out_entries: + xfarray_destroy(pp->pptr_entries); out_pp: kvfree(pp); return error; diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 4db762480b8d4..97a106519b531 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -1544,6 +1544,9 @@ DEFINE_EVENT(xchk_pptr_class, name, \ DEFINE_XCHK_PPTR_EVENT(xchk_dir_defer); DEFINE_XCHK_PPTR_EVENT(xchk_dir_slowpath); DEFINE_XCHK_PPTR_EVENT(xchk_dir_ultraslowpath); +DEFINE_XCHK_PPTR_EVENT(xchk_parent_defer); +DEFINE_XCHK_PPTR_EVENT(xchk_parent_slowpath); +DEFINE_XCHK_PPTR_EVENT(xchk_parent_ultraslowpath); /* repair tracepoints */ #if IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) From e7420e75ef04787bc51688fc9bbca7da4d164a1e Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:06 -0700 Subject: [PATCH 0459/1702] xfs: remove some boilerplate from xfs_attr_set In preparation for online/offline repair wanting to use xfs_attr_set, move some of the boilerplate out of this function into the callers. Repair can initialize the da_args completely, and the userspace flag handling/twisting goes away once we move it to xfs_attr_change. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 33 ++++++++++++--------------------- fs/xfs/scrub/attr_repair.c | 4 ++++ fs/xfs/xfs_xattr.c | 24 ++++++++++++++++++++++-- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 8c283e5c24702..df8418671c379 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -948,6 +948,16 @@ xfs_attr_lookup( return error; } +/* + * Make a change to the xattr structure. + * + * The caller must have initialized @args, attached dquots, and must not hold + * any ILOCKs. + * + * Returns -EEXIST for XFS_ATTRUPDATE_CREATE if the name already exists. + * Returns -ENOATTR for XFS_ATTRUPDATE_REMOVE if the name does not exist. + * Returns 0 on success, or a negative errno if something else went wrong. + */ int xfs_attr_set( struct xfs_da_args *args, @@ -961,27 +971,7 @@ xfs_attr_set( int rmt_blks = 0; unsigned int total; - if (xfs_is_shutdown(dp->i_mount)) - return -EIO; - - error = xfs_qm_dqattach(dp); - if (error) - return error; - - if (!args->owner) - args->owner = args->dp->i_ino; - args->geo = mp->m_attr_geo; - args->whichfork = XFS_ATTR_FORK; - xfs_attr_sethash(args); - - /* - * We have no control over the attribute names that userspace passes us - * to remove, so we have to allow the name lookup prior to attribute - * removal to fail as well. Preserve the logged flag, since we need - * to pass that through to the logging code. - */ - args->op_flags = XFS_DA_OP_OKNOENT | - (args->op_flags & XFS_DA_OP_LOGGED); + ASSERT(!args->trans); switch (op) { case XFS_ATTRUPDATE_UPSERT: @@ -1076,6 +1066,7 @@ xfs_attr_set( error = xfs_trans_commit(args->trans); out_unlock: xfs_iunlock(dp, XFS_ILOCK_EXCL); + args->trans = NULL; return error; out_trans_cancel: diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c index 8b89c112c492f..67c0ec0d1dbba 100644 --- a/fs/xfs/scrub/attr_repair.c +++ b/fs/xfs/scrub/attr_repair.c @@ -558,6 +558,9 @@ xrep_xattr_insert_rec( .namelen = key->namelen, .valuelen = key->valuelen, .owner = rx->sc->ip->i_ino, + .geo = rx->sc->mp->m_attr_geo, + .whichfork = XFS_ATTR_FORK, + .op_flags = XFS_DA_OP_OKNOENT, }; struct xchk_xattr_buf *ab = rx->sc->buf; int error; @@ -602,6 +605,7 @@ xrep_xattr_insert_rec( * xfs_attr_set creates and commits its own transaction. If the attr * already exists, we'll just drop it during the rebuild. */ + xfs_attr_sethash(&args); error = xfs_attr_set(&args, XFS_ATTRUPDATE_CREATE); if (error == -EEXIST) error = 0; diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c index b43f7081b0f46..bbdbe9026658d 100644 --- a/fs/xfs/xfs_xattr.c +++ b/fs/xfs/xfs_xattr.c @@ -17,6 +17,7 @@ #include "xfs_acl.h" #include "xfs_log.h" #include "xfs_xattr.h" +#include "xfs_quota.h" #include @@ -70,7 +71,9 @@ xfs_attr_want_log_assist( /* * Set or remove an xattr, having grabbed the appropriate logging resources - * prior to calling libxfs. + * prior to calling libxfs. Callers of this function are only required to + * initialize the inode, attr_filter, name, namelen, value, and valuelen fields + * of @args. */ int xfs_attr_change( @@ -80,7 +83,19 @@ xfs_attr_change( struct xfs_mount *mp = args->dp->i_mount; int error; - ASSERT(!(args->op_flags & XFS_DA_OP_LOGGED)); + if (xfs_is_shutdown(mp)) + return -EIO; + + error = xfs_qm_dqattach(args->dp); + if (error) + return error; + + /* + * We have no control over the attribute names that userspace passes us + * to remove, so we have to allow the name lookup prior to attribute + * removal to fail as well. + */ + args->op_flags = XFS_DA_OP_OKNOENT; if (xfs_attr_want_log_assist(mp)) { error = xfs_attr_grab_log_assist(mp); @@ -90,6 +105,11 @@ xfs_attr_change( args->op_flags |= XFS_DA_OP_LOGGED; } + args->owner = args->dp->i_ino; + args->geo = mp->m_attr_geo; + args->whichfork = XFS_ATTR_FORK; + xfs_attr_sethash(args); + return xfs_attr_set(args, op); } From bf61c36a45d4c215994699a7a06a00c58d22e8a2 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:07 -0700 Subject: [PATCH 0460/1702] xfs: make the reserved block permission flag explicit in xfs_attr_set Make the use of reserved blocks an explicit parameter to xfs_attr_set. Userspace setting XFS_ATTR_ROOT attrs should continue to be able to use it, but for online repairs we can back out and therefore do not care. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 6 +++--- fs/xfs/libxfs/xfs_attr.h | 2 +- fs/xfs/scrub/attr_repair.c | 2 +- fs/xfs/xfs_xattr.c | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index df8418671c379..c98145596f029 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -952,7 +952,7 @@ xfs_attr_lookup( * Make a change to the xattr structure. * * The caller must have initialized @args, attached dquots, and must not hold - * any ILOCKs. + * any ILOCKs. Reserved data blocks may be used if @rsvd is set. * * Returns -EEXIST for XFS_ATTRUPDATE_CREATE if the name already exists. * Returns -ENOATTR for XFS_ATTRUPDATE_REMOVE if the name does not exist. @@ -961,12 +961,12 @@ xfs_attr_lookup( int xfs_attr_set( struct xfs_da_args *args, - enum xfs_attr_update op) + enum xfs_attr_update op, + bool rsvd) { struct xfs_inode *dp = args->dp; struct xfs_mount *mp = dp->i_mount; struct xfs_trans_res tres; - bool rsvd = (args->attr_filter & XFS_ATTR_ROOT); int error, local; int rmt_blks = 0; unsigned int total; diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h index d12583dd7eecb..43dee4cbaab25 100644 --- a/fs/xfs/libxfs/xfs_attr.h +++ b/fs/xfs/libxfs/xfs_attr.h @@ -558,7 +558,7 @@ enum xfs_attr_update { XFS_ATTRUPDATE_REPLACE, /* set value, fail if attr does not exist */ }; -int xfs_attr_set(struct xfs_da_args *args, enum xfs_attr_update op); +int xfs_attr_set(struct xfs_da_args *args, enum xfs_attr_update op, bool rsvd); int xfs_attr_set_iter(struct xfs_attr_intent *attr); int xfs_attr_remove_iter(struct xfs_attr_intent *attr); bool xfs_attr_check_namespace(unsigned int attr_flags); diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c index 67c0ec0d1dbba..cbcc446d5119b 100644 --- a/fs/xfs/scrub/attr_repair.c +++ b/fs/xfs/scrub/attr_repair.c @@ -606,7 +606,7 @@ xrep_xattr_insert_rec( * already exists, we'll just drop it during the rebuild. */ xfs_attr_sethash(&args); - error = xfs_attr_set(&args, XFS_ATTRUPDATE_CREATE); + error = xfs_attr_set(&args, XFS_ATTRUPDATE_CREATE, false); if (error == -EEXIST) error = 0; diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c index bbdbe9026658d..ab3d22f662f28 100644 --- a/fs/xfs/xfs_xattr.c +++ b/fs/xfs/xfs_xattr.c @@ -110,7 +110,7 @@ xfs_attr_change( args->whichfork = XFS_ATTR_FORK; xfs_attr_sethash(args); - return xfs_attr_set(args, op); + return xfs_attr_set(args, op, args->attr_filter & XFS_ATTR_ROOT); } From 77ede5f44b0d86c2ec812442846f512884009766 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:05 -0700 Subject: [PATCH 0461/1702] xfs: walk directory parent pointers to determine backref count If the filesystem has parent pointers enabled, walk the parent pointers of subdirectories to determine the true backref count. In theory each subdir should have a single parent reachable via dotdot, but in the case of (corrupt) subdirs with multiple parents, we need to keep the link counts high enough that the directory loop detector will be able to correct the multiple parents problems. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/common.h | 1 + fs/xfs/scrub/nlinks.c | 85 +++++++++++++++++++++++++++++++++++- fs/xfs/scrub/nlinks_repair.c | 2 + fs/xfs/scrub/parent.c | 61 ++++++++++++++++++++++++++ fs/xfs/scrub/trace.c | 1 + fs/xfs/scrub/trace.h | 28 ++++++++++++ 6 files changed, 177 insertions(+), 1 deletion(-) diff --git a/fs/xfs/scrub/common.h b/fs/xfs/scrub/common.h index 89f7bbec887ed..e00466f404829 100644 --- a/fs/xfs/scrub/common.h +++ b/fs/xfs/scrub/common.h @@ -212,6 +212,7 @@ static inline bool xchk_skip_xref(struct xfs_scrub_metadata *sm) } bool xchk_dir_looks_zapped(struct xfs_inode *dp); +bool xchk_pptr_looks_zapped(struct xfs_inode *ip); #ifdef CONFIG_XFS_ONLINE_REPAIR /* Decide if a repair is required. */ diff --git a/fs/xfs/scrub/nlinks.c b/fs/xfs/scrub/nlinks.c index fcb9c473f372e..d27b32e6f33db 100644 --- a/fs/xfs/scrub/nlinks.c +++ b/fs/xfs/scrub/nlinks.c @@ -18,6 +18,7 @@ #include "xfs_dir2.h" #include "xfs_dir2_priv.h" #include "xfs_ag.h" +#include "xfs_parent.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/repair.h" @@ -29,6 +30,7 @@ #include "scrub/trace.h" #include "scrub/readdir.h" #include "scrub/tempfile.h" +#include "scrub/listxattr.h" /* * Live Inode Link Count Checking @@ -272,12 +274,17 @@ xchk_nlinks_collect_dirent( * number of parents of the root directory. * * Otherwise, increment the number of backrefs pointing back to ino. + * + * If the filesystem has parent pointers, we walk the pptrs to + * determine the backref count. */ if (dotdot) { if (dp == sc->mp->m_rootip) error = xchk_nlinks_update_incore(xnc, ino, 1, 0, 0); - else + else if (!xfs_has_parent(sc->mp)) error = xchk_nlinks_update_incore(xnc, ino, 0, 1, 0); + else + error = 0; if (error) goto out_unlock; } @@ -314,6 +321,61 @@ xchk_nlinks_collect_dirent( return error; } +/* Bump the backref count for the inode referenced by this parent pointer. */ +STATIC int +xchk_nlinks_collect_pptr( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xfs_name xname = { + .name = name, + .len = namelen, + }; + struct xchk_nlink_ctrs *xnc = priv; + const struct xfs_parent_rec *pptr_rec = value; + xfs_ino_t parent_ino; + int error; + + /* Update the shadow link counts if we haven't already failed. */ + + if (xchk_iscan_aborted(&xnc->collect_iscan)) { + error = -ECANCELED; + goto out_incomplete; + } + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, &parent_ino, NULL); + if (error) + return error; + + trace_xchk_nlinks_collect_pptr(sc->mp, ip, &xname, pptr_rec); + + mutex_lock(&xnc->lock); + + error = xchk_nlinks_update_incore(xnc, parent_ino, 0, 1, 0); + if (error) + goto out_unlock; + + mutex_unlock(&xnc->lock); + return 0; + +out_unlock: + mutex_unlock(&xnc->lock); + xchk_iscan_abort(&xnc->collect_iscan); +out_incomplete: + xchk_set_incomplete(sc); + return error; +} + /* Walk a directory to bump the observed link counts of the children. */ STATIC int xchk_nlinks_collect_dir( @@ -360,6 +422,27 @@ xchk_nlinks_collect_dir( if (error) goto out_abort; + /* Walk the parent pointers to get real backref counts. */ + if (xfs_has_parent(sc->mp)) { + /* + * If the extended attributes look as though they has been + * zapped by the inode record repair code, we cannot scan for + * parent pointers. + */ + if (xchk_pptr_looks_zapped(dp)) { + error = -EBUSY; + goto out_unlock; + } + + error = xchk_xattr_walk(sc, dp, xchk_nlinks_collect_pptr, xnc); + if (error == -ECANCELED) { + error = 0; + goto out_unlock; + } + if (error) + goto out_abort; + } + xchk_iscan_mark_visited(&xnc->collect_iscan, dp); goto out_unlock; diff --git a/fs/xfs/scrub/nlinks_repair.c b/fs/xfs/scrub/nlinks_repair.c index 83f8637bb08fd..78d0f650fe897 100644 --- a/fs/xfs/scrub/nlinks_repair.c +++ b/fs/xfs/scrub/nlinks_repair.c @@ -18,6 +18,8 @@ #include "xfs_ialloc.h" #include "xfs_sb.h" #include "xfs_ag.h" +#include "xfs_dir2.h" +#include "xfs_parent.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/repair.h" diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c index f14e6f643fb60..068691434be17 100644 --- a/fs/xfs/scrub/parent.c +++ b/fs/xfs/scrub/parent.c @@ -873,3 +873,64 @@ xchk_parent( return error; } + +/* + * Decide if this file's extended attributes (and therefore its parent + * pointers) have been zapped to satisfy the inode and ifork verifiers. + * Checking and repairing should be postponed until the extended attribute + * structure is fixed. + */ +bool +xchk_pptr_looks_zapped( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + struct inode *inode = VFS_I(ip); + + ASSERT(xfs_has_parent(mp)); + + /* + * Temporary files that cannot be linked into the directory tree do not + * have attr forks because they cannot ever have parents. + */ + if (inode->i_nlink == 0 && !(inode->i_state & I_LINKABLE)) + return false; + + /* + * Directory tree roots do not have parents, so the expected outcome + * of a parent pointer scan is always the empty set. It's safe to scan + * them even if the attr fork was zapped. + */ + if (ip == mp->m_rootip) + return false; + + /* + * Metadata inodes are all rooted in the superblock and do not have + * any parents. Hence the attr fork will not be initialized, but + * there are no parent pointers that might have been zapped. + */ + if (xfs_is_metadata_inode(ip)) + return false; + + /* + * Linked and linkable non-rootdir files should always have an + * attribute fork because that is where parent pointers are + * stored. If the fork is absent, something is amiss. + */ + if (!xfs_inode_has_attr_fork(ip)) + return true; + + /* Repair zapped this file's attr fork a short time ago */ + if (xfs_ifork_zapped(ip, XFS_ATTR_FORK)) + return true; + + /* + * If the dinode repair found a bad attr fork, it will reset the fork + * to extents format with zero records and wait for the bmapbta + * scrubber to reconstruct the block mappings. The extended attribute + * structure always contain some content when parent pointers are + * enabled, so this is a clear sign of a zapped attr fork. + */ + return ip->i_af.if_format == XFS_DINODE_FMT_EXTENTS && + ip->i_af.if_nextents == 0; +} diff --git a/fs/xfs/scrub/trace.c b/fs/xfs/scrub/trace.c index b2ce7b22cad34..4a8cc2c98d997 100644 --- a/fs/xfs/scrub/trace.c +++ b/fs/xfs/scrub/trace.c @@ -19,6 +19,7 @@ #include "xfs_da_format.h" #include "xfs_dir2.h" #include "xfs_rmap.h" +#include "xfs_parent.h" #include "scrub/scrub.h" #include "scrub/xfile.h" #include "scrub/xfarray.h" diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 97a106519b531..3e726610b9e32 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -26,6 +26,7 @@ struct xchk_iscan; struct xchk_nlink; struct xchk_fscounters; struct xfs_rmap_update_params; +struct xfs_parent_rec; /* * ftrace's __print_symbolic requires that all enum values be wrapped in the @@ -1363,6 +1364,33 @@ TRACE_EVENT(xchk_nlinks_collect_dirent, __get_str(name)) ); +TRACE_EVENT(xchk_nlinks_collect_pptr, + TP_PROTO(struct xfs_mount *mp, struct xfs_inode *dp, + const struct xfs_name *name, + const struct xfs_parent_rec *pptr), + TP_ARGS(mp, dp, name, pptr), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dir) + __field(xfs_ino_t, ino) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->dir = dp->i_ino; + __entry->ino = be64_to_cpu(pptr->p_ino); + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d dir 0x%llx -> ino 0x%llx name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dir, + __entry->ino, + __entry->namelen, + __get_str(name)) +); + TRACE_EVENT(xchk_nlinks_collect_metafile, TP_PROTO(struct xfs_mount *mp, xfs_ino_t ino), TP_ARGS(mp, ino), From 086e934fe9c741f25a269ae74cc891eaf3f5c4e2 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:08 -0700 Subject: [PATCH 0462/1702] xfs: salvage parent pointers when rebuilding xattr structures When we're salvaging extended attributes, make sure we validate the ones that claim to be parent pointers before adding them to the salvage pile. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/attr_repair.c | 36 +++++++++++++++++++++++++++--------- fs/xfs/scrub/trace.h | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c index cbcc446d5119b..48443e88e298b 100644 --- a/fs/xfs/scrub/attr_repair.c +++ b/fs/xfs/scrub/attr_repair.c @@ -28,6 +28,7 @@ #include "xfs_exchmaps.h" #include "xfs_exchrange.h" #include "xfs_acl.h" +#include "xfs_parent.h" #include "scrub/xfs_scrub.h" #include "scrub/scrub.h" #include "scrub/common.h" @@ -127,6 +128,9 @@ xrep_xattr_want_salvage( return false; if (valuelen > XATTR_SIZE_MAX || valuelen < 0) return false; + if (attr_flags & XFS_ATTR_PARENT) + return xfs_parent_valuecheck(rx->sc->mp, value, valuelen); + return true; } @@ -154,14 +158,21 @@ xrep_xattr_salvage_key( * Truncate the name to the first character that would trip namecheck. * If we no longer have a name after that, ignore this attribute. */ - while (i < namelen && name[i] != 0) - i++; - if (i == 0) - return 0; - key.namelen = i; + if (flags & XFS_ATTR_PARENT) { + key.namelen = namelen; + + trace_xrep_xattr_salvage_pptr(rx->sc->ip, flags, name, + key.namelen, value, valuelen); + } else { + while (i < namelen && name[i] != 0) + i++; + if (i == 0) + return 0; + key.namelen = i; - trace_xrep_xattr_salvage_rec(rx->sc->ip, flags, name, key.namelen, - valuelen); + trace_xrep_xattr_salvage_rec(rx->sc->ip, flags, name, + key.namelen, valuelen); + } error = xfblob_store(rx->xattr_blobs, &key.name_cookie, name, key.namelen); @@ -598,8 +609,15 @@ xrep_xattr_insert_rec( ab->name[key->namelen] = 0; - trace_xrep_xattr_insert_rec(rx->sc->tempip, key->flags, ab->name, - key->namelen, key->valuelen); + if (key->flags & XFS_ATTR_PARENT) { + trace_xrep_xattr_insert_pptr(rx->sc->tempip, key->flags, + ab->name, key->namelen, ab->value, + key->valuelen); + args.op_flags |= XFS_DA_OP_LOGGED; + } else { + trace_xrep_xattr_insert_rec(rx->sc->tempip, key->flags, + ab->name, key->namelen, key->valuelen); + } /* * xfs_attr_set creates and commits its own transaction. If the attr diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 3e726610b9e32..4b968df3d840c 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2540,6 +2540,44 @@ DEFINE_EVENT(xrep_xattr_salvage_class, name, \ DEFINE_XREP_XATTR_SALVAGE_EVENT(xrep_xattr_salvage_rec); DEFINE_XREP_XATTR_SALVAGE_EVENT(xrep_xattr_insert_rec); +DECLARE_EVENT_CLASS(xrep_pptr_salvage_class, + TP_PROTO(struct xfs_inode *ip, unsigned int flags, const void *name, + unsigned int namelen, const void *value, unsigned int valuelen), + TP_ARGS(ip, flags, name, namelen, value, valuelen), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __field(unsigned int, namelen) + __dynamic_array(char, name, namelen) + ), + TP_fast_assign( + const struct xfs_parent_rec *rec = value; + + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = ip->i_ino; + __entry->parent_ino = be64_to_cpu(rec->p_ino); + __entry->parent_gen = be32_to_cpu(rec->p_gen); + __entry->namelen = namelen; + memcpy(__get_str(name), name, namelen); + ), + TP_printk("dev %d:%d ino 0x%llx parent_ino 0x%llx parent_gen 0x%x name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->parent_ino, + __entry->parent_gen, + __entry->namelen, + __get_str(name)) +) +#define DEFINE_XREP_PPTR_SALVAGE_EVENT(name) \ +DEFINE_EVENT(xrep_pptr_salvage_class, name, \ + TP_PROTO(struct xfs_inode *ip, unsigned int flags, const void *name, \ + unsigned int namelen, const void *value, unsigned int valuelen), \ + TP_ARGS(ip, flags, name, namelen, value, valuelen)) +DEFINE_XREP_PPTR_SALVAGE_EVENT(xrep_xattr_salvage_pptr); +DEFINE_XREP_PPTR_SALVAGE_EVENT(xrep_xattr_insert_pptr); + TRACE_EVENT(xrep_xattr_class, TP_PROTO(struct xfs_inode *ip, struct xfs_inode *arg_ip), TP_ARGS(ip, arg_ip), From 59a2af9086f0d60fc8de7346da67db7d764c7221 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:05 -0700 Subject: [PATCH 0463/1702] xfs: check parent pointer xattrs when scrubbing Check parent pointer xattrs as part of scrubbing xattrs. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/attr.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index 393ed36709b3a..b550f3e34ffc7 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -17,6 +17,7 @@ #include "xfs_attr.h" #include "xfs_attr_leaf.h" #include "xfs_attr_sf.h" +#include "xfs_parent.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/dabtree.h" @@ -208,6 +209,13 @@ xchk_xattr_actor( return -ECANCELED; } + /* Check parent pointer record. */ + if ((attr_flags & XFS_ATTR_PARENT) && + !xfs_parent_valuecheck(sc->mp, value, valuelen)) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, args.blkno); + return -ECANCELED; + } + /* * Try to allocate enough memory to extract the attr value. If that * doesn't work, return -EDEADLOCK as a signal to try again with a @@ -219,6 +227,14 @@ xchk_xattr_actor( if (error) return error; + /* + * Parent pointers are matched on attr name and value, so we must + * supply the xfs_parent_rec here when confirming that the dabtree + * indexing works correctly. + */ + if (attr_flags & XFS_ATTR_PARENT) + memcpy(ab->value, value, valuelen); + args.value = ab->value; /* From 5769aa41ee34d4d1cc2b35376107b8e9694698f0 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:09 -0700 Subject: [PATCH 0464/1702] xfs: add raw parent pointer apis to support repair Add a couple of utility functions to set or remove parent pointers from a file. These functions will be used by repair code, hence they skip the xattr logging that regular parent pointer updates use. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_dir2.c | 2 +- fs/xfs/libxfs/xfs_dir2.h | 2 +- fs/xfs/libxfs/xfs_parent.c | 64 ++++++++++++++++++++++++++++++++++++++ fs/xfs/libxfs/xfs_parent.h | 6 ++++ 4 files changed, 72 insertions(+), 2 deletions(-) diff --git a/fs/xfs/libxfs/xfs_dir2.c b/fs/xfs/libxfs/xfs_dir2.c index 9da99fa20c759..7634344dc5153 100644 --- a/fs/xfs/libxfs/xfs_dir2.c +++ b/fs/xfs/libxfs/xfs_dir2.c @@ -434,7 +434,7 @@ int xfs_dir_removename( struct xfs_trans *tp, struct xfs_inode *dp, - struct xfs_name *name, + const struct xfs_name *name, xfs_ino_t ino, xfs_extlen_t total) /* bmap's total block count */ { diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h index eb3a5c35025b5..b580a78bcf4fc 100644 --- a/fs/xfs/libxfs/xfs_dir2.h +++ b/fs/xfs/libxfs/xfs_dir2.h @@ -58,7 +58,7 @@ extern int xfs_dir_lookup(struct xfs_trans *tp, struct xfs_inode *dp, const struct xfs_name *name, xfs_ino_t *inum, struct xfs_name *ci_name); extern int xfs_dir_removename(struct xfs_trans *tp, struct xfs_inode *dp, - struct xfs_name *name, xfs_ino_t ino, + const struct xfs_name *name, xfs_ino_t ino, xfs_extlen_t tot); extern int xfs_dir_replace(struct xfs_trans *tp, struct xfs_inode *dp, const struct xfs_name *name, xfs_ino_t inum, diff --git a/fs/xfs/libxfs/xfs_parent.c b/fs/xfs/libxfs/xfs_parent.c index 1e53df5f83323..69366c44a7015 100644 --- a/fs/xfs/libxfs/xfs_parent.c +++ b/fs/xfs/libxfs/xfs_parent.c @@ -313,3 +313,67 @@ xfs_parent_lookup( xfs_parent_da_args_init(scratch, tp, pptr, ip, ip->i_ino, parent_name); return xfs_attr_get_ilocked(scratch); } + +/* Sanity-check a parent pointer before we try to perform repairs. */ +static inline bool +xfs_parent_sanity_check( + struct xfs_mount *mp, + const struct xfs_name *parent_name, + const struct xfs_parent_rec *pptr) +{ + if (!xfs_parent_namecheck(XFS_ATTR_PARENT, parent_name->name, + parent_name->len)) + return false; + + if (!xfs_parent_valuecheck(mp, pptr, sizeof(*pptr))) + return false; + + return true; +} + + +/* + * Attach the parent pointer (@parent_name -> @pptr) to @ip immediately. + * Caller must not have a transaction or hold the ILOCK. This is for + * specialized repair functions only. The scratchpad need not be initialized. + */ +int +xfs_parent_set( + struct xfs_inode *ip, + xfs_ino_t owner, + const struct xfs_name *parent_name, + struct xfs_parent_rec *pptr, + struct xfs_da_args *scratch) +{ + if (!xfs_parent_sanity_check(ip->i_mount, parent_name, pptr)) { + ASSERT(0); + return -EFSCORRUPTED; + } + + memset(scratch, 0, sizeof(struct xfs_da_args)); + xfs_parent_da_args_init(scratch, NULL, pptr, ip, owner, parent_name); + return xfs_attr_set(scratch, XFS_ATTRUPDATE_CREATE, false); +} + +/* + * Remove the parent pointer (@parent_name -> @pptr) from @ip immediately. + * Caller must not have a transaction or hold the ILOCK. This is for + * specialized repair functions only. The scratchpad need not be initialized. + */ +int +xfs_parent_unset( + struct xfs_inode *ip, + xfs_ino_t owner, + const struct xfs_name *parent_name, + struct xfs_parent_rec *pptr, + struct xfs_da_args *scratch) +{ + if (!xfs_parent_sanity_check(ip->i_mount, parent_name, pptr)) { + ASSERT(0); + return -EFSCORRUPTED; + } + + memset(scratch, 0, sizeof(struct xfs_da_args)); + xfs_parent_da_args_init(scratch, NULL, pptr, ip, owner, parent_name); + return xfs_attr_set(scratch, XFS_ATTRUPDATE_REMOVE, false); +} diff --git a/fs/xfs/libxfs/xfs_parent.h b/fs/xfs/libxfs/xfs_parent.h index 97788582321a6..b8036527cdc73 100644 --- a/fs/xfs/libxfs/xfs_parent.h +++ b/fs/xfs/libxfs/xfs_parent.h @@ -100,5 +100,11 @@ int xfs_parent_from_attr(struct xfs_mount *mp, unsigned int attr_flags, int xfs_parent_lookup(struct xfs_trans *tp, struct xfs_inode *ip, const struct xfs_name *name, struct xfs_parent_rec *pptr, struct xfs_da_args *scratch); +int xfs_parent_set(struct xfs_inode *ip, xfs_ino_t owner, + const struct xfs_name *name, struct xfs_parent_rec *pptr, + struct xfs_da_args *scratch); +int xfs_parent_unset(struct xfs_inode *ip, xfs_ino_t owner, + const struct xfs_name *name, struct xfs_parent_rec *pptr, + struct xfs_da_args *scratch); #endif /* __XFS_PARENT_H__ */ From f88320b698ad099a2f742adfb9f87177bfffe0c5 Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Tue, 23 Apr 2024 07:15:51 -0700 Subject: [PATCH 0465/1702] RDMA/mana_ib: Fix missing ret value Set ret to -ENODEV when netdev_master_upper_dev_get_rcu returns NULL. Fixes: 8b184e4f1c32 ("RDMA/mana_ib: Enable RoCE on port 1") Link: https://lore.kernel.org/r/1713881751-21621-1-git-send-email-kotaranov@linux.microsoft.com Signed-off-by: Konstantin Taranov Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mana/device.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c index fca4d0d85c645..7e09ceb3da537 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -87,6 +87,7 @@ static int mana_ib_probe(struct auxiliary_device *adev, upper_ndev = netdev_master_upper_dev_get_rcu(mc->ports[0]); if (!upper_ndev) { rcu_read_unlock(); + ret = -ENODEV; ibdev_err(&dev->ib_dev, "Failed to get master netdev"); goto free_ib_device; } From ee2929118884d7f6e8af35e166d5bc01379bf63f Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Tue, 16 Apr 2024 17:18:59 +0200 Subject: [PATCH 0466/1702] dt-bindings: interrupt-controller: renesas,irqc: Add r8a779g0 support Document support for the Interrupt Controller for External Devices (INT-EX) in the Renesas R-Car V4M (R8A779H0) SoC. Signed-off-by: Geert Uytterhoeven Acked-by: Conor Dooley Link: https://lore.kernel.org/r/264cffccfbb1f92657420f5f869236b06a97d958.1713280616.git.geert+renesas@glider.be Signed-off-by: Rob Herring (Arm) --- .../devicetree/bindings/interrupt-controller/renesas,irqc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/interrupt-controller/renesas,irqc.yaml b/Documentation/devicetree/bindings/interrupt-controller/renesas,irqc.yaml index b417341fc8ae0..fb3c29e813499 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/renesas,irqc.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/renesas,irqc.yaml @@ -39,6 +39,7 @@ properties: - renesas,intc-ex-r8a779a0 # R-Car V3U - renesas,intc-ex-r8a779f0 # R-Car S4-8 - renesas,intc-ex-r8a779g0 # R-Car V4H + - renesas,intc-ex-r8a779h0 # R-Car V4M - const: renesas,irqc '#interrupt-cells': From d47bca77bf3ab475c33b3929c33c80aeb49df35c Mon Sep 17 00:00:00 2001 From: Chris Morgan Date: Thu, 18 Apr 2024 13:16:14 -0500 Subject: [PATCH 0467/1702] dt-bindings: irq: sun7i-nmi: Add binding for the H616 NMI controller Add binding for the H616 NMI controller. Signed-off-by: Chris Morgan Acked-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240418181615.1370179-2-macroalpha82@gmail.com Signed-off-by: Rob Herring (Arm) --- .../interrupt-controller/allwinner,sun7i-a20-sc-nmi.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun7i-a20-sc-nmi.yaml b/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun7i-a20-sc-nmi.yaml index 83603180d8d9c..f49b43f45f3d9 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun7i-a20-sc-nmi.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun7i-a20-sc-nmi.yaml @@ -25,12 +25,12 @@ properties: - const: allwinner,sun6i-a31-sc-nmi deprecated: true - const: allwinner,sun7i-a20-sc-nmi - - items: - - const: allwinner,sun8i-v3s-nmi - - const: allwinner,sun9i-a80-nmi - const: allwinner,sun9i-a80-nmi - items: - - const: allwinner,sun50i-a100-nmi + - enum: + - allwinner,sun8i-v3s-nmi + - allwinner,sun50i-a100-nmi + - allwinner,sun50i-h616-nmi - const: allwinner,sun9i-a80-nmi reg: From 638887e128d422f727c128a1f60967fe091f47cf Mon Sep 17 00:00:00 2001 From: Andrew Jeffery Date: Wed, 3 Apr 2024 12:34:39 +1030 Subject: [PATCH 0468/1702] dt-bindings: watchdog: aspeed,ast2400-wdt: Convert to DT schema Squash warnings such as: ``` arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-galaxy100.dtb: /ahb/apb@1e600000/watchdog@1e785000: failed to match any schema with compatible: ['aspeed,ast2400-wdt'] ``` The schema binding additionally defines the clocks property over the prose binding to align with use of the node in the DTS files. Signed-off-by: Andrew Jeffery Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20240403020439.418788-1-andrew@codeconstruct.com.au Signed-off-by: Rob Herring (Arm) --- .../bindings/watchdog/aspeed,ast2400-wdt.yaml | 142 ++++++++++++++++++ .../bindings/watchdog/aspeed-wdt.txt | 73 --------- 2 files changed, 142 insertions(+), 73 deletions(-) create mode 100644 Documentation/devicetree/bindings/watchdog/aspeed,ast2400-wdt.yaml delete mode 100644 Documentation/devicetree/bindings/watchdog/aspeed-wdt.txt diff --git a/Documentation/devicetree/bindings/watchdog/aspeed,ast2400-wdt.yaml b/Documentation/devicetree/bindings/watchdog/aspeed,ast2400-wdt.yaml new file mode 100644 index 0000000000000..be78a9865584d --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/aspeed,ast2400-wdt.yaml @@ -0,0 +1,142 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/watchdog/aspeed,ast2400-wdt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Aspeed watchdog timer controllers + +maintainers: + - Andrew Jeffery + +properties: + compatible: + enum: + - aspeed,ast2400-wdt + - aspeed,ast2500-wdt + - aspeed,ast2600-wdt + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + description: > + The clock used to drive the watchdog counter. From the AST2500 no source + other than the 1MHz clock can be selected, so the clocks property is + optional. + + aspeed,reset-type: + $ref: /schemas/types.yaml#/definitions/string + enum: + - cpu + - soc + - system + - none + default: system + description: > + The watchdog can be programmed to generate one of three different types of + reset when a timeout occcurs. + + Specifying 'cpu' will only reset the processor on a timeout event. + + Specifying 'soc' will reset a configurable subset of the SoC's controllers + on a timeout event. Controllers critical to the SoC's operation may remain + untouched. The set of SoC controllers to reset may be specified via the + aspeed,reset-mask property if the node has the aspeed,ast2500-wdt or + aspeed,ast2600-wdt compatible. + + Specifying 'system' will reset all controllers on a timeout event, as if + EXTRST had been asserted. + + Specifying 'none' will cause the timeout event to have no reset effect. + Another watchdog engine on the chip must be used for chip reset operations. + + aspeed,alt-boot: + $ref: /schemas/types.yaml#/definitions/flag + description: > + Direct the watchdog to configure the SoC to boot from the alternative boot + region if a timeout occurs. + + aspeed,external-signal: + $ref: /schemas/types.yaml#/definitions/flag + description: > + Assert the timeout event on an external signal pin associated with the + watchdog controller instance. The pin must be muxed appropriately. + + aspeed,ext-pulse-duration: + $ref: /schemas/types.yaml#/definitions/uint32 + description: > + The duration, in microseconds, of the pulse emitted on the external signal + pin. + + aspeed,ext-push-pull: + $ref: /schemas/types.yaml#/definitions/flag + description: > + If aspeed,external-signal is specified in the node, set the external + signal pin's drive type to push-pull. If aspeed,ext-push-pull is not + specified then the pin is configured as open-drain. + + aspeed,ext-active-high: + $ref: /schemas/types.yaml#/definitions/flag + description: > + If both aspeed,external-signal and aspeed,ext-push-pull are specified in + the node, set the pulse polarity to active-high. If aspeed,ext-active-high + is not specified then the pin is configured as active-low. + + aspeed,reset-mask: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 2 + description: > + A bitmask indicating which peripherals will be reset if the watchdog + timer expires. On AST2500 SoCs this should be a single word defined using + the AST2500_WDT_RESET_* macros; on AST2600 SoCs this should be a two-word + array with the first word defined using the AST2600_WDT_RESET1_* macros, + and the second word defined using the AST2600_WDT_RESET2_* macros. + +required: + - compatible + - reg + +allOf: + - if: + anyOf: + - required: + - aspeed,ext-push-pull + - required: + - aspeed,ext-active-high + - required: + - aspeed,reset-mask + then: + properties: + compatible: + enum: + - aspeed,ast2500-wdt + - aspeed,ast2600-wdt + - if: + required: + - aspeed,ext-active-high + then: + required: + - aspeed,ext-push-pull + +additionalProperties: false + +examples: + - | + watchdog@1e785000 { + compatible = "aspeed,ast2400-wdt"; + reg = <0x1e785000 0x1c>; + aspeed,reset-type = "system"; + aspeed,external-signal; + }; + - | + #include + watchdog@1e785040 { + compatible = "aspeed,ast2600-wdt"; + reg = <0x1e785040 0x40>; + aspeed,reset-type = "soc"; + aspeed,reset-mask = ; + }; diff --git a/Documentation/devicetree/bindings/watchdog/aspeed-wdt.txt b/Documentation/devicetree/bindings/watchdog/aspeed-wdt.txt deleted file mode 100644 index 3208adb3e52e8..0000000000000 --- a/Documentation/devicetree/bindings/watchdog/aspeed-wdt.txt +++ /dev/null @@ -1,73 +0,0 @@ -Aspeed Watchdog Timer - -Required properties: - - compatible: must be one of: - - "aspeed,ast2400-wdt" - - "aspeed,ast2500-wdt" - - "aspeed,ast2600-wdt" - - - reg: physical base address of the controller and length of memory mapped - region - -Optional properties: - - - aspeed,reset-type = "cpu|soc|system|none" - - Reset behavior - Whenever a timeout occurs the watchdog can be programmed - to generate one of three different, mutually exclusive, types of resets. - - Type "none" can be specified to indicate that no resets are to be done. - This is useful in situations where another watchdog engine on chip is - to perform the reset. - - If 'aspeed,reset-type=' is not specified the default is to enable system - reset. - - Reset types: - - - cpu: Reset CPU on watchdog timeout - - - soc: Reset 'System on Chip' on watchdog timeout - - - system: Reset system on watchdog timeout - - - none: No reset is performed on timeout. Assumes another watchdog - engine is responsible for this. - - - aspeed,alt-boot: If property is present then boot from alternate block. - - aspeed,external-signal: If property is present then signal is sent to - external reset counter (only WDT1 and WDT2). If not - specified no external signal is sent. - - aspeed,ext-pulse-duration: External signal pulse duration in microseconds - -Optional properties for AST2500-compatible watchdogs: - - aspeed,ext-push-pull: If aspeed,external-signal is present, set the pin's - drive type to push-pull. The default is open-drain. - - aspeed,ext-active-high: If aspeed,external-signal is present and and the pin - is configured as push-pull, then set the pulse - polarity to active-high. The default is active-low. - -Optional properties for AST2500- and AST2600-compatible watchdogs: - - aspeed,reset-mask: A bitmask indicating which peripherals will be reset if - the watchdog timer expires. On AST2500 this should be a - single word defined using the AST2500_WDT_RESET_* macros; - on AST2600 this should be a two-word array with the first - word defined using the AST2600_WDT_RESET1_* macros and the - second word defined using the AST2600_WDT_RESET2_* macros. - -Examples: - - wdt1: watchdog@1e785000 { - compatible = "aspeed,ast2400-wdt"; - reg = <0x1e785000 0x1c>; - aspeed,reset-type = "system"; - aspeed,external-signal; - }; - - #include - wdt2: watchdog@1e785040 { - compatible = "aspeed,ast2600-wdt"; - reg = <0x1e785040 0x40>; - aspeed,reset-mask = ; - }; From 669430b183fccb3a8b39000a1e9dfb3a2d3028ce Mon Sep 17 00:00:00 2001 From: Saravana Kannan Date: Wed, 17 Apr 2024 13:07:37 -0700 Subject: [PATCH 0469/1702] of: property: fw_devlink: Add support for "power-supplies" binding Add support for parsing power-supplies binding so that fw_devlink can enforce the dependency. Signed-off-by: Saravana Kannan Cc: Sebastian Reichel Link: https://lore.kernel.org/r/20240417200738.1370896-1-saravanak@google.com Signed-off-by: Rob Herring (Arm) --- drivers/of/property.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/of/property.c b/drivers/of/property.c index c1be194412bf6..4ede45001025c 100644 --- a/drivers/of/property.c +++ b/drivers/of/property.c @@ -1242,6 +1242,7 @@ DEFINE_SIMPLE_PROP(backlight, "backlight", NULL) DEFINE_SIMPLE_PROP(panel, "panel", NULL) DEFINE_SIMPLE_PROP(msi_parent, "msi-parent", "#msi-cells") DEFINE_SIMPLE_PROP(post_init_providers, "post-init-providers", NULL) +DEFINE_SIMPLE_PROP(power_supplies, "power-supplies", NULL) DEFINE_SUFFIX_PROP(regulators, "-supply", NULL) DEFINE_SUFFIX_PROP(gpio, "-gpio", "#gpio-cells") @@ -1347,6 +1348,7 @@ static const struct supplier_bindings of_supplier_bindings[] = { { .parse_prop = parse_backlight, }, { .parse_prop = parse_panel, }, { .parse_prop = parse_msi_parent, }, + { .parse_prop = parse_power_supplies, }, { .parse_prop = parse_gpio_compat, }, { .parse_prop = parse_interrupts, }, { .parse_prop = parse_regulators, }, From 849ed9d414d04e369bccc2278d75becde9e40e0f Mon Sep 17 00:00:00 2001 From: Luca Weiss Date: Sun, 18 Feb 2024 21:57:25 +0100 Subject: [PATCH 0470/1702] dt-bindings: clock: qcom,hfpll: Convert to YAML Convert the .txt documentation to .yaml with some adjustments. * APQ8064/IPQ8064/MSM8960 compatibles are dropped since their HFPLLs are a part of GCC so there is no need for a separate compat entry. * Change the MSM8974 compatible to follow the updated naming schema. Theis compatible is not used upstream yet. * Add qcs404-hfpll. QCS404 currently uses qcom,hfpll. Mark that as deprecated since every SoC appears to need different driver data so "qcom,hfpll" makes no sense to keep Signed-off-by: Luca Weiss Reviewed-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240218-hfpll-yaml-v2-1-31543e0d6261@z3ntu.xyz Signed-off-by: Bjorn Andersson --- .../devicetree/bindings/clock/qcom,hfpll.txt | 63 ----------------- .../devicetree/bindings/clock/qcom,hfpll.yaml | 69 +++++++++++++++++++ 2 files changed, 69 insertions(+), 63 deletions(-) delete mode 100644 Documentation/devicetree/bindings/clock/qcom,hfpll.txt create mode 100644 Documentation/devicetree/bindings/clock/qcom,hfpll.yaml diff --git a/Documentation/devicetree/bindings/clock/qcom,hfpll.txt b/Documentation/devicetree/bindings/clock/qcom,hfpll.txt deleted file mode 100644 index 5769cbbe76bea..0000000000000 --- a/Documentation/devicetree/bindings/clock/qcom,hfpll.txt +++ /dev/null @@ -1,63 +0,0 @@ -High-Frequency PLL (HFPLL) - -PROPERTIES - -- compatible: - Usage: required - Value type: : - shall contain only one of the following. The generic - compatible "qcom,hfpll" should be also included. - - "qcom,hfpll-ipq8064", "qcom,hfpll" - "qcom,hfpll-apq8064", "qcom,hfpll" - "qcom,hfpll-msm8974", "qcom,hfpll" - "qcom,hfpll-msm8960", "qcom,hfpll" - "qcom,msm8976-hfpll-a53", "qcom,hfpll" - "qcom,msm8976-hfpll-a72", "qcom,hfpll" - "qcom,msm8976-hfpll-cci", "qcom,hfpll" - -- reg: - Usage: required - Value type: - Definition: address and size of HPLL registers. An optional second - element specifies the address and size of the alias - register region. - -- clocks: - Usage: required - Value type: - Definition: reference to the xo clock. - -- clock-names: - Usage: required - Value type: - Definition: must be "xo". - -- clock-output-names: - Usage: required - Value type: - Definition: Name of the PLL. Typically hfpllX where X is a CPU number - starting at 0. Otherwise hfpll_Y where Y is more specific - such as "l2". - -Example: - -1) An HFPLL for the L2 cache. - - clock-controller@f9016000 { - compatible = "qcom,hfpll-ipq8064", "qcom,hfpll"; - reg = <0xf9016000 0x30>; - clocks = <&xo_board>; - clock-names = "xo"; - clock-output-names = "hfpll_l2"; - }; - -2) An HFPLL for CPU0. This HFPLL has the alias register region. - - clock-controller@f908a000 { - compatible = "qcom,hfpll-ipq8064", "qcom,hfpll"; - reg = <0xf908a000 0x30>, <0xf900a000 0x30>; - clocks = <&xo_board>; - clock-names = "xo"; - clock-output-names = "hfpll0"; - }; diff --git a/Documentation/devicetree/bindings/clock/qcom,hfpll.yaml b/Documentation/devicetree/bindings/clock/qcom,hfpll.yaml new file mode 100644 index 0000000000000..8cb1c164f760f --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,hfpll.yaml @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,hfpll.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm High-Frequency PLL + +maintainers: + - Bjorn Andersson + +description: + The HFPLL is used as CPU PLL on various Qualcomm SoCs. + +properties: + compatible: + oneOf: + - enum: + - qcom,msm8974-hfpll + - qcom,msm8976-hfpll-a53 + - qcom,msm8976-hfpll-a72 + - qcom,msm8976-hfpll-cci + - qcom,qcs404-hfpll + - const: qcom,hfpll + deprecated: true + + reg: + items: + - description: HFPLL registers + - description: Alias register region + minItems: 1 + + '#clock-cells': + const: 0 + + clocks: + items: + - description: board XO clock + + clock-names: + items: + - const: xo + + clock-output-names: + description: + Name of the PLL. Typically hfpllX where X is a CPU number starting at 0. + Otherwise hfpll_Y where Y is more specific such as "l2". + maxItems: 1 + +required: + - compatible + - reg + - '#clock-cells' + - clocks + - clock-names + - clock-output-names + +additionalProperties: false + +examples: + - | + clock-controller@f908a000 { + compatible = "qcom,msm8974-hfpll"; + reg = <0xf908a000 0x30>, <0xf900a000 0x30>; + #clock-cells = <0>; + clock-output-names = "hfpll0"; + clocks = <&xo_board>; + clock-names = "xo"; + }; From 3db0f3b9ff5adb6a5e8564a32fadb2af1216810d Mon Sep 17 00:00:00 2001 From: Luca Weiss Date: Sun, 18 Feb 2024 21:57:26 +0100 Subject: [PATCH 0471/1702] clk: qcom: hfpll: Add QCS404-specific compatible It doesn't appear that the configuration is for the HFPLL is generic, so add a qcs404-specific compatible and rename the existing struct to qcs404. Keep qcom,hfpll in the driver for compatibility with old dtbs. Signed-off-by: Luca Weiss Link: https://lore.kernel.org/r/20240218-hfpll-yaml-v2-2-31543e0d6261@z3ntu.xyz Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/hfpll.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/clk/qcom/hfpll.c b/drivers/clk/qcom/hfpll.c index dac27e31ef609..b0b0cb074b4ab 100644 --- a/drivers/clk/qcom/hfpll.c +++ b/drivers/clk/qcom/hfpll.c @@ -14,7 +14,7 @@ #include "clk-regmap.h" #include "clk-hfpll.h" -static const struct hfpll_data hdata = { +static const struct hfpll_data qcs404 = { .mode_reg = 0x00, .l_reg = 0x04, .m_reg = 0x08, @@ -84,10 +84,12 @@ static const struct hfpll_data msm8976_cci = { }; static const struct of_device_id qcom_hfpll_match_table[] = { - { .compatible = "qcom,hfpll", .data = &hdata }, { .compatible = "qcom,msm8976-hfpll-a53", .data = &msm8976_a53 }, { .compatible = "qcom,msm8976-hfpll-a72", .data = &msm8976_a72 }, { .compatible = "qcom,msm8976-hfpll-cci", .data = &msm8976_cci }, + { .compatible = "qcom,qcs404-hfpll", .data = &qcs404 }, + /* Deprecated in bindings */ + { .compatible = "qcom,hfpll", .data = &qcs404 }, { } }; MODULE_DEVICE_TABLE(of, qcom_hfpll_match_table); From 72a5425adc7ade7f7808e85d949dde26c05ec127 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sat, 20 Apr 2024 03:50:04 +0100 Subject: [PATCH 0472/1702] ext2: Remove call to folio_set_error() Nobody checks this flag on ext2 folios, stop setting it. Cc: Jan Kara Cc: linux-ext4@vger.kernel.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Jan Kara Message-Id: <20240420025029.2166544-10-willy@infradead.org> --- fs/ext2/dir.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/ext2/dir.c b/fs/ext2/dir.c index 4fb155b5a958f..087457061c6ef 100644 --- a/fs/ext2/dir.c +++ b/fs/ext2/dir.c @@ -175,7 +175,6 @@ static bool ext2_check_folio(struct folio *folio, int quiet, char *kaddr) (unsigned long) le32_to_cpu(p->inode)); } fail: - folio_set_error(folio); return false; } From 91ba8cb1b09393b1c2222082e087f066fe13fd9b Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sat, 20 Apr 2024 03:50:08 +0100 Subject: [PATCH 0473/1702] isofs: Remove calls to set/clear the error flag Nobody checks the error flag on isofs folios, so stop setting and clearing it. Cc: Jan Kara Signed-off-by: Matthew Wilcox (Oracle) Acked-by: Jan Kara Signed-off-by: Jan Kara Message-Id: <20240420025029.2166544-14-willy@infradead.org> --- fs/isofs/compress.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fs/isofs/compress.c b/fs/isofs/compress.c index c4da3f634b92e..34d5baa5d88a0 100644 --- a/fs/isofs/compress.c +++ b/fs/isofs/compress.c @@ -346,8 +346,6 @@ static int zisofs_read_folio(struct file *file, struct folio *folio) for (i = 0; i < pcount; i++, index++) { if (i != full_page) pages[i] = grab_cache_page_nowait(mapping, index); - if (pages[i]) - ClearPageError(pages[i]); } err = zisofs_fill_pages(inode, full_page, pcount, pages); @@ -356,8 +354,6 @@ static int zisofs_read_folio(struct file *file, struct folio *folio) for (i = 0; i < pcount; i++) { if (pages[i]) { flush_dcache_page(pages[i]); - if (i == full_page && err) - SetPageError(pages[i]); unlock_page(pages[i]); if (i != full_page) put_page(pages[i]); From 76fc23b695f4717bb5e7b616eaad7d6213fdea9f Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:10 -0700 Subject: [PATCH 0474/1702] xfs: repair directories by scanning directory parent pointers For filesystems with parent pointers, scan the entire filesystem looking for parent pointers that target the directory we're rebuilding instead of trying to salvage whatever we can from the directory data blocks. This will be more robust than salvaging, but there's more code to come. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/dir_repair.c | 347 +++++++++++++++++++++++++++++++++++++- 1 file changed, 341 insertions(+), 6 deletions(-) diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index 575397aef1f7a..4e6b0e88b9960 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -28,6 +28,7 @@ #include "xfs_exchmaps.h" #include "xfs_exchrange.h" #include "xfs_ag.h" +#include "xfs_parent.h" #include "scrub/xfs_scrub.h" #include "scrub/scrub.h" #include "scrub/common.h" @@ -43,6 +44,7 @@ #include "scrub/reap.h" #include "scrub/findparent.h" #include "scrub/orphanage.h" +#include "scrub/listxattr.h" /* * Directory Repair @@ -57,6 +59,15 @@ * being repaired and the temporary directory, and will later become important * for parent pointer scanning. * + * If parent pointers are enabled on this filesystem, we instead reconstruct + * the directory by visiting each parent pointer of each file in the filesystem + * and translating the relevant parent pointer records into dirents. In this + * case, it is advantageous to stash all directory entries created from parent + * pointers for a single child file before replaying them into the temporary + * directory. To save memory, the live filesystem scan reuses the findparent + * fields. Directory repair chooses either parent pointer scanning or + * directory entry salvaging, but not both. + * * Directory entries added to the temporary directory do not elevate the link * counts of the inodes found. When salvaging completes, the remaining stashed * entries are replayed to the temporary directory. An atomic mapping exchange @@ -112,7 +123,15 @@ struct xrep_dir { /* * Information used to scan the filesystem to find the inumber of the - * dotdot entry for this directory. + * dotdot entry for this directory. For directory salvaging when + * parent pointers are not enabled, we use the findparent_* functions + * on this object and access only the parent_ino field directly. + * + * When parent pointers are enabled, however, the pptr scanner uses the + * iscan, hooks, lock, and parent_ino fields of this object directly. + * @pscan.lock coordinates access to dir_entries, dir_names, + * parent_ino, subdirs, dirents, and args. This reduces the memory + * requirements of this structure. */ struct xrep_parent_scan_info pscan; @@ -763,28 +782,35 @@ xrep_dir_replay_updates( int error; /* Add all the salvaged dirents to the temporary directory. */ + mutex_lock(&rd->pscan.lock); foreach_xfarray_idx(rd->dir_entries, array_cur) { struct xrep_dirent dirent; error = xfarray_load(rd->dir_entries, array_cur, &dirent); if (error) - return error; + goto out_unlock; error = xfblob_loadname(rd->dir_names, dirent.name_cookie, &rd->xname, dirent.namelen); if (error) - return error; + goto out_unlock; rd->xname.type = dirent.ftype; + mutex_unlock(&rd->pscan.lock); error = xrep_dir_replay_update(rd, &rd->xname, &dirent); if (error) return error; + mutex_lock(&rd->pscan.lock); } /* Empty out both arrays now that we've added the entries. */ xfarray_truncate(rd->dir_entries); xfblob_truncate(rd->dir_names); + mutex_unlock(&rd->pscan.lock); return 0; +out_unlock: + mutex_unlock(&rd->pscan.lock); + return error; } /* @@ -995,6 +1021,269 @@ xrep_dir_salvage_entries( } +/* + * Examine a parent pointer of a file. If it leads us back to the directory + * that we're rebuilding, create an incore dirent from the parent pointer and + * stash it. + */ +STATIC int +xrep_dir_scan_pptr( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xfs_name xname = { + .name = name, + .len = namelen, + .type = xfs_mode_to_ftype(VFS_I(ip)->i_mode), + }; + xfs_ino_t parent_ino; + uint32_t parent_gen; + struct xrep_dir *rd = priv; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + /* + * Ignore parent pointers that point back to a different dir, list the + * wrong generation number, or are invalid. + */ + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, &parent_ino, &parent_gen); + if (error) + return error; + + if (parent_ino != sc->ip->i_ino || + parent_gen != VFS_I(sc->ip)->i_generation) + return 0; + + mutex_lock(&rd->pscan.lock); + error = xrep_dir_stash_createname(rd, &xname, ip->i_ino); + mutex_unlock(&rd->pscan.lock); + return error; +} + +/* + * If this child dirent points to the directory being repaired, remember that + * fact so that we can reset the dotdot entry if necessary. + */ +STATIC int +xrep_dir_scan_dirent( + struct xfs_scrub *sc, + struct xfs_inode *dp, + xfs_dir2_dataptr_t dapos, + const struct xfs_name *name, + xfs_ino_t ino, + void *priv) +{ + struct xrep_dir *rd = priv; + + /* Dirent doesn't point to this directory. */ + if (ino != rd->sc->ip->i_ino) + return 0; + + /* Ignore garbage inum. */ + if (!xfs_verify_dir_ino(rd->sc->mp, ino)) + return 0; + + /* No weird looking names. */ + if (name->len >= MAXNAMELEN || name->len <= 0) + return 0; + + /* Don't pick up dot or dotdot entries; we only want child dirents. */ + if (xfs_dir2_samename(name, &xfs_name_dotdot) || + xfs_dir2_samename(name, &xfs_name_dot)) + return 0; + + trace_xrep_dir_stash_createname(sc->tempip, &xfs_name_dotdot, + dp->i_ino); + + xrep_findparent_scan_found(&rd->pscan, dp->i_ino); + return 0; +} + +/* + * Decide if we want to look for child dirents or parent pointers in this file. + * Skip the dir being repaired and any files being used to stage repairs. + */ +static inline bool +xrep_dir_want_scan( + struct xrep_dir *rd, + const struct xfs_inode *ip) +{ + return ip != rd->sc->ip && !xrep_is_tempfile(ip); +} + +/* + * Take ILOCK on a file that we want to scan. + * + * Select ILOCK_EXCL if the file is a directory with an unloaded data bmbt or + * has an unloaded attr bmbt. Otherwise, take ILOCK_SHARED. + */ +static inline unsigned int +xrep_dir_scan_ilock( + struct xrep_dir *rd, + struct xfs_inode *ip) +{ + uint lock_mode = XFS_ILOCK_SHARED; + + /* Need to take the shared ILOCK to advance the iscan cursor. */ + if (!xrep_dir_want_scan(rd, ip)) + goto lock; + + if (S_ISDIR(VFS_I(ip)->i_mode) && xfs_need_iread_extents(&ip->i_df)) { + lock_mode = XFS_ILOCK_EXCL; + goto lock; + } + + if (xfs_inode_has_attr_fork(ip) && xfs_need_iread_extents(&ip->i_af)) + lock_mode = XFS_ILOCK_EXCL; + +lock: + xfs_ilock(ip, lock_mode); + return lock_mode; +} + +/* + * Scan this file for relevant child dirents or parent pointers that point to + * the directory we're rebuilding. + */ +STATIC int +xrep_dir_scan_file( + struct xrep_dir *rd, + struct xfs_inode *ip) +{ + unsigned int lock_mode; + int error = 0; + + lock_mode = xrep_dir_scan_ilock(rd, ip); + + if (!xrep_dir_want_scan(rd, ip)) + goto scan_done; + + /* + * If the extended attributes look as though they has been zapped by + * the inode record repair code, we cannot scan for parent pointers. + */ + if (xchk_pptr_looks_zapped(ip)) { + error = -EBUSY; + goto scan_done; + } + + error = xchk_xattr_walk(rd->sc, ip, xrep_dir_scan_pptr, rd); + if (error) + goto scan_done; + + if (S_ISDIR(VFS_I(ip)->i_mode)) { + /* + * If the directory looks as though it has been zapped by the + * inode record repair code, we cannot scan for child dirents. + */ + if (xchk_dir_looks_zapped(ip)) { + error = -EBUSY; + goto scan_done; + } + + error = xchk_dir_walk(rd->sc, ip, xrep_dir_scan_dirent, rd); + if (error) + goto scan_done; + } + +scan_done: + xchk_iscan_mark_visited(&rd->pscan.iscan, ip); + xfs_iunlock(ip, lock_mode); + return error; +} + +/* + * Scan all files in the filesystem for parent pointers that we can turn into + * replacement dirents, and a dirent that we can use to set the dotdot pointer. + */ +STATIC int +xrep_dir_scan_dirtree( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + struct xfs_inode *ip; + int error; + + /* Roots of directory trees are their own parents. */ + if (sc->ip == sc->mp->m_rootip) + xrep_findparent_scan_found(&rd->pscan, sc->ip->i_ino); + + /* + * Filesystem scans are time consuming. Drop the directory ILOCK and + * all other resources for the duration of the scan and hope for the + * best. The live update hooks will keep our scan information up to + * date even though we've dropped the locks. + */ + xchk_trans_cancel(sc); + if (sc->ilock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) + xchk_iunlock(sc, sc->ilock_flags & (XFS_ILOCK_SHARED | + XFS_ILOCK_EXCL)); + error = xchk_trans_alloc_empty(sc); + if (error) + return error; + + while ((error = xchk_iscan_iter(&rd->pscan.iscan, &ip)) == 1) { + bool flush; + + error = xrep_dir_scan_file(rd, ip); + xchk_irele(sc, ip); + if (error) + break; + + /* Flush stashed dirent updates to constrain memory usage. */ + mutex_lock(&rd->pscan.lock); + flush = xrep_dir_want_flush_stashed(rd); + mutex_unlock(&rd->pscan.lock); + if (flush) { + xchk_trans_cancel(sc); + + error = xrep_tempfile_iolock_polled(sc); + if (error) + break; + + error = xrep_dir_replay_updates(rd); + xrep_tempfile_iounlock(sc); + if (error) + break; + + error = xchk_trans_alloc_empty(sc); + if (error) + break; + } + + if (xchk_should_terminate(sc, &error)) + break; + } + xchk_iscan_iter_finish(&rd->pscan.iscan); + if (error) { + /* + * If we couldn't grab an inode that was busy with a state + * change, change the error code so that we exit to userspace + * as quickly as possible. + */ + if (error == -EBUSY) + return -ECANCELED; + return error; + } + + /* + * Cancel the empty transaction so that we can (later) use the atomic + * file mapping exchange functions to lock files and commit the new + * directory. + */ + xchk_trans_cancel(rd->sc); + return 0; +} + /* * Free all the directory blocks and reset the data fork. The caller must * join the inode to the transaction. This function returns with the inode @@ -1194,6 +1483,45 @@ xrep_dir_set_nlink( return 0; } +/* + * Finish replaying stashed dirent updates, allocate a transaction for + * exchanging data fork mappings, and take the ILOCKs of both directories + * before we commit the new directory structure. + */ +STATIC int +xrep_dir_finalize_tempdir( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + int error; + + if (!xfs_has_parent(sc->mp)) + return xrep_tempexch_trans_alloc(sc, XFS_DATA_FORK, &rd->tx); + + /* + * Repair relies on the ILOCK to quiesce all possible dirent updates. + * Replay all queued dirent updates into the tempdir before exchanging + * the contents, even if that means dropping the ILOCKs and the + * transaction. + */ + do { + error = xrep_dir_replay_updates(rd); + if (error) + return error; + + error = xrep_tempexch_trans_alloc(sc, XFS_DATA_FORK, &rd->tx); + if (error) + return error; + + if (xfarray_length(rd->dir_entries) == 0) + break; + + xchk_trans_cancel(sc); + xrep_tempfile_iunlock_both(sc); + } while (!xchk_should_terminate(sc, &error)); + return error; +} + /* Exchange the temporary directory's data fork with the one being repaired. */ STATIC int xrep_dir_swap( @@ -1296,8 +1624,12 @@ xrep_dir_rebuild_tree( if (error) return error; - /* Allocate transaction and ILOCK the scrub file and the temp file. */ - error = xrep_tempexch_trans_alloc(sc, XFS_DATA_FORK, &rd->tx); + /* + * Allocate transaction, lock inodes, and make sure that we've replayed + * all the stashed dirent updates to the tempdir. After this point, + * we're ready to exchange data fork mappings. + */ + error = xrep_dir_finalize_tempdir(rd); if (error) return error; @@ -1482,7 +1814,10 @@ xrep_directory( if (error) return error; - error = xrep_dir_salvage_entries(rd); + if (xfs_has_parent(sc->mp)) + error = xrep_dir_scan_dirtree(rd); + else + error = xrep_dir_salvage_entries(rd); if (error) goto out_teardown; From 8559b21a64d983315bdf1bd9f8dfdf732c56d057 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:10 -0700 Subject: [PATCH 0475/1702] xfs: implement live updates for directory repairs While we're scanning the filesystem for parent pointers that we can turn into dirents, we cannot hold the IOLOCK or ILOCK of the directory being repaired. Therefore, we need to set up a dirent hook so that we can keep the temporary directory up to date with the rest of the filesystem. Hence we add the ability to *remove* entries from the temporary dir. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/dir_repair.c | 218 +++++++++++++++++++++++++++++++++++--- fs/xfs/scrub/findparent.c | 10 +- fs/xfs/scrub/findparent.h | 10 +- fs/xfs/scrub/trace.h | 2 + 4 files changed, 218 insertions(+), 22 deletions(-) diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index 4e6b0e88b9960..60e31da4451e8 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -85,6 +85,12 @@ * other threads. */ +/* Create a dirent in the tempdir. */ +#define XREP_DIRENT_ADD (1) + +/* Remove a dirent from the tempdir. */ +#define XREP_DIRENT_REMOVE (2) + /* Directory entry to be restored in the new directory. */ struct xrep_dirent { /* Cookie for retrieval of the dirent name. */ @@ -98,6 +104,9 @@ struct xrep_dirent { /* File type of the dirent. */ uint8_t ftype; + + /* XREP_DIRENT_{ADD,REMOVE} */ + uint8_t action; }; /* @@ -339,6 +348,7 @@ xrep_dir_stash_createname( xfs_ino_t ino) { struct xrep_dirent dirent = { + .action = XREP_DIRENT_ADD, .ino = ino, .namelen = name->len, .ftype = name->type, @@ -354,6 +364,33 @@ xrep_dir_stash_createname( return xfarray_append(rd->dir_entries, &dirent); } +/* + * Remember that we want to remove a dirent from the tempdir. These stashed + * actions will be replayed later. + */ +STATIC int +xrep_dir_stash_removename( + struct xrep_dir *rd, + const struct xfs_name *name, + xfs_ino_t ino) +{ + struct xrep_dirent dirent = { + .action = XREP_DIRENT_REMOVE, + .ino = ino, + .namelen = name->len, + .ftype = name->type, + }; + int error; + + trace_xrep_dir_stash_removename(rd->sc->tempip, name, ino); + + error = xfblob_storename(rd->dir_names, &dirent.name_cookie, name); + if (error) + return error; + + return xfarray_append(rd->dir_entries, &dirent); +} + /* Allocate an in-core record to hold entries while we rebuild the dir data. */ STATIC int xrep_dir_salvage_entry( @@ -705,6 +742,43 @@ xrep_dir_replay_createname( return xfs_dir2_node_addname(&rd->args); } +/* Replay a stashed removename onto the temporary directory. */ +STATIC int +xrep_dir_replay_removename( + struct xrep_dir *rd, + const struct xfs_name *name, + xfs_extlen_t total) +{ + struct xfs_inode *dp = rd->args.dp; + bool is_block, is_leaf; + int error; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + + xrep_dir_init_args(rd, dp, name); + rd->args.op_flags = 0; + rd->args.total = total; + + trace_xrep_dir_replay_removename(dp, name, 0); + + if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) + return xfs_dir2_sf_removename(&rd->args); + + error = xfs_dir2_isblock(&rd->args, &is_block); + if (error) + return error; + if (is_block) + return xfs_dir2_block_removename(&rd->args); + + error = xfs_dir2_isleaf(&rd->args, &is_leaf); + if (error) + return error; + if (is_leaf) + return xfs_dir2_leaf_removename(&rd->args); + + return xfs_dir2_node_removename(&rd->args); +} + /* * Add this stashed incore directory entry to the temporary directory. * The caller must hold the tempdir's IOLOCK, must not hold any ILOCKs, and @@ -732,26 +806,64 @@ xrep_dir_replay_update( xrep_tempfile_ilock(rd->sc); xfs_trans_ijoin(rd->sc->tp, rd->sc->tempip, 0); - /* - * Create a replacement dirent in the temporary directory. Note that - * _createname doesn't check for existing entries. There shouldn't be - * any in the temporary dir, but we'll verify this in debug mode. - */ + switch (dirent->action) { + case XREP_DIRENT_ADD: + /* + * Create a replacement dirent in the temporary directory. + * Note that _createname doesn't check for existing entries. + * There shouldn't be any in the temporary dir, but we'll + * verify this in debug mode. + */ #ifdef DEBUG - error = xchk_dir_lookup(rd->sc, rd->sc->tempip, xname, &ino); - if (error != -ENOENT) { - ASSERT(error != -ENOENT); - goto out_cancel; - } + error = xchk_dir_lookup(rd->sc, rd->sc->tempip, xname, &ino); + if (error != -ENOENT) { + ASSERT(error != -ENOENT); + goto out_cancel; + } #endif - error = xrep_dir_replay_createname(rd, xname, dirent->ino, resblks); - if (error) - goto out_cancel; + error = xrep_dir_replay_createname(rd, xname, dirent->ino, + resblks); + if (error) + goto out_cancel; + + if (xname->type == XFS_DIR3_FT_DIR) + rd->subdirs++; + rd->dirents++; + break; + case XREP_DIRENT_REMOVE: + /* + * Remove a dirent from the temporary directory. Note that + * _removename doesn't check the inode target of the exist + * entry. There should be a perfect match in the temporary + * dir, but we'll verify this in debug mode. + */ +#ifdef DEBUG + error = xchk_dir_lookup(rd->sc, rd->sc->tempip, xname, &ino); + if (error) { + ASSERT(error != 0); + goto out_cancel; + } + if (ino != dirent->ino) { + ASSERT(ino == dirent->ino); + error = -EIO; + goto out_cancel; + } +#endif - if (xname->type == XFS_DIR3_FT_DIR) - rd->subdirs++; - rd->dirents++; + error = xrep_dir_replay_removename(rd, xname, resblks); + if (error) + goto out_cancel; + + if (xname->type == XFS_DIR3_FT_DIR) + rd->subdirs--; + rd->dirents--; + break; + default: + ASSERT(0); + error = -EIO; + goto out_cancel; + } /* Commit and unlock. */ error = xrep_trans_commit(rd->sc); @@ -1284,6 +1396,71 @@ xrep_dir_scan_dirtree( return 0; } +/* + * Capture dirent updates being made by other threads which are relevant to the + * directory being repaired. + */ +STATIC int +xrep_dir_live_update( + struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct xfs_dir_update_params *p = data; + struct xrep_dir *rd; + struct xfs_scrub *sc; + int error = 0; + + rd = container_of(nb, struct xrep_dir, pscan.dhook.dirent_hook.nb); + sc = rd->sc; + + /* + * This thread updated a child dirent in the directory that we're + * rebuilding. Stash the update for replay against the temporary + * directory. + */ + if (p->dp->i_ino == sc->ip->i_ino && + xchk_iscan_want_live_update(&rd->pscan.iscan, p->ip->i_ino)) { + mutex_lock(&rd->pscan.lock); + if (p->delta > 0) + error = xrep_dir_stash_createname(rd, p->name, + p->ip->i_ino); + else + error = xrep_dir_stash_removename(rd, p->name, + p->ip->i_ino); + mutex_unlock(&rd->pscan.lock); + if (error) + goto out_abort; + } + + /* + * This thread updated another directory's child dirent that points to + * the directory that we're rebuilding, so remember the new dotdot + * target. + */ + if (p->ip->i_ino == sc->ip->i_ino && + xchk_iscan_want_live_update(&rd->pscan.iscan, p->dp->i_ino)) { + if (p->delta > 0) { + trace_xrep_dir_stash_createname(sc->tempip, + &xfs_name_dotdot, + p->dp->i_ino); + + xrep_findparent_scan_found(&rd->pscan, p->dp->i_ino); + } else { + trace_xrep_dir_stash_removename(sc->tempip, + &xfs_name_dotdot, + rd->pscan.parent_ino); + + xrep_findparent_scan_found(&rd->pscan, NULLFSINO); + } + } + + return NOTIFY_DONE; +out_abort: + xchk_iscan_abort(&rd->pscan.iscan); + return NOTIFY_DONE; +} + /* * Free all the directory blocks and reset the data fork. The caller must * join the inode to the transaction. This function returns with the inode @@ -1633,6 +1810,9 @@ xrep_dir_rebuild_tree( if (error) return error; + if (xchk_iscan_aborted(&rd->pscan.iscan)) + return -ECANCELED; + /* * Exchange the tempdir's data fork with the file being repaired. This * recreates the transaction and re-takes the ILOCK in the scrub @@ -1688,7 +1868,11 @@ xrep_dir_setup_scan( if (error) goto out_xfarray; - error = xrep_findparent_scan_start(sc, &rd->pscan); + if (xfs_has_parent(sc->mp)) + error = __xrep_findparent_scan_start(sc, &rd->pscan, + xrep_dir_live_update); + else + error = xrep_findparent_scan_start(sc, &rd->pscan); if (error) goto out_xfblob; diff --git a/fs/xfs/scrub/findparent.c b/fs/xfs/scrub/findparent.c index 712dd73e4789f..c78422ad757bf 100644 --- a/fs/xfs/scrub/findparent.c +++ b/fs/xfs/scrub/findparent.c @@ -238,9 +238,10 @@ xrep_findparent_live_update( * will be called when there is a dotdot update for the inode being repaired. */ int -xrep_findparent_scan_start( +__xrep_findparent_scan_start( struct xfs_scrub *sc, - struct xrep_parent_scan_info *pscan) + struct xrep_parent_scan_info *pscan, + notifier_fn_t custom_fn) { int error; @@ -262,7 +263,10 @@ xrep_findparent_scan_start( * ILOCK, which means that any in-progress inode updates will finish * before we can scan the inode. */ - xfs_dir_hook_setup(&pscan->dhook, xrep_findparent_live_update); + if (custom_fn) + xfs_dir_hook_setup(&pscan->dhook, custom_fn); + else + xfs_dir_hook_setup(&pscan->dhook, xrep_findparent_live_update); error = xfs_dir_hook_add(sc->mp, &pscan->dhook); if (error) goto out_iscan; diff --git a/fs/xfs/scrub/findparent.h b/fs/xfs/scrub/findparent.h index 501f99d3164ed..d998c7a88152c 100644 --- a/fs/xfs/scrub/findparent.h +++ b/fs/xfs/scrub/findparent.h @@ -24,8 +24,14 @@ struct xrep_parent_scan_info { bool lookup_parent; }; -int xrep_findparent_scan_start(struct xfs_scrub *sc, - struct xrep_parent_scan_info *pscan); +int __xrep_findparent_scan_start(struct xfs_scrub *sc, + struct xrep_parent_scan_info *pscan, + notifier_fn_t custom_fn); +static inline int xrep_findparent_scan_start(struct xfs_scrub *sc, + struct xrep_parent_scan_info *pscan) +{ + return __xrep_findparent_scan_start(sc, pscan, NULL); +} int xrep_findparent_scan(struct xrep_parent_scan_info *pscan); void xrep_findparent_scan_teardown(struct xrep_parent_scan_info *pscan); diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 4b968df3d840c..64db413b18884 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2692,6 +2692,8 @@ DEFINE_XREP_DIRENT_EVENT(xrep_dir_salvage_entry); DEFINE_XREP_DIRENT_EVENT(xrep_dir_stash_createname); DEFINE_XREP_DIRENT_EVENT(xrep_dir_replay_createname); DEFINE_XREP_DIRENT_EVENT(xrep_adoption_reparent); +DEFINE_XREP_DIRENT_EVENT(xrep_dir_stash_removename); +DEFINE_XREP_DIRENT_EVENT(xrep_dir_replay_removename); DECLARE_EVENT_CLASS(xrep_adoption_class, TP_PROTO(struct xfs_inode *dp, struct xfs_inode *ip, bool moved), From e5d7ce0364d8ee6821fd93814885c3bef775b9c3 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:11 -0700 Subject: [PATCH 0476/1702] xfs: replay unlocked parent pointer updates that accrue during xattr repair There are a few places where the extended attribute repair code drops the ILOCK to apply stashed xattrs to the temporary file. Although setxattr and removexattr are still locked out because we retain our hold on the IOLOCK, this doesn't prevent renames from updating parent pointers, because the VFS doesn't take i_rwsem on children that are being moved. Therefore, set up a dirent hook to capture parent pointer updates for this file, and replay(?) the updates. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/attr_repair.c | 438 ++++++++++++++++++++++++++++++++++++- fs/xfs/scrub/trace.h | 73 +++++++ 2 files changed, 509 insertions(+), 2 deletions(-) diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c index 48443e88e298b..87f6c9cb54eb6 100644 --- a/fs/xfs/scrub/attr_repair.c +++ b/fs/xfs/scrub/attr_repair.c @@ -96,6 +96,52 @@ struct xrep_xattr { /* Number of attributes that we are salvaging. */ unsigned long long attrs_found; + + /* Can we flush stashed attrs to the tempfile? */ + bool can_flush; + + /* Did the live update fail, and hence the repair is now out of date? */ + bool live_update_aborted; + + /* Lock protecting parent pointer updates */ + struct mutex lock; + + /* Fixed-size array of xrep_xattr_pptr structures. */ + struct xfarray *pptr_recs; + + /* Blobs containing parent pointer names. */ + struct xfblob *pptr_names; + + /* Hook to capture parent pointer updates. */ + struct xfs_dir_hook dhook; + + /* Scratch buffer for capturing parent pointers. */ + struct xfs_da_args pptr_args; + + /* Name buffer */ + struct xfs_name xname; + char namebuf[MAXNAMELEN]; +}; + +/* Create a parent pointer in the tempfile. */ +#define XREP_XATTR_PPTR_ADD (1) + +/* Remove a parent pointer from the tempfile. */ +#define XREP_XATTR_PPTR_REMOVE (2) + +/* A stashed parent pointer update. */ +struct xrep_xattr_pptr { + /* Cookie for retrieval of the pptr name. */ + xfblob_cookie name_cookie; + + /* Parent pointer record. */ + struct xfs_parent_rec pptr_rec; + + /* Length of the pptr name. */ + uint8_t namelen; + + /* XREP_XATTR_PPTR_{ADD,REMOVE} */ + uint8_t action; }; /* Set up to recreate the extended attributes. */ @@ -103,6 +149,9 @@ int xrep_setup_xattr( struct xfs_scrub *sc) { + if (xfs_has_parent(sc->mp)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); + return xrep_tempfile_create(sc, S_IFREG); } @@ -711,11 +760,122 @@ xrep_xattr_want_flush_stashed( { unsigned long long bytes; + if (!rx->can_flush) + return false; + bytes = xfarray_bytes(rx->xattr_records) + xfblob_bytes(rx->xattr_blobs); return bytes > XREP_XATTR_MAX_STASH_BYTES; } +/* + * Did we observe rename changing parent pointer xattrs while we were flushing + * salvaged attrs? + */ +static inline bool +xrep_xattr_saw_pptr_conflict( + struct xrep_xattr *rx) +{ + bool ret; + + ASSERT(rx->can_flush); + + if (!xfs_has_parent(rx->sc->mp)) + return false; + + xfs_assert_ilocked(rx->sc->ip, XFS_ILOCK_EXCL); + + mutex_lock(&rx->lock); + ret = xfarray_bytes(rx->pptr_recs) > 0; + mutex_unlock(&rx->lock); + + return ret; +} + +/* + * Reset the entire repair state back to initial conditions, now that we've + * detected a parent pointer update to the attr structure while we were + * flushing salvaged attrs. See the locking notes in dir_repair.c for more + * information on why this is all necessary. + */ +STATIC int +xrep_xattr_full_reset( + struct xrep_xattr *rx) +{ + struct xfs_scrub *sc = rx->sc; + struct xfs_attr_sf_hdr *hdr; + struct xfs_ifork *ifp = &sc->tempip->i_af; + int error; + + trace_xrep_xattr_full_reset(sc->ip, sc->tempip); + + /* The temporary file's data fork had better not be in btree format. */ + if (sc->tempip->i_df.if_format == XFS_DINODE_FMT_BTREE) { + ASSERT(0); + return -EIO; + } + + /* + * We begin in transaction context with sc->ip ILOCKed but not joined + * to the transaction. To reset to the initial state, we must hold + * sc->ip's ILOCK to prevent rename from updating parent pointer + * information and the tempfile's ILOCK to clear its contents. + */ + xchk_iunlock(rx->sc, XFS_ILOCK_EXCL); + xrep_tempfile_ilock_both(sc); + xfs_trans_ijoin(sc->tp, sc->ip, 0); + xfs_trans_ijoin(sc->tp, sc->tempip, 0); + + /* + * Free all the blocks of the attr fork of the temp file, and reset + * it back to local format. + */ + if (xfs_ifork_has_extents(&sc->tempip->i_af)) { + error = xrep_reap_ifork(sc, sc->tempip, XFS_ATTR_FORK); + if (error) + return error; + + ASSERT(ifp->if_bytes == 0); + ifp->if_format = XFS_DINODE_FMT_LOCAL; + xfs_idata_realloc(sc->tempip, sizeof(*hdr), XFS_ATTR_FORK); + } + + /* Reinitialize the attr fork to an empty shortform structure. */ + hdr = ifp->if_data; + memset(hdr, 0, sizeof(*hdr)); + hdr->totsize = cpu_to_be16(sizeof(*hdr)); + xfs_trans_log_inode(sc->tp, sc->tempip, XFS_ILOG_CORE | XFS_ILOG_ADATA); + + /* + * Roll this transaction to commit our reset ondisk. The tempfile + * should no longer be joined to the transaction, so we drop its ILOCK. + * This should leave us in transaction context with sc->ip ILOCKed but + * not joined to the transaction. + */ + error = xrep_roll_trans(sc); + if (error) + return error; + xrep_tempfile_iunlock(sc); + + /* + * Erase any accumulated parent pointer updates now that we've erased + * the tempfile's attr fork. We're resetting the entire repair state + * back to where we were initially, except now we won't flush salvaged + * xattrs until the very end. + */ + mutex_lock(&rx->lock); + xfarray_truncate(rx->pptr_recs); + xfblob_truncate(rx->pptr_names); + mutex_unlock(&rx->lock); + + rx->can_flush = false; + rx->attrs_found = 0; + + ASSERT(xfarray_bytes(rx->xattr_records) == 0); + ASSERT(xfblob_bytes(rx->xattr_blobs) == 0); + return 0; +} + /* Extract as many attribute keys and values as we can. */ STATIC int xrep_xattr_recover( @@ -730,6 +890,7 @@ xrep_xattr_recover( int nmap; int error; +restart: /* * Iterate each xattr leaf block in the attr fork to scan them for any * attributes that we might salvage. @@ -768,6 +929,14 @@ xrep_xattr_recover( error = xrep_xattr_flush_stashed(rx); if (error) return error; + + if (xrep_xattr_saw_pptr_conflict(rx)) { + error = xrep_xattr_full_reset(rx); + if (error) + return error; + + goto restart; + } } } } @@ -927,6 +1096,180 @@ xrep_xattr_salvage_attributes( return xrep_xattr_flush_stashed(rx); } +/* + * Add this stashed incore parent pointer to the temporary file. The caller + * must hold the tempdir's IOLOCK, must not hold any ILOCKs, and must not be in + * transaction context. + */ +STATIC int +xrep_xattr_replay_pptr_update( + struct xrep_xattr *rx, + const struct xfs_name *xname, + struct xrep_xattr_pptr *pptr) +{ + struct xfs_scrub *sc = rx->sc; + int error; + + switch (pptr->action) { + case XREP_XATTR_PPTR_ADD: + /* Create parent pointer. */ + trace_xrep_xattr_replay_parentadd(sc->tempip, xname, + &pptr->pptr_rec); + + error = xfs_parent_set(sc->tempip, sc->ip->i_ino, xname, + &pptr->pptr_rec, &rx->pptr_args); + ASSERT(error != -EEXIST); + return error; + case XREP_XATTR_PPTR_REMOVE: + /* Remove parent pointer. */ + trace_xrep_xattr_replay_parentremove(sc->tempip, xname, + &pptr->pptr_rec); + + error = xfs_parent_unset(sc->tempip, sc->ip->i_ino, xname, + &pptr->pptr_rec, &rx->pptr_args); + ASSERT(error != -ENOATTR); + return error; + } + + ASSERT(0); + return -EIO; +} + +/* + * Flush stashed parent pointer updates that have been recorded by the scanner. + * This is done to reduce the memory requirements of the xattr rebuild, since + * files can have a lot of hardlinks and the fs can be busy. + * + * Caller must not hold transactions or ILOCKs. Caller must hold the tempfile + * IOLOCK. + */ +STATIC int +xrep_xattr_replay_pptr_updates( + struct xrep_xattr *rx) +{ + xfarray_idx_t array_cur; + int error; + + mutex_lock(&rx->lock); + foreach_xfarray_idx(rx->pptr_recs, array_cur) { + struct xrep_xattr_pptr pptr; + + error = xfarray_load(rx->pptr_recs, array_cur, &pptr); + if (error) + goto out_unlock; + + error = xfblob_loadname(rx->pptr_names, pptr.name_cookie, + &rx->xname, pptr.namelen); + if (error) + goto out_unlock; + mutex_unlock(&rx->lock); + + error = xrep_xattr_replay_pptr_update(rx, &rx->xname, &pptr); + if (error) + return error; + + mutex_lock(&rx->lock); + } + + /* Empty out both arrays now that we've added the entries. */ + xfarray_truncate(rx->pptr_recs); + xfblob_truncate(rx->pptr_names); + mutex_unlock(&rx->lock); + return 0; +out_unlock: + mutex_unlock(&rx->lock); + return error; +} + +/* + * Remember that we want to create a parent pointer in the tempfile. These + * stashed actions will be replayed later. + */ +STATIC int +xrep_xattr_stash_parentadd( + struct xrep_xattr *rx, + const struct xfs_name *name, + const struct xfs_inode *dp) +{ + struct xrep_xattr_pptr pptr = { + .action = XREP_XATTR_PPTR_ADD, + .namelen = name->len, + }; + int error; + + trace_xrep_xattr_stash_parentadd(rx->sc->tempip, dp, name); + + xfs_inode_to_parent_rec(&pptr.pptr_rec, dp); + error = xfblob_storename(rx->pptr_names, &pptr.name_cookie, name); + if (error) + return error; + + return xfarray_append(rx->pptr_recs, &pptr); +} + +/* + * Remember that we want to remove a parent pointer from the tempfile. These + * stashed actions will be replayed later. + */ +STATIC int +xrep_xattr_stash_parentremove( + struct xrep_xattr *rx, + const struct xfs_name *name, + const struct xfs_inode *dp) +{ + struct xrep_xattr_pptr pptr = { + .action = XREP_XATTR_PPTR_REMOVE, + .namelen = name->len, + }; + int error; + + trace_xrep_xattr_stash_parentremove(rx->sc->tempip, dp, name); + + xfs_inode_to_parent_rec(&pptr.pptr_rec, dp); + error = xfblob_storename(rx->pptr_names, &pptr.name_cookie, name); + if (error) + return error; + + return xfarray_append(rx->pptr_recs, &pptr); +} + +/* + * Capture dirent updates being made by other threads. We will have to replay + * the parent pointer updates before exchanging attr forks. + */ +STATIC int +xrep_xattr_live_dirent_update( + struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct xfs_dir_update_params *p = data; + struct xrep_xattr *rx; + struct xfs_scrub *sc; + int error; + + rx = container_of(nb, struct xrep_xattr, dhook.dirent_hook.nb); + sc = rx->sc; + + /* + * This thread updated a dirent that points to the file that we're + * repairing, so stash the update for replay against the temporary + * file. + */ + if (p->ip->i_ino != sc->ip->i_ino) + return NOTIFY_DONE; + + mutex_lock(&rx->lock); + if (p->delta > 0) + error = xrep_xattr_stash_parentadd(rx, p->name, p->dp); + else + error = xrep_xattr_stash_parentremove(rx, p->name, p->dp); + if (error) + rx->live_update_aborted = true; + mutex_unlock(&rx->lock); + return NOTIFY_DONE; +} + /* * Prepare both inodes' attribute forks for an exchange. Promote the tempfile * from short format to leaf format, and if the file being repaired has a short @@ -1030,6 +1373,45 @@ xrep_xattr_swap( return xrep_tempexch_contents(sc, tx); } +/* + * Finish replaying stashed parent pointer updates, allocate a transaction for + * exchanging extent mappings, and take the ILOCKs of both files before we + * commit the new extended attribute structure. + */ +STATIC int +xrep_xattr_finalize_tempfile( + struct xrep_xattr *rx) +{ + struct xfs_scrub *sc = rx->sc; + int error; + + if (!xfs_has_parent(sc->mp)) + return xrep_tempexch_trans_alloc(sc, XFS_ATTR_FORK, &rx->tx); + + /* + * Repair relies on the ILOCK to quiesce all possible xattr updates. + * Replay all queued parent pointer updates into the tempfile before + * exchanging the contents, even if that means dropping the ILOCKs and + * the transaction. + */ + do { + error = xrep_xattr_replay_pptr_updates(rx); + if (error) + return error; + + error = xrep_tempexch_trans_alloc(sc, XFS_ATTR_FORK, &rx->tx); + if (error) + return error; + + if (xfarray_length(rx->pptr_recs) == 0) + break; + + xchk_trans_cancel(sc); + xrep_tempfile_iunlock_both(sc); + } while (!xchk_should_terminate(sc, &error)); + return error; +} + /* * Exchange the new extended attribute data (which we created in the tempfile) * with the file being repaired. @@ -1082,8 +1464,12 @@ xrep_xattr_rebuild_tree( if (error) return error; - /* Allocate exchange transaction and lock both inodes. */ - error = xrep_tempexch_trans_alloc(rx->sc, XFS_ATTR_FORK, &rx->tx); + /* + * Allocate transaction, lock inodes, and make sure that we've replayed + * all the stashed parent pointer updates to the temp file. After this + * point, we're ready to exchange attr fork mappings. + */ + error = xrep_xattr_finalize_tempfile(rx); if (error) return error; @@ -1124,8 +1510,15 @@ STATIC void xrep_xattr_teardown( struct xrep_xattr *rx) { + if (xfs_has_parent(rx->sc->mp)) + xfs_dir_hook_del(rx->sc->mp, &rx->dhook); + if (rx->pptr_names) + xfblob_destroy(rx->pptr_names); + if (rx->pptr_recs) + xfarray_destroy(rx->pptr_recs); xfblob_destroy(rx->xattr_blobs); xfarray_destroy(rx->xattr_records); + mutex_destroy(&rx->lock); kfree(rx); } @@ -1144,6 +1537,10 @@ xrep_xattr_setup_scan( if (!rx) return -ENOMEM; rx->sc = sc; + rx->can_flush = true; + rx->xname.name = rx->namebuf; + + mutex_init(&rx->lock); /* * Allocate enough memory to handle loading local attr values from the @@ -1171,11 +1568,43 @@ xrep_xattr_setup_scan( if (error) goto out_keys; + if (xfs_has_parent(sc->mp)) { + ASSERT(sc->flags & XCHK_FSGATES_DIRENTS); + + descr = xchk_xfile_ino_descr(sc, + "xattr retained parent pointer entries"); + error = xfarray_create(descr, 0, + sizeof(struct xrep_xattr_pptr), + &rx->pptr_recs); + kfree(descr); + if (error) + goto out_values; + + descr = xchk_xfile_ino_descr(sc, + "xattr retained parent pointer names"); + error = xfblob_create(descr, &rx->pptr_names); + kfree(descr); + if (error) + goto out_pprecs; + + xfs_dir_hook_setup(&rx->dhook, xrep_xattr_live_dirent_update); + error = xfs_dir_hook_add(sc->mp, &rx->dhook); + if (error) + goto out_ppnames; + } + *rxp = rx; return 0; +out_ppnames: + xfblob_destroy(rx->pptr_names); +out_pprecs: + xfarray_destroy(rx->pptr_recs); +out_values: + xfblob_destroy(rx->xattr_blobs); out_keys: xfarray_destroy(rx->xattr_records); out_rx: + mutex_destroy(&rx->lock); kfree(rx); return error; } @@ -1212,6 +1641,11 @@ xrep_xattr( if (error) goto out_scan; + if (rx->live_update_aborted) { + error = -EIO; + goto out_scan; + } + /* Last chance to abort before we start committing fixes. */ if (xchk_should_terminate(sc, &error)) goto out_scan; diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 64db413b18884..68532f686eeb1 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2602,6 +2602,43 @@ DEFINE_EVENT(xrep_xattr_class, name, \ TP_ARGS(ip, arg_ip)) DEFINE_XREP_XATTR_EVENT(xrep_xattr_rebuild_tree); DEFINE_XREP_XATTR_EVENT(xrep_xattr_reset_fork); +DEFINE_XREP_XATTR_EVENT(xrep_xattr_full_reset); + +DECLARE_EVENT_CLASS(xrep_xattr_pptr_scan_class, + TP_PROTO(struct xfs_inode *ip, const struct xfs_inode *dp, + const struct xfs_name *name), + TP_ARGS(ip, dp, name), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = ip->i_ino; + __entry->parent_ino = dp->i_ino; + __entry->parent_gen = VFS_IC(dp)->i_generation; + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d ino 0x%llx parent_ino 0x%llx parent_gen 0x%x name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->parent_ino, + __entry->parent_gen, + __entry->namelen, + __get_str(name)) +) +#define DEFINE_XREP_XATTR_PPTR_SCAN_EVENT(name) \ +DEFINE_EVENT(xrep_xattr_pptr_scan_class, name, \ + TP_PROTO(struct xfs_inode *ip, const struct xfs_inode *dp, \ + const struct xfs_name *name), \ + TP_ARGS(ip, dp, name)) +DEFINE_XREP_XATTR_PPTR_SCAN_EVENT(xrep_xattr_stash_parentadd); +DEFINE_XREP_XATTR_PPTR_SCAN_EVENT(xrep_xattr_stash_parentremove); TRACE_EVENT(xrep_dir_recover_dirblock, TP_PROTO(struct xfs_inode *dp, xfs_dablk_t dabno, uint32_t magic, @@ -2748,6 +2785,42 @@ DEFINE_XREP_PARENT_SALVAGE_EVENT(xrep_dir_salvaged_parent); DEFINE_XREP_PARENT_SALVAGE_EVENT(xrep_findparent_dirent); DEFINE_XREP_PARENT_SALVAGE_EVENT(xrep_findparent_from_dcache); +DECLARE_EVENT_CLASS(xrep_pptr_class, + TP_PROTO(struct xfs_inode *ip, const struct xfs_name *name, + const struct xfs_parent_rec *pptr), + TP_ARGS(ip, name, pptr), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = ip->i_ino; + __entry->parent_ino = be64_to_cpu(pptr->p_ino); + __entry->parent_gen = be32_to_cpu(pptr->p_gen); + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d ino 0x%llx parent_ino 0x%llx parent_gen 0x%x name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->parent_ino, + __entry->parent_gen, + __entry->namelen, + __get_str(name)) +) +#define DEFINE_XREP_PPTR_EVENT(name) \ +DEFINE_EVENT(xrep_pptr_class, name, \ + TP_PROTO(struct xfs_inode *ip, const struct xfs_name *name, \ + const struct xfs_parent_rec *pptr), \ + TP_ARGS(ip, name, pptr)) +DEFINE_XREP_PPTR_EVENT(xrep_xattr_replay_parentadd); +DEFINE_XREP_PPTR_EVENT(xrep_xattr_replay_parentremove); + TRACE_EVENT(xrep_nlinks_set_record, TP_PROTO(struct xfs_mount *mp, xfs_ino_t ino, const struct xchk_nlink *obs), From b334f7fab57a07fbece40648b4a22ab3f173bd48 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:12 -0700 Subject: [PATCH 0477/1702] xfs: repair directory parent pointers by scanning for dirents If parent pointers are enabled on the filesystem, we can repair the entire dataset by walking the directories of the filesystem looking for dirents that we can turn into parent pointers. Once we have a full incore dataset, we'll figure out what to do with it, but that's for a subsequent patch. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/parent_repair.c | 414 ++++++++++++++++++++++++++++++++++- fs/xfs/scrub/trace.h | 36 +++ 2 files changed, 447 insertions(+), 3 deletions(-) diff --git a/fs/xfs/scrub/parent_repair.c b/fs/xfs/scrub/parent_repair.c index 63590e1b35060..b4084a9f0e9c8 100644 --- a/fs/xfs/scrub/parent_repair.c +++ b/fs/xfs/scrub/parent_repair.c @@ -24,6 +24,7 @@ #include "xfs_trans_space.h" #include "xfs_health.h" #include "xfs_exchmaps.h" +#include "xfs_parent.h" #include "scrub/xfs_scrub.h" #include "scrub/scrub.h" #include "scrub/common.h" @@ -34,6 +35,9 @@ #include "scrub/readdir.h" #include "scrub/tempfile.h" #include "scrub/orphanage.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" /* * Repairing The Directory Parent Pointer @@ -49,14 +53,61 @@ * See the section on locking issues in dir_repair.c for more information about * conflicts with the VFS. The findparent code wll keep our incore parent * inode up to date. + * + * If parent pointers are enabled, we instead reconstruct the parent pointer + * information by visiting every directory entry of every directory in the + * system and translating the relevant dirents into parent pointers. In this + * case, it is advantageous to stash all parent pointers created from dirents + * from a single parent file before replaying them into the temporary file. To + * save memory, the live filesystem scan reuses the findparent object. Parent + * pointer repair chooses either directory scanning or findparent, but not + * both. + * + * When salvaging completes, the remaining stashed entries are replayed to the + * temporary file. All non-parent pointer extended attributes are copied to + * the temporary file's extended attributes. An atomic extent swap is used to + * commit the new directory blocks to the directory being repaired. This will + * disrupt attrmulti cursors. + */ + +/* A stashed parent pointer update. */ +struct xrep_pptr { + /* Cookie for retrieval of the pptr name. */ + xfblob_cookie name_cookie; + + /* Parent pointer record. */ + struct xfs_parent_rec pptr_rec; + + /* Length of the pptr name. */ + uint8_t namelen; +}; + +/* + * Stash up to 8 pages of recovered parent pointers in pptr_recs and + * pptr_names before we write them to the temp file. */ +#define XREP_PARENT_MAX_STASH_BYTES (PAGE_SIZE * 8) struct xrep_parent { struct xfs_scrub *sc; + /* Fixed-size array of xrep_pptr structures. */ + struct xfarray *pptr_recs; + + /* Blobs containing parent pointer names. */ + struct xfblob *pptr_names; + /* * Information used to scan the filesystem to find the inumber of the - * dotdot entry for this directory. + * dotdot entry for this directory. On filesystems without parent + * pointers, we use the findparent_* functions on this object and + * access only the parent_ino field directly. + * + * When parent pointers are enabled, the directory entry scanner uses + * the iscan, hooks, and lock fields of this object directly. + * @pscan.lock coordinates access to pptr_recs, pptr_names, pptr, and + * pptr_scratch. This reduces the memory requirements of this + * structure. */ struct xrep_parent_scan_info pscan; @@ -66,6 +117,9 @@ struct xrep_parent { /* Directory entry name, plus the trailing null. */ struct xfs_name xname; unsigned char namebuf[MAXNAMELEN]; + + /* Scratch buffer for scanning pptr xattrs */ + struct xfs_da_args pptr_args; }; /* Tear down all the incore stuff we created. */ @@ -74,6 +128,12 @@ xrep_parent_teardown( struct xrep_parent *rp) { xrep_findparent_scan_teardown(&rp->pscan); + if (rp->pptr_names) + xfblob_destroy(rp->pptr_names); + rp->pptr_names = NULL; + if (rp->pptr_recs) + xfarray_destroy(rp->pptr_recs); + rp->pptr_recs = NULL; } /* Set up for a parent repair. */ @@ -82,6 +142,7 @@ xrep_setup_parent( struct xfs_scrub *sc) { struct xrep_parent *rp; + int error; xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); @@ -92,6 +153,10 @@ xrep_setup_parent( rp->xname.name = rp->namebuf; sc->buf = rp; + error = xrep_tempfile_create(sc, S_IFREG); + if (error) + return error; + return xrep_orphanage_try_create(sc); } @@ -147,6 +212,307 @@ xrep_parent_find_dotdot( return error; } +/* + * Add this stashed incore parent pointer to the temporary file. + * The caller must hold the tempdir's IOLOCK, must not hold any ILOCKs, and + * must not be in transaction context. + */ +STATIC int +xrep_parent_replay_update( + struct xrep_parent *rp, + const struct xfs_name *xname, + struct xrep_pptr *pptr) +{ + struct xfs_scrub *sc = rp->sc; + + /* Create parent pointer. */ + trace_xrep_parent_replay_parentadd(sc->tempip, xname, &pptr->pptr_rec); + + return xfs_parent_set(sc->tempip, sc->ip->i_ino, xname, + &pptr->pptr_rec, &rp->pptr_args); +} + +/* + * Flush stashed parent pointer updates that have been recorded by the scanner. + * This is done to reduce the memory requirements of the parent pointer + * rebuild, since files can have a lot of hardlinks and the fs can be busy. + * + * Caller must not hold transactions or ILOCKs. Caller must hold the tempfile + * IOLOCK. + */ +STATIC int +xrep_parent_replay_updates( + struct xrep_parent *rp) +{ + xfarray_idx_t array_cur; + int error; + + mutex_lock(&rp->pscan.lock); + foreach_xfarray_idx(rp->pptr_recs, array_cur) { + struct xrep_pptr pptr; + + error = xfarray_load(rp->pptr_recs, array_cur, &pptr); + if (error) + goto out_unlock; + + error = xfblob_loadname(rp->pptr_names, pptr.name_cookie, + &rp->xname, pptr.namelen); + if (error) + goto out_unlock; + rp->xname.len = pptr.namelen; + mutex_unlock(&rp->pscan.lock); + + error = xrep_parent_replay_update(rp, &rp->xname, &pptr); + if (error) + return error; + + mutex_lock(&rp->pscan.lock); + } + + /* Empty out both arrays now that we've added the entries. */ + xfarray_truncate(rp->pptr_recs); + xfblob_truncate(rp->pptr_names); + mutex_unlock(&rp->pscan.lock); + return 0; +out_unlock: + mutex_unlock(&rp->pscan.lock); + return error; +} + +/* + * Remember that we want to create a parent pointer in the tempfile. These + * stashed actions will be replayed later. + */ +STATIC int +xrep_parent_stash_parentadd( + struct xrep_parent *rp, + const struct xfs_name *name, + const struct xfs_inode *dp) +{ + struct xrep_pptr pptr = { + .namelen = name->len, + }; + int error; + + trace_xrep_parent_stash_parentadd(rp->sc->tempip, dp, name); + + xfs_inode_to_parent_rec(&pptr.pptr_rec, dp); + error = xfblob_storename(rp->pptr_names, &pptr.name_cookie, name); + if (error) + return error; + + return xfarray_append(rp->pptr_recs, &pptr); +} + +/* + * Examine an entry of a directory. If this dirent leads us back to the file + * whose parent pointers we're rebuilding, add a pptr to the temporary + * directory. + */ +STATIC int +xrep_parent_scan_dirent( + struct xfs_scrub *sc, + struct xfs_inode *dp, + xfs_dir2_dataptr_t dapos, + const struct xfs_name *name, + xfs_ino_t ino, + void *priv) +{ + struct xrep_parent *rp = priv; + int error; + + /* Dirent doesn't point to this directory. */ + if (ino != rp->sc->ip->i_ino) + return 0; + + /* No weird looking names. */ + if (name->len == 0 || !xfs_dir2_namecheck(name->name, name->len)) + return -EFSCORRUPTED; + + /* No mismatching ftypes. */ + if (name->type != xfs_mode_to_ftype(VFS_I(sc->ip)->i_mode)) + return -EFSCORRUPTED; + + /* Don't pick up dot or dotdot entries; we only want child dirents. */ + if (xfs_dir2_samename(name, &xfs_name_dotdot) || + xfs_dir2_samename(name, &xfs_name_dot)) + return 0; + + /* + * Transform this dirent into a parent pointer and queue it for later + * addition to the temporary file. + */ + mutex_lock(&rp->pscan.lock); + error = xrep_parent_stash_parentadd(rp, name, dp); + mutex_unlock(&rp->pscan.lock); + return error; +} + +/* + * Decide if we want to look for dirents in this directory. Skip the file + * being repaired and any files being used to stage repairs. + */ +static inline bool +xrep_parent_want_scan( + struct xrep_parent *rp, + const struct xfs_inode *ip) +{ + return ip != rp->sc->ip && !xrep_is_tempfile(ip); +} + +/* + * Take ILOCK on a file that we want to scan. + * + * Select ILOCK_EXCL if the file is a directory with an unloaded data bmbt. + * Otherwise, take ILOCK_SHARED. + */ +static inline unsigned int +xrep_parent_scan_ilock( + struct xrep_parent *rp, + struct xfs_inode *ip) +{ + uint lock_mode = XFS_ILOCK_SHARED; + + /* Still need to take the shared ILOCK to advance the iscan cursor. */ + if (!xrep_parent_want_scan(rp, ip)) + goto lock; + + if (S_ISDIR(VFS_I(ip)->i_mode) && xfs_need_iread_extents(&ip->i_df)) { + lock_mode = XFS_ILOCK_EXCL; + goto lock; + } + +lock: + xfs_ilock(ip, lock_mode); + return lock_mode; +} + +/* + * Scan this file for relevant child dirents that point to the file whose + * parent pointers we're rebuilding. + */ +STATIC int +xrep_parent_scan_file( + struct xrep_parent *rp, + struct xfs_inode *ip) +{ + unsigned int lock_mode; + int error = 0; + + lock_mode = xrep_parent_scan_ilock(rp, ip); + + if (!xrep_parent_want_scan(rp, ip)) + goto scan_done; + + if (S_ISDIR(VFS_I(ip)->i_mode)) { + /* + * If the directory looks as though it has been zapped by the + * inode record repair code, we cannot scan for child dirents. + */ + if (xchk_dir_looks_zapped(ip)) { + error = -EBUSY; + goto scan_done; + } + + error = xchk_dir_walk(rp->sc, ip, xrep_parent_scan_dirent, rp); + if (error) + goto scan_done; + } + +scan_done: + xchk_iscan_mark_visited(&rp->pscan.iscan, ip); + xfs_iunlock(ip, lock_mode); + return error; +} + +/* Decide if we've stashed too much pptr data in memory. */ +static inline bool +xrep_parent_want_flush_stashed( + struct xrep_parent *rp) +{ + unsigned long long bytes; + + bytes = xfarray_bytes(rp->pptr_recs) + xfblob_bytes(rp->pptr_names); + return bytes > XREP_PARENT_MAX_STASH_BYTES; +} + +/* + * Scan all directories in the filesystem to look for dirents that we can turn + * into parent pointers. + */ +STATIC int +xrep_parent_scan_dirtree( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + struct xfs_inode *ip; + int error; + + /* + * Filesystem scans are time consuming. Drop the file ILOCK and all + * other resources for the duration of the scan and hope for the best. + * The live update hooks will keep our scan information up to date. + */ + xchk_trans_cancel(sc); + if (sc->ilock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) + xchk_iunlock(sc, sc->ilock_flags & (XFS_ILOCK_SHARED | + XFS_ILOCK_EXCL)); + error = xchk_trans_alloc_empty(sc); + if (error) + return error; + + while ((error = xchk_iscan_iter(&rp->pscan.iscan, &ip)) == 1) { + bool flush; + + error = xrep_parent_scan_file(rp, ip); + xchk_irele(sc, ip); + if (error) + break; + + /* Flush stashed pptr updates to constrain memory usage. */ + mutex_lock(&rp->pscan.lock); + flush = xrep_parent_want_flush_stashed(rp); + mutex_unlock(&rp->pscan.lock); + if (flush) { + xchk_trans_cancel(sc); + + error = xrep_tempfile_iolock_polled(sc); + if (error) + break; + + error = xrep_parent_replay_updates(rp); + xrep_tempfile_iounlock(sc); + if (error) + break; + + error = xchk_trans_alloc_empty(sc); + if (error) + break; + } + + if (xchk_should_terminate(sc, &error)) + break; + } + xchk_iscan_iter_finish(&rp->pscan.iscan); + if (error) { + /* + * If we couldn't grab an inode that was busy with a state + * change, change the error code so that we exit to userspace + * as quickly as possible. + */ + if (error == -EBUSY) + return -ECANCELED; + return error; + } + + /* + * Cancel the empty transaction so that we can (later) use the atomic + * extent swap helpers to lock files and commit the new directory. + */ + xchk_trans_cancel(rp->sc); + return 0; +} + /* Reset a directory's dotdot entry, if needed. */ STATIC int xrep_parent_reset_dotdot( @@ -298,8 +664,39 @@ xrep_parent_setup_scan( struct xrep_parent *rp) { struct xfs_scrub *sc = rp->sc; + char *descr; + int error; - return xrep_findparent_scan_start(sc, &rp->pscan); + if (!xfs_has_parent(sc->mp)) + return xrep_findparent_scan_start(sc, &rp->pscan); + + /* Set up some staging memory for logging parent pointer updates. */ + descr = xchk_xfile_ino_descr(sc, "parent pointer entries"); + error = xfarray_create(descr, 0, sizeof(struct xrep_pptr), + &rp->pptr_recs); + kfree(descr); + if (error) + return error; + + descr = xchk_xfile_ino_descr(sc, "parent pointer names"); + error = xfblob_create(descr, &rp->pptr_names); + kfree(descr); + if (error) + goto out_recs; + + error = xrep_findparent_scan_start(sc, &rp->pscan); + if (error) + goto out_names; + + return 0; + +out_names: + xfblob_destroy(rp->pptr_names); + rp->pptr_names = NULL; +out_recs: + xfarray_destroy(rp->pptr_recs); + rp->pptr_recs = NULL; + return error; } int @@ -309,11 +706,22 @@ xrep_parent( struct xrep_parent *rp = sc->buf; int error; + /* + * When the parent pointers feature is enabled, repairs are committed + * by atomically committing a new xattr structure and reaping the old + * attr fork. Reaping requires rmap to be enabled. + */ + if (xfs_has_parent(sc->mp) && !xfs_has_rmapbt(sc->mp)) + return -EOPNOTSUPP; + error = xrep_parent_setup_scan(rp); if (error) return error; - error = xrep_parent_find_dotdot(rp); + if (xfs_has_parent(sc->mp)) + error = xrep_parent_scan_dirtree(rp); + else + error = xrep_parent_find_dotdot(rp); if (error) goto out_teardown; diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 68532f686eeb1..10c2a8d10058b 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2820,6 +2820,42 @@ DEFINE_EVENT(xrep_pptr_class, name, \ TP_ARGS(ip, name, pptr)) DEFINE_XREP_PPTR_EVENT(xrep_xattr_replay_parentadd); DEFINE_XREP_PPTR_EVENT(xrep_xattr_replay_parentremove); +DEFINE_XREP_PPTR_EVENT(xrep_parent_replay_parentadd); + +DECLARE_EVENT_CLASS(xrep_pptr_scan_class, + TP_PROTO(struct xfs_inode *ip, const struct xfs_inode *dp, + const struct xfs_name *name), + TP_ARGS(ip, dp, name), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = ip->i_ino; + __entry->parent_ino = dp->i_ino; + __entry->parent_gen = VFS_IC(dp)->i_generation; + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d ino 0x%llx parent_ino 0x%llx parent_gen 0x%x name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->parent_ino, + __entry->parent_gen, + __entry->namelen, + __get_str(name)) +) +#define DEFINE_XREP_PPTR_SCAN_EVENT(name) \ +DEFINE_EVENT(xrep_pptr_scan_class, name, \ + TP_PROTO(struct xfs_inode *ip, const struct xfs_inode *dp, \ + const struct xfs_name *name), \ + TP_ARGS(ip, dp, name)) +DEFINE_XREP_PPTR_SCAN_EVENT(xrep_parent_stash_parentadd); TRACE_EVENT(xrep_nlinks_set_record, TP_PROTO(struct xfs_mount *mp, xfs_ino_t ino, From 65a1fb7a11291f361d36e6ebf3bb5e60e9ca8d13 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:13 -0700 Subject: [PATCH 0478/1702] xfs: implement live updates for parent pointer repairs While we're scanning the filesystem for dirents that we can turn into parent pointers, we cannot hold the IOLOCK or ILOCK of the file being repaired. Therefore, we need to set up a dirent hook so that we can keep the temporary file's parent pionters up to date with the rest of the filesystem. Hence we add the ability to *remove* pptrs from the temporary file. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/parent_repair.c | 103 +++++++++++++++++++++++++++++++++-- fs/xfs/scrub/trace.h | 2 + 2 files changed, 100 insertions(+), 5 deletions(-) diff --git a/fs/xfs/scrub/parent_repair.c b/fs/xfs/scrub/parent_repair.c index b4084a9f0e9c8..311bc7990d7c7 100644 --- a/fs/xfs/scrub/parent_repair.c +++ b/fs/xfs/scrub/parent_repair.c @@ -70,6 +70,12 @@ * disrupt attrmulti cursors. */ +/* Create a parent pointer in the tempfile. */ +#define XREP_PPTR_ADD (1) + +/* Remove a parent pointer from the tempfile. */ +#define XREP_PPTR_REMOVE (2) + /* A stashed parent pointer update. */ struct xrep_pptr { /* Cookie for retrieval of the pptr name. */ @@ -80,6 +86,9 @@ struct xrep_pptr { /* Length of the pptr name. */ uint8_t namelen; + + /* XREP_PPTR_{ADD,REMOVE} */ + uint8_t action; }; /* @@ -225,11 +234,25 @@ xrep_parent_replay_update( { struct xfs_scrub *sc = rp->sc; - /* Create parent pointer. */ - trace_xrep_parent_replay_parentadd(sc->tempip, xname, &pptr->pptr_rec); + switch (pptr->action) { + case XREP_PPTR_ADD: + /* Create parent pointer. */ + trace_xrep_parent_replay_parentadd(sc->tempip, xname, + &pptr->pptr_rec); + + return xfs_parent_set(sc->tempip, sc->ip->i_ino, xname, + &pptr->pptr_rec, &rp->pptr_args); + case XREP_PPTR_REMOVE: + /* Remove parent pointer. */ + trace_xrep_parent_replay_parentremove(sc->tempip, xname, + &pptr->pptr_rec); + + return xfs_parent_unset(sc->tempip, sc->ip->i_ino, xname, + &pptr->pptr_rec, &rp->pptr_args); + } - return xfs_parent_set(sc->tempip, sc->ip->i_ino, xname, - &pptr->pptr_rec, &rp->pptr_args); + ASSERT(0); + return -EIO; } /* @@ -290,6 +313,7 @@ xrep_parent_stash_parentadd( const struct xfs_inode *dp) { struct xrep_pptr pptr = { + .action = XREP_PPTR_ADD, .namelen = name->len, }; int error; @@ -304,6 +328,32 @@ xrep_parent_stash_parentadd( return xfarray_append(rp->pptr_recs, &pptr); } +/* + * Remember that we want to remove a parent pointer from the tempfile. These + * stashed actions will be replayed later. + */ +STATIC int +xrep_parent_stash_parentremove( + struct xrep_parent *rp, + const struct xfs_name *name, + const struct xfs_inode *dp) +{ + struct xrep_pptr pptr = { + .action = XREP_PPTR_REMOVE, + .namelen = name->len, + }; + int error; + + trace_xrep_parent_stash_parentremove(rp->sc->tempip, dp, name); + + xfs_inode_to_parent_rec(&pptr.pptr_rec, dp); + error = xfblob_storename(rp->pptr_names, &pptr.name_cookie, name); + if (error) + return error; + + return xfarray_append(rp->pptr_recs, &pptr); +} + /* * Examine an entry of a directory. If this dirent leads us back to the file * whose parent pointers we're rebuilding, add a pptr to the temporary @@ -513,6 +563,48 @@ xrep_parent_scan_dirtree( return 0; } +/* + * Capture dirent updates being made by other threads which are relevant to the + * file being repaired. + */ +STATIC int +xrep_parent_live_update( + struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct xfs_dir_update_params *p = data; + struct xrep_parent *rp; + struct xfs_scrub *sc; + int error; + + rp = container_of(nb, struct xrep_parent, pscan.dhook.dirent_hook.nb); + sc = rp->sc; + + /* + * This thread updated a dirent that points to the file that we're + * repairing, so stash the update for replay against the temporary + * file. + */ + if (p->ip->i_ino == sc->ip->i_ino && + xchk_iscan_want_live_update(&rp->pscan.iscan, p->dp->i_ino)) { + mutex_lock(&rp->pscan.lock); + if (p->delta > 0) + error = xrep_parent_stash_parentadd(rp, p->name, p->dp); + else + error = xrep_parent_stash_parentremove(rp, p->name, + p->dp); + mutex_unlock(&rp->pscan.lock); + if (error) + goto out_abort; + } + + return NOTIFY_DONE; +out_abort: + xchk_iscan_abort(&rp->pscan.iscan); + return NOTIFY_DONE; +} + /* Reset a directory's dotdot entry, if needed. */ STATIC int xrep_parent_reset_dotdot( @@ -684,7 +776,8 @@ xrep_parent_setup_scan( if (error) goto out_recs; - error = xrep_findparent_scan_start(sc, &rp->pscan); + error = __xrep_findparent_scan_start(sc, &rp->pscan, + xrep_parent_live_update); if (error) goto out_names; diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 10c2a8d10058b..3e0cd482379c6 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2821,6 +2821,7 @@ DEFINE_EVENT(xrep_pptr_class, name, \ DEFINE_XREP_PPTR_EVENT(xrep_xattr_replay_parentadd); DEFINE_XREP_PPTR_EVENT(xrep_xattr_replay_parentremove); DEFINE_XREP_PPTR_EVENT(xrep_parent_replay_parentadd); +DEFINE_XREP_PPTR_EVENT(xrep_parent_replay_parentremove); DECLARE_EVENT_CLASS(xrep_pptr_scan_class, TP_PROTO(struct xfs_inode *ip, const struct xfs_inode *dp, @@ -2856,6 +2857,7 @@ DEFINE_EVENT(xrep_pptr_scan_class, name, \ const struct xfs_name *name), \ TP_ARGS(ip, dp, name)) DEFINE_XREP_PPTR_SCAN_EVENT(xrep_parent_stash_parentadd); +DEFINE_XREP_PPTR_SCAN_EVENT(xrep_parent_stash_parentremove); TRACE_EVENT(xrep_nlinks_set_record, TP_PROTO(struct xfs_mount *mp, xfs_ino_t ino, From 13db7007892694c891fc37feccbd2ac8f227af78 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:14 -0700 Subject: [PATCH 0479/1702] xfs: remove pointless unlocked assertion Remove this assertion about the inode not having an attr fork from xfs_bmap_add_attrfork because the function handles that case just fine. Weirder still, the function actually /requires/ the caller not to hold the ILOCK, which means that its accesses are not stabilized. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_bmap.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 8a1446e025e07..1f528cf2d9069 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -1041,8 +1041,6 @@ xfs_bmap_add_attrfork( int logflags; /* logging flags */ int error; /* error return value */ - ASSERT(xfs_inode_has_attr_fork(ip) == 0); - mp = ip->i_mount; ASSERT(!XFS_NOT_DQATTACHED(mp, ip)); From 55edcd1f86474f973fccf5c5ccc8bc7908893142 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:15 -0700 Subject: [PATCH 0480/1702] xfs: split xfs_bmap_add_attrfork into two pieces Split this function into two pieces -- one to make the actual changes to the inode core to add the attr fork, and another one to deal with getting the transaction and locking the inodes. The next couple of patches will need this to be split into two. One patch implements committing new parent pointer recordsets to damaged files. If one file has an attr fork and the other does not, we have to create the missing attr fork before the atomic swap transaction, and can use the behavior encoded in the current xfs_bmap_add_attrfork. The second patch adapts /lost+found adoptions to handle parent pointers correctly. The adoption process will add a parent pointer to a child that is being moved to /lost+found, but this requires that the attr fork already exists. We don't know if we're actually going to commit the adoption until we've already reserved a transaction and taken the ILOCKs, which means that we must have a way to bypass the start of the current xfs_bmap_add_attrfork. Therefore, create xfs_attr_add_fork as the helper that creates a transaction and takes locks; and make xfs_bmap_add_attrfork the function that updates the inode core and allocates the incore attr fork. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 39 ++++++++++++++++++++++++++++++++++++++- fs/xfs/libxfs/xfs_bmap.c | 36 ++++++++++-------------------------- fs/xfs/libxfs/xfs_bmap.h | 3 ++- 3 files changed, 50 insertions(+), 28 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index c98145596f029..ab6ec2f15d766 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -948,6 +948,43 @@ xfs_attr_lookup( return error; } +STATIC int +xfs_attr_add_fork( + struct xfs_inode *ip, /* incore inode pointer */ + int size, /* space new attribute needs */ + int rsvd) /* xact may use reserved blks */ +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; /* transaction pointer */ + unsigned int blks; /* space reservation */ + int error; /* error return value */ + + ASSERT(!XFS_NOT_DQATTACHED(mp, ip)); + + blks = XFS_ADDAFORK_SPACE_RES(mp); + + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_addafork, blks, 0, + rsvd, &tp); + if (error) + return error; + + if (xfs_inode_has_attr_fork(ip)) + goto trans_cancel; + + error = xfs_bmap_add_attrfork(tp, ip, size, rsvd); + if (error) + goto trans_cancel; + + error = xfs_trans_commit(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; + +trans_cancel: + xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + /* * Make a change to the xattr structure. * @@ -989,7 +1026,7 @@ xfs_attr_set( xfs_attr_sf_entsize_byname(args->namelen, args->valuelen); - error = xfs_bmap_add_attrfork(dp, sf_size, rsvd); + error = xfs_attr_add_fork(dp, sf_size, rsvd); if (error) return error; } diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 1f528cf2d9069..6053f5e5c71ee 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -1025,38 +1025,29 @@ xfs_bmap_set_attrforkoff( } /* - * Convert inode from non-attributed to attributed. - * Must not be in a transaction, ip must not be locked. + * Convert inode from non-attributed to attributed. Caller must hold the + * ILOCK_EXCL and the file cannot have an attr fork. */ int /* error code */ xfs_bmap_add_attrfork( - xfs_inode_t *ip, /* incore inode pointer */ + struct xfs_trans *tp, + struct xfs_inode *ip, /* incore inode pointer */ int size, /* space new attribute needs */ int rsvd) /* xact may use reserved blks */ { - xfs_mount_t *mp; /* mount structure */ - xfs_trans_t *tp; /* transaction pointer */ - int blks; /* space reservation */ + struct xfs_mount *mp = tp->t_mountp; int version = 1; /* superblock attr version */ int logflags; /* logging flags */ int error; /* error return value */ - mp = ip->i_mount; + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); ASSERT(!XFS_NOT_DQATTACHED(mp, ip)); - - blks = XFS_ADDAFORK_SPACE_RES(mp); - - error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_addafork, blks, 0, - rsvd, &tp); - if (error) - return error; - if (xfs_inode_has_attr_fork(ip)) - goto trans_cancel; + ASSERT(!xfs_inode_has_attr_fork(ip)); xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); error = xfs_bmap_set_attrforkoff(ip, size, &version); if (error) - goto trans_cancel; + return error; xfs_ifork_init_attr(ip, XFS_DINODE_FMT_EXTENTS, 0); logflags = 0; @@ -1077,7 +1068,7 @@ xfs_bmap_add_attrfork( if (logflags) xfs_trans_log_inode(tp, ip, logflags); if (error) - goto trans_cancel; + return error; if (!xfs_has_attr(mp) || (!xfs_has_attr2(mp) && version == 2)) { bool log_sb = false; @@ -1096,14 +1087,7 @@ xfs_bmap_add_attrfork( xfs_log_sb(tp); } - error = xfs_trans_commit(tp); - xfs_iunlock(ip, XFS_ILOCK_EXCL); - return error; - -trans_cancel: - xfs_trans_cancel(tp); - xfs_iunlock(ip, XFS_ILOCK_EXCL); - return error; + return 0; } /* diff --git a/fs/xfs/libxfs/xfs_bmap.h b/fs/xfs/libxfs/xfs_bmap.h index 32fb2a455c294..e98849eb9bbae 100644 --- a/fs/xfs/libxfs/xfs_bmap.h +++ b/fs/xfs/libxfs/xfs_bmap.h @@ -176,7 +176,8 @@ int xfs_bmap_longest_free_extent(struct xfs_perag *pag, void xfs_trim_extent(struct xfs_bmbt_irec *irec, xfs_fileoff_t bno, xfs_filblks_t len); unsigned int xfs_bmap_compute_attr_offset(struct xfs_mount *mp); -int xfs_bmap_add_attrfork(struct xfs_inode *ip, int size, int rsvd); +int xfs_bmap_add_attrfork(struct xfs_trans *tp, struct xfs_inode *ip, + int size, int rsvd); void xfs_bmap_local_to_extents_empty(struct xfs_trans *tp, struct xfs_inode *ip, int whichfork); int xfs_bmap_local_to_extents(struct xfs_trans *tp, struct xfs_inode *ip, From 6efbbdeb140603351e1413aee79e789bf2279a2b Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:15 -0700 Subject: [PATCH 0481/1702] xfs: add a per-leaf block callback to xchk_xattr_walk Add a second callback function to xchk_xattr_walk so that we can do something in between attr leaf blocks. This will be used by the next patch to see if we should flush cached parent pointer updates to constrain memory usage. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/attr.c | 2 +- fs/xfs/scrub/dir_repair.c | 2 +- fs/xfs/scrub/listxattr.c | 10 +++++++++- fs/xfs/scrub/listxattr.h | 4 +++- fs/xfs/scrub/nlinks.c | 3 ++- fs/xfs/scrub/parent.c | 7 ++++--- 6 files changed, 20 insertions(+), 8 deletions(-) diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index b550f3e34ffc7..708334f9b2bd1 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -675,7 +675,7 @@ xchk_xattr( * iteration, which doesn't really follow the usual buffer * locking order. */ - error = xchk_xattr_walk(sc, sc->ip, xchk_xattr_actor, NULL); + error = xchk_xattr_walk(sc, sc->ip, xchk_xattr_actor, NULL, NULL); if (!xchk_fblock_process_error(sc, XFS_ATTR_FORK, 0, &error)) return error; diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index 60e31da4451e8..e968150fe0f06 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -1288,7 +1288,7 @@ xrep_dir_scan_file( goto scan_done; } - error = xchk_xattr_walk(rd->sc, ip, xrep_dir_scan_pptr, rd); + error = xchk_xattr_walk(rd->sc, ip, xrep_dir_scan_pptr, NULL, rd); if (error) goto scan_done; diff --git a/fs/xfs/scrub/listxattr.c b/fs/xfs/scrub/listxattr.c index cbe5911ecbbcf..256ff7700c942 100644 --- a/fs/xfs/scrub/listxattr.c +++ b/fs/xfs/scrub/listxattr.c @@ -221,6 +221,7 @@ xchk_xattr_walk_node( struct xfs_scrub *sc, struct xfs_inode *ip, xchk_xattr_fn attr_fn, + xchk_xattrleaf_fn leaf_fn, void *priv) { struct xfs_attr3_icleaf_hdr leafhdr; @@ -252,6 +253,12 @@ xchk_xattr_walk_node( xfs_trans_brelse(sc->tp, leaf_bp); + if (leaf_fn) { + error = leaf_fn(sc, priv); + if (error) + goto out_bitmap; + } + /* Make sure we haven't seen this new leaf already. */ len = 1; if (xdab_bitmap_test(&seen_dablks, leafhdr.forw, &len)) { @@ -288,6 +295,7 @@ xchk_xattr_walk( struct xfs_scrub *sc, struct xfs_inode *ip, xchk_xattr_fn attr_fn, + xchk_xattrleaf_fn leaf_fn, void *priv) { int error; @@ -308,5 +316,5 @@ xchk_xattr_walk( if (xfs_attr_is_leaf(ip)) return xchk_xattr_walk_leaf(sc, ip, attr_fn, priv); - return xchk_xattr_walk_node(sc, ip, attr_fn, priv); + return xchk_xattr_walk_node(sc, ip, attr_fn, leaf_fn, priv); } diff --git a/fs/xfs/scrub/listxattr.h b/fs/xfs/scrub/listxattr.h index 48fe89d05946b..703cfb7b14cfd 100644 --- a/fs/xfs/scrub/listxattr.h +++ b/fs/xfs/scrub/listxattr.h @@ -11,7 +11,9 @@ typedef int (*xchk_xattr_fn)(struct xfs_scrub *sc, struct xfs_inode *ip, unsigned int namelen, const void *value, unsigned int valuelen, void *priv); +typedef int (*xchk_xattrleaf_fn)(struct xfs_scrub *sc, void *priv); + int xchk_xattr_walk(struct xfs_scrub *sc, struct xfs_inode *ip, - xchk_xattr_fn attr_fn, void *priv); + xchk_xattr_fn attr_fn, xchk_xattrleaf_fn leaf_fn, void *priv); #endif /* __XFS_SCRUB_LISTXATTR_H__ */ diff --git a/fs/xfs/scrub/nlinks.c b/fs/xfs/scrub/nlinks.c index d27b32e6f33db..80aee30886c45 100644 --- a/fs/xfs/scrub/nlinks.c +++ b/fs/xfs/scrub/nlinks.c @@ -434,7 +434,8 @@ xchk_nlinks_collect_dir( goto out_unlock; } - error = xchk_xattr_walk(sc, dp, xchk_nlinks_collect_pptr, xnc); + error = xchk_xattr_walk(sc, dp, xchk_nlinks_collect_pptr, NULL, + xnc); if (error == -ECANCELED) { error = 0; goto out_unlock; diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c index 068691434be17..733c410a22797 100644 --- a/fs/xfs/scrub/parent.c +++ b/fs/xfs/scrub/parent.c @@ -317,7 +317,7 @@ xchk_parent_pptr_and_dotdot( return 0; /* Otherwise, walk the pptrs again, and check. */ - error = xchk_xattr_walk(sc, sc->ip, xchk_parent_scan_dotdot, pp); + error = xchk_xattr_walk(sc, sc->ip, xchk_parent_scan_dotdot, NULL, pp); if (error == -ECANCELED) { /* Found a parent pointer that matches dotdot. */ return 0; @@ -699,7 +699,8 @@ xchk_parent_count_pptrs( */ if (pp->need_revalidate) { pp->pptrs_found = 0; - error = xchk_xattr_walk(sc, sc->ip, xchk_parent_count_pptr, pp); + error = xchk_xattr_walk(sc, sc->ip, xchk_parent_count_pptr, + NULL, pp); if (error == -EFSCORRUPTED) { /* Found a bad parent pointer */ xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); @@ -758,7 +759,7 @@ xchk_parent_pptr( if (error) goto out_entries; - error = xchk_xattr_walk(sc, sc->ip, xchk_parent_scan_attr, pp); + error = xchk_xattr_walk(sc, sc->ip, xchk_parent_scan_attr, NULL, pp); if (error == -ECANCELED) { error = 0; goto out_names; From a26dc21309af68623b82b4e366cbbeb5a85ce65b Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:16 -0700 Subject: [PATCH 0482/1702] xfs: actually rebuild the parent pointer xattrs Once we've assembled all the parent pointers for a file, we need to commit the new dataset atomically to that file. Parent pointer records are embedded in the xattr structure, which means that we must write a new extended attribute structure, again, atomically. Therefore, we must copy the non-parent-pointer attributes from the file being repaired into the temporary file's extended attributes and then call the atomic extent swap mechanism to exchange the blocks. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 2 +- fs/xfs/libxfs/xfs_attr.h | 1 + fs/xfs/scrub/attr_repair.c | 4 +- fs/xfs/scrub/attr_repair.h | 4 + fs/xfs/scrub/findparent.c | 2 + fs/xfs/scrub/parent_repair.c | 709 ++++++++++++++++++++++++++++++++++- fs/xfs/scrub/trace.h | 2 + 7 files changed, 701 insertions(+), 23 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index ab6ec2f15d766..1c2a27fce08a9 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -948,7 +948,7 @@ xfs_attr_lookup( return error; } -STATIC int +int xfs_attr_add_fork( struct xfs_inode *ip, /* incore inode pointer */ int size, /* space new attribute needs */ diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h index 43dee4cbaab25..088cb7b301680 100644 --- a/fs/xfs/libxfs/xfs_attr.h +++ b/fs/xfs/libxfs/xfs_attr.h @@ -648,5 +648,6 @@ int __init xfs_attr_intent_init_cache(void); void xfs_attr_intent_destroy_cache(void); int xfs_attr_sf_totsize(struct xfs_inode *dp); +int xfs_attr_add_fork(struct xfs_inode *ip, int size, int rsvd); #endif /* __XFS_ATTR_H__ */ diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c index 87f6c9cb54eb6..e059813b92b70 100644 --- a/fs/xfs/scrub/attr_repair.c +++ b/fs/xfs/scrub/attr_repair.c @@ -1030,7 +1030,7 @@ xrep_xattr_reset_fork( * fork. The caller must ILOCK the tempfile and join it to the transaction. * This function returns with the inode joined to a clean scrub transaction. */ -STATIC int +int xrep_xattr_reset_tempfile_fork( struct xfs_scrub *sc) { @@ -1336,7 +1336,7 @@ xrep_xattr_swap_prep( } /* Exchange the temporary file's attribute fork with the one being repaired. */ -STATIC int +int xrep_xattr_swap( struct xfs_scrub *sc, struct xrep_tempexch *tx) diff --git a/fs/xfs/scrub/attr_repair.h b/fs/xfs/scrub/attr_repair.h index 0a9ffa7cfa906..979729bd4a5f8 100644 --- a/fs/xfs/scrub/attr_repair.h +++ b/fs/xfs/scrub/attr_repair.h @@ -6,6 +6,10 @@ #ifndef __XFS_SCRUB_ATTR_REPAIR_H__ #define __XFS_SCRUB_ATTR_REPAIR_H__ +struct xrep_tempexch; + +int xrep_xattr_swap(struct xfs_scrub *sc, struct xrep_tempexch *tx); int xrep_xattr_reset_fork(struct xfs_scrub *sc); +int xrep_xattr_reset_tempfile_fork(struct xfs_scrub *sc); #endif /* __XFS_SCRUB_ATTR_REPAIR_H__ */ diff --git a/fs/xfs/scrub/findparent.c b/fs/xfs/scrub/findparent.c index c78422ad757bf..01766041ba2cd 100644 --- a/fs/xfs/scrub/findparent.c +++ b/fs/xfs/scrub/findparent.c @@ -24,6 +24,7 @@ #include "xfs_trans_space.h" #include "xfs_health.h" #include "xfs_exchmaps.h" +#include "xfs_parent.h" #include "scrub/xfs_scrub.h" #include "scrub/scrub.h" #include "scrub/common.h" @@ -33,6 +34,7 @@ #include "scrub/findparent.h" #include "scrub/readdir.h" #include "scrub/tempfile.h" +#include "scrub/listxattr.h" /* * Finding the Parent of a Directory diff --git a/fs/xfs/scrub/parent_repair.c b/fs/xfs/scrub/parent_repair.c index 311bc7990d7c7..28e9746c06631 100644 --- a/fs/xfs/scrub/parent_repair.c +++ b/fs/xfs/scrub/parent_repair.c @@ -25,6 +25,8 @@ #include "xfs_health.h" #include "xfs_exchmaps.h" #include "xfs_parent.h" +#include "xfs_attr.h" +#include "xfs_bmap.h" #include "scrub/xfs_scrub.h" #include "scrub/scrub.h" #include "scrub/common.h" @@ -34,10 +36,13 @@ #include "scrub/findparent.h" #include "scrub/readdir.h" #include "scrub/tempfile.h" +#include "scrub/tempexch.h" #include "scrub/orphanage.h" #include "scrub/xfile.h" #include "scrub/xfarray.h" #include "scrub/xfblob.h" +#include "scrub/attr_repair.h" +#include "scrub/listxattr.h" /* * Repairing The Directory Parent Pointer @@ -65,9 +70,9 @@ * * When salvaging completes, the remaining stashed entries are replayed to the * temporary file. All non-parent pointer extended attributes are copied to - * the temporary file's extended attributes. An atomic extent swap is used to - * commit the new directory blocks to the directory being repaired. This will - * disrupt attrmulti cursors. + * the temporary file's extended attributes. An atomic file mapping exchange + * is used to commit the new xattr blocks to the file being repaired. This + * will disrupt attrmulti cursors. */ /* Create a parent pointer in the tempfile. */ @@ -106,6 +111,23 @@ struct xrep_parent { /* Blobs containing parent pointer names. */ struct xfblob *pptr_names; + /* xattr keys */ + struct xfarray *xattr_records; + + /* xattr values */ + struct xfblob *xattr_blobs; + + /* Scratch buffers for saving extended attributes */ + unsigned char *xattr_name; + void *xattr_value; + unsigned int xattr_value_sz; + + /* + * Information used to exchange the attr fork mappings, if the fs + * supports parent pointers. + */ + struct xrep_tempexch tx; + /* * Information used to scan the filesystem to find the inumber of the * dotdot entry for this directory. On filesystems without parent @@ -117,6 +139,8 @@ struct xrep_parent { * @pscan.lock coordinates access to pptr_recs, pptr_names, pptr, and * pptr_scratch. This reduces the memory requirements of this * structure. + * + * The lock also controls access to xattr_records and xattr_blobs(?) */ struct xrep_parent_scan_info pscan; @@ -129,14 +153,48 @@ struct xrep_parent { /* Scratch buffer for scanning pptr xattrs */ struct xfs_da_args pptr_args; + + /* Have we seen any live updates of parent pointers recently? */ + bool saw_pptr_updates; }; +struct xrep_parent_xattr { + /* Cookie for retrieval of the xattr name. */ + xfblob_cookie name_cookie; + + /* Cookie for retrieval of the xattr value. */ + xfblob_cookie value_cookie; + + /* XFS_ATTR_* flags */ + int flags; + + /* Length of the value and name. */ + uint32_t valuelen; + uint16_t namelen; +}; + +/* + * Stash up to 8 pages of attrs in xattr_records/xattr_blobs before we write + * them to the temp file. + */ +#define XREP_PARENT_XATTR_MAX_STASH_BYTES (PAGE_SIZE * 8) + /* Tear down all the incore stuff we created. */ static void xrep_parent_teardown( struct xrep_parent *rp) { xrep_findparent_scan_teardown(&rp->pscan); + kvfree(rp->xattr_name); + rp->xattr_name = NULL; + kvfree(rp->xattr_value); + rp->xattr_value = NULL; + if (rp->xattr_blobs) + xfblob_destroy(rp->xattr_blobs); + rp->xattr_blobs = NULL; + if (rp->xattr_records) + xfarray_destroy(rp->xattr_records); + rp->xattr_records = NULL; if (rp->pptr_names) xfblob_destroy(rp->pptr_names); rp->pptr_names = NULL; @@ -556,10 +614,11 @@ xrep_parent_scan_dirtree( } /* - * Cancel the empty transaction so that we can (later) use the atomic - * extent swap helpers to lock files and commit the new directory. + * Retake sc->ip's ILOCK now that we're done flushing stashed parent + * pointers. We end this function with an empty transaction and the + * ILOCK. */ - xchk_trans_cancel(rp->sc); + xchk_ilock(rp->sc, XFS_ILOCK_EXCL); return 0; } @@ -594,6 +653,8 @@ xrep_parent_live_update( else error = xrep_parent_stash_parentremove(rp, p->name, p->dp); + if (!error) + rp->saw_pptr_updates = true; mutex_unlock(&rp->pscan.lock); if (error) goto out_abort; @@ -648,6 +709,55 @@ xrep_parent_reset_dotdot( return xfs_trans_roll(&sc->tp); } +/* Pass back the parent inumber if this a parent pointer */ +STATIC int +xrep_parent_lookup_pptr( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + xfs_ino_t *inop = priv; + xfs_ino_t parent_ino; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, &parent_ino, NULL); + if (error) + return error; + + *inop = parent_ino; + return -ECANCELED; +} + +/* + * Find the first parent of the scrub target by walking parent pointers for + * the purpose of deciding if we're going to move it to the orphanage. + * We don't care if the attr fork is zapped. + */ +STATIC int +xrep_parent_lookup_pptrs( + struct xfs_scrub *sc, + xfs_ino_t *inop) +{ + int error; + + *inop = NULLFSINO; + + error = xchk_xattr_walk(sc, sc->ip, xrep_parent_lookup_pptr, NULL, + inop); + if (error && error != -ECANCELED) + return error; + return 0; +} + /* * Move the current file to the orphanage. * @@ -664,14 +774,26 @@ xrep_parent_move_to_orphanage( xfs_ino_t orig_parent, new_parent; int error; - /* - * We are about to drop the ILOCK on sc->ip to lock the orphanage and - * prepare for the adoption. Therefore, look up the old dotdot entry - * for sc->ip so that we can compare it after we re-lock sc->ip. - */ - error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, &orig_parent); - if (error) - return error; + if (S_ISDIR(VFS_I(sc->ip)->i_mode)) { + /* + * We are about to drop the ILOCK on sc->ip to lock the + * orphanage and prepare for the adoption. Therefore, look up + * the old dotdot entry for sc->ip so that we can compare it + * after we re-lock sc->ip. + */ + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, + &orig_parent); + if (error) + return error; + } else { + /* + * We haven't dropped the ILOCK since we committed the new + * xattr structure (and hence the new parent pointer records), + * which means that the file cannot have been moved in the + * directory tree, and there are no parents. + */ + orig_parent = NULLFSINO; + } /* * Drop the ILOCK on the scrub target and commit the transaction. @@ -704,9 +826,14 @@ xrep_parent_move_to_orphanage( * Now that we've reacquired the ILOCK on sc->ip, look up the dotdot * entry again. If the parent changed or the child was unlinked while * the child directory was unlocked, we don't need to move the child to - * the orphanage after all. + * the orphanage after all. For a non-directory, we have to scan for + * the first parent pointer to see if one has been added. */ - error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, &new_parent); + if (S_ISDIR(VFS_I(sc->ip)->i_mode)) + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, + &new_parent); + else + error = xrep_parent_lookup_pptrs(sc, &new_parent); if (error) return error; @@ -733,6 +860,488 @@ xrep_parent_move_to_orphanage( return 0; } +/* Ensure that the xattr value buffer is large enough. */ +STATIC int +xrep_parent_alloc_xattr_value( + struct xrep_parent *rp, + size_t bufsize) +{ + void *new_val; + + if (rp->xattr_value_sz >= bufsize) + return 0; + + if (rp->xattr_value) { + kvfree(rp->xattr_value); + rp->xattr_value = NULL; + rp->xattr_value_sz = 0; + } + + new_val = kvmalloc(bufsize, XCHK_GFP_FLAGS); + if (!new_val) + return -ENOMEM; + + rp->xattr_value = new_val; + rp->xattr_value_sz = bufsize; + return 0; +} + +/* Retrieve the (remote) value of a non-pptr xattr. */ +STATIC int +xrep_parent_fetch_xattr_remote( + struct xrep_parent *rp, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + unsigned int valuelen) +{ + struct xfs_scrub *sc = rp->sc; + struct xfs_da_args args = { + .attr_filter = attr_flags & XFS_ATTR_NSP_ONDISK_MASK, + .geo = sc->mp->m_attr_geo, + .whichfork = XFS_ATTR_FORK, + .dp = ip, + .name = name, + .namelen = namelen, + .trans = sc->tp, + .valuelen = valuelen, + .owner = ip->i_ino, + }; + int error; + + /* + * If we need a larger value buffer, try to allocate one. If that + * fails, return with -EDEADLOCK to try harder. + */ + error = xrep_parent_alloc_xattr_value(rp, valuelen); + if (error == -ENOMEM) + return -EDEADLOCK; + if (error) + return error; + + args.value = rp->xattr_value; + xfs_attr_sethash(&args); + return xfs_attr_get_ilocked(&args); +} + +/* Stash non-pptr attributes for later replay into the temporary file. */ +STATIC int +xrep_parent_stash_xattr( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xrep_parent_xattr key = { + .valuelen = valuelen, + .namelen = namelen, + .flags = attr_flags & XFS_ATTR_NSP_ONDISK_MASK, + }; + struct xrep_parent *rp = priv; + int error; + + if (attr_flags & (XFS_ATTR_INCOMPLETE | XFS_ATTR_PARENT)) + return 0; + + if (!value) { + error = xrep_parent_fetch_xattr_remote(rp, ip, attr_flags, + name, namelen, valuelen); + if (error) + return error; + + value = rp->xattr_value; + } + + trace_xrep_parent_stash_xattr(rp->sc->tempip, key.flags, (void *)name, + key.namelen, key.valuelen); + + error = xfblob_store(rp->xattr_blobs, &key.name_cookie, name, + key.namelen); + if (error) + return error; + + error = xfblob_store(rp->xattr_blobs, &key.value_cookie, value, + key.valuelen); + if (error) + return error; + + return xfarray_append(rp->xattr_records, &key); +} + +/* Insert one xattr key/value. */ +STATIC int +xrep_parent_insert_xattr( + struct xrep_parent *rp, + const struct xrep_parent_xattr *key) +{ + struct xfs_da_args args = { + .dp = rp->sc->tempip, + .attr_filter = key->flags, + .namelen = key->namelen, + .valuelen = key->valuelen, + .owner = rp->sc->ip->i_ino, + .geo = rp->sc->mp->m_attr_geo, + .whichfork = XFS_ATTR_FORK, + .op_flags = XFS_DA_OP_OKNOENT, + }; + int error; + + ASSERT(!(key->flags & XFS_ATTR_PARENT)); + + /* + * Grab pointers to the scrub buffer so that we can use them to insert + * attrs into the temp file. + */ + args.name = rp->xattr_name; + args.value = rp->xattr_value; + + /* + * The attribute name is stored near the end of the in-core buffer, + * though we reserve one more byte to ensure null termination. + */ + rp->xattr_name[XATTR_NAME_MAX] = 0; + + error = xfblob_load(rp->xattr_blobs, key->name_cookie, rp->xattr_name, + key->namelen); + if (error) + return error; + + error = xfblob_free(rp->xattr_blobs, key->name_cookie); + if (error) + return error; + + error = xfblob_load(rp->xattr_blobs, key->value_cookie, args.value, + key->valuelen); + if (error) + return error; + + error = xfblob_free(rp->xattr_blobs, key->value_cookie); + if (error) + return error; + + rp->xattr_name[key->namelen] = 0; + + trace_xrep_parent_insert_xattr(rp->sc->tempip, key->flags, + rp->xattr_name, key->namelen, key->valuelen); + + xfs_attr_sethash(&args); + return xfs_attr_set(&args, XFS_ATTRUPDATE_UPSERT, false); +} + +/* + * Periodically flush salvaged attributes to the temporary file. This is done + * to reduce the memory requirements of the xattr rebuild because files can + * contain millions of attributes. + */ +STATIC int +xrep_parent_flush_xattrs( + struct xrep_parent *rp) +{ + xfarray_idx_t array_cur; + int error; + + /* + * Entering this function, the scrub context has a reference to the + * inode being repaired, the temporary file, and the empty scrub + * transaction that we created for the xattr scan. We hold ILOCK_EXCL + * on the inode being repaired. + * + * To constrain kernel memory use, we occasionally flush salvaged + * xattrs from the xfarray and xfblob structures into the temporary + * file in preparation for exchanging the xattr structures at the end. + * Updating the temporary file requires a transaction, so we commit the + * scrub transaction and drop the ILOCK so that xfs_attr_set can + * allocate whatever transaction it wants. + * + * We still hold IOLOCK_EXCL on the inode being repaired, which + * prevents anyone from adding xattrs (or parent pointers) while we're + * flushing. + */ + xchk_trans_cancel(rp->sc); + xchk_iunlock(rp->sc, XFS_ILOCK_EXCL); + + /* + * Take the IOLOCK of the temporary file while we modify xattrs. This + * isn't strictly required because the temporary file is never revealed + * to userspace, but we follow the same locking rules. We still hold + * sc->ip's IOLOCK. + */ + error = xrep_tempfile_iolock_polled(rp->sc); + if (error) + return error; + + /* Add all the salvaged attrs to the temporary file. */ + foreach_xfarray_idx(rp->xattr_records, array_cur) { + struct xrep_parent_xattr key; + + error = xfarray_load(rp->xattr_records, array_cur, &key); + if (error) + return error; + + error = xrep_parent_insert_xattr(rp, &key); + if (error) + return error; + } + + /* Empty out both arrays now that we've added the entries. */ + xfarray_truncate(rp->xattr_records); + xfblob_truncate(rp->xattr_blobs); + + xrep_tempfile_iounlock(rp->sc); + + /* Recreate the empty transaction and relock the inode. */ + error = xchk_trans_alloc_empty(rp->sc); + if (error) + return error; + xchk_ilock(rp->sc, XFS_ILOCK_EXCL); + return 0; +} + +/* Decide if we've stashed too much xattr data in memory. */ +static inline bool +xrep_parent_want_flush_xattrs( + struct xrep_parent *rp) +{ + unsigned long long bytes; + + bytes = xfarray_bytes(rp->xattr_records) + + xfblob_bytes(rp->xattr_blobs); + return bytes > XREP_PARENT_XATTR_MAX_STASH_BYTES; +} + +/* Flush staged attributes to the temporary file if we're over the limit. */ +STATIC int +xrep_parent_try_flush_xattrs( + struct xfs_scrub *sc, + void *priv) +{ + struct xrep_parent *rp = priv; + int error; + + if (!xrep_parent_want_flush_xattrs(rp)) + return 0; + + error = xrep_parent_flush_xattrs(rp); + if (error) + return error; + + /* + * If there were any parent pointer updates to the xattr structure + * while we dropped the ILOCK, the xattr structure is now stale. + * Signal to the attr copy process that we need to start over, but + * this time without opportunistic attr flushing. + * + * This is unlikely to happen, so we're ok with restarting the copy. + */ + mutex_lock(&rp->pscan.lock); + if (rp->saw_pptr_updates) + error = -ESTALE; + mutex_unlock(&rp->pscan.lock); + return error; +} + +/* Copy all the non-pptr extended attributes into the temporary file. */ +STATIC int +xrep_parent_copy_xattrs( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + int error; + + /* + * Clear the pptr updates flag. We hold sc->ip ILOCKed, so there + * can't be any parent pointer updates in progress. + */ + mutex_lock(&rp->pscan.lock); + rp->saw_pptr_updates = false; + mutex_unlock(&rp->pscan.lock); + + /* Copy xattrs, stopping periodically to flush the incore buffers. */ + error = xchk_xattr_walk(sc, sc->ip, xrep_parent_stash_xattr, + xrep_parent_try_flush_xattrs, rp); + if (error && error != -ESTALE) + return error; + + if (error == -ESTALE) { + /* + * The xattr copy collided with a parent pointer update. + * Restart the copy, but this time hold the ILOCK all the way + * to the end to lock out any directory parent pointer updates. + */ + error = xchk_xattr_walk(sc, sc->ip, xrep_parent_stash_xattr, + NULL, rp); + if (error) + return error; + } + + /* Flush any remaining stashed xattrs to the temporary file. */ + if (xfarray_bytes(rp->xattr_records) == 0) + return 0; + + return xrep_parent_flush_xattrs(rp); +} + +/* + * Ensure that @sc->ip and @sc->tempip both have attribute forks before we head + * into the attr fork exchange transaction. All files on a filesystem with + * parent pointers must have an attr fork because the parent pointer code does + * not itself add attribute forks. + * + * Note: Unlinkable unlinked files don't need one, but the overhead of having + * an unnecessary attr fork is not justified by the additional code complexity + * that would be needed to track that state correctly. + */ +STATIC int +xrep_parent_ensure_attr_fork( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + int error; + + error = xfs_attr_add_fork(sc->tempip, + sizeof(struct xfs_attr_sf_hdr), 1); + if (error) + return error; + return xfs_attr_add_fork(sc->ip, sizeof(struct xfs_attr_sf_hdr), 1); +} + +/* + * Finish replaying stashed parent pointer updates, allocate a transaction for + * exchanging extent mappings, and take the ILOCKs of both files before we + * commit the new attribute structure. + */ +STATIC int +xrep_parent_finalize_tempfile( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + int error; + + /* + * Repair relies on the ILOCK to quiesce all possible xattr updates. + * Replay all queued parent pointer updates into the tempfile before + * exchanging the contents, even if that means dropping the ILOCKs and + * the transaction. + */ + do { + error = xrep_parent_replay_updates(rp); + if (error) + return error; + + error = xrep_parent_ensure_attr_fork(rp); + if (error) + return error; + + error = xrep_tempexch_trans_alloc(sc, XFS_ATTR_FORK, &rp->tx); + if (error) + return error; + + if (xfarray_length(rp->pptr_recs) == 0) + break; + + xchk_trans_cancel(sc); + xrep_tempfile_iunlock_both(sc); + } while (!xchk_should_terminate(sc, &error)); + return error; +} + +/* + * Replay all the stashed parent pointers into the temporary file, copy all + * the non-pptr xattrs from the file being repaired into the temporary file, + * and exchange the attr fork contents atomically. + */ +STATIC int +xrep_parent_rebuild_pptrs( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + xfs_ino_t parent_ino = NULLFSINO; + int error; + + /* + * Copy non-ppttr xattrs from the file being repaired into the + * temporary file's xattr structure. We hold sc->ip's IOLOCK, which + * prevents setxattr/removexattr calls from occurring, but renames + * update the parent pointers without holding IOLOCK. If we detect + * stale attr structures, we restart the scan but only flush at the + * end. + */ + error = xrep_parent_copy_xattrs(rp); + if (error) + return error; + + /* + * Cancel the empty transaction that we used to walk and copy attrs, + * and drop the ILOCK so that we can take the IOLOCK on the temporary + * file. We still hold sc->ip's IOLOCK. + */ + xchk_trans_cancel(sc); + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + error = xrep_tempfile_iolock_polled(sc); + if (error) + return error; + + /* + * Allocate transaction, lock inodes, and make sure that we've replayed + * all the stashed pptr updates to the tempdir. After this point, + * we're ready to exchange the attr fork mappings. + */ + error = xrep_parent_finalize_tempfile(rp); + if (error) + return error; + + /* Last chance to abort before we start committing pptr fixes. */ + if (xchk_should_terminate(sc, &error)) + return error; + + if (xchk_iscan_aborted(&rp->pscan.iscan)) + return -ECANCELED; + + /* + * Exchange the attr fork contents and junk the old attr fork contents, + * which are now in the tempfile. + */ + error = xrep_xattr_swap(sc, &rp->tx); + if (error) + return error; + error = xrep_xattr_reset_tempfile_fork(sc); + if (error) + return error; + + /* + * Roll to get a transaction without any inodes joined to it. Then we + * can drop the tempfile's ILOCK and IOLOCK before doing more work on + * the scrub target file. + */ + error = xfs_trans_roll(&sc->tp); + if (error) + return error; + xrep_tempfile_iunlock(sc); + xrep_tempfile_iounlock(sc); + + /* + * We've committed the new parent pointers. Find at least one parent + * so that we can decide if we're moving this file to the orphanage. + * For this purpose, root directories are their own parents. + */ + if (sc->ip == sc->mp->m_rootip) { + xrep_findparent_scan_found(&rp->pscan, sc->ip->i_ino); + } else { + error = xrep_parent_lookup_pptrs(sc, &parent_ino); + if (error) + return error; + if (parent_ino != NULLFSINO) + xrep_findparent_scan_found(&rp->pscan, parent_ino); + } + return 0; +} + /* * Commit the new parent pointer structure (currently only the dotdot entry) to * the file that we're repairing. @@ -741,13 +1350,24 @@ STATIC int xrep_parent_rebuild_tree( struct xrep_parent *rp) { + int error; + + if (xfs_has_parent(rp->sc->mp)) { + error = xrep_parent_rebuild_pptrs(rp); + if (error) + return error; + } + if (rp->pscan.parent_ino == NULLFSINO) { if (xrep_orphanage_can_adopt(rp->sc)) return xrep_parent_move_to_orphanage(rp); return -EFSCORRUPTED; } - return xrep_parent_reset_dotdot(rp); + if (S_ISDIR(VFS_I(rp->sc->ip)->i_mode)) + return xrep_parent_reset_dotdot(rp); + + return 0; } /* Set up the filesystem scan so we can look for parents. */ @@ -757,18 +1377,39 @@ xrep_parent_setup_scan( { struct xfs_scrub *sc = rp->sc; char *descr; + struct xfs_da_geometry *geo = sc->mp->m_attr_geo; + int max_len; int error; if (!xfs_has_parent(sc->mp)) return xrep_findparent_scan_start(sc, &rp->pscan); + /* Buffers for copying non-pptr attrs to the tempfile */ + rp->xattr_name = kvmalloc(XATTR_NAME_MAX + 1, XCHK_GFP_FLAGS); + if (!rp->xattr_name) + return -ENOMEM; + + /* + * Allocate enough memory to handle loading local attr values from the + * xfblob data while flushing stashed attrs to the temporary file. + * We only realloc the buffer when salvaging remote attr values, so + * TRY_HARDER means we allocate the maximal attr value size. + */ + if (sc->flags & XCHK_TRY_HARDER) + max_len = XATTR_SIZE_MAX; + else + max_len = xfs_attr_leaf_entsize_local_max(geo->blksize); + error = xrep_parent_alloc_xattr_value(rp, max_len); + if (error) + goto out_xattr_name; + /* Set up some staging memory for logging parent pointer updates. */ descr = xchk_xfile_ino_descr(sc, "parent pointer entries"); error = xfarray_create(descr, 0, sizeof(struct xrep_pptr), &rp->pptr_recs); kfree(descr); if (error) - return error; + goto out_xattr_value; descr = xchk_xfile_ino_descr(sc, "parent pointer names"); error = xfblob_create(descr, &rp->pptr_names); @@ -776,19 +1417,47 @@ xrep_parent_setup_scan( if (error) goto out_recs; + /* Set up some storage for copying attrs before the mapping exchange */ + descr = xchk_xfile_ino_descr(sc, + "parent pointer retained xattr entries"); + error = xfarray_create(descr, 0, sizeof(struct xrep_parent_xattr), + &rp->xattr_records); + kfree(descr); + if (error) + goto out_names; + + descr = xchk_xfile_ino_descr(sc, + "parent pointer retained xattr values"); + error = xfblob_create(descr, &rp->xattr_blobs); + kfree(descr); + if (error) + goto out_attr_keys; + error = __xrep_findparent_scan_start(sc, &rp->pscan, xrep_parent_live_update); if (error) - goto out_names; + goto out_attr_values; return 0; +out_attr_values: + xfblob_destroy(rp->xattr_blobs); + rp->xattr_blobs = NULL; +out_attr_keys: + xfarray_destroy(rp->xattr_records); + rp->xattr_records = NULL; out_names: xfblob_destroy(rp->pptr_names); rp->pptr_names = NULL; out_recs: xfarray_destroy(rp->pptr_recs); rp->pptr_recs = NULL; +out_xattr_value: + kvfree(rp->xattr_value); + rp->xattr_value = NULL; +out_xattr_name: + kvfree(rp->xattr_name); + rp->xattr_name = NULL; return error; } @@ -818,7 +1487,7 @@ xrep_parent( if (error) goto out_teardown; - /* Last chance to abort before we start committing fixes. */ + /* Last chance to abort before we start committing dotdot fixes. */ if (xchk_should_terminate(sc, &error)) goto out_teardown; diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 3e0cd482379c6..ecfaa4b88910f 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2539,6 +2539,8 @@ DEFINE_EVENT(xrep_xattr_salvage_class, name, \ TP_ARGS(ip, flags, name, namelen, valuelen)) DEFINE_XREP_XATTR_SALVAGE_EVENT(xrep_xattr_salvage_rec); DEFINE_XREP_XATTR_SALVAGE_EVENT(xrep_xattr_insert_rec); +DEFINE_XREP_XATTR_SALVAGE_EVENT(xrep_parent_stash_xattr); +DEFINE_XREP_XATTR_SALVAGE_EVENT(xrep_parent_insert_xattr); DECLARE_EVENT_CLASS(xrep_pptr_salvage_class, TP_PROTO(struct xfs_inode *ip, unsigned int flags, const void *name, From 7be3d20bbeda6602d6c5f67ec2b8f189a07ea0e3 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:17 -0700 Subject: [PATCH 0483/1702] xfs: adapt the orphanage code to handle parent pointers Adapt the orphanage's adoption code to update the child file's parent pointers as part of the reparenting process. Also ensure that the child has an attr fork to receive the parent pointer update, since the runtime code assumes one exists. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/orphanage.c | 38 ++++++++++++++++++++++++++++++++++++++ fs/xfs/scrub/orphanage.h | 3 +++ fs/xfs/scrub/scrub.c | 2 ++ 3 files changed, 43 insertions(+) diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c index 94bcc2799188f..b2f905924d0d8 100644 --- a/fs/xfs/scrub/orphanage.c +++ b/fs/xfs/scrub/orphanage.c @@ -19,6 +19,8 @@ #include "xfs_icache.h" #include "xfs_bmap.h" #include "xfs_bmap_btree.h" +#include "xfs_parent.h" +#include "xfs_attr_sf.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/repair.h" @@ -330,6 +332,8 @@ xrep_adoption_trans_alloc( if (S_ISDIR(VFS_I(sc->ip)->i_mode)) child_blkres = xfs_rename_space_res(mp, 0, false, xfs_name_dotdot.len, false); + if (xfs_has_parent(mp)) + child_blkres += XFS_ADDAFORK_SPACE_RES(mp); adopt->child_blkres = child_blkres; /* @@ -503,6 +507,19 @@ xrep_adoption_zap_dcache( dput(d_orphanage); } +/* + * If we have to add an attr fork ahead of a parent pointer update, how much + * space should we ask for? + */ +static inline int +xrep_adoption_attr_sizeof( + const struct xrep_adoption *adopt) +{ + return sizeof(struct xfs_attr_sf_hdr) + + xfs_attr_sf_entsize_byname(sizeof(struct xfs_parent_rec), + adopt->xname->len); +} + /* * Move the current file to the orphanage under the computed name. * @@ -524,6 +541,19 @@ xrep_adoption_move( if (error) return error; + /* + * If this filesystem has parent pointers, ensure that the file being + * moved to the orphanage has an attribute fork. This is required + * because the parent pointer code does not itself add attr forks. + */ + if (!xfs_inode_has_attr_fork(sc->ip) && xfs_has_parent(sc->mp)) { + int sf_size = xrep_adoption_attr_sizeof(adopt); + + error = xfs_bmap_add_attrfork(sc->tp, sc->ip, sf_size, true); + if (error) + return error; + } + /* Create the new name in the orphanage. */ error = xfs_dir_createname(sc->tp, sc->orphanage, adopt->xname, sc->ip->i_ino, adopt->orphanage_blkres); @@ -548,6 +578,14 @@ xrep_adoption_move( return error; } + /* Add a parent pointer from the file back to the lost+found. */ + if (xfs_has_parent(sc->mp)) { + error = xfs_parent_addname(sc->tp, &adopt->ppargs, + sc->orphanage, adopt->xname, sc->ip); + if (error) + return error; + } + /* * Notify dirent hooks that we moved the file to /lost+found, and * finish all the deferred work so that we know the adoption is fully diff --git a/fs/xfs/scrub/orphanage.h b/fs/xfs/scrub/orphanage.h index 319179ab788d3..beb6b686784e6 100644 --- a/fs/xfs/scrub/orphanage.h +++ b/fs/xfs/scrub/orphanage.h @@ -54,6 +54,9 @@ struct xrep_adoption { /* Name used for the adoption. */ struct xfs_name *xname; + /* Parent pointer context tracking */ + struct xfs_parent_args ppargs; + /* Block reservations for orphanage and child (if directory). */ unsigned int orphanage_blkres; unsigned int child_blkres; diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index ebb06838c31be..7b1f1abdc7a98 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -19,6 +19,8 @@ #include "xfs_rmap.h" #include "xfs_exchrange.h" #include "xfs_exchmaps.h" +#include "xfs_dir2.h" +#include "xfs_parent.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/trace.h" From 928b721a11789a9363d6d7c32a1f3166a79f3b5f Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:20 -0700 Subject: [PATCH 0484/1702] xfs: teach online scrub to find directory tree structure problems Create a new scrubber that detects corruptions within the directory tree structure itself. It can detect directories with multiple parents; loops within the directory tree; and directory loops not accessible from the root. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/libxfs/xfs_fs.h | 3 +- fs/xfs/scrub/common.h | 1 + fs/xfs/scrub/dirtree.c | 795 ++++++++++++++++++++++++++++++++++++++ fs/xfs/scrub/dirtree.h | 129 +++++++ fs/xfs/scrub/ino_bitmap.h | 37 ++ fs/xfs/scrub/scrub.c | 7 + fs/xfs/scrub/scrub.h | 1 + fs/xfs/scrub/stats.c | 1 + fs/xfs/scrub/trace.c | 4 + fs/xfs/scrub/trace.h | 190 ++++++++- fs/xfs/scrub/xfarray.h | 1 + 12 files changed, 1168 insertions(+), 2 deletions(-) create mode 100644 fs/xfs/scrub/dirtree.c create mode 100644 fs/xfs/scrub/dirtree.h create mode 100644 fs/xfs/scrub/ino_bitmap.h diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index af99a455ce4db..8ec0dd257a984 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -163,6 +163,7 @@ xfs-y += $(addprefix scrub/, \ common.o \ dabtree.o \ dir.o \ + dirtree.o \ fscounters.o \ health.o \ ialloc.o \ diff --git a/fs/xfs/libxfs/xfs_fs.h b/fs/xfs/libxfs/xfs_fs.h index dd13bfa500f2e..85f2a7e20ac00 100644 --- a/fs/xfs/libxfs/xfs_fs.h +++ b/fs/xfs/libxfs/xfs_fs.h @@ -719,9 +719,10 @@ struct xfs_scrub_metadata { #define XFS_SCRUB_TYPE_QUOTACHECK 25 /* quota counters */ #define XFS_SCRUB_TYPE_NLINKS 26 /* inode link counts */ #define XFS_SCRUB_TYPE_HEALTHY 27 /* everything checked out ok */ +#define XFS_SCRUB_TYPE_DIRTREE 28 /* directory tree structure */ /* Number of scrub subcommands. */ -#define XFS_SCRUB_TYPE_NR 28 +#define XFS_SCRUB_TYPE_NR 29 /* i: Repair this metadata. */ #define XFS_SCRUB_IFLAG_REPAIR (1u << 0) diff --git a/fs/xfs/scrub/common.h b/fs/xfs/scrub/common.h index e00466f404829..39465e39dc5fd 100644 --- a/fs/xfs/scrub/common.h +++ b/fs/xfs/scrub/common.h @@ -92,6 +92,7 @@ int xchk_setup_directory(struct xfs_scrub *sc); int xchk_setup_xattr(struct xfs_scrub *sc); int xchk_setup_symlink(struct xfs_scrub *sc); int xchk_setup_parent(struct xfs_scrub *sc); +int xchk_setup_dirtree(struct xfs_scrub *sc); #ifdef CONFIG_XFS_RT int xchk_setup_rtbitmap(struct xfs_scrub *sc); int xchk_setup_rtsummary(struct xfs_scrub *sc); diff --git a/fs/xfs/scrub/dirtree.c b/fs/xfs/scrub/dirtree.c new file mode 100644 index 0000000000000..807c8a8ca7d4a --- /dev/null +++ b/fs/xfs/scrub/dirtree.c @@ -0,0 +1,795 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2023-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_attr.h" +#include "xfs_parent.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/bitmap.h" +#include "scrub/ino_bitmap.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" +#include "scrub/listxattr.h" +#include "scrub/trace.h" +#include "scrub/dirtree.h" + +/* + * Directory Tree Structure Validation + * =================================== + * + * Validating the tree qualities of the directory tree structure can be + * difficult. If the tree is frozen, running a depth (or breadth) first search + * and marking a bitmap suffices to determine if there is a cycle. XORing the + * mark bitmap with the inode bitmap afterwards tells us if there are + * disconnected cycles. If the tree is not frozen, directory updates can move + * subtrees across the scanner wavefront, which complicates the design greatly. + * + * Directory parent pointers change that by enabling an incremental approach to + * validation of the tree structure. Instead of using one thread to scan the + * entire filesystem, we instead can have multiple threads walking individual + * subdirectories upwards to the root. In a perfect world, the IOLOCK would + * suffice to stabilize two directories in a parent -> child relationship. + * Unfortunately, the VFS does not take the IOLOCK when moving a child + * subdirectory, so we instead synchronize on ILOCK and use dirent update hooks + * to detect a race. If a race occurs in a path, we restart the scan. + * + * If the walk terminates without reaching the root, we know the path is + * disconnected and ought to be attached to the lost and found. If on the walk + * we find the same subdir that we're scanning, we know this is a cycle and + * should delete an incoming edge. If we find multiple paths to the root, we + * know to delete an incoming edge. + * + * There are two big hitches with this approach: first, all file link counts + * must be correct to prevent other writers from doing the wrong thing with the + * directory tree structure. Second, because we're walking upwards in a tree + * of arbitrary depth, we cannot hold all the ILOCKs. Instead, we will use a + * directory update hook to invalidate the scan results if one of the paths + * we've scanned has changed. + */ + +/* Clean up the dirtree checking resources. */ +STATIC void +xchk_dirtree_buf_cleanup( + void *buf) +{ + struct xchk_dirtree *dl = buf; + struct xchk_dirpath *path, *n; + + xchk_dirtree_for_each_path_safe(dl, path, n) { + list_del_init(&path->list); + xino_bitmap_destroy(&path->seen_inodes); + kfree(path); + } + + xfblob_destroy(dl->path_names); + xfarray_destroy(dl->path_steps); + mutex_destroy(&dl->lock); +} + +/* Set us up to look for directory loops. */ +int +xchk_setup_dirtree( + struct xfs_scrub *sc) +{ + struct xchk_dirtree *dl; + char *descr; + int error; + + dl = kvzalloc(sizeof(struct xchk_dirtree), XCHK_GFP_FLAGS); + if (!dl) + return -ENOMEM; + dl->sc = sc; + dl->xname.name = dl->namebuf; + INIT_LIST_HEAD(&dl->path_list); + dl->root_ino = NULLFSINO; + + mutex_init(&dl->lock); + + descr = xchk_xfile_ino_descr(sc, "dirtree path steps"); + error = xfarray_create(descr, 0, sizeof(struct xchk_dirpath_step), + &dl->path_steps); + kfree(descr); + if (error) + goto out_dl; + + descr = xchk_xfile_ino_descr(sc, "dirtree path names"); + error = xfblob_create(descr, &dl->path_names); + kfree(descr); + if (error) + goto out_steps; + + error = xchk_setup_inode_contents(sc, 0); + if (error) + goto out_names; + + sc->buf = dl; + sc->buf_cleanup = xchk_dirtree_buf_cleanup; + return 0; + +out_names: + xfblob_destroy(dl->path_names); +out_steps: + xfarray_destroy(dl->path_steps); +out_dl: + mutex_destroy(&dl->lock); + kvfree(dl); + return error; +} + +/* + * Add the parent pointer described by @dl->pptr to the given path as a new + * step. Returns -ELNRNG if the path is too deep. + */ +STATIC int +xchk_dirpath_append( + struct xchk_dirtree *dl, + struct xfs_inode *ip, + struct xchk_dirpath *path, + const struct xfs_name *name, + const struct xfs_parent_rec *pptr) +{ + struct xchk_dirpath_step step = { + .pptr_rec = *pptr, /* struct copy */ + .name_len = name->len, + }; + int error; + + /* + * If this path is more than 2 billion steps long, this directory tree + * is too far gone to fix. + */ + if (path->nr_steps >= XFS_MAXLINK) + return -ELNRNG; + + error = xfblob_storename(dl->path_names, &step.name_cookie, name); + if (error) + return error; + + error = xino_bitmap_set(&path->seen_inodes, ip->i_ino); + if (error) + return error; + + error = xfarray_append(dl->path_steps, &step); + if (error) + return error; + + path->nr_steps++; + return 0; +} + +/* + * Create an xchk_path for each parent pointer of the directory that we're + * scanning. For each path created, we will eventually try to walk towards the + * root with the goal of deleting all parents except for one that leads to the + * root. + * + * Returns -EFSCORRUPTED to signal that the inode being scanned has a corrupt + * parent pointer and hence there's no point in continuing; or -ENOSR if there + * are too many parent pointers for this directory. + */ +STATIC int +xchk_dirtree_create_path( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xfs_name xname = { + .name = name, + .len = namelen, + }; + struct xchk_dirtree *dl = priv; + struct xchk_dirpath *path; + const struct xfs_parent_rec *rec = value; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, NULL, NULL); + if (error) + return error; + + /* + * If there are more than 2 billion actual parent pointers for this + * subdirectory, this fs is too far gone to fix. + */ + if (dl->nr_paths >= XFS_MAXLINK) + return -ENOSR; + + trace_xchk_dirtree_create_path(sc, ip, dl->nr_paths, &xname, rec); + + /* + * Create a new xchk_path structure to remember this parent pointer + * and record the first name step. + */ + path = kmalloc(sizeof(struct xchk_dirpath), XCHK_GFP_FLAGS); + if (!path) + return -ENOMEM; + + INIT_LIST_HEAD(&path->list); + xino_bitmap_init(&path->seen_inodes); + path->nr_steps = 0; + path->outcome = XCHK_DIRPATH_SCANNING; + + error = xchk_dirpath_append(dl, sc->ip, path, &xname, rec); + if (error) + goto out_path; + + path->first_step = xfarray_length(dl->path_steps) - 1; + path->second_step = XFARRAY_NULLIDX; + path->path_nr = dl->nr_paths; + + list_add_tail(&path->list, &dl->path_list); + dl->nr_paths++; + return 0; +out_path: + kfree(path); + return error; +} + +/* + * Validate that the first step of this path still has a corresponding + * parent pointer in @sc->ip. We probably dropped @sc->ip's ILOCK while + * walking towards the roots, which is why this is necessary. + * + * This function has a side effect of loading the first parent pointer of this + * path into the parent pointer scratch pad. This prepares us to walk up the + * directory tree towards the root. Returns -ESTALE if the scan data is now + * out of date. + */ +STATIC int +xchk_dirpath_revalidate( + struct xchk_dirtree *dl, + struct xchk_dirpath *path) +{ + struct xfs_scrub *sc = dl->sc; + int error; + + /* + * Look up the parent pointer that corresponds to the start of this + * path. If the parent pointer has disappeared on us, dump all the + * scan results and try again. + */ + error = xfs_parent_lookup(sc->tp, sc->ip, &dl->xname, &dl->pptr_rec, + &dl->pptr_args); + if (error == -ENOATTR) { + trace_xchk_dirpath_disappeared(dl->sc, sc->ip, path->path_nr, + path->first_step, &dl->xname, &dl->pptr_rec); + dl->stale = true; + return -ESTALE; + } + + return error; +} + +/* + * Walk the parent pointers of a directory at the end of a path and record + * the parent that we find in @dl->xname/pptr_rec. + */ +STATIC int +xchk_dirpath_find_next_step( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xchk_dirtree *dl = priv; + const struct xfs_parent_rec *rec = value; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, NULL, NULL); + if (error) + return error; + + /* + * If we've already set @dl->pptr_rec, then this directory has multiple + * parents. Signal this back to the caller via -EMLINK. + */ + if (dl->parents_found > 0) + return -EMLINK; + + dl->parents_found++; + memcpy(dl->namebuf, name, namelen); + dl->xname.len = namelen; + dl->pptr_rec = *rec; /* struct copy */ + return 0; +} + +/* Set and log the outcome of a path walk. */ +static inline void +xchk_dirpath_set_outcome( + struct xchk_dirtree *dl, + struct xchk_dirpath *path, + enum xchk_dirpath_outcome outcome) +{ + trace_xchk_dirpath_set_outcome(dl->sc, path->path_nr, path->nr_steps, + outcome); + + path->outcome = outcome; +} + +/* + * Scan the directory at the end of this path for its parent directory link. + * If we find one, extend the path. Returns -ESTALE if the scan data out of + * date. Returns -EFSCORRUPTED if the parent pointer is bad; or -ELNRNG if + * the path got too deep. + */ +STATIC int +xchk_dirpath_step_up( + struct xchk_dirtree *dl, + struct xchk_dirpath *path) +{ + struct xfs_scrub *sc = dl->sc; + struct xfs_inode *dp; + xfs_ino_t parent_ino = be64_to_cpu(dl->pptr_rec.p_ino); + unsigned int lock_mode; + int error; + + /* Grab and lock the parent directory. */ + error = xchk_iget(sc, parent_ino, &dp); + if (error) + return error; + + lock_mode = xfs_ilock_attr_map_shared(dp); + mutex_lock(&dl->lock); + + if (dl->stale) { + error = -ESTALE; + goto out_scanlock; + } + + /* We've reached the root directory; the path is ok. */ + if (parent_ino == dl->root_ino) { + xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_OK); + error = 0; + goto out_scanlock; + } + + /* + * The inode being scanned is its own distant ancestor! Get rid of + * this path. + */ + if (parent_ino == sc->ip->i_ino) { + xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); + error = 0; + goto out_scanlock; + } + + /* + * We've seen this inode before during the path walk. There's a loop + * above us in the directory tree. This probably means that we cannot + * continue, but let's keep walking paths to get a full picture. + */ + if (xino_bitmap_test(&path->seen_inodes, parent_ino)) { + xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_LOOP); + error = 0; + goto out_scanlock; + } + + /* The handle encoded in the parent pointer must match. */ + if (VFS_I(dp)->i_generation != be32_to_cpu(dl->pptr_rec.p_gen)) { + trace_xchk_dirpath_badgen(dl->sc, dp, path->path_nr, + path->nr_steps, &dl->xname, &dl->pptr_rec); + error = -EFSCORRUPTED; + goto out_scanlock; + } + + /* Parent pointer must point up to a directory. */ + if (!S_ISDIR(VFS_I(dp)->i_mode)) { + trace_xchk_dirpath_nondir_parent(dl->sc, dp, path->path_nr, + path->nr_steps, &dl->xname, &dl->pptr_rec); + error = -EFSCORRUPTED; + goto out_scanlock; + } + + /* Parent cannot be an unlinked directory. */ + if (VFS_I(dp)->i_nlink == 0) { + trace_xchk_dirpath_unlinked_parent(dl->sc, dp, path->path_nr, + path->nr_steps, &dl->xname, &dl->pptr_rec); + error = -EFSCORRUPTED; + goto out_scanlock; + } + + /* + * If the extended attributes look as though they has been zapped by + * the inode record repair code, we cannot scan for parent pointers. + */ + if (xchk_pptr_looks_zapped(dp)) { + error = -EBUSY; + xchk_set_incomplete(sc); + goto out_scanlock; + } + + /* + * Walk the parent pointers of @dp to find the parent of this directory + * to find the next step in our walk. If we find that @dp has exactly + * one parent, the parent pointer information will be stored in + * @dl->pptr_rec. This prepares us for the next step of the walk. + */ + mutex_unlock(&dl->lock); + dl->parents_found = 0; + error = xchk_xattr_walk(sc, dp, xchk_dirpath_find_next_step, NULL, dl); + mutex_lock(&dl->lock); + if (error == -EFSCORRUPTED || error == -EMLINK || + (!error && dl->parents_found == 0)) { + /* + * Further up the directory tree from @sc->ip, we found a + * corrupt parent pointer, multiple parent pointers while + * finding this directory's parent, or zero parents despite + * having a nonzero link count. Keep looking for other paths. + */ + xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_CORRUPT); + error = 0; + goto out_scanlock; + } + if (error) + goto out_scanlock; + + if (dl->stale) { + error = -ESTALE; + goto out_scanlock; + } + + trace_xchk_dirpath_found_next_step(sc, dp, path->path_nr, + path->nr_steps, &dl->xname, &dl->pptr_rec); + + /* Append to the path steps */ + error = xchk_dirpath_append(dl, dp, path, &dl->xname, &dl->pptr_rec); + if (error) + goto out_scanlock; + + if (path->second_step == XFARRAY_NULLIDX) + path->second_step = xfarray_length(dl->path_steps) - 1; + +out_scanlock: + mutex_unlock(&dl->lock); + xfs_iunlock(dp, lock_mode); + xchk_irele(sc, dp); + return error; +} + +/* + * Walk the directory tree upwards towards what is hopefully the root + * directory, recording path steps as we go. The current path components are + * stored in dl->pptr_rec and dl->xname. + * + * Returns -ESTALE if the scan data are out of date. Returns -EFSCORRUPTED + * only if the direct parent pointer of @sc->ip associated with this path is + * corrupt. + */ +STATIC int +xchk_dirpath_walk_upwards( + struct xchk_dirtree *dl, + struct xchk_dirpath *path) +{ + struct xfs_scrub *sc = dl->sc; + int error; + + ASSERT(sc->ilock_flags & XFS_ILOCK_EXCL); + + /* Reload the start of this path and make sure it's still there. */ + error = xchk_dirpath_revalidate(dl, path); + if (error) + return error; + + trace_xchk_dirpath_walk_upwards(sc, sc->ip, path->path_nr, &dl->xname, + &dl->pptr_rec); + + /* + * The inode being scanned is its own direct ancestor! + * Get rid of this path. + */ + if (be64_to_cpu(dl->pptr_rec.p_ino) == sc->ip->i_ino) { + xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); + return 0; + } + + /* + * Drop ILOCK_EXCL on the inode being scanned. We still hold + * IOLOCK_EXCL on it, so it cannot move around or be renamed. + * + * Beyond this point we're walking up the directory tree, which means + * that we can acquire and drop the ILOCK on an alias of sc->ip. The + * ILOCK state is no longer tracked in the scrub context. Hence we + * must drop @sc->ip's ILOCK during the walk. + */ + mutex_unlock(&dl->lock); + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + /* + * Take the first step in the walk towards the root by checking the + * start of this path, which is a direct parent pointer of @sc->ip. + * If we see any kind of error here (including corruptions), the parent + * pointer of @sc->ip is corrupt. Stop the whole scan. + */ + error = xchk_dirpath_step_up(dl, path); + if (error) { + xchk_ilock(sc, XFS_ILOCK_EXCL); + mutex_lock(&dl->lock); + return error; + } + + /* + * Take steps upward from the second step in this path towards the + * root. If we hit corruption errors here, there's a problem + * *somewhere* in the path, but we don't need to stop scanning. + */ + while (!error && path->outcome == XCHK_DIRPATH_SCANNING) + error = xchk_dirpath_step_up(dl, path); + + /* Retake the locks we had, mark paths, etc. */ + xchk_ilock(sc, XFS_ILOCK_EXCL); + mutex_lock(&dl->lock); + if (error == -EFSCORRUPTED) { + xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_CORRUPT); + error = 0; + } + if (!error && dl->stale) + return -ESTALE; + return error; +} + +/* Delete all the collected path information. */ +STATIC void +xchk_dirtree_reset( + void *buf) +{ + struct xchk_dirtree *dl = buf; + struct xchk_dirpath *path, *n; + + ASSERT(dl->sc->ilock_flags & XFS_ILOCK_EXCL); + + xchk_dirtree_for_each_path_safe(dl, path, n) { + list_del_init(&path->list); + xino_bitmap_destroy(&path->seen_inodes); + kfree(path); + } + dl->nr_paths = 0; + + xfarray_truncate(dl->path_steps); + xfblob_truncate(dl->path_names); + + dl->stale = false; +} + +/* + * Load the name/pptr from the first step in this path into @dl->pptr_rec and + * @dl->xname. + */ +STATIC int +xchk_dirtree_load_path( + struct xchk_dirtree *dl, + struct xchk_dirpath *path) +{ + struct xchk_dirpath_step step; + int error; + + error = xfarray_load(dl->path_steps, path->first_step, &step); + if (error) + return error; + + error = xfblob_loadname(dl->path_names, step.name_cookie, &dl->xname, + step.name_len); + if (error) + return error; + + dl->pptr_rec = step.pptr_rec; /* struct copy */ + return 0; +} + +/* + * For each parent pointer of this subdir, trace a path upwards towards the + * root directory and record what we find. Returns 0 for success; + * -EFSCORRUPTED if walking the parent pointers of @sc->ip failed, -ELNRNG if a + * path was too deep; -ENOSR if there were too many parent pointers; or + * a negative errno. + */ +STATIC int +xchk_dirtree_find_paths_to_root( + struct xchk_dirtree *dl) +{ + struct xfs_scrub *sc = dl->sc; + struct xchk_dirpath *path; + int error = 0; + + do { + if (xchk_should_terminate(sc, &error)) + return error; + + xchk_dirtree_reset(dl); + + /* + * If the extended attributes look as though they has been + * zapped by the inode record repair code, we cannot scan for + * parent pointers. + */ + if (xchk_pptr_looks_zapped(sc->ip)) { + xchk_set_incomplete(sc); + return -EBUSY; + } + + /* + * Create path walk contexts for each parent of the directory + * that is being scanned. Directories are supposed to have + * only one parent, but this is how we detect multiple parents. + */ + error = xchk_xattr_walk(sc, sc->ip, xchk_dirtree_create_path, + NULL, dl); + if (error) + return error; + + xchk_dirtree_for_each_path(dl, path) { + /* Load path components into dl->pptr/xname */ + error = xchk_dirtree_load_path(dl, path); + if (error) + return error; + + /* + * Try to walk up each path to the root. This enables + * us to find directory loops in ancestors, and the + * like. + */ + error = xchk_dirpath_walk_upwards(dl, path); + if (error == -EFSCORRUPTED) { + /* + * A parent pointer of @sc->ip is bad, don't + * bother continuing. + */ + break; + } + if (error == -ESTALE) { + /* This had better be an invalidation. */ + ASSERT(dl->stale); + break; + } + if (error) + return error; + } + } while (dl->stale); + + return error; +} + +/* + * Figure out what to do with the paths we tried to find. Do not call this + * if the scan results are stale. + */ +STATIC void +xchk_dirtree_evaluate( + struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc) +{ + struct xchk_dirpath *path; + + ASSERT(!dl->stale); + + /* Scan the paths we have to decide what to do. */ + memset(oc, 0, sizeof(struct xchk_dirtree_outcomes)); + xchk_dirtree_for_each_path(dl, path) { + trace_xchk_dirpath_evaluate_path(dl->sc, path->path_nr, + path->nr_steps, path->outcome); + + switch (path->outcome) { + case XCHK_DIRPATH_SCANNING: + /* shouldn't get here */ + ASSERT(0); + break; + case XCHK_DIRPATH_DELETE: + /* This one is already going away. */ + oc->bad++; + break; + case XCHK_DIRPATH_CORRUPT: + case XCHK_DIRPATH_LOOP: + /* Couldn't find the end of this path. */ + oc->suspect++; + break; + case XCHK_DIRPATH_STALE: + /* shouldn't get here either */ + ASSERT(0); + break; + case XCHK_DIRPATH_OK: + /* This path got all the way to the root. */ + oc->good++; + break; + } + } + + trace_xchk_dirtree_evaluate(dl, oc); +} + +/* Look for directory loops. */ +int +xchk_dirtree( + struct xfs_scrub *sc) +{ + struct xchk_dirtree_outcomes oc; + struct xchk_dirtree *dl = sc->buf; + int error; + + /* + * Nondirectories do not point downwards to other files, so they cannot + * cause a cycle in the directory tree. + */ + if (!S_ISDIR(VFS_I(sc->ip)->i_mode)) + return -ENOENT; + + ASSERT(xfs_has_parent(sc->mp)); + + /* Find the root of the directory tree. */ + dl->root_ino = sc->mp->m_rootip->i_ino; + + trace_xchk_dirtree_start(sc->ip, sc->sm, 0); + + mutex_lock(&dl->lock); + + /* Trace each parent pointer's path to the root. */ + error = xchk_dirtree_find_paths_to_root(dl); + if (error == -EFSCORRUPTED || error == -ELNRNG || error == -ENOSR) { + /* + * Don't bother walking the paths if the xattr structure or the + * parent pointers are corrupt; this scan cannot be completed + * without full information. + */ + xchk_ino_xref_set_corrupt(sc, sc->ip->i_ino); + error = 0; + goto out_scanlock; + } + if (error == -EBUSY) { + /* + * We couldn't scan some directory's parent pointers because + * the attr fork looked like it had been zapped. The + * scan was marked incomplete, so no further error code + * is necessary. + */ + error = 0; + goto out_scanlock; + } + if (error) + goto out_scanlock; + + /* Assess what we found in our path evaluation. */ + xchk_dirtree_evaluate(dl, &oc); + if (xchk_dirtree_parentless(dl)) { + if (oc.good || oc.bad || oc.suspect) + xchk_ino_set_corrupt(sc, sc->ip->i_ino); + } else { + if (oc.bad || oc.good + oc.suspect != 1) + xchk_ino_set_corrupt(sc, sc->ip->i_ino); + if (oc.suspect) + xchk_ino_xref_set_corrupt(sc, sc->ip->i_ino); + } + +out_scanlock: + mutex_unlock(&dl->lock); + trace_xchk_dirtree_done(sc->ip, sc->sm, error); + return error; +} diff --git a/fs/xfs/scrub/dirtree.h b/fs/xfs/scrub/dirtree.h new file mode 100644 index 0000000000000..50fefd64ae508 --- /dev/null +++ b/fs/xfs/scrub/dirtree.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2023-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_DIRTREE_H__ +#define __XFS_SCRUB_DIRTREE_H__ + +/* + * Each of these represents one parent pointer path step in a chain going + * up towards the directory tree root. These are stored inside an xfarray. + */ +struct xchk_dirpath_step { + /* Directory entry name associated with this parent link. */ + xfblob_cookie name_cookie; + unsigned int name_len; + + /* Handle of the parent directory. */ + struct xfs_parent_rec pptr_rec; +}; + +enum xchk_dirpath_outcome { + XCHK_DIRPATH_SCANNING = 0, /* still being put together */ + XCHK_DIRPATH_DELETE, /* delete this path */ + XCHK_DIRPATH_CORRUPT, /* corruption detected in path */ + XCHK_DIRPATH_LOOP, /* cycle detected further up */ + XCHK_DIRPATH_STALE, /* path is stale */ + XCHK_DIRPATH_OK, /* path reaches the root */ +}; + +/* + * Each of these represents one parent pointer path out of the directory being + * scanned. These exist in-core, and hopefully there aren't more than a + * handful of them. + */ +struct xchk_dirpath { + struct list_head list; + + /* Index of the first step in this path. */ + xfarray_idx_t first_step; + + /* Index of the second step in this path. */ + xfarray_idx_t second_step; + + /* Inodes seen while walking this path. */ + struct xino_bitmap seen_inodes; + + /* Number of steps in this path. */ + unsigned int nr_steps; + + /* Which path is this? */ + unsigned int path_nr; + + /* What did we conclude from following this path? */ + enum xchk_dirpath_outcome outcome; +}; + +struct xchk_dirtree_outcomes { + /* Number of XCHK_DIRPATH_DELETE */ + unsigned int bad; + + /* Number of XCHK_DIRPATH_CORRUPT or XCHK_DIRPATH_LOOP */ + unsigned int suspect; + + /* Number of XCHK_DIRPATH_OK */ + unsigned int good; +}; + +struct xchk_dirtree { + struct xfs_scrub *sc; + + /* Root inode that we're looking for. */ + xfs_ino_t root_ino; + + /* Scratch buffer for scanning pptr xattrs */ + struct xfs_parent_rec pptr_rec; + struct xfs_da_args pptr_args; + + /* Name buffer */ + struct xfs_name xname; + char namebuf[MAXNAMELEN]; + + /* lock for everything below here */ + struct mutex lock; + + /* + * All path steps observed during this scan. Each of the path + * steps for a particular pathwalk are recorded in sequential + * order in the xfarray. A pathwalk ends either with a step + * pointing to the root directory (success) or pointing to NULLFSINO + * (loop detected, empty dir detected, etc). + */ + struct xfarray *path_steps; + + /* All names observed during this scan. */ + struct xfblob *path_names; + + /* All paths being tracked by this scanner. */ + struct list_head path_list; + + /* Number of paths in path_list. */ + unsigned int nr_paths; + + /* Number of parents found by a pptr scan. */ + unsigned int parents_found; + + /* Have the path data been invalidated by a concurrent update? */ + bool stale:1; +}; + +#define xchk_dirtree_for_each_path_safe(dl, path, n) \ + list_for_each_entry_safe((path), (n), &(dl)->path_list, list) + +#define xchk_dirtree_for_each_path(dl, path) \ + list_for_each_entry((path), &(dl)->path_list, list) + +static inline bool +xchk_dirtree_parentless(const struct xchk_dirtree *dl) +{ + struct xfs_scrub *sc = dl->sc; + + if (sc->ip == sc->mp->m_rootip) + return true; + if (VFS_I(sc->ip)->i_nlink == 0) + return true; + return false; +} + +#endif /* __XFS_SCRUB_DIRTREE_H__ */ diff --git a/fs/xfs/scrub/ino_bitmap.h b/fs/xfs/scrub/ino_bitmap.h new file mode 100644 index 0000000000000..1300833679abf --- /dev/null +++ b/fs/xfs/scrub/ino_bitmap.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2023-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_INO_BITMAP_H__ +#define __XFS_SCRUB_INO_BITMAP_H__ + +/* Bitmaps, but for type-checked for xfs_ino_t */ + +struct xino_bitmap { + struct xbitmap64 inobitmap; +}; + +static inline void xino_bitmap_init(struct xino_bitmap *bitmap) +{ + xbitmap64_init(&bitmap->inobitmap); +} + +static inline void xino_bitmap_destroy(struct xino_bitmap *bitmap) +{ + xbitmap64_destroy(&bitmap->inobitmap); +} + +static inline int xino_bitmap_set(struct xino_bitmap *bitmap, xfs_ino_t ino) +{ + return xbitmap64_set(&bitmap->inobitmap, ino, 1); +} + +static inline int xino_bitmap_test(struct xino_bitmap *bitmap, xfs_ino_t ino) +{ + uint64_t len = 1; + + return xbitmap64_test(&bitmap->inobitmap, ino, &len); +} + +#endif /* __XFS_SCRUB_INO_BITMAP_H__ */ diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 7b1f1abdc7a98..8f1431db77395 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -436,6 +436,13 @@ static const struct xchk_meta_ops meta_scrub_ops[] = { .scrub = xchk_health_record, .repair = xrep_notsupported, }, + [XFS_SCRUB_TYPE_DIRTREE] = { /* directory tree structure */ + .type = ST_INODE, + .setup = xchk_setup_dirtree, + .scrub = xchk_dirtree, + .has = xfs_has_parent, + .repair = xrep_notsupported, + }, }; static int diff --git a/fs/xfs/scrub/scrub.h b/fs/xfs/scrub/scrub.h index 54a4242bc79cf..3910270471462 100644 --- a/fs/xfs/scrub/scrub.h +++ b/fs/xfs/scrub/scrub.h @@ -185,6 +185,7 @@ int xchk_directory(struct xfs_scrub *sc); int xchk_xattr(struct xfs_scrub *sc); int xchk_symlink(struct xfs_scrub *sc); int xchk_parent(struct xfs_scrub *sc); +int xchk_dirtree(struct xfs_scrub *sc); #ifdef CONFIG_XFS_RT int xchk_rtbitmap(struct xfs_scrub *sc); int xchk_rtsummary(struct xfs_scrub *sc); diff --git a/fs/xfs/scrub/stats.c b/fs/xfs/scrub/stats.c index 42cafbed94ac6..7996c23354763 100644 --- a/fs/xfs/scrub/stats.c +++ b/fs/xfs/scrub/stats.c @@ -79,6 +79,7 @@ static const char *name_map[XFS_SCRUB_TYPE_NR] = { [XFS_SCRUB_TYPE_FSCOUNTERS] = "fscounters", [XFS_SCRUB_TYPE_QUOTACHECK] = "quotacheck", [XFS_SCRUB_TYPE_NLINKS] = "nlinks", + [XFS_SCRUB_TYPE_DIRTREE] = "dirtree", }; /* Format the scrub stats into a text buffer, similar to pcp style. */ diff --git a/fs/xfs/scrub/trace.c b/fs/xfs/scrub/trace.c index 4a8cc2c98d997..4470ad0533b81 100644 --- a/fs/xfs/scrub/trace.c +++ b/fs/xfs/scrub/trace.c @@ -28,6 +28,10 @@ #include "scrub/orphanage.h" #include "scrub/nlinks.h" #include "scrub/fscounters.h" +#include "scrub/bitmap.h" +#include "scrub/ino_bitmap.h" +#include "scrub/xfblob.h" +#include "scrub/dirtree.h" /* Figure out which block the btree cursor was pointing to. */ static inline xfs_fsblock_t diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index ecfaa4b88910f..c474bcd7d54b7 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -27,6 +27,9 @@ struct xchk_nlink; struct xchk_fscounters; struct xfs_rmap_update_params; struct xfs_parent_rec; +enum xchk_dirpath_outcome; +struct xchk_dirtree; +struct xchk_dirtree_outcomes; /* * ftrace's __print_symbolic requires that all enum values be wrapped in the @@ -65,6 +68,7 @@ TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_FSCOUNTERS); TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_QUOTACHECK); TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_NLINKS); TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_HEALTHY); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_DIRTREE); #define XFS_SCRUB_TYPE_STRINGS \ { XFS_SCRUB_TYPE_PROBE, "probe" }, \ @@ -94,7 +98,8 @@ TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_HEALTHY); { XFS_SCRUB_TYPE_FSCOUNTERS, "fscounters" }, \ { XFS_SCRUB_TYPE_QUOTACHECK, "quotacheck" }, \ { XFS_SCRUB_TYPE_NLINKS, "nlinks" }, \ - { XFS_SCRUB_TYPE_HEALTHY, "healthy" } + { XFS_SCRUB_TYPE_HEALTHY, "healthy" }, \ + { XFS_SCRUB_TYPE_DIRTREE, "dirtree" } #define XFS_SCRUB_FLAG_STRINGS \ { XFS_SCRUB_IFLAG_REPAIR, "repair" }, \ @@ -171,6 +176,8 @@ DEFINE_EVENT(xchk_class, name, \ DEFINE_SCRUB_EVENT(xchk_start); DEFINE_SCRUB_EVENT(xchk_done); DEFINE_SCRUB_EVENT(xchk_deadlock_retry); +DEFINE_SCRUB_EVENT(xchk_dirtree_start); +DEFINE_SCRUB_EVENT(xchk_dirtree_done); DEFINE_SCRUB_EVENT(xrep_attempt); DEFINE_SCRUB_EVENT(xrep_done); @@ -1576,6 +1583,187 @@ DEFINE_XCHK_PPTR_EVENT(xchk_parent_defer); DEFINE_XCHK_PPTR_EVENT(xchk_parent_slowpath); DEFINE_XCHK_PPTR_EVENT(xchk_parent_ultraslowpath); +DECLARE_EVENT_CLASS(xchk_dirtree_class, + TP_PROTO(struct xfs_scrub *sc, struct xfs_inode *ip, + unsigned int path_nr, const struct xfs_name *name, + const struct xfs_parent_rec *pptr), + TP_ARGS(sc, ip, path_nr, name, pptr), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, path_nr) + __field(xfs_ino_t, child_ino) + __field(unsigned int, child_gen) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->path_nr = path_nr; + __entry->child_ino = ip->i_ino; + __entry->child_gen = VFS_I(ip)->i_generation; + __entry->parent_ino = be64_to_cpu(pptr->p_ino); + __entry->parent_gen = be32_to_cpu(pptr->p_gen); + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d path %u child_ino 0x%llx child_gen 0x%x parent_ino 0x%llx parent_gen 0x%x name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->path_nr, + __entry->child_ino, + __entry->child_gen, + __entry->parent_ino, + __entry->parent_gen, + __entry->namelen, + __get_str(name)) +); +#define DEFINE_XCHK_DIRTREE_EVENT(name) \ +DEFINE_EVENT(xchk_dirtree_class, name, \ + TP_PROTO(struct xfs_scrub *sc, struct xfs_inode *ip, \ + unsigned int path_nr, const struct xfs_name *name, \ + const struct xfs_parent_rec *pptr), \ + TP_ARGS(sc, ip, path_nr, name, pptr)) +DEFINE_XCHK_DIRTREE_EVENT(xchk_dirtree_create_path); +DEFINE_XCHK_DIRTREE_EVENT(xchk_dirpath_walk_upwards); + +DECLARE_EVENT_CLASS(xchk_dirpath_class, + TP_PROTO(struct xfs_scrub *sc, struct xfs_inode *ip, + unsigned int path_nr, unsigned int step_nr, + const struct xfs_name *name, + const struct xfs_parent_rec *pptr), + TP_ARGS(sc, ip, path_nr, step_nr, name, pptr), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, path_nr) + __field(unsigned int, step_nr) + __field(xfs_ino_t, child_ino) + __field(unsigned int, child_gen) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->path_nr = path_nr; + __entry->step_nr = step_nr; + __entry->child_ino = ip->i_ino; + __entry->child_gen = VFS_I(ip)->i_generation; + __entry->parent_ino = be64_to_cpu(pptr->p_ino); + __entry->parent_gen = be32_to_cpu(pptr->p_gen); + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d path %u step %u child_ino 0x%llx child_gen 0x%x parent_ino 0x%llx parent_gen 0x%x name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->path_nr, + __entry->step_nr, + __entry->child_ino, + __entry->child_gen, + __entry->parent_ino, + __entry->parent_gen, + __entry->namelen, + __get_str(name)) +); +#define DEFINE_XCHK_DIRPATH_EVENT(name) \ +DEFINE_EVENT(xchk_dirpath_class, name, \ + TP_PROTO(struct xfs_scrub *sc, struct xfs_inode *ip, \ + unsigned int path_nr, unsigned int step_nr, \ + const struct xfs_name *name, \ + const struct xfs_parent_rec *pptr), \ + TP_ARGS(sc, ip, path_nr, step_nr, name, pptr)) +DEFINE_XCHK_DIRPATH_EVENT(xchk_dirpath_disappeared); +DEFINE_XCHK_DIRPATH_EVENT(xchk_dirpath_badgen); +DEFINE_XCHK_DIRPATH_EVENT(xchk_dirpath_nondir_parent); +DEFINE_XCHK_DIRPATH_EVENT(xchk_dirpath_unlinked_parent); +DEFINE_XCHK_DIRPATH_EVENT(xchk_dirpath_found_next_step); + +TRACE_DEFINE_ENUM(XCHK_DIRPATH_SCANNING); +TRACE_DEFINE_ENUM(XCHK_DIRPATH_DELETE); +TRACE_DEFINE_ENUM(XCHK_DIRPATH_CORRUPT); +TRACE_DEFINE_ENUM(XCHK_DIRPATH_LOOP); +TRACE_DEFINE_ENUM(XCHK_DIRPATH_STALE); +TRACE_DEFINE_ENUM(XCHK_DIRPATH_OK); + +#define XCHK_DIRPATH_OUTCOME_STRINGS \ + { XCHK_DIRPATH_SCANNING, "scanning" }, \ + { XCHK_DIRPATH_DELETE, "delete" }, \ + { XCHK_DIRPATH_CORRUPT, "corrupt" }, \ + { XCHK_DIRPATH_LOOP, "loop" }, \ + { XCHK_DIRPATH_STALE, "stale" }, \ + { XCHK_DIRPATH_OK, "ok" } + +DECLARE_EVENT_CLASS(xchk_dirpath_outcome_class, + TP_PROTO(struct xfs_scrub *sc, unsigned long long path_nr, + unsigned int nr_steps, \ + unsigned int outcome), + TP_ARGS(sc, path_nr, nr_steps, outcome), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long long, path_nr) + __field(unsigned int, nr_steps) + __field(unsigned int, outcome) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->path_nr = path_nr; + __entry->nr_steps = nr_steps; + __entry->outcome = outcome; + ), + TP_printk("dev %d:%d path %llu steps %u outcome %s", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->path_nr, + __entry->nr_steps, + __print_symbolic(__entry->outcome, XCHK_DIRPATH_OUTCOME_STRINGS)) +); +#define DEFINE_XCHK_DIRPATH_OUTCOME_EVENT(name) \ +DEFINE_EVENT(xchk_dirpath_outcome_class, name, \ + TP_PROTO(struct xfs_scrub *sc, unsigned long long path_nr, \ + unsigned int nr_steps, \ + unsigned int outcome), \ + TP_ARGS(sc, path_nr, nr_steps, outcome)) +DEFINE_XCHK_DIRPATH_OUTCOME_EVENT(xchk_dirpath_set_outcome); +DEFINE_XCHK_DIRPATH_OUTCOME_EVENT(xchk_dirpath_evaluate_path); + +DECLARE_EVENT_CLASS(xchk_dirtree_evaluate_class, + TP_PROTO(const struct xchk_dirtree *dl, + const struct xchk_dirtree_outcomes *oc), + TP_ARGS(dl, oc), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_ino_t, rootino) + __field(unsigned int, nr_paths) + __field(unsigned int, bad) + __field(unsigned int, suspect) + __field(unsigned int, good) + ), + TP_fast_assign( + __entry->dev = dl->sc->mp->m_super->s_dev; + __entry->ino = dl->sc->ip->i_ino; + __entry->rootino = dl->root_ino; + __entry->nr_paths = dl->nr_paths; + __entry->bad = oc->bad; + __entry->suspect = oc->suspect; + __entry->good = oc->good; + ), + TP_printk("dev %d:%d ino 0x%llx rootino 0x%llx nr_paths %u bad %u suspect %u good %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->rootino, + __entry->nr_paths, + __entry->bad, + __entry->suspect, + __entry->good) +); +#define DEFINE_XCHK_DIRTREE_EVALUATE_EVENT(name) \ +DEFINE_EVENT(xchk_dirtree_evaluate_class, name, \ + TP_PROTO(const struct xchk_dirtree *dl, \ + const struct xchk_dirtree_outcomes *oc), \ + TP_ARGS(dl, oc)) +DEFINE_XCHK_DIRTREE_EVALUATE_EVENT(xchk_dirtree_evaluate); + /* repair tracepoints */ #if IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) diff --git a/fs/xfs/scrub/xfarray.h b/fs/xfs/scrub/xfarray.h index 3b10a58e9f146..8f54c8fc888fa 100644 --- a/fs/xfs/scrub/xfarray.h +++ b/fs/xfs/scrub/xfarray.h @@ -8,6 +8,7 @@ /* xfile array index type, along with cursor initialization */ typedef uint64_t xfarray_idx_t; +#define XFARRAY_NULLIDX ((__force xfarray_idx_t)-1ULL) #define XFARRAY_CURSOR_INIT ((__force xfarray_idx_t)0) /* Iterate each index of an xfile array. */ From 3f50ddbf4b470f8566f99a83597da50180d937c0 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:18 -0700 Subject: [PATCH 0485/1702] xfs: repair link count of nondirectories after rebuilding parent pointers Since the parent pointer scrubber does not exhaustively search the filesystem for missing parent pointers, it doesn't have a good way to determine that there are pointers missing from an otherwise uncorrupt xattr structure. Instead, for nondirectories it employs a heuristic of comparing the file link count to the number of parent pointers found. However, we don't want this heuristic flagging a false corruption after a repair has actually scanned the entire filesystem to rebuild the parent pointers. Therefore, reset the file link count in this one case because we actually know the correct link count. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/parent_repair.c | 107 +++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/fs/xfs/scrub/parent_repair.c b/fs/xfs/scrub/parent_repair.c index 28e9746c06631..ee88ce5a12b83 100644 --- a/fs/xfs/scrub/parent_repair.c +++ b/fs/xfs/scrub/parent_repair.c @@ -27,6 +27,7 @@ #include "xfs_parent.h" #include "xfs_attr.h" #include "xfs_bmap.h" +#include "xfs_ag.h" #include "scrub/xfs_scrub.h" #include "scrub/scrub.h" #include "scrub/common.h" @@ -156,6 +157,9 @@ struct xrep_parent { /* Have we seen any live updates of parent pointers recently? */ bool saw_pptr_updates; + + /* Number of parents we found after all other repairs */ + unsigned long long parents; }; struct xrep_parent_xattr { @@ -1370,6 +1374,102 @@ xrep_parent_rebuild_tree( return 0; } +/* Count the number of parent pointers. */ +STATIC int +xrep_parent_count_pptr( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xrep_parent *rp = priv; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, NULL, NULL); + if (error) + return error; + + rp->parents++; + return 0; +} + +/* + * After all parent pointer rebuilding and adoption activity completes, reset + * the link count of this nondirectory, having scanned the fs to rebuild all + * parent pointers. + */ +STATIC int +xrep_parent_set_nondir_nlink( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + struct xfs_inode *ip = sc->ip; + struct xfs_perag *pag; + bool joined = false; + int error; + + /* Count parent pointers so we can reset the file link count. */ + rp->parents = 0; + error = xchk_xattr_walk(sc, ip, xrep_parent_count_pptr, NULL, rp); + if (error) + return error; + + if (rp->parents > 0 && xfs_inode_on_unlinked_list(ip)) { + xfs_trans_ijoin(sc->tp, sc->ip, 0); + joined = true; + + /* + * The file is on the unlinked list but we found parents. + * Remove the file from the unlinked list. + */ + pag = xfs_perag_get(sc->mp, XFS_INO_TO_AGNO(sc->mp, ip->i_ino)); + if (!pag) { + ASSERT(0); + return -EFSCORRUPTED; + } + + error = xfs_iunlink_remove(sc->tp, pag, ip); + xfs_perag_put(pag); + if (error) + return error; + } else if (rp->parents == 0 && !xfs_inode_on_unlinked_list(ip)) { + xfs_trans_ijoin(sc->tp, sc->ip, 0); + joined = true; + + /* + * The file is not on the unlinked list but we found no + * parents. Add the file to the unlinked list. + */ + error = xfs_iunlink(sc->tp, ip); + if (error) + return error; + } + + /* Set the correct link count. */ + if (VFS_I(ip)->i_nlink != rp->parents) { + if (!joined) { + xfs_trans_ijoin(sc->tp, sc->ip, 0); + joined = true; + } + + set_nlink(VFS_I(ip), min_t(unsigned long long, rp->parents, + XFS_NLINK_PINNED)); + } + + /* Log the inode to keep it moving forward if we dirtied anything. */ + if (joined) + xfs_trans_log_inode(sc->tp, ip, XFS_ILOG_CORE); + return 0; +} + /* Set up the filesystem scan so we can look for parents. */ STATIC int xrep_parent_setup_scan( @@ -1494,6 +1594,13 @@ xrep_parent( error = xrep_parent_rebuild_tree(rp); if (error) goto out_teardown; + if (xfs_has_parent(sc->mp) && !S_ISDIR(VFS_I(sc->ip)->i_mode)) { + error = xrep_parent_set_nondir_nlink(rp); + if (error) + goto out_teardown; + } + + error = xrep_defer_finish(sc); out_teardown: xrep_parent_teardown(rp); From d54c5ac80f8f180eb064ca255d61a879aef6f06d Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:21 -0700 Subject: [PATCH 0486/1702] xfs: invalidate dirloop scrub path data when concurrent updates happen Add a dirent update hook so that we can detect directory tree updates that affect any of the paths found by this scrubber and force it to rescan. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/dirtree.c | 160 ++++++++++++++++++++++++++++++++++++++++- fs/xfs/scrub/dirtree.h | 20 ++++++ fs/xfs/scrub/trace.h | 65 +++++++++++++++++ 3 files changed, 244 insertions(+), 1 deletion(-) diff --git a/fs/xfs/scrub/dirtree.c b/fs/xfs/scrub/dirtree.c index 807c8a8ca7d4a..ecc56eb5ed270 100644 --- a/fs/xfs/scrub/dirtree.c +++ b/fs/xfs/scrub/dirtree.c @@ -70,6 +70,9 @@ xchk_dirtree_buf_cleanup( struct xchk_dirtree *dl = buf; struct xchk_dirpath *path, *n; + if (dl->scan_ino != NULLFSINO) + xfs_dir_hook_del(dl->sc->mp, &dl->dhook); + xchk_dirtree_for_each_path_safe(dl, path, n) { list_del_init(&path->list); xino_bitmap_destroy(&path->seen_inodes); @@ -90,13 +93,17 @@ xchk_setup_dirtree( char *descr; int error; + xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); + dl = kvzalloc(sizeof(struct xchk_dirtree), XCHK_GFP_FLAGS); if (!dl) return -ENOMEM; dl->sc = sc; dl->xname.name = dl->namebuf; + dl->hook_xname.name = dl->hook_namebuf; INIT_LIST_HEAD(&dl->path_list); dl->root_ino = NULLFSINO; + dl->scan_ino = NULLFSINO; mutex_init(&dl->lock); @@ -558,6 +565,133 @@ xchk_dirpath_walk_upwards( return error; } +/* + * Decide if this path step has been touched by this live update. Returns + * 1 for yes, 0 for no, or a negative errno. + */ +STATIC int +xchk_dirpath_step_is_stale( + struct xchk_dirtree *dl, + struct xchk_dirpath *path, + unsigned int step_nr, + xfarray_idx_t step_idx, + struct xfs_dir_update_params *p, + xfs_ino_t *cursor) +{ + struct xchk_dirpath_step step; + xfs_ino_t child_ino = *cursor; + int error; + + error = xfarray_load(dl->path_steps, step_idx, &step); + if (error) + return error; + *cursor = be64_to_cpu(step.pptr_rec.p_ino); + + /* + * If the parent and child being updated are not the ones mentioned in + * this path step, the scan data is still ok. + */ + if (p->ip->i_ino != child_ino || p->dp->i_ino != *cursor) + return 0; + + /* + * If the dirent name lengths or byte sequences are different, the scan + * data is still ok. + */ + if (p->name->len != step.name_len) + return 0; + + error = xfblob_loadname(dl->path_names, step.name_cookie, + &dl->hook_xname, step.name_len); + if (error) + return error; + + if (memcmp(dl->hook_xname.name, p->name->name, p->name->len) != 0) + return 0; + + /* Exact match, scan data is out of date. */ + trace_xchk_dirpath_changed(dl->sc, path->path_nr, step_nr, p->dp, + p->ip, p->name); + return 1; +} + +/* + * Decide if this path has been touched by this live update. Returns 1 for + * yes, 0 for no, or a negative errno. + */ +STATIC int +xchk_dirpath_is_stale( + struct xchk_dirtree *dl, + struct xchk_dirpath *path, + struct xfs_dir_update_params *p) +{ + xfs_ino_t cursor = dl->scan_ino; + xfarray_idx_t idx = path->first_step; + unsigned int i; + int ret; + + /* + * The child being updated has not been seen by this path at all; this + * path cannot be stale. + */ + if (!xino_bitmap_test(&path->seen_inodes, p->ip->i_ino)) + return 0; + + ret = xchk_dirpath_step_is_stale(dl, path, 0, idx, p, &cursor); + if (ret != 0) + return ret; + + for (i = 1, idx = path->second_step; i < path->nr_steps; i++, idx++) { + ret = xchk_dirpath_step_is_stale(dl, path, i, idx, p, &cursor); + if (ret != 0) + return ret; + } + + return 0; +} + +/* + * Decide if a directory update from the regular filesystem touches any of the + * paths we've scanned, and invalidate the scan data if true. + */ +STATIC int +xchk_dirtree_live_update( + struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct xfs_dir_update_params *p = data; + struct xchk_dirtree *dl; + struct xchk_dirpath *path; + int ret; + + dl = container_of(nb, struct xchk_dirtree, dhook.dirent_hook.nb); + + trace_xchk_dirtree_live_update(dl->sc, p->dp, action, p->ip, p->delta, + p->name); + + mutex_lock(&dl->lock); + + if (dl->stale || dl->aborted) + goto out_unlock; + + xchk_dirtree_for_each_path(dl, path) { + ret = xchk_dirpath_is_stale(dl, path, p); + if (ret < 0) { + dl->aborted = true; + break; + } + if (ret == 1) { + dl->stale = true; + break; + } + } + +out_unlock: + mutex_unlock(&dl->lock); + return NOTIFY_DONE; +} + /* Delete all the collected path information. */ STATIC void xchk_dirtree_reset( @@ -673,6 +807,8 @@ xchk_dirtree_find_paths_to_root( } if (error) return error; + if (dl->aborted) + return 0; } } while (dl->stale); @@ -744,11 +880,28 @@ xchk_dirtree( ASSERT(xfs_has_parent(sc->mp)); - /* Find the root of the directory tree. */ + /* + * Find the root of the directory tree. Remember which directory to + * scan, because the hook doesn't detach until after sc->ip gets + * released during teardown. + */ dl->root_ino = sc->mp->m_rootip->i_ino; + dl->scan_ino = sc->ip->i_ino; trace_xchk_dirtree_start(sc->ip, sc->sm, 0); + /* + * Hook into the directory entry code so that we can capture updates to + * paths that we have already scanned. The scanner thread takes each + * directory's ILOCK, which means that any in-progress directory update + * will finish before we can scan the directory. + */ + ASSERT(sc->flags & XCHK_FSGATES_DIRENTS); + xfs_dir_hook_setup(&dl->dhook, xchk_dirtree_live_update); + error = xfs_dir_hook_add(sc->mp, &dl->dhook); + if (error) + goto out; + mutex_lock(&dl->lock); /* Trace each parent pointer's path to the root. */ @@ -775,6 +928,10 @@ xchk_dirtree( } if (error) goto out_scanlock; + if (dl->aborted) { + xchk_set_incomplete(sc); + goto out_scanlock; + } /* Assess what we found in our path evaluation. */ xchk_dirtree_evaluate(dl, &oc); @@ -790,6 +947,7 @@ xchk_dirtree( out_scanlock: mutex_unlock(&dl->lock); +out: trace_xchk_dirtree_done(sc->ip, sc->sm, error); return error; } diff --git a/fs/xfs/scrub/dirtree.h b/fs/xfs/scrub/dirtree.h index 50fefd64ae508..2ddbcf43c2915 100644 --- a/fs/xfs/scrub/dirtree.h +++ b/fs/xfs/scrub/dirtree.h @@ -72,6 +72,13 @@ struct xchk_dirtree { /* Root inode that we're looking for. */ xfs_ino_t root_ino; + /* + * This is the inode that we're scanning. The live update hook can + * continue to be called after xchk_teardown drops sc->ip but before + * it calls buf_cleanup, so we keep a copy. + */ + xfs_ino_t scan_ino; + /* Scratch buffer for scanning pptr xattrs */ struct xfs_parent_rec pptr_rec; struct xfs_da_args pptr_args; @@ -80,9 +87,19 @@ struct xchk_dirtree { struct xfs_name xname; char namebuf[MAXNAMELEN]; + /* + * Hook into directory updates so that we can receive live updates + * from other writer threads. + */ + struct xfs_dir_hook dhook; + /* lock for everything below here */ struct mutex lock; + /* buffer for the live update functions to use for dirent names */ + struct xfs_name hook_xname; + unsigned char hook_namebuf[MAXNAMELEN]; + /* * All path steps observed during this scan. Each of the path * steps for a particular pathwalk are recorded in sequential @@ -106,6 +123,9 @@ struct xchk_dirtree { /* Have the path data been invalidated by a concurrent update? */ bool stale:1; + + /* Has the scan been aborted? */ + bool aborted:1; }; #define xchk_dirtree_for_each_path_safe(dl, path, n) \ diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index c474bcd7d54b7..509b6f4fd0cd3 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -1764,6 +1764,71 @@ DEFINE_EVENT(xchk_dirtree_evaluate_class, name, \ TP_ARGS(dl, oc)) DEFINE_XCHK_DIRTREE_EVALUATE_EVENT(xchk_dirtree_evaluate); +TRACE_EVENT(xchk_dirpath_changed, + TP_PROTO(struct xfs_scrub *sc, unsigned int path_nr, + unsigned int step_nr, const struct xfs_inode *dp, + const struct xfs_inode *ip, const struct xfs_name *xname), + TP_ARGS(sc, path_nr, step_nr, dp, ip, xname), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, path_nr) + __field(unsigned int, step_nr) + __field(xfs_ino_t, child_ino) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, namelen) + __dynamic_array(char, name, xname->len) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->path_nr = path_nr; + __entry->step_nr = step_nr; + __entry->child_ino = ip->i_ino; + __entry->parent_ino = dp->i_ino; + __entry->namelen = xname->len; + memcpy(__get_str(name), xname->name, xname->len); + ), + TP_printk("dev %d:%d path %u step %u child_ino 0x%llx parent_ino 0x%llx name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->path_nr, + __entry->step_nr, + __entry->child_ino, + __entry->parent_ino, + __entry->namelen, + __get_str(name)) +); + +TRACE_EVENT(xchk_dirtree_live_update, + TP_PROTO(struct xfs_scrub *sc, const struct xfs_inode *dp, + int action, const struct xfs_inode *ip, int delta, + const struct xfs_name *xname), + TP_ARGS(sc, dp, action, ip, delta, xname), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, parent_ino) + __field(int, action) + __field(xfs_ino_t, child_ino) + __field(int, delta) + __field(unsigned int, namelen) + __dynamic_array(char, name, xname->len) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->parent_ino = dp->i_ino; + __entry->action = action; + __entry->child_ino = ip->i_ino; + __entry->delta = delta; + __entry->namelen = xname->len; + memcpy(__get_str(name), xname->name, xname->len); + ), + TP_printk("dev %d:%d parent_ino 0x%llx child_ino 0x%llx nlink_delta %d name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->parent_ino, + __entry->child_ino, + __entry->delta, + __entry->namelen, + __get_str(name)) +); + /* repair tracepoints */ #if IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) From 327ed702d84034879572812f580cb769848af7ae Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:19 -0700 Subject: [PATCH 0487/1702] xfs: inode repair should ensure there's an attr fork to store parent pointers The runtime parent pointer update code expects that any file being moved around the directory tree already has an attr fork. However, if we had to rebuild an inode core record, there's a chance that we zeroed forkoff as part of the inode to pass the iget verifiers. Therefore, if we performed any repairs on an inode core, ensure that the inode has a nonzero forkoff before unlocking the inode. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/inode_repair.c | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/fs/xfs/scrub/inode_repair.c b/fs/xfs/scrub/inode_repair.c index e3b74ea50fdef..daf9f1ee7c2cb 100644 --- a/fs/xfs/scrub/inode_repair.c +++ b/fs/xfs/scrub/inode_repair.c @@ -1736,6 +1736,44 @@ xrep_inode_extsize( } } +/* Ensure this file has an attr fork if it needs to hold a parent pointer. */ +STATIC int +xrep_inode_pptr( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_inode *ip = sc->ip; + struct inode *inode = VFS_I(ip); + + if (!xfs_has_parent(mp)) + return 0; + + /* + * Unlinked inodes that cannot be added to the directory tree will not + * have a parent pointer. + */ + if (inode->i_nlink == 0 && !(inode->i_state & I_LINKABLE)) + return 0; + + /* The root directory doesn't have a parent pointer. */ + if (ip == mp->m_rootip) + return 0; + + /* + * Metadata inodes are rooted in the superblock and do not have any + * parents. + */ + if (xfs_is_metadata_inode(ip)) + return 0; + + /* Inode already has an attr fork; no further work possible here. */ + if (xfs_inode_has_attr_fork(ip)) + return 0; + + return xfs_bmap_add_attrfork(sc->tp, ip, + sizeof(struct xfs_attr_sf_hdr), true); +} + /* Fix any irregularities in an inode that the verifiers don't catch. */ STATIC int xrep_inode_problems( @@ -1744,6 +1782,9 @@ xrep_inode_problems( int error; error = xrep_inode_blockcounts(sc); + if (error) + return error; + error = xrep_inode_pptr(sc); if (error) return error; xrep_inode_timestamps(sc->ip); From 271557de7cbfdecb08e89ae1ca74647ceb57224f Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:23 -0700 Subject: [PATCH 0488/1702] xfs: reduce the rate of cond_resched calls inside scrub We really don't want to call cond_resched every single time we go through a loop in scrub -- there may be billions of records, and probing into the scheduler itself has overhead. Reduce this overhead by only calling cond_resched 10x per second; and add a counter so that we only check jiffies once every 1000 records or so. Surprisingly, this reduces scrub-only fstests runtime by about 2%. I used the bmapinflate xfs_db command to produce a billion-extent file and this stupid gadget reduced the scrub runtime by about 4%. From a stupid microbenchmark of calling these things 1 billion times, I estimate that cond_resched costs about 5.5ns per call; jiffes costs about 0.3ns per read; and fatal_signal_pending costs about 0.4ns per call. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/common.h | 25 ----------------- fs/xfs/scrub/scrub.c | 1 + fs/xfs/scrub/scrub.h | 64 ++++++++++++++++++++++++++++++++++++++++++ fs/xfs/scrub/xfarray.c | 10 +++---- fs/xfs/scrub/xfarray.h | 3 ++ fs/xfs/scrub/xfile.c | 2 +- 6 files changed, 74 insertions(+), 31 deletions(-) diff --git a/fs/xfs/scrub/common.h b/fs/xfs/scrub/common.h index 39465e39dc5fd..3d5f1f6b4b7bf 100644 --- a/fs/xfs/scrub/common.h +++ b/fs/xfs/scrub/common.h @@ -6,31 +6,6 @@ #ifndef __XFS_SCRUB_COMMON_H__ #define __XFS_SCRUB_COMMON_H__ -/* - * We /could/ terminate a scrub/repair operation early. If we're not - * in a good place to continue (fatal signal, etc.) then bail out. - * Note that we're careful not to make any judgements about *error. - */ -static inline bool -xchk_should_terminate( - struct xfs_scrub *sc, - int *error) -{ - /* - * If preemption is disabled, we need to yield to the scheduler every - * few seconds so that we don't run afoul of the soft lockup watchdog - * or RCU stall detector. - */ - cond_resched(); - - if (fatal_signal_pending(current)) { - if (*error == 0) - *error = -EINTR; - return true; - } - return false; -} - int xchk_trans_alloc(struct xfs_scrub *sc, uint resblks); int xchk_trans_alloc_empty(struct xfs_scrub *sc); void xchk_trans_cancel(struct xfs_scrub *sc); diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index e813b66b603a1..4a81f828f9f13 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -620,6 +620,7 @@ xfs_scrub_metadata( sc->sm = sm; sc->ops = &meta_scrub_ops[sm->sm_type]; sc->sick_mask = xchk_health_mask_for_scrub_type(sm->sm_type); + sc->relax = INIT_XCHK_RELAX; retry_op: /* * When repairs are allowed, prevent freezing or readonly remount while diff --git a/fs/xfs/scrub/scrub.h b/fs/xfs/scrub/scrub.h index 3910270471462..4e7e3edb6350c 100644 --- a/fs/xfs/scrub/scrub.h +++ b/fs/xfs/scrub/scrub.h @@ -8,6 +8,49 @@ struct xfs_scrub; +struct xchk_relax { + unsigned long next_resched; + unsigned int resched_nr; + bool interruptible; +}; + +/* Yield to the scheduler at most 10x per second. */ +#define XCHK_RELAX_NEXT (jiffies + (HZ / 10)) + +#define INIT_XCHK_RELAX \ + (struct xchk_relax){ \ + .next_resched = XCHK_RELAX_NEXT, \ + .resched_nr = 0, \ + .interruptible = true, \ + } + +/* + * Relax during a scrub operation and exit if there's a fatal signal pending. + * + * If preemption is disabled, we need to yield to the scheduler every now and + * then so that we don't run afoul of the soft lockup watchdog or RCU stall + * detector. cond_resched calls are somewhat expensive (~5ns) so we want to + * ratelimit this to 10x per second. Amortize the cost of the other checks by + * only doing it once every 100 calls. + */ +static inline int xchk_maybe_relax(struct xchk_relax *widget) +{ + /* Amortize the cost of scheduling and checking signals. */ + if (likely(++widget->resched_nr < 100)) + return 0; + widget->resched_nr = 0; + + if (unlikely(widget->next_resched <= jiffies)) { + cond_resched(); + widget->next_resched = XCHK_RELAX_NEXT; + } + + if (widget->interruptible && fatal_signal_pending(current)) + return -EINTR; + + return 0; +} + /* * Standard flags for allocating memory within scrub. NOFS context is * configured by the process allocation scope. Scrub and repair must be able @@ -123,6 +166,9 @@ struct xfs_scrub { */ unsigned int sick_mask; + /* next time we want to cond_resched() */ + struct xchk_relax relax; + /* State tracking for single-AG operations. */ struct xchk_ag sa; }; @@ -167,6 +213,24 @@ struct xfs_scrub_subord *xchk_scrub_create_subord(struct xfs_scrub *sc, unsigned int subtype); void xchk_scrub_free_subord(struct xfs_scrub_subord *sub); +/* + * We /could/ terminate a scrub/repair operation early. If we're not + * in a good place to continue (fatal signal, etc.) then bail out. + * Note that we're careful not to make any judgements about *error. + */ +static inline bool +xchk_should_terminate( + struct xfs_scrub *sc, + int *error) +{ + if (xchk_maybe_relax(&sc->relax)) { + if (*error == 0) + *error = -EINTR; + return true; + } + return false; +} + /* Metadata scrubbers */ int xchk_tester(struct xfs_scrub *sc); int xchk_superblock(struct xfs_scrub *sc); diff --git a/fs/xfs/scrub/xfarray.c b/fs/xfs/scrub/xfarray.c index b65cd3fc5ac9b..9185ae7088d49 100644 --- a/fs/xfs/scrub/xfarray.c +++ b/fs/xfs/scrub/xfarray.c @@ -7,9 +7,9 @@ #include "xfs_fs.h" #include "xfs_shared.h" #include "xfs_format.h" +#include "scrub/scrub.h" #include "scrub/xfile.h" #include "scrub/xfarray.h" -#include "scrub/scrub.h" #include "scrub/trace.h" /* @@ -486,6 +486,9 @@ xfarray_sortinfo_alloc( xfarray_sortinfo_lo(si)[0] = 0; xfarray_sortinfo_hi(si)[0] = array->nr - 1; + si->relax = INIT_XCHK_RELAX; + if (flags & XFARRAY_SORT_KILLABLE) + si->relax.interruptible = false; trace_xfarray_sort(si, nr_bytes); *infop = si; @@ -503,10 +506,7 @@ xfarray_sort_terminated( * few seconds so that we don't run afoul of the soft lockup watchdog * or RCU stall detector. */ - cond_resched(); - - if ((si->flags & XFARRAY_SORT_KILLABLE) && - fatal_signal_pending(current)) { + if (xchk_maybe_relax(&si->relax)) { if (*error == 0) *error = -EINTR; return true; diff --git a/fs/xfs/scrub/xfarray.h b/fs/xfs/scrub/xfarray.h index 8f54c8fc888fa..5eeeeed13ae24 100644 --- a/fs/xfs/scrub/xfarray.h +++ b/fs/xfs/scrub/xfarray.h @@ -127,6 +127,9 @@ struct xfarray_sortinfo { /* XFARRAY_SORT_* flags; see below. */ unsigned int flags; + /* next time we want to cond_resched() */ + struct xchk_relax relax; + /* Cache a folio here for faster scanning for pivots */ struct folio *folio; diff --git a/fs/xfs/scrub/xfile.c b/fs/xfs/scrub/xfile.c index 4e254a0ba0036..d848222f802ba 100644 --- a/fs/xfs/scrub/xfile.c +++ b/fs/xfs/scrub/xfile.c @@ -10,9 +10,9 @@ #include "xfs_log_format.h" #include "xfs_trans_resv.h" #include "xfs_mount.h" +#include "scrub/scrub.h" #include "scrub/xfile.h" #include "scrub/xfarray.h" -#include "scrub/scrub.h" #include "scrub/trace.h" #include From 37056912d5721324ac28787a4f903798f7361099 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:21 -0700 Subject: [PATCH 0489/1702] xfs: report directory tree corruption in the health information Report directories that are the source of corruption in the directory tree. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_fs.h | 1 + fs/xfs/libxfs/xfs_health.h | 4 +++- fs/xfs/scrub/health.c | 1 + fs/xfs/xfs_health.c | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_fs.h b/fs/xfs/libxfs/xfs_fs.h index 85f2a7e20ac00..7ae1912cdbfc2 100644 --- a/fs/xfs/libxfs/xfs_fs.h +++ b/fs/xfs/libxfs/xfs_fs.h @@ -411,6 +411,7 @@ struct xfs_bulkstat { #define XFS_BS_SICK_XATTR (1 << 5) /* extended attributes */ #define XFS_BS_SICK_SYMLINK (1 << 6) /* symbolic link remote target */ #define XFS_BS_SICK_PARENT (1 << 7) /* parent pointers */ +#define XFS_BS_SICK_DIRTREE (1 << 8) /* directory tree structure */ /* * Project quota id helpers (previously projid was 16bit only diff --git a/fs/xfs/libxfs/xfs_health.h b/fs/xfs/libxfs/xfs_health.h index 3c64b5f9bd681..b0edb4288e592 100644 --- a/fs/xfs/libxfs/xfs_health.h +++ b/fs/xfs/libxfs/xfs_health.h @@ -95,6 +95,7 @@ struct xfs_da_args; /* Don't propagate sick status to ag health summary during inactivation */ #define XFS_SICK_INO_FORGET (1 << 12) +#define XFS_SICK_INO_DIRTREE (1 << 13) /* directory tree structure */ /* Primary evidence of health problems in a given group. */ #define XFS_SICK_FS_PRIMARY (XFS_SICK_FS_COUNTERS | \ @@ -125,7 +126,8 @@ struct xfs_da_args; XFS_SICK_INO_DIR | \ XFS_SICK_INO_XATTR | \ XFS_SICK_INO_SYMLINK | \ - XFS_SICK_INO_PARENT) + XFS_SICK_INO_PARENT | \ + XFS_SICK_INO_DIRTREE) #define XFS_SICK_INO_ZAPPED (XFS_SICK_INO_BMBTD_ZAPPED | \ XFS_SICK_INO_BMBTA_ZAPPED | \ diff --git a/fs/xfs/scrub/health.c b/fs/xfs/scrub/health.c index 9020a6bef7f14..b712a8bd34f54 100644 --- a/fs/xfs/scrub/health.c +++ b/fs/xfs/scrub/health.c @@ -108,6 +108,7 @@ static const struct xchk_health_map type_to_health_flag[XFS_SCRUB_TYPE_NR] = { [XFS_SCRUB_TYPE_FSCOUNTERS] = { XHG_FS, XFS_SICK_FS_COUNTERS }, [XFS_SCRUB_TYPE_QUOTACHECK] = { XHG_FS, XFS_SICK_FS_QUOTACHECK }, [XFS_SCRUB_TYPE_NLINKS] = { XHG_FS, XFS_SICK_FS_NLINKS }, + [XFS_SCRUB_TYPE_DIRTREE] = { XHG_INO, XFS_SICK_INO_DIRTREE }, }; /* Return the health status mask for this scrub type. */ diff --git a/fs/xfs/xfs_health.c b/fs/xfs/xfs_health.c index b39f959146bc1..10f116d093a22 100644 --- a/fs/xfs/xfs_health.c +++ b/fs/xfs/xfs_health.c @@ -470,6 +470,7 @@ static const struct ioctl_sick_map ino_map[] = { { XFS_SICK_INO_BMBTA_ZAPPED, XFS_BS_SICK_BMBTA }, { XFS_SICK_INO_DIR_ZAPPED, XFS_BS_SICK_DIR }, { XFS_SICK_INO_SYMLINK_ZAPPED, XFS_BS_SICK_SYMLINK }, + { XFS_SICK_INO_DIRTREE, XFS_BS_SICK_DIRTREE }, { 0, 0 }, }; From be7cf174e908b1f350dd3ae4fbdf335f22af3273 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:24 -0700 Subject: [PATCH 0490/1702] xfs: move xfs_ioc_scrub_metadata to scrub.c Move the scrub ioctl handler to scrub.c to keep the code together and to reduce unnecessary code when CONFIG_XFS_ONLINE_SCRUB=n. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/scrub.c | 27 ++++++++++++++++++++++++++- fs/xfs/scrub/xfs_scrub.h | 4 ++-- fs/xfs/xfs_ioctl.c | 24 ------------------------ 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 4a81f828f9f13..1456cc11c406d 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -578,7 +578,7 @@ xchk_scrub_create_subord( } /* Dispatch metadata scrubbing. */ -int +STATIC int xfs_scrub_metadata( struct file *file, struct xfs_scrub_metadata *sm) @@ -724,3 +724,28 @@ xfs_scrub_metadata( run.retries++; goto retry_op; } + +/* Scrub one aspect of one piece of metadata. */ +int +xfs_ioc_scrub_metadata( + struct file *file, + void __user *arg) +{ + struct xfs_scrub_metadata scrub; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (copy_from_user(&scrub, arg, sizeof(scrub))) + return -EFAULT; + + error = xfs_scrub_metadata(file, &scrub); + if (error) + return error; + + if (copy_to_user(arg, &scrub, sizeof(scrub))) + return -EFAULT; + + return 0; +} diff --git a/fs/xfs/scrub/xfs_scrub.h b/fs/xfs/scrub/xfs_scrub.h index a39befa743ce0..02c930f175d0b 100644 --- a/fs/xfs/scrub/xfs_scrub.h +++ b/fs/xfs/scrub/xfs_scrub.h @@ -7,9 +7,9 @@ #define __XFS_SCRUB_H__ #ifndef CONFIG_XFS_ONLINE_SCRUB -# define xfs_scrub_metadata(file, sm) (-ENOTTY) +# define xfs_ioc_scrub_metadata(f, a) (-ENOTTY) #else -int xfs_scrub_metadata(struct file *file, struct xfs_scrub_metadata *sm); +int xfs_ioc_scrub_metadata(struct file *file, void __user *arg); #endif /* CONFIG_XFS_ONLINE_SCRUB */ #endif /* __XFS_SCRUB_H__ */ diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 0e97070abe802..857b19428984a 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -1055,30 +1055,6 @@ xfs_ioc_getfsmap( return error; } -STATIC int -xfs_ioc_scrub_metadata( - struct file *file, - void __user *arg) -{ - struct xfs_scrub_metadata scrub; - int error; - - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; - - if (copy_from_user(&scrub, arg, sizeof(scrub))) - return -EFAULT; - - error = xfs_scrub_metadata(file, &scrub); - if (error) - return error; - - if (copy_to_user(arg, &scrub, sizeof(scrub))) - return -EFAULT; - - return 0; -} - int xfs_ioc_swapext( xfs_swapext_t *sxp) From 3f31406aef493b3f19020909d29974e28253f91c Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:22 -0700 Subject: [PATCH 0491/1702] xfs: fix corruptions in the directory tree Repair corruptions in the directory tree itself. Cycles are broken by removing an incoming parent->child link. Multiply-owned directories are fixed by pruning the extra parent -> child links Disconnected subtrees are reconnected to the lost and found. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 1 + fs/xfs/scrub/dirtree.c | 38 +- fs/xfs/scrub/dirtree.h | 29 ++ fs/xfs/scrub/dirtree_repair.c | 821 ++++++++++++++++++++++++++++++++++ fs/xfs/scrub/orphanage.c | 6 + fs/xfs/scrub/orphanage.h | 8 + fs/xfs/scrub/repair.h | 4 + fs/xfs/scrub/scrub.c | 2 +- fs/xfs/scrub/trace.h | 23 +- fs/xfs/xfs_inode.c | 2 +- fs/xfs/xfs_inode.h | 1 + 11 files changed, 927 insertions(+), 8 deletions(-) create mode 100644 fs/xfs/scrub/dirtree_repair.c diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index 8ec0dd257a984..d1ce1213797be 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -204,6 +204,7 @@ xfs-y += $(addprefix scrub/, \ bmap_repair.o \ cow_repair.o \ dir_repair.o \ + dirtree_repair.o \ findparent.o \ fscounters_repair.o \ ialloc_repair.o \ diff --git a/fs/xfs/scrub/dirtree.c b/fs/xfs/scrub/dirtree.c index ecc56eb5ed270..bde58fb561ea1 100644 --- a/fs/xfs/scrub/dirtree.c +++ b/fs/xfs/scrub/dirtree.c @@ -26,6 +26,8 @@ #include "scrub/xfblob.h" #include "scrub/listxattr.h" #include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/orphanage.h" #include "scrub/dirtree.h" /* @@ -95,6 +97,12 @@ xchk_setup_dirtree( xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); + if (xchk_could_repair(sc)) { + error = xrep_setup_dirtree(sc); + if (error) + return error; + } + dl = kvzalloc(sizeof(struct xchk_dirtree), XCHK_GFP_FLAGS); if (!dl) return -ENOMEM; @@ -104,6 +112,7 @@ xchk_setup_dirtree( INIT_LIST_HEAD(&dl->path_list); dl->root_ino = NULLFSINO; dl->scan_ino = NULLFSINO; + dl->parent_ino = NULLFSINO; mutex_init(&dl->lock); @@ -142,7 +151,7 @@ xchk_setup_dirtree( * Add the parent pointer described by @dl->pptr to the given path as a new * step. Returns -ELNRNG if the path is too deep. */ -STATIC int +int xchk_dirpath_append( struct xchk_dirtree *dl, struct xfs_inode *ip, @@ -609,6 +618,22 @@ xchk_dirpath_step_is_stale( if (memcmp(dl->hook_xname.name, p->name->name, p->name->len) != 0) return 0; + /* + * If the update comes from the repair code itself, walk the state + * machine forward. + */ + if (p->ip->i_ino == dl->scan_ino && + path->outcome == XREP_DIRPATH_ADOPTING) { + xchk_dirpath_set_outcome(dl, path, XREP_DIRPATH_ADOPTED); + return 0; + } + + if (p->ip->i_ino == dl->scan_ino && + path->outcome == XREP_DIRPATH_DELETING) { + xchk_dirpath_set_outcome(dl, path, XREP_DIRPATH_DELETED); + return 0; + } + /* Exact match, scan data is out of date. */ trace_xchk_dirpath_changed(dl->sc, path->path_nr, step_nr, p->dp, p->ip, p->name); @@ -747,7 +772,7 @@ xchk_dirtree_load_path( * path was too deep; -ENOSR if there were too many parent pointers; or * a negative errno. */ -STATIC int +int xchk_dirtree_find_paths_to_root( struct xchk_dirtree *dl) { @@ -819,7 +844,7 @@ xchk_dirtree_find_paths_to_root( * Figure out what to do with the paths we tried to find. Do not call this * if the scan results are stale. */ -STATIC void +void xchk_dirtree_evaluate( struct xchk_dirtree *dl, struct xchk_dirtree_outcomes *oc) @@ -856,6 +881,13 @@ xchk_dirtree_evaluate( /* This path got all the way to the root. */ oc->good++; break; + case XREP_DIRPATH_DELETING: + case XREP_DIRPATH_DELETED: + case XREP_DIRPATH_ADOPTING: + case XREP_DIRPATH_ADOPTED: + /* These should not be in progress! */ + ASSERT(0); + break; } } diff --git a/fs/xfs/scrub/dirtree.h b/fs/xfs/scrub/dirtree.h index 2ddbcf43c2915..1e1686365c61c 100644 --- a/fs/xfs/scrub/dirtree.h +++ b/fs/xfs/scrub/dirtree.h @@ -26,6 +26,11 @@ enum xchk_dirpath_outcome { XCHK_DIRPATH_LOOP, /* cycle detected further up */ XCHK_DIRPATH_STALE, /* path is stale */ XCHK_DIRPATH_OK, /* path reaches the root */ + + XREP_DIRPATH_DELETING, /* path is being deleted */ + XREP_DIRPATH_DELETED, /* path has been deleted */ + XREP_DIRPATH_ADOPTING, /* path is being adopted */ + XREP_DIRPATH_ADOPTED, /* path has been adopted */ }; /* @@ -64,6 +69,9 @@ struct xchk_dirtree_outcomes { /* Number of XCHK_DIRPATH_OK */ unsigned int good; + + /* Directory needs to be added to lost+found */ + bool needs_adoption; }; struct xchk_dirtree { @@ -79,6 +87,14 @@ struct xchk_dirtree { */ xfs_ino_t scan_ino; + /* + * If we start deleting redundant paths to this subdirectory, this is + * the inode number of the surviving parent and the dotdot entry will + * be set to this value. If the value is NULLFSINO, then use @root_ino + * as a stand-in until the orphanage can adopt the subdirectory. + */ + xfs_ino_t parent_ino; + /* Scratch buffer for scanning pptr xattrs */ struct xfs_parent_rec pptr_rec; struct xfs_da_args pptr_args; @@ -87,12 +103,18 @@ struct xchk_dirtree { struct xfs_name xname; char namebuf[MAXNAMELEN]; + /* Information for reparenting this directory. */ + struct xrep_adoption adoption; + /* * Hook into directory updates so that we can receive live updates * from other writer threads. */ struct xfs_dir_hook dhook; + /* Parent pointer update arguments. */ + struct xfs_parent_args ppargs; + /* lock for everything below here */ struct mutex lock; @@ -146,4 +168,11 @@ xchk_dirtree_parentless(const struct xchk_dirtree *dl) return false; } +int xchk_dirtree_find_paths_to_root(struct xchk_dirtree *dl); +int xchk_dirpath_append(struct xchk_dirtree *dl, struct xfs_inode *ip, + struct xchk_dirpath *path, const struct xfs_name *name, + const struct xfs_parent_rec *pptr); +void xchk_dirtree_evaluate(struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc); + #endif /* __XFS_SCRUB_DIRTREE_H__ */ diff --git a/fs/xfs/scrub/dirtree_repair.c b/fs/xfs/scrub/dirtree_repair.c new file mode 100644 index 0000000000000..5c04e70ba9518 --- /dev/null +++ b/fs/xfs/scrub/dirtree_repair.c @@ -0,0 +1,821 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2023-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_trans_space.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_attr.h" +#include "xfs_parent.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/bitmap.h" +#include "scrub/ino_bitmap.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" +#include "scrub/listxattr.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/orphanage.h" +#include "scrub/dirtree.h" +#include "scrub/readdir.h" + +/* + * Directory Tree Structure Repairs + * ================================ + * + * If we decide that the directory being scanned is participating in a + * directory loop, the only change we can make is to remove directory entries + * pointing down to @sc->ip. If that leaves it with no parents, the directory + * should be adopted by the orphanage. + */ + +/* Set up to repair directory loops. */ +int +xrep_setup_dirtree( + struct xfs_scrub *sc) +{ + return xrep_orphanage_try_create(sc); +} + +/* Change the outcome of this path. */ +static inline void +xrep_dirpath_set_outcome( + struct xchk_dirtree *dl, + struct xchk_dirpath *path, + enum xchk_dirpath_outcome outcome) +{ + trace_xrep_dirpath_set_outcome(dl->sc, path->path_nr, path->nr_steps, + outcome); + + path->outcome = outcome; +} + +/* Delete all paths. */ +STATIC void +xrep_dirtree_delete_all_paths( + struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc) +{ + struct xchk_dirpath *path; + + xchk_dirtree_for_each_path(dl, path) { + switch (path->outcome) { + case XCHK_DIRPATH_CORRUPT: + case XCHK_DIRPATH_LOOP: + oc->suspect--; + oc->bad++; + xrep_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); + break; + case XCHK_DIRPATH_OK: + oc->good--; + oc->bad++; + xrep_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); + break; + default: + break; + } + } + + ASSERT(oc->suspect == 0); + ASSERT(oc->good == 0); +} + +/* Since this is the surviving path, set the dotdot entry to this value. */ +STATIC void +xrep_dirpath_retain_parent( + struct xchk_dirtree *dl, + struct xchk_dirpath *path) +{ + struct xchk_dirpath_step step; + int error; + + error = xfarray_load(dl->path_steps, path->first_step, &step); + if (error) + return; + + dl->parent_ino = be64_to_cpu(step.pptr_rec.p_ino); +} + +/* Find the one surviving path so we know how to set dotdot. */ +STATIC void +xrep_dirtree_find_surviving_path( + struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc) +{ + struct xchk_dirpath *path; + bool foundit = false; + + xchk_dirtree_for_each_path(dl, path) { + switch (path->outcome) { + case XCHK_DIRPATH_CORRUPT: + case XCHK_DIRPATH_LOOP: + case XCHK_DIRPATH_OK: + if (!foundit) { + xrep_dirpath_retain_parent(dl, path); + foundit = true; + continue; + } + ASSERT(foundit == false); + break; + default: + break; + } + } + + ASSERT(oc->suspect + oc->good == 1); +} + +/* Delete all paths except for the one good one. */ +STATIC void +xrep_dirtree_keep_one_good_path( + struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc) +{ + struct xchk_dirpath *path; + bool foundit = false; + + xchk_dirtree_for_each_path(dl, path) { + switch (path->outcome) { + case XCHK_DIRPATH_CORRUPT: + case XCHK_DIRPATH_LOOP: + oc->suspect--; + oc->bad++; + xrep_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); + break; + case XCHK_DIRPATH_OK: + if (!foundit) { + xrep_dirpath_retain_parent(dl, path); + foundit = true; + continue; + } + oc->good--; + oc->bad++; + xrep_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); + break; + default: + break; + } + } + + ASSERT(oc->suspect == 0); + ASSERT(oc->good < 2); +} + +/* Delete all paths except for one suspect one. */ +STATIC void +xrep_dirtree_keep_one_suspect_path( + struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc) +{ + struct xchk_dirpath *path; + bool foundit = false; + + xchk_dirtree_for_each_path(dl, path) { + switch (path->outcome) { + case XCHK_DIRPATH_CORRUPT: + case XCHK_DIRPATH_LOOP: + if (!foundit) { + xrep_dirpath_retain_parent(dl, path); + foundit = true; + continue; + } + oc->suspect--; + oc->bad++; + xrep_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); + break; + case XCHK_DIRPATH_OK: + ASSERT(0); + break; + default: + break; + } + } + + ASSERT(oc->suspect == 1); + ASSERT(oc->good == 0); +} + +/* + * Figure out what to do with the paths we tried to find. Returns -EDEADLOCK + * if the scan results have become stale. + */ +STATIC void +xrep_dirtree_decide_fate( + struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc) +{ + xchk_dirtree_evaluate(dl, oc); + + /* Parentless directories should not have any paths at all. */ + if (xchk_dirtree_parentless(dl)) { + xrep_dirtree_delete_all_paths(dl, oc); + return; + } + + /* One path is exactly the number of paths we want. */ + if (oc->good + oc->suspect == 1) { + xrep_dirtree_find_surviving_path(dl, oc); + return; + } + + /* Zero paths means we should reattach the subdir to the orphanage. */ + if (oc->good + oc->suspect == 0) { + if (dl->sc->orphanage) + oc->needs_adoption = true; + return; + } + + /* + * Otherwise, this subdirectory has too many parents. If there's at + * least one good path, keep it and delete the others. + */ + if (oc->good > 0) { + xrep_dirtree_keep_one_good_path(dl, oc); + return; + } + + /* + * There are no good paths and there are too many suspect paths. + * Keep the first suspect path and delete the rest. + */ + xrep_dirtree_keep_one_suspect_path(dl, oc); +} + +/* + * Load the first step of this path into @step and @dl->xname/pptr + * for later repair work. + */ +STATIC int +xrep_dirtree_prep_path( + struct xchk_dirtree *dl, + struct xchk_dirpath *path, + struct xchk_dirpath_step *step) +{ + int error; + + error = xfarray_load(dl->path_steps, path->first_step, step); + if (error) + return error; + + error = xfblob_loadname(dl->path_names, step->name_cookie, &dl->xname, + step->name_len); + if (error) + return error; + + dl->pptr_rec = step->pptr_rec; /* struct copy */ + return 0; +} + +/* Delete the VFS dentry for a removed child. */ +STATIC int +xrep_dirtree_purge_dentry( + struct xchk_dirtree *dl, + struct xfs_inode *dp, + const struct xfs_name *name) +{ + struct qstr qname = QSTR_INIT(name->name, name->len); + struct dentry *parent_dentry, *child_dentry; + int error = 0; + + /* + * Find the dentry for the parent directory. If there isn't one, we're + * done. Caller already holds i_rwsem for parent and child. + */ + parent_dentry = d_find_alias(VFS_I(dp)); + if (!parent_dentry) + return 0; + + /* The VFS thinks the parent is a directory, right? */ + if (!d_is_dir(parent_dentry)) { + ASSERT(d_is_dir(parent_dentry)); + error = -EFSCORRUPTED; + goto out_dput_parent; + } + + /* + * Try to find the dirent pointing to the child. If there isn't one, + * we're done. + */ + qname.hash = full_name_hash(parent_dentry, name->name, name->len); + child_dentry = d_lookup(parent_dentry, &qname); + if (!child_dentry) { + error = 0; + goto out_dput_parent; + } + + trace_xrep_dirtree_delete_child(dp->i_mount, child_dentry); + + /* Child is not a directory? We're screwed. */ + if (!d_is_dir(child_dentry)) { + ASSERT(d_is_dir(child_dentry)); + error = -EFSCORRUPTED; + goto out_dput_child; + } + + /* Replace the child dentry with a negative one. */ + d_delete(child_dentry); + +out_dput_child: + dput(child_dentry); +out_dput_parent: + dput(parent_dentry); + return error; +} + +/* + * Prepare to delete a link by taking the IOLOCK of the parent and the child + * (scrub target). Caller must hold IOLOCK_EXCL on @sc->ip. Returns 0 if we + * took both locks, or a negative errno if we couldn't lock the parent in time. + */ +static inline int +xrep_dirtree_unlink_iolock( + struct xfs_scrub *sc, + struct xfs_inode *dp) +{ + int error; + + ASSERT(sc->ilock_flags & XFS_IOLOCK_EXCL); + + if (xfs_ilock_nowait(dp, XFS_IOLOCK_EXCL)) + return 0; + + xchk_iunlock(sc, XFS_IOLOCK_EXCL); + do { + xfs_ilock(dp, XFS_IOLOCK_EXCL); + if (xchk_ilock_nowait(sc, XFS_IOLOCK_EXCL)) + break; + xfs_iunlock(dp, XFS_IOLOCK_EXCL); + + if (xchk_should_terminate(sc, &error)) { + xchk_ilock(sc, XFS_IOLOCK_EXCL); + return error; + } + + delay(1); + } while (1); + + return 0; +} + +/* + * Remove a link from the directory tree and update the dcache. Returns + * -ESTALE if the scan data are now out of date. + */ +STATIC int +xrep_dirtree_unlink( + struct xchk_dirtree *dl, + struct xfs_inode *dp, + struct xchk_dirpath *path, + struct xchk_dirpath_step *step) +{ + struct xfs_scrub *sc = dl->sc; + struct xfs_mount *mp = sc->mp; + xfs_ino_t dotdot_ino; + xfs_ino_t parent_ino = dl->parent_ino; + unsigned int resblks; + int dontcare; + int error; + + /* Take IOLOCK_EXCL of the parent and child. */ + error = xrep_dirtree_unlink_iolock(sc, dp); + if (error) + return error; + + /* + * Create the transaction that we need to sever the path. Ignore + * EDQUOT and ENOSPC being returned via nospace_error because the + * directory code can handle a reservationless update. + */ + resblks = xfs_remove_space_res(mp, step->name_len); + error = xfs_trans_alloc_dir(dp, &M_RES(mp)->tr_remove, sc->ip, + &resblks, &sc->tp, &dontcare); + if (error) + goto out_iolock; + + /* + * Cancel if someone invalidate the paths while we were trying to get + * the ILOCK. + */ + mutex_lock(&dl->lock); + if (dl->stale) { + mutex_unlock(&dl->lock); + error = -ESTALE; + goto out_trans_cancel; + } + xrep_dirpath_set_outcome(dl, path, XREP_DIRPATH_DELETING); + mutex_unlock(&dl->lock); + + trace_xrep_dirtree_delete_path(dl->sc, sc->ip, path->path_nr, + &dl->xname, &dl->pptr_rec); + + /* + * Decide if we need to reset the dotdot entry. Rules: + * + * - If there's a surviving parent, we want dotdot to point there. + * - If we don't have any surviving parents, then point dotdot at the + * root dir. + * - If dotdot is already set to the value we want, pass in NULLFSINO + * for no change necessary. + * + * Do this /before/ we dirty anything, in case the dotdot lookup + * fails. + */ + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, &dotdot_ino); + if (error) + goto out_trans_cancel; + if (parent_ino == NULLFSINO) + parent_ino = dl->root_ino; + if (dotdot_ino == parent_ino) + parent_ino = NULLFSINO; + + /* Drop the link from sc->ip's dotdot entry. */ + error = xfs_droplink(sc->tp, dp); + if (error) + goto out_trans_cancel; + + /* Reset the dotdot entry to a surviving parent. */ + if (parent_ino != NULLFSINO) { + error = xfs_dir_replace(sc->tp, sc->ip, &xfs_name_dotdot, + parent_ino, 0); + if (error) + goto out_trans_cancel; + } + + /* Drop the link from dp to sc->ip. */ + error = xfs_droplink(sc->tp, sc->ip); + if (error) + goto out_trans_cancel; + + error = xfs_dir_removename(sc->tp, dp, &dl->xname, sc->ip->i_ino, + resblks); + if (error) { + ASSERT(error != -ENOENT); + goto out_trans_cancel; + } + + if (xfs_has_parent(sc->mp)) { + error = xfs_parent_removename(sc->tp, &dl->ppargs, dp, + &dl->xname, sc->ip); + if (error) + goto out_trans_cancel; + } + + /* + * Notify dirent hooks that we removed the bad link, invalidate the + * dcache, and commit the repair. + */ + xfs_dir_update_hook(dp, sc->ip, -1, &dl->xname); + error = xrep_dirtree_purge_dentry(dl, dp, &dl->xname); + if (error) + goto out_trans_cancel; + + error = xrep_trans_commit(sc); + goto out_ilock; + +out_trans_cancel: + xchk_trans_cancel(sc); +out_ilock: + xfs_iunlock(sc->ip, XFS_ILOCK_EXCL); + xfs_iunlock(dp, XFS_ILOCK_EXCL); +out_iolock: + xfs_iunlock(dp, XFS_IOLOCK_EXCL); + return error; +} + +/* + * Delete a directory entry that points to this directory. Returns -ESTALE + * if the scan data are now out of date. + */ +STATIC int +xrep_dirtree_delete_path( + struct xchk_dirtree *dl, + struct xchk_dirpath *path) +{ + struct xchk_dirpath_step step; + struct xfs_scrub *sc = dl->sc; + struct xfs_inode *dp; + int error; + + /* + * Load the parent pointer and directory inode for this path, then + * drop the scan lock, the ILOCK, and the transaction so that + * _delete_path can reserve the proper transaction. This sets up + * @dl->xname for the deletion. + */ + error = xrep_dirtree_prep_path(dl, path, &step); + if (error) + return error; + + error = xchk_iget(sc, be64_to_cpu(step.pptr_rec.p_ino), &dp); + if (error) + return error; + + mutex_unlock(&dl->lock); + xchk_trans_cancel(sc); + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + /* Delete the directory link and release the parent. */ + error = xrep_dirtree_unlink(dl, dp, path, &step); + xchk_irele(sc, dp); + + /* + * Retake all the resources we had at the beginning even if the repair + * failed or the scan data are now stale. This keeps things simple for + * the caller. + */ + xchk_trans_alloc_empty(sc); + xchk_ilock(sc, XFS_ILOCK_EXCL); + mutex_lock(&dl->lock); + + if (!error && dl->stale) + error = -ESTALE; + return error; +} + +/* Add a new path to represent our in-progress adoption. */ +STATIC int +xrep_dirtree_create_adoption_path( + struct xchk_dirtree *dl) +{ + struct xfs_scrub *sc = dl->sc; + struct xchk_dirpath *path; + int error; + + /* + * We should have capped the number of paths at XFS_MAXLINK-1 in the + * scanner. + */ + if (dl->nr_paths > XFS_MAXLINK) { + ASSERT(dl->nr_paths <= XFS_MAXLINK); + return -EFSCORRUPTED; + } + + /* + * Create a new xchk_path structure to remember this parent pointer + * and record the first name step. + */ + path = kmalloc(sizeof(struct xchk_dirpath), XCHK_GFP_FLAGS); + if (!path) + return -ENOMEM; + + INIT_LIST_HEAD(&path->list); + xino_bitmap_init(&path->seen_inodes); + path->nr_steps = 0; + path->outcome = XREP_DIRPATH_ADOPTING; + + /* + * Record the new link that we just created in the orphanage. Because + * adoption is the last repair that we perform, we don't bother filling + * in the path all the way back to the root. + */ + xfs_inode_to_parent_rec(&dl->pptr_rec, sc->orphanage); + + error = xino_bitmap_set(&path->seen_inodes, sc->orphanage->i_ino); + if (error) + goto out_path; + + trace_xrep_dirtree_create_adoption(sc, sc->ip, dl->nr_paths, + &dl->xname, &dl->pptr_rec); + + error = xchk_dirpath_append(dl, sc->ip, path, &dl->xname, + &dl->pptr_rec); + if (error) + goto out_path; + + path->first_step = xfarray_length(dl->path_steps) - 1; + path->second_step = XFARRAY_NULLIDX; + path->path_nr = dl->nr_paths; + + list_add_tail(&path->list, &dl->path_list); + dl->nr_paths++; + return 0; + +out_path: + kfree(path); + return error; +} + +/* + * Prepare to move a file to the orphanage by taking the IOLOCK of the + * orphanage and the child (scrub target). Caller must hold IOLOCK_EXCL on + * @sc->ip. Returns 0 if we took both locks, or a negative errno if we + * couldn't lock the orphanage in time. + */ +static inline int +xrep_dirtree_adopt_iolock( + struct xfs_scrub *sc) +{ + int error; + + ASSERT(sc->ilock_flags & XFS_IOLOCK_EXCL); + + if (xrep_orphanage_ilock_nowait(sc, XFS_IOLOCK_EXCL)) + return 0; + + xchk_iunlock(sc, XFS_IOLOCK_EXCL); + do { + xrep_orphanage_ilock(sc, XFS_IOLOCK_EXCL); + if (xchk_ilock_nowait(sc, XFS_IOLOCK_EXCL)) + break; + xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + + if (xchk_should_terminate(sc, &error)) { + xchk_ilock(sc, XFS_IOLOCK_EXCL); + return error; + } + + delay(1); + } while (1); + + return 0; +} + +/* + * Reattach this orphaned directory to the orphanage. Do not call this with + * any resources held. Returns -ESTALE if the scan data have become out of + * date. + */ +STATIC int +xrep_dirtree_adopt( + struct xchk_dirtree *dl) +{ + struct xfs_scrub *sc = dl->sc; + int error; + + /* Take the IOLOCK of the orphanage and the scrub target. */ + error = xrep_dirtree_adopt_iolock(sc); + if (error) + return error; + + /* + * Set up for an adoption. The directory tree fixer runs after the + * link counts have been corrected. Therefore, we must bump the + * child's link count since there will be no further opportunity to fix + * errors. + */ + error = xrep_adoption_trans_alloc(sc, &dl->adoption); + if (error) + goto out_iolock; + dl->adoption.bump_child_nlink = true; + + /* Figure out what name we're going to use here. */ + error = xrep_adoption_compute_name(&dl->adoption, &dl->xname); + if (error) + goto out_trans; + + /* + * Now that we have a proposed name for the orphanage entry, create + * a faux path so that the live update hook will see it. + */ + mutex_lock(&dl->lock); + if (dl->stale) { + mutex_unlock(&dl->lock); + error = -ESTALE; + goto out_trans; + } + error = xrep_dirtree_create_adoption_path(dl); + mutex_unlock(&dl->lock); + if (error) + goto out_trans; + + /* Reparent the directory. */ + error = xrep_adoption_move(&dl->adoption); + if (error) + goto out_trans; + + /* + * Commit the name and release all inode locks except for the scrub + * target's IOLOCK. + */ + error = xrep_trans_commit(sc); + goto out_ilock; + +out_trans: + xchk_trans_cancel(sc); +out_ilock: + xchk_iunlock(sc, XFS_ILOCK_EXCL); + xrep_orphanage_iunlock(sc, XFS_ILOCK_EXCL); +out_iolock: + xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + return error; +} + +/* + * This newly orphaned directory needs to be adopted by the orphanage. + * Make this happen. + */ +STATIC int +xrep_dirtree_move_to_orphanage( + struct xchk_dirtree *dl) +{ + struct xfs_scrub *sc = dl->sc; + int error; + + /* + * Start by dropping all the resources that we hold so that we can grab + * all the resources that we need for the adoption. + */ + mutex_unlock(&dl->lock); + xchk_trans_cancel(sc); + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + /* Perform the adoption. */ + error = xrep_dirtree_adopt(dl); + + /* + * Retake all the resources we had at the beginning even if the repair + * failed or the scan data are now stale. This keeps things simple for + * the caller. + */ + xchk_trans_alloc_empty(sc); + xchk_ilock(sc, XFS_ILOCK_EXCL); + mutex_lock(&dl->lock); + + if (!error && dl->stale) + error = -ESTALE; + return error; +} + +/* + * Try to fix all the problems. Returns -ESTALE if the scan data have become + * out of date. + */ +STATIC int +xrep_dirtree_fix_problems( + struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc) +{ + struct xchk_dirpath *path; + int error; + + /* Delete all the paths we don't want. */ + xchk_dirtree_for_each_path(dl, path) { + if (path->outcome != XCHK_DIRPATH_DELETE) + continue; + + error = xrep_dirtree_delete_path(dl, path); + if (error) + return error; + } + + /* Reparent this directory to the orphanage. */ + if (oc->needs_adoption) { + if (xrep_orphanage_can_adopt(dl->sc)) + return xrep_dirtree_move_to_orphanage(dl); + return -EFSCORRUPTED; + } + + return 0; +} + +/* Fix directory loops involving this directory. */ +int +xrep_dirtree( + struct xfs_scrub *sc) +{ + struct xchk_dirtree *dl = sc->buf; + struct xchk_dirtree_outcomes oc; + int error; + + /* + * Prepare to fix the directory tree by retaking the scan lock. The + * order of resource acquisition is still IOLOCK -> transaction -> + * ILOCK -> scan lock. + */ + mutex_lock(&dl->lock); + do { + /* + * Decide what we're going to do, then do it. An -ESTALE + * return here means the scan results are invalid and we have + * to walk again. + */ + if (!dl->stale) { + xrep_dirtree_decide_fate(dl, &oc); + + trace_xrep_dirtree_decided_fate(dl, &oc); + + error = xrep_dirtree_fix_problems(dl, &oc); + if (!error || error != -ESTALE) + break; + } + error = xchk_dirtree_find_paths_to_root(dl); + if (error == -ELNRNG || error == -ENOSR) + error = -EFSCORRUPTED; + } while (!error); + mutex_unlock(&dl->lock); + + return error; +} diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c index b2f905924d0d8..b1c6c60ee1da6 100644 --- a/fs/xfs/scrub/orphanage.c +++ b/fs/xfs/scrub/orphanage.c @@ -570,6 +570,12 @@ xrep_adoption_move( xfs_bumplink(sc->tp, sc->orphanage); xfs_trans_log_inode(sc->tp, sc->orphanage, XFS_ILOG_CORE); + /* Bump the link count of the child. */ + if (adopt->bump_child_nlink) { + xfs_bumplink(sc->tp, sc->ip); + xfs_trans_log_inode(sc->tp, sc->ip, XFS_ILOG_CORE); + } + /* Replace the dotdot entry if the child is a subdirectory. */ if (isdir) { error = xfs_dir_replace(sc->tp, sc->ip, &xfs_name_dotdot, diff --git a/fs/xfs/scrub/orphanage.h b/fs/xfs/scrub/orphanage.h index beb6b686784e6..7c7a2e7d81dbd 100644 --- a/fs/xfs/scrub/orphanage.h +++ b/fs/xfs/scrub/orphanage.h @@ -60,6 +60,14 @@ struct xrep_adoption { /* Block reservations for orphanage and child (if directory). */ unsigned int orphanage_blkres; unsigned int child_blkres; + + /* + * Does the caller want us to bump the child link count? This is not + * needed when reattaching files that have become disconnected but have + * nlink > 1. It is necessary when changing the directory tree + * structure. + */ + bool bump_child_nlink:1; }; bool xrep_orphanage_can_adopt(struct xfs_scrub *sc); diff --git a/fs/xfs/scrub/repair.h b/fs/xfs/scrub/repair.h index 622eb486a16fb..0e0dc2bf985c2 100644 --- a/fs/xfs/scrub/repair.h +++ b/fs/xfs/scrub/repair.h @@ -95,6 +95,7 @@ int xrep_setup_directory(struct xfs_scrub *sc); int xrep_setup_parent(struct xfs_scrub *sc); int xrep_setup_nlinks(struct xfs_scrub *sc); int xrep_setup_symlink(struct xfs_scrub *sc, unsigned int *resblks); +int xrep_setup_dirtree(struct xfs_scrub *sc); /* Repair setup functions */ int xrep_setup_ag_allocbt(struct xfs_scrub *sc); @@ -132,6 +133,7 @@ int xrep_xattr(struct xfs_scrub *sc); int xrep_directory(struct xfs_scrub *sc); int xrep_parent(struct xfs_scrub *sc); int xrep_symlink(struct xfs_scrub *sc); +int xrep_dirtree(struct xfs_scrub *sc); #ifdef CONFIG_XFS_RT int xrep_rtbitmap(struct xfs_scrub *sc); @@ -205,6 +207,7 @@ xrep_setup_nothing( #define xrep_setup_directory xrep_setup_nothing #define xrep_setup_parent xrep_setup_nothing #define xrep_setup_nlinks xrep_setup_nothing +#define xrep_setup_dirtree xrep_setup_nothing #define xrep_setup_inode(sc, imap) ((void)0) @@ -239,6 +242,7 @@ static inline int xrep_setup_symlink(struct xfs_scrub *sc, unsigned int *x) #define xrep_directory xrep_notsupported #define xrep_parent xrep_notsupported #define xrep_symlink xrep_notsupported +#define xrep_dirtree xrep_notsupported #endif /* CONFIG_XFS_ONLINE_REPAIR */ diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 8f1431db77395..e813b66b603a1 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -441,7 +441,7 @@ static const struct xchk_meta_ops meta_scrub_ops[] = { .setup = xchk_setup_dirtree, .scrub = xchk_dirtree, .has = xfs_has_parent, - .repair = xrep_notsupported, + .repair = xrep_dirtree, }, }; diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 509b6f4fd0cd3..b3756722bee1d 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -1685,6 +1685,10 @@ TRACE_DEFINE_ENUM(XCHK_DIRPATH_CORRUPT); TRACE_DEFINE_ENUM(XCHK_DIRPATH_LOOP); TRACE_DEFINE_ENUM(XCHK_DIRPATH_STALE); TRACE_DEFINE_ENUM(XCHK_DIRPATH_OK); +TRACE_DEFINE_ENUM(XREP_DIRPATH_DELETING); +TRACE_DEFINE_ENUM(XREP_DIRPATH_DELETED); +TRACE_DEFINE_ENUM(XREP_DIRPATH_ADOPTING); +TRACE_DEFINE_ENUM(XREP_DIRPATH_ADOPTED); #define XCHK_DIRPATH_OUTCOME_STRINGS \ { XCHK_DIRPATH_SCANNING, "scanning" }, \ @@ -1692,7 +1696,11 @@ TRACE_DEFINE_ENUM(XCHK_DIRPATH_OK); { XCHK_DIRPATH_CORRUPT, "corrupt" }, \ { XCHK_DIRPATH_LOOP, "loop" }, \ { XCHK_DIRPATH_STALE, "stale" }, \ - { XCHK_DIRPATH_OK, "ok" } + { XCHK_DIRPATH_OK, "ok" }, \ + { XREP_DIRPATH_DELETING, "deleting" }, \ + { XREP_DIRPATH_DELETED, "deleted" }, \ + { XREP_DIRPATH_ADOPTING, "adopting" }, \ + { XREP_DIRPATH_ADOPTED, "adopted" } DECLARE_EVENT_CLASS(xchk_dirpath_outcome_class, TP_PROTO(struct xfs_scrub *sc, unsigned long long path_nr, @@ -1738,6 +1746,7 @@ DECLARE_EVENT_CLASS(xchk_dirtree_evaluate_class, __field(unsigned int, bad) __field(unsigned int, suspect) __field(unsigned int, good) + __field(bool, needs_adoption) ), TP_fast_assign( __entry->dev = dl->sc->mp->m_super->s_dev; @@ -1747,15 +1756,17 @@ DECLARE_EVENT_CLASS(xchk_dirtree_evaluate_class, __entry->bad = oc->bad; __entry->suspect = oc->suspect; __entry->good = oc->good; + __entry->needs_adoption = oc->needs_adoption ? 1 : 0; ), - TP_printk("dev %d:%d ino 0x%llx rootino 0x%llx nr_paths %u bad %u suspect %u good %u", + TP_printk("dev %d:%d ino 0x%llx rootino 0x%llx nr_paths %u bad %u suspect %u good %u adopt? %d", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, __entry->rootino, __entry->nr_paths, __entry->bad, __entry->suspect, - __entry->good) + __entry->good, + __entry->needs_adoption) ); #define DEFINE_XCHK_DIRTREE_EVALUATE_EVENT(name) \ DEFINE_EVENT(xchk_dirtree_evaluate_class, name, \ @@ -3181,6 +3192,7 @@ DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_check_child); DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_check_alias); DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_check_dentry); DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_invalidate_child); +DEFINE_REPAIR_DENTRY_EVENT(xrep_dirtree_delete_child); TRACE_EVENT(xrep_symlink_salvage_target, TP_PROTO(struct xfs_inode *ip, char *target, unsigned int targetlen), @@ -3483,6 +3495,11 @@ TRACE_EVENT(xrep_iunlink_commit_bucket, __entry->agino) ); +DEFINE_XCHK_DIRPATH_OUTCOME_EVENT(xrep_dirpath_set_outcome); +DEFINE_XCHK_DIRTREE_EVENT(xrep_dirtree_delete_path); +DEFINE_XCHK_DIRTREE_EVENT(xrep_dirtree_create_adoption); +DEFINE_XCHK_DIRTREE_EVALUATE_EVENT(xrep_dirtree_decided_fate); + #endif /* IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) */ #endif /* _TRACE_XFS_SCRUB_TRACE_H */ diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index ebe2ce9bd9eee..58fb7a5062e1e 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -893,7 +893,7 @@ xfs_init_new_inode( * link count to go to zero, move the inode to AGI unlinked list so that it can * be freed when the last active reference goes away via xfs_inactive(). */ -static int /* error */ +int xfs_droplink( struct xfs_trans *tp, struct xfs_inode *ip) diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index 04a91e312993b..9fd4d29a57137 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -626,6 +626,7 @@ void xfs_end_io(struct work_struct *work); int xfs_ilock2_io_mmap(struct xfs_inode *ip1, struct xfs_inode *ip2); void xfs_iunlock2_io_mmap(struct xfs_inode *ip1, struct xfs_inode *ip2); void xfs_iunlock2_remapping(struct xfs_inode *ip1, struct xfs_inode *ip2); +int xfs_droplink(struct xfs_trans *tp, struct xfs_inode *ip); void xfs_bumplink(struct xfs_trans *tp, struct xfs_inode *ip); void xfs_lock_inodes(struct xfs_inode **ips, int inodes, uint lock_mode); void xfs_sort_inodes(struct xfs_inode **i_tab, unsigned int num_inodes); From b27ce0da60a523fc32e3795f96b2de5490642235 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:26 -0700 Subject: [PATCH 0492/1702] xfs: use dontcache for grabbing inodes during scrub Back when I wrote commit a03297a0ca9f2, I had thought that we'd be doing users a favor by only marking inodes dontcache at the end of a scrub operation, and only if there's only one reference to that inode. This was more or less true back when I_DONTCACHE was an XFS iflag and the only thing it did was change the outcome of xfs_fs_drop_inode to 1. Note: If there are dentries pointing to the inode when scrub finishes, the inode will have positive i_count and stay around in cache until dentry reclaim. But now we have d_mark_dontcache, which cause the inode *and* the dentries attached to it all to be marked I_DONTCACHE, which means that we drop the dentries ASAP, which drops the inode ASAP. This is bad if scrub found problems with the inode, because now they can be scheduled for inactivation, which can cause inodegc to trip on it and shut down the filesystem. Even if the inode isn't bad, this is still suboptimal because phases 3-7 each initiate inode scans. Dropping the inode immediately during phase 3 is silly because phase 5 will reload it and drop it immediately, etc. It's fine to mark the inodes dontcache, but if there have been accesses to the file that set up dentries, we should keep them. I validated this by setting up ftrace to capture xfs_iget_recycle* tracepoints and ran xfs/285 for 30 seconds. With current djwong-wtf I saw ~30,000 recycle events. I then dropped the d_mark_dontcache calls and set XFS_IGET_DONTCACHE, and the recycle events dropped to ~5,000 per 30 seconds. Therefore, grab the inode with XFS_IGET_DONTCACHE, which only has the effect of setting I_DONTCACHE for cache misses. Remove the d_mark_dontcache call that can happen in xchk_irele. Fixes: a03297a0ca9f2 ("xfs: manage inode DONTCACHE status at irele time") Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/common.c | 12 +++--------- fs/xfs/scrub/iscan.c | 13 +++++++++++-- fs/xfs/scrub/scrub.h | 7 +++++++ 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/fs/xfs/scrub/common.c b/fs/xfs/scrub/common.c index a7d3a22796620..1ad8ec63a7f44 100644 --- a/fs/xfs/scrub/common.c +++ b/fs/xfs/scrub/common.c @@ -783,7 +783,7 @@ xchk_iget( { ASSERT(sc->tp != NULL); - return xfs_iget(sc->mp, sc->tp, inum, XFS_IGET_UNTRUSTED, 0, ipp); + return xfs_iget(sc->mp, sc->tp, inum, XCHK_IGET_FLAGS, 0, ipp); } /* @@ -834,8 +834,8 @@ xchk_iget_agi( if (error) return error; - error = xfs_iget(mp, tp, inum, - XFS_IGET_NORETRY | XFS_IGET_UNTRUSTED, 0, ipp); + error = xfs_iget(mp, tp, inum, XFS_IGET_NORETRY | XCHK_IGET_FLAGS, 0, + ipp); if (error == -EAGAIN) { /* * The inode may be in core but temporarily unavailable and may @@ -1062,12 +1062,6 @@ xchk_irele( spin_lock(&VFS_I(ip)->i_lock); VFS_I(ip)->i_state &= ~I_DONTCACHE; spin_unlock(&VFS_I(ip)->i_lock); - } else if (atomic_read(&VFS_I(ip)->i_count) == 1) { - /* - * If this is the last reference to the inode and the caller - * permits it, set DONTCACHE to avoid thrashing. - */ - d_mark_dontcache(VFS_I(ip)); } xfs_irele(ip); diff --git a/fs/xfs/scrub/iscan.c b/fs/xfs/scrub/iscan.c index c380207702e26..cf9d983667cec 100644 --- a/fs/xfs/scrub/iscan.c +++ b/fs/xfs/scrub/iscan.c @@ -407,6 +407,15 @@ xchk_iscan_iget_retry( return -EAGAIN; } +/* + * For an inode scan, we hold the AGI and want to try to grab a batch of + * inodes. Holding the AGI prevents inodegc from clearing freed inodes, + * so we must use noretry here. For every inode after the first one in the + * batch, we don't want to wait, so we use retry there too. Finally, use + * dontcache to avoid polluting the cache. + */ +#define ISCAN_IGET_FLAGS (XFS_IGET_NORETRY | XFS_IGET_DONTCACHE) + /* * Grab an inode as part of an inode scan. While scanning this inode, the * caller must ensure that no other threads can modify the inode until a call @@ -434,7 +443,7 @@ xchk_iscan_iget( ASSERT(iscan->__inodes[0] == NULL); /* Fill the first slot in the inode array. */ - error = xfs_iget(sc->mp, sc->tp, ino, XFS_IGET_NORETRY, 0, + error = xfs_iget(sc->mp, sc->tp, ino, ISCAN_IGET_FLAGS, 0, &iscan->__inodes[idx]); trace_xchk_iscan_iget(iscan, error); @@ -507,7 +516,7 @@ xchk_iscan_iget( ASSERT(iscan->__inodes[idx] == NULL); - error = xfs_iget(sc->mp, sc->tp, ino, XFS_IGET_NORETRY, 0, + error = xfs_iget(sc->mp, sc->tp, ino, ISCAN_IGET_FLAGS, 0, &iscan->__inodes[idx]); if (error) break; diff --git a/fs/xfs/scrub/scrub.h b/fs/xfs/scrub/scrub.h index 4e7e3edb6350c..1da10182f7f42 100644 --- a/fs/xfs/scrub/scrub.h +++ b/fs/xfs/scrub/scrub.h @@ -60,6 +60,13 @@ static inline int xchk_maybe_relax(struct xchk_relax *widget) #define XCHK_GFP_FLAGS ((__force gfp_t)(GFP_KERNEL | __GFP_NOWARN | \ __GFP_RETRY_MAYFAIL)) +/* + * For opening files by handle for fsck operations, we don't trust the inumber + * or the allocation state; therefore, perform an untrusted lookup. We don't + * want these inodes to pollute the cache, so mark them for immediate removal. + */ +#define XCHK_IGET_FLAGS (XFS_IGET_UNTRUSTED | XFS_IGET_DONTCACHE) + /* Type info and names for the scrub types. */ enum xchk_type { ST_NONE = 1, /* disabled */ From 66917537522312a6e462787f0f347d6998cf7038 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:27 -0700 Subject: [PATCH 0493/1702] xfs: drop the scrub file's iolock when transaction allocation fails If the transaction allocation in the !orphanage_available case of xrep_nlinks_repair_inode fails, we need to drop the IOLOCK of the file being scrubbed before exiting. Found by fuzzing u3.sfdir3.list[1].name = zeroes in xfs/1546. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/nlinks_repair.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/xfs/scrub/nlinks_repair.c b/fs/xfs/scrub/nlinks_repair.c index 78d0f650fe897..b3e707f47b7b5 100644 --- a/fs/xfs/scrub/nlinks_repair.c +++ b/fs/xfs/scrub/nlinks_repair.c @@ -138,8 +138,10 @@ xrep_nlinks_repair_inode( error = xfs_trans_alloc(mp, &M_RES(mp)->tr_link, 0, 0, 0, &sc->tp); - if (error) + if (error) { + xchk_iunlock(sc, XFS_IOLOCK_EXCL); return error; + } xchk_ilock(sc, XFS_ILOCK_EXCL); xfs_trans_ijoin(sc->tp, ip, 0); From c77b37584c2d1054452853e47e42c7350b8fe687 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:25 -0700 Subject: [PATCH 0494/1702] xfs: introduce vectored scrub mode Introduce a variant on XFS_SCRUB_METADATA that allows for a vectored mode. The caller specifies the principal metadata object that they want to scrub (allocation group, inode, etc.) once, followed by an array of scrub types they want called on that object. The kernel runs the scrub operations and writes the output flags and errno code to the corresponding array element. A new pseudo scrub type BARRIER is introduced to force the kernel to return to userspace if any corruptions have been found when scrubbing the previous scrub types in the array. This enables userspace to schedule, for example, the sequence: 1. data fork 2. barrier 3. directory If the data fork scrub is clean, then the kernel will perform the directory scrub. If not, the barrier in 2 will exit back to userspace. The alternative would have been an interface where userspace passes a pointer to an empty buffer, and the kernel formats that with xfs_scrub_vecs that tell userspace what it scrubbed and what the outcome was. With that the kernel would have to communicate that the buffer needed to have been at least X size, even though for our cases XFS_SCRUB_TYPE_NR + 2 would always be enough. Compared to that, this design keeps all the dependency policy and ordering logic in userspace where it already resides instead of duplicating it in the kernel. The downside of that is that it needs the barrier logic. When running fstests in "rebuild all metadata after each test" mode, I observed a 10% reduction in runtime due to fewer transitions across the system call boundary. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_fs.h | 33 +++++++++ fs/xfs/scrub/scrub.c | 149 +++++++++++++++++++++++++++++++++++++++ fs/xfs/scrub/trace.h | 79 ++++++++++++++++++++- fs/xfs/scrub/xfs_scrub.h | 2 + fs/xfs/xfs_ioctl.c | 2 + 5 files changed, 264 insertions(+), 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_fs.h b/fs/xfs/libxfs/xfs_fs.h index 7ae1912cdbfc2..97996cb79aaaa 100644 --- a/fs/xfs/libxfs/xfs_fs.h +++ b/fs/xfs/libxfs/xfs_fs.h @@ -725,6 +725,15 @@ struct xfs_scrub_metadata { /* Number of scrub subcommands. */ #define XFS_SCRUB_TYPE_NR 29 +/* + * This special type code only applies to the vectored scrub implementation. + * + * If any of the previous scrub vectors recorded runtime errors or have + * sv_flags bits set that match the OFLAG bits in the barrier vector's + * sv_flags, set the barrier's sv_ret to -ECANCELED and return to userspace. + */ +#define XFS_SCRUB_TYPE_BARRIER (0xFFFFFFFF) + /* i: Repair this metadata. */ #define XFS_SCRUB_IFLAG_REPAIR (1u << 0) @@ -769,6 +778,29 @@ struct xfs_scrub_metadata { XFS_SCRUB_OFLAG_NO_REPAIR_NEEDED) #define XFS_SCRUB_FLAGS_ALL (XFS_SCRUB_FLAGS_IN | XFS_SCRUB_FLAGS_OUT) +/* Vectored scrub calls to reduce the number of kernel transitions. */ + +struct xfs_scrub_vec { + __u32 sv_type; /* XFS_SCRUB_TYPE_* */ + __u32 sv_flags; /* XFS_SCRUB_FLAGS_* */ + __s32 sv_ret; /* 0 or a negative error code */ + __u32 sv_reserved; /* must be zero */ +}; + +/* Vectored metadata scrub control structure. */ +struct xfs_scrub_vec_head { + __u64 svh_ino; /* inode number. */ + __u32 svh_gen; /* inode generation. */ + __u32 svh_agno; /* ag number. */ + __u32 svh_flags; /* XFS_SCRUB_VEC_FLAGS_* */ + __u16 svh_rest_us; /* wait this much time between vector items */ + __u16 svh_nr; /* number of svh_vectors */ + __u64 svh_reserved; /* must be zero */ + __u64 svh_vectors; /* pointer to buffer of xfs_scrub_vec */ +}; + +#define XFS_SCRUB_VEC_FLAGS_ALL (0) + /* * ioctl limits */ @@ -928,6 +960,7 @@ struct xfs_getparents_by_handle { #define XFS_IOC_AG_GEOMETRY _IOWR('X', 61, struct xfs_ag_geometry) #define XFS_IOC_GETPARENTS _IOWR('X', 62, struct xfs_getparents) #define XFS_IOC_GETPARENTS_BY_HANDLE _IOWR('X', 63, struct xfs_getparents_by_handle) +#define XFS_IOC_SCRUBV_METADATA _IOWR('X', 64, struct xfs_scrub_vec_head) /* * ioctl commands that replace IRIX syssgi()'s diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 1456cc11c406d..78b00ab85c9c9 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -21,6 +21,7 @@ #include "xfs_exchmaps.h" #include "xfs_dir2.h" #include "xfs_parent.h" +#include "xfs_icache.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/trace.h" @@ -749,3 +750,151 @@ xfs_ioc_scrub_metadata( return 0; } + +/* Decide if there have been any scrub failures up to this point. */ +static inline int +xfs_scrubv_check_barrier( + struct xfs_mount *mp, + const struct xfs_scrub_vec *vectors, + const struct xfs_scrub_vec *stop_vec) +{ + const struct xfs_scrub_vec *v; + __u32 failmask; + + failmask = stop_vec->sv_flags & XFS_SCRUB_FLAGS_OUT; + + for (v = vectors; v < stop_vec; v++) { + if (v->sv_type == XFS_SCRUB_TYPE_BARRIER) + continue; + + /* + * Runtime errors count as a previous failure, except the ones + * used to ask userspace to retry. + */ + switch (v->sv_ret) { + case -EBUSY: + case -ENOENT: + case -EUSERS: + case 0: + break; + default: + return -ECANCELED; + } + + /* + * If any of the out-flags on the scrub vector match the mask + * that was set on the barrier vector, that's a previous fail. + */ + if (v->sv_flags & failmask) + return -ECANCELED; + } + + return 0; +} + +/* Vectored scrub implementation to reduce ioctl calls. */ +int +xfs_ioc_scrubv_metadata( + struct file *file, + void __user *arg) +{ + struct xfs_scrub_vec_head head; + struct xfs_scrub_vec_head __user *uhead = arg; + struct xfs_scrub_vec *vectors; + struct xfs_scrub_vec __user *uvectors; + struct xfs_inode *ip_in = XFS_I(file_inode(file)); + struct xfs_mount *mp = ip_in->i_mount; + struct xfs_scrub_vec *v; + size_t vec_bytes; + unsigned int i; + int error = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (copy_from_user(&head, uhead, sizeof(head))) + return -EFAULT; + + if (head.svh_reserved) + return -EINVAL; + if (head.svh_flags & ~XFS_SCRUB_VEC_FLAGS_ALL) + return -EINVAL; + if (head.svh_nr == 0) + return 0; + + vec_bytes = array_size(head.svh_nr, sizeof(struct xfs_scrub_vec)); + if (vec_bytes > PAGE_SIZE) + return -ENOMEM; + + uvectors = (void __user *)(uintptr_t)head.svh_vectors; + vectors = memdup_user(uvectors, vec_bytes); + if (IS_ERR(vectors)) + return PTR_ERR(vectors); + + trace_xchk_scrubv_start(ip_in, &head); + + for (i = 0, v = vectors; i < head.svh_nr; i++, v++) { + if (v->sv_reserved) { + error = -EINVAL; + goto out_free; + } + + if (v->sv_type == XFS_SCRUB_TYPE_BARRIER && + (v->sv_flags & ~XFS_SCRUB_FLAGS_OUT)) { + error = -EINVAL; + goto out_free; + } + + trace_xchk_scrubv_item(mp, &head, i, v); + } + + /* Run all the scrubbers. */ + for (i = 0, v = vectors; i < head.svh_nr; i++, v++) { + struct xfs_scrub_metadata sm = { + .sm_type = v->sv_type, + .sm_flags = v->sv_flags, + .sm_ino = head.svh_ino, + .sm_gen = head.svh_gen, + .sm_agno = head.svh_agno, + }; + + if (v->sv_type == XFS_SCRUB_TYPE_BARRIER) { + v->sv_ret = xfs_scrubv_check_barrier(mp, vectors, v); + if (v->sv_ret) { + trace_xchk_scrubv_barrier_fail(mp, &head, i, v); + break; + } + + continue; + } + + v->sv_ret = xfs_scrub_metadata(file, &sm); + v->sv_flags = sm.sm_flags; + + trace_xchk_scrubv_outcome(mp, &head, i, v); + + if (head.svh_rest_us) { + ktime_t expires; + + expires = ktime_add_ns(ktime_get(), + head.svh_rest_us * 1000); + set_current_state(TASK_KILLABLE); + schedule_hrtimeout(&expires, HRTIMER_MODE_ABS); + } + + if (fatal_signal_pending(current)) { + error = -EINTR; + goto out_free; + } + } + + if (copy_to_user(uvectors, vectors, vec_bytes) || + copy_to_user(uhead, &head, sizeof(head))) { + error = -EFAULT; + goto out_free; + } + +out_free: + kfree(vectors); + return error; +} diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index b3756722bee1d..8ce74bd8530a6 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -69,6 +69,7 @@ TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_QUOTACHECK); TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_NLINKS); TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_HEALTHY); TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_DIRTREE); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_BARRIER); #define XFS_SCRUB_TYPE_STRINGS \ { XFS_SCRUB_TYPE_PROBE, "probe" }, \ @@ -99,7 +100,8 @@ TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_DIRTREE); { XFS_SCRUB_TYPE_QUOTACHECK, "quotacheck" }, \ { XFS_SCRUB_TYPE_NLINKS, "nlinks" }, \ { XFS_SCRUB_TYPE_HEALTHY, "healthy" }, \ - { XFS_SCRUB_TYPE_DIRTREE, "dirtree" } + { XFS_SCRUB_TYPE_DIRTREE, "dirtree" }, \ + { XFS_SCRUB_TYPE_BARRIER, "barrier" } #define XFS_SCRUB_FLAG_STRINGS \ { XFS_SCRUB_IFLAG_REPAIR, "repair" }, \ @@ -208,6 +210,81 @@ DEFINE_EVENT(xchk_fsgate_class, name, \ DEFINE_SCRUB_FSHOOK_EVENT(xchk_fsgates_enable); DEFINE_SCRUB_FSHOOK_EVENT(xchk_fsgates_disable); +DECLARE_EVENT_CLASS(xchk_vector_head_class, + TP_PROTO(struct xfs_inode *ip, struct xfs_scrub_vec_head *vhead), + TP_ARGS(ip, vhead), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_agnumber_t, agno) + __field(xfs_ino_t, inum) + __field(unsigned int, gen) + __field(unsigned int, flags) + __field(unsigned short, rest_us) + __field(unsigned short, nr_vecs) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = ip->i_ino; + __entry->agno = vhead->svh_agno; + __entry->inum = vhead->svh_ino; + __entry->gen = vhead->svh_gen; + __entry->flags = vhead->svh_flags; + __entry->rest_us = vhead->svh_rest_us; + __entry->nr_vecs = vhead->svh_nr; + ), + TP_printk("dev %d:%d ino 0x%llx agno 0x%x inum 0x%llx gen 0x%x flags 0x%x rest_us %u nr_vecs %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->agno, + __entry->inum, + __entry->gen, + __entry->flags, + __entry->rest_us, + __entry->nr_vecs) +) +#define DEFINE_SCRUBV_HEAD_EVENT(name) \ +DEFINE_EVENT(xchk_vector_head_class, name, \ + TP_PROTO(struct xfs_inode *ip, struct xfs_scrub_vec_head *vhead), \ + TP_ARGS(ip, vhead)) + +DEFINE_SCRUBV_HEAD_EVENT(xchk_scrubv_start); + +DECLARE_EVENT_CLASS(xchk_vector_class, + TP_PROTO(struct xfs_mount *mp, struct xfs_scrub_vec_head *vhead, + unsigned int vec_nr, struct xfs_scrub_vec *v), + TP_ARGS(mp, vhead, vec_nr, v), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, vec_nr) + __field(unsigned int, vec_type) + __field(unsigned int, vec_flags) + __field(int, vec_ret) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->vec_nr = vec_nr; + __entry->vec_type = v->sv_type; + __entry->vec_flags = v->sv_flags; + __entry->vec_ret = v->sv_ret; + ), + TP_printk("dev %d:%d vec[%u] type %s flags %s ret %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->vec_nr, + __print_symbolic(__entry->vec_type, XFS_SCRUB_TYPE_STRINGS), + __print_flags(__entry->vec_flags, "|", XFS_SCRUB_FLAG_STRINGS), + __entry->vec_ret) +) +#define DEFINE_SCRUBV_EVENT(name) \ +DEFINE_EVENT(xchk_vector_class, name, \ + TP_PROTO(struct xfs_mount *mp, struct xfs_scrub_vec_head *vhead, \ + unsigned int vec_nr, struct xfs_scrub_vec *v), \ + TP_ARGS(mp, vhead, vec_nr, v)) + +DEFINE_SCRUBV_EVENT(xchk_scrubv_barrier_fail); +DEFINE_SCRUBV_EVENT(xchk_scrubv_item); +DEFINE_SCRUBV_EVENT(xchk_scrubv_outcome); + TRACE_EVENT(xchk_op_error, TP_PROTO(struct xfs_scrub *sc, xfs_agnumber_t agno, xfs_agblock_t bno, int error, void *ret_ip), diff --git a/fs/xfs/scrub/xfs_scrub.h b/fs/xfs/scrub/xfs_scrub.h index 02c930f175d0b..f17173b83e6f3 100644 --- a/fs/xfs/scrub/xfs_scrub.h +++ b/fs/xfs/scrub/xfs_scrub.h @@ -8,8 +8,10 @@ #ifndef CONFIG_XFS_ONLINE_SCRUB # define xfs_ioc_scrub_metadata(f, a) (-ENOTTY) +# define xfs_ioc_scrubv_metadata(f, a) (-ENOTTY) #else int xfs_ioc_scrub_metadata(struct file *file, void __user *arg); +int xfs_ioc_scrubv_metadata(struct file *file, void __user *arg); #endif /* CONFIG_XFS_ONLINE_SCRUB */ #endif /* __XFS_SCRUB_H__ */ diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 857b19428984a..f0117188f3021 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -1413,6 +1413,8 @@ xfs_file_ioctl( case FS_IOC_GETFSMAP: return xfs_ioc_getfsmap(ip, arg); + case XFS_IOC_SCRUBV_METADATA: + return xfs_ioc_scrubv_metadata(filp, arg); case XFS_IOC_SCRUB_METADATA: return xfs_ioc_scrub_metadata(filp, arg); From 4ad350ac58627bfe81f71f43f6738e36b4eb75c6 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:26 -0700 Subject: [PATCH 0495/1702] xfs: only iget the file once when doing vectored scrub-by-handle If a program wants us to perform a scrub on a file handle and the fd passed to ioctl() is not the file referenced in the handle, iget the file once and pass it into the scrub code. This amortizes the untrusted iget lookup over /all/ the scrubbers mentioned in the scrubv call. When running fstests in "rebuild all metadata after each test" mode, I observed a 10% reduction in runtime on account of avoiding repeated inobt lookups. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/scrub.c | 45 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 78b00ab85c9c9..43af5ce1f99f0 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -792,6 +792,37 @@ xfs_scrubv_check_barrier( return 0; } +/* + * If the caller provided us with a nonzero inode number that isn't the ioctl + * file, try to grab a reference to it to eliminate all further untrusted inode + * lookups. If we can't get the inode, let each scrub function try again. + */ +STATIC struct xfs_inode * +xchk_scrubv_open_by_handle( + struct xfs_mount *mp, + const struct xfs_scrub_vec_head *head) +{ + struct xfs_trans *tp; + struct xfs_inode *ip; + int error; + + error = xfs_trans_alloc_empty(mp, &tp); + if (error) + return NULL; + + error = xfs_iget(mp, tp, head->svh_ino, XCHK_IGET_FLAGS, 0, &ip); + xfs_trans_cancel(tp); + if (error) + return NULL; + + if (VFS_I(ip)->i_generation != head->svh_gen) { + xfs_irele(ip); + return NULL; + } + + return ip; +} + /* Vectored scrub implementation to reduce ioctl calls. */ int xfs_ioc_scrubv_metadata( @@ -804,6 +835,7 @@ xfs_ioc_scrubv_metadata( struct xfs_scrub_vec __user *uvectors; struct xfs_inode *ip_in = XFS_I(file_inode(file)); struct xfs_mount *mp = ip_in->i_mount; + struct xfs_inode *handle_ip = NULL; struct xfs_scrub_vec *v; size_t vec_bytes; unsigned int i; @@ -848,6 +880,17 @@ xfs_ioc_scrubv_metadata( trace_xchk_scrubv_item(mp, &head, i, v); } + /* + * If the caller wants us to do a scrub-by-handle and the file used to + * call the ioctl is not the same file, load the incore inode and pin + * it across all the scrubv actions to avoid repeated UNTRUSTED + * lookups. The reference is not passed to deeper layers of scrub + * because each scrubber gets to decide its own strategy and return + * values for getting an inode. + */ + if (head.svh_ino && head.svh_ino != ip_in->i_ino) + handle_ip = xchk_scrubv_open_by_handle(mp, &head); + /* Run all the scrubbers. */ for (i = 0, v = vectors; i < head.svh_nr; i++, v++) { struct xfs_scrub_metadata sm = { @@ -895,6 +938,8 @@ xfs_ioc_scrubv_metadata( } out_free: + if (handle_ip) + xfs_irele(handle_ip); kfree(vectors); return error; } From b44bfc06958f49ccb611632a4fb7c7df4fdcbc06 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:28 -0700 Subject: [PATCH 0496/1702] xfs: fix iunlock calls in xrep_adoption_trans_alloc If the transaction allocation in xrep_adoption_trans_alloc fails, we should drop only the locks that we took. In this case this is ILOCK_EXCL of both the orphanage and the file being repaired. Dropping any IOLOCK here is incorrect. Found by fuzzing u3.sfdir3.list[1].name = zeroes in xfs/1546. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/orphanage.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c index b1c6c60ee1da6..2b142e6de8f3f 100644 --- a/fs/xfs/scrub/orphanage.c +++ b/fs/xfs/scrub/orphanage.c @@ -382,7 +382,7 @@ xrep_adoption_trans_alloc( out_cancel: xchk_trans_cancel(sc); xrep_orphanage_iunlock(sc, XFS_ILOCK_EXCL); - xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + xchk_iunlock(sc, XFS_ILOCK_EXCL); return error; } From 6d335233fe6952189c949d65ab16d92afb0b8fb4 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:29 -0700 Subject: [PATCH 0497/1702] xfs: exchange-range for repairs is no longer dynamic The atomic file exchange-range functionality is now a permanent filesystem feature instead of a dynamic log-incompat feature. It cannot be turned on at runtime, so we no longer need the XCHK_FSGATES flags and whatnot that supported it. Remove the flag and the enable function, and move the xfs_has_exchange_range checks to the start of the repair functions. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/attr_repair.c | 3 +++ fs/xfs/scrub/dir_repair.c | 3 +++ fs/xfs/scrub/parent_repair.c | 10 +++++++--- fs/xfs/scrub/rtsummary_repair.c | 10 ++++------ fs/xfs/scrub/scrub.c | 8 +++----- fs/xfs/scrub/scrub.h | 7 ------- fs/xfs/scrub/symlink_repair.c | 3 +++ fs/xfs/scrub/tempexch.h | 1 - fs/xfs/scrub/tempfile.c | 24 ++---------------------- fs/xfs/scrub/trace.h | 1 - 10 files changed, 25 insertions(+), 45 deletions(-) diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c index e059813b92b70..c7eb94069cafc 100644 --- a/fs/xfs/scrub/attr_repair.c +++ b/fs/xfs/scrub/attr_repair.c @@ -1630,6 +1630,9 @@ xrep_xattr( /* The rmapbt is required to reap the old attr fork. */ if (!xfs_has_rmapbt(sc->mp)) return -EOPNOTSUPP; + /* We require atomic file exchange range to rebuild anything. */ + if (!xfs_has_exchange_range(sc->mp)) + return -EOPNOTSUPP; error = xrep_xattr_setup_scan(sc, &rx); if (error) diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index e968150fe0f06..6ad40f8aafb82 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -1993,6 +1993,9 @@ xrep_directory( /* The rmapbt is required to reap the old data fork. */ if (!xfs_has_rmapbt(sc->mp)) return -EOPNOTSUPP; + /* We require atomic file exchange range to rebuild anything. */ + if (!xfs_has_exchange_range(sc->mp)) + return -EOPNOTSUPP; error = xrep_dir_setup_scan(rd); if (error) diff --git a/fs/xfs/scrub/parent_repair.c b/fs/xfs/scrub/parent_repair.c index ee88ce5a12b83..7b42b7f65a0bd 100644 --- a/fs/xfs/scrub/parent_repair.c +++ b/fs/xfs/scrub/parent_repair.c @@ -1571,10 +1571,14 @@ xrep_parent( /* * When the parent pointers feature is enabled, repairs are committed * by atomically committing a new xattr structure and reaping the old - * attr fork. Reaping requires rmap to be enabled. + * attr fork. Reaping requires rmap and exchange-range to be enabled. */ - if (xfs_has_parent(sc->mp) && !xfs_has_rmapbt(sc->mp)) - return -EOPNOTSUPP; + if (xfs_has_parent(sc->mp)) { + if (!xfs_has_rmapbt(sc->mp)) + return -EOPNOTSUPP; + if (!xfs_has_exchange_range(sc->mp)) + return -EOPNOTSUPP; + } error = xrep_parent_setup_scan(rp); if (error) diff --git a/fs/xfs/scrub/rtsummary_repair.c b/fs/xfs/scrub/rtsummary_repair.c index c8bb6c4f15d05..d9e971c4c79fb 100644 --- a/fs/xfs/scrub/rtsummary_repair.c +++ b/fs/xfs/scrub/rtsummary_repair.c @@ -62,12 +62,7 @@ xrep_setup_rtsummary( return -EOPNOTSUPP; rts->resblks += blocks; - - /* - * Grab support for atomic file content exchanges before we allocate - * any transactions or grab ILOCKs. - */ - return xrep_tempexch_enable(sc); + return 0; } static int @@ -111,6 +106,9 @@ xrep_rtsummary( /* We require the rmapbt to rebuild anything. */ if (!xfs_has_rmapbt(mp)) return -EOPNOTSUPP; + /* We require atomic file exchange range to rebuild anything. */ + if (!xfs_has_exchange_range(mp)) + return -EOPNOTSUPP; /* Walk away if we disagree on the size of the rt bitmap. */ if (rts->rbmblocks != mp->m_sb.sb_rbmblocks) diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 43af5ce1f99f0..c013f0ba4f36b 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -154,15 +154,14 @@ xchk_probe( /* Scrub setup and teardown */ -#define FSGATES_MASK (XCHK_FSGATES_ALL | XREP_FSGATES_ALL) static inline void xchk_fsgates_disable( struct xfs_scrub *sc) { - if (!(sc->flags & FSGATES_MASK)) + if (!(sc->flags & XCHK_FSGATES_ALL)) return; - trace_xchk_fsgates_disable(sc, sc->flags & FSGATES_MASK); + trace_xchk_fsgates_disable(sc, sc->flags & XCHK_FSGATES_ALL); if (sc->flags & XCHK_FSGATES_DRAIN) xfs_drain_wait_disable(); @@ -176,9 +175,8 @@ xchk_fsgates_disable( if (sc->flags & XCHK_FSGATES_RMAP) xfs_rmap_hook_disable(); - sc->flags &= ~FSGATES_MASK; + sc->flags &= ~XCHK_FSGATES_ALL; } -#undef FSGATES_MASK /* Free the resources associated with a scrub subtype. */ void diff --git a/fs/xfs/scrub/scrub.h b/fs/xfs/scrub/scrub.h index 1da10182f7f42..1bc33f010d0e7 100644 --- a/fs/xfs/scrub/scrub.h +++ b/fs/xfs/scrub/scrub.h @@ -188,7 +188,6 @@ struct xfs_scrub { #define XCHK_FSGATES_QUOTA (1U << 4) /* quota live update enabled */ #define XCHK_FSGATES_DIRENTS (1U << 5) /* directory live update enabled */ #define XCHK_FSGATES_RMAP (1U << 6) /* rmapbt live update enabled */ -#define XREP_FSGATES_EXCHANGE_RANGE (1U << 29) /* uses file content exchange */ #define XREP_RESET_PERAG_RESV (1U << 30) /* must reset AG space reservation */ #define XREP_ALREADY_FIXED (1U << 31) /* checking our repair work */ @@ -203,12 +202,6 @@ struct xfs_scrub { XCHK_FSGATES_DIRENTS | \ XCHK_FSGATES_RMAP) -/* - * The sole XREP_FSGATES* flag reflects a log intent item that is protected - * by a log-incompat feature flag. No code patching in use here. - */ -#define XREP_FSGATES_ALL (XREP_FSGATES_EXCHANGE_RANGE) - struct xfs_scrub_subord { struct xfs_scrub sc; struct xfs_scrub *parent_sc; diff --git a/fs/xfs/scrub/symlink_repair.c b/fs/xfs/scrub/symlink_repair.c index c8b5a5b878ac9..d015a86ef460f 100644 --- a/fs/xfs/scrub/symlink_repair.c +++ b/fs/xfs/scrub/symlink_repair.c @@ -490,6 +490,9 @@ xrep_symlink( /* The rmapbt is required to reap the old data fork. */ if (!xfs_has_rmapbt(sc->mp)) return -EOPNOTSUPP; + /* We require atomic file exchange range to rebuild anything. */ + if (!xfs_has_exchange_range(sc->mp)) + return -EOPNOTSUPP; ASSERT(sc->ilock_flags & XFS_ILOCK_EXCL); diff --git a/fs/xfs/scrub/tempexch.h b/fs/xfs/scrub/tempexch.h index c1dd4adec4f11..995ba187c5aa6 100644 --- a/fs/xfs/scrub/tempexch.h +++ b/fs/xfs/scrub/tempexch.h @@ -11,7 +11,6 @@ struct xrep_tempexch { struct xfs_exchmaps_req req; }; -int xrep_tempexch_enable(struct xfs_scrub *sc); int xrep_tempexch_trans_reserve(struct xfs_scrub *sc, int whichfork, struct xrep_tempexch *ti); int xrep_tempexch_trans_alloc(struct xfs_scrub *sc, int whichfork, diff --git a/fs/xfs/scrub/tempfile.c b/fs/xfs/scrub/tempfile.c index ddbcccb3dba13..b747b625c5ee4 100644 --- a/fs/xfs/scrub/tempfile.c +++ b/fs/xfs/scrub/tempfile.c @@ -486,23 +486,6 @@ xrep_tempfile_roll_trans( return 0; } -/* Enable file content exchanges. */ -int -xrep_tempexch_enable( - struct xfs_scrub *sc) -{ - if (sc->flags & XREP_FSGATES_EXCHANGE_RANGE) - return 0; - - if (!xfs_has_exchange_range(sc->mp)) - return -EOPNOTSUPP; - - trace_xchk_fsgates_enable(sc, XREP_FSGATES_EXCHANGE_RANGE); - - sc->flags |= XREP_FSGATES_EXCHANGE_RANGE; - return 0; -} - /* * Fill out the mapping exchange request in preparation for atomically * committing the contents of a metadata file that we've rebuilt in the temp @@ -745,6 +728,7 @@ xrep_tempexch_trans_alloc( int error; ASSERT(sc->tp == NULL); + ASSERT(xfs_has_exchange_range(sc->mp)); error = xrep_tempexch_prep_request(sc, whichfork, tx); if (error) @@ -757,10 +741,6 @@ xrep_tempexch_trans_alloc( if (xfs_has_lazysbcount(sc->mp)) flags |= XFS_TRANS_RES_FDBLKS; - error = xrep_tempexch_enable(sc); - if (error) - return error; - error = xfs_trans_alloc(sc->mp, &M_RES(sc->mp)->tr_itruncate, tx->req.resblks, 0, flags, &sc->tp); if (error) @@ -785,7 +765,7 @@ xrep_tempexch_contents( { int error; - ASSERT(sc->flags & XREP_FSGATES_EXCHANGE_RANGE); + ASSERT(xfs_has_exchange_range(sc->mp)); xfs_exchange_mappings(sc->tp, &tx->req); error = xfs_defer_finish(&sc->tp); diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 8ce74bd8530a6..4b945007ca6ca 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -122,7 +122,6 @@ TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_BARRIER); { XCHK_FSGATES_QUOTA, "fsgates_quota" }, \ { XCHK_FSGATES_DIRENTS, "fsgates_dirents" }, \ { XCHK_FSGATES_RMAP, "fsgates_rmap" }, \ - { XREP_FSGATES_EXCHANGE_RANGE, "fsgates_exchrange" }, \ { XREP_RESET_PERAG_RESV, "reset_perag_resv" }, \ { XREP_ALREADY_FIXED, "already_fixed" } From 5e1c7d0b29f7e05b01e448d2579a469cf3a0d350 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 22 Apr 2024 09:48:30 -0700 Subject: [PATCH 0498/1702] xfs: invalidate dentries for a file before moving it to the orphanage Invalidate the cached dentries that point to the file that we're moving to lost+found before we actually move it. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/scrub/orphanage.c | 47 +++++++++++++++++----------------------- fs/xfs/scrub/trace.h | 2 -- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c index 2b142e6de8f3f..7148d8362db83 100644 --- a/fs/xfs/scrub/orphanage.c +++ b/fs/xfs/scrub/orphanage.c @@ -434,16 +434,17 @@ xrep_adoption_check_dcache( { struct qstr qname = QSTR_INIT(adopt->xname->name, adopt->xname->len); + struct xfs_scrub *sc = adopt->sc; struct dentry *d_orphanage, *d_child; int error = 0; - d_orphanage = d_find_alias(VFS_I(adopt->sc->orphanage)); + d_orphanage = d_find_alias(VFS_I(sc->orphanage)); if (!d_orphanage) return 0; d_child = d_hash_and_lookup(d_orphanage, &qname); if (d_child) { - trace_xrep_adoption_check_child(adopt->sc->mp, d_child); + trace_xrep_adoption_check_child(sc->mp, d_child); if (d_is_positive(d_child)) { ASSERT(d_is_negative(d_child)); @@ -454,33 +455,15 @@ xrep_adoption_check_dcache( } dput(d_orphanage); - if (error) - return error; - - /* - * Do we need to update d_parent of the dentry for the file being - * repaired? There shouldn't be a hashed dentry with a parent since - * the file had nonzero nlink but wasn't connected to any parent dir. - */ - d_child = d_find_alias(VFS_I(adopt->sc->ip)); - if (!d_child) - return 0; - - trace_xrep_adoption_check_alias(adopt->sc->mp, d_child); - - if (d_child->d_parent && !d_unhashed(d_child)) { - ASSERT(d_child->d_parent == NULL || d_unhashed(d_child)); - error = -EFSCORRUPTED; - } - - dput(d_child); return error; } /* - * Remove all negative dentries from the dcache. There should not be any - * positive entries, since we've maintained our lock on the orphanage - * directory. + * Invalidate all dentries for the name that was added to the orphanage + * directory, and all dentries pointing to the child inode that was moved. + * + * There should not be any positive entries for the name, since we've + * maintained our lock on the orphanage directory. */ static void xrep_adoption_zap_dcache( @@ -488,15 +471,17 @@ xrep_adoption_zap_dcache( { struct qstr qname = QSTR_INIT(adopt->xname->name, adopt->xname->len); + struct xfs_scrub *sc = adopt->sc; struct dentry *d_orphanage, *d_child; - d_orphanage = d_find_alias(VFS_I(adopt->sc->orphanage)); + /* Invalidate all dentries for the adoption name */ + d_orphanage = d_find_alias(VFS_I(sc->orphanage)); if (!d_orphanage) return; d_child = d_hash_and_lookup(d_orphanage, &qname); while (d_child != NULL) { - trace_xrep_adoption_invalidate_child(adopt->sc->mp, d_child); + trace_xrep_adoption_invalidate_child(sc->mp, d_child); ASSERT(d_is_negative(d_child)); d_invalidate(d_child); @@ -505,6 +490,14 @@ xrep_adoption_zap_dcache( } dput(d_orphanage); + + /* Invalidate all the dentries pointing down to this file. */ + while ((d_child = d_find_alias(VFS_I(sc->ip))) != NULL) { + trace_xrep_adoption_invalidate_child(sc->mp, d_child); + + d_invalidate(d_child); + dput(d_child); + } } /* diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 4b945007ca6ca..e27daa51cab64 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -3265,8 +3265,6 @@ DEFINE_EVENT(xrep_dentry_class, name, \ TP_PROTO(struct xfs_mount *mp, const struct dentry *dentry), \ TP_ARGS(mp, dentry)) DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_check_child); -DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_check_alias); -DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_check_dentry); DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_invalidate_child); DEFINE_REPAIR_DENTRY_EVENT(xrep_dirtree_delete_child); From 08e012a62de877e77d7d44d5bddace63d760741b Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Wed, 24 Apr 2024 10:06:38 +0800 Subject: [PATCH 0499/1702] xfs: Remove unused function xrep_dir_self_parent The function are defined in the dir_repair.c file, but not called elsewhere, so delete the unused function. fs/xfs/scrub/dir_repair.c:186:1: warning: unused function 'xrep_dir_self_parent'. Reported-by: Abaci Robot Closes: https://bugzilla.openanolis.cn/show_bug.cgi?id=8867 Signed-off-by: Jiapeng Chong Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/scrub/dir_repair.c | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index 6ad40f8aafb82..fa1a9954d48d9 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -204,27 +204,6 @@ xrep_setup_directory( return 0; } -/* - * If we're the root of a directory tree, we are our own parent. If we're an - * unlinked directory, the parent /won't/ have a link to us. Set the parent - * directory to the root for both cases. Returns NULLFSINO if we don't know - * what to do. - */ -static inline xfs_ino_t -xrep_dir_self_parent( - struct xrep_dir *rd) -{ - struct xfs_scrub *sc = rd->sc; - - if (sc->ip->i_ino == sc->mp->m_sb.sb_rootino) - return sc->mp->m_sb.sb_rootino; - - if (VFS_I(sc->ip)->i_nlink == 0) - return sc->mp->m_sb.sb_rootino; - - return NULLFSINO; -} - /* * Look up the dotdot entry and confirm that it's really the parent. * Returns NULLFSINO if we don't know what to do. From 11a921909fea230cf7afcd6842a9452f3720b61b Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Tue, 27 Jun 2023 15:30:19 +0200 Subject: [PATCH 0500/1702] kernel misc: Remove the now superfluous sentinel elements from ctl_table array This commit comes at the tail end of a greater effort to remove the empty elements at the end of the ctl_table arrays (sentinels) which will reduce the overall build time size of the kernel and run time memory bloat by ~64 bytes per sentinel (further information Link : https://lore.kernel.org/all/ZO5Yx5JFogGi%2FcBo@bombadil.infradead.org/) Remove the sentinel from ctl_table arrays. Reduce by one the values used to compare the size of the adjusted arrays. Signed-off-by: Joel Granados --- kernel/acct.c | 1 - kernel/exit.c | 1 - kernel/hung_task.c | 1 - kernel/kexec_core.c | 1 - kernel/latencytop.c | 1 - kernel/panic.c | 1 - kernel/pid_namespace.c | 1 - kernel/pid_sysctl.h | 1 - kernel/reboot.c | 1 - kernel/signal.c | 1 - kernel/stackleak.c | 1 - kernel/sysctl.c | 2 -- kernel/ucount.c | 3 +-- kernel/utsname_sysctl.c | 1 - kernel/watchdog.c | 2 -- 15 files changed, 1 insertion(+), 18 deletions(-) diff --git a/kernel/acct.c b/kernel/acct.c index 986c8214dabf2..179848ad33e97 100644 --- a/kernel/acct.c +++ b/kernel/acct.c @@ -84,7 +84,6 @@ static struct ctl_table kern_acct_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, - { } }; static __init int kernel_acct_sysctls_init(void) diff --git a/kernel/exit.c b/kernel/exit.c index 41a12630cbbc9..cd3aa9042f1a1 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -94,7 +94,6 @@ static struct ctl_table kern_exit_table[] = { .mode = 0644, .proc_handler = proc_douintvec, }, - { } }; static __init int kernel_exit_sysctls_init(void) diff --git a/kernel/hung_task.c b/kernel/hung_task.c index b2fc2727d6544..1d92016b0b3c4 100644 --- a/kernel/hung_task.c +++ b/kernel/hung_task.c @@ -314,7 +314,6 @@ static struct ctl_table hung_task_sysctls[] = { .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_NEG_ONE, }, - {} }; static void __init hung_task_sysctl_init(void) diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 0e96f6b243447..9112d69d68b0b 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -948,7 +948,6 @@ static struct ctl_table kexec_core_sysctls[] = { .mode = 0644, .proc_handler = kexec_limit_handler, }, - { } }; static int __init kexec_core_sysctl_init(void) diff --git a/kernel/latencytop.c b/kernel/latencytop.c index 781249098cb6e..84c53285f4999 100644 --- a/kernel/latencytop.c +++ b/kernel/latencytop.c @@ -85,7 +85,6 @@ static struct ctl_table latencytop_sysctl[] = { .mode = 0644, .proc_handler = sysctl_latencytop, }, - {} }; #endif diff --git a/kernel/panic.c b/kernel/panic.c index 747c3f3d289a2..8bff183d6180e 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -100,7 +100,6 @@ static struct ctl_table kern_panic_table[] = { .mode = 0644, .proc_handler = proc_douintvec, }, - { } }; static __init int kernel_panic_sysctls_init(void) diff --git a/kernel/pid_namespace.c b/kernel/pid_namespace.c index 7ade20e952321..dc48fecfa1dce 100644 --- a/kernel/pid_namespace.c +++ b/kernel/pid_namespace.c @@ -307,7 +307,6 @@ static struct ctl_table pid_ns_ctl_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = &pid_max, }, - { } }; #endif /* CONFIG_CHECKPOINT_RESTORE */ diff --git a/kernel/pid_sysctl.h b/kernel/pid_sysctl.h index 2ee41a3a1dfde..fe9fb991dc422 100644 --- a/kernel/pid_sysctl.h +++ b/kernel/pid_sysctl.h @@ -41,7 +41,6 @@ static struct ctl_table pid_ns_ctl_table_vm[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_TWO, }, - { } }; static inline void register_pid_ns_sysctl_table_vm(void) { diff --git a/kernel/reboot.c b/kernel/reboot.c index 22c16e2564cca..f05dbde2c93fe 100644 --- a/kernel/reboot.c +++ b/kernel/reboot.c @@ -1295,7 +1295,6 @@ static struct ctl_table kern_reboot_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, - { } }; static void __init kernel_reboot_sysctls_init(void) diff --git a/kernel/signal.c b/kernel/signal.c index 7bdbcf1b78d0f..01c4c46a51a82 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -4840,7 +4840,6 @@ static struct ctl_table signal_debug_table[] = { .proc_handler = proc_dointvec }, #endif - { } }; static int __init init_signal_sysctls(void) diff --git a/kernel/stackleak.c b/kernel/stackleak.c index 34c9d81eea940..d099f3affcf1a 100644 --- a/kernel/stackleak.c +++ b/kernel/stackleak.c @@ -54,7 +54,6 @@ static struct ctl_table stackleak_sysctls[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, - {} }; static int __init stackleak_sysctls_init(void) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 81cc974913bb9..e0b917328cf99 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2034,7 +2034,6 @@ static struct ctl_table kern_table[] = { .extra2 = SYSCTL_INT_MAX, }, #endif - { } }; static struct ctl_table vm_table[] = { @@ -2240,7 +2239,6 @@ static struct ctl_table vm_table[] = { .extra2 = (void *)&mmap_rnd_compat_bits_max, }, #endif - { } }; int __init sysctl_init_bases(void) diff --git a/kernel/ucount.c b/kernel/ucount.c index 4aa6166cb8563..e196da0204dcb 100644 --- a/kernel/ucount.c +++ b/kernel/ucount.c @@ -87,7 +87,6 @@ static struct ctl_table user_table[] = { UCOUNT_ENTRY("max_fanotify_groups"), UCOUNT_ENTRY("max_fanotify_marks"), #endif - { } }; #endif /* CONFIG_SYSCTL */ @@ -96,7 +95,7 @@ bool setup_userns_sysctls(struct user_namespace *ns) #ifdef CONFIG_SYSCTL struct ctl_table *tbl; - BUILD_BUG_ON(ARRAY_SIZE(user_table) != UCOUNT_COUNTS + 1); + BUILD_BUG_ON(ARRAY_SIZE(user_table) != UCOUNT_COUNTS); setup_sysctl_set(&ns->set, &set_root, set_is_seen); tbl = kmemdup(user_table, sizeof(user_table), GFP_KERNEL); if (tbl) { diff --git a/kernel/utsname_sysctl.c b/kernel/utsname_sysctl.c index 019e3a1566cf5..76a772072557f 100644 --- a/kernel/utsname_sysctl.c +++ b/kernel/utsname_sysctl.c @@ -120,7 +120,6 @@ static struct ctl_table uts_kern_table[] = { .proc_handler = proc_do_uts_string, .poll = &domainname_poll, }, - {} }; #ifdef CONFIG_PROC_SYSCTL diff --git a/kernel/watchdog.c b/kernel/watchdog.c index d7b2125503af3..4e472d416525b 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -950,7 +950,6 @@ static struct ctl_table watchdog_sysctls[] = { }, #endif /* CONFIG_SMP */ #endif - {} }; static struct ctl_table watchdog_hardlockup_sysctl[] = { @@ -963,7 +962,6 @@ static struct ctl_table watchdog_hardlockup_sysctl[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, - {} }; static void __init watchdog_sysctl_init(void) From 7fd9c63f8777094514b46834c561647040f12fbb Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Tue, 27 Jun 2023 15:30:24 +0200 Subject: [PATCH 0501/1702] umh: Remove the now superfluous sentinel elements from ctl_table array This commit comes at the tail end of a greater effort to remove the empty elements at the end of the ctl_table arrays (sentinels) which will reduce the overall build time size of the kernel and run time memory bloat by ~64 bytes per sentinel (further information Link : https://lore.kernel.org/all/ZO5Yx5JFogGi%2FcBo@bombadil.infradead.org/) Remove sentinel element from usermodehelper_table Signed-off-by: Joel Granados --- kernel/umh.c | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/umh.c b/kernel/umh.c index 1b13c5d346248..598b3ffe15224 100644 --- a/kernel/umh.c +++ b/kernel/umh.c @@ -560,7 +560,6 @@ static struct ctl_table usermodehelper_table[] = { .mode = 0600, .proc_handler = proc_cap_handler, }, - { } }; static int __init init_umh_sysctls(void) From 66f20b11d3a31bcfe2f64e6a91238bd71b7eb72f Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Tue, 27 Jun 2023 15:30:23 +0200 Subject: [PATCH 0502/1702] ftrace: Remove the now superfluous sentinel elements from ctl_table array This commit comes at the tail end of a greater effort to remove the empty elements at the end of the ctl_table arrays (sentinels) which will reduce the overall build time size of the kernel and run time memory bloat by ~64 bytes per sentinel (further information Link : https://lore.kernel.org/all/ZO5Yx5JFogGi%2FcBo@bombadil.infradead.org/) Remove sentinel elements from ftrace_sysctls and user_event_sysctls Acked-by: Masami Hiramatsu (Google) Acked-by: Steven Rostedt (Google) Signed-off-by: Joel Granados --- kernel/trace/ftrace.c | 1 - kernel/trace/trace_events_user.c | 1 - 2 files changed, 2 deletions(-) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index da1710499698b..6cec53aa45a6d 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -8270,7 +8270,6 @@ static struct ctl_table ftrace_sysctls[] = { .mode = 0644, .proc_handler = ftrace_enable_sysctl, }, - {} }; static int __init ftrace_sysctl_init(void) diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index 70d428c394b68..304ceed9fd7d4 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -2833,7 +2833,6 @@ static struct ctl_table user_event_sysctls[] = { .mode = 0644, .proc_handler = set_max_user_events_sysctl, }, - {} }; static int __init trace_events_user_init(void) From fe6fc8e11b5a118a299c0d8f06407794173e6aa9 Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Tue, 27 Jun 2023 15:30:23 +0200 Subject: [PATCH 0503/1702] timekeeping: Remove the now superfluous sentinel elements from ctl_table array This commit comes at the tail end of a greater effort to remove the empty elements at the end of the ctl_table arrays (sentinels) which will reduce the overall build time size of the kernel and run time memory bloat by ~64 bytes per sentinel (further information Link : https://lore.kernel.org/all/ZO5Yx5JFogGi%2FcBo@bombadil.infradead.org/) Remove sentinel element from time_sysctl Signed-off-by: Joel Granados --- kernel/time/timer.c | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/time/timer.c b/kernel/time/timer.c index dee29f1f5b75f..9d107f4b506c6 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -312,7 +312,6 @@ static struct ctl_table timer_sysctl[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, - {} }; static int __init timer_sysctl_init(void) From e822582effc6bc00da9b28cf814935a6be070504 Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Tue, 27 Jun 2023 15:30:23 +0200 Subject: [PATCH 0504/1702] seccomp: Remove the now superfluous sentinel elements from ctl_table array This commit comes at the tail end of a greater effort to remove the empty elements at the end of the ctl_table arrays (sentinels) which will reduce the overall build time size of the kernel and run time memory bloat by ~64 bytes per sentinel (further information Link : https://lore.kernel.org/all/ZO5Yx5JFogGi%2FcBo@bombadil.infradead.org/) Remove sentinel element from seccomp_sysctl_table. Acked-by: Kees Cook Signed-off-by: Joel Granados --- kernel/seccomp.c | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/seccomp.c b/kernel/seccomp.c index aca7b437882e9..7ed72723fb8a1 100644 --- a/kernel/seccomp.c +++ b/kernel/seccomp.c @@ -2445,7 +2445,6 @@ static struct ctl_table seccomp_sysctl_table[] = { .mode = 0644, .proc_handler = seccomp_actions_logged_handler, }, - { } }; static int __init seccomp_sysctl_init(void) From f532376e881fb370bc2901a9e1cf0b7e461638ad Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Tue, 27 Jun 2023 15:30:21 +0200 Subject: [PATCH 0505/1702] scheduler: Remove the now superfluous sentinel elements from ctl_table array This commit comes at the tail end of a greater effort to remove the empty elements at the end of the ctl_table arrays (sentinels) which will reduce the overall build time size of the kernel and run time memory bloat by ~64 bytes per sentinel (further information Link : https://lore.kernel.org/all/ZO5Yx5JFogGi%2FcBo@bombadil.infradead.org/) rm sentinel element from ctl_table arrays Acked-by: Peter Zijlstra (Intel) Tested-by: Valentin Schneider Reviewed-by: Valentin Schneider Signed-off-by: Joel Granados --- kernel/sched/autogroup.c | 1 - kernel/sched/core.c | 1 - kernel/sched/deadline.c | 1 - kernel/sched/fair.c | 1 - kernel/sched/rt.c | 1 - kernel/sched/topology.c | 1 - 6 files changed, 6 deletions(-) diff --git a/kernel/sched/autogroup.c b/kernel/sched/autogroup.c index 991fc90025357..db68a964e34e2 100644 --- a/kernel/sched/autogroup.c +++ b/kernel/sched/autogroup.c @@ -19,7 +19,6 @@ static struct ctl_table sched_autogroup_sysctls[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, - {} }; static void __init sched_autogroup_sysctl_init(void) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 7019a40457a6d..7ce76620a308a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4741,7 +4741,6 @@ static struct ctl_table sched_core_sysctls[] = { .extra2 = SYSCTL_FOUR, }, #endif /* CONFIG_NUMA_BALANCING */ - {} }; static int __init sched_core_sysctl_init(void) { diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index a04a436af8cc4..c75d1307d86d5 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -43,7 +43,6 @@ static struct ctl_table sched_dl_sysctls[] = { .proc_handler = proc_douintvec_minmax, .extra2 = (void *)&sysctl_sched_dl_period_max, }, - {} }; static int __init sched_dl_sysctl_init(void) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 03be0d1330a6b..4ac2cf7a918eb 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -157,7 +157,6 @@ static struct ctl_table sched_fair_sysctls[] = { .extra1 = SYSCTL_ZERO, }, #endif /* CONFIG_NUMA_BALANCING */ - {} }; static int __init sched_fair_sysctl_init(void) diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 3261b067b67e2..aa4c1c874fa44 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -56,7 +56,6 @@ static struct ctl_table sched_rt_sysctls[] = { .mode = 0644, .proc_handler = sched_rr_handler, }, - {} }; static int __init sched_rt_sysctl_init(void) diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 99ea5986038ce..42c22648d124f 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -322,7 +322,6 @@ static struct ctl_table sched_energy_aware_sysctls[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, - {} }; static int __init sched_energy_aware_sysctl_init(void) From f842d9a96e6993763090960d1ab0f763374a99c0 Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Tue, 27 Jun 2023 15:30:21 +0200 Subject: [PATCH 0506/1702] printk: Remove the now superfluous sentinel elements from ctl_table array This commit comes at the tail end of a greater effort to remove the empty elements at the end of the ctl_table arrays (sentinels) which will reduce the overall build time size of the kernel and run time memory bloat by ~64 bytes per sentinel (further information Link : https://lore.kernel.org/all/ZO5Yx5JFogGi%2FcBo@bombadil.infradead.org/) rm sentinel element from printk_sysctls Reviewed-by: Petr Mladek Signed-off-by: Joel Granados --- kernel/printk/sysctl.c | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/printk/sysctl.c b/kernel/printk/sysctl.c index c228343eeb975..3e47dedce9e52 100644 --- a/kernel/printk/sysctl.c +++ b/kernel/printk/sysctl.c @@ -76,7 +76,6 @@ static struct ctl_table printk_sysctls[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_TWO, }, - {} }; void __init printk_sysctl_init(void) From f884cd38625c403077a724417efc963487893559 Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Tue, 27 Jun 2023 15:30:20 +0200 Subject: [PATCH 0507/1702] kprobes: Remove the now superfluous sentinel elements from ctl_table array This commit comes at the tail end of a greater effort to remove the empty elements at the end of the ctl_table arrays (sentinels) which will reduce the overall build time size of the kernel and run time memory bloat by ~64 bytes per sentinel (further information Link : https://lore.kernel.org/all/ZO5Yx5JFogGi%2FcBo@bombadil.infradead.org/) Remove sentinel element from kprobe_sysclts Acked-by: Masami Hiramatsu (Google) Signed-off-by: Joel Granados --- kernel/kprobes.c | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 9d9095e817928..85af0e05a38f0 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -968,7 +968,6 @@ static struct ctl_table kprobe_sysctls[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, - {} }; static void __init kprobe_sysctls_init(void) From f15843f725a55d7232aa9b699be52c2c3da06982 Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Tue, 27 Jun 2023 15:30:19 +0200 Subject: [PATCH 0508/1702] delayacct: Remove the now superfluous sentinel elements from ctl_table array This commit comes at the tail end of a greater effort to remove the empty elements at the end of the ctl_table arrays (sentinels) which will reduce the overall build time size of the kernel and run time memory bloat by ~64 bytes per sentinel (further information Link : https://lore.kernel.org/all/ZO5Yx5JFogGi%2FcBo@bombadil.infradead.org/) Remove sentinel element from kern_delayacct_table Signed-off-by: Joel Granados --- kernel/delayacct.c | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/delayacct.c b/kernel/delayacct.c index 6f0c358e73d80..e039b0f99a0ba 100644 --- a/kernel/delayacct.c +++ b/kernel/delayacct.c @@ -74,7 +74,6 @@ static struct ctl_table kern_delayacct_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, - { } }; static __init int kernel_delayacct_sysctls_init(void) From 1adb825af946fa6b2104c1713ad4999283e09fbc Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Tue, 27 Jun 2023 15:30:19 +0200 Subject: [PATCH 0509/1702] bpf: Remove the now superfluous sentinel elements from ctl_table array This commit comes at the tail end of a greater effort to remove the empty elements at the end of the ctl_table arrays (sentinels) which will reduce the overall build time size of the kernel and run time memory bloat by ~64 bytes per sentinel (further information Link : https://lore.kernel.org/all/ZO5Yx5JFogGi%2FcBo@bombadil.infradead.org/) Remove sentinel element from bpf_syscall_table. Acked-by: Andrii Nakryiko Signed-off-by: Joel Granados --- kernel/bpf/syscall.c | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index ae2ff73bde7e7..c7e805087b062 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -5979,7 +5979,6 @@ static struct ctl_table bpf_syscall_table[] = { .mode = 0644, .proc_handler = bpf_stats_handler, }, - { } }; static int __init bpf_syscall_sysctl_init(void) From 520713a93d550406dae14d49cdb8778d70cecdfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Fri, 15 Mar 2024 19:11:30 +0100 Subject: [PATCH 0510/1702] sysctl: treewide: drop unused argument ctl_table_root::set_ownership(table) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the 'table' argument from set_ownership as it is never used. This change is a step towards putting "struct ctl_table" into .rodata and eventually having sysctl core only use "const struct ctl_table". The patch was created with the following coccinelle script: @@ identifier func, head, table, uid, gid; @@ void func( struct ctl_table_header *head, - struct ctl_table *table, kuid_t *uid, kgid_t *gid) { ... } No additional occurrences of 'set_ownership' were found after doing a tree-wide search. Reviewed-by: Joel Granados Signed-off-by: Thomas Weißschuh Signed-off-by: Joel Granados --- fs/proc/proc_sysctl.c | 2 +- include/linux/sysctl.h | 1 - ipc/ipc_sysctl.c | 3 +-- ipc/mq_sysctl.c | 3 +-- net/sysctl_net.c | 1 - 5 files changed, 3 insertions(+), 7 deletions(-) diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 37cde0efee570..ed3a41ed97055 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -480,7 +480,7 @@ static struct inode *proc_sys_make_inode(struct super_block *sb, } if (root->set_ownership) - root->set_ownership(head, table, &inode->i_uid, &inode->i_gid); + root->set_ownership(head, &inode->i_uid, &inode->i_gid); else { inode->i_uid = GLOBAL_ROOT_UID; inode->i_gid = GLOBAL_ROOT_GID; diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index ee7d33b89e9ef..60333a6b9370a 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -205,7 +205,6 @@ struct ctl_table_root { struct ctl_table_set default_set; struct ctl_table_set *(*lookup)(struct ctl_table_root *root); void (*set_ownership)(struct ctl_table_header *head, - struct ctl_table *table, kuid_t *uid, kgid_t *gid); int (*permissions)(struct ctl_table_header *head, struct ctl_table *table); }; diff --git a/ipc/ipc_sysctl.c b/ipc/ipc_sysctl.c index 45cb1dabce29a..1a5085e5b1785 100644 --- a/ipc/ipc_sysctl.c +++ b/ipc/ipc_sysctl.c @@ -192,7 +192,6 @@ static int set_is_seen(struct ctl_table_set *set) } static void ipc_set_ownership(struct ctl_table_header *head, - struct ctl_table *table, kuid_t *uid, kgid_t *gid) { struct ipc_namespace *ns = @@ -224,7 +223,7 @@ static int ipc_permissions(struct ctl_table_header *head, struct ctl_table *tabl kuid_t ns_root_uid; kgid_t ns_root_gid; - ipc_set_ownership(head, table, &ns_root_uid, &ns_root_gid); + ipc_set_ownership(head, &ns_root_uid, &ns_root_gid); if (uid_eq(current_euid(), ns_root_uid)) mode >>= 6; diff --git a/ipc/mq_sysctl.c b/ipc/mq_sysctl.c index 21fba3a6edaf7..6bb1c5397c69b 100644 --- a/ipc/mq_sysctl.c +++ b/ipc/mq_sysctl.c @@ -78,7 +78,6 @@ static int set_is_seen(struct ctl_table_set *set) } static void mq_set_ownership(struct ctl_table_header *head, - struct ctl_table *table, kuid_t *uid, kgid_t *gid) { struct ipc_namespace *ns = @@ -97,7 +96,7 @@ static int mq_permissions(struct ctl_table_header *head, struct ctl_table *table kuid_t ns_root_uid; kgid_t ns_root_gid; - mq_set_ownership(head, table, &ns_root_uid, &ns_root_gid); + mq_set_ownership(head, &ns_root_uid, &ns_root_gid); if (uid_eq(current_euid(), ns_root_uid)) mode >>= 6; diff --git a/net/sysctl_net.c b/net/sysctl_net.c index 051ed5f6fc937..a0a7a79991f9f 100644 --- a/net/sysctl_net.c +++ b/net/sysctl_net.c @@ -54,7 +54,6 @@ static int net_ctl_permissions(struct ctl_table_header *head, } static void net_ctl_set_ownership(struct ctl_table_header *head, - struct ctl_table *table, kuid_t *uid, kgid_t *gid) { struct net *net = container_of(head->set, struct net, sysctls); From 795f90c6f13c30484ff10355a6775979f57f78cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Fri, 15 Mar 2024 19:11:31 +0100 Subject: [PATCH 0511/1702] sysctl: treewide: constify argument ctl_table_root::permissions(table) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The permissions callback should not modify the ctl_table. Enforce this expectation via the typesystem. This is a step to put "struct ctl_table" into .rodata throughout the kernel. The patch was created with the following coccinelle script: @@ identifier func, head, ctl; @@ int func( struct ctl_table_header *head, - struct ctl_table *ctl) + const struct ctl_table *ctl) { ... } (insert_entry() from fs/proc/proc_sysctl.c is a false-positive) No additional occurrences of '.permissions =' were found after a tree-wide search for places missed by the conccinelle script. Reviewed-by: Joel Granados Signed-off-by: Thomas Weißschuh Signed-off-by: Joel Granados --- include/linux/sysctl.h | 2 +- ipc/ipc_sysctl.c | 2 +- ipc/mq_sysctl.c | 2 +- kernel/ucount.c | 2 +- net/sysctl_net.c | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 60333a6b9370a..f9214de0490c4 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -206,7 +206,7 @@ struct ctl_table_root { struct ctl_table_set *(*lookup)(struct ctl_table_root *root); void (*set_ownership)(struct ctl_table_header *head, kuid_t *uid, kgid_t *gid); - int (*permissions)(struct ctl_table_header *head, struct ctl_table *table); + int (*permissions)(struct ctl_table_header *head, const struct ctl_table *table); }; #define register_sysctl(path, table) \ diff --git a/ipc/ipc_sysctl.c b/ipc/ipc_sysctl.c index 1a5085e5b1785..19b2a67aef406 100644 --- a/ipc/ipc_sysctl.c +++ b/ipc/ipc_sysctl.c @@ -204,7 +204,7 @@ static void ipc_set_ownership(struct ctl_table_header *head, *gid = gid_valid(ns_root_gid) ? ns_root_gid : GLOBAL_ROOT_GID; } -static int ipc_permissions(struct ctl_table_header *head, struct ctl_table *table) +static int ipc_permissions(struct ctl_table_header *head, const struct ctl_table *table) { int mode = table->mode; diff --git a/ipc/mq_sysctl.c b/ipc/mq_sysctl.c index 6bb1c5397c69b..43c0825da9e8b 100644 --- a/ipc/mq_sysctl.c +++ b/ipc/mq_sysctl.c @@ -90,7 +90,7 @@ static void mq_set_ownership(struct ctl_table_header *head, *gid = gid_valid(ns_root_gid) ? ns_root_gid : GLOBAL_ROOT_GID; } -static int mq_permissions(struct ctl_table_header *head, struct ctl_table *table) +static int mq_permissions(struct ctl_table_header *head, const struct ctl_table *table) { int mode = table->mode; kuid_t ns_root_uid; diff --git a/kernel/ucount.c b/kernel/ucount.c index e196da0204dcb..4d5b9c12c0142 100644 --- a/kernel/ucount.c +++ b/kernel/ucount.c @@ -38,7 +38,7 @@ static int set_is_seen(struct ctl_table_set *set) } static int set_permissions(struct ctl_table_header *head, - struct ctl_table *table) + const struct ctl_table *table) { struct user_namespace *user_ns = container_of(head->set, struct user_namespace, set); diff --git a/net/sysctl_net.c b/net/sysctl_net.c index a0a7a79991f9f..f5017012a0493 100644 --- a/net/sysctl_net.c +++ b/net/sysctl_net.c @@ -40,7 +40,7 @@ static int is_seen(struct ctl_table_set *set) /* Return standard mode bits for table entry. */ static int net_ctl_permissions(struct ctl_table_header *head, - struct ctl_table *table) + const struct ctl_table *table) { struct net *net = container_of(head->set, struct net, sysctls); From eb32d3adef461bc523c97321f19209f55512ee4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Fri, 22 Mar 2024 18:05:56 +0100 Subject: [PATCH 0512/1702] sysctl: drop sysctl_is_perm_empty_ctl_table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is used only twice and those callers are simpler with sysctl_is_perm_empty_ctl_header(). So use this sibling function. This is part of an effort to constify definition of struct ctl_table. For this effort the mutable member 'type' is moved from struct ctl_table to struct ctl_table_header. Unifying the macros sysctl_is_perm_empty_ctl_* makes this easier. Signed-off-by: Thomas Weißschuh Signed-off-by: Joel Granados --- fs/proc/proc_sysctl.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index ed3a41ed97055..3f8abb6d2ad54 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -48,10 +48,8 @@ struct ctl_table_header *register_sysctl_mount_point(const char *path) } EXPORT_SYMBOL(register_sysctl_mount_point); -#define sysctl_is_perm_empty_ctl_table(tptr) \ - (tptr[0].type == SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY) #define sysctl_is_perm_empty_ctl_header(hptr) \ - (sysctl_is_perm_empty_ctl_table(hptr->ctl_table)) + (hptr->ctl_table[0].type == SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY) #define sysctl_set_perm_empty_ctl_header(hptr) \ (hptr->ctl_table[0].type = SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY) #define sysctl_clear_perm_empty_ctl_header(hptr) \ @@ -233,7 +231,7 @@ static int insert_header(struct ctl_dir *dir, struct ctl_table_header *header) /* Am I creating a permanently empty directory? */ if (header->ctl_table_size > 0 && - sysctl_is_perm_empty_ctl_table(header->ctl_table)) { + sysctl_is_perm_empty_ctl_header(header)) { if (!RB_EMPTY_ROOT(&dir->root)) return -EINVAL; sysctl_set_perm_empty_ctl_header(dir_h); @@ -1204,7 +1202,7 @@ static bool get_links(struct ctl_dir *dir, struct ctl_table *entry, *link; if (header->ctl_table_size == 0 || - sysctl_is_perm_empty_ctl_table(header->ctl_table)) + sysctl_is_perm_empty_ctl_header(header)) return true; /* Are there links available for every entry in table? */ From 4a7b29f6509470a7e63dd67e516268fbef226194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Fri, 22 Mar 2024 18:05:57 +0100 Subject: [PATCH 0513/1702] sysctl: move sysctl type to ctl_table_header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the SYSCTL_TABLE_TYPE_{DEFAULT,PERMANENTLY_EMPTY} enums from ctl_table to ctl_table_header. Removing the mutable member is necessary to constify static instances of struct ctl_table. Move the initialization of the sysctl_mount_point type into init_header() where all the other header fields are also initialized. As a side-effect the memory usage of the sysctl core is reduced. Each ctl_table_header instance can manage multiple ctl_table instances and is only allocated when the table is actually registered. This saves 8 bytes of memory per ctl_table on 64bit, 4 due to the enum field itself and 4 due to padding. Signed-off-by: Thomas Weißschuh Signed-off-by: Joel Granados --- fs/proc/proc_sysctl.c | 10 ++++++---- include/linux/sysctl.h | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 3f8abb6d2ad54..a847b0bc63c45 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -31,7 +31,7 @@ static const struct inode_operations proc_sys_dir_operations; /* Support for permanently empty directories */ static struct ctl_table sysctl_mount_point[] = { - {.type = SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY } + { } }; /** @@ -49,11 +49,11 @@ struct ctl_table_header *register_sysctl_mount_point(const char *path) EXPORT_SYMBOL(register_sysctl_mount_point); #define sysctl_is_perm_empty_ctl_header(hptr) \ - (hptr->ctl_table[0].type == SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY) + (hptr->type == SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY) #define sysctl_set_perm_empty_ctl_header(hptr) \ - (hptr->ctl_table[0].type = SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY) + (hptr->type = SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY) #define sysctl_clear_perm_empty_ctl_header(hptr) \ - (hptr->ctl_table[0].type = SYSCTL_TABLE_TYPE_DEFAULT) + (hptr->type = SYSCTL_TABLE_TYPE_DEFAULT) void proc_sys_poll_notify(struct ctl_table_poll *poll) { @@ -208,6 +208,8 @@ static void init_header(struct ctl_table_header *head, node++; } } + if (table == sysctl_mount_point) + sysctl_set_perm_empty_ctl_header(head); } static void erase_header(struct ctl_table_header *head) diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index f9214de0490c4..47bd28ffa88f4 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -137,17 +137,6 @@ struct ctl_table { void *data; int maxlen; umode_t mode; - /** - * enum type - Enumeration to differentiate between ctl target types - * @SYSCTL_TABLE_TYPE_DEFAULT: ctl target with no special considerations - * @SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY: Used to identify a permanently - * empty directory target to serve - * as mount point. - */ - enum { - SYSCTL_TABLE_TYPE_DEFAULT, - SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY - } type; proc_handler *proc_handler; /* Callback for text formatting */ struct ctl_table_poll *poll; void *extra1; @@ -188,6 +177,17 @@ struct ctl_table_header { struct ctl_dir *parent; struct ctl_node *node; struct hlist_head inodes; /* head for proc_inode->sysctl_inodes */ + /** + * enum type - Enumeration to differentiate between ctl target types + * @SYSCTL_TABLE_TYPE_DEFAULT: ctl target with no special considerations + * @SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY: Used to identify a permanently + * empty directory target to serve + * as mount point. + */ + enum { + SYSCTL_TABLE_TYPE_DEFAULT, + SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY, + } type; }; struct ctl_dir { From a35dd3a786f57903151b18275b1eed105084cf72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Fri, 22 Mar 2024 18:05:58 +0100 Subject: [PATCH 0514/1702] sysctl: drop now unnecessary out-of-bounds check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the now unneeded check for ctl_table_size; it is safe to do so as sysctl_set_perm_empty_ctl_header() does not access the ctl_table member anymore. This also makes the element of sysctl_mount_point unnecessary, so drop it at the same time. Signed-off-by: Thomas Weißschuh Signed-off-by: Joel Granados --- fs/proc/proc_sysctl.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index a847b0bc63c45..b1c2c0b821161 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -30,9 +30,7 @@ static const struct file_operations proc_sys_dir_file_operations; static const struct inode_operations proc_sys_dir_operations; /* Support for permanently empty directories */ -static struct ctl_table sysctl_mount_point[] = { - { } -}; +static struct ctl_table sysctl_mount_point[] = { }; /** * register_sysctl_mount_point() - registers a sysctl mount point @@ -232,8 +230,7 @@ static int insert_header(struct ctl_dir *dir, struct ctl_table_header *header) return -EROFS; /* Am I creating a permanently empty directory? */ - if (header->ctl_table_size > 0 && - sysctl_is_perm_empty_ctl_header(header)) { + if (sysctl_is_perm_empty_ctl_header(header)) { if (!RB_EMPTY_ROOT(&dir->root)) return -EINVAL; sysctl_set_perm_empty_ctl_header(dir_h); From cd27553b0dee6fdc4a2535ab9fc3c8fbdd811d13 Mon Sep 17 00:00:00 2001 From: Paul Barker Date: Wed, 17 Apr 2024 12:41:31 +0100 Subject: [PATCH 0515/1702] pinctrl: renesas: rzg2l: Limit 2.5V power supply to Ethernet interfaces The RZ/G3S SoC supports configurable supply voltages for several of its I/O interfaces. All of these interfaces support both 1.8V and 3.3V supplies, but only the Ethernet and XSPI interfaces support a 2.5V supply. Voltage selection for the XSPI interface is not yet supported, so this leaves only the Ethernet interfaces currently supporting selection of a 2.5V supply. So we need to return an error if there is an attempt to select a 2.5V supply for any non-Ethernet interface. Fixes: 51996952b8b5 ("pinctrl: renesas: rzg2l: Add support to select power source for Ethernet pins") Signed-off-by: Paul Barker Reviewed-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20240417114132.6605-1-paul.barker.ct@bp.renesas.com Signed-off-by: Geert Uytterhoeven --- drivers/pinctrl/renesas/pinctrl-rzg2l.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pinctrl/renesas/pinctrl-rzg2l.c b/drivers/pinctrl/renesas/pinctrl-rzg2l.c index dbcf009837ef8..c3256bfde5023 100644 --- a/drivers/pinctrl/renesas/pinctrl-rzg2l.c +++ b/drivers/pinctrl/renesas/pinctrl-rzg2l.c @@ -892,6 +892,8 @@ static int rzg2l_set_power_source(struct rzg2l_pinctrl *pctrl, u32 pin, u32 caps val = PVDD_1800; break; case 2500: + if (!(caps & (PIN_CFG_IO_VMC_ETH0 | PIN_CFG_IO_VMC_ETH1))) + return -EINVAL; val = PVDD_2500; break; case 3300: From 1b1f8cc20429e27c1199d4b2846862da8665561b Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 13 Apr 2024 18:19:40 +0200 Subject: [PATCH 0516/1702] clk: renesas: r8a7740: Remove unused div4_clk.flags field In "struct div4_clk", the "flags" field is unused. Remove it, and update the "div4_clks" array accordingly. Found with cppcheck, unusedStructMember. Signed-off-by: Christophe JAILLET Reviewed-by: Geert Uytterhoeven Tested-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/658e6b1b23d5b66646bb830361b8c55ccf797771.1713025170.git.christophe.jaillet@wanadoo.fr Signed-off-by: Geert Uytterhoeven --- drivers/clk/renesas/clk-r8a7740.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/drivers/clk/renesas/clk-r8a7740.c b/drivers/clk/renesas/clk-r8a7740.c index 3ee3f57e4e9ae..325394b6e55e2 100644 --- a/drivers/clk/renesas/clk-r8a7740.c +++ b/drivers/clk/renesas/clk-r8a7740.c @@ -32,22 +32,21 @@ struct div4_clk { const char *name; unsigned int reg; unsigned int shift; - int flags; }; static struct div4_clk div4_clks[] = { - { "i", CPG_FRQCRA, 20, CLK_ENABLE_ON_INIT }, - { "zg", CPG_FRQCRA, 16, CLK_ENABLE_ON_INIT }, - { "b", CPG_FRQCRA, 8, CLK_ENABLE_ON_INIT }, - { "m1", CPG_FRQCRA, 4, CLK_ENABLE_ON_INIT }, - { "hp", CPG_FRQCRB, 4, 0 }, - { "hpp", CPG_FRQCRC, 20, 0 }, - { "usbp", CPG_FRQCRC, 16, 0 }, - { "s", CPG_FRQCRC, 12, 0 }, - { "zb", CPG_FRQCRC, 8, 0 }, - { "m3", CPG_FRQCRC, 4, 0 }, - { "cp", CPG_FRQCRC, 0, 0 }, - { NULL, 0, 0, 0 }, + { "i", CPG_FRQCRA, 20 }, + { "zg", CPG_FRQCRA, 16 }, + { "b", CPG_FRQCRA, 8 }, + { "m1", CPG_FRQCRA, 4 }, + { "hp", CPG_FRQCRB, 4 }, + { "hpp", CPG_FRQCRC, 20 }, + { "usbp", CPG_FRQCRC, 16 }, + { "s", CPG_FRQCRC, 12 }, + { "zb", CPG_FRQCRC, 8 }, + { "m3", CPG_FRQCRC, 4 }, + { "cp", CPG_FRQCRC, 0 }, + { NULL, 0, 0 }, }; static const struct clk_div_table div4_div_table[] = { From d0d4585222d0e652ca33f03921d5574963d372ec Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Thu, 25 Apr 2024 10:03:18 +0200 Subject: [PATCH 0517/1702] clk: renesas: shmobile: Remove unused CLK_ENABLE_ON_INIT CLK_ENABLE_ON_INIT is a relic from the old SH clock framework. It is not used on SH/R-Mobile ARM drivers. Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/f01e60a1007afe9385ddc10c4665752857ba4135.1714032122.git.geert+renesas@glider.be --- drivers/clk/renesas/clk-r8a73a4.c | 2 -- drivers/clk/renesas/clk-r8a7740.c | 2 -- drivers/clk/renesas/clk-sh73a0.c | 2 -- 3 files changed, 6 deletions(-) diff --git a/drivers/clk/renesas/clk-r8a73a4.c b/drivers/clk/renesas/clk-r8a73a4.c index f45c2c45808be..4b1815147f776 100644 --- a/drivers/clk/renesas/clk-r8a73a4.c +++ b/drivers/clk/renesas/clk-r8a73a4.c @@ -30,8 +30,6 @@ struct r8a73a4_cpg { #define CPG_PLL2HCR 0xe4 #define CPG_PLL2SCR 0xf4 -#define CLK_ENABLE_ON_INIT BIT(0) - struct div4_clk { const char *name; unsigned int reg; diff --git a/drivers/clk/renesas/clk-r8a7740.c b/drivers/clk/renesas/clk-r8a7740.c index 325394b6e55e2..22e9be7240bb2 100644 --- a/drivers/clk/renesas/clk-r8a7740.c +++ b/drivers/clk/renesas/clk-r8a7740.c @@ -26,8 +26,6 @@ struct r8a7740_cpg { #define CPG_USBCKCR 0x8c #define CPG_FRQCRC 0xe0 -#define CLK_ENABLE_ON_INIT BIT(0) - struct div4_clk { const char *name; unsigned int reg; diff --git a/drivers/clk/renesas/clk-sh73a0.c b/drivers/clk/renesas/clk-sh73a0.c index 8c51090f13e13..47fc99ccd2838 100644 --- a/drivers/clk/renesas/clk-sh73a0.c +++ b/drivers/clk/renesas/clk-sh73a0.c @@ -34,8 +34,6 @@ struct sh73a0_cpg { #define CPG_DSI0PHYCR 0x6c #define CPG_DSI1PHYCR 0x70 -#define CLK_ENABLE_ON_INIT BIT(0) - struct div4_clk { const char *name; const char *parent; From 2174035a7f1148a52f5a3f371f04224168b5b00a Mon Sep 17 00:00:00 2001 From: Jaegeuk Kim Date: Fri, 5 Apr 2024 01:15:42 +0000 Subject: [PATCH 0518/1702] f2fs: clear writeback when compression failed Let's stop issuing compressed writes and clear their writeback flags. Reviewed-by: Daeho Jeong Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/compress.c | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index d67c471ab5df6..b12d3a49bfdac 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -1031,6 +1031,31 @@ static void set_cluster_writeback(struct compress_ctx *cc) } } +static void cancel_cluster_writeback(struct compress_ctx *cc, + struct compress_io_ctx *cic, int submitted) +{ + int i; + + /* Wait for submitted IOs. */ + if (submitted > 1) { + f2fs_submit_merged_write(F2FS_I_SB(cc->inode), DATA); + while (atomic_read(&cic->pending_pages) != + (cc->valid_nr_cpages - submitted + 1)) + f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT); + } + + /* Cancel writeback and stay locked. */ + for (i = 0; i < cc->cluster_size; i++) { + if (i < submitted) { + inode_inc_dirty_pages(cc->inode); + lock_page(cc->rpages[i]); + } + clear_page_private_gcing(cc->rpages[i]); + if (folio_test_writeback(page_folio(cc->rpages[i]))) + end_page_writeback(cc->rpages[i]); + } +} + static void set_cluster_dirty(struct compress_ctx *cc) { int i; @@ -1232,7 +1257,6 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, .page = NULL, .encrypted_page = NULL, .compressed_page = NULL, - .submitted = 0, .io_type = io_type, .io_wbc = wbc, .encrypted = fscrypt_inode_uses_fs_layer_crypto(cc->inode) ? @@ -1358,7 +1382,16 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, fio.compressed_page = cc->cpages[i - 1]; cc->cpages[i - 1] = NULL; + fio.submitted = 0; f2fs_outplace_write_data(&dn, &fio); + if (unlikely(!fio.submitted)) { + cancel_cluster_writeback(cc, cic, i); + + /* To call fscrypt_finalize_bounce_page */ + i = cc->valid_nr_cpages; + *submitted = 0; + goto out_destroy_crypt; + } (*submitted)++; unlock_continue: inode_dec_dirty_pages(cc->inode); @@ -1392,8 +1425,11 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc, out_destroy_crypt: page_array_free(cc->inode, cic->rpages, cc->cluster_size); - for (--i; i >= 0; i--) + for (--i; i >= 0; i--) { + if (!cc->cpages[i]) + continue; fscrypt_finalize_bounce_page(&cc->cpages[i]); + } out_put_cic: kmem_cache_free(cic_entry_slab, cic); out_put_dnode: From b864ddb57eb00c4ea1e6801c7b2f70f1db2a7f4b Mon Sep 17 00:00:00 2001 From: Jaegeuk Kim Date: Tue, 9 Apr 2024 20:34:11 +0000 Subject: [PATCH 0519/1702] f2fs: fix false alarm on invalid block address f2fs_ra_meta_pages can try to read ahead on invalid block address which is not the corruption case. Cc: # v6.9+ Bugzilla: https://bugzilla.kernel.org/show_bug.cgi?id=218770 Fixes: 31f85ccc84b8 ("f2fs: unify the error handling of f2fs_is_valid_blkaddr") Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/checkpoint.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index 5d05a413f4510..55d444bec5c06 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -179,22 +179,22 @@ static bool __f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi, break; case META_SIT: if (unlikely(blkaddr >= SIT_BLK_CNT(sbi))) - goto err; + goto check_only; break; case META_SSA: if (unlikely(blkaddr >= MAIN_BLKADDR(sbi) || blkaddr < SM_I(sbi)->ssa_blkaddr)) - goto err; + goto check_only; break; case META_CP: if (unlikely(blkaddr >= SIT_I(sbi)->sit_base_addr || blkaddr < __start_cp_addr(sbi))) - goto err; + goto check_only; break; case META_POR: if (unlikely(blkaddr >= MAX_BLKADDR(sbi) || blkaddr < MAIN_BLKADDR(sbi))) - goto err; + goto check_only; break; case DATA_GENERIC: case DATA_GENERIC_ENHANCE: @@ -228,6 +228,7 @@ static bool __f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi, return true; err: f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); +check_only: return false; } From 3763f9effcdccc38bde614c911f331203f204a01 Mon Sep 17 00:00:00 2001 From: Wu Bo Date: Tue, 23 Apr 2024 05:27:59 -0600 Subject: [PATCH 0520/1702] f2fs: use helper to print zone condition To make code clean, use blk_zone_cond_str() to print debug information. Signed-off-by: Wu Bo Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/segment.c | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 2206199e80998..5d6dae4861f38 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -5025,17 +5025,6 @@ static int sanity_check_curseg(struct f2fs_sb_info *sbi) } #ifdef CONFIG_BLK_DEV_ZONED -static const char *f2fs_zone_status[BLK_ZONE_COND_OFFLINE + 1] = { - [BLK_ZONE_COND_NOT_WP] = "NOT_WP", - [BLK_ZONE_COND_EMPTY] = "EMPTY", - [BLK_ZONE_COND_IMP_OPEN] = "IMPLICIT_OPEN", - [BLK_ZONE_COND_EXP_OPEN] = "EXPLICIT_OPEN", - [BLK_ZONE_COND_CLOSED] = "CLOSED", - [BLK_ZONE_COND_READONLY] = "READONLY", - [BLK_ZONE_COND_FULL] = "FULL", - [BLK_ZONE_COND_OFFLINE] = "OFFLINE", -}; - static int check_zone_write_pointer(struct f2fs_sb_info *sbi, struct f2fs_dev_info *fdev, struct blk_zone *zone) @@ -5066,7 +5055,7 @@ static int check_zone_write_pointer(struct f2fs_sb_info *sbi, if (IS_CURSEC(sbi, GET_SEC_FROM_SEG(sbi, zone_segno))) { f2fs_notice(sbi, "Open zones: valid block[0x%x,0x%x] cond[%s]", zone_segno, valid_block_cnt, - f2fs_zone_status[zone->cond]); + blk_zone_cond_str(zone->cond)); return 0; } @@ -5077,7 +5066,7 @@ static int check_zone_write_pointer(struct f2fs_sb_info *sbi, if (!valid_block_cnt) { f2fs_notice(sbi, "Zone without valid block has non-zero write " "pointer. Reset the write pointer: cond[%s]", - f2fs_zone_status[zone->cond]); + blk_zone_cond_str(zone->cond)); ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block, zone->len >> log_sectors_per_block); if (ret) @@ -5095,7 +5084,7 @@ static int check_zone_write_pointer(struct f2fs_sb_info *sbi, */ f2fs_notice(sbi, "Valid blocks are not aligned with write " "pointer: valid block[0x%x,0x%x] cond[%s]", - zone_segno, valid_block_cnt, f2fs_zone_status[zone->cond]); + zone_segno, valid_block_cnt, blk_zone_cond_str(zone->cond)); nofs_flags = memalloc_nofs_save(); ret = blkdev_zone_mgmt(fdev->bdev, REQ_OP_ZONE_FINISH, From b6cc692ac67a75b97a2d524d8e2fd2d59b20248a Mon Sep 17 00:00:00 2001 From: Claudiu Beznea Date: Mon, 22 Apr 2024 13:53:48 +0300 Subject: [PATCH 0521/1702] dt-bindings: clock: r9a07g043-cpg: Add power domain IDs Add power domain IDs for the RZ/G2UL (R9A07G043) SoC. Signed-off-by: Claudiu Beznea Acked-by: Rob Herring Reviewed-by: Geert Uytterhoeven Reviewed-by: Ulf Hansson Link: https://lore.kernel.org/r/20240422105355.1622177-2-claudiu.beznea.uj@bp.renesas.com Signed-off-by: Geert Uytterhoeven --- include/dt-bindings/clock/r9a07g043-cpg.h | 52 +++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/include/dt-bindings/clock/r9a07g043-cpg.h b/include/dt-bindings/clock/r9a07g043-cpg.h index a64139fec8152..1319933437779 100644 --- a/include/dt-bindings/clock/r9a07g043-cpg.h +++ b/include/dt-bindings/clock/r9a07g043-cpg.h @@ -200,5 +200,57 @@ #define R9A07G043_AX45MP_CORE0_RESETN 78 /* RZ/Five Only */ #define R9A07G043_IAX45_RESETN 79 /* RZ/Five Only */ +/* Power domain IDs. */ +#define R9A07G043_PD_ALWAYS_ON 0 +#define R9A07G043_PD_GIC 1 /* RZ/G2UL Only */ +#define R9A07G043_PD_IA55 2 /* RZ/G2UL Only */ +#define R9A07G043_PD_MHU 3 /* RZ/G2UL Only */ +#define R9A07G043_PD_CORESIGHT 4 /* RZ/G2UL Only */ +#define R9A07G043_PD_SYC 5 /* RZ/G2UL Only */ +#define R9A07G043_PD_DMAC 6 +#define R9A07G043_PD_GTM0 7 +#define R9A07G043_PD_GTM1 8 +#define R9A07G043_PD_GTM2 9 +#define R9A07G043_PD_MTU 10 +#define R9A07G043_PD_POE3 11 +#define R9A07G043_PD_WDT0 12 +#define R9A07G043_PD_SPI 13 +#define R9A07G043_PD_SDHI0 14 +#define R9A07G043_PD_SDHI1 15 +#define R9A07G043_PD_ISU 16 /* RZ/G2UL Only */ +#define R9A07G043_PD_CRU 17 /* RZ/G2UL Only */ +#define R9A07G043_PD_LCDC 18 /* RZ/G2UL Only */ +#define R9A07G043_PD_SSI0 19 +#define R9A07G043_PD_SSI1 20 +#define R9A07G043_PD_SSI2 21 +#define R9A07G043_PD_SSI3 22 +#define R9A07G043_PD_SRC 23 +#define R9A07G043_PD_USB0 24 +#define R9A07G043_PD_USB1 25 +#define R9A07G043_PD_USB_PHY 26 +#define R9A07G043_PD_ETHER0 27 +#define R9A07G043_PD_ETHER1 28 +#define R9A07G043_PD_I2C0 29 +#define R9A07G043_PD_I2C1 30 +#define R9A07G043_PD_I2C2 31 +#define R9A07G043_PD_I2C3 32 +#define R9A07G043_PD_SCIF0 33 +#define R9A07G043_PD_SCIF1 34 +#define R9A07G043_PD_SCIF2 35 +#define R9A07G043_PD_SCIF3 36 +#define R9A07G043_PD_SCIF4 37 +#define R9A07G043_PD_SCI0 38 +#define R9A07G043_PD_SCI1 39 +#define R9A07G043_PD_IRDA 40 +#define R9A07G043_PD_RSPI0 41 +#define R9A07G043_PD_RSPI1 42 +#define R9A07G043_PD_RSPI2 43 +#define R9A07G043_PD_CANFD 44 +#define R9A07G043_PD_ADC 45 +#define R9A07G043_PD_TSU 46 +#define R9A07G043_PD_PLIC 47 /* RZ/Five Only */ +#define R9A07G043_PD_IAX45 48 /* RZ/Five Only */ +#define R9A07G043_PD_NCEPLDM 49 /* RZ/Five Only */ +#define R9A07G043_PD_NCEPLMT 50 /* RZ/Five Only */ #endif /* __DT_BINDINGS_CLOCK_R9A07G043_CPG_H__ */ From d744e4567419e26025f42e8b1f13c0b021d62819 Mon Sep 17 00:00:00 2001 From: Claudiu Beznea Date: Mon, 22 Apr 2024 13:53:49 +0300 Subject: [PATCH 0522/1702] dt-bindings: clock: r9a07g044-cpg: Add power domain IDs Add power domain IDs for the RZ/G2L (R9A07G044) SoC. Signed-off-by: Claudiu Beznea Reviewed-by: Geert Uytterhoeven Acked-by: Rob Herring Reviewed-by: Ulf Hansson Link: https://lore.kernel.org/r/20240422105355.1622177-3-claudiu.beznea.uj@bp.renesas.com Signed-off-by: Geert Uytterhoeven --- include/dt-bindings/clock/r9a07g044-cpg.h | 58 +++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/include/dt-bindings/clock/r9a07g044-cpg.h b/include/dt-bindings/clock/r9a07g044-cpg.h index 0bb17ff1a01a7..e209f96f92b7e 100644 --- a/include/dt-bindings/clock/r9a07g044-cpg.h +++ b/include/dt-bindings/clock/r9a07g044-cpg.h @@ -217,4 +217,62 @@ #define R9A07G044_ADC_ADRST_N 82 #define R9A07G044_TSU_PRESETN 83 +/* Power domain IDs. */ +#define R9A07G044_PD_ALWAYS_ON 0 +#define R9A07G044_PD_GIC 1 +#define R9A07G044_PD_IA55 2 +#define R9A07G044_PD_MHU 3 +#define R9A07G044_PD_CORESIGHT 4 +#define R9A07G044_PD_SYC 5 +#define R9A07G044_PD_DMAC 6 +#define R9A07G044_PD_GTM0 7 +#define R9A07G044_PD_GTM1 8 +#define R9A07G044_PD_GTM2 9 +#define R9A07G044_PD_MTU 10 +#define R9A07G044_PD_POE3 11 +#define R9A07G044_PD_GPT 12 +#define R9A07G044_PD_POEGA 13 +#define R9A07G044_PD_POEGB 14 +#define R9A07G044_PD_POEGC 15 +#define R9A07G044_PD_POEGD 16 +#define R9A07G044_PD_WDT0 17 +#define R9A07G044_PD_WDT1 18 +#define R9A07G044_PD_SPI 19 +#define R9A07G044_PD_SDHI0 20 +#define R9A07G044_PD_SDHI1 21 +#define R9A07G044_PD_3DGE 22 +#define R9A07G044_PD_ISU 23 +#define R9A07G044_PD_VCPL4 24 +#define R9A07G044_PD_CRU 25 +#define R9A07G044_PD_MIPI_DSI 26 +#define R9A07G044_PD_LCDC 27 +#define R9A07G044_PD_SSI0 28 +#define R9A07G044_PD_SSI1 29 +#define R9A07G044_PD_SSI2 30 +#define R9A07G044_PD_SSI3 31 +#define R9A07G044_PD_SRC 32 +#define R9A07G044_PD_USB0 33 +#define R9A07G044_PD_USB1 34 +#define R9A07G044_PD_USB_PHY 35 +#define R9A07G044_PD_ETHER0 36 +#define R9A07G044_PD_ETHER1 37 +#define R9A07G044_PD_I2C0 38 +#define R9A07G044_PD_I2C1 39 +#define R9A07G044_PD_I2C2 40 +#define R9A07G044_PD_I2C3 41 +#define R9A07G044_PD_SCIF0 42 +#define R9A07G044_PD_SCIF1 43 +#define R9A07G044_PD_SCIF2 44 +#define R9A07G044_PD_SCIF3 45 +#define R9A07G044_PD_SCIF4 46 +#define R9A07G044_PD_SCI0 47 +#define R9A07G044_PD_SCI1 48 +#define R9A07G044_PD_IRDA 49 +#define R9A07G044_PD_RSPI0 50 +#define R9A07G044_PD_RSPI1 51 +#define R9A07G044_PD_RSPI2 52 +#define R9A07G044_PD_CANFD 53 +#define R9A07G044_PD_ADC 54 +#define R9A07G044_PD_TSU 55 + #endif /* __DT_BINDINGS_CLOCK_R9A07G044_CPG_H__ */ From 5b9979fda3058c786706d86b6e0ecb8e71be5a84 Mon Sep 17 00:00:00 2001 From: Claudiu Beznea Date: Mon, 22 Apr 2024 13:53:50 +0300 Subject: [PATCH 0523/1702] dt-bindings: clock: r9a07g054-cpg: Add power domain IDs Add power domain IDs for the RZ/V2L (R9A07G054) SoC. Signed-off-by: Claudiu Beznea Reviewed-by: Geert Uytterhoeven Acked-by: Rob Herring Reviewed-by: Ulf Hansson Link: https://lore.kernel.org/r/20240422105355.1622177-4-claudiu.beznea.uj@bp.renesas.com Signed-off-by: Geert Uytterhoeven --- include/dt-bindings/clock/r9a07g054-cpg.h | 58 +++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/include/dt-bindings/clock/r9a07g054-cpg.h b/include/dt-bindings/clock/r9a07g054-cpg.h index 43f4dbda872cc..2c99f89397c4c 100644 --- a/include/dt-bindings/clock/r9a07g054-cpg.h +++ b/include/dt-bindings/clock/r9a07g054-cpg.h @@ -226,4 +226,62 @@ #define R9A07G054_TSU_PRESETN 83 #define R9A07G054_STPAI_ARESETN 84 +/* Power domain IDs. */ +#define R9A07G054_PD_ALWAYS_ON 0 +#define R9A07G054_PD_GIC 1 +#define R9A07G054_PD_IA55 2 +#define R9A07G054_PD_MHU 3 +#define R9A07G054_PD_CORESIGHT 4 +#define R9A07G054_PD_SYC 5 +#define R9A07G054_PD_DMAC 6 +#define R9A07G054_PD_GTM0 7 +#define R9A07G054_PD_GTM1 8 +#define R9A07G054_PD_GTM2 9 +#define R9A07G054_PD_MTU 10 +#define R9A07G054_PD_POE3 11 +#define R9A07G054_PD_GPT 12 +#define R9A07G054_PD_POEGA 13 +#define R9A07G054_PD_POEGB 14 +#define R9A07G054_PD_POEGC 15 +#define R9A07G054_PD_POEGD 16 +#define R9A07G054_PD_WDT0 17 +#define R9A07G054_PD_WDT1 18 +#define R9A07G054_PD_SPI 19 +#define R9A07G054_PD_SDHI0 20 +#define R9A07G054_PD_SDHI1 21 +#define R9A07G054_PD_3DGE 22 +#define R9A07G054_PD_ISU 23 +#define R9A07G054_PD_VCPL4 24 +#define R9A07G054_PD_CRU 25 +#define R9A07G054_PD_MIPI_DSI 26 +#define R9A07G054_PD_LCDC 27 +#define R9A07G054_PD_SSI0 28 +#define R9A07G054_PD_SSI1 29 +#define R9A07G054_PD_SSI2 30 +#define R9A07G054_PD_SSI3 31 +#define R9A07G054_PD_SRC 32 +#define R9A07G054_PD_USB0 33 +#define R9A07G054_PD_USB1 34 +#define R9A07G054_PD_USB_PHY 35 +#define R9A07G054_PD_ETHER0 36 +#define R9A07G054_PD_ETHER1 37 +#define R9A07G054_PD_I2C0 38 +#define R9A07G054_PD_I2C1 39 +#define R9A07G054_PD_I2C2 40 +#define R9A07G054_PD_I2C3 41 +#define R9A07G054_PD_SCIF0 42 +#define R9A07G054_PD_SCIF1 43 +#define R9A07G054_PD_SCIF2 44 +#define R9A07G054_PD_SCIF3 45 +#define R9A07G054_PD_SCIF4 46 +#define R9A07G054_PD_SCI0 47 +#define R9A07G054_PD_SCI1 48 +#define R9A07G054_PD_IRDA 49 +#define R9A07G054_PD_RSPI0 50 +#define R9A07G054_PD_RSPI1 51 +#define R9A07G054_PD_RSPI2 52 +#define R9A07G054_PD_CANFD 53 +#define R9A07G054_PD_ADC 54 +#define R9A07G054_PD_TSU 55 + #endif /* __DT_BINDINGS_CLOCK_R9A07G054_CPG_H__ */ From 2d03ce9cd7bdd1e29772f8ecf7769535361c7eaa Mon Sep 17 00:00:00 2001 From: Claudiu Beznea Date: Mon, 22 Apr 2024 13:53:51 +0300 Subject: [PATCH 0524/1702] dt-bindings: clock: r9a08g045-cpg: Add power domain IDs Add power domain IDs for the RZ/G3S (R9A08G045) SoC. Signed-off-by: Claudiu Beznea Reviewed-by: Geert Uytterhoeven Acked-by: Rob Herring Reviewed-by: Ulf Hansson Link: https://lore.kernel.org/r/20240422105355.1622177-5-claudiu.beznea.uj@bp.renesas.com Signed-off-by: Geert Uytterhoeven --- include/dt-bindings/clock/r9a08g045-cpg.h | 70 +++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/include/dt-bindings/clock/r9a08g045-cpg.h b/include/dt-bindings/clock/r9a08g045-cpg.h index 410725b778a86..8281e9caf3a9e 100644 --- a/include/dt-bindings/clock/r9a08g045-cpg.h +++ b/include/dt-bindings/clock/r9a08g045-cpg.h @@ -239,4 +239,74 @@ #define R9A08G045_I3C_PRESETN 92 #define R9A08G045_VBAT_BRESETN 93 +/* Power domain IDs. */ +#define R9A08G045_PD_ALWAYS_ON 0 +#define R9A08G045_PD_GIC 1 +#define R9A08G045_PD_IA55 2 +#define R9A08G045_PD_MHU 3 +#define R9A08G045_PD_CORESIGHT 4 +#define R9A08G045_PD_SYC 5 +#define R9A08G045_PD_DMAC 6 +#define R9A08G045_PD_GTM0 7 +#define R9A08G045_PD_GTM1 8 +#define R9A08G045_PD_GTM2 9 +#define R9A08G045_PD_GTM3 10 +#define R9A08G045_PD_GTM4 11 +#define R9A08G045_PD_GTM5 12 +#define R9A08G045_PD_GTM6 13 +#define R9A08G045_PD_GTM7 14 +#define R9A08G045_PD_MTU 15 +#define R9A08G045_PD_POE3 16 +#define R9A08G045_PD_GPT 17 +#define R9A08G045_PD_POEGA 18 +#define R9A08G045_PD_POEGB 19 +#define R9A08G045_PD_POEGC 20 +#define R9A08G045_PD_POEGD 21 +#define R9A08G045_PD_WDT0 22 +#define R9A08G045_PD_XSPI 23 +#define R9A08G045_PD_SDHI0 24 +#define R9A08G045_PD_SDHI1 25 +#define R9A08G045_PD_SDHI2 26 +#define R9A08G045_PD_SSI0 27 +#define R9A08G045_PD_SSI1 28 +#define R9A08G045_PD_SSI2 29 +#define R9A08G045_PD_SSI3 30 +#define R9A08G045_PD_SRC 31 +#define R9A08G045_PD_USB0 32 +#define R9A08G045_PD_USB1 33 +#define R9A08G045_PD_USB_PHY 34 +#define R9A08G045_PD_ETHER0 35 +#define R9A08G045_PD_ETHER1 36 +#define R9A08G045_PD_I2C0 37 +#define R9A08G045_PD_I2C1 38 +#define R9A08G045_PD_I2C2 39 +#define R9A08G045_PD_I2C3 40 +#define R9A08G045_PD_SCIF0 41 +#define R9A08G045_PD_SCIF1 42 +#define R9A08G045_PD_SCIF2 43 +#define R9A08G045_PD_SCIF3 44 +#define R9A08G045_PD_SCIF4 45 +#define R9A08G045_PD_SCIF5 46 +#define R9A08G045_PD_SCI0 47 +#define R9A08G045_PD_SCI1 48 +#define R9A08G045_PD_IRDA 49 +#define R9A08G045_PD_RSPI0 50 +#define R9A08G045_PD_RSPI1 51 +#define R9A08G045_PD_RSPI2 52 +#define R9A08G045_PD_RSPI3 53 +#define R9A08G045_PD_RSPI4 54 +#define R9A08G045_PD_CANFD 55 +#define R9A08G045_PD_ADC 56 +#define R9A08G045_PD_TSU 57 +#define R9A08G045_PD_OCTA 58 +#define R9A08G045_PD_PDM 59 +#define R9A08G045_PD_PCI 60 +#define R9A08G045_PD_SPDIF 61 +#define R9A08G045_PD_I3C 62 +#define R9A08G045_PD_VBAT 63 + +#define R9A08G045_PD_DDR 64 +#define R9A08G045_PD_TZCDDR 65 +#define R9A08G045_PD_OTFDE_DDR 66 + #endif /* __DT_BINDINGS_CLOCK_R9A08G045_CPG_H__ */ From f33dca9ed6f41c8acf2c17c402738deddb7d7c28 Mon Sep 17 00:00:00 2001 From: Claudiu Beznea Date: Mon, 22 Apr 2024 13:53:52 +0300 Subject: [PATCH 0525/1702] dt-bindings: clock: renesas,rzg2l-cpg: Update #power-domain-cells = <1> for RZ/G3S The driver will be modified (in the next commits) to be able to specify individual power domain IDs for each IP. The driver will still support #power-domain-cells = <0>, thus, previous users are not affected. The #power-domain-cells = <1> has been instantiated only for RZ/G3S at the moment, as individual platform clock drivers need to be adapted for this to be supported on the rest of the SoCs. Also, the description for #power-domain-cells is updated with links to per-SoC power domain IDs. Signed-off-by: Claudiu Beznea Reviewed-by: Rob Herring Reviewed-by: Geert Uytterhoeven Reviewed-by: Ulf Hansson Link: https://lore.kernel.org/r/20240422105355.1622177-6-claudiu.beznea.uj@bp.renesas.com Signed-off-by: Geert Uytterhoeven --- .../bindings/clock/renesas,rzg2l-cpg.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/clock/renesas,rzg2l-cpg.yaml b/Documentation/devicetree/bindings/clock/renesas,rzg2l-cpg.yaml index 80a8c7114c310..4e3b0c45124ae 100644 --- a/Documentation/devicetree/bindings/clock/renesas,rzg2l-cpg.yaml +++ b/Documentation/devicetree/bindings/clock/renesas,rzg2l-cpg.yaml @@ -57,7 +57,8 @@ properties: can be power-managed through Module Standby should refer to the CPG device node in their "power-domains" property, as documented by the generic PM Domain bindings in Documentation/devicetree/bindings/power/power-domain.yaml. - const: 0 + The power domain specifiers defined in could + be used to reference individual CPG power domains. '#reset-cells': description: @@ -76,6 +77,21 @@ required: additionalProperties: false +allOf: + - if: + properties: + compatible: + contains: + const: renesas,r9a08g045-cpg + then: + properties: + '#power-domain-cells': + const: 1 + else: + properties: + '#power-domain-cells': + const: 0 + examples: - | cpg: clock-controller@11010000 { From 0c8a59b3113ef6184d1e4cf2fb911c641d5172e3 Mon Sep 17 00:00:00 2001 From: Claudiu Beznea Date: Mon, 22 Apr 2024 13:53:53 +0300 Subject: [PATCH 0526/1702] clk: renesas: rzg2l: Extend power domain support RZ/{G2L, V2L, G3S}-based CPG versions have support for saving extra power when clocks are disabled by activating module standby. This is done through MSTOP-specific registers that are part of CPG. Each individual module has one or more bits associated with one MSTOP register (see table "Registers for Module Standby Mode" from HW manuals). Hardware manual associates modules' clocks with one or more MSTOP bits. There are 3 mappings available (identified by researching RZ/G2L, RZ/G3S, RZ/V2L HW manuals): case 1: N clocks mapped to N MSTOP bits (with N={0, ..., X}) case 2: N clocks mapped to 1 MSTOP bit (with N={0, ..., X}) case 3: N clocks mapped to M MSTOP bits (with N={0, ..., X}, M={0, ..., Y}) Case 3 has been currently identified on RZ/V2L for the VCPL4 module. To cover all three cases, the individual platform drivers will provide the clock driver with MSTOP register offsets and associated bits in this register as a bitmask, and the clock driver will apply this bitmask to the proper MSTOP register. The MSTOP support was implemented through power domains. Platform-specific clock drivers will register an array of type struct rzg2l_cpg_pm_domain_init_data, which will be used to instantiate properly the power domains. Signed-off-by: Claudiu Beznea Reviewed-by: Geert Uytterhoeven Reviewed-by: Ulf Hansson Link: https://lore.kernel.org/r/20240422105355.1622177-7-claudiu.beznea.uj@bp.renesas.com Signed-off-by: Geert Uytterhoeven --- drivers/clk/renesas/rzg2l-cpg.c | 199 +++++++++++++++++++++++++++++--- drivers/clk/renesas/rzg2l-cpg.h | 67 +++++++++++ 2 files changed, 252 insertions(+), 14 deletions(-) diff --git a/drivers/clk/renesas/rzg2l-cpg.c b/drivers/clk/renesas/rzg2l-cpg.c index 3d2daa4ba2a4b..04b78064d4e01 100644 --- a/drivers/clk/renesas/rzg2l-cpg.c +++ b/drivers/clk/renesas/rzg2l-cpg.c @@ -139,7 +139,6 @@ struct rzg2l_pll5_mux_dsi_div_param { * @num_resets: Number of Module Resets in info->resets[] * @last_dt_core_clk: ID of the last Core Clock exported to DT * @info: Pointer to platform data - * @genpd: PM domain * @mux_dsi_div_params: pll5 mux and dsi div parameters */ struct rzg2l_cpg_priv { @@ -156,8 +155,6 @@ struct rzg2l_cpg_priv { const struct rzg2l_cpg_info *info; - struct generic_pm_domain genpd; - struct rzg2l_pll5_mux_dsi_div_param mux_dsi_div_params; }; @@ -1559,9 +1556,34 @@ static bool rzg2l_cpg_is_pm_clk(struct rzg2l_cpg_priv *priv, return true; } +/** + * struct rzg2l_cpg_pm_domains - RZ/G2L PM domains data structure + * @onecell_data: cell data + * @domains: generic PM domains + */ +struct rzg2l_cpg_pm_domains { + struct genpd_onecell_data onecell_data; + struct generic_pm_domain *domains[]; +}; + +/** + * struct rzg2l_cpg_pd - RZ/G2L power domain data structure + * @genpd: generic PM domain + * @priv: pointer to CPG private data structure + * @conf: CPG PM domain configuration info + * @id: RZ/G2L power domain ID + */ +struct rzg2l_cpg_pd { + struct generic_pm_domain genpd; + struct rzg2l_cpg_priv *priv; + struct rzg2l_cpg_pm_domain_conf conf; + u16 id; +}; + static int rzg2l_cpg_attach_dev(struct generic_pm_domain *domain, struct device *dev) { - struct rzg2l_cpg_priv *priv = container_of(domain, struct rzg2l_cpg_priv, genpd); + struct rzg2l_cpg_pd *pd = container_of(domain, struct rzg2l_cpg_pd, genpd); + struct rzg2l_cpg_priv *priv = pd->priv; struct device_node *np = dev->of_node; struct of_phandle_args clkspec; bool once = true; @@ -1617,31 +1639,180 @@ static void rzg2l_cpg_detach_dev(struct generic_pm_domain *unused, struct device } static void rzg2l_cpg_genpd_remove(void *data) +{ + struct genpd_onecell_data *celldata = data; + + for (unsigned int i = 0; i < celldata->num_domains; i++) + pm_genpd_remove(celldata->domains[i]); +} + +static void rzg2l_cpg_genpd_remove_simple(void *data) { pm_genpd_remove(data); } +static int rzg2l_cpg_power_on(struct generic_pm_domain *domain) +{ + struct rzg2l_cpg_pd *pd = container_of(domain, struct rzg2l_cpg_pd, genpd); + struct rzg2l_cpg_reg_conf mstop = pd->conf.mstop; + struct rzg2l_cpg_priv *priv = pd->priv; + + /* Set MSTOP. */ + if (mstop.mask) + writel(mstop.mask << 16, priv->base + mstop.off); + + return 0; +} + +static int rzg2l_cpg_power_off(struct generic_pm_domain *domain) +{ + struct rzg2l_cpg_pd *pd = container_of(domain, struct rzg2l_cpg_pd, genpd); + struct rzg2l_cpg_reg_conf mstop = pd->conf.mstop; + struct rzg2l_cpg_priv *priv = pd->priv; + + /* Set MSTOP. */ + if (mstop.mask) + writel(mstop.mask | (mstop.mask << 16), priv->base + mstop.off); + + return 0; +} + +static int __init rzg2l_cpg_pd_setup(struct rzg2l_cpg_pd *pd, bool always_on) +{ + struct dev_power_governor *governor; + + pd->genpd.flags |= GENPD_FLAG_PM_CLK | GENPD_FLAG_ACTIVE_WAKEUP; + pd->genpd.attach_dev = rzg2l_cpg_attach_dev; + pd->genpd.detach_dev = rzg2l_cpg_detach_dev; + if (always_on) { + pd->genpd.flags |= GENPD_FLAG_ALWAYS_ON; + governor = &pm_domain_always_on_gov; + } else { + pd->genpd.power_on = rzg2l_cpg_power_on; + pd->genpd.power_off = rzg2l_cpg_power_off; + governor = &simple_qos_governor; + } + + return pm_genpd_init(&pd->genpd, governor, !always_on); +} + static int __init rzg2l_cpg_add_clk_domain(struct rzg2l_cpg_priv *priv) { struct device *dev = priv->dev; struct device_node *np = dev->of_node; - struct generic_pm_domain *genpd = &priv->genpd; + struct rzg2l_cpg_pd *pd; + int ret; + + pd = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL); + if (!pd) + return -ENOMEM; + + pd->genpd.name = np->name; + pd->priv = priv; + ret = rzg2l_cpg_pd_setup(pd, true); + if (ret) + return ret; + + ret = devm_add_action_or_reset(dev, rzg2l_cpg_genpd_remove_simple, &pd->genpd); + if (ret) + return ret; + + return of_genpd_add_provider_simple(np, &pd->genpd); +} + +static struct generic_pm_domain * +rzg2l_cpg_pm_domain_xlate(const struct of_phandle_args *spec, void *data) +{ + struct generic_pm_domain *domain = ERR_PTR(-ENOENT); + struct genpd_onecell_data *genpd = data; + + if (spec->args_count != 1) + return ERR_PTR(-EINVAL); + + for (unsigned int i = 0; i < genpd->num_domains; i++) { + struct rzg2l_cpg_pd *pd = container_of(genpd->domains[i], struct rzg2l_cpg_pd, + genpd); + + if (pd->id == spec->args[0]) { + domain = &pd->genpd; + break; + } + } + + return domain; +} + +static int __init rzg2l_cpg_add_pm_domains(struct rzg2l_cpg_priv *priv) +{ + const struct rzg2l_cpg_info *info = priv->info; + struct device *dev = priv->dev; + struct device_node *np = dev->of_node; + struct rzg2l_cpg_pm_domains *domains; + struct generic_pm_domain *parent; + u32 ncells; int ret; - genpd->name = np->name; - genpd->flags = GENPD_FLAG_PM_CLK | GENPD_FLAG_ALWAYS_ON | - GENPD_FLAG_ACTIVE_WAKEUP; - genpd->attach_dev = rzg2l_cpg_attach_dev; - genpd->detach_dev = rzg2l_cpg_detach_dev; - ret = pm_genpd_init(genpd, &pm_domain_always_on_gov, false); + ret = of_property_read_u32(np, "#power-domain-cells", &ncells); + if (ret) + return ret; + + /* For backward compatibility. */ + if (!ncells) + return rzg2l_cpg_add_clk_domain(priv); + + domains = devm_kzalloc(dev, struct_size(domains, domains, info->num_pm_domains), + GFP_KERNEL); + if (!domains) + return -ENOMEM; + + domains->onecell_data.domains = domains->domains; + domains->onecell_data.num_domains = info->num_pm_domains; + domains->onecell_data.xlate = rzg2l_cpg_pm_domain_xlate; + + ret = devm_add_action_or_reset(dev, rzg2l_cpg_genpd_remove, &domains->onecell_data); if (ret) return ret; - ret = devm_add_action_or_reset(dev, rzg2l_cpg_genpd_remove, genpd); + for (unsigned int i = 0; i < info->num_pm_domains; i++) { + bool always_on = !!(info->pm_domains[i].flags & RZG2L_PD_F_ALWAYS_ON); + struct rzg2l_cpg_pd *pd; + + pd = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL); + if (!pd) + return -ENOMEM; + + pd->genpd.name = info->pm_domains[i].name; + pd->conf = info->pm_domains[i].conf; + pd->id = info->pm_domains[i].id; + pd->priv = priv; + + ret = rzg2l_cpg_pd_setup(pd, always_on); + if (ret) + return ret; + + if (always_on) { + ret = rzg2l_cpg_power_on(&pd->genpd); + if (ret) + return ret; + } + + domains->domains[i] = &pd->genpd; + /* Parent should be on the very first entry of info->pm_domains[]. */ + if (!i) { + parent = &pd->genpd; + continue; + } + + ret = pm_genpd_add_subdomain(parent, &pd->genpd); + if (ret) + return ret; + } + + ret = of_genpd_add_provider_onecell(np, &domains->onecell_data); if (ret) return ret; - return of_genpd_add_provider_simple(np, genpd); + return 0; } static int __init rzg2l_cpg_probe(struct platform_device *pdev) @@ -1697,7 +1868,7 @@ static int __init rzg2l_cpg_probe(struct platform_device *pdev) if (error) return error; - error = rzg2l_cpg_add_clk_domain(priv); + error = rzg2l_cpg_add_pm_domains(priv); if (error) return error; diff --git a/drivers/clk/renesas/rzg2l-cpg.h b/drivers/clk/renesas/rzg2l-cpg.h index 6e38c8fc888c8..ecfe7e7ea8a17 100644 --- a/drivers/clk/renesas/rzg2l-cpg.h +++ b/drivers/clk/renesas/rzg2l-cpg.h @@ -27,6 +27,18 @@ #define CPG_PL6_ETH_SSEL (0x418) #define CPG_PL5_SDIV (0x420) #define CPG_RST_MON (0x680) +#define CPG_BUS_ACPU_MSTOP (0xB60) +#define CPG_BUS_MCPU1_MSTOP (0xB64) +#define CPG_BUS_MCPU2_MSTOP (0xB68) +#define CPG_BUS_PERI_COM_MSTOP (0xB6C) +#define CPG_BUS_PERI_CPU_MSTOP (0xB70) +#define CPG_BUS_PERI_DDR_MSTOP (0xB74) +#define CPG_BUS_REG0_MSTOP (0xB7C) +#define CPG_BUS_REG1_MSTOP (0xB80) +#define CPG_BUS_TZCDDR_MSTOP (0xB84) +#define CPG_MHU_MSTOP (0xB88) +#define CPG_BUS_MCPU3_MSTOP (0xB90) +#define CPG_BUS_PERI_CPU2_MSTOP (0xB94) #define CPG_OTHERFUNC1_REG (0xBE8) #define CPG_SIPLL5_STBY_RESETB BIT(0) @@ -234,6 +246,55 @@ struct rzg2l_reset { #define DEF_RST(_id, _off, _bit) \ DEF_RST_MON(_id, _off, _bit, -1) +/** + * struct rzg2l_cpg_reg_conf - RZ/G2L register configuration data structure + * @off: register offset + * @mask: register mask + */ +struct rzg2l_cpg_reg_conf { + u16 off; + u16 mask; +}; + +#define DEF_REG_CONF(_off, _mask) ((struct rzg2l_cpg_reg_conf) { .off = (_off), .mask = (_mask) }) + +/** + * struct rzg2l_cpg_pm_domain_conf - PM domain configuration data structure + * @mstop: MSTOP register configuration + */ +struct rzg2l_cpg_pm_domain_conf { + struct rzg2l_cpg_reg_conf mstop; +}; + +/** + * struct rzg2l_cpg_pm_domain_init_data - PM domain init data + * @name: PM domain name + * @conf: PM domain configuration + * @flags: RZG2L PM domain flags (see RZG2L_PD_F_*) + * @id: PM domain ID (similar to the ones defined in + * include/dt-bindings/clock/-cpg.h) + */ +struct rzg2l_cpg_pm_domain_init_data { + const char * const name; + struct rzg2l_cpg_pm_domain_conf conf; + u32 flags; + u16 id; +}; + +#define DEF_PD(_name, _id, _mstop_conf, _flags) \ + { \ + .name = (_name), \ + .id = (_id), \ + .conf = { \ + .mstop = (_mstop_conf), \ + }, \ + .flags = (_flags), \ + } + +/* Power domain flags. */ +#define RZG2L_PD_F_ALWAYS_ON BIT(0) +#define RZG2L_PD_F_NONE (0) + /** * struct rzg2l_cpg_info - SoC-specific CPG Description * @@ -252,6 +313,8 @@ struct rzg2l_reset { * @crit_mod_clks: Array with Module Clock IDs of critical clocks that * should not be disabled without a knowledgeable driver * @num_crit_mod_clks: Number of entries in crit_mod_clks[] + * @pm_domains: PM domains init data array + * @num_pm_domains: Number of PM domains * @has_clk_mon_regs: Flag indicating whether the SoC has CLK_MON registers */ struct rzg2l_cpg_info { @@ -278,6 +341,10 @@ struct rzg2l_cpg_info { const unsigned int *crit_mod_clks; unsigned int num_crit_mod_clks; + /* Power domain. */ + const struct rzg2l_cpg_pm_domain_init_data *pm_domains; + unsigned int num_pm_domains; + bool has_clk_mon_regs; }; From 5add5ebc4e35a703a49976abfd82e708d9aea4ad Mon Sep 17 00:00:00 2001 From: Claudiu Beznea Date: Mon, 22 Apr 2024 13:53:54 +0300 Subject: [PATCH 0527/1702] clk: renesas: r9a08g045: Add support for power domains Instantiate power domains for the currently enabled IPs of the R9A08G045 SoC. Signed-off-by: Claudiu Beznea Reviewed-by: Geert Uytterhoeven Reviewed-by: Ulf Hansson Link: https://lore.kernel.org/r/20240422105355.1622177-8-claudiu.beznea.uj@bp.renesas.com Signed-off-by: Geert Uytterhoeven --- drivers/clk/renesas/r9a08g045-cpg.c | 41 +++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/drivers/clk/renesas/r9a08g045-cpg.c b/drivers/clk/renesas/r9a08g045-cpg.c index c3e6da2de197f..b068733b145f4 100644 --- a/drivers/clk/renesas/r9a08g045-cpg.c +++ b/drivers/clk/renesas/r9a08g045-cpg.c @@ -240,6 +240,43 @@ static const unsigned int r9a08g045_crit_mod_clks[] __initconst = { MOD_CLK_BASE + R9A08G045_DMAC_ACLK, }; +static const struct rzg2l_cpg_pm_domain_init_data r9a08g045_pm_domains[] = { + /* Keep always-on domain on the first position for proper domains registration. */ + DEF_PD("always-on", R9A08G045_PD_ALWAYS_ON, + DEF_REG_CONF(0, 0), + RZG2L_PD_F_ALWAYS_ON), + DEF_PD("gic", R9A08G045_PD_GIC, + DEF_REG_CONF(CPG_BUS_ACPU_MSTOP, BIT(3)), + RZG2L_PD_F_ALWAYS_ON), + DEF_PD("ia55", R9A08G045_PD_IA55, + DEF_REG_CONF(CPG_BUS_PERI_CPU_MSTOP, BIT(13)), + RZG2L_PD_F_ALWAYS_ON), + DEF_PD("dmac", R9A08G045_PD_DMAC, + DEF_REG_CONF(CPG_BUS_REG1_MSTOP, GENMASK(3, 0)), + RZG2L_PD_F_ALWAYS_ON), + DEF_PD("wdt0", R9A08G045_PD_WDT0, + DEF_REG_CONF(CPG_BUS_REG0_MSTOP, BIT(0)), + RZG2L_PD_F_NONE), + DEF_PD("sdhi0", R9A08G045_PD_SDHI0, + DEF_REG_CONF(CPG_BUS_PERI_COM_MSTOP, BIT(0)), + RZG2L_PD_F_NONE), + DEF_PD("sdhi1", R9A08G045_PD_SDHI1, + DEF_REG_CONF(CPG_BUS_PERI_COM_MSTOP, BIT(1)), + RZG2L_PD_F_NONE), + DEF_PD("sdhi2", R9A08G045_PD_SDHI2, + DEF_REG_CONF(CPG_BUS_PERI_COM_MSTOP, BIT(11)), + RZG2L_PD_F_NONE), + DEF_PD("eth0", R9A08G045_PD_ETHER0, + DEF_REG_CONF(CPG_BUS_PERI_COM_MSTOP, BIT(2)), + RZG2L_PD_F_NONE), + DEF_PD("eth1", R9A08G045_PD_ETHER1, + DEF_REG_CONF(CPG_BUS_PERI_COM_MSTOP, BIT(3)), + RZG2L_PD_F_NONE), + DEF_PD("scif0", R9A08G045_PD_SCIF0, + DEF_REG_CONF(CPG_BUS_MCPU2_MSTOP, BIT(1)), + RZG2L_PD_F_NONE), +}; + const struct rzg2l_cpg_info r9a08g045_cpg_info = { /* Core Clocks */ .core_clks = r9a08g045_core_clks, @@ -260,5 +297,9 @@ const struct rzg2l_cpg_info r9a08g045_cpg_info = { .resets = r9a08g045_resets, .num_resets = R9A08G045_VBAT_BRESETN + 1, /* Last reset ID + 1 */ + /* Power domains */ + .pm_domains = r9a08g045_pm_domains, + .num_pm_domains = ARRAY_SIZE(r9a08g045_pm_domains), + .has_clk_mon_regs = true, }; From 5b0a67008b0d608bb7585fe4c3c0c09d5ceaf1c9 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Sat, 16 Mar 2024 12:52:41 +0100 Subject: [PATCH 0528/1702] mm: remove guard around pgd_offset_k() macro The last architecture redefining pgd_offset_k() was IA64 and it was removed by commit cf8e8658100d ("arch: Remove Itanium (IA-64) architecture") There is no need anymore to guard generic version of pgd_offset_k() with #ifndef pgd_offset_k Link: https://lkml.kernel.org/r/59d3f47d5615d18cca1986f269be2fcb3df34556.1710589838.git.christophe.leroy@csgroup.eu Signed-off-by: Christophe Leroy Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 85fc7554cd52b..bd9c7180718cd 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -149,9 +149,7 @@ static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address) * a shortcut which implies the use of the kernel's pgd, instead * of a process's */ -#ifndef pgd_offset_k #define pgd_offset_k(address) pgd_offset(&init_mm, (address)) -#endif /* * In many cases it is known that a virtual address is mapped at PMD or PTE From 91b71e78b8e478e1a9af36b15ff1d38810572866 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Sat, 16 Mar 2024 01:58:03 +0000 Subject: [PATCH 0529/1702] mm: memcg: add NULL check to obj_cgroup_put() 9 out of 16 callers perform a NULL check before calling obj_cgroup_put(). Move the NULL check in the function, similar to mem_cgroup_put(). The unlikely() NULL check in current_objcg_update() was left alone to avoid dropping the unlikey() annotation as this a fast path. Link: https://lkml.kernel.org/r/20240316015803.2777252-1-yosryahmed@google.com Signed-off-by: Yosry Ahmed Acked-by: Johannes Weiner Acked-by: Roman Gushchin Cc: Michal Hocko Cc: Muchun Song Cc: Shakeel Butt Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 3 ++- kernel/bpf/memalloc.c | 6 ++---- mm/memcontrol.c | 18 ++++++------------ mm/zswap.c | 3 +-- 4 files changed, 11 insertions(+), 19 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 394fd0a887ae7..b6264796815d4 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -818,7 +818,8 @@ static inline void obj_cgroup_get_many(struct obj_cgroup *objcg, static inline void obj_cgroup_put(struct obj_cgroup *objcg) { - percpu_ref_put(&objcg->refcnt); + if (objcg) + percpu_ref_put(&objcg->refcnt); } static inline bool mem_cgroup_tryget(struct mem_cgroup *memcg) diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c index 550f02e2cb132..a546aba46d5da 100644 --- a/kernel/bpf/memalloc.c +++ b/kernel/bpf/memalloc.c @@ -759,8 +759,7 @@ void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma) rcu_in_progress += atomic_read(&c->call_rcu_ttrace_in_progress); rcu_in_progress += atomic_read(&c->call_rcu_in_progress); } - if (ma->objcg) - obj_cgroup_put(ma->objcg); + obj_cgroup_put(ma->objcg); destroy_mem_alloc(ma, rcu_in_progress); } if (ma->caches) { @@ -776,8 +775,7 @@ void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma) rcu_in_progress += atomic_read(&c->call_rcu_in_progress); } } - if (ma->objcg) - obj_cgroup_put(ma->objcg); + obj_cgroup_put(ma->objcg); destroy_mem_alloc(ma, rcu_in_progress); } } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index fabce2b50c695..786c0fae6e309 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2369,8 +2369,7 @@ static void drain_local_stock(struct work_struct *dummy) clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); local_unlock_irqrestore(&memcg_stock.stock_lock, flags); - if (old) - obj_cgroup_put(old); + obj_cgroup_put(old); } /* @@ -3145,8 +3144,7 @@ static struct obj_cgroup *current_objcg_update(void) if (old) { old = (struct obj_cgroup *) ((unsigned long)old & ~CURRENT_OBJCG_UPDATE_FLAG); - if (old) - obj_cgroup_put(old); + obj_cgroup_put(old); old = NULL; } @@ -3418,8 +3416,7 @@ void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, mod_objcg_mlstate(objcg, pgdat, idx, nr); local_unlock_irqrestore(&memcg_stock.stock_lock, flags); - if (old) - obj_cgroup_put(old); + obj_cgroup_put(old); } static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes) @@ -3546,8 +3543,7 @@ static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes, } local_unlock_irqrestore(&memcg_stock.stock_lock, flags); - if (old) - obj_cgroup_put(old); + obj_cgroup_put(old); if (nr_pages) obj_cgroup_uncharge_pages(objcg, nr_pages); @@ -5468,8 +5464,7 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg) { int node; - if (memcg->orig_objcg) - obj_cgroup_put(memcg->orig_objcg); + obj_cgroup_put(memcg->orig_objcg); for_each_node(node) free_mem_cgroup_per_node_info(memcg, node); @@ -6620,8 +6615,7 @@ static void mem_cgroup_exit(struct task_struct *task) objcg = (struct obj_cgroup *) ((unsigned long)objcg & ~CURRENT_OBJCG_UPDATE_FLAG); - if (objcg) - obj_cgroup_put(objcg); + obj_cgroup_put(objcg); /* * Some kernel allocations can happen after this point, diff --git a/mm/zswap.c b/mm/zswap.c index 6f8850c44b616..2817e2791c135 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1618,8 +1618,7 @@ bool zswap_store(struct folio *folio) freepage: zswap_entry_cache_free(entry); reject: - if (objcg) - obj_cgroup_put(objcg); + obj_cgroup_put(objcg); check_old: /* * If the zswap store fails or zswap is disabled, we must invalidate the From fa9fcd8bb6e4901e5ed0c7972547403d4c8dfec8 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 14 Mar 2024 17:13:00 +0100 Subject: [PATCH 0530/1702] mm/madvise: don't perform madvise VMA walk for MADV_POPULATE_(READ|WRITE) We changed faultin_page_range() to no longer consume a VMA, because faultin_page_range() might internally release the mm lock to lookup the VMA again -- required to cleanly handle VM_FAULT_RETRY. But independent of that, __get_user_pages() will always lookup the VMA itself. Now that we let __get_user_pages() just handle VMA checks in a way that is suitable for MADV_POPULATE_(READ|WRITE), the VMA walk in madvise() is just overhead. So let's just call madvise_populate() on the full range instead. There is one change in behavior: madvise_walk_vmas() would skip any VMA holes, and if everything succeeded, it would return -ENOMEM after processing all VMAs. However, for MADV_POPULATE_(READ|WRITE) it's unlikely for the caller to notice any difference: -ENOMEM might either indicate that there were VMA holes or that populating page tables failed because there was not enough memory. So it's unlikely that user space will notice the difference, and that special handling likely only makes sense for some other madvise() actions. Further, we'd already fail with -ENOMEM early in the past if looking up the VMA after dropping the MM lock failed because of concurrent VMA modifications. So let's just keep it simple and avoid the madvise VMA walk, and consistently fail early if we find a VMA hole. Link: https://lkml.kernel.org/r/20240314161300.382526-3-david@redhat.com Signed-off-by: David Hildenbrand Cc: Darrick J. Wong Cc: Hugh Dickins Cc: Jason Gunthorpe Cc: John Hubbard Signed-off-by: Andrew Morton --- mm/madvise.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/mm/madvise.c b/mm/madvise.c index 1a073fcc4c0c0..a2dd70c4a2e6b 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -901,26 +901,19 @@ static long madvise_dontneed_free(struct vm_area_struct *vma, return -EINVAL; } -static long madvise_populate(struct vm_area_struct *vma, - struct vm_area_struct **prev, - unsigned long start, unsigned long end, - int behavior) +static long madvise_populate(struct mm_struct *mm, unsigned long start, + unsigned long end, int behavior) { const bool write = behavior == MADV_POPULATE_WRITE; - struct mm_struct *mm = vma->vm_mm; int locked = 1; long pages; - *prev = vma; - while (start < end) { /* Populate (prefault) page tables readable/writable. */ pages = faultin_page_range(mm, start, end, write, &locked); if (!locked) { mmap_read_lock(mm); locked = 1; - *prev = NULL; - vma = NULL; } if (pages < 0) { switch (pages) { @@ -1021,9 +1014,6 @@ static int madvise_vma_behavior(struct vm_area_struct *vma, case MADV_DONTNEED: case MADV_DONTNEED_LOCKED: return madvise_dontneed_free(vma, prev, start, end, behavior); - case MADV_POPULATE_READ: - case MADV_POPULATE_WRITE: - return madvise_populate(vma, prev, start, end, behavior); case MADV_NORMAL: new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ; break; @@ -1425,8 +1415,16 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh end = start + len; blk_start_plug(&plug); - error = madvise_walk_vmas(mm, start, end, behavior, - madvise_vma_behavior); + switch (behavior) { + case MADV_POPULATE_READ: + case MADV_POPULATE_WRITE: + error = madvise_populate(mm, start, end, behavior); + break; + default: + error = madvise_walk_vmas(mm, start, end, behavior, + madvise_vma_behavior); + break; + } blk_finish_plug(&plug); if (write) mmap_write_unlock(mm); From 13e860961fd4505d7247ba69e8516f577c2dee5a Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Thu, 14 Mar 2024 17:52:50 +0530 Subject: [PATCH 0531/1702] selftests/mm: virtual_address_range: Switch to ksft_exit_fail_msg mmap() must not succeed in validate_lower_address_hint(), for if it does, it is a bug in mmap() itself. Reflect this behaviour with ksft_exit_fail_msg(). While at it, do some formatting changes. Link: https://lkml.kernel.org/r/20240314122250.68534-1-dev.jain@arm.com Signed-off-by: Dev Jain Reviewed-by: Muhammad Usama Anjum Cc: Anshuman Khandual Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/virtual_address_range.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/mm/virtual_address_range.c b/tools/testing/selftests/mm/virtual_address_range.c index 7bcf8d48256a6..426ddfc345fbd 100644 --- a/tools/testing/selftests/mm/virtual_address_range.c +++ b/tools/testing/selftests/mm/virtual_address_range.c @@ -85,7 +85,7 @@ static int validate_lower_address_hint(void) char *ptr; ptr = mmap((void *) (1UL << 45), MAP_CHUNK_SIZE, PROT_READ | - PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (ptr == MAP_FAILED) return 0; @@ -105,13 +105,11 @@ int main(int argc, char *argv[]) for (i = 0; i < NR_CHUNKS_LOW; i++) { ptr[i] = mmap(NULL, MAP_CHUNK_SIZE, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (ptr[i] == MAP_FAILED) { - if (validate_lower_address_hint()) { - ksft_test_result_skip("Memory constraint not fulfilled\n"); - ksft_finished(); - } + if (validate_lower_address_hint()) + ksft_exit_fail_msg("mmap unexpectedly succeeded with hint\n"); break; } @@ -127,7 +125,7 @@ int main(int argc, char *argv[]) for (i = 0; i < NR_CHUNKS_HIGH; i++) { hint = hind_addr(); hptr[i] = mmap(hint, MAP_CHUNK_SIZE, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (hptr[i] == MAP_FAILED) break; From 55f77df7d715110299f12c27f4365bd6332d1adb Mon Sep 17 00:00:00 2001 From: Lucas Stach Date: Mon, 18 Mar 2024 21:07:36 +0100 Subject: [PATCH 0532/1702] mm: page_alloc: control latency caused by zone PCP draining Patch series "mm/treewide: Remove pXd_huge() API", v2. In previous work [1], we removed the pXd_large() API, which is arch specific. This patchset further removes the hugetlb pXd_huge() API. Hugetlb was never special on creating huge mappings when compared with other huge mappings. Having a standalone API just to detect such pgtable entries is more or less redundant, especially after the pXd_leaf() API set is introduced with/without CONFIG_HUGETLB_PAGE. When looking at this problem, a few issues are also exposed that we don't have a clear definition of the *_huge() variance API. This patchset started by cleaning these issues first, then replace all *_huge() users to use *_leaf(), then drop all *_huge() code. On x86/sparc, swap entries will be reported "true" in pXd_huge(), while for all the rest archs they're reported "false" instead. This part is done in patch 1-5, in which I suspect patch 1 can be seen as a bug fix, but I'll leave that to hmm experts to decide. Besides, there are three archs (arm, arm64, powerpc) that have slightly different definitions between the *_huge() v.s. *_leaf() variances. I tackled them separately so that it'll be easier for arch experts to chim in when necessary. This part is done in patch 6-9. The final patches 10-14 do the rest on the final removal, since *_leaf() will be the ultimate API in the future, and we seem to have quite some confusions on how *_huge() APIs can be defined, provide a rich comment for *_leaf() API set to define them properly to avoid future misuse, and hopefully that'll also help new archs to start support huge mappings and avoid traps (like either swap entries, or PROT_NONE entry checks). [1] https://lore.kernel.org/r/20240305043750.93762-1-peterx@redhat.com This patch (of 14): When the complete PCP is drained a much larger number of pages than the usual batch size might be freed at once, causing large IRQ and preemption latency spikes, as they are all freed while holding the pcp and zone spinlocks. To avoid those latency spikes, limit the number of pages freed in a single bulk operation to common batch limits. Link: https://lkml.kernel.org/r/20240318200404.448346-1-peterx@redhat.com Link: https://lkml.kernel.org/r/20240318200736.2835502-1-l.stach@pengutronix.de Signed-off-by: Lucas Stach Signed-off-by: Peter Xu Cc: Christophe Leroy Cc: Jason Gunthorpe Cc: "Matthew Wilcox (Oracle)" Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Alistair Popple Cc: Andreas Larsson Cc: "Aneesh Kumar K.V" Cc: Arnd Bergmann Cc: Bjorn Andersson Cc: Borislav Petkov Cc: Catalin Marinas Cc: Dave Hansen Cc: David S. Miller Cc: Fabio Estevam Cc: Ingo Molnar Cc: Konrad Dybcio Cc: Krzysztof Kozlowski Cc: Mark Salter Cc: Michael Ellerman Cc: Naoya Horiguchi Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Russell King Cc: Shawn Guo Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton --- mm/page_alloc.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 14d39f34d3367..5083ac034d26e 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2216,12 +2216,15 @@ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) */ static void drain_pages_zone(unsigned int cpu, struct zone *zone) { - struct per_cpu_pages *pcp; + struct per_cpu_pages *pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); + int count = READ_ONCE(pcp->count); + + while (count) { + int to_drain = min(count, pcp->batch << CONFIG_PCP_BATCH_SCALE_MAX); + count -= to_drain; - pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); - if (pcp->count) { spin_lock(&pcp->lock); - free_pcppages_bulk(zone, pcp->count, pcp, 0); + free_pcppages_bulk(zone, to_drain, pcp, 0); spin_unlock(&pcp->lock); } } From 9abc71b47b7dcb81bd87fece43058d686b46e465 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 18 Mar 2024 16:03:51 -0400 Subject: [PATCH 0533/1702] mm/hmm: process pud swap entry without pud_huge() Swap pud entries do not always return true for pud_huge() for all archs. x86 and sparc (so far) allow it, but all the rest do not accept a swap entry to be reported as pud_huge(). So it's not safe to check swap entries within pud_huge(). Check swap entries before pud_huge(), so it should be always safe. This is the only place in the kernel that (IMHO, wrongly) relies on pud_huge() to return true on pud swap entries. The plan is to cleanup pXd_huge() to only report non-swap mappings for all archs. Link: https://lkml.kernel.org/r/20240318200404.448346-2-peterx@redhat.com Signed-off-by: Peter Xu Reviewed-by: Jason Gunthorpe Cc: Alistair Popple Cc: Andreas Larsson Cc: "Aneesh Kumar K.V" Cc: Arnd Bergmann Cc: Bjorn Andersson Cc: Borislav Petkov Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: David S. Miller Cc: Fabio Estevam Cc: Ingo Molnar Cc: Konrad Dybcio Cc: Krzysztof Kozlowski Cc: Lucas Stach Cc: Mark Salter Cc: "Matthew Wilcox (Oracle)" Cc: Michael Ellerman Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Naoya Horiguchi Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Russell King Cc: Shawn Guo Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton --- mm/hmm.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/mm/hmm.c b/mm/hmm.c index 277ddcab4947d..c95b9ec5d95fe 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -424,7 +424,7 @@ static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end, walk->action = ACTION_CONTINUE; pud = READ_ONCE(*pudp); - if (pud_none(pud)) { + if (!pud_present(pud)) { spin_unlock(ptl); return hmm_vma_walk_hole(start, end, -1, walk); } @@ -435,11 +435,6 @@ static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end, unsigned long *hmm_pfns; unsigned long cpu_flags; - if (!pud_present(pud)) { - spin_unlock(ptl); - return hmm_vma_walk_hole(start, end, -1, walk); - } - i = (addr - range->start) >> PAGE_SHIFT; npages = (end - addr) >> PAGE_SHIFT; hmm_pfns = &range->hmm_pfns[i]; From e6fd5564c07c3c749ff3d1b2aa35540b4047e395 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 18 Mar 2024 16:03:52 -0400 Subject: [PATCH 0534/1702] mm/gup: cache p4d in follow_p4d_mask() Add a variable to cache p4d in follow_p4d_mask(). It's a good practise to make sure all the following checks will have a consistent view of the entry. Link: https://lkml.kernel.org/r/20240318200404.448346-3-peterx@redhat.com Signed-off-by: Peter Xu Cc: Alistair Popple Cc: Andreas Larsson Cc: "Aneesh Kumar K.V" Cc: Arnd Bergmann Cc: Bjorn Andersson Cc: Borislav Petkov Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: David S. Miller Cc: Fabio Estevam Cc: Ingo Molnar Cc: Jason Gunthorpe Cc: Konrad Dybcio Cc: Krzysztof Kozlowski Cc: Lucas Stach Cc: Mark Salter Cc: "Matthew Wilcox (Oracle)" Cc: Michael Ellerman Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Naoya Horiguchi Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Russell King Cc: Shawn Guo Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton --- mm/gup.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index 1611e73b1121b..19f6a458683d4 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -772,16 +772,17 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma, unsigned int flags, struct follow_page_context *ctx) { - p4d_t *p4d; + p4d_t *p4dp, p4d; - p4d = p4d_offset(pgdp, address); - if (p4d_none(*p4d)) + p4dp = p4d_offset(pgdp, address); + p4d = READ_ONCE(*p4dp); + if (p4d_none(p4d)) return no_page_table(vma, flags); - BUILD_BUG_ON(p4d_huge(*p4d)); - if (unlikely(p4d_bad(*p4d))) + BUILD_BUG_ON(p4d_huge(p4d)); + if (unlikely(p4d_bad(p4d))) return no_page_table(vma, flags); - return follow_pud_mask(vma, address, p4d, flags, ctx); + return follow_pud_mask(vma, address, p4dp, flags, ctx); } /** From 089f92141ed0327c7dcde0141de351a21c63e091 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 18 Mar 2024 16:03:53 -0400 Subject: [PATCH 0535/1702] mm/gup: check p4d presence before going on Currently there should have no p4d swap entries so it may not matter much, however this may help us to rule out swap entries in pXd_huge() API, which will include p4d_huge(). The p4d_present() checks make it 100% clear that we won't rely on p4d_huge() for swap entries. Link: https://lkml.kernel.org/r/20240318200404.448346-4-peterx@redhat.com Signed-off-by: Peter Xu Cc: Alistair Popple Cc: Andreas Larsson Cc: "Aneesh Kumar K.V" Cc: Arnd Bergmann Cc: Bjorn Andersson Cc: Borislav Petkov Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: David S. Miller Cc: Fabio Estevam Cc: Ingo Molnar Cc: Jason Gunthorpe Cc: Konrad Dybcio Cc: Krzysztof Kozlowski Cc: Lucas Stach Cc: Mark Salter Cc: "Matthew Wilcox (Oracle)" Cc: Michael Ellerman Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Naoya Horiguchi Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Russell King Cc: Shawn Guo Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton --- mm/gup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index 19f6a458683d4..7ed2c99fb72c9 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -776,7 +776,7 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma, p4dp = p4d_offset(pgdp, address); p4d = READ_ONCE(*p4dp); - if (p4d_none(p4d)) + if (!p4d_present(p4d)) return no_page_table(vma, flags); BUILD_BUG_ON(p4d_huge(p4d)); if (unlikely(p4d_bad(p4d))) @@ -3081,7 +3081,7 @@ static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned lo p4d_t p4d = READ_ONCE(*p4dp); next = p4d_addr_end(addr, end); - if (p4d_none(p4d)) + if (!p4d_present(p4d)) return 0; BUILD_BUG_ON(p4d_huge(p4d)); if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) { From d0973cb9b4758cc62bcc12d382bd7015a3dcefca Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 18 Mar 2024 16:03:54 -0400 Subject: [PATCH 0536/1702] mm/x86: change pXd_huge() behavior to exclude swap entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch partly reverts below commits: 3a194f3f8ad0 ("mm/hugetlb: make pud_huge() and follow_huge_pud() aware of non-present pud entry") cbef8478bee5 ("mm/hugetlb: pmd_huge() returns true for non-present hugepage") Right now, pXd_huge() definition across kernel is unclear. We have two groups that think differently on swap entries: - x86/sparc: Allow pXd_huge() to accept swap entries - all the rest: Doesn't allow pXd_huge() to accept swap entries This is so confusing. Since the sparc helpers seem to be added in 2016, which is after x86's (2015), so sparc could have followed a trend. x86 proposed such swap handling in 2015 to resolve hugetlb swap entries hit in GUP, but now GUP guards swap entries with !pXd_present() in all layers so we should be safe. We should define this API properly, one way or another, rather than keep them defined differently across archs. Gut feeling tells me that pXd_huge() shouldn't include swap entries, and it turns out that I am not the only one thinking so, the question was raised when the current pmd_huge() for x86 was proposed by Ville Syrjälä: https://lore.kernel.org/all/Y2WQ7I4LXh8iUIRd@intel.com/ I might also be missing something obvious, but why is it even necessary to treat PRESENT==0+PSE==0 as a huge entry? It is also questioned when Jason Gunthorpe reviewed the other patchset on swap entry handlings: https://lore.kernel.org/all/20240221125753.GQ13330@nvidia.com/ Revert its meaning back to original. It shouldn't have any functional change as we should be ready with guards on !pXd_present() explicitly everywhere. Note that I also dropped the "#if CONFIG_PGTABLE_LEVELS > 2", it was there probably because it was breaking things when 3a194f3f8ad0 was proposed, according to the report here: https://lore.kernel.org/all/Y2LYXItKQyaJTv8j@intel.com/ Now we shouldn't need that. Instead of reverting to _PAGE_PSE raw check, leverage pXd_leaf(). Link: https://lkml.kernel.org/r/20240318200404.448346-5-peterx@redhat.com Signed-off-by: Peter Xu Cc: Naoya Horiguchi Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: Alistair Popple Cc: Andreas Larsson Cc: "Aneesh Kumar K.V" Cc: Arnd Bergmann Cc: Bjorn Andersson Cc: Catalin Marinas Cc: Christophe Leroy Cc: David S. Miller Cc: Fabio Estevam Cc: Jason Gunthorpe Cc: Konrad Dybcio Cc: Krzysztof Kozlowski Cc: Lucas Stach Cc: Mark Salter Cc: "Matthew Wilcox (Oracle)" Cc: Michael Ellerman Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Russell King Cc: Shawn Guo Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/x86/mm/hugetlbpage.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/arch/x86/mm/hugetlbpage.c b/arch/x86/mm/hugetlbpage.c index 5804bbae4f012..8362953a24ce2 100644 --- a/arch/x86/mm/hugetlbpage.c +++ b/arch/x86/mm/hugetlbpage.c @@ -20,29 +20,19 @@ #include /* - * pmd_huge() returns 1 if @pmd is hugetlb related entry, that is normal - * hugetlb entry or non-present (migration or hwpoisoned) hugetlb entry. - * Otherwise, returns 0. + * pmd_huge() returns 1 if @pmd is hugetlb related entry. */ int pmd_huge(pmd_t pmd) { - return !pmd_none(pmd) && - (pmd_val(pmd) & (_PAGE_PRESENT|_PAGE_PSE)) != _PAGE_PRESENT; + return pmd_leaf(pmd); } /* - * pud_huge() returns 1 if @pud is hugetlb related entry, that is normal - * hugetlb entry or non-present (migration or hwpoisoned) hugetlb entry. - * Otherwise, returns 0. + * pud_huge() returns 1 if @pud is hugetlb related entry. */ int pud_huge(pud_t pud) { -#if CONFIG_PGTABLE_LEVELS > 2 - return !pud_none(pud) && - (pud_val(pud) & (_PAGE_PRESENT|_PAGE_PSE)) != _PAGE_PRESENT; -#else - return 0; -#endif + return pud_leaf(pud); } #ifdef CONFIG_HUGETLB_PAGE From ae798490ec20005a6113eff176d6ec803cb514cb Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 18 Mar 2024 16:03:55 -0400 Subject: [PATCH 0537/1702] mm/sparc: change pXd_huge() behavior to exclude swap entries Please refer to the previous patch on the reasoning for x86. Now sparc is the only architecture that will allow swap entries to be reported as pXd_huge(). After this patch, all architectures should forbid swap entries in pXd_huge(). [akpm@linux-foundation.org: s/;;/;/, per Muchun] Link: https://lkml.kernel.org/r/20240318200404.448346-6-peterx@redhat.com Signed-off-by: Peter Xu Cc: David S. Miller Cc: Andreas Larsson Cc: Alistair Popple Cc: "Aneesh Kumar K.V" Cc: Arnd Bergmann Cc: Bjorn Andersson Cc: Borislav Petkov Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: Fabio Estevam Cc: Ingo Molnar Cc: Jason Gunthorpe Cc: Konrad Dybcio Cc: Krzysztof Kozlowski Cc: Lucas Stach Cc: Mark Salter Cc: "Matthew Wilcox (Oracle)" Cc: Michael Ellerman Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Naoya Horiguchi Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Russell King Cc: Shawn Guo Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/sparc/mm/hugetlbpage.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c index b432500c13a5d..532143f1bf634 100644 --- a/arch/sparc/mm/hugetlbpage.c +++ b/arch/sparc/mm/hugetlbpage.c @@ -409,14 +409,12 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, int pmd_huge(pmd_t pmd) { - return !pmd_none(pmd) && - (pmd_val(pmd) & (_PAGE_VALID|_PAGE_PMD_HUGE)) != _PAGE_VALID; + return pmd_leaf(pmd); } int pud_huge(pud_t pud) { - return !pud_none(pud) && - (pud_val(pud) & (_PAGE_VALID|_PAGE_PUD_HUGE)) != _PAGE_VALID; + return pud_leaf(pud); } static void hugetlb_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, From 7966a2b76f117b650b42a94522e748889ce0794c Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 18 Mar 2024 16:03:56 -0400 Subject: [PATCH 0538/1702] mm/arm: use macros to define pmd/pud helpers It's already confusing that ARM 2-level v.s. 3-level defines SECT bit differently on pmd/puds. Always use a macro which is much clearer. Link: https://lkml.kernel.org/r/20240318200404.448346-7-peterx@redhat.com Signed-off-by: Peter Xu Cc: Russell King Cc: Shawn Guo Cc: Krzysztof Kozlowski Cc: Bjorn Andersson Cc: Arnd Bergmann Cc: Konrad Dybcio Cc: Fabio Estevam Cc: Alistair Popple Cc: Andreas Larsson Cc: "Aneesh Kumar K.V" Cc: Borislav Petkov Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: David S. Miller Cc: Ingo Molnar Cc: Jason Gunthorpe Cc: Lucas Stach Cc: Mark Salter Cc: "Matthew Wilcox (Oracle)" Cc: Michael Ellerman Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Naoya Horiguchi Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/arm/include/asm/pgtable-2level.h | 4 ++-- arch/arm/include/asm/pgtable-3level-hwdef.h | 1 + arch/arm/include/asm/pgtable-3level.h | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/arm/include/asm/pgtable-2level.h b/arch/arm/include/asm/pgtable-2level.h index b0a262566eb95..4245c2e747203 100644 --- a/arch/arm/include/asm/pgtable-2level.h +++ b/arch/arm/include/asm/pgtable-2level.h @@ -213,8 +213,8 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) #define pmd_pfn(pmd) (__phys_to_pfn(pmd_val(pmd) & PHYS_MASK)) -#define pmd_leaf(pmd) (pmd_val(pmd) & 2) -#define pmd_bad(pmd) (pmd_val(pmd) & 2) +#define pmd_leaf(pmd) (pmd_val(pmd) & PMD_TYPE_SECT) +#define pmd_bad(pmd) pmd_leaf(pmd) #define pmd_present(pmd) (pmd_val(pmd)) #define copy_pmd(pmdpd,pmdps) \ diff --git a/arch/arm/include/asm/pgtable-3level-hwdef.h b/arch/arm/include/asm/pgtable-3level-hwdef.h index 2f35b4eddaa80..e7b666cf00603 100644 --- a/arch/arm/include/asm/pgtable-3level-hwdef.h +++ b/arch/arm/include/asm/pgtable-3level-hwdef.h @@ -14,6 +14,7 @@ * + Level 1/2 descriptor * - common */ +#define PUD_TABLE_BIT (_AT(pmdval_t, 1) << 1) #define PMD_TYPE_MASK (_AT(pmdval_t, 3) << 0) #define PMD_TYPE_FAULT (_AT(pmdval_t, 0) << 0) #define PMD_TYPE_TABLE (_AT(pmdval_t, 3) << 0) diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h index 4b1d9eb3908a2..e7aecbef75c98 100644 --- a/arch/arm/include/asm/pgtable-3level.h +++ b/arch/arm/include/asm/pgtable-3level.h @@ -112,7 +112,7 @@ #ifndef __ASSEMBLY__ #define pud_none(pud) (!pud_val(pud)) -#define pud_bad(pud) (!(pud_val(pud) & 2)) +#define pud_bad(pud) (!(pud_val(pud) & PUD_TABLE_BIT)) #define pud_present(pud) (pud_val(pud)) #define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ PMD_TYPE_TABLE) @@ -137,7 +137,7 @@ static inline pmd_t *pud_pgtable(pud_t pud) return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK); } -#define pmd_bad(pmd) (!(pmd_val(pmd) & 2)) +#define pmd_bad(pmd) (!(pmd_val(pmd) & PMD_TABLE_BIT)) #define copy_pmd(pmdpd,pmdps) \ do { \ From 6818135dea59f085335737629c410c2b8c6b2eb7 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 18 Mar 2024 16:03:57 -0400 Subject: [PATCH 0539/1702] mm/arm: redefine pmd_huge() with pmd_leaf() Most of the archs already define these two APIs the same way. ARM is more complicated in two aspects: - For pXd_huge() it's always checking against !PXD_TABLE_BIT, while for pXd_leaf() it's always checking against PXD_TYPE_SECT. - SECT/TABLE bits are defined differently on 2-level v.s. 3-level ARM pgtables, which makes the whole thing even harder to follow. Luckily, the second complexity should be hidden by the pmd_leaf() implementation against 2-level v.s. 3-level headers. Invoke pmd_leaf() directly for pmd_huge(), to remove the first part of complexity. This prepares to drop pXd_huge() API globally. When at it, drop the obsolete comments - it's outdated. Link: https://lkml.kernel.org/r/20240318200404.448346-8-peterx@redhat.com Signed-off-by: Peter Xu Cc: Russell King Cc: Shawn Guo Cc: Krzysztof Kozlowski Cc: Bjorn Andersson Cc: Arnd Bergmann Cc: Konrad Dybcio Cc: Fabio Estevam Cc: Alistair Popple Cc: Andreas Larsson Cc: "Aneesh Kumar K.V" Cc: Borislav Petkov Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: David S. Miller Cc: Ingo Molnar Cc: Jason Gunthorpe Cc: Lucas Stach Cc: Mark Salter Cc: "Matthew Wilcox (Oracle)" Cc: Michael Ellerman Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Naoya Horiguchi Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/arm/mm/hugetlbpage.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/arch/arm/mm/hugetlbpage.c b/arch/arm/mm/hugetlbpage.c index dd7a0277c5c09..c2fa643f6bb5e 100644 --- a/arch/arm/mm/hugetlbpage.c +++ b/arch/arm/mm/hugetlbpage.c @@ -18,11 +18,6 @@ #include #include -/* - * On ARM, huge pages are backed by pmd's rather than pte's, so we do a lot - * of type casting from pmd_t * to pte_t *. - */ - int pud_huge(pud_t pud) { return 0; @@ -30,5 +25,5 @@ int pud_huge(pud_t pud) int pmd_huge(pmd_t pmd) { - return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT); + return pmd_leaf(pmd); } From 961a6ee5c7759afce06c27a83b36756d7b81a784 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 18 Mar 2024 16:03:58 -0400 Subject: [PATCH 0540/1702] mm/arm64: merge pXd_huge() and pXd_leaf() definitions Unlike most archs, aarch64 defines pXd_huge() and pXd_leaf() slightly differently. Redefine the pXd_huge() with pXd_leaf(). There used to be two traps for old aarch64 definitions over these APIs that I found when reading the code around, they're: (1) 4797ec2dc83a ("arm64: fix pud_huge() for 2-level pagetables") (2) 23bc8f69f0ec ("arm64: mm: fix p?d_leaf()") Define pXd_huge() with the current pXd_leaf() will make sure (2) isn't a problem (on PROT_NONE checks). To make sure it also works for (1), we move over the __PAGETABLE_PMD_FOLDED check to pud_leaf(), allowing it to constantly returning "false" for 2-level pgtables, which looks even safer to cover both now. Link: https://lkml.kernel.org/r/20240318200404.448346-9-peterx@redhat.com Signed-off-by: Peter Xu Cc: Muchun Song Cc: Mark Salter Cc: Catalin Marinas Cc: Will Deacon Cc: Alistair Popple Cc: Andreas Larsson Cc: "Aneesh Kumar K.V" Cc: Arnd Bergmann Cc: Bjorn Andersson Cc: Borislav Petkov Cc: Christophe Leroy Cc: Dave Hansen Cc: David S. Miller Cc: Fabio Estevam Cc: Ingo Molnar Cc: Jason Gunthorpe Cc: Konrad Dybcio Cc: Krzysztof Kozlowski Cc: Lucas Stach Cc: "Matthew Wilcox (Oracle)" Cc: Michael Ellerman Cc: Mike Rapoport (IBM) Cc: Naoya Horiguchi Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Russell King Cc: Shawn Guo Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- arch/arm64/include/asm/pgtable.h | 4 ++++ arch/arm64/mm/hugetlbpage.c | 8 ++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index afdd56d26ad70..ec406e761e041 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -709,7 +709,11 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) #define pud_none(pud) (!pud_val(pud)) #define pud_bad(pud) (!pud_table(pud)) #define pud_present(pud) pte_present(pud_pte(pud)) +#ifndef __PAGETABLE_PMD_FOLDED #define pud_leaf(pud) (pud_present(pud) && !pud_table(pud)) +#else +#define pud_leaf(pud) false +#endif #define pud_valid(pud) pte_valid(pud_pte(pud)) #define pud_user(pud) pte_user(pud_pte(pud)) #define pud_user_exec(pud) pte_user_exec(pud_pte(pud)) diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index 0f0e10bb0a954..1234bbaef5bf4 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -81,16 +81,12 @@ bool arch_hugetlb_migration_supported(struct hstate *h) int pmd_huge(pmd_t pmd) { - return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT); + return pmd_leaf(pmd); } int pud_huge(pud_t pud) { -#ifndef __PAGETABLE_PMD_FOLDED - return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT); -#else - return 0; -#endif + return pud_leaf(pud); } static int find_num_contig(struct mm_struct *mm, unsigned long addr, From 460b9adc05d7d331cc6d0311cff7412b689e8729 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 18 Mar 2024 16:03:59 -0400 Subject: [PATCH 0541/1702] mm/powerpc: redefine pXd_huge() with pXd_leaf() PowerPC book3s 4K mostly has the same definition on both, except pXd_huge() constantly returns 0 for hash MMUs. As Michael Ellerman pointed out [1], it is safe to check _PAGE_PTE on hash MMUs, as the bit will never be set so it will keep returning false. As a reference, __p[mu]d_mkhuge() will trigger a BUG_ON trying to create such huge mappings for 4K hash MMUs. Meanwhile, the major powerpc hugetlb pgtable walker __find_linux_pte() already used pXd_leaf() to check leaf hugetlb mappings. The goal should be that we will have one API pXd_leaf() to detect all kinds of huge mappings (hugepd is still special in this case, though). AFAICT we need to use the pXd_leaf() impl (rather than pXd_huge()'s) to make sure ie. THPs on hash MMU will also return true. This helps to simplify a follow up patch to drop pXd_huge() treewide. NOTE: *_leaf() definition need to be moved before the inclusion of asm/book3s/64/pgtable-4k.h, which defines pXd_huge() with it. [1] https://lore.kernel.org/r/87v85zo6w7.fsf@mail.lhotse Link: https://lkml.kernel.org/r/20240318200404.448346-10-peterx@redhat.com Signed-off-by: Peter Xu Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Christophe Leroy Cc: "Aneesh Kumar K.V" Cc: "Naveen N. Rao" Cc: Alistair Popple Cc: Andreas Larsson Cc: Arnd Bergmann Cc: Bjorn Andersson Cc: Borislav Petkov Cc: Catalin Marinas Cc: Dave Hansen Cc: David S. Miller Cc: Fabio Estevam Cc: Ingo Molnar Cc: Jason Gunthorpe Cc: Konrad Dybcio Cc: Krzysztof Kozlowski Cc: Lucas Stach Cc: Mark Salter Cc: "Matthew Wilcox (Oracle)" Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Naoya Horiguchi Cc: Russell King Cc: Shawn Guo Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton --- .../include/asm/book3s/64/pgtable-4k.h | 14 ++-------- arch/powerpc/include/asm/book3s/64/pgtable.h | 27 +++++++++---------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/64/pgtable-4k.h b/arch/powerpc/include/asm/book3s/64/pgtable-4k.h index 48f21820afe20..92545981bb49d 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable-4k.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable-4k.h @@ -8,22 +8,12 @@ #ifdef CONFIG_HUGETLB_PAGE static inline int pmd_huge(pmd_t pmd) { - /* - * leaf pte for huge page - */ - if (radix_enabled()) - return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE)); - return 0; + return pmd_leaf(pmd); } static inline int pud_huge(pud_t pud) { - /* - * leaf pte for huge page - */ - if (radix_enabled()) - return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE)); - return 0; + return pud_leaf(pud); } /* diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index fac5615e6bc57..8f9432e3855ae 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -262,6 +262,18 @@ extern unsigned long __kernel_io_end; extern struct page *vmemmap; extern unsigned long pci_io_base; + +#define pmd_leaf pmd_leaf +static inline bool pmd_leaf(pmd_t pmd) +{ + return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE)); +} + +#define pud_leaf pud_leaf +static inline bool pud_leaf(pud_t pud) +{ + return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE)); +} #endif /* __ASSEMBLY__ */ #include @@ -1426,20 +1438,5 @@ static inline bool is_pte_rw_upgrade(unsigned long old_val, unsigned long new_va return false; } -/* - * Like pmd_huge(), but works regardless of config options - */ -#define pmd_leaf pmd_leaf -static inline bool pmd_leaf(pmd_t pmd) -{ - return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE)); -} - -#define pud_leaf pud_leaf -static inline bool pud_leaf(pud_t pud) -{ - return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE)); -} - #endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ */ From 7db86dc389aac35425e3f858ab5c12e1e877c4ff Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 18 Mar 2024 16:04:00 -0400 Subject: [PATCH 0542/1702] mm/gup: merge pXd huge mapping checks Huge mapping checks in GUP are slightly redundant and can be simplified. pXd_huge() now is the same as pXd_leaf(). pmd_trans_huge() and pXd_devmap() should both imply pXd_leaf(). Time to merge them into one. Link: https://lkml.kernel.org/r/20240318200404.448346-11-peterx@redhat.com Signed-off-by: Peter Xu Reviewed-by: Jason Gunthorpe Cc: Alistair Popple Cc: Andreas Larsson Cc: "Aneesh Kumar K.V" Cc: Arnd Bergmann Cc: Bjorn Andersson Cc: Borislav Petkov Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: David S. Miller Cc: Fabio Estevam Cc: Ingo Molnar Cc: Konrad Dybcio Cc: Krzysztof Kozlowski Cc: Lucas Stach Cc: Mark Salter Cc: "Matthew Wilcox (Oracle)" Cc: Michael Ellerman Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Naoya Horiguchi Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Russell King Cc: Shawn Guo Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton --- mm/gup.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index 7ed2c99fb72c9..d89377b87311b 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -3017,8 +3017,7 @@ static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned lo if (!pmd_present(pmd)) return 0; - if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) || - pmd_devmap(pmd))) { + if (unlikely(pmd_leaf(pmd))) { /* See gup_pte_range() */ if (pmd_protnone(pmd)) return 0; @@ -3055,7 +3054,7 @@ static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned lo next = pud_addr_end(addr, end); if (unlikely(!pud_present(pud))) return 0; - if (unlikely(pud_huge(pud) || pud_devmap(pud))) { + if (unlikely(pud_leaf(pud))) { if (!gup_huge_pud(pud, pudp, addr, next, flags, pages, nr)) return 0; @@ -3108,7 +3107,7 @@ static void gup_pgd_range(unsigned long addr, unsigned long end, next = pgd_addr_end(addr, end); if (pgd_none(pgd)) return; - if (unlikely(pgd_huge(pgd))) { + if (unlikely(pgd_leaf(pgd))) { if (!gup_huge_pgd(pgd, pgdp, addr, next, flags, pages, nr)) return; From 1965e933ddeba5e27e68ed80cfc7241931f70d4c Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 18 Mar 2024 16:04:01 -0400 Subject: [PATCH 0543/1702] mm/treewide: replace pXd_huge() with pXd_leaf() Now after we're sure all pXd_huge() definitions are the same as pXd_leaf(), reuse it. Luckily, pXd_huge() isn't widely used. Link: https://lkml.kernel.org/r/20240318200404.448346-12-peterx@redhat.com Signed-off-by: Peter Xu Cc: Alistair Popple Cc: Andreas Larsson Cc: "Aneesh Kumar K.V" Cc: Arnd Bergmann Cc: Bjorn Andersson Cc: Borislav Petkov Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: David S. Miller Cc: Fabio Estevam Cc: Ingo Molnar Cc: Jason Gunthorpe Cc: Konrad Dybcio Cc: Krzysztof Kozlowski Cc: Lucas Stach Cc: Mark Salter Cc: "Matthew Wilcox (Oracle)" Cc: Michael Ellerman Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Naoya Horiguchi Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Russell King Cc: Shawn Guo Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/arm/include/asm/pgtable-3level.h | 2 +- arch/arm64/include/asm/pgtable.h | 2 +- arch/arm64/mm/hugetlbpage.c | 4 ++-- arch/loongarch/mm/hugetlbpage.c | 2 +- arch/mips/mm/tlb-r4k.c | 2 +- arch/powerpc/mm/pgtable_64.c | 6 +++--- arch/x86/mm/pgtable.c | 4 ++-- mm/gup.c | 4 ++-- mm/hmm.c | 2 +- mm/memory.c | 2 +- 10 files changed, 15 insertions(+), 15 deletions(-) diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h index e7aecbef75c98..9e3c44f0aea22 100644 --- a/arch/arm/include/asm/pgtable-3level.h +++ b/arch/arm/include/asm/pgtable-3level.h @@ -190,7 +190,7 @@ static inline pte_t pte_mkspecial(pte_t pte) #define pmd_dirty(pmd) (pmd_isset((pmd), L_PMD_SECT_DIRTY)) #define pmd_hugewillfault(pmd) (!pmd_young(pmd) || !pmd_write(pmd)) -#define pmd_thp_or_huge(pmd) (pmd_huge(pmd) || pmd_trans_huge(pmd)) +#define pmd_thp_or_huge(pmd) (pmd_leaf(pmd) || pmd_trans_huge(pmd)) #ifdef CONFIG_TRANSPARENT_HUGEPAGE #define pmd_trans_huge(pmd) (pmd_val(pmd) && !pmd_table(pmd)) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index ec406e761e041..a5c84f38c5283 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -517,7 +517,7 @@ static inline pmd_t pmd_mkinvalid(pmd_t pmd) return pmd; } -#define pmd_thp_or_huge(pmd) (pmd_huge(pmd) || pmd_trans_huge(pmd)) +#define pmd_thp_or_huge(pmd) (pmd_leaf(pmd) || pmd_trans_huge(pmd)) #define pmd_write(pmd) pte_write(pmd_pte(pmd)) diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index 1234bbaef5bf4..f494fc31201f1 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -321,7 +321,7 @@ pte_t *huge_pte_offset(struct mm_struct *mm, if (sz != PUD_SIZE && pud_none(pud)) return NULL; /* hugepage or swap? */ - if (pud_huge(pud) || !pud_present(pud)) + if (pud_leaf(pud) || !pud_present(pud)) return (pte_t *)pudp; /* table; check the next level */ @@ -333,7 +333,7 @@ pte_t *huge_pte_offset(struct mm_struct *mm, if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) && pmd_none(pmd)) return NULL; - if (pmd_huge(pmd) || !pmd_present(pmd)) + if (pmd_leaf(pmd) || !pmd_present(pmd)) return (pte_t *)pmdp; if (sz == CONT_PTE_SIZE) diff --git a/arch/loongarch/mm/hugetlbpage.c b/arch/loongarch/mm/hugetlbpage.c index 1e76fcb83093d..a4e78e74aa217 100644 --- a/arch/loongarch/mm/hugetlbpage.c +++ b/arch/loongarch/mm/hugetlbpage.c @@ -64,7 +64,7 @@ uint64_t pmd_to_entrylo(unsigned long pmd_val) { uint64_t val; /* PMD as PTE. Must be huge page */ - if (!pmd_huge(__pmd(pmd_val))) + if (!pmd_leaf(__pmd(pmd_val))) panic("%s", __func__); val = pmd_val ^ _PAGE_HUGE; diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c index 4106084e57d72..76f3b9c0a9f0c 100644 --- a/arch/mips/mm/tlb-r4k.c +++ b/arch/mips/mm/tlb-r4k.c @@ -326,7 +326,7 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) idx = read_c0_index(); #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT /* this could be a huge page */ - if (pmd_huge(*pmdp)) { + if (pmd_leaf(*pmdp)) { unsigned long lo; write_c0_pagemask(PM_HUGE_MASK); ptep = (pte_t *)pmdp; diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c index 9b99113cb51a8..6621cfc3baf86 100644 --- a/arch/powerpc/mm/pgtable_64.c +++ b/arch/powerpc/mm/pgtable_64.c @@ -102,7 +102,7 @@ struct page *p4d_page(p4d_t p4d) { if (p4d_leaf(p4d)) { if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP)) - VM_WARN_ON(!p4d_huge(p4d)); + VM_WARN_ON(!p4d_leaf(p4d)); return pte_page(p4d_pte(p4d)); } return virt_to_page(p4d_pgtable(p4d)); @@ -113,7 +113,7 @@ struct page *pud_page(pud_t pud) { if (pud_leaf(pud)) { if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP)) - VM_WARN_ON(!pud_huge(pud)); + VM_WARN_ON(!pud_leaf(pud)); return pte_page(pud_pte(pud)); } return virt_to_page(pud_pgtable(pud)); @@ -132,7 +132,7 @@ struct page *pmd_page(pmd_t pmd) * enabled so these checks can't be used. */ if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP)) - VM_WARN_ON(!(pmd_leaf(pmd) || pmd_huge(pmd))); + VM_WARN_ON(!pmd_leaf(pmd)); return pte_page(pmd_pte(pmd)); } return virt_to_page(pmd_page_vaddr(pmd)); diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index d007591b80597..94767c82fc0d7 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -731,7 +731,7 @@ int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot) return 0; /* Bail out if we are we on a populated non-leaf entry: */ - if (pud_present(*pud) && !pud_huge(*pud)) + if (pud_present(*pud) && !pud_leaf(*pud)) return 0; set_pte((pte_t *)pud, pfn_pte( @@ -760,7 +760,7 @@ int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot) } /* Bail out if we are we on a populated non-leaf entry: */ - if (pmd_present(*pmd) && !pmd_huge(*pmd)) + if (pmd_present(*pmd) && !pmd_leaf(*pmd)) return 0; set_pte((pte_t *)pmd, pfn_pte( diff --git a/mm/gup.c b/mm/gup.c index d89377b87311b..8433d3dc31fce 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -778,7 +778,7 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma, p4d = READ_ONCE(*p4dp); if (!p4d_present(p4d)) return no_page_table(vma, flags); - BUILD_BUG_ON(p4d_huge(p4d)); + BUILD_BUG_ON(p4d_leaf(p4d)); if (unlikely(p4d_bad(p4d))) return no_page_table(vma, flags); @@ -3082,7 +3082,7 @@ static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned lo next = p4d_addr_end(addr, end); if (!p4d_present(p4d)) return 0; - BUILD_BUG_ON(p4d_huge(p4d)); + BUILD_BUG_ON(p4d_leaf(p4d)); if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) { if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr, P4D_SHIFT, next, flags, pages, nr)) diff --git a/mm/hmm.c b/mm/hmm.c index c95b9ec5d95fe..93aebd9cc130e 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -429,7 +429,7 @@ static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end, return hmm_vma_walk_hole(start, end, -1, walk); } - if (pud_huge(pud) && pud_devmap(pud)) { + if (pud_leaf(pud) && pud_devmap(pud)) { unsigned long i, npages, pfn; unsigned int required_fault; unsigned long *hmm_pfns; diff --git a/mm/memory.c b/mm/memory.c index d2155ced45f8f..e0576f2a8d236 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2765,7 +2765,7 @@ static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud, unsigned long next; int err = 0; - BUG_ON(pud_huge(*pud)); + BUG_ON(pud_leaf(*pud)); if (create) { pmd = pmd_alloc_track(mm, pud, addr, mask); From 9636f055dae19ffae9d8bf21a68261b58c769987 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 18 Mar 2024 16:04:02 -0400 Subject: [PATCH 0544/1702] mm/treewide: remove pXd_huge() This API is not used anymore, drop it for the whole tree. Link: https://lkml.kernel.org/r/20240318200404.448346-13-peterx@redhat.com Signed-off-by: Peter Xu Cc: Alistair Popple Cc: Andreas Larsson Cc: "Aneesh Kumar K.V" Cc: Arnd Bergmann Cc: Bjorn Andersson Cc: Borislav Petkov Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: David S. Miller Cc: Fabio Estevam Cc: Ingo Molnar Cc: Jason Gunthorpe Cc: Konrad Dybcio Cc: Krzysztof Kozlowski Cc: Lucas Stach Cc: Mark Salter Cc: "Matthew Wilcox (Oracle)" Cc: Michael Ellerman Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Naoya Horiguchi Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Russell King Cc: Shawn Guo Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/arm/mm/Makefile | 1 - arch/arm/mm/hugetlbpage.c | 29 ------------------- arch/arm64/mm/hugetlbpage.c | 10 ------- arch/loongarch/mm/hugetlbpage.c | 10 ------- arch/mips/include/asm/pgtable-32.h | 2 +- arch/mips/include/asm/pgtable-64.h | 2 +- arch/mips/mm/hugetlbpage.c | 10 ------- arch/parisc/mm/hugetlbpage.c | 11 ------- .../include/asm/book3s/64/pgtable-4k.h | 10 ------- .../include/asm/book3s/64/pgtable-64k.h | 25 ---------------- arch/powerpc/include/asm/nohash/pgtable.h | 10 ------- arch/riscv/mm/hugetlbpage.c | 10 ------- arch/s390/mm/hugetlbpage.c | 10 ------- arch/sh/mm/hugetlbpage.c | 10 ------- arch/sparc/mm/hugetlbpage.c | 10 ------- arch/x86/mm/hugetlbpage.c | 16 ---------- include/linux/hugetlb.h | 24 --------------- 17 files changed, 2 insertions(+), 198 deletions(-) delete mode 100644 arch/arm/mm/hugetlbpage.c diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile index 71b858c9b10c4..1779e12db0850 100644 --- a/arch/arm/mm/Makefile +++ b/arch/arm/mm/Makefile @@ -21,7 +21,6 @@ KASAN_SANITIZE_physaddr.o := n obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_ALIGNMENT_TRAP) += alignment.o -obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_ARM_PV_FIXUP) += pv-fixup-asm.o obj-$(CONFIG_CPU_ABRT_NOMMU) += abort-nommu.o diff --git a/arch/arm/mm/hugetlbpage.c b/arch/arm/mm/hugetlbpage.c deleted file mode 100644 index c2fa643f6bb5e..0000000000000 --- a/arch/arm/mm/hugetlbpage.c +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * arch/arm/mm/hugetlbpage.c - * - * Copyright (C) 2012 ARM Ltd. - * - * Based on arch/x86/include/asm/hugetlb.h and Bill Carson's patches - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -int pud_huge(pud_t pud) -{ - return 0; -} - -int pmd_huge(pmd_t pmd) -{ - return pmd_leaf(pmd); -} diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index f494fc31201f1..ca58210d6c07d 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -79,16 +79,6 @@ bool arch_hugetlb_migration_supported(struct hstate *h) } #endif -int pmd_huge(pmd_t pmd) -{ - return pmd_leaf(pmd); -} - -int pud_huge(pud_t pud) -{ - return pud_leaf(pud); -} - static int find_num_contig(struct mm_struct *mm, unsigned long addr, pte_t *ptep, size_t *pgsize) { diff --git a/arch/loongarch/mm/hugetlbpage.c b/arch/loongarch/mm/hugetlbpage.c index a4e78e74aa217..12222c56cb594 100644 --- a/arch/loongarch/mm/hugetlbpage.c +++ b/arch/loongarch/mm/hugetlbpage.c @@ -50,16 +50,6 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, return (pte_t *) pmd; } -int pmd_huge(pmd_t pmd) -{ - return (pmd_val(pmd) & _PAGE_HUGE) != 0; -} - -int pud_huge(pud_t pud) -{ - return (pud_val(pud) & _PAGE_HUGE) != 0; -} - uint64_t pmd_to_entrylo(unsigned long pmd_val) { uint64_t val; diff --git a/arch/mips/include/asm/pgtable-32.h b/arch/mips/include/asm/pgtable-32.h index 0e196650f4f48..92b7591aac2ac 100644 --- a/arch/mips/include/asm/pgtable-32.h +++ b/arch/mips/include/asm/pgtable-32.h @@ -129,7 +129,7 @@ static inline int pmd_none(pmd_t pmd) static inline int pmd_bad(pmd_t pmd) { #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT - /* pmd_huge(pmd) but inline */ + /* pmd_leaf(pmd) but inline */ if (unlikely(pmd_val(pmd) & _PAGE_HUGE)) return 0; #endif diff --git a/arch/mips/include/asm/pgtable-64.h b/arch/mips/include/asm/pgtable-64.h index 20ca48c1b6063..7c28510b3768f 100644 --- a/arch/mips/include/asm/pgtable-64.h +++ b/arch/mips/include/asm/pgtable-64.h @@ -245,7 +245,7 @@ static inline int pmd_none(pmd_t pmd) static inline int pmd_bad(pmd_t pmd) { #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT - /* pmd_huge(pmd) but inline */ + /* pmd_leaf(pmd) but inline */ if (unlikely(pmd_val(pmd) & _PAGE_HUGE)) return 0; #endif diff --git a/arch/mips/mm/hugetlbpage.c b/arch/mips/mm/hugetlbpage.c index 7eaff5b078739..0b9e15555b59b 100644 --- a/arch/mips/mm/hugetlbpage.c +++ b/arch/mips/mm/hugetlbpage.c @@ -57,13 +57,3 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, } return (pte_t *) pmd; } - -int pmd_huge(pmd_t pmd) -{ - return (pmd_val(pmd) & _PAGE_HUGE) != 0; -} - -int pud_huge(pud_t pud) -{ - return (pud_val(pud) & _PAGE_HUGE) != 0; -} diff --git a/arch/parisc/mm/hugetlbpage.c b/arch/parisc/mm/hugetlbpage.c index a9f7e21f66567..0356199bd9e7e 100644 --- a/arch/parisc/mm/hugetlbpage.c +++ b/arch/parisc/mm/hugetlbpage.c @@ -180,14 +180,3 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma, } return changed; } - - -int pmd_huge(pmd_t pmd) -{ - return 0; -} - -int pud_huge(pud_t pud) -{ - return 0; -} diff --git a/arch/powerpc/include/asm/book3s/64/pgtable-4k.h b/arch/powerpc/include/asm/book3s/64/pgtable-4k.h index 92545981bb49d..baf934578c3a5 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable-4k.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable-4k.h @@ -6,16 +6,6 @@ */ #ifndef __ASSEMBLY__ #ifdef CONFIG_HUGETLB_PAGE -static inline int pmd_huge(pmd_t pmd) -{ - return pmd_leaf(pmd); -} - -static inline int pud_huge(pud_t pud) -{ - return pud_leaf(pud); -} - /* * With radix , we have hugepage ptes in the pud and pmd entries. We don't * need to setup hugepage directory for them. Our pte and page directory format diff --git a/arch/powerpc/include/asm/book3s/64/pgtable-64k.h b/arch/powerpc/include/asm/book3s/64/pgtable-64k.h index ced7ee8b42fc3..6ac73da7b80e3 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable-64k.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable-64k.h @@ -4,31 +4,6 @@ #ifndef __ASSEMBLY__ #ifdef CONFIG_HUGETLB_PAGE -/* - * We have PGD_INDEX_SIZ = 12 and PTE_INDEX_SIZE = 8, so that we can have - * 16GB hugepage pte in PGD and 16MB hugepage pte at PMD; - * - * Defined in such a way that we can optimize away code block at build time - * if CONFIG_HUGETLB_PAGE=n. - * - * returns true for pmd migration entries, THP, devmap, hugetlb - * But compile time dependent on CONFIG_HUGETLB_PAGE - */ -static inline int pmd_huge(pmd_t pmd) -{ - /* - * leaf pte for huge page - */ - return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE)); -} - -static inline int pud_huge(pud_t pud) -{ - /* - * leaf pte for huge page - */ - return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE)); -} /* * With 64k page size, we have hugepage ptes in the pgd and pmd entries. We don't diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index 427db14292c9e..f5f39d4f03c82 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -351,16 +351,6 @@ static inline int hugepd_ok(hugepd_t hpd) #endif } -static inline int pmd_huge(pmd_t pmd) -{ - return 0; -} - -static inline int pud_huge(pud_t pud) -{ - return 0; -} - #define is_hugepd(hpd) (hugepd_ok(hpd)) #endif diff --git a/arch/riscv/mm/hugetlbpage.c b/arch/riscv/mm/hugetlbpage.c index 5ef2a6891158a..0ebd968b33c99 100644 --- a/arch/riscv/mm/hugetlbpage.c +++ b/arch/riscv/mm/hugetlbpage.c @@ -399,16 +399,6 @@ static bool is_napot_size(unsigned long size) #endif /*CONFIG_RISCV_ISA_SVNAPOT*/ -int pud_huge(pud_t pud) -{ - return pud_leaf(pud); -} - -int pmd_huge(pmd_t pmd) -{ - return pmd_leaf(pmd); -} - static bool __hugetlb_valid_size(unsigned long size) { if (size == HPAGE_SIZE) diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c index c2e8242bd15dd..ca43b6fce71cf 100644 --- a/arch/s390/mm/hugetlbpage.c +++ b/arch/s390/mm/hugetlbpage.c @@ -233,16 +233,6 @@ pte_t *huge_pte_offset(struct mm_struct *mm, return (pte_t *) pmdp; } -int pmd_huge(pmd_t pmd) -{ - return pmd_leaf(pmd); -} - -int pud_huge(pud_t pud) -{ - return pud_leaf(pud); -} - bool __init arch_hugetlb_valid_size(unsigned long size) { if (MACHINE_HAS_EDAT1 && size == PMD_SIZE) diff --git a/arch/sh/mm/hugetlbpage.c b/arch/sh/mm/hugetlbpage.c index 6cb0ad73dbb9e..ff209b55285ad 100644 --- a/arch/sh/mm/hugetlbpage.c +++ b/arch/sh/mm/hugetlbpage.c @@ -70,13 +70,3 @@ pte_t *huge_pte_offset(struct mm_struct *mm, return pte; } - -int pmd_huge(pmd_t pmd) -{ - return 0; -} - -int pud_huge(pud_t pud) -{ - return 0; -} diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c index 532143f1bf634..8ed5bdf95d253 100644 --- a/arch/sparc/mm/hugetlbpage.c +++ b/arch/sparc/mm/hugetlbpage.c @@ -407,16 +407,6 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, return entry; } -int pmd_huge(pmd_t pmd) -{ - return pmd_leaf(pmd); -} - -int pud_huge(pud_t pud) -{ - return pud_leaf(pud); -} - static void hugetlb_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, unsigned long addr) { diff --git a/arch/x86/mm/hugetlbpage.c b/arch/x86/mm/hugetlbpage.c index 8362953a24ce2..dab6db288e5b2 100644 --- a/arch/x86/mm/hugetlbpage.c +++ b/arch/x86/mm/hugetlbpage.c @@ -19,22 +19,6 @@ #include #include -/* - * pmd_huge() returns 1 if @pmd is hugetlb related entry. - */ -int pmd_huge(pmd_t pmd) -{ - return pmd_leaf(pmd); -} - -/* - * pud_huge() returns 1 if @pud is hugetlb related entry. - */ -int pud_huge(pud_t pud) -{ - return pud_leaf(pud); -} - #ifdef CONFIG_HUGETLB_PAGE static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file, unsigned long addr, unsigned long len, diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 77b30a8c6076b..300de33c6fde0 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -272,13 +272,9 @@ void hugetlb_vma_unlock_write(struct vm_area_struct *vma); int hugetlb_vma_trylock_write(struct vm_area_struct *vma); void hugetlb_vma_assert_locked(struct vm_area_struct *vma); void hugetlb_vma_lock_release(struct kref *kref); - -int pmd_huge(pmd_t pmd); -int pud_huge(pud_t pud); long hugetlb_change_protection(struct vm_area_struct *vma, unsigned long address, unsigned long end, pgprot_t newprot, unsigned long cp_flags); - bool is_hugetlb_entry_migration(pte_t pte); bool is_hugetlb_entry_hwpoisoned(pte_t pte); void hugetlb_unshare_all_pmds(struct vm_area_struct *vma); @@ -399,16 +395,6 @@ static inline void hugetlb_vma_assert_locked(struct vm_area_struct *vma) { } -static inline int pmd_huge(pmd_t pmd) -{ - return 0; -} - -static inline int pud_huge(pud_t pud) -{ - return 0; -} - static inline int is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, unsigned long len) { @@ -493,16 +479,6 @@ static inline vm_fault_t hugetlb_fault(struct mm_struct *mm, static inline void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) { } #endif /* !CONFIG_HUGETLB_PAGE */ -/* - * hugepages at page global directory. If arch support - * hugepages at pgd level, they need to define this. - */ -#ifndef pgd_huge -#define pgd_huge(x) 0 -#endif -#ifndef p4d_huge -#define p4d_huge(x) 0 -#endif #ifndef pgd_write static inline int pgd_write(pgd_t pgd) From 502016e33ae6474177097bc43ee26bad8f4d6919 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 18 Mar 2024 16:04:03 -0400 Subject: [PATCH 0545/1702] mm/arm: remove pmd_thp_or_huge() ARM/ARM64 used to define pmd_thp_or_huge(). Now this macro is completely redundant. Remove it and use pmd_leaf(). Link: https://lkml.kernel.org/r/20240318200404.448346-14-peterx@redhat.com Signed-off-by: Peter Xu Cc: Mark Salter Cc: Catalin Marinas Cc: Will Deacon Cc: Russell King Cc: Shawn Guo Cc: Krzysztof Kozlowski Cc: Bjorn Andersson Cc: Arnd Bergmann Cc: Konrad Dybcio Cc: Fabio Estevam Cc: Alistair Popple Cc: Andreas Larsson Cc: "Aneesh Kumar K.V" Cc: Borislav Petkov Cc: Christophe Leroy Cc: Dave Hansen Cc: David S. Miller Cc: Ingo Molnar Cc: Jason Gunthorpe Cc: Lucas Stach Cc: "Matthew Wilcox (Oracle)" Cc: Michael Ellerman Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Naoya Horiguchi Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- arch/arm/include/asm/pgtable-2level.h | 1 - arch/arm/include/asm/pgtable-3level.h | 1 - arch/arm/lib/uaccess_with_memcpy.c | 4 ++-- arch/arm64/include/asm/pgtable.h | 2 -- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/arch/arm/include/asm/pgtable-2level.h b/arch/arm/include/asm/pgtable-2level.h index 4245c2e747203..6b5392e20f411 100644 --- a/arch/arm/include/asm/pgtable-2level.h +++ b/arch/arm/include/asm/pgtable-2level.h @@ -241,7 +241,6 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) * define empty stubs for use by pin_page_for_write. */ #define pmd_hugewillfault(pmd) (0) -#define pmd_thp_or_huge(pmd) (0) #endif /* __ASSEMBLY__ */ diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h index 9e3c44f0aea22..fa5939eb9864e 100644 --- a/arch/arm/include/asm/pgtable-3level.h +++ b/arch/arm/include/asm/pgtable-3level.h @@ -190,7 +190,6 @@ static inline pte_t pte_mkspecial(pte_t pte) #define pmd_dirty(pmd) (pmd_isset((pmd), L_PMD_SECT_DIRTY)) #define pmd_hugewillfault(pmd) (!pmd_young(pmd) || !pmd_write(pmd)) -#define pmd_thp_or_huge(pmd) (pmd_leaf(pmd) || pmd_trans_huge(pmd)) #ifdef CONFIG_TRANSPARENT_HUGEPAGE #define pmd_trans_huge(pmd) (pmd_val(pmd) && !pmd_table(pmd)) diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c index 2f6163f05e935..c0ac7796d7750 100644 --- a/arch/arm/lib/uaccess_with_memcpy.c +++ b/arch/arm/lib/uaccess_with_memcpy.c @@ -56,10 +56,10 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp) * to see that it's still huge and whether or not we will * need to fault on write. */ - if (unlikely(pmd_thp_or_huge(*pmd))) { + if (unlikely(pmd_leaf(*pmd))) { ptl = ¤t->mm->page_table_lock; spin_lock(ptl); - if (unlikely(!pmd_thp_or_huge(*pmd) + if (unlikely(!pmd_leaf(*pmd) || pmd_hugewillfault(*pmd))) { spin_unlock(ptl); return 0; diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index a5c84f38c5283..6870b60158fc8 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -517,8 +517,6 @@ static inline pmd_t pmd_mkinvalid(pmd_t pmd) return pmd; } -#define pmd_thp_or_huge(pmd) (pmd_leaf(pmd) || pmd_trans_huge(pmd)) - #define pmd_write(pmd) pte_write(pmd_pte(pmd)) #define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT)) From 64078b3d57dd9aaa4dde4a4f5851a53a79ac4d42 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 18 Mar 2024 16:04:04 -0400 Subject: [PATCH 0546/1702] mm: document pXd_leaf() API There's one small section already, but since we're going to remove pXd_huge(), that comment may start to obsolete. Rewrite that section with more information, hopefully with that the API is crystal clear on what it implies. Link: https://lkml.kernel.org/r/20240318200404.448346-15-peterx@redhat.com Signed-off-by: Peter Xu Reviewed-by: Jason Gunthorpe Cc: Alistair Popple Cc: Andreas Larsson Cc: "Aneesh Kumar K.V" Cc: Arnd Bergmann Cc: Bjorn Andersson Cc: Borislav Petkov Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: David S. Miller Cc: Fabio Estevam Cc: Ingo Molnar Cc: Konrad Dybcio Cc: Krzysztof Kozlowski Cc: Lucas Stach Cc: Mark Salter Cc: "Matthew Wilcox (Oracle)" Cc: Michael Ellerman Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Naoya Horiguchi Cc: "Naveen N. Rao" Cc: Nicholas Piggin Cc: Russell King Cc: Shawn Guo Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index bd9c7180718cd..2a1c044ae4672 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1768,11 +1768,25 @@ typedef unsigned int pgtbl_mod_mask; #endif /* - * p?d_leaf() - true if this entry is a final mapping to a physical address. - * This differs from p?d_huge() by the fact that they are always available (if - * the architecture supports large pages at the appropriate level) even - * if CONFIG_HUGETLB_PAGE is not defined. - * Only meaningful when called on a valid entry. + * pXd_leaf() is the API to check whether a pgtable entry is a huge page + * mapping. It should work globally across all archs, without any + * dependency on CONFIG_* options. For architectures that do not support + * huge mappings on specific levels, below fallbacks will be used. + * + * A leaf pgtable entry should always imply the following: + * + * - It is a "present" entry. IOW, before using this API, please check it + * with pXd_present() first. NOTE: it may not always mean the "present + * bit" is set. For example, PROT_NONE entries are always "present". + * + * - It should _never_ be a swap entry of any type. Above "present" check + * should have guarded this, but let's be crystal clear on this. + * + * - It should contain a huge PFN, which points to a huge page larger than + * PAGE_SIZE of the platform. The PFN format isn't important here. + * + * - It should cover all kinds of huge mappings (e.g., pXd_trans_huge(), + * pXd_devmap(), or hugetlb mappings). */ #ifndef pgd_leaf #define pgd_leaf(x) false From 91cdcd8d624bfdf05e8db9d572759516f3c786b8 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Tue, 12 Mar 2024 11:34:11 -0400 Subject: [PATCH 0547/1702] mm: zswap: optimize zswap pool size tracking Profiling the munmap() of a zswapped memory region shows 60% of the total cycles currently going into updating the zswap_pool_total_size. There are three consumers of this counter: - store, to enforce the globally configured pool limit - meminfo & debugfs, to report the size to the user - shrink, to determine the batch size for each cycle Instead of aggregating everytime an entry enters or exits the zswap pool, aggregate the value from the zpools on-demand: - Stores aggregate the counter anyway upon success. Aggregating to check the limit instead is the same amount of work. - Meminfo & debugfs might benefit somewhat from a pre-aggregated counter, but aren't exactly hotpaths. - Shrinking can aggregate once for every cycle instead of doing it for every freed entry. As the shrinker might work on tens or hundreds of objects per scan cycle, this is a large reduction in aggregations. The paths that benefit dramatically are swapin, swapoff, and unmaps. There could be millions of pages being processed until somebody asks for the pool size again. This eliminates the pool size updates from those paths entirely. Top profile entries for a 24G range munmap(), before: 38.54% zswap-unmap [kernel.kallsyms] [k] zs_zpool_total_size 12.51% zswap-unmap [kernel.kallsyms] [k] zpool_get_total_size 9.10% zswap-unmap [kernel.kallsyms] [k] zswap_update_total_size 2.95% zswap-unmap [kernel.kallsyms] [k] obj_cgroup_uncharge_zswap 2.88% zswap-unmap [kernel.kallsyms] [k] __slab_free 2.86% zswap-unmap [kernel.kallsyms] [k] xas_store and after: 7.70% zswap-unmap [kernel.kallsyms] [k] __slab_free 7.16% zswap-unmap [kernel.kallsyms] [k] obj_cgroup_uncharge_zswap 6.74% zswap-unmap [kernel.kallsyms] [k] xas_store It was also briefly considered to move to a single atomic in zswap that is updated by the backends, since zswap only cares about the sum of all pools anyway. However, zram directly needs per-pool information out of zsmalloc. To keep the backend from having to update two atomics every time, I opted for the lazy aggregation instead for now. Link: https://lkml.kernel.org/r/20240312153901.3441-1-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Acked-by: Yosry Ahmed Reviewed-by: Chengming Zhou Reviewed-by: Nhat Pham Signed-off-by: Andrew Morton --- fs/proc/meminfo.c | 3 +- include/linux/zswap.h | 2 +- mm/zswap.c | 101 +++++++++++++++++++++--------------------- 3 files changed, 52 insertions(+), 54 deletions(-) diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index 45af9a989d404..245171d9164be 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c @@ -89,8 +89,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v) show_val_kb(m, "SwapTotal: ", i.totalswap); show_val_kb(m, "SwapFree: ", i.freeswap); #ifdef CONFIG_ZSWAP - seq_printf(m, "Zswap: %8lu kB\n", - (unsigned long)(zswap_pool_total_size >> 10)); + show_val_kb(m, "Zswap: ", zswap_total_pages()); seq_printf(m, "Zswapped: %8lu kB\n", (unsigned long)atomic_read(&zswap_stored_pages) << (PAGE_SHIFT - 10)); diff --git a/include/linux/zswap.h b/include/linux/zswap.h index 341aea4900704..2a85b941db975 100644 --- a/include/linux/zswap.h +++ b/include/linux/zswap.h @@ -7,7 +7,6 @@ struct lruvec; -extern u64 zswap_pool_total_size; extern atomic_t zswap_stored_pages; #ifdef CONFIG_ZSWAP @@ -27,6 +26,7 @@ struct zswap_lruvec_state { atomic_long_t nr_zswap_protected; }; +unsigned long zswap_total_pages(void); bool zswap_store(struct folio *folio); bool zswap_load(struct folio *folio); void zswap_invalidate(swp_entry_t swp); diff --git a/mm/zswap.c b/mm/zswap.c index 2817e2791c135..a56b76852664c 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -43,8 +43,6 @@ /********************************* * statistics **********************************/ -/* Total bytes used by the compressed storage */ -u64 zswap_pool_total_size; /* The number of compressed pages currently stored in zswap */ atomic_t zswap_stored_pages = ATOMIC_INIT(0); /* The number of same-value filled pages currently stored in zswap */ @@ -265,45 +263,6 @@ static inline struct zswap_tree *swap_zswap_tree(swp_entry_t swp) pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \ zpool_get_type((p)->zpools[0])) -static bool zswap_is_full(void) -{ - return totalram_pages() * zswap_max_pool_percent / 100 < - DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE); -} - -static bool zswap_can_accept(void) -{ - return totalram_pages() * zswap_accept_thr_percent / 100 * - zswap_max_pool_percent / 100 > - DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE); -} - -static u64 get_zswap_pool_size(struct zswap_pool *pool) -{ - u64 pool_size = 0; - int i; - - for (i = 0; i < ZSWAP_NR_ZPOOLS; i++) - pool_size += zpool_get_total_size(pool->zpools[i]); - - return pool_size; -} - -static void zswap_update_total_size(void) -{ - struct zswap_pool *pool; - u64 total = 0; - - rcu_read_lock(); - - list_for_each_entry_rcu(pool, &zswap_pools, list) - total += get_zswap_pool_size(pool); - - rcu_read_unlock(); - - zswap_pool_total_size = total; -} - /********************************* * pool functions **********************************/ @@ -541,6 +500,33 @@ static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor) return NULL; } +static unsigned long zswap_max_pages(void) +{ + return totalram_pages() * zswap_max_pool_percent / 100; +} + +static unsigned long zswap_accept_thr_pages(void) +{ + return zswap_max_pages() * zswap_accept_thr_percent / 100; +} + +unsigned long zswap_total_pages(void) +{ + struct zswap_pool *pool; + u64 total = 0; + + rcu_read_lock(); + list_for_each_entry_rcu(pool, &zswap_pools, list) { + int i; + + for (i = 0; i < ZSWAP_NR_ZPOOLS; i++) + total += zpool_get_total_size(pool->zpools[i]); + } + rcu_read_unlock(); + + return total >> PAGE_SHIFT; +} + /********************************* * param callbacks **********************************/ @@ -913,7 +899,6 @@ static void zswap_entry_free(struct zswap_entry *entry) } zswap_entry_cache_free(entry); atomic_dec(&zswap_stored_pages); - zswap_update_total_size(); } /* @@ -1344,7 +1329,7 @@ static unsigned long zswap_shrinker_count(struct shrinker *shrinker, nr_backing = memcg_page_state(memcg, MEMCG_ZSWAP_B) >> PAGE_SHIFT; nr_stored = memcg_page_state(memcg, MEMCG_ZSWAPPED); } else { - nr_backing = zswap_pool_total_size >> PAGE_SHIFT; + nr_backing = zswap_total_pages(); nr_stored = atomic_read(&zswap_nr_stored); } @@ -1412,6 +1397,10 @@ static void shrink_worker(struct work_struct *w) { struct mem_cgroup *memcg; int ret, failures = 0; + unsigned long thr; + + /* Reclaim down to the accept threshold */ + thr = zswap_accept_thr_pages(); /* global reclaim will select cgroup in a round-robin fashion. */ do { @@ -1459,10 +1448,9 @@ static void shrink_worker(struct work_struct *w) break; if (ret && ++failures == MAX_RECLAIM_RETRIES) break; - resched: cond_resched(); - } while (!zswap_can_accept()); + } while (zswap_total_pages() > thr); } static int zswap_is_page_same_filled(void *ptr, unsigned long *value) @@ -1503,6 +1491,7 @@ bool zswap_store(struct folio *folio) struct zswap_entry *entry, *dupentry; struct obj_cgroup *objcg = NULL; struct mem_cgroup *memcg = NULL; + unsigned long max_pages, cur_pages; VM_WARN_ON_ONCE(!folio_test_locked(folio)); VM_WARN_ON_ONCE(!folio_test_swapcache(folio)); @@ -1514,6 +1503,7 @@ bool zswap_store(struct folio *folio) if (!zswap_enabled) goto check_old; + /* Check cgroup limits */ objcg = get_obj_cgroup_from_folio(folio); if (objcg && !obj_cgroup_may_zswap(objcg)) { memcg = get_mem_cgroup_from_objcg(objcg); @@ -1524,15 +1514,18 @@ bool zswap_store(struct folio *folio) mem_cgroup_put(memcg); } - /* reclaim space if needed */ - if (zswap_is_full()) { + /* Check global limits */ + cur_pages = zswap_total_pages(); + max_pages = zswap_max_pages(); + + if (cur_pages >= max_pages) { zswap_pool_limit_hit++; zswap_pool_reached_full = true; goto shrink; } if (zswap_pool_reached_full) { - if (!zswap_can_accept()) + if (cur_pages > zswap_accept_thr_pages()) goto shrink; else zswap_pool_reached_full = false; @@ -1608,7 +1601,6 @@ bool zswap_store(struct folio *folio) /* update stats */ atomic_inc(&zswap_stored_pages); - zswap_update_total_size(); count_vm_event(ZSWPOUT); return true; @@ -1752,6 +1744,13 @@ void zswap_swapoff(int type) static struct dentry *zswap_debugfs_root; +static int debugfs_get_total_size(void *data, u64 *val) +{ + *val = zswap_total_pages() * PAGE_SIZE; + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(total_size_fops, debugfs_get_total_size, NULL, "%llu\n"); + static int zswap_debugfs_init(void) { if (!debugfs_initialized()) @@ -1773,8 +1772,8 @@ static int zswap_debugfs_init(void) zswap_debugfs_root, &zswap_reject_compress_poor); debugfs_create_u64("written_back_pages", 0444, zswap_debugfs_root, &zswap_written_back_pages); - debugfs_create_u64("pool_total_size", 0444, - zswap_debugfs_root, &zswap_pool_total_size); + debugfs_create_file("pool_total_size", 0444, + zswap_debugfs_root, NULL, &total_size_fops); debugfs_create_atomic_t("stored_pages", 0444, zswap_debugfs_root, &zswap_stored_pages); debugfs_create_atomic_t("same_filled_pages", 0444, From 4196b48ddd382419e51658bf94a25af195ba9450 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Tue, 12 Mar 2024 11:34:12 -0400 Subject: [PATCH 0548/1702] mm: zpool: return pool size in pages All zswap backends track their pool sizes in pages. Currently they multiply by PAGE_SIZE for zswap, only for zswap to divide again in order to do limit math. Report pages directly. Link: https://lkml.kernel.org/r/20240312153901.3441-2-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Acked-by: Yosry Ahmed Reviewed-by: Chengming Zhou Reviewed-by: Nhat Pham Cc: Yosry Ahmed Signed-off-by: Andrew Morton --- include/linux/zpool.h | 4 ++-- mm/z3fold.c | 10 +++++----- mm/zbud.c | 10 +++++----- mm/zpool.c | 10 +++++----- mm/zsmalloc.c | 6 +++--- mm/zswap.c | 6 +++--- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/include/linux/zpool.h b/include/linux/zpool.h index 3296438eec062..a67d62b796980 100644 --- a/include/linux/zpool.h +++ b/include/linux/zpool.h @@ -53,7 +53,7 @@ void *zpool_map_handle(struct zpool *pool, unsigned long handle, void zpool_unmap_handle(struct zpool *pool, unsigned long handle); -u64 zpool_get_total_size(struct zpool *pool); +u64 zpool_get_total_pages(struct zpool *pool); /** @@ -91,7 +91,7 @@ struct zpool_driver { enum zpool_mapmode mm); void (*unmap)(void *pool, unsigned long handle); - u64 (*total_size)(void *pool); + u64 (*total_pages)(void *pool); }; void zpool_register_driver(struct zpool_driver *driver); diff --git a/mm/z3fold.c b/mm/z3fold.c index 7ab05621052dc..2ebfed32871bf 100644 --- a/mm/z3fold.c +++ b/mm/z3fold.c @@ -1237,12 +1237,12 @@ static void z3fold_unmap(struct z3fold_pool *pool, unsigned long handle) } /** - * z3fold_get_pool_size() - gets the z3fold pool size in pages + * z3fold_get_pool_pages() - gets the z3fold pool size in pages * @pool: pool whose size is being queried * * Returns: size in pages of the given pool. */ -static u64 z3fold_get_pool_size(struct z3fold_pool *pool) +static u64 z3fold_get_pool_pages(struct z3fold_pool *pool) { return atomic64_read(&pool->pages_nr); } @@ -1402,9 +1402,9 @@ static void z3fold_zpool_unmap(void *pool, unsigned long handle) z3fold_unmap(pool, handle); } -static u64 z3fold_zpool_total_size(void *pool) +static u64 z3fold_zpool_total_pages(void *pool) { - return z3fold_get_pool_size(pool) * PAGE_SIZE; + return z3fold_get_pool_pages(pool); } static struct zpool_driver z3fold_zpool_driver = { @@ -1417,7 +1417,7 @@ static struct zpool_driver z3fold_zpool_driver = { .free = z3fold_zpool_free, .map = z3fold_zpool_map, .unmap = z3fold_zpool_unmap, - .total_size = z3fold_zpool_total_size, + .total_pages = z3fold_zpool_total_pages, }; MODULE_ALIAS("zpool-z3fold"); diff --git a/mm/zbud.c b/mm/zbud.c index 2190cc1f37b3d..e9836fff94381 100644 --- a/mm/zbud.c +++ b/mm/zbud.c @@ -365,13 +365,13 @@ static void zbud_unmap(struct zbud_pool *pool, unsigned long handle) } /** - * zbud_get_pool_size() - gets the zbud pool size in pages + * zbud_get_pool_pages() - gets the zbud pool size in pages * @pool: pool whose size is being queried * * Returns: size in pages of the given pool. The pool lock need not be * taken to access pages_nr. */ -static u64 zbud_get_pool_size(struct zbud_pool *pool) +static u64 zbud_get_pool_pages(struct zbud_pool *pool) { return pool->pages_nr; } @@ -410,9 +410,9 @@ static void zbud_zpool_unmap(void *pool, unsigned long handle) zbud_unmap(pool, handle); } -static u64 zbud_zpool_total_size(void *pool) +static u64 zbud_zpool_total_pages(void *pool) { - return zbud_get_pool_size(pool) * PAGE_SIZE; + return zbud_get_pool_pages(pool); } static struct zpool_driver zbud_zpool_driver = { @@ -425,7 +425,7 @@ static struct zpool_driver zbud_zpool_driver = { .free = zbud_zpool_free, .map = zbud_zpool_map, .unmap = zbud_zpool_unmap, - .total_size = zbud_zpool_total_size, + .total_pages = zbud_zpool_total_pages, }; MODULE_ALIAS("zpool-zbud"); diff --git a/mm/zpool.c b/mm/zpool.c index 846410479c2f4..b9fda1fa857da 100644 --- a/mm/zpool.c +++ b/mm/zpool.c @@ -321,16 +321,16 @@ void zpool_unmap_handle(struct zpool *zpool, unsigned long handle) } /** - * zpool_get_total_size() - The total size of the pool + * zpool_get_total_pages() - The total size of the pool * @zpool: The zpool to check * - * This returns the total size in bytes of the pool. + * This returns the total size in pages of the pool. * - * Returns: Total size of the zpool in bytes. + * Returns: Total size of the zpool in pages. */ -u64 zpool_get_total_size(struct zpool *zpool) +u64 zpool_get_total_pages(struct zpool *zpool) { - return zpool->driver->total_size(zpool->pool); + return zpool->driver->total_pages(zpool->pool); } /** diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 7d7cb3eaabe02..b42d3545ca856 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -399,9 +399,9 @@ static void zs_zpool_unmap(void *pool, unsigned long handle) zs_unmap_object(pool, handle); } -static u64 zs_zpool_total_size(void *pool) +static u64 zs_zpool_total_pages(void *pool) { - return zs_get_total_pages(pool) << PAGE_SHIFT; + return zs_get_total_pages(pool); } static struct zpool_driver zs_zpool_driver = { @@ -414,7 +414,7 @@ static struct zpool_driver zs_zpool_driver = { .free = zs_zpool_free, .map = zs_zpool_map, .unmap = zs_zpool_unmap, - .total_size = zs_zpool_total_size, + .total_pages = zs_zpool_total_pages, }; MODULE_ALIAS("zpool-zsmalloc"); diff --git a/mm/zswap.c b/mm/zswap.c index a56b76852664c..8721980a22740 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -513,18 +513,18 @@ static unsigned long zswap_accept_thr_pages(void) unsigned long zswap_total_pages(void) { struct zswap_pool *pool; - u64 total = 0; + unsigned long total = 0; rcu_read_lock(); list_for_each_entry_rcu(pool, &zswap_pools, list) { int i; for (i = 0; i < ZSWAP_NR_ZPOOLS; i++) - total += zpool_get_total_size(pool->zpools[i]); + total += zpool_get_total_pages(pool->zpools[i]); } rcu_read_unlock(); - return total >> PAGE_SHIFT; + return total; } /********************************* From c2af060d1c18beaec56351cf9c9bcbbc5af341a3 Mon Sep 17 00:00:00 2001 From: Duoming Zhou Date: Tue, 12 Mar 2024 08:59:05 +0800 Subject: [PATCH 0549/1702] lib/test_hmm.c: handle src_pfns and dst_pfns allocation failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kcalloc() in dmirror_device_evict_chunk() will return null if the physical memory has run out. As a result, if src_pfns or dst_pfns is dereferenced, the null pointer dereference bug will happen. Moreover, the device is going away. If the kcalloc() fails, the pages mapping a chunk could not be evicted. So add a __GFP_NOFAIL flag in kcalloc(). Finally, as there is no need to have physically contiguous memory, Switch kcalloc() to kvcalloc() in order to avoid failing allocations. Link: https://lkml.kernel.org/r/20240312005905.9939-1-duoming@zju.edu.cn Fixes: b2ef9f5a5cb3 ("mm/hmm/test: add selftest driver for HMM") Signed-off-by: Duoming Zhou Cc: Jérôme Glisse Signed-off-by: Andrew Morton --- lib/test_hmm.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 717dcb8301273..b823ba7cb6a15 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -1226,8 +1226,8 @@ static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk) unsigned long *src_pfns; unsigned long *dst_pfns; - src_pfns = kcalloc(npages, sizeof(*src_pfns), GFP_KERNEL); - dst_pfns = kcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL); + src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); + dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); migrate_device_range(src_pfns, start_pfn, npages); for (i = 0; i < npages; i++) { @@ -1250,8 +1250,8 @@ static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk) } migrate_device_pages(src_pfns, dst_pfns, npages); migrate_device_finalize(src_pfns, dst_pfns, npages); - kfree(src_pfns); - kfree(dst_pfns); + kvfree(src_pfns); + kvfree(dst_pfns); } /* Removes free pages from the free list so they can't be re-allocated */ From fea68a75651cfc3458318416a1a2b45161cd22c2 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Mon, 11 Mar 2024 23:52:10 +0000 Subject: [PATCH 0550/1702] mm: zswap: remove unnecessary check in zswap_find_zpool() zswap_find_zpool() checks if ZSWAP_NR_ZPOOLS > 1, which is always true. This is a remnant from a patch version that had ZSWAP_NR_ZPOOLS as a config option and never made it upstream. Remove the unnecessary check. Link: https://lkml.kernel.org/r/20240311235210.2937484-1-yosryahmed@google.com Signed-off-by: Yosry Ahmed Reviewed-by: Chengming Zhou Reviewed-by: Nhat Pham Acked-by: Johannes Weiner Cc: Yosry Ahmed Signed-off-by: Andrew Morton --- mm/zswap.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index 8721980a22740..e435dc7c043b7 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -871,12 +871,7 @@ static void zswap_entry_cache_free(struct zswap_entry *entry) static struct zpool *zswap_find_zpool(struct zswap_entry *entry) { - int i = 0; - - if (ZSWAP_NR_ZPOOLS > 1) - i = hash_ptr(entry, ilog2(ZSWAP_NR_ZPOOLS)); - - return entry->pool->zpools[i]; + return entry->pool->zpools[hash_ptr(entry, ilog2(ZSWAP_NR_ZPOOLS))]; } /* From f8fd525ba3a298d5791536bd8ca39be7a7103296 Mon Sep 17 00:00:00 2001 From: Donet Tom Date: Fri, 8 Mar 2024 09:15:37 -0600 Subject: [PATCH 0551/1702] mm/mempolicy: use numa_node_id() instead of cpu_to_node() Patch series "Allow migrate on protnone reference with MPOL_PREFERRED_MANY policy:, v4. This patchset is to optimize the cross-socket memory access with MPOL_PREFERRED_MANY policy. To test this patch we ran the following test on a 3 node system. Node 0 - 2GB - Tier 1 Node 1 - 11GB - Tier 1 Node 6 - 10GB - Tier 2 Below changes are made to memcached to set the memory policy, It select Node0 and Node1 as preferred nodes. #include #include unsigned long nodemask; int ret; nodemask = 0x03; ret = set_mempolicy(MPOL_PREFERRED_MANY | MPOL_F_NUMA_BALANCING, &nodemask, 10); /* If MPOL_F_NUMA_BALANCING isn't supported, * fall back to MPOL_PREFERRED_MANY */ if (ret < 0 && errno == EINVAL){ printf("set mem policy normal\n"); ret = set_mempolicy(MPOL_PREFERRED_MANY, &nodemask, 10); } if (ret < 0) { perror("Failed to call set_mempolicy"); exit(-1); } Test Procedure: =============== 1. Make sure memory tiring and demotion are enabled. 2. Start memcached. # ./memcached -b 100000 -m 204800 -u root -c 1000000 -t 7 -d -s "/tmp/memcached.sock" 3. Run memtier_benchmark to store 3200000 keys. #./memtier_benchmark -S "/tmp/memcached.sock" --protocol=memcache_binary --threads=1 --pipeline=1 --ratio=1:0 --key-pattern=S:S --key-minimum=1 --key-maximum=3200000 -n allkeys -c 1 -R -x 1 -d 1024 4. Start a memory eater on node 0 and 1. This will demote all memcached pages to node 6. 5. Make sure all the memcached pages got demoted to lower tier by reading /proc//numa_maps. # cat /proc/2771/numa_maps --- default anon=1009 dirty=1009 active=0 N6=1009 kernelpagesize_kB=64 default anon=1009 dirty=1009 active=0 N6=1009 kernelpagesize_kB=64 --- 6. Kill memory eater. 7. Read the pgpromote_success counter. 8. Start reading the keys by running memtier_benchmark. #./memtier_benchmark -S "/tmp/memcached.sock" --protocol=memcache_binary --pipeline=1 --distinct-client-seed --ratio=0:3 --key-pattern=R:R --key-minimum=1 --key-maximum=3200000 -n allkeys --threads=64 -c 1 -R -x 6 9. Read the pgpromote_success counter. Test Results: ============= Without Patch ------------------ 1. pgpromote_success before test Node 0: pgpromote_success 11 Node 1: pgpromote_success 140974 pgpromote_success after test Node 0: pgpromote_success 11 Node 1: pgpromote_success 140974 2. Memtier-benchmark result. AGGREGATED AVERAGE RESULTS (6 runs) ================================================================== Type Ops/sec Hits/sec Misses/sec Avg. Latency p50 Latency ------------------------------------------------------------------ Sets 0.00 --- --- --- --- Gets 305792.03 305791.93 0.10 0.18949 0.16700 Waits 0.00 --- --- --- --- Totals 305792.03 305791.93 0.10 0.18949 0.16700 ====================================== p99 Latency p99.9 Latency KB/sec ------------------------------------- --- --- 0.00 0.44700 1.71100 11542.69 --- --- --- 0.44700 1.71100 11542.69 With Patch --------------- 1. pgpromote_success before test Node 0: pgpromote_success 5 Node 1: pgpromote_success 89386 pgpromote_success after test Node 0: pgpromote_success 57895 Node 1: pgpromote_success 141463 2. Memtier-benchmark result. AGGREGATED AVERAGE RESULTS (6 runs) ==================================================================== Type Ops/sec Hits/sec Misses/sec Avg. Latency p50 Latency -------------------------------------------------------------------- Sets 0.00 --- --- --- --- Gets 521942.24 521942.07 0.17 0.11459 0.10300 Waits 0.00 --- --- --- --- Totals 521942.24 521942.07 0.17 0.11459 0.10300 ======================================= p99 Latency p99.9 Latency KB/sec --------------------------------------- --- --- 0.00 0.23100 0.31900 19701.68 --- --- --- 0.23100 0.31900 19701.68 Test Result Analysis: ===================== 1. With patch we could observe pages are getting promoted. 2. Memtier-benchmark results shows that, with the patch, performance has increased more than 50%. Ops/sec without fix - 305792.03 Ops/sec with fix - 521942.24 This patch (of 2): Instead of using 'cpu_to_node()', we use 'numa_node_id()', which is quicker. smp_processor_id is guaranteed to be stable in the 'mpol_misplaced()' function because it is called with ptl held. lockdep_assert_held was added to ensure that. No functional change in this patch. [donettom@linux.ibm.com: add "* @vmf: structure describing the fault" comment] Link: https://lkml.kernel.org/r/d8b993ea9dccfac0bc3ed61d3a81f4ac5f376e46.1711002865.git.donettom@linux.ibm.com Link: https://lkml.kernel.org/r/cover.1711373653.git.donettom@linux.ibm.com Link: https://lkml.kernel.org/r/6059f034f436734b472d066db69676fb3a459864.1711373653.git.donettom@linux.ibm.com Link: https://lkml.kernel.org/r/cover.1709909210.git.donettom@linux.ibm.com Link: https://lkml.kernel.org/r/744646531af02cc687cde8ae788fb1779e99d02c.1709909210.git.donettom@linux.ibm.com Signed-off-by: Aneesh Kumar K.V (IBM) Signed-off-by: Donet Tom Cc: Andrea Arcangeli Cc: Dan Williams Cc: Dave Hansen Cc: Feng Tang Cc: Huang, Ying Cc: Hugh Dickins Cc: Ingo Molnar Cc: Johannes Weiner Cc: Kefeng Wang Cc: "Matthew Wilcox (Oracle)" Cc: Mel Gorman Cc: Michal Hocko Cc: Peter Zijlstra Cc: Rik van Riel Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/mempolicy.h | 5 +++-- mm/huge_memory.c | 2 +- mm/internal.h | 2 +- mm/memory.c | 8 +++++--- mm/mempolicy.c | 14 ++++++++++---- 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h index 931b118336f48..1add16f216124 100644 --- a/include/linux/mempolicy.h +++ b/include/linux/mempolicy.h @@ -167,7 +167,8 @@ extern void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol); /* Check if a vma is migratable */ extern bool vma_migratable(struct vm_area_struct *vma); -int mpol_misplaced(struct folio *, struct vm_area_struct *, unsigned long); +int mpol_misplaced(struct folio *folio, struct vm_fault *vmf, + unsigned long addr); extern void mpol_put_task_policy(struct task_struct *); static inline bool mpol_is_preferred_many(struct mempolicy *pol) @@ -282,7 +283,7 @@ static inline int mpol_parse_str(char *str, struct mempolicy **mpol) #endif static inline int mpol_misplaced(struct folio *folio, - struct vm_area_struct *vma, + struct vm_fault *vmf, unsigned long address) { return -1; /* no node preference */ diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 89f58c7603b25..dd88bb3ea196e 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1754,7 +1754,7 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) */ if (node_is_toptier(nid)) last_cpupid = folio_last_cpupid(folio); - target_nid = numa_migrate_prep(folio, vma, haddr, nid, &flags); + target_nid = numa_migrate_prep(folio, vmf, haddr, nid, &flags); if (target_nid == NUMA_NO_NODE) { folio_put(folio); goto out_map; diff --git a/mm/internal.h b/mm/internal.h index 07ad2675a88b4..85c3db43454de 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1087,7 +1087,7 @@ void vunmap_range_noflush(unsigned long start, unsigned long end); void __vunmap_range_noflush(unsigned long start, unsigned long end); -int numa_migrate_prep(struct folio *folio, struct vm_area_struct *vma, +int numa_migrate_prep(struct folio *folio, struct vm_fault *vmf, unsigned long addr, int page_nid, int *flags); void free_zone_device_page(struct page *page); diff --git a/mm/memory.c b/mm/memory.c index e0576f2a8d236..c859a09b4f722 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5035,9 +5035,11 @@ static vm_fault_t do_fault(struct vm_fault *vmf) return ret; } -int numa_migrate_prep(struct folio *folio, struct vm_area_struct *vma, +int numa_migrate_prep(struct folio *folio, struct vm_fault *vmf, unsigned long addr, int page_nid, int *flags) { + struct vm_area_struct *vma = vmf->vma; + folio_get(folio); /* Record the current PID acceesing VMA */ @@ -5049,7 +5051,7 @@ int numa_migrate_prep(struct folio *folio, struct vm_area_struct *vma, *flags |= TNF_FAULT_LOCAL; } - return mpol_misplaced(folio, vma, addr); + return mpol_misplaced(folio, vmf, addr); } static vm_fault_t do_numa_page(struct vm_fault *vmf) @@ -5123,7 +5125,7 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) last_cpupid = (-1 & LAST_CPUPID_MASK); else last_cpupid = folio_last_cpupid(folio); - target_nid = numa_migrate_prep(folio, vma, vmf->address, nid, &flags); + target_nid = numa_migrate_prep(folio, vmf, vmf->address, nid, &flags); if (target_nid == NUMA_NO_NODE) { folio_put(folio); goto out_map; diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 0fe77738d971d..aa48376e2d341 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -2718,7 +2718,7 @@ static void sp_free(struct sp_node *n) * mpol_misplaced - check whether current folio node is valid in policy * * @folio: folio to be checked - * @vma: vm area where folio mapped + * @vmf: structure describing the fault * @addr: virtual address in @vma for shared policy lookup and interleave policy * * Lookup current policy node id for vma,addr and "compare to" folio's @@ -2728,18 +2728,24 @@ static void sp_free(struct sp_node *n) * Return: NUMA_NO_NODE if the page is in a node that is valid for this * policy, or a suitable node ID to allocate a replacement folio from. */ -int mpol_misplaced(struct folio *folio, struct vm_area_struct *vma, +int mpol_misplaced(struct folio *folio, struct vm_fault *vmf, unsigned long addr) { struct mempolicy *pol; pgoff_t ilx; struct zoneref *z; int curnid = folio_nid(folio); + struct vm_area_struct *vma = vmf->vma; int thiscpu = raw_smp_processor_id(); - int thisnid = cpu_to_node(thiscpu); + int thisnid = numa_node_id(); int polnid = NUMA_NO_NODE; int ret = NUMA_NO_NODE; + /* + * Make sure ptl is held so that we don't preempt and we + * have a stable smp processor id + */ + lockdep_assert_held(vmf->ptl); pol = get_vma_policy(vma, addr, folio_order(folio), &ilx); if (!(pol->flags & MPOL_F_MOF)) goto out; @@ -2781,7 +2787,7 @@ int mpol_misplaced(struct folio *folio, struct vm_area_struct *vma, if (node_isset(curnid, pol->nodes)) goto out; z = first_zones_zonelist( - node_zonelist(numa_node_id(), GFP_HIGHUSER), + node_zonelist(thisnid, GFP_HIGHUSER), gfp_zone(GFP_HIGHUSER), &pol->nodes); polnid = zone_to_nid(z->zone); From 133d04b1eee9901b57973220f2d7801f0e84c890 Mon Sep 17 00:00:00 2001 From: Donet Tom Date: Fri, 8 Mar 2024 09:15:38 -0600 Subject: [PATCH 0552/1702] mm/numa_balancing: allow migrate on protnone reference with MPOL_PREFERRED_MANY policy commit bda420b98505 ("numa balancing: migrate on fault among multiple bound nodes") added support for migrate on protnone reference with MPOL_BIND memory policy. This allowed numa fault migration when the executing node is part of the policy mask for MPOL_BIND. This patch extends migration support to MPOL_PREFERRED_MANY policy. Currently, we cannot specify MPOL_PREFERRED_MANY with the mempolicy flag MPOL_F_NUMA_BALANCING. This causes issues when we want to use NUMA_BALANCING_MEMORY_TIERING. To effectively use the slow memory tier, the kernel should not allocate pages from the slower memory tier via allocation control zonelist fallback. Instead, we should move cold pages from the faster memory node via memory demotion. For a page allocation, kswapd is only woken up after we try to allocate pages from all nodes in the allocation zone list. This implies that, without using memory policies, we will end up allocating hot pages in the slower memory tier. MPOL_PREFERRED_MANY was added by commit b27abaccf8e8 ("mm/mempolicy: add MPOL_PREFERRED_MANY for multiple preferred nodes") to allow better allocation control when we have memory tiers in the system. With MPOL_PREFERRED_MANY, the user can use a policy node mask consisting only of faster memory nodes. When we fail to allocate pages from the faster memory node, kswapd would be woken up, allowing demotion of cold pages to slower memory nodes. With the current kernel, such usage of memory policies implies we can't do page promotion from a slower memory tier to a faster memory tier using numa fault. This patch fixes this issue. For MPOL_PREFERRED_MANY, if the executing node is in the policy node mask, we allow numa migration to the executing nodes. If the executing node is not in the policy node mask, we do not allow numa migration. Example: On a 2-sockets system, NUMA node N0, N1 and N2 are in socket 0, N3 in socket 1. N0, N1 and N3 have fast memory and CPU, while N2 has slow memory and no CPU. For a workload, we may use MPOL_PREFERRED_MANY with nodemask N0 and N1 set because the workload runs on CPUs of socket 0 at most times. Then, even if the workload runs on CPUs of N3 occasionally, we will not try to migrate the workload pages from N2 to N3 because users may want to avoid cross-socket access as much as possible in the long term. In below table, Process is the Process executing node and Curr Loc Pgs is the numa node where page present(folio node) =========================================================== Process Policy Curr Loc Pgs Observation ----------------------------------------------------------- N0 N0 N1 N1 Pages Migrated from N1 to N0 N0 N0 N1 N2 Pages Migrated from N2 to N0 N0 N0 N1 N3 Pages Migrated from N3 to N0 N3 N0 N1 N0 Pages NOT Migrated to N3 N3 N0 N1 N1 Pages NOT Migrated to N3 N3 N0 N1 N2 Pages NOT Migrated to N3 ------------------------------------------------------------ Link: https://lkml.kernel.org/r/158acc57319129aa46d50fd64c9330f3e7c7b4bf.1711373653.git.donettom@linux.ibm.com Link: https://lkml.kernel.org/r/369d6a58758396335fd1176d97bbca4e7730d75a.1709909210.git.donettom@linux.ibm.com Signed-off-by: Aneesh Kumar K.V (IBM) Signed-off-by: Donet Tom Cc: Andrea Arcangeli Cc: Dan Williams Cc: Dave Hansen Cc: Feng Tang Cc: Huang, Ying Cc: Hugh Dickins Cc: Ingo Molnar Cc: Johannes Weiner Cc: Kefeng Wang Cc: "Matthew Wilcox (Oracle)" Cc: Mel Gorman Cc: Michal Hocko Cc: Peter Zijlstra Cc: Rik van Riel Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/mempolicy.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index aa48376e2d341..13100a290918f 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -1504,9 +1504,10 @@ static inline int sanitize_mpol_flags(int *mode, unsigned short *flags) if ((*flags & MPOL_F_STATIC_NODES) && (*flags & MPOL_F_RELATIVE_NODES)) return -EINVAL; if (*flags & MPOL_F_NUMA_BALANCING) { - if (*mode != MPOL_BIND) + if (*mode == MPOL_BIND || *mode == MPOL_PREFERRED_MANY) + *flags |= (MPOL_F_MOF | MPOL_F_MORON); + else return -EINVAL; - *flags |= (MPOL_F_MOF | MPOL_F_MORON); } return 0; } @@ -2770,15 +2771,26 @@ int mpol_misplaced(struct folio *folio, struct vm_fault *vmf, break; case MPOL_BIND: - /* Optimize placement among multiple nodes via NUMA balancing */ + case MPOL_PREFERRED_MANY: + /* + * Even though MPOL_PREFERRED_MANY can allocate pages outside + * policy nodemask we don't allow numa migration to nodes + * outside policy nodemask for now. This is done so that if we + * want demotion to slow memory to happen, before allocating + * from some DRAM node say 'x', we will end up using a + * MPOL_PREFERRED_MANY mask excluding node 'x'. In such scenario + * we should not promote to node 'x' from slow memory node. + */ if (pol->flags & MPOL_F_MORON) { + /* + * Optimize placement among multiple nodes + * via NUMA balancing + */ if (node_isset(thisnid, pol->nodes)) break; goto out; } - fallthrough; - case MPOL_PREFERRED_MANY: /* * use current page if in policy nodemask, * else select nearest allowed node, if any. From 2ccd48ce35e87f09472b42dda96fbf7b5165f3c3 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Mon, 11 Mar 2024 19:43:46 +0000 Subject: [PATCH 0553/1702] percpu: clean up all mappings when pcpu_map_pages() fails In pcpu_map_pages(), if __pcpu_map_pages() fails on a CPU, we call __pcpu_unmap_pages() to clean up mappings on all CPUs where mappings were created, but not on the CPU where __pcpu_map_pages() fails. __pcpu_map_pages() and __pcpu_unmap_pages() are wrappers around vmap_pages_range_noflush() and vunmap_range_noflush(). All other callers of vmap_pages_range_noflush() call vunmap_range_noflush() when mapping fails, except pcpu_map_pages(). The reason could be that partial mappings may be left behind from a failed mapping attempt. Call __pcpu_unmap_pages() for the failed CPU as well in pcpu_map_pages(). This was found by code inspection, no failures or bugs were observed. Link: https://lkml.kernel.org/r/20240311194346.2291333-1-yosryahmed@google.com Signed-off-by: Yosry Ahmed Acked-by: Dennis Zhou Cc: Christoph Lameter (Ampere) Cc: Tejun Heo Signed-off-by: Andrew Morton --- mm/percpu-vm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/percpu-vm.c b/mm/percpu-vm.c index 2054c9213c433..cd69caf6aa8d8 100644 --- a/mm/percpu-vm.c +++ b/mm/percpu-vm.c @@ -231,10 +231,10 @@ static int pcpu_map_pages(struct pcpu_chunk *chunk, return 0; err: for_each_possible_cpu(tcpu) { - if (tcpu == cpu) - break; __pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start), page_end - page_start); + if (tcpu == cpu) + break; } pcpu_post_unmap_tlb_flush(chunk, page_start, page_end); return err; From 51a7bf0238c265a34d7d27c489eb4cd52e083e87 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Mon, 25 Mar 2024 22:41:49 -0700 Subject: [PATCH 0554/1702] scripts/kernel-doc: drop "_noprof" on function prototypes Memory profiling introduces macros as hooks for function-level allocation profiling[1]. Memory allocation functions that are profiled are named like xyz_alloc() for API access to the function. xyz_alloc() then calls xyz_alloc_noprof() to do the allocation work. The kernel-doc comments for the memory allocation functions are introduced with the xyz_alloc() function names but the function implementations are the xyz_alloc_noprof() names. This causes kernel-doc warnings for mismatched documentation and function prototype names. By dropping the "_noprof" part of the function name, the kernel-doc function name matches the function prototype name, so the warnings are resolved. [1] https://lore.kernel.org/all/20240321163705.3067592-1-surenb@google.com/ Link: https://lkml.kernel.org/r/20240326054149.2121-1-rdunlap@infradead.org Signed-off-by: Randy Dunlap Reported-by: Stephen Rothwell Closes: https://lore.kernel.org/all/20240325123603.1bdd6588@canb.auug.org.au/ Tested-by: Suren Baghdasaryan Cc: Jonathan Corbet Cc: Kent Overstreet Signed-off-by: Andrew Morton --- scripts/kernel-doc | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index cb1be22afc65f..b463acecad401 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1723,6 +1723,7 @@ sub dump_function($$) { $prototype =~ s/__must_check +//; $prototype =~ s/__weak +//; $prototype =~ s/__sched +//; + $prototype =~ s/_noprof//; $prototype =~ s/__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +//; $prototype =~ s/__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +//; $prototype =~ s/__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +//; From 0069455bcbf9ea73ffe4553ed6d2b4e4cad703de Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 21 Mar 2024 09:36:23 -0700 Subject: [PATCH 0555/1702] fix missing vmalloc.h includes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "Memory allocation profiling", v6. Overview: Low overhead [1] per-callsite memory allocation profiling. Not just for debug kernels, overhead low enough to be deployed in production. Example output: root@moria-kvm:~# sort -rn /proc/allocinfo 127664128 31168 mm/page_ext.c:270 func:alloc_page_ext 56373248 4737 mm/slub.c:2259 func:alloc_slab_page 14880768 3633 mm/readahead.c:247 func:page_cache_ra_unbounded 14417920 3520 mm/mm_init.c:2530 func:alloc_large_system_hash 13377536 234 block/blk-mq.c:3421 func:blk_mq_alloc_rqs 11718656 2861 mm/filemap.c:1919 func:__filemap_get_folio 9192960 2800 kernel/fork.c:307 func:alloc_thread_stack_node 4206592 4 net/netfilter/nf_conntrack_core.c:2567 func:nf_ct_alloc_hashtable 4136960 1010 drivers/staging/ctagmod/ctagmod.c:20 [ctagmod] func:ctagmod_start 3940352 962 mm/memory.c:4214 func:alloc_anon_folio 2894464 22613 fs/kernfs/dir.c:615 func:__kernfs_new_node ... Usage: kconfig options: - CONFIG_MEM_ALLOC_PROFILING - CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT - CONFIG_MEM_ALLOC_PROFILING_DEBUG adds warnings for allocations that weren't accounted because of a missing annotation sysctl: /proc/sys/vm/mem_profiling Runtime info: /proc/allocinfo Notes: [1]: Overhead To measure the overhead we are comparing the following configurations: (1) Baseline with CONFIG_MEMCG_KMEM=n (2) Disabled by default (CONFIG_MEM_ALLOC_PROFILING=y && CONFIG_MEM_ALLOC_PROFILING_BY_DEFAULT=n) (3) Enabled by default (CONFIG_MEM_ALLOC_PROFILING=y && CONFIG_MEM_ALLOC_PROFILING_BY_DEFAULT=y) (4) Enabled at runtime (CONFIG_MEM_ALLOC_PROFILING=y && CONFIG_MEM_ALLOC_PROFILING_BY_DEFAULT=n && /proc/sys/vm/mem_profiling=1) (5) Baseline with CONFIG_MEMCG_KMEM=y && allocating with __GFP_ACCOUNT (6) Disabled by default (CONFIG_MEM_ALLOC_PROFILING=y && CONFIG_MEM_ALLOC_PROFILING_BY_DEFAULT=n) && CONFIG_MEMCG_KMEM=y (7) Enabled by default (CONFIG_MEM_ALLOC_PROFILING=y && CONFIG_MEM_ALLOC_PROFILING_BY_DEFAULT=y) && CONFIG_MEMCG_KMEM=y Performance overhead: To evaluate performance we implemented an in-kernel test executing multiple get_free_page/free_page and kmalloc/kfree calls with allocation sizes growing from 8 to 240 bytes with CPU frequency set to max and CPU affinity set to a specific CPU to minimize the noise. Below are results from running the test on Ubuntu 22.04.2 LTS with 6.8.0-rc1 kernel on 56 core Intel Xeon: kmalloc pgalloc (1 baseline) 6.764s 16.902s (2 default disabled) 6.793s (+0.43%) 17.007s (+0.62%) (3 default enabled) 7.197s (+6.40%) 23.666s (+40.02%) (4 runtime enabled) 7.405s (+9.48%) 23.901s (+41.41%) (5 memcg) 13.388s (+97.94%) 48.460s (+186.71%) (6 def disabled+memcg) 13.332s (+97.10%) 48.105s (+184.61%) (7 def enabled+memcg) 13.446s (+98.78%) 54.963s (+225.18%) Memory overhead: Kernel size: text data bss dec diff (1) 26515311 18890222 17018880 62424413 (2) 26524728 19423818 16740352 62688898 264485 (3) 26524724 19423818 16740352 62688894 264481 (4) 26524728 19423818 16740352 62688898 264485 (5) 26541782 18964374 16957440 62463596 39183 Memory consumption on a 56 core Intel CPU with 125GB of memory: Code tags: 192 kB PageExts: 262144 kB (256MB) SlabExts: 9876 kB (9.6MB) PcpuExts: 512 kB (0.5MB) Total overhead is 0.2% of total memory. Benchmarks: Hackbench tests run 100 times: hackbench -s 512 -l 200 -g 15 -f 25 -P baseline disabled profiling enabled profiling avg 0.3543 0.3559 (+0.0016) 0.3566 (+0.0023) stdev 0.0137 0.0188 0.0077 hackbench -l 10000 baseline disabled profiling enabled profiling avg 6.4218 6.4306 (+0.0088) 6.5077 (+0.0859) stdev 0.0933 0.0286 0.0489 stress-ng tests: stress-ng --class memory --seq 4 -t 60 stress-ng --class cpu --seq 4 -t 60 Results posted at: https://evilpiepirate.org/~kent/memalloc_prof_v4_stress-ng/ [2] https://lore.kernel.org/all/20240306182440.2003814-1-surenb@google.com/ This patch (of 37): The next patch drops vmalloc.h from a system header in order to fix a circular dependency; this adds it to all the files that were pulling it in implicitly. [kent.overstreet@linux.dev: fix arch/alpha/lib/memcpy.c] Link: https://lkml.kernel.org/r/20240327002152.3339937-1-kent.overstreet@linux.dev [surenb@google.com: fix arch/x86/mm/numa_32.c] Link: https://lkml.kernel.org/r/20240402180933.1663992-1-surenb@google.com [kent.overstreet@linux.dev: a few places were depending on sizes.h] Link: https://lkml.kernel.org/r/20240404034744.1664840-1-kent.overstreet@linux.dev [arnd@arndb.de: fix mm/kasan/hw_tags.c] Link: https://lkml.kernel.org/r/20240404124435.3121534-1-arnd@kernel.org [surenb@google.com: fix arc build] Link: https://lkml.kernel.org/r/20240405225115.431056-1-surenb@google.com Link: https://lkml.kernel.org/r/20240321163705.3067592-1-surenb@google.com Link: https://lkml.kernel.org/r/20240321163705.3067592-2-surenb@google.com Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Signed-off-by: Arnd Bergmann Reviewed-by: Pasha Tatashin Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- arch/alpha/lib/checksum.c | 1 + arch/alpha/lib/fpreg.c | 1 + arch/arc/include/asm/mmu-arcv2.h | 2 ++ arch/arm/kernel/irq.c | 1 + arch/arm/kernel/traps.c | 1 + arch/arm64/kernel/efi.c | 1 + arch/loongarch/include/asm/kfence.h | 1 + arch/powerpc/kernel/iommu.c | 1 + arch/powerpc/mm/mem.c | 1 + arch/riscv/kernel/elf_kexec.c | 1 + arch/riscv/kernel/probes/kprobes.c | 1 + arch/s390/kernel/cert_store.c | 1 + arch/s390/kernel/ipl.c | 1 + arch/x86/include/asm/io.h | 1 + arch/x86/kernel/cpu/sgx/main.c | 1 + arch/x86/kernel/irq_64.c | 1 + arch/x86/mm/fault.c | 1 + arch/x86/mm/numa_32.c | 1 + drivers/accel/ivpu/ivpu_mmu_context.c | 1 + drivers/gpu/drm/gma500/mmu.c | 1 + drivers/gpu/drm/i915/gem/i915_gem_pages.c | 1 + drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c | 1 + drivers/gpu/drm/i915/gt/shmem_utils.c | 1 + drivers/gpu/drm/i915/gvt/firmware.c | 1 + drivers/gpu/drm/i915/gvt/gtt.c | 1 + drivers/gpu/drm/i915/gvt/handlers.c | 1 + drivers/gpu/drm/i915/gvt/mmio.c | 1 + drivers/gpu/drm/i915/gvt/vgpu.c | 1 + drivers/gpu/drm/i915/intel_gvt.c | 1 + drivers/gpu/drm/imagination/pvr_vm_mips.c | 1 + drivers/gpu/drm/mediatek/mtk_drm_gem.c | 1 + drivers/gpu/drm/omapdrm/omap_gem.c | 1 + drivers/gpu/drm/v3d/v3d_bo.c | 1 + drivers/gpu/drm/vmwgfx/vmwgfx_binding.c | 1 + drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c | 1 + drivers/gpu/drm/vmwgfx/vmwgfx_devcaps.c | 1 + drivers/gpu/drm/vmwgfx/vmwgfx_drv.c | 1 + drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c | 1 + drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c | 1 + drivers/gpu/drm/xen/xen_drm_front_gem.c | 1 + drivers/hwtracing/coresight/coresight-trbe.c | 1 + drivers/net/ethernet/marvell/octeon_ep/octep_pfvf_mbox.c | 1 + drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_mbox.c | 1 + drivers/net/ethernet/microsoft/mana/hw_channel.c | 1 + drivers/platform/x86/uv_sysfs.c | 1 + drivers/scsi/mpi3mr/mpi3mr_transport.c | 2 ++ drivers/vfio/pci/pds/dirty.c | 1 + drivers/virt/acrn/mm.c | 1 + drivers/virtio/virtio_mem.c | 1 + include/asm-generic/io.h | 1 + include/linux/io.h | 1 + include/linux/pds/pds_common.h | 2 ++ include/rdma/rdmavt_qp.h | 1 + mm/debug_vm_pgtable.c | 1 + mm/kasan/hw_tags.c | 1 + sound/pci/hda/cs35l41_hda.c | 1 + 56 files changed, 59 insertions(+) diff --git a/arch/alpha/lib/checksum.c b/arch/alpha/lib/checksum.c index 3f35c3ed69488..c29b98ef9c826 100644 --- a/arch/alpha/lib/checksum.c +++ b/arch/alpha/lib/checksum.c @@ -14,6 +14,7 @@ #include #include +#include static inline unsigned short from64to16(unsigned long x) { diff --git a/arch/alpha/lib/fpreg.c b/arch/alpha/lib/fpreg.c index 7c08b225261c4..3d32165043f8b 100644 --- a/arch/alpha/lib/fpreg.c +++ b/arch/alpha/lib/fpreg.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #if defined(CONFIG_ALPHA_EV6) || defined(CONFIG_ALPHA_EV67) diff --git a/arch/arc/include/asm/mmu-arcv2.h b/arch/arc/include/asm/mmu-arcv2.h index ed9036d4ede31..d85dc07219071 100644 --- a/arch/arc/include/asm/mmu-arcv2.h +++ b/arch/arc/include/asm/mmu-arcv2.h @@ -9,6 +9,8 @@ #ifndef _ASM_ARC_MMU_ARCV2_H #define _ASM_ARC_MMU_ARCV2_H +#include + /* * TLB Management regs */ diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index fe28fc1f759d9..dab42d066d068 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 72c82a4d63ac2..480e307501bb4 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index 9afcc690fe73c..4a92096db34e0 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include diff --git a/arch/loongarch/include/asm/kfence.h b/arch/loongarch/include/asm/kfence.h index a6a5760da3a33..92636e82957c7 100644 --- a/arch/loongarch/include/asm/kfence.h +++ b/arch/loongarch/include/asm/kfence.h @@ -10,6 +10,7 @@ #define _ASM_LOONGARCH_KFENCE_H #include +#include #include #include diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index 1185efebf032b..65468d0829e1c 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index 3a440004b97d2..a197d4c2244b3 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include diff --git a/arch/riscv/kernel/elf_kexec.c b/arch/riscv/kernel/elf_kexec.c index 54260c16f9912..11c0d2e0becfe 100644 --- a/arch/riscv/kernel/elf_kexec.c +++ b/arch/riscv/kernel/elf_kexec.c @@ -19,6 +19,7 @@ #include #include #include +#include #include int arch_kimage_file_post_load_cleanup(struct kimage *image) diff --git a/arch/riscv/kernel/probes/kprobes.c b/arch/riscv/kernel/probes/kprobes.c index 2f08c14a933d0..71a8b8945b26f 100644 --- a/arch/riscv/kernel/probes/kprobes.c +++ b/arch/riscv/kernel/probes/kprobes.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/s390/kernel/cert_store.c b/arch/s390/kernel/cert_store.c index 554447768bddc..bf983513dd333 100644 --- a/arch/s390/kernel/cert_store.c +++ b/arch/s390/kernel/cert_store.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 1486350a41775..5d6d381aa0be4 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h index 294cd2a408181..7452fc193b4f9 100644 --- a/arch/x86/include/asm/io.h +++ b/arch/x86/include/asm/io.h @@ -42,6 +42,7 @@ #include #include #include +#include #define build_mmio_read(name, size, type, reg, barrier) \ static inline type name(const volatile void __iomem *addr) \ diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c index 166692f2d5011..27892e57c4ef9 100644 --- a/arch/x86/kernel/cpu/sgx/main.c +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include "driver.h" #include "encl.h" diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c index fe0c859873d17..ade0043ce56ef 100644 --- a/arch/x86/kernel/irq_64.c +++ b/arch/x86/kernel/irq_64.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 622d12ec7f085..a4cc20d0036d4 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -20,6 +20,7 @@ #include /* efi_crash_gracefully_on_page_fault()*/ #include #include /* find_and_lock_vma() */ +#include #include /* boot_cpu_has, ... */ #include /* dotraplinkage, ... */ diff --git a/arch/x86/mm/numa_32.c b/arch/x86/mm/numa_32.c index 025fd7ea5d69f..65fda406e6f26 100644 --- a/arch/x86/mm/numa_32.c +++ b/arch/x86/mm/numa_32.c @@ -24,6 +24,7 @@ #include #include +#include #include #include "numa_internal.h" diff --git a/drivers/accel/ivpu/ivpu_mmu_context.c b/drivers/accel/ivpu/ivpu_mmu_context.c index fe61612992364..128aef8e5a199 100644 --- a/drivers/accel/ivpu/ivpu_mmu_context.c +++ b/drivers/accel/ivpu/ivpu_mmu_context.c @@ -6,6 +6,7 @@ #include #include #include +#include #include diff --git a/drivers/gpu/drm/gma500/mmu.c b/drivers/gpu/drm/gma500/mmu.c index a70b01ccdf702..4d78b33eaa823 100644 --- a/drivers/gpu/drm/gma500/mmu.c +++ b/drivers/gpu/drm/gma500/mmu.c @@ -5,6 +5,7 @@ **************************************************************************/ #include +#include #include "mmu.h" #include "psb_drv.h" diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index 0ba955611dfb5..8780aa2431053 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -5,6 +5,7 @@ */ #include +#include #include "gt/intel_gt.h" #include "gt/intel_tlb.h" diff --git a/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c b/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c index b2a5882b8f81a..0756570187392 100644 --- a/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c +++ b/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c @@ -4,6 +4,7 @@ * Copyright © 2016 Intel Corporation */ +#include #include "mock_dmabuf.h" static struct sg_table *mock_map_dma_buf(struct dma_buf_attachment *attachment, diff --git a/drivers/gpu/drm/i915/gt/shmem_utils.c b/drivers/gpu/drm/i915/gt/shmem_utils.c index bccc3a1200bc6..1fb6ff77fd899 100644 --- a/drivers/gpu/drm/i915/gt/shmem_utils.c +++ b/drivers/gpu/drm/i915/gt/shmem_utils.c @@ -7,6 +7,7 @@ #include #include #include +#include #include "i915_drv.h" #include "gem/i915_gem_object.h" diff --git a/drivers/gpu/drm/i915/gvt/firmware.c b/drivers/gpu/drm/i915/gvt/firmware.c index 4dd52ac2043e7..d800d267f0e9b 100644 --- a/drivers/gpu/drm/i915/gvt/firmware.c +++ b/drivers/gpu/drm/i915/gvt/firmware.c @@ -30,6 +30,7 @@ #include #include +#include #include "i915_drv.h" #include "gvt.h" diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index 094fca9b0e73d..58cca4906f413 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -39,6 +39,7 @@ #include "trace.h" #include "gt/intel_gt_regs.h" +#include #if defined(VERBOSE_DEBUG) #define gvt_vdbg_mm(fmt, args...) gvt_dbg_mm(fmt, ##args) diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index efcb00472be24..ea9c300927671 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -52,6 +52,7 @@ #include "display/skl_watermark_regs.h" #include "display/vlv_dsi_pll_regs.h" #include "gt/intel_gt_regs.h" +#include /* XXX FIXME i915 has changed PP_XXX definition */ #define PCH_PP_STATUS _MMIO(0xc7200) diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c index 5b5def6ddef7a..780762f28aa4f 100644 --- a/drivers/gpu/drm/i915/gvt/mmio.c +++ b/drivers/gpu/drm/i915/gvt/mmio.c @@ -33,6 +33,7 @@ * */ +#include #include "i915_drv.h" #include "i915_reg.h" #include "gvt.h" diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index 08ad1bd651f10..63c751ca41196 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -34,6 +34,7 @@ #include "i915_drv.h" #include "gvt.h" #include "i915_pvinfo.h" +#include void populate_pvinfo_page(struct intel_vgpu *vgpu) { diff --git a/drivers/gpu/drm/i915/intel_gvt.c b/drivers/gpu/drm/i915/intel_gvt.c index 9b6d87c8b5831..5a01d60e51861 100644 --- a/drivers/gpu/drm/i915/intel_gvt.c +++ b/drivers/gpu/drm/i915/intel_gvt.c @@ -28,6 +28,7 @@ #include "gt/intel_context.h" #include "gt/intel_ring.h" #include "gt/shmem_utils.h" +#include /** * DOC: Intel GVT-g host support diff --git a/drivers/gpu/drm/imagination/pvr_vm_mips.c b/drivers/gpu/drm/imagination/pvr_vm_mips.c index b7fef3c797e6c..6563dcde109cd 100644 --- a/drivers/gpu/drm/imagination/pvr_vm_mips.c +++ b/drivers/gpu/drm/imagination/pvr_vm_mips.c @@ -14,6 +14,7 @@ #include #include #include +#include /** * pvr_vm_mips_init() - Initialise MIPS FW pagetable diff --git a/drivers/gpu/drm/mediatek/mtk_drm_gem.c b/drivers/gpu/drm/mediatek/mtk_drm_gem.c index 4f2e3feabc0f8..3e519869b632c 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_gem.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_gem.c @@ -4,6 +4,7 @@ */ #include +#include #include #include diff --git a/drivers/gpu/drm/omapdrm/omap_gem.c b/drivers/gpu/drm/omapdrm/omap_gem.c index 3421e8389222a..9ea0c64c26b5c 100644 --- a/drivers/gpu/drm/omapdrm/omap_gem.c +++ b/drivers/gpu/drm/omapdrm/omap_gem.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include diff --git a/drivers/gpu/drm/v3d/v3d_bo.c b/drivers/gpu/drm/v3d/v3d_bo.c index a07ede668cc16..a165cbcdd27b9 100644 --- a/drivers/gpu/drm/v3d/v3d_bo.c +++ b/drivers/gpu/drm/v3d/v3d_bo.c @@ -21,6 +21,7 @@ #include #include +#include #include "v3d_drv.h" #include "uapi/drm/v3d_drm.h" diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c b/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c index ae2de914eb890..2731f6ded1c28 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c @@ -54,6 +54,7 @@ #include "vmwgfx_drv.h" #include "vmwgfx_binding.h" #include "device_include/svga3d_reg.h" +#include #define VMW_BINDING_RT_BIT 0 #define VMW_BINDING_PS_BIT 1 diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c index 195ff8792e5ab..dd4ca6a9c690b 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c @@ -31,6 +31,7 @@ #include #include +#include bool vmw_supports_3d(struct vmw_private *dev_priv) { diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_devcaps.c b/drivers/gpu/drm/vmwgfx/vmwgfx_devcaps.c index 829df395c2ed7..6e6beff9e2621 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_devcaps.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_devcaps.c @@ -25,6 +25,7 @@ * **************************************************************************/ +#include #include "vmwgfx_devcaps.h" #include "vmwgfx_drv.h" diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c index 0a304706e0132..f8504a2e049e0 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c @@ -53,6 +53,7 @@ #include #include #include +#include #define VMWGFX_DRIVER_DESC "Linux drm driver for VMware graphics devices" diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c index cc3086e649eb5..2e52d73eba484 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c @@ -35,6 +35,7 @@ #include #include +#include /* * Helper macro to get dx_ctx_node if available otherwise print an error diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c index a1da5678c7314..835d1eed8dd93 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c @@ -31,6 +31,7 @@ #include #include +#include int vmw_getparam_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) diff --git a/drivers/gpu/drm/xen/xen_drm_front_gem.c b/drivers/gpu/drm/xen/xen_drm_front_gem.c index 3ad2b4cfd1f0b..63112ed975c47 100644 --- a/drivers/gpu/drm/xen/xen_drm_front_gem.c +++ b/drivers/gpu/drm/xen/xen_drm_front_gem.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 6136776482e6d..96a32b2136699 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -17,6 +17,7 @@ #include #include +#include #include "coresight-self-hosted-trace.h" #include "coresight-trbe.h" diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_pfvf_mbox.c b/drivers/net/ethernet/marvell/octeon_ep/octep_pfvf_mbox.c index 2e2c3be8a0b42..e6eb98d70f3c4 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_pfvf_mbox.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_pfvf_mbox.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "octep_config.h" #include "octep_main.h" diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_mbox.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_mbox.c index 2eab21e43048a..445b626efe11d 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_mbox.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_mbox.c @@ -7,6 +7,7 @@ #include #include #include +#include #include "octep_vf_config.h" #include "octep_vf_main.h" diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c index 2729a2c5acf9c..11021c34e47e7 100644 --- a/drivers/net/ethernet/microsoft/mana/hw_channel.c +++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c @@ -3,6 +3,7 @@ #include #include +#include static int mana_hwc_get_msg_index(struct hw_channel_context *hwc, u16 *msg_id) { diff --git a/drivers/platform/x86/uv_sysfs.c b/drivers/platform/x86/uv_sysfs.c index 38d1b692d3c0a..40e0108771893 100644 --- a/drivers/platform/x86/uv_sysfs.c +++ b/drivers/platform/x86/uv_sysfs.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/scsi/mpi3mr/mpi3mr_transport.c b/drivers/scsi/mpi3mr/mpi3mr_transport.c index d32ad46318cb0..dabb91f0f75df 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_transport.c +++ b/drivers/scsi/mpi3mr/mpi3mr_transport.c @@ -7,6 +7,8 @@ * */ +#include + #include "mpi3mr.h" /** diff --git a/drivers/vfio/pci/pds/dirty.c b/drivers/vfio/pci/pds/dirty.c index 68e8f006dfdbf..c51f5e4c3dd6d 100644 --- a/drivers/vfio/pci/pds/dirty.c +++ b/drivers/vfio/pci/pds/dirty.c @@ -3,6 +3,7 @@ #include #include +#include #include #include diff --git a/drivers/virt/acrn/mm.c b/drivers/virt/acrn/mm.c index fa5d9ca6be570..c088ee1f11804 100644 --- a/drivers/virt/acrn/mm.c +++ b/drivers/virt/acrn/mm.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "acrn_drv.h" diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 8e32232944423..e8355f55a8f7e 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -21,6 +21,7 @@ #include #include #include +#include #include diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index bac63e874c7bf..d4e1f042c9eb3 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -9,6 +9,7 @@ #include /* I/O is all done through memory accesses */ #include /* for memset() and memcpy() */ +#include #include #include diff --git a/include/linux/io.h b/include/linux/io.h index 235ba7d80a8f0..e8ee40ee1843b 100644 --- a/include/linux/io.h +++ b/include/linux/io.h @@ -6,6 +6,7 @@ #ifndef _LINUX_IO_H #define _LINUX_IO_H +#include #include #include #include diff --git a/include/linux/pds/pds_common.h b/include/linux/pds/pds_common.h index 30581e2e04cc1..5802e1deef24b 100644 --- a/include/linux/pds/pds_common.h +++ b/include/linux/pds/pds_common.h @@ -4,6 +4,8 @@ #ifndef _PDS_COMMON_H_ #define _PDS_COMMON_H_ +#include + #define PDS_CORE_DRV_NAME "pds_core" /* the device's internal addressing uses up to 52 bits */ diff --git a/include/rdma/rdmavt_qp.h b/include/rdma/rdmavt_qp.h index 2e58d5e6ac0e4..d678929441933 100644 --- a/include/rdma/rdmavt_qp.h +++ b/include/rdma/rdmavt_qp.h @@ -11,6 +11,7 @@ #include #include #include +#include /* * Atomic bit definitions for r_aflags. */ diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 65c19025da3df..f1c9a2c5abc04 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c index 2b994092a2d43..9958ebc15d383 100644 --- a/mm/kasan/hw_tags.c +++ b/mm/kasan/hw_tags.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "kasan.h" diff --git a/sound/pci/hda/cs35l41_hda.c b/sound/pci/hda/cs35l41_hda.c index d3fa6e136744d..990b5bd717a1c 100644 --- a/sound/pci/hda/cs35l41_hda.c +++ b/sound/pci/hda/cs35l41_hda.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "hda_local.h" #include "hda_auto_parser.h" #include "hda_jack.h" From 690da22dbfa85aa9f3ce06f40656c0b67e53cc80 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 21 Mar 2024 09:36:24 -0700 Subject: [PATCH 0556/1702] asm-generic/io.h: kill vmalloc.h dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Needed to avoid a new circular dependency with the memory allocation profiling series. Naturally, a whole bunch of files needed to include vmalloc.h that were previously getting it implicitly. Link: https://lkml.kernel.org/r/20240321163705.3067592-3-surenb@google.com Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Reviewed-by: Pasha Tatashin Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/asm-generic/io.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index d4e1f042c9eb3..80de699bf6af4 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -992,7 +992,6 @@ static inline void iowrite64_rep(volatile void __iomem *addr, #ifdef __KERNEL__ -#include #define __io_virt(x) ((void __force *)(x)) /* From 9ea9cd8e61a1f8dd304a3c8531b20f27ed5446d5 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 21 Mar 2024 09:36:25 -0700 Subject: [PATCH 0557/1702] mm/slub: mark slab_free_freelist_hook() __always_inline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It seems we need to be more forceful with the compiler on this one. This is done for performance reasons only. Link: https://lkml.kernel.org/r/20240321163705.3067592-4-surenb@google.com Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Reviewed-by: Kees Cook Reviewed-by: Pasha Tatashin Reviewed-by: Vlastimil Babka Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Peter Zijlstra Cc: Tejun Heo Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- mm/slub.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 1bb2a93cf7b6a..bc9f40889834d 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2106,9 +2106,9 @@ bool slab_free_hook(struct kmem_cache *s, void *x, bool init) return !kasan_slab_free(s, x, init); } -static inline bool slab_free_freelist_hook(struct kmem_cache *s, - void **head, void **tail, - int *cnt) +static __fastpath_inline +bool slab_free_freelist_hook(struct kmem_cache *s, void **head, void **tail, + int *cnt) { void *object; From a7f13d0f4b02fcc5ab2be5a31efd7ce8add9d10a Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 21 Mar 2024 09:36:26 -0700 Subject: [PATCH 0558/1702] scripts/kallysms: always include __start and __stop symbols MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These symbols are used to denote section boundaries: by always including them we can unify loading sections from modules with loading built-in sections, which leads to some significant cleanup. Link: https://lkml.kernel.org/r/20240321163705.3067592-5-surenb@google.com Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Reviewed-by: Kees Cook Reviewed-by: Pasha Tatashin Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- scripts/kallsyms.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c index 653b92f6d4c8f..47978efe4797c 100644 --- a/scripts/kallsyms.c +++ b/scripts/kallsyms.c @@ -204,6 +204,11 @@ static int symbol_in_range(const struct sym_entry *s, return 0; } +static bool string_starts_with(const char *s, const char *prefix) +{ + return strncmp(s, prefix, strlen(prefix)) == 0; +} + static int symbol_valid(const struct sym_entry *s) { const char *name = sym_name(s); @@ -211,6 +216,14 @@ static int symbol_valid(const struct sym_entry *s) /* if --all-symbols is not specified, then symbols outside the text * and inittext sections are discarded */ if (!all_symbols) { + /* + * Symbols starting with __start and __stop are used to denote + * section boundaries, and should always be included: + */ + if (string_starts_with(name, "__start_") || + string_starts_with(name, "__stop_")) + return 1; + if (symbol_in_range(s, text_ranges, ARRAY_SIZE(text_ranges)) == 0) return 0; From a5674119f0faa46f6ebdfcfa92342535b7f54e67 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 21 Mar 2024 09:36:27 -0700 Subject: [PATCH 0559/1702] fs: convert alloc_inode_sb() to a macro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We're introducing alloc tagging, which tracks memory allocations by callsite. Converting alloc_inode_sb() to a macro means allocations will be tracked by its caller, which is a bit more useful. Link: https://lkml.kernel.org/r/20240321163705.3067592-6-surenb@google.com Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Cc: Alexander Viro Reviewed-by: Kees Cook Reviewed-by: Pasha Tatashin Tested-by: Kees Cook Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/fs.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/include/linux/fs.h b/include/linux/fs.h index 8dfd53b52744a..4f2e2e5b6ab11 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3085,11 +3085,7 @@ int setattr_should_drop_sgid(struct mnt_idmap *idmap, * This must be used for allocating filesystems specific inodes to set * up the inode reclaim context correctly. */ -static inline void * -alloc_inode_sb(struct super_block *sb, struct kmem_cache *cache, gfp_t gfp) -{ - return kmem_cache_alloc_lru(cache, &sb->s_inode_lru, gfp); -} +#define alloc_inode_sb(_sb, _cache, _gfp) kmem_cache_alloc_lru(_cache, &_sb->s_inode_lru, _gfp) extern void __insert_inode_hash(struct inode *, unsigned long hashval); static inline void insert_inode_hash(struct inode *inode) From 21c690a349baab895dc68ab70d291e1598d7109d Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:28 -0700 Subject: [PATCH 0560/1702] mm: introduce slabobj_ext to support slab object extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently slab pages can store only vectors of obj_cgroup pointers in page->memcg_data. Introduce slabobj_ext structure to allow more data to be stored for each slab object. Wrap obj_cgroup into slabobj_ext to support current functionality while allowing to extend slabobj_ext in the future. Link: https://lkml.kernel.org/r/20240321163705.3067592-7-surenb@google.com Signed-off-by: Suren Baghdasaryan Reviewed-by: Pasha Tatashin Reviewed-by: Vlastimil Babka Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Kent Overstreet Cc: Miguel Ojeda Cc: Peter Zijlstra Cc: Tejun Heo Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 20 +++++--- include/linux/mm_types.h | 4 +- init/Kconfig | 4 ++ mm/kfence/core.c | 14 ++--- mm/kfence/kfence.h | 4 +- mm/memcontrol.c | 56 +++----------------- mm/page_owner.c | 2 +- mm/slab.h | 52 ++++++++++--------- mm/slub.c | 101 +++++++++++++++++++++++++++++-------- 9 files changed, 145 insertions(+), 112 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index b6264796815d4..99f423742324a 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -349,8 +349,8 @@ struct mem_cgroup { extern struct mem_cgroup *root_mem_cgroup; enum page_memcg_data_flags { - /* page->memcg_data is a pointer to an objcgs vector */ - MEMCG_DATA_OBJCGS = (1UL << 0), + /* page->memcg_data is a pointer to an slabobj_ext vector */ + MEMCG_DATA_OBJEXTS = (1UL << 0), /* page has been accounted as a non-slab kernel page */ MEMCG_DATA_KMEM = (1UL << 1), /* the next bit after the last actual flag */ @@ -388,7 +388,7 @@ static inline struct mem_cgroup *__folio_memcg(struct folio *folio) unsigned long memcg_data = folio->memcg_data; VM_BUG_ON_FOLIO(folio_test_slab(folio), folio); - VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio); + VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio); VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio); return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); @@ -409,7 +409,7 @@ static inline struct obj_cgroup *__folio_objcg(struct folio *folio) unsigned long memcg_data = folio->memcg_data; VM_BUG_ON_FOLIO(folio_test_slab(folio), folio); - VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio); + VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio); VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio); return (struct obj_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); @@ -506,7 +506,7 @@ static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) */ unsigned long memcg_data = READ_ONCE(folio->memcg_data); - if (memcg_data & MEMCG_DATA_OBJCGS) + if (memcg_data & MEMCG_DATA_OBJEXTS) return NULL; if (memcg_data & MEMCG_DATA_KMEM) { @@ -552,7 +552,7 @@ static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *ob static inline bool folio_memcg_kmem(struct folio *folio) { VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page); - VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJCGS, folio); + VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJEXTS, folio); return folio->memcg_data & MEMCG_DATA_KMEM; } @@ -1633,6 +1633,14 @@ unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, } #endif /* CONFIG_MEMCG */ +/* + * Extended information for slab objects stored as an array in page->memcg_data + * if MEMCG_DATA_OBJEXTS is set. + */ +struct slabobj_ext { + struct obj_cgroup *objcg; +} __aligned(8); + static inline void __inc_lruvec_kmem_state(void *p, enum node_stat_item idx) { __mod_lruvec_kmem_state(p, idx, 1); diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 5240bd7bca338..4ae4684d1add2 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -169,7 +169,7 @@ struct page { /* Usage count. *DO NOT USE DIRECTLY*. See page_ref.h */ atomic_t _refcount; -#ifdef CONFIG_MEMCG +#ifdef CONFIG_SLAB_OBJ_EXT unsigned long memcg_data; #endif @@ -331,7 +331,7 @@ struct folio { }; atomic_t _mapcount; atomic_t _refcount; -#ifdef CONFIG_MEMCG +#ifdef CONFIG_SLAB_OBJ_EXT unsigned long memcg_data; #endif #if defined(WANT_PAGE_VIRTUAL) diff --git a/init/Kconfig b/init/Kconfig index aa02aec6aa7d2..10d4a638d9aeb 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -929,6 +929,9 @@ config NUMA_BALANCING_DEFAULT_ENABLED If set, automatic NUMA balancing will be enabled if running on a NUMA machine. +config SLAB_OBJ_EXT + bool + menuconfig CGROUPS bool "Control Group support" select KERNFS @@ -962,6 +965,7 @@ config MEMCG bool "Memory controller" select PAGE_COUNTER select EVENTFD + select SLAB_OBJ_EXT help Provides control over the memory footprint of tasks in a cgroup. diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 8350f5c06f2e0..964b8482275ba 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -595,9 +595,9 @@ static unsigned long kfence_init_pool(void) continue; __folio_set_slab(slab_folio(slab)); -#ifdef CONFIG_MEMCG - slab->memcg_data = (unsigned long)&kfence_metadata_init[i / 2 - 1].objcg | - MEMCG_DATA_OBJCGS; +#ifdef CONFIG_MEMCG_KMEM + slab->obj_exts = (unsigned long)&kfence_metadata_init[i / 2 - 1].obj_exts | + MEMCG_DATA_OBJEXTS; #endif } @@ -645,8 +645,8 @@ static unsigned long kfence_init_pool(void) if (!i || (i % 2)) continue; -#ifdef CONFIG_MEMCG - slab->memcg_data = 0; +#ifdef CONFIG_MEMCG_KMEM + slab->obj_exts = 0; #endif __folio_clear_slab(slab_folio(slab)); } @@ -1139,8 +1139,8 @@ void __kfence_free(void *addr) { struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr); -#ifdef CONFIG_MEMCG - KFENCE_WARN_ON(meta->objcg); +#ifdef CONFIG_MEMCG_KMEM + KFENCE_WARN_ON(meta->obj_exts.objcg); #endif /* * If the objects of the cache are SLAB_TYPESAFE_BY_RCU, defer freeing diff --git a/mm/kfence/kfence.h b/mm/kfence/kfence.h index f46fbb03062b9..084f5f36e8e76 100644 --- a/mm/kfence/kfence.h +++ b/mm/kfence/kfence.h @@ -97,8 +97,8 @@ struct kfence_metadata { struct kfence_track free_track; /* For updating alloc_covered on frees. */ u32 alloc_stack_hash; -#ifdef CONFIG_MEMCG - struct obj_cgroup *objcg; +#ifdef CONFIG_MEMCG_KMEM + struct slabobj_ext obj_exts; #endif }; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 786c0fae6e309..896b4bf05b9cf 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2977,13 +2977,6 @@ void mem_cgroup_commit_charge(struct folio *folio, struct mem_cgroup *memcg) } #ifdef CONFIG_MEMCG_KMEM -/* - * The allocated objcg pointers array is not accounted directly. - * Moreover, it should not come from DMA buffer and is not readily - * reclaimable. So those GFP bits should be masked off. - */ -#define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | \ - __GFP_ACCOUNT | __GFP_NOFAIL) /* * mod_objcg_mlstate() may be called with irq enabled, so @@ -3003,62 +2996,27 @@ static inline void mod_objcg_mlstate(struct obj_cgroup *objcg, rcu_read_unlock(); } -int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s, - gfp_t gfp, bool new_slab) -{ - unsigned int objects = objs_per_slab(s, slab); - unsigned long memcg_data; - void *vec; - - gfp &= ~OBJCGS_CLEAR_MASK; - vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp, - slab_nid(slab)); - if (!vec) - return -ENOMEM; - - memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS; - if (new_slab) { - /* - * If the slab is brand new and nobody can yet access its - * memcg_data, no synchronization is required and memcg_data can - * be simply assigned. - */ - slab->memcg_data = memcg_data; - } else if (cmpxchg(&slab->memcg_data, 0, memcg_data)) { - /* - * If the slab is already in use, somebody can allocate and - * assign obj_cgroups in parallel. In this case the existing - * objcg vector should be reused. - */ - kfree(vec); - return 0; - } - - kmemleak_not_leak(vec); - return 0; -} - static __always_inline struct mem_cgroup *mem_cgroup_from_obj_folio(struct folio *folio, void *p) { /* * Slab objects are accounted individually, not per-page. * Memcg membership data for each individual object is saved in - * slab->memcg_data. + * slab->obj_exts. */ if (folio_test_slab(folio)) { - struct obj_cgroup **objcgs; + struct slabobj_ext *obj_exts; struct slab *slab; unsigned int off; slab = folio_slab(folio); - objcgs = slab_objcgs(slab); - if (!objcgs) + obj_exts = slab_obj_exts(slab); + if (!obj_exts) return NULL; off = obj_to_index(slab->slab_cache, slab, p); - if (objcgs[off]) - return obj_cgroup_memcg(objcgs[off]); + if (obj_exts[off].objcg) + return obj_cgroup_memcg(obj_exts[off].objcg); return NULL; } @@ -3066,7 +3024,7 @@ struct mem_cgroup *mem_cgroup_from_obj_folio(struct folio *folio, void *p) /* * folio_memcg_check() is used here, because in theory we can encounter * a folio where the slab flag has been cleared already, but - * slab->memcg_data has not been freed yet + * slab->obj_exts has not been freed yet * folio_memcg_check() will guarantee that a proper memory * cgroup pointer or NULL will be returned. */ diff --git a/mm/page_owner.c b/mm/page_owner.c index 742f432e5bf06..75c23302868a9 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -515,7 +515,7 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, if (!memcg_data) goto out_unlock; - if (memcg_data & MEMCG_DATA_OBJCGS) + if (memcg_data & MEMCG_DATA_OBJEXTS) ret += scnprintf(kbuf + ret, count - ret, "Slab cache page\n"); diff --git a/mm/slab.h b/mm/slab.h index d2bc9b1912229..1c16dc8344fac 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -87,8 +87,8 @@ struct slab { unsigned int __unused; atomic_t __page_refcount; -#ifdef CONFIG_MEMCG - unsigned long memcg_data; +#ifdef CONFIG_SLAB_OBJ_EXT + unsigned long obj_exts; #endif }; @@ -97,8 +97,8 @@ struct slab { SLAB_MATCH(flags, __page_flags); SLAB_MATCH(compound_head, slab_cache); /* Ensure bit 0 is clear */ SLAB_MATCH(_refcount, __page_refcount); -#ifdef CONFIG_MEMCG -SLAB_MATCH(memcg_data, memcg_data); +#ifdef CONFIG_SLAB_OBJ_EXT +SLAB_MATCH(memcg_data, obj_exts); #endif #undef SLAB_MATCH static_assert(sizeof(struct slab) <= sizeof(struct page)); @@ -536,42 +536,44 @@ static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t fla return false; } -#ifdef CONFIG_MEMCG_KMEM +#ifdef CONFIG_SLAB_OBJ_EXT + /* - * slab_objcgs - get the object cgroups vector associated with a slab + * slab_obj_exts - get the pointer to the slab object extension vector + * associated with a slab. * @slab: a pointer to the slab struct * - * Returns a pointer to the object cgroups vector associated with the slab, + * Returns a pointer to the object extension vector associated with the slab, * or NULL if no such vector has been associated yet. */ -static inline struct obj_cgroup **slab_objcgs(struct slab *slab) +static inline struct slabobj_ext *slab_obj_exts(struct slab *slab) { - unsigned long memcg_data = READ_ONCE(slab->memcg_data); + unsigned long obj_exts = READ_ONCE(slab->obj_exts); - VM_BUG_ON_PAGE(memcg_data && !(memcg_data & MEMCG_DATA_OBJCGS), +#ifdef CONFIG_MEMCG + VM_BUG_ON_PAGE(obj_exts && !(obj_exts & MEMCG_DATA_OBJEXTS), slab_page(slab)); - VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, slab_page(slab)); + VM_BUG_ON_PAGE(obj_exts & MEMCG_DATA_KMEM, slab_page(slab)); - return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); + return (struct slabobj_ext *)(obj_exts & ~MEMCG_DATA_FLAGS_MASK); +#else + return (struct slabobj_ext *)obj_exts; +#endif } -int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s, - gfp_t gfp, bool new_slab); -void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, - enum node_stat_item idx, int nr); -#else /* CONFIG_MEMCG_KMEM */ -static inline struct obj_cgroup **slab_objcgs(struct slab *slab) +#else /* CONFIG_SLAB_OBJ_EXT */ + +static inline struct slabobj_ext *slab_obj_exts(struct slab *slab) { return NULL; } -static inline int memcg_alloc_slab_cgroups(struct slab *slab, - struct kmem_cache *s, gfp_t gfp, - bool new_slab) -{ - return 0; -} -#endif /* CONFIG_MEMCG_KMEM */ +#endif /* CONFIG_SLAB_OBJ_EXT */ + +#ifdef CONFIG_MEMCG_KMEM +void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, + enum node_stat_item idx, int nr); +#endif size_t __ksize(const void *objp); diff --git a/mm/slub.c b/mm/slub.c index bc9f40889834d..5c896c76812d8 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1871,13 +1871,78 @@ static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s) NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B; } -#ifdef CONFIG_MEMCG_KMEM -static inline void memcg_free_slab_cgroups(struct slab *slab) +#ifdef CONFIG_SLAB_OBJ_EXT + +/* + * The allocated objcg pointers array is not accounted directly. + * Moreover, it should not come from DMA buffer and is not readily + * reclaimable. So those GFP bits should be masked off. + */ +#define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | \ + __GFP_ACCOUNT | __GFP_NOFAIL) + +static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, + gfp_t gfp, bool new_slab) { - kfree(slab_objcgs(slab)); - slab->memcg_data = 0; + unsigned int objects = objs_per_slab(s, slab); + unsigned long obj_exts; + void *vec; + + gfp &= ~OBJCGS_CLEAR_MASK; + vec = kcalloc_node(objects, sizeof(struct slabobj_ext), gfp, + slab_nid(slab)); + if (!vec) + return -ENOMEM; + + obj_exts = (unsigned long)vec; +#ifdef CONFIG_MEMCG + obj_exts |= MEMCG_DATA_OBJEXTS; +#endif + if (new_slab) { + /* + * If the slab is brand new and nobody can yet access its + * obj_exts, no synchronization is required and obj_exts can + * be simply assigned. + */ + slab->obj_exts = obj_exts; + } else if (cmpxchg(&slab->obj_exts, 0, obj_exts)) { + /* + * If the slab is already in use, somebody can allocate and + * assign slabobj_exts in parallel. In this case the existing + * objcg vector should be reused. + */ + kfree(vec); + return 0; + } + + kmemleak_not_leak(vec); + return 0; } +static inline void free_slab_obj_exts(struct slab *slab) +{ + struct slabobj_ext *obj_exts; + + obj_exts = slab_obj_exts(slab); + if (!obj_exts) + return; + + kfree(obj_exts); + slab->obj_exts = 0; +} +#else /* CONFIG_SLAB_OBJ_EXT */ +static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, + gfp_t gfp, bool new_slab) +{ + return 0; +} + +static inline void free_slab_obj_exts(struct slab *slab) +{ +} +#endif /* CONFIG_SLAB_OBJ_EXT */ + +#ifdef CONFIG_MEMCG_KMEM static inline size_t obj_full_size(struct kmem_cache *s) { /* @@ -1956,15 +2021,15 @@ static void __memcg_slab_post_alloc_hook(struct kmem_cache *s, if (likely(p[i])) { slab = virt_to_slab(p[i]); - if (!slab_objcgs(slab) && - memcg_alloc_slab_cgroups(slab, s, flags, false)) { + if (!slab_obj_exts(slab) && + alloc_slab_obj_exts(slab, s, flags, false)) { obj_cgroup_uncharge(objcg, obj_full_size(s)); continue; } off = obj_to_index(s, slab, p[i]); obj_cgroup_get(objcg); - slab_objcgs(slab)[off] = objcg; + slab_obj_exts(slab)[off].objcg = objcg; mod_objcg_state(objcg, slab_pgdat(slab), cache_vmstat_idx(s), obj_full_size(s)); } else { @@ -1985,18 +2050,18 @@ void memcg_slab_post_alloc_hook(struct kmem_cache *s, struct obj_cgroup *objcg, static void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, int objects, - struct obj_cgroup **objcgs) + struct slabobj_ext *obj_exts) { for (int i = 0; i < objects; i++) { struct obj_cgroup *objcg; unsigned int off; off = obj_to_index(s, slab, p[i]); - objcg = objcgs[off]; + objcg = obj_exts[off].objcg; if (!objcg) continue; - objcgs[off] = NULL; + obj_exts[off].objcg = NULL; obj_cgroup_uncharge(objcg, obj_full_size(s)); mod_objcg_state(objcg, slab_pgdat(slab), cache_vmstat_idx(s), -obj_full_size(s)); @@ -2008,16 +2073,16 @@ static __fastpath_inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, int objects) { - struct obj_cgroup **objcgs; + struct slabobj_ext *obj_exts; if (!memcg_kmem_online()) return; - objcgs = slab_objcgs(slab); - if (likely(!objcgs)) + obj_exts = slab_obj_exts(slab); + if (likely(!obj_exts)) return; - __memcg_slab_free_hook(s, slab, p, objects, objcgs); + __memcg_slab_free_hook(s, slab, p, objects, obj_exts); } static inline @@ -2028,10 +2093,6 @@ void memcg_slab_alloc_error_hook(struct kmem_cache *s, int objects, obj_cgroup_uncharge(objcg, objects * obj_full_size(s)); } #else /* CONFIG_MEMCG_KMEM */ -static inline void memcg_free_slab_cgroups(struct slab *slab) -{ -} - static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s, struct list_lru *lru, struct obj_cgroup **objcgp, @@ -2298,7 +2359,7 @@ static __always_inline void account_slab(struct slab *slab, int order, struct kmem_cache *s, gfp_t gfp) { if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT)) - memcg_alloc_slab_cgroups(slab, s, gfp, true); + alloc_slab_obj_exts(slab, s, gfp, true); mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), PAGE_SIZE << order); @@ -2308,7 +2369,7 @@ static __always_inline void unaccount_slab(struct slab *slab, int order, struct kmem_cache *s) { if (memcg_kmem_online()) - memcg_free_slab_cgroups(slab); + free_slab_obj_exts(slab); mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), -(PAGE_SIZE << order)); From 768c33be1b3166791bd554abd105154719eba43d Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:29 -0700 Subject: [PATCH 0561/1702] mm: introduce __GFP_NO_OBJ_EXT flag to selectively prevent slabobj_ext creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce __GFP_NO_OBJ_EXT flag in order to prevent recursive allocations when allocating slabobj_ext on a slab. Link: https://lkml.kernel.org/r/20240321163705.3067592-8-surenb@google.com Signed-off-by: Suren Baghdasaryan Reviewed-by: Kees Cook Reviewed-by: Pasha Tatashin Reviewed-by: Vlastimil Babka Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Kent Overstreet Cc: Miguel Ojeda Cc: Peter Zijlstra Cc: Tejun Heo Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/gfp_types.h | 11 +++++++++++ mm/slub.c | 2 ++ 2 files changed, 13 insertions(+) diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h index 13becafe41df0..313be4ad79fd9 100644 --- a/include/linux/gfp_types.h +++ b/include/linux/gfp_types.h @@ -54,6 +54,9 @@ enum { #endif #ifdef CONFIG_LOCKDEP ___GFP_NOLOCKDEP_BIT, +#endif +#ifdef CONFIG_SLAB_OBJ_EXT + ___GFP_NO_OBJ_EXT_BIT, #endif ___GFP_LAST_BIT }; @@ -95,6 +98,11 @@ enum { #else #define ___GFP_NOLOCKDEP 0 #endif +#ifdef CONFIG_SLAB_OBJ_EXT +#define ___GFP_NO_OBJ_EXT BIT(___GFP_NO_OBJ_EXT_BIT) +#else +#define ___GFP_NO_OBJ_EXT 0 +#endif /* * Physical address zone modifiers (see linux/mmzone.h - low four bits) @@ -135,12 +143,15 @@ enum { * node with no fallbacks or placement policy enforcements. * * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg. + * + * %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension. */ #define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE) #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) #define __GFP_HARDWALL ((__force gfp_t)___GFP_HARDWALL) #define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE) #define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT) +#define __GFP_NO_OBJ_EXT ((__force gfp_t)___GFP_NO_OBJ_EXT) /** * DOC: Watermark modifiers diff --git a/mm/slub.c b/mm/slub.c index 5c896c76812d8..2cb53642a091e 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1889,6 +1889,8 @@ static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, void *vec; gfp &= ~OBJCGS_CLEAR_MASK; + /* Prevent recursive extension vector allocation */ + gfp |= __GFP_NO_OBJ_EXT; vec = kcalloc_node(objects, sizeof(struct slabobj_ext), gfp, slab_nid(slab)); if (!vec) From 45012241ec5d5870d986e66c8ce20a0f6212e6f8 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:30 -0700 Subject: [PATCH 0562/1702] mm/slab: introduce SLAB_NO_OBJ_EXT to avoid obj_ext creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slab extension objects can't be allocated before slab infrastructure is initialized. Some caches, like kmem_cache and kmem_cache_node, are created before slab infrastructure is initialized. Objects from these caches can't have extension objects. Introduce SLAB_NO_OBJ_EXT slab flag to mark these caches and avoid creating extensions for objects allocated from these slabs. Link: https://lkml.kernel.org/r/20240321163705.3067592-9-surenb@google.com Signed-off-by: Suren Baghdasaryan Reviewed-by: Kees Cook Reviewed-by: Pasha Tatashin Reviewed-by: Vlastimil Babka Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Kent Overstreet Cc: Miguel Ojeda Cc: Peter Zijlstra Cc: Tejun Heo Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/slab.h | 10 ++++++++++ mm/slub.c | 5 +++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/include/linux/slab.h b/include/linux/slab.h index e53cbfa183255..68ff754b85a4a 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -56,6 +56,9 @@ enum _slab_flag_bits { #endif _SLAB_OBJECT_POISON, _SLAB_CMPXCHG_DOUBLE, +#ifdef CONFIG_SLAB_OBJ_EXT + _SLAB_NO_OBJ_EXT, +#endif _SLAB_FLAGS_LAST_BIT }; @@ -202,6 +205,13 @@ enum _slab_flag_bits { #endif #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ +/* Slab created using create_boot_cache */ +#ifdef CONFIG_SLAB_OBJ_EXT +#define SLAB_NO_OBJ_EXT __SLAB_FLAG_BIT(_SLAB_NO_OBJ_EXT) +#else +#define SLAB_NO_OBJ_EXT __SLAB_FLAG_UNUSED +#endif + /* * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. * diff --git a/mm/slub.c b/mm/slub.c index 2cb53642a091e..666dcc3b8a268 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -5693,7 +5693,8 @@ void __init kmem_cache_init(void) node_set(node, slab_nodes); create_boot_cache(kmem_cache_node, "kmem_cache_node", - sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN, 0, 0); + sizeof(struct kmem_cache_node), + SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0); hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI); @@ -5703,7 +5704,7 @@ void __init kmem_cache_init(void) create_boot_cache(kmem_cache, "kmem_cache", offsetof(struct kmem_cache, node) + nr_node_ids * sizeof(struct kmem_cache_node *), - SLAB_HWCACHE_ALIGN, 0, 0); + SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0); kmem_cache = bootstrap(&boot_kmem_cache); kmem_cache_node = bootstrap(&boot_kmem_cache_node); From 53ce720359b8f4b531c5b8bd7fa06d74bca45f0a Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:31 -0700 Subject: [PATCH 0563/1702] slab: objext: introduce objext_flags as extension to page_memcg_data_flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce objext_flags to store additional objext flags unrelated to memcg. Link: https://lkml.kernel.org/r/20240321163705.3067592-10-surenb@google.com Signed-off-by: Suren Baghdasaryan Reviewed-by: Kees Cook Reviewed-by: Pasha Tatashin Reviewed-by: Vlastimil Babka Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Kent Overstreet Cc: Miguel Ojeda Cc: Peter Zijlstra Cc: Tejun Heo Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 29 ++++++++++++++++++++++------- mm/slab.h | 5 +---- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 99f423742324a..12afc2647cf0e 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -357,7 +357,22 @@ enum page_memcg_data_flags { __NR_MEMCG_DATA_FLAGS = (1UL << 2), }; -#define MEMCG_DATA_FLAGS_MASK (__NR_MEMCG_DATA_FLAGS - 1) +#define __FIRST_OBJEXT_FLAG __NR_MEMCG_DATA_FLAGS + +#else /* CONFIG_MEMCG */ + +#define __FIRST_OBJEXT_FLAG (1UL << 0) + +#endif /* CONFIG_MEMCG */ + +enum objext_flags { + /* the next bit after the last actual flag */ + __NR_OBJEXTS_FLAGS = __FIRST_OBJEXT_FLAG, +}; + +#define OBJEXTS_FLAGS_MASK (__NR_OBJEXTS_FLAGS - 1) + +#ifdef CONFIG_MEMCG static inline bool folio_memcg_kmem(struct folio *folio); @@ -391,7 +406,7 @@ static inline struct mem_cgroup *__folio_memcg(struct folio *folio) VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio); VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio); - return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); + return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK); } /* @@ -412,7 +427,7 @@ static inline struct obj_cgroup *__folio_objcg(struct folio *folio) VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio); VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio); - return (struct obj_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); + return (struct obj_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK); } /* @@ -469,11 +484,11 @@ static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio) if (memcg_data & MEMCG_DATA_KMEM) { struct obj_cgroup *objcg; - objcg = (void *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); + objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); return obj_cgroup_memcg(objcg); } - return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); + return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK); } /* @@ -512,11 +527,11 @@ static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) if (memcg_data & MEMCG_DATA_KMEM) { struct obj_cgroup *objcg; - objcg = (void *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); + objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); return obj_cgroup_memcg(objcg); } - return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); + return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK); } static inline struct mem_cgroup *page_memcg_check(struct page *page) diff --git a/mm/slab.h b/mm/slab.h index 1c16dc8344fac..65db525e93af3 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -554,11 +554,8 @@ static inline struct slabobj_ext *slab_obj_exts(struct slab *slab) VM_BUG_ON_PAGE(obj_exts && !(obj_exts & MEMCG_DATA_OBJEXTS), slab_page(slab)); VM_BUG_ON_PAGE(obj_exts & MEMCG_DATA_KMEM, slab_page(slab)); - - return (struct slabobj_ext *)(obj_exts & ~MEMCG_DATA_FLAGS_MASK); -#else - return (struct slabobj_ext *)obj_exts; #endif + return (struct slabobj_ext *)(obj_exts & ~OBJEXTS_FLAGS_MASK); } #else /* CONFIG_SLAB_OBJ_EXT */ From 916cc5167cc6f81651f4460d9a35c939ad18ecff Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:32 -0700 Subject: [PATCH 0564/1702] lib: code tagging framework MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add basic infrastructure to support code tagging which stores tag common information consisting of the module name, function, file name and line number. Provide functions to register a new code tag type and navigate between code tags. Link: https://lkml.kernel.org/r/20240321163705.3067592-11-surenb@google.com Co-developed-by: Kent Overstreet Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/codetag.h | 68 +++++++++++++ lib/Kconfig.debug | 4 + lib/Makefile | 1 + lib/codetag.c | 219 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 292 insertions(+) create mode 100644 include/linux/codetag.h create mode 100644 lib/codetag.c diff --git a/include/linux/codetag.h b/include/linux/codetag.h new file mode 100644 index 0000000000000..7734269cdb63f --- /dev/null +++ b/include/linux/codetag.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * code tagging framework + */ +#ifndef _LINUX_CODETAG_H +#define _LINUX_CODETAG_H + +#include + +struct codetag_iterator; +struct codetag_type; +struct codetag_module; +struct seq_buf; +struct module; + +/* + * An instance of this structure is created in a special ELF section at every + * code location being tagged. At runtime, the special section is treated as + * an array of these. + */ +struct codetag { + unsigned int flags; /* used in later patches */ + unsigned int lineno; + const char *modname; + const char *function; + const char *filename; +} __aligned(8); + +union codetag_ref { + struct codetag *ct; +}; + +struct codetag_type_desc { + const char *section; + size_t tag_size; +}; + +struct codetag_iterator { + struct codetag_type *cttype; + struct codetag_module *cmod; + unsigned long mod_id; + struct codetag *ct; +}; + +#ifdef MODULE +#define CT_MODULE_NAME KBUILD_MODNAME +#else +#define CT_MODULE_NAME NULL +#endif + +#define CODE_TAG_INIT { \ + .modname = CT_MODULE_NAME, \ + .function = __func__, \ + .filename = __FILE__, \ + .lineno = __LINE__, \ + .flags = 0, \ +} + +void codetag_lock_module_list(struct codetag_type *cttype, bool lock); +struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype); +struct codetag *codetag_next_ct(struct codetag_iterator *iter); + +void codetag_to_text(struct seq_buf *out, struct codetag *ct); + +struct codetag_type * +codetag_register_type(const struct codetag_type_desc *desc); + +#endif /* _LINUX_CODETAG_H */ diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index c63a5fbf1f1c2..015fc6ee9849a 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -968,6 +968,10 @@ config DEBUG_STACKOVERFLOW If in doubt, say "N". +config CODE_TAGGING + bool + select KALLSYMS + source "lib/Kconfig.kasan" source "lib/Kconfig.kfence" source "lib/Kconfig.kmsan" diff --git a/lib/Makefile b/lib/Makefile index ffc6b2341b45a..910335da8f133 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -233,6 +233,7 @@ obj-$(CONFIG_OF_RECONFIG_NOTIFIER_ERROR_INJECT) += \ of-reconfig-notifier-error-inject.o obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o +obj-$(CONFIG_CODE_TAGGING) += codetag.o lib-$(CONFIG_GENERIC_BUG) += bug.o obj-$(CONFIG_HAVE_ARCH_TRACEHOOK) += syscall.o diff --git a/lib/codetag.c b/lib/codetag.c new file mode 100644 index 0000000000000..8b5b89ad508df --- /dev/null +++ b/lib/codetag.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include + +struct codetag_type { + struct list_head link; + unsigned int count; + struct idr mod_idr; + struct rw_semaphore mod_lock; /* protects mod_idr */ + struct codetag_type_desc desc; +}; + +struct codetag_range { + struct codetag *start; + struct codetag *stop; +}; + +struct codetag_module { + struct module *mod; + struct codetag_range range; +}; + +static DEFINE_MUTEX(codetag_lock); +static LIST_HEAD(codetag_types); + +void codetag_lock_module_list(struct codetag_type *cttype, bool lock) +{ + if (lock) + down_read(&cttype->mod_lock); + else + up_read(&cttype->mod_lock); +} + +struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype) +{ + struct codetag_iterator iter = { + .cttype = cttype, + .cmod = NULL, + .mod_id = 0, + .ct = NULL, + }; + + return iter; +} + +static inline struct codetag *get_first_module_ct(struct codetag_module *cmod) +{ + return cmod->range.start < cmod->range.stop ? cmod->range.start : NULL; +} + +static inline +struct codetag *get_next_module_ct(struct codetag_iterator *iter) +{ + struct codetag *res = (struct codetag *) + ((char *)iter->ct + iter->cttype->desc.tag_size); + + return res < iter->cmod->range.stop ? res : NULL; +} + +struct codetag *codetag_next_ct(struct codetag_iterator *iter) +{ + struct codetag_type *cttype = iter->cttype; + struct codetag_module *cmod; + struct codetag *ct; + + lockdep_assert_held(&cttype->mod_lock); + + if (unlikely(idr_is_empty(&cttype->mod_idr))) + return NULL; + + ct = NULL; + while (true) { + cmod = idr_find(&cttype->mod_idr, iter->mod_id); + + /* If module was removed move to the next one */ + if (!cmod) + cmod = idr_get_next_ul(&cttype->mod_idr, + &iter->mod_id); + + /* Exit if no more modules */ + if (!cmod) + break; + + if (cmod != iter->cmod) { + iter->cmod = cmod; + ct = get_first_module_ct(cmod); + } else + ct = get_next_module_ct(iter); + + if (ct) + break; + + iter->mod_id++; + } + + iter->ct = ct; + return ct; +} + +void codetag_to_text(struct seq_buf *out, struct codetag *ct) +{ + if (ct->modname) + seq_buf_printf(out, "%s:%u [%s] func:%s", + ct->filename, ct->lineno, + ct->modname, ct->function); + else + seq_buf_printf(out, "%s:%u func:%s", + ct->filename, ct->lineno, ct->function); +} + +static inline size_t range_size(const struct codetag_type *cttype, + const struct codetag_range *range) +{ + return ((char *)range->stop - (char *)range->start) / + cttype->desc.tag_size; +} + +#ifdef CONFIG_MODULES +static void *get_symbol(struct module *mod, const char *prefix, const char *name) +{ + DECLARE_SEQ_BUF(sb, KSYM_NAME_LEN); + const char *buf; + + seq_buf_printf(&sb, "%s%s", prefix, name); + if (seq_buf_has_overflowed(&sb)) + return NULL; + + buf = seq_buf_str(&sb); + return mod ? + (void *)find_kallsyms_symbol_value(mod, buf) : + (void *)kallsyms_lookup_name(buf); +} + +static struct codetag_range get_section_range(struct module *mod, + const char *section) +{ + return (struct codetag_range) { + get_symbol(mod, "__start_", section), + get_symbol(mod, "__stop_", section), + }; +} + +static int codetag_module_init(struct codetag_type *cttype, struct module *mod) +{ + struct codetag_range range; + struct codetag_module *cmod; + int err; + + range = get_section_range(mod, cttype->desc.section); + if (!range.start || !range.stop) { + pr_warn("Failed to load code tags of type %s from the module %s\n", + cttype->desc.section, + mod ? mod->name : "(built-in)"); + return -EINVAL; + } + + /* Ignore empty ranges */ + if (range.start == range.stop) + return 0; + + BUG_ON(range.start > range.stop); + + cmod = kmalloc(sizeof(*cmod), GFP_KERNEL); + if (unlikely(!cmod)) + return -ENOMEM; + + cmod->mod = mod; + cmod->range = range; + + down_write(&cttype->mod_lock); + err = idr_alloc(&cttype->mod_idr, cmod, 0, 0, GFP_KERNEL); + if (err >= 0) + cttype->count += range_size(cttype, &range); + up_write(&cttype->mod_lock); + + if (err < 0) { + kfree(cmod); + return err; + } + + return 0; +} + +#else /* CONFIG_MODULES */ +static int codetag_module_init(struct codetag_type *cttype, struct module *mod) { return 0; } +#endif /* CONFIG_MODULES */ + +struct codetag_type * +codetag_register_type(const struct codetag_type_desc *desc) +{ + struct codetag_type *cttype; + int err; + + BUG_ON(desc->tag_size <= 0); + + cttype = kzalloc(sizeof(*cttype), GFP_KERNEL); + if (unlikely(!cttype)) + return ERR_PTR(-ENOMEM); + + cttype->desc = *desc; + idr_init(&cttype->mod_idr); + init_rwsem(&cttype->mod_lock); + + err = codetag_module_init(cttype, NULL); + if (unlikely(err)) { + kfree(cttype); + return ERR_PTR(err); + } + + mutex_lock(&codetag_lock); + list_add_tail(&cttype->link, &codetag_types); + mutex_unlock(&codetag_lock); + + return cttype; +} From a473573964e51dcb6efc182f773cd3924be4a184 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:33 -0700 Subject: [PATCH 0565/1702] lib: code tagging module support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for code tagging from dynamically loaded modules. Link: https://lkml.kernel.org/r/20240321163705.3067592-12-surenb@google.com Signed-off-by: Suren Baghdasaryan Co-developed-by: Kent Overstreet Signed-off-by: Kent Overstreet Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/codetag.h | 12 +++++++++ kernel/module/main.c | 4 +++ lib/codetag.c | 58 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/include/linux/codetag.h b/include/linux/codetag.h index 7734269cdb63f..c44f5b83f24d6 100644 --- a/include/linux/codetag.h +++ b/include/linux/codetag.h @@ -33,6 +33,10 @@ union codetag_ref { struct codetag_type_desc { const char *section; size_t tag_size; + void (*module_load)(struct codetag_type *cttype, + struct codetag_module *cmod); + void (*module_unload)(struct codetag_type *cttype, + struct codetag_module *cmod); }; struct codetag_iterator { @@ -65,4 +69,12 @@ void codetag_to_text(struct seq_buf *out, struct codetag *ct); struct codetag_type * codetag_register_type(const struct codetag_type_desc *desc); +#if defined(CONFIG_CODE_TAGGING) && defined(CONFIG_MODULES) +void codetag_load_module(struct module *mod); +void codetag_unload_module(struct module *mod); +#else +static inline void codetag_load_module(struct module *mod) {} +static inline void codetag_unload_module(struct module *mod) {} +#endif + #endif /* _LINUX_CODETAG_H */ diff --git a/kernel/module/main.c b/kernel/module/main.c index e1e8a7a9d6c19..ffa6b3e9cb43b 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -56,6 +56,7 @@ #include #include #include +#include #include #include #include "internal.h" @@ -1242,6 +1243,7 @@ static void free_module(struct module *mod) { trace_module_free(mod); + codetag_unload_module(mod); mod_sysfs_teardown(mod); /* @@ -2995,6 +2997,8 @@ static int load_module(struct load_info *info, const char __user *uargs, /* Get rid of temporary copy. */ free_copy(info, flags); + codetag_load_module(mod); + /* Done! */ trace_module_load(mod); diff --git a/lib/codetag.c b/lib/codetag.c index 8b5b89ad508df..54d2828eba256 100644 --- a/lib/codetag.c +++ b/lib/codetag.c @@ -124,15 +124,20 @@ static void *get_symbol(struct module *mod, const char *prefix, const char *name { DECLARE_SEQ_BUF(sb, KSYM_NAME_LEN); const char *buf; + void *ret; seq_buf_printf(&sb, "%s%s", prefix, name); if (seq_buf_has_overflowed(&sb)) return NULL; buf = seq_buf_str(&sb); - return mod ? + preempt_disable(); + ret = mod ? (void *)find_kallsyms_symbol_value(mod, buf) : (void *)kallsyms_lookup_name(buf); + preempt_enable(); + + return ret; } static struct codetag_range get_section_range(struct module *mod, @@ -173,8 +178,11 @@ static int codetag_module_init(struct codetag_type *cttype, struct module *mod) down_write(&cttype->mod_lock); err = idr_alloc(&cttype->mod_idr, cmod, 0, 0, GFP_KERNEL); - if (err >= 0) + if (err >= 0) { cttype->count += range_size(cttype, &range); + if (cttype->desc.module_load) + cttype->desc.module_load(cttype, cmod); + } up_write(&cttype->mod_lock); if (err < 0) { @@ -185,6 +193,52 @@ static int codetag_module_init(struct codetag_type *cttype, struct module *mod) return 0; } +void codetag_load_module(struct module *mod) +{ + struct codetag_type *cttype; + + if (!mod) + return; + + mutex_lock(&codetag_lock); + list_for_each_entry(cttype, &codetag_types, link) + codetag_module_init(cttype, mod); + mutex_unlock(&codetag_lock); +} + +void codetag_unload_module(struct module *mod) +{ + struct codetag_type *cttype; + + if (!mod) + return; + + mutex_lock(&codetag_lock); + list_for_each_entry(cttype, &codetag_types, link) { + struct codetag_module *found = NULL; + struct codetag_module *cmod; + unsigned long mod_id, tmp; + + down_write(&cttype->mod_lock); + idr_for_each_entry_ul(&cttype->mod_idr, cmod, tmp, mod_id) { + if (cmod->mod && cmod->mod == mod) { + found = cmod; + break; + } + } + if (found) { + if (cttype->desc.module_unload) + cttype->desc.module_unload(cttype, cmod); + + cttype->count -= range_size(cttype, &cmod->range); + idr_remove(&cttype->mod_idr, mod_id); + kfree(cmod); + } + up_write(&cttype->mod_lock); + } + mutex_unlock(&codetag_lock); +} + #else /* CONFIG_MODULES */ static int codetag_module_init(struct codetag_type *cttype, struct module *mod) { return 0; } #endif /* CONFIG_MODULES */ From 47a92dfbe01f41bcbf359250ccb3caa589763abf Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:34 -0700 Subject: [PATCH 0566/1702] lib: prevent module unloading if memory is not freed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skip freeing module's data section if there are non-zero allocation tags because otherwise, once these allocations are freed, the access to their code tag would cause UAF. Link: https://lkml.kernel.org/r/20240321163705.3067592-13-surenb@google.com Signed-off-by: Suren Baghdasaryan Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Kent Overstreet Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/codetag.h | 6 +++--- kernel/module/main.c | 27 +++++++++++++++++++-------- lib/codetag.c | 11 ++++++++--- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/include/linux/codetag.h b/include/linux/codetag.h index c44f5b83f24d6..bfd0ba5c4185c 100644 --- a/include/linux/codetag.h +++ b/include/linux/codetag.h @@ -35,7 +35,7 @@ struct codetag_type_desc { size_t tag_size; void (*module_load)(struct codetag_type *cttype, struct codetag_module *cmod); - void (*module_unload)(struct codetag_type *cttype, + bool (*module_unload)(struct codetag_type *cttype, struct codetag_module *cmod); }; @@ -71,10 +71,10 @@ codetag_register_type(const struct codetag_type_desc *desc); #if defined(CONFIG_CODE_TAGGING) && defined(CONFIG_MODULES) void codetag_load_module(struct module *mod); -void codetag_unload_module(struct module *mod); +bool codetag_unload_module(struct module *mod); #else static inline void codetag_load_module(struct module *mod) {} -static inline void codetag_unload_module(struct module *mod) {} +static inline bool codetag_unload_module(struct module *mod) { return true; } #endif #endif /* _LINUX_CODETAG_H */ diff --git a/kernel/module/main.c b/kernel/module/main.c index ffa6b3e9cb43b..2d25eebc549db 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -1211,15 +1211,19 @@ static void *module_memory_alloc(unsigned int size, enum mod_mem_type type) return module_alloc(size); } -static void module_memory_free(void *ptr, enum mod_mem_type type) +static void module_memory_free(void *ptr, enum mod_mem_type type, + bool unload_codetags) { + if (!unload_codetags && mod_mem_type_is_core_data(type)) + return; + if (mod_mem_use_vmalloc(type)) vfree(ptr); else module_memfree(ptr); } -static void free_mod_mem(struct module *mod) +static void free_mod_mem(struct module *mod, bool unload_codetags) { for_each_mod_mem_type(type) { struct module_memory *mod_mem = &mod->mem[type]; @@ -1230,20 +1234,27 @@ static void free_mod_mem(struct module *mod) /* Free lock-classes; relies on the preceding sync_rcu(). */ lockdep_free_key_range(mod_mem->base, mod_mem->size); if (mod_mem->size) - module_memory_free(mod_mem->base, type); + module_memory_free(mod_mem->base, type, + unload_codetags); } /* MOD_DATA hosts mod, so free it at last */ lockdep_free_key_range(mod->mem[MOD_DATA].base, mod->mem[MOD_DATA].size); - module_memory_free(mod->mem[MOD_DATA].base, MOD_DATA); + module_memory_free(mod->mem[MOD_DATA].base, MOD_DATA, unload_codetags); } /* Free a module, remove from lists, etc. */ static void free_module(struct module *mod) { + bool unload_codetags; + trace_module_free(mod); - codetag_unload_module(mod); + unload_codetags = codetag_unload_module(mod); + if (!unload_codetags) + pr_warn("%s: memory allocation(s) from the module still alive, cannot unload cleanly\n", + mod->name); + mod_sysfs_teardown(mod); /* @@ -1285,7 +1296,7 @@ static void free_module(struct module *mod) kfree(mod->args); percpu_modfree(mod); - free_mod_mem(mod); + free_mod_mem(mod, unload_codetags); } void *__symbol_get(const char *symbol) @@ -2298,7 +2309,7 @@ static int move_module(struct module *mod, struct load_info *info) return 0; out_enomem: for (t--; t >= 0; t--) - module_memory_free(mod->mem[t].base, t); + module_memory_free(mod->mem[t].base, t, true); return ret; } @@ -2428,7 +2439,7 @@ static void module_deallocate(struct module *mod, struct load_info *info) percpu_modfree(mod); module_arch_freeing_init(mod); - free_mod_mem(mod); + free_mod_mem(mod, true); } int __weak module_finalize(const Elf_Ehdr *hdr, diff --git a/lib/codetag.c b/lib/codetag.c index 54d2828eba256..408062f722ced 100644 --- a/lib/codetag.c +++ b/lib/codetag.c @@ -5,6 +5,7 @@ #include #include #include +#include struct codetag_type { struct list_head link; @@ -206,12 +207,13 @@ void codetag_load_module(struct module *mod) mutex_unlock(&codetag_lock); } -void codetag_unload_module(struct module *mod) +bool codetag_unload_module(struct module *mod) { struct codetag_type *cttype; + bool unload_ok = true; if (!mod) - return; + return true; mutex_lock(&codetag_lock); list_for_each_entry(cttype, &codetag_types, link) { @@ -228,7 +230,8 @@ void codetag_unload_module(struct module *mod) } if (found) { if (cttype->desc.module_unload) - cttype->desc.module_unload(cttype, cmod); + if (!cttype->desc.module_unload(cttype, cmod)) + unload_ok = false; cttype->count -= range_size(cttype, &cmod->range); idr_remove(&cttype->mod_idr, mod_id); @@ -237,6 +240,8 @@ void codetag_unload_module(struct module *mod) up_write(&cttype->mod_lock); } mutex_unlock(&codetag_lock); + + return unload_ok; } #else /* CONFIG_MODULES */ From 22d407b164ff79de42d21f37d99f9ee7abdd51c8 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:35 -0700 Subject: [PATCH 0567/1702] lib: add allocation tagging support for memory allocation profiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce CONFIG_MEM_ALLOC_PROFILING which provides definitions to easily instrument memory allocators. It registers an "alloc_tags" codetag type with /proc/allocinfo interface to output allocation tag information when the feature is enabled. CONFIG_MEM_ALLOC_PROFILING_DEBUG is provided for debugging the memory allocation profiling instrumentation. Memory allocation profiling can be enabled or disabled at runtime using /proc/sys/vm/mem_profiling sysctl when CONFIG_MEM_ALLOC_PROFILING_DEBUG=n. CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT enables memory allocation profiling by default. [surenb@google.com: Documentation/filesystems/proc.rst: fix allocinfo title] Link: https://lkml.kernel.org/r/20240326073813.727090-1-surenb@google.com [surenb@google.com: do limited memory accounting for modules with ARCH_NEEDS_WEAK_PER_CPU] Link: https://lkml.kernel.org/r/20240402180933.1663992-2-surenb@google.com [klarasmodin@gmail.com: explicitly include irqflags.h in alloc_tag.h] Link: https://lkml.kernel.org/r/20240407133252.173636-1-klarasmodin@gmail.com [surenb@google.com: fix alloc_tag_init() to prevent passing NULL to PTR_ERR()] Link: https://lkml.kernel.org/r/20240417003349.2520094-1-surenb@google.com Link: https://lkml.kernel.org/r/20240321163705.3067592-14-surenb@google.com Signed-off-by: Suren Baghdasaryan Co-developed-by: Kent Overstreet Signed-off-by: Kent Overstreet Signed-off-by: Klara Modin Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- Documentation/admin-guide/sysctl/vm.rst | 16 +++ Documentation/filesystems/proc.rst | 29 +++++ include/asm-generic/codetag.lds.h | 14 +++ include/asm-generic/vmlinux.lds.h | 3 + include/linux/alloc_tag.h | 156 ++++++++++++++++++++++++ include/linux/sched.h | 24 ++++ lib/Kconfig.debug | 25 ++++ lib/Makefile | 2 + lib/alloc_tag.c | 152 +++++++++++++++++++++++ scripts/module.lds.S | 7 ++ 10 files changed, 428 insertions(+) create mode 100644 include/asm-generic/codetag.lds.h create mode 100644 include/linux/alloc_tag.h create mode 100644 lib/alloc_tag.c diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index c59889de122b9..e86c968a7a0ec 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -43,6 +43,7 @@ Currently, these files are in /proc/sys/vm: - legacy_va_layout - lowmem_reserve_ratio - max_map_count +- mem_profiling (only if CONFIG_MEM_ALLOC_PROFILING=y) - memory_failure_early_kill - memory_failure_recovery - min_free_kbytes @@ -425,6 +426,21 @@ e.g., up to one or two maps per allocation. The default value is 65530. +mem_profiling +============== + +Enable memory profiling (when CONFIG_MEM_ALLOC_PROFILING=y) + +1: Enable memory profiling. + +0: Disable memory profiling. + +Enabling memory profiling introduces a small performance overhead for all +memory allocations. + +The default value depends on CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT. + + memory_failure_early_kill: ========================== diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst index c6a6b9df21049..245269dd6e028 100644 --- a/Documentation/filesystems/proc.rst +++ b/Documentation/filesystems/proc.rst @@ -688,6 +688,7 @@ files are there, and which are missing. ============ =============================================================== File Content ============ =============================================================== + allocinfo Memory allocations profiling information apm Advanced power management info bootconfig Kernel command line obtained from boot config, and, if there were kernel parameters from the @@ -953,6 +954,34 @@ also be allocatable although a lot of filesystem metadata may have to be reclaimed to achieve this. +allocinfo +~~~~~~~~~ + +Provides information about memory allocations at all locations in the code +base. Each allocation in the code is identified by its source file, line +number, module (if originates from a loadable module) and the function calling +the allocation. The number of bytes allocated and number of calls at each +location are reported. + +Example output. + +:: + + > sort -rn /proc/allocinfo + 127664128 31168 mm/page_ext.c:270 func:alloc_page_ext + 56373248 4737 mm/slub.c:2259 func:alloc_slab_page + 14880768 3633 mm/readahead.c:247 func:page_cache_ra_unbounded + 14417920 3520 mm/mm_init.c:2530 func:alloc_large_system_hash + 13377536 234 block/blk-mq.c:3421 func:blk_mq_alloc_rqs + 11718656 2861 mm/filemap.c:1919 func:__filemap_get_folio + 9192960 2800 kernel/fork.c:307 func:alloc_thread_stack_node + 4206592 4 net/netfilter/nf_conntrack_core.c:2567 func:nf_ct_alloc_hashtable + 4136960 1010 drivers/staging/ctagmod/ctagmod.c:20 [ctagmod] func:ctagmod_start + 3940352 962 mm/memory.c:4214 func:alloc_anon_folio + 2894464 22613 fs/kernfs/dir.c:615 func:__kernfs_new_node + ... + + meminfo ~~~~~~~ diff --git a/include/asm-generic/codetag.lds.h b/include/asm-generic/codetag.lds.h new file mode 100644 index 0000000000000..64f536b803802 --- /dev/null +++ b/include/asm-generic/codetag.lds.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_GENERIC_CODETAG_LDS_H +#define __ASM_GENERIC_CODETAG_LDS_H + +#define SECTION_WITH_BOUNDARIES(_name) \ + . = ALIGN(8); \ + __start_##_name = .; \ + KEEP(*(_name)) \ + __stop_##_name = .; + +#define CODETAG_SECTIONS() \ + SECTION_WITH_BOUNDARIES(alloc_tags) + +#endif /* __ASM_GENERIC_CODETAG_LDS_H */ diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index f7749d0f2562f..3e4497b5135a1 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -50,6 +50,8 @@ * [__nosave_begin, __nosave_end] for the nosave data */ +#include + #ifndef LOAD_OFFSET #define LOAD_OFFSET 0 #endif @@ -366,6 +368,7 @@ . = ALIGN(8); \ BOUNDED_SECTION_BY(__dyndbg_classes, ___dyndbg_classes) \ BOUNDED_SECTION_BY(__dyndbg, ___dyndbg) \ + CODETAG_SECTIONS() \ LIKELY_PROFILE() \ BRANCH_PROFILE() \ TRACE_PRINTKS() \ diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h new file mode 100644 index 0000000000000..13561223fdabd --- /dev/null +++ b/include/linux/alloc_tag.h @@ -0,0 +1,156 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * allocation tagging + */ +#ifndef _LINUX_ALLOC_TAG_H +#define _LINUX_ALLOC_TAG_H + +#include +#include +#include +#include +#include +#include +#include +#include + +struct alloc_tag_counters { + u64 bytes; + u64 calls; +}; + +/* + * An instance of this structure is created in a special ELF section at every + * allocation callsite. At runtime, the special section is treated as + * an array of these. Embedded codetag utilizes codetag framework. + */ +struct alloc_tag { + struct codetag ct; + struct alloc_tag_counters __percpu *counters; +} __aligned(8); + +#ifdef CONFIG_MEM_ALLOC_PROFILING + +static inline struct alloc_tag *ct_to_alloc_tag(struct codetag *ct) +{ + return container_of(ct, struct alloc_tag, ct); +} + +#ifdef ARCH_NEEDS_WEAK_PER_CPU +/* + * When percpu variables are required to be defined as weak, static percpu + * variables can't be used inside a function (see comments for DECLARE_PER_CPU_SECTION). + * Instead we will accound all module allocations to a single counter. + */ +DECLARE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag); + +#define DEFINE_ALLOC_TAG(_alloc_tag) \ + static struct alloc_tag _alloc_tag __used __aligned(8) \ + __section("alloc_tags") = { \ + .ct = CODE_TAG_INIT, \ + .counters = &_shared_alloc_tag }; + +#else /* ARCH_NEEDS_WEAK_PER_CPU */ + +#define DEFINE_ALLOC_TAG(_alloc_tag) \ + static DEFINE_PER_CPU(struct alloc_tag_counters, _alloc_tag_cntr); \ + static struct alloc_tag _alloc_tag __used __aligned(8) \ + __section("alloc_tags") = { \ + .ct = CODE_TAG_INIT, \ + .counters = &_alloc_tag_cntr }; + +#endif /* ARCH_NEEDS_WEAK_PER_CPU */ + +DECLARE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, + mem_alloc_profiling_key); + +static inline bool mem_alloc_profiling_enabled(void) +{ + return static_branch_maybe(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, + &mem_alloc_profiling_key); +} + +static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) +{ + struct alloc_tag_counters v = { 0, 0 }; + struct alloc_tag_counters *counter; + int cpu; + + for_each_possible_cpu(cpu) { + counter = per_cpu_ptr(tag->counters, cpu); + v.bytes += counter->bytes; + v.calls += counter->calls; + } + + return v; +} + +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG +static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) +{ + WARN_ONCE(ref && ref->ct, + "alloc_tag was not cleared (got tag for %s:%u)\n", + ref->ct->filename, ref->ct->lineno); + + WARN_ONCE(!tag, "current->alloc_tag not set"); +} + +static inline void alloc_tag_sub_check(union codetag_ref *ref) +{ + WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n"); +} +#else +static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {} +static inline void alloc_tag_sub_check(union codetag_ref *ref) {} +#endif + +/* Caller should verify both ref and tag to be valid */ +static inline void __alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag) +{ + ref->ct = &tag->ct; + /* + * We need in increment the call counter every time we have a new + * allocation or when we split a large allocation into smaller ones. + * Each new reference for every sub-allocation needs to increment call + * counter because when we free each part the counter will be decremented. + */ + this_cpu_inc(tag->counters->calls); +} + +static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes) +{ + alloc_tag_add_check(ref, tag); + if (!ref || !tag) + return; + + __alloc_tag_ref_set(ref, tag); + this_cpu_add(tag->counters->bytes, bytes); +} + +static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) +{ + struct alloc_tag *tag; + + alloc_tag_sub_check(ref); + if (!ref || !ref->ct) + return; + + tag = ct_to_alloc_tag(ref->ct); + + this_cpu_sub(tag->counters->bytes, bytes); + this_cpu_dec(tag->counters->calls); + + ref->ct = NULL; +} + +#else /* CONFIG_MEM_ALLOC_PROFILING */ + +#define DEFINE_ALLOC_TAG(_alloc_tag) +static inline bool mem_alloc_profiling_enabled(void) { return false; } +static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, + size_t bytes) {} +static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {} + +#endif /* CONFIG_MEM_ALLOC_PROFILING */ + +#endif /* _LINUX_ALLOC_TAG_H */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 3c2abbc587b49..4118b3f959c32 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -770,6 +770,10 @@ struct task_struct { unsigned int flags; unsigned int ptrace; +#ifdef CONFIG_MEM_ALLOC_PROFILING + struct alloc_tag *alloc_tag; +#endif + #ifdef CONFIG_SMP int on_cpu; struct __call_single_node wake_entry; @@ -810,6 +814,7 @@ struct task_struct { struct task_group *sched_task_group; #endif + #ifdef CONFIG_UCLAMP_TASK /* * Clamp values requested for a scheduling entity. @@ -2187,4 +2192,23 @@ static inline int sched_core_idle_cpu(int cpu) { return idle_cpu(cpu); } extern void sched_set_stop_task(int cpu, struct task_struct *stop); +#ifdef CONFIG_MEM_ALLOC_PROFILING +static inline struct alloc_tag *alloc_tag_save(struct alloc_tag *tag) +{ + swap(current->alloc_tag, tag); + return tag; +} + +static inline void alloc_tag_restore(struct alloc_tag *tag, struct alloc_tag *old) +{ +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG + WARN(current->alloc_tag != tag, "current->alloc_tag was changed:\n"); +#endif + current->alloc_tag = old; +} +#else +#define alloc_tag_save(_tag) NULL +#define alloc_tag_restore(_tag, _old) do {} while (0) +#endif + #endif diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 015fc6ee9849a..fa7aa32ba11a2 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -972,6 +972,31 @@ config CODE_TAGGING bool select KALLSYMS +config MEM_ALLOC_PROFILING + bool "Enable memory allocation profiling" + default n + depends on PROC_FS + depends on !DEBUG_FORCE_WEAK_PER_CPU + select CODE_TAGGING + help + Track allocation source code and record total allocation size + initiated at that code location. The mechanism can be used to track + memory leaks with a low performance and memory impact. + +config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT + bool "Enable memory allocation profiling by default" + default y + depends on MEM_ALLOC_PROFILING + +config MEM_ALLOC_PROFILING_DEBUG + bool "Memory allocation profiler debugging" + default n + depends on MEM_ALLOC_PROFILING + select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT + help + Adds warnings with helpful error messages for memory allocation + profiling. + source "lib/Kconfig.kasan" source "lib/Kconfig.kfence" source "lib/Kconfig.kmsan" diff --git a/lib/Makefile b/lib/Makefile index 910335da8f133..2f4e17bfb2990 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -234,6 +234,8 @@ obj-$(CONFIG_OF_RECONFIG_NOTIFIER_ERROR_INJECT) += \ obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o obj-$(CONFIG_CODE_TAGGING) += codetag.o +obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o + lib-$(CONFIG_GENERIC_BUG) += bug.o obj-$(CONFIG_HAVE_ARCH_TRACEHOOK) += syscall.o diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c new file mode 100644 index 0000000000000..331dd17650f32 --- /dev/null +++ b/lib/alloc_tag.c @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include +#include + +static struct codetag_type *alloc_tag_cttype; + +DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag); +EXPORT_SYMBOL(_shared_alloc_tag); + +DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, + mem_alloc_profiling_key); + +static void *allocinfo_start(struct seq_file *m, loff_t *pos) +{ + struct codetag_iterator *iter; + struct codetag *ct; + loff_t node = *pos; + + iter = kzalloc(sizeof(*iter), GFP_KERNEL); + m->private = iter; + if (!iter) + return NULL; + + codetag_lock_module_list(alloc_tag_cttype, true); + *iter = codetag_get_ct_iter(alloc_tag_cttype); + while ((ct = codetag_next_ct(iter)) != NULL && node) + node--; + + return ct ? iter : NULL; +} + +static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos) +{ + struct codetag_iterator *iter = (struct codetag_iterator *)arg; + struct codetag *ct = codetag_next_ct(iter); + + (*pos)++; + if (!ct) + return NULL; + + return iter; +} + +static void allocinfo_stop(struct seq_file *m, void *arg) +{ + struct codetag_iterator *iter = (struct codetag_iterator *)m->private; + + if (iter) { + codetag_lock_module_list(alloc_tag_cttype, false); + kfree(iter); + } +} + +static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct) +{ + struct alloc_tag *tag = ct_to_alloc_tag(ct); + struct alloc_tag_counters counter = alloc_tag_read(tag); + s64 bytes = counter.bytes; + + seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls); + codetag_to_text(out, ct); + seq_buf_putc(out, ' '); + seq_buf_putc(out, '\n'); +} + +static int allocinfo_show(struct seq_file *m, void *arg) +{ + struct codetag_iterator *iter = (struct codetag_iterator *)arg; + char *bufp; + size_t n = seq_get_buf(m, &bufp); + struct seq_buf buf; + + seq_buf_init(&buf, bufp, n); + alloc_tag_to_text(&buf, iter->ct); + seq_commit(m, seq_buf_used(&buf)); + return 0; +} + +static const struct seq_operations allocinfo_seq_op = { + .start = allocinfo_start, + .next = allocinfo_next, + .stop = allocinfo_stop, + .show = allocinfo_show, +}; + +static void __init procfs_init(void) +{ + proc_create_seq("allocinfo", 0444, NULL, &allocinfo_seq_op); +} + +static bool alloc_tag_module_unload(struct codetag_type *cttype, + struct codetag_module *cmod) +{ + struct codetag_iterator iter = codetag_get_ct_iter(cttype); + struct alloc_tag_counters counter; + bool module_unused = true; + struct alloc_tag *tag; + struct codetag *ct; + + for (ct = codetag_next_ct(&iter); ct; ct = codetag_next_ct(&iter)) { + if (iter.cmod != cmod) + continue; + + tag = ct_to_alloc_tag(ct); + counter = alloc_tag_read(tag); + + if (WARN(counter.bytes, + "%s:%u module %s func:%s has %llu allocated at module unload", + ct->filename, ct->lineno, ct->modname, ct->function, counter.bytes)) + module_unused = false; + } + + return module_unused; +} + +static struct ctl_table memory_allocation_profiling_sysctls[] = { + { + .procname = "mem_profiling", + .data = &mem_alloc_profiling_key, +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG + .mode = 0444, +#else + .mode = 0644, +#endif + .proc_handler = proc_do_static_key, + }, + { } +}; + +static int __init alloc_tag_init(void) +{ + const struct codetag_type_desc desc = { + .section = "alloc_tags", + .tag_size = sizeof(struct alloc_tag), + .module_unload = alloc_tag_module_unload, + }; + + alloc_tag_cttype = codetag_register_type(&desc); + if (IS_ERR(alloc_tag_cttype)) + return PTR_ERR(alloc_tag_cttype); + + register_sysctl_init("vm", memory_allocation_profiling_sysctls); + procfs_init(); + + return 0; +} +module_init(alloc_tag_init); diff --git a/scripts/module.lds.S b/scripts/module.lds.S index bf5bcf2836d81..45c67a0994f3e 100644 --- a/scripts/module.lds.S +++ b/scripts/module.lds.S @@ -9,6 +9,8 @@ #define DISCARD_EH_FRAME *(.eh_frame) #endif +#include + SECTIONS { /DISCARD/ : { *(.discard) @@ -47,12 +49,17 @@ SECTIONS { .data : { *(.data .data.[0-9a-zA-Z_]*) *(.data..L*) + CODETAG_SECTIONS() } .rodata : { *(.rodata .rodata.[0-9a-zA-Z_]*) *(.rodata..L*) } +#else + .data : { + CODETAG_SECTIONS() + } #endif } From dcfe378c81f72f146890ce1dcfdcc742d3b66924 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:36 -0700 Subject: [PATCH 0568/1702] lib: introduce support for page allocation tagging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce helper functions to easily instrument page allocators by storing a pointer to the allocation tag associated with the code that allocated the page in a page_ext field. Link: https://lkml.kernel.org/r/20240321163705.3067592-15-surenb@google.com Signed-off-by: Suren Baghdasaryan Co-developed-by: Kent Overstreet Signed-off-by: Kent Overstreet Reviewed-by: Vlastimil Babka Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/page_ext.h | 1 - include/linux/pgalloc_tag.h | 78 +++++++++++++++++++++++++++++++++++++ lib/Kconfig.debug | 1 + lib/alloc_tag.c | 17 ++++++++ mm/mm_init.c | 1 + mm/page_alloc.c | 4 ++ mm/page_ext.c | 4 ++ 7 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 include/linux/pgalloc_tag.h diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h index be98564191e67..07e0656898f98 100644 --- a/include/linux/page_ext.h +++ b/include/linux/page_ext.h @@ -4,7 +4,6 @@ #include #include -#include struct pglist_data; diff --git a/include/linux/pgalloc_tag.h b/include/linux/pgalloc_tag.h new file mode 100644 index 0000000000000..66bd021eb46ea --- /dev/null +++ b/include/linux/pgalloc_tag.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * page allocation tagging + */ +#ifndef _LINUX_PGALLOC_TAG_H +#define _LINUX_PGALLOC_TAG_H + +#include + +#ifdef CONFIG_MEM_ALLOC_PROFILING + +#include + +extern struct page_ext_operations page_alloc_tagging_ops; +extern struct page_ext *page_ext_get(struct page *page); +extern void page_ext_put(struct page_ext *page_ext); + +static inline union codetag_ref *codetag_ref_from_page_ext(struct page_ext *page_ext) +{ + return (void *)page_ext + page_alloc_tagging_ops.offset; +} + +static inline struct page_ext *page_ext_from_codetag_ref(union codetag_ref *ref) +{ + return (void *)ref - page_alloc_tagging_ops.offset; +} + +/* Should be called only if mem_alloc_profiling_enabled() */ +static inline union codetag_ref *get_page_tag_ref(struct page *page) +{ + if (page) { + struct page_ext *page_ext = page_ext_get(page); + + if (page_ext) + return codetag_ref_from_page_ext(page_ext); + } + return NULL; +} + +static inline void put_page_tag_ref(union codetag_ref *ref) +{ + page_ext_put(page_ext_from_codetag_ref(ref)); +} + +static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, + unsigned int nr) +{ + if (mem_alloc_profiling_enabled()) { + union codetag_ref *ref = get_page_tag_ref(page); + + if (ref) { + alloc_tag_add(ref, task->alloc_tag, PAGE_SIZE * nr); + put_page_tag_ref(ref); + } + } +} + +static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) +{ + if (mem_alloc_profiling_enabled()) { + union codetag_ref *ref = get_page_tag_ref(page); + + if (ref) { + alloc_tag_sub(ref, PAGE_SIZE * nr); + put_page_tag_ref(ref); + } + } +} + +#else /* CONFIG_MEM_ALLOC_PROFILING */ + +static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, + unsigned int nr) {} +static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {} + +#endif /* CONFIG_MEM_ALLOC_PROFILING */ + +#endif /* _LINUX_PGALLOC_TAG_H */ diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index fa7aa32ba11a2..a8f77de5e8ea8 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -978,6 +978,7 @@ config MEM_ALLOC_PROFILING depends on PROC_FS depends on !DEBUG_FORCE_WEAK_PER_CPU select CODE_TAGGING + select PAGE_EXTENSION help Track allocation source code and record total allocation size initiated at that code location. The mechanism can be used to track diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c index 331dd17650f32..59bb71036837b 100644 --- a/lib/alloc_tag.c +++ b/lib/alloc_tag.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -118,6 +119,22 @@ static bool alloc_tag_module_unload(struct codetag_type *cttype, return module_unused; } +static __init bool need_page_alloc_tagging(void) +{ + return true; +} + +static __init void init_page_alloc_tagging(void) +{ +} + +struct page_ext_operations page_alloc_tagging_ops = { + .size = sizeof(union codetag_ref), + .need = need_page_alloc_tagging, + .init = init_page_alloc_tagging, +}; +EXPORT_SYMBOL(page_alloc_tagging_ops); + static struct ctl_table memory_allocation_profiling_sysctls[] = { { .procname = "mem_profiling", diff --git a/mm/mm_init.c b/mm/mm_init.c index 549e76af8f82a..2fd9bf044a79c 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 5083ac034d26e..3fd273c22749f 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -54,6 +54,7 @@ #include #include #include +#include #include #include "internal.h" #include "shuffle.h" @@ -1101,6 +1102,7 @@ __always_inline bool free_pages_prepare(struct page *page, /* Do not let hwpoison pages hit pcplists/buddy */ reset_page_owner(page, order); page_table_check_free(page, order); + pgalloc_tag_sub(page, 1 << order); return false; } @@ -1140,6 +1142,7 @@ __always_inline bool free_pages_prepare(struct page *page, page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; reset_page_owner(page, order); page_table_check_free(page, order); + pgalloc_tag_sub(page, 1 << order); if (!PageHighMem(page)) { debug_check_no_locks_freed(page_address(page), @@ -1533,6 +1536,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order, set_page_owner(page, order, gfp_flags); page_table_check_alloc(page, order); + pgalloc_tag_add(page, current, 1 << order); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, diff --git a/mm/page_ext.c b/mm/page_ext.c index 4548fcc66d74d..3c58fe8a24df4 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -10,6 +10,7 @@ #include #include #include +#include /* * struct page extension @@ -82,6 +83,9 @@ static struct page_ext_operations *page_ext_ops[] __initdata = { #if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT) &page_idle_ops, #endif +#ifdef CONFIG_MEM_ALLOC_PROFILING + &page_alloc_tagging_ops, +#endif #ifdef CONFIG_PAGE_TABLE_CHECK &page_table_check_ops, #endif From 8d469d0bee74d3f2d02f0b232933a3f084d9cbf7 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:37 -0700 Subject: [PATCH 0569/1702] lib: introduce early boot parameter to avoid page_ext memory overhead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The highest memory overhead from memory allocation profiling comes from page_ext objects. This overhead exists even if the feature is disabled but compiled-in. To avoid it, introduce an early boot parameter that prevents page_ext object creation. The new boot parameter is a tri-state with possible values of 0|1|never. When it is set to "never" the memory allocation profiling support is disabled, and overhead is minimized (currently no page_ext objects are allocated, in the future more overhead might be eliminated). As a result we also lose ability to enable memory allocation profiling at runtime (because there is no space to store alloctag references). Runtime sysctrl becomes read-only if the early boot parameter was set to "never". Note that the default value of this boot parameter depends on the CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT configuration. When CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT=n the boot parameter is set to "never", therefore eliminating any overhead. CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT=y results in boot parameter being set to 1 (enabled). This allows distributions to avoid any overhead by setting CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT=n config and with no changes to the kernel command line. We reuse sysctl.vm.mem_profiling boot parameter name in order to avoid introducing yet another control. This change turns it into a tri-state early boot parameter. Link: https://lkml.kernel.org/r/20240321163705.3067592-16-surenb@google.com Signed-off-by: Suren Baghdasaryan Reviewed-by: Vlastimil Babka Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Kent Overstreet Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- lib/alloc_tag.c | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c index 59bb71036837b..6bd8e85beaf72 100644 --- a/lib/alloc_tag.c +++ b/lib/alloc_tag.c @@ -119,9 +119,46 @@ static bool alloc_tag_module_unload(struct codetag_type *cttype, return module_unused; } +#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT +static bool mem_profiling_support __meminitdata = true; +#else +static bool mem_profiling_support __meminitdata; +#endif + +static int __init setup_early_mem_profiling(char *str) +{ + bool enable; + + if (!str || !str[0]) + return -EINVAL; + + if (!strncmp(str, "never", 5)) { + enable = false; + mem_profiling_support = false; + } else { + int res; + + res = kstrtobool(str, &enable); + if (res) + return res; + + mem_profiling_support = true; + } + + if (enable != static_key_enabled(&mem_alloc_profiling_key)) { + if (enable) + static_branch_enable(&mem_alloc_profiling_key); + else + static_branch_disable(&mem_alloc_profiling_key); + } + + return 0; +} +early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling); + static __init bool need_page_alloc_tagging(void) { - return true; + return mem_profiling_support; } static __init void init_page_alloc_tagging(void) @@ -161,6 +198,8 @@ static int __init alloc_tag_init(void) if (IS_ERR(alloc_tag_cttype)) return PTR_ERR(alloc_tag_cttype); + if (!mem_profiling_support) + memory_allocation_profiling_sysctls[0].mode = 0444; register_sysctl_init("vm", memory_allocation_profiling_sysctls); procfs_init(); From ccdabb1d7f7a09810495ac57c0154a9913791dd2 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:38 -0700 Subject: [PATCH 0570/1702] mm: percpu: increase PERCPU_MODULE_RESERVE to accommodate allocation tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As each allocation tag generates a per-cpu variable, more space is required to store them. Increase PERCPU_MODULE_RESERVE to provide enough area. A better long-term solution would be to allocate this memory dynamically. [surenb@google.com: increase PERCPU_MODULE_RESERVE to accommodate allocation tags] Link: https://lkml.kernel.org/r/20240406214044.1114406-1-surenb@google.com Link: https://lkml.kernel.org/r/20240321163705.3067592-17-surenb@google.com Signed-off-by: Suren Baghdasaryan Signed-off-by: Kent Overstreet Tested-by: Kees Cook Cc: Peter Zijlstra Cc: Tejun Heo Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/percpu.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 8c677f185901b..9dfa7b526e70c 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -14,7 +14,11 @@ /* enough to cover all DEFINE_PER_CPUs in modules */ #ifdef CONFIG_MODULES +#ifdef CONFIG_MEM_ALLOC_PROFILING +#define PERCPU_MODULE_RESERVE (8 << 13) +#else #define PERCPU_MODULE_RESERVE (8 << 10) +#endif #else #define PERCPU_MODULE_RESERVE 0 #endif From 8a2f11878771da65b8ac135c73b47dae13afbd62 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:39 -0700 Subject: [PATCH 0571/1702] change alloc_pages name in dma_map_ops to avoid name conflicts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After redefining alloc_pages, all uses of that name are being replaced. Change the conflicting names to prevent preprocessor from replacing them when it's not intended. Link: https://lkml.kernel.org/r/20240321163705.3067592-18-surenb@google.com Signed-off-by: Suren Baghdasaryan Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Kent Overstreet Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- arch/alpha/kernel/pci_iommu.c | 2 +- arch/mips/jazz/jazzdma.c | 2 +- arch/powerpc/kernel/dma-iommu.c | 2 +- arch/powerpc/platforms/ps3/system-bus.c | 4 ++-- arch/powerpc/platforms/pseries/vio.c | 2 +- arch/x86/kernel/amd_gart_64.c | 2 +- drivers/iommu/dma-iommu.c | 2 +- drivers/parisc/ccio-dma.c | 2 +- drivers/parisc/sba_iommu.c | 2 +- drivers/xen/grant-dma-ops.c | 2 +- drivers/xen/swiotlb-xen.c | 2 +- include/linux/dma-map-ops.h | 2 +- kernel/dma/mapping.c | 4 ++-- 13 files changed, 15 insertions(+), 15 deletions(-) diff --git a/arch/alpha/kernel/pci_iommu.c b/arch/alpha/kernel/pci_iommu.c index c81183935e970..7fcf3e9b71030 100644 --- a/arch/alpha/kernel/pci_iommu.c +++ b/arch/alpha/kernel/pci_iommu.c @@ -929,7 +929,7 @@ const struct dma_map_ops alpha_pci_ops = { .dma_supported = alpha_pci_supported, .mmap = dma_common_mmap, .get_sgtable = dma_common_get_sgtable, - .alloc_pages = dma_common_alloc_pages, + .alloc_pages_op = dma_common_alloc_pages, .free_pages = dma_common_free_pages, }; EXPORT_SYMBOL(alpha_pci_ops); diff --git a/arch/mips/jazz/jazzdma.c b/arch/mips/jazz/jazzdma.c index eabddb89d221f..c97b089b99029 100644 --- a/arch/mips/jazz/jazzdma.c +++ b/arch/mips/jazz/jazzdma.c @@ -617,7 +617,7 @@ const struct dma_map_ops jazz_dma_ops = { .sync_sg_for_device = jazz_dma_sync_sg_for_device, .mmap = dma_common_mmap, .get_sgtable = dma_common_get_sgtable, - .alloc_pages = dma_common_alloc_pages, + .alloc_pages_op = dma_common_alloc_pages, .free_pages = dma_common_free_pages, }; EXPORT_SYMBOL(jazz_dma_ops); diff --git a/arch/powerpc/kernel/dma-iommu.c b/arch/powerpc/kernel/dma-iommu.c index 8920862ffd791..f0ae39e77e374 100644 --- a/arch/powerpc/kernel/dma-iommu.c +++ b/arch/powerpc/kernel/dma-iommu.c @@ -216,6 +216,6 @@ const struct dma_map_ops dma_iommu_ops = { .get_required_mask = dma_iommu_get_required_mask, .mmap = dma_common_mmap, .get_sgtable = dma_common_get_sgtable, - .alloc_pages = dma_common_alloc_pages, + .alloc_pages_op = dma_common_alloc_pages, .free_pages = dma_common_free_pages, }; diff --git a/arch/powerpc/platforms/ps3/system-bus.c b/arch/powerpc/platforms/ps3/system-bus.c index d6b5f5ecd5152..56dc6b29a3e76 100644 --- a/arch/powerpc/platforms/ps3/system-bus.c +++ b/arch/powerpc/platforms/ps3/system-bus.c @@ -695,7 +695,7 @@ static const struct dma_map_ops ps3_sb_dma_ops = { .unmap_page = ps3_unmap_page, .mmap = dma_common_mmap, .get_sgtable = dma_common_get_sgtable, - .alloc_pages = dma_common_alloc_pages, + .alloc_pages_op = dma_common_alloc_pages, .free_pages = dma_common_free_pages, }; @@ -709,7 +709,7 @@ static const struct dma_map_ops ps3_ioc0_dma_ops = { .unmap_page = ps3_unmap_page, .mmap = dma_common_mmap, .get_sgtable = dma_common_get_sgtable, - .alloc_pages = dma_common_alloc_pages, + .alloc_pages_op = dma_common_alloc_pages, .free_pages = dma_common_free_pages, }; diff --git a/arch/powerpc/platforms/pseries/vio.c b/arch/powerpc/platforms/pseries/vio.c index 90ff85c879bfe..477c1d5e1737b 100644 --- a/arch/powerpc/platforms/pseries/vio.c +++ b/arch/powerpc/platforms/pseries/vio.c @@ -611,7 +611,7 @@ static const struct dma_map_ops vio_dma_mapping_ops = { .get_required_mask = dma_iommu_get_required_mask, .mmap = dma_common_mmap, .get_sgtable = dma_common_get_sgtable, - .alloc_pages = dma_common_alloc_pages, + .alloc_pages_op = dma_common_alloc_pages, .free_pages = dma_common_free_pages, }; diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c index 2ae98f754e591..c884deca839b2 100644 --- a/arch/x86/kernel/amd_gart_64.c +++ b/arch/x86/kernel/amd_gart_64.c @@ -676,7 +676,7 @@ static const struct dma_map_ops gart_dma_ops = { .get_sgtable = dma_common_get_sgtable, .dma_supported = dma_direct_supported, .get_required_mask = dma_direct_get_required_mask, - .alloc_pages = dma_direct_alloc_pages, + .alloc_pages_op = dma_direct_alloc_pages, .free_pages = dma_direct_free_pages, }; diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index e4cb26f6a9434..07d087eecc17c 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1723,7 +1723,7 @@ static const struct dma_map_ops iommu_dma_ops = { .flags = DMA_F_PCI_P2PDMA_SUPPORTED, .alloc = iommu_dma_alloc, .free = iommu_dma_free, - .alloc_pages = dma_common_alloc_pages, + .alloc_pages_op = dma_common_alloc_pages, .free_pages = dma_common_free_pages, .alloc_noncontiguous = iommu_dma_alloc_noncontiguous, .free_noncontiguous = iommu_dma_free_noncontiguous, diff --git a/drivers/parisc/ccio-dma.c b/drivers/parisc/ccio-dma.c index 9ce0d20a6c581..feef537257d05 100644 --- a/drivers/parisc/ccio-dma.c +++ b/drivers/parisc/ccio-dma.c @@ -1022,7 +1022,7 @@ static const struct dma_map_ops ccio_ops = { .map_sg = ccio_map_sg, .unmap_sg = ccio_unmap_sg, .get_sgtable = dma_common_get_sgtable, - .alloc_pages = dma_common_alloc_pages, + .alloc_pages_op = dma_common_alloc_pages, .free_pages = dma_common_free_pages, }; diff --git a/drivers/parisc/sba_iommu.c b/drivers/parisc/sba_iommu.c index 784037837f65e..fc3863c09f83d 100644 --- a/drivers/parisc/sba_iommu.c +++ b/drivers/parisc/sba_iommu.c @@ -1090,7 +1090,7 @@ static const struct dma_map_ops sba_ops = { .map_sg = sba_map_sg, .unmap_sg = sba_unmap_sg, .get_sgtable = dma_common_get_sgtable, - .alloc_pages = dma_common_alloc_pages, + .alloc_pages_op = dma_common_alloc_pages, .free_pages = dma_common_free_pages, }; diff --git a/drivers/xen/grant-dma-ops.c b/drivers/xen/grant-dma-ops.c index 76f6f26265a3b..29257d2639dbf 100644 --- a/drivers/xen/grant-dma-ops.c +++ b/drivers/xen/grant-dma-ops.c @@ -282,7 +282,7 @@ static int xen_grant_dma_supported(struct device *dev, u64 mask) static const struct dma_map_ops xen_grant_dma_ops = { .alloc = xen_grant_dma_alloc, .free = xen_grant_dma_free, - .alloc_pages = xen_grant_dma_alloc_pages, + .alloc_pages_op = xen_grant_dma_alloc_pages, .free_pages = xen_grant_dma_free_pages, .mmap = dma_common_mmap, .get_sgtable = dma_common_get_sgtable, diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 0e6c6c25d154f..1c4ef5111651d 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -403,7 +403,7 @@ const struct dma_map_ops xen_swiotlb_dma_ops = { .dma_supported = xen_swiotlb_dma_supported, .mmap = dma_common_mmap, .get_sgtable = dma_common_get_sgtable, - .alloc_pages = dma_common_alloc_pages, + .alloc_pages_op = dma_common_alloc_pages, .free_pages = dma_common_free_pages, .max_mapping_size = swiotlb_max_mapping_size, }; diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 4abc60f042092..9ee319851b5f6 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -29,7 +29,7 @@ struct dma_map_ops { unsigned long attrs); void (*free)(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, unsigned long attrs); - struct page *(*alloc_pages)(struct device *dev, size_t size, + struct page *(*alloc_pages_op)(struct device *dev, size_t size, dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp); void (*free_pages)(struct device *dev, size_t size, struct page *vaddr, diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 58db8fd70471a..5e2d51e1cdf69 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -570,9 +570,9 @@ static struct page *__dma_alloc_pages(struct device *dev, size_t size, size = PAGE_ALIGN(size); if (dma_alloc_direct(dev, ops)) return dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp); - if (!ops->alloc_pages) + if (!ops->alloc_pages_op) return NULL; - return ops->alloc_pages(dev, size, dma_handle, dir, gfp); + return ops->alloc_pages_op(dev, size, dma_handle, dir, gfp); } struct page *dma_alloc_pages(struct device *dev, size_t size, From b951aaff503502a7fe066eeed2744ba8a6413c89 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:40 -0700 Subject: [PATCH 0572/1702] mm: enable page allocation tagging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Redefine page allocators to record allocation tags upon their invocation. Instrument post_alloc_hook and free_pages_prepare to modify current allocation tag. [surenb@google.com: undo _noprof additions in the documentation] Link: https://lkml.kernel.org/r/20240326231453.1206227-3-surenb@google.com Link: https://lkml.kernel.org/r/20240321163705.3067592-19-surenb@google.com Signed-off-by: Suren Baghdasaryan Co-developed-by: Kent Overstreet Signed-off-by: Kent Overstreet Reviewed-by: Kees Cook Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/alloc_tag.h | 14 +++++ include/linux/gfp.h | 126 ++++++++++++++++++++++++-------------- include/linux/pagemap.h | 9 ++- mm/compaction.c | 7 ++- mm/filemap.c | 6 +- mm/mempolicy.c | 46 +++++++------- mm/page_alloc.c | 52 ++++++++-------- 7 files changed, 157 insertions(+), 103 deletions(-) diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h index 13561223fdabd..8932187124dab 100644 --- a/include/linux/alloc_tag.h +++ b/include/linux/alloc_tag.h @@ -153,4 +153,18 @@ static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {} #endif /* CONFIG_MEM_ALLOC_PROFILING */ +#define alloc_hooks_tag(_tag, _do_alloc) \ +({ \ + struct alloc_tag * __maybe_unused _old = alloc_tag_save(_tag); \ + typeof(_do_alloc) _res = _do_alloc; \ + alloc_tag_restore(_tag, _old); \ + _res; \ +}) + +#define alloc_hooks(_do_alloc) \ +({ \ + DEFINE_ALLOC_TAG(_alloc_tag); \ + alloc_hooks_tag(&_alloc_tag, _do_alloc); \ +}) + #endif /* _LINUX_ALLOC_TAG_H */ diff --git a/include/linux/gfp.h b/include/linux/gfp.h index c775ea3c60155..450c2cbcf04b8 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -6,6 +6,8 @@ #include #include +#include +#include struct vm_area_struct; struct mempolicy; @@ -175,42 +177,46 @@ static inline void arch_free_page(struct page *page, int order) { } static inline void arch_alloc_page(struct page *page, int order) { } #endif -struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid, +struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid, nodemask_t *nodemask); -struct folio *__folio_alloc(gfp_t gfp, unsigned int order, int preferred_nid, +#define __alloc_pages(...) alloc_hooks(__alloc_pages_noprof(__VA_ARGS__)) + +struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid, nodemask_t *nodemask); +#define __folio_alloc(...) alloc_hooks(__folio_alloc_noprof(__VA_ARGS__)) -unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, +unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid, nodemask_t *nodemask, int nr_pages, struct list_head *page_list, struct page **page_array); +#define __alloc_pages_bulk(...) alloc_hooks(alloc_pages_bulk_noprof(__VA_ARGS__)) -unsigned long alloc_pages_bulk_array_mempolicy(gfp_t gfp, +unsigned long alloc_pages_bulk_array_mempolicy_noprof(gfp_t gfp, unsigned long nr_pages, struct page **page_array); +#define alloc_pages_bulk_array_mempolicy(...) \ + alloc_hooks(alloc_pages_bulk_array_mempolicy_noprof(__VA_ARGS__)) /* Bulk allocate order-0 pages */ -static inline unsigned long -alloc_pages_bulk_list(gfp_t gfp, unsigned long nr_pages, struct list_head *list) -{ - return __alloc_pages_bulk(gfp, numa_mem_id(), NULL, nr_pages, list, NULL); -} +#define alloc_pages_bulk_list(_gfp, _nr_pages, _list) \ + __alloc_pages_bulk(_gfp, numa_mem_id(), NULL, _nr_pages, _list, NULL) -static inline unsigned long -alloc_pages_bulk_array(gfp_t gfp, unsigned long nr_pages, struct page **page_array) -{ - return __alloc_pages_bulk(gfp, numa_mem_id(), NULL, nr_pages, NULL, page_array); -} +#define alloc_pages_bulk_array(_gfp, _nr_pages, _page_array) \ + __alloc_pages_bulk(_gfp, numa_mem_id(), NULL, _nr_pages, NULL, _page_array) static inline unsigned long -alloc_pages_bulk_array_node(gfp_t gfp, int nid, unsigned long nr_pages, struct page **page_array) +alloc_pages_bulk_array_node_noprof(gfp_t gfp, int nid, unsigned long nr_pages, + struct page **page_array) { if (nid == NUMA_NO_NODE) nid = numa_mem_id(); - return __alloc_pages_bulk(gfp, nid, NULL, nr_pages, NULL, page_array); + return alloc_pages_bulk_noprof(gfp, nid, NULL, nr_pages, NULL, page_array); } +#define alloc_pages_bulk_array_node(...) \ + alloc_hooks(alloc_pages_bulk_array_node_noprof(__VA_ARGS__)) + static inline void warn_if_node_offline(int this_node, gfp_t gfp_mask) { gfp_t warn_gfp = gfp_mask & (__GFP_THISNODE|__GFP_NOWARN); @@ -230,82 +236,104 @@ static inline void warn_if_node_offline(int this_node, gfp_t gfp_mask) * online. For more general interface, see alloc_pages_node(). */ static inline struct page * -__alloc_pages_node(int nid, gfp_t gfp_mask, unsigned int order) +__alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order) { VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); warn_if_node_offline(nid, gfp_mask); - return __alloc_pages(gfp_mask, order, nid, NULL); + return __alloc_pages_noprof(gfp_mask, order, nid, NULL); } +#define __alloc_pages_node(...) alloc_hooks(__alloc_pages_node_noprof(__VA_ARGS__)) + static inline -struct folio *__folio_alloc_node(gfp_t gfp, unsigned int order, int nid) +struct folio *__folio_alloc_node_noprof(gfp_t gfp, unsigned int order, int nid) { VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); warn_if_node_offline(nid, gfp); - return __folio_alloc(gfp, order, nid, NULL); + return __folio_alloc_noprof(gfp, order, nid, NULL); } +#define __folio_alloc_node(...) alloc_hooks(__folio_alloc_node_noprof(__VA_ARGS__)) + /* * Allocate pages, preferring the node given as nid. When nid == NUMA_NO_NODE, * prefer the current CPU's closest node. Otherwise node must be valid and * online. */ -static inline struct page *alloc_pages_node(int nid, gfp_t gfp_mask, - unsigned int order) +static inline struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, + unsigned int order) { if (nid == NUMA_NO_NODE) nid = numa_mem_id(); - return __alloc_pages_node(nid, gfp_mask, order); + return __alloc_pages_node_noprof(nid, gfp_mask, order); } +#define alloc_pages_node(...) alloc_hooks(alloc_pages_node_noprof(__VA_ARGS__)) + #ifdef CONFIG_NUMA -struct page *alloc_pages(gfp_t gfp, unsigned int order); -struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, +struct page *alloc_pages_noprof(gfp_t gfp, unsigned int order); +struct page *alloc_pages_mpol_noprof(gfp_t gfp, unsigned int order, struct mempolicy *mpol, pgoff_t ilx, int nid); -struct folio *folio_alloc(gfp_t gfp, unsigned int order); -struct folio *vma_alloc_folio(gfp_t gfp, int order, struct vm_area_struct *vma, +struct folio *folio_alloc_noprof(gfp_t gfp, unsigned int order); +struct folio *vma_alloc_folio_noprof(gfp_t gfp, int order, struct vm_area_struct *vma, unsigned long addr, bool hugepage); #else -static inline struct page *alloc_pages(gfp_t gfp_mask, unsigned int order) +static inline struct page *alloc_pages_noprof(gfp_t gfp_mask, unsigned int order) { - return alloc_pages_node(numa_node_id(), gfp_mask, order); + return alloc_pages_node_noprof(numa_node_id(), gfp_mask, order); } -static inline struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, +static inline struct page *alloc_pages_mpol_noprof(gfp_t gfp, unsigned int order, struct mempolicy *mpol, pgoff_t ilx, int nid) { - return alloc_pages(gfp, order); + return alloc_pages_noprof(gfp, order); } -static inline struct folio *folio_alloc(gfp_t gfp, unsigned int order) +static inline struct folio *folio_alloc_noprof(gfp_t gfp, unsigned int order) { return __folio_alloc_node(gfp, order, numa_node_id()); } -#define vma_alloc_folio(gfp, order, vma, addr, hugepage) \ - folio_alloc(gfp, order) +#define vma_alloc_folio_noprof(gfp, order, vma, addr, hugepage) \ + folio_alloc_noprof(gfp, order) #endif + +#define alloc_pages(...) alloc_hooks(alloc_pages_noprof(__VA_ARGS__)) +#define alloc_pages_mpol(...) alloc_hooks(alloc_pages_mpol_noprof(__VA_ARGS__)) +#define folio_alloc(...) alloc_hooks(folio_alloc_noprof(__VA_ARGS__)) +#define vma_alloc_folio(...) alloc_hooks(vma_alloc_folio_noprof(__VA_ARGS__)) + #define alloc_page(gfp_mask) alloc_pages(gfp_mask, 0) -static inline struct page *alloc_page_vma(gfp_t gfp, + +static inline struct page *alloc_page_vma_noprof(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr) { - struct folio *folio = vma_alloc_folio(gfp, 0, vma, addr, false); + struct folio *folio = vma_alloc_folio_noprof(gfp, 0, vma, addr, false); return &folio->page; } +#define alloc_page_vma(...) alloc_hooks(alloc_page_vma_noprof(__VA_ARGS__)) + +extern unsigned long get_free_pages_noprof(gfp_t gfp_mask, unsigned int order); +#define __get_free_pages(...) alloc_hooks(get_free_pages_noprof(__VA_ARGS__)) -extern unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order); -extern unsigned long get_zeroed_page(gfp_t gfp_mask); +extern unsigned long get_zeroed_page_noprof(gfp_t gfp_mask); +#define get_zeroed_page(...) alloc_hooks(get_zeroed_page_noprof(__VA_ARGS__)) + +void *alloc_pages_exact_noprof(size_t size, gfp_t gfp_mask) __alloc_size(1); +#define alloc_pages_exact(...) alloc_hooks(alloc_pages_exact_noprof(__VA_ARGS__)) -void *alloc_pages_exact(size_t size, gfp_t gfp_mask) __alloc_size(1); void free_pages_exact(void *virt, size_t size); -__meminit void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2); -#define __get_free_page(gfp_mask) \ - __get_free_pages((gfp_mask), 0) +__meminit void *alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2); +#define alloc_pages_exact_nid(...) \ + alloc_hooks(alloc_pages_exact_nid_noprof(__VA_ARGS__)) + +#define __get_free_page(gfp_mask) \ + __get_free_pages((gfp_mask), 0) -#define __get_dma_pages(gfp_mask, order) \ - __get_free_pages((gfp_mask) | GFP_DMA, (order)) +#define __get_dma_pages(gfp_mask, order) \ + __get_free_pages((gfp_mask) | GFP_DMA, (order)) extern void __free_pages(struct page *page, unsigned int order); extern void free_pages(unsigned long addr, unsigned int order); @@ -374,10 +402,14 @@ extern gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma); #ifdef CONFIG_CONTIG_ALLOC /* The below functions must be run on a range from a single zone. */ -extern int alloc_contig_range(unsigned long start, unsigned long end, +extern int alloc_contig_range_noprof(unsigned long start, unsigned long end, unsigned migratetype, gfp_t gfp_mask); -extern struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, - int nid, nodemask_t *nodemask); +#define alloc_contig_range(...) alloc_hooks(alloc_contig_range_noprof(__VA_ARGS__)) + +extern struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask, + int nid, nodemask_t *nodemask); +#define alloc_contig_pages(...) alloc_hooks(alloc_contig_pages_noprof(__VA_ARGS__)) + #endif void free_contig_range(unsigned long pfn, unsigned long nr_pages); diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 2df35e65557d2..35636e67e2e14 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -542,14 +542,17 @@ static inline void *detach_page_private(struct page *page) #endif #ifdef CONFIG_NUMA -struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order); +struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order); #else -static inline struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order) +static inline struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order) { - return folio_alloc(gfp, order); + return folio_alloc_noprof(gfp, order); } #endif +#define filemap_alloc_folio(...) \ + alloc_hooks(filemap_alloc_folio_noprof(__VA_ARGS__)) + static inline struct page *__page_cache_alloc(gfp_t gfp) { return &filemap_alloc_folio(gfp, 0)->page; diff --git a/mm/compaction.c b/mm/compaction.c index 807b58e6eb68b..70b01190d2f3f 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -1851,7 +1851,7 @@ static void isolate_freepages(struct compact_control *cc) * This is a migrate-callback that "allocates" freepages by taking pages * from the isolated freelists in the block we are migrating to. */ -static struct folio *compaction_alloc(struct folio *src, unsigned long data) +static struct folio *compaction_alloc_noprof(struct folio *src, unsigned long data) { struct compact_control *cc = (struct compact_control *)data; struct folio *dst; @@ -1898,6 +1898,11 @@ static struct folio *compaction_alloc(struct folio *src, unsigned long data) return page_rmappable_folio(&dst->page); } +static struct folio *compaction_alloc(struct folio *src, unsigned long data) +{ + return alloc_hooks(compaction_alloc_noprof(src, data)); +} + /* * This is a migrate-callback that "frees" freepages back to the isolated * freelist. All pages on the freelist are from the same zone, so there is no diff --git a/mm/filemap.c b/mm/filemap.c index 30de18c4fd28a..60890758d6605 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -966,7 +966,7 @@ int filemap_add_folio(struct address_space *mapping, struct folio *folio, EXPORT_SYMBOL_GPL(filemap_add_folio); #ifdef CONFIG_NUMA -struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order) +struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order) { int n; struct folio *folio; @@ -981,9 +981,9 @@ struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order) return folio; } - return folio_alloc(gfp, order); + return folio_alloc_noprof(gfp, order); } -EXPORT_SYMBOL(filemap_alloc_folio); +EXPORT_SYMBOL(filemap_alloc_folio_noprof); #endif /* diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 13100a290918f..0b3def99174a1 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -2201,9 +2201,9 @@ static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order, */ preferred_gfp = gfp | __GFP_NOWARN; preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); - page = __alloc_pages(preferred_gfp, order, nid, nodemask); + page = __alloc_pages_noprof(preferred_gfp, order, nid, nodemask); if (!page) - page = __alloc_pages(gfp, order, nid, NULL); + page = __alloc_pages_noprof(gfp, order, nid, NULL); return page; } @@ -2218,7 +2218,7 @@ static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order, * * Return: The page on success or NULL if allocation fails. */ -struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, +struct page *alloc_pages_mpol_noprof(gfp_t gfp, unsigned int order, struct mempolicy *pol, pgoff_t ilx, int nid) { nodemask_t *nodemask; @@ -2249,7 +2249,7 @@ struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, * First, try to allocate THP only on local node, but * don't reclaim unnecessarily, just compact. */ - page = __alloc_pages_node(nid, + page = __alloc_pages_node_noprof(nid, gfp | __GFP_THISNODE | __GFP_NORETRY, order); if (page || !(gfp & __GFP_DIRECT_RECLAIM)) return page; @@ -2262,7 +2262,7 @@ struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, } } - page = __alloc_pages(gfp, order, nid, nodemask); + page = __alloc_pages_noprof(gfp, order, nid, nodemask); if (unlikely(pol->mode == MPOL_INTERLEAVE) && page) { /* skip NUMA_INTERLEAVE_HIT update if numa stats is disabled */ @@ -2293,7 +2293,7 @@ struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, * * Return: The folio on success or NULL if allocation fails. */ -struct folio *vma_alloc_folio(gfp_t gfp, int order, struct vm_area_struct *vma, +struct folio *vma_alloc_folio_noprof(gfp_t gfp, int order, struct vm_area_struct *vma, unsigned long addr, bool hugepage) { struct mempolicy *pol; @@ -2301,12 +2301,12 @@ struct folio *vma_alloc_folio(gfp_t gfp, int order, struct vm_area_struct *vma, struct page *page; pol = get_vma_policy(vma, addr, order, &ilx); - page = alloc_pages_mpol(gfp | __GFP_COMP, order, - pol, ilx, numa_node_id()); + page = alloc_pages_mpol_noprof(gfp | __GFP_COMP, order, + pol, ilx, numa_node_id()); mpol_cond_put(pol); return page_rmappable_folio(page); } -EXPORT_SYMBOL(vma_alloc_folio); +EXPORT_SYMBOL(vma_alloc_folio_noprof); /** * alloc_pages - Allocate pages. @@ -2322,7 +2322,7 @@ EXPORT_SYMBOL(vma_alloc_folio); * flags are used. * Return: The page on success or NULL if allocation fails. */ -struct page *alloc_pages(gfp_t gfp, unsigned int order) +struct page *alloc_pages_noprof(gfp_t gfp, unsigned int order) { struct mempolicy *pol = &default_policy; @@ -2333,16 +2333,16 @@ struct page *alloc_pages(gfp_t gfp, unsigned int order) if (!in_interrupt() && !(gfp & __GFP_THISNODE)) pol = get_task_policy(current); - return alloc_pages_mpol(gfp, order, - pol, NO_INTERLEAVE_INDEX, numa_node_id()); + return alloc_pages_mpol_noprof(gfp, order, pol, NO_INTERLEAVE_INDEX, + numa_node_id()); } -EXPORT_SYMBOL(alloc_pages); +EXPORT_SYMBOL(alloc_pages_noprof); -struct folio *folio_alloc(gfp_t gfp, unsigned int order) +struct folio *folio_alloc_noprof(gfp_t gfp, unsigned int order) { - return page_rmappable_folio(alloc_pages(gfp | __GFP_COMP, order)); + return page_rmappable_folio(alloc_pages_noprof(gfp | __GFP_COMP, order)); } -EXPORT_SYMBOL(folio_alloc); +EXPORT_SYMBOL(folio_alloc_noprof); static unsigned long alloc_pages_bulk_array_interleave(gfp_t gfp, struct mempolicy *pol, unsigned long nr_pages, @@ -2361,13 +2361,13 @@ static unsigned long alloc_pages_bulk_array_interleave(gfp_t gfp, for (i = 0; i < nodes; i++) { if (delta) { - nr_allocated = __alloc_pages_bulk(gfp, + nr_allocated = alloc_pages_bulk_noprof(gfp, interleave_nodes(pol), NULL, nr_pages_per_node + 1, NULL, page_array); delta--; } else { - nr_allocated = __alloc_pages_bulk(gfp, + nr_allocated = alloc_pages_bulk_noprof(gfp, interleave_nodes(pol), NULL, nr_pages_per_node, NULL, page_array); } @@ -2504,11 +2504,11 @@ static unsigned long alloc_pages_bulk_array_preferred_many(gfp_t gfp, int nid, preferred_gfp = gfp | __GFP_NOWARN; preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); - nr_allocated = __alloc_pages_bulk(preferred_gfp, nid, &pol->nodes, + nr_allocated = alloc_pages_bulk_noprof(preferred_gfp, nid, &pol->nodes, nr_pages, NULL, page_array); if (nr_allocated < nr_pages) - nr_allocated += __alloc_pages_bulk(gfp, numa_node_id(), NULL, + nr_allocated += alloc_pages_bulk_noprof(gfp, numa_node_id(), NULL, nr_pages - nr_allocated, NULL, page_array + nr_allocated); return nr_allocated; @@ -2520,7 +2520,7 @@ static unsigned long alloc_pages_bulk_array_preferred_many(gfp_t gfp, int nid, * It can accelerate memory allocation especially interleaving * allocate memory. */ -unsigned long alloc_pages_bulk_array_mempolicy(gfp_t gfp, +unsigned long alloc_pages_bulk_array_mempolicy_noprof(gfp_t gfp, unsigned long nr_pages, struct page **page_array) { struct mempolicy *pol = &default_policy; @@ -2544,8 +2544,8 @@ unsigned long alloc_pages_bulk_array_mempolicy(gfp_t gfp, nid = numa_node_id(); nodemask = policy_nodemask(gfp, pol, NO_INTERLEAVE_INDEX, &nid); - return __alloc_pages_bulk(gfp, nid, nodemask, - nr_pages, NULL, page_array); + return alloc_pages_bulk_noprof(gfp, nid, nodemask, + nr_pages, NULL, page_array); } int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 3fd273c22749f..d3afaf8f983ab 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4391,7 +4391,7 @@ static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, * * Returns the number of pages on the list or array. */ -unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, +unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid, nodemask_t *nodemask, int nr_pages, struct list_head *page_list, struct page **page_array) @@ -4527,7 +4527,7 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, pcp_trylock_finish(UP_flags); failed: - page = __alloc_pages(gfp, 0, preferred_nid, nodemask); + page = __alloc_pages_noprof(gfp, 0, preferred_nid, nodemask); if (page) { if (page_list) list_add(&page->lru, page_list); @@ -4538,13 +4538,13 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, goto out; } -EXPORT_SYMBOL_GPL(__alloc_pages_bulk); +EXPORT_SYMBOL_GPL(alloc_pages_bulk_noprof); /* * This is the 'heart' of the zoned buddy allocator. */ -struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid, - nodemask_t *nodemask) +struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, + int preferred_nid, nodemask_t *nodemask) { struct page *page; unsigned int alloc_flags = ALLOC_WMARK_LOW; @@ -4606,38 +4606,38 @@ struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid, return page; } -EXPORT_SYMBOL(__alloc_pages); +EXPORT_SYMBOL(__alloc_pages_noprof); -struct folio *__folio_alloc(gfp_t gfp, unsigned int order, int preferred_nid, +struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid, nodemask_t *nodemask) { - struct page *page = __alloc_pages(gfp | __GFP_COMP, order, + struct page *page = __alloc_pages_noprof(gfp | __GFP_COMP, order, preferred_nid, nodemask); return page_rmappable_folio(page); } -EXPORT_SYMBOL(__folio_alloc); +EXPORT_SYMBOL(__folio_alloc_noprof); /* * Common helper functions. Never use with __GFP_HIGHMEM because the returned * address cannot represent highmem pages. Use alloc_pages and then kmap if * you need to access high mem. */ -unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) +unsigned long get_free_pages_noprof(gfp_t gfp_mask, unsigned int order) { struct page *page; - page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order); + page = alloc_pages_noprof(gfp_mask & ~__GFP_HIGHMEM, order); if (!page) return 0; return (unsigned long) page_address(page); } -EXPORT_SYMBOL(__get_free_pages); +EXPORT_SYMBOL(get_free_pages_noprof); -unsigned long get_zeroed_page(gfp_t gfp_mask) +unsigned long get_zeroed_page_noprof(gfp_t gfp_mask) { - return __get_free_page(gfp_mask | __GFP_ZERO); + return get_free_pages_noprof(gfp_mask | __GFP_ZERO, 0); } -EXPORT_SYMBOL(get_zeroed_page); +EXPORT_SYMBOL(get_zeroed_page_noprof); /** * __free_pages - Free pages allocated with alloc_pages(). @@ -4853,7 +4853,7 @@ static void *make_alloc_exact(unsigned long addr, unsigned int order, * * Return: pointer to the allocated area or %NULL in case of error. */ -void *alloc_pages_exact(size_t size, gfp_t gfp_mask) +void *alloc_pages_exact_noprof(size_t size, gfp_t gfp_mask) { unsigned int order = get_order(size); unsigned long addr; @@ -4861,10 +4861,10 @@ void *alloc_pages_exact(size_t size, gfp_t gfp_mask) if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM))) gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM); - addr = __get_free_pages(gfp_mask, order); + addr = get_free_pages_noprof(gfp_mask, order); return make_alloc_exact(addr, order, size); } -EXPORT_SYMBOL(alloc_pages_exact); +EXPORT_SYMBOL(alloc_pages_exact_noprof); /** * alloc_pages_exact_nid - allocate an exact number of physically-contiguous @@ -4878,7 +4878,7 @@ EXPORT_SYMBOL(alloc_pages_exact); * * Return: pointer to the allocated area or %NULL in case of error. */ -void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) +void * __meminit alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) { unsigned int order = get_order(size); struct page *p; @@ -4886,7 +4886,7 @@ void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM))) gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM); - p = alloc_pages_node(nid, gfp_mask, order); + p = alloc_pages_node_noprof(nid, gfp_mask, order); if (!p) return NULL; return make_alloc_exact((unsigned long)page_address(p), order, size); @@ -6343,7 +6343,7 @@ int __alloc_contig_migrate_range(struct compact_control *cc, * pages which PFN is in [start, end) are allocated for the caller and * need to be freed with free_contig_range(). */ -int alloc_contig_range(unsigned long start, unsigned long end, +int alloc_contig_range_noprof(unsigned long start, unsigned long end, unsigned migratetype, gfp_t gfp_mask) { unsigned long outer_start, outer_end; @@ -6467,15 +6467,15 @@ int alloc_contig_range(unsigned long start, unsigned long end, undo_isolate_page_range(start, end, migratetype); return ret; } -EXPORT_SYMBOL(alloc_contig_range); +EXPORT_SYMBOL(alloc_contig_range_noprof); static int __alloc_contig_pages(unsigned long start_pfn, unsigned long nr_pages, gfp_t gfp_mask) { unsigned long end_pfn = start_pfn + nr_pages; - return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE, - gfp_mask); + return alloc_contig_range_noprof(start_pfn, end_pfn, MIGRATE_MOVABLE, + gfp_mask); } static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn, @@ -6530,8 +6530,8 @@ static bool zone_spans_last_pfn(const struct zone *zone, * * Return: pointer to contiguous pages on success, or NULL if not successful. */ -struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, - int nid, nodemask_t *nodemask) +struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask, + int nid, nodemask_t *nodemask) { unsigned long ret, pfn, flags; struct zonelist *zonelist; From be25d1d4e822a703b19043baff916aec6ac4362d Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:41 -0700 Subject: [PATCH 0573/1702] mm: create new codetag references during page splitting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a high-order page is split into smaller ones, each newly split page should get its codetag. After the split each split page will be referencing the original codetag. The codetag's "bytes" counter remains the same because the amount of allocated memory has not changed, however the "calls" counter gets increased to keep the counter correct when these individual pages get freed. Link: https://lkml.kernel.org/r/20240321163705.3067592-20-surenb@google.com Signed-off-by: Suren Baghdasaryan Reviewed-by: Vlastimil Babka Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Kent Overstreet Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/alloc_tag.h | 9 +++++++++ include/linux/pgalloc_tag.h | 30 ++++++++++++++++++++++++++++++ mm/huge_memory.c | 2 ++ mm/page_alloc.c | 2 ++ 4 files changed, 43 insertions(+) diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h index 8932187124dab..2aea7afa1fab9 100644 --- a/include/linux/alloc_tag.h +++ b/include/linux/alloc_tag.h @@ -117,6 +117,15 @@ static inline void __alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag this_cpu_inc(tag->counters->calls); } +static inline void alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag) +{ + alloc_tag_add_check(ref, tag); + if (!ref || !tag) + return; + + __alloc_tag_ref_set(ref, tag); +} + static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes) { alloc_tag_add_check(ref, tag); diff --git a/include/linux/pgalloc_tag.h b/include/linux/pgalloc_tag.h index 66bd021eb46ea..093edf98c3d7e 100644 --- a/include/linux/pgalloc_tag.h +++ b/include/linux/pgalloc_tag.h @@ -67,11 +67,41 @@ static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) } } +static inline void pgalloc_tag_split(struct page *page, unsigned int nr) +{ + int i; + struct page_ext *page_ext; + union codetag_ref *ref; + struct alloc_tag *tag; + + if (!mem_alloc_profiling_enabled()) + return; + + page_ext = page_ext_get(page); + if (unlikely(!page_ext)) + return; + + ref = codetag_ref_from_page_ext(page_ext); + if (!ref->ct) + goto out; + + tag = ct_to_alloc_tag(ref->ct); + page_ext = page_ext_next(page_ext); + for (i = 1; i < nr; i++) { + /* Set new reference to point to the original tag */ + alloc_tag_ref_set(codetag_ref_from_page_ext(page_ext), tag); + page_ext = page_ext_next(page_ext); + } +out: + page_ext_put(page_ext); +} + #else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, unsigned int nr) {} static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {} +static inline void pgalloc_tag_split(struct page *page, unsigned int nr) {} #endif /* CONFIG_MEM_ALLOC_PROFILING */ diff --git a/mm/huge_memory.c b/mm/huge_memory.c index dd88bb3ea196e..25191ab70631b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -2946,6 +2947,7 @@ static void __split_huge_page(struct page *page, struct list_head *list, /* Caller disabled irqs, so they are still disabled here */ split_page_owner(head, order, new_order); + pgalloc_tag_split(head, 1 << order); /* See comment in __split_huge_page_tail() */ if (folio_test_anon(folio)) { diff --git a/mm/page_alloc.c b/mm/page_alloc.c index d3afaf8f983ab..e453ee22d4893 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2630,6 +2630,7 @@ void split_page(struct page *page, unsigned int order) for (i = 1; i < (1 << order); i++) set_page_refcounted(page + i); split_page_owner(page, order, 0); + pgalloc_tag_split(page, 1 << order); split_page_memcg(page, order, 0); } EXPORT_SYMBOL_GPL(split_page); @@ -4827,6 +4828,7 @@ static void *make_alloc_exact(unsigned long addr, unsigned int order, struct page *last = page + nr; split_page_owner(page, order, 0); + pgalloc_tag_split(page, 1 << order); split_page_memcg(page, order, 0); while (page < --last) set_page_refcounted(last); From cc92eba1c88b1f74e0f044df2738f4e4b22f1e4e Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:42 -0700 Subject: [PATCH 0574/1702] mm: fix non-compound multi-order memory accounting in __free_pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a non-compound multi-order page is freed, it is possible that a speculative reference keeps the page pinned. In this case we free all pages except for the first page, which will be freed later by the last put_page(). However the page passed to put_page() is indistinguishable from an order-0 page, so it cannot do the accounting, just as it cannot free the subsequent pages. Do the accounting here, where we free the pages. Link: https://lkml.kernel.org/r/20240321163705.3067592-21-surenb@google.com Reported-by: Vlastimil Babka Signed-off-by: Suren Baghdasaryan Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Kent Overstreet Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/pgalloc_tag.h | 24 ++++++++++++++++++++++++ mm/page_alloc.c | 5 ++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/include/linux/pgalloc_tag.h b/include/linux/pgalloc_tag.h index 093edf98c3d7e..50d212330bbbe 100644 --- a/include/linux/pgalloc_tag.h +++ b/include/linux/pgalloc_tag.h @@ -96,12 +96,36 @@ static inline void pgalloc_tag_split(struct page *page, unsigned int nr) page_ext_put(page_ext); } +static inline struct alloc_tag *pgalloc_tag_get(struct page *page) +{ + struct alloc_tag *tag = NULL; + + if (mem_alloc_profiling_enabled()) { + union codetag_ref *ref = get_page_tag_ref(page); + + alloc_tag_sub_check(ref); + if (ref && ref->ct) + tag = ct_to_alloc_tag(ref->ct); + put_page_tag_ref(ref); + } + + return tag; +} + +static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) +{ + if (mem_alloc_profiling_enabled() && tag) + this_cpu_sub(tag->counters->bytes, PAGE_SIZE * nr); +} + #else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, unsigned int nr) {} static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {} static inline void pgalloc_tag_split(struct page *page, unsigned int nr) {} +static inline struct alloc_tag *pgalloc_tag_get(struct page *page) { return NULL; } +static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {} #endif /* CONFIG_MEM_ALLOC_PROFILING */ diff --git a/mm/page_alloc.c b/mm/page_alloc.c index e453ee22d4893..e1241ecef2716 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4664,12 +4664,15 @@ void __free_pages(struct page *page, unsigned int order) { /* get PageHead before we drop reference */ int head = PageHead(page); + struct alloc_tag *tag = pgalloc_tag_get(page); if (put_page_testzero(page)) free_the_page(page, order); - else if (!head) + else if (!head) { + pgalloc_tag_sub_pages(tag, (1 << order) - 1); while (order-- > 0) free_the_page(page + (1 << order), order); + } } EXPORT_SYMBOL(__free_pages); From 26865a1bfae00cf7b060b6356016e3cf0cd13eff Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:43 -0700 Subject: [PATCH 0575/1702] mm/page_ext: enable early_page_ext when CONFIG_MEM_ALLOC_PROFILING_DEBUG=y MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For all page allocations to be tagged, page_ext has to be initialized before the first page allocation. Early tasks allocate their stacks using page allocator before alloc_node_page_ext() initializes page_ext area, unless early_page_ext is enabled. Therefore these allocations will generate a warning when CONFIG_MEM_ALLOC_PROFILING_DEBUG is enabled. Enable early_page_ext whenever CONFIG_MEM_ALLOC_PROFILING_DEBUG=y to ensure page_ext initialization prior to any page allocation. This will have all the negative effects associated with early_page_ext, such as possible longer boot time, therefore we enable it only when debugging with CONFIG_MEM_ALLOC_PROFILING_DEBUG enabled and not universally for CONFIG_MEM_ALLOC_PROFILING. Link: https://lkml.kernel.org/r/20240321163705.3067592-22-surenb@google.com Signed-off-by: Suren Baghdasaryan Reviewed-by: Vlastimil Babka Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Kent Overstreet Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- mm/page_ext.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mm/page_ext.c b/mm/page_ext.c index 3c58fe8a24df4..e7d8f1a5589ed 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -95,7 +95,16 @@ unsigned long page_ext_size; static unsigned long total_usage; +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG +/* + * To ensure correct allocation tagging for pages, page_ext should be available + * before the first page allocation. Otherwise early task stacks will be + * allocated before page_ext initialization and missing tags will be flagged. + */ +bool early_page_ext __meminitdata = true; +#else bool early_page_ext __meminitdata; +#endif static int __init setup_early_page_ext(char *str) { early_page_ext = true; From c789b5fe38f386bbdec8c9c41e9af52d3775e16c Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:44 -0700 Subject: [PATCH 0576/1702] lib: add codetag reference into slabobj_ext MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To store code tag for every slab object, a codetag reference is embedded into slabobj_ext when CONFIG_MEM_ALLOC_PROFILING=y. Link: https://lkml.kernel.org/r/20240321163705.3067592-23-surenb@google.com Signed-off-by: Suren Baghdasaryan Co-developed-by: Kent Overstreet Signed-off-by: Kent Overstreet Reviewed-by: Vlastimil Babka Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 5 +++++ lib/Kconfig.debug | 1 + 2 files changed, 6 insertions(+) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 12afc2647cf0e..24a6df30be49e 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1653,7 +1653,12 @@ unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order, * if MEMCG_DATA_OBJEXTS is set. */ struct slabobj_ext { +#ifdef CONFIG_MEMCG_KMEM struct obj_cgroup *objcg; +#endif +#ifdef CONFIG_MEM_ALLOC_PROFILING + union codetag_ref ref; +#endif } __aligned(8); static inline void __inc_lruvec_kmem_state(void *p, enum node_stat_item idx) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index a8f77de5e8ea8..9d2685ec45924 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -979,6 +979,7 @@ config MEM_ALLOC_PROFILING depends on !DEBUG_FORCE_WEAK_PER_CPU select CODE_TAGGING select PAGE_EXTENSION + select SLAB_OBJ_EXT help Track allocation source code and record total allocation size initiated at that code location. The mechanism can be used to track From 4b8736964640fe160724e7135dc62883bddcdace Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:45 -0700 Subject: [PATCH 0577/1702] mm/slab: add allocation accounting into slab allocation and free paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Account slab allocations using codetag reference embedded into slabobj_ext. Link: https://lkml.kernel.org/r/20240321163705.3067592-24-surenb@google.com Signed-off-by: Suren Baghdasaryan Co-developed-by: Kent Overstreet Signed-off-by: Kent Overstreet Reviewed-by: Kees Cook Reviewed-by: Vlastimil Babka Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- mm/slub.c | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/mm/slub.c b/mm/slub.c index 666dcc3b8a268..5840ab9633195 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1932,7 +1932,68 @@ static inline void free_slab_obj_exts(struct slab *slab) kfree(obj_exts); slab->obj_exts = 0; } + +static inline bool need_slab_obj_ext(void) +{ + if (mem_alloc_profiling_enabled()) + return true; + + /* + * CONFIG_MEMCG_KMEM creates vector of obj_cgroup objects conditionally + * inside memcg_slab_post_alloc_hook. No other users for now. + */ + return false; +} + +static inline struct slabobj_ext * +prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p) +{ + struct slab *slab; + + if (!p) + return NULL; + + if (s->flags & SLAB_NO_OBJ_EXT) + return NULL; + + if (flags & __GFP_NO_OBJ_EXT) + return NULL; + + slab = virt_to_slab(p); + if (!slab_obj_exts(slab) && + WARN(alloc_slab_obj_exts(slab, s, flags, false), + "%s, %s: Failed to create slab extension vector!\n", + __func__, s->name)) + return NULL; + + return slab_obj_exts(slab) + obj_to_index(s, slab, p); +} + +static inline void +alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, + int objects) +{ +#ifdef CONFIG_MEM_ALLOC_PROFILING + struct slabobj_ext *obj_exts; + int i; + + if (!mem_alloc_profiling_enabled()) + return; + + obj_exts = slab_obj_exts(slab); + if (!obj_exts) + return; + + for (i = 0; i < objects; i++) { + unsigned int off = obj_to_index(s, slab, p[i]); + + alloc_tag_sub(&obj_exts[off].ref, s->size); + } +#endif +} + #else /* CONFIG_SLAB_OBJ_EXT */ + static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, gfp_t gfp, bool new_slab) { @@ -1942,6 +2003,24 @@ static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, static inline void free_slab_obj_exts(struct slab *slab) { } + +static inline bool need_slab_obj_ext(void) +{ + return false; +} + +static inline struct slabobj_ext * +prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p) +{ + return NULL; +} + +static inline void +alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, + int objects) +{ +} + #endif /* CONFIG_SLAB_OBJ_EXT */ #ifdef CONFIG_MEMCG_KMEM @@ -2370,7 +2449,7 @@ static __always_inline void account_slab(struct slab *slab, int order, static __always_inline void unaccount_slab(struct slab *slab, int order, struct kmem_cache *s) { - if (memcg_kmem_online()) + if (memcg_kmem_online() || need_slab_obj_ext()) free_slab_obj_exts(slab); mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), @@ -3823,6 +3902,7 @@ void slab_post_alloc_hook(struct kmem_cache *s, struct obj_cgroup *objcg, unsigned int orig_size) { unsigned int zero_size = s->object_size; + struct slabobj_ext *obj_exts; bool kasan_init = init; size_t i; gfp_t init_flags = flags & gfp_allowed_mask; @@ -3865,6 +3945,18 @@ void slab_post_alloc_hook(struct kmem_cache *s, struct obj_cgroup *objcg, kmemleak_alloc_recursive(p[i], s->object_size, 1, s->flags, init_flags); kmsan_slab_alloc(s, p[i], init_flags); + if (need_slab_obj_ext()) { + obj_exts = prepare_slab_obj_exts_hook(s, flags, p[i]); +#ifdef CONFIG_MEM_ALLOC_PROFILING + /* + * Currently obj_exts is used only for allocation profiling. + * If other users appear then mem_alloc_profiling_enabled() + * check should be added before alloc_tag_add(). + */ + if (likely(obj_exts)) + alloc_tag_add(&obj_exts->ref, current->alloc_tag, s->size); +#endif + } } memcg_slab_post_alloc_hook(s, objcg, flags, size, p); @@ -4339,6 +4431,7 @@ void slab_free(struct kmem_cache *s, struct slab *slab, void *object, unsigned long addr) { memcg_slab_free_hook(s, slab, &object, 1); + alloc_tagging_slab_free_hook(s, slab, &object, 1); if (likely(slab_free_hook(s, object, slab_want_init_on_free(s)))) do_slab_free(s, slab, object, object, 1, addr); @@ -4349,6 +4442,7 @@ void slab_free_bulk(struct kmem_cache *s, struct slab *slab, void *head, void *tail, void **p, int cnt, unsigned long addr) { memcg_slab_free_hook(s, slab, p, cnt); + alloc_tagging_slab_free_hook(s, slab, p, cnt); /* * With KASAN enabled slab_free_freelist_hook modifies the freelist * to remove objects, whose reuse must be delayed. From 53ed0af4964229595b60594b35334d006d411ef0 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 21 Mar 2024 09:36:46 -0700 Subject: [PATCH 0578/1702] rust: add a rust helper for krealloc() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Memory allocation profiling is turning krealloc() into a nontrivial macro - so for now, we need a helper for it. Until we have proper support on the rust side for memory allocation profiling this does mean that all Rust allocations will be accounted to the helper. Link: https://lkml.kernel.org/r/20240321163705.3067592-25-surenb@google.com Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Reviewed-by: Alice Ryhl Acked-by: Miguel Ojeda Tested-by: Kees Cook Cc: Alex Gaynor Cc: Wedson Almeida Filho Cc: Boqun Feng Cc: Gary Guo Cc: "Björn Roy Baron" Cc: Benno Lossin Cc: Andreas Hindborg Cc: Alexander Viro Cc: Christoph Lameter Cc: Dennis Zhou Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- rust/helpers.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rust/helpers.c b/rust/helpers.c index 70e59efd92bc4..858d802abd115 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -157,6 +158,13 @@ void rust_helper_init_work_with_key(struct work_struct *work, work_func_t func, } EXPORT_SYMBOL_GPL(rust_helper_init_work_with_key); +void * __must_check __realloc_size(2) +rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags) +{ + return krealloc(objp, new_size, flags); +} +EXPORT_SYMBOL_GPL(rust_helper_krealloc); + /* * `bindgen` binds the C `size_t` type as the Rust `usize` type, so we can * use it in contexts where Rust expects a `usize` like slice (array) indices. From 7bd230a26648ac68ab3731ebbc449090f0ac6a37 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:47 -0700 Subject: [PATCH 0579/1702] mm/slab: enable slab allocation tagging for kmalloc and friends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Redefine kmalloc, krealloc, kzalloc, kcalloc, etc. to record allocations and deallocations done by these functions. [surenb@google.com: undo _noprof additions in the documentation] Link: https://lkml.kernel.org/r/20240326231453.1206227-7-surenb@google.com [rdunlap@infradead.org: fix kcalloc() kernel-doc warnings] Link: https://lkml.kernel.org/r/20240327044649.9199-1-rdunlap@infradead.org Link: https://lkml.kernel.org/r/20240321163705.3067592-26-surenb@google.com Signed-off-by: Suren Baghdasaryan Co-developed-by: Kent Overstreet Signed-off-by: Kent Overstreet Signed-off-by: Randy Dunlap Reviewed-by: Kees Cook Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/fortify-string.h | 5 +- include/linux/slab.h | 169 +++++++++++++++++---------------- include/linux/string.h | 4 +- mm/slab_common.c | 6 +- mm/slub.c | 50 +++++----- mm/util.c | 16 ++-- 6 files changed, 127 insertions(+), 123 deletions(-) diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h index 6aeebe0a67770..b24d62bad0b32 100644 --- a/include/linux/fortify-string.h +++ b/include/linux/fortify-string.h @@ -725,9 +725,9 @@ __FORTIFY_INLINE void *memchr_inv(const void * const POS0 p, int c, size_t size) return __real_memchr_inv(p, c, size); } -extern void *__real_kmemdup(const void *src, size_t len, gfp_t gfp) __RENAME(kmemdup) +extern void *__real_kmemdup(const void *src, size_t len, gfp_t gfp) __RENAME(kmemdup_noprof) __realloc_size(2); -__FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp) +__FORTIFY_INLINE void *kmemdup_noprof(const void * const POS0 p, size_t size, gfp_t gfp) { const size_t p_size = __struct_size(p); @@ -737,6 +737,7 @@ __FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp fortify_panic(FORTIFY_FUNC_kmemdup, FORTIFY_READ, p_size, size, NULL); return __real_kmemdup(p, size, gfp); } +#define kmemdup(...) alloc_hooks(kmemdup_noprof(__VA_ARGS__)) /** * strcpy - Copy a string into another string buffer diff --git a/include/linux/slab.h b/include/linux/slab.h index 68ff754b85a4a..c0be1cd03cf6a 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -271,7 +271,10 @@ int kmem_cache_shrink(struct kmem_cache *s); /* * Common kmalloc functions provided by all allocators */ -void * __must_check krealloc(const void *objp, size_t new_size, gfp_t flags) __realloc_size(2); +void * __must_check krealloc_noprof(const void *objp, size_t new_size, + gfp_t flags) __realloc_size(2); +#define krealloc(...) alloc_hooks(krealloc_noprof(__VA_ARGS__)) + void kfree(const void *objp); void kfree_sensitive(const void *objp); size_t __ksize(const void *objp); @@ -523,7 +526,10 @@ static __always_inline unsigned int __kmalloc_index(size_t size, static_assert(PAGE_SHIFT <= 20); #define kmalloc_index(s) __kmalloc_index(s, true) -void *__kmalloc(size_t size, gfp_t flags) __assume_kmalloc_alignment __alloc_size(1); +#include + +void *__kmalloc_noprof(size_t size, gfp_t flags) __assume_kmalloc_alignment __alloc_size(1); +#define __kmalloc(...) alloc_hooks(__kmalloc_noprof(__VA_ARGS__)) /** * kmem_cache_alloc - Allocate an object @@ -535,9 +541,14 @@ void *__kmalloc(size_t size, gfp_t flags) __assume_kmalloc_alignment __alloc_siz * * Return: pointer to the new object or %NULL in case of error */ -void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) __assume_slab_alignment __malloc; -void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru, - gfp_t gfpflags) __assume_slab_alignment __malloc; +void *kmem_cache_alloc_noprof(struct kmem_cache *cachep, + gfp_t flags) __assume_slab_alignment __malloc; +#define kmem_cache_alloc(...) alloc_hooks(kmem_cache_alloc_noprof(__VA_ARGS__)) + +void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru, + gfp_t gfpflags) __assume_slab_alignment __malloc; +#define kmem_cache_alloc_lru(...) alloc_hooks(kmem_cache_alloc_lru_noprof(__VA_ARGS__)) + void kmem_cache_free(struct kmem_cache *s, void *objp); /* @@ -548,29 +559,40 @@ void kmem_cache_free(struct kmem_cache *s, void *objp); * Note that interrupts must be enabled when calling these functions. */ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p); -int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, void **p); + +int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, void **p); +#define kmem_cache_alloc_bulk(...) alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__)) static __always_inline void kfree_bulk(size_t size, void **p) { kmem_cache_free_bulk(NULL, size, p); } -void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment +void *__kmalloc_node_noprof(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment __alloc_size(1); -void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) __assume_slab_alignment - __malloc; +#define __kmalloc_node(...) alloc_hooks(__kmalloc_node_noprof(__VA_ARGS__)) + +void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t flags, + int node) __assume_slab_alignment __malloc; +#define kmem_cache_alloc_node(...) alloc_hooks(kmem_cache_alloc_node_noprof(__VA_ARGS__)) -void *kmalloc_trace(struct kmem_cache *s, gfp_t flags, size_t size) +void *kmalloc_trace_noprof(struct kmem_cache *s, gfp_t flags, size_t size) __assume_kmalloc_alignment __alloc_size(3); -void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, - int node, size_t size) __assume_kmalloc_alignment +void *kmalloc_node_trace_noprof(struct kmem_cache *s, gfp_t gfpflags, + int node, size_t size) __assume_kmalloc_alignment __alloc_size(4); -void *kmalloc_large(size_t size, gfp_t flags) __assume_page_alignment +#define kmalloc_trace(...) alloc_hooks(kmalloc_trace_noprof(__VA_ARGS__)) + +#define kmalloc_node_trace(...) alloc_hooks(kmalloc_node_trace_noprof(__VA_ARGS__)) + +void *kmalloc_large_noprof(size_t size, gfp_t flags) __assume_page_alignment __alloc_size(1); +#define kmalloc_large(...) alloc_hooks(kmalloc_large_noprof(__VA_ARGS__)) -void *kmalloc_large_node(size_t size, gfp_t flags, int node) __assume_page_alignment +void *kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) __assume_page_alignment __alloc_size(1); +#define kmalloc_large_node(...) alloc_hooks(kmalloc_large_node_noprof(__VA_ARGS__)) /** * kmalloc - allocate kernel memory @@ -626,37 +648,39 @@ void *kmalloc_large_node(size_t size, gfp_t flags, int node) __assume_page_align * Try really hard to succeed the allocation but fail * eventually. */ -static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags) +static __always_inline __alloc_size(1) void *kmalloc_noprof(size_t size, gfp_t flags) { if (__builtin_constant_p(size) && size) { unsigned int index; if (size > KMALLOC_MAX_CACHE_SIZE) - return kmalloc_large(size, flags); + return kmalloc_large_noprof(size, flags); index = kmalloc_index(size); - return kmalloc_trace( + return kmalloc_trace_noprof( kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index], flags, size); } - return __kmalloc(size, flags); + return __kmalloc_noprof(size, flags); } +#define kmalloc(...) alloc_hooks(kmalloc_noprof(__VA_ARGS__)) -static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) +static __always_inline __alloc_size(1) void *kmalloc_node_noprof(size_t size, gfp_t flags, int node) { if (__builtin_constant_p(size) && size) { unsigned int index; if (size > KMALLOC_MAX_CACHE_SIZE) - return kmalloc_large_node(size, flags, node); + return kmalloc_large_node_noprof(size, flags, node); index = kmalloc_index(size); - return kmalloc_node_trace( + return kmalloc_node_trace_noprof( kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index], flags, node, size); } - return __kmalloc_node(size, flags, node); + return __kmalloc_node_noprof(size, flags, node); } +#define kmalloc_node(...) alloc_hooks(kmalloc_node_noprof(__VA_ARGS__)) /** * kmalloc_array - allocate memory for an array. @@ -664,16 +688,17 @@ static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t fla * @size: element size. * @flags: the type of memory to allocate (see kmalloc). */ -static inline __alloc_size(1, 2) void *kmalloc_array(size_t n, size_t size, gfp_t flags) +static inline __alloc_size(1, 2) void *kmalloc_array_noprof(size_t n, size_t size, gfp_t flags) { size_t bytes; if (unlikely(check_mul_overflow(n, size, &bytes))) return NULL; if (__builtin_constant_p(n) && __builtin_constant_p(size)) - return kmalloc(bytes, flags); - return __kmalloc(bytes, flags); + return kmalloc_noprof(bytes, flags); + return kmalloc_noprof(bytes, flags); } +#define kmalloc_array(...) alloc_hooks(kmalloc_array_noprof(__VA_ARGS__)) /** * krealloc_array - reallocate memory for an array. @@ -682,18 +707,19 @@ static inline __alloc_size(1, 2) void *kmalloc_array(size_t n, size_t size, gfp_ * @new_size: new size of a single member of the array * @flags: the type of memory to allocate (see kmalloc) */ -static inline __realloc_size(2, 3) void * __must_check krealloc_array(void *p, - size_t new_n, - size_t new_size, - gfp_t flags) +static inline __realloc_size(2, 3) void * __must_check krealloc_array_noprof(void *p, + size_t new_n, + size_t new_size, + gfp_t flags) { size_t bytes; if (unlikely(check_mul_overflow(new_n, new_size, &bytes))) return NULL; - return krealloc(p, bytes, flags); + return krealloc_noprof(p, bytes, flags); } +#define krealloc_array(...) alloc_hooks(krealloc_array_noprof(__VA_ARGS__)) /** * kcalloc - allocate memory for an array. The memory is set to zero. @@ -701,16 +727,12 @@ static inline __realloc_size(2, 3) void * __must_check krealloc_array(void *p, * @size: element size. * @flags: the type of memory to allocate (see kmalloc). */ -static inline __alloc_size(1, 2) void *kcalloc(size_t n, size_t size, gfp_t flags) -{ - return kmalloc_array(n, size, flags | __GFP_ZERO); -} +#define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO) -void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node, +void *kmalloc_node_track_caller_noprof(size_t size, gfp_t flags, int node, unsigned long caller) __alloc_size(1); -#define kmalloc_node_track_caller(size, flags, node) \ - __kmalloc_node_track_caller(size, flags, node, \ - _RET_IP_) +#define kmalloc_node_track_caller(...) \ + alloc_hooks(kmalloc_node_track_caller_noprof(__VA_ARGS__, _RET_IP_)) /* * kmalloc_track_caller is a special version of kmalloc that records the @@ -720,11 +742,9 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node, * allocator where we care about the real place the memory allocation * request comes from. */ -#define kmalloc_track_caller(size, flags) \ - __kmalloc_node_track_caller(size, flags, \ - NUMA_NO_NODE, _RET_IP_) +#define kmalloc_track_caller(...) kmalloc_node_track_caller(__VA_ARGS__, NUMA_NO_NODE) -static inline __alloc_size(1, 2) void *kmalloc_array_node(size_t n, size_t size, gfp_t flags, +static inline __alloc_size(1, 2) void *kmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node) { size_t bytes; @@ -732,75 +752,56 @@ static inline __alloc_size(1, 2) void *kmalloc_array_node(size_t n, size_t size, if (unlikely(check_mul_overflow(n, size, &bytes))) return NULL; if (__builtin_constant_p(n) && __builtin_constant_p(size)) - return kmalloc_node(bytes, flags, node); - return __kmalloc_node(bytes, flags, node); + return kmalloc_node_noprof(bytes, flags, node); + return __kmalloc_node_noprof(bytes, flags, node); } +#define kmalloc_array_node(...) alloc_hooks(kmalloc_array_node_noprof(__VA_ARGS__)) -static inline __alloc_size(1, 2) void *kcalloc_node(size_t n, size_t size, gfp_t flags, int node) -{ - return kmalloc_array_node(n, size, flags | __GFP_ZERO, node); -} +#define kcalloc_node(_n, _size, _flags, _node) \ + kmalloc_array_node(_n, _size, (_flags) | __GFP_ZERO, _node) /* * Shortcuts */ -static inline void *kmem_cache_zalloc(struct kmem_cache *k, gfp_t flags) -{ - return kmem_cache_alloc(k, flags | __GFP_ZERO); -} +#define kmem_cache_zalloc(_k, _flags) kmem_cache_alloc(_k, (_flags)|__GFP_ZERO) /** * kzalloc - allocate memory. The memory is set to zero. * @size: how many bytes of memory are required. * @flags: the type of memory to allocate (see kmalloc). */ -static inline __alloc_size(1) void *kzalloc(size_t size, gfp_t flags) +static inline __alloc_size(1) void *kzalloc_noprof(size_t size, gfp_t flags) { - return kmalloc(size, flags | __GFP_ZERO); + return kmalloc_noprof(size, flags | __GFP_ZERO); } +#define kzalloc(...) alloc_hooks(kzalloc_noprof(__VA_ARGS__)) +#define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__GFP_ZERO, _node) -/** - * kzalloc_node - allocate zeroed memory from a particular memory node. - * @size: how many bytes of memory are required. - * @flags: the type of memory to allocate (see kmalloc). - * @node: memory node from which to allocate - */ -static inline __alloc_size(1) void *kzalloc_node(size_t size, gfp_t flags, int node) -{ - return kmalloc_node(size, flags | __GFP_ZERO, node); -} +extern void *kvmalloc_node_noprof(size_t size, gfp_t flags, int node) __alloc_size(1); +#define kvmalloc_node(...) alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__)) -extern void *kvmalloc_node(size_t size, gfp_t flags, int node) __alloc_size(1); -static inline __alloc_size(1) void *kvmalloc(size_t size, gfp_t flags) -{ - return kvmalloc_node(size, flags, NUMA_NO_NODE); -} -static inline __alloc_size(1) void *kvzalloc_node(size_t size, gfp_t flags, int node) -{ - return kvmalloc_node(size, flags | __GFP_ZERO, node); -} -static inline __alloc_size(1) void *kvzalloc(size_t size, gfp_t flags) -{ - return kvmalloc(size, flags | __GFP_ZERO); -} +#define kvmalloc(_size, _flags) kvmalloc_node(_size, _flags, NUMA_NO_NODE) +#define kvzalloc(_size, _flags) kvmalloc(_size, _flags|__GFP_ZERO) + +#define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, _flags|__GFP_ZERO, _node) -static inline __alloc_size(1, 2) void *kvmalloc_array(size_t n, size_t size, gfp_t flags) +static inline __alloc_size(1, 2) void *kvmalloc_array_noprof(size_t n, size_t size, gfp_t flags) { size_t bytes; if (unlikely(check_mul_overflow(n, size, &bytes))) return NULL; - return kvmalloc(bytes, flags); + return kvmalloc_node_noprof(bytes, flags, NUMA_NO_NODE); } -static inline __alloc_size(1, 2) void *kvcalloc(size_t n, size_t size, gfp_t flags) -{ - return kvmalloc_array(n, size, flags | __GFP_ZERO); -} +#define kvmalloc_array(...) alloc_hooks(kvmalloc_array_noprof(__VA_ARGS__)) +#define kvcalloc(_n, _size, _flags) kvmalloc_array(_n, _size, _flags|__GFP_ZERO) -extern void *kvrealloc(const void *p, size_t oldsize, size_t newsize, gfp_t flags) +extern void *kvrealloc_noprof(const void *p, size_t oldsize, size_t newsize, gfp_t flags) __realloc_size(3); +#define kvrealloc(...) alloc_hooks(kvrealloc_noprof(__VA_ARGS__)) + extern void kvfree(const void *addr); DEFINE_FREE(kvfree, void *, if (_T) kvfree(_T)) diff --git a/include/linux/string.h b/include/linux/string.h index 9ba8b4597009c..793c27ad7c0d2 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -282,7 +282,9 @@ extern void kfree_const(const void *x); extern char *kstrdup(const char *s, gfp_t gfp) __malloc; extern const char *kstrdup_const(const char *s, gfp_t gfp); extern char *kstrndup(const char *s, size_t len, gfp_t gfp); -extern void *kmemdup(const void *src, size_t len, gfp_t gfp) __realloc_size(2); +extern void *kmemdup_noprof(const void *src, size_t len, gfp_t gfp) __realloc_size(2); +#define kmemdup(...) alloc_hooks(kmemdup_noprof(__VA_ARGS__)) + extern void *kvmemdup(const void *src, size_t len, gfp_t gfp) __realloc_size(2); extern char *kmemdup_nul(const char *s, size_t len, gfp_t gfp); extern void *kmemdup_array(const void *src, size_t element_size, size_t count, gfp_t gfp); diff --git a/mm/slab_common.c b/mm/slab_common.c index f5234672f03ce..3179a6aeffc56 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1189,7 +1189,7 @@ __do_krealloc(const void *p, size_t new_size, gfp_t flags) return (void *)p; } - ret = kmalloc_track_caller(new_size, flags); + ret = kmalloc_node_track_caller_noprof(new_size, flags, NUMA_NO_NODE, _RET_IP_); if (ret && p) { /* Disable KASAN checks as the object's redzone is accessed. */ kasan_disable_current(); @@ -1213,7 +1213,7 @@ __do_krealloc(const void *p, size_t new_size, gfp_t flags) * * Return: pointer to the allocated memory or %NULL in case of error */ -void *krealloc(const void *p, size_t new_size, gfp_t flags) +void *krealloc_noprof(const void *p, size_t new_size, gfp_t flags) { void *ret; @@ -1228,7 +1228,7 @@ void *krealloc(const void *p, size_t new_size, gfp_t flags) return ret; } -EXPORT_SYMBOL(krealloc); +EXPORT_SYMBOL(krealloc_noprof); /** * kfree_sensitive - Clear sensitive information in memory before freeing diff --git a/mm/slub.c b/mm/slub.c index 5840ab9633195..0fc891d44cffa 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4002,7 +4002,7 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list return object; } -void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) +void *kmem_cache_alloc_noprof(struct kmem_cache *s, gfp_t gfpflags) { void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_, s->object_size); @@ -4011,9 +4011,9 @@ void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) return ret; } -EXPORT_SYMBOL(kmem_cache_alloc); +EXPORT_SYMBOL(kmem_cache_alloc_noprof); -void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru, +void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru, gfp_t gfpflags) { void *ret = slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _RET_IP_, @@ -4023,7 +4023,7 @@ void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru, return ret; } -EXPORT_SYMBOL(kmem_cache_alloc_lru); +EXPORT_SYMBOL(kmem_cache_alloc_lru_noprof); /** * kmem_cache_alloc_node - Allocate an object on the specified node @@ -4038,7 +4038,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_lru); * * Return: pointer to the new object or %NULL in case of error */ -void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) +void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node) { void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size); @@ -4046,7 +4046,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) return ret; } -EXPORT_SYMBOL(kmem_cache_alloc_node); +EXPORT_SYMBOL(kmem_cache_alloc_node_noprof); /* * To avoid unnecessary overhead, we pass through large allocation requests @@ -4063,7 +4063,7 @@ static void *__kmalloc_large_node(size_t size, gfp_t flags, int node) flags = kmalloc_fix_flags(flags); flags |= __GFP_COMP; - folio = (struct folio *)alloc_pages_node(node, flags, order); + folio = (struct folio *)alloc_pages_node_noprof(node, flags, order); if (folio) { ptr = folio_address(folio); lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, @@ -4078,7 +4078,7 @@ static void *__kmalloc_large_node(size_t size, gfp_t flags, int node) return ptr; } -void *kmalloc_large(size_t size, gfp_t flags) +void *kmalloc_large_noprof(size_t size, gfp_t flags) { void *ret = __kmalloc_large_node(size, flags, NUMA_NO_NODE); @@ -4086,9 +4086,9 @@ void *kmalloc_large(size_t size, gfp_t flags) flags, NUMA_NO_NODE); return ret; } -EXPORT_SYMBOL(kmalloc_large); +EXPORT_SYMBOL(kmalloc_large_noprof); -void *kmalloc_large_node(size_t size, gfp_t flags, int node) +void *kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) { void *ret = __kmalloc_large_node(size, flags, node); @@ -4096,7 +4096,7 @@ void *kmalloc_large_node(size_t size, gfp_t flags, int node) flags, node); return ret; } -EXPORT_SYMBOL(kmalloc_large_node); +EXPORT_SYMBOL(kmalloc_large_node_noprof); static __always_inline void *__do_kmalloc_node(size_t size, gfp_t flags, int node, @@ -4123,26 +4123,26 @@ void *__do_kmalloc_node(size_t size, gfp_t flags, int node, return ret; } -void *__kmalloc_node(size_t size, gfp_t flags, int node) +void *__kmalloc_node_noprof(size_t size, gfp_t flags, int node) { return __do_kmalloc_node(size, flags, node, _RET_IP_); } -EXPORT_SYMBOL(__kmalloc_node); +EXPORT_SYMBOL(__kmalloc_node_noprof); -void *__kmalloc(size_t size, gfp_t flags) +void *__kmalloc_noprof(size_t size, gfp_t flags) { return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_); } -EXPORT_SYMBOL(__kmalloc); +EXPORT_SYMBOL(__kmalloc_noprof); -void *__kmalloc_node_track_caller(size_t size, gfp_t flags, - int node, unsigned long caller) +void *kmalloc_node_track_caller_noprof(size_t size, gfp_t flags, + int node, unsigned long caller) { return __do_kmalloc_node(size, flags, node, caller); } -EXPORT_SYMBOL(__kmalloc_node_track_caller); +EXPORT_SYMBOL(kmalloc_node_track_caller_noprof); -void *kmalloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) +void *kmalloc_trace_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size) { void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_, size); @@ -4152,9 +4152,9 @@ void *kmalloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) ret = kasan_kmalloc(s, ret, size, gfpflags); return ret; } -EXPORT_SYMBOL(kmalloc_trace); +EXPORT_SYMBOL(kmalloc_trace_noprof); -void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, +void *kmalloc_node_trace_noprof(struct kmem_cache *s, gfp_t gfpflags, int node, size_t size) { void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size); @@ -4164,7 +4164,7 @@ void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, ret = kasan_kmalloc(s, ret, size, gfpflags); return ret; } -EXPORT_SYMBOL(kmalloc_node_trace); +EXPORT_SYMBOL(kmalloc_node_trace_noprof); static noinline void free_to_partial_list( struct kmem_cache *s, struct slab *slab, @@ -4769,8 +4769,8 @@ static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, #endif /* CONFIG_SLUB_TINY */ /* Note that interrupts must be enabled when calling this function. */ -int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, - void **p) +int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, + void **p) { int i; struct obj_cgroup *objcg = NULL; @@ -4798,7 +4798,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, return i; } -EXPORT_SYMBOL(kmem_cache_alloc_bulk); +EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof); /* diff --git a/mm/util.c b/mm/util.c index 669397235787b..cee3563d2071b 100644 --- a/mm/util.c +++ b/mm/util.c @@ -124,16 +124,16 @@ EXPORT_SYMBOL(kstrndup); * Return: newly allocated copy of @src or %NULL in case of error, * result is physically contiguous. Use kfree() to free. */ -void *kmemdup(const void *src, size_t len, gfp_t gfp) +void *kmemdup_noprof(const void *src, size_t len, gfp_t gfp) { void *p; - p = kmalloc_track_caller(len, gfp); + p = kmalloc_node_track_caller_noprof(len, gfp, NUMA_NO_NODE, _RET_IP_); if (p) memcpy(p, src, len); return p; } -EXPORT_SYMBOL(kmemdup); +EXPORT_SYMBOL(kmemdup_noprof); /** * kmemdup_array - duplicate a given array. @@ -609,7 +609,7 @@ EXPORT_SYMBOL(vm_mmap); * * Return: pointer to the allocated memory of %NULL in case of failure */ -void *kvmalloc_node(size_t size, gfp_t flags, int node) +void *kvmalloc_node_noprof(size_t size, gfp_t flags, int node) { gfp_t kmalloc_flags = flags; void *ret; @@ -631,7 +631,7 @@ void *kvmalloc_node(size_t size, gfp_t flags, int node) kmalloc_flags &= ~__GFP_NOFAIL; } - ret = kmalloc_node(size, kmalloc_flags, node); + ret = kmalloc_node_noprof(size, kmalloc_flags, node); /* * It doesn't really make sense to fallback to vmalloc for sub page @@ -660,7 +660,7 @@ void *kvmalloc_node(size_t size, gfp_t flags, int node) flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, node, __builtin_return_address(0)); } -EXPORT_SYMBOL(kvmalloc_node); +EXPORT_SYMBOL(kvmalloc_node_noprof); /** * kvfree() - Free memory. @@ -699,7 +699,7 @@ void kvfree_sensitive(const void *addr, size_t len) } EXPORT_SYMBOL(kvfree_sensitive); -void *kvrealloc(const void *p, size_t oldsize, size_t newsize, gfp_t flags) +void *kvrealloc_noprof(const void *p, size_t oldsize, size_t newsize, gfp_t flags) { void *newp; @@ -712,7 +712,7 @@ void *kvrealloc(const void *p, size_t oldsize, size_t newsize, gfp_t flags) kvfree(p); return newp; } -EXPORT_SYMBOL(kvrealloc); +EXPORT_SYMBOL(kvrealloc_noprof); /** * __vmalloc_array - allocate memory for a virtually contiguous array. From e26d8769da6d4c7319fdd4ddb497caa7d87a6c5c Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 21 Mar 2024 09:36:48 -0700 Subject: [PATCH 0580/1702] mempool: hook up to memory allocation profiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds hooks to mempools for correctly annotating mempool-backed allocations at the correct source line, so they show up correctly in /sys/kernel/debug/allocations. Various inline functions are converted to wrappers so that we can invoke alloc_hooks() in fewer places. [surenb@google.com: undo _noprof additions in the documentation] Link: https://lkml.kernel.org/r/20240326231453.1206227-4-surenb@google.com [surenb@google.com: add missing mempool_create_node documentation] Link: https://lkml.kernel.org/r/20240402180835.1661905-1-surenb@google.com Link: https://lkml.kernel.org/r/20240321163705.3067592-27-surenb@google.com Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/mempool.h | 73 ++++++++++++++++++++--------------------- mm/mempool.c | 36 +++++++++----------- 2 files changed, 50 insertions(+), 59 deletions(-) diff --git a/include/linux/mempool.h b/include/linux/mempool.h index 16c5cc807ff6b..7b151441341b4 100644 --- a/include/linux/mempool.h +++ b/include/linux/mempool.h @@ -5,6 +5,8 @@ #ifndef _LINUX_MEMPOOL_H #define _LINUX_MEMPOOL_H +#include +#include #include #include @@ -39,18 +41,32 @@ void mempool_exit(mempool_t *pool); int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn, mempool_free_t *free_fn, void *pool_data, gfp_t gfp_mask, int node_id); -int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn, + +int mempool_init_noprof(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn, mempool_free_t *free_fn, void *pool_data); +#define mempool_init(...) \ + alloc_hooks(mempool_init_noprof(__VA_ARGS__)) extern mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn, mempool_free_t *free_fn, void *pool_data); -extern mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn, + +extern mempool_t *mempool_create_node_noprof(int min_nr, mempool_alloc_t *alloc_fn, mempool_free_t *free_fn, void *pool_data, gfp_t gfp_mask, int nid); +#define mempool_create_node(...) \ + alloc_hooks(mempool_create_node_noprof(__VA_ARGS__)) + +#define mempool_create(_min_nr, _alloc_fn, _free_fn, _pool_data) \ + mempool_create_node(_min_nr, _alloc_fn, _free_fn, _pool_data, \ + GFP_KERNEL, NUMA_NO_NODE) extern int mempool_resize(mempool_t *pool, int new_min_nr); extern void mempool_destroy(mempool_t *pool); -extern void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask) __malloc; + +extern void *mempool_alloc_noprof(mempool_t *pool, gfp_t gfp_mask) __malloc; +#define mempool_alloc(...) \ + alloc_hooks(mempool_alloc_noprof(__VA_ARGS__)) + extern void *mempool_alloc_preallocated(mempool_t *pool) __malloc; extern void mempool_free(void *element, mempool_t *pool); @@ -62,19 +78,10 @@ extern void mempool_free(void *element, mempool_t *pool); void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data); void mempool_free_slab(void *element, void *pool_data); -static inline int -mempool_init_slab_pool(mempool_t *pool, int min_nr, struct kmem_cache *kc) -{ - return mempool_init(pool, min_nr, mempool_alloc_slab, - mempool_free_slab, (void *) kc); -} - -static inline mempool_t * -mempool_create_slab_pool(int min_nr, struct kmem_cache *kc) -{ - return mempool_create(min_nr, mempool_alloc_slab, mempool_free_slab, - (void *) kc); -} +#define mempool_init_slab_pool(_pool, _min_nr, _kc) \ + mempool_init(_pool, (_min_nr), mempool_alloc_slab, mempool_free_slab, (void *)(_kc)) +#define mempool_create_slab_pool(_min_nr, _kc) \ + mempool_create((_min_nr), mempool_alloc_slab, mempool_free_slab, (void *)(_kc)) /* * a mempool_alloc_t and a mempool_free_t to kmalloc and kfree the @@ -83,17 +90,12 @@ mempool_create_slab_pool(int min_nr, struct kmem_cache *kc) void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data); void mempool_kfree(void *element, void *pool_data); -static inline int mempool_init_kmalloc_pool(mempool_t *pool, int min_nr, size_t size) -{ - return mempool_init(pool, min_nr, mempool_kmalloc, - mempool_kfree, (void *) size); -} - -static inline mempool_t *mempool_create_kmalloc_pool(int min_nr, size_t size) -{ - return mempool_create(min_nr, mempool_kmalloc, mempool_kfree, - (void *) size); -} +#define mempool_init_kmalloc_pool(_pool, _min_nr, _size) \ + mempool_init(_pool, (_min_nr), mempool_kmalloc, mempool_kfree, \ + (void *)(unsigned long)(_size)) +#define mempool_create_kmalloc_pool(_min_nr, _size) \ + mempool_create((_min_nr), mempool_kmalloc, mempool_kfree, \ + (void *)(unsigned long)(_size)) void *mempool_kvmalloc(gfp_t gfp_mask, void *pool_data); void mempool_kvfree(void *element, void *pool_data); @@ -115,16 +117,11 @@ static inline mempool_t *mempool_create_kvmalloc_pool(int min_nr, size_t size) void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data); void mempool_free_pages(void *element, void *pool_data); -static inline int mempool_init_page_pool(mempool_t *pool, int min_nr, int order) -{ - return mempool_init(pool, min_nr, mempool_alloc_pages, - mempool_free_pages, (void *)(long)order); -} - -static inline mempool_t *mempool_create_page_pool(int min_nr, int order) -{ - return mempool_create(min_nr, mempool_alloc_pages, mempool_free_pages, - (void *)(long)order); -} +#define mempool_init_page_pool(_pool, _min_nr, _order) \ + mempool_init(_pool, (_min_nr), mempool_alloc_pages, \ + mempool_free_pages, (void *)(long)(_order)) +#define mempool_create_page_pool(_min_nr, _order) \ + mempool_create((_min_nr), mempool_alloc_pages, \ + mempool_free_pages, (void *)(long)(_order)) #endif /* _LINUX_MEMPOOL_H */ diff --git a/mm/mempool.c b/mm/mempool.c index 076c736f5f1ff..6ece63a00acf5 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -240,22 +240,24 @@ EXPORT_SYMBOL(mempool_init_node); * * Return: %0 on success, negative error code otherwise. */ -int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn, - mempool_free_t *free_fn, void *pool_data) +int mempool_init_noprof(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn, + mempool_free_t *free_fn, void *pool_data) { return mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data, GFP_KERNEL, NUMA_NO_NODE); } -EXPORT_SYMBOL(mempool_init); +EXPORT_SYMBOL(mempool_init_noprof); /** - * mempool_create - create a memory pool + * mempool_create_node - create a memory pool * @min_nr: the minimum number of elements guaranteed to be * allocated for this pool. * @alloc_fn: user-defined element-allocation function. * @free_fn: user-defined element-freeing function. * @pool_data: optional private data available to the user-defined functions. + * @gfp_mask: memory allocation flags + * @node_id: numa node to allocate on * * this function creates and allocates a guaranteed size, preallocated * memory pool. The pool can be used from the mempool_alloc() and mempool_free() @@ -265,17 +267,9 @@ EXPORT_SYMBOL(mempool_init); * * Return: pointer to the created memory pool object or %NULL on error. */ -mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn, - mempool_free_t *free_fn, void *pool_data) -{ - return mempool_create_node(min_nr, alloc_fn, free_fn, pool_data, - GFP_KERNEL, NUMA_NO_NODE); -} -EXPORT_SYMBOL(mempool_create); - -mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn, - mempool_free_t *free_fn, void *pool_data, - gfp_t gfp_mask, int node_id) +mempool_t *mempool_create_node_noprof(int min_nr, mempool_alloc_t *alloc_fn, + mempool_free_t *free_fn, void *pool_data, + gfp_t gfp_mask, int node_id) { mempool_t *pool; @@ -291,7 +285,7 @@ mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn, return pool; } -EXPORT_SYMBOL(mempool_create_node); +EXPORT_SYMBOL(mempool_create_node_noprof); /** * mempool_resize - resize an existing memory pool @@ -387,7 +381,7 @@ EXPORT_SYMBOL(mempool_resize); * * Return: pointer to the allocated element or %NULL on error. */ -void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask) +void *mempool_alloc_noprof(mempool_t *pool, gfp_t gfp_mask) { void *element; unsigned long flags; @@ -454,7 +448,7 @@ void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask) finish_wait(&pool->wait, &wait); goto repeat_alloc; } -EXPORT_SYMBOL(mempool_alloc); +EXPORT_SYMBOL(mempool_alloc_noprof); /** * mempool_alloc_preallocated - allocate an element from preallocated elements @@ -562,7 +556,7 @@ void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data) { struct kmem_cache *mem = pool_data; VM_BUG_ON(mem->ctor); - return kmem_cache_alloc(mem, gfp_mask); + return kmem_cache_alloc_noprof(mem, gfp_mask); } EXPORT_SYMBOL(mempool_alloc_slab); @@ -580,7 +574,7 @@ EXPORT_SYMBOL(mempool_free_slab); void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data) { size_t size = (size_t)pool_data; - return kmalloc(size, gfp_mask); + return kmalloc_noprof(size, gfp_mask); } EXPORT_SYMBOL(mempool_kmalloc); @@ -610,7 +604,7 @@ EXPORT_SYMBOL(mempool_kvfree); void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data) { int order = (int)(long)pool_data; - return alloc_pages(gfp_mask, order); + return alloc_pages_noprof(gfp_mask, order); } EXPORT_SYMBOL(mempool_alloc_pages); From 8f30d2660a38bef82af039065bd383fa73b35e39 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 21 Mar 2024 09:36:49 -0700 Subject: [PATCH 0581/1702] mm: percpu: introduce pcpuobj_ext MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upcoming alloc tagging patches require a place to stash per-allocation metadata. We already do this when memcg is enabled, so this patch generalizes the obj_cgroup * vector in struct pcpu_chunk by creating a pcpu_obj_ext type, which we will be adding to in an upcoming patch - similarly to the previous slabobj_ext patch. Link: https://lkml.kernel.org/r/20240321163705.3067592-28-surenb@google.com Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Tested-by: Kees Cook Cc: Dennis Zhou Cc: Tejun Heo Cc: Christoph Lameter Cc: linux-mm@kvack.org Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- mm/percpu-internal.h | 19 +++++++++++++++++-- mm/percpu.c | 30 +++++++++++++++--------------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/mm/percpu-internal.h b/mm/percpu-internal.h index cdd0aa597a819..e62d582f4bf31 100644 --- a/mm/percpu-internal.h +++ b/mm/percpu-internal.h @@ -32,6 +32,16 @@ struct pcpu_block_md { int nr_bits; /* total bits responsible for */ }; +struct pcpuobj_ext { +#ifdef CONFIG_MEMCG_KMEM + struct obj_cgroup *cgroup; +#endif +}; + +#ifdef CONFIG_MEMCG_KMEM +#define NEED_PCPUOBJ_EXT +#endif + struct pcpu_chunk { #ifdef CONFIG_PERCPU_STATS int nr_alloc; /* # of allocations */ @@ -64,8 +74,8 @@ struct pcpu_chunk { int end_offset; /* additional area required to have the region end page aligned */ -#ifdef CONFIG_MEMCG_KMEM - struct obj_cgroup **obj_cgroups; /* vector of object cgroups */ +#ifdef NEED_PCPUOBJ_EXT + struct pcpuobj_ext *obj_exts; /* vector of object cgroups */ #endif int nr_pages; /* # of pages served by this chunk */ @@ -74,6 +84,11 @@ struct pcpu_chunk { unsigned long populated[]; /* populated bitmap */ }; +static inline bool need_pcpuobj_ext(void) +{ + return !mem_cgroup_kmem_disabled(); +} + extern spinlock_t pcpu_lock; extern struct list_head *pcpu_chunk_lists; diff --git a/mm/percpu.c b/mm/percpu.c index 4e11fc1e6deff..2e5edaad9cc3e 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -1392,9 +1392,9 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr, panic("%s: Failed to allocate %zu bytes\n", __func__, alloc_size); -#ifdef CONFIG_MEMCG_KMEM +#ifdef NEED_PCPUOBJ_EXT /* first chunk is free to use */ - chunk->obj_cgroups = NULL; + chunk->obj_exts = NULL; #endif pcpu_init_md_blocks(chunk); @@ -1463,12 +1463,12 @@ static struct pcpu_chunk *pcpu_alloc_chunk(gfp_t gfp) if (!chunk->md_blocks) goto md_blocks_fail; -#ifdef CONFIG_MEMCG_KMEM - if (!mem_cgroup_kmem_disabled()) { - chunk->obj_cgroups = +#ifdef NEED_PCPUOBJ_EXT + if (need_pcpuobj_ext()) { + chunk->obj_exts = pcpu_mem_zalloc(pcpu_chunk_map_bits(chunk) * - sizeof(struct obj_cgroup *), gfp); - if (!chunk->obj_cgroups) + sizeof(struct pcpuobj_ext), gfp); + if (!chunk->obj_exts) goto objcg_fail; } #endif @@ -1480,7 +1480,7 @@ static struct pcpu_chunk *pcpu_alloc_chunk(gfp_t gfp) return chunk; -#ifdef CONFIG_MEMCG_KMEM +#ifdef NEED_PCPUOBJ_EXT objcg_fail: pcpu_mem_free(chunk->md_blocks); #endif @@ -1498,8 +1498,8 @@ static void pcpu_free_chunk(struct pcpu_chunk *chunk) { if (!chunk) return; -#ifdef CONFIG_MEMCG_KMEM - pcpu_mem_free(chunk->obj_cgroups); +#ifdef NEED_PCPUOBJ_EXT + pcpu_mem_free(chunk->obj_exts); #endif pcpu_mem_free(chunk->md_blocks); pcpu_mem_free(chunk->bound_map); @@ -1646,9 +1646,9 @@ static void pcpu_memcg_post_alloc_hook(struct obj_cgroup *objcg, if (!objcg) return; - if (likely(chunk && chunk->obj_cgroups)) { + if (likely(chunk && chunk->obj_exts)) { obj_cgroup_get(objcg); - chunk->obj_cgroups[off >> PCPU_MIN_ALLOC_SHIFT] = objcg; + chunk->obj_exts[off >> PCPU_MIN_ALLOC_SHIFT].cgroup = objcg; rcu_read_lock(); mod_memcg_state(obj_cgroup_memcg(objcg), MEMCG_PERCPU_B, @@ -1663,13 +1663,13 @@ static void pcpu_memcg_free_hook(struct pcpu_chunk *chunk, int off, size_t size) { struct obj_cgroup *objcg; - if (unlikely(!chunk->obj_cgroups)) + if (unlikely(!chunk->obj_exts)) return; - objcg = chunk->obj_cgroups[off >> PCPU_MIN_ALLOC_SHIFT]; + objcg = chunk->obj_exts[off >> PCPU_MIN_ALLOC_SHIFT].cgroup; if (!objcg) return; - chunk->obj_cgroups[off >> PCPU_MIN_ALLOC_SHIFT] = NULL; + chunk->obj_exts[off >> PCPU_MIN_ALLOC_SHIFT].cgroup = NULL; obj_cgroup_uncharge(objcg, pcpu_obj_full_size(size)); From 60fa4a9e232317217aaf8bb956079f6b8c79fb28 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 21 Mar 2024 09:36:50 -0700 Subject: [PATCH 0582/1702] mm: percpu: add codetag reference into pcpuobj_ext MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To store codetag for every per-cpu allocation, a codetag reference is embedded into pcpuobj_ext when CONFIG_MEM_ALLOC_PROFILING=y. Hooks to use the newly introduced codetag are added. Link: https://lkml.kernel.org/r/20240321163705.3067592-29-surenb@google.com Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- mm/percpu-internal.h | 11 +++++++++-- mm/percpu.c | 26 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/mm/percpu-internal.h b/mm/percpu-internal.h index e62d582f4bf31..7e42f0ca3b7bb 100644 --- a/mm/percpu-internal.h +++ b/mm/percpu-internal.h @@ -36,9 +36,12 @@ struct pcpuobj_ext { #ifdef CONFIG_MEMCG_KMEM struct obj_cgroup *cgroup; #endif +#ifdef CONFIG_MEM_ALLOC_PROFILING + union codetag_ref tag; +#endif }; -#ifdef CONFIG_MEMCG_KMEM +#if defined(CONFIG_MEMCG_KMEM) || defined(CONFIG_MEM_ALLOC_PROFILING) #define NEED_PCPUOBJ_EXT #endif @@ -86,7 +89,11 @@ struct pcpu_chunk { static inline bool need_pcpuobj_ext(void) { - return !mem_cgroup_kmem_disabled(); + if (IS_ENABLED(CONFIG_MEM_ALLOC_PROFILING)) + return true; + if (!mem_cgroup_kmem_disabled()) + return true; + return false; } extern spinlock_t pcpu_lock; diff --git a/mm/percpu.c b/mm/percpu.c index 2e5edaad9cc3e..90e9e4004ac97 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -1699,6 +1699,32 @@ static void pcpu_memcg_free_hook(struct pcpu_chunk *chunk, int off, size_t size) } #endif /* CONFIG_MEMCG_KMEM */ +#ifdef CONFIG_MEM_ALLOC_PROFILING +static void pcpu_alloc_tag_alloc_hook(struct pcpu_chunk *chunk, int off, + size_t size) +{ + if (mem_alloc_profiling_enabled() && likely(chunk->obj_exts)) { + alloc_tag_add(&chunk->obj_exts[off >> PCPU_MIN_ALLOC_SHIFT].tag, + current->alloc_tag, size); + } +} + +static void pcpu_alloc_tag_free_hook(struct pcpu_chunk *chunk, int off, size_t size) +{ + if (mem_alloc_profiling_enabled() && likely(chunk->obj_exts)) + alloc_tag_sub(&chunk->obj_exts[off >> PCPU_MIN_ALLOC_SHIFT].tag, size); +} +#else +static void pcpu_alloc_tag_alloc_hook(struct pcpu_chunk *chunk, int off, + size_t size) +{ +} + +static void pcpu_alloc_tag_free_hook(struct pcpu_chunk *chunk, int off, size_t size) +{ +} +#endif + /** * pcpu_alloc - the percpu allocator * @size: size of area to allocate in bytes From 24e44cc22aa3112082f2ee23137d048c73ca96d5 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:51 -0700 Subject: [PATCH 0583/1702] mm: percpu: enable per-cpu allocation tagging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Redefine __alloc_percpu, __alloc_percpu_gfp and __alloc_reserved_percpu to record allocations and deallocations done by these functions. [surenb@google.com: undo _noprof additions in the documentation] Link: https://lkml.kernel.org/r/20240326231453.1206227-6-surenb@google.com Link: https://lkml.kernel.org/r/20240321163705.3067592-30-surenb@google.com Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/percpu.h | 23 +++++++++++----- mm/percpu.c | 62 ++++-------------------------------------- 2 files changed, 22 insertions(+), 63 deletions(-) diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 9dfa7b526e70c..13a82f11e4fd2 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -2,6 +2,7 @@ #ifndef __LINUX_PERCPU_H #define __LINUX_PERCPU_H +#include #include #include #include @@ -9,6 +10,7 @@ #include #include #include +#include #include @@ -125,7 +127,6 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); #endif -extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align) __alloc_size(1); extern bool __is_kernel_percpu_address(unsigned long addr, unsigned long *can_addr); extern bool is_kernel_percpu_address(unsigned long addr); @@ -133,14 +134,16 @@ extern bool is_kernel_percpu_address(unsigned long addr); extern void __init setup_per_cpu_areas(void); #endif -extern void __percpu *__alloc_percpu_gfp(size_t size, size_t align, gfp_t gfp) __alloc_size(1); -extern void __percpu *__alloc_percpu(size_t size, size_t align) __alloc_size(1); -extern void free_percpu(void __percpu *__pdata); +extern void __percpu *pcpu_alloc_noprof(size_t size, size_t align, bool reserved, + gfp_t gfp) __alloc_size(1); extern size_t pcpu_alloc_size(void __percpu *__pdata); -DEFINE_FREE(free_percpu, void __percpu *, free_percpu(_T)) - -extern phys_addr_t per_cpu_ptr_to_phys(void *addr); +#define __alloc_percpu_gfp(_size, _align, _gfp) \ + alloc_hooks(pcpu_alloc_noprof(_size, _align, false, _gfp)) +#define __alloc_percpu(_size, _align) \ + alloc_hooks(pcpu_alloc_noprof(_size, _align, false, GFP_KERNEL)) +#define __alloc_reserved_percpu(_size, _align) \ + alloc_hooks(pcpu_alloc_noprof(_size, _align, true, GFP_KERNEL)) #define alloc_percpu_gfp(type, gfp) \ (typeof(type) __percpu *)__alloc_percpu_gfp(sizeof(type), \ @@ -149,6 +152,12 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr); (typeof(type) __percpu *)__alloc_percpu(sizeof(type), \ __alignof__(type)) +extern void free_percpu(void __percpu *__pdata); + +DEFINE_FREE(free_percpu, void __percpu *, free_percpu(_T)) + +extern phys_addr_t per_cpu_ptr_to_phys(void *addr); + extern unsigned long pcpu_nr_pages(void); #endif /* __LINUX_PERCPU_H */ diff --git a/mm/percpu.c b/mm/percpu.c index 90e9e4004ac97..474e3683b74d1 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -1740,7 +1740,7 @@ static void pcpu_alloc_tag_free_hook(struct pcpu_chunk *chunk, int off, size_t s * RETURNS: * Percpu pointer to the allocated area on success, NULL on failure. */ -static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved, +void __percpu *pcpu_alloc_noprof(size_t size, size_t align, bool reserved, gfp_t gfp) { gfp_t pcpu_gfp; @@ -1907,6 +1907,8 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved, pcpu_memcg_post_alloc_hook(objcg, chunk, off, size); + pcpu_alloc_tag_alloc_hook(chunk, off, size); + return ptr; fail_unlock: @@ -1935,61 +1937,7 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved, return NULL; } - -/** - * __alloc_percpu_gfp - allocate dynamic percpu area - * @size: size of area to allocate in bytes - * @align: alignment of area (max PAGE_SIZE) - * @gfp: allocation flags - * - * Allocate zero-filled percpu area of @size bytes aligned at @align. If - * @gfp doesn't contain %GFP_KERNEL, the allocation doesn't block and can - * be called from any context but is a lot more likely to fail. If @gfp - * has __GFP_NOWARN then no warning will be triggered on invalid or failed - * allocation requests. - * - * RETURNS: - * Percpu pointer to the allocated area on success, NULL on failure. - */ -void __percpu *__alloc_percpu_gfp(size_t size, size_t align, gfp_t gfp) -{ - return pcpu_alloc(size, align, false, gfp); -} -EXPORT_SYMBOL_GPL(__alloc_percpu_gfp); - -/** - * __alloc_percpu - allocate dynamic percpu area - * @size: size of area to allocate in bytes - * @align: alignment of area (max PAGE_SIZE) - * - * Equivalent to __alloc_percpu_gfp(size, align, %GFP_KERNEL). - */ -void __percpu *__alloc_percpu(size_t size, size_t align) -{ - return pcpu_alloc(size, align, false, GFP_KERNEL); -} -EXPORT_SYMBOL_GPL(__alloc_percpu); - -/** - * __alloc_reserved_percpu - allocate reserved percpu area - * @size: size of area to allocate in bytes - * @align: alignment of area (max PAGE_SIZE) - * - * Allocate zero-filled percpu area of @size bytes aligned at @align - * from reserved percpu area if arch has set it up; otherwise, - * allocation is served from the same dynamic area. Might sleep. - * Might trigger writeouts. - * - * CONTEXT: - * Does GFP_KERNEL allocation. - * - * RETURNS: - * Percpu pointer to the allocated area on success, NULL on failure. - */ -void __percpu *__alloc_reserved_percpu(size_t size, size_t align) -{ - return pcpu_alloc(size, align, true, GFP_KERNEL); -} +EXPORT_SYMBOL_GPL(pcpu_alloc_noprof); /** * pcpu_balance_free - manage the amount of free chunks @@ -2328,6 +2276,8 @@ void free_percpu(void __percpu *ptr) spin_lock_irqsave(&pcpu_lock, flags); size = pcpu_free_area(chunk, off); + pcpu_alloc_tag_free_hook(chunk, off, size); + pcpu_memcg_free_hook(chunk, off, size); /* From 88ae5fb755b0d45f790b392f578924d1ca5693a7 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 21 Mar 2024 09:36:52 -0700 Subject: [PATCH 0584/1702] mm: vmalloc: enable memory allocation profiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This wrapps all external vmalloc allocation functions with the alloc_hooks() wrapper, and switches internal allocations to _noprof variants where appropriate, for the new memory allocation profiling feature. [surenb@google.com: arch/um: fix forward declaration for vmalloc] Link: https://lkml.kernel.org/r/20240326073750.726636-1-surenb@google.com [surenb@google.com: undo _noprof additions in the documentation] Link: https://lkml.kernel.org/r/20240326231453.1206227-5-surenb@google.com Link: https://lkml.kernel.org/r/20240321163705.3067592-31-surenb@google.com Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- arch/um/include/shared/um_malloc.h | 3 +- drivers/staging/media/atomisp/pci/hmm/hmm.c | 2 +- include/linux/vmalloc.h | 60 ++++++++++++----- kernel/kallsyms_selftest.c | 2 +- mm/nommu.c | 56 ++++++++-------- mm/util.c | 18 ++--- mm/vmalloc.c | 74 ++++++++++----------- 7 files changed, 123 insertions(+), 92 deletions(-) diff --git a/arch/um/include/shared/um_malloc.h b/arch/um/include/shared/um_malloc.h index 13da93284c2c7..bf503658f08eb 100644 --- a/arch/um/include/shared/um_malloc.h +++ b/arch/um/include/shared/um_malloc.h @@ -11,7 +11,8 @@ extern void *uml_kmalloc(int size, int flags); extern void kfree(const void *ptr); -extern void *vmalloc(unsigned long size); +extern void *vmalloc_noprof(unsigned long size); +#define vmalloc(...) vmalloc_noprof(__VA_ARGS__) extern void vfree(void *ptr); #endif /* __UM_MALLOC_H__ */ diff --git a/drivers/staging/media/atomisp/pci/hmm/hmm.c b/drivers/staging/media/atomisp/pci/hmm/hmm.c index bb12644fd033b..3e2899ad85174 100644 --- a/drivers/staging/media/atomisp/pci/hmm/hmm.c +++ b/drivers/staging/media/atomisp/pci/hmm/hmm.c @@ -205,7 +205,7 @@ static ia_css_ptr __hmm_alloc(size_t bytes, enum hmm_bo_type type, } dev_dbg(atomisp_dev, "pages: 0x%08x (%zu bytes), type: %d, vmalloc %p\n", - bo->start, bytes, type, vmalloc); + bo->start, bytes, type, vmalloc_noprof); return bo->start; diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 98ea90e90439d..e4a631ec430bb 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -2,6 +2,8 @@ #ifndef _LINUX_VMALLOC_H #define _LINUX_VMALLOC_H +#include +#include #include #include #include @@ -138,26 +140,54 @@ extern unsigned long vmalloc_nr_pages(void); static inline unsigned long vmalloc_nr_pages(void) { return 0; } #endif -extern void *vmalloc(unsigned long size) __alloc_size(1); -extern void *vzalloc(unsigned long size) __alloc_size(1); -extern void *vmalloc_user(unsigned long size) __alloc_size(1); -extern void *vmalloc_node(unsigned long size, int node) __alloc_size(1); -extern void *vzalloc_node(unsigned long size, int node) __alloc_size(1); -extern void *vmalloc_32(unsigned long size) __alloc_size(1); -extern void *vmalloc_32_user(unsigned long size) __alloc_size(1); -extern void *__vmalloc(unsigned long size, gfp_t gfp_mask) __alloc_size(1); -extern void *__vmalloc_node_range(unsigned long size, unsigned long align, +extern void *vmalloc_noprof(unsigned long size) __alloc_size(1); +#define vmalloc(...) alloc_hooks(vmalloc_noprof(__VA_ARGS__)) + +extern void *vzalloc_noprof(unsigned long size) __alloc_size(1); +#define vzalloc(...) alloc_hooks(vzalloc_noprof(__VA_ARGS__)) + +extern void *vmalloc_user_noprof(unsigned long size) __alloc_size(1); +#define vmalloc_user(...) alloc_hooks(vmalloc_user_noprof(__VA_ARGS__)) + +extern void *vmalloc_node_noprof(unsigned long size, int node) __alloc_size(1); +#define vmalloc_node(...) alloc_hooks(vmalloc_node_noprof(__VA_ARGS__)) + +extern void *vzalloc_node_noprof(unsigned long size, int node) __alloc_size(1); +#define vzalloc_node(...) alloc_hooks(vzalloc_node_noprof(__VA_ARGS__)) + +extern void *vmalloc_32_noprof(unsigned long size) __alloc_size(1); +#define vmalloc_32(...) alloc_hooks(vmalloc_32_noprof(__VA_ARGS__)) + +extern void *vmalloc_32_user_noprof(unsigned long size) __alloc_size(1); +#define vmalloc_32_user(...) alloc_hooks(vmalloc_32_user_noprof(__VA_ARGS__)) + +extern void *__vmalloc_noprof(unsigned long size, gfp_t gfp_mask) __alloc_size(1); +#define __vmalloc(...) alloc_hooks(__vmalloc_noprof(__VA_ARGS__)) + +extern void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, unsigned long start, unsigned long end, gfp_t gfp_mask, pgprot_t prot, unsigned long vm_flags, int node, const void *caller) __alloc_size(1); -void *__vmalloc_node(unsigned long size, unsigned long align, gfp_t gfp_mask, +#define __vmalloc_node_range(...) alloc_hooks(__vmalloc_node_range_noprof(__VA_ARGS__)) + +void *__vmalloc_node_noprof(unsigned long size, unsigned long align, gfp_t gfp_mask, int node, const void *caller) __alloc_size(1); -void *vmalloc_huge(unsigned long size, gfp_t gfp_mask) __alloc_size(1); +#define __vmalloc_node(...) alloc_hooks(__vmalloc_node_noprof(__VA_ARGS__)) + +void *vmalloc_huge_noprof(unsigned long size, gfp_t gfp_mask) __alloc_size(1); +#define vmalloc_huge(...) alloc_hooks(vmalloc_huge_noprof(__VA_ARGS__)) + +extern void *__vmalloc_array_noprof(size_t n, size_t size, gfp_t flags) __alloc_size(1, 2); +#define __vmalloc_array(...) alloc_hooks(__vmalloc_array_noprof(__VA_ARGS__)) + +extern void *vmalloc_array_noprof(size_t n, size_t size) __alloc_size(1, 2); +#define vmalloc_array(...) alloc_hooks(vmalloc_array_noprof(__VA_ARGS__)) + +extern void *__vcalloc_noprof(size_t n, size_t size, gfp_t flags) __alloc_size(1, 2); +#define __vcalloc(...) alloc_hooks(__vcalloc_noprof(__VA_ARGS__)) -extern void *__vmalloc_array(size_t n, size_t size, gfp_t flags) __alloc_size(1, 2); -extern void *vmalloc_array(size_t n, size_t size) __alloc_size(1, 2); -extern void *__vcalloc(size_t n, size_t size, gfp_t flags) __alloc_size(1, 2); -extern void *vcalloc(size_t n, size_t size) __alloc_size(1, 2); +extern void *vcalloc_noprof(size_t n, size_t size) __alloc_size(1, 2); +#define vcalloc(...) alloc_hooks(vcalloc_noprof(__VA_ARGS__)) extern void vfree(const void *addr); extern void vfree_atomic(const void *addr); diff --git a/kernel/kallsyms_selftest.c b/kernel/kallsyms_selftest.c index 8a689b4ff4f98..2f84896a7bcbd 100644 --- a/kernel/kallsyms_selftest.c +++ b/kernel/kallsyms_selftest.c @@ -82,7 +82,7 @@ static struct test_item test_items[] = { ITEM_FUNC(kallsyms_test_func_static), ITEM_FUNC(kallsyms_test_func), ITEM_FUNC(kallsyms_test_func_weak), - ITEM_FUNC(vmalloc), + ITEM_FUNC(vmalloc_noprof), ITEM_FUNC(vfree), #ifdef CONFIG_KALLSYMS_ALL ITEM_DATA(kallsyms_test_var_bss_static), diff --git a/mm/nommu.c b/mm/nommu.c index 5ec8f44e7ce97..88b646ea6d30c 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -137,28 +137,28 @@ void vfree(const void *addr) } EXPORT_SYMBOL(vfree); -void *__vmalloc(unsigned long size, gfp_t gfp_mask) +void *__vmalloc_noprof(unsigned long size, gfp_t gfp_mask) { /* * You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc() * returns only a logical address. */ - return kmalloc(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM); + return kmalloc_noprof(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM); } -EXPORT_SYMBOL(__vmalloc); +EXPORT_SYMBOL(__vmalloc_noprof); -void *__vmalloc_node_range(unsigned long size, unsigned long align, +void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, unsigned long start, unsigned long end, gfp_t gfp_mask, pgprot_t prot, unsigned long vm_flags, int node, const void *caller) { - return __vmalloc(size, gfp_mask); + return __vmalloc_noprof(size, gfp_mask); } -void *__vmalloc_node(unsigned long size, unsigned long align, gfp_t gfp_mask, +void *__vmalloc_node_noprof(unsigned long size, unsigned long align, gfp_t gfp_mask, int node, const void *caller) { - return __vmalloc(size, gfp_mask); + return __vmalloc_noprof(size, gfp_mask); } static void *__vmalloc_user_flags(unsigned long size, gfp_t flags) @@ -179,11 +179,11 @@ static void *__vmalloc_user_flags(unsigned long size, gfp_t flags) return ret; } -void *vmalloc_user(unsigned long size) +void *vmalloc_user_noprof(unsigned long size) { return __vmalloc_user_flags(size, GFP_KERNEL | __GFP_ZERO); } -EXPORT_SYMBOL(vmalloc_user); +EXPORT_SYMBOL(vmalloc_user_noprof); struct page *vmalloc_to_page(const void *addr) { @@ -217,13 +217,13 @@ long vread_iter(struct iov_iter *iter, const char *addr, size_t count) * For tight control over page level allocator and protection flags * use __vmalloc() instead. */ -void *vmalloc(unsigned long size) +void *vmalloc_noprof(unsigned long size) { - return __vmalloc(size, GFP_KERNEL); + return __vmalloc_noprof(size, GFP_KERNEL); } -EXPORT_SYMBOL(vmalloc); +EXPORT_SYMBOL(vmalloc_noprof); -void *vmalloc_huge(unsigned long size, gfp_t gfp_mask) __weak __alias(__vmalloc); +void *vmalloc_huge_noprof(unsigned long size, gfp_t gfp_mask) __weak __alias(__vmalloc_noprof); /* * vzalloc - allocate virtually contiguous memory with zero fill @@ -237,11 +237,11 @@ void *vmalloc_huge(unsigned long size, gfp_t gfp_mask) __weak __alias(__vmalloc) * For tight control over page level allocator and protection flags * use __vmalloc() instead. */ -void *vzalloc(unsigned long size) +void *vzalloc_noprof(unsigned long size) { - return __vmalloc(size, GFP_KERNEL | __GFP_ZERO); + return __vmalloc_noprof(size, GFP_KERNEL | __GFP_ZERO); } -EXPORT_SYMBOL(vzalloc); +EXPORT_SYMBOL(vzalloc_noprof); /** * vmalloc_node - allocate memory on a specific node @@ -254,11 +254,11 @@ EXPORT_SYMBOL(vzalloc); * For tight control over page level allocator and protection flags * use __vmalloc() instead. */ -void *vmalloc_node(unsigned long size, int node) +void *vmalloc_node_noprof(unsigned long size, int node) { - return vmalloc(size); + return vmalloc_noprof(size); } -EXPORT_SYMBOL(vmalloc_node); +EXPORT_SYMBOL(vmalloc_node_noprof); /** * vzalloc_node - allocate memory on a specific node with zero fill @@ -272,11 +272,11 @@ EXPORT_SYMBOL(vmalloc_node); * For tight control over page level allocator and protection flags * use __vmalloc() instead. */ -void *vzalloc_node(unsigned long size, int node) +void *vzalloc_node_noprof(unsigned long size, int node) { - return vzalloc(size); + return vzalloc_noprof(size); } -EXPORT_SYMBOL(vzalloc_node); +EXPORT_SYMBOL(vzalloc_node_noprof); /** * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) @@ -285,11 +285,11 @@ EXPORT_SYMBOL(vzalloc_node); * Allocate enough 32bit PA addressable pages to cover @size from the * page level allocator and map them into contiguous kernel virtual space. */ -void *vmalloc_32(unsigned long size) +void *vmalloc_32_noprof(unsigned long size) { - return __vmalloc(size, GFP_KERNEL); + return __vmalloc_noprof(size, GFP_KERNEL); } -EXPORT_SYMBOL(vmalloc_32); +EXPORT_SYMBOL(vmalloc_32_noprof); /** * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory @@ -301,15 +301,15 @@ EXPORT_SYMBOL(vmalloc_32); * VM_USERMAP is set on the corresponding VMA so that subsequent calls to * remap_vmalloc_range() are permissible. */ -void *vmalloc_32_user(unsigned long size) +void *vmalloc_32_user_noprof(unsigned long size) { /* * We'll have to sort out the ZONE_DMA bits for 64-bit, * but for now this can simply use vmalloc_user() directly. */ - return vmalloc_user(size); + return vmalloc_user_noprof(size); } -EXPORT_SYMBOL(vmalloc_32_user); +EXPORT_SYMBOL(vmalloc_32_user_noprof); void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot) { diff --git a/mm/util.c b/mm/util.c index cee3563d2071b..a9e911b22b992 100644 --- a/mm/util.c +++ b/mm/util.c @@ -656,7 +656,7 @@ void *kvmalloc_node_noprof(size_t size, gfp_t flags, int node) * about the resulting pointer, and cannot play * protection games. */ - return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END, + return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END, flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, node, __builtin_return_address(0)); } @@ -720,7 +720,7 @@ EXPORT_SYMBOL(kvrealloc_noprof); * @size: element size. * @flags: the type of memory to allocate (see kmalloc). */ -void *__vmalloc_array(size_t n, size_t size, gfp_t flags) +void *__vmalloc_array_noprof(size_t n, size_t size, gfp_t flags) { size_t bytes; @@ -728,18 +728,18 @@ void *__vmalloc_array(size_t n, size_t size, gfp_t flags) return NULL; return __vmalloc(bytes, flags); } -EXPORT_SYMBOL(__vmalloc_array); +EXPORT_SYMBOL(__vmalloc_array_noprof); /** * vmalloc_array - allocate memory for a virtually contiguous array. * @n: number of elements. * @size: element size. */ -void *vmalloc_array(size_t n, size_t size) +void *vmalloc_array_noprof(size_t n, size_t size) { return __vmalloc_array(n, size, GFP_KERNEL); } -EXPORT_SYMBOL(vmalloc_array); +EXPORT_SYMBOL(vmalloc_array_noprof); /** * __vcalloc - allocate and zero memory for a virtually contiguous array. @@ -747,22 +747,22 @@ EXPORT_SYMBOL(vmalloc_array); * @size: element size. * @flags: the type of memory to allocate (see kmalloc). */ -void *__vcalloc(size_t n, size_t size, gfp_t flags) +void *__vcalloc_noprof(size_t n, size_t size, gfp_t flags) { return __vmalloc_array(n, size, flags | __GFP_ZERO); } -EXPORT_SYMBOL(__vcalloc); +EXPORT_SYMBOL(__vcalloc_noprof); /** * vcalloc - allocate and zero memory for a virtually contiguous array. * @n: number of elements. * @size: element size. */ -void *vcalloc(size_t n, size_t size) +void *vcalloc_noprof(size_t n, size_t size) { return __vmalloc_array(n, size, GFP_KERNEL | __GFP_ZERO); } -EXPORT_SYMBOL(vcalloc); +EXPORT_SYMBOL(vcalloc_noprof); struct anon_vma *folio_anon_vma(struct folio *folio) { diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 68fa001648cc1..61af431bb6b82 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3523,12 +3523,12 @@ vm_area_alloc_pages(gfp_t gfp, int nid, * but mempolicy wants to alloc memory by interleaving. */ if (IS_ENABLED(CONFIG_NUMA) && nid == NUMA_NO_NODE) - nr = alloc_pages_bulk_array_mempolicy(bulk_gfp, + nr = alloc_pages_bulk_array_mempolicy_noprof(bulk_gfp, nr_pages_request, pages + nr_allocated); else - nr = alloc_pages_bulk_array_node(bulk_gfp, nid, + nr = alloc_pages_bulk_array_node_noprof(bulk_gfp, nid, nr_pages_request, pages + nr_allocated); @@ -3558,9 +3558,9 @@ vm_area_alloc_pages(gfp_t gfp, int nid, break; if (nid == NUMA_NO_NODE) - page = alloc_pages(alloc_gfp, order); + page = alloc_pages_noprof(alloc_gfp, order); else - page = alloc_pages_node(nid, alloc_gfp, order); + page = alloc_pages_node_noprof(nid, alloc_gfp, order); if (unlikely(!page)) { if (!nofail) break; @@ -3617,10 +3617,10 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, /* Please note that the recursion is strictly bounded. */ if (array_size > PAGE_SIZE) { - area->pages = __vmalloc_node(array_size, 1, nested_gfp, node, + area->pages = __vmalloc_node_noprof(array_size, 1, nested_gfp, node, area->caller); } else { - area->pages = kmalloc_node(array_size, nested_gfp, node); + area->pages = kmalloc_node_noprof(array_size, nested_gfp, node); } if (!area->pages) { @@ -3730,7 +3730,7 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, * * Return: the address of the area or %NULL on failure */ -void *__vmalloc_node_range(unsigned long size, unsigned long align, +void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, unsigned long start, unsigned long end, gfp_t gfp_mask, pgprot_t prot, unsigned long vm_flags, int node, const void *caller) @@ -3877,10 +3877,10 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align, * * Return: pointer to the allocated memory or %NULL on error */ -void *__vmalloc_node(unsigned long size, unsigned long align, +void *__vmalloc_node_noprof(unsigned long size, unsigned long align, gfp_t gfp_mask, int node, const void *caller) { - return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, + return __vmalloc_node_range_noprof(size, align, VMALLOC_START, VMALLOC_END, gfp_mask, PAGE_KERNEL, 0, node, caller); } /* @@ -3889,15 +3889,15 @@ void *__vmalloc_node(unsigned long size, unsigned long align, * than that. */ #ifdef CONFIG_TEST_VMALLOC_MODULE -EXPORT_SYMBOL_GPL(__vmalloc_node); +EXPORT_SYMBOL_GPL(__vmalloc_node_noprof); #endif -void *__vmalloc(unsigned long size, gfp_t gfp_mask) +void *__vmalloc_noprof(unsigned long size, gfp_t gfp_mask) { - return __vmalloc_node(size, 1, gfp_mask, NUMA_NO_NODE, + return __vmalloc_node_noprof(size, 1, gfp_mask, NUMA_NO_NODE, __builtin_return_address(0)); } -EXPORT_SYMBOL(__vmalloc); +EXPORT_SYMBOL(__vmalloc_noprof); /** * vmalloc - allocate virtually contiguous memory @@ -3911,12 +3911,12 @@ EXPORT_SYMBOL(__vmalloc); * * Return: pointer to the allocated memory or %NULL on error */ -void *vmalloc(unsigned long size) +void *vmalloc_noprof(unsigned long size) { - return __vmalloc_node(size, 1, GFP_KERNEL, NUMA_NO_NODE, + return __vmalloc_node_noprof(size, 1, GFP_KERNEL, NUMA_NO_NODE, __builtin_return_address(0)); } -EXPORT_SYMBOL(vmalloc); +EXPORT_SYMBOL(vmalloc_noprof); /** * vmalloc_huge - allocate virtually contiguous memory, allow huge pages @@ -3930,13 +3930,13 @@ EXPORT_SYMBOL(vmalloc); * * Return: pointer to the allocated memory or %NULL on error */ -void *vmalloc_huge(unsigned long size, gfp_t gfp_mask) +void *vmalloc_huge_noprof(unsigned long size, gfp_t gfp_mask) { - return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END, + return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END, gfp_mask, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, NUMA_NO_NODE, __builtin_return_address(0)); } -EXPORT_SYMBOL_GPL(vmalloc_huge); +EXPORT_SYMBOL_GPL(vmalloc_huge_noprof); /** * vzalloc - allocate virtually contiguous memory with zero fill @@ -3951,12 +3951,12 @@ EXPORT_SYMBOL_GPL(vmalloc_huge); * * Return: pointer to the allocated memory or %NULL on error */ -void *vzalloc(unsigned long size) +void *vzalloc_noprof(unsigned long size) { - return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE, + return __vmalloc_node_noprof(size, 1, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE, __builtin_return_address(0)); } -EXPORT_SYMBOL(vzalloc); +EXPORT_SYMBOL(vzalloc_noprof); /** * vmalloc_user - allocate zeroed virtually contiguous memory for userspace @@ -3967,14 +3967,14 @@ EXPORT_SYMBOL(vzalloc); * * Return: pointer to the allocated memory or %NULL on error */ -void *vmalloc_user(unsigned long size) +void *vmalloc_user_noprof(unsigned long size) { - return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END, + return __vmalloc_node_range_noprof(size, SHMLBA, VMALLOC_START, VMALLOC_END, GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL, VM_USERMAP, NUMA_NO_NODE, __builtin_return_address(0)); } -EXPORT_SYMBOL(vmalloc_user); +EXPORT_SYMBOL(vmalloc_user_noprof); /** * vmalloc_node - allocate memory on a specific node @@ -3989,12 +3989,12 @@ EXPORT_SYMBOL(vmalloc_user); * * Return: pointer to the allocated memory or %NULL on error */ -void *vmalloc_node(unsigned long size, int node) +void *vmalloc_node_noprof(unsigned long size, int node) { - return __vmalloc_node(size, 1, GFP_KERNEL, node, + return __vmalloc_node_noprof(size, 1, GFP_KERNEL, node, __builtin_return_address(0)); } -EXPORT_SYMBOL(vmalloc_node); +EXPORT_SYMBOL(vmalloc_node_noprof); /** * vzalloc_node - allocate memory on a specific node with zero fill @@ -4007,12 +4007,12 @@ EXPORT_SYMBOL(vmalloc_node); * * Return: pointer to the allocated memory or %NULL on error */ -void *vzalloc_node(unsigned long size, int node) +void *vzalloc_node_noprof(unsigned long size, int node) { - return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, node, + return __vmalloc_node_noprof(size, 1, GFP_KERNEL | __GFP_ZERO, node, __builtin_return_address(0)); } -EXPORT_SYMBOL(vzalloc_node); +EXPORT_SYMBOL(vzalloc_node_noprof); #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32) #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL) @@ -4035,12 +4035,12 @@ EXPORT_SYMBOL(vzalloc_node); * * Return: pointer to the allocated memory or %NULL on error */ -void *vmalloc_32(unsigned long size) +void *vmalloc_32_noprof(unsigned long size) { - return __vmalloc_node(size, 1, GFP_VMALLOC32, NUMA_NO_NODE, + return __vmalloc_node_noprof(size, 1, GFP_VMALLOC32, NUMA_NO_NODE, __builtin_return_address(0)); } -EXPORT_SYMBOL(vmalloc_32); +EXPORT_SYMBOL(vmalloc_32_noprof); /** * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory @@ -4051,14 +4051,14 @@ EXPORT_SYMBOL(vmalloc_32); * * Return: pointer to the allocated memory or %NULL on error */ -void *vmalloc_32_user(unsigned long size) +void *vmalloc_32_user_noprof(unsigned long size) { - return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END, + return __vmalloc_node_range_noprof(size, SHMLBA, VMALLOC_START, VMALLOC_END, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, VM_USERMAP, NUMA_NO_NODE, __builtin_return_address(0)); } -EXPORT_SYMBOL(vmalloc_32_user); +EXPORT_SYMBOL(vmalloc_32_user_noprof); /* * Atomically zero bytes in the iterator. From 9e54dd8b64dccdf5336172bf905017a6f97ee18d Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 21 Mar 2024 09:36:53 -0700 Subject: [PATCH 0585/1702] rhashtable: plumb through alloc tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This gives better memory allocation profiling results; rhashtable allocations will be accounted to the code that initialized the rhashtable. [surenb@google.com: undo _noprof additions in the documentation] Link: https://lkml.kernel.org/r/20240326231453.1206227-1-surenb@google.com Link: https://lkml.kernel.org/r/20240321163705.3067592-32-surenb@google.com Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/alloc_tag.h | 3 +++ include/linux/rhashtable-types.h | 11 +++++++++-- lib/rhashtable.c | 22 ++++++++++++++-------- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h index 2aea7afa1fab9..0567766e50bdb 100644 --- a/include/linux/alloc_tag.h +++ b/include/linux/alloc_tag.h @@ -152,6 +152,8 @@ static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) ref->ct = NULL; } +#define alloc_tag_record(p) ((p) = current->alloc_tag) + #else /* CONFIG_MEM_ALLOC_PROFILING */ #define DEFINE_ALLOC_TAG(_alloc_tag) @@ -159,6 +161,7 @@ static inline bool mem_alloc_profiling_enabled(void) { return false; } static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes) {} static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {} +#define alloc_tag_record(p) do {} while (0) #endif /* CONFIG_MEM_ALLOC_PROFILING */ diff --git a/include/linux/rhashtable-types.h b/include/linux/rhashtable-types.h index b6f3797277ff8..015c8298bebc4 100644 --- a/include/linux/rhashtable-types.h +++ b/include/linux/rhashtable-types.h @@ -9,6 +9,7 @@ #ifndef _LINUX_RHASHTABLE_TYPES_H #define _LINUX_RHASHTABLE_TYPES_H +#include #include #include #include @@ -88,6 +89,9 @@ struct rhashtable { struct mutex mutex; spinlock_t lock; atomic_t nelems; +#ifdef CONFIG_MEM_ALLOC_PROFILING + struct alloc_tag *alloc_tag; +#endif }; /** @@ -127,9 +131,12 @@ struct rhashtable_iter { bool end_of_table; }; -int rhashtable_init(struct rhashtable *ht, +int rhashtable_init_noprof(struct rhashtable *ht, const struct rhashtable_params *params); -int rhltable_init(struct rhltable *hlt, +#define rhashtable_init(...) alloc_hooks(rhashtable_init_noprof(__VA_ARGS__)) + +int rhltable_init_noprof(struct rhltable *hlt, const struct rhashtable_params *params); +#define rhltable_init(...) alloc_hooks(rhltable_init_noprof(__VA_ARGS__)) #endif /* _LINUX_RHASHTABLE_TYPES_H */ diff --git a/lib/rhashtable.c b/lib/rhashtable.c index 6ae2ba8e06a21..dbbed19f8fff9 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -130,7 +130,8 @@ static union nested_table *nested_table_alloc(struct rhashtable *ht, if (ntbl) return ntbl; - ntbl = kzalloc(PAGE_SIZE, GFP_ATOMIC); + ntbl = alloc_hooks_tag(ht->alloc_tag, + kmalloc_noprof(PAGE_SIZE, GFP_ATOMIC|__GFP_ZERO)); if (ntbl && leaf) { for (i = 0; i < PAGE_SIZE / sizeof(ntbl[0]); i++) @@ -157,7 +158,8 @@ static struct bucket_table *nested_bucket_table_alloc(struct rhashtable *ht, size = sizeof(*tbl) + sizeof(tbl->buckets[0]); - tbl = kzalloc(size, gfp); + tbl = alloc_hooks_tag(ht->alloc_tag, + kmalloc_noprof(size, gfp|__GFP_ZERO)); if (!tbl) return NULL; @@ -181,7 +183,9 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht, int i; static struct lock_class_key __key; - tbl = kvzalloc(struct_size(tbl, buckets, nbuckets), gfp); + tbl = alloc_hooks_tag(ht->alloc_tag, + kvmalloc_node_noprof(struct_size(tbl, buckets, nbuckets), + gfp|__GFP_ZERO, NUMA_NO_NODE)); size = nbuckets; @@ -1016,7 +1020,7 @@ static u32 rhashtable_jhash2(const void *key, u32 length, u32 seed) * .obj_hashfn = my_hash_fn, * }; */ -int rhashtable_init(struct rhashtable *ht, +int rhashtable_init_noprof(struct rhashtable *ht, const struct rhashtable_params *params) { struct bucket_table *tbl; @@ -1031,6 +1035,8 @@ int rhashtable_init(struct rhashtable *ht, spin_lock_init(&ht->lock); memcpy(&ht->p, params, sizeof(*params)); + alloc_tag_record(ht->alloc_tag); + if (params->min_size) ht->p.min_size = roundup_pow_of_two(params->min_size); @@ -1076,7 +1082,7 @@ int rhashtable_init(struct rhashtable *ht, return 0; } -EXPORT_SYMBOL_GPL(rhashtable_init); +EXPORT_SYMBOL_GPL(rhashtable_init_noprof); /** * rhltable_init - initialize a new hash list table @@ -1087,15 +1093,15 @@ EXPORT_SYMBOL_GPL(rhashtable_init); * * See documentation for rhashtable_init. */ -int rhltable_init(struct rhltable *hlt, const struct rhashtable_params *params) +int rhltable_init_noprof(struct rhltable *hlt, const struct rhashtable_params *params) { int err; - err = rhashtable_init(&hlt->ht, params); + err = rhashtable_init_noprof(&hlt->ht, params); hlt->ht.rhlist = true; return err; } -EXPORT_SYMBOL_GPL(rhltable_init); +EXPORT_SYMBOL_GPL(rhltable_init_noprof); static void rhashtable_free_one(struct rhashtable *ht, struct rhash_head *obj, void (*free_fn)(void *ptr, void *arg), From 1438d349d16b78d88f9e978a4a5496f078c8191b Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:54 -0700 Subject: [PATCH 0586/1702] lib: add memory allocations report in show_mem() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Include allocations in show_mem reports. Link: https://lkml.kernel.org/r/20240321163705.3067592-33-surenb@google.com Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Reviewed-by: Vlastimil Babka Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/alloc_tag.h | 7 +++++++ include/linux/codetag.h | 1 + lib/alloc_tag.c | 38 ++++++++++++++++++++++++++++++++++++++ lib/codetag.c | 5 +++++ mm/show_mem.c | 26 ++++++++++++++++++++++++++ 5 files changed, 77 insertions(+) diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h index 0567766e50bdb..d18388c0e9e19 100644 --- a/include/linux/alloc_tag.h +++ b/include/linux/alloc_tag.h @@ -31,6 +31,13 @@ struct alloc_tag { #ifdef CONFIG_MEM_ALLOC_PROFILING +struct codetag_bytes { + struct codetag *ct; + s64 bytes; +}; + +size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep); + static inline struct alloc_tag *ct_to_alloc_tag(struct codetag *ct) { return container_of(ct, struct alloc_tag, ct); diff --git a/include/linux/codetag.h b/include/linux/codetag.h index bfd0ba5c4185c..c2a579ccd4558 100644 --- a/include/linux/codetag.h +++ b/include/linux/codetag.h @@ -61,6 +61,7 @@ struct codetag_iterator { } void codetag_lock_module_list(struct codetag_type *cttype, bool lock); +bool codetag_trylock_module_list(struct codetag_type *cttype); struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype); struct codetag *codetag_next_ct(struct codetag_iterator *iter); diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c index 6bd8e85beaf72..26af9982ddc44 100644 --- a/lib/alloc_tag.c +++ b/lib/alloc_tag.c @@ -89,6 +89,44 @@ static const struct seq_operations allocinfo_seq_op = { .show = allocinfo_show, }; +size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep) +{ + struct codetag_iterator iter; + struct codetag *ct; + struct codetag_bytes n; + unsigned int i, nr = 0; + + if (can_sleep) + codetag_lock_module_list(alloc_tag_cttype, true); + else if (!codetag_trylock_module_list(alloc_tag_cttype)) + return 0; + + iter = codetag_get_ct_iter(alloc_tag_cttype); + while ((ct = codetag_next_ct(&iter))) { + struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct)); + + n.ct = ct; + n.bytes = counter.bytes; + + for (i = 0; i < nr; i++) + if (n.bytes > tags[i].bytes) + break; + + if (i < count) { + nr -= nr == count; + memmove(&tags[i + 1], + &tags[i], + sizeof(tags[0]) * (nr - i)); + nr++; + tags[i] = n; + } + } + + codetag_lock_module_list(alloc_tag_cttype, false); + + return nr; +} + static void __init procfs_init(void) { proc_create_seq("allocinfo", 0444, NULL, &allocinfo_seq_op); diff --git a/lib/codetag.c b/lib/codetag.c index 408062f722ced..5ace625f2328f 100644 --- a/lib/codetag.c +++ b/lib/codetag.c @@ -36,6 +36,11 @@ void codetag_lock_module_list(struct codetag_type *cttype, bool lock) up_read(&cttype->mod_lock); } +bool codetag_trylock_module_list(struct codetag_type *cttype) +{ + return down_read_trylock(&cttype->mod_lock) != 0; +} + struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype) { struct codetag_iterator iter = { diff --git a/mm/show_mem.c b/mm/show_mem.c index 8dcfafbd283c1..bdb439551eefc 100644 --- a/mm/show_mem.c +++ b/mm/show_mem.c @@ -423,4 +423,30 @@ void __show_mem(unsigned int filter, nodemask_t *nodemask, int max_zone_idx) #ifdef CONFIG_MEMORY_FAILURE printk("%lu pages hwpoisoned\n", atomic_long_read(&num_poisoned_pages)); #endif +#ifdef CONFIG_MEM_ALLOC_PROFILING + { + struct codetag_bytes tags[10]; + size_t i, nr; + + nr = alloc_tag_top_users(tags, ARRAY_SIZE(tags), false); + if (nr) { + pr_notice("Memory allocations:\n"); + for (i = 0; i < nr; i++) { + struct codetag *ct = tags[i].ct; + struct alloc_tag *tag = ct_to_alloc_tag(ct); + struct alloc_tag_counters counter = alloc_tag_read(tag); + + /* Same as alloc_tag_to_text() but w/o intermediate buffer */ + if (ct->modname) + pr_notice("%12lli %8llu %s:%u [%s] func:%s\n", + counter.bytes, counter.calls, ct->filename, + ct->lineno, ct->modname, ct->function); + else + pr_notice("%12lli %8llu %s:%u func:%s\n", + counter.bytes, counter.calls, ct->filename, + ct->lineno, ct->function); + } + } + } +#endif } From 239d6c96d86f8a42d799b9a85c09cc7f36aef3f8 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:55 -0700 Subject: [PATCH 0587/1702] codetag: debug: skip objext checking when it's for objext itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit objext objects are created with __GFP_NO_OBJ_EXT flag and therefore have no corresponding objext themselves (otherwise we would get an infinite recursion). When freeing these objects their codetag will be empty and when CONFIG_MEM_ALLOC_PROFILING_DEBUG is enabled this will lead to false warnings. Introduce CODETAG_EMPTY special codetag value to mark allocations which intentionally lack codetag to avoid these warnings. Set objext codetags to CODETAG_EMPTY before freeing to indicate that the codetag is expected to be empty. Link: https://lkml.kernel.org/r/20240321163705.3067592-34-surenb@google.com Signed-off-by: Suren Baghdasaryan Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Kent Overstreet Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/alloc_tag.h | 26 ++++++++++++++++++++++++++ mm/slub.c | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h index d18388c0e9e19..3af5d6a5ced42 100644 --- a/include/linux/alloc_tag.h +++ b/include/linux/alloc_tag.h @@ -29,6 +29,27 @@ struct alloc_tag { struct alloc_tag_counters __percpu *counters; } __aligned(8); +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG + +#define CODETAG_EMPTY ((void *)1) + +static inline bool is_codetag_empty(union codetag_ref *ref) +{ + return ref->ct == CODETAG_EMPTY; +} + +static inline void set_codetag_empty(union codetag_ref *ref) +{ + if (ref) + ref->ct = CODETAG_EMPTY; +} + +#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ + +static inline bool is_codetag_empty(union codetag_ref *ref) { return false; } + +#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ + #ifdef CONFIG_MEM_ALLOC_PROFILING struct codetag_bytes { @@ -151,6 +172,11 @@ static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) if (!ref || !ref->ct) return; + if (is_codetag_empty(ref)) { + ref->ct = NULL; + return; + } + tag = ct_to_alloc_tag(ref->ct); this_cpu_sub(tag->counters->bytes, bytes); diff --git a/mm/slub.c b/mm/slub.c index 0fc891d44cffa..00f461826462f 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1873,6 +1873,30 @@ static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s) #ifdef CONFIG_SLAB_OBJ_EXT +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG + +static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) +{ + struct slabobj_ext *slab_exts; + struct slab *obj_exts_slab; + + obj_exts_slab = virt_to_slab(obj_exts); + slab_exts = slab_obj_exts(obj_exts_slab); + if (slab_exts) { + unsigned int offs = obj_to_index(obj_exts_slab->slab_cache, + obj_exts_slab, obj_exts); + /* codetag should be NULL */ + WARN_ON(slab_exts[offs].ref.ct); + set_codetag_empty(&slab_exts[offs].ref); + } +} + +#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ + +static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) {} + +#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ + /* * The allocated objcg pointers array is not accounted directly. * Moreover, it should not come from DMA buffer and is not readily @@ -1913,6 +1937,7 @@ static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, * assign slabobj_exts in parallel. In this case the existing * objcg vector should be reused. */ + mark_objexts_empty(vec); kfree(vec); return 0; } @@ -1929,6 +1954,14 @@ static inline void free_slab_obj_exts(struct slab *slab) if (!obj_exts) return; + /* + * obj_exts was created with __GFP_NO_OBJ_EXT flag, therefore its + * corresponding extension will be NULL. alloc_tag_sub() will throw a + * warning if slab has extensions but the extension of an object is + * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that + * the extension for obj_exts is expected to be NULL. + */ + mark_objexts_empty(obj_exts); kfree(obj_exts); slab->obj_exts = 0; } From d224eb0287fbd84f4f13eca042c7f08f87138f3b Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:56 -0700 Subject: [PATCH 0588/1702] codetag: debug: mark codetags for reserved pages as empty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To avoid debug warnings while freeing reserved pages which were not allocated with usual allocators, mark their codetags as empty before freeing. Link: https://lkml.kernel.org/r/20240321163705.3067592-35-surenb@google.com Signed-off-by: Suren Baghdasaryan Reviewed-by: Kees Cook Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Kent Overstreet Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/alloc_tag.h | 1 + include/linux/mm.h | 9 +++++++++ include/linux/pgalloc_tag.h | 2 ++ mm/mm_init.c | 12 +++++++++++- 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h index 3af5d6a5ced42..afc9e259a2d35 100644 --- a/include/linux/alloc_tag.h +++ b/include/linux/alloc_tag.h @@ -47,6 +47,7 @@ static inline void set_codetag_empty(union codetag_ref *ref) #else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ static inline bool is_codetag_empty(union codetag_ref *ref) { return false; } +static inline void set_codetag_empty(union codetag_ref *ref) {} #endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ diff --git a/include/linux/mm.h b/include/linux/mm.h index b6bdaa18b9e9d..0dbd737cfabdc 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -3134,6 +3135,14 @@ extern void reserve_bootmem_region(phys_addr_t start, /* Free the reserved page into the buddy system, so it gets managed. */ static inline void free_reserved_page(struct page *page) { + if (mem_alloc_profiling_enabled()) { + union codetag_ref *ref = get_page_tag_ref(page); + + if (ref) { + set_codetag_empty(ref); + put_page_tag_ref(ref); + } + } ClearPageReserved(page); init_page_count(page); __free_page(page); diff --git a/include/linux/pgalloc_tag.h b/include/linux/pgalloc_tag.h index 50d212330bbbe..62d8dad74b373 100644 --- a/include/linux/pgalloc_tag.h +++ b/include/linux/pgalloc_tag.h @@ -120,6 +120,8 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) #else /* CONFIG_MEM_ALLOC_PROFILING */ +static inline union codetag_ref *get_page_tag_ref(struct page *page) { return NULL; } +static inline void put_page_tag_ref(union codetag_ref *ref) {} static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, unsigned int nr) {} static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {} diff --git a/mm/mm_init.c b/mm/mm_init.c index 2fd9bf044a79c..f45c2b32ba826 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2567,7 +2567,6 @@ void __init set_dma_reserve(unsigned long new_dma_reserve) void __init memblock_free_pages(struct page *page, unsigned long pfn, unsigned int order) { - if (IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) { int nid = early_pfn_to_nid(pfn); @@ -2579,6 +2578,17 @@ void __init memblock_free_pages(struct page *page, unsigned long pfn, /* KMSAN will take care of these pages. */ return; } + + /* pages were reserved and not allocated */ + if (mem_alloc_profiling_enabled()) { + union codetag_ref *ref = get_page_tag_ref(page); + + if (ref) { + set_codetag_empty(ref); + put_page_tag_ref(ref); + } + } + __free_pages_core(page, order); } From 09c46563ff6d5f090211e48ff1fdba0ec7f4c97f Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 21 Mar 2024 09:36:57 -0700 Subject: [PATCH 0589/1702] codetag: debug: introduce OBJEXTS_ALLOC_FAIL to mark failed slab_ext allocations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If slabobj_ext vector allocation for a slab object fails and later on it succeeds for another object in the same slab, the slabobj_ext for the original object will be NULL and will be flagged in case when CONFIG_MEM_ALLOC_PROFILING_DEBUG is enabled. Mark failed slabobj_ext vector allocations using a new objext_flags flag stored in the lower bits of slab->obj_exts. When new allocation succeeds it marks all tag references in the same slabobj_ext vector as empty to avoid warnings implemented by CONFIG_MEM_ALLOC_PROFILING_DEBUG checks. Link: https://lkml.kernel.org/r/20240321163705.3067592-36-surenb@google.com Signed-off-by: Suren Baghdasaryan Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Kent Overstreet Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 4 +++- mm/slub.c | 46 ++++++++++++++++++++++++++++++++------ 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 24a6df30be49e..8f332b4ae84ca 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -366,8 +366,10 @@ enum page_memcg_data_flags { #endif /* CONFIG_MEMCG */ enum objext_flags { + /* slabobj_ext vector failed to allocate */ + OBJEXTS_ALLOC_FAIL = __FIRST_OBJEXT_FLAG, /* the next bit after the last actual flag */ - __NR_OBJEXTS_FLAGS = __FIRST_OBJEXT_FLAG, + __NR_OBJEXTS_FLAGS = (__FIRST_OBJEXT_FLAG << 1), }; #define OBJEXTS_FLAGS_MASK (__NR_OBJEXTS_FLAGS - 1) diff --git a/mm/slub.c b/mm/slub.c index 00f461826462f..be047279c9e97 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1891,9 +1891,33 @@ static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) } } +static inline void mark_failed_objexts_alloc(struct slab *slab) +{ + slab->obj_exts = OBJEXTS_ALLOC_FAIL; +} + +static inline void handle_failed_objexts_alloc(unsigned long obj_exts, + struct slabobj_ext *vec, unsigned int objects) +{ + /* + * If vector previously failed to allocate then we have live + * objects with no tag reference. Mark all references in this + * vector as empty to avoid warnings later on. + */ + if (obj_exts & OBJEXTS_ALLOC_FAIL) { + unsigned int i; + + for (i = 0; i < objects; i++) + set_codetag_empty(&vec[i].ref); + } +} + #else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) {} +static inline void mark_failed_objexts_alloc(struct slab *slab) {} +static inline void handle_failed_objexts_alloc(unsigned long obj_exts, + struct slabobj_ext *vec, unsigned int objects) {} #endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ @@ -1909,29 +1933,37 @@ static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, gfp_t gfp, bool new_slab) { unsigned int objects = objs_per_slab(s, slab); - unsigned long obj_exts; - void *vec; + unsigned long new_exts; + unsigned long old_exts; + struct slabobj_ext *vec; gfp &= ~OBJCGS_CLEAR_MASK; /* Prevent recursive extension vector allocation */ gfp |= __GFP_NO_OBJ_EXT; vec = kcalloc_node(objects, sizeof(struct slabobj_ext), gfp, slab_nid(slab)); - if (!vec) + if (!vec) { + /* Mark vectors which failed to allocate */ + if (new_slab) + mark_failed_objexts_alloc(slab); + return -ENOMEM; + } - obj_exts = (unsigned long)vec; + new_exts = (unsigned long)vec; #ifdef CONFIG_MEMCG - obj_exts |= MEMCG_DATA_OBJEXTS; + new_exts |= MEMCG_DATA_OBJEXTS; #endif + old_exts = slab->obj_exts; + handle_failed_objexts_alloc(old_exts, vec, objects); if (new_slab) { /* * If the slab is brand new and nobody can yet access its * obj_exts, no synchronization is required and obj_exts can * be simply assigned. */ - slab->obj_exts = obj_exts; - } else if (cmpxchg(&slab->obj_exts, 0, obj_exts)) { + slab->obj_exts = new_exts; + } else if (cmpxchg(&slab->obj_exts, old_exts, new_exts) != old_exts) { /* * If the slab is already in use, somebody can allocate and * assign slabobj_exts in parallel. In this case the existing From 9f0ee883ec7fa10f4e0a9f46e6b0eb4dd106ec93 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 21 Mar 2024 09:36:58 -0700 Subject: [PATCH 0590/1702] MAINTAINERS: add entries for code tagging and memory allocation profiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new code & libraries added are being maintained - mark them as such. [lbulwahn@redhat.com: MAINTAINERS: improve entries] Link: https://lkml.kernel.org/r/20240411064717.51140-1-lukas.bulwahn@redhat.com Link: https://lkml.kernel.org/r/20240321163705.3067592-37-surenb@google.com Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Signed-off-by: Lukas Bulwahn Reviewed-by: Kees Cook Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- MAINTAINERS | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index ee9cc2b40409e..11dbcd764967f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5252,6 +5252,14 @@ S: Supported F: Documentation/process/code-of-conduct-interpretation.rst F: Documentation/process/code-of-conduct.rst +CODE TAGGING +M: Suren Baghdasaryan +M: Kent Overstreet +S: Maintained +F: include/asm-generic/codetag.lds.h +F: include/linux/codetag.h +F: lib/codetag.c + COMEDI DRIVERS M: Ian Abbott M: H Hartley Sweeten @@ -14151,6 +14159,16 @@ F: mm/memblock.c F: mm/mm_init.c F: tools/testing/memblock/ +MEMORY ALLOCATION PROFILING +M: Suren Baghdasaryan +M: Kent Overstreet +L: linux-mm@kvack.org +S: Maintained +F: Documentation/mm/allocation-profiling.rst +F: include/linux/alloc_tag.h +F: include/linux/pgalloc_tag.h +F: lib/alloc_tag.c + MEMORY CONTROLLER DRIVERS M: Krzysztof Kozlowski L: linux-kernel@vger.kernel.org From ebdf9ad4ca9897768bcb7dad6581369693c81fe0 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 21 Mar 2024 09:36:59 -0700 Subject: [PATCH 0591/1702] memprofiling: documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provide documentation for memory allocation profiling. Link: https://lkml.kernel.org/r/20240321163705.3067592-38-surenb@google.com Signed-off-by: Kent Overstreet Signed-off-by: Suren Baghdasaryan Tested-by: Kees Cook Cc: Alexander Viro Cc: Alex Gaynor Cc: Alice Ryhl Cc: Andreas Hindborg Cc: Benno Lossin Cc: "Björn Roy Baron" Cc: Boqun Feng Cc: Christoph Lameter Cc: Dennis Zhou Cc: Gary Guo Cc: Miguel Ojeda Cc: Pasha Tatashin Cc: Peter Zijlstra Cc: Tejun Heo Cc: Vlastimil Babka Cc: Wedson Almeida Filho Signed-off-by: Andrew Morton --- Documentation/mm/allocation-profiling.rst | 100 ++++++++++++++++++++++ Documentation/mm/index.rst | 1 + 2 files changed, 101 insertions(+) create mode 100644 Documentation/mm/allocation-profiling.rst diff --git a/Documentation/mm/allocation-profiling.rst b/Documentation/mm/allocation-profiling.rst new file mode 100644 index 0000000000000..d3b733b41ae65 --- /dev/null +++ b/Documentation/mm/allocation-profiling.rst @@ -0,0 +1,100 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=========================== +MEMORY ALLOCATION PROFILING +=========================== + +Low overhead (suitable for production) accounting of all memory allocations, +tracked by file and line number. + +Usage: +kconfig options: +- CONFIG_MEM_ALLOC_PROFILING + +- CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT + +- CONFIG_MEM_ALLOC_PROFILING_DEBUG + adds warnings for allocations that weren't accounted because of a + missing annotation + +Boot parameter: + sysctl.vm.mem_profiling=0|1|never + + When set to "never", memory allocation profiling overhead is minimized and it + cannot be enabled at runtime (sysctl becomes read-only). + When CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT=y, default value is "1". + When CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT=n, default value is "never". + +sysctl: + /proc/sys/vm/mem_profiling + +Runtime info: + /proc/allocinfo + +Example output:: + + root@moria-kvm:~# sort -g /proc/allocinfo|tail|numfmt --to=iec + 2.8M 22648 fs/kernfs/dir.c:615 func:__kernfs_new_node + 3.8M 953 mm/memory.c:4214 func:alloc_anon_folio + 4.0M 1010 drivers/staging/ctagmod/ctagmod.c:20 [ctagmod] func:ctagmod_start + 4.1M 4 net/netfilter/nf_conntrack_core.c:2567 func:nf_ct_alloc_hashtable + 6.0M 1532 mm/filemap.c:1919 func:__filemap_get_folio + 8.8M 2785 kernel/fork.c:307 func:alloc_thread_stack_node + 13M 234 block/blk-mq.c:3421 func:blk_mq_alloc_rqs + 14M 3520 mm/mm_init.c:2530 func:alloc_large_system_hash + 15M 3656 mm/readahead.c:247 func:page_cache_ra_unbounded + 55M 4887 mm/slub.c:2259 func:alloc_slab_page + 122M 31168 mm/page_ext.c:270 func:alloc_page_ext + +=================== +Theory of operation +=================== + +Memory allocation profiling builds off of code tagging, which is a library for +declaring static structs (that typically describe a file and line number in +some way, hence code tagging) and then finding and operating on them at runtime, +- i.e. iterating over them to print them in debugfs/procfs. + +To add accounting for an allocation call, we replace it with a macro +invocation, alloc_hooks(), that +- declares a code tag +- stashes a pointer to it in task_struct +- calls the real allocation function +- and finally, restores the task_struct alloc tag pointer to its previous value. + +This allows for alloc_hooks() calls to be nested, with the most recent one +taking effect. This is important for allocations internal to the mm/ code that +do not properly belong to the outer allocation context and should be counted +separately: for example, slab object extension vectors, or when the slab +allocates pages from the page allocator. + +Thus, proper usage requires determining which function in an allocation call +stack should be tagged. There are many helper functions that essentially wrap +e.g. kmalloc() and do a little more work, then are called in multiple places; +we'll generally want the accounting to happen in the callers of these helpers, +not in the helpers themselves. + +To fix up a given helper, for example foo(), do the following: +- switch its allocation call to the _noprof() version, e.g. kmalloc_noprof() + +- rename it to foo_noprof() + +- define a macro version of foo() like so: + + #define foo(...) alloc_hooks(foo_noprof(__VA_ARGS__)) + +It's also possible to stash a pointer to an alloc tag in your own data structures. + +Do this when you're implementing a generic data structure that does allocations +"on behalf of" some other code - for example, the rhashtable code. This way, +instead of seeing a large line in /proc/allocinfo for rhashtable.c, we can +break it out by rhashtable type. + +To do so: +- Hook your data structure's init function, like any other allocation function. + +- Within your init function, use the convenience macro alloc_tag_record() to + record alloc tag in your data structure. + +- Then, use the following form for your allocations: + alloc_hooks_tag(ht->your_saved_tag, kmalloc_noprof(...)) diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst index 31d2ac3064387..48b9b559ca7b4 100644 --- a/Documentation/mm/index.rst +++ b/Documentation/mm/index.rst @@ -26,6 +26,7 @@ see the :doc:`admin guide <../admin-guide/mm/index>`. page_cache shmfs oom + allocation-profiling Legacy Documentation ==================== From 2c321f3f70bc284510598f712b702ce8d60c4d14 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Sun, 14 Apr 2024 19:07:31 -0700 Subject: [PATCH 0592/1702] mm: change inlined allocation helpers to account at the call site Main goal of memory allocation profiling patchset is to provide accounting that is cheap enough to run in production. To achieve that we inject counters using codetags at the allocation call sites to account every time allocation is made. This injection allows us to perform accounting efficiently because injected counters are immediately available as opposed to the alternative methods, such as using _RET_IP_, which would require counter lookup and appropriate locking that makes accounting much more expensive. This method requires all allocation functions to inject separate counters at their call sites so that their callers can be individually accounted. Counter injection is implemented by allocation hooks which should wrap all allocation functions. Inlined functions which perform allocations but do not use allocation hooks are directly charged for the allocations they perform. In most cases these functions are just specialized allocation wrappers used from multiple places to allocate objects of a specific type. It would be more useful to do the accounting at their call sites instead. Instrument these helpers to do accounting at the call site. Simple inlined allocation wrappers are converted directly into macros. More complex allocators or allocators with documentation are converted into _noprof versions and allocation hooks are added. This allows memory allocation profiling mechanism to charge allocations to the callers of these functions. Link: https://lkml.kernel.org/r/20240415020731.1152108-1-surenb@google.com Signed-off-by: Suren Baghdasaryan Acked-by: Jan Kara [jbd2] Cc: Anna Schumaker Cc: Arnd Bergmann Cc: Benjamin Tissoires Cc: Christoph Lameter Cc: David Rientjes Cc: David S. Miller Cc: Dennis Zhou Cc: Eric Dumazet Cc: Herbert Xu Cc: Jakub Kicinski Cc: Jakub Sitnicki Cc: Jiri Kosina Cc: Joerg Roedel Cc: Joonsoo Kim Cc: Kent Overstreet Cc: Matthew Wilcox (Oracle) Cc: Paolo Abeni Cc: Pekka Enberg Cc: Tejun Heo Cc: Theodore Ts'o Cc: Trond Myklebust Cc: Vlastimil Babka Cc: Will Deacon Signed-off-by: Andrew Morton --- drivers/iommu/amd/amd_iommu.h | 5 ++-- fs/nfs/iostat.h | 5 +--- include/acpi/platform/aclinuxex.h | 19 +++++--------- include/asm-generic/pgalloc.h | 35 +++++++++++++++---------- include/crypto/hash.h | 7 ++--- include/crypto/internal/acompress.h | 5 ++-- include/crypto/skcipher.h | 7 ++--- include/linux/bpf.h | 33 ++++++----------------- include/linux/bpfptr.h | 5 ++-- include/linux/dma-fence-chain.h | 6 ++--- include/linux/hid_bpf.h | 6 ++--- include/linux/jbd2.h | 12 +++------ include/linux/mm.h | 5 ++-- include/linux/mm_types.h | 5 ++-- include/linux/percpu.h | 3 +++ include/linux/ptr_ring.h | 28 +++++++++++--------- include/linux/skb_array.h | 19 ++++++++------ include/linux/skbuff.h | 20 ++++++-------- include/linux/skmsg.h | 8 +++--- include/linux/slab.h | 5 ++++ include/linux/sockptr.h | 10 ++++--- include/net/netlabel.h | 16 ++++++----- include/net/netlink.h | 5 ++-- include/net/request_sock.h | 5 ++-- include/net/tcx.h | 5 ++-- net/sunrpc/auth_gss/auth_gss_internal.h | 6 +++-- 26 files changed, 142 insertions(+), 143 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index f482aab420f78..52575ba9a141f 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -134,13 +134,14 @@ static inline int get_pci_sbdf_id(struct pci_dev *pdev) return PCI_SEG_DEVID_TO_SBDF(seg, devid); } -static inline void *alloc_pgtable_page(int nid, gfp_t gfp) +static inline void *alloc_pgtable_page_noprof(int nid, gfp_t gfp) { struct page *page; - page = alloc_pages_node(nid, gfp | __GFP_ZERO, 0); + page = alloc_pages_node_noprof(nid, gfp | __GFP_ZERO, 0); return page ? page_address(page) : NULL; } +#define alloc_pgtable_page(...) alloc_hooks(alloc_pgtable_page_noprof(__VA_ARGS__)) /* * This must be called after device probe completes. During probe diff --git a/fs/nfs/iostat.h b/fs/nfs/iostat.h index 5aa776b5a3e7f..b17a9eb9b1488 100644 --- a/fs/nfs/iostat.h +++ b/fs/nfs/iostat.h @@ -46,10 +46,7 @@ static inline void nfs_add_stats(const struct inode *inode, nfs_add_server_stats(NFS_SERVER(inode), stat, addend); } -static inline struct nfs_iostats __percpu *nfs_alloc_iostats(void) -{ - return alloc_percpu(struct nfs_iostats); -} +#define nfs_alloc_iostats() alloc_percpu(struct nfs_iostats) static inline void nfs_free_iostats(struct nfs_iostats __percpu *stats) { diff --git a/include/acpi/platform/aclinuxex.h b/include/acpi/platform/aclinuxex.h index 600d4e2641dae..62cac266a1c89 100644 --- a/include/acpi/platform/aclinuxex.h +++ b/include/acpi/platform/aclinuxex.h @@ -47,26 +47,19 @@ acpi_status acpi_os_terminate(void); * However, boot has (system_state != SYSTEM_RUNNING) * to quiet __might_sleep() in kmalloc() and resume does not. */ -static inline void *acpi_os_allocate(acpi_size size) -{ - return kmalloc(size, irqs_disabled()? GFP_ATOMIC : GFP_KERNEL); -} +#define acpi_os_allocate(_size) \ + kmalloc(_size, irqs_disabled() ? GFP_ATOMIC : GFP_KERNEL) -static inline void *acpi_os_allocate_zeroed(acpi_size size) -{ - return kzalloc(size, irqs_disabled()? GFP_ATOMIC : GFP_KERNEL); -} +#define acpi_os_allocate_zeroed(_size) \ + kzalloc(_size, irqs_disabled() ? GFP_ATOMIC : GFP_KERNEL) static inline void acpi_os_free(void *memory) { kfree(memory); } -static inline void *acpi_os_acquire_object(acpi_cache_t * cache) -{ - return kmem_cache_zalloc(cache, - irqs_disabled()? GFP_ATOMIC : GFP_KERNEL); -} +#define acpi_os_acquire_object(_cache) \ + kmem_cache_zalloc(_cache, irqs_disabled() ? GFP_ATOMIC : GFP_KERNEL) static inline acpi_thread_id acpi_os_get_thread_id(void) { diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h index 879e5f8aa5e92..7c48f5fbf8aa7 100644 --- a/include/asm-generic/pgalloc.h +++ b/include/asm-generic/pgalloc.h @@ -16,15 +16,16 @@ * * Return: pointer to the allocated memory or %NULL on error */ -static inline pte_t *__pte_alloc_one_kernel(struct mm_struct *mm) +static inline pte_t *__pte_alloc_one_kernel_noprof(struct mm_struct *mm) { - struct ptdesc *ptdesc = pagetable_alloc(GFP_PGTABLE_KERNEL & + struct ptdesc *ptdesc = pagetable_alloc_noprof(GFP_PGTABLE_KERNEL & ~__GFP_HIGHMEM, 0); if (!ptdesc) return NULL; return ptdesc_address(ptdesc); } +#define __pte_alloc_one_kernel(...) alloc_hooks(__pte_alloc_one_kernel_noprof(__VA_ARGS__)) #ifndef __HAVE_ARCH_PTE_ALLOC_ONE_KERNEL /** @@ -33,10 +34,11 @@ static inline pte_t *__pte_alloc_one_kernel(struct mm_struct *mm) * * Return: pointer to the allocated memory or %NULL on error */ -static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) +static inline pte_t *pte_alloc_one_kernel_noprof(struct mm_struct *mm) { - return __pte_alloc_one_kernel(mm); + return __pte_alloc_one_kernel_noprof(mm); } +#define pte_alloc_one_kernel(...) alloc_hooks(pte_alloc_one_kernel_noprof(__VA_ARGS__)) #endif /** @@ -61,11 +63,11 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) * * Return: `struct page` referencing the ptdesc or %NULL on error */ -static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp) +static inline pgtable_t __pte_alloc_one_noprof(struct mm_struct *mm, gfp_t gfp) { struct ptdesc *ptdesc; - ptdesc = pagetable_alloc(gfp, 0); + ptdesc = pagetable_alloc_noprof(gfp, 0); if (!ptdesc) return NULL; if (!pagetable_pte_ctor(ptdesc)) { @@ -75,6 +77,7 @@ static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp) return ptdesc_page(ptdesc); } +#define __pte_alloc_one(...) alloc_hooks(__pte_alloc_one_noprof(__VA_ARGS__)) #ifndef __HAVE_ARCH_PTE_ALLOC_ONE /** @@ -85,10 +88,11 @@ static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp) * * Return: `struct page` referencing the ptdesc or %NULL on error */ -static inline pgtable_t pte_alloc_one(struct mm_struct *mm) +static inline pgtable_t pte_alloc_one_noprof(struct mm_struct *mm) { - return __pte_alloc_one(mm, GFP_PGTABLE_USER); + return __pte_alloc_one_noprof(mm, GFP_PGTABLE_USER); } +#define pte_alloc_one(...) alloc_hooks(pte_alloc_one_noprof(__VA_ARGS__)) #endif /* @@ -124,14 +128,14 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page) * * Return: pointer to the allocated memory or %NULL on error */ -static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) +static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) { struct ptdesc *ptdesc; gfp_t gfp = GFP_PGTABLE_USER; if (mm == &init_mm) gfp = GFP_PGTABLE_KERNEL; - ptdesc = pagetable_alloc(gfp, 0); + ptdesc = pagetable_alloc_noprof(gfp, 0); if (!ptdesc) return NULL; if (!pagetable_pmd_ctor(ptdesc)) { @@ -140,6 +144,7 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) } return ptdesc_address(ptdesc); } +#define pmd_alloc_one(...) alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__)) #endif #ifndef __HAVE_ARCH_PMD_FREE @@ -157,7 +162,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) #if CONFIG_PGTABLE_LEVELS > 3 -static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr) +static inline pud_t *__pud_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) { gfp_t gfp = GFP_PGTABLE_USER; struct ptdesc *ptdesc; @@ -166,13 +171,14 @@ static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr) gfp = GFP_PGTABLE_KERNEL; gfp &= ~__GFP_HIGHMEM; - ptdesc = pagetable_alloc(gfp, 0); + ptdesc = pagetable_alloc_noprof(gfp, 0); if (!ptdesc) return NULL; pagetable_pud_ctor(ptdesc); return ptdesc_address(ptdesc); } +#define __pud_alloc_one(...) alloc_hooks(__pud_alloc_one_noprof(__VA_ARGS__)) #ifndef __HAVE_ARCH_PUD_ALLOC_ONE /** @@ -184,10 +190,11 @@ static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr) * * Return: pointer to the allocated memory or %NULL on error */ -static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) +static inline pud_t *pud_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) { - return __pud_alloc_one(mm, addr); + return __pud_alloc_one_noprof(mm, addr); } +#define pud_alloc_one(...) alloc_hooks(pud_alloc_one_noprof(__VA_ARGS__)) #endif static inline void __pud_free(struct mm_struct *mm, pud_t *pud) diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 5d61f576cfc86..e5181cc9b7c56 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -578,19 +578,20 @@ static inline void ahash_request_set_tfm(struct ahash_request *req, * * Return: allocated request handle in case of success, or NULL if out of memory */ -static inline struct ahash_request *ahash_request_alloc( +static inline struct ahash_request *ahash_request_alloc_noprof( struct crypto_ahash *tfm, gfp_t gfp) { struct ahash_request *req; - req = kmalloc(sizeof(struct ahash_request) + - crypto_ahash_reqsize(tfm), gfp); + req = kmalloc_noprof(sizeof(struct ahash_request) + + crypto_ahash_reqsize(tfm), gfp); if (likely(req)) ahash_request_set_tfm(req, tfm); return req; } +#define ahash_request_alloc(...) alloc_hooks(ahash_request_alloc_noprof(__VA_ARGS__)) /** * ahash_request_free() - zeroize and free the request data structure diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 4ac46bafba9d7..2a67793f52ad8 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -69,15 +69,16 @@ static inline void acomp_request_complete(struct acomp_req *req, crypto_request_complete(&req->base, err); } -static inline struct acomp_req *__acomp_request_alloc(struct crypto_acomp *tfm) +static inline struct acomp_req *__acomp_request_alloc_noprof(struct crypto_acomp *tfm) { struct acomp_req *req; - req = kzalloc(sizeof(*req) + crypto_acomp_reqsize(tfm), GFP_KERNEL); + req = kzalloc_noprof(sizeof(*req) + crypto_acomp_reqsize(tfm), GFP_KERNEL); if (likely(req)) acomp_request_set_tfm(req, tfm); return req; } +#define __acomp_request_alloc(...) alloc_hooks(__acomp_request_alloc_noprof(__VA_ARGS__)) static inline void __acomp_request_free(struct acomp_req *req) { diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h index c8857d7bdb37f..6c5330e316b0e 100644 --- a/include/crypto/skcipher.h +++ b/include/crypto/skcipher.h @@ -861,19 +861,20 @@ static inline struct skcipher_request *skcipher_request_cast( * * Return: allocated request handle in case of success, or NULL if out of memory */ -static inline struct skcipher_request *skcipher_request_alloc( +static inline struct skcipher_request *skcipher_request_alloc_noprof( struct crypto_skcipher *tfm, gfp_t gfp) { struct skcipher_request *req; - req = kmalloc(sizeof(struct skcipher_request) + - crypto_skcipher_reqsize(tfm), gfp); + req = kmalloc_noprof(sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(tfm), gfp); if (likely(req)) skcipher_request_set_tfm(req, tfm); return req; } +#define skcipher_request_alloc(...) alloc_hooks(skcipher_request_alloc_noprof(__VA_ARGS__)) /** * skcipher_request_free() - zeroize and free request data structure diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 890e152d553ea..a86da1f38b3c3 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2244,31 +2244,14 @@ void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size, void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align, gfp_t flags); #else -static inline void * -bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, - int node) -{ - return kmalloc_node(size, flags, node); -} - -static inline void * -bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) -{ - return kzalloc(size, flags); -} - -static inline void * -bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size, gfp_t flags) -{ - return kvcalloc(n, size, flags); -} - -static inline void __percpu * -bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align, - gfp_t flags) -{ - return __alloc_percpu_gfp(size, align, flags); -} +#define bpf_map_kmalloc_node(_map, _size, _flags, _node) \ + kmalloc_node(_size, _flags, _node) +#define bpf_map_kzalloc(_map, _size, _flags) \ + kzalloc(_size, _flags) +#define bpf_map_kvcalloc(_map, _n, _size, _flags) \ + kvcalloc(_n, _size, _flags) +#define bpf_map_alloc_percpu(_map, _size, _align, _flags) \ + __alloc_percpu_gfp(_size, _align, _flags) #endif static inline int diff --git a/include/linux/bpfptr.h b/include/linux/bpfptr.h index 79b2f78eec1a0..1af241525a17c 100644 --- a/include/linux/bpfptr.h +++ b/include/linux/bpfptr.h @@ -65,9 +65,9 @@ static inline int copy_to_bpfptr_offset(bpfptr_t dst, size_t offset, return copy_to_sockptr_offset((sockptr_t) dst, offset, src, size); } -static inline void *kvmemdup_bpfptr(bpfptr_t src, size_t len) +static inline void *kvmemdup_bpfptr_noprof(bpfptr_t src, size_t len) { - void *p = kvmalloc(len, GFP_USER | __GFP_NOWARN); + void *p = kvmalloc_noprof(len, GFP_USER | __GFP_NOWARN); if (!p) return ERR_PTR(-ENOMEM); @@ -77,6 +77,7 @@ static inline void *kvmemdup_bpfptr(bpfptr_t src, size_t len) } return p; } +#define kvmemdup_bpfptr(...) alloc_hooks(kvmemdup_bpfptr_noprof(__VA_ARGS__)) static inline long strncpy_from_bpfptr(char *dst, bpfptr_t src, size_t count) { diff --git a/include/linux/dma-fence-chain.h b/include/linux/dma-fence-chain.h index 4bdf0b96da283..ad9e2506c2f40 100644 --- a/include/linux/dma-fence-chain.h +++ b/include/linux/dma-fence-chain.h @@ -86,10 +86,8 @@ dma_fence_chain_contained(struct dma_fence *fence) * * Returns a new struct dma_fence_chain object or NULL on failure. */ -static inline struct dma_fence_chain *dma_fence_chain_alloc(void) -{ - return kmalloc(sizeof(struct dma_fence_chain), GFP_KERNEL); -}; +#define dma_fence_chain_alloc() \ + ((struct dma_fence_chain *)kmalloc(sizeof(struct dma_fence_chain), GFP_KERNEL)) /** * dma_fence_chain_free diff --git a/include/linux/hid_bpf.h b/include/linux/hid_bpf.h index 7118ac28d4687..ca70eeb6393e7 100644 --- a/include/linux/hid_bpf.h +++ b/include/linux/hid_bpf.h @@ -149,10 +149,8 @@ static inline int hid_bpf_connect_device(struct hid_device *hdev) { return 0; } static inline void hid_bpf_disconnect_device(struct hid_device *hdev) {} static inline void hid_bpf_destroy_device(struct hid_device *hid) {} static inline void hid_bpf_device_init(struct hid_device *hid) {} -static inline u8 *call_hid_bpf_rdesc_fixup(struct hid_device *hdev, u8 *rdesc, unsigned int *size) -{ - return kmemdup(rdesc, *size, GFP_KERNEL); -} +#define call_hid_bpf_rdesc_fixup(_hdev, _rdesc, _size) \ + ((u8 *)kmemdup(_rdesc, *(_size), GFP_KERNEL)) #endif /* CONFIG_HID_BPF */ diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h index 971f3e826e152..9512fe3326684 100644 --- a/include/linux/jbd2.h +++ b/include/linux/jbd2.h @@ -1586,10 +1586,8 @@ void jbd2_journal_put_journal_head(struct journal_head *jh); */ extern struct kmem_cache *jbd2_handle_cache; -static inline handle_t *jbd2_alloc_handle(gfp_t gfp_flags) -{ - return kmem_cache_zalloc(jbd2_handle_cache, gfp_flags); -} +#define jbd2_alloc_handle(_gfp_flags) \ + ((handle_t *)kmem_cache_zalloc(jbd2_handle_cache, _gfp_flags)) static inline void jbd2_free_handle(handle_t *handle) { @@ -1602,10 +1600,8 @@ static inline void jbd2_free_handle(handle_t *handle) */ extern struct kmem_cache *jbd2_inode_cache; -static inline struct jbd2_inode *jbd2_alloc_inode(gfp_t gfp_flags) -{ - return kmem_cache_alloc(jbd2_inode_cache, gfp_flags); -} +#define jbd2_alloc_inode(_gfp_flags) \ + ((struct jbd2_inode *)kmem_cache_alloc(jbd2_inode_cache, _gfp_flags)) static inline void jbd2_free_inode(struct jbd2_inode *jinode) { diff --git a/include/linux/mm.h b/include/linux/mm.h index 0dbd737cfabdc..60eabc6c8e007 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2860,12 +2860,13 @@ static inline bool pagetable_is_reserved(struct ptdesc *pt) * * Return: The ptdesc describing the allocated page tables. */ -static inline struct ptdesc *pagetable_alloc(gfp_t gfp, unsigned int order) +static inline struct ptdesc *pagetable_alloc_noprof(gfp_t gfp, unsigned int order) { - struct page *page = alloc_pages(gfp | __GFP_COMP, order); + struct page *page = alloc_pages_noprof(gfp | __GFP_COMP, order); return page_ptdesc(page); } +#define pagetable_alloc(...) alloc_hooks(pagetable_alloc_noprof(__VA_ARGS__)) /** * pagetable_free - Free pagetables diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 4ae4684d1add2..6fef56938a175 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1170,14 +1170,15 @@ static inline void mm_init_cid(struct mm_struct *mm) cpumask_clear(mm_cidmask(mm)); } -static inline int mm_alloc_cid(struct mm_struct *mm) +static inline int mm_alloc_cid_noprof(struct mm_struct *mm) { - mm->pcpu_cid = alloc_percpu(struct mm_cid); + mm->pcpu_cid = alloc_percpu_noprof(struct mm_cid); if (!mm->pcpu_cid) return -ENOMEM; mm_init_cid(mm); return 0; } +#define mm_alloc_cid(...) alloc_hooks(mm_alloc_cid_noprof(__VA_ARGS__)) static inline void mm_destroy_cid(struct mm_struct *mm) { diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 13a82f11e4fd2..03053de557cf6 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -151,6 +151,9 @@ extern size_t pcpu_alloc_size(void __percpu *__pdata); #define alloc_percpu(type) \ (typeof(type) __percpu *)__alloc_percpu(sizeof(type), \ __alignof__(type)) +#define alloc_percpu_noprof(type) \ + ((typeof(type) __percpu *)pcpu_alloc_noprof(sizeof(type), \ + __alignof__(type), false, GFP_KERNEL)) extern void free_percpu(void __percpu *__pdata); diff --git a/include/linux/ptr_ring.h b/include/linux/ptr_ring.h index 808f9d3ee5465..fd037c127bb07 100644 --- a/include/linux/ptr_ring.h +++ b/include/linux/ptr_ring.h @@ -464,11 +464,11 @@ static inline int ptr_ring_consume_batched_bh(struct ptr_ring *r, /* Not all gfp_t flags (besides GFP_KERNEL) are allowed. See * documentation for vmalloc for which of them are legal. */ -static inline void **__ptr_ring_init_queue_alloc(unsigned int size, gfp_t gfp) +static inline void **__ptr_ring_init_queue_alloc_noprof(unsigned int size, gfp_t gfp) { if (size > KMALLOC_MAX_SIZE / sizeof(void *)) return NULL; - return kvmalloc_array(size, sizeof(void *), gfp | __GFP_ZERO); + return kvmalloc_array_noprof(size, sizeof(void *), gfp | __GFP_ZERO); } static inline void __ptr_ring_set_size(struct ptr_ring *r, int size) @@ -484,9 +484,9 @@ static inline void __ptr_ring_set_size(struct ptr_ring *r, int size) r->batch = 1; } -static inline int ptr_ring_init(struct ptr_ring *r, int size, gfp_t gfp) +static inline int ptr_ring_init_noprof(struct ptr_ring *r, int size, gfp_t gfp) { - r->queue = __ptr_ring_init_queue_alloc(size, gfp); + r->queue = __ptr_ring_init_queue_alloc_noprof(size, gfp); if (!r->queue) return -ENOMEM; @@ -497,6 +497,7 @@ static inline int ptr_ring_init(struct ptr_ring *r, int size, gfp_t gfp) return 0; } +#define ptr_ring_init(...) alloc_hooks(ptr_ring_init_noprof(__VA_ARGS__)) /* * Return entries into ring. Destroy entries that don't fit. @@ -587,11 +588,11 @@ static inline void **__ptr_ring_swap_queue(struct ptr_ring *r, void **queue, * In particular if you consume ring in interrupt or BH context, you must * disable interrupts/BH when doing so. */ -static inline int ptr_ring_resize(struct ptr_ring *r, int size, gfp_t gfp, +static inline int ptr_ring_resize_noprof(struct ptr_ring *r, int size, gfp_t gfp, void (*destroy)(void *)) { unsigned long flags; - void **queue = __ptr_ring_init_queue_alloc(size, gfp); + void **queue = __ptr_ring_init_queue_alloc_noprof(size, gfp); void **old; if (!queue) @@ -609,6 +610,7 @@ static inline int ptr_ring_resize(struct ptr_ring *r, int size, gfp_t gfp, return 0; } +#define ptr_ring_resize(...) alloc_hooks(ptr_ring_resize_noprof(__VA_ARGS__)) /* * Note: producer lock is nested within consumer lock, so if you @@ -616,21 +618,21 @@ static inline int ptr_ring_resize(struct ptr_ring *r, int size, gfp_t gfp, * In particular if you consume ring in interrupt or BH context, you must * disable interrupts/BH when doing so. */ -static inline int ptr_ring_resize_multiple(struct ptr_ring **rings, - unsigned int nrings, - int size, - gfp_t gfp, void (*destroy)(void *)) +static inline int ptr_ring_resize_multiple_noprof(struct ptr_ring **rings, + unsigned int nrings, + int size, + gfp_t gfp, void (*destroy)(void *)) { unsigned long flags; void ***queues; int i; - queues = kmalloc_array(nrings, sizeof(*queues), gfp); + queues = kmalloc_array_noprof(nrings, sizeof(*queues), gfp); if (!queues) goto noqueues; for (i = 0; i < nrings; ++i) { - queues[i] = __ptr_ring_init_queue_alloc(size, gfp); + queues[i] = __ptr_ring_init_queue_alloc_noprof(size, gfp); if (!queues[i]) goto nomem; } @@ -660,6 +662,8 @@ static inline int ptr_ring_resize_multiple(struct ptr_ring **rings, noqueues: return -ENOMEM; } +#define ptr_ring_resize_multiple(...) \ + alloc_hooks(ptr_ring_resize_multiple_noprof(__VA_ARGS__)) static inline void ptr_ring_cleanup(struct ptr_ring *r, void (*destroy)(void *)) { diff --git a/include/linux/skb_array.h b/include/linux/skb_array.h index e2d45b7cb6198..926496c9cc9c3 100644 --- a/include/linux/skb_array.h +++ b/include/linux/skb_array.h @@ -177,10 +177,11 @@ static inline int skb_array_peek_len_any(struct skb_array *a) return PTR_RING_PEEK_CALL_ANY(&a->ring, __skb_array_len_with_tag); } -static inline int skb_array_init(struct skb_array *a, int size, gfp_t gfp) +static inline int skb_array_init_noprof(struct skb_array *a, int size, gfp_t gfp) { - return ptr_ring_init(&a->ring, size, gfp); + return ptr_ring_init_noprof(&a->ring, size, gfp); } +#define skb_array_init(...) alloc_hooks(skb_array_init_noprof(__VA_ARGS__)) static void __skb_array_destroy_skb(void *ptr) { @@ -198,15 +199,17 @@ static inline int skb_array_resize(struct skb_array *a, int size, gfp_t gfp) return ptr_ring_resize(&a->ring, size, gfp, __skb_array_destroy_skb); } -static inline int skb_array_resize_multiple(struct skb_array **rings, - int nrings, unsigned int size, - gfp_t gfp) +static inline int skb_array_resize_multiple_noprof(struct skb_array **rings, + int nrings, unsigned int size, + gfp_t gfp) { BUILD_BUG_ON(offsetof(struct skb_array, ring)); - return ptr_ring_resize_multiple((struct ptr_ring **)rings, - nrings, size, gfp, - __skb_array_destroy_skb); + return ptr_ring_resize_multiple_noprof((struct ptr_ring **)rings, + nrings, size, gfp, + __skb_array_destroy_skb); } +#define skb_array_resize_multiple(...) \ + alloc_hooks(skb_array_resize_multiple_noprof(__VA_ARGS__)) static inline void skb_array_cleanup(struct skb_array *a) { diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 9d24aec064e88..b72920c9887b7 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3371,7 +3371,7 @@ void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason); * * %NULL is returned if there is no free memory. */ -static inline struct page *__dev_alloc_pages(gfp_t gfp_mask, +static inline struct page *__dev_alloc_pages_noprof(gfp_t gfp_mask, unsigned int order) { /* This piece of code contains several assumptions. @@ -3384,13 +3384,11 @@ static inline struct page *__dev_alloc_pages(gfp_t gfp_mask, */ gfp_mask |= __GFP_COMP | __GFP_MEMALLOC; - return alloc_pages_node(NUMA_NO_NODE, gfp_mask, order); + return alloc_pages_node_noprof(NUMA_NO_NODE, gfp_mask, order); } +#define __dev_alloc_pages(...) alloc_hooks(__dev_alloc_pages_noprof(__VA_ARGS__)) -static inline struct page *dev_alloc_pages(unsigned int order) -{ - return __dev_alloc_pages(GFP_ATOMIC | __GFP_NOWARN, order); -} +#define dev_alloc_pages(_order) __dev_alloc_pages(GFP_ATOMIC | __GFP_NOWARN, _order) /** * __dev_alloc_page - allocate a page for network Rx @@ -3400,15 +3398,13 @@ static inline struct page *dev_alloc_pages(unsigned int order) * * %NULL is returned if there is no free memory. */ -static inline struct page *__dev_alloc_page(gfp_t gfp_mask) +static inline struct page *__dev_alloc_page_noprof(gfp_t gfp_mask) { - return __dev_alloc_pages(gfp_mask, 0); + return __dev_alloc_pages_noprof(gfp_mask, 0); } +#define __dev_alloc_page(...) alloc_hooks(__dev_alloc_page_noprof(__VA_ARGS__)) -static inline struct page *dev_alloc_page(void) -{ - return dev_alloc_pages(0); -} +#define dev_alloc_page() dev_alloc_pages(0) /** * dev_page_is_reusable - check whether a page can be reused for network Rx diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index e65ec3fd27998..78efc5b20284a 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -410,11 +410,9 @@ void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock); int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock, struct sk_msg *msg); -static inline struct sk_psock_link *sk_psock_init_link(void) -{ - return kzalloc(sizeof(struct sk_psock_link), - GFP_ATOMIC | __GFP_NOWARN); -} +#define sk_psock_init_link() \ + ((struct sk_psock_link *)kzalloc(sizeof(struct sk_psock_link), \ + GFP_ATOMIC | __GFP_NOWARN)) static inline void sk_psock_free_link(struct sk_psock_link *link) { diff --git a/include/linux/slab.h b/include/linux/slab.h index c0be1cd03cf6a..4cc37ef22aaed 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -744,6 +744,9 @@ void *kmalloc_node_track_caller_noprof(size_t size, gfp_t flags, int node, */ #define kmalloc_track_caller(...) kmalloc_node_track_caller(__VA_ARGS__, NUMA_NO_NODE) +#define kmalloc_track_caller_noprof(...) \ + kmalloc_node_track_caller_noprof(__VA_ARGS__, NUMA_NO_NODE, _RET_IP_) + static inline __alloc_size(1, 2) void *kmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node) { @@ -781,6 +784,7 @@ extern void *kvmalloc_node_noprof(size_t size, gfp_t flags, int node) __alloc_si #define kvmalloc_node(...) alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__)) #define kvmalloc(_size, _flags) kvmalloc_node(_size, _flags, NUMA_NO_NODE) +#define kvmalloc_noprof(_size, _flags) kvmalloc_node_noprof(_size, _flags, NUMA_NO_NODE) #define kvzalloc(_size, _flags) kvmalloc(_size, _flags|__GFP_ZERO) #define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, _flags|__GFP_ZERO, _node) @@ -797,6 +801,7 @@ static inline __alloc_size(1, 2) void *kvmalloc_array_noprof(size_t n, size_t si #define kvmalloc_array(...) alloc_hooks(kvmalloc_array_noprof(__VA_ARGS__)) #define kvcalloc(_n, _size, _flags) kvmalloc_array(_n, _size, _flags|__GFP_ZERO) +#define kvcalloc_noprof(_n, _size, _flags) kvmalloc_array_noprof(_n, _size, _flags|__GFP_ZERO) extern void *kvrealloc_noprof(const void *p, size_t oldsize, size_t newsize, gfp_t flags) __realloc_size(3); diff --git a/include/linux/sockptr.h b/include/linux/sockptr.h index 317200cd3a603..fc5a206c40435 100644 --- a/include/linux/sockptr.h +++ b/include/linux/sockptr.h @@ -117,9 +117,9 @@ static inline int copy_to_sockptr(sockptr_t dst, const void *src, size_t size) return copy_to_sockptr_offset(dst, 0, src, size); } -static inline void *memdup_sockptr(sockptr_t src, size_t len) +static inline void *memdup_sockptr_noprof(sockptr_t src, size_t len) { - void *p = kmalloc_track_caller(len, GFP_USER | __GFP_NOWARN); + void *p = kmalloc_track_caller_noprof(len, GFP_USER | __GFP_NOWARN); if (!p) return ERR_PTR(-ENOMEM); @@ -129,10 +129,11 @@ static inline void *memdup_sockptr(sockptr_t src, size_t len) } return p; } +#define memdup_sockptr(...) alloc_hooks(memdup_sockptr_noprof(__VA_ARGS__)) -static inline void *memdup_sockptr_nul(sockptr_t src, size_t len) +static inline void *memdup_sockptr_nul_noprof(sockptr_t src, size_t len) { - char *p = kmalloc_track_caller(len + 1, GFP_KERNEL); + char *p = kmalloc_track_caller_noprof(len + 1, GFP_KERNEL); if (!p) return ERR_PTR(-ENOMEM); @@ -143,6 +144,7 @@ static inline void *memdup_sockptr_nul(sockptr_t src, size_t len) p[len] = '\0'; return p; } +#define memdup_sockptr_nul(...) alloc_hooks(memdup_sockptr_nul_noprof(__VA_ARGS__)) static inline long strncpy_from_sockptr(char *dst, sockptr_t src, size_t count) { diff --git a/include/net/netlabel.h b/include/net/netlabel.h index f3ab0b8a4b18a..c31bd96dafdba 100644 --- a/include/net/netlabel.h +++ b/include/net/netlabel.h @@ -274,15 +274,17 @@ struct netlbl_calipso_ops { * on success, NULL on failure. * */ -static inline struct netlbl_lsm_cache *netlbl_secattr_cache_alloc(gfp_t flags) +static inline struct netlbl_lsm_cache *netlbl_secattr_cache_alloc_noprof(gfp_t flags) { struct netlbl_lsm_cache *cache; - cache = kzalloc(sizeof(*cache), flags); + cache = kzalloc_noprof(sizeof(*cache), flags); if (cache) refcount_set(&cache->refcount, 1); return cache; } +#define netlbl_secattr_cache_alloc(...) \ + alloc_hooks(netlbl_secattr_cache_alloc_noprof(__VA_ARGS__)) /** * netlbl_secattr_cache_free - Frees a netlbl_lsm_cache struct @@ -311,10 +313,11 @@ static inline void netlbl_secattr_cache_free(struct netlbl_lsm_cache *cache) * on failure. * */ -static inline struct netlbl_lsm_catmap *netlbl_catmap_alloc(gfp_t flags) +static inline struct netlbl_lsm_catmap *netlbl_catmap_alloc_noprof(gfp_t flags) { - return kzalloc(sizeof(struct netlbl_lsm_catmap), flags); + return kzalloc_noprof(sizeof(struct netlbl_lsm_catmap), flags); } +#define netlbl_catmap_alloc(...) alloc_hooks(netlbl_catmap_alloc_noprof(__VA_ARGS__)) /** * netlbl_catmap_free - Free a LSM secattr catmap @@ -376,10 +379,11 @@ static inline void netlbl_secattr_destroy(struct netlbl_lsm_secattr *secattr) * pointer on success, or NULL on failure. * */ -static inline struct netlbl_lsm_secattr *netlbl_secattr_alloc(gfp_t flags) +static inline struct netlbl_lsm_secattr *netlbl_secattr_alloc_noprof(gfp_t flags) { - return kzalloc(sizeof(struct netlbl_lsm_secattr), flags); + return kzalloc_noprof(sizeof(struct netlbl_lsm_secattr), flags); } +#define netlbl_secattr_alloc(...) alloc_hooks(netlbl_secattr_alloc_noprof(__VA_ARGS__)) /** * netlbl_secattr_free - Frees a netlbl_lsm_secattr struct diff --git a/include/net/netlink.h b/include/net/netlink.h index c19ff921b661a..972b5484fa6fc 100644 --- a/include/net/netlink.h +++ b/include/net/netlink.h @@ -1891,10 +1891,11 @@ static inline struct nla_bitfield32 nla_get_bitfield32(const struct nlattr *nla) * @src: netlink attribute to duplicate from * @gfp: GFP mask */ -static inline void *nla_memdup(const struct nlattr *src, gfp_t gfp) +static inline void *nla_memdup_noprof(const struct nlattr *src, gfp_t gfp) { - return kmemdup(nla_data(src), nla_len(src), gfp); + return kmemdup_noprof(nla_data(src), nla_len(src), gfp); } +#define nla_memdup(...) alloc_hooks(nla_memdup_noprof(__VA_ARGS__)) /** * nla_nest_start_noflag - Start a new level of nested attributes diff --git a/include/net/request_sock.h b/include/net/request_sock.h index 004e651e6067e..29495c331d20e 100644 --- a/include/net/request_sock.h +++ b/include/net/request_sock.h @@ -127,12 +127,12 @@ static inline struct sock *skb_steal_sock(struct sk_buff *skb, } static inline struct request_sock * -reqsk_alloc(const struct request_sock_ops *ops, struct sock *sk_listener, +reqsk_alloc_noprof(const struct request_sock_ops *ops, struct sock *sk_listener, bool attach_listener) { struct request_sock *req; - req = kmem_cache_alloc(ops->slab, GFP_ATOMIC | __GFP_NOWARN); + req = kmem_cache_alloc_noprof(ops->slab, GFP_ATOMIC | __GFP_NOWARN); if (!req) return NULL; req->rsk_listener = NULL; @@ -157,6 +157,7 @@ reqsk_alloc(const struct request_sock_ops *ops, struct sock *sk_listener, return req; } +#define reqsk_alloc(...) alloc_hooks(reqsk_alloc_noprof(__VA_ARGS__)) static inline void __reqsk_free(struct request_sock *req) { diff --git a/include/net/tcx.h b/include/net/tcx.h index 04be9377785d7..72a3e75e539fb 100644 --- a/include/net/tcx.h +++ b/include/net/tcx.h @@ -75,9 +75,9 @@ tcx_entry_fetch(struct net_device *dev, bool ingress) return rcu_dereference_rtnl(dev->tcx_egress); } -static inline struct bpf_mprog_entry *tcx_entry_create(void) +static inline struct bpf_mprog_entry *tcx_entry_create_noprof(void) { - struct tcx_entry *tcx = kzalloc(sizeof(*tcx), GFP_KERNEL); + struct tcx_entry *tcx = kzalloc_noprof(sizeof(*tcx), GFP_KERNEL); if (tcx) { bpf_mprog_bundle_init(&tcx->bundle); @@ -85,6 +85,7 @@ static inline struct bpf_mprog_entry *tcx_entry_create(void) } return NULL; } +#define tcx_entry_create(...) alloc_hooks(tcx_entry_create_noprof(__VA_ARGS__)) static inline void tcx_entry_free(struct bpf_mprog_entry *entry) { diff --git a/net/sunrpc/auth_gss/auth_gss_internal.h b/net/sunrpc/auth_gss/auth_gss_internal.h index c53b329092d40..4ebc1b7043d91 100644 --- a/net/sunrpc/auth_gss/auth_gss_internal.h +++ b/net/sunrpc/auth_gss/auth_gss_internal.h @@ -23,7 +23,7 @@ simple_get_bytes(const void *p, const void *end, void *res, size_t len) } static inline const void * -simple_get_netobj(const void *p, const void *end, struct xdr_netobj *dest) +simple_get_netobj_noprof(const void *p, const void *end, struct xdr_netobj *dest) { const void *q; unsigned int len; @@ -35,7 +35,7 @@ simple_get_netobj(const void *p, const void *end, struct xdr_netobj *dest) if (unlikely(q > end || q < p)) return ERR_PTR(-EFAULT); if (len) { - dest->data = kmemdup(p, len, GFP_KERNEL); + dest->data = kmemdup_noprof(p, len, GFP_KERNEL); if (unlikely(dest->data == NULL)) return ERR_PTR(-ENOMEM); } else @@ -43,3 +43,5 @@ simple_get_netobj(const void *p, const void *end, struct xdr_netobj *dest) dest->len = len; return q; } + +#define simple_get_netobj(...) alloc_hooks(simple_get_netobj_noprof(__VA_ARGS__)) From e13106952faad91c6e492bf23b7cbdf1b1c269ce Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 25 Apr 2024 13:08:50 -0700 Subject: [PATCH 0593/1702] alloc_tag: Tighten file permissions on /proc/allocinfo The /proc/allocinfo file exposes a tremendous about of information about kernel build details, memory allocations (obviously), and potentially even image layout (due to ordering). As this is intended to be consumed by system owners (like /proc/slabinfo), use the same file permissions as there: 0400. Link: https://lkml.kernel.org/r/20240425200844.work.184-kees@kernel.org Signed-off-by: Kees Cook Cc: Kent Overstreet Cc: Suren Baghdasaryan Signed-off-by: Andrew Morton --- lib/alloc_tag.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c index 26af9982ddc44..531dbe2f54563 100644 --- a/lib/alloc_tag.c +++ b/lib/alloc_tag.c @@ -129,7 +129,7 @@ size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sl static void __init procfs_init(void) { - proc_create_seq("allocinfo", 0444, NULL, &allocinfo_seq_op); + proc_create_seq("allocinfo", 0400, NULL, &allocinfo_seq_op); } static bool alloc_tag_module_unload(struct codetag_type *cttype, From 3b89ec41747a6b6b8c7b6ad4fe13e063cb6dfe7f Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 25 Apr 2024 13:55:23 -0700 Subject: [PATCH 0594/1702] mm/slub: avoid recursive loop with kmemleak The system will immediate fill up stack and crash when both CONFIG_DEBUG_KMEMLEAK and CONFIG_MEM_ALLOC_PROFILING are enabled. Avoid allocation tagging of kmemleak caches, otherwise recursive allocation tracking occurs. Link: https://lkml.kernel.org/r/20240425205516.work.220-kees@kernel.org Fixes: 279bb991b4d9 ("mm/slab: add allocation accounting into slab allocation and free paths") Signed-off-by: Kees Cook Cc: Catalin Marinas Cc: Christoph Lameter Cc: David Rientjes Cc: Hyeonggon Yoo <42.hyeyoo@gmail.com> Cc: Joonsoo Kim Cc: Kent Overstreet Cc: Pekka Enberg Cc: Roman Gushchin Cc: Suren Baghdasaryan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/kmemleak.c | 4 ++-- mm/slub.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 6a540c2b27c52..d2037073ed919 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -463,7 +463,7 @@ static struct kmemleak_object *mem_pool_alloc(gfp_t gfp) /* try the slab allocator first */ if (object_cache) { - object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp)); + object = kmem_cache_alloc_noprof(object_cache, gfp_kmemleak_mask(gfp)); if (object) return object; } @@ -947,7 +947,7 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp) untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); if (scan_area_cache) - area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp)); + area = kmem_cache_alloc_noprof(scan_area_cache, gfp_kmemleak_mask(gfp)); raw_spin_lock_irqsave(&object->lock, flags); if (!area) { diff --git a/mm/slub.c b/mm/slub.c index be047279c9e97..6437746be03d8 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2018,7 +2018,7 @@ prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p) if (!p) return NULL; - if (s->flags & SLAB_NO_OBJ_EXT) + if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE)) return NULL; if (flags & __GFP_NO_OBJ_EXT) From b7b098cf00a2b65d5654a86dc8edf82f125289c1 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 21 Mar 2024 14:24:39 +0000 Subject: [PATCH 0595/1702] mm: always initialise folio->_deferred_list Patch series "Various significant MM patches". These patches all interact in annoying ways which make it tricky to send them out in any way other than a big batch, even though there's not really an overarching theme to connect them. The big effects of this patch series are: - folio_test_hugetlb() becomes reliable, even when called without a page reference - We free up PG_slab, and we could always use more page flags - We no longer need to check PageSlab before calling page_mapcount() This patch (of 9): For compound pages which are at least order-2 (and hence have a deferred_list), initialise it and then we can check at free that the page is not part of a deferred list. We recently found this useful to rule out a source of corruption. [peterx@redhat.com: always initialise folio->_deferred_list] Link: https://lkml.kernel.org/r/20240417211836.2742593-2-peterx@redhat.com Link: https://lkml.kernel.org/r/20240321142448.1645400-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240321142448.1645400-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Peter Xu Reviewed-by: David Hildenbrand Acked-by: Vlastimil Babka Cc: Miaohe Lin Cc: Muchun Song Cc: Oscar Salvador Signed-off-by: Andrew Morton --- mm/huge_memory.c | 2 -- mm/hugetlb.c | 3 ++- mm/internal.h | 2 ++ mm/memcontrol.c | 3 +++ mm/page_alloc.c | 9 +++++---- 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 25191ab70631b..4cc7133aaa4be 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -793,8 +793,6 @@ void folio_prep_large_rmappable(struct folio *folio) { if (!folio || !folio_test_large(folio)) return; - if (folio_order(folio) > 1) - INIT_LIST_HEAD(&folio->_deferred_list); folio_set_large_rmappable(folio); } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index ce7be5c244429..378181547b7bb 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1796,7 +1796,8 @@ static void __update_and_free_hugetlb_folio(struct hstate *h, destroy_compound_gigantic_folio(folio, huge_page_order(h)); free_gigantic_folio(folio, huge_page_order(h)); } else { - __free_pages(&folio->page, huge_page_order(h)); + INIT_LIST_HEAD(&folio->_deferred_list); + folio_put(folio); } } diff --git a/mm/internal.h b/mm/internal.h index 85c3db43454de..5c0c57c9cd196 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -525,6 +525,8 @@ static inline void prep_compound_head(struct page *page, unsigned int order) atomic_set(&folio->_entire_mapcount, -1); atomic_set(&folio->_nr_pages_mapped, 0); atomic_set(&folio->_pincount, 0); + if (order > 1) + INIT_LIST_HEAD(&folio->_deferred_list); } static inline void prep_compound_tail(struct page *head, int tail_idx) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 896b4bf05b9cf..45dd209012827 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -7400,6 +7400,9 @@ static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug) struct obj_cgroup *objcg; VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); + VM_BUG_ON_FOLIO(folio_order(folio) > 1 && + !folio_test_hugetlb(folio) && + !list_empty(&folio->_deferred_list), folio); /* * Nobody should be changing or seriously looking at diff --git a/mm/page_alloc.c b/mm/page_alloc.c index e1241ecef2716..7e8f4b751801d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1007,10 +1007,11 @@ static int free_tail_page_prepare(struct page *head_page, struct page *page) } break; case 2: - /* - * the second tail page: ->mapping is - * deferred_list.next -- ignore value. - */ + /* the second tail page: deferred_list overlaps ->mapping */ + if (unlikely(!list_empty(&folio->_deferred_list))) { + bad_page(page, "on deferred list"); + goto out; + } break; default: if (page->mapping != TAIL_MAPPING) { From 85edc15a4c606094a14c36ebf5bceea7f9a3e395 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 21 Mar 2024 14:24:41 +0000 Subject: [PATCH 0596/1702] mm: remove folio_prep_large_rmappable() Now that prep_compound_page() initialises folio->_deferred_list, folio_prep_large_rmappable()'s only purpose is to set the large_rmappable flag, so inline it into the two callers. Take the opportunity to convert the large_rmappable definition from PAGEFLAG to FOLIO_FLAG and remove the existance of PageTestLargeRmappable and friends. Link: https://lkml.kernel.org/r/20240321142448.1645400-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Acked-by: Vlastimil Babka Cc: Miaohe Lin Cc: Muchun Song Cc: Oscar Salvador Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 3 --- include/linux/page-flags.h | 4 ++-- mm/huge_memory.c | 9 +-------- mm/internal.h | 3 ++- 4 files changed, 5 insertions(+), 14 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index de0c891050769..0e16451adaba3 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -263,7 +263,6 @@ unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); -void folio_prep_large_rmappable(struct folio *folio); bool can_split_folio(struct folio *folio, int *pextra_pins); int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, unsigned int new_order); @@ -411,8 +410,6 @@ static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, return 0; } -static inline void folio_prep_large_rmappable(struct folio *folio) {} - #define transparent_hugepage_flags 0UL #define thp_get_unmapped_area NULL diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 4bf1c25fd1dc5..6fb3cd42ee596 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -868,9 +868,9 @@ static inline void ClearPageCompound(struct page *page) BUG_ON(!PageHead(page)); ClearPageHead(page); } -PAGEFLAG(LargeRmappable, large_rmappable, PF_SECOND) +FOLIO_FLAG(large_rmappable, FOLIO_SECOND_PAGE) #else -TESTPAGEFLAG_FALSE(LargeRmappable, large_rmappable) +FOLIO_FLAG_FALSE(large_rmappable) #endif #define PG_head_mask ((1UL << PG_head)) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 4cc7133aaa4be..16b2c5622fb1b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -789,13 +789,6 @@ struct deferred_split *get_deferred_split_queue(struct folio *folio) } #endif -void folio_prep_large_rmappable(struct folio *folio) -{ - if (!folio || !folio_test_large(folio)) - return; - folio_set_large_rmappable(folio); -} - static inline bool is_transparent_hugepage(struct folio *folio) { if (!folio_test_large(folio)) @@ -2862,7 +2855,7 @@ static void __split_huge_page_tail(struct folio *folio, int tail, clear_compound_head(page_tail); if (new_order) { prep_compound_page(page_tail, new_order); - folio_prep_large_rmappable(new_folio); + folio_set_large_rmappable(new_folio); } /* Finally unfreeze refcount. Additional reference from page cache. */ diff --git a/mm/internal.h b/mm/internal.h index 5c0c57c9cd196..ab8250d8a5911 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -513,7 +513,8 @@ static inline struct folio *page_rmappable_folio(struct page *page) { struct folio *folio = (struct folio *)page; - folio_prep_large_rmappable(folio); + if (folio && folio_test_large(folio)) + folio_set_large_rmappable(folio); return folio; } From 8682a7be36d8c6ebd484753034a716a13f8a1f54 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 21 Mar 2024 14:24:44 +0000 Subject: [PATCH 0597/1702] mm: remove a call to compound_head() from is_page_hwpoison() We can call it only once instead of twice. Link: https://lkml.kernel.org/r/20240321142448.1645400-7-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Reviewed-by: Miaohe Lin Acked-by: Vlastimil Babka Cc: Muchun Song Cc: Oscar Salvador Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 6fb3cd42ee596..94eb8a11a3212 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -1065,11 +1065,14 @@ static inline bool PageHuge(const struct page *page) * best effort only and inherently racy: there is no way to synchronize with * failing hardware. */ -static inline bool is_page_hwpoison(struct page *page) +static inline bool is_page_hwpoison(const struct page *page) { + const struct folio *folio; + if (PageHWPoison(page)) return true; - return PageHuge(page) && PageHWPoison(compound_head(page)); + folio = page_folio(page); + return folio_test_hugetlb(folio) && PageHWPoison(&folio->page); } extern bool is_free_buddy_page(struct page *page); From 46df8e73a4a3f1445f2a8429111e72ede1f4d291 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 21 Mar 2024 14:24:45 +0000 Subject: [PATCH 0598/1702] mm: free up PG_slab Reclaim the Slab page flag by using a spare bit in PageType. We are perennially short of page flags for various purposes, and now that the original SLAB allocator has been retired, SLUB does not use the mapcount/page_type field. This lets us remove a number of special cases for ignoring mapcount on Slab pages. [willy@infradead.org: update vmcoreinfo] Link: https://lkml.kernel.org/r/ZgGV-O8WYQ_83kxp@casper.infradead.org Link: https://lkml.kernel.org/r/20240321142448.1645400-8-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Acked-by: David Hildenbrand Cc: Miaohe Lin Cc: Muchun Song Cc: Oscar Salvador Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 21 +++++++++++++++++---- include/trace/events/mmflags.h | 2 +- kernel/vmcore_info.c | 3 ++- mm/memory-failure.c | 9 --------- mm/slab.h | 2 +- 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 94eb8a11a3212..73e0b17c77283 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -109,7 +109,6 @@ enum pageflags { PG_active, PG_workingset, PG_error, - PG_slab, PG_owner_priv_1, /* Owner use. If pagecache, fs may use*/ PG_arch_1, PG_reserved, @@ -524,7 +523,6 @@ PAGEFLAG(Active, active, PF_HEAD) __CLEARPAGEFLAG(Active, active, PF_HEAD) TESTCLEARFLAG(Active, active, PF_HEAD) PAGEFLAG(Workingset, workingset, PF_HEAD) TESTCLEARFLAG(Workingset, workingset, PF_HEAD) -__PAGEFLAG(Slab, slab, PF_NO_TAIL) PAGEFLAG(Checked, checked, PF_NO_COMPOUND) /* Used by some filesystems */ /* Xen */ @@ -931,7 +929,7 @@ PAGEFLAG_FALSE(HasHWPoisoned, has_hwpoisoned) #endif /* - * For pages that are never mapped to userspace (and aren't PageSlab), + * For pages that are never mapped to userspace, * page_type may be used. Because it is initialised to -1, we invert the * sense of the bit, so __SetPageFoo *clears* the bit used for PageFoo, and * __ClearPageFoo *sets* the bit used for PageFoo. We reserve a few high and @@ -947,6 +945,7 @@ PAGEFLAG_FALSE(HasHWPoisoned, has_hwpoisoned) #define PG_table 0x00000200 #define PG_guard 0x00000400 #define PG_hugetlb 0x00000800 +#define PG_slab 0x00001000 #define PageType(page, flag) \ ((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE) @@ -1041,6 +1040,20 @@ PAGE_TYPE_OPS(Table, table, pgtable) */ PAGE_TYPE_OPS(Guard, guard, guard) +FOLIO_TYPE_OPS(slab, slab) + +/** + * PageSlab - Determine if the page belongs to the slab allocator + * @page: The page to test. + * + * Context: Any context. + * Return: True for slab pages, false for any other kind of page. + */ +static inline bool PageSlab(const struct page *page) +{ + return folio_test_slab(page_folio(page)); +} + #ifdef CONFIG_HUGETLB_PAGE FOLIO_TYPE_OPS(hugetlb, hugetlb) #else @@ -1121,7 +1134,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) (1UL << PG_lru | 1UL << PG_locked | \ 1UL << PG_private | 1UL << PG_private_2 | \ 1UL << PG_writeback | 1UL << PG_reserved | \ - 1UL << PG_slab | 1UL << PG_active | \ + 1UL << PG_active | \ 1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK) /* diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index d55e53ac91bd2..e46d6e82765e7 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -107,7 +107,6 @@ DEF_PAGEFLAG_NAME(lru), \ DEF_PAGEFLAG_NAME(active), \ DEF_PAGEFLAG_NAME(workingset), \ - DEF_PAGEFLAG_NAME(slab), \ DEF_PAGEFLAG_NAME(owner_priv_1), \ DEF_PAGEFLAG_NAME(arch_1), \ DEF_PAGEFLAG_NAME(reserved), \ @@ -135,6 +134,7 @@ IF_HAVE_PG_ARCH_X(arch_3) #define DEF_PAGETYPE_NAME(_name) { PG_##_name, __stringify(_name) } #define __def_pagetype_names \ + DEF_PAGETYPE_NAME(slab), \ DEF_PAGETYPE_NAME(hugetlb), \ DEF_PAGETYPE_NAME(offline), \ DEF_PAGETYPE_NAME(guard), \ diff --git a/kernel/vmcore_info.c b/kernel/vmcore_info.c index 23c125c2e2436..1d5eadd9dd61c 100644 --- a/kernel/vmcore_info.c +++ b/kernel/vmcore_info.c @@ -198,7 +198,8 @@ static int __init crash_save_vmcoreinfo_init(void) VMCOREINFO_NUMBER(PG_private); VMCOREINFO_NUMBER(PG_swapcache); VMCOREINFO_NUMBER(PG_swapbacked); - VMCOREINFO_NUMBER(PG_slab); +#define PAGE_SLAB_MAPCOUNT_VALUE (~PG_slab) + VMCOREINFO_NUMBER(PAGE_SLAB_MAPCOUNT_VALUE); #ifdef CONFIG_MEMORY_FAILURE VMCOREINFO_NUMBER(PG_hwpoison); #endif diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 9e62a00b46dde..0a7a8a4ba421d 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1251,7 +1251,6 @@ static int me_huge_page(struct page_state *ps, struct page *p) #define mlock (1UL << PG_mlocked) #define lru (1UL << PG_lru) #define head (1UL << PG_head) -#define slab (1UL << PG_slab) #define reserved (1UL << PG_reserved) static struct page_state error_states[] = { @@ -1261,13 +1260,6 @@ static struct page_state error_states[] = { * PG_buddy pages only make a small fraction of all free pages. */ - /* - * Could in theory check if slab page is free or if we can drop - * currently unused objects without touching them. But just - * treat it as standard kernel for now. - */ - { slab, slab, MF_MSG_SLAB, me_kernel }, - { head, head, MF_MSG_HUGE, me_huge_page }, { sc|dirty, sc|dirty, MF_MSG_DIRTY_SWAPCACHE, me_swapcache_dirty }, @@ -1294,7 +1286,6 @@ static struct page_state error_states[] = { #undef mlock #undef lru #undef head -#undef slab #undef reserved static void update_per_node_mf_stats(unsigned long pfn, diff --git a/mm/slab.h b/mm/slab.h index 65db525e93af3..1343bfa12cee9 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -84,8 +84,8 @@ struct slab { }; struct rcu_head rcu_head; }; - unsigned int __unused; + unsigned int __page_type; atomic_t __page_refcount; #ifdef CONFIG_SLAB_OBJ_EXT unsigned long obj_exts; From 8f790d0c7cfed047a1f7aad3fcddd7a979bf7232 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 21 Mar 2024 14:24:46 +0000 Subject: [PATCH 0599/1702] mm: improve dumping of mapcount and page_type For pages that have a page_type, set the mapcount to 0, which will reduce the confusion in people reading page dumps ("Why does this page have a mapcount of -128?"). Now that hugetlbfs is a page_type, read the entire_mapcount for any large folio; this is fine for all folios as no user reuses the entire_mapcount field. For pages which do not have a page type, do not print it to reduce clutter. Link: https://lkml.kernel.org/r/20240321142448.1645400-9-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Acked-by: Vlastimil Babka Cc: Miaohe Lin Cc: Muchun Song Cc: Oscar Salvador Signed-off-by: Andrew Morton --- mm/debug.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/mm/debug.c b/mm/debug.c index c1c1a6a484e4c..e8a96b8b71973 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -55,18 +55,14 @@ static void __dump_folio(struct folio *folio, struct page *page, unsigned long pfn, unsigned long idx) { struct address_space *mapping = folio_mapping(folio); - int mapcount = 0; + int mapcount = atomic_read(&page->_mapcount) + 1; char *type = ""; - /* - * page->_mapcount space in struct page is used by slab pages to - * encode own info, and we must avoid calling page_folio() again. - */ - if (!folio_test_slab(folio)) { - mapcount = atomic_read(&page->_mapcount) + 1; - if (folio_test_large(folio)) - mapcount += folio_entire_mapcount(folio); - } + /* Open-code page_mapcount() to avoid looking up a stale folio */ + if (mapcount < 0) + mapcount = 0; + if (folio_test_large(folio)) + mapcount += folio_entire_mapcount(folio); pr_warn("page: refcount:%d mapcount:%d mapping:%p index:%#lx pfn:%#lx\n", folio_ref_count(folio), mapcount, mapping, @@ -99,7 +95,8 @@ static void __dump_folio(struct folio *folio, struct page *page, */ pr_warn("%sflags: %pGp%s\n", type, &folio->flags, is_migrate_cma_folio(folio, pfn) ? " CMA" : ""); - pr_warn("page_type: %pGt\n", &folio->page.page_type); + if (page_has_type(&folio->page)) + pr_warn("page_type: %pGt\n", &folio->page.page_type); print_hex_dump(KERN_WARNING, "raw: ", DUMP_PREFIX_NONE, 32, sizeof(unsigned long), page, From 42a346b41c5b17d2fd609279e0dd0a4257d8fba0 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 21 Mar 2024 14:24:47 +0000 Subject: [PATCH 0600/1702] hugetlb: remove mention of destructors We no longer have destructors or dtors, merely a page flag (technically a page type flag, but that's an implementation detail). Remove __clear_hugetlb_destructor, fix up comments and the occasional variable name. Link: https://lkml.kernel.org/r/20240321142448.1645400-10-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Acked-by: Vlastimil Babka Cc: Miaohe Lin Cc: Muchun Song Cc: Oscar Salvador Signed-off-by: Andrew Morton --- mm/hugetlb.c | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 378181547b7bb..e4dd0c8024892 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1619,19 +1619,11 @@ static inline void destroy_compound_gigantic_folio(struct folio *folio, unsigned int order) { } #endif -static inline void __clear_hugetlb_destructor(struct hstate *h, - struct folio *folio) -{ - lockdep_assert_held(&hugetlb_lock); - - __folio_clear_hugetlb(folio); -} - /* * Remove hugetlb folio from lists. - * If vmemmap exists for the folio, update dtor so that the folio appears - * as just a compound page. Otherwise, wait until after allocating vmemmap - * to update dtor. + * If vmemmap exists for the folio, clear the hugetlb flag so that the + * folio appears as just a compound page. Otherwise, wait until after + * allocating vmemmap to clear the flag. * * A reference is held on the folio, except in the case of demote. * @@ -1662,12 +1654,12 @@ static void __remove_hugetlb_folio(struct hstate *h, struct folio *folio, } /* - * We can only clear the hugetlb destructor after allocating vmemmap + * We can only clear the hugetlb flag after allocating vmemmap * pages. Otherwise, someone (memory error handling) may try to write * to tail struct pages. */ if (!folio_test_hugetlb_vmemmap_optimized(folio)) - __clear_hugetlb_destructor(h, folio); + __folio_clear_hugetlb(folio); /* * In the case of demote we do not ref count the page as it will soon @@ -1741,7 +1733,7 @@ static void add_hugetlb_folio(struct hstate *h, struct folio *folio, static void __update_and_free_hugetlb_folio(struct hstate *h, struct folio *folio) { - bool clear_dtor = folio_test_hugetlb_vmemmap_optimized(folio); + bool clear_flag = folio_test_hugetlb_vmemmap_optimized(folio); if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported()) return; @@ -1754,11 +1746,11 @@ static void __update_and_free_hugetlb_folio(struct hstate *h, return; /* - * If folio is not vmemmap optimized (!clear_dtor), then the folio + * If folio is not vmemmap optimized (!clear_flag), then the folio * is no longer identified as a hugetlb page. hugetlb_vmemmap_restore_folio * can only be passed hugetlb pages and will BUG otherwise. */ - if (clear_dtor && hugetlb_vmemmap_restore_folio(h, folio)) { + if (clear_flag && hugetlb_vmemmap_restore_folio(h, folio)) { spin_lock_irq(&hugetlb_lock); /* * If we cannot allocate vmemmap pages, just refuse to free the @@ -1779,11 +1771,11 @@ static void __update_and_free_hugetlb_folio(struct hstate *h, /* * If vmemmap pages were allocated above, then we need to clear the - * hugetlb destructor under the hugetlb lock. + * hugetlb flag under the hugetlb lock. */ if (folio_test_hugetlb(folio)) { spin_lock_irq(&hugetlb_lock); - __clear_hugetlb_destructor(h, folio); + __folio_clear_hugetlb(folio); spin_unlock_irq(&hugetlb_lock); } @@ -1885,7 +1877,7 @@ static void bulk_vmemmap_restore_error(struct hstate *h, list_for_each_entry_safe(folio, t_folio, non_hvo_folios, lru) { list_del(&folio->lru); spin_lock_irq(&hugetlb_lock); - __clear_hugetlb_destructor(h, folio); + __folio_clear_hugetlb(folio); spin_unlock_irq(&hugetlb_lock); update_and_free_hugetlb_folio(h, folio, false); cond_resched(); @@ -1910,7 +1902,7 @@ static void bulk_vmemmap_restore_error(struct hstate *h, } else { list_del(&folio->lru); spin_lock_irq(&hugetlb_lock); - __clear_hugetlb_destructor(h, folio); + __folio_clear_hugetlb(folio); spin_unlock_irq(&hugetlb_lock); update_and_free_hugetlb_folio(h, folio, false); cond_resched(); @@ -1943,14 +1935,14 @@ static void update_and_free_pages_bulk(struct hstate *h, * should only be pages on the non_hvo_folios list. * Do note that the non_hvo_folios list could be empty. * Without HVO enabled, ret will be 0 and there is no need to call - * __clear_hugetlb_destructor as this was done previously. + * __folio_clear_hugetlb as this was done previously. */ VM_WARN_ON(!list_empty(folio_list)); VM_WARN_ON(ret < 0); if (!list_empty(&non_hvo_folios) && ret) { spin_lock_irq(&hugetlb_lock); list_for_each_entry(folio, &non_hvo_folios, lru) - __clear_hugetlb_destructor(h, folio); + __folio_clear_hugetlb(folio); spin_unlock_irq(&hugetlb_lock); } @@ -1975,7 +1967,7 @@ void free_huge_folio(struct folio *folio) { /* * Can't pass hstate in here because it is called from the - * compound page destructor. + * generic mm code. */ struct hstate *h = folio_hstate(folio); int nid = folio_nid(folio); @@ -2125,7 +2117,7 @@ static bool __prep_compound_gigantic_folio(struct folio *folio, set_compound_head(p, &folio->page); } __folio_set_head(folio); - /* we rely on prep_new_hugetlb_folio to set the destructor */ + /* we rely on prep_new_hugetlb_folio to set the hugetlb flag */ folio_set_order(folio, order); atomic_set(&folio->_entire_mapcount, -1); atomic_set(&folio->_nr_pages_mapped, 0); From 0104096498858346e3ac0668add1bf682461d4aa Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Thu, 21 Mar 2024 16:05:22 +0530 Subject: [PATCH 0601/1702] selftests/mm: confirm VA exhaustion without reliance on correctness of mmap() Currently, VA exhaustion is being checked by passing a hint to mmap() and expecting it to fail. While populating the lower VA space, mmap() fails because we have exhausted the space. Then, in validate_lower_address_hint(), because mmap() fails, we confirm that we have indeed exhausted the space. There is a circular logic involved here. Assume that there is a bug in mmap(), also assume that it exists independent of whether you pass a hint address or not; that for some reason it is not able to find a 1GB chunk. My idea is to assert the exhaustion against some other method. This patch makes a stricter test by successful write() calls from /proc/self/maps to a dump file, confirming that a free chunk is indeed not available. [dev.jain@arm.com: replace SZ_1GB with MAP_CHUNK_SIZE, tidy-up] Link: https://lkml.kernel.org/r/20240325042653.867055-1-dev.jain@arm.com Link: https://lkml.kernel.org/r/20240321103522.516097-1-dev.jain@arm.com Signed-off-by: Dev Jain Cc: Anshuman Khandual Cc: Shuah Khan Signed-off-by: Andrew Morton --- .../selftests/mm/virtual_address_range.c | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tools/testing/selftests/mm/virtual_address_range.c b/tools/testing/selftests/mm/virtual_address_range.c index 426ddfc345fbd..4e4c1e311247f 100644 --- a/tools/testing/selftests/mm/virtual_address_range.c +++ b/tools/testing/selftests/mm/virtual_address_range.c @@ -12,6 +12,8 @@ #include #include #include +#include + #include "../kselftest.h" /* @@ -93,6 +95,66 @@ static int validate_lower_address_hint(void) return 1; } +static int validate_complete_va_space(void) +{ + unsigned long start_addr, end_addr, prev_end_addr; + char line[400]; + char prot[6]; + FILE *file; + int fd; + + fd = open("va_dump", O_CREAT | O_WRONLY, 0600); + unlink("va_dump"); + if (fd < 0) { + ksft_test_result_skip("cannot create or open dump file\n"); + ksft_finished(); + } + + file = fopen("/proc/self/maps", "r"); + if (file == NULL) + ksft_exit_fail_msg("cannot open /proc/self/maps\n"); + + prev_end_addr = 0; + while (fgets(line, sizeof(line), file)) { + unsigned long hop; + + if (sscanf(line, "%lx-%lx %s[rwxp-]", + &start_addr, &end_addr, prot) != 3) + ksft_exit_fail_msg("cannot parse /proc/self/maps\n"); + + /* end of userspace mappings; ignore vsyscall mapping */ + if (start_addr & (1UL << 63)) + return 0; + + /* /proc/self/maps must have gaps less than MAP_CHUNK_SIZE */ + if (start_addr - prev_end_addr >= MAP_CHUNK_SIZE) + return 1; + + prev_end_addr = end_addr; + + if (prot[0] != 'r') + continue; + + /* + * Confirm whether MAP_CHUNK_SIZE chunk can be found or not. + * If write succeeds, no need to check MAP_CHUNK_SIZE - 1 + * addresses after that. If the address was not held by this + * process, write would fail with errno set to EFAULT. + * Anyways, if write returns anything apart from 1, exit the + * program since that would mean a bug in /proc/self/maps. + */ + hop = 0; + while (start_addr + hop < end_addr) { + if (write(fd, (void *)(start_addr + hop), 1) != 1) + return 1; + lseek(fd, 0, SEEK_SET); + + hop += MAP_CHUNK_SIZE; + } + } + return 0; +} + int main(int argc, char *argv[]) { char *ptr[NR_CHUNKS_LOW]; @@ -133,6 +195,10 @@ int main(int argc, char *argv[]) validate_addr(hptr[i], 1); } hchunks = i; + if (validate_complete_va_space()) { + ksft_test_result_fail("BUG in mmap() or /proc/self/maps\n"); + ksft_finished(); + } for (i = 0; i < lchunks; i++) munmap(ptr[i], MAP_CHUNK_SIZE); From eff201b8755bbc95a65165960a6464ca2ae18587 Mon Sep 17 00:00:00 2001 From: Hao Ge Date: Thu, 21 Mar 2024 11:22:56 +0800 Subject: [PATCH 0602/1702] mm/page-flags: make __PageMovable return bool Make __PageMovable() return bool like __folio_test_movable(). Link: https://lkml.kernel.org/r/20240321032256.82063-1-gehao@kylinos.cn Signed-off-by: Hao Ge Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 73e0b17c77283..9c7ca28ac84ca 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -707,7 +707,7 @@ static __always_inline bool __folio_test_movable(const struct folio *folio) PAGE_MAPPING_MOVABLE; } -static __always_inline int __PageMovable(const struct page *page) +static __always_inline bool __PageMovable(const struct page *page) { return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) == PAGE_MAPPING_MOVABLE; From 64c2e895df14c642fbb05e5ac7f55044cd4913a3 Mon Sep 17 00:00:00 2001 From: Hao Ge Date: Thu, 21 Mar 2024 11:07:12 +0800 Subject: [PATCH 0603/1702] mm/page-flags: make PageMappingFlags return bool Make PageMappingFlags() return bool like folio_mapping_flags(). Link: https://lkml.kernel.org/r/20240321030712.80618-1-gehao@kylinos.cn Signed-off-by: Hao Ge Reviewed-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 9c7ca28ac84ca..0163d6da6283c 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -686,7 +686,7 @@ static __always_inline bool folio_mapping_flags(const struct folio *folio) return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) != 0; } -static __always_inline int PageMappingFlags(const struct page *page) +static __always_inline bool PageMappingFlags(const struct page *page) { return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0; } From 2e47a445d7b3904d0dd6a1401357e3d5ac1a6440 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Thu, 21 Mar 2024 17:50:47 -0400 Subject: [PATCH 0604/1702] selftests/mm: run_vmtests.sh: fix hugetlb mem size calculation The script calculates a mininum required size of hugetlb memories, but it'll stop working with <1MB huge page sizes, reporting all zeros even if huge pages are available. In reality, the calculation doesn't really need to be as complicated either. Make it simpler and work for KB-level hugepages too. [peterx@redhat.com: run_vmtests.sh: fix hugetlb mem size calculation] Link: https://lkml.kernel.org/r/20240403200324.1603493-1-peterx@redhat.com Link: https://lkml.kernel.org/r/20240321215047.678172-1-peterx@redhat.com Signed-off-by: Peter Xu Reviewed-by: David Hildenbrand Reviewed-by: Muchun Song Reviewed-by: Muhammad Usama Anjum Tested-by: Ryan Roberts Cc: Nico Pache Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/run_vmtests.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 4bdb3a0c7a606..3157204b90476 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -152,9 +152,13 @@ done < /proc/meminfo # both of these requirements into account and attempt to increase # number of huge pages available. nr_cpus=$(nproc) -hpgsize_MB=$((hpgsize_KB / 1024)) -half_ufd_size_MB=$((((nr_cpus * hpgsize_MB + 127) / 128) * 128)) -needmem_KB=$((half_ufd_size_MB * 2 * 1024)) +uffd_min_KB=$((hpgsize_KB * nr_cpus * 2)) +hugetlb_min_KB=$((256 * 1024)) +if [[ $uffd_min_KB -gt $hugetlb_min_KB ]]; then + needmem_KB=$uffd_min_KB +else + needmem_KB=$hugetlb_min_KB +fi # set proper nr_hugepages if [ -n "$freepgs" ] && [ -n "$hpgsize_KB" ]; then @@ -294,7 +298,8 @@ CATEGORY="userfaultfd" run_test ./uffd-unit-tests uffd_stress_bin=./uffd-stress CATEGORY="userfaultfd" run_test ${uffd_stress_bin} anon 20 16 # Hugetlb tests require source and destination huge pages. Pass in half -# the size ($half_ufd_size_MB), which is used for *each*. +# the size of the free pages we have, which is used for *each*. +half_ufd_size_MB=$((freepgs / 2)) CATEGORY="userfaultfd" run_test ${uffd_stress_bin} hugetlb "$half_ufd_size_MB" 32 CATEGORY="userfaultfd" run_test ${uffd_stress_bin} hugetlb-private "$half_ufd_size_MB" 32 CATEGORY="userfaultfd" run_test ${uffd_stress_bin} shmem 20 16 From 17edeb5d3f761c20fd28f6002f5a9faa53c0a0d8 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 20 Mar 2024 14:02:06 -0400 Subject: [PATCH 0605/1702] mm: page_alloc: remove pcppage migratetype caching Patch series "mm: page_alloc: freelist migratetype hygiene", v4. The page allocator's mobility grouping is intended to keep unmovable pages separate from reclaimable/compactable ones to allow on-demand defragmentation for higher-order allocations and huge pages. Currently, there are several places where accidental type mixing occurs: an allocation asks for a page of a certain migratetype and receives another. This ruins pageblocks for compaction, which in turn makes allocating huge pages more expensive and less reliable. The series addresses those causes. The last patch adds type checks on all freelist movements to prevent new violations being introduced. The benefits can be seen in a mixed workload that stresses the machine with a memcache-type workload and a kernel build job while periodically attempting to allocate batches of THP. The following data is aggregated over 50 consecutive defconfig builds: VANILLA PATCHED Hugealloc Time mean 165843.93 ( +0.00%) 113025.88 ( -31.85%) Hugealloc Time stddev 158957.35 ( +0.00%) 114716.07 ( -27.83%) Kbuild Real time 310.24 ( +0.00%) 300.73 ( -3.06%) Kbuild User time 1271.13 ( +0.00%) 1259.42 ( -0.92%) Kbuild System time 582.02 ( +0.00%) 559.79 ( -3.81%) THP fault alloc 30585.14 ( +0.00%) 40853.62 ( +33.57%) THP fault fallback 36626.46 ( +0.00%) 26357.62 ( -28.04%) THP fault fail rate % 54.49 ( +0.00%) 39.22 ( -27.53%) Pagealloc fallback 1328.00 ( +0.00%) 1.00 ( -99.85%) Pagealloc type mismatch 181009.50 ( +0.00%) 0.00 ( -100.00%) Direct compact stall 434.56 ( +0.00%) 257.66 ( -40.61%) Direct compact fail 421.70 ( +0.00%) 249.94 ( -40.63%) Direct compact success 12.86 ( +0.00%) 7.72 ( -37.09%) Direct compact success rate % 2.86 ( +0.00%) 2.82 ( -0.96%) Compact daemon scanned migrate 3370059.62 ( +0.00%) 3612054.76 ( +7.18%) Compact daemon scanned free 7718439.20 ( +0.00%) 5386385.02 ( -30.21%) Compact direct scanned migrate 309248.62 ( +0.00%) 176721.04 ( -42.85%) Compact direct scanned free 433582.84 ( +0.00%) 315727.66 ( -27.18%) Compact migrate scanned daemon % 91.20 ( +0.00%) 94.48 ( +3.56%) Compact free scanned daemon % 94.58 ( +0.00%) 94.42 ( -0.16%) Compact total migrate scanned 3679308.24 ( +0.00%) 3788775.80 ( +2.98%) Compact total free scanned 8152022.04 ( +0.00%) 5702112.68 ( -30.05%) Alloc stall 872.04 ( +0.00%) 5156.12 ( +490.71%) Pages kswapd scanned 510645.86 ( +0.00%) 3394.94 ( -99.33%) Pages kswapd reclaimed 134811.62 ( +0.00%) 2701.26 ( -98.00%) Pages direct scanned 99546.06 ( +0.00%) 376407.52 ( +278.12%) Pages direct reclaimed 62123.40 ( +0.00%) 289535.70 ( +366.06%) Pages total scanned 610191.92 ( +0.00%) 379802.46 ( -37.76%) Pages scanned kswapd % 76.36 ( +0.00%) 0.10 ( -98.58%) Swap out 12057.54 ( +0.00%) 15022.98 ( +24.59%) Swap in 209.16 ( +0.00%) 256.48 ( +22.52%) File refaults 17701.64 ( +0.00%) 11765.40 ( -33.53%) Huge page success rate is higher, allocation latencies are shorter and more predictable. Stealing (fallback) rate is drastically reduced. Notably, while the vanilla kernel keeps doing fallbacks on an ongoing basis, the patched kernel enters a steady state once the distribution of block types is adequate for the workload. Steals over 50 runs: VANILLA PATCHED 1504.0 227.0 1557.0 6.0 1391.0 13.0 1080.0 26.0 1057.0 40.0 1156.0 6.0 805.0 46.0 736.0 20.0 1747.0 2.0 1699.0 34.0 1269.0 13.0 1858.0 12.0 907.0 4.0 727.0 2.0 563.0 2.0 3094.0 2.0 10211.0 3.0 2621.0 1.0 5508.0 2.0 1060.0 2.0 538.0 3.0 5773.0 2.0 2199.0 0.0 3781.0 2.0 1387.0 1.0 4977.0 0.0 2865.0 1.0 1814.0 1.0 3739.0 1.0 6857.0 0.0 382.0 0.0 407.0 1.0 3784.0 0.0 297.0 0.0 298.0 0.0 6636.0 0.0 4188.0 0.0 242.0 0.0 9960.0 0.0 5816.0 0.0 354.0 0.0 287.0 0.0 261.0 0.0 140.0 1.0 2065.0 0.0 312.0 0.0 331.0 0.0 164.0 0.0 465.0 1.0 219.0 0.0 Type mismatches are down too. Those count every time an allocation request asks for one migratetype and gets another. This can still occur minimally in the patched kernel due to non-stealing fallbacks, but it's quite rare and follows the pattern of overall fallbacks - once the block type distribution settles, mismatches cease as well: VANILLA: PATCHED: 182602.0 268.0 135794.0 20.0 88619.0 19.0 95973.0 0.0 129590.0 0.0 129298.0 0.0 147134.0 0.0 230854.0 0.0 239709.0 0.0 137670.0 0.0 132430.0 0.0 65712.0 0.0 57901.0 0.0 67506.0 0.0 63565.0 4.0 34806.0 0.0 42962.0 0.0 32406.0 0.0 38668.0 0.0 61356.0 0.0 57800.0 0.0 41435.0 0.0 83456.0 0.0 65048.0 0.0 28955.0 0.0 47597.0 0.0 75117.0 0.0 55564.0 0.0 38280.0 0.0 52404.0 0.0 26264.0 0.0 37538.0 0.0 19671.0 0.0 30936.0 0.0 26933.0 0.0 16962.0 0.0 44554.0 0.0 46352.0 0.0 24995.0 0.0 35152.0 0.0 12823.0 0.0 21583.0 0.0 18129.0 0.0 31693.0 0.0 28745.0 0.0 33308.0 0.0 31114.0 0.0 35034.0 0.0 12111.0 0.0 24885.0 0.0 Compaction work is markedly reduced despite much better THP rates. In the vanilla kernel, reclaim seems to have been driven primarily by watermark boosting that happens as a result of fallbacks. With those all but eliminated, watermarks average lower and kswapd does less work. The uptick in direct reclaim is because THP requests have to fend for themselves more often - which is intended policy right now. Aggregate reclaim activity is lowered significantly, though. This patch (of 10): The idea behind the cache is to save get_pageblock_migratetype() lookups during bulk freeing. A microbenchmark suggests this isn't helping, though. The pcp migratetype can get stale, which means that bulk freeing has an extra branch to check if the pageblock was isolated while on the pcp. While the variance overlaps, the cache write and the branch seem to make this a net negative. The following test allocates and frees batches of 10,000 pages (~3x the pcp high marks to trigger flushing): Before: 8,668.48 msec task-clock # 99.735 CPUs utilized ( +- 2.90% ) 19 context-switches # 4.341 /sec ( +- 3.24% ) 0 cpu-migrations # 0.000 /sec 17,440 page-faults # 3.984 K/sec ( +- 2.90% ) 41,758,692,473 cycles # 9.541 GHz ( +- 2.90% ) 126,201,294,231 instructions # 5.98 insn per cycle ( +- 2.90% ) 25,348,098,335 branches # 5.791 G/sec ( +- 2.90% ) 33,436,921 branch-misses # 0.26% of all branches ( +- 2.90% ) 0.0869148 +- 0.0000302 seconds time elapsed ( +- 0.03% ) After: 8,444.81 msec task-clock # 99.726 CPUs utilized ( +- 2.90% ) 22 context-switches # 5.160 /sec ( +- 3.23% ) 0 cpu-migrations # 0.000 /sec 17,443 page-faults # 4.091 K/sec ( +- 2.90% ) 40,616,738,355 cycles # 9.527 GHz ( +- 2.90% ) 126,383,351,792 instructions # 6.16 insn per cycle ( +- 2.90% ) 25,224,985,153 branches # 5.917 G/sec ( +- 2.90% ) 32,236,793 branch-misses # 0.25% of all branches ( +- 2.90% ) 0.0846799 +- 0.0000412 seconds time elapsed ( +- 0.05% ) A side effect is that this also ensures that pages whose pageblock gets stolen while on the pcplist end up on the right freelist and we don't perform potentially type-incompatible buddy merges (or skip merges when we shouldn't), which is likely beneficial to long-term fragmentation management, although the effects would be harder to measure. Settle for simpler and faster code as justification here. Link: https://lkml.kernel.org/r/20240320180429.678181-1-hannes@cmpxchg.org Link: https://lkml.kernel.org/r/20240320180429.678181-2-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Acked-by: Zi Yan Reviewed-by: Vlastimil Babka Acked-by: Mel Gorman Tested-by: "Huang, Ying" Tested-by: Baolin Wang Cc: David Hildenbrand Signed-off-by: Andrew Morton --- mm/page_alloc.c | 66 +++++++++++-------------------------------------- 1 file changed, 14 insertions(+), 52 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 7e8f4b751801d..56a341c8b3ac2 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -207,24 +207,6 @@ EXPORT_SYMBOL(node_states); gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; -/* - * A cached value of the page's pageblock's migratetype, used when the page is - * put on a pcplist. Used to avoid the pageblock migratetype lookup when - * freeing from pcplists in most cases, at the cost of possibly becoming stale. - * Also the migratetype set in the page does not necessarily match the pcplist - * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any - * other index - this ensures that it will be put on the correct CMA freelist. - */ -static inline int get_pcppage_migratetype(struct page *page) -{ - return page->index; -} - -static inline void set_pcppage_migratetype(struct page *page, int migratetype) -{ - page->index = migratetype; -} - #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE unsigned int pageblock_order __read_mostly; #endif @@ -1195,7 +1177,6 @@ static void free_pcppages_bulk(struct zone *zone, int count, { unsigned long flags; unsigned int order; - bool isolated_pageblocks; struct page *page; /* @@ -1208,7 +1189,6 @@ static void free_pcppages_bulk(struct zone *zone, int count, pindex = pindex - 1; spin_lock_irqsave(&zone->lock, flags); - isolated_pageblocks = has_isolate_pageblock(zone); while (count > 0) { struct list_head *list; @@ -1224,23 +1204,19 @@ static void free_pcppages_bulk(struct zone *zone, int count, order = pindex_to_order(pindex); nr_pages = 1 << order; do { + unsigned long pfn; int mt; page = list_last_entry(list, struct page, pcp_list); - mt = get_pcppage_migratetype(page); + pfn = page_to_pfn(page); + mt = get_pfnblock_migratetype(page, pfn); /* must delete to avoid corrupting pcp list */ list_del(&page->pcp_list); count -= nr_pages; pcp->count -= nr_pages; - /* MIGRATE_ISOLATE page should not go to pcplists */ - VM_BUG_ON_PAGE(is_migrate_isolate(mt), page); - /* Pageblock could have been isolated meanwhile */ - if (unlikely(isolated_pageblocks)) - mt = get_pageblock_migratetype(page); - - __free_one_page(page, page_to_pfn(page), zone, order, mt, FPI_NONE); + __free_one_page(page, pfn, zone, order, mt, FPI_NONE); trace_mm_page_pcpu_drain(page, order, mt); } while (count > 0 && !list_empty(list)); } @@ -1580,7 +1556,6 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, continue; del_page_from_free_list(page, zone, current_order); expand(zone, page, order, current_order, migratetype); - set_pcppage_migratetype(page, migratetype); trace_mm_page_alloc_zone_locked(page, order, migratetype, pcp_allowed_order(order) && migratetype < MIGRATE_PCPTYPES); @@ -2151,7 +2126,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, * pages are ordered properly. */ list_add_tail(&page->pcp_list, list); - if (is_migrate_cma(get_pcppage_migratetype(page))) + if (is_migrate_cma(get_pageblock_migratetype(page))) __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, -(1 << order)); } @@ -2347,19 +2322,6 @@ void drain_all_pages(struct zone *zone) __drain_all_pages(zone, false); } -static bool free_unref_page_prepare(struct page *page, unsigned long pfn, - unsigned int order) -{ - int migratetype; - - if (!free_pages_prepare(page, order)) - return false; - - migratetype = get_pfnblock_migratetype(page, pfn); - set_pcppage_migratetype(page, migratetype); - return true; -} - static int nr_pcp_free(struct per_cpu_pages *pcp, int batch, int high, bool free_high) { int min_nr_free, max_nr_free; @@ -2492,7 +2454,7 @@ void free_unref_page(struct page *page, unsigned int order) unsigned long pfn = page_to_pfn(page); int migratetype, pcpmigratetype; - if (!free_unref_page_prepare(page, pfn, order)) + if (!free_pages_prepare(page, order)) return; /* @@ -2502,7 +2464,7 @@ void free_unref_page(struct page *page, unsigned int order) * get those areas back if necessary. Otherwise, we may have to free * excessively into the page allocator */ - migratetype = pcpmigratetype = get_pcppage_migratetype(page); + migratetype = pcpmigratetype = get_pfnblock_migratetype(page, pfn); if (unlikely(migratetype >= MIGRATE_PCPTYPES)) { if (unlikely(is_migrate_isolate(migratetype))) { free_one_page(page_zone(page), page, pfn, order, migratetype, FPI_NONE); @@ -2541,14 +2503,14 @@ void free_unref_folios(struct folio_batch *folios) if (order > 0 && folio_test_large_rmappable(folio)) folio_undo_large_rmappable(folio); - if (!free_unref_page_prepare(&folio->page, pfn, order)) + if (!free_pages_prepare(&folio->page, order)) continue; /* * Free isolated folios and orders not handled on the PCP * directly to the allocator, see comment in free_unref_page. */ - migratetype = get_pcppage_migratetype(&folio->page); + migratetype = get_pfnblock_migratetype(&folio->page, pfn); if (!pcp_allowed_order(order) || is_migrate_isolate(migratetype)) { free_one_page(folio_zone(folio), &folio->page, pfn, @@ -2565,10 +2527,11 @@ void free_unref_folios(struct folio_batch *folios) for (i = 0; i < folios->nr; i++) { struct folio *folio = folios->folios[i]; struct zone *zone = folio_zone(folio); + unsigned long pfn = folio_pfn(folio); unsigned int order = (unsigned long)folio->private; folio->private = NULL; - migratetype = get_pcppage_migratetype(&folio->page); + migratetype = get_pfnblock_migratetype(&folio->page, pfn); /* Different zone requires a different pcp lock */ if (zone != locked_zone) { @@ -2585,9 +2548,8 @@ void free_unref_folios(struct folio_batch *folios) pcp = pcp_spin_trylock(zone->per_cpu_pageset); if (unlikely(!pcp)) { pcp_trylock_finish(UP_flags); - free_one_page(zone, &folio->page, - folio_pfn(folio), order, - migratetype, FPI_NONE); + free_one_page(zone, &folio->page, pfn, + order, migratetype, FPI_NONE); locked_zone = NULL; continue; } @@ -2757,7 +2719,7 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, } } __mod_zone_freepage_state(zone, -(1 << order), - get_pcppage_migratetype(page)); + get_pageblock_migratetype(page)); spin_unlock_irqrestore(&zone->lock, flags); } while (check_new_pages(page, order)); From 9cbe97bad5cd75b5b493734bd2695febb8e95281 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 20 Mar 2024 14:02:07 -0400 Subject: [PATCH 0606/1702] mm: page_alloc: optimize free_unref_folios() Move direct freeing of isolated pages to the lock-breaking block in the second loop. This saves an unnecessary migratetype reassessment. Minor comment and local variable scoping cleanups. Link: https://lkml.kernel.org/r/20240320180429.678181-3-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Suggested-by: Vlastimil Babka Tested-by: "Huang, Ying" Reviewed-by: Vlastimil Babka Tested-by: Baolin Wang Cc: David Hildenbrand Cc: Mel Gorman Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/page_alloc.c | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 56a341c8b3ac2..5fa3d534df2f9 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2493,7 +2493,7 @@ void free_unref_folios(struct folio_batch *folios) unsigned long __maybe_unused UP_flags; struct per_cpu_pages *pcp = NULL; struct zone *locked_zone = NULL; - int i, j, migratetype; + int i, j; /* Prepare folios for freeing */ for (i = 0, j = 0; i < folios->nr; i++) { @@ -2505,14 +2505,15 @@ void free_unref_folios(struct folio_batch *folios) folio_undo_large_rmappable(folio); if (!free_pages_prepare(&folio->page, order)) continue; - /* - * Free isolated folios and orders not handled on the PCP - * directly to the allocator, see comment in free_unref_page. + * Free orders not handled on the PCP directly to the + * allocator. */ - migratetype = get_pfnblock_migratetype(&folio->page, pfn); - if (!pcp_allowed_order(order) || - is_migrate_isolate(migratetype)) { + if (!pcp_allowed_order(order)) { + int migratetype; + + migratetype = get_pfnblock_migratetype(&folio->page, + pfn); free_one_page(folio_zone(folio), &folio->page, pfn, order, migratetype, FPI_NONE); continue; @@ -2529,15 +2530,29 @@ void free_unref_folios(struct folio_batch *folios) struct zone *zone = folio_zone(folio); unsigned long pfn = folio_pfn(folio); unsigned int order = (unsigned long)folio->private; + int migratetype; folio->private = NULL; migratetype = get_pfnblock_migratetype(&folio->page, pfn); /* Different zone requires a different pcp lock */ - if (zone != locked_zone) { + if (zone != locked_zone || + is_migrate_isolate(migratetype)) { if (pcp) { pcp_spin_unlock(pcp); pcp_trylock_finish(UP_flags); + locked_zone = NULL; + pcp = NULL; + } + + /* + * Free isolated pages directly to the + * allocator, see comment in free_unref_page. + */ + if (is_migrate_isolate(migratetype)) { + free_one_page(zone, &folio->page, pfn, + order, migratetype, FPI_NONE); + continue; } /* @@ -2550,7 +2565,6 @@ void free_unref_folios(struct folio_batch *folios) pcp_trylock_finish(UP_flags); free_one_page(zone, &folio->page, pfn, order, migratetype, FPI_NONE); - locked_zone = NULL; continue; } locked_zone = zone; From e6cf9e1c4cde8a53385423ecb8ca581097f42e02 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 20 Mar 2024 14:02:08 -0400 Subject: [PATCH 0607/1702] mm: page_alloc: fix up block types when merging compatible blocks The buddy allocator coalesces compatible blocks during freeing, but it doesn't update the types of the subblocks to match. When an allocation later breaks the chunk down again, its pieces will be put on freelists of the wrong type. This encourages incompatible page mixing (ask for one type, get another), and thus long-term fragmentation. Update the subblocks when merging a larger chunk, such that a later expand() will maintain freelist type hygiene. Link: https://lkml.kernel.org/r/20240320180429.678181-4-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka Acked-by: Mel Gorman Tested-by: "Huang, Ying" Tested-by: Baolin Wang Cc: David Hildenbrand Signed-off-by: Andrew Morton --- mm/page_alloc.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 5fa3d534df2f9..2c03336def76f 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -786,10 +786,17 @@ static inline void __free_one_page(struct page *page, */ int buddy_mt = get_pfnblock_migratetype(buddy, buddy_pfn); - if (migratetype != buddy_mt - && (!migratetype_is_mergeable(migratetype) || - !migratetype_is_mergeable(buddy_mt))) - goto done_merging; + if (migratetype != buddy_mt) { + if (!migratetype_is_mergeable(migratetype) || + !migratetype_is_mergeable(buddy_mt)) + goto done_merging; + /* + * Match buddy type. This ensures that + * an expand() down the line puts the + * sub-blocks on the right freelists. + */ + set_pageblock_migratetype(buddy, migratetype); + } } /* From b54ccd3c6bacbc571f7e61797fb5ff9fe3861413 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 20 Mar 2024 14:02:09 -0400 Subject: [PATCH 0608/1702] mm: page_alloc: move free pages when converting block during isolation When claiming a block during compaction isolation, move any remaining free pages to the correct freelists as well, instead of stranding them on the wrong list. Otherwise, this encourages incompatible page mixing down the line, and thus long-term fragmentation. Link: https://lkml.kernel.org/r/20240320180429.678181-5-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Reviewed-by: Zi Yan Reviewed-by: Vlastimil Babka Acked-by: Mel Gorman Tested-by: "Huang, Ying" Tested-by: Baolin Wang Cc: David Hildenbrand Signed-off-by: Andrew Morton --- mm/page_alloc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 2c03336def76f..dcacb86efd298 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2653,9 +2653,12 @@ int __isolate_free_page(struct page *page, unsigned int order) * Only change normal pageblocks (i.e., they can merge * with others) */ - if (migratetype_is_mergeable(mt)) + if (migratetype_is_mergeable(mt)) { set_pageblock_migratetype(page, MIGRATE_MOVABLE); + move_freepages_block(zone, page, + MIGRATE_MOVABLE, NULL); + } } } From 2dd482ba627de15d67f0c0ed445133c8ae9b201b Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 20 Mar 2024 14:02:10 -0400 Subject: [PATCH 0609/1702] mm: page_alloc: fix move_freepages_block() range error When a block is partially outside the zone of the cursor page, the function cuts the range to the pivot page instead of the zone start. This can leave large parts of the block behind, which encourages incompatible page mixing down the line (ask for one type, get another), and thus long-term fragmentation. This triggers reliably on the first block in the DMA zone, whose start_pfn is 1. The block is stolen, but everything before the pivot page (which was often hundreds of pages) is left on the old list. Link: https://lkml.kernel.org/r/20240320180429.678181-6-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Reviewed-by: Vlastimil Babka Tested-by: Baolin Wang Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Mel Gorman Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/page_alloc.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index dcacb86efd298..0e223d5b94fa5 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1650,9 +1650,15 @@ int move_freepages_block(struct zone *zone, struct page *page, start_pfn = pageblock_start_pfn(pfn); end_pfn = pageblock_end_pfn(pfn) - 1; - /* Do not cross zone boundaries */ + /* + * The caller only has the lock for @zone, don't touch ranges + * that straddle into other zones. While we could move part of + * the range that's inside the zone, this call is usually + * accompanied by other operations such as migratetype updates + * which also should be locked. + */ if (!zone_spans_pfn(zone, start_pfn)) - start_pfn = pfn; + return 0; if (!zone_spans_pfn(zone, end_pfn)) return 0; From c0cd6f557b9090525d288806cccbc73440ac235a Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 20 Mar 2024 14:02:11 -0400 Subject: [PATCH 0610/1702] mm: page_alloc: fix freelist movement during block conversion Currently, page block type conversion during fallbacks, atomic reservations and isolation can strand various amounts of free pages on incorrect freelists. For example, fallback stealing moves free pages in the block to the new type's freelists, but then may not actually claim the block for that type if there aren't enough compatible pages already allocated. In all cases, free page moving might fail if the block straddles more than one zone, in which case no free pages are moved at all, but the block type is changed anyway. This is detrimental to type hygiene on the freelists. It encourages incompatible page mixing down the line (ask for one type, get another) and thus contributes to long-term fragmentation. Split the process into a proper transaction: check first if conversion will happen, then try to move the free pages, and only if that was successful convert the block to the new type. [baolin.wang@linux.alibaba.com: fix allocation failures with CONFIG_CMA] Link: https://lkml.kernel.org/r/a97697e0-45b0-4f71-b087-fdc7a1d43c0e@linux.alibaba.com Link: https://lkml.kernel.org/r/20240320180429.678181-7-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Signed-off-by: Baolin Wang Tested-by: "Huang, Ying" Reviewed-by: Vlastimil Babka Tested-by: Baolin Wang Cc: David Hildenbrand Cc: Mel Gorman Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/page-isolation.h | 3 +- mm/page_alloc.c | 174 ++++++++++++++++++++------------- mm/page_isolation.c | 22 +++-- 3 files changed, 121 insertions(+), 78 deletions(-) diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h index 4ac34392823a9..8550b3c914809 100644 --- a/include/linux/page-isolation.h +++ b/include/linux/page-isolation.h @@ -34,8 +34,7 @@ static inline bool is_migrate_isolate(int migratetype) #define REPORT_FAILURE 0x2 void set_pageblock_migratetype(struct page *page, int migratetype); -int move_freepages_block(struct zone *zone, struct page *page, - int migratetype, int *num_movable); +int move_freepages_block(struct zone *zone, struct page *page, int migratetype); int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, int migratetype, int flags, gfp_t gfp_flags); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 0e223d5b94fa5..a7cfe65e45c19 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1601,9 +1601,8 @@ static inline struct page *__rmqueue_cma_fallback(struct zone *zone, * Note that start_page and end_pages are not aligned on a pageblock * boundary. If alignment is required, use move_freepages_block() */ -static int move_freepages(struct zone *zone, - unsigned long start_pfn, unsigned long end_pfn, - int migratetype, int *num_movable) +static int move_freepages(struct zone *zone, unsigned long start_pfn, + unsigned long end_pfn, int migratetype) { struct page *page; unsigned long pfn; @@ -1613,14 +1612,6 @@ static int move_freepages(struct zone *zone, for (pfn = start_pfn; pfn <= end_pfn;) { page = pfn_to_page(pfn); if (!PageBuddy(page)) { - /* - * We assume that pages that could be isolated for - * migration are movable. But we don't actually try - * isolating, as that would be expensive. - */ - if (num_movable && - (PageLRU(page) || __PageMovable(page))) - (*num_movable)++; pfn++; continue; } @@ -1638,17 +1629,16 @@ static int move_freepages(struct zone *zone, return pages_moved; } -int move_freepages_block(struct zone *zone, struct page *page, - int migratetype, int *num_movable) +static bool prep_move_freepages_block(struct zone *zone, struct page *page, + unsigned long *start_pfn, + unsigned long *end_pfn, + int *num_free, int *num_movable) { - unsigned long start_pfn, end_pfn, pfn; - - if (num_movable) - *num_movable = 0; + unsigned long pfn, start, end; pfn = page_to_pfn(page); - start_pfn = pageblock_start_pfn(pfn); - end_pfn = pageblock_end_pfn(pfn) - 1; + start = pageblock_start_pfn(pfn); + end = pageblock_end_pfn(pfn) - 1; /* * The caller only has the lock for @zone, don't touch ranges @@ -1657,13 +1647,50 @@ int move_freepages_block(struct zone *zone, struct page *page, * accompanied by other operations such as migratetype updates * which also should be locked. */ - if (!zone_spans_pfn(zone, start_pfn)) - return 0; - if (!zone_spans_pfn(zone, end_pfn)) - return 0; + if (!zone_spans_pfn(zone, start)) + return false; + if (!zone_spans_pfn(zone, end)) + return false; + + *start_pfn = start; + *end_pfn = end; - return move_freepages(zone, start_pfn, end_pfn, migratetype, - num_movable); + if (num_free) { + *num_free = 0; + *num_movable = 0; + for (pfn = start; pfn <= end;) { + page = pfn_to_page(pfn); + if (PageBuddy(page)) { + int nr = 1 << buddy_order(page); + + *num_free += nr; + pfn += nr; + continue; + } + /* + * We assume that pages that could be isolated for + * migration are movable. But we don't actually try + * isolating, as that would be expensive. + */ + if (PageLRU(page) || __PageMovable(page)) + (*num_movable)++; + pfn++; + } + } + + return true; +} + +int move_freepages_block(struct zone *zone, struct page *page, + int migratetype) +{ + unsigned long start_pfn, end_pfn; + + if (!prep_move_freepages_block(zone, page, &start_pfn, &end_pfn, + NULL, NULL)) + return -1; + + return move_freepages(zone, start_pfn, end_pfn, migratetype); } static void change_pageblock_range(struct page *pageblock_page, @@ -1748,33 +1775,37 @@ static inline bool boost_watermark(struct zone *zone) } /* - * This function implements actual steal behaviour. If order is large enough, - * we can steal whole pageblock. If not, we first move freepages in this - * pageblock to our migratetype and determine how many already-allocated pages - * are there in the pageblock with a compatible migratetype. If at least half - * of pages are free or compatible, we can change migratetype of the pageblock - * itself, so pages freed in the future will be put on the correct free list. + * This function implements actual steal behaviour. If order is large enough, we + * can claim the whole pageblock for the requested migratetype. If not, we check + * the pageblock for constituent pages; if at least half of the pages are free + * or compatible, we can still claim the whole block, so pages freed in the + * future will be put on the correct free list. Otherwise, we isolate exactly + * the order we need from the fallback block and leave its migratetype alone. */ -static void steal_suitable_fallback(struct zone *zone, struct page *page, - unsigned int alloc_flags, int start_type, bool whole_block) +static struct page * +steal_suitable_fallback(struct zone *zone, struct page *page, + int current_order, int order, int start_type, + unsigned int alloc_flags, bool whole_block) { - unsigned int current_order = buddy_order(page); int free_pages, movable_pages, alike_pages; - int old_block_type; + unsigned long start_pfn, end_pfn; + int block_type; - old_block_type = get_pageblock_migratetype(page); + block_type = get_pageblock_migratetype(page); /* * This can happen due to races and we want to prevent broken * highatomic accounting. */ - if (is_migrate_highatomic(old_block_type)) + if (is_migrate_highatomic(block_type)) goto single_page; /* Take ownership for orders >= pageblock_order */ if (current_order >= pageblock_order) { + del_page_from_free_list(page, zone, current_order); change_pageblock_range(page, current_order, start_type); - goto single_page; + expand(zone, page, order, current_order, start_type); + return page; } /* @@ -1789,10 +1820,9 @@ static void steal_suitable_fallback(struct zone *zone, struct page *page, if (!whole_block) goto single_page; - free_pages = move_freepages_block(zone, page, start_type, - &movable_pages); /* moving whole block can fail due to zone boundary conditions */ - if (!free_pages) + if (!prep_move_freepages_block(zone, page, &start_pfn, &end_pfn, + &free_pages, &movable_pages)) goto single_page; /* @@ -1810,7 +1840,7 @@ static void steal_suitable_fallback(struct zone *zone, struct page *page, * vice versa, be conservative since we can't distinguish the * exact migratetype of non-movable pages. */ - if (old_block_type == MIGRATE_MOVABLE) + if (block_type == MIGRATE_MOVABLE) alike_pages = pageblock_nr_pages - (free_pages + movable_pages); else @@ -1821,13 +1851,16 @@ static void steal_suitable_fallback(struct zone *zone, struct page *page, * compatible migratability as our allocation, claim the whole block. */ if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || - page_group_by_mobility_disabled) + page_group_by_mobility_disabled) { + move_freepages(zone, start_pfn, end_pfn, start_type); set_pageblock_migratetype(page, start_type); - - return; + return __rmqueue_smallest(zone, order, start_type); + } single_page: - move_to_free_list(page, zone, current_order, start_type); + del_page_from_free_list(page, zone, current_order); + expand(zone, page, order, current_order, block_type); + return page; } /* @@ -1895,9 +1928,10 @@ static void reserve_highatomic_pageblock(struct page *page, struct zone *zone) mt = get_pageblock_migratetype(page); /* Only reserve normal pageblocks (i.e., they can merge with others) */ if (migratetype_is_mergeable(mt)) { - zone->nr_reserved_highatomic += pageblock_nr_pages; - set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); - move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL); + if (move_freepages_block(zone, page, MIGRATE_HIGHATOMIC) != -1) { + set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); + zone->nr_reserved_highatomic += pageblock_nr_pages; + } } out_unlock: @@ -1922,7 +1956,7 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, struct zone *zone; struct page *page; int order; - bool ret; + int ret; for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx, ac->nodemask) { @@ -1971,10 +2005,14 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, * of pageblocks that cannot be completely freed * may increase. */ + ret = move_freepages_block(zone, page, ac->migratetype); + /* + * Reserving this block already succeeded, so this should + * not fail on zone boundaries. + */ + WARN_ON_ONCE(ret == -1); set_pageblock_migratetype(page, ac->migratetype); - ret = move_freepages_block(zone, page, ac->migratetype, - NULL); - if (ret) { + if (ret > 0) { spin_unlock_irqrestore(&zone->lock, flags); return ret; } @@ -1995,7 +2033,7 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, * deviation from the rest of this file, to make the for loop * condition simpler. */ -static __always_inline bool +static __always_inline struct page * __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, unsigned int alloc_flags) { @@ -2042,7 +2080,7 @@ __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, goto do_steal; } - return false; + return NULL; find_smallest: for (current_order = order; current_order < NR_PAGE_ORDERS; current_order++) { @@ -2062,14 +2100,14 @@ __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, do_steal: page = get_page_from_free_area(area, fallback_mt); - steal_suitable_fallback(zone, page, alloc_flags, start_migratetype, - can_steal); + /* take off list, maybe claim block, expand remainder */ + page = steal_suitable_fallback(zone, page, current_order, order, + start_migratetype, alloc_flags, can_steal); trace_mm_page_alloc_extfrag(page, order, current_order, start_migratetype, fallback_mt); - return true; - + return page; } /* @@ -2096,15 +2134,15 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype, return page; } } -retry: + page = __rmqueue_smallest(zone, order, migratetype); if (unlikely(!page)) { if (alloc_flags & ALLOC_CMA) page = __rmqueue_cma_fallback(zone, order); - if (!page && __rmqueue_fallback(zone, order, migratetype, - alloc_flags)) - goto retry; + if (!page) + page = __rmqueue_fallback(zone, order, migratetype, + alloc_flags); } return page; } @@ -2659,12 +2697,10 @@ int __isolate_free_page(struct page *page, unsigned int order) * Only change normal pageblocks (i.e., they can merge * with others) */ - if (migratetype_is_mergeable(mt)) { - set_pageblock_migratetype(page, - MIGRATE_MOVABLE); - move_freepages_block(zone, page, - MIGRATE_MOVABLE, NULL); - } + if (migratetype_is_mergeable(mt) && + move_freepages_block(zone, page, + MIGRATE_MOVABLE) != -1) + set_pageblock_migratetype(page, MIGRATE_MOVABLE); } } diff --git a/mm/page_isolation.c b/mm/page_isolation.c index a5c8fa4c2a75c..71539d7b96cf9 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -178,15 +178,18 @@ static int set_migratetype_isolate(struct page *page, int migratetype, int isol_ unmovable = has_unmovable_pages(check_unmovable_start, check_unmovable_end, migratetype, isol_flags); if (!unmovable) { - unsigned long nr_pages; + int nr_pages; int mt = get_pageblock_migratetype(page); + nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE); + /* Block spans zone boundaries? */ + if (nr_pages == -1) { + spin_unlock_irqrestore(&zone->lock, flags); + return -EBUSY; + } + __mod_zone_freepage_state(zone, -nr_pages, mt); set_pageblock_migratetype(page, MIGRATE_ISOLATE); zone->nr_isolate_pageblock++; - nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE, - NULL); - - __mod_zone_freepage_state(zone, -nr_pages, mt); spin_unlock_irqrestore(&zone->lock, flags); return 0; } @@ -206,7 +209,7 @@ static int set_migratetype_isolate(struct page *page, int migratetype, int isol_ static void unset_migratetype_isolate(struct page *page, int migratetype) { struct zone *zone; - unsigned long flags, nr_pages; + unsigned long flags; bool isolated_page = false; unsigned int order; struct page *buddy; @@ -252,7 +255,12 @@ static void unset_migratetype_isolate(struct page *page, int migratetype) * allocation. */ if (!isolated_page) { - nr_pages = move_freepages_block(zone, page, migratetype, NULL); + int nr_pages = move_freepages_block(zone, page, migratetype); + /* + * Isolating this block already succeeded, so this + * should not fail on zone boundaries. + */ + WARN_ON_ONCE(nr_pages == -1); __mod_zone_freepage_state(zone, nr_pages, migratetype); } set_pageblock_migratetype(page, migratetype); From 55612e80e722ac554cc5e80df05555b4f8d40c37 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 20 Mar 2024 14:02:12 -0400 Subject: [PATCH 0611/1702] mm: page_alloc: close migratetype race between freeing and stealing There are three freeing paths that read the page's migratetype optimistically before grabbing the zone lock. When this races with block stealing, those pages go on the wrong freelist. The paths in question are: - when freeing >costly orders that aren't THP - when freeing pages to the buddy upon pcp lock contention - when freeing pages that are isolated - when freeing pages initially during boot - when freeing the remainder in alloc_pages_exact() - when "accepting" unaccepted VM host memory before first use - when freeing pages during unpoisoning None of these are so hot that they would need this optimization at the cost of hampering defrag efforts. Especially when contrasted with the fact that the most common buddy freeing path - free_pcppages_bulk - is checking the migratetype under the zone->lock just fine. In addition, isolated pages need to look up the migratetype under the lock anyway, which adds branches to the locked section, and results in a double lookup when the pages are in fact isolated. Move the lookups into the lock. Link: https://lkml.kernel.org/r/20240320180429.678181-8-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Reported-by: Vlastimil Babka Reviewed-by: Vlastimil Babka Tested-by: Baolin Wang Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Mel Gorman Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/page_alloc.c | 52 ++++++++++++++++++------------------------------- 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index a7cfe65e45c19..289dcb4347192 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1231,18 +1231,15 @@ static void free_pcppages_bulk(struct zone *zone, int count, spin_unlock_irqrestore(&zone->lock, flags); } -static void free_one_page(struct zone *zone, - struct page *page, unsigned long pfn, - unsigned int order, - int migratetype, fpi_t fpi_flags) +static void free_one_page(struct zone *zone, struct page *page, + unsigned long pfn, unsigned int order, + fpi_t fpi_flags) { unsigned long flags; + int migratetype; spin_lock_irqsave(&zone->lock, flags); - if (unlikely(has_isolate_pageblock(zone) || - is_migrate_isolate(migratetype))) { - migratetype = get_pfnblock_migratetype(page, pfn); - } + migratetype = get_pfnblock_migratetype(page, pfn); __free_one_page(page, pfn, zone, order, migratetype, fpi_flags); spin_unlock_irqrestore(&zone->lock, flags); } @@ -1250,21 +1247,13 @@ static void free_one_page(struct zone *zone, static void __free_pages_ok(struct page *page, unsigned int order, fpi_t fpi_flags) { - int migratetype; unsigned long pfn = page_to_pfn(page); struct zone *zone = page_zone(page); if (!free_pages_prepare(page, order)) return; - /* - * Calling get_pfnblock_migratetype() without spin_lock_irqsave() here - * is used to avoid calling get_pfnblock_migratetype() under the lock. - * This will reduce the lock holding time. - */ - migratetype = get_pfnblock_migratetype(page, pfn); - - free_one_page(zone, page, pfn, order, migratetype, fpi_flags); + free_one_page(zone, page, pfn, order, fpi_flags); __count_vm_events(PGFREE, 1 << order); } @@ -2503,7 +2492,7 @@ void free_unref_page(struct page *page, unsigned int order) struct per_cpu_pages *pcp; struct zone *zone; unsigned long pfn = page_to_pfn(page); - int migratetype, pcpmigratetype; + int migratetype; if (!free_pages_prepare(page, order)) return; @@ -2515,23 +2504,23 @@ void free_unref_page(struct page *page, unsigned int order) * get those areas back if necessary. Otherwise, we may have to free * excessively into the page allocator */ - migratetype = pcpmigratetype = get_pfnblock_migratetype(page, pfn); + migratetype = get_pfnblock_migratetype(page, pfn); if (unlikely(migratetype >= MIGRATE_PCPTYPES)) { if (unlikely(is_migrate_isolate(migratetype))) { - free_one_page(page_zone(page), page, pfn, order, migratetype, FPI_NONE); + free_one_page(page_zone(page), page, pfn, order, FPI_NONE); return; } - pcpmigratetype = MIGRATE_MOVABLE; + migratetype = MIGRATE_MOVABLE; } zone = page_zone(page); pcp_trylock_prepare(UP_flags); pcp = pcp_spin_trylock(zone->per_cpu_pageset); if (pcp) { - free_unref_page_commit(zone, pcp, page, pcpmigratetype, order); + free_unref_page_commit(zone, pcp, page, migratetype, order); pcp_spin_unlock(pcp); } else { - free_one_page(zone, page, pfn, order, migratetype, FPI_NONE); + free_one_page(zone, page, pfn, order, FPI_NONE); } pcp_trylock_finish(UP_flags); } @@ -2561,12 +2550,8 @@ void free_unref_folios(struct folio_batch *folios) * allocator. */ if (!pcp_allowed_order(order)) { - int migratetype; - - migratetype = get_pfnblock_migratetype(&folio->page, - pfn); - free_one_page(folio_zone(folio), &folio->page, pfn, - order, migratetype, FPI_NONE); + free_one_page(folio_zone(folio), &folio->page, + pfn, order, FPI_NONE); continue; } folio->private = (void *)(unsigned long)order; @@ -2602,7 +2587,7 @@ void free_unref_folios(struct folio_batch *folios) */ if (is_migrate_isolate(migratetype)) { free_one_page(zone, &folio->page, pfn, - order, migratetype, FPI_NONE); + order, FPI_NONE); continue; } @@ -2615,7 +2600,7 @@ void free_unref_folios(struct folio_batch *folios) if (unlikely(!pcp)) { pcp_trylock_finish(UP_flags); free_one_page(zone, &folio->page, pfn, - order, migratetype, FPI_NONE); + order, FPI_NONE); continue; } locked_zone = zone; @@ -6798,13 +6783,14 @@ bool take_page_off_buddy(struct page *page) bool put_page_back_buddy(struct page *page) { struct zone *zone = page_zone(page); - unsigned long pfn = page_to_pfn(page); unsigned long flags; - int migratetype = get_pfnblock_migratetype(page, pfn); bool ret = false; spin_lock_irqsave(&zone->lock, flags); if (put_page_testzero(page)) { + unsigned long pfn = page_to_pfn(page); + int migratetype = get_pfnblock_migratetype(page, pfn); + ClearPageHWPoisonTakenOff(page); __free_one_page(page, pfn, zone, 0, migratetype, FPI_NONE); if (TestClearPageHWPoison(page)) { From f37c0f6876a8eabe1477c87860460bc181f6cdbb Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Wed, 20 Mar 2024 14:02:13 -0400 Subject: [PATCH 0612/1702] mm: page_alloc: set migratetype inside move_freepages() This avoids changing migratetype after move_freepages() or move_freepages_block(), which is error prone. It also prepares for upcoming changes to fix move_freepages() not moving free pages partially in the range. Link: https://lkml.kernel.org/r/20240320180429.678181-9-hannes@cmpxchg.org Signed-off-by: Zi Yan Signed-off-by: Johannes Weiner Reviewed-by: Vlastimil Babka Tested-by: Baolin Wang Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Mel Gorman Signed-off-by: Andrew Morton --- mm/page_alloc.c | 27 +++++++++++++-------------- mm/page_isolation.c | 7 +++---- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 289dcb4347192..fee52ce8ab2d4 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1586,9 +1586,8 @@ static inline struct page *__rmqueue_cma_fallback(struct zone *zone, #endif /* - * Move the free pages in a range to the freelist tail of the requested type. - * Note that start_page and end_pages are not aligned on a pageblock - * boundary. If alignment is required, use move_freepages_block() + * Change the type of a block and move all its free pages to that + * type's freelist. */ static int move_freepages(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn, int migratetype) @@ -1598,6 +1597,9 @@ static int move_freepages(struct zone *zone, unsigned long start_pfn, unsigned int order; int pages_moved = 0; + VM_WARN_ON(start_pfn & (pageblock_nr_pages - 1)); + VM_WARN_ON(start_pfn + pageblock_nr_pages - 1 != end_pfn); + for (pfn = start_pfn; pfn <= end_pfn;) { page = pfn_to_page(pfn); if (!PageBuddy(page)) { @@ -1615,6 +1617,8 @@ static int move_freepages(struct zone *zone, unsigned long start_pfn, pages_moved += 1 << order; } + set_pageblock_migratetype(pfn_to_page(start_pfn), migratetype); + return pages_moved; } @@ -1842,7 +1846,6 @@ steal_suitable_fallback(struct zone *zone, struct page *page, if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || page_group_by_mobility_disabled) { move_freepages(zone, start_pfn, end_pfn, start_type); - set_pageblock_migratetype(page, start_type); return __rmqueue_smallest(zone, order, start_type); } @@ -1916,12 +1919,10 @@ static void reserve_highatomic_pageblock(struct page *page, struct zone *zone) /* Yoink! */ mt = get_pageblock_migratetype(page); /* Only reserve normal pageblocks (i.e., they can merge with others) */ - if (migratetype_is_mergeable(mt)) { - if (move_freepages_block(zone, page, MIGRATE_HIGHATOMIC) != -1) { - set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC); + if (migratetype_is_mergeable(mt)) + if (move_freepages_block(zone, page, + MIGRATE_HIGHATOMIC) != -1) zone->nr_reserved_highatomic += pageblock_nr_pages; - } - } out_unlock: spin_unlock_irqrestore(&zone->lock, flags); @@ -2000,7 +2001,6 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, * not fail on zone boundaries. */ WARN_ON_ONCE(ret == -1); - set_pageblock_migratetype(page, ac->migratetype); if (ret > 0) { spin_unlock_irqrestore(&zone->lock, flags); return ret; @@ -2682,10 +2682,9 @@ int __isolate_free_page(struct page *page, unsigned int order) * Only change normal pageblocks (i.e., they can merge * with others) */ - if (migratetype_is_mergeable(mt) && - move_freepages_block(zone, page, - MIGRATE_MOVABLE) != -1) - set_pageblock_migratetype(page, MIGRATE_MOVABLE); + if (migratetype_is_mergeable(mt)) + move_freepages_block(zone, page, + MIGRATE_MOVABLE); } } diff --git a/mm/page_isolation.c b/mm/page_isolation.c index 71539d7b96cf9..f84f0981b2dfa 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -188,7 +188,6 @@ static int set_migratetype_isolate(struct page *page, int migratetype, int isol_ return -EBUSY; } __mod_zone_freepage_state(zone, -nr_pages, mt); - set_pageblock_migratetype(page, MIGRATE_ISOLATE); zone->nr_isolate_pageblock++; spin_unlock_irqrestore(&zone->lock, flags); return 0; @@ -262,10 +261,10 @@ static void unset_migratetype_isolate(struct page *page, int migratetype) */ WARN_ON_ONCE(nr_pages == -1); __mod_zone_freepage_state(zone, nr_pages, migratetype); - } - set_pageblock_migratetype(page, migratetype); - if (isolated_page) + } else { + set_pageblock_migratetype(page, migratetype); __putback_isolated_page(page, order, migratetype); + } zone->nr_isolate_pageblock--; out: spin_unlock_irqrestore(&zone->lock, flags); From fd919a85cd55be5d00a6a7372071f44c8eafb825 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 20 Mar 2024 14:02:14 -0400 Subject: [PATCH 0613/1702] mm: page_isolation: prepare for hygienic freelists Page isolation currently sets MIGRATE_ISOLATE on a block, then drops zone->lock and scans the block for straddling buddies to split up. Because this happens non-atomically wrt the page allocator, it's possible for allocations to get a buddy whose first block is a regular pcp migratetype but whose tail is isolated. This means that in certain cases memory can still be allocated after isolation. It will also trigger the freelist type hygiene warnings in subsequent patches. start_isolate_page_range() isolate_single_pageblock() set_migratetype_isolate(tail) lock zone->lock move_freepages_block(tail) // nop set_pageblock_migratetype(tail) unlock zone->lock __rmqueue_smallest() del_page_from_freelist(head) expand(head, head_mt) WARN(head_mt != tail_mt) start_pfn = ALIGN_DOWN(MAX_ORDER_NR_PAGES) for (pfn = start_pfn, pfn < end_pfn) if (PageBuddy()) split_free_page(head) Introduce a variant of move_freepages_block() provided by the allocator specifically for page isolation; it moves free pages, converts the block, and handles the splitting of straddling buddies while holding zone->lock. The allocator knows that pageblocks and buddies are always naturally aligned, which means that buddies can only straddle blocks if they're actually >pageblock_order. This means the search-and-split part can be simplified compared to what page isolation used to do. Also tighten up the page isolation code around the expectations of which pages can be large, and how they are freed. Based on extensive discussions with and invaluable input from Zi Yan. [hannes@cmpxchg.org: work around older gcc warning] Link: https://lkml.kernel.org/r/20240321142426.GB777580@cmpxchg.org Link: https://lkml.kernel.org/r/20240320180429.678181-10-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Reviewed-by: Vlastimil Babka Tested-by: Baolin Wang Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Mel Gorman Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/page-isolation.h | 4 +- mm/internal.h | 4 - mm/page_alloc.c | 204 +++++++++++++++++++-------------- mm/page_isolation.c | 106 ++++++----------- 4 files changed, 155 insertions(+), 163 deletions(-) diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h index 8550b3c914809..c16db00670907 100644 --- a/include/linux/page-isolation.h +++ b/include/linux/page-isolation.h @@ -34,7 +34,9 @@ static inline bool is_migrate_isolate(int migratetype) #define REPORT_FAILURE 0x2 void set_pageblock_migratetype(struct page *page, int migratetype); -int move_freepages_block(struct zone *zone, struct page *page, int migratetype); + +bool move_freepages_block_isolate(struct zone *zone, struct page *page, + int migratetype); int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, int migratetype, int flags, gfp_t gfp_flags); diff --git a/mm/internal.h b/mm/internal.h index ab8250d8a5911..cbc40f07537ca 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -562,10 +562,6 @@ extern void *memmap_alloc(phys_addr_t size, phys_addr_t align, void memmap_init_range(unsigned long, int, unsigned long, unsigned long, unsigned long, enum meminit_context, struct vmem_altmap *, int); - -int split_free_page(struct page *free_page, - unsigned int order, unsigned long split_pfn_offset); - #if defined CONFIG_COMPACTION || defined CONFIG_CMA /* diff --git a/mm/page_alloc.c b/mm/page_alloc.c index fee52ce8ab2d4..fe7853eb3ea0c 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -833,64 +833,6 @@ static inline void __free_one_page(struct page *page, page_reporting_notify_free(order); } -/** - * split_free_page() -- split a free page at split_pfn_offset - * @free_page: the original free page - * @order: the order of the page - * @split_pfn_offset: split offset within the page - * - * Return -ENOENT if the free page is changed, otherwise 0 - * - * It is used when the free page crosses two pageblocks with different migratetypes - * at split_pfn_offset within the page. The split free page will be put into - * separate migratetype lists afterwards. Otherwise, the function achieves - * nothing. - */ -int split_free_page(struct page *free_page, - unsigned int order, unsigned long split_pfn_offset) -{ - struct zone *zone = page_zone(free_page); - unsigned long free_page_pfn = page_to_pfn(free_page); - unsigned long pfn; - unsigned long flags; - int free_page_order; - int mt; - int ret = 0; - - if (split_pfn_offset == 0) - return ret; - - spin_lock_irqsave(&zone->lock, flags); - - if (!PageBuddy(free_page) || buddy_order(free_page) != order) { - ret = -ENOENT; - goto out; - } - - mt = get_pfnblock_migratetype(free_page, free_page_pfn); - if (likely(!is_migrate_isolate(mt))) - __mod_zone_freepage_state(zone, -(1UL << order), mt); - - del_page_from_free_list(free_page, zone, order); - for (pfn = free_page_pfn; - pfn < free_page_pfn + (1UL << order);) { - int mt = get_pfnblock_migratetype(pfn_to_page(pfn), pfn); - - free_page_order = min_t(unsigned int, - pfn ? __ffs(pfn) : order, - __fls(split_pfn_offset)); - __free_one_page(pfn_to_page(pfn), pfn, zone, free_page_order, - mt, FPI_NONE); - pfn += 1UL << free_page_order; - split_pfn_offset -= (1UL << free_page_order); - /* we have done the first part, now switch to second part */ - if (split_pfn_offset == 0) - split_pfn_offset = (1UL << order) - (pfn - free_page_pfn); - } -out: - spin_unlock_irqrestore(&zone->lock, flags); - return ret; -} /* * A bad page could be due to a number of fields. Instead of multiple branches, * try and check multiple fields with one check. The caller must do a detailed @@ -1674,8 +1616,8 @@ static bool prep_move_freepages_block(struct zone *zone, struct page *page, return true; } -int move_freepages_block(struct zone *zone, struct page *page, - int migratetype) +static int move_freepages_block(struct zone *zone, struct page *page, + int migratetype) { unsigned long start_pfn, end_pfn; @@ -1686,6 +1628,123 @@ int move_freepages_block(struct zone *zone, struct page *page, return move_freepages(zone, start_pfn, end_pfn, migratetype); } +#ifdef CONFIG_MEMORY_ISOLATION +/* Look for a buddy that straddles start_pfn */ +static unsigned long find_large_buddy(unsigned long start_pfn) +{ + int order = 0; + struct page *page; + unsigned long pfn = start_pfn; + + while (!PageBuddy(page = pfn_to_page(pfn))) { + /* Nothing found */ + if (++order > MAX_PAGE_ORDER) + return start_pfn; + pfn &= ~0UL << order; + } + + /* + * Found a preceding buddy, but does it straddle? + */ + if (pfn + (1 << buddy_order(page)) > start_pfn) + return pfn; + + /* Nothing found */ + return start_pfn; +} + +/* Split a multi-block free page into its individual pageblocks */ +static void split_large_buddy(struct zone *zone, struct page *page, + unsigned long pfn, int order) +{ + unsigned long end_pfn = pfn + (1 << order); + + VM_WARN_ON_ONCE(order <= pageblock_order); + VM_WARN_ON_ONCE(pfn & (pageblock_nr_pages - 1)); + + /* Caller removed page from freelist, buddy info cleared! */ + VM_WARN_ON_ONCE(PageBuddy(page)); + + while (pfn != end_pfn) { + int mt = get_pfnblock_migratetype(page, pfn); + + __free_one_page(page, pfn, zone, pageblock_order, mt, FPI_NONE); + pfn += pageblock_nr_pages; + page = pfn_to_page(pfn); + } +} + +/** + * move_freepages_block_isolate - move free pages in block for page isolation + * @zone: the zone + * @page: the pageblock page + * @migratetype: migratetype to set on the pageblock + * + * This is similar to move_freepages_block(), but handles the special + * case encountered in page isolation, where the block of interest + * might be part of a larger buddy spanning multiple pageblocks. + * + * Unlike the regular page allocator path, which moves pages while + * stealing buddies off the freelist, page isolation is interested in + * arbitrary pfn ranges that may have overlapping buddies on both ends. + * + * This function handles that. Straddling buddies are split into + * individual pageblocks. Only the block of interest is moved. + * + * Returns %true if pages could be moved, %false otherwise. + */ +bool move_freepages_block_isolate(struct zone *zone, struct page *page, + int migratetype) +{ + unsigned long start_pfn, end_pfn, pfn; + int nr_moved, mt; + + if (!prep_move_freepages_block(zone, page, &start_pfn, &end_pfn, + NULL, NULL)) + return false; + + /* No splits needed if buddies can't span multiple blocks */ + if (pageblock_order == MAX_PAGE_ORDER) + goto move; + + /* We're a tail block in a larger buddy */ + pfn = find_large_buddy(start_pfn); + if (pfn != start_pfn) { + struct page *buddy = pfn_to_page(pfn); + int order = buddy_order(buddy); + int mt = get_pfnblock_migratetype(buddy, pfn); + + if (!is_migrate_isolate(mt)) + __mod_zone_freepage_state(zone, -(1UL << order), mt); + del_page_from_free_list(buddy, zone, order); + set_pageblock_migratetype(page, migratetype); + split_large_buddy(zone, buddy, pfn, order); + return true; + } + + /* We're the starting block of a larger buddy */ + if (PageBuddy(page) && buddy_order(page) > pageblock_order) { + int mt = get_pfnblock_migratetype(page, pfn); + int order = buddy_order(page); + + if (!is_migrate_isolate(mt)) + __mod_zone_freepage_state(zone, -(1UL << order), mt); + del_page_from_free_list(page, zone, order); + set_pageblock_migratetype(page, migratetype); + split_large_buddy(zone, page, pfn, order); + return true; + } +move: + mt = get_pfnblock_migratetype(page, start_pfn); + nr_moved = move_freepages(zone, start_pfn, end_pfn, migratetype); + if (!is_migrate_isolate(mt)) + __mod_zone_freepage_state(zone, -nr_moved, mt); + else if (!is_migrate_isolate(migratetype)) + __mod_zone_freepage_state(zone, nr_moved, migratetype); + return true; +} +#endif /* CONFIG_MEMORY_ISOLATION */ + static void change_pageblock_range(struct page *pageblock_page, int start_order, int migratetype) { @@ -6365,7 +6424,6 @@ int alloc_contig_range_noprof(unsigned long start, unsigned long end, unsigned migratetype, gfp_t gfp_mask) { unsigned long outer_start, outer_end; - int order; int ret = 0; struct compact_control cc = { @@ -6438,29 +6496,7 @@ int alloc_contig_range_noprof(unsigned long start, unsigned long end, * We don't have to hold zone->lock here because the pages are * isolated thus they won't get removed from buddy. */ - - order = 0; - outer_start = start; - while (!PageBuddy(pfn_to_page(outer_start))) { - if (++order > MAX_PAGE_ORDER) { - outer_start = start; - break; - } - outer_start &= ~0UL << order; - } - - if (outer_start != start) { - order = buddy_order(pfn_to_page(outer_start)); - - /* - * outer_start page could be small order buddy page and - * it doesn't include start page. Adjust outer_start - * in this case to report failed page properly - * on tracepoint in test_pages_isolated() - */ - if (outer_start + (1UL << order) <= start) - outer_start = start; - } + outer_start = find_large_buddy(start); /* Make sure the range is really isolated. */ if (test_pages_isolated(outer_start, end, 0)) { diff --git a/mm/page_isolation.c b/mm/page_isolation.c index f84f0981b2dfa..042937d5abe4b 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -178,16 +178,10 @@ static int set_migratetype_isolate(struct page *page, int migratetype, int isol_ unmovable = has_unmovable_pages(check_unmovable_start, check_unmovable_end, migratetype, isol_flags); if (!unmovable) { - int nr_pages; - int mt = get_pageblock_migratetype(page); - - nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE); - /* Block spans zone boundaries? */ - if (nr_pages == -1) { + if (!move_freepages_block_isolate(zone, page, MIGRATE_ISOLATE)) { spin_unlock_irqrestore(&zone->lock, flags); return -EBUSY; } - __mod_zone_freepage_state(zone, -nr_pages, mt); zone->nr_isolate_pageblock++; spin_unlock_irqrestore(&zone->lock, flags); return 0; @@ -254,13 +248,11 @@ static void unset_migratetype_isolate(struct page *page, int migratetype) * allocation. */ if (!isolated_page) { - int nr_pages = move_freepages_block(zone, page, migratetype); /* * Isolating this block already succeeded, so this * should not fail on zone boundaries. */ - WARN_ON_ONCE(nr_pages == -1); - __mod_zone_freepage_state(zone, nr_pages, migratetype); + WARN_ON_ONCE(!move_freepages_block_isolate(zone, page, migratetype)); } else { set_pageblock_migratetype(page, migratetype); __putback_isolated_page(page, order, migratetype); @@ -374,26 +366,29 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags, VM_BUG_ON(!page); pfn = page_to_pfn(page); - /* - * start_pfn is MAX_ORDER_NR_PAGES aligned, if there is any - * free pages in [start_pfn, boundary_pfn), its head page will - * always be in the range. - */ + if (PageBuddy(page)) { int order = buddy_order(page); - if (pfn + (1UL << order) > boundary_pfn) { - /* free page changed before split, check it again */ - if (split_free_page(page, order, boundary_pfn - pfn)) - continue; - } + /* move_freepages_block_isolate() handled this */ + VM_WARN_ON_ONCE(pfn + (1 << order) > boundary_pfn); pfn += 1UL << order; continue; } + /* - * migrate compound pages then let the free page handling code - * above do the rest. If migration is not possible, just fail. + * If a compound page is straddling our block, attempt + * to migrate it out of the way. + * + * We don't have to worry about this creating a large + * free page that straddles into our block: gigantic + * pages are freed as order-0 chunks, and LRU pages + * (currently) do not exceed pageblock_order. + * + * The block of interest has already been marked + * MIGRATE_ISOLATE above, so when migration is done it + * will free its pages onto the correct freelists. */ if (PageCompound(page)) { struct page *head = compound_head(page); @@ -404,16 +399,10 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags, pfn = head_pfn + nr_pages; continue; } + #if defined CONFIG_COMPACTION || defined CONFIG_CMA - /* - * hugetlb, lru compound (THP), and movable compound pages - * can be migrated. Otherwise, fail the isolation. - */ - if (PageHuge(page) || PageLRU(page) || __PageMovable(page)) { - int order; - unsigned long outer_pfn; + if (PageHuge(page)) { int page_mt = get_pageblock_migratetype(page); - bool isolate_page = !is_migrate_isolate_page(page); struct compact_control cc = { .nr_migratepages = 0, .order = -1, @@ -426,56 +415,25 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags, }; INIT_LIST_HEAD(&cc.migratepages); - /* - * XXX: mark the page as MIGRATE_ISOLATE so that - * no one else can grab the freed page after migration. - * Ideally, the page should be freed as two separate - * pages to be added into separate migratetype free - * lists. - */ - if (isolate_page) { - ret = set_migratetype_isolate(page, page_mt, - flags, head_pfn, head_pfn + nr_pages); - if (ret) - goto failed; - } - ret = __alloc_contig_migrate_range(&cc, head_pfn, head_pfn + nr_pages, page_mt); - - /* - * restore the page's migratetype so that it can - * be split into separate migratetype free lists - * later. - */ - if (isolate_page) - unset_migratetype_isolate(page, page_mt); - if (ret) goto failed; - /* - * reset pfn to the head of the free page, so - * that the free page handling code above can split - * the free page to the right migratetype list. - * - * head_pfn is not used here as a hugetlb page order - * can be bigger than MAX_PAGE_ORDER, but after it is - * freed, the free page order is not. Use pfn within - * the range to find the head of the free page. - */ - order = 0; - outer_pfn = pfn; - while (!PageBuddy(pfn_to_page(outer_pfn))) { - /* stop if we cannot find the free page */ - if (++order > MAX_PAGE_ORDER) - goto failed; - outer_pfn &= ~0UL << order; - } - pfn = outer_pfn; + pfn = head_pfn + nr_pages; continue; - } else + } + + /* + * These pages are movable too, but they're + * not expected to exceed pageblock_order. + * + * Let us know when they do, so we can add + * proper free and split handling for them. + */ + VM_WARN_ON_ONCE_PAGE(PageLRU(page), page); + VM_WARN_ON_ONCE_PAGE(__PageMovable(page), page); #endif - goto failed; + goto failed; } pfn++; From e0932b6c1f942fa747258e152cdce0d0b2b5be5c Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 20 Mar 2024 14:02:15 -0400 Subject: [PATCH 0614/1702] mm: page_alloc: consolidate free page accounting Free page accounting currently happens a bit too high up the call stack, where it has to deal with guard pages, compaction capturing, block stealing and even page isolation. This is subtle and fragile, and makes it difficult to hack on the code. Now that type violations on the freelists have been fixed, push the accounting down to where pages enter and leave the freelist. [hannes@cmpxchg.org: undo unrelated drive-by line wrap] Link: https://lkml.kernel.org/r/20240327185736.GA7597@cmpxchg.org [hannes@cmpxchg.org: remove unused page parameter from account_freepages()] Link: https://lkml.kernel.org/r/20240327185831.GB7597@cmpxchg.org [baolin.wang@linux.alibaba.com: fix free page accounting] Link: https://lkml.kernel.org/r/a2a48baca69f103aa431fd201f8a06e3b95e203d.1712648441.git.baolin.wang@linux.alibaba.com [andriy.shevchenko@linux.intel.com: avoid defining unused function] Link: https://lkml.kernel.org/r/20240423161506.2637177-1-andriy.shevchenko@linux.intel.com Link: https://lkml.kernel.org/r/20240320180429.678181-11-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Signed-off-by: Andy Shevchenko Signed-off-by: Baolin Wang Reviewed-by: Vlastimil Babka Tested-by: Baolin Wang Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Mel Gorman Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/mm.h | 18 ++-- include/linux/vmstat.h | 8 -- mm/debug_page_alloc.c | 12 +-- mm/internal.h | 5 -- mm/page_alloc.c | 192 +++++++++++++++++++++++------------------ 5 files changed, 118 insertions(+), 117 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 60eabc6c8e007..1588fe15a38e3 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3797,24 +3797,22 @@ static inline bool page_is_guard(struct page *page) return PageGuard(page); } -bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order, - int migratetype); +bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order); static inline bool set_page_guard(struct zone *zone, struct page *page, - unsigned int order, int migratetype) + unsigned int order) { if (!debug_guardpage_enabled()) return false; - return __set_page_guard(zone, page, order, migratetype); + return __set_page_guard(zone, page, order); } -void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order, - int migratetype); +void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order); static inline void clear_page_guard(struct zone *zone, struct page *page, - unsigned int order, int migratetype) + unsigned int order) { if (!debug_guardpage_enabled()) return; - __clear_page_guard(zone, page, order, migratetype); + __clear_page_guard(zone, page, order); } #else /* CONFIG_DEBUG_PAGEALLOC */ @@ -3824,9 +3822,9 @@ static inline unsigned int debug_guardpage_minorder(void) { return 0; } static inline bool debug_guardpage_enabled(void) { return false; } static inline bool page_is_guard(struct page *page) { return false; } static inline bool set_page_guard(struct zone *zone, struct page *page, - unsigned int order, int migratetype) { return false; } + unsigned int order) { return false; } static inline void clear_page_guard(struct zone *zone, struct page *page, - unsigned int order, int migratetype) {} + unsigned int order) {} #endif /* CONFIG_DEBUG_PAGEALLOC */ #ifdef __HAVE_ARCH_GATE_AREA diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h index 343906a98d6ee..735eae6e272cb 100644 --- a/include/linux/vmstat.h +++ b/include/linux/vmstat.h @@ -487,14 +487,6 @@ static inline void node_stat_sub_folio(struct folio *folio, mod_node_page_state(folio_pgdat(folio), item, -folio_nr_pages(folio)); } -static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages, - int migratetype) -{ - __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages); - if (is_migrate_cma(migratetype)) - __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages); -} - extern const char * const vmstat_text[]; static inline const char *zone_stat_name(enum zone_stat_item item) diff --git a/mm/debug_page_alloc.c b/mm/debug_page_alloc.c index 6755f0c9d4a39..d46acf989dde8 100644 --- a/mm/debug_page_alloc.c +++ b/mm/debug_page_alloc.c @@ -32,8 +32,7 @@ static int __init debug_guardpage_minorder_setup(char *buf) } early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup); -bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order, - int migratetype) +bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order) { if (order >= debug_guardpage_minorder()) return false; @@ -41,19 +40,12 @@ bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order, __SetPageGuard(page); INIT_LIST_HEAD(&page->buddy_list); set_page_private(page, order); - /* Guard pages are not available for any usage */ - if (!is_migrate_isolate(migratetype)) - __mod_zone_freepage_state(zone, -(1 << order), migratetype); return true; } -void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order, - int migratetype) +void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order) { __ClearPageGuard(page); - set_page_private(page, 0); - if (!is_migrate_isolate(migratetype)) - __mod_zone_freepage_state(zone, (1 << order), migratetype); } diff --git a/mm/internal.h b/mm/internal.h index cbc40f07537ca..fb219e31f0f0b 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1039,11 +1039,6 @@ static inline bool is_migrate_highatomic(enum migratetype migratetype) return migratetype == MIGRATE_HIGHATOMIC; } -static inline bool is_migrate_highatomic_page(struct page *page) -{ - return get_pageblock_migratetype(page) == MIGRATE_HIGHATOMIC; -} - void setup_zone_pageset(struct zone *zone); struct migration_target_control { diff --git a/mm/page_alloc.c b/mm/page_alloc.c index fe7853eb3ea0c..b3b98a91d6a75 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -643,23 +643,33 @@ compaction_capture(struct capture_control *capc, struct page *page, } #endif /* CONFIG_COMPACTION */ -/* Used for pages not on another list */ -static inline void add_to_free_list(struct page *page, struct zone *zone, - unsigned int order, int migratetype) +static inline void account_freepages(struct zone *zone, int nr_pages, + int migratetype) { - struct free_area *area = &zone->free_area[order]; + if (is_migrate_isolate(migratetype)) + return; - list_add(&page->buddy_list, &area->free_list[migratetype]); - area->nr_free++; + __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages); + + if (is_migrate_cma(migratetype)) + __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages); } /* Used for pages not on another list */ -static inline void add_to_free_list_tail(struct page *page, struct zone *zone, - unsigned int order, int migratetype) +static inline void __add_to_free_list(struct page *page, struct zone *zone, + unsigned int order, int migratetype, + bool tail) { struct free_area *area = &zone->free_area[order]; - list_add_tail(&page->buddy_list, &area->free_list[migratetype]); + VM_WARN_ONCE(get_pageblock_migratetype(page) != migratetype, + "page type is %lu, passed migratetype is %d (nr=%d)\n", + get_pageblock_migratetype(page), migratetype, 1 << order); + + if (tail) + list_add_tail(&page->buddy_list, &area->free_list[migratetype]); + else + list_add(&page->buddy_list, &area->free_list[migratetype]); area->nr_free++; } @@ -669,16 +679,28 @@ static inline void add_to_free_list_tail(struct page *page, struct zone *zone, * allocation again (e.g., optimization for memory onlining). */ static inline void move_to_free_list(struct page *page, struct zone *zone, - unsigned int order, int migratetype) + unsigned int order, int old_mt, int new_mt) { struct free_area *area = &zone->free_area[order]; - list_move_tail(&page->buddy_list, &area->free_list[migratetype]); + /* Free page moving can fail, so it happens before the type update */ + VM_WARN_ONCE(get_pageblock_migratetype(page) != old_mt, + "page type is %lu, passed migratetype is %d (nr=%d)\n", + get_pageblock_migratetype(page), old_mt, 1 << order); + + list_move_tail(&page->buddy_list, &area->free_list[new_mt]); + + account_freepages(zone, -(1 << order), old_mt); + account_freepages(zone, 1 << order, new_mt); } -static inline void del_page_from_free_list(struct page *page, struct zone *zone, - unsigned int order) +static inline void __del_page_from_free_list(struct page *page, struct zone *zone, + unsigned int order, int migratetype) { + VM_WARN_ONCE(get_pageblock_migratetype(page) != migratetype, + "page type is %lu, passed migratetype is %d (nr=%d)\n", + get_pageblock_migratetype(page), migratetype, 1 << order); + /* clear reported state and update reported page count */ if (page_reported(page)) __ClearPageReported(page); @@ -689,6 +711,13 @@ static inline void del_page_from_free_list(struct page *page, struct zone *zone, zone->free_area[order].nr_free--; } +static inline void del_page_from_free_list(struct page *page, struct zone *zone, + unsigned int order, int migratetype) +{ + __del_page_from_free_list(page, zone, order, migratetype); + account_freepages(zone, -(1 << order), migratetype); +} + static inline struct page *get_page_from_free_area(struct free_area *area, int migratetype) { @@ -760,16 +789,16 @@ static inline void __free_one_page(struct page *page, VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); VM_BUG_ON(migratetype == -1); - if (likely(!is_migrate_isolate(migratetype))) - __mod_zone_freepage_state(zone, 1 << order, migratetype); - VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); VM_BUG_ON_PAGE(bad_range(zone, page), page); + account_freepages(zone, 1 << order, migratetype); + while (order < MAX_PAGE_ORDER) { + int buddy_mt = migratetype; + if (compaction_capture(capc, page, order, migratetype)) { - __mod_zone_freepage_state(zone, -(1 << order), - migratetype); + account_freepages(zone, -(1 << order), migratetype); return; } @@ -784,19 +813,12 @@ static inline void __free_one_page(struct page *page, * pageblock isolation could cause incorrect freepage or CMA * accounting or HIGHATOMIC accounting. */ - int buddy_mt = get_pfnblock_migratetype(buddy, buddy_pfn); + buddy_mt = get_pfnblock_migratetype(buddy, buddy_pfn); - if (migratetype != buddy_mt) { - if (!migratetype_is_mergeable(migratetype) || - !migratetype_is_mergeable(buddy_mt)) - goto done_merging; - /* - * Match buddy type. This ensures that - * an expand() down the line puts the - * sub-blocks on the right freelists. - */ - set_pageblock_migratetype(buddy, migratetype); - } + if (migratetype != buddy_mt && + (!migratetype_is_mergeable(migratetype) || + !migratetype_is_mergeable(buddy_mt))) + goto done_merging; } /* @@ -804,9 +826,19 @@ static inline void __free_one_page(struct page *page, * merge with it and move up one order. */ if (page_is_guard(buddy)) - clear_page_guard(zone, buddy, order, migratetype); + clear_page_guard(zone, buddy, order); else - del_page_from_free_list(buddy, zone, order); + __del_page_from_free_list(buddy, zone, order, buddy_mt); + + if (unlikely(buddy_mt != migratetype)) { + /* + * Match buddy type. This ensures that an + * expand() down the line puts the sub-blocks + * on the right freelists. + */ + set_pageblock_migratetype(buddy, migratetype); + } + combined_pfn = buddy_pfn & pfn; page = page + (combined_pfn - pfn); pfn = combined_pfn; @@ -823,10 +855,7 @@ static inline void __free_one_page(struct page *page, else to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); - if (to_tail) - add_to_free_list_tail(page, zone, order, migratetype); - else - add_to_free_list(page, zone, order, migratetype); + __add_to_free_list(page, zone, order, migratetype, to_tail); /* Notify page reporting subsystem of freed page */ if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY)) @@ -1318,10 +1347,10 @@ static inline void expand(struct zone *zone, struct page *page, * Corresponding page table entries will not be touched, * pages will stay not present in virtual address space */ - if (set_page_guard(zone, &page[size], high, migratetype)) + if (set_page_guard(zone, &page[size], high)) continue; - add_to_free_list(&page[size], zone, high, migratetype); + add_to_free_list(&page[size], zone, high, migratetype, false); set_buddy_order(&page[size], high); } } @@ -1492,7 +1521,7 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, page = get_page_from_free_area(area, migratetype); if (!page) continue; - del_page_from_free_list(page, zone, current_order); + del_page_from_free_list(page, zone, current_order, migratetype); expand(zone, page, order, current_order, migratetype); trace_mm_page_alloc_zone_locked(page, order, migratetype, pcp_allowed_order(order) && @@ -1532,7 +1561,7 @@ static inline struct page *__rmqueue_cma_fallback(struct zone *zone, * type's freelist. */ static int move_freepages(struct zone *zone, unsigned long start_pfn, - unsigned long end_pfn, int migratetype) + unsigned long end_pfn, int old_mt, int new_mt) { struct page *page; unsigned long pfn; @@ -1554,12 +1583,14 @@ static int move_freepages(struct zone *zone, unsigned long start_pfn, VM_BUG_ON_PAGE(page_zone(page) != zone, page); order = buddy_order(page); - move_to_free_list(page, zone, order, migratetype); + + move_to_free_list(page, zone, order, old_mt, new_mt); + pfn += 1 << order; pages_moved += 1 << order; } - set_pageblock_migratetype(pfn_to_page(start_pfn), migratetype); + set_pageblock_migratetype(pfn_to_page(start_pfn), new_mt); return pages_moved; } @@ -1617,7 +1648,7 @@ static bool prep_move_freepages_block(struct zone *zone, struct page *page, } static int move_freepages_block(struct zone *zone, struct page *page, - int migratetype) + int old_mt, int new_mt) { unsigned long start_pfn, end_pfn; @@ -1625,7 +1656,7 @@ static int move_freepages_block(struct zone *zone, struct page *page, NULL, NULL)) return -1; - return move_freepages(zone, start_pfn, end_pfn, migratetype); + return move_freepages(zone, start_pfn, end_pfn, old_mt, new_mt); } #ifdef CONFIG_MEMORY_ISOLATION @@ -1697,7 +1728,6 @@ bool move_freepages_block_isolate(struct zone *zone, struct page *page, int migratetype) { unsigned long start_pfn, end_pfn, pfn; - int nr_moved, mt; if (!prep_move_freepages_block(zone, page, &start_pfn, &end_pfn, NULL, NULL)) @@ -1712,11 +1742,9 @@ bool move_freepages_block_isolate(struct zone *zone, struct page *page, if (pfn != start_pfn) { struct page *buddy = pfn_to_page(pfn); int order = buddy_order(buddy); - int mt = get_pfnblock_migratetype(buddy, pfn); - if (!is_migrate_isolate(mt)) - __mod_zone_freepage_state(zone, -(1UL << order), mt); - del_page_from_free_list(buddy, zone, order); + del_page_from_free_list(buddy, zone, order, + get_pfnblock_migratetype(buddy, pfn)); set_pageblock_migratetype(page, migratetype); split_large_buddy(zone, buddy, pfn, order); return true; @@ -1724,23 +1752,17 @@ bool move_freepages_block_isolate(struct zone *zone, struct page *page, /* We're the starting block of a larger buddy */ if (PageBuddy(page) && buddy_order(page) > pageblock_order) { - int mt = get_pfnblock_migratetype(page, pfn); int order = buddy_order(page); - if (!is_migrate_isolate(mt)) - __mod_zone_freepage_state(zone, -(1UL << order), mt); - del_page_from_free_list(page, zone, order); + del_page_from_free_list(page, zone, order, + get_pfnblock_migratetype(page, pfn)); set_pageblock_migratetype(page, migratetype); split_large_buddy(zone, page, pfn, order); return true; } move: - mt = get_pfnblock_migratetype(page, start_pfn); - nr_moved = move_freepages(zone, start_pfn, end_pfn, migratetype); - if (!is_migrate_isolate(mt)) - __mod_zone_freepage_state(zone, -nr_moved, mt); - else if (!is_migrate_isolate(migratetype)) - __mod_zone_freepage_state(zone, nr_moved, migratetype); + move_freepages(zone, start_pfn, end_pfn, + get_pfnblock_migratetype(page, start_pfn), migratetype); return true; } #endif /* CONFIG_MEMORY_ISOLATION */ @@ -1854,7 +1876,7 @@ steal_suitable_fallback(struct zone *zone, struct page *page, /* Take ownership for orders >= pageblock_order */ if (current_order >= pageblock_order) { - del_page_from_free_list(page, zone, current_order); + del_page_from_free_list(page, zone, current_order, block_type); change_pageblock_range(page, current_order, start_type); expand(zone, page, order, current_order, start_type); return page; @@ -1904,12 +1926,12 @@ steal_suitable_fallback(struct zone *zone, struct page *page, */ if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || page_group_by_mobility_disabled) { - move_freepages(zone, start_pfn, end_pfn, start_type); + move_freepages(zone, start_pfn, end_pfn, block_type, start_type); return __rmqueue_smallest(zone, order, start_type); } single_page: - del_page_from_free_list(page, zone, current_order); + del_page_from_free_list(page, zone, current_order, block_type); expand(zone, page, order, current_order, block_type); return page; } @@ -1979,7 +2001,7 @@ static void reserve_highatomic_pageblock(struct page *page, struct zone *zone) mt = get_pageblock_migratetype(page); /* Only reserve normal pageblocks (i.e., they can merge with others) */ if (migratetype_is_mergeable(mt)) - if (move_freepages_block(zone, page, + if (move_freepages_block(zone, page, mt, MIGRATE_HIGHATOMIC) != -1) zone->nr_reserved_highatomic += pageblock_nr_pages; @@ -2020,11 +2042,13 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, spin_lock_irqsave(&zone->lock, flags); for (order = 0; order < NR_PAGE_ORDERS; order++) { struct free_area *area = &(zone->free_area[order]); + int mt; page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); if (!page) continue; + mt = get_pageblock_migratetype(page); /* * In page freeing path, migratetype change is racy so * we can counter several free pages in a pageblock @@ -2032,7 +2056,7 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, * from highatomic to ac->migratetype. So we should * adjust the count once. */ - if (is_migrate_highatomic_page(page)) { + if (is_migrate_highatomic(mt)) { /* * It should never happen but changes to * locking could inadvertently allow a per-cpu @@ -2054,7 +2078,8 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, * of pageblocks that cannot be completely freed * may increase. */ - ret = move_freepages_block(zone, page, ac->migratetype); + ret = move_freepages_block(zone, page, mt, + ac->migratetype); /* * Reserving this block already succeeded, so this should * not fail on zone boundaries. @@ -2225,12 +2250,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, * pages are ordered properly. */ list_add_tail(&page->pcp_list, list); - if (is_migrate_cma(get_pageblock_migratetype(page))) - __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, - -(1 << order)); } - - __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); spin_unlock_irqrestore(&zone->lock, flags); return i; @@ -2723,11 +2743,9 @@ int __isolate_free_page(struct page *page, unsigned int order) watermark = zone->_watermark[WMARK_MIN] + (1UL << order); if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) return 0; - - __mod_zone_freepage_state(zone, -(1UL << order), mt); } - del_page_from_free_list(page, zone, order); + del_page_from_free_list(page, zone, order, mt); /* * Set the pageblock if the isolated page is at least half of a @@ -2742,7 +2760,7 @@ int __isolate_free_page(struct page *page, unsigned int order) * with others) */ if (migratetype_is_mergeable(mt)) - move_freepages_block(zone, page, + move_freepages_block(zone, page, mt, MIGRATE_MOVABLE); } } @@ -2827,8 +2845,6 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, return NULL; } } - __mod_zone_freepage_state(zone, -(1 << order), - get_pageblock_migratetype(page)); spin_unlock_irqrestore(&zone->lock, flags); } while (check_new_pages(page, order)); @@ -6716,8 +6732,9 @@ void __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) BUG_ON(page_count(page)); BUG_ON(!PageBuddy(page)); + VM_WARN_ON(get_pageblock_migratetype(page) != MIGRATE_ISOLATE); order = buddy_order(page); - del_page_from_free_list(page, zone, order); + del_page_from_free_list(page, zone, order, MIGRATE_ISOLATE); pfn += (1 << order); } spin_unlock_irqrestore(&zone->lock, flags); @@ -6745,6 +6762,14 @@ bool is_free_buddy_page(struct page *page) EXPORT_SYMBOL(is_free_buddy_page); #ifdef CONFIG_MEMORY_FAILURE +static inline void add_to_free_list(struct page *page, struct zone *zone, + unsigned int order, int migratetype, + bool tail) +{ + __add_to_free_list(page, zone, order, migratetype, tail); + account_freepages(zone, 1 << order, migratetype); +} + /* * Break down a higher-order page in sub-pages, and keep our target out of * buddy allocator. @@ -6767,10 +6792,10 @@ static void break_down_buddy_pages(struct zone *zone, struct page *page, current_buddy = page + size; } - if (set_page_guard(zone, current_buddy, high, migratetype)) + if (set_page_guard(zone, current_buddy, high)) continue; - add_to_free_list(current_buddy, zone, high, migratetype); + add_to_free_list(current_buddy, zone, high, migratetype, false); set_buddy_order(current_buddy, high); } } @@ -6796,12 +6821,11 @@ bool take_page_off_buddy(struct page *page) int migratetype = get_pfnblock_migratetype(page_head, pfn_head); - del_page_from_free_list(page_head, zone, page_order); + del_page_from_free_list(page_head, zone, page_order, + migratetype); break_down_buddy_pages(zone, page_head, page, 0, page_order, migratetype); SetPageHWPoisonTakenOff(page); - if (!is_migrate_isolate(migratetype)) - __mod_zone_freepage_state(zone, -1, migratetype); ret = true; break; } @@ -6909,7 +6933,7 @@ static bool try_to_accept_memory_one(struct zone *zone) list_del(&page->lru); last = list_empty(&zone->unaccepted_pages); - __mod_zone_freepage_state(zone, -MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE); + account_freepages(zone, -MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE); __mod_zone_page_state(zone, NR_UNACCEPTED, -MAX_ORDER_NR_PAGES); spin_unlock_irqrestore(&zone->lock, flags); @@ -6961,7 +6985,7 @@ static bool __free_unaccepted(struct page *page) spin_lock_irqsave(&zone->lock, flags); first = list_empty(&zone->unaccepted_pages); list_add_tail(&page->lru, &zone->unaccepted_pages); - __mod_zone_freepage_state(zone, MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE); + account_freepages(zone, MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE); __mod_zone_page_state(zone, NR_UNACCEPTED, MAX_ORDER_NR_PAGES); spin_unlock_irqrestore(&zone->lock, flags); From e1f42a577f63647dadf1abe4583053c03d6be045 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Thu, 25 Apr 2024 20:56:04 -0700 Subject: [PATCH 0615/1702] mm: page_alloc: change move_freepages() to __move_freepages_block() The function is now supposed to be called only on a single pageblock and checks start_pfn and end_pfn accordingly. Rename it to make this more obvious and drop the end_pfn parameter which can be determined trivially and none of the callers use it for anything else. Also make the (now internal) end_pfn exclusive, which is more common. Link: https://lkml.kernel.org/r/81b1d642-2ec0-49f5-89fc-19a3828419ff@suse.cz Signed-off-by: Vlastimil Babka Reviewed-by: Zi Yan Acked-by: Johannes Weiner Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Mel Gorman Signed-off-by: Andrew Morton --- mm/page_alloc.c | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index b3b98a91d6a75..489dd74d2232a 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1560,18 +1560,18 @@ static inline struct page *__rmqueue_cma_fallback(struct zone *zone, * Change the type of a block and move all its free pages to that * type's freelist. */ -static int move_freepages(struct zone *zone, unsigned long start_pfn, - unsigned long end_pfn, int old_mt, int new_mt) +static int __move_freepages_block(struct zone *zone, unsigned long start_pfn, + int old_mt, int new_mt) { struct page *page; - unsigned long pfn; + unsigned long pfn, end_pfn; unsigned int order; int pages_moved = 0; VM_WARN_ON(start_pfn & (pageblock_nr_pages - 1)); - VM_WARN_ON(start_pfn + pageblock_nr_pages - 1 != end_pfn); + end_pfn = pageblock_end_pfn(start_pfn); - for (pfn = start_pfn; pfn <= end_pfn;) { + for (pfn = start_pfn; pfn < end_pfn;) { page = pfn_to_page(pfn); if (!PageBuddy(page)) { pfn++; @@ -1597,14 +1597,13 @@ static int move_freepages(struct zone *zone, unsigned long start_pfn, static bool prep_move_freepages_block(struct zone *zone, struct page *page, unsigned long *start_pfn, - unsigned long *end_pfn, int *num_free, int *num_movable) { unsigned long pfn, start, end; pfn = page_to_pfn(page); start = pageblock_start_pfn(pfn); - end = pageblock_end_pfn(pfn) - 1; + end = pageblock_end_pfn(pfn); /* * The caller only has the lock for @zone, don't touch ranges @@ -1615,16 +1614,15 @@ static bool prep_move_freepages_block(struct zone *zone, struct page *page, */ if (!zone_spans_pfn(zone, start)) return false; - if (!zone_spans_pfn(zone, end)) + if (!zone_spans_pfn(zone, end - 1)) return false; *start_pfn = start; - *end_pfn = end; if (num_free) { *num_free = 0; *num_movable = 0; - for (pfn = start; pfn <= end;) { + for (pfn = start; pfn < end;) { page = pfn_to_page(pfn); if (PageBuddy(page)) { int nr = 1 << buddy_order(page); @@ -1650,13 +1648,12 @@ static bool prep_move_freepages_block(struct zone *zone, struct page *page, static int move_freepages_block(struct zone *zone, struct page *page, int old_mt, int new_mt) { - unsigned long start_pfn, end_pfn; + unsigned long start_pfn; - if (!prep_move_freepages_block(zone, page, &start_pfn, &end_pfn, - NULL, NULL)) + if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL)) return -1; - return move_freepages(zone, start_pfn, end_pfn, old_mt, new_mt); + return __move_freepages_block(zone, start_pfn, old_mt, new_mt); } #ifdef CONFIG_MEMORY_ISOLATION @@ -1727,10 +1724,9 @@ static void split_large_buddy(struct zone *zone, struct page *page, bool move_freepages_block_isolate(struct zone *zone, struct page *page, int migratetype) { - unsigned long start_pfn, end_pfn, pfn; + unsigned long start_pfn, pfn; - if (!prep_move_freepages_block(zone, page, &start_pfn, &end_pfn, - NULL, NULL)) + if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL)) return false; /* No splits needed if buddies can't span multiple blocks */ @@ -1761,8 +1757,9 @@ bool move_freepages_block_isolate(struct zone *zone, struct page *page, return true; } move: - move_freepages(zone, start_pfn, end_pfn, - get_pfnblock_migratetype(page, start_pfn), migratetype); + __move_freepages_block(zone, start_pfn, + get_pfnblock_migratetype(page, start_pfn), + migratetype); return true; } #endif /* CONFIG_MEMORY_ISOLATION */ @@ -1862,7 +1859,7 @@ steal_suitable_fallback(struct zone *zone, struct page *page, unsigned int alloc_flags, bool whole_block) { int free_pages, movable_pages, alike_pages; - unsigned long start_pfn, end_pfn; + unsigned long start_pfn; int block_type; block_type = get_pageblock_migratetype(page); @@ -1895,8 +1892,8 @@ steal_suitable_fallback(struct zone *zone, struct page *page, goto single_page; /* moving whole block can fail due to zone boundary conditions */ - if (!prep_move_freepages_block(zone, page, &start_pfn, &end_pfn, - &free_pages, &movable_pages)) + if (!prep_move_freepages_block(zone, page, &start_pfn, &free_pages, + &movable_pages)) goto single_page; /* @@ -1926,7 +1923,7 @@ steal_suitable_fallback(struct zone *zone, struct page *page, */ if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || page_group_by_mobility_disabled) { - move_freepages(zone, start_pfn, end_pfn, block_type, start_type); + __move_freepages_block(zone, start_pfn, block_type, start_type); return __rmqueue_smallest(zone, order, start_type); } From 883dd161e9a83e188487debc562b1928917a4b39 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 27 Mar 2024 15:01:11 -0400 Subject: [PATCH 0616/1702] mm: page_alloc: batch vmstat updates in expand() expand() currently updates vmstat for every subpage. This is unnecessary, since they're all of the same zone and migratetype. Count added pages locally, then do a single vmstat update. Link: https://lkml.kernel.org/r/20240327190111.GC7597@cmpxchg.org Signed-off-by: Johannes Weiner Suggested-by: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/page_alloc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 489dd74d2232a..382d1c98f8e52 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1335,6 +1335,7 @@ static inline void expand(struct zone *zone, struct page *page, int low, int high, int migratetype) { unsigned long size = 1 << high; + unsigned long nr_added = 0; while (high > low) { high--; @@ -1350,9 +1351,11 @@ static inline void expand(struct zone *zone, struct page *page, if (set_page_guard(zone, &page[size], high)) continue; - add_to_free_list(&page[size], zone, high, migratetype, false); + __add_to_free_list(&page[size], zone, high, migratetype, false); set_buddy_order(&page[size], high); + nr_added += size; } + account_freepages(zone, nr_added, migratetype); } static void check_new_page_bad(struct page *page) From cc9bc36ebef70f3437caf712574478d5b95b25d4 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Fri, 22 Mar 2024 00:10:01 +0000 Subject: [PATCH 0617/1702] mm: zswap: remove nr_zswap_stored atomic nr_stored was introduced by commit b5ba474f3f51 ("zswap: shrink zswap pool based on memory pressure") as a per zswap_pool counter of the number of stored pages that are not same-filled pages. It is used in zswap_shrinker_count() to scale the number of freeable compressed pages by the compression ratio. That is, to reduce the amount of writeback from zswap with higher compression ratios as the ROI from IO diminishes. Later on, commit bf9b7df23cb3 ("mm/zswap: global lru and shrinker shared by all zswap_pools") made the shrinker global (not per zswap_pool), and replaced nr_stored with nr_zswap_stored (initially introduced as zswap.nr_stored), which is now a global counter. The counter is now awfully close to zswap_stored_pages. The only difference is that the latter also includes same-filled pages. Also, when memcgs are enabled, we use memcg_page_state(memcg, MEMCG_ZSWAPPED), which includes same-filled pages anyway (i.e. equivalent to zswap_stored_pages). Use zswap_stored_pages instead in zswap_shrinker_count() to keep things consistent whether memcgs are enabled or not, and add a comment about the number of freeable pages possibly being scaled down more than it should if we have lots of same-filled pages (i.e. inflated compression ratio). Remove nr_zswap_stored and one atomic operation in the store and free paths. Link: https://lkml.kernel.org/r/20240322001001.1562517-1-yosryahmed@google.com Signed-off-by: Yosry Ahmed Reviewed-by: Nhat Pham Acked-by: Johannes Weiner Reviewed-by: Chengming Zhou Signed-off-by: Andrew Morton --- mm/zswap.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index e435dc7c043b7..01fe081474fff 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -181,8 +181,6 @@ struct zswap_pool { /* Global LRU lists shared by all zswap pools. */ static struct list_lru zswap_list_lru; -/* counter of pages stored in all zswap pools. */ -static atomic_t zswap_nr_stored = ATOMIC_INIT(0); /* The lock protects zswap_next_shrink updates. */ static DEFINE_SPINLOCK(zswap_shrink_lock); @@ -885,7 +883,6 @@ static void zswap_entry_free(struct zswap_entry *entry) else { zswap_lru_del(&zswap_list_lru, entry); zpool_free(zswap_find_zpool(entry), entry->handle); - atomic_dec(&zswap_nr_stored); zswap_pool_put(entry->pool); } if (entry->objcg) { @@ -1325,7 +1322,7 @@ static unsigned long zswap_shrinker_count(struct shrinker *shrinker, nr_stored = memcg_page_state(memcg, MEMCG_ZSWAPPED); } else { nr_backing = zswap_total_pages(); - nr_stored = atomic_read(&zswap_nr_stored); + nr_stored = atomic_read(&zswap_stored_pages); } if (!nr_stored) @@ -1345,6 +1342,11 @@ static unsigned long zswap_shrinker_count(struct shrinker *shrinker, * This ensures that the better zswap compresses memory, the fewer * pages we will evict to swap (as it will otherwise incur IO for * relatively small memory saving). + * + * The memory saving factor calculated here takes same-filled pages into + * account, but those are not freeable since they almost occupy no + * space. Hence, we may scale nr_freeable down a little bit more than we + * should if we have a lot of same-filled pages. */ return mult_frac(nr_freeable, nr_backing, nr_stored); } @@ -1590,7 +1592,6 @@ bool zswap_store(struct folio *folio) if (entry->length) { INIT_LIST_HEAD(&entry->lru); zswap_lru_add(&zswap_list_lru, entry); - atomic_inc(&zswap_nr_stored); } spin_unlock(&tree->lock); From b04da042208dc0de0de084a2a478d38ee0e51cce Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Thu, 7 Mar 2024 14:05:47 -0500 Subject: [PATCH 0618/1702] mm/kmemleak: compact kmemleak_object further Patch series "mm/kmemleak: Minor cleanup & performance tuning". This series contains 2 simple cleanup patches to slightly reduce memory and performance overhead. This patch (of 2): With commit 56a61617dd22 ("mm: use stack_depot for recording kmemleak's backtrace"), the size of kmemleak_object has been reduced by 128 bytes for 64-bit arches. The replacement "depot_stack_handle_t trace_handle" is actually just 4 bytes long leaving a hole of 4 bytes. By moving up trace_handle to another existing 4-byte hold, we can save 8 more bytes from kmemleak_object reducing its overall size from 248 to 240 bytes. Link: https://lkml.kernel.org/r/20240307190548.963626-1-longman@redhat.com Link: https://lkml.kernel.org/r/20240307190548.963626-2-longman@redhat.com Signed-off-by: Waiman Long Acked-by: Catalin Marinas Signed-off-by: Andrew Morton --- mm/kmemleak.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/kmemleak.c b/mm/kmemleak.c index d2037073ed919..fdcf01f622028 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -158,9 +158,9 @@ struct kmemleak_object { int count; /* checksum for detecting modified objects */ u32 checksum; + depot_stack_handle_t trace_handle; /* memory ranges to be scanned inside an object (empty for all) */ struct hlist_head area_list; - depot_stack_handle_t trace_handle; unsigned long jiffies; /* creation timestamp */ pid_t pid; /* pid of the current task */ char comm[TASK_COMM_LEN]; /* executable name */ From c8d36bc2df89dbb7f27cc560a0c5252a6dd16c10 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Thu, 7 Mar 2024 14:05:48 -0500 Subject: [PATCH 0619/1702] mm/kmemleak: disable KASAN instrumentation in kmemleak Kmemleak ia a memory leak checker. KASAN is also a memory checker but it focuses more on finding out-of-bounds and use-after-free bugs. Since kmemleak is inherently slow especially on systems with large number of CPUs, adding KASAN instrumentation will make it slower even more. As kmemleak is not for production use, the utility of enabling KASAN there is questionable. This patch disables KASAN instrumentation for configurations that enable both of them to slightly reduce performance overhead. Link: https://lkml.kernel.org/r/20240307190548.963626-3-longman@redhat.com Signed-off-by: Waiman Long Acked-by: Catalin Marinas Signed-off-by: Andrew Morton --- mm/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/Makefile b/mm/Makefile index 4abb40b911ec4..e164eed35d467 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -5,6 +5,7 @@ KASAN_SANITIZE_slab_common.o := n KASAN_SANITIZE_slub.o := n +KASAN_SANITIZE_kmemleak.o := n KCSAN_SANITIZE_kmemleak.o := n # These produce frequent data race reports: most of them are due to races on From aaab830ad887629156ef17097c2ad24ce6fb8177 Mon Sep 17 00:00:00 2001 From: rulinhuang Date: Wed, 6 Mar 2024 21:14:40 -0500 Subject: [PATCH 0620/1702] mm/vmalloc: eliminated the lock contention from twice to once When allocating a new memory area where the mapping address range is known, it is observed that the vmap_node->busy.lock is acquired twice. The first acquisition occurs in the alloc_vmap_area() function when inserting the vm area into the vm mapping red-black tree. The second acquisition occurs in the setup_vmalloc_vm() function when updating the properties of the vm, such as flags and address, etc. Combine these two operations together in alloc_vmap_area(), which improves scalability when the vmap_node->busy.lock is contended. By doing so, the need to acquire the lock twice can also be eliminated to once. With the above change, tested on intel sapphire rapids platform(224 vcpu), a 4% performance improvement is gained on stress-ng/pthread(https://github.com/ColinIanKing/stress-ng), which is the stress test of thread creations. Link: https://lkml.kernel.org/r/20240307021440.64967-1-rulin.huang@intel.com Co-developed-by: "Chen, Tim C" Signed-off-by: "Chen, Tim C" Co-developed-by: "King, Colin" Signed-off-by: "King, Colin" Signed-off-by: rulinhuang Reviewed-by: Baoquan He Reviewed-by: Uladzislau Rezki (Sony) Cc: Tim Chen Cc: Christoph Hellwig Cc: Lorenzo Stoakes Cc: Wangyang Guo Signed-off-by: Andrew Morton --- mm/vmalloc.c | 50 ++++++++++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 61af431bb6b82..e30acfb2cd8ba 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -1926,15 +1926,26 @@ node_alloc(unsigned long size, unsigned long align, return va; } +static inline void setup_vmalloc_vm(struct vm_struct *vm, + struct vmap_area *va, unsigned long flags, const void *caller) +{ + vm->flags = flags; + vm->addr = (void *)va->va_start; + vm->size = va->va_end - va->va_start; + vm->caller = caller; + va->vm = vm; +} + /* * Allocate a region of KVA of the specified size and alignment, within the - * vstart and vend. + * vstart and vend. If vm is passed in, the two will also be bound. */ static struct vmap_area *alloc_vmap_area(unsigned long size, unsigned long align, unsigned long vstart, unsigned long vend, int node, gfp_t gfp_mask, - unsigned long va_flags) + unsigned long va_flags, struct vm_struct *vm, + unsigned long flags, const void *caller) { struct vmap_node *vn; struct vmap_area *va; @@ -1997,6 +2008,9 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, va->vm = NULL; va->flags = (va_flags | vn_id); + if (vm) + setup_vmalloc_vm(vm, va, flags, caller); + vn = addr_to_node(va->va_start); spin_lock(&vn->busy.lock); @@ -2574,7 +2588,8 @@ static void *new_vmap_block(unsigned int order, gfp_t gfp_mask) va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE, VMALLOC_START, VMALLOC_END, node, gfp_mask, - VMAP_RAM|VMAP_BLOCK); + VMAP_RAM|VMAP_BLOCK, NULL, + 0, NULL); if (IS_ERR(va)) { kfree(vb); return ERR_CAST(va); @@ -2931,7 +2946,8 @@ void *vm_map_ram(struct page **pages, unsigned int count, int node) struct vmap_area *va; va = alloc_vmap_area(size, PAGE_SIZE, VMALLOC_START, VMALLOC_END, - node, GFP_KERNEL, VMAP_RAM); + node, GFP_KERNEL, VMAP_RAM, + NULL, 0, NULL); if (IS_ERR(va)) return NULL; @@ -3034,26 +3050,6 @@ void __init vm_area_register_early(struct vm_struct *vm, size_t align) kasan_populate_early_vm_area_shadow(vm->addr, vm->size); } -static inline void setup_vmalloc_vm_locked(struct vm_struct *vm, - struct vmap_area *va, unsigned long flags, const void *caller) -{ - vm->flags = flags; - vm->addr = (void *)va->va_start; - vm->size = va->va_end - va->va_start; - vm->caller = caller; - va->vm = vm; -} - -static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va, - unsigned long flags, const void *caller) -{ - struct vmap_node *vn = addr_to_node(va->va_start); - - spin_lock(&vn->busy.lock); - setup_vmalloc_vm_locked(vm, va, flags, caller); - spin_unlock(&vn->busy.lock); -} - static void clear_vm_uninitialized_flag(struct vm_struct *vm) { /* @@ -3090,14 +3086,12 @@ static struct vm_struct *__get_vm_area_node(unsigned long size, if (!(flags & VM_NO_GUARD)) size += PAGE_SIZE; - va = alloc_vmap_area(size, align, start, end, node, gfp_mask, 0); + va = alloc_vmap_area(size, align, start, end, node, gfp_mask, 0, area, flags, caller); if (IS_ERR(va)) { kfree(area); return NULL; } - setup_vmalloc_vm(area, va, flags, caller); - /* * Mark pages for non-VM_ALLOC mappings as accessible. Do it now as a * best-effort approach, as they can be mapped outside of vmalloc code. @@ -4672,7 +4666,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, spin_lock(&vn->busy.lock); insert_vmap_area(vas[area], &vn->busy.root, &vn->busy.head); - setup_vmalloc_vm_locked(vms[area], vas[area], VM_ALLOC, + setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC, pcpu_get_vm_areas); spin_unlock(&vn->busy.lock); } From e42dfe4e0a51b476dcc6f1461c51fdb1b76573aa Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Wed, 6 Mar 2024 18:13:26 +0800 Subject: [PATCH 0621/1702] mm: record the migration reason for struct migration_target_control Patch series "make the hugetlb migration strategy consistent", v2. As discussed in previous thread [1], there is an inconsistency when handling hugetlb migration. When handling the migration of freed hugetlb, it prevents fallback to other NUMA nodes in alloc_and_dissolve_hugetlb_folio(). However, when dealing with in-use hugetlb, it allows fallback to other NUMA nodes in alloc_hugetlb_folio_nodemask(), which can break the per-node hugetlb pool and might result in unexpected failures when node bound workloads doesn't get what is asssumed available. This patchset tries to make the hugetlb migration strategy more clear and consistent. Please find details in each patch. [1] https://lore.kernel.org/all/6f26ce22d2fcd523418a085f2c588fe0776d46e7.1706794035.git.baolin.wang@linux.alibaba.com/ This patch (of 2): To support different hugetlb allocation strategies during hugetlb migration based on various migration reasons, record the migration reason in the migration_target_control structure as a preparation. Link: https://lkml.kernel.org/r/cover.1709719720.git.baolin.wang@linux.alibaba.com Link: https://lkml.kernel.org/r/7b95d4981e07211f57139fc5b1f7ce91b920cee4.1709719720.git.baolin.wang@linux.alibaba.com Signed-off-by: Baolin Wang Reviewed-by: Oscar Salvador Cc: David Hildenbrand Cc: Miaohe Lin Cc: Michal Hocko Cc: Muchun Song Cc: Naoya Horiguchi Signed-off-by: Andrew Morton --- mm/gup.c | 1 + mm/internal.h | 1 + mm/memory-failure.c | 1 + mm/memory_hotplug.c | 1 + mm/mempolicy.c | 1 + mm/migrate.c | 1 + mm/page_alloc.c | 1 + mm/vmscan.c | 3 ++- 8 files changed, 9 insertions(+), 1 deletion(-) diff --git a/mm/gup.c b/mm/gup.c index 8433d3dc31fce..6d8d15f8c7f9a 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2145,6 +2145,7 @@ static int migrate_longterm_unpinnable_pages( struct migration_target_control mtc = { .nid = NUMA_NO_NODE, .gfp_mask = GFP_USER | __GFP_NOWARN, + .reason = MR_LONGTERM_PIN, }; if (migrate_pages(movable_page_list, alloc_migration_target, diff --git a/mm/internal.h b/mm/internal.h index fb219e31f0f0b..63bdac6d04130 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1045,6 +1045,7 @@ struct migration_target_control { int nid; /* preferred node id */ nodemask_t *nmask; gfp_t gfp_mask; + enum migrate_reason reason; }; /* diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 0a7a8a4ba421d..9e50586f2e37e 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -2669,6 +2669,7 @@ static int soft_offline_in_use_page(struct page *page) struct migration_target_control mtc = { .nid = NUMA_NO_NODE, .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, + .reason = MR_MEMORY_FAILURE, }; if (!huge && folio_test_large(folio)) { diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index a444e2d7dd2bf..b79ba36e09e03 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1841,6 +1841,7 @@ static void do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) struct migration_target_control mtc = { .nmask = &nmask, .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, + .reason = MR_MEMORY_HOTPLUG, }; int ret; diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 0b3def99174a1..e128e6b7bbcbe 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -1070,6 +1070,7 @@ static long migrate_to_node(struct mm_struct *mm, int source, int dest, struct migration_target_control mtc = { .nid = dest, .gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE, + .reason = MR_SYSCALL, }; nodes_clear(nmask); diff --git a/mm/migrate.c b/mm/migrate.c index 73a052a382f13..bde63010a3cf9 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2060,6 +2060,7 @@ static int do_move_pages_to_node(struct list_head *pagelist, int node) struct migration_target_control mtc = { .nid = node, .gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE, + .reason = MR_SYSCALL, }; err = migrate_pages(pagelist, alloc_migration_target, NULL, diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 382d1c98f8e52..daab8cab91ccf 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6351,6 +6351,7 @@ int __alloc_contig_migrate_range(struct compact_control *cc, struct migration_target_control mtc = { .nid = zone_to_nid(cc->zone), .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, + .reason = MR_CONTIG_RANGE, }; struct page *page; unsigned long total_mapped = 0; diff --git a/mm/vmscan.c b/mm/vmscan.c index 3ef654addd44c..289121e76753f 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -967,7 +967,8 @@ static unsigned int demote_folio_list(struct list_head *demote_folios, .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN | __GFP_NOMEMALLOC | GFP_NOWAIT, .nid = target_nid, - .nmask = &allowed_mask + .nmask = &allowed_mask, + .reason = MR_DEMOTION, }; if (list_empty(demote_folios)) From 42d0c3fbb5811fbfb663d8ede1d7ffba02e7ae18 Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Wed, 6 Mar 2024 18:13:27 +0800 Subject: [PATCH 0622/1702] mm: hugetlb: make the hugetlb migration strategy consistent As discussed in previous thread [1], there is an inconsistency when handing hugetlb migration. When handling the migration of freed hugetlb, it prevents fallback to other NUMA nodes in alloc_and_dissolve_hugetlb_folio(). However, when dealing with in-use hugetlb, it allows fallback to other NUMA nodes in alloc_hugetlb_folio_nodemask(), which can break the per-node hugetlb pool and might result in unexpected failures when node bound workloads doesn't get what is asssumed available. To make hugetlb migration strategy more clear, we should list all the scenarios of hugetlb migration and analyze whether allocation fallback is permitted: 1) Memory offline: will call dissolve_free_huge_pages() to free the freed hugetlb, and call do_migrate_range() to migrate the in-use hugetlb. Both can break the per-node hugetlb pool, but as this is an explicit offlining operation, no better choice. So should allow the hugetlb allocation fallback. 2) Memory failure: same as memory offline. Should allow fallback to a different node might be the only option to handle it, otherwise the impact of poisoned memory can be amplified. 3) Longterm pinning: will call migrate_longterm_unpinnable_pages() to migrate in-use and not-longterm-pinnable hugetlb, which can break the per-node pool. But we should fail to longterm pinning if can not allocate on current node to avoid breaking the per-node pool. 4) Syscalls (mbind, migrate_pages, move_pages): these are explicit users operation to move pages to other nodes, so fallback to other nodes should not be prohibited. 5) alloc_contig_range: used by CMA allocation and virtio-mem fake-offline to allocate given range of pages. Now the freed hugetlb migration is not allowed to fallback, to keep consistency, the in-use hugetlb migration should be also not allowed to fallback. 6) alloc_contig_pages: used by kfence, pgtable_debug etc. The strategy should be consistent with that of alloc_contig_range(). Based on the analysis of the various scenarios above, introducing a new helper to determine whether fallback is permitted according to the migration reason.. [1] https://lore.kernel.org/all/6f26ce22d2fcd523418a085f2c588fe0776d46e7.1706794035.git.baolin.wang@linux.alibaba.com/ Link: https://lkml.kernel.org/r/3519fcd41522817307a05b40fb551e2e17e68101.1709719720.git.baolin.wang@linux.alibaba.com Signed-off-by: Baolin Wang Cc: David Hildenbrand Cc: Miaohe Lin Cc: Michal Hocko Cc: Muchun Song Cc: Naoya Horiguchi Cc: Oscar Salvador Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 35 +++++++++++++++++++++++++++++++++-- mm/hugetlb.c | 14 ++++++++++++-- mm/mempolicy.c | 3 ++- mm/migrate.c | 3 ++- 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 300de33c6fde0..d748628efc5ec 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -723,7 +723,8 @@ int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list); struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, unsigned long addr, int avoid_reserve); struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, - nodemask_t *nmask, gfp_t gfp_mask); + nodemask_t *nmask, gfp_t gfp_mask, + bool allow_alloc_fallback); int hugetlb_add_to_page_cache(struct folio *folio, struct address_space *mapping, pgoff_t idx); void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma, @@ -946,6 +947,30 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask) return modified_mask; } +static inline bool htlb_allow_alloc_fallback(int reason) +{ + bool allowed_fallback = false; + + /* + * Note: the memory offline, memory failure and migration syscalls will + * be allowed to fallback to other nodes due to lack of a better chioce, + * that might break the per-node hugetlb pool. While other cases will + * set the __GFP_THISNODE to avoid breaking the per-node hugetlb pool. + */ + switch (reason) { + case MR_MEMORY_HOTPLUG: + case MR_MEMORY_FAILURE: + case MR_SYSCALL: + case MR_MEMPOLICY_MBIND: + allowed_fallback = true; + break; + default: + break; + } + + return allowed_fallback; +} + static inline spinlock_t *huge_pte_lockptr(struct hstate *h, struct mm_struct *mm, pte_t *pte) { @@ -1041,7 +1066,8 @@ static inline struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, static inline struct folio * alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, - nodemask_t *nmask, gfp_t gfp_mask) + nodemask_t *nmask, gfp_t gfp_mask, + bool allow_alloc_fallback) { return NULL; } @@ -1157,6 +1183,11 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask) return 0; } +static inline bool htlb_allow_alloc_fallback(int reason) +{ + return false; +} + static inline spinlock_t *huge_pte_lockptr(struct hstate *h, struct mm_struct *mm, pte_t *pte) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index e4dd0c8024892..2e984554ec633 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2598,7 +2598,7 @@ struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h, /* folio migration callback function */ struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, - nodemask_t *nmask, gfp_t gfp_mask) + nodemask_t *nmask, gfp_t gfp_mask, bool allow_alloc_fallback) { spin_lock_irq(&hugetlb_lock); if (available_huge_pages(h)) { @@ -2613,6 +2613,10 @@ struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, } spin_unlock_irq(&hugetlb_lock); + /* We cannot fallback to other nodes, as we could break the per-node pool. */ + if (!allow_alloc_fallback) + gfp_mask |= __GFP_THISNODE; + return alloc_migrate_hugetlb_folio(h, gfp_mask, preferred_nid, nmask); } @@ -6636,7 +6640,13 @@ static struct folio *alloc_hugetlb_folio_vma(struct hstate *h, gfp_mask = htlb_alloc_mask(h); node = huge_node(vma, address, gfp_mask, &mpol, &nodemask); - folio = alloc_hugetlb_folio_nodemask(h, node, nodemask, gfp_mask); + /* + * This is used to allocate a temporary hugetlb to hold the copied + * content, which will then be copied again to the final hugetlb + * consuming a reservation. Set the alloc_fallback to false to indicate + * that breaking the per-node hugetlb pool is not allowed in this case. + */ + folio = alloc_hugetlb_folio_nodemask(h, node, nodemask, gfp_mask, false); mpol_cond_put(mpol); return folio; diff --git a/mm/mempolicy.c b/mm/mempolicy.c index e128e6b7bbcbe..72626b2cefa94 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -1228,7 +1228,8 @@ static struct folio *alloc_migration_target_by_mpol(struct folio *src, h = folio_hstate(src); gfp = htlb_alloc_mask(h); nodemask = policy_nodemask(gfp, pol, ilx, &nid); - return alloc_hugetlb_folio_nodemask(h, nid, nodemask, gfp); + return alloc_hugetlb_folio_nodemask(h, nid, nodemask, gfp, + htlb_allow_alloc_fallback(MR_MEMPOLICY_MBIND)); } if (folio_test_large(src)) diff --git a/mm/migrate.c b/mm/migrate.c index bde63010a3cf9..ab9856f5931b7 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2022,7 +2022,8 @@ struct folio *alloc_migration_target(struct folio *src, unsigned long private) gfp_mask = htlb_modify_alloc_mask(h, gfp_mask); return alloc_hugetlb_folio_nodemask(h, nid, - mtc->nmask, gfp_mask); + mtc->nmask, gfp_mask, + htlb_allow_alloc_fallback(mtc->reason)); } if (folio_test_large(src)) { From 353dc187840100fabeb946bb9573bc5ca5e04fcb Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Wed, 6 Mar 2024 18:13:28 +0800 Subject: [PATCH 0623/1702] docs: hugetlbpage.rst: add hugetlb migration description Add some description of the hugetlb migration strategy. Link: https://lkml.kernel.org/r/63fb16e7a4ebc5cb69ce655af86e29b2d8e9ba34.1709719720.git.baolin.wang@linux.alibaba.com Signed-off-by: Baolin Wang Reviewed-by: Oscar Salvador Cc: David Hildenbrand Cc: Miaohe Lin Cc: Michal Hocko Cc: Muchun Song Cc: Naoya Horiguchi Signed-off-by: Andrew Morton --- Documentation/admin-guide/mm/hugetlbpage.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Documentation/admin-guide/mm/hugetlbpage.rst b/Documentation/admin-guide/mm/hugetlbpage.rst index e4d4b4a8dc973..f34a0d798d5b5 100644 --- a/Documentation/admin-guide/mm/hugetlbpage.rst +++ b/Documentation/admin-guide/mm/hugetlbpage.rst @@ -376,6 +376,13 @@ Note that the number of overcommit and reserve pages remain global quantities, as we don't know until fault time, when the faulting task's mempolicy is applied, from which node the huge page allocation will be attempted. +The hugetlb may be migrated between the per-node hugepages pool in the following +scenarios: memory offline, memory failure, longterm pinning, syscalls(mbind, +migrate_pages and move_pages), alloc_contig_range() and alloc_contig_pages(). +Now only memory offline, memory failure and syscalls allow fallbacking to allocate +a new hugetlb on a different node if the current node is unable to allocate during +hugetlb migration, that means these 3 cases can break the per-node hugepages pool. + .. _using_huge_pages: Using Huge Pages From 02d7d31ae47030919f421ce43d71abca150365f6 Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Fri, 22 Mar 2024 17:35:51 +0530 Subject: [PATCH 0624/1702] selftests/mm: parse VMA range in one go Use sscanf() to directly parse the VMA range. No functional change is intended. Link: https://lkml.kernel.org/r/20240322120551.818764-1-dev.jain@arm.com Signed-off-by: Dev Jain Cc: Anshuman Khandual Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/mlock2-tests.c | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/tools/testing/selftests/mm/mlock2-tests.c b/tools/testing/selftests/mm/mlock2-tests.c index 26f744188ad0c..7f0d50fa361dc 100644 --- a/tools/testing/selftests/mm/mlock2-tests.c +++ b/tools/testing/selftests/mm/mlock2-tests.c @@ -20,8 +20,6 @@ static int get_vm_area(unsigned long addr, struct vm_boundaries *area) FILE *file; int ret = 1; char line[1024] = {0}; - char *end_addr; - char *stop; unsigned long start; unsigned long end; @@ -37,21 +35,10 @@ static int get_vm_area(unsigned long addr, struct vm_boundaries *area) memset(area, 0, sizeof(struct vm_boundaries)); while(fgets(line, 1024, file)) { - end_addr = strchr(line, '-'); - if (!end_addr) { + if (sscanf(line, "%lx-%lx", &start, &end) != 2) { ksft_print_msg("cannot parse /proc/self/maps\n"); goto out; } - *end_addr = '\0'; - end_addr++; - stop = strchr(end_addr, ' '); - if (!stop) { - ksft_print_msg("cannot parse /proc/self/maps\n"); - goto out; - } - - sscanf(line, "%lx", &start); - sscanf(end_addr, "%lx", &end); if (start <= addr && end > addr) { area->start = start; From f238b8c33c6738f146bbfbb09b78870ea157c2b7 Mon Sep 17 00:00:00 2001 From: Barry Song Date: Sat, 23 Mar 2024 00:41:36 +1300 Subject: [PATCH 0625/1702] arm64: mm: swap: support THP_SWAP on hardware with MTE Commit d0637c505f8a1 ("arm64: enable THP_SWAP for arm64") brings up THP_SWAP on ARM64, but it doesn't enable THP_SWP on hardware with MTE as the MTE code works with the assumption tags save/restore is always handling a folio with only one page. The limitation should be removed as more and more ARM64 SoCs have this feature. Co-existence of MTE and THP_SWAP becomes more and more important. This patch makes MTE tags saving support large folios, then we don't need to split large folios into base pages for swapping out on ARM64 SoCs with MTE any more. arch_prepare_to_swap() should take folio rather than page as parameter because we support THP swap-out as a whole. It saves tags for all pages in a large folio. As now we are restoring tags based-on folio, in arch_swap_restore(), we may increase some extra loops and early-exitings while refaulting a large folio which is still in swapcache in do_swap_page(). In case a large folio has nr pages, do_swap_page() will only set the PTE of the particular page which is causing the page fault. Thus do_swap_page() runs nr times, and each time, arch_swap_restore() will loop nr times for those subpages in the folio. So right now the algorithmic complexity becomes O(nr^2). Once we support mapping large folios in do_swap_page(), extra loops and early-exitings will decrease while not being completely removed as a large folio might get partially tagged in corner cases such as, 1. a large folio in swapcache can be partially unmapped, thus, MTE tags for the unmapped pages will be invalidated; 2. users might use mprotect() to set MTEs on a part of a large folio. arch_thp_swp_supported() is dropped since ARM64 MTE was the only one who needed it. Link: https://lkml.kernel.org/r/20240322114136.61386-2-21cnbao@gmail.com Signed-off-by: Barry Song Reviewed-by: Steven Price Acked-by: Chris Li Reviewed-by: Ryan Roberts Cc: Catalin Marinas Cc: Will Deacon Cc: Mark Rutland Cc: David Hildenbrand Cc: Kemeng Shi Cc: "Matthew Wilcox (Oracle)" Cc: Anshuman Khandual Cc: Peter Collingbourne Cc: Yosry Ahmed Cc: Peter Xu Cc: Lorenzo Stoakes Cc: "Mike Rapoport (IBM)" Cc: Hugh Dickins Cc: "Aneesh Kumar K.V" Cc: Rick Edgecombe Signed-off-by: Andrew Morton --- arch/arm64/include/asm/pgtable.h | 19 ++------------ arch/arm64/mm/mteswap.c | 45 ++++++++++++++++++++++++++++++++ include/linux/huge_mm.h | 12 --------- include/linux/pgtable.h | 2 +- mm/internal.h | 14 ++++++++++ mm/memory.c | 2 +- mm/page_io.c | 2 +- mm/shmem.c | 2 +- mm/swap_slots.c | 2 +- mm/swapfile.c | 2 +- 10 files changed, 67 insertions(+), 35 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 6870b60158fc8..9fd8613b2db2a 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -49,12 +49,6 @@ __flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1) #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -static inline bool arch_thp_swp_supported(void) -{ - return !system_supports_mte(); -} -#define arch_thp_swp_supported arch_thp_swp_supported - /* * Outside of a few very special situations (e.g. hibernation), we always * use broadcast TLB invalidation instructions, therefore a spurious page @@ -1282,12 +1276,7 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, #ifdef CONFIG_ARM64_MTE #define __HAVE_ARCH_PREPARE_TO_SWAP -static inline int arch_prepare_to_swap(struct page *page) -{ - if (system_supports_mte()) - return mte_save_tags(page); - return 0; -} +extern int arch_prepare_to_swap(struct folio *folio); #define __HAVE_ARCH_SWAP_INVALIDATE static inline void arch_swap_invalidate_page(int type, pgoff_t offset) @@ -1303,11 +1292,7 @@ static inline void arch_swap_invalidate_area(int type) } #define __HAVE_ARCH_SWAP_RESTORE -static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio) -{ - if (system_supports_mte()) - mte_restore_tags(entry, &folio->page); -} +extern void arch_swap_restore(swp_entry_t entry, struct folio *folio); #endif /* CONFIG_ARM64_MTE */ diff --git a/arch/arm64/mm/mteswap.c b/arch/arm64/mm/mteswap.c index a31833e3ddc54..63e8d72f202a3 100644 --- a/arch/arm64/mm/mteswap.c +++ b/arch/arm64/mm/mteswap.c @@ -68,6 +68,13 @@ void mte_invalidate_tags(int type, pgoff_t offset) mte_free_tag_storage(tags); } +static inline void __mte_invalidate_tags(struct page *page) +{ + swp_entry_t entry = page_swap_entry(page); + + mte_invalidate_tags(swp_type(entry), swp_offset(entry)); +} + void mte_invalidate_tags_area(int type) { swp_entry_t entry = swp_entry(type, 0); @@ -83,3 +90,41 @@ void mte_invalidate_tags_area(int type) } xa_unlock(&mte_pages); } + +int arch_prepare_to_swap(struct folio *folio) +{ + long i, nr; + int err; + + if (!system_supports_mte()) + return 0; + + nr = folio_nr_pages(folio); + + for (i = 0; i < nr; i++) { + err = mte_save_tags(folio_page(folio, i)); + if (err) + goto out; + } + return 0; + +out: + while (i--) + __mte_invalidate_tags(folio_page(folio, i)); + return err; +} + +void arch_swap_restore(swp_entry_t entry, struct folio *folio) +{ + long i, nr; + + if (!system_supports_mte()) + return; + + nr = folio_nr_pages(folio); + + for (i = 0; i < nr; i++) { + mte_restore_tags(entry, folio_page(folio, i)); + entry.val++; + } +} diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 0e16451adaba3..7576025db55df 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -532,16 +532,4 @@ static inline int split_folio_to_order(struct folio *folio, int new_order) #define split_folio_to_list(f, l) split_folio_to_list_to_order(f, l, 0) #define split_folio(f) split_folio_to_order(f, 0) -/* - * archs that select ARCH_WANTS_THP_SWAP but don't support THP_SWP due to - * limitations in the implementation like arm64 MTE can override this to - * false - */ -#ifndef arch_thp_swp_supported -static inline bool arch_thp_swp_supported(void) -{ - return true; -} -#endif - #endif /* _LINUX_HUGE_MM_H */ diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 2a1c044ae4672..600e17d036599 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1050,7 +1050,7 @@ static inline int arch_unmap_one(struct mm_struct *mm, * prototypes must be defined in the arch-specific asm/pgtable.h file. */ #ifndef __HAVE_ARCH_PREPARE_TO_SWAP -static inline int arch_prepare_to_swap(struct page *page) +static inline int arch_prepare_to_swap(struct folio *folio) { return 0; } diff --git a/mm/internal.h b/mm/internal.h index 63bdac6d04130..6c8d3844b6a3b 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -76,6 +76,20 @@ static inline int folio_nr_pages_mapped(struct folio *folio) return atomic_read(&folio->_nr_pages_mapped) & FOLIO_PAGES_MAPPED; } +/* + * Retrieve the first entry of a folio based on a provided entry within the + * folio. We cannot rely on folio->swap as there is no guarantee that it has + * been initialized. Used for calling arch_swap_restore() + */ +static inline swp_entry_t folio_swap(swp_entry_t entry, struct folio *folio) +{ + swp_entry_t swap = { + .val = ALIGN_DOWN(entry.val, folio_nr_pages(folio)), + }; + + return swap; +} + static inline void *folio_raw_mapping(struct folio *folio) { unsigned long mapping = (unsigned long)folio->mapping; diff --git a/mm/memory.c b/mm/memory.c index c859a09b4f722..805cebb6fd72e 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4190,7 +4190,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) * when reading from swap. This metadata may be indexed by swap entry * so this must be called before swap_free(). */ - arch_swap_restore(entry, folio); + arch_swap_restore(folio_swap(entry, folio), folio); /* * Remove the swap entry and conditionally try to free up the swapcache. diff --git a/mm/page_io.c b/mm/page_io.c index ae2b49055e435..a9a7c236aeccf 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -189,7 +189,7 @@ int swap_writepage(struct page *page, struct writeback_control *wbc) * Arch code may have to preserve more data than just the page * contents, e.g. memory tags. */ - ret = arch_prepare_to_swap(&folio->page); + ret = arch_prepare_to_swap(folio); if (ret) { folio_mark_dirty(folio); folio_unlock(folio); diff --git a/mm/shmem.c b/mm/shmem.c index 94ab99b6b574a..98985179f495d 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1907,7 +1907,7 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index, * Some architectures may have to restore extra metadata to the * folio after reading from swap. */ - arch_swap_restore(swap, folio); + arch_swap_restore(folio_swap(swap, folio), folio); if (shmem_should_replace_folio(folio, gfp)) { error = shmem_replace_folio(&folio, gfp, info, index); diff --git a/mm/swap_slots.c b/mm/swap_slots.c index 90973ce7881db..53abeaf1371d0 100644 --- a/mm/swap_slots.c +++ b/mm/swap_slots.c @@ -310,7 +310,7 @@ swp_entry_t folio_alloc_swap(struct folio *folio) entry.val = 0; if (folio_test_large(folio)) { - if (IS_ENABLED(CONFIG_THP_SWAP) && arch_thp_swp_supported()) + if (IS_ENABLED(CONFIG_THP_SWAP)) get_swap_pages(1, &entry, folio_nr_pages(folio)); goto out; } diff --git a/mm/swapfile.c b/mm/swapfile.c index 4919423cce76a..5e6d2304a2a47 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1806,7 +1806,7 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, * when reading from swap. This metadata may be indexed by swap entry * so this must be called before swap_free(). */ - arch_swap_restore(entry, folio); + arch_swap_restore(folio_swap(entry, folio), folio); dec_mm_counter(vma->vm_mm, MM_SWAPENTS); inc_mm_counter(vma->vm_mm, MM_ANONPAGES); From 0fd44ab213bcfb26c47eedaa0985e4b5dbf0a494 Mon Sep 17 00:00:00 2001 From: Liu Shixin Date: Fri, 22 Mar 2024 17:35:54 +0800 Subject: [PATCH 0626/1702] mm/readahead: break read-ahead loop if filemap_add_folio return -ENOMEM Patch series "Fix I/O high when memory almost met memcg limit", v2. Recently, when install package in a docker which almost reached its memory limit, the installer has no respond severely for more than 15 minutes. During this period, I/O stays high(~1G/s) and influence the whole machine. I've constructed a use case as follows: 1. create a docker: $ cat test.sh #!/bin/bash docker rm centos7 --force docker create --name centos7 --memory 4G --memory-swap 6G centos:7 /usr/sbin/init docker start centos7 sleep 1 docker cp ./alloc_page centos7:/ docker cp ./reproduce.sh centos7:/ docker exec -it centos7 /bin/bash 2. try reproduce the problem in docker: $ cat reproduce.sh #!/bin/bash while true; do flag=$(ps -ef | grep -v grep | grep alloc_page| wc -l) if [ "$flag" -eq 0 ]; then /alloc_page & fi sleep 30 start_time=$(date +%s) yum install -y expect > /dev/null 2>&1 end_time=$(date +%s) elapsed_time=$((end_time - start_time)) echo "$elapsed_time seconds" yum remove -y expect > /dev/null 2>&1 done $ cat alloc_page.c: #include #include #include #include #define SIZE 1*1024*1024 //1M int main() { void *addr = NULL; int i; for (i = 0; i < 1024 * 6 - 50;i++) { addr = (void *)malloc(SIZE); if (!addr) return -1; memset(addr, 0, SIZE); } sleep(99999); return 0; } We found that this problem is caused by a lot ot meaningless read-ahead. Since the docker is almost met memory limit, the page will be reclaimed immediately after read-ahead and will read-ahead again immediately. The program is executed slowly and waste a lot of I/O resource. These two patch aim to break the read-ahead in above scenario. [1] https://lore.kernel.org/linux-mm/c2f4a2fa-3bde-72ce-66f5-db81a373fdbc@huawei.com/T/ [2] https://lore.kernel.org/all/20240201100835.1626685-1-liushixin2@huawei.com/ [3] https://lore.kernel.org/all/20240201173130.frpaqpy7iyzias5j@quack3/ This patch (of 2): When filemap_add_folio() return -ENOMEM, break read-ahead loop like what filemap_alloc_folio() does. Link: https://lkml.kernel.org/r/20240322093555.226789-1-liushixin2@huawei.com Link: https://lkml.kernel.org/r/20240322093555.226789-2-liushixin2@huawei.com Signed-off-by: Liu Shixin Signed-off-by: Jinjiang Tu Reviewed-by: Jan Kara Cc: Al Viro Cc: Christian Brauner Cc: Liu Shixin Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/readahead.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mm/readahead.c b/mm/readahead.c index 130c0e7df99f5..63d6000103f0e 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -228,6 +228,7 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, */ for (i = 0; i < nr_to_read; i++) { struct folio *folio = xa_load(&mapping->i_pages, index + i); + int ret; if (folio && !xa_is_value(folio)) { /* @@ -247,9 +248,12 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, folio = filemap_alloc_folio(gfp_mask, 0); if (!folio) break; - if (filemap_add_folio(mapping, folio, index + i, - gfp_mask) < 0) { + + ret = filemap_add_folio(mapping, folio, index + i, gfp_mask); + if (ret < 0) { folio_put(folio); + if (ret == -ENOMEM) + break; read_pages(ractl); ractl->_index++; i = ractl->_index + ractl->_nr_pages - index - 1; From 5c46d5319bde73075c5f2cefd555426848eb506f Mon Sep 17 00:00:00 2001 From: Liu Shixin Date: Fri, 22 Mar 2024 17:35:55 +0800 Subject: [PATCH 0627/1702] mm/filemap: don't decrease mmap_miss when folio has workingset flag If there are too many folios that are recently evicted in a file, then they will probably continue to be evicted. In such situation, there is no positive effect to read-ahead this file since it is only a waste of IO. The mmap_miss is increased in do_sync_mmap_readahead() and decreased in both do_async_mmap_readahead() and filemap_map_pages(). In order to skip read-ahead in above scenario, the mmap_miss have to increased exceed MMAP_LOTSAMISS. This can be done by stop decreased mmap_miss when folio has workingset flag. The async path is not to care because in above scenario, it's hard to run into the async path. [liushixin2@huawei.com: add comments] Link: https://lkml.kernel.org/r/20240326065026.1910584-1-liushixin2@huawei.com Link: https://lkml.kernel.org/r/20240322093555.226789-3-liushixin2@huawei.com Signed-off-by: Liu Shixin Reviewed-by: Jan Kara Cc: Al Viro Cc: Christian Brauner Cc: Jinjiang Tu Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/filemap.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 60890758d6605..480ae65898031 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -3492,7 +3492,15 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf, if (PageHWPoison(page + count)) goto skip; - (*mmap_miss)++; + /* + * If there are too many folios that are recently evicted + * in a file, they will probably continue to be evicted. + * In such situation, read-ahead is only a waste of IO. + * Don't decrease mmap_miss in this scenario to make sure + * we can stop read-ahead. + */ + if (!folio_test_workingset(folio)) + (*mmap_miss)++; /* * NOTE: If there're PTE markers, we'll leave them to be @@ -3541,7 +3549,9 @@ static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, if (PageHWPoison(page)) return ret; - (*mmap_miss)++; + /* See comment of filemap_map_folio_range() */ + if (!folio_test_workingset(folio)) + (*mmap_miss)++; /* * NOTE: If there're PTE markers, we'll leave them to be From 4b68a773a7ceb3467efec308963140d3e217ec59 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Sat, 9 Mar 2024 12:44:54 +0800 Subject: [PATCH 0628/1702] mm/vmalloc.c: optimize to reduce arguments of alloc_vmap_area() If called by __get_vm_area_node(), by open coding the field assignments of 'struct vm_struct *vm', and move the vm->flags and vm->caller assignments into __get_vm_area_node(), the passed in arguments 'flags' and 'caller' can be removed. This alleviates overloaded arguments passed in for alloc_vmap_area(). Link: https://lkml.kernel.org/r/20240309044454.648888-1-bhe@redhat.com Signed-off-by: Baoquan He Reviewed-by: Uladzislau Rezki (Sony) Signed-off-by: Andrew Morton --- mm/vmalloc.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index e30acfb2cd8ba..6856fad1a8f8c 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -1944,8 +1944,7 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, unsigned long align, unsigned long vstart, unsigned long vend, int node, gfp_t gfp_mask, - unsigned long va_flags, struct vm_struct *vm, - unsigned long flags, const void *caller) + unsigned long va_flags, struct vm_struct *vm) { struct vmap_node *vn; struct vmap_area *va; @@ -2008,8 +2007,11 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, va->vm = NULL; va->flags = (va_flags | vn_id); - if (vm) - setup_vmalloc_vm(vm, va, flags, caller); + if (vm) { + vm->addr = (void *)va->va_start; + vm->size = va->va_end - va->va_start; + va->vm = vm; + } vn = addr_to_node(va->va_start); @@ -2588,8 +2590,7 @@ static void *new_vmap_block(unsigned int order, gfp_t gfp_mask) va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE, VMALLOC_START, VMALLOC_END, node, gfp_mask, - VMAP_RAM|VMAP_BLOCK, NULL, - 0, NULL); + VMAP_RAM|VMAP_BLOCK, NULL); if (IS_ERR(va)) { kfree(vb); return ERR_CAST(va); @@ -2947,7 +2948,7 @@ void *vm_map_ram(struct page **pages, unsigned int count, int node) va = alloc_vmap_area(size, PAGE_SIZE, VMALLOC_START, VMALLOC_END, node, GFP_KERNEL, VMAP_RAM, - NULL, 0, NULL); + NULL); if (IS_ERR(va)) return NULL; @@ -3086,7 +3087,10 @@ static struct vm_struct *__get_vm_area_node(unsigned long size, if (!(flags & VM_NO_GUARD)) size += PAGE_SIZE; - va = alloc_vmap_area(size, align, start, end, node, gfp_mask, 0, area, flags, caller); + area->flags = flags; + area->caller = caller; + + va = alloc_vmap_area(size, align, start, end, node, gfp_mask, 0, area); if (IS_ERR(va)) { kfree(area); return NULL; From 73bc32875ee9b1881dd780308c6793fe463fe803 Mon Sep 17 00:00:00 2001 From: Barry Song Date: Wed, 6 Mar 2024 22:52:19 +1300 Subject: [PATCH 0629/1702] mm: hold PTL from the first PTE while reclaiming a large folio Within try_to_unmap_one(), page_vma_mapped_walk() races with other PTE modifications preceded by pte clear. While iterating over PTEs of a large folio, it only starts acquiring PTL from the first valid (present) PTE. PTE modifications can temporarily set PTEs to pte_none. Consequently, the initial PTEs of a large folio might be skipped in try_to_unmap_one(). For example, for an anon folio, if we skip PTE0, we may have PTE0 which is still present, while PTE1 ~ PTE(nr_pages - 1) are swap entries after try_to_unmap_one(). So folio will be still mapped, the folio fails to be reclaimed and is put back to LRU in this round. This also breaks up PTEs optimization such as CONT-PTE on this large folio and may lead to accident folio_split() afterwards. And since a part of PTEs are now swap entries, accessing those parts will introduce overhead - do_swap_page. Although the kernel can withstand all of the above issues, the situation still seems quite awkward and warrants making it more ideal. The same race also occurs with small folios, but they have only one PTE, thus, it won't be possible for them to be partially unmapped. This patch holds PTL from PTE0, allowing us to avoid reading PTE values that are in the process of being transformed. With stable PTE values, we can ensure that this large folio is either completely reclaimed or that all PTEs remain untouched in this round. A corner case is that if we hold PTL from PTE0 and most initial PTEs have been really unmapped before that, we may increase the duration of holding PTL. Thus we only apply this optimization to folios which are still entirely mapped (not in deferred_split list). [akpm@linux-foundation.org: rewrap comment, per Matthew] Link: https://lkml.kernel.org/r/20240306095219.71086-1-21cnbao@gmail.com Signed-off-by: Barry Song Acked-by: David Hildenbrand Cc: Hugh Dickins Cc: Chris Li Cc: Chuanhua Han Cc: Gao Xiang Cc: Huang, Ying Cc: Hugh Dickins Cc: Kefeng Wang Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Ryan Roberts Cc: Yang Shi Cc: Yu Zhao Signed-off-by: Andrew Morton --- mm/vmscan.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mm/vmscan.c b/mm/vmscan.c index 289121e76753f..7554adc9c824e 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1257,6 +1257,20 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, if (folio_test_pmd_mappable(folio)) flags |= TTU_SPLIT_HUGE_PMD; + /* + * Without TTU_SYNC, try_to_unmap will only begin to + * hold PTL from the first present PTE within a large + * folio. Some initial PTEs might be skipped due to + * races with parallel PTE writes in which PTEs can be + * cleared temporarily before being written new present + * values. This will lead to a large folio is still + * mapped while some subpages have been partially + * unmapped after try_to_unmap; TTU_SYNC helps + * try_to_unmap acquire PTL from the first PTE, + * eliminating the influence of temporary PTE values. + */ + if (folio_test_large(folio) && list_empty(&folio->_deferred_list)) + flags |= TTU_SYNC; try_to_unmap(folio, flags); if (folio_mapped(folio)) { From 7262f208ca681385d133844be8a58d9b4ca185f7 Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Fri, 22 Mar 2024 15:33:04 -0400 Subject: [PATCH 0630/1702] mm/migrate: split source folio if it is on deferred split list If the source folio is on deferred split list, it is likely some subpages are not used. Split it before migration to avoid migrating unused subpages. Commit 616b8371539a6 ("mm: thp: enable thp migration in generic path") did not check if a THP is on deferred split list before migration, thus, the destination THP is never put on deferred split list even if the source THP might be. The opportunity of reclaiming free pages in a partially mapped THP during deferred list scanning is lost, but no other harmful consequence is present[1]. [1]: https://lore.kernel.org/linux-mm/03CE3A00-917C-48CC-8E1C-6A98713C817C@nvidia.com/ [zi.yan@sent.com: fix an error in migrate_misplaced_folio()] Link: https://lkml.kernel.org/r/20240326150031.569387-1-zi.yan@sent.com Link: https://lkml.kernel.org/r/20240322193304.522496-1-zi.yan@sent.com Fixes: 616b8371539a ("mm: thp: enable thp migration in generic path") Signed-off-by: Zi Yan Tested-by: Baolin Wang Reviewed-by: Baolin Wang Acked-by: David Hildenbrand Cc: Huang, Ying Cc: Kirill A. Shutemov Cc: Matthew Wilcox (Oracle) Cc: Ryan Roberts Cc: SeongJae Park Cc: Yang Shi Cc: Yin Fengwei Signed-off-by: Andrew Morton --- mm/migrate.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/mm/migrate.c b/mm/migrate.c index ab9856f5931b7..63c97bb639d75 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1652,6 +1652,29 @@ static int migrate_pages_batch(struct list_head *from, cond_resched(); + /* + * The rare folio on the deferred split list should + * be split now. It should not count as a failure. + * Only check it without removing it from the list. + * Since the folio can be on deferred_split_scan() + * local list and removing it can cause the local list + * corruption. Folio split process below can handle it + * with the help of folio_ref_freeze(). + * + * nr_pages > 2 is needed to avoid checking order-1 + * page cache folios. They exist, in contrast to + * non-existent order-1 anonymous folios, and do not + * use _deferred_list. + */ + if (nr_pages > 2 && + !list_empty(&folio->_deferred_list)) { + if (try_split_folio(folio, split_folios) == 0) { + stats->nr_thp_split += is_thp; + stats->nr_split++; + continue; + } + } + /* * Large folio migration might be unsupported or * the allocation might be failed so we should retry From ebb34f78d72c2320620ba6d55cb22a52949047a1 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 27 Feb 2024 21:15:48 +0100 Subject: [PATCH 0631/1702] mm: convert folio_estimated_sharers() to folio_likely_mapped_shared() Callers of folio_estimated_sharers() only care about "mapped shared vs. mapped exclusively", not the exact estimate of sharers. Let's consolidate and unify the condition users are checking. While at it clarify the semantics and extend the discussion on the fuzziness. Use the "likely mapped shared" terminology to better express what the (adjusted) function actually checks. Whether a partially-mappable folio is more likely to not be partially mapped than partially mapped is debatable. In the future, we might be able to improve our estimate for partially-mappable folios, though. Note that we will now consistently detect "mapped shared" only if the first subpage is actually mapped multiple times. When the first subpage is not mapped, we will consistently detect it as "mapped exclusively". This change should currently only affect the usage in madvise_free_pte_range() and queue_folios_pte_range() for large folios: if the first page was already unmapped, we would have skipped the folio. [david@redhat.com: folio_likely_mapped_shared() kerneldoc fixup] Link: https://lkml.kernel.org/r/dd0ad9f2-2d7a-45f3-9ba3-979488c7dd27@redhat.com Link: https://lkml.kernel.org/r/20240227201548.857831-1-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Khalid Aziz Acked-by: Barry Song Reviewed-by: Vishal Moola (Oracle) Reviewed-by: Ryan Roberts Reviewed-by: Zi Yan Signed-off-by: Andrew Morton --- include/linux/mm.h | 48 ++++++++++++++++++++++++++++++++++++---------- mm/huge_memory.c | 2 +- mm/madvise.c | 6 +++--- mm/memory.c | 2 +- mm/mempolicy.c | 14 ++++++-------- mm/migrate.c | 8 ++++---- 6 files changed, 53 insertions(+), 27 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 1588fe15a38e3..fe66b515aaab3 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2165,21 +2165,49 @@ static inline size_t folio_size(struct folio *folio) } /** - * folio_estimated_sharers - Estimate the number of sharers of a folio. + * folio_likely_mapped_shared - Estimate if the folio is mapped into the page + * tables of more than one MM * @folio: The folio. * - * folio_estimated_sharers() aims to serve as a function to efficiently - * estimate the number of processes sharing a folio. This is done by - * looking at the precise mapcount of the first subpage in the folio, and - * assuming the other subpages are the same. This may not be true for large - * folios. If you want exact mapcounts for exact calculations, look at - * page_mapcount() or folio_total_mapcount(). + * This function checks if the folio is currently mapped into more than one + * MM ("mapped shared"), or if the folio is only mapped into a single MM + * ("mapped exclusively"). * - * Return: The estimated number of processes sharing a folio. + * As precise information is not easily available for all folios, this function + * estimates the number of MMs ("sharers") that are currently mapping a folio + * using the number of times the first page of the folio is currently mapped + * into page tables. + * + * For small anonymous folios (except KSM folios) and anonymous hugetlb folios, + * the return value will be exactly correct, because they can only be mapped + * at most once into an MM, and they cannot be partially mapped. + * + * For other folios, the result can be fuzzy: + * #. For partially-mappable large folios (THP), the return value can wrongly + * indicate "mapped exclusively" (false negative) when the folio is + * only partially mapped into at least one MM. + * #. For pagecache folios (including hugetlb), the return value can wrongly + * indicate "mapped shared" (false positive) when two VMAs in the same MM + * cover the same file range. + * #. For (small) KSM folios, the return value can wrongly indicate "mapped + * shared" (false negative), when the folio is mapped multiple times into + * the same MM. + * + * Further, this function only considers current page table mappings that + * are tracked using the folio mapcount(s). + * + * This function does not consider: + * #. If the folio might get mapped in the (near) future (e.g., swapcache, + * pagecache, temporary unmapping for migration). + * #. If the folio is mapped differently (VM_PFNMAP). + * #. If hugetlb page table sharing applies. Callers might want to check + * hugetlb_pmd_shared(). + * + * Return: Whether the folio is estimated to be mapped into more than one MM. */ -static inline int folio_estimated_sharers(struct folio *folio) +static inline bool folio_likely_mapped_shared(struct folio *folio) { - return page_mapcount(folio_page(folio, 0)); + return page_mapcount(folio_page(folio, 0)) > 1; } #ifndef HAVE_ARCH_MAKE_PAGE_ACCESSIBLE diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 16b2c5622fb1b..1170fc22ed893 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1821,7 +1821,7 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, * If other processes are mapping this folio, we couldn't discard * the folio unless they all do MADV_FREE so let's skip the folio. */ - if (folio_estimated_sharers(folio) != 1) + if (folio_likely_mapped_shared(folio)) goto out; if (!folio_trylock(folio)) diff --git a/mm/madvise.c b/mm/madvise.c index a2dd70c4a2e6b..7625830d6ae91 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -366,7 +366,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, folio = pfn_folio(pmd_pfn(orig_pmd)); /* Do not interfere with other mappings of this folio */ - if (folio_estimated_sharers(folio) != 1) + if (folio_likely_mapped_shared(folio)) goto huge_unlock; if (pageout_anon_only_filter && !folio_test_anon(folio)) @@ -453,7 +453,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, if (folio_test_large(folio)) { int err; - if (folio_estimated_sharers(folio) > 1) + if (folio_likely_mapped_shared(folio)) break; if (pageout_anon_only_filter && !folio_test_anon(folio)) break; @@ -677,7 +677,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, if (folio_test_large(folio)) { int err; - if (folio_estimated_sharers(folio) != 1) + if (folio_likely_mapped_shared(folio)) break; if (!folio_trylock(folio)) break; diff --git a/mm/memory.c b/mm/memory.c index 805cebb6fd72e..bcd29104f5ea8 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5112,7 +5112,7 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) * Flag if the folio is shared between multiple address spaces. This * is later used when determining whether to group tasks together */ - if (folio_estimated_sharers(folio) > 1 && (vma->vm_flags & VM_SHARED)) + if (folio_likely_mapped_shared(folio) && (vma->vm_flags & VM_SHARED)) flags |= TNF_SHARED; nid = folio_nid(folio); diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 72626b2cefa94..913cff5da5a30 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -642,12 +642,11 @@ static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask, * Unless MPOL_MF_MOVE_ALL, we try to avoid migrating a shared folio. * Choosing not to migrate a shared folio is not counted as a failure. * - * To check if the folio is shared, ideally we want to make sure - * every page is mapped to the same process. Doing that is very - * expensive, so check the estimated sharers of the folio instead. + * See folio_likely_mapped_shared() on possible imprecision when we + * cannot easily detect if a folio is shared. */ if ((flags & MPOL_MF_MOVE_ALL) || - (folio_estimated_sharers(folio) == 1 && !hugetlb_pmd_shared(pte))) + (!folio_likely_mapped_shared(folio) && !hugetlb_pmd_shared(pte))) if (!isolate_hugetlb(folio, qp->pagelist)) qp->nr_failed++; unlock: @@ -1032,11 +1031,10 @@ static bool migrate_folio_add(struct folio *folio, struct list_head *foliolist, * Unless MPOL_MF_MOVE_ALL, we try to avoid migrating a shared folio. * Choosing not to migrate a shared folio is not counted as a failure. * - * To check if the folio is shared, ideally we want to make sure - * every page is mapped to the same process. Doing that is very - * expensive, so check the estimated sharers of the folio instead. + * See folio_likely_mapped_shared() on possible imprecision when we + * cannot easily detect if a folio is shared. */ - if ((flags & MPOL_MF_MOVE_ALL) || folio_estimated_sharers(folio) == 1) { + if ((flags & MPOL_MF_MOVE_ALL) || !folio_likely_mapped_shared(folio)) { if (folio_isolate_lru(folio)) { list_add_tail(&folio->lru, foliolist); node_stat_mod_folio(folio, diff --git a/mm/migrate.c b/mm/migrate.c index 63c97bb639d75..a31aa75d223d0 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2593,11 +2593,11 @@ int migrate_misplaced_folio(struct folio *folio, struct vm_area_struct *vma, /* * Don't migrate file folios that are mapped in multiple processes * with execute permissions as they are probably shared libraries. - * To check if the folio is shared, ideally we want to make sure - * every page is mapped to the same process. Doing that is very - * expensive, so check the estimated mapcount of the folio instead. + * + * See folio_likely_mapped_shared() on possible imprecision when we + * cannot easily detect if a folio is shared. */ - if (folio_estimated_sharers(folio) != 1 && folio_is_file_lru(folio) && + if (folio_likely_mapped_shared(folio) && folio_is_file_lru(folio) && (vma->vm_flags & VM_EXEC)) goto out; From de60fd8ddeda2b41fbe11df11733838c5f684616 Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Tue, 16 Apr 2024 01:18:53 +0800 Subject: [PATCH 0632/1702] mm/filemap: return early if failed to allocate memory for split MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "mm/filemap: optimize folio adding and splitting", v4. Currently, at least 3 tree walks are needed for filemap folio adding if the folio is previously evicted. One for getting the order of current slot, one for ranged conflict check, and one for another order retrieving. If a split is needed, more walks are needed. This series is trying to merge these walks, and speed up filemap_add_folio, I see a 7.5% - 12.5% performance gain for fio stress test. So instead of doing multiple tree walks, do one optimism range check with lock hold, and exit if raced with another insertion. If a shadow exists, check it with a new xas_get_order helper before releasing the lock to avoid redundant tree walks for getting its order. Drop the lock and do the allocation only if a split is needed. In the best case, it only need to walk the tree once. If it needs to alloc and split, 3 walks are issued (One for first ranged conflict check and order retrieving, one for the second check after allocation, one for the insert after split). Testing with 4K pages, in an 8G cgroup, with 16G brd as block device: echo 3 > /proc/sys/vm/drop_caches fio -name=cached --numjobs=16 --filename=/mnt/test.img \ --buffered=1 --ioengine=mmap --rw=randread --time_based \ --ramp_time=30s --runtime=5m --group_reporting Before: bw ( MiB/s): min= 1027, max= 3520, per=100.00%, avg=2445.02, stdev=18.90, samples=8691 iops : min=263001, max=901288, avg=625924.36, stdev=4837.28, samples=8691 After (+7.3%): bw ( MiB/s): min= 493, max= 3947, per=100.00%, avg=2625.56, stdev=25.74, samples=8651 iops : min=126454, max=1010681, avg=672142.61, stdev=6590.48, samples=8651 Test result with THP (do a THP randread then switch to 4K page in hope it issues a lot of splitting): echo 3 > /proc/sys/vm/drop_caches fio -name=cached --numjobs=16 --filename=/mnt/test.img \ --buffered=1 --ioengine=mmap -thp=1 --readonly \ --rw=randread --time_based --ramp_time=30s --runtime=10m \ --group_reporting fio -name=cached --numjobs=16 --filename=/mnt/test.img \ --buffered=1 --ioengine=mmap \ --rw=randread --time_based --runtime=5s --group_reporting Before: bw ( KiB/s): min= 4141, max=14202, per=100.00%, avg=7935.51, stdev=96.85, samples=18976 iops : min= 1029, max= 3548, avg=1979.52, stdev=24.23, samples=18976· READ: bw=4545B/s (4545B/s), 4545B/s-4545B/s (4545B/s-4545B/s), io=64.0KiB (65.5kB), run=14419-14419msec After (+10.4%): bw ( KiB/s): min= 4611, max=15370, per=100.00%, avg=8928.74, stdev=105.17, samples=19146 iops : min= 1151, max= 3842, avg=2231.27, stdev=26.29, samples=19146 READ: bw=4635B/s (4635B/s), 4635B/s-4635B/s (4635B/s-4635B/s), io=64.0KiB (65.5kB), run=14137-14137msec The performance is better for both 4K (+7.5%) and THP (+12.5%) cached read. This patch (of 4): xas_split_alloc could fail with NOMEM, and in such case, it should abort early instead of keep going and fail the xas_split below. Link: https://lkml.kernel.org/r/20240416071722.45997-1-ryncsn@gmail.com Link: https://lkml.kernel.org/r/20240415171857.19244-1-ryncsn@gmail.com Link: https://lkml.kernel.org/r/20240415171857.19244-2-ryncsn@gmail.com Signed-off-by: Kairui Song Acked-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/filemap.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mm/filemap.c b/mm/filemap.c index 480ae65898031..12089c24abfbd 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -880,9 +880,12 @@ noinline int __filemap_add_folio(struct address_space *mapping, unsigned int order = xa_get_order(xas.xa, xas.xa_index); void *entry, *old = NULL; - if (order > folio_order(folio)) + if (order > folio_order(folio)) { xas_split_alloc(&xas, xa_load(xas.xa, xas.xa_index), order, gfp); + if (xas_error(&xas)) + goto error; + } xas_lock_irq(&xas); xas_for_each_conflict(&xas, entry) { old = entry; From b2ebcf9d3d5a0108f640d8c8200ece8848045725 Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Tue, 16 Apr 2024 01:18:54 +0800 Subject: [PATCH 0633/1702] mm/filemap: clean up hugetlb exclusion code __filemap_add_folio only has two callers, one never passes hugetlb folio and one always passes in hugetlb folio. So move the hugetlb related cgroup charging out of it to make the code cleaner. Link: https://lkml.kernel.org/r/20240415171857.19244-3-ryncsn@gmail.com Signed-off-by: Kairui Song Acked-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/filemap.c | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 12089c24abfbd..17a66ea544e71 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -853,20 +853,12 @@ noinline int __filemap_add_folio(struct address_space *mapping, { XA_STATE(xas, &mapping->i_pages, index); bool huge = folio_test_hugetlb(folio); - bool charged = false; - long nr = 1; + long nr; VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); VM_BUG_ON_FOLIO(folio_test_swapbacked(folio), folio); mapping_set_update(&xas, mapping); - if (!huge) { - int error = mem_cgroup_charge(folio, NULL, gfp); - if (error) - return error; - charged = true; - } - VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio); xas_set_order(&xas, index, folio_order(folio)); nr = folio_nr_pages(folio); @@ -931,8 +923,6 @@ noinline int __filemap_add_folio(struct address_space *mapping, trace_mm_filemap_add_to_page_cache(folio); return 0; error: - if (charged) - mem_cgroup_uncharge(folio); folio->mapping = NULL; /* Leave page->index set: truncation relies upon it */ folio_put_refs(folio, nr); @@ -946,11 +936,16 @@ int filemap_add_folio(struct address_space *mapping, struct folio *folio, void *shadow = NULL; int ret; + ret = mem_cgroup_charge(folio, NULL, gfp); + if (ret) + return ret; + __folio_set_locked(folio); ret = __filemap_add_folio(mapping, folio, index, gfp, &shadow); - if (unlikely(ret)) + if (unlikely(ret)) { + mem_cgroup_uncharge(folio); __folio_clear_locked(folio); - else { + } else { /* * The folio might have been evicted from cache only * recently, in which case it should be activated like From a4864671ca0bf51c8e78242951741df52c06766f Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Tue, 16 Apr 2024 01:18:55 +0800 Subject: [PATCH 0634/1702] lib/xarray: introduce a new helper xas_get_order It can be used after xas_load to check the order of loaded entries. Compared to xa_get_order, it saves an XA_STATE and avoid a rewalk. Added new test for xas_get_order, to make the test work, we have to export xas_get_order with EXPORT_SYMBOL_GPL. Also fix a sparse warning by checking the slot value with xa_entry instead of accessing it directly, as suggested by Matthew Wilcox. [kasong@tencent.com: simplify comment, sparse warning fix, per Matthew Wilcox] Link: https://lkml.kernel.org/r/20240416071722.45997-4-ryncsn@gmail.com Link: https://lkml.kernel.org/r/20240415171857.19244-4-ryncsn@gmail.com Signed-off-by: Kairui Song Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/xarray.h | 6 ++++++ lib/test_xarray.c | 34 +++++++++++++++++++++++++++++ lib/xarray.c | 49 ++++++++++++++++++++++++++---------------- 3 files changed, 71 insertions(+), 18 deletions(-) diff --git a/include/linux/xarray.h b/include/linux/xarray.h index cb571dfcf4b16..d9d479334c9e6 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -1548,6 +1548,7 @@ void xas_create_range(struct xa_state *); #ifdef CONFIG_XARRAY_MULTI int xa_get_order(struct xarray *, unsigned long index); +int xas_get_order(struct xa_state *xas); void xas_split(struct xa_state *, void *entry, unsigned int order); void xas_split_alloc(struct xa_state *, void *entry, unsigned int order, gfp_t); #else @@ -1556,6 +1557,11 @@ static inline int xa_get_order(struct xarray *xa, unsigned long index) return 0; } +static inline int xas_get_order(struct xa_state *xas) +{ + return 0; +} + static inline void xas_split(struct xa_state *xas, void *entry, unsigned int order) { diff --git a/lib/test_xarray.c b/lib/test_xarray.c index ebe2af2e072db..0efde8f934908 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -1984,6 +1984,39 @@ static noinline void check_get_order(struct xarray *xa) } } +static noinline void check_xas_get_order(struct xarray *xa) +{ + XA_STATE(xas, xa, 0); + + unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 20 : 1; + unsigned int order; + unsigned long i, j; + + for (order = 0; order < max_order; order++) { + for (i = 0; i < 10; i++) { + xas_set_order(&xas, i << order, order); + do { + xas_lock(&xas); + xas_store(&xas, xa_mk_value(i)); + xas_unlock(&xas); + } while (xas_nomem(&xas, GFP_KERNEL)); + + for (j = i << order; j < (i + 1) << order; j++) { + xas_set_order(&xas, j, 0); + rcu_read_lock(); + xas_load(&xas); + XA_BUG_ON(xa, xas_get_order(&xas) != order); + rcu_read_unlock(); + } + + xas_lock(&xas); + xas_set_order(&xas, i << order, order); + xas_store(&xas, NULL); + xas_unlock(&xas); + } + } +} + static noinline void check_destroy(struct xarray *xa) { unsigned long index; @@ -2035,6 +2068,7 @@ static int xarray_checks(void) check_multi_store(&array); check_multi_store_advanced(&array); check_get_order(&array); + check_xas_get_order(&array); check_xa_alloc(); check_find(&array); check_find_entry(&array); diff --git a/lib/xarray.c b/lib/xarray.c index 39f07bfc4dcca..da79128ad754f 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1750,39 +1750,52 @@ void *xa_store_range(struct xarray *xa, unsigned long first, EXPORT_SYMBOL(xa_store_range); /** - * xa_get_order() - Get the order of an entry. - * @xa: XArray. - * @index: Index of the entry. + * xas_get_order() - Get the order of an entry. + * @xas: XArray operation state. + * + * Called after xas_load, the xas should not be in an error state. * * Return: A number between 0 and 63 indicating the order of the entry. */ -int xa_get_order(struct xarray *xa, unsigned long index) +int xas_get_order(struct xa_state *xas) { - XA_STATE(xas, xa, index); - void *entry; int order = 0; - rcu_read_lock(); - entry = xas_load(&xas); - - if (!entry) - goto unlock; - - if (!xas.xa_node) - goto unlock; + if (!xas->xa_node) + return 0; for (;;) { - unsigned int slot = xas.xa_offset + (1 << order); + unsigned int slot = xas->xa_offset + (1 << order); if (slot >= XA_CHUNK_SIZE) break; - if (!xa_is_sibling(xas.xa_node->slots[slot])) + if (!xa_is_sibling(xa_entry(xas->xa, xas->xa_node, slot))) break; order++; } - order += xas.xa_node->shift; -unlock: + order += xas->xa_node->shift; + return order; +} +EXPORT_SYMBOL_GPL(xas_get_order); + +/** + * xa_get_order() - Get the order of an entry. + * @xa: XArray. + * @index: Index of the entry. + * + * Return: A number between 0 and 63 indicating the order of the entry. + */ +int xa_get_order(struct xarray *xa, unsigned long index) +{ + XA_STATE(xas, xa, index); + int order = 0; + void *entry; + + rcu_read_lock(); + entry = xas_load(&xas); + if (entry) + order = xas_get_order(&xas); rcu_read_unlock(); return order; From 6758c1128ceb45d1a35298912b974eb4895b7dd9 Mon Sep 17 00:00:00 2001 From: Kairui Song Date: Tue, 16 Apr 2024 01:18:56 +0800 Subject: [PATCH 0635/1702] mm/filemap: optimize filemap folio adding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of doing multiple tree walks, do one optimism range check with lock hold, and exit if raced with another insertion. If a shadow exists, check it with a new xas_get_order helper before releasing the lock to avoid redundant tree walks for getting its order. Drop the lock and do the allocation only if a split is needed. In the best case, it only need to walk the tree once. If it needs to alloc and split, 3 walks are issued (One for first ranged conflict check and order retrieving, one for the second check after allocation, one for the insert after split). Testing with 4K pages, in an 8G cgroup, with 16G brd as block device: echo 3 > /proc/sys/vm/drop_caches fio -name=cached --numjobs=16 --filename=/mnt/test.img \ --buffered=1 --ioengine=mmap --rw=randread --time_based \ --ramp_time=30s --runtime=5m --group_reporting Before: bw ( MiB/s): min= 1027, max= 3520, per=100.00%, avg=2445.02, stdev=18.90, samples=8691 iops : min=263001, max=901288, avg=625924.36, stdev=4837.28, samples=8691 After (+7.3%): bw ( MiB/s): min= 493, max= 3947, per=100.00%, avg=2625.56, stdev=25.74, samples=8651 iops : min=126454, max=1010681, avg=672142.61, stdev=6590.48, samples=8651 Test result with THP (do a THP randread then switch to 4K page in hope it issues a lot of splitting): echo 3 > /proc/sys/vm/drop_caches fio -name=cached --numjobs=16 --filename=/mnt/test.img \ --buffered=1 --ioengine=mmap -thp=1 --readonly \ --rw=randread --time_based --ramp_time=30s --runtime=10m \ --group_reporting fio -name=cached --numjobs=16 --filename=/mnt/test.img \ --buffered=1 --ioengine=mmap \ --rw=randread --time_based --runtime=5s --group_reporting Before: bw ( KiB/s): min= 4141, max=14202, per=100.00%, avg=7935.51, stdev=96.85, samples=18976 iops : min= 1029, max= 3548, avg=1979.52, stdev=24.23, samples=18976· READ: bw=4545B/s (4545B/s), 4545B/s-4545B/s (4545B/s-4545B/s), io=64.0KiB (65.5kB), run=14419-14419msec After (+12.5%): bw ( KiB/s): min= 4611, max=15370, per=100.00%, avg=8928.74, stdev=105.17, samples=19146 iops : min= 1151, max= 3842, avg=2231.27, stdev=26.29, samples=19146 READ: bw=4635B/s (4635B/s), 4635B/s-4635B/s (4635B/s-4635B/s), io=64.0KiB (65.5kB), run=14137-14137msec The performance is better for both 4K (+7.5%) and THP (+12.5%) cached read. Link: https://lkml.kernel.org/r/20240415171857.19244-5-ryncsn@gmail.com Signed-off-by: Kairui Song Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- lib/test_xarray.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++ mm/filemap.c | 56 ++++++++++++++++++++++++++++++++------------ 2 files changed, 100 insertions(+), 15 deletions(-) diff --git a/lib/test_xarray.c b/lib/test_xarray.c index 0efde8f934908..8732a311f6139 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -2017,6 +2017,64 @@ static noinline void check_xas_get_order(struct xarray *xa) } } +static noinline void check_xas_conflict_get_order(struct xarray *xa) +{ + XA_STATE(xas, xa, 0); + + void *entry; + int only_once; + unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 20 : 1; + unsigned int order; + unsigned long i, j, k; + + for (order = 0; order < max_order; order++) { + for (i = 0; i < 10; i++) { + xas_set_order(&xas, i << order, order); + do { + xas_lock(&xas); + xas_store(&xas, xa_mk_value(i)); + xas_unlock(&xas); + } while (xas_nomem(&xas, GFP_KERNEL)); + + /* + * Ensure xas_get_order works with xas_for_each_conflict. + */ + j = i << order; + for (k = 0; k < order; k++) { + only_once = 0; + xas_set_order(&xas, j + (1 << k), k); + xas_lock(&xas); + xas_for_each_conflict(&xas, entry) { + XA_BUG_ON(xa, entry != xa_mk_value(i)); + XA_BUG_ON(xa, xas_get_order(&xas) != order); + only_once++; + } + XA_BUG_ON(xa, only_once != 1); + xas_unlock(&xas); + } + + if (order < max_order - 1) { + only_once = 0; + xas_set_order(&xas, (i & ~1UL) << order, order + 1); + xas_lock(&xas); + xas_for_each_conflict(&xas, entry) { + XA_BUG_ON(xa, entry != xa_mk_value(i)); + XA_BUG_ON(xa, xas_get_order(&xas) != order); + only_once++; + } + XA_BUG_ON(xa, only_once != 1); + xas_unlock(&xas); + } + + xas_set_order(&xas, i << order, order); + xas_lock(&xas); + xas_store(&xas, NULL); + xas_unlock(&xas); + } + } +} + + static noinline void check_destroy(struct xarray *xa) { unsigned long index; @@ -2069,6 +2127,7 @@ static int xarray_checks(void) check_multi_store_advanced(&array); check_get_order(&array); check_xas_get_order(&array); + check_xas_conflict_get_order(&array); check_xa_alloc(); check_find(&array); check_find_entry(&array); diff --git a/mm/filemap.c b/mm/filemap.c index 17a66ea544e71..7b0b2229d4ed2 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -852,7 +852,9 @@ noinline int __filemap_add_folio(struct address_space *mapping, struct folio *folio, pgoff_t index, gfp_t gfp, void **shadowp) { XA_STATE(xas, &mapping->i_pages, index); - bool huge = folio_test_hugetlb(folio); + void *alloced_shadow = NULL; + int alloced_order = 0; + bool huge; long nr; VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); @@ -861,6 +863,7 @@ noinline int __filemap_add_folio(struct address_space *mapping, VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio); xas_set_order(&xas, index, folio_order(folio)); + huge = folio_test_hugetlb(folio); nr = folio_nr_pages(folio); gfp &= GFP_RECLAIM_MASK; @@ -868,16 +871,10 @@ noinline int __filemap_add_folio(struct address_space *mapping, folio->mapping = mapping; folio->index = xas.xa_index; - do { - unsigned int order = xa_get_order(xas.xa, xas.xa_index); + for (;;) { + int order = -1, split_order = 0; void *entry, *old = NULL; - if (order > folio_order(folio)) { - xas_split_alloc(&xas, xa_load(xas.xa, xas.xa_index), - order, gfp); - if (xas_error(&xas)) - goto error; - } xas_lock_irq(&xas); xas_for_each_conflict(&xas, entry) { old = entry; @@ -885,19 +882,33 @@ noinline int __filemap_add_folio(struct address_space *mapping, xas_set_err(&xas, -EEXIST); goto unlock; } + /* + * If a larger entry exists, + * it will be the first and only entry iterated. + */ + if (order == -1) + order = xas_get_order(&xas); + } + + /* entry may have changed before we re-acquire the lock */ + if (alloced_order && (old != alloced_shadow || order != alloced_order)) { + xas_destroy(&xas); + alloced_order = 0; } if (old) { - if (shadowp) - *shadowp = old; - /* entry may have been split before we acquired lock */ - order = xa_get_order(xas.xa, xas.xa_index); - if (order > folio_order(folio)) { + if (order > 0 && order > folio_order(folio)) { /* How to handle large swap entries? */ BUG_ON(shmem_mapping(mapping)); + if (!alloced_order) { + split_order = order; + goto unlock; + } xas_split(&xas, old, order); xas_reset(&xas); } + if (shadowp) + *shadowp = old; } xas_store(&xas, folio); @@ -913,9 +924,24 @@ noinline int __filemap_add_folio(struct address_space *mapping, __lruvec_stat_mod_folio(folio, NR_FILE_THPS, nr); } + unlock: xas_unlock_irq(&xas); - } while (xas_nomem(&xas, gfp)); + + /* split needed, alloc here and retry. */ + if (split_order) { + xas_split_alloc(&xas, old, split_order, gfp); + if (xas_error(&xas)) + goto error; + alloced_shadow = old; + alloced_order = split_order; + xas_reset(&xas); + continue; + } + + if (!xas_nomem(&xas, gfp)) + break; + } if (xas_error(&xas)) goto error; From fdb022f6e930152ba36fdac7c5caf3ee44306215 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 25 Mar 2024 22:56:41 +0800 Subject: [PATCH 0636/1702] x86: remove unneeded memblock_find_dma_reserve() Patch series "mm/mm_init.c: refactor free_area_init_core()". In function free_area_init_core(), the code calculating zone->managed_pages and the subtracting dma_reserve from DMA zone looks very confusing. From git history, the code calculating zone->managed_pages was for zone->present_pages originally. The early rough assignment is for optimize zone's pcp and water mark setting. Later, managed_pages was introduced into zone to represent the number of managed pages by buddy. Now, zone->managed_pages is zeroed out and reset in mem_init() when calling memblock_free_all(). zone's pcp and wmark setting relying on actual zone->managed_pages are done later than mem_init() invocation. So we don't need rush to early calculate and set zone->managed_pages, just set it as zone->present_pages, will adjust it in mem_init(). And also add a new function calc_nr_kernel_pages() to count up free but not reserved pages in memblock, then assign it to nr_all_pages and nr_kernel_pages after memmap pages are allocated. This patch (of 6): Variable dma_reserve and its usage was introduced in commit 0e0b864e069c ("[PATCH] Account for memmap and optionally the kernel image as holes"). Its original purpose was to accounting for the reserved pages in DMA zone to make DMA zone's watermarks calculation more accurate on x86. However, currently there's zone->managed_pages to account for all available pages for buddy, zone->present_pages to account for all present physical pages in zone. What is more important, on x86, calculating and setting the zone->managed_pages is a temporary move, all zone's managed_pages will be zeroed out and reset to the actual value according to how many pages are added to buddy allocator in mem_init(). Before mem_init(), no buddy alloction is requested. And zone's pcp and watermark setting are all done after mem_init(). So, no need to worry about the DMA zone's setting accuracy during free_area_init(). Hence, remove memblock_find_dma_reserve() to stop calculating and setting dma_reserve. Link: https://lkml.kernel.org/r/20240325145646.1044760-1-bhe@redhat.com Link: https://lkml.kernel.org/r/20240325145646.1044760-2-bhe@redhat.com Signed-off-by: Baoquan He Reviewed-by: Mike Rapoport (IBM) Signed-off-by: Andrew Morton --- arch/x86/include/asm/pgtable.h | 1 - arch/x86/kernel/setup.c | 2 -- arch/x86/mm/init.c | 47 ---------------------------------- 3 files changed, 50 deletions(-) diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 315535ffb2582..cefc7a84f7a44 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -1200,7 +1200,6 @@ static inline int pgd_none(pgd_t pgd) extern int direct_gbpages; void init_mem_mapping(void); void early_alloc_pgt_buf(void); -extern void memblock_find_dma_reserve(void); void __init poking_init(void); unsigned long init_memory_mapping(unsigned long start, unsigned long end, pgprot_t prot); diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index e125e059e2c45..47e7fcdbdacd5 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -1107,8 +1107,6 @@ void __init setup_arch(char **cmdline_p) */ arch_reserve_crashkernel(); - memblock_find_dma_reserve(); - if (!early_xdbc_setup_hardware()) early_xdbc_register_console(); diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index 679893ea5e687..615f0bf4bda61 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -990,53 +990,6 @@ void __init free_initrd_mem(unsigned long start, unsigned long end) } #endif -/* - * Calculate the precise size of the DMA zone (first 16 MB of RAM), - * and pass it to the MM layer - to help it set zone watermarks more - * accurately. - * - * Done on 64-bit systems only for the time being, although 32-bit systems - * might benefit from this as well. - */ -void __init memblock_find_dma_reserve(void) -{ -#ifdef CONFIG_X86_64 - u64 nr_pages = 0, nr_free_pages = 0; - unsigned long start_pfn, end_pfn; - phys_addr_t start_addr, end_addr; - int i; - u64 u; - - /* - * Iterate over all memory ranges (free and reserved ones alike), - * to calculate the total number of pages in the first 16 MB of RAM: - */ - nr_pages = 0; - for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) { - start_pfn = min(start_pfn, MAX_DMA_PFN); - end_pfn = min(end_pfn, MAX_DMA_PFN); - - nr_pages += end_pfn - start_pfn; - } - - /* - * Iterate over free memory ranges to calculate the number of free - * pages in the DMA zone, while not counting potential partial - * pages at the beginning or the end of the range: - */ - nr_free_pages = 0; - for_each_free_mem_range(u, NUMA_NO_NODE, MEMBLOCK_NONE, &start_addr, &end_addr, NULL) { - start_pfn = min_t(unsigned long, PFN_UP(start_addr), MAX_DMA_PFN); - end_pfn = min_t(unsigned long, PFN_DOWN(end_addr), MAX_DMA_PFN); - - if (start_pfn < end_pfn) - nr_free_pages += end_pfn - start_pfn; - } - - set_dma_reserve(nr_pages - nr_free_pages); -#endif -} - void __init zone_sizes_init(void) { unsigned long max_zone_pfns[MAX_NR_ZONES]; From 6600a6b10c3210280e7623b8ba7feac9619cc2cd Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 25 Mar 2024 22:56:42 +0800 Subject: [PATCH 0637/1702] mm/mm_init.c: remove the useless dma_reserve Now nobody calls set_dma_reserve() to set value for dma_reserve, remove set_dma_reserve(), global variable dma_reserve and the codes using it. Link: https://lkml.kernel.org/r/20240325145646.1044760-3-bhe@redhat.com Signed-off-by: Baoquan He Reviewed-by: Mike Rapoport (IBM) Signed-off-by: Andrew Morton --- include/linux/mm.h | 1 - mm/mm_init.c | 23 ----------------------- 2 files changed, 24 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index fe66b515aaab3..22ae6ab621da2 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3250,7 +3250,6 @@ static inline int early_pfn_to_nid(unsigned long pfn) extern int __meminit early_pfn_to_nid(unsigned long pfn); #endif -extern void set_dma_reserve(unsigned long new_dma_reserve); extern void mem_init(void); extern void __init mmap_init(void); diff --git a/mm/mm_init.c b/mm/mm_init.c index f45c2b32ba826..91bb8600c0fa2 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -227,7 +227,6 @@ static unsigned long required_movablecore_percent __initdata; static unsigned long nr_kernel_pages __initdata; static unsigned long nr_all_pages __initdata; -static unsigned long dma_reserve __initdata; static bool deferred_struct_pages __meminitdata; @@ -1584,12 +1583,6 @@ static void __init free_area_init_core(struct pglist_data *pgdat) zone_names[j], memmap_pages, freesize); } - /* Account for reserved pages */ - if (j == 0 && freesize > dma_reserve) { - freesize -= dma_reserve; - pr_debug(" %s zone: %lu pages reserved\n", zone_names[0], dma_reserve); - } - if (!is_highmem_idx(j)) nr_kernel_pages += freesize; /* Charge for highmem memmap if there are enough kernel pages */ @@ -2548,22 +2541,6 @@ void *__init alloc_large_system_hash(const char *tablename, return table; } -/** - * set_dma_reserve - set the specified number of pages reserved in the first zone - * @new_dma_reserve: The number of pages to mark reserved - * - * The per-cpu batchsize and zone watermarks are determined by managed_pages. - * In the DMA zone, a significant percentage may be consumed by kernel image - * and other unfreeable allocations which can skew the watermarks badly. This - * function may optionally be used to account for unfreeable pages in the - * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and - * smaller per-cpu batchsize. - */ -void __init set_dma_reserve(unsigned long new_dma_reserve) -{ - dma_reserve = new_dma_reserve; -} - void __init memblock_free_pages(struct page *page, unsigned long pfn, unsigned int order) { From 8ad4184985e844ab5848dbf8b7b02f4e14b5d010 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 25 Mar 2024 22:56:43 +0800 Subject: [PATCH 0638/1702] mm/mm_init.c: add new function calc_nr_all_pages() This is a preparation to calculate nr_kernel_pages and nr_all_pages, both of which will be used later in alloc_large_system_hash(). nr_all_pages counts up all free but not reserved memory in memblock allocator, including HIGHMEM memory. While nr_kernel_pages counts up all free but not reserved low memory in memblock allocator, excluding HIGHMEM memory. Link: https://lkml.kernel.org/r/20240325145646.1044760-4-bhe@redhat.com Signed-off-by: Baoquan He Reviewed-by: Mike Rapoport (IBM) Signed-off-by: Andrew Morton --- mm/mm_init.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/mm/mm_init.c b/mm/mm_init.c index 91bb8600c0fa2..ae91499564974 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1265,6 +1265,30 @@ static void __init reset_memoryless_node_totalpages(struct pglist_data *pgdat) pr_debug("On node %d totalpages: 0\n", pgdat->node_id); } +static void __init calc_nr_kernel_pages(void) +{ + unsigned long start_pfn, end_pfn; + phys_addr_t start_addr, end_addr; + u64 u; +#ifdef CONFIG_HIGHMEM + unsigned long high_zone_low = arch_zone_lowest_possible_pfn[ZONE_HIGHMEM]; +#endif + + for_each_free_mem_range(u, NUMA_NO_NODE, MEMBLOCK_NONE, &start_addr, &end_addr, NULL) { + start_pfn = PFN_UP(start_addr); + end_pfn = PFN_DOWN(end_addr); + + if (start_pfn < end_pfn) { + nr_all_pages += end_pfn - start_pfn; +#ifdef CONFIG_HIGHMEM + start_pfn = clamp(start_pfn, 0, high_zone_low); + end_pfn = clamp(end_pfn, 0, high_zone_low); +#endif + nr_kernel_pages += end_pfn - start_pfn; + } + } +} + static void __init calculate_node_totalpages(struct pglist_data *pgdat, unsigned long node_start_pfn, unsigned long node_end_pfn) From 0ac5e785dcb797a3f0af1b205ce48134e186e88f Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 25 Mar 2024 22:56:44 +0800 Subject: [PATCH 0639/1702] mm/mm_init.c: remove meaningless calculation of zone->managed_pages in free_area_init_core() Currently, in free_area_init_core(), when initialize zone's field, a rough value is set to zone->managed_pages. That value is calculated by (zone->present_pages - memmap_pages). In the meantime, add the value to nr_all_pages and nr_kernel_pages which represent all free pages of system (only low memory or including HIGHMEM memory separately). Both of them are gonna be used in alloc_large_system_hash(). However, the rough calculation and setting of zone->managed_pages is meaningless because a) memmap pages are allocated on units of node in sparse_init() or alloc_node_mem_map(pgdat); The simple (zone->present_pages - memmap_pages) is too rough to make sense for zone; b) the set zone->managed_pages will be zeroed out and reset with acutal value in mem_init() via memblock_free_all(). Before the resetting, no buddy allocation request is issued. Here, remove the meaningless and complicated calculation of (zone->present_pages - memmap_pages), directly set zone->managed_pages as zone->present_pages for now. It will be adjusted in mem_init(). And also remove the assignment of nr_all_pages and nr_kernel_pages in free_area_init_core(). Instead, call the newly added calc_nr_kernel_pages() to count up all free but not reserved memory in memblock and assign to nr_all_pages and nr_kernel_pages. The counting excludes memmap_pages, and other kernel used data, which is more accurate than old way and simpler, and can also cover the ppc required arch_reserved_kernel_pages() case. And also clean up the outdated code comment above free_area_init_core(). And free_area_init_core() is easy to understand now, no need to add words to explain. [bhe@redhat.com: initialize zone->managed_pages as zone->present_pages for now] Link: https://lkml.kernel.org/r/ZgU0bsJ2FEjykvju@MiWiFi-R3L-srv Link: https://lkml.kernel.org/r/20240325145646.1044760-5-bhe@redhat.com Signed-off-by: Baoquan He Reviewed-by: Mike Rapoport (IBM) Signed-off-by: Andrew Morton --- mm/mm_init.c | 46 +++++----------------------------------------- 1 file changed, 5 insertions(+), 41 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index ae91499564974..b211b30231cb4 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1566,15 +1566,6 @@ void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) } #endif -/* - * Set up the zone data structures: - * - mark all pages reserved - * - mark all memory queues empty - * - clear the memory bitmaps - * - * NOTE: pgdat should get zeroed by caller. - * NOTE: this function is only called during early init. - */ static void __init free_area_init_core(struct pglist_data *pgdat) { enum zone_type j; @@ -1585,41 +1576,13 @@ static void __init free_area_init_core(struct pglist_data *pgdat) for (j = 0; j < MAX_NR_ZONES; j++) { struct zone *zone = pgdat->node_zones + j; - unsigned long size, freesize, memmap_pages; - - size = zone->spanned_pages; - freesize = zone->present_pages; - - /* - * Adjust freesize so that it accounts for how much memory - * is used by this zone for memmap. This affects the watermark - * and per-cpu initialisations - */ - memmap_pages = calc_memmap_size(size, freesize); - if (!is_highmem_idx(j)) { - if (freesize >= memmap_pages) { - freesize -= memmap_pages; - if (memmap_pages) - pr_debug(" %s zone: %lu pages used for memmap\n", - zone_names[j], memmap_pages); - } else - pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n", - zone_names[j], memmap_pages, freesize); - } - - if (!is_highmem_idx(j)) - nr_kernel_pages += freesize; - /* Charge for highmem memmap if there are enough kernel pages */ - else if (nr_kernel_pages > memmap_pages * 2) - nr_kernel_pages -= memmap_pages; - nr_all_pages += freesize; + unsigned long size = zone->spanned_pages; /* - * Set an approximate value for lowmem here, it will be adjusted - * when the bootmem allocator frees pages into the buddy system. - * And all highmem pages will be managed by the buddy system. + * Initialize zone->managed_pages as 0 , it will be reset + * when memblock allocator frees pages into buddy system. */ - zone_init_internals(zone, j, nid, freesize); + zone_init_internals(zone, j, nid, zone->present_pages); if (!size) continue; @@ -1916,6 +1879,7 @@ void __init free_area_init(unsigned long *max_zone_pfn) check_for_memory(pgdat); } + calc_nr_kernel_pages(); memmap_init(); /* disable hash distribution for systems with a single node */ From 90e796e22e35af0d19874c36fa4a22709aec1659 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 25 Mar 2024 22:56:45 +0800 Subject: [PATCH 0640/1702] mm/mm_init.c: remove unneeded calc_memmap_size() Nobody calls calc_memmap_size() now. Link: https://lkml.kernel.org/r/20240325145646.1044760-6-bhe@redhat.com Signed-off-by: Baoquan He Reviewed-by: Mike Rapoport (IBM) Signed-off-by: Andrew Morton --- mm/mm_init.c | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index b211b30231cb4..8c261572ca6ea 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1332,26 +1332,6 @@ static void __init calculate_node_totalpages(struct pglist_data *pgdat, pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); } -static unsigned long __init calc_memmap_size(unsigned long spanned_pages, - unsigned long present_pages) -{ - unsigned long pages = spanned_pages; - - /* - * Provide a more accurate estimation if there are holes within - * the zone and SPARSEMEM is in use. If there are holes within the - * zone, each populated memory region may cost us one or two extra - * memmap pages due to alignment because memmap pages for each - * populated regions may not be naturally aligned on page boundary. - * So the (present_pages >> 4) heuristic is a tradeoff for that. - */ - if (spanned_pages > present_pages + (present_pages >> 4) && - IS_ENABLED(CONFIG_SPARSEMEM)) - pages = present_pages; - - return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; -} - #ifdef CONFIG_TRANSPARENT_HUGEPAGE static void pgdat_init_split_queue(struct pglist_data *pgdat) { From 0b52663f7547520fbf54e2c07f61f8bd1b5cb6eb Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 25 Mar 2024 22:56:46 +0800 Subject: [PATCH 0641/1702] mm/mm_init.c: remove arch_reserved_kernel_pages() Since the current calculation of calc_nr_kernel_pages() has taken into consideration of kernel reserved memory, no need to have arch_reserved_kernel_pages() any more. Link: https://lkml.kernel.org/r/20240325145646.1044760-7-bhe@redhat.com Signed-off-by: Baoquan He Reviewed-by: Mike Rapoport (IBM) Signed-off-by: Andrew Morton --- arch/powerpc/include/asm/mmu.h | 4 ---- arch/powerpc/kernel/fadump.c | 5 ----- include/linux/mm.h | 3 --- mm/mm_init.c | 12 ------------ 4 files changed, 24 deletions(-) diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h index 3b72c7ed24cfd..aa5c0fd5edb1c 100644 --- a/arch/powerpc/include/asm/mmu.h +++ b/arch/powerpc/include/asm/mmu.h @@ -406,9 +406,5 @@ extern void *abatron_pteptrs[2]; #include #endif -#if defined(CONFIG_FA_DUMP) || defined(CONFIG_PRESERVE_FA_DUMP) -#define __HAVE_ARCH_RESERVED_KERNEL_PAGES -#endif - #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_MMU_H_ */ diff --git a/arch/powerpc/kernel/fadump.c b/arch/powerpc/kernel/fadump.c index d14eda1e85896..ae8c7619e597d 100644 --- a/arch/powerpc/kernel/fadump.c +++ b/arch/powerpc/kernel/fadump.c @@ -1735,8 +1735,3 @@ static void __init fadump_reserve_crash_area(u64 base) memblock_reserve(mstart, msize); } } - -unsigned long __init arch_reserved_kernel_pages(void) -{ - return memblock_reserved_size() / PAGE_SIZE; -} diff --git a/include/linux/mm.h b/include/linux/mm.h index 22ae6ab621da2..dd47ba7a1c041 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3261,9 +3261,6 @@ static inline void show_mem(void) extern long si_mem_available(void); extern void si_meminfo(struct sysinfo * val); extern void si_meminfo_node(struct sysinfo *val, int nid); -#ifdef __HAVE_ARCH_RESERVED_KERNEL_PAGES -extern unsigned long arch_reserved_kernel_pages(void); -#endif extern __printf(3, 4) void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...); diff --git a/mm/mm_init.c b/mm/mm_init.c index 8c261572ca6ea..0d32bcc301e23 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2374,17 +2374,6 @@ void __init page_alloc_init_late(void) page_alloc_sysctl_init(); } -#ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES -/* - * Returns the number of pages that arch has reserved but - * is not known to alloc_large_system_hash(). - */ -static unsigned long __init arch_reserved_kernel_pages(void) -{ - return 0; -} -#endif - /* * Adaptive scale is meant to reduce sizes of hash tables on large memory * machines. As memory size is increased the scale is also increased but at @@ -2427,7 +2416,6 @@ void *__init alloc_large_system_hash(const char *tablename, if (!numentries) { /* round applicable memory size up to nearest megabyte */ numentries = nr_kernel_pages; - numentries -= arch_reserved_kernel_pages(); /* It isn't necessary when PAGE_SIZE >= 1MB */ if (PAGE_SIZE < SZ_1M) From d4e6b397be1b403991a028ea59dbe61dedc9bcc0 Mon Sep 17 00:00:00 2001 From: Yajun Deng Date: Mon, 25 Mar 2024 14:32:58 +0800 Subject: [PATCH 0642/1702] mm/mmap: convert all mas except mas_detach to vma iterator There are two types of iterators mas and vmi in the current code. If the maple tree comes from the mm structure, we can use the vma iterator. Avoid using mas directly as possible. Keep using mas for the mt_detach tree, since it doesn't come from the mm structure. Remove as many uses of mas as possible, but we will still have a few that must be passed through in unmap_vmas() and free_pgtables(). Also introduce vma_iter_reset, vma_iter_{prev, next}_range_limit and vma_iter_area_{lowest, highest} helper functions for using the vma interator. Link: https://lkml.kernel.org/r/20240325063258.1437618-1-yajun.deng@linux.dev Signed-off-by: Yajun Deng Tested-by: Helge Deller [parisc] Reviewed-by: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/internal.h | 29 +++++++++++++ mm/mmap.c | 113 +++++++++++++++++++++++++------------------------- 2 files changed, 85 insertions(+), 57 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 6c8d3844b6a3b..5dbfa1c12e890 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1254,6 +1254,35 @@ static inline void vma_iter_config(struct vma_iterator *vmi, __mas_set_range(&vmi->mas, index, last - 1); } +static inline void vma_iter_reset(struct vma_iterator *vmi) +{ + mas_reset(&vmi->mas); +} + +static inline +struct vm_area_struct *vma_iter_prev_range_limit(struct vma_iterator *vmi, unsigned long min) +{ + return mas_prev_range(&vmi->mas, min); +} + +static inline +struct vm_area_struct *vma_iter_next_range_limit(struct vma_iterator *vmi, unsigned long max) +{ + return mas_next_range(&vmi->mas, max); +} + +static inline int vma_iter_area_lowest(struct vma_iterator *vmi, unsigned long min, + unsigned long max, unsigned long size) +{ + return mas_empty_area(&vmi->mas, min, max - 1, size); +} + +static inline int vma_iter_area_highest(struct vma_iterator *vmi, unsigned long min, + unsigned long max, unsigned long size) +{ + return mas_empty_area_rev(&vmi->mas, min, max - 1, size); +} + /* * VMA Iterator functions shared between nommu and mmap */ diff --git a/mm/mmap.c b/mm/mmap.c index 6dbda99a47da1..77a625e13ec12 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1114,21 +1114,21 @@ static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_ */ struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma) { - MA_STATE(mas, &vma->vm_mm->mm_mt, vma->vm_end, vma->vm_end); struct anon_vma *anon_vma = NULL; struct vm_area_struct *prev, *next; + VMA_ITERATOR(vmi, vma->vm_mm, vma->vm_end); /* Try next first. */ - next = mas_walk(&mas); + next = vma_iter_load(&vmi); if (next) { anon_vma = reusable_anon_vma(next, vma, next); if (anon_vma) return anon_vma; } - prev = mas_prev(&mas, 0); + prev = vma_prev(&vmi); VM_BUG_ON_VMA(prev != vma, vma); - prev = mas_prev(&mas, 0); + prev = vma_prev(&vmi); /* Try prev next. */ if (prev) anon_vma = reusable_anon_vma(prev, prev, vma); @@ -1576,8 +1576,7 @@ static unsigned long unmapped_area(struct vm_unmapped_area_info *info) unsigned long length, gap; unsigned long low_limit, high_limit; struct vm_area_struct *tmp; - - MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); + VMA_ITERATOR(vmi, current->mm, 0); /* Adjust search length to account for worst case alignment overhead */ length = info->length + info->align_mask; @@ -1589,23 +1588,23 @@ static unsigned long unmapped_area(struct vm_unmapped_area_info *info) low_limit = mmap_min_addr; high_limit = info->high_limit; retry: - if (mas_empty_area(&mas, low_limit, high_limit - 1, length)) + if (vma_iter_area_lowest(&vmi, low_limit, high_limit, length)) return -ENOMEM; - gap = mas.index; + gap = vma_iter_addr(&vmi); gap += (info->align_offset - gap) & info->align_mask; - tmp = mas_next(&mas, ULONG_MAX); + tmp = vma_next(&vmi); if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */ if (vm_start_gap(tmp) < gap + length - 1) { low_limit = tmp->vm_end; - mas_reset(&mas); + vma_iter_reset(&vmi); goto retry; } } else { - tmp = mas_prev(&mas, 0); + tmp = vma_prev(&vmi); if (tmp && vm_end_gap(tmp) > gap) { low_limit = vm_end_gap(tmp); - mas_reset(&mas); + vma_iter_reset(&vmi); goto retry; } } @@ -1628,8 +1627,8 @@ static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) unsigned long length, gap, gap_end; unsigned long low_limit, high_limit; struct vm_area_struct *tmp; + VMA_ITERATOR(vmi, current->mm, 0); - MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); /* Adjust search length to account for worst case alignment overhead */ length = info->length + info->align_mask; if (length < info->length) @@ -1640,24 +1639,24 @@ static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) low_limit = mmap_min_addr; high_limit = info->high_limit; retry: - if (mas_empty_area_rev(&mas, low_limit, high_limit - 1, length)) + if (vma_iter_area_highest(&vmi, low_limit, high_limit, length)) return -ENOMEM; - gap = mas.last + 1 - info->length; + gap = vma_iter_end(&vmi) - info->length; gap -= (gap - info->align_offset) & info->align_mask; - gap_end = mas.last; - tmp = mas_next(&mas, ULONG_MAX); + gap_end = vma_iter_end(&vmi); + tmp = vma_next(&vmi); if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */ - if (vm_start_gap(tmp) <= gap_end) { + if (vm_start_gap(tmp) < gap_end) { high_limit = vm_start_gap(tmp); - mas_reset(&mas); + vma_iter_reset(&vmi); goto retry; } } else { - tmp = mas_prev(&mas, 0); + tmp = vma_prev(&vmi); if (tmp && vm_end_gap(tmp) > gap) { high_limit = tmp->vm_start; - mas_reset(&mas); + vma_iter_reset(&vmi); goto retry; } } @@ -1912,12 +1911,12 @@ find_vma_prev(struct mm_struct *mm, unsigned long addr, struct vm_area_struct **pprev) { struct vm_area_struct *vma; - MA_STATE(mas, &mm->mm_mt, addr, addr); + VMA_ITERATOR(vmi, mm, addr); - vma = mas_walk(&mas); - *pprev = mas_prev(&mas, 0); + vma = vma_iter_load(&vmi); + *pprev = vma_prev(&vmi); if (!vma) - vma = mas_next(&mas, ULONG_MAX); + vma = vma_next(&vmi); return vma; } @@ -1971,7 +1970,7 @@ static int expand_upwards(struct vm_area_struct *vma, unsigned long address) struct vm_area_struct *next; unsigned long gap_addr; int error = 0; - MA_STATE(mas, &mm->mm_mt, vma->vm_start, address); + VMA_ITERATOR(vmi, mm, vma->vm_start); if (!(vma->vm_flags & VM_GROWSUP)) return -EFAULT; @@ -1997,15 +1996,15 @@ static int expand_upwards(struct vm_area_struct *vma, unsigned long address) } if (next) - mas_prev_range(&mas, address); + vma_iter_prev_range_limit(&vmi, address); - __mas_set_range(&mas, vma->vm_start, address - 1); - if (mas_preallocate(&mas, vma, GFP_KERNEL)) + vma_iter_config(&vmi, vma->vm_start, address); + if (vma_iter_prealloc(&vmi, vma)) return -ENOMEM; /* We must make sure the anon_vma is allocated. */ if (unlikely(anon_vma_prepare(vma))) { - mas_destroy(&mas); + vma_iter_free(&vmi); return -ENOMEM; } @@ -2045,7 +2044,7 @@ static int expand_upwards(struct vm_area_struct *vma, unsigned long address) anon_vma_interval_tree_pre_update_vma(vma); vma->vm_end = address; /* Overwrite old entry in mtree. */ - mas_store_prealloc(&mas, vma); + vma_iter_store(&vmi, vma); anon_vma_interval_tree_post_update_vma(vma); spin_unlock(&mm->page_table_lock); @@ -2054,7 +2053,7 @@ static int expand_upwards(struct vm_area_struct *vma, unsigned long address) } } anon_vma_unlock_write(vma->anon_vma); - mas_destroy(&mas); + vma_iter_free(&vmi); validate_mm(mm); return error; } @@ -2067,9 +2066,9 @@ static int expand_upwards(struct vm_area_struct *vma, unsigned long address) int expand_downwards(struct vm_area_struct *vma, unsigned long address) { struct mm_struct *mm = vma->vm_mm; - MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_start); struct vm_area_struct *prev; int error = 0; + VMA_ITERATOR(vmi, mm, vma->vm_start); if (!(vma->vm_flags & VM_GROWSDOWN)) return -EFAULT; @@ -2079,7 +2078,7 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) return -EPERM; /* Enforce stack_guard_gap */ - prev = mas_prev(&mas, 0); + prev = vma_prev(&vmi); /* Check that both stack segments have the same anon_vma? */ if (prev) { if (!(prev->vm_flags & VM_GROWSDOWN) && @@ -2089,15 +2088,15 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) } if (prev) - mas_next_range(&mas, vma->vm_start); + vma_iter_next_range_limit(&vmi, vma->vm_start); - __mas_set_range(&mas, address, vma->vm_end - 1); - if (mas_preallocate(&mas, vma, GFP_KERNEL)) + vma_iter_config(&vmi, address, vma->vm_end); + if (vma_iter_prealloc(&vmi, vma)) return -ENOMEM; /* We must make sure the anon_vma is allocated. */ if (unlikely(anon_vma_prepare(vma))) { - mas_destroy(&mas); + vma_iter_free(&vmi); return -ENOMEM; } @@ -2138,7 +2137,7 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) vma->vm_start = address; vma->vm_pgoff -= grow; /* Overwrite old entry in mtree. */ - mas_store_prealloc(&mas, vma); + vma_iter_store(&vmi, vma); anon_vma_interval_tree_post_update_vma(vma); spin_unlock(&mm->page_table_lock); @@ -2147,7 +2146,7 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) } } anon_vma_unlock_write(vma->anon_vma); - mas_destroy(&mas); + vma_iter_free(&vmi); validate_mm(mm); return error; } @@ -3242,7 +3241,7 @@ void exit_mmap(struct mm_struct *mm) struct mmu_gather tlb; struct vm_area_struct *vma; unsigned long nr_accounted = 0; - MA_STATE(mas, &mm->mm_mt, 0, 0); + VMA_ITERATOR(vmi, mm, 0); int count = 0; /* mm's last user has gone, and its about to be pulled down */ @@ -3251,7 +3250,7 @@ void exit_mmap(struct mm_struct *mm) mmap_read_lock(mm); arch_exit_mmap(mm); - vma = mas_find(&mas, ULONG_MAX); + vma = vma_next(&vmi); if (!vma || unlikely(xa_is_zero(vma))) { /* Can happen if dup_mmap() received an OOM */ mmap_read_unlock(mm); @@ -3264,7 +3263,7 @@ void exit_mmap(struct mm_struct *mm) tlb_gather_mmu_fullmm(&tlb, mm); /* update_hiwater_rss(mm) here? but nobody should be looking */ /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */ - unmap_vmas(&tlb, &mas, vma, 0, ULONG_MAX, ULONG_MAX, false); + unmap_vmas(&tlb, &vmi.mas, vma, 0, ULONG_MAX, ULONG_MAX, false); mmap_read_unlock(mm); /* @@ -3274,8 +3273,8 @@ void exit_mmap(struct mm_struct *mm) set_bit(MMF_OOM_SKIP, &mm->flags); mmap_write_lock(mm); mt_clear_in_rcu(&mm->mm_mt); - mas_set(&mas, vma->vm_end); - free_pgtables(&tlb, &mas, vma, FIRST_USER_ADDRESS, + vma_iter_set(&vmi, vma->vm_end); + free_pgtables(&tlb, &vmi.mas, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING, true); tlb_finish_mmu(&tlb); @@ -3284,14 +3283,14 @@ void exit_mmap(struct mm_struct *mm) * enabled, without holding any MM locks besides the unreachable * mmap_write_lock. */ - mas_set(&mas, vma->vm_end); + vma_iter_set(&vmi, vma->vm_end); do { if (vma->vm_flags & VM_ACCOUNT) nr_accounted += vma_pages(vma); remove_vma(vma, true); count++; cond_resched(); - vma = mas_find(&mas, ULONG_MAX); + vma = vma_next(&vmi); } while (vma && likely(!xa_is_zero(vma))); BUG_ON(count != mm->map_count); @@ -3713,7 +3712,7 @@ int mm_take_all_locks(struct mm_struct *mm) { struct vm_area_struct *vma; struct anon_vma_chain *avc; - MA_STATE(mas, &mm->mm_mt, 0, 0); + VMA_ITERATOR(vmi, mm, 0); mmap_assert_write_locked(mm); @@ -3725,14 +3724,14 @@ int mm_take_all_locks(struct mm_struct *mm) * being written to until mmap_write_unlock() or mmap_write_downgrade() * is reached. */ - mas_for_each(&mas, vma, ULONG_MAX) { + for_each_vma(vmi, vma) { if (signal_pending(current)) goto out_unlock; vma_start_write(vma); } - mas_set(&mas, 0); - mas_for_each(&mas, vma, ULONG_MAX) { + vma_iter_init(&vmi, mm, 0); + for_each_vma(vmi, vma) { if (signal_pending(current)) goto out_unlock; if (vma->vm_file && vma->vm_file->f_mapping && @@ -3740,8 +3739,8 @@ int mm_take_all_locks(struct mm_struct *mm) vm_lock_mapping(mm, vma->vm_file->f_mapping); } - mas_set(&mas, 0); - mas_for_each(&mas, vma, ULONG_MAX) { + vma_iter_init(&vmi, mm, 0); + for_each_vma(vmi, vma) { if (signal_pending(current)) goto out_unlock; if (vma->vm_file && vma->vm_file->f_mapping && @@ -3749,8 +3748,8 @@ int mm_take_all_locks(struct mm_struct *mm) vm_lock_mapping(mm, vma->vm_file->f_mapping); } - mas_set(&mas, 0); - mas_for_each(&mas, vma, ULONG_MAX) { + vma_iter_init(&vmi, mm, 0); + for_each_vma(vmi, vma) { if (signal_pending(current)) goto out_unlock; if (vma->anon_vma) @@ -3809,12 +3808,12 @@ void mm_drop_all_locks(struct mm_struct *mm) { struct vm_area_struct *vma; struct anon_vma_chain *avc; - MA_STATE(mas, &mm->mm_mt, 0, 0); + VMA_ITERATOR(vmi, mm, 0); mmap_assert_write_locked(mm); BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); - mas_for_each(&mas, vma, ULONG_MAX) { + for_each_vma(vmi, vma) { if (vma->anon_vma) list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) vm_unlock_anon_vma(avc->anon_vma); From a8353dc98f3ae570297e5e25cc05fc7d6b7f0e7b Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Sun, 24 Mar 2024 21:44:52 -0700 Subject: [PATCH 0643/1702] huge_memory.c: document huge page splitting rules more thoroughly 1. Add information about the behavior of huge page splitting, with respect to page/folio refcounts, and gup/pup pins. 2. Update and clarify the existing documentation, to compensate for the ravages of time and code change. Link: https://lkml.kernel.org/r/20240325044452.217463-1-jhubbard@nvidia.com Signed-off-by: John Hubbard Reviewed-by: Zi Yan Reviewed-by: David Hildenbrand Cc: Matthew Wilcox Signed-off-by: Andrew Morton --- mm/huge_memory.c | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 1170fc22ed893..75ad971ca45ea 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3006,28 +3006,40 @@ bool can_split_folio(struct folio *folio, int *pextra_pins) } /* - * This function splits huge page into pages in @new_order. @page can point to - * any subpage of huge page to split. Split doesn't change the position of - * @page. + * This function splits a large folio into smaller folios of order @new_order. + * @page can point to any page of the large folio to split. The split operation + * does not change the position of @page. * - * NOTE: order-1 anonymous folio is not supported because _deferred_list, - * which is used by partially mapped folios, is stored in subpage 2 and an - * order-1 folio only has subpage 0 and 1. File-backed order-1 folios are OK, - * since they do not use _deferred_list. + * Prerequisites: * - * Only caller must hold pin on the @page, otherwise split fails with -EBUSY. - * The huge page must be locked. + * 1) The caller must hold a reference on the @page's owning folio, also known + * as the large folio. + * + * 2) The large folio must be locked. + * + * 3) The folio must not be pinned. Any unexpected folio references, including + * GUP pins, will result in the folio not getting split; instead, the caller + * will receive an -EBUSY. + * + * 4) @new_order > 1, usually. Splitting to order-1 anonymous folios is not + * supported for non-file-backed folios, because folio->_deferred_list, which + * is used by partially mapped folios, is stored in subpage 2, but an order-1 + * folio only has subpages 0 and 1. File-backed order-1 folios are supported, + * since they do not use _deferred_list. + * + * After splitting, the caller's folio reference will be transferred to @page, + * resulting in a raised refcount of @page after this call. The other pages may + * be freed if they are not mapped. * * If @list is null, tail pages will be added to LRU list, otherwise, to @list. * - * Pages in new_order will inherit mapping, flags, and so on from the hugepage. + * Pages in @new_order will inherit the mapping, flags, and so on from the + * huge page. * - * GUP pin and PG_locked transferred to @page or the compound page @page belongs - * to. Rest subpages can be freed if they are not mapped. + * Returns 0 if the huge page was split successfully. * - * Returns 0 if the hugepage is split successfully. - * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under - * us. + * Returns -EBUSY if @page's folio is pinned, or if the anon_vma disappeared + * from under us. */ int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, unsigned int new_order) From 85109a8a9a10602749b2693ddb9417b3d14754a5 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Mon, 25 Mar 2024 11:56:35 +0800 Subject: [PATCH 0644/1702] mm: backing-dev: use group allocation/free of per-cpu counters API Use group allocation/free of per-cpu counters api to accelerate wb_init/exit() and simplify code. Link: https://lkml.kernel.org/r/20240325035635.49342-1-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Cc: Dennis Zhou Signed-off-by: Andrew Morton --- mm/backing-dev.c | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/mm/backing-dev.c b/mm/backing-dev.c index 5f2be8c8df11f..5fa3666356f95 100644 --- a/mm/backing-dev.c +++ b/mm/backing-dev.c @@ -388,7 +388,7 @@ static void wb_update_bandwidth_workfn(struct work_struct *work) static int wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi, gfp_t gfp) { - int i, err; + int err; memset(wb, 0, sizeof(*wb)); @@ -416,18 +416,10 @@ static int wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi, if (err) return err; - for (i = 0; i < NR_WB_STAT_ITEMS; i++) { - err = percpu_counter_init(&wb->stat[i], 0, gfp); - if (err) - goto out_destroy_stat; - } - - return 0; + err = percpu_counter_init_many(wb->stat, 0, gfp, NR_WB_STAT_ITEMS); + if (err) + fprop_local_destroy_percpu(&wb->completions); -out_destroy_stat: - while (i--) - percpu_counter_destroy(&wb->stat[i]); - fprop_local_destroy_percpu(&wb->completions); return err; } @@ -460,13 +452,8 @@ static void wb_shutdown(struct bdi_writeback *wb) static void wb_exit(struct bdi_writeback *wb) { - int i; - WARN_ON(delayed_work_pending(&wb->dwork)); - - for (i = 0; i < NR_WB_STAT_ITEMS; i++) - percpu_counter_destroy(&wb->stat[i]); - + percpu_counter_destroy_many(wb->stat, NR_WB_STAT_ITEMS); fprop_local_destroy_percpu(&wb->completions); } From 1b265da7ea1e1ae997fa119c2846bb389eb39c6b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 25 Mar 2024 07:45:40 +0800 Subject: [PATCH 0645/1702] virt: acrn: stop using follow_pfn Patch series "remove follow_pfn". This series open codes follow_pfn in the only remaining caller, although the code there remains questionable. It then also moves follow_phys into the only user and simplifies it a bit. This patch (of 3): Switch from follow_pfn to follow_pte so that we can get rid of follow_pfn. Note that this doesn't fix any of the pre-existing raciness and lack of permission checking in the code. Link: https://lkml.kernel.org/r/20240324234542.2038726-1-hch@lst.de Link: https://lkml.kernel.org/r/20240324234542.2038726-2-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: David Hildenbrand Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fei Li Cc: Peter Zijlstra Cc: Ingo Molnar Signed-off-by: Andrew Morton --- drivers/virt/acrn/mm.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/virt/acrn/mm.c b/drivers/virt/acrn/mm.c index c088ee1f11804..b30077baf352b 100644 --- a/drivers/virt/acrn/mm.c +++ b/drivers/virt/acrn/mm.c @@ -172,18 +172,24 @@ int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) mmap_read_lock(current->mm); vma = vma_lookup(current->mm, memmap->vma_base); if (vma && ((vma->vm_flags & VM_PFNMAP) != 0)) { + spinlock_t *ptl; + pte_t *ptep; + if ((memmap->vma_base + memmap->len) > vma->vm_end) { mmap_read_unlock(current->mm); return -EINVAL; } - ret = follow_pfn(vma, memmap->vma_base, &pfn); - mmap_read_unlock(current->mm); + ret = follow_pte(vma->vm_mm, memmap->vma_base, &ptep, &ptl); if (ret < 0) { + mmap_read_unlock(current->mm); dev_dbg(acrn_dev.this_device, "Failed to lookup PFN at VMA:%pK.\n", (void *)memmap->vma_base); return ret; } + pfn = pte_pfn(ptep_get(ptep)); + pte_unmap_unlock(ptep, ptl); + mmap_read_unlock(current->mm); return acrn_mm_region_add(vm, memmap->user_vm_pa, PFN_PHYS(pfn), memmap->len, From cb10c28ac82c9b7a5e9b3b1dc7157036c20c36dd Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 25 Mar 2024 07:45:41 +0800 Subject: [PATCH 0646/1702] mm: remove follow_pfn Remove follow_pfn now that the last user is gone. Link: https://lkml.kernel.org/r/20240324234542.2038726-3-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: David Hildenbrand Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fei Li Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Nathan Chancellor Signed-off-by: Andrew Morton --- include/linux/mm.h | 2 -- mm/memory.c | 36 ++---------------------------------- mm/nommu.c | 21 --------------------- 3 files changed, 2 insertions(+), 57 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index dd47ba7a1c041..cb8e6158b5775 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2424,8 +2424,6 @@ int copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma); int follow_pte(struct mm_struct *mm, unsigned long address, pte_t **ptepp, spinlock_t **ptlp); -int follow_pfn(struct vm_area_struct *vma, unsigned long address, - unsigned long *pfn); int follow_phys(struct vm_area_struct *vma, unsigned long address, unsigned int flags, unsigned long *prot, resource_size_t *phys); int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, diff --git a/mm/memory.c b/mm/memory.c index bcd29104f5ea8..652370703e41a 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5884,8 +5884,8 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) * Only IO mappings and raw PFN mappings are allowed. The mmap semaphore * should be taken for read. * - * KVM uses this function. While it is arguably less bad than ``follow_pfn``, - * it is not a good general-purpose API. + * KVM uses this function. While it is arguably less bad than the historic + * ``follow_pfn``, it is not a good general-purpose API. * * Return: zero on success, -ve otherwise. */ @@ -5927,38 +5927,6 @@ int follow_pte(struct mm_struct *mm, unsigned long address, } EXPORT_SYMBOL_GPL(follow_pte); -/** - * follow_pfn - look up PFN at a user virtual address - * @vma: memory mapping - * @address: user virtual address - * @pfn: location to store found PFN - * - * Only IO mappings and raw PFN mappings are allowed. - * - * This function does not allow the caller to read the permissions - * of the PTE. Do not use it. - * - * Return: zero and the pfn at @pfn on success, -ve otherwise. - */ -int follow_pfn(struct vm_area_struct *vma, unsigned long address, - unsigned long *pfn) -{ - int ret = -EINVAL; - spinlock_t *ptl; - pte_t *ptep; - - if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) - return ret; - - ret = follow_pte(vma->vm_mm, address, &ptep, &ptl); - if (ret) - return ret; - *pfn = pte_pfn(ptep_get(ptep)); - pte_unmap_unlock(ptep, ptl); - return 0; -} -EXPORT_SYMBOL(follow_pfn); - #ifdef CONFIG_HAVE_IOREMAP_PROT int follow_phys(struct vm_area_struct *vma, unsigned long address, unsigned int flags, diff --git a/mm/nommu.c b/mm/nommu.c index 88b646ea6d30c..331d2f778695c 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -110,27 +110,6 @@ unsigned int kobjsize(const void *objp) return page_size(page); } -/** - * follow_pfn - look up PFN at a user virtual address - * @vma: memory mapping - * @address: user virtual address - * @pfn: location to store found PFN - * - * Only IO mappings and raw PFN mappings are allowed. - * - * Returns zero and the pfn at @pfn on success, -ve otherwise. - */ -int follow_pfn(struct vm_area_struct *vma, unsigned long address, - unsigned long *pfn) -{ - if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) - return -EINVAL; - - *pfn = address >> PAGE_SHIFT; - return 0; -} -EXPORT_SYMBOL(follow_pfn); - void vfree(const void *addr) { kfree(addr); From 5b34b76cb0cd8a21dee5c7677eae98480b0d05cc Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 25 Mar 2024 07:45:42 +0800 Subject: [PATCH 0647/1702] mm: move follow_phys to arch/x86/mm/pat/memtype.c follow_phys is only used by two callers in arch/x86/mm/pat/memtype.c. Move it there and hardcode the two arguments that get the same values passed by both callers. [david@redhat.com: conflict resolutions] Link: https://lkml.kernel.org/r/20240403212131.929421-4-david@redhat.com Link: https://lkml.kernel.org/r/20240324234542.2038726-4-hch@lst.de Signed-off-by: Christoph Hellwig Signed-off-by: David Hildenbrand Reviewed-by: David Hildenbrand Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fei Li Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Nathan Chancellor Signed-off-by: Andrew Morton --- arch/x86/mm/pat/memtype.c | 29 ++++++++++++++++++++++++++++- include/linux/mm.h | 2 -- mm/memory.c | 32 -------------------------------- 3 files changed, 28 insertions(+), 35 deletions(-) diff --git a/arch/x86/mm/pat/memtype.c b/arch/x86/mm/pat/memtype.c index 36b603d0cddef..d01c3b0bd6eb1 100644 --- a/arch/x86/mm/pat/memtype.c +++ b/arch/x86/mm/pat/memtype.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include @@ -947,6 +948,32 @@ static void free_pfn_range(u64 paddr, unsigned long size) memtype_free(paddr, paddr + size); } +static int follow_phys(struct vm_area_struct *vma, unsigned long *prot, + resource_size_t *phys) +{ + pte_t *ptep, pte; + spinlock_t *ptl; + + if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) + return -EINVAL; + + if (follow_pte(vma->vm_mm, vma->vm_start, &ptep, &ptl)) + return -EINVAL; + + pte = ptep_get(ptep); + + /* Never return PFNs of anon folios in COW mappings. */ + if (vm_normal_folio(vma, vma->vm_start, pte)) { + pte_unmap_unlock(ptep, ptl); + return -EINVAL; + } + + *prot = pgprot_val(pte_pgprot(pte)); + *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT; + pte_unmap_unlock(ptep, ptl); + return 0; +} + static int get_pat_info(struct vm_area_struct *vma, resource_size_t *paddr, pgprot_t *pgprot) { @@ -964,7 +991,7 @@ static int get_pat_info(struct vm_area_struct *vma, resource_size_t *paddr, * detect the PFN. If we need the cachemode as well, we're out of luck * for now and have to fail fork(). */ - if (!follow_phys(vma, vma->vm_start, 0, &prot, paddr)) { + if (!follow_phys(vma, &prot, paddr)) { if (pgprot) *pgprot = __pgprot(prot); return 0; diff --git a/include/linux/mm.h b/include/linux/mm.h index cb8e6158b5775..5dc65618e3863 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2424,8 +2424,6 @@ int copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma); int follow_pte(struct mm_struct *mm, unsigned long address, pte_t **ptepp, spinlock_t **ptlp); -int follow_phys(struct vm_area_struct *vma, unsigned long address, - unsigned int flags, unsigned long *prot, resource_size_t *phys); int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, void *buf, int len, int write); diff --git a/mm/memory.c b/mm/memory.c index 652370703e41a..62ee4a15092ae 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5928,38 +5928,6 @@ int follow_pte(struct mm_struct *mm, unsigned long address, EXPORT_SYMBOL_GPL(follow_pte); #ifdef CONFIG_HAVE_IOREMAP_PROT -int follow_phys(struct vm_area_struct *vma, - unsigned long address, unsigned int flags, - unsigned long *prot, resource_size_t *phys) -{ - int ret = -EINVAL; - pte_t *ptep, pte; - spinlock_t *ptl; - - if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) - goto out; - - if (follow_pte(vma->vm_mm, address, &ptep, &ptl)) - goto out; - pte = ptep_get(ptep); - - /* Never return PFNs of anon folios in COW mappings. */ - if (vm_normal_folio(vma, address, pte)) - goto unlock; - - if ((flags & FOLL_WRITE) && !pte_write(pte)) - goto unlock; - - *prot = pgprot_val(pte_pgprot(pte)); - *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT; - - ret = 0; -unlock: - pte_unmap_unlock(ptep, ptl); -out: - return ret; -} - /** * generic_access_phys - generic implementation for iomem mmap access * @vma: the vma to access From c139ca42f5740f0c94a001eee82dafada72c19ee Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 26 Mar 2024 15:32:09 +0100 Subject: [PATCH 0648/1702] selftests/memfd_secret: add vmsplice() test Let's add a simple reproducer for a scenario where GUP-fast could succeed on secretmem folios, making vmsplice() succeed instead of failing. The reproducer is based on a reproducer [1] by Miklos Szeredi. We want to perform two tests: vmsplice() when a fresh page was just faulted in, and vmsplice() on an existing page after munmap() that would drain certain LRU caches/batches in the kernel. In an ideal world, we could use fallocate(FALLOC_FL_PUNCH_HOLE) / MADV_REMOVE to remove any existing page. As that is currently not possible, run the test before any other tests that would allocate memory in the secretmem fd. Perform the ftruncate() only once, and check the return value. [1] https://lkml.kernel.org/r/CAJfpegt3UCsMmxd0taOY11Uaw5U=eS1fE5dn0wZX3HF0oy8-oQ@mail.gmail.com Link: https://lkml.kernel.org/r/20240326143210.291116-3-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Mike Rapoport (IBM) Cc: Lorenzo Stoakes Cc: Miklos Szeredi Cc: xingwei lee Cc: yue sun Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/memfd_secret.c | 51 ++++++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/mm/memfd_secret.c b/tools/testing/selftests/mm/memfd_secret.c index 9b298f6a04b37..9a0597310a765 100644 --- a/tools/testing/selftests/mm/memfd_secret.c +++ b/tools/testing/selftests/mm/memfd_secret.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "../kselftest.h" @@ -83,6 +84,45 @@ static void test_mlock_limit(int fd) pass("mlock limit is respected\n"); } +static void test_vmsplice(int fd, const char *desc) +{ + ssize_t transferred; + struct iovec iov; + int pipefd[2]; + char *mem; + + if (pipe(pipefd)) { + fail("pipe failed: %s\n", strerror(errno)); + return; + } + + mem = mmap(NULL, page_size, prot, mode, fd, 0); + if (mem == MAP_FAILED) { + fail("Unable to mmap secret memory\n"); + goto close_pipe; + } + + /* + * vmsplice() may use GUP-fast, which must also fail. Prefault the + * page table, so GUP-fast could find it. + */ + memset(mem, PATTERN, page_size); + + iov.iov_base = mem; + iov.iov_len = page_size; + transferred = vmsplice(pipefd[1], &iov, 1, 0); + + if (transferred < 0 && errno == EFAULT) + pass("vmsplice is blocked as expected with %s\n", desc); + else + fail("vmsplice: unexpected memory access with %s\n", desc); + + munmap(mem, page_size); +close_pipe: + close(pipefd[0]); + close(pipefd[1]); +} + static void try_process_vm_read(int fd, int pipefd[2]) { struct iovec liov, riov; @@ -187,7 +227,6 @@ static void test_remote_access(int fd, const char *name, return; } - ftruncate(fd, page_size); memset(mem, PATTERN, page_size); if (write(pipefd[1], &mem, sizeof(mem)) < 0) { @@ -258,7 +297,7 @@ static void prepare(void) strerror(errno)); } -#define NUM_TESTS 4 +#define NUM_TESTS 6 int main(int argc, char *argv[]) { @@ -277,9 +316,17 @@ int main(int argc, char *argv[]) ksft_exit_fail_msg("memfd_secret failed: %s\n", strerror(errno)); } + if (ftruncate(fd, page_size)) + ksft_exit_fail_msg("ftruncate failed: %s\n", strerror(errno)); test_mlock_limit(fd); test_file_apis(fd); + /* + * We have to run the first vmsplice test before any secretmem page was + * allocated for this fd. + */ + test_vmsplice(fd, "fresh page"); + test_vmsplice(fd, "existing page"); test_process_vm_read(fd); test_ptrace(fd); From f002882ca369aba3eece5006f3346ccf75ede7c5 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 26 Mar 2024 15:32:10 +0100 Subject: [PATCH 0649/1702] mm: merge folio_is_secretmem() and folio_fast_pin_allowed() into gup_fast_folio_allowed() folio_is_secretmem() is currently only used during GUP-fast. Nowadays, folio_fast_pin_allowed() performs similar checks during GUP-fast and contains a lot of careful handling -- READ_ONCE() -- , sanity checks -- lockdep_assert_irqs_disabled() -- and helpful comments on how this handling is safe and correct. So let's merge folio_is_secretmem() into folio_fast_pin_allowed(). Rename folio_fast_pin_allowed() to gup_fast_folio_allowed(), to better match the new semantics. Link: https://lkml.kernel.org/r/20240326143210.291116-4-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Mike Rapoport (IBM) Cc: David Hildenbrand Cc: Lorenzo Stoakes Cc: Miklos Szeredi Cc: xingwei lee Cc: yue sun Signed-off-by: Andrew Morton --- include/linux/secretmem.h | 21 ++--------------- mm/gup.c | 48 +++++++++++++++++++++++---------------- 2 files changed, 30 insertions(+), 39 deletions(-) diff --git a/include/linux/secretmem.h b/include/linux/secretmem.h index acf7e1a3f3def..e918f96881f56 100644 --- a/include/linux/secretmem.h +++ b/include/linux/secretmem.h @@ -6,25 +6,8 @@ extern const struct address_space_operations secretmem_aops; -static inline bool folio_is_secretmem(struct folio *folio) +static inline bool secretmem_mapping(struct address_space *mapping) { - struct address_space *mapping; - - /* - * Using folio_mapping() is quite slow because of the actual call - * instruction. - * We know that secretmem pages are not compound, so we can - * save a couple of cycles here. - */ - if (folio_test_large(folio)) - return false; - - mapping = (struct address_space *) - ((unsigned long)folio->mapping & ~PAGE_MAPPING_FLAGS); - - if (!mapping || mapping != folio->mapping) - return false; - return mapping->a_ops == &secretmem_aops; } @@ -38,7 +21,7 @@ static inline bool vma_is_secretmem(struct vm_area_struct *vma) return false; } -static inline bool folio_is_secretmem(struct folio *folio) +static inline bool secretmem_mapping(struct address_space *mapping) { return false; } diff --git a/mm/gup.c b/mm/gup.c index 6d8d15f8c7f9a..7b21fcda3a981 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2468,12 +2468,14 @@ EXPORT_SYMBOL(get_user_pages_unlocked); #ifdef CONFIG_HAVE_FAST_GUP /* - * Used in the GUP-fast path to determine whether a pin is permitted for a - * specific folio. + * Used in the GUP-fast path to determine whether GUP is permitted to work on + * a specific folio. * * This call assumes the caller has pinned the folio, that the lowest page table * level still points to this folio, and that interrupts have been disabled. * + * GUP-fast must reject all secretmem folios. + * * Writing to pinned file-backed dirty tracked folios is inherently problematic * (see comment describing the writable_file_mapping_allowed() function). We * therefore try to avoid the most egregious case of a long-term mapping doing @@ -2483,25 +2485,34 @@ EXPORT_SYMBOL(get_user_pages_unlocked); * in the fast path, so instead we whitelist known good cases and if in doubt, * fall back to the slow path. */ -static bool folio_fast_pin_allowed(struct folio *folio, unsigned int flags) +static bool gup_fast_folio_allowed(struct folio *folio, unsigned int flags) { + bool reject_file_backed = false; struct address_space *mapping; + bool check_secretmem = false; unsigned long mapping_flags; /* * If we aren't pinning then no problematic write can occur. A long term * pin is the most egregious case so this is the one we disallow. */ - if ((flags & (FOLL_PIN | FOLL_LONGTERM | FOLL_WRITE)) != + if ((flags & (FOLL_PIN | FOLL_LONGTERM | FOLL_WRITE)) == (FOLL_PIN | FOLL_LONGTERM | FOLL_WRITE)) - return true; + reject_file_backed = true; + + /* We hold a folio reference, so we can safely access folio fields. */ - /* The folio is pinned, so we can safely access folio fields. */ + /* secretmem folios are always order-0 folios. */ + if (IS_ENABLED(CONFIG_SECRETMEM) && !folio_test_large(folio)) + check_secretmem = true; + + if (!reject_file_backed && !check_secretmem) + return true; if (WARN_ON_ONCE(folio_test_slab(folio))) return false; - /* hugetlb mappings do not require dirty-tracking. */ + /* hugetlb neither requires dirty-tracking nor can be secretmem. */ if (folio_test_hugetlb(folio)) return true; @@ -2537,10 +2548,12 @@ static bool folio_fast_pin_allowed(struct folio *folio, unsigned int flags) /* * At this point, we know the mapping is non-null and points to an - * address_space object. The only remaining whitelisted file system is - * shmem. + * address_space object. */ - return shmem_mapping(mapping); + if (check_secretmem && secretmem_mapping(mapping)) + return false; + /* The only remaining allowed file system is shmem. */ + return !reject_file_backed || shmem_mapping(mapping); } static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start, @@ -2626,18 +2639,13 @@ static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, if (!folio) goto pte_unmap; - if (unlikely(folio_is_secretmem(folio))) { - gup_put_folio(folio, 1, flags); - goto pte_unmap; - } - if (unlikely(pmd_val(pmd) != pmd_val(*pmdp)) || unlikely(pte_val(pte) != pte_val(ptep_get(ptep)))) { gup_put_folio(folio, 1, flags); goto pte_unmap; } - if (!folio_fast_pin_allowed(folio, flags)) { + if (!gup_fast_folio_allowed(folio, flags)) { gup_put_folio(folio, 1, flags); goto pte_unmap; } @@ -2834,7 +2842,7 @@ static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, return 0; } - if (!folio_fast_pin_allowed(folio, flags)) { + if (!gup_fast_folio_allowed(folio, flags)) { gup_put_folio(folio, refs, flags); return 0; } @@ -2905,7 +2913,7 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, return 0; } - if (!folio_fast_pin_allowed(folio, flags)) { + if (!gup_fast_folio_allowed(folio, flags)) { gup_put_folio(folio, refs, flags); return 0; } @@ -2949,7 +2957,7 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, return 0; } - if (!folio_fast_pin_allowed(folio, flags)) { + if (!gup_fast_folio_allowed(folio, flags)) { gup_put_folio(folio, refs, flags); return 0; } @@ -2994,7 +3002,7 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr, return 0; } - if (!folio_fast_pin_allowed(folio, flags)) { + if (!gup_fast_folio_allowed(folio, flags)) { gup_put_folio(folio, refs, flags); return 0; } From f4b6680973d5f94845ca34dcc7126a769d48c0d4 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 17:10:23 +0000 Subject: [PATCH 0650/1702] sh: remove use of PG_arch_1 on individual pages Patch series "Various page->flags cleanups". The first two patches are bug fixes, although I'm not sure that either architecture will have noticed. There aren't a lot of uses of page->flags left! The big build-up here is to reworking stable_page_flags(), which will definitely be a user-visible change. I think a welcome one, given the special case we had to spread the Slab flag into all tail pages. This patch (of 10): Since switching to the new page table range API, we do not set the PG_arch_1 (aka dcache clean) flag on tail pages, only on the folio. Test it on the folio. Also use page_mapped() instead of page_mapcount() as it is more efficient. [akpm@linux-foundation.org: fix folio_flags call] Link: https://lkml.kernel.org/r/20240326171045.410737-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240326171045.410737-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- arch/sh/mm/cache-sh4.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/sh/mm/cache-sh4.c b/arch/sh/mm/cache-sh4.c index 862046f26981b..9a1e581cd192d 100644 --- a/arch/sh/mm/cache-sh4.c +++ b/arch/sh/mm/cache-sh4.c @@ -241,13 +241,14 @@ static void sh4_flush_cache_page(void *args) if ((vma->vm_mm == current->active_mm)) vaddr = NULL; else { + struct folio *folio = page_folio(page); /* * Use kmap_coherent or kmap_atomic to do flushes for * another ASID than the current one. */ map_coherent = (current_cpu_data.dcache.n_aliases && - test_bit(PG_dcache_clean, &page->flags) && - page_mapcount(page)); + test_bit(PG_dcache_clean, folio_flags(folio, 0)) && + page_mapped(page)); if (map_coherent) vaddr = kmap_coherent(page, address); else From fa92722e382b3926a371d7432bbfde5efee2f2be Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 17:10:24 +0000 Subject: [PATCH 0651/1702] xtensa: remove uses of PG_arch_1 on individual pages Since switching to the new page table range API, we disregard the PG_arch_1 (aka dcache dirty) flag on tail pages, and only pay attention to it on the folio. Fix these two missed spots where we were setting it on arbitrary pages. Link: https://lkml.kernel.org/r/20240326171045.410737-3-willy@infradead.org Reported-by: Svetly Todorov Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Svetly Todorov [xtensa] Signed-off-by: Andrew Morton --- arch/xtensa/mm/cache.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/xtensa/mm/cache.c b/arch/xtensa/mm/cache.c index 7ec66a79f4723..23be0e7516ced 100644 --- a/arch/xtensa/mm/cache.c +++ b/arch/xtensa/mm/cache.c @@ -87,12 +87,13 @@ static inline void *coherent_kvaddr(struct page *page, unsigned long base, void clear_user_highpage(struct page *page, unsigned long vaddr) { + struct folio *folio = page_folio(page); unsigned long paddr; void *kvaddr = coherent_kvaddr(page, TLBTEMP_BASE_1, vaddr, &paddr); preempt_disable(); kmap_invalidate_coherent(page, vaddr); - set_bit(PG_arch_1, &page->flags); + set_bit(PG_arch_1, folio_flags(folio, 0)); clear_page_alias(kvaddr, paddr); preempt_enable(); } @@ -101,6 +102,7 @@ EXPORT_SYMBOL(clear_user_highpage); void copy_user_highpage(struct page *dst, struct page *src, unsigned long vaddr, struct vm_area_struct *vma) { + struct folio *folio = page_folio(dst); unsigned long dst_paddr, src_paddr; void *dst_vaddr = coherent_kvaddr(dst, TLBTEMP_BASE_1, vaddr, &dst_paddr); @@ -109,7 +111,7 @@ void copy_user_highpage(struct page *dst, struct page *src, preempt_disable(); kmap_invalidate_coherent(dst, vaddr); - set_bit(PG_arch_1, &dst->flags); + set_bit(PG_arch_1, folio_flags(folio, 0)); copy_page_alias(dst_vaddr, src_vaddr, dst_paddr, src_paddr); preempt_enable(); } From 6e65aa55cdf4c7cc0490b156b4aae2035c157140 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 17:10:25 +0000 Subject: [PATCH 0652/1702] mm: make page_ext_get() take a const argument In order to constify other functions, we need page_ext_get() to be const. This is no problem as lookup_page_ext() already takes a const argument. Link: https://lkml.kernel.org/r/20240326171045.410737-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/page_ext.h | 4 ++-- include/linux/pgalloc_tag.h | 2 -- mm/page_ext.c | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h index 07e0656898f98..e4b48a0dda244 100644 --- a/include/linux/page_ext.h +++ b/include/linux/page_ext.h @@ -77,7 +77,7 @@ static inline void page_ext_init(void) } #endif -extern struct page_ext *page_ext_get(struct page *page); +extern struct page_ext *page_ext_get(const struct page *page); extern void page_ext_put(struct page_ext *page_ext); static inline void *page_ext_data(struct page_ext *page_ext, @@ -117,7 +117,7 @@ static inline void page_ext_init_flatmem(void) { } -static inline struct page_ext *page_ext_get(struct page *page) +static inline struct page_ext *page_ext_get(const struct page *page) { return NULL; } diff --git a/include/linux/pgalloc_tag.h b/include/linux/pgalloc_tag.h index 62d8dad74b373..86ba5d33e43bd 100644 --- a/include/linux/pgalloc_tag.h +++ b/include/linux/pgalloc_tag.h @@ -12,8 +12,6 @@ #include extern struct page_ext_operations page_alloc_tagging_ops; -extern struct page_ext *page_ext_get(struct page *page); -extern void page_ext_put(struct page_ext *page_ext); static inline union codetag_ref *codetag_ref_from_page_ext(struct page_ext *page_ext) { diff --git a/mm/page_ext.c b/mm/page_ext.c index e7d8f1a5589ed..95dd8ffeaf811 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -514,7 +514,7 @@ void __meminit pgdat_page_ext_init(struct pglist_data *pgdat) * Context: Any context. Caller may not sleep until they have called * page_ext_put(). */ -struct page_ext *page_ext_get(struct page *page) +struct page_ext *page_ext_get(const struct page *page) { struct page_ext *page_ext; From e3089fd0b0199a368db4039122b241139cc665f3 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 17:10:26 +0000 Subject: [PATCH 0653/1702] mm: make folio_test_idle and folio_test_young take a const argument If these functions are defined in page-flags.h, they already take a const argument; make it true for these alternate definitions too. Link: https://lkml.kernel.org/r/20240326171045.410737-5-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/page_idle.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/linux/page_idle.h b/include/linux/page_idle.h index d8f3448406430..511e22ef459fc 100644 --- a/include/linux/page_idle.h +++ b/include/linux/page_idle.h @@ -13,7 +13,7 @@ * If there is not enough space to store Idle and Young bits in page flags, use * page ext flags instead. */ -static inline bool folio_test_young(struct folio *folio) +static inline bool folio_test_young(const struct folio *folio) { struct page_ext *page_ext = page_ext_get(&folio->page); bool page_young; @@ -52,7 +52,7 @@ static inline bool folio_test_clear_young(struct folio *folio) return page_young; } -static inline bool folio_test_idle(struct folio *folio) +static inline bool folio_test_idle(const struct folio *folio) { struct page_ext *page_ext = page_ext_get(&folio->page); bool page_idle; @@ -60,7 +60,7 @@ static inline bool folio_test_idle(struct folio *folio) if (unlikely(!page_ext)) return false; - page_idle = test_bit(PAGE_EXT_IDLE, &page_ext->flags); + page_idle = test_bit(PAGE_EXT_IDLE, &page_ext->flags); page_ext_put(page_ext); return page_idle; @@ -91,7 +91,7 @@ static inline void folio_clear_idle(struct folio *folio) #else /* !CONFIG_PAGE_IDLE_FLAG */ -static inline bool folio_test_young(struct folio *folio) +static inline bool folio_test_young(const struct folio *folio) { return false; } @@ -105,7 +105,7 @@ static inline bool folio_test_clear_young(struct folio *folio) return false; } -static inline bool folio_test_idle(struct folio *folio) +static inline bool folio_test_idle(const struct folio *folio) { return false; } From 2ace5a670e2a0f3a6029bc6cf91dc14202074e7a Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 17:10:27 +0000 Subject: [PATCH 0654/1702] mm: make is_free_buddy_page() take a const argument This function does not modify its argument; let the callers know that so they can make better optimisation decisions. Link: https://lkml.kernel.org/r/20240326171045.410737-6-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 2 +- mm/page_alloc.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 0163d6da6283c..eaecf544039fe 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -1088,7 +1088,7 @@ static inline bool is_page_hwpoison(const struct page *page) return folio_test_hugetlb(folio) && PageHWPoison(&folio->page); } -extern bool is_free_buddy_page(struct page *page); +bool is_free_buddy_page(const struct page *page); PAGEFLAG(Isolated, isolated, PF_ANY); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index daab8cab91ccf..6a8593aef2977 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6745,16 +6745,16 @@ void __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) /* * This function returns a stable result only if called under zone lock. */ -bool is_free_buddy_page(struct page *page) +bool is_free_buddy_page(const struct page *page) { unsigned long pfn = page_to_pfn(page); unsigned int order; for (order = 0; order < NR_PAGE_ORDERS; order++) { - struct page *page_head = page - (pfn & ((1 << order) - 1)); + const struct page *head = page - (pfn & ((1 << order) - 1)); - if (PageBuddy(page_head) && - buddy_order_unsafe(page_head) >= order) + if (PageBuddy(head) && + buddy_order_unsafe(head) >= order) break; } From b84fd2835c70e0149e2522fd746d3fb7049a1e19 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 17:10:28 +0000 Subject: [PATCH 0655/1702] mm: make page_mapped() take a const argument None of the functions called by page_mapped() modify the page or folio, so mark them all as const. Link: https://lkml.kernel.org/r/20240326171045.410737-7-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/mm.h | 12 ++++++------ mm/internal.h | 7 ++++--- mm/rmap.c | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 5dc65618e3863..9c0e441664d48 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1200,7 +1200,7 @@ static inline int is_vmalloc_or_module_addr(const void *x) * debugging purposes - it does not include PTE-mapped sub-pages; look * at folio_mapcount() or page_mapcount() instead. */ -static inline int folio_entire_mapcount(struct folio *folio) +static inline int folio_entire_mapcount(const struct folio *folio) { VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); return atomic_read(&folio->_entire_mapcount) + 1; @@ -1240,7 +1240,7 @@ static inline int page_mapcount(struct page *page) return mapcount; } -int folio_total_mapcount(struct folio *folio); +int folio_total_mapcount(const struct folio *folio); /** * folio_mapcount() - Calculate the number of mappings of this folio. @@ -1253,14 +1253,14 @@ int folio_total_mapcount(struct folio *folio); * * Return: The number of times this folio is mapped. */ -static inline int folio_mapcount(struct folio *folio) +static inline int folio_mapcount(const struct folio *folio) { if (likely(!folio_test_large(folio))) return atomic_read(&folio->_mapcount) + 1; return folio_total_mapcount(folio); } -static inline bool folio_large_is_mapped(struct folio *folio) +static inline bool folio_large_is_mapped(const struct folio *folio) { /* * Reading _entire_mapcount below could be omitted if hugetlb @@ -1288,7 +1288,7 @@ static inline bool folio_mapped(struct folio *folio) * For compound page it returns true if any sub-page of compound page is mapped, * even if this particular sub-page is not itself mapped by any PTE or PMD. */ -static inline bool page_mapped(struct page *page) +static inline bool page_mapped(const struct page *page) { if (likely(!PageCompound(page))) return atomic_read(&page->_mapcount) >= 0; @@ -2070,7 +2070,7 @@ static inline void set_page_links(struct page *page, enum zone_type zone, * * Return: A positive power of two. */ -static inline long folio_nr_pages(struct folio *folio) +static inline long folio_nr_pages(const struct folio *folio) { if (!folio_test_large(folio)) return 1; diff --git a/mm/internal.h b/mm/internal.h index 5dbfa1c12e890..8e11f7b2da216 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -71,7 +71,7 @@ void page_writeback_init(void); * How many individual pages have an elevated _mapcount. Excludes * the folio's entire_mapcount. */ -static inline int folio_nr_pages_mapped(struct folio *folio) +static inline int folio_nr_pages_mapped(const struct folio *folio) { return atomic_read(&folio->_nr_pages_mapped) & FOLIO_PAGES_MAPPED; } @@ -81,7 +81,8 @@ static inline int folio_nr_pages_mapped(struct folio *folio) * folio. We cannot rely on folio->swap as there is no guarantee that it has * been initialized. Used for calling arch_swap_restore() */ -static inline swp_entry_t folio_swap(swp_entry_t entry, struct folio *folio) +static inline swp_entry_t folio_swap(swp_entry_t entry, + const struct folio *folio) { swp_entry_t swap = { .val = ALIGN_DOWN(entry.val, folio_nr_pages(folio)), @@ -90,7 +91,7 @@ static inline swp_entry_t folio_swap(swp_entry_t entry, struct folio *folio) return swap; } -static inline void *folio_raw_mapping(struct folio *folio) +static inline void *folio_raw_mapping(const struct folio *folio) { unsigned long mapping = (unsigned long)folio->mapping; diff --git a/mm/rmap.c b/mm/rmap.c index 3746a55310183..d52759aa3ff77 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1134,7 +1134,7 @@ int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff, return page_vma_mkclean_one(&pvmw); } -int folio_total_mapcount(struct folio *folio) +int folio_total_mapcount(const struct folio *folio) { int mapcount = folio_entire_mapcount(folio); int nr_pages; From 51718e25c53f58fe9fa515f219367632cce4914d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 17:10:29 +0000 Subject: [PATCH 0656/1702] mm: convert arch_clear_hugepage_flags to take a folio All implementations that aren't no-ops just set a bit in the flags, and we want to use the folio flags rather than the page flags for that. Rename it to arch_clear_hugetlb_flags() while we're touching it so nobody thinks it's used for THP. [willy@infradead.org: fix arm64 build] Link: https://lkml.kernel.org/r/ZgQvNKGdlDkwhQEX@casper.infradead.org Link: https://lkml.kernel.org/r/20240326171045.410737-8-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- arch/arm/include/asm/hugetlb.h | 6 +++--- arch/arm64/include/asm/hugetlb.h | 6 +++--- arch/riscv/include/asm/hugetlb.h | 6 +++--- arch/s390/include/asm/hugetlb.h | 6 +++--- arch/sh/include/asm/hugetlb.h | 6 +++--- include/linux/hugetlb.h | 6 +++--- mm/hugetlb.c | 4 ++-- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/arch/arm/include/asm/hugetlb.h b/arch/arm/include/asm/hugetlb.h index a3a82b7158d4c..b766c4b373f64 100644 --- a/arch/arm/include/asm/hugetlb.h +++ b/arch/arm/include/asm/hugetlb.h @@ -15,10 +15,10 @@ #include #include -static inline void arch_clear_hugepage_flags(struct page *page) +static inline void arch_clear_hugetlb_flags(struct folio *folio) { - clear_bit(PG_dcache_clean, &page->flags); + clear_bit(PG_dcache_clean, &folio->flags); } -#define arch_clear_hugepage_flags arch_clear_hugepage_flags +#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags #endif /* _ASM_ARM_HUGETLB_H */ diff --git a/arch/arm64/include/asm/hugetlb.h b/arch/arm64/include/asm/hugetlb.h index 2ddc33d93b13b..3954cbd2ff56b 100644 --- a/arch/arm64/include/asm/hugetlb.h +++ b/arch/arm64/include/asm/hugetlb.h @@ -18,11 +18,11 @@ extern bool arch_hugetlb_migration_supported(struct hstate *h); #endif -static inline void arch_clear_hugepage_flags(struct page *page) +static inline void arch_clear_hugetlb_flags(struct folio *folio) { - clear_bit(PG_dcache_clean, &page->flags); + clear_bit(PG_dcache_clean, &folio->flags); } -#define arch_clear_hugepage_flags arch_clear_hugepage_flags +#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags); #define arch_make_huge_pte arch_make_huge_pte diff --git a/arch/riscv/include/asm/hugetlb.h b/arch/riscv/include/asm/hugetlb.h index 22deb7a2a6ec4..b1ce97a9dbfce 100644 --- a/arch/riscv/include/asm/hugetlb.h +++ b/arch/riscv/include/asm/hugetlb.h @@ -5,11 +5,11 @@ #include #include -static inline void arch_clear_hugepage_flags(struct page *page) +static inline void arch_clear_hugetlb_flags(struct folio *folio) { - clear_bit(PG_dcache_clean, &page->flags); + clear_bit(PG_dcache_clean, &folio->flags); } -#define arch_clear_hugepage_flags arch_clear_hugepage_flags +#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags #ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION bool arch_hugetlb_migration_supported(struct hstate *h); diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h index deb198a610395..ce5f4fe8be4dc 100644 --- a/arch/s390/include/asm/hugetlb.h +++ b/arch/s390/include/asm/hugetlb.h @@ -39,11 +39,11 @@ static inline int prepare_hugepage_range(struct file *file, return 0; } -static inline void arch_clear_hugepage_flags(struct page *page) +static inline void arch_clear_hugetlb_flags(struct folio *folio) { - clear_bit(PG_arch_1, &page->flags); + clear_bit(PG_arch_1, &folio->flags); } -#define arch_clear_hugepage_flags arch_clear_hugepage_flags +#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long sz) diff --git a/arch/sh/include/asm/hugetlb.h b/arch/sh/include/asm/hugetlb.h index 4d3ba39e681c4..75028bd568ba5 100644 --- a/arch/sh/include/asm/hugetlb.h +++ b/arch/sh/include/asm/hugetlb.h @@ -27,11 +27,11 @@ static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, return *ptep; } -static inline void arch_clear_hugepage_flags(struct page *page) +static inline void arch_clear_hugetlb_flags(struct folio *folio) { - clear_bit(PG_dcache_clean, &page->flags); + clear_bit(PG_dcache_clean, &folio->flags); } -#define arch_clear_hugepage_flags arch_clear_hugepage_flags +#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags #include diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index d748628efc5ec..03fc6d6250680 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -836,9 +836,9 @@ static inline int is_hugepage_only_range(struct mm_struct *mm, #define is_hugepage_only_range is_hugepage_only_range #endif -#ifndef arch_clear_hugepage_flags -static inline void arch_clear_hugepage_flags(struct page *page) { } -#define arch_clear_hugepage_flags arch_clear_hugepage_flags +#ifndef arch_clear_hugetlb_flags +static inline void arch_clear_hugetlb_flags(struct folio *folio) { } +#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags #endif #ifndef arch_make_huge_pte diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 2e984554ec633..88bb38df1701b 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1726,7 +1726,7 @@ static void add_hugetlb_folio(struct hstate *h, struct folio *folio, */ return; - arch_clear_hugepage_flags(&folio->page); + arch_clear_hugetlb_flags(folio); enqueue_hugetlb_folio(h, folio); } @@ -2024,7 +2024,7 @@ void free_huge_folio(struct folio *folio) spin_unlock_irqrestore(&hugetlb_lock, flags); update_and_free_hugetlb_folio(h, folio, true); } else { - arch_clear_hugepage_flags(&folio->page); + arch_clear_hugetlb_flags(folio); enqueue_hugetlb_folio(h, folio); spin_unlock_irqrestore(&hugetlb_lock, flags); } From 5e0debe012f31d6409b840920d0c37396d32ce30 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 17:10:30 +0000 Subject: [PATCH 0657/1702] slub: remove use of page->flags Use slub->__page_flags instead. We can also remove the assertion that it's not a tail page as struct slab never points to a tail page. Link: https://lkml.kernel.org/r/20240326171045.410737-9-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/slub.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 6437746be03d8..e09d8260adff5 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -616,18 +616,12 @@ slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) */ static __always_inline void slab_lock(struct slab *slab) { - struct page *page = slab_page(slab); - - VM_BUG_ON_PAGE(PageTail(page), page); - bit_spin_lock(PG_locked, &page->flags); + bit_spin_lock(PG_locked, &slab->__page_flags); } static __always_inline void slab_unlock(struct slab *slab) { - struct page *page = slab_page(slab); - - VM_BUG_ON_PAGE(PageTail(page), page); - bit_spin_unlock(PG_locked, &page->flags); + bit_spin_unlock(PG_locked, &slab->__page_flags); } static inline bool From 4dc7d37370951fe86216f03a4e0a6909f9b90a8c Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 17:10:31 +0000 Subject: [PATCH 0658/1702] remove references to page->flags in documentation Mostly rewording, but remove entirely the copy of page_fixed_fake_head() in the documentation; we can refer people to the actual source if necessary. Link: https://lkml.kernel.org/r/20240326171045.410737-10-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- .../admin-guide/cgroup-v1/memory.rst | 4 ++-- Documentation/mm/vmemmap_dedup.rst | 22 +------------------ .../translations/zh_CN/core-api/cachetlb.rst | 2 +- mm/migrate.c | 2 +- mm/rmap.c | 4 ++-- 5 files changed, 7 insertions(+), 27 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index ca7d9402f6be1..46110e6a31bbd 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -300,14 +300,14 @@ When oom event notifier is registered, event will be delivered. Lock order is as follows:: - Page lock (PG_locked bit of page->flags) + folio_lock mm->page_table_lock or split pte_lock folio_memcg_lock (memcg->move_lock) mapping->i_pages lock lruvec->lru_lock. Per-node-per-memcgroup LRU (cgroup's private LRU) is guarded by -lruvec->lru_lock; PG_lru bit of page->flags is cleared before +lruvec->lru_lock; the folio LRU flag is cleared before isolating a page from its LRU under lruvec->lru_lock. .. _cgroup-v1-memory-kernel-extension: diff --git a/Documentation/mm/vmemmap_dedup.rst b/Documentation/mm/vmemmap_dedup.rst index 593ede6d314ba..b4a55b6569fa5 100644 --- a/Documentation/mm/vmemmap_dedup.rst +++ b/Documentation/mm/vmemmap_dedup.rst @@ -180,27 +180,7 @@ this correctly. There is only **one** head ``struct page``, the tail ``struct page`` with ``PG_head`` are fake head ``struct page``. We need an approach to distinguish between those two different types of ``struct page`` so that ``compound_head()`` can return the real head ``struct page`` when the -parameter is the tail ``struct page`` but with ``PG_head``. The following code -snippet describes how to distinguish between real and fake head ``struct page``. - -.. code-block:: c - - if (test_bit(PG_head, &page->flags)) { - unsigned long head = READ_ONCE(page[1].compound_head); - - if (head & 1) { - if (head == (unsigned long)page + 1) - /* head struct page */ - else - /* tail struct page */ - } else { - /* head struct page */ - } - } - -We can safely access the field of the **page[1]** with ``PG_head`` because the -page is a compound page composed with at least two contiguous pages. -The implementation refers to ``page_fixed_fake_head()``. +parameter is the tail ``struct page`` but with ``PG_head``. Device DAX ========== diff --git a/Documentation/translations/zh_CN/core-api/cachetlb.rst b/Documentation/translations/zh_CN/core-api/cachetlb.rst index b4a76ec75daa5..64295c61d1c13 100644 --- a/Documentation/translations/zh_CN/core-api/cachetlb.rst +++ b/Documentation/translations/zh_CN/core-api/cachetlb.rst @@ -260,7 +260,7 @@ HyperSparc cpu就是这样一个具有这种属性的cpu。 如果D-cache别名不是一个问题，这个程序可以简单地定义为该架构上 的nop。 - 在page->flags (PG_arch_1)中有一个位是“架构私有”。内核保证， + 在folio->flags (PG_arch_1)中有一个位是“架构私有”。内核保证， 对于分页缓存的页面，当这样的页面第一次进入分页缓存时，它将清除 这个位。 diff --git a/mm/migrate.c b/mm/migrate.c index a31aa75d223d0..285072bca29ce 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -113,7 +113,7 @@ bool isolate_movable_page(struct page *page, isolate_mode_t mode) if (!mops->isolate_page(&folio->page, mode)) goto out_no_isolated; - /* Driver shouldn't use PG_isolated bit of page->flags */ + /* Driver shouldn't use the isolated flag */ WARN_ON_ONCE(folio_test_isolated(folio)); folio_set_isolated(folio); folio_unlock(folio); diff --git a/mm/rmap.c b/mm/rmap.c index d52759aa3ff77..5ee9e338d09b5 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -23,7 +23,7 @@ * inode->i_rwsem (while writing or truncating, not reading or faulting) * mm->mmap_lock * mapping->invalidate_lock (in filemap_fault) - * page->flags PG_locked (lock_page) + * folio_lock * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share, see hugetlbfs below) * vma_start_write * mapping->i_mmap_rwsem @@ -50,7 +50,7 @@ * hugetlb_fault_mutex (hugetlbfs specific page fault mutex) * vma_lock (hugetlb specific lock for pmd_sharing) * mapping->i_mmap_rwsem (also used for hugetlb pmd sharing) - * page->flags PG_locked (lock_page) + * folio_lock */ #include From dee3d0bef2b00772be430425832ead6aa9d707f9 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 17:10:32 +0000 Subject: [PATCH 0659/1702] proc: rewrite stable_page_flags() Reduce the usage of PageFlag tests and reduce the number of compound_head() calls. For multi-page folios, we'll now show all pages as having the flags that apply to them, e.g. if it's dirty, all pages will have the dirty flag set instead of just the head page. The mapped flag is still per page, as is the hwpoison flag. [willy@infradead.org: fix up some bits vs masks] Link: https://lkml.kernel.org/r/20240403173112.1450721-1-willy@infradead.org [willy@infradead.org: fix warnings] Link: https://lkml.kernel.org/r/ZhBPtCYfSuFuUMEz@casper.infradead.org Link: https://lkml.kernel.org/r/20240326171045.410737-11-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Svetly Todorov Signed-off-by: Andrew Morton --- fs/proc/page.c | 69 ++++++++++++++++++---------------- include/linux/huge_mm.h | 4 +- include/linux/page-flags.h | 2 +- tools/cgroup/memcg_slabinfo.py | 5 +-- 4 files changed, 42 insertions(+), 38 deletions(-) diff --git a/fs/proc/page.c b/fs/proc/page.c index 9223856c934b4..05120263af2a4 100644 --- a/fs/proc/page.c +++ b/fs/proc/page.c @@ -107,10 +107,13 @@ static inline u64 kpf_copy_bit(u64 kflags, int ubit, int kbit) return ((kflags >> kbit) & 1) << ubit; } -u64 stable_page_flags(struct page *page) +u64 stable_page_flags(const struct page *page) { - u64 k; - u64 u; + const struct folio *folio; + unsigned long k; + unsigned long mapping; + bool is_anon; + u64 u = 0; /* * pseudo flag: KPF_NOPAGE @@ -118,49 +121,47 @@ u64 stable_page_flags(struct page *page) */ if (!page) return 1 << KPF_NOPAGE; + folio = page_folio(page); - k = page->flags; - u = 0; + k = folio->flags; + mapping = (unsigned long)folio->mapping; + is_anon = mapping & PAGE_MAPPING_ANON; /* * pseudo flags for the well known (anonymous) memory mapped pages */ if (page_mapped(page)) u |= 1 << KPF_MMAP; - if (PageAnon(page)) + if (is_anon) { u |= 1 << KPF_ANON; - if (PageKsm(page)) - u |= 1 << KPF_KSM; + if (mapping & PAGE_MAPPING_KSM) + u |= 1 << KPF_KSM; + } /* * compound pages: export both head/tail info * they together define a compound page's start/end pos and order */ - if (PageHead(page)) - u |= 1 << KPF_COMPOUND_HEAD; - if (PageTail(page)) + if (page == &folio->page) + u |= kpf_copy_bit(k, KPF_COMPOUND_HEAD, PG_head); + else u |= 1 << KPF_COMPOUND_TAIL; - if (PageHuge(page)) + if (folio_test_hugetlb(folio)) u |= 1 << KPF_HUGE; /* - * PageTransCompound can be true for non-huge compound pages (slab - * pages or pages allocated by drivers with __GFP_COMP) because it - * just checks PG_head/PG_tail, so we need to check PageLRU/PageAnon + * We need to check PageLRU/PageAnon * to make sure a given page is a thp, not a non-huge compound page. */ - else if (PageTransCompound(page)) { - struct page *head = compound_head(page); - - if (PageLRU(head) || PageAnon(head)) + else if (folio_test_large(folio)) { + if ((k & (1 << PG_lru)) || is_anon) u |= 1 << KPF_THP; - else if (is_huge_zero_page(head)) { + else if (is_huge_zero_page(&folio->page)) { u |= 1 << KPF_ZERO_PAGE; u |= 1 << KPF_THP; } } else if (is_zero_pfn(page_to_pfn(page))) u |= 1 << KPF_ZERO_PAGE; - /* * Caveats on high order pages: PG_buddy and PG_slab will only be set * on the head page. @@ -174,16 +175,17 @@ u64 stable_page_flags(struct page *page) u |= 1 << KPF_OFFLINE; if (PageTable(page)) u |= 1 << KPF_PGTABLE; + if (folio_test_slab(folio)) + u |= 1 << KPF_SLAB; - if (page_is_idle(page)) +#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT) + u |= kpf_copy_bit(k, KPF_IDLE, PG_idle); +#else + if (folio_test_idle(folio)) u |= 1 << KPF_IDLE; +#endif u |= kpf_copy_bit(k, KPF_LOCKED, PG_locked); - - u |= kpf_copy_bit(k, KPF_SLAB, PG_slab); - if (PageTail(page) && PageSlab(page)) - u |= 1 << KPF_SLAB; - u |= kpf_copy_bit(k, KPF_ERROR, PG_error); u |= kpf_copy_bit(k, KPF_DIRTY, PG_dirty); u |= kpf_copy_bit(k, KPF_UPTODATE, PG_uptodate); @@ -194,7 +196,8 @@ u64 stable_page_flags(struct page *page) u |= kpf_copy_bit(k, KPF_ACTIVE, PG_active); u |= kpf_copy_bit(k, KPF_RECLAIM, PG_reclaim); - if (PageSwapCache(page)) +#define SWAPCACHE ((1 << PG_swapbacked) | (1 << PG_swapcache)) + if ((k & SWAPCACHE) == SWAPCACHE) u |= 1 << KPF_SWAPCACHE; u |= kpf_copy_bit(k, KPF_SWAPBACKED, PG_swapbacked); @@ -202,7 +205,10 @@ u64 stable_page_flags(struct page *page) u |= kpf_copy_bit(k, KPF_MLOCKED, PG_mlocked); #ifdef CONFIG_MEMORY_FAILURE - u |= kpf_copy_bit(k, KPF_HWPOISON, PG_hwpoison); + if (u & (1 << KPF_HUGE)) + u |= kpf_copy_bit(k, KPF_HWPOISON, PG_hwpoison); + else + u |= kpf_copy_bit(page->flags, KPF_HWPOISON, PG_hwpoison); #endif #ifdef CONFIG_ARCH_USES_PG_UNCACHED @@ -228,7 +234,6 @@ static ssize_t kpageflags_read(struct file *file, char __user *buf, { const unsigned long max_dump_pfn = get_max_dump_pfn(); u64 __user *out = (u64 __user *)buf; - struct page *ppage; unsigned long src = *ppos; unsigned long pfn; ssize_t ret = 0; @@ -245,9 +250,9 @@ static ssize_t kpageflags_read(struct file *file, char __user *buf, * TODO: ZONE_DEVICE support requires to identify * memmaps that were actually initialized. */ - ppage = pfn_to_online_page(pfn); + struct page *page = pfn_to_online_page(pfn); - if (put_user(stable_page_flags(ppage), out)) { + if (put_user(stable_page_flags(page), out)) { ret = -EFAULT; break; } diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 7576025db55df..1540a1481daf5 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -351,7 +351,7 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf); extern struct page *huge_zero_page; extern unsigned long huge_zero_pfn; -static inline bool is_huge_zero_page(struct page *page) +static inline bool is_huge_zero_page(const struct page *page) { return READ_ONCE(huge_zero_page) == page; } @@ -480,7 +480,7 @@ static inline vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) return 0; } -static inline bool is_huge_zero_page(struct page *page) +static inline bool is_huge_zero_page(const struct page *page) { return false; } diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index eaecf544039fe..888353c209c03 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -734,7 +734,7 @@ static __always_inline bool PageKsm(const struct page *page) TESTPAGEFLAG_FALSE(Ksm, ksm) #endif -u64 stable_page_flags(struct page *page); +u64 stable_page_flags(const struct page *page); /** * folio_xor_flags_has_waiters - Change some folio flags. diff --git a/tools/cgroup/memcg_slabinfo.py b/tools/cgroup/memcg_slabinfo.py index 1d3a90d93fe26..270c28a0d0980 100644 --- a/tools/cgroup/memcg_slabinfo.py +++ b/tools/cgroup/memcg_slabinfo.py @@ -146,12 +146,11 @@ def detect_kernel_config(): def for_each_slab(prog): - PGSlab = 1 << prog.constant('PG_slab') - PGHead = 1 << prog.constant('PG_head') + PGSlab = ~prog.constant('PG_slab') for page in for_each_page(prog): try: - if page.flags.value_() & PGSlab: + if page.page_type.value_() == PGSlab: yield cast('struct slab *', page) except FaultError: pass From 9f9796b413d3c417f34cae427c4e47bfdd3a7454 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Tue, 26 Mar 2024 11:37:38 +0100 Subject: [PATCH 0660/1702] mm, slab: move memcg charging to post-alloc hook Patch series "memcg_kmem hooks refactoring", v3. This patch (of 2): The MEMCG_KMEM integration with slab currently relies on two hooks during allocation. memcg_slab_pre_alloc_hook() determines the objcg and charges it, and memcg_slab_post_alloc_hook() assigns the objcg pointer to the allocated object(s). As Linus pointed out, this is unnecessarily complex. Failing to charge due to memcg limits should be rare, so we can optimistically allocate the object(s) and do the charging together with assigning the objcg pointer in a single post_alloc hook. In the rare case the charging fails, we can free the object(s) back. This simplifies the code (no need to pass around the objcg pointer) and potentially allows to separate charging from allocation in cases where it's common that the allocation would be immediately freed, and the memcg handling overhead could be saved. [vbabka@suse.cz: fix call to memcg_alloc_abort_single()] Link: https://lkml.kernel.org/r/4af50be2-4109-45e5-8a36-2136252a635e@suse.cz [roman.gushchin@linux.dev: comment fixup] Link: https://lkml.kernel.org/r/Zg2LsNm6twOmG69l@P9FQF9L96D.corp.robot.car Link: https://lkml.kernel.org/r/20240326-slab-memcg-v3-0-d85d2563287a@suse.cz Link: https://lkml.kernel.org/r/20240326-slab-memcg-v3-1-d85d2563287a@suse.cz Signed-off-by: Vlastimil Babka Signed-off-by: Roman Gushchin Suggested-by: Linus Torvalds Link: https://lore.kernel.org/all/CAHk-=whYOOdM7jWy5jdrAm8LxcgCMFyk2bt8fYYvZzM4U-zAQA@mail.gmail.com/ Reviewed-by: Roman Gushchin Reviewed-by: Chengming Zhou Cc: Al Viro Cc: Christian Brauner Cc: Christoph Lameter Cc: Chuck Lever Cc: David Rientjes Cc: Hyeonggon Yoo <42.hyeyoo@gmail.com> Cc: Jan Kara Cc: Jeff Layton Cc: Johannes Weiner Cc: Joonsoo Kim Cc: Josh Poimboeuf Cc: Kees Cook Cc: Michal Hocko Cc: Muchun Song Cc: Pekka Enberg Cc: Shakeel Butt Cc: Aishwarya TCV Signed-off-by: Andrew Morton --- mm/memcontrol.c | 2 +- mm/slub.c | 180 +++++++++++++++++++++--------------------------- 2 files changed, 78 insertions(+), 104 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 45dd209012827..a2aa2ff6cf778 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -350,7 +350,7 @@ static void memcg_reparent_objcgs(struct mem_cgroup *memcg, /* * A lot of the calls to the cache allocation functions are expected to be - * inlined by the compiler. Since the calls to memcg_slab_pre_alloc_hook() are + * inlined by the compiler. Since the calls to memcg_slab_post_alloc_hook() are * conditional to this static branch, we'll have to allow modules that does * kmem_cache_alloc and the such to see this symbol as well */ diff --git a/mm/slub.c b/mm/slub.c index e09d8260adff5..f83efdbe15b7e 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2092,23 +2092,36 @@ static inline size_t obj_full_size(struct kmem_cache *s) return s->size + sizeof(struct obj_cgroup *); } -/* - * Returns false if the allocation should fail. - */ -static bool __memcg_slab_pre_alloc_hook(struct kmem_cache *s, - struct list_lru *lru, - struct obj_cgroup **objcgp, - size_t objects, gfp_t flags) +static bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, + struct list_lru *lru, + gfp_t flags, size_t size, + void **p) { + struct obj_cgroup *objcg; + struct slab *slab; + unsigned long off; + size_t i; + /* * The obtained objcg pointer is safe to use within the current scope, * defined by current task or set_active_memcg() pair. * obj_cgroup_get() is used to get a permanent reference. */ - struct obj_cgroup *objcg = current_obj_cgroup(); + objcg = current_obj_cgroup(); if (!objcg) return true; + /* + * slab_alloc_node() avoids the NULL check, so we might be called with a + * single NULL object. kmem_cache_alloc_bulk() aborts if it can't fill + * the whole requested size. + * return success as there's nothing to free back + */ + if (unlikely(*p == NULL)) + return true; + + flags &= gfp_allowed_mask; + if (lru) { int ret; struct mem_cgroup *memcg; @@ -2121,71 +2134,51 @@ static bool __memcg_slab_pre_alloc_hook(struct kmem_cache *s, return false; } - if (obj_cgroup_charge(objcg, flags, objects * obj_full_size(s))) + if (obj_cgroup_charge(objcg, flags, size * obj_full_size(s))) return false; - *objcgp = objcg; + for (i = 0; i < size; i++) { + slab = virt_to_slab(p[i]); + + if (!slab_obj_exts(slab) && + alloc_slab_obj_exts(slab, s, flags, false)) { + obj_cgroup_uncharge(objcg, obj_full_size(s)); + continue; + } + + off = obj_to_index(s, slab, p[i]); + obj_cgroup_get(objcg); + slab_obj_exts(slab)[off].objcg = objcg; + mod_objcg_state(objcg, slab_pgdat(slab), + cache_vmstat_idx(s), obj_full_size(s)); + } + return true; } -/* - * Returns false if the allocation should fail. - */ +static void memcg_alloc_abort_single(struct kmem_cache *s, void *object); + static __fastpath_inline -bool memcg_slab_pre_alloc_hook(struct kmem_cache *s, struct list_lru *lru, - struct obj_cgroup **objcgp, size_t objects, - gfp_t flags) +bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, + gfp_t flags, size_t size, void **p) { - if (!memcg_kmem_online()) + if (likely(!memcg_kmem_online())) return true; if (likely(!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT))) return true; - return likely(__memcg_slab_pre_alloc_hook(s, lru, objcgp, objects, - flags)); -} - -static void __memcg_slab_post_alloc_hook(struct kmem_cache *s, - struct obj_cgroup *objcg, - gfp_t flags, size_t size, - void **p) -{ - struct slab *slab; - unsigned long off; - size_t i; - - flags &= gfp_allowed_mask; - - for (i = 0; i < size; i++) { - if (likely(p[i])) { - slab = virt_to_slab(p[i]); - - if (!slab_obj_exts(slab) && - alloc_slab_obj_exts(slab, s, flags, false)) { - obj_cgroup_uncharge(objcg, obj_full_size(s)); - continue; - } + if (likely(__memcg_slab_post_alloc_hook(s, lru, flags, size, p))) + return true; - off = obj_to_index(s, slab, p[i]); - obj_cgroup_get(objcg); - slab_obj_exts(slab)[off].objcg = objcg; - mod_objcg_state(objcg, slab_pgdat(slab), - cache_vmstat_idx(s), obj_full_size(s)); - } else { - obj_cgroup_uncharge(objcg, obj_full_size(s)); - } + if (likely(size == 1)) { + memcg_alloc_abort_single(s, *p); + *p = NULL; + } else { + kmem_cache_free_bulk(s, size, p); } -} - -static __fastpath_inline -void memcg_slab_post_alloc_hook(struct kmem_cache *s, struct obj_cgroup *objcg, - gfp_t flags, size_t size, void **p) -{ - if (likely(!memcg_kmem_online() || !objcg)) - return; - return __memcg_slab_post_alloc_hook(s, objcg, flags, size, p); + return false; } static void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, @@ -2224,40 +2217,19 @@ void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, __memcg_slab_free_hook(s, slab, p, objects, obj_exts); } - -static inline -void memcg_slab_alloc_error_hook(struct kmem_cache *s, int objects, - struct obj_cgroup *objcg) -{ - if (objcg) - obj_cgroup_uncharge(objcg, objects * obj_full_size(s)); -} #else /* CONFIG_MEMCG_KMEM */ -static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s, - struct list_lru *lru, - struct obj_cgroup **objcgp, - size_t objects, gfp_t flags) -{ - return true; -} - -static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s, - struct obj_cgroup *objcg, +static inline bool memcg_slab_post_alloc_hook(struct kmem_cache *s, + struct list_lru *lru, gfp_t flags, size_t size, void **p) { + return true; } static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, int objects) { } - -static inline -void memcg_slab_alloc_error_hook(struct kmem_cache *s, int objects, - struct obj_cgroup *objcg) -{ -} #endif /* CONFIG_MEMCG_KMEM */ /* @@ -3937,10 +3909,7 @@ noinline int should_failslab(struct kmem_cache *s, gfp_t gfpflags) ALLOW_ERROR_INJECTION(should_failslab, ERRNO); static __fastpath_inline -struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, - struct list_lru *lru, - struct obj_cgroup **objcgp, - size_t size, gfp_t flags) +struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags) { flags &= gfp_allowed_mask; @@ -3949,14 +3918,11 @@ struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, if (unlikely(should_failslab(s, flags))) return NULL; - if (unlikely(!memcg_slab_pre_alloc_hook(s, lru, objcgp, size, flags))) - return NULL; - return s; } static __fastpath_inline -void slab_post_alloc_hook(struct kmem_cache *s, struct obj_cgroup *objcg, +bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, gfp_t flags, size_t size, void **p, bool init, unsigned int orig_size) { @@ -4018,7 +3984,7 @@ void slab_post_alloc_hook(struct kmem_cache *s, struct obj_cgroup *objcg, } } - memcg_slab_post_alloc_hook(s, objcg, flags, size, p); + return memcg_slab_post_alloc_hook(s, lru, flags, size, p); } /* @@ -4035,10 +4001,9 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list gfp_t gfpflags, int node, unsigned long addr, size_t orig_size) { void *object; - struct obj_cgroup *objcg = NULL; bool init = false; - s = slab_pre_alloc_hook(s, lru, &objcg, 1, gfpflags); + s = slab_pre_alloc_hook(s, gfpflags); if (unlikely(!s)) return NULL; @@ -4055,8 +4020,10 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list /* * When init equals 'true', like for kzalloc() family, only * @orig_size bytes might be zeroed instead of s->object_size + * In case this fails due to memcg_slab_post_alloc_hook(), + * object is set to NULL */ - slab_post_alloc_hook(s, objcg, gfpflags, 1, &object, init, orig_size); + slab_post_alloc_hook(s, lru, gfpflags, 1, &object, init, orig_size); return object; } @@ -4496,6 +4463,16 @@ void slab_free(struct kmem_cache *s, struct slab *slab, void *object, do_slab_free(s, slab, object, object, 1, addr); } +#ifdef CONFIG_MEMCG_KMEM +/* Do not inline the rare memcg charging failed path into the allocation path */ +static noinline +void memcg_alloc_abort_single(struct kmem_cache *s, void *object) +{ + if (likely(slab_free_hook(s, object, slab_want_init_on_free(s)))) + do_slab_free(s, virt_to_slab(object), object, object, 1, _RET_IP_); +} +#endif + static __fastpath_inline void slab_free_bulk(struct kmem_cache *s, struct slab *slab, void *head, void *tail, void **p, int cnt, unsigned long addr) @@ -4832,29 +4809,26 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, void **p) { int i; - struct obj_cgroup *objcg = NULL; if (!size) return 0; - /* memcg and kmem_cache debug support */ - s = slab_pre_alloc_hook(s, NULL, &objcg, size, flags); + s = slab_pre_alloc_hook(s, flags); if (unlikely(!s)) return 0; i = __kmem_cache_alloc_bulk(s, flags, size, p); + if (unlikely(i == 0)) + return 0; /* * memcg and kmem_cache debug support and memory initialization. * Done outside of the IRQ disabled fastpath loop. */ - if (likely(i != 0)) { - slab_post_alloc_hook(s, objcg, flags, size, p, - slab_want_init_on_alloc(flags, s), s->object_size); - } else { - memcg_slab_alloc_error_hook(s, size, objcg); + if (unlikely(!slab_post_alloc_hook(s, NULL, flags, size, p, + slab_want_init_on_alloc(flags, s), s->object_size))) { + return 0; } - return i; } EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof); From e6100a4590bfc0a91c8e882f73538c9c0d987a46 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Tue, 26 Mar 2024 11:37:39 +0100 Subject: [PATCH 0661/1702] mm, slab: move slab_memcg hooks to mm/memcontrol.c The hooks make multiple calls to functions in mm/memcontrol.c, including to th current_obj_cgroup() marked __always_inline. It might be faster to make a single call to the hook in mm/memcontrol.c instead. The hooks also don't use almost anything from mm/slub.c. obj_full_size() can move with the hooks and cache_vmstat_idx() to the internal mm/slab.h Link: https://lkml.kernel.org/r/20240326-slab-memcg-v3-2-d85d2563287a@suse.cz Signed-off-by: Vlastimil Babka Reviewed-by: Roman Gushchin Cc: Al Viro Cc: Chengming Zhou Cc: Christian Brauner Cc: Christoph Lameter Cc: Chuck Lever Cc: David Rientjes Cc: Hyeonggon Yoo <42.hyeyoo@gmail.com> Cc: Jan Kara Cc: Jeff Layton Cc: Johannes Weiner Cc: Joonsoo Kim Cc: Josh Poimboeuf Cc: Kees Cook Cc: Linus Torvalds Cc: Michal Hocko Cc: Muchun Song Cc: Pekka Enberg Cc: Shakeel Butt Signed-off-by: Andrew Morton --- mm/memcontrol.c | 90 ++++++++++++++++++++++++++++++++++++++++++ mm/slab.h | 13 ++++++ mm/slub.c | 103 +----------------------------------------------- 3 files changed, 105 insertions(+), 101 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index a2aa2ff6cf778..29fa5b17a1ef5 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3556,6 +3556,96 @@ void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size) refill_obj_stock(objcg, size, true); } +static inline size_t obj_full_size(struct kmem_cache *s) +{ + /* + * For each accounted object there is an extra space which is used + * to store obj_cgroup membership. Charge it too. + */ + return s->size + sizeof(struct obj_cgroup *); +} + +bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, + gfp_t flags, size_t size, void **p) +{ + struct obj_cgroup *objcg; + struct slab *slab; + unsigned long off; + size_t i; + + /* + * The obtained objcg pointer is safe to use within the current scope, + * defined by current task or set_active_memcg() pair. + * obj_cgroup_get() is used to get a permanent reference. + */ + objcg = current_obj_cgroup(); + if (!objcg) + return true; + + /* + * slab_alloc_node() avoids the NULL check, so we might be called with a + * single NULL object. kmem_cache_alloc_bulk() aborts if it can't fill + * the whole requested size. + * return success as there's nothing to free back + */ + if (unlikely(*p == NULL)) + return true; + + flags &= gfp_allowed_mask; + + if (lru) { + int ret; + struct mem_cgroup *memcg; + + memcg = get_mem_cgroup_from_objcg(objcg); + ret = memcg_list_lru_alloc(memcg, lru, flags); + css_put(&memcg->css); + + if (ret) + return false; + } + + if (obj_cgroup_charge(objcg, flags, size * obj_full_size(s))) + return false; + + for (i = 0; i < size; i++) { + slab = virt_to_slab(p[i]); + + if (!slab_obj_exts(slab) && + alloc_slab_obj_exts(slab, s, flags, false)) { + obj_cgroup_uncharge(objcg, obj_full_size(s)); + continue; + } + + off = obj_to_index(s, slab, p[i]); + obj_cgroup_get(objcg); + slab_obj_exts(slab)[off].objcg = objcg; + mod_objcg_state(objcg, slab_pgdat(slab), + cache_vmstat_idx(s), obj_full_size(s)); + } + + return true; +} + +void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, + void **p, int objects, struct slabobj_ext *obj_exts) +{ + for (int i = 0; i < objects; i++) { + struct obj_cgroup *objcg; + unsigned int off; + + off = obj_to_index(s, slab, p[i]); + objcg = obj_exts[off].objcg; + if (!objcg) + continue; + + obj_exts[off].objcg = NULL; + obj_cgroup_uncharge(objcg, obj_full_size(s)); + mod_objcg_state(objcg, slab_pgdat(slab), cache_vmstat_idx(s), + -obj_full_size(s)); + obj_cgroup_put(objcg); + } +} #endif /* CONFIG_MEMCG_KMEM */ /* diff --git a/mm/slab.h b/mm/slab.h index 1343bfa12cee9..411251b9bdd15 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -558,6 +558,9 @@ static inline struct slabobj_ext *slab_obj_exts(struct slab *slab) return (struct slabobj_ext *)(obj_exts & ~OBJEXTS_FLAGS_MASK); } +int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, + gfp_t gfp, bool new_slab); + #else /* CONFIG_SLAB_OBJ_EXT */ static inline struct slabobj_ext *slab_obj_exts(struct slab *slab) @@ -567,7 +570,17 @@ static inline struct slabobj_ext *slab_obj_exts(struct slab *slab) #endif /* CONFIG_SLAB_OBJ_EXT */ +static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s) +{ + return (s->flags & SLAB_RECLAIM_ACCOUNT) ? + NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B; +} + #ifdef CONFIG_MEMCG_KMEM +bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, + gfp_t flags, size_t size, void **p); +void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, + void **p, int objects, struct slabobj_ext *obj_exts); void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, enum node_stat_item idx, int nr); #endif diff --git a/mm/slub.c b/mm/slub.c index f83efdbe15b7e..3e33ff900d35e 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1859,12 +1859,6 @@ static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, #endif #endif /* CONFIG_SLUB_DEBUG */ -static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s) -{ - return (s->flags & SLAB_RECLAIM_ACCOUNT) ? - NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B; -} - #ifdef CONFIG_SLAB_OBJ_EXT #ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG @@ -1923,8 +1917,8 @@ static inline void handle_failed_objexts_alloc(unsigned long obj_exts, #define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | \ __GFP_ACCOUNT | __GFP_NOFAIL) -static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, - gfp_t gfp, bool new_slab) +int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, + gfp_t gfp, bool new_slab) { unsigned int objects = objs_per_slab(s, slab); unsigned long new_exts; @@ -2083,78 +2077,6 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, #endif /* CONFIG_SLAB_OBJ_EXT */ #ifdef CONFIG_MEMCG_KMEM -static inline size_t obj_full_size(struct kmem_cache *s) -{ - /* - * For each accounted object there is an extra space which is used - * to store obj_cgroup membership. Charge it too. - */ - return s->size + sizeof(struct obj_cgroup *); -} - -static bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, - struct list_lru *lru, - gfp_t flags, size_t size, - void **p) -{ - struct obj_cgroup *objcg; - struct slab *slab; - unsigned long off; - size_t i; - - /* - * The obtained objcg pointer is safe to use within the current scope, - * defined by current task or set_active_memcg() pair. - * obj_cgroup_get() is used to get a permanent reference. - */ - objcg = current_obj_cgroup(); - if (!objcg) - return true; - - /* - * slab_alloc_node() avoids the NULL check, so we might be called with a - * single NULL object. kmem_cache_alloc_bulk() aborts if it can't fill - * the whole requested size. - * return success as there's nothing to free back - */ - if (unlikely(*p == NULL)) - return true; - - flags &= gfp_allowed_mask; - - if (lru) { - int ret; - struct mem_cgroup *memcg; - - memcg = get_mem_cgroup_from_objcg(objcg); - ret = memcg_list_lru_alloc(memcg, lru, flags); - css_put(&memcg->css); - - if (ret) - return false; - } - - if (obj_cgroup_charge(objcg, flags, size * obj_full_size(s))) - return false; - - for (i = 0; i < size; i++) { - slab = virt_to_slab(p[i]); - - if (!slab_obj_exts(slab) && - alloc_slab_obj_exts(slab, s, flags, false)) { - obj_cgroup_uncharge(objcg, obj_full_size(s)); - continue; - } - - off = obj_to_index(s, slab, p[i]); - obj_cgroup_get(objcg); - slab_obj_exts(slab)[off].objcg = objcg; - mod_objcg_state(objcg, slab_pgdat(slab), - cache_vmstat_idx(s), obj_full_size(s)); - } - - return true; -} static void memcg_alloc_abort_single(struct kmem_cache *s, void *object); @@ -2181,27 +2103,6 @@ bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, return false; } -static void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, - void **p, int objects, - struct slabobj_ext *obj_exts) -{ - for (int i = 0; i < objects; i++) { - struct obj_cgroup *objcg; - unsigned int off; - - off = obj_to_index(s, slab, p[i]); - objcg = obj_exts[off].objcg; - if (!objcg) - continue; - - obj_exts[off].objcg = NULL; - obj_cgroup_uncharge(objcg, obj_full_size(s)); - mod_objcg_state(objcg, slab_pgdat(slab), cache_vmstat_idx(s), - -obj_full_size(s)); - obj_cgroup_put(objcg); - } -} - static __fastpath_inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, int objects) From 850ed20539a4cb4188ad664259f4c53206991d86 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Tue, 26 Mar 2024 14:11:27 +0800 Subject: [PATCH 0662/1702] mm: move array mem_section init code out of memory_present() Patch series "mm/init: minor clean up and improvement". These are all observed when going through code flow during mm init. This patch (of 7): When CONFIG_SPARSEMEM_EXTREME is enabled, mem_section need be initialized to point at a two-dimensional array, and its 1st dimension of length NR_SECTION_ROOTS will be dynamically allocated. Once the allocation is done, it's available for all nodes. So take the 1st dimension of mem_section initialization out of memory_present()(), and put it into memblocks_present() which is a more appripriate place. Link: https://lkml.kernel.org/r/20240326061134.1055295-1-bhe@redhat.com Link: https://lkml.kernel.org/r/20240326061134.1055295-2-bhe@redhat.com Signed-off-by: Baoquan He Cc: Mel Gorman Cc: "Mike Rapoport (IBM)" Signed-off-by: Andrew Morton --- mm/sparse.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/mm/sparse.c b/mm/sparse.c index aed0951b87fa0..46e88549d1a6a 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -226,19 +226,6 @@ static void __init memory_present(int nid, unsigned long start, unsigned long en { unsigned long pfn; -#ifdef CONFIG_SPARSEMEM_EXTREME - if (unlikely(!mem_section)) { - unsigned long size, align; - - size = sizeof(struct mem_section *) * NR_SECTION_ROOTS; - align = 1 << (INTERNODE_CACHE_SHIFT); - mem_section = memblock_alloc(size, align); - if (!mem_section) - panic("%s: Failed to allocate %lu bytes align=0x%lx\n", - __func__, size, align); - } -#endif - start &= PAGE_SECTION_MASK; mminit_validate_memmodel_limits(&start, &end); for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { @@ -267,6 +254,19 @@ static void __init memblocks_present(void) unsigned long start, end; int i, nid; +#ifdef CONFIG_SPARSEMEM_EXTREME + if (unlikely(!mem_section)) { + unsigned long size, align; + + size = sizeof(struct mem_section *) * NR_SECTION_ROOTS; + align = 1 << (INTERNODE_CACHE_SHIFT); + mem_section = memblock_alloc(size, align); + if (!mem_section) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, size, align); + } +#endif + for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) memory_present(nid, start, end); } From c091dd963f99a4ae1a5c31bff68e5c2ebd786b07 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Tue, 26 Mar 2024 14:11:28 +0800 Subject: [PATCH 0663/1702] mm/init: remove the unnecessary special treatment for memory-less node Because memory-less node's ->node_present_pages and its zone's ->present_pages are all 0, the judgement before calling node_set_state() to set N_MEMORY, N_HIGH_MEMORY, N_NORMAL_MEMORY for node is enough to skip memory-less node. The 'continue;' statement inside for_each_node() loop of free_area_init() is gilding the lily. Here, remove the special handling to make memory-less node share the same code flow as normal node. And also rephrase the code comments above the 'continue' statement and move them above above line 'if (pgdat->node_present_pages)'. [bhe@redhat.com: redo code comments, per Mike] Link: https://lkml.kernel.org/r/ZhYJAVQRYJSTKZng@MiWiFi-R3L-srv Link: https://lkml.kernel.org/r/20240326061134.1055295-3-bhe@redhat.com Signed-off-by: Baoquan He Cc: Mel Gorman Cc: "Mike Rapoport (IBM)" Signed-off-by: Andrew Morton --- mm/mm_init.c | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 0d32bcc301e23..0fe466abde438 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1835,28 +1835,23 @@ void __init free_area_init(unsigned long *max_zone_pfn) panic("Cannot allocate %zuB for node %d.\n", sizeof(*pgdat), nid); arch_refresh_nodedata(nid, pgdat); - free_area_init_node(nid); - - /* - * We do not want to confuse userspace by sysfs - * files/directories for node without any memory - * attached to it, so this node is not marked as - * N_MEMORY and not marked online so that no sysfs - * hierarchy will be created via register_one_node for - * it. The pgdat will get fully initialized by - * hotadd_init_pgdat() when memory is hotplugged into - * this node. - */ - continue; } pgdat = NODE_DATA(nid); free_area_init_node(nid); - /* Any memory on that node */ - if (pgdat->node_present_pages) + /* + * No sysfs hierarcy will be created via register_one_node() + *for memory-less node because here it's not marked as N_MEMORY + *and won't be set online later. The benefit is userspace + *program won't be confused by sysfs files/directories of + *memory-less node. The pgdat will get fully initialized by + *hotadd_init_pgdat() when memory is hotplugged into this node. + */ + if (pgdat->node_present_pages) { node_set_state(nid, N_MEMORY); - check_for_memory(pgdat); + check_for_memory(pgdat); + } } calc_nr_kernel_pages(); From b6dd94596f7fafd162da236b438e31b12d352187 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Tue, 26 Mar 2024 14:11:29 +0800 Subject: [PATCH 0664/1702] mm: make __absent_pages_in_range() as static It's only called in mm/mm_init.c now. Link: https://lkml.kernel.org/r/20240326061134.1055295-4-bhe@redhat.com Signed-off-by: Baoquan He Reviewed-by: Mike Rapoport (IBM) Cc: Mel Gorman Signed-off-by: Andrew Morton --- include/linux/mm.h | 2 -- mm/mm_init.c | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 9c0e441664d48..00a46a5d5c9d6 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3229,8 +3229,6 @@ static inline unsigned long get_num_physpages(void) */ void free_area_init(unsigned long *max_zone_pfn); unsigned long node_map_pfn_alignment(void); -unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn, - unsigned long end_pfn); extern unsigned long absent_pages_in_range(unsigned long start_pfn, unsigned long end_pfn); extern void get_pfn_range_for_nid(unsigned int nid, diff --git a/mm/mm_init.c b/mm/mm_init.c index 0fe466abde438..19c78b54fc8a5 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1144,7 +1144,7 @@ static void __init adjust_zone_range_for_zone_movable(int nid, * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, * then all holes in the requested range will be accounted for. */ -unsigned long __init __absent_pages_in_range(int nid, +static unsigned long __init __absent_pages_in_range(int nid, unsigned long range_start_pfn, unsigned long range_end_pfn) { From bb8ea62daa7cfd634d1a43815666948311074d0c Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Tue, 26 Mar 2024 14:11:30 +0800 Subject: [PATCH 0665/1702] mm/page_alloc.c: remove unneeded codes in !NUMA version of build_zonelists() When CONFIG_NUMA=n, MAX_NUMNODES is always 1 because Kconfig item NODES_SHIFT depends on NUMA. So in !NUMA version of build_zonelists(), no need to bother with the two for loop because code execution won't enter them ever. Here, remove those unneeded codes in !NUMA version of build_zonelists(). [bhe@redhat.com: remove unused locals] Link: https://lkml.kernel.org/r/ZgQL1WOf9K88nLpQ@MiWiFi-R3L-srv Link: https://lkml.kernel.org/r/20240326061134.1055295-5-bhe@redhat.com Signed-off-by: Baoquan He Cc: Mel Gorman Cc: "Mike Rapoport (IBM)" Signed-off-by: Andrew Morton --- mm/page_alloc.c | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 6a8593aef2977..d932f406c0c1a 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5280,37 +5280,13 @@ static void setup_min_slab_ratio(void); static void build_zonelists(pg_data_t *pgdat) { - int node, local_node; struct zoneref *zonerefs; int nr_zones; - local_node = pgdat->node_id; - zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; nr_zones = build_zonerefs_node(pgdat, zonerefs); zonerefs += nr_zones; - /* - * Now we build the zonelist so that it contains the zones - * of all the other nodes. - * We don't want to pressure a particular node, so when - * building the zones for node N, we make sure that the - * zones coming right after the local ones are those from - * node N+1 (modulo N) - */ - for (node = local_node + 1; node < MAX_NUMNODES; node++) { - if (!node_online(node)) - continue; - nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); - zonerefs += nr_zones; - } - for (node = 0; node < local_node; node++) { - if (!node_online(node)) - continue; - nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs); - zonerefs += nr_zones; - } - zonerefs->zone = NULL; zonerefs->zone_idx = 0; } From f55d3471b78a15e8ec1db88fe12456aad70600a4 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Tue, 26 Mar 2024 14:11:31 +0800 Subject: [PATCH 0666/1702] mm/mm_init.c: remove the outdated code comment above deferred_grow_zone() The noinline attribute has been taken off in commit 9420f89db2dd ("mm: move most of core MM initialization to mm/mm_init.c"). So remove the unneeded code comment above deferred_grow_zone(). And also remove the unneeded bracket in deferred_init_pages(). Link: https://lkml.kernel.org/r/20240326061134.1055295-6-bhe@redhat.com Signed-off-by: Baoquan He Reviewed-by: Mike Rapoport (IBM) Cc: Mel Gorman Signed-off-by: Andrew Morton --- mm/mm_init.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 19c78b54fc8a5..d01912b8a5973 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2014,7 +2014,7 @@ static unsigned long __init deferred_init_pages(struct zone *zone, __init_single_page(page, pfn, zid, nid); nr_pages++; } - return (nr_pages); + return nr_pages; } /* @@ -2216,10 +2216,6 @@ static int __init deferred_init_memmap(void *data) * Return true when zone was grown, otherwise return false. We return true even * when we grow less than requested, to let the caller decide if there are * enough pages to satisfy the allocation. - * - * Note: We use noinline because this function is needed only during boot, and - * it is called from a __ref function _deferred_grow_zone. This way we are - * making sure that it is not inlined into permanent text section. */ bool __init deferred_grow_zone(struct zone *zone, unsigned int order) { From 96a5c186efff9906c58fa24ab05d0cc3ee000a00 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Tue, 26 Mar 2024 14:11:32 +0800 Subject: [PATCH 0667/1702] mm/page_alloc.c: don't show protection in zone's ->lowmem_reserve[] for empty zone On one node, for lower zone's ->lowmem_reserve[], it will show how much memory is reserved in this lower zone to avoid excessive page allocation from the relevant higher zone's fallback allocation. However, currently lower zone's lowmem_reserve[] element will be filled even though the relevant higher zone is empty. That doesnt' make sense and can cause confusion. E.g on node 0 of one system as below, it has zone DMA/DMA32/NORMAL/MOVABLE/DEVICE, among them zone MOVABLE/DEVICE are the highest and both are empty. In zone DMA/DMA32's protection array, we can see that it has value for zone MOVABLE and DEVICE. Node 0, zone DMA ...... pages free 2816 boost 0 min 7 low 10 high 13 spanned 4095 present 3998 managed 3840 cma 0 protection: (0, 1582, 23716, 23716, 23716) ...... Node 0, zone DMA32 pages free 403269 boost 0 min 753 low 1158 high 1563 spanned 1044480 present 487039 managed 405070 cma 0 protection: (0, 0, 22134, 22134, 22134) ...... Node 0, zone Normal pages free 5423879 boost 0 min 10539 low 16205 high 21871 spanned 5767168 present 5767168 managed 5666438 cma 0 protection: (0, 0, 0, 0, 0) ...... Node 0, zone Movable pages free 0 boost 0 min 32 low 32 high 32 spanned 0 present 0 managed 0 cma 0 protection: (0, 0, 0, 0, 0) Node 0, zone Device pages free 0 boost 0 min 0 low 0 high 0 spanned 0 present 0 managed 0 cma 0 protection: (0, 0, 0, 0, 0) Here, clear out the element value in lower zone's ->lowmem_reserve[] if the relevant higher zone is empty. And also replace space with tab in _deferred_grow_zone() Link: https://lkml.kernel.org/r/20240326061134.1055295-7-bhe@redhat.com Signed-off-by: Baoquan He Cc: Mel Gorman Cc: "Mike Rapoport (IBM)" Signed-off-by: Andrew Morton --- mm/page_alloc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index d932f406c0c1a..875cec8c77089 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -315,7 +315,7 @@ static inline bool deferred_pages_enabled(void) static bool __ref _deferred_grow_zone(struct zone *zone, unsigned int order) { - return deferred_grow_zone(zone, order); + return deferred_grow_zone(zone, order); } #else static inline bool deferred_pages_enabled(void) @@ -5903,10 +5903,11 @@ static void setup_per_zone_lowmem_reserve(void) for (j = i + 1; j < MAX_NR_ZONES; j++) { struct zone *upper_zone = &pgdat->node_zones[j]; + bool empty = !zone_managed_pages(upper_zone); managed_pages += zone_managed_pages(upper_zone); - if (clear) + if (clear || empty) zone->lowmem_reserve[j] = 0; else zone->lowmem_reserve[j] = managed_pages / ratio; From 0aac45663a6d8803e8436dbaca884e0712970c7a Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Tue, 26 Mar 2024 14:11:33 +0800 Subject: [PATCH 0668/1702] mm/page_alloc.c: change the array-length to MIGRATE_PCPTYPES Earlier, in commit 1dd214b8f21c ("mm: page_alloc: avoid merging non-fallbackable pageblocks with others"), migrate type MIGRATE_CMA and MIGRATE_ISOLATE are removed from fallbacks list since they are never used. Later on, in commit ("aa02d3c174ab mm/page_alloc: reduce fallbacks to (MIGRATE_PCPTYPES - 1)"), the array column size is reduced to 'MIGRATE_PCPTYPES - 1'. In fact, the array row size need be reduced to MIGRATE_PCPTYPES too since it's only covering rows of the number MIGRATE_PCPTYPES. Even though the current code has handled cases when the migratetype is CMA, HIGHATOMIC and MEMORY_ISOLATION, making the row size right is still good to avoid future error and confusion. Link: https://lkml.kernel.org/r/20240326061134.1055295-8-bhe@redhat.com Signed-off-by: Baoquan He Cc: Mel Gorman Cc: "Mike Rapoport (IBM)" Signed-off-by: Andrew Morton --- mm/page_alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 875cec8c77089..47ab0297838a6 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1542,7 +1542,7 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, * * The other migratetypes do not have fallbacks. */ -static int fallbacks[MIGRATE_TYPES][MIGRATE_PCPTYPES - 1] = { +static int fallbacks[MIGRATE_PCPTYPES][MIGRATE_PCPTYPES - 1] = { [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE }, [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE }, [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE }, From 796c2c23e14e754b1b72bed450f03d6f69f29358 Mon Sep 17 00:00:00 2001 From: Chris Li Date: Tue, 26 Mar 2024 11:35:43 -0700 Subject: [PATCH 0669/1702] zswap: replace RB tree with xarray Very deep RB tree requires rebalance at times. That contributes to the zswap fault latencies. Xarray does not need to perform tree rebalance. Replacing RB tree to xarray can have some small performance gain. One small difference is that xarray insert might fail with ENOMEM, while RB tree insert does not allocate additional memory. The zswap_entry size will reduce a bit due to removing the RB node, which has two pointers and a color field. Xarray store the pointer in the xarray tree rather than the zswap_entry. Every entry has one pointer from the xarray tree. Overall, switching to xarray should save some memory, if the swap entries are densely packed. Notice the zswap_rb_search and zswap_rb_insert often followed by zswap_rb_erase. Use xa_erase and xa_store directly. That saves one tree lookup as well. Remove zswap_invalidate_entry due to no need to call zswap_rb_erase any more. Use zswap_free_entry instead. The "struct zswap_tree" has been replaced by "struct xarray". The tree spin lock has transferred to the xarray lock. Run the kernel build testing 5 times for each version, averages: (memory.max=2GB, zswap shrinker and writeback enabled, one 50GB swapfile, 24 HT core, 32 jobs) mm-unstable-4aaccadb5c04 xarray v9 user 3548.902 3534.375 sys 522.232 520.976 real 202.796 200.864 [chrisl@kernel.org: restore original comment "erase" to "invalidate"] Link: https://lkml.kernel.org/r/20240326-zswap-xarray-v10-1-bf698417c968@kernel.org Link: https://lkml.kernel.org/r/20240326-zswap-xarray-v9-1-d2891a65dfc7@kernel.org Signed-off-by: Chris Li Acked-by: Yosry Ahmed Acked-by: Johannes Weiner Reviewed-by: Nhat Pham Cc: Barry Song Cc: Chengming Zhou Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/zswap.c | 183 +++++++++++++++++------------------------------------ 1 file changed, 57 insertions(+), 126 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index 01fe081474fff..e6cf6282f5fd0 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -194,7 +193,6 @@ static struct shrinker *zswap_shrinker; * This structure contains the metadata for tracking a single compressed * page within zswap. * - * rbnode - links the entry into red-black tree for the appropriate swap type * swpentry - associated swap entry, the offset indexes into the red-black tree * length - the length in bytes of the compressed page data. Needed during * decompression. For a same value filled page length is 0, and both @@ -206,7 +204,6 @@ static struct shrinker *zswap_shrinker; * lru - handle to the pool's lru used to evict pages. */ struct zswap_entry { - struct rb_node rbnode; swp_entry_t swpentry; unsigned int length; struct zswap_pool *pool; @@ -218,12 +215,7 @@ struct zswap_entry { struct list_head lru; }; -struct zswap_tree { - struct rb_root rbroot; - spinlock_t lock; -}; - -static struct zswap_tree *zswap_trees[MAX_SWAPFILES]; +static struct xarray *zswap_trees[MAX_SWAPFILES]; static unsigned int nr_zswap_trees[MAX_SWAPFILES]; /* RCU-protected iteration */ @@ -251,7 +243,7 @@ static bool zswap_has_pool; * helpers and fwd declarations **********************************/ -static inline struct zswap_tree *swap_zswap_tree(swp_entry_t swp) +static inline struct xarray *swap_zswap_tree(swp_entry_t swp) { return &zswap_trees[swp_type(swp)][swp_offset(swp) >> SWAP_ADDRESS_SPACE_SHIFT]; @@ -790,63 +782,6 @@ void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg) spin_unlock(&zswap_shrink_lock); } -/********************************* -* rbtree functions -**********************************/ -static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset) -{ - struct rb_node *node = root->rb_node; - struct zswap_entry *entry; - pgoff_t entry_offset; - - while (node) { - entry = rb_entry(node, struct zswap_entry, rbnode); - entry_offset = swp_offset(entry->swpentry); - if (entry_offset > offset) - node = node->rb_left; - else if (entry_offset < offset) - node = node->rb_right; - else - return entry; - } - return NULL; -} - -/* - * In the case that a entry with the same offset is found, a pointer to - * the existing entry is stored in dupentry and the function returns -EEXIST - */ -static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry, - struct zswap_entry **dupentry) -{ - struct rb_node **link = &root->rb_node, *parent = NULL; - struct zswap_entry *myentry; - pgoff_t myentry_offset, entry_offset = swp_offset(entry->swpentry); - - while (*link) { - parent = *link; - myentry = rb_entry(parent, struct zswap_entry, rbnode); - myentry_offset = swp_offset(myentry->swpentry); - if (myentry_offset > entry_offset) - link = &(*link)->rb_left; - else if (myentry_offset < entry_offset) - link = &(*link)->rb_right; - else { - *dupentry = myentry; - return -EEXIST; - } - } - rb_link_node(&entry->rbnode, parent, link); - rb_insert_color(&entry->rbnode, root); - return 0; -} - -static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry) -{ - rb_erase(&entry->rbnode, root); - RB_CLEAR_NODE(&entry->rbnode); -} - /********************************* * zswap entry functions **********************************/ @@ -858,7 +793,6 @@ static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp, int nid) entry = kmem_cache_alloc_node(zswap_entry_cache, gfp, nid); if (!entry) return NULL; - RB_CLEAR_NODE(&entry->rbnode); return entry; } @@ -893,17 +827,6 @@ static void zswap_entry_free(struct zswap_entry *entry) atomic_dec(&zswap_stored_pages); } -/* - * The caller hold the tree lock and search the entry from the tree, - * so it must be on the tree, remove it from the tree and free it. - */ -static void zswap_invalidate_entry(struct zswap_tree *tree, - struct zswap_entry *entry) -{ - zswap_rb_erase(&tree->rbroot, entry); - zswap_entry_free(entry); -} - /********************************* * compressed storage functions **********************************/ @@ -1103,7 +1026,8 @@ static void zswap_decompress(struct zswap_entry *entry, struct page *page) static int zswap_writeback_entry(struct zswap_entry *entry, swp_entry_t swpentry) { - struct zswap_tree *tree; + struct xarray *tree; + pgoff_t offset = swp_offset(swpentry); struct folio *folio; struct mempolicy *mpol; bool folio_was_allocated; @@ -1140,19 +1064,13 @@ static int zswap_writeback_entry(struct zswap_entry *entry, * be dereferenced. */ tree = swap_zswap_tree(swpentry); - spin_lock(&tree->lock); - if (zswap_rb_search(&tree->rbroot, swp_offset(swpentry)) != entry) { - spin_unlock(&tree->lock); + if (entry != xa_cmpxchg(tree, offset, entry, NULL, GFP_KERNEL)) { delete_from_swap_cache(folio); folio_unlock(folio); folio_put(folio); return -ENOMEM; } - /* Safe to deref entry after the entry is verified above. */ - zswap_rb_erase(&tree->rbroot, entry); - spin_unlock(&tree->lock); - zswap_decompress(entry, &folio->page); count_vm_event(ZSWPWB); @@ -1484,8 +1402,8 @@ bool zswap_store(struct folio *folio) { swp_entry_t swp = folio->swap; pgoff_t offset = swp_offset(swp); - struct zswap_tree *tree = swap_zswap_tree(swp); - struct zswap_entry *entry, *dupentry; + struct xarray *tree = swap_zswap_tree(swp); + struct zswap_entry *entry, *old; struct obj_cgroup *objcg = NULL; struct mem_cgroup *memcg = NULL; unsigned long max_pages, cur_pages; @@ -1573,27 +1491,43 @@ bool zswap_store(struct folio *folio) insert_entry: entry->swpentry = swp; entry->objcg = objcg; + + old = xa_store(tree, offset, entry, GFP_KERNEL); + if (xa_is_err(old)) { + int err = xa_err(old); + + WARN_ONCE(err != -ENOMEM, "unexpected xarray error: %d\n", err); + zswap_reject_alloc_fail++; + goto store_failed; + } + + /* + * We may have had an existing entry that became stale when + * the folio was redirtied and now the new version is being + * swapped out. Get rid of the old. + */ + if (old) + zswap_entry_free(old); + if (objcg) { obj_cgroup_charge_zswap(objcg, entry->length); - /* Account before objcg ref is moved to tree */ count_objcg_event(objcg, ZSWPOUT); } - /* map */ - spin_lock(&tree->lock); /* - * The folio may have been dirtied again, invalidate the - * possibly stale entry before inserting the new entry. + * We finish initializing the entry while it's already in xarray. + * This is safe because: + * + * 1. Concurrent stores and invalidations are excluded by folio lock. + * + * 2. Writeback is excluded by the entry not being on the LRU yet. + * The publishing order matters to prevent writeback from seeing + * an incoherent entry. */ - if (zswap_rb_insert(&tree->rbroot, entry, &dupentry) == -EEXIST) { - zswap_invalidate_entry(tree, dupentry); - WARN_ON(zswap_rb_insert(&tree->rbroot, entry, &dupentry)); - } if (entry->length) { INIT_LIST_HEAD(&entry->lru); zswap_lru_add(&zswap_list_lru, entry); } - spin_unlock(&tree->lock); /* update stats */ atomic_inc(&zswap_stored_pages); @@ -1601,8 +1535,14 @@ bool zswap_store(struct folio *folio) return true; +store_failed: + if (!entry->length) + atomic_dec(&zswap_same_filled_pages); + else { + zpool_free(zswap_find_zpool(entry), entry->handle); put_pool: - zswap_pool_put(entry->pool); + zswap_pool_put(entry->pool); + } freepage: zswap_entry_cache_free(entry); reject: @@ -1613,11 +1553,9 @@ bool zswap_store(struct folio *folio) * possibly stale entry which was previously stored at this offset. * Otherwise, writeback could overwrite the new data in the swapfile. */ - spin_lock(&tree->lock); - entry = zswap_rb_search(&tree->rbroot, offset); + entry = xa_erase(tree, offset); if (entry) - zswap_invalidate_entry(tree, entry); - spin_unlock(&tree->lock); + zswap_entry_free(entry); return false; shrink: @@ -1631,18 +1569,12 @@ bool zswap_load(struct folio *folio) pgoff_t offset = swp_offset(swp); struct page *page = &folio->page; bool swapcache = folio_test_swapcache(folio); - struct zswap_tree *tree = swap_zswap_tree(swp); + struct xarray *tree = swap_zswap_tree(swp); struct zswap_entry *entry; u8 *dst; VM_WARN_ON_ONCE(!folio_test_locked(folio)); - spin_lock(&tree->lock); - entry = zswap_rb_search(&tree->rbroot, offset); - if (!entry) { - spin_unlock(&tree->lock); - return false; - } /* * When reading into the swapcache, invalidate our entry. The * swapcache can be the authoritative owner of the page and @@ -1656,8 +1588,12 @@ bool zswap_load(struct folio *folio) * the fault fails. We remain the primary owner of the entry.) */ if (swapcache) - zswap_rb_erase(&tree->rbroot, entry); - spin_unlock(&tree->lock); + entry = xa_erase(tree, offset); + else + entry = xa_load(tree, offset); + + if (!entry) + return false; if (entry->length) zswap_decompress(entry, page); @@ -1682,19 +1618,17 @@ bool zswap_load(struct folio *folio) void zswap_invalidate(swp_entry_t swp) { pgoff_t offset = swp_offset(swp); - struct zswap_tree *tree = swap_zswap_tree(swp); + struct xarray *tree = swap_zswap_tree(swp); struct zswap_entry *entry; - spin_lock(&tree->lock); - entry = zswap_rb_search(&tree->rbroot, offset); + entry = xa_erase(tree, offset); if (entry) - zswap_invalidate_entry(tree, entry); - spin_unlock(&tree->lock); + zswap_entry_free(entry); } int zswap_swapon(int type, unsigned long nr_pages) { - struct zswap_tree *trees, *tree; + struct xarray *trees, *tree; unsigned int nr, i; nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES); @@ -1704,11 +1638,8 @@ int zswap_swapon(int type, unsigned long nr_pages) return -ENOMEM; } - for (i = 0; i < nr; i++) { - tree = trees + i; - tree->rbroot = RB_ROOT; - spin_lock_init(&tree->lock); - } + for (i = 0; i < nr; i++) + xa_init(trees + i); nr_zswap_trees[type] = nr; zswap_trees[type] = trees; @@ -1717,7 +1648,7 @@ int zswap_swapon(int type, unsigned long nr_pages) void zswap_swapoff(int type) { - struct zswap_tree *trees = zswap_trees[type]; + struct xarray *trees = zswap_trees[type]; unsigned int i; if (!trees) @@ -1725,7 +1656,7 @@ void zswap_swapoff(int type) /* try_to_unuse() invalidated all the entries already */ for (i = 0; i < nr_zswap_trees[type]; i++) - WARN_ON_ONCE(!RB_EMPTY_ROOT(&trees[i].rbroot)); + WARN_ON_ONCE(!xa_empty(trees + i)); kvfree(trees); nr_zswap_trees[type] = 0; From 4d30eac3744d195ac43fbc36117d1e5de447b1ca Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 20:28:21 +0000 Subject: [PATCH 0670/1702] sparc: use is_huge_zero_pmd() Patch series "Convert huge_zero_page to huge_zero_folio". Almost all the callers of is_huge_zero_page() already have a folio. And they should -- is_huge_zero_page() will return false for tail pages, even if they're tail pages of the huge zero page. That's confusing, and one of the benefits of the folio conversion is to get rid of this confusion. This patch (of 8): There's no need to convert to a page, much less a folio. We can tell from the pmd whether it is a huge zero page or not. Saves 60 bytes of text. Link: https://lkml.kernel.org/r/20240326202833.523759-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240326202833.523759-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Acked-by: David Hildenbrand Signed-off-by: Andrew Morton --- arch/sparc/mm/tlb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c index b44d79d778c71..19642f7ffb52a 100644 --- a/arch/sparc/mm/tlb.c +++ b/arch/sparc/mm/tlb.c @@ -183,12 +183,12 @@ static void __set_pmd_acct(struct mm_struct *mm, unsigned long addr, * hugetlb_pte_count. */ if (pmd_val(pmd) & _PAGE_PMD_HUGE) { - if (is_huge_zero_page(pmd_page(pmd))) + if (is_huge_zero_pmd(pmd)) mm->context.hugetlb_pte_count++; else mm->context.thp_pte_count++; } else { - if (is_huge_zero_page(pmd_page(orig))) + if (is_huge_zero_pmd(orig)) mm->context.hugetlb_pte_count--; else mm->context.thp_pte_count--; @@ -259,7 +259,7 @@ pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, * Sanity check pmd before doing the actual decrement. */ if ((pmd_val(entry) & _PAGE_PMD_HUGE) && - !is_huge_zero_page(pmd_page(entry))) + !is_huge_zero_pmd(entry)) (vma->vm_mm)->context.thp_pte_count--; return old; From 5beaee54a324ba1fe307e341ec825d5d099f4091 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 20:28:22 +0000 Subject: [PATCH 0671/1702] mm: add is_huge_zero_folio() This is the folio equivalent of is_huge_zero_page(). It doesn't add any efficiency, but it does prevent the caller from passing a tail page and getting confused when the predicate returns false. Link: https://lkml.kernel.org/r/20240326202833.523759-3-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- fs/proc/page.c | 2 +- include/linux/huge_mm.h | 10 ++++++++++ mm/huge_memory.c | 6 +++--- mm/mempolicy.c | 2 +- mm/swap.c | 2 +- mm/swap_state.c | 2 +- mm/userfaultfd.c | 2 +- 7 files changed, 18 insertions(+), 8 deletions(-) diff --git a/fs/proc/page.c b/fs/proc/page.c index 05120263af2a4..2fb64bdb64eb1 100644 --- a/fs/proc/page.c +++ b/fs/proc/page.c @@ -155,7 +155,7 @@ u64 stable_page_flags(const struct page *page) else if (folio_test_large(folio)) { if ((k & (1 << PG_lru)) || is_anon) u |= 1 << KPF_THP; - else if (is_huge_zero_page(&folio->page)) { + else if (is_huge_zero_folio(folio)) { u |= 1 << KPF_ZERO_PAGE; u |= 1 << KPF_THP; } diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 1540a1481daf5..600c6008262b2 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -356,6 +356,11 @@ static inline bool is_huge_zero_page(const struct page *page) return READ_ONCE(huge_zero_page) == page; } +static inline bool is_huge_zero_folio(const struct folio *folio) +{ + return READ_ONCE(huge_zero_page) == &folio->page; +} + static inline bool is_huge_zero_pmd(pmd_t pmd) { return pmd_present(pmd) && READ_ONCE(huge_zero_pfn) == pmd_pfn(pmd); @@ -485,6 +490,11 @@ static inline bool is_huge_zero_page(const struct page *page) return false; } +static inline bool is_huge_zero_folio(const struct folio *folio) +{ + return false; +} + static inline bool is_huge_zero_pmd(pmd_t pmd) { return false; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 75ad971ca45ea..5c043c7b50624 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -789,12 +789,12 @@ struct deferred_split *get_deferred_split_queue(struct folio *folio) } #endif -static inline bool is_transparent_hugepage(struct folio *folio) +static inline bool is_transparent_hugepage(const struct folio *folio) { if (!folio_test_large(folio)) return false; - return is_huge_zero_page(&folio->page) || + return is_huge_zero_folio(folio) || folio_test_large_rmappable(folio); } @@ -3085,7 +3085,7 @@ int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, } - is_hzp = is_huge_zero_page(&folio->page); + is_hzp = is_huge_zero_folio(folio); if (is_hzp) { pr_warn_ratelimited("Called split_huge_page for huge zero page\n"); return -EBUSY; diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 913cff5da5a30..5743028a63a51 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -510,7 +510,7 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk) return; } folio = pfn_folio(pmd_pfn(*pmd)); - if (is_huge_zero_page(&folio->page)) { + if (is_huge_zero_folio(folio)) { walk->action = ACTION_CONTINUE; return; } diff --git a/mm/swap.c b/mm/swap.c index 500a09a48dfd3..f72364e92d5f4 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -985,7 +985,7 @@ void folios_put_refs(struct folio_batch *folios, unsigned int *refs) struct folio *folio = folios->folios[i]; unsigned int nr_refs = refs ? refs[i] : 1; - if (is_huge_zero_page(&folio->page)) + if (is_huge_zero_folio(folio)) continue; if (folio_is_zone_device(folio)) { diff --git a/mm/swap_state.c b/mm/swap_state.c index bfc7e8c58a6d3..2deac23633cdd 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -301,7 +301,7 @@ void free_page_and_swap_cache(struct page *page) struct folio *folio = page_folio(page); free_swap_cache(folio); - if (!is_huge_zero_page(page)) + if (!is_huge_zero_folio(folio)) folio_put(folio); } diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 3c3539c573e7f..a0ec14553fbea 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1664,7 +1664,7 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, !pmd_none(dst_pmdval)) { struct folio *folio = pfn_folio(pmd_pfn(*src_pmd)); - if (!folio || (!is_huge_zero_page(&folio->page) && + if (!folio || (!is_huge_zero_folio(folio) && !PageAnonExclusive(&folio->page))) { spin_unlock(ptl); err = -EBUSY; From e06d03d5590ae1c257b8aa2cfbfe6765e0755c14 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 20:28:23 +0000 Subject: [PATCH 0672/1702] mm: add pmd_folio() Convert directly from a pmd to a folio without going through another representation first. For now this is just a slightly shorter way to write it, but it might end up being more efficient later. Link: https://lkml.kernel.org/r/20240326202833.523759-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 2 ++ mm/huge_memory.c | 6 +++--- mm/madvise.c | 2 +- mm/mempolicy.c | 2 +- mm/mlock.c | 2 +- mm/userfaultfd.c | 2 +- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 600e17d036599..09c85c7bf9c2c 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -50,6 +50,8 @@ #define pmd_pgtable(pmd) pmd_page(pmd) #endif +#define pmd_folio(pmd) page_folio(pmd_page(pmd)) + /* * A page table page can be thought of an array like this: pXd_t[PTRS_PER_PxD] * diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 5c043c7b50624..712263e3b1f6e 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1816,7 +1816,7 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, goto out; } - folio = pfn_folio(pmd_pfn(orig_pmd)); + folio = pmd_folio(orig_pmd); /* * If other processes are mapping this folio, we couldn't discard * the folio unless they all do MADV_FREE so let's skip the folio. @@ -2086,7 +2086,7 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, if (pmd_protnone(*pmd)) goto unlock; - folio = page_folio(pmd_page(*pmd)); + folio = pmd_folio(*pmd); toptier = node_is_toptier(folio_nid(folio)); /* * Skip scanning top tier node if normal numa @@ -2663,7 +2663,7 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, * It's safe to call pmd_page when folio is set because it's * guaranteed that pmd is present. */ - if (folio && folio != page_folio(pmd_page(*pmd))) + if (folio && folio != pmd_folio(*pmd)) goto out; __split_huge_pmd_locked(vma, pmd, range.start, freeze); } diff --git a/mm/madvise.c b/mm/madvise.c index 7625830d6ae91..1f77a51baaac2 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -363,7 +363,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, goto huge_unlock; } - folio = pfn_folio(pmd_pfn(orig_pmd)); + folio = pmd_folio(orig_pmd); /* Do not interfere with other mappings of this folio */ if (folio_likely_mapped_shared(folio)) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 5743028a63a51..aec756ae56377 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -509,7 +509,7 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk) qp->nr_failed++; return; } - folio = pfn_folio(pmd_pfn(*pmd)); + folio = pmd_folio(*pmd); if (is_huge_zero_folio(folio)) { walk->action = ACTION_CONTINUE; return; diff --git a/mm/mlock.c b/mm/mlock.c index 1ed2f2ab37cd1..30b51cdea89de 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -378,7 +378,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, goto out; if (is_huge_zero_pmd(*pmd)) goto out; - folio = page_folio(pmd_page(*pmd)); + folio = pmd_folio(*pmd); if (vma->vm_flags & VM_LOCKED) mlock_folio(folio); else diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index a0ec14553fbea..b70618e8dcd24 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1662,7 +1662,7 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, /* Check if we can move the pmd without splitting it. */ if (move_splits_huge_pmd(dst_addr, src_addr, src_start + len) || !pmd_none(dst_pmdval)) { - struct folio *folio = pfn_folio(pmd_pfn(*src_pmd)); + struct folio *folio = pmd_folio(*src_pmd); if (!folio || (!is_huge_zero_folio(folio) && !PageAnonExclusive(&folio->page))) { From b002a7b0a58aceaed0e3c5dd80d6f549df6490b2 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 20:28:24 +0000 Subject: [PATCH 0673/1702] mm: convert migrate_vma_collect_pmd to use a folio Convert the pmd directly to a folio and use it. Turns four calls to compound_head() into one. Link: https://lkml.kernel.org/r/20240326202833.523759-5-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- mm/migrate_device.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mm/migrate_device.c b/mm/migrate_device.c index b6c27c76e1a0b..d40b46ae9d65b 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -71,7 +71,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, return migrate_vma_collect_hole(start, end, -1, walk); if (pmd_trans_huge(*pmdp)) { - struct page *page; + struct folio *folio; ptl = pmd_lock(mm, pmdp); if (unlikely(!pmd_trans_huge(*pmdp))) { @@ -79,21 +79,21 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, goto again; } - page = pmd_page(*pmdp); - if (is_huge_zero_page(page)) { + folio = pmd_folio(*pmdp); + if (is_huge_zero_folio(folio)) { spin_unlock(ptl); split_huge_pmd(vma, pmdp, addr); } else { int ret; - get_page(page); + folio_get(folio); spin_unlock(ptl); - if (unlikely(!trylock_page(page))) + if (unlikely(!folio_trylock(folio))) return migrate_vma_collect_skip(start, end, walk); - ret = split_huge_page(page); - unlock_page(page); - put_page(page); + ret = split_folio(folio); + folio_unlock(folio); + folio_put(folio); if (ret) return migrate_vma_collect_skip(start, end, walk); From 5691753d73a233a97c13f7d389f15d20596d062d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 20:28:25 +0000 Subject: [PATCH 0674/1702] mm: convert huge_zero_page to huge_zero_folio With all callers of is_huge_zero_page() converted, we can now switch the huge_zero_page itself from being a compound page to a folio. Link: https://lkml.kernel.org/r/20240326202833.523759-6-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 21 ++++++++------------- mm/huge_memory.c | 28 ++++++++++++++-------------- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 600c6008262b2..7ba59ba363549 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -348,17 +348,12 @@ struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr, vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf); -extern struct page *huge_zero_page; +extern struct folio *huge_zero_folio; extern unsigned long huge_zero_pfn; -static inline bool is_huge_zero_page(const struct page *page) -{ - return READ_ONCE(huge_zero_page) == page; -} - static inline bool is_huge_zero_folio(const struct folio *folio) { - return READ_ONCE(huge_zero_page) == &folio->page; + return READ_ONCE(huge_zero_folio) == folio; } static inline bool is_huge_zero_pmd(pmd_t pmd) @@ -371,9 +366,14 @@ static inline bool is_huge_zero_pud(pud_t pud) return false; } -struct page *mm_get_huge_zero_page(struct mm_struct *mm); +struct folio *mm_get_huge_zero_folio(struct mm_struct *mm); void mm_put_huge_zero_page(struct mm_struct *mm); +static inline struct page *mm_get_huge_zero_page(struct mm_struct *mm) +{ + return &mm_get_huge_zero_folio(mm)->page; +} + #define mk_huge_pmd(page, prot) pmd_mkhuge(mk_pmd(page, prot)) static inline bool thp_migration_supported(void) @@ -485,11 +485,6 @@ static inline vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) return 0; } -static inline bool is_huge_zero_page(const struct page *page) -{ - return false; -} - static inline bool is_huge_zero_folio(const struct folio *folio) { return false; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 712263e3b1f6e..6d510d635b1d9 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -74,7 +74,7 @@ static unsigned long deferred_split_scan(struct shrinker *shrink, struct shrink_control *sc); static atomic_t huge_zero_refcount; -struct page *huge_zero_page __read_mostly; +struct folio *huge_zero_folio __read_mostly; unsigned long huge_zero_pfn __read_mostly = ~0UL; unsigned long huge_anon_orders_always __read_mostly; unsigned long huge_anon_orders_madvise __read_mostly; @@ -192,24 +192,24 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, static bool get_huge_zero_page(void) { - struct page *zero_page; + struct folio *zero_folio; retry: if (likely(atomic_inc_not_zero(&huge_zero_refcount))) return true; - zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, + zero_folio = folio_alloc((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, HPAGE_PMD_ORDER); - if (!zero_page) { + if (!zero_folio) { count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); return false; } preempt_disable(); - if (cmpxchg(&huge_zero_page, NULL, zero_page)) { + if (cmpxchg(&huge_zero_folio, NULL, zero_folio)) { preempt_enable(); - __free_pages(zero_page, compound_order(zero_page)); + folio_put(zero_folio); goto retry; } - WRITE_ONCE(huge_zero_pfn, page_to_pfn(zero_page)); + WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio)); /* We take additional reference here. It will be put back by shrinker */ atomic_set(&huge_zero_refcount, 2); @@ -227,10 +227,10 @@ static void put_huge_zero_page(void) BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); } -struct page *mm_get_huge_zero_page(struct mm_struct *mm) +struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) { if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) - return READ_ONCE(huge_zero_page); + return READ_ONCE(huge_zero_folio); if (!get_huge_zero_page()) return NULL; @@ -238,7 +238,7 @@ struct page *mm_get_huge_zero_page(struct mm_struct *mm) if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) put_huge_zero_page(); - return READ_ONCE(huge_zero_page); + return READ_ONCE(huge_zero_folio); } void mm_put_huge_zero_page(struct mm_struct *mm) @@ -258,10 +258,10 @@ static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink, struct shrink_control *sc) { if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { - struct page *zero_page = xchg(&huge_zero_page, NULL); - BUG_ON(zero_page == NULL); + struct folio *zero_folio = xchg(&huge_zero_folio, NULL); + BUG_ON(zero_folio == NULL); WRITE_ONCE(huge_zero_pfn, ~0UL); - __free_pages(zero_page, compound_order(zero_page)); + folio_put(zero_folio); return HPAGE_PMD_NR; } @@ -1340,7 +1340,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, * since we already have a zero page to copy. It just takes a * reference. */ - mm_get_huge_zero_page(dst_mm); + mm_get_huge_zero_folio(dst_mm); goto out_zero_page; } From e28833bc4ace001e77201d7b69ac389233482864 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 20:28:26 +0000 Subject: [PATCH 0675/1702] mm: convert do_huge_pmd_anonymous_page to huge_zero_folio Use folios more widely. Link: https://lkml.kernel.org/r/20240326202833.523759-7-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- mm/huge_memory.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 6d510d635b1d9..098b8ad4bcc74 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -971,14 +971,14 @@ gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma) } /* Caller must hold page table lock. */ -static void set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, +static void set_huge_zero_folio(pgtable_t pgtable, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, - struct page *zero_page) + struct folio *zero_folio) { pmd_t entry; if (!pmd_none(*pmd)) return; - entry = mk_pmd(zero_page, vma->vm_page_prot); + entry = mk_pmd(&zero_folio->page, vma->vm_page_prot); entry = pmd_mkhuge(entry); pgtable_trans_huge_deposit(mm, pmd, pgtable); set_pmd_at(mm, haddr, pmd, entry); @@ -1002,13 +1002,14 @@ vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) !mm_forbids_zeropage(vma->vm_mm) && transparent_hugepage_use_zero_page()) { pgtable_t pgtable; - struct page *zero_page; + struct folio *zero_folio; vm_fault_t ret; + pgtable = pte_alloc_one(vma->vm_mm); if (unlikely(!pgtable)) return VM_FAULT_OOM; - zero_page = mm_get_huge_zero_page(vma->vm_mm); - if (unlikely(!zero_page)) { + zero_folio = mm_get_huge_zero_folio(vma->vm_mm); + if (unlikely(!zero_folio)) { pte_free(vma->vm_mm, pgtable); count_vm_event(THP_FAULT_FALLBACK); return VM_FAULT_FALLBACK; @@ -1026,8 +1027,8 @@ vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) ret = handle_userfault(vmf, VM_UFFD_MISSING); VM_BUG_ON(ret & VM_FAULT_FALLBACK); } else { - set_huge_zero_page(pgtable, vma->vm_mm, vma, - haddr, vmf->pmd, zero_page); + set_huge_zero_folio(pgtable, vma->vm_mm, vma, + haddr, vmf->pmd, zero_folio); update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); spin_unlock(vmf->ptl); } @@ -1336,9 +1337,9 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, */ if (is_huge_zero_pmd(pmd)) { /* - * get_huge_zero_page() will never allocate a new page here, - * since we already have a zero page to copy. It just takes a - * reference. + * mm_get_huge_zero_folio() will never allocate a new + * folio here, since we already have a zero page to + * copy. It just takes a reference. */ mm_get_huge_zero_folio(dst_mm); goto out_zero_page; From c93012d849c9e3fab9540458623b98794c4bf0a1 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 20:28:27 +0000 Subject: [PATCH 0676/1702] dax: use huge_zero_folio Convert from huge_zero_page to huge_zero_folio. Link: https://lkml.kernel.org/r/20240326202833.523759-8-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- fs/dax.c | 14 +++++++------- include/trace/events/fs_dax.h | 16 ++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/fs/dax.c b/fs/dax.c index 423fc1607dfae..becb4a6920c6a 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1207,17 +1207,17 @@ static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf, struct vm_area_struct *vma = vmf->vma; struct inode *inode = mapping->host; pgtable_t pgtable = NULL; - struct page *zero_page; + struct folio *zero_folio; spinlock_t *ptl; pmd_t pmd_entry; pfn_t pfn; - zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm); + zero_folio = mm_get_huge_zero_folio(vmf->vma->vm_mm); - if (unlikely(!zero_page)) + if (unlikely(!zero_folio)) goto fallback; - pfn = page_to_pfn_t(zero_page); + pfn = page_to_pfn_t(&zero_folio->page); *entry = dax_insert_entry(xas, vmf, iter, *entry, pfn, DAX_PMD | DAX_ZERO_PAGE); @@ -1237,17 +1237,17 @@ static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf, pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); mm_inc_nr_ptes(vma->vm_mm); } - pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot); + pmd_entry = mk_pmd(&zero_folio->page, vmf->vma->vm_page_prot); pmd_entry = pmd_mkhuge(pmd_entry); set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry); spin_unlock(ptl); - trace_dax_pmd_load_hole(inode, vmf, zero_page, *entry); + trace_dax_pmd_load_hole(inode, vmf, zero_folio, *entry); return VM_FAULT_NOPAGE; fallback: if (pgtable) pte_free(vma->vm_mm, pgtable); - trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, *entry); + trace_dax_pmd_load_hole_fallback(inode, vmf, zero_folio, *entry); return VM_FAULT_FALLBACK; } #else diff --git a/include/trace/events/fs_dax.h b/include/trace/events/fs_dax.h index 97b09fcf7e528..86fe6aecff1e7 100644 --- a/include/trace/events/fs_dax.h +++ b/include/trace/events/fs_dax.h @@ -62,14 +62,14 @@ DEFINE_PMD_FAULT_EVENT(dax_pmd_fault_done); DECLARE_EVENT_CLASS(dax_pmd_load_hole_class, TP_PROTO(struct inode *inode, struct vm_fault *vmf, - struct page *zero_page, + struct folio *zero_folio, void *radix_entry), - TP_ARGS(inode, vmf, zero_page, radix_entry), + TP_ARGS(inode, vmf, zero_folio, radix_entry), TP_STRUCT__entry( __field(unsigned long, ino) __field(unsigned long, vm_flags) __field(unsigned long, address) - __field(struct page *, zero_page) + __field(struct folio *, zero_folio) __field(void *, radix_entry) __field(dev_t, dev) ), @@ -78,17 +78,17 @@ DECLARE_EVENT_CLASS(dax_pmd_load_hole_class, __entry->ino = inode->i_ino; __entry->vm_flags = vmf->vma->vm_flags; __entry->address = vmf->address; - __entry->zero_page = zero_page; + __entry->zero_folio = zero_folio; __entry->radix_entry = radix_entry; ), - TP_printk("dev %d:%d ino %#lx %s address %#lx zero_page %p " + TP_printk("dev %d:%d ino %#lx %s address %#lx zero_folio %p " "radix_entry %#lx", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, __entry->vm_flags & VM_SHARED ? "shared" : "private", __entry->address, - __entry->zero_page, + __entry->zero_folio, (unsigned long)__entry->radix_entry ) ) @@ -96,8 +96,8 @@ DECLARE_EVENT_CLASS(dax_pmd_load_hole_class, #define DEFINE_PMD_LOAD_HOLE_EVENT(name) \ DEFINE_EVENT(dax_pmd_load_hole_class, name, \ TP_PROTO(struct inode *inode, struct vm_fault *vmf, \ - struct page *zero_page, void *radix_entry), \ - TP_ARGS(inode, vmf, zero_page, radix_entry)) + struct folio *zero_folio, void *radix_entry), \ + TP_ARGS(inode, vmf, zero_folio, radix_entry)) DEFINE_PMD_LOAD_HOLE_EVENT(dax_pmd_load_hole); DEFINE_PMD_LOAD_HOLE_EVENT(dax_pmd_load_hole_fallback); From 632230ff193954b514c908fdb45320f64ac9dc72 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 26 Mar 2024 20:28:28 +0000 Subject: [PATCH 0677/1702] mm: rename mm_put_huge_zero_page to mm_put_huge_zero_folio Also remove mm_get_huge_zero_page() now it has no users. Link: https://lkml.kernel.org/r/20240326202833.523759-9-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 9 ++------- kernel/fork.c | 2 +- mm/huge_memory.c | 2 +- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 7ba59ba363549..9d11e886aa9a9 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -367,12 +367,7 @@ static inline bool is_huge_zero_pud(pud_t pud) } struct folio *mm_get_huge_zero_folio(struct mm_struct *mm); -void mm_put_huge_zero_page(struct mm_struct *mm); - -static inline struct page *mm_get_huge_zero_page(struct mm_struct *mm) -{ - return &mm_get_huge_zero_folio(mm)->page; -} +void mm_put_huge_zero_folio(struct mm_struct *mm); #define mk_huge_pmd(page, prot) pmd_mkhuge(mk_pmd(page, prot)) @@ -500,7 +495,7 @@ static inline bool is_huge_zero_pud(pud_t pud) return false; } -static inline void mm_put_huge_zero_page(struct mm_struct *mm) +static inline void mm_put_huge_zero_folio(struct mm_struct *mm) { return; } diff --git a/kernel/fork.c b/kernel/fork.c index aebb3e6c96dc6..99076dbe27d83 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1344,7 +1344,7 @@ static inline void __mmput(struct mm_struct *mm) ksm_exit(mm); khugepaged_exit(mm); /* must run before exit_mmap */ exit_mmap(mm); - mm_put_huge_zero_page(mm); + mm_put_huge_zero_folio(mm); set_mm_exe_file(mm, NULL); if (!list_empty(&mm->mmlist)) { spin_lock(&mmlist_lock); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 098b8ad4bcc74..c4820cd4749e0 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -241,7 +241,7 @@ struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) return READ_ONCE(huge_zero_folio); } -void mm_put_huge_zero_page(struct mm_struct *mm) +void mm_put_huge_zero_folio(struct mm_struct *mm) { if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) put_huge_zero_page(); From ac3830c3b2666d08063f633a95cb838b7f8db90e Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 27 Mar 2024 11:23:20 -0400 Subject: [PATCH 0678/1702] mm/Kconfig: CONFIG_PGTABLE_HAS_HUGE_LEAVES Patch series "mm/gup: Unify hugetlb, part 2", v4. The series removes the hugetlb slow gup path after a previous refactor work [1], so that slow gup now uses the exact same path to process all kinds of memory including hugetlb. For the long term, we may want to remove most, if not all, call sites of huge_pte_offset(). It'll be ideal if that API can be completely dropped from arch hugetlb API. This series is one small step towards merging hugetlb specific codes into generic mm paths. From that POV, this series removes one reference to huge_pte_offset() out of many others. One goal of such a route is that we can reconsider merging hugetlb features like High Granularity Mapping (HGM). It was not accepted in the past because it may add lots of hugetlb specific codes and make the mm code even harder to maintain. With a merged codeset, features like HGM can hopefully share some code with THP, legacy (PMD+) or modern (continuous PTEs). To make it work, the generic slow gup code will need to at least understand hugepd, which is already done like so in fast-gup. Due to the specialty of hugepd to be software-only solution (no hardware recognizes the hugepd format, so it's purely artificial structures), there's chance we can merge some or all hugepd formats with cont_pte in the future. That question is yet unsettled from Power side to have an acknowledgement. As of now for this series, I kept the hugepd handling because we may still need to do so before getting a clearer picture of the future of hugepd. The other reason is simply that we did it already for fast-gup and most codes are still around to be reused. It'll make more sense to keep slow/fast gup behave the same before a decision is made to remove hugepd. There's one major difference for slow-gup on cont_pte / cont_pmd handling, currently supported on three architectures (aarch64, riscv, ppc). Before the series, slow gup will be able to recognize e.g. cont_pte entries with the help of huge_pte_offset() when hstate is around. Now it's gone but still working, by looking up pgtable entries one by one. It's not ideal, but hopefully this change should not affect yet on major workloads. There's some more information in the commit message of the last patch. If this would be a concern, we can consider teaching slow gup to recognize cont pte/pmd entries, and that should recover the lost performance. But I doubt its necessity for now, so I kept it as simple as it can be. Patch layout ============= Patch 1-8: Preparation works, or cleanups in relevant code paths Patch 9-11: Teach slow gup with all kinds of huge entries (pXd, hugepd) Patch 12: Drop hugetlb_follow_page_mask() More information can be found in the commit messages of each patch. [1] https://lore.kernel.org/all/20230628215310.73782-1-peterx@redhat.com [2] https://lore.kernel.org/r/20240321215047.678172-1-peterx@redhat.com Introduce a config option that will be selected as long as huge leaves are involved in pgtable (thp or hugetlbfs). It would be useful to mark any code with this new config that can process either hugetlb or thp pages in any level that is higher than pte level. Link: https://lkml.kernel.org/r/20240327152332.950956-1-peterx@redhat.com Link: https://lkml.kernel.org/r/20240327152332.950956-2-peterx@redhat.com Signed-off-by: Peter Xu Reviewed-by: Jason Gunthorpe Tested-by: Ryan Roberts Cc: Andrea Arcangeli Cc: Andrew Jones Cc: Aneesh Kumar K.V (IBM) Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: James Houghton Cc: John Hubbard Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Michael Ellerman Cc: "Mike Rapoport (IBM)" Cc: Muchun Song Cc: Rik van Riel Cc: Vlastimil Babka Cc: Yang Shi Signed-off-by: Andrew Morton --- mm/Kconfig | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mm/Kconfig b/mm/Kconfig index b1448aa81e15f..f0ed3168db004 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -849,6 +849,12 @@ config READ_ONLY_THP_FOR_FS endif # TRANSPARENT_HUGEPAGE +# +# The architecture supports pgtable leaves that is larger than PAGE_SIZE +# +config PGTABLE_HAS_HUGE_LEAVES + def_bool TRANSPARENT_HUGEPAGE || HUGETLB_PAGE + # # UP and nommu archs use km based percpu allocator # From 24334e78e8e3ca6c8e51dad67d779b1ed40988a6 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 27 Mar 2024 11:23:21 -0400 Subject: [PATCH 0679/1702] mm/hugetlb: declare hugetlbfs_pagecache_present() non-static It will be used outside hugetlb.c soon. Link: https://lkml.kernel.org/r/20240327152332.950956-3-peterx@redhat.com Signed-off-by: Peter Xu Tested-by: Ryan Roberts Cc: Andrea Arcangeli Cc: Andrew Jones Cc: Aneesh Kumar K.V (IBM) Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: James Houghton Cc: Jason Gunthorpe Cc: John Hubbard Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Michael Ellerman Cc: "Mike Rapoport (IBM)" Cc: Muchun Song Cc: Rik van Riel Cc: Vlastimil Babka Cc: Yang Shi Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 9 +++++++++ mm/hugetlb.c | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 03fc6d6250680..a7988c78d69fa 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -174,6 +174,9 @@ u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx); pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, pud_t *pud); +bool hugetlbfs_pagecache_present(struct hstate *h, + struct vm_area_struct *vma, + unsigned long address); struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage); @@ -1228,6 +1231,12 @@ static inline void hugetlb_register_node(struct node *node) static inline void hugetlb_unregister_node(struct node *node) { } + +static inline bool hugetlbfs_pagecache_present( + struct hstate *h, struct vm_area_struct *vma, unsigned long address) +{ + return false; +} #endif /* CONFIG_HUGETLB_PAGE */ static inline spinlock_t *huge_pte_lock(struct hstate *h, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 88bb38df1701b..740435567be86 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6113,8 +6113,8 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, /* * Return whether there is a pagecache page to back given address within VMA. */ -static bool hugetlbfs_pagecache_present(struct hstate *h, - struct vm_area_struct *vma, unsigned long address) +bool hugetlbfs_pagecache_present(struct hstate *h, + struct vm_area_struct *vma, unsigned long address) { struct address_space *mapping = vma->vm_file->f_mapping; pgoff_t idx = linear_page_index(vma, address); From b979db1611a63b207dbc47706de989ac113ea898 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 27 Mar 2024 11:23:22 -0400 Subject: [PATCH 0680/1702] mm: make HPAGE_PXD_* macros even if !THP These macros can be helpful when we plan to merge hugetlb code into generic code. Move them out and define them as long as PGTABLE_HAS_HUGE_LEAVES is selected, because there are systems that only define HUGETLB_PAGE not THP. One note here is HPAGE_PMD_SHIFT must be defined even if PMD_SHIFT is not defined (e.g. !CONFIG_MMU case); it (or in other forms, like HPAGE_PMD_NR) is already used in lots of common codes without ifdef guards. Use the old trick to let complations work. Here we only need to differenciate HPAGE_PXD_SHIFT definitions. All the rest macros will be defined based on it. When at it, move HPAGE_PMD_NR / HPAGE_PMD_ORDER over together. Link: https://lkml.kernel.org/r/20240327152332.950956-4-peterx@redhat.com Signed-off-by: Peter Xu Tested-by: Ryan Roberts Cc: Andrea Arcangeli Cc: Andrew Jones Cc: Aneesh Kumar K.V (IBM) Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: James Houghton Cc: Jason Gunthorpe Cc: John Hubbard Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Michael Ellerman Cc: "Mike Rapoport (IBM)" Cc: Muchun Song Cc: Rik van Riel Cc: Vlastimil Babka Cc: Yang Shi Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 9d11e886aa9a9..ff72ee19a1254 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -64,9 +64,6 @@ ssize_t single_hugepage_flag_show(struct kobject *kobj, enum transparent_hugepage_flag flag); extern struct kobj_attribute shmem_enabled_attr; -#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT) -#define HPAGE_PMD_NR (1< Date: Wed, 27 Mar 2024 11:23:23 -0400 Subject: [PATCH 0681/1702] mm: introduce vma_pgtable_walk_{begin|end}() Introduce per-vma begin()/end() helpers for pgtable walks. This is a preparation work to merge hugetlb pgtable walkers with generic mm. The helpers need to be called before and after a pgtable walk, will start to be needed if the pgtable walker code supports hugetlb pages. It's a hook point for any type of VMA, but for now only hugetlb uses it to stablize the pgtable pages from getting away (due to possible pmd unsharing). Link: https://lkml.kernel.org/r/20240327152332.950956-5-peterx@redhat.com Signed-off-by: Peter Xu Reviewed-by: Christoph Hellwig Reviewed-by: Muchun Song Tested-by: Ryan Roberts Cc: Andrea Arcangeli Cc: Andrew Jones Cc: Aneesh Kumar K.V (IBM) Cc: Axel Rasmussen Cc: Christophe Leroy Cc: David Hildenbrand Cc: James Houghton Cc: Jason Gunthorpe Cc: John Hubbard Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Michael Ellerman Cc: "Mike Rapoport (IBM)" Cc: Rik van Riel Cc: Vlastimil Babka Cc: Yang Shi Signed-off-by: Andrew Morton --- include/linux/mm.h | 3 +++ mm/memory.c | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/include/linux/mm.h b/include/linux/mm.h index 00a46a5d5c9d6..3dda0dbcae144 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4230,4 +4230,7 @@ static inline bool pfn_is_unaccepted_memory(unsigned long pfn) return range_contains_unaccepted_memory(paddr, paddr + PAGE_SIZE); } +void vma_pgtable_walk_begin(struct vm_area_struct *vma); +void vma_pgtable_walk_end(struct vm_area_struct *vma); + #endif /* _LINUX_MM_H */ diff --git a/mm/memory.c b/mm/memory.c index 62ee4a15092ae..0b92336bcebde 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6378,3 +6378,15 @@ void ptlock_free(struct ptdesc *ptdesc) kmem_cache_free(page_ptl_cachep, ptdesc->ptl); } #endif + +void vma_pgtable_walk_begin(struct vm_area_struct *vma) +{ + if (is_vm_hugetlb_page(vma)) + hugetlb_vma_lock_read(vma); +} + +void vma_pgtable_walk_end(struct vm_area_struct *vma) +{ + if (is_vm_hugetlb_page(vma)) + hugetlb_vma_unlock_read(vma); +} From 35a76f5c0863fc6b570bf02ab6dbf4cd65d1e001 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 27 Mar 2024 11:23:24 -0400 Subject: [PATCH 0682/1702] mm/arch: provide pud_pfn() fallback The comment in the code explains the reasons. We took a different approach comparing to pmd_pfn() by providing a fallback function. Another option is to provide some lower level config options (compare to HUGETLB_PAGE or THP) to identify which layer an arch can support for such huge mappings. However that can be an overkill. [peterx@redhat.com: fix loongson defconfig] Link: https://lkml.kernel.org/r/20240403013249.1418299-4-peterx@redhat.com Link: https://lkml.kernel.org/r/20240327152332.950956-6-peterx@redhat.com Signed-off-by: Peter Xu Reviewed-by: Jason Gunthorpe Tested-by: Ryan Roberts Cc: Mike Rapoport (IBM) Cc: Matthew Wilcox Cc: Andrea Arcangeli Cc: Andrew Jones Cc: Aneesh Kumar K.V (IBM) Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: James Houghton Cc: John Hubbard Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Michael Ellerman Cc: Muchun Song Cc: Rik van Riel Cc: Vlastimil Babka Cc: Yang Shi Signed-off-by: Andrew Morton --- arch/riscv/include/asm/pgtable.h | 1 + arch/s390/include/asm/pgtable.h | 1 + arch/sparc/include/asm/pgtable_64.h | 1 + arch/x86/include/asm/pgtable.h | 1 + include/linux/pgtable.h | 14 ++++++++++++++ 5 files changed, 18 insertions(+) diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 9f8ea0e33eb10..661b2b4fe7589 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -648,6 +648,7 @@ static inline unsigned long pmd_pfn(pmd_t pmd) #define __pud_to_phys(pud) (__page_val_to_pfn(pud_val(pud)) << PAGE_SHIFT) +#define pud_pfn pud_pfn static inline unsigned long pud_pfn(pud_t pud) { return ((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT); diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 60950e7a25f58..2cb2a2e7b34b8 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -1405,6 +1405,7 @@ static inline unsigned long pud_deref(pud_t pud) return (unsigned long)__va(pud_val(pud) & origin_mask); } +#define pud_pfn pud_pfn static inline unsigned long pud_pfn(pud_t pud) { return __pa(pud_deref(pud)) >> PAGE_SHIFT; diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index 4d1bafaba942d..26efc9bb644aa 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -875,6 +875,7 @@ static inline bool pud_leaf(pud_t pud) return pte_val(pte) & _PAGE_PMD_HUGE; } +#define pud_pfn pud_pfn static inline unsigned long pud_pfn(pud_t pud) { pte_t pte = __pte(pud_val(pud)); diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index cefc7a84f7a44..273f7557218cd 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -234,6 +234,7 @@ static inline unsigned long pmd_pfn(pmd_t pmd) return (pfn & pmd_pfn_mask(pmd)) >> PAGE_SHIFT; } +#define pud_pfn pud_pfn static inline unsigned long pud_pfn(pud_t pud) { phys_addr_t pfn = pud_val(pud); diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 09c85c7bf9c2c..f108c7a3c1d9e 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1819,6 +1819,20 @@ typedef unsigned int pgtbl_mod_mask; #define pte_leaf_size(x) PAGE_SIZE #endif +/* + * We always define pmd_pfn for all archs as it's used in lots of generic + * code. Now it happens too for pud_pfn (and can happen for larger + * mappings too in the future; we're not there yet). Instead of defining + * it for all archs (like pmd_pfn), provide a fallback. + * + * Note that returning 0 here means any arch that didn't define this can + * get severely wrong when it hits a real pud leaf. It's arch's + * responsibility to properly define it when a huge pud is possible. + */ +#ifndef pud_pfn +#define pud_pfn(x) 0 +#endif + /* * Some architectures have MMUs that are configurable or selectable at boot * time. These lead to variable PTRS_PER_x. For statically allocated arrays it From 607c63195d6388c4b70bea104708221392564893 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 27 Mar 2024 11:23:25 -0400 Subject: [PATCH 0683/1702] mm/gup: drop gup_fast_folio_allowed() in hugepd processing Hugepd format for GUP is only used in PowerPC with hugetlbfs. There are some kernel usage of hugepd (can refer to hugepd_populate_kernel() for PPC_8XX), however those pages are not candidates for GUP. Commit a6e79df92e4a ("mm/gup: disallow FOLL_LONGTERM GUP-fast writing to file-backed mappings") added a check to fail gup-fast if there's potential risk of violating GUP over writeback file systems. That should never apply to hugepd. Considering that hugepd is an old format (and even software-only), there's no plan to extend hugepd into other file typed memories that is prone to the same issue. Drop that check, not only because it'll never be true for hugepd per any known plan, but also it paves way for reusing the function outside fast-gup. To make sure we'll still remember this issue just in case hugepd will be extended to support non-hugetlbfs memories, add a rich comment above gup_huge_pd(), explaining the issue with proper references. [akpm@linux-foundation.org: fix comment, per David] Link: https://lkml.kernel.org/r/20240327152332.950956-7-peterx@redhat.com Signed-off-by: Peter Xu Tested-by: Ryan Roberts Cc: Christoph Hellwig Cc: Lorenzo Stoakes Cc: Michael Ellerman Cc: Andrea Arcangeli Cc: Andrew Jones Cc: Aneesh Kumar K.V (IBM) Cc: Axel Rasmussen Cc: Christophe Leroy Cc: David Hildenbrand Cc: James Houghton Cc: Jason Gunthorpe Cc: John Hubbard Cc: Kirill A. Shutemov Cc: Matthew Wilcox (Oracle) Cc: "Mike Rapoport (IBM)" Cc: Muchun Song Cc: Rik van Riel Cc: Vlastimil Babka Cc: Yang Shi Signed-off-by: Andrew Morton --- mm/gup.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index 7b21fcda3a981..1cc3240e49001 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2842,11 +2842,6 @@ static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, return 0; } - if (!gup_fast_folio_allowed(folio, flags)) { - gup_put_folio(folio, refs, flags); - return 0; - } - if (!pte_write(pte) && gup_must_unshare(NULL, flags, &folio->page)) { gup_put_folio(folio, refs, flags); return 0; @@ -2857,6 +2852,14 @@ static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, return 1; } +/* + * NOTE: currently GUP for a hugepd is only possible on hugetlbfs file + * systems on Power, which does not have issue with folio writeback against + * GUP updates. When hugepd will be extended to support non-hugetlbfs or + * even anonymous memory, we need to do extra check as what we do with most + * of the other folios. See writable_file_mapping_allowed() and + * gup_fast_folio_allowed() for more information. + */ static int gup_huge_pd(hugepd_t hugepd, unsigned long addr, unsigned int pdshift, unsigned long end, unsigned int flags, struct page **pages, int *nr) From f3c94c625fe38823cc8010e60b9cc23fbf058ee3 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 27 Mar 2024 11:23:26 -0400 Subject: [PATCH 0684/1702] mm/gup: refactor record_subpages() to find 1st small page All the fast-gup functions take a tail page to operate, always need to do page mask calculations before feeding that into record_subpages(). Merge that logic into record_subpages(), so that it will do the nth_page() calculation. Link: https://lkml.kernel.org/r/20240327152332.950956-8-peterx@redhat.com Signed-off-by: Peter Xu Reviewed-by: Jason Gunthorpe Tested-by: Ryan Roberts Cc: Andrea Arcangeli Cc: Andrew Jones Cc: Aneesh Kumar K.V (IBM) Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: James Houghton Cc: John Hubbard Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Michael Ellerman Cc: "Mike Rapoport (IBM)" Cc: Muchun Song Cc: Rik van Riel Cc: Vlastimil Babka Cc: Yang Shi Signed-off-by: Andrew Morton --- mm/gup.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index 1cc3240e49001..e83e262ea8e95 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2789,13 +2789,16 @@ static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr, } #endif -static int record_subpages(struct page *page, unsigned long addr, - unsigned long end, struct page **pages) +static int record_subpages(struct page *page, unsigned long sz, + unsigned long addr, unsigned long end, + struct page **pages) { + struct page *start_page; int nr; + start_page = nth_page(page, (addr & (sz - 1)) >> PAGE_SHIFT); for (nr = 0; addr != end; nr++, addr += PAGE_SIZE) - pages[nr] = nth_page(page, nr); + pages[nr] = nth_page(start_page, nr); return nr; } @@ -2830,8 +2833,8 @@ static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, /* hugepages are never "special" */ VM_BUG_ON(!pfn_valid(pte_pfn(pte))); - page = nth_page(pte_page(pte), (addr & (sz - 1)) >> PAGE_SHIFT); - refs = record_subpages(page, addr, end, pages + *nr); + page = pte_page(pte); + refs = record_subpages(page, sz, addr, end, pages + *nr); folio = try_grab_folio(page, refs, flags); if (!folio) @@ -2904,8 +2907,8 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, pages, nr); } - page = nth_page(pmd_page(orig), (addr & ~PMD_MASK) >> PAGE_SHIFT); - refs = record_subpages(page, addr, end, pages + *nr); + page = pmd_page(orig); + refs = record_subpages(page, PMD_SIZE, addr, end, pages + *nr); folio = try_grab_folio(page, refs, flags); if (!folio) @@ -2948,8 +2951,8 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, pages, nr); } - page = nth_page(pud_page(orig), (addr & ~PUD_MASK) >> PAGE_SHIFT); - refs = record_subpages(page, addr, end, pages + *nr); + page = pud_page(orig); + refs = record_subpages(page, PUD_SIZE, addr, end, pages + *nr); folio = try_grab_folio(page, refs, flags); if (!folio) @@ -2988,8 +2991,8 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr, BUILD_BUG_ON(pgd_devmap(orig)); - page = nth_page(pgd_page(orig), (addr & ~PGDIR_MASK) >> PAGE_SHIFT); - refs = record_subpages(page, addr, end, pages + *nr); + page = pgd_page(orig); + refs = record_subpages(page, PGDIR_SIZE, addr, end, pages + *nr); folio = try_grab_folio(page, refs, flags); if (!folio) From 878b0c4516210b613468fea827f4cc0f71b87970 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 27 Mar 2024 11:23:27 -0400 Subject: [PATCH 0685/1702] mm/gup: handle hugetlb for no_page_table() no_page_table() is not yet used for hugetlb code paths. Make it prepared. The major difference here is hugetlb will return -EFAULT as long as page cache does not exist, even if VM_SHARED. See hugetlb_follow_page_mask(). Pass "address" into no_page_table() too, as hugetlb will need it. Link: https://lkml.kernel.org/r/20240327152332.950956-9-peterx@redhat.com Signed-off-by: Peter Xu Reviewed-by: Christoph Hellwig Reviewed-by: Jason Gunthorpe Tested-by: Ryan Roberts Cc: Andrea Arcangeli Cc: Andrew Jones Cc: Aneesh Kumar K.V (IBM) Cc: Axel Rasmussen Cc: Christophe Leroy Cc: David Hildenbrand Cc: James Houghton Cc: John Hubbard Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Michael Ellerman Cc: "Mike Rapoport (IBM)" Cc: Muchun Song Cc: Rik van Riel Cc: Vlastimil Babka Cc: Yang Shi Signed-off-by: Andrew Morton --- mm/gup.c | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index e83e262ea8e95..1a1e459fd1382 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -501,19 +501,27 @@ static inline void mm_set_has_pinned_flag(unsigned long *mm_flags) #ifdef CONFIG_MMU static struct page *no_page_table(struct vm_area_struct *vma, - unsigned int flags) + unsigned int flags, unsigned long address) { + if (!(flags & FOLL_DUMP)) + return NULL; + /* - * When core dumping an enormous anonymous area that nobody - * has touched so far, we don't want to allocate unnecessary pages or + * When core dumping, we don't want to allocate unnecessary pages or * page tables. Return error instead of NULL to skip handle_mm_fault, * then get_dump_page() will return NULL to leave a hole in the dump. * But we can only make this optimization where a hole would surely * be zero-filled if handle_mm_fault() actually did handle it. */ - if ((flags & FOLL_DUMP) && - (vma_is_anonymous(vma) || !vma->vm_ops->fault)) + if (is_vm_hugetlb_page(vma)) { + struct hstate *h = hstate_vma(vma); + + if (!hugetlbfs_pagecache_present(h, vma, address)) + return ERR_PTR(-EFAULT); + } else if ((vma_is_anonymous(vma) || !vma->vm_ops->fault)) { return ERR_PTR(-EFAULT); + } + return NULL; } @@ -593,7 +601,7 @@ static struct page *follow_page_pte(struct vm_area_struct *vma, ptep = pte_offset_map_lock(mm, pmd, address, &ptl); if (!ptep) - return no_page_table(vma, flags); + return no_page_table(vma, flags, address); pte = ptep_get(ptep); if (!pte_present(pte)) goto no_page; @@ -685,7 +693,7 @@ static struct page *follow_page_pte(struct vm_area_struct *vma, pte_unmap_unlock(ptep, ptl); if (!pte_none(pte)) return NULL; - return no_page_table(vma, flags); + return no_page_table(vma, flags, address); } static struct page *follow_pmd_mask(struct vm_area_struct *vma, @@ -701,27 +709,27 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma, pmd = pmd_offset(pudp, address); pmdval = pmdp_get_lockless(pmd); if (pmd_none(pmdval)) - return no_page_table(vma, flags); + return no_page_table(vma, flags, address); if (!pmd_present(pmdval)) - return no_page_table(vma, flags); + return no_page_table(vma, flags, address); if (pmd_devmap(pmdval)) { ptl = pmd_lock(mm, pmd); page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap); spin_unlock(ptl); if (page) return page; - return no_page_table(vma, flags); + return no_page_table(vma, flags, address); } if (likely(!pmd_trans_huge(pmdval))) return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); if (pmd_protnone(pmdval) && !gup_can_follow_protnone(vma, flags)) - return no_page_table(vma, flags); + return no_page_table(vma, flags, address); ptl = pmd_lock(mm, pmd); if (unlikely(!pmd_present(*pmd))) { spin_unlock(ptl); - return no_page_table(vma, flags); + return no_page_table(vma, flags, address); } if (unlikely(!pmd_trans_huge(*pmd))) { spin_unlock(ptl); @@ -752,17 +760,17 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma, pud = pud_offset(p4dp, address); if (pud_none(*pud)) - return no_page_table(vma, flags); + return no_page_table(vma, flags, address); if (pud_devmap(*pud)) { ptl = pud_lock(mm, pud); page = follow_devmap_pud(vma, address, pud, flags, &ctx->pgmap); spin_unlock(ptl); if (page) return page; - return no_page_table(vma, flags); + return no_page_table(vma, flags, address); } if (unlikely(pud_bad(*pud))) - return no_page_table(vma, flags); + return no_page_table(vma, flags, address); return follow_pmd_mask(vma, address, pud, flags, ctx); } @@ -777,10 +785,10 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma, p4dp = p4d_offset(pgdp, address); p4d = READ_ONCE(*p4dp); if (!p4d_present(p4d)) - return no_page_table(vma, flags); + return no_page_table(vma, flags, address); BUILD_BUG_ON(p4d_leaf(p4d)); if (unlikely(p4d_bad(p4d))) - return no_page_table(vma, flags); + return no_page_table(vma, flags, address); return follow_pud_mask(vma, address, p4dp, flags, ctx); } @@ -830,7 +838,7 @@ static struct page *follow_page_mask(struct vm_area_struct *vma, pgd = pgd_offset(mm, address); if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) - return no_page_table(vma, flags); + return no_page_table(vma, flags, address); return follow_p4d_mask(vma, address, pgd, flags, ctx); } From caf8cab7985794d7cc8b85143b11e258ea5c8a44 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 27 Mar 2024 11:23:28 -0400 Subject: [PATCH 0686/1702] mm/gup: cache *pudp in follow_pud_mask() Introduce "pud_t pud" in the function, so the code won't dereference *pudp multiple time. Not only because that looks less straightforward, but also because if the dereference really happened, it's not clear whether there can be race to see different *pudp values if it's being modified at the same time. Link: https://lkml.kernel.org/r/20240327152332.950956-10-peterx@redhat.com Signed-off-by: Peter Xu Acked-by: James Houghton Reviewed-by: Jason Gunthorpe Tested-by: Ryan Roberts Cc: Andrea Arcangeli Cc: Andrew Jones Cc: Aneesh Kumar K.V (IBM) Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: John Hubbard Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Michael Ellerman Cc: "Mike Rapoport (IBM)" Cc: Muchun Song Cc: Rik van Riel Cc: Vlastimil Babka Cc: Yang Shi Signed-off-by: Andrew Morton --- mm/gup.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index 1a1e459fd1382..39224b5fe62f1 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -753,26 +753,27 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma, unsigned int flags, struct follow_page_context *ctx) { - pud_t *pud; + pud_t *pudp, pud; spinlock_t *ptl; struct page *page; struct mm_struct *mm = vma->vm_mm; - pud = pud_offset(p4dp, address); - if (pud_none(*pud)) + pudp = pud_offset(p4dp, address); + pud = READ_ONCE(*pudp); + if (pud_none(pud)) return no_page_table(vma, flags, address); - if (pud_devmap(*pud)) { - ptl = pud_lock(mm, pud); - page = follow_devmap_pud(vma, address, pud, flags, &ctx->pgmap); + if (pud_devmap(pud)) { + ptl = pud_lock(mm, pudp); + page = follow_devmap_pud(vma, address, pudp, flags, &ctx->pgmap); spin_unlock(ptl); if (page) return page; return no_page_table(vma, flags, address); } - if (unlikely(pud_bad(*pud))) + if (unlikely(pud_bad(pud))) return no_page_table(vma, flags, address); - return follow_pmd_mask(vma, address, pud, flags, ctx); + return follow_pmd_mask(vma, address, pudp, flags, ctx); } static struct page *follow_p4d_mask(struct vm_area_struct *vma, From 1b1676180246232308885c4f37fee01cf898fdb2 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 27 Mar 2024 11:23:29 -0400 Subject: [PATCH 0687/1702] mm/gup: handle huge pud for follow_pud_mask() Teach follow_pud_mask() to be able to handle normal PUD pages like hugetlb. Rename follow_devmap_pud() to follow_huge_pud() so that it can process either huge devmap or hugetlb. Move it out of TRANSPARENT_HUGEPAGE_PUD and and huge_memory.c (which relies on CONFIG_THP). Switch to pud_leaf() to detect both cases in the slow gup. In the new follow_huge_pud(), taking care of possible CoR for hugetlb if necessary. touch_pud() needs to be moved out of huge_memory.c to be accessable from gup.c even if !THP. Since at it, optimize the non-present check by adding a pud_present() early check before taking the pgtable lock, failing the follow_page() early if PUD is not present: that is required by both devmap or hugetlb. Use pud_huge() to also cover the pud_devmap() case. One more trivial thing to mention is, introduce "pud_t pud" in the code paths along the way, so the code doesn't dereference *pudp multiple time. Not only because that looks less straightforward, but also because if the dereference really happened, it's not clear whether there can be race to see different *pudp values when it's being modified at the same time. Setting ctx->page_mask properly for a PUD entry. As a side effect, this patch should also be able to optimize devmap GUP on PUD to be able to jump over the whole PUD range, but not yet verified. Hugetlb already can do so prior to this patch. Link: https://lkml.kernel.org/r/20240327152332.950956-11-peterx@redhat.com Signed-off-by: Peter Xu Reviewed-by: Jason Gunthorpe Tested-by: Ryan Roberts Cc: Andrea Arcangeli Cc: Andrew Jones Cc: Aneesh Kumar K.V (IBM) Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: James Houghton Cc: John Hubbard Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Michael Ellerman Cc: "Mike Rapoport (IBM)" Cc: Muchun Song Cc: Rik van Riel Cc: Vlastimil Babka Cc: Yang Shi Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 8 ----- mm/gup.c | 70 +++++++++++++++++++++++++++++++++++++++-- mm/huge_memory.c | 47 ++------------------------- mm/internal.h | 2 ++ 4 files changed, 71 insertions(+), 56 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index ff72ee19a1254..ab26d9e65ec3b 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -351,8 +351,6 @@ static inline bool folio_test_pmd_mappable(struct folio *folio) struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd, int flags, struct dev_pagemap **pgmap); -struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr, - pud_t *pud, int flags, struct dev_pagemap **pgmap); vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf); @@ -507,12 +505,6 @@ static inline struct page *follow_devmap_pmd(struct vm_area_struct *vma, return NULL; } -static inline struct page *follow_devmap_pud(struct vm_area_struct *vma, - unsigned long addr, pud_t *pud, int flags, struct dev_pagemap **pgmap) -{ - return NULL; -} - static inline bool thp_migration_supported(void) { return false; diff --git a/mm/gup.c b/mm/gup.c index 39224b5fe62f1..2b06d59f2fa3b 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -525,6 +525,70 @@ static struct page *no_page_table(struct vm_area_struct *vma, return NULL; } +#ifdef CONFIG_PGTABLE_HAS_HUGE_LEAVES +static struct page *follow_huge_pud(struct vm_area_struct *vma, + unsigned long addr, pud_t *pudp, + int flags, struct follow_page_context *ctx) +{ + struct mm_struct *mm = vma->vm_mm; + struct page *page; + pud_t pud = *pudp; + unsigned long pfn = pud_pfn(pud); + int ret; + + assert_spin_locked(pud_lockptr(mm, pudp)); + + if ((flags & FOLL_WRITE) && !pud_write(pud)) + return NULL; + + if (!pud_present(pud)) + return NULL; + + pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT; + + if (IS_ENABLED(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) && + pud_devmap(pud)) { + /* + * device mapped pages can only be returned if the caller + * will manage the page reference count. + * + * At least one of FOLL_GET | FOLL_PIN must be set, so + * assert that here: + */ + if (!(flags & (FOLL_GET | FOLL_PIN))) + return ERR_PTR(-EEXIST); + + if (flags & FOLL_TOUCH) + touch_pud(vma, addr, pudp, flags & FOLL_WRITE); + + ctx->pgmap = get_dev_pagemap(pfn, ctx->pgmap); + if (!ctx->pgmap) + return ERR_PTR(-EFAULT); + } + + page = pfn_to_page(pfn); + + if (!pud_devmap(pud) && !pud_write(pud) && + gup_must_unshare(vma, flags, page)) + return ERR_PTR(-EMLINK); + + ret = try_grab_page(page, flags); + if (ret) + page = ERR_PTR(ret); + else + ctx->page_mask = HPAGE_PUD_NR - 1; + + return page; +} +#else /* CONFIG_PGTABLE_HAS_HUGE_LEAVES */ +static struct page *follow_huge_pud(struct vm_area_struct *vma, + unsigned long addr, pud_t *pudp, + int flags, struct follow_page_context *ctx) +{ + return NULL; +} +#endif /* CONFIG_PGTABLE_HAS_HUGE_LEAVES */ + static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address, pte_t *pte, unsigned int flags) { @@ -760,11 +824,11 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma, pudp = pud_offset(p4dp, address); pud = READ_ONCE(*pudp); - if (pud_none(pud)) + if (!pud_present(pud)) return no_page_table(vma, flags, address); - if (pud_devmap(pud)) { + if (pud_leaf(pud)) { ptl = pud_lock(mm, pudp); - page = follow_devmap_pud(vma, address, pudp, flags, &ctx->pgmap); + page = follow_huge_pud(vma, address, pudp, flags, ctx); spin_unlock(ptl); if (page) return page; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index c4820cd4749e0..249318f367e4c 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1378,8 +1378,8 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, } #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD -static void touch_pud(struct vm_area_struct *vma, unsigned long addr, - pud_t *pud, bool write) +void touch_pud(struct vm_area_struct *vma, unsigned long addr, + pud_t *pud, bool write) { pud_t _pud; @@ -1391,49 +1391,6 @@ static void touch_pud(struct vm_area_struct *vma, unsigned long addr, update_mmu_cache_pud(vma, addr, pud); } -struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr, - pud_t *pud, int flags, struct dev_pagemap **pgmap) -{ - unsigned long pfn = pud_pfn(*pud); - struct mm_struct *mm = vma->vm_mm; - struct page *page; - int ret; - - assert_spin_locked(pud_lockptr(mm, pud)); - - if (flags & FOLL_WRITE && !pud_write(*pud)) - return NULL; - - if (pud_present(*pud) && pud_devmap(*pud)) - /* pass */; - else - return NULL; - - if (flags & FOLL_TOUCH) - touch_pud(vma, addr, pud, flags & FOLL_WRITE); - - /* - * device mapped pages can only be returned if the - * caller will manage the page reference count. - * - * At least one of FOLL_GET | FOLL_PIN must be set, so assert that here: - */ - if (!(flags & (FOLL_GET | FOLL_PIN))) - return ERR_PTR(-EEXIST); - - pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT; - *pgmap = get_dev_pagemap(pfn, *pgmap); - if (!*pgmap) - return ERR_PTR(-EFAULT); - page = pfn_to_page(pfn); - - ret = try_grab_page(page, flags); - if (ret) - page = ERR_PTR(ret); - - return page; -} - int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, pud_t *dst_pud, pud_t *src_pud, unsigned long addr, struct vm_area_struct *vma) diff --git a/mm/internal.h b/mm/internal.h index 8e11f7b2da216..d06072f06a8d7 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1112,6 +1112,8 @@ int __must_check try_grab_page(struct page *page, unsigned int flags); /* * mm/huge_memory.c */ +void touch_pud(struct vm_area_struct *vma, unsigned long addr, + pud_t *pud, bool write); struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd, unsigned int flags); From 4418c522f683f2d73e9573847e98904c2b777654 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 27 Mar 2024 11:23:30 -0400 Subject: [PATCH 0688/1702] mm/gup: handle huge pmd for follow_pmd_mask() Replace pmd_trans_huge() with pmd_leaf() to also cover pmd_huge() as long as enabled. FOLL_TOUCH and FOLL_SPLIT_PMD only apply to THP, not yet huge. Since now follow_trans_huge_pmd() can process hugetlb pages, renaming it into follow_huge_pmd() to match what it does. Move it into gup.c so not depend on CONFIG_THP. When at it, move the ctx->page_mask setup into follow_huge_pmd(), only set it when the page is valid. It was not a bug to set it before even if GUP failed (page==NULL), because follow_page_mask() callers always ignores page_mask if so. But doing so makes the code cleaner. [peterx@redhat.com: allow follow_pmd_mask() to take hugetlb tail pages] Link: https://lkml.kernel.org/r/20240403013249.1418299-3-peterx@redhat.com Link: https://lkml.kernel.org/r/20240327152332.950956-12-peterx@redhat.com Signed-off-by: Peter Xu Reviewed-by: Jason Gunthorpe Tested-by: Ryan Roberts Cc: Andrea Arcangeli Cc: Andrew Jones Cc: Aneesh Kumar K.V (IBM) Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: James Houghton Cc: John Hubbard Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Michael Ellerman Cc: "Mike Rapoport (IBM)" Cc: Muchun Song Cc: Rik van Riel Cc: Vlastimil Babka Cc: Yang Shi Signed-off-by: Andrew Morton --- mm/gup.c | 104 ++++++++++++++++++++++++++++++++++++++++++++--- mm/huge_memory.c | 86 +-------------------------------------- mm/internal.h | 5 +-- 3 files changed, 102 insertions(+), 93 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index 2b06d59f2fa3b..f7028c698c265 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -580,6 +580,90 @@ static struct page *follow_huge_pud(struct vm_area_struct *vma, return page; } + +/* FOLL_FORCE can write to even unwritable PMDs in COW mappings. */ +static inline bool can_follow_write_pmd(pmd_t pmd, struct page *page, + struct vm_area_struct *vma, + unsigned int flags) +{ + /* If the pmd is writable, we can write to the page. */ + if (pmd_write(pmd)) + return true; + + /* Maybe FOLL_FORCE is set to override it? */ + if (!(flags & FOLL_FORCE)) + return false; + + /* But FOLL_FORCE has no effect on shared mappings */ + if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED)) + return false; + + /* ... or read-only private ones */ + if (!(vma->vm_flags & VM_MAYWRITE)) + return false; + + /* ... or already writable ones that just need to take a write fault */ + if (vma->vm_flags & VM_WRITE) + return false; + + /* + * See can_change_pte_writable(): we broke COW and could map the page + * writable if we have an exclusive anonymous page ... + */ + if (!page || !PageAnon(page) || !PageAnonExclusive(page)) + return false; + + /* ... and a write-fault isn't required for other reasons. */ + if (vma_soft_dirty_enabled(vma) && !pmd_soft_dirty(pmd)) + return false; + return !userfaultfd_huge_pmd_wp(vma, pmd); +} + +static struct page *follow_huge_pmd(struct vm_area_struct *vma, + unsigned long addr, pmd_t *pmd, + unsigned int flags, + struct follow_page_context *ctx) +{ + struct mm_struct *mm = vma->vm_mm; + pmd_t pmdval = *pmd; + struct page *page; + int ret; + + assert_spin_locked(pmd_lockptr(mm, pmd)); + + page = pmd_page(pmdval); + if ((flags & FOLL_WRITE) && + !can_follow_write_pmd(pmdval, page, vma, flags)) + return NULL; + + /* Avoid dumping huge zero page */ + if ((flags & FOLL_DUMP) && is_huge_zero_pmd(pmdval)) + return ERR_PTR(-EFAULT); + + if (pmd_protnone(*pmd) && !gup_can_follow_protnone(vma, flags)) + return NULL; + + if (!pmd_write(pmdval) && gup_must_unshare(vma, flags, page)) + return ERR_PTR(-EMLINK); + + VM_BUG_ON_PAGE((flags & FOLL_PIN) && PageAnon(page) && + !PageAnonExclusive(page), page); + + ret = try_grab_page(page, flags); + if (ret) + return ERR_PTR(ret); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (pmd_trans_huge(pmdval) && (flags & FOLL_TOUCH)) + touch_pmd(vma, addr, pmd, flags & FOLL_WRITE); +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + + page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT; + ctx->page_mask = HPAGE_PMD_NR - 1; + + return page; +} + #else /* CONFIG_PGTABLE_HAS_HUGE_LEAVES */ static struct page *follow_huge_pud(struct vm_area_struct *vma, unsigned long addr, pud_t *pudp, @@ -587,6 +671,14 @@ static struct page *follow_huge_pud(struct vm_area_struct *vma, { return NULL; } + +static struct page *follow_huge_pmd(struct vm_area_struct *vma, + unsigned long addr, pmd_t *pmd, + unsigned int flags, + struct follow_page_context *ctx) +{ + return NULL; +} #endif /* CONFIG_PGTABLE_HAS_HUGE_LEAVES */ static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address, @@ -784,31 +876,31 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma, return page; return no_page_table(vma, flags, address); } - if (likely(!pmd_trans_huge(pmdval))) + if (likely(!pmd_leaf(pmdval))) return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); if (pmd_protnone(pmdval) && !gup_can_follow_protnone(vma, flags)) return no_page_table(vma, flags, address); ptl = pmd_lock(mm, pmd); - if (unlikely(!pmd_present(*pmd))) { + pmdval = *pmd; + if (unlikely(!pmd_present(pmdval))) { spin_unlock(ptl); return no_page_table(vma, flags, address); } - if (unlikely(!pmd_trans_huge(*pmd))) { + if (unlikely(!pmd_leaf(pmdval))) { spin_unlock(ptl); return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); } - if (flags & FOLL_SPLIT_PMD) { + if (pmd_trans_huge(pmdval) && (flags & FOLL_SPLIT_PMD)) { spin_unlock(ptl); split_huge_pmd(vma, pmd, address); /* If pmd was left empty, stuff a page table in there quickly */ return pte_alloc(mm, pmd) ? ERR_PTR(-ENOMEM) : follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); } - page = follow_trans_huge_pmd(vma, address, pmd, flags); + page = follow_huge_pmd(vma, address, pmd, flags, ctx); spin_unlock(ptl); - ctx->page_mask = HPAGE_PMD_NR - 1; return page; } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 249318f367e4c..157cee64850c8 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1221,8 +1221,8 @@ vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write) EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud); #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ -static void touch_pmd(struct vm_area_struct *vma, unsigned long addr, - pmd_t *pmd, bool write) +void touch_pmd(struct vm_area_struct *vma, unsigned long addr, + pmd_t *pmd, bool write) { pmd_t _pmd; @@ -1577,88 +1577,6 @@ static inline bool can_change_pmd_writable(struct vm_area_struct *vma, return pmd_dirty(pmd); } -/* FOLL_FORCE can write to even unwritable PMDs in COW mappings. */ -static inline bool can_follow_write_pmd(pmd_t pmd, struct page *page, - struct vm_area_struct *vma, - unsigned int flags) -{ - /* If the pmd is writable, we can write to the page. */ - if (pmd_write(pmd)) - return true; - - /* Maybe FOLL_FORCE is set to override it? */ - if (!(flags & FOLL_FORCE)) - return false; - - /* But FOLL_FORCE has no effect on shared mappings */ - if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED)) - return false; - - /* ... or read-only private ones */ - if (!(vma->vm_flags & VM_MAYWRITE)) - return false; - - /* ... or already writable ones that just need to take a write fault */ - if (vma->vm_flags & VM_WRITE) - return false; - - /* - * See can_change_pte_writable(): we broke COW and could map the page - * writable if we have an exclusive anonymous page ... - */ - if (!page || !PageAnon(page) || !PageAnonExclusive(page)) - return false; - - /* ... and a write-fault isn't required for other reasons. */ - if (vma_soft_dirty_enabled(vma) && !pmd_soft_dirty(pmd)) - return false; - return !userfaultfd_huge_pmd_wp(vma, pmd); -} - -struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, - unsigned long addr, - pmd_t *pmd, - unsigned int flags) -{ - struct mm_struct *mm = vma->vm_mm; - struct page *page; - int ret; - - assert_spin_locked(pmd_lockptr(mm, pmd)); - - page = pmd_page(*pmd); - VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page); - - if ((flags & FOLL_WRITE) && - !can_follow_write_pmd(*pmd, page, vma, flags)) - return NULL; - - /* Avoid dumping huge zero page */ - if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd)) - return ERR_PTR(-EFAULT); - - if (pmd_protnone(*pmd) && !gup_can_follow_protnone(vma, flags)) - return NULL; - - if (!pmd_write(*pmd) && gup_must_unshare(vma, flags, page)) - return ERR_PTR(-EMLINK); - - VM_BUG_ON_PAGE((flags & FOLL_PIN) && PageAnon(page) && - !PageAnonExclusive(page), page); - - ret = try_grab_page(page, flags); - if (ret) - return ERR_PTR(ret); - - if (flags & FOLL_TOUCH) - touch_pmd(vma, addr, pmd, flags & FOLL_WRITE); - - page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT; - VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page); - - return page; -} - /* NUMA hinting page fault entry point for trans huge pmds */ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) { diff --git a/mm/internal.h b/mm/internal.h index d06072f06a8d7..5933a8e7d2dd5 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1114,9 +1114,8 @@ int __must_check try_grab_page(struct page *page, unsigned int flags); */ void touch_pud(struct vm_area_struct *vma, unsigned long addr, pud_t *pud, bool write); -struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, - unsigned long addr, pmd_t *pmd, - unsigned int flags); +void touch_pmd(struct vm_area_struct *vma, unsigned long addr, + pmd_t *pmd, bool write); /* * mm/mmap.c From a12083d721d703f985f4403d6b333cc449f838f6 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 27 Mar 2024 11:23:31 -0400 Subject: [PATCH 0689/1702] mm/gup: handle hugepd for follow_page() Hugepd is only used in PowerPC so far on 4K page size kernels where hash mmu is used. follow_page_mask() used to leverage hugetlb APIs to access hugepd entries. Teach follow_page_mask() itself on hugepd. With previous refactors on fast-gup gup_huge_pd(), most of the code can be leveraged. There's something not needed for follow page, for example, gup_hugepte() tries to detect pgtable entry change which will never happen with slow gup (which has the pgtable lock held), but that's not a problem to check. Since follow_page() always only fetch one page, set the end to "address + PAGE_SIZE" should suffice. We will still do the pgtable walk once for each hugetlb page by setting ctx->page_mask properly. One thing worth mentioning is that some level of pgtable's _bad() helper will report is_hugepd() entries as TRUE on Power8 hash MMUs. I think it at least applies to PUD on Power8 with 4K pgsize. It means feeding a hugepd entry to pud_bad() will report a false positive. Let's leave that for now because it can be arch-specific where I am a bit declined to touch. In this patch it's not a problem as long as hugepd is detected before any bad pgtable entries. To allow slow gup like follow_*_page() to access hugepd helpers, hugepd codes are moved to the top. Besides that, the helper record_subpages() will be used by either hugepd or fast-gup now. To avoid "unused function" warnings we must provide a "#ifdef" for it, unfortunately. Link: https://lkml.kernel.org/r/20240327152332.950956-13-peterx@redhat.com Signed-off-by: Peter Xu Tested-by: Ryan Roberts Cc: Andrea Arcangeli Cc: Andrew Jones Cc: Aneesh Kumar K.V (IBM) Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: James Houghton Cc: Jason Gunthorpe Cc: John Hubbard Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Michael Ellerman Cc: "Mike Rapoport (IBM)" Cc: Muchun Song Cc: Rik van Riel Cc: Vlastimil Babka Cc: Yang Shi Signed-off-by: Andrew Morton --- mm/gup.c | 269 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 163 insertions(+), 106 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index f7028c698c265..a572a1169aa72 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -500,6 +500,149 @@ static inline void mm_set_has_pinned_flag(unsigned long *mm_flags) } #ifdef CONFIG_MMU + +#if defined(CONFIG_ARCH_HAS_HUGEPD) || defined(CONFIG_HAVE_FAST_GUP) +static int record_subpages(struct page *page, unsigned long sz, + unsigned long addr, unsigned long end, + struct page **pages) +{ + struct page *start_page; + int nr; + + start_page = nth_page(page, (addr & (sz - 1)) >> PAGE_SHIFT); + for (nr = 0; addr != end; nr++, addr += PAGE_SIZE) + pages[nr] = nth_page(start_page, nr); + + return nr; +} +#endif /* CONFIG_ARCH_HAS_HUGEPD || CONFIG_HAVE_FAST_GUP */ + +#ifdef CONFIG_ARCH_HAS_HUGEPD +static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end, + unsigned long sz) +{ + unsigned long __boundary = (addr + sz) & ~(sz-1); + return (__boundary - 1 < end - 1) ? __boundary : end; +} + +static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, + unsigned long end, unsigned int flags, + struct page **pages, int *nr) +{ + unsigned long pte_end; + struct page *page; + struct folio *folio; + pte_t pte; + int refs; + + pte_end = (addr + sz) & ~(sz-1); + if (pte_end < end) + end = pte_end; + + pte = huge_ptep_get(ptep); + + if (!pte_access_permitted(pte, flags & FOLL_WRITE)) + return 0; + + /* hugepages are never "special" */ + VM_BUG_ON(!pfn_valid(pte_pfn(pte))); + + page = pte_page(pte); + refs = record_subpages(page, sz, addr, end, pages + *nr); + + folio = try_grab_folio(page, refs, flags); + if (!folio) + return 0; + + if (unlikely(pte_val(pte) != pte_val(ptep_get(ptep)))) { + gup_put_folio(folio, refs, flags); + return 0; + } + + if (!pte_write(pte) && gup_must_unshare(NULL, flags, &folio->page)) { + gup_put_folio(folio, refs, flags); + return 0; + } + + *nr += refs; + folio_set_referenced(folio); + return 1; +} + +/* + * NOTE: currently GUP for a hugepd is only possible on hugetlbfs file + * systems on Power, which does not have issue with folio writeback against + * GUP updates. When hugepd will be extended to support non-hugetlbfs or + * even anonymous memory, we need to do extra check as what we do with most + * of the other folios. See writable_file_mapping_allowed() and + * gup_fast_folio_allowed() for more information. + */ +static int gup_huge_pd(hugepd_t hugepd, unsigned long addr, + unsigned int pdshift, unsigned long end, unsigned int flags, + struct page **pages, int *nr) +{ + pte_t *ptep; + unsigned long sz = 1UL << hugepd_shift(hugepd); + unsigned long next; + + ptep = hugepte_offset(hugepd, addr, pdshift); + do { + next = hugepte_addr_end(addr, end, sz); + if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr)) + return 0; + } while (ptep++, addr = next, addr != end); + + return 1; +} + +static struct page *follow_hugepd(struct vm_area_struct *vma, hugepd_t hugepd, + unsigned long addr, unsigned int pdshift, + unsigned int flags, + struct follow_page_context *ctx) +{ + struct page *page; + struct hstate *h; + spinlock_t *ptl; + int nr = 0, ret; + pte_t *ptep; + + /* Only hugetlb supports hugepd */ + if (WARN_ON_ONCE(!is_vm_hugetlb_page(vma))) + return ERR_PTR(-EFAULT); + + h = hstate_vma(vma); + ptep = hugepte_offset(hugepd, addr, pdshift); + ptl = huge_pte_lock(h, vma->vm_mm, ptep); + ret = gup_huge_pd(hugepd, addr, pdshift, addr + PAGE_SIZE, + flags, &page, &nr); + spin_unlock(ptl); + + if (ret) { + WARN_ON_ONCE(nr != 1); + ctx->page_mask = (1U << huge_page_order(h)) - 1; + return page; + } + + return NULL; +} +#else /* CONFIG_ARCH_HAS_HUGEPD */ +static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr, + unsigned int pdshift, unsigned long end, unsigned int flags, + struct page **pages, int *nr) +{ + return 0; +} + +static struct page *follow_hugepd(struct vm_area_struct *vma, hugepd_t hugepd, + unsigned long addr, unsigned int pdshift, + unsigned int flags, + struct follow_page_context *ctx) +{ + return NULL; +} +#endif /* CONFIG_ARCH_HAS_HUGEPD */ + + static struct page *no_page_table(struct vm_area_struct *vma, unsigned int flags, unsigned long address) { @@ -868,6 +1011,9 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma, return no_page_table(vma, flags, address); if (!pmd_present(pmdval)) return no_page_table(vma, flags, address); + if (unlikely(is_hugepd(__hugepd(pmd_val(pmdval))))) + return follow_hugepd(vma, __hugepd(pmd_val(pmdval)), + address, PMD_SHIFT, flags, ctx); if (pmd_devmap(pmdval)) { ptl = pmd_lock(mm, pmd); page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap); @@ -918,6 +1064,9 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma, pud = READ_ONCE(*pudp); if (!pud_present(pud)) return no_page_table(vma, flags, address); + if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) + return follow_hugepd(vma, __hugepd(pud_val(pud)), + address, PUD_SHIFT, flags, ctx); if (pud_leaf(pud)) { ptl = pud_lock(mm, pudp); page = follow_huge_pud(vma, address, pudp, flags, ctx); @@ -941,10 +1090,13 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma, p4dp = p4d_offset(pgdp, address); p4d = READ_ONCE(*p4dp); - if (!p4d_present(p4d)) - return no_page_table(vma, flags, address); BUILD_BUG_ON(p4d_leaf(p4d)); - if (unlikely(p4d_bad(p4d))) + + if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) + return follow_hugepd(vma, __hugepd(p4d_val(p4d)), + address, P4D_SHIFT, flags, ctx); + + if (!p4d_present(p4d) || p4d_bad(p4d)) return no_page_table(vma, flags, address); return follow_pud_mask(vma, address, p4dp, flags, ctx); @@ -994,10 +1146,15 @@ static struct page *follow_page_mask(struct vm_area_struct *vma, pgd = pgd_offset(mm, address); - if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) - return no_page_table(vma, flags, address); + if (unlikely(is_hugepd(__hugepd(pgd_val(*pgd))))) + page = follow_hugepd(vma, __hugepd(pgd_val(*pgd)), + address, PGDIR_SHIFT, flags, ctx); + else if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) + page = no_page_table(vma, flags, address); + else + page = follow_p4d_mask(vma, address, pgd, flags, ctx); - return follow_p4d_mask(vma, address, pgd, flags, ctx); + return page; } struct page *follow_page(struct vm_area_struct *vma, unsigned long address, @@ -2954,106 +3111,6 @@ static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr, } #endif -static int record_subpages(struct page *page, unsigned long sz, - unsigned long addr, unsigned long end, - struct page **pages) -{ - struct page *start_page; - int nr; - - start_page = nth_page(page, (addr & (sz - 1)) >> PAGE_SHIFT); - for (nr = 0; addr != end; nr++, addr += PAGE_SIZE) - pages[nr] = nth_page(start_page, nr); - - return nr; -} - -#ifdef CONFIG_ARCH_HAS_HUGEPD -static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end, - unsigned long sz) -{ - unsigned long __boundary = (addr + sz) & ~(sz-1); - return (__boundary - 1 < end - 1) ? __boundary : end; -} - -static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, - unsigned long end, unsigned int flags, - struct page **pages, int *nr) -{ - unsigned long pte_end; - struct page *page; - struct folio *folio; - pte_t pte; - int refs; - - pte_end = (addr + sz) & ~(sz-1); - if (pte_end < end) - end = pte_end; - - pte = huge_ptep_get(ptep); - - if (!pte_access_permitted(pte, flags & FOLL_WRITE)) - return 0; - - /* hugepages are never "special" */ - VM_BUG_ON(!pfn_valid(pte_pfn(pte))); - - page = pte_page(pte); - refs = record_subpages(page, sz, addr, end, pages + *nr); - - folio = try_grab_folio(page, refs, flags); - if (!folio) - return 0; - - if (unlikely(pte_val(pte) != pte_val(ptep_get(ptep)))) { - gup_put_folio(folio, refs, flags); - return 0; - } - - if (!pte_write(pte) && gup_must_unshare(NULL, flags, &folio->page)) { - gup_put_folio(folio, refs, flags); - return 0; - } - - *nr += refs; - folio_set_referenced(folio); - return 1; -} - -/* - * NOTE: currently GUP for a hugepd is only possible on hugetlbfs file - * systems on Power, which does not have issue with folio writeback against - * GUP updates. When hugepd will be extended to support non-hugetlbfs or - * even anonymous memory, we need to do extra check as what we do with most - * of the other folios. See writable_file_mapping_allowed() and - * gup_fast_folio_allowed() for more information. - */ -static int gup_huge_pd(hugepd_t hugepd, unsigned long addr, - unsigned int pdshift, unsigned long end, unsigned int flags, - struct page **pages, int *nr) -{ - pte_t *ptep; - unsigned long sz = 1UL << hugepd_shift(hugepd); - unsigned long next; - - ptep = hugepte_offset(hugepd, addr, pdshift); - do { - next = hugepte_addr_end(addr, end, sz); - if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr)) - return 0; - } while (ptep++, addr = next, addr != end); - - return 1; -} -#else -static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr, - unsigned int pdshift, unsigned long end, unsigned int flags, - struct page **pages, int *nr) -{ - return 0; -} -#endif /* CONFIG_ARCH_HAS_HUGEPD */ - static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, unsigned long end, unsigned int flags, struct page **pages, int *nr) From 9cb28da54643ad464c47585cd5866c30b0218e67 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 27 Mar 2024 11:23:32 -0400 Subject: [PATCH 0690/1702] mm/gup: handle hugetlb in the generic follow_page_mask code Now follow_page() is ready to handle hugetlb pages in whatever form, and over all architectures. Switch to the generic code path. Time to retire hugetlb_follow_page_mask(), following the previous retirement of follow_hugetlb_page() in 4849807114b8. There may be a slight difference of how the loops run when processing slow GUP over a large hugetlb range on cont_pte/cont_pmd supported archs: each loop of __get_user_pages() will resolve one pgtable entry with the patch applied, rather than relying on the size of hugetlb hstate, the latter may cover multiple entries in one loop. A quick performance test on an aarch64 VM on M1 chip shows 15% degrade over a tight loop of slow gup after the path switched. That shouldn't be a problem because slow-gup should not be a hot path for GUP in general: when page is commonly present, fast-gup will already succeed, while when the page is indeed missing and require a follow up page fault, the slow gup degrade will probably buried in the fault paths anyway. It also explains why slow gup for THP used to be very slow before 57edfcfd3419 ("mm/gup: accelerate thp gup even for "pages != NULL"") lands, the latter not part of a performance analysis but a side benefit. If the performance will be a concern, we can consider handle CONT_PTE in follow_page(). Before that is justified to be necessary, keep everything clean and simple. Link: https://lkml.kernel.org/r/20240327152332.950956-14-peterx@redhat.com Signed-off-by: Peter Xu Reviewed-by: Jason Gunthorpe Tested-by: Ryan Roberts Cc: Andrea Arcangeli Cc: Andrew Jones Cc: Aneesh Kumar K.V (IBM) Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: James Houghton Cc: John Hubbard Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Michael Ellerman Cc: "Mike Rapoport (IBM)" Cc: Muchun Song Cc: Rik van Riel Cc: Vlastimil Babka Cc: Yang Shi Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 7 ---- mm/gup.c | 15 +++------ mm/hugetlb.c | 71 ----------------------------------------- 3 files changed, 5 insertions(+), 88 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index a7988c78d69fa..3f3e628802792 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -328,13 +328,6 @@ static inline void hugetlb_zap_end( { } -static inline struct page *hugetlb_follow_page_mask( - struct vm_area_struct *vma, unsigned long address, unsigned int flags, - unsigned int *page_mask) -{ - BUILD_BUG(); /* should never be compiled in if !CONFIG_HUGETLB_PAGE*/ -} - static inline int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, struct vm_area_struct *dst_vma, diff --git a/mm/gup.c b/mm/gup.c index a572a1169aa72..fbf80e16bc949 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1132,18 +1132,11 @@ static struct page *follow_page_mask(struct vm_area_struct *vma, { pgd_t *pgd; struct mm_struct *mm = vma->vm_mm; + struct page *page; - ctx->page_mask = 0; - - /* - * Call hugetlb_follow_page_mask for hugetlb vmas as it will use - * special hugetlb page table walking code. This eliminates the - * need to check for hugetlb entries in the general walking code. - */ - if (is_vm_hugetlb_page(vma)) - return hugetlb_follow_page_mask(vma, address, flags, - &ctx->page_mask); + vma_pgtable_walk_begin(vma); + ctx->page_mask = 0; pgd = pgd_offset(mm, address); if (unlikely(is_hugepd(__hugepd(pgd_val(*pgd))))) @@ -1154,6 +1147,8 @@ static struct page *follow_page_mask(struct vm_area_struct *vma, else page = follow_p4d_mask(vma, address, pgd, flags, ctx); + vma_pgtable_walk_end(vma); + return page; } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 740435567be86..8af2fd48f785d 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6876,77 +6876,6 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, } #endif /* CONFIG_USERFAULTFD */ -struct page *hugetlb_follow_page_mask(struct vm_area_struct *vma, - unsigned long address, unsigned int flags, - unsigned int *page_mask) -{ - struct hstate *h = hstate_vma(vma); - struct mm_struct *mm = vma->vm_mm; - unsigned long haddr = address & huge_page_mask(h); - struct page *page = NULL; - spinlock_t *ptl; - pte_t *pte, entry; - int ret; - - hugetlb_vma_lock_read(vma); - pte = hugetlb_walk(vma, haddr, huge_page_size(h)); - if (!pte) - goto out_unlock; - - ptl = huge_pte_lock(h, mm, pte); - entry = huge_ptep_get(pte); - if (pte_present(entry)) { - page = pte_page(entry); - - if (!huge_pte_write(entry)) { - if (flags & FOLL_WRITE) { - page = NULL; - goto out; - } - - if (gup_must_unshare(vma, flags, page)) { - /* Tell the caller to do unsharing */ - page = ERR_PTR(-EMLINK); - goto out; - } - } - - page = nth_page(page, ((address & ~huge_page_mask(h)) >> PAGE_SHIFT)); - - /* - * Note that page may be a sub-page, and with vmemmap - * optimizations the page struct may be read only. - * try_grab_page() will increase the ref count on the - * head page, so this will be OK. - * - * try_grab_page() should always be able to get the page here, - * because we hold the ptl lock and have verified pte_present(). - */ - ret = try_grab_page(page, flags); - - if (WARN_ON_ONCE(ret)) { - page = ERR_PTR(ret); - goto out; - } - - *page_mask = (1U << huge_page_order(h)) - 1; - } -out: - spin_unlock(ptl); -out_unlock: - hugetlb_vma_unlock_read(vma); - - /* - * Fixup retval for dump requests: if pagecache doesn't exist, - * don't try to allocate a new page but just skip it. - */ - if (!page && (flags & FOLL_DUMP) && - !hugetlbfs_pagecache_present(h, vma, address)) - page = ERR_PTR(-EFAULT); - - return page; -} - long hugetlb_change_protection(struct vm_area_struct *vma, unsigned long address, unsigned long end, pgprot_t newprot, unsigned long cp_flags) From c0bff412e67b781d761e330ff9578aa9ed2be79e Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Tue, 2 Apr 2024 21:32:47 -0400 Subject: [PATCH 0691/1702] mm: allow anon exclusive check over hugetlb tail pages PageAnonExclusive() used to forbid tail pages for hugetlbfs, as that used to be called mostly in hugetlb specific paths and the head page was guaranteed. As we move forward towards merging hugetlb paths into generic mm, we may start to pass in tail hugetlb pages (when with cont-pte/cont-pmd huge pages) for such check. Allow it to properly fetch the head, in which case the anon-exclusiveness of the head will always represents the tail page. There's already a sign of it when we look at the GUP-fast which already contain the hugetlb processing altogether: we used to have a specific commit 5805192c7b72 ("mm/gup: handle cont-PTE hugetlb pages correctly in gup_must_unshare() via GUP-fast") covering that area. Now with this more generic change, that can also go away. [akpm@linux-foundation.org: simplify PageAnonExclusive(), per Matthew] Link: https://lkml.kernel.org/r/Zg3u5Sh9EbbYPhaI@casper.infradead.org Link: https://lkml.kernel.org/r/20240403013249.1418299-2-peterx@redhat.com Signed-off-by: Peter Xu Acked-by: David Hildenbrand Cc: Huacai Chen Cc: Jason Gunthorpe Cc: Matthew Wilcox (Oracle) Cc: Nathan Chancellor Cc: Ryan Roberts Cc: WANG Xuerui Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 7 ++++++- mm/internal.h | 10 ---------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 888353c209c03..7577fe7debafc 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -1095,7 +1095,12 @@ PAGEFLAG(Isolated, isolated, PF_ANY); static __always_inline int PageAnonExclusive(const struct page *page) { VM_BUG_ON_PGFLAGS(!PageAnon(page), page); - VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page); + /* + * HugeTLB stores this information on the head page; THP keeps it per + * page + */ + if (PageHuge(page)) + page = compound_head(page); return test_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags); } diff --git a/mm/internal.h b/mm/internal.h index 5933a8e7d2dd5..6614ba4ca9dec 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1203,16 +1203,6 @@ static inline bool gup_must_unshare(struct vm_area_struct *vma, if (IS_ENABLED(CONFIG_HAVE_FAST_GUP)) smp_rmb(); - /* - * During GUP-fast we might not get called on the head page for a - * hugetlb page that is mapped using cont-PTE, because GUP-fast does - * not work with the abstracted hugetlb PTEs that always point at the - * head page. For hugetlb, PageAnonExclusive only applies on the head - * page (as it cannot be partially COW-shared), so lookup the head page. - */ - if (unlikely(!PageHead(page) && PageHuge(page))) - page = compound_head(page); - /* * Note that PageKsm() pages cannot be exclusive, and consequently, * cannot get pinned. From ba168b52bf8ecb877af294769b0ab32b06bd9293 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 27 Mar 2024 19:06:59 +0000 Subject: [PATCH 0692/1702] mm: use rwsem assertion macros for mmap_lock This slightly strengthens our write assertion when lockdep is disabled. It also downgrades us from BUG_ON to WARN_ON, but I think that's an improvement. I don't think dumping the mm_struct was all that valuable; the call chain is what's important. Link: https://lkml.kernel.org/r/20240327190701.1082560-1-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/mmap_lock.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index 8d38dcb6d044c..de9dc20b01ba7 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -60,16 +60,14 @@ static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write) #endif /* CONFIG_TRACING */ -static inline void mmap_assert_locked(struct mm_struct *mm) +static inline void mmap_assert_locked(const struct mm_struct *mm) { - lockdep_assert_held(&mm->mmap_lock); - VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); + rwsem_assert_held(&mm->mmap_lock); } -static inline void mmap_assert_write_locked(struct mm_struct *mm) +static inline void mmap_assert_write_locked(const struct mm_struct *mm) { - lockdep_assert_held_write(&mm->mmap_lock); - VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); + rwsem_assert_held_write(&mm->mmap_lock); } #ifdef CONFIG_PER_VMA_LOCK From 07db63a2161a66a2198bc20c7580dd95af887ff4 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 27 Mar 2024 18:54:29 +0000 Subject: [PATCH 0693/1702] filemap: remove __set_page_dirty() All callers have been converted to use folios; remove this wrapper. Link: https://lkml.kernel.org/r/20240327185447.1076689-1-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/pagemap.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 35636e67e2e14..ec8b83928c306 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1147,11 +1147,6 @@ void folio_end_writeback(struct folio *folio); void wait_for_stable_page(struct page *page); void folio_wait_stable(struct folio *folio); void __folio_mark_dirty(struct folio *folio, struct address_space *, int warn); -static inline void __set_page_dirty(struct page *page, - struct address_space *mapping, int warn) -{ - __folio_mark_dirty(page_folio(page), mapping, warn); -} void folio_account_cleaned(struct folio *folio, struct bdi_writeback *wb); void __folio_cancel_dirty(struct folio *folio); static inline void folio_cancel_dirty(struct folio *folio) From 3b612c8f069fecb84af277c8b90e14832749ceca Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 27 Mar 2024 15:35:48 +0100 Subject: [PATCH 0694/1702] mm: optimize CONFIG_PER_VMA_LOCK member placement in vm_area_struct Currently, we end up wasting some memory in each vm_area_struct. Pahole states that: [...] int vm_lock_seq; /* 40 4 */ /* XXX 4 bytes hole, try to pack */ struct vma_lock * vm_lock; /* 48 8 */ bool detached; /* 56 1 */ /* XXX 7 bytes hole, try to pack */ [...] Let's reduce the holes and memory wastage by moving the bool: [...] bool detached; /* 40 1 */ /* XXX 3 bytes hole, try to pack */ int vm_lock_seq; /* 44 4 */ struct vma_lock * vm_lock; /* 48 8 */ [...] Effectively shrinking the vm_area_struct with CONFIG_PER_VMA_LOCK by 8 byte. Likely, we could place "detached" in the lowest bit of vm_lock, but at least on 64bit that won't really make a difference, so keep it simple. Link: https://lkml.kernel.org/r/20240327143548.744070-1-david@redhat.com Signed-off-by: David Hildenbrand Cc: Suren Baghdasaryan Signed-off-by: Andrew Morton --- include/linux/mm_types.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 6fef56938a175..fee6561feb7eb 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -671,6 +671,9 @@ struct vm_area_struct { }; #ifdef CONFIG_PER_VMA_LOCK + /* Flag to indicate areas detached from the mm->mm_mt tree */ + bool detached; + /* * Can only be written (using WRITE_ONCE()) while holding both: * - mmap_lock (in write mode) @@ -687,9 +690,6 @@ struct vm_area_struct { */ int vm_lock_seq; struct vma_lock *vm_lock; - - /* Flag to indicate areas detached from the mm->mm_mt tree */ - bool detached; #endif /* From 82a616d0f33b77b1cadd0652efbe11874771320f Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 27 Mar 2024 15:33:01 +0100 Subject: [PATCH 0695/1702] mm: remove "prot" parameter from move_pte() The "prot" parameter is unused, and using it instead of what's stored in that particular PTE would very likely be wrong. Let's simply remove it. Link: https://lkml.kernel.org/r/20240327143301.741807-1-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Vishal Moola (Oracle) Cc: "David S. Miller" Cc: Andreas Larsson Signed-off-by: Andrew Morton --- arch/sparc/include/asm/pgtable_64.h | 2 +- include/linux/pgtable.h | 2 +- mm/mremap.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index 26efc9bb644aa..3fe429d73a65b 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -957,7 +957,7 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr, #ifdef DCACHE_ALIASING_POSSIBLE #define __HAVE_ARCH_MOVE_PTE -#define move_pte(pte, prot, old_addr, new_addr) \ +#define move_pte(pte, old_addr, new_addr) \ ({ \ pte_t newpte = (pte); \ if (tlb_type != hypervisor && pte_present(pte)) { \ diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index f108c7a3c1d9e..a3fc8150b047d 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1079,7 +1079,7 @@ static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio) #endif #ifndef __HAVE_ARCH_MOVE_PTE -#define move_pte(pte, prot, old_addr, new_addr) (pte) +#define move_pte(pte, old_addr, new_addr) (pte) #endif #ifndef pte_accessible diff --git a/mm/mremap.c b/mm/mremap.c index 38d98465f3d82..f5aba752d35f7 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -205,7 +205,7 @@ static int move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, */ if (pte_present(pte)) force_flush = true; - pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr); + pte = move_pte(pte, old_addr, new_addr); pte = move_soft_dirty_pte(pte); set_pte_at(mm, new_addr, new_pte, pte); } From 08b8247ebd2b70367ab3a87657efb237662bd8f6 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 27 Mar 2024 22:30:08 +0800 Subject: [PATCH 0696/1702] mm: remove __set_page_dirty_nobuffers() There are no more callers of __set_page_dirty_nobuffers(), remove it. Link: https://lkml.kernel.org/r/20240327143008.3739435-1-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviewed-by: David Hildenbrand Reviewed-by: Matthew Wilcox (Oracle) Reviewed-by: Vishal Moola (Oracle) Signed-off-by: Andrew Morton --- include/linux/pagemap.h | 1 - mm/folio-compat.c | 6 ------ 2 files changed, 7 deletions(-) diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index ec8b83928c306..65e332df4d769 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1158,7 +1158,6 @@ static inline void folio_cancel_dirty(struct folio *folio) bool folio_clear_dirty_for_io(struct folio *folio); bool clear_page_dirty_for_io(struct page *page); void folio_invalidate(struct folio *folio, size_t offset, size_t length); -int __set_page_dirty_nobuffers(struct page *page); bool noop_dirty_folio(struct address_space *mapping, struct folio *folio); #ifdef CONFIG_MIGRATION diff --git a/mm/folio-compat.c b/mm/folio-compat.c index 50412014f16f7..f31e0ce65b11d 100644 --- a/mm/folio-compat.c +++ b/mm/folio-compat.c @@ -58,12 +58,6 @@ bool set_page_dirty(struct page *page) } EXPORT_SYMBOL(set_page_dirty); -int __set_page_dirty_nobuffers(struct page *page) -{ - return filemap_dirty_folio(page_mapping(page), page_folio(page)); -} -EXPORT_SYMBOL(__set_page_dirty_nobuffers); - bool clear_page_dirty_for_io(struct page *page) { return folio_clear_dirty_for_io(page_folio(page)); From afd584398b51eba12d0ffa4e0e0a493323993103 Mon Sep 17 00:00:00 2001 From: ZhangPeng Date: Wed, 27 Mar 2024 17:08:35 +0800 Subject: [PATCH 0697/1702] userfaultfd: early return in dup_userfaultfd() When vma->vm_userfaultfd_ctx.ctx is NULL, vma->vm_flags should have cleared __VM_UFFD_FLAGS. Therefore, there is no need to down_write or clear the flag, which will affect fork performance. Fix this by returning early if octx is NULL in dup_userfaultfd(). By applying this patch we can get a 1.3% performance improvement for lmbench fork_prot. Results are as follows: base early return Process fork+exit: 419.1106 413.4804 Link: https://lkml.kernel.org/r/20240327090835.3232629-1-zhangpeng362@huawei.com Signed-off-by: ZhangPeng Reviewed-by: Peter Xu Cc: Axel Rasmussen Cc: David Hildenbrand Cc: Kefeng Wang Cc: Liam R. Howlett Cc: Lokesh Gidra Cc: Nanyong Sun Cc: Suren Baghdasaryan Signed-off-by: Andrew Morton --- fs/userfaultfd.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 60dcfafdc11a8..3e6ddda6f1596 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -657,7 +657,10 @@ int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs) struct userfaultfd_fork_ctx *fctx; octx = vma->vm_userfaultfd_ctx.ctx; - if (!octx || !(octx->features & UFFD_FEATURE_EVENT_FORK)) { + if (!octx) + return 0; + + if (!(octx->features & UFFD_FEATURE_EVENT_FORK)) { vma_start_write(vma); vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; userfaultfd_set_vm_flags(vma, vma->vm_flags & ~__VM_UFFD_FLAGS); From 5def1e0f476da713141269d86815aba7c2817cb5 Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Mon, 25 Mar 2024 19:16:43 -0700 Subject: [PATCH 0698/1702] proc: refactor pde_get_unmapped_area as prep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "Cover a guard gap corner case", v4. In working on x86’s shadow stack feature, I came across some limitations around the kernel’s handling of guard gaps. AFAICT these limitations are not too important for the traditional stack usage of guard gaps, but have bigger impact on shadow stack’s usage. And now in addition to x86, we have two other architectures implementing shadow stack like features that plan to use guard gaps. I wanted to see about addressing them, but I have not worked on mmap() placement related code before, so would greatly appreciate if people could take a look and point me in the right direction. The nature of the limitations of concern is as follows. In order to ensure guard gaps between mappings, mmap() would need to consider two things: 1. That the new mapping isn’t placed in an any existing mapping’s guard gap. 2. That the new mapping isn’t placed such that any existing mappings are not in *its* guard gaps Currently mmap never considers (2), and (1) is not considered in some situations. When not passing an address hint, or passing one without MAP_FIXED_NOREPLACE, (1) is enforced. With MAP_FIXED_NOREPLACE, (1) is not enforced. With MAP_FIXED, (1) is not considered, but this seems to be expected since MAP_FIXED can already clobber existing mappings. For MAP_FIXED_NOREPLACE I would have guessed it should respect the guard gaps of existing mappings, but it is probably a little ambiguous. In this series I just tried to add enforcement of (2) for the normal (no address hint) case and only for the newer shadow stack memory (not stacks). The reason is that with the no-address-hint situation, landing next to a guard gap could come up naturally and so be more influencable by attackers such that two shadow stacks could be adjacent without a guard gap. Where as the address-hint scenarios would require more control - being able to call mmap() with specific arguments. As for why not just fix the other corner cases anyway, I thought it might have some greater possibility of affecting existing apps. This patch (of 14): Future changes will perform a treewide change to remove the indirect branch that is involved in calling mm->get_unmapped_area(). After doing this, the function will no longer be able to be handled as a function pointer. To make the treewide change diff cleaner and easier to review, refactor pde_get_unmapped_area() such that mm->get_unmapped_area() is called without being stored in a local function pointer. With this in refactoring, follow on changes will be able to simply replace the call site with a future function that calls it directly. Link: https://lkml.kernel.org/r/20240326021656.202649-1-rick.p.edgecombe@intel.com Link: https://lkml.kernel.org/r/20240326021656.202649-2-rick.p.edgecombe@intel.com Signed-off-by: Rick Edgecombe Cc: Andy Lutomirski Cc: Borislav Petkov (AMD) Cc: Christophe Leroy Cc: Dave Hansen Cc: Deepak Gupta Cc: H. Peter Anvin (Intel) Cc: Ingo Molnar Cc: Kees Cook Cc: Kirill A. Shutemov Cc: Liam R. Howlett Cc: Mark Brown Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Alexei Starovoitov Cc: Aneesh Kumar K.V Cc: Dan Williams Cc: Guo Ren Cc: Helge Deller Cc: "James E.J. Bottomley" Cc: Michael Ellerman Cc: Naveen N. Rao Cc: Nicholas Piggin Signed-off-by: Andrew Morton --- fs/proc/inode.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/fs/proc/inode.c b/fs/proc/inode.c index dcd513dccf55c..75396a24fd8cd 100644 --- a/fs/proc/inode.c +++ b/fs/proc/inode.c @@ -451,15 +451,12 @@ pde_get_unmapped_area(struct proc_dir_entry *pde, struct file *file, unsigned lo unsigned long len, unsigned long pgoff, unsigned long flags) { - typeof_member(struct proc_ops, proc_get_unmapped_area) get_area; + if (pde->proc_ops->proc_get_unmapped_area) + return pde->proc_ops->proc_get_unmapped_area(file, orig_addr, len, pgoff, flags); - get_area = pde->proc_ops->proc_get_unmapped_area; #ifdef CONFIG_MMU - if (!get_area) - get_area = current->mm->get_unmapped_area; + return current->mm->get_unmapped_area(file, orig_addr, len, pgoff, flags); #endif - if (get_area) - return get_area(file, orig_addr, len, pgoff, flags); return orig_addr; } From 529ce23a764f25d172198b4c6ba90f1e2ad17f93 Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Mon, 25 Mar 2024 19:16:44 -0700 Subject: [PATCH 0699/1702] mm: switch mm->get_unmapped_area() to a flag The mm_struct contains a function pointer *get_unmapped_area(), which is set to either arch_get_unmapped_area() or arch_get_unmapped_area_topdown() during the initialization of the mm. Since the function pointer only ever points to two functions that are named the same across all arch's, a function pointer is not really required. In addition future changes will want to add versions of the functions that take additional arguments. So to save a pointers worth of bytes in mm_struct, and prevent adding additional function pointers to mm_struct in future changes, remove it and keep the information about which get_unmapped_area() to use in a flag. Add the new flag to MMF_INIT_MASK so it doesn't get clobbered on fork by mmf_init_flags(). Most MM flags get clobbered on fork. In the pre-existing behavior mm->get_unmapped_area() would get copied to the new mm in dup_mm(), so not clobbering the flag preserves the existing behavior around inheriting the topdown-ness. Introduce a helper, mm_get_unmapped_area(), to easily convert code that refers to the old function pointer to instead select and call either arch_get_unmapped_area() or arch_get_unmapped_area_topdown() based on the flag. Then drop the mm->get_unmapped_area() function pointer. Leave the get_unmapped_area() pointer in struct file_operations alone. The main purpose of this change is to reorganize in preparation for future changes, but it also converts the calls of mm->get_unmapped_area() from indirect branches into a direct ones. The stress-ng bigheap benchmark calls realloc a lot, which calls through get_unmapped_area() in the kernel. On x86, the change yielded a ~1% improvement there on a retpoline config. In testing a few x86 configs, removing the pointer unfortunately didn't result in any actual size reductions in the compiled layout of mm_struct. But depending on compiler or arch alignment requirements, the change could shrink the size of mm_struct. Link: https://lkml.kernel.org/r/20240326021656.202649-3-rick.p.edgecombe@intel.com Signed-off-by: Rick Edgecombe Acked-by: Dave Hansen Acked-by: Liam R. Howlett Reviewed-by: Kirill A. Shutemov Acked-by: Alexei Starovoitov Cc: Dan Williams Cc: Andy Lutomirski Cc: Aneesh Kumar K.V Cc: Borislav Petkov (AMD) Cc: Christophe Leroy Cc: Deepak Gupta Cc: Guo Ren Cc: Helge Deller Cc: H. Peter Anvin (Intel) Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Kees Cook Cc: Mark Brown Cc: Michael Ellerman Cc: Naveen N. Rao Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- arch/s390/mm/hugetlbpage.c | 2 +- arch/s390/mm/mmap.c | 4 ++-- arch/sparc/kernel/sys_sparc_64.c | 15 ++++++--------- arch/sparc/mm/hugetlbpage.c | 2 +- arch/x86/kernel/cpu/sgx/driver.c | 2 +- arch/x86/mm/hugetlbpage.c | 2 +- arch/x86/mm/mmap.c | 4 ++-- drivers/char/mem.c | 2 +- drivers/dax/device.c | 6 +++--- fs/hugetlbfs/inode.c | 4 ++-- fs/proc/inode.c | 3 ++- fs/ramfs/file-mmu.c | 2 +- include/linux/mm_types.h | 6 +----- include/linux/sched/coredump.h | 5 ++++- include/linux/sched/mm.h | 5 +++++ io_uring/io_uring.c | 2 +- kernel/bpf/arena.c | 2 +- kernel/bpf/syscall.c | 2 +- mm/debug.c | 6 ------ mm/huge_memory.c | 9 ++++----- mm/mmap.c | 21 ++++++++++++++++++--- mm/shmem.c | 11 +++++------ mm/util.c | 6 +++--- 23 files changed, 66 insertions(+), 57 deletions(-) diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c index ca43b6fce71cf..7d948e243f4b2 100644 --- a/arch/s390/mm/hugetlbpage.c +++ b/arch/s390/mm/hugetlbpage.c @@ -318,7 +318,7 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, goto check_asce_limit; } - if (mm->get_unmapped_area == arch_get_unmapped_area) + if (!test_bit(MMF_TOPDOWN, &mm->flags)) addr = hugetlb_get_unmapped_area_bottomup(file, addr, len, pgoff, flags); else diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c index b14fc0887654d..6b2e4436ad4a8 100644 --- a/arch/s390/mm/mmap.c +++ b/arch/s390/mm/mmap.c @@ -185,10 +185,10 @@ void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) */ if (mmap_is_legacy(rlim_stack)) { mm->mmap_base = mmap_base_legacy(random_factor); - mm->get_unmapped_area = arch_get_unmapped_area; + clear_bit(MMF_TOPDOWN, &mm->flags); } else { mm->mmap_base = mmap_base(random_factor, rlim_stack); - mm->get_unmapped_area = arch_get_unmapped_area_topdown; + set_bit(MMF_TOPDOWN, &mm->flags); } } diff --git a/arch/sparc/kernel/sys_sparc_64.c b/arch/sparc/kernel/sys_sparc_64.c index 1e9a9e016237b..1dbf7211666ea 100644 --- a/arch/sparc/kernel/sys_sparc_64.c +++ b/arch/sparc/kernel/sys_sparc_64.c @@ -218,14 +218,10 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, unsigned long get_fb_unmapped_area(struct file *filp, unsigned long orig_addr, unsigned long len, unsigned long pgoff, unsigned long flags) { unsigned long align_goal, addr = -ENOMEM; - unsigned long (*get_area)(struct file *, unsigned long, - unsigned long, unsigned long, unsigned long); - - get_area = current->mm->get_unmapped_area; if (flags & MAP_FIXED) { /* Ok, don't mess with it. */ - return get_area(NULL, orig_addr, len, pgoff, flags); + return mm_get_unmapped_area(current->mm, NULL, orig_addr, len, pgoff, flags); } flags &= ~MAP_SHARED; @@ -238,7 +234,8 @@ unsigned long get_fb_unmapped_area(struct file *filp, unsigned long orig_addr, u align_goal = (64UL * 1024); do { - addr = get_area(NULL, orig_addr, len + (align_goal - PAGE_SIZE), pgoff, flags); + addr = mm_get_unmapped_area(current->mm, NULL, orig_addr, + len + (align_goal - PAGE_SIZE), pgoff, flags); if (!(addr & ~PAGE_MASK)) { addr = (addr + (align_goal - 1UL)) & ~(align_goal - 1UL); break; @@ -256,7 +253,7 @@ unsigned long get_fb_unmapped_area(struct file *filp, unsigned long orig_addr, u * be obtained. */ if (addr & ~PAGE_MASK) - addr = get_area(NULL, orig_addr, len, pgoff, flags); + addr = mm_get_unmapped_area(current->mm, NULL, orig_addr, len, pgoff, flags); return addr; } @@ -292,7 +289,7 @@ void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) gap == RLIM_INFINITY || sysctl_legacy_va_layout) { mm->mmap_base = TASK_UNMAPPED_BASE + random_factor; - mm->get_unmapped_area = arch_get_unmapped_area; + clear_bit(MMF_TOPDOWN, &mm->flags); } else { /* We know it's 32-bit */ unsigned long task_size = STACK_TOP32; @@ -303,7 +300,7 @@ void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) gap = (task_size / 6 * 5); mm->mmap_base = PAGE_ALIGN(task_size - gap - random_factor); - mm->get_unmapped_area = arch_get_unmapped_area_topdown; + set_bit(MMF_TOPDOWN, &mm->flags); } } diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c index 8ed5bdf95d253..c23012e3a3537 100644 --- a/arch/sparc/mm/hugetlbpage.c +++ b/arch/sparc/mm/hugetlbpage.c @@ -123,7 +123,7 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr, (!vma || addr + len <= vm_start_gap(vma))) return addr; } - if (mm->get_unmapped_area == arch_get_unmapped_area) + if (!test_bit(MMF_TOPDOWN, &mm->flags)) return hugetlb_get_unmapped_area_bottomup(file, addr, len, pgoff, flags); else diff --git a/arch/x86/kernel/cpu/sgx/driver.c b/arch/x86/kernel/cpu/sgx/driver.c index 262f5fb18d74d..22b65a5f5ec6c 100644 --- a/arch/x86/kernel/cpu/sgx/driver.c +++ b/arch/x86/kernel/cpu/sgx/driver.c @@ -113,7 +113,7 @@ static unsigned long sgx_get_unmapped_area(struct file *file, if (flags & MAP_FIXED) return addr; - return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); + return mm_get_unmapped_area(current->mm, file, addr, len, pgoff, flags); } #ifdef CONFIG_COMPAT diff --git a/arch/x86/mm/hugetlbpage.c b/arch/x86/mm/hugetlbpage.c index dab6db288e5b2..06ca9a60bac2a 100644 --- a/arch/x86/mm/hugetlbpage.c +++ b/arch/x86/mm/hugetlbpage.c @@ -115,7 +115,7 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr, } get_unmapped_area: - if (mm->get_unmapped_area == arch_get_unmapped_area) + if (!test_bit(MMF_TOPDOWN, &mm->flags)) return hugetlb_get_unmapped_area_bottomup(file, addr, len, pgoff, flags); else diff --git a/arch/x86/mm/mmap.c b/arch/x86/mm/mmap.c index c90c20904a607..a2cabb1c81e1a 100644 --- a/arch/x86/mm/mmap.c +++ b/arch/x86/mm/mmap.c @@ -129,9 +129,9 @@ static void arch_pick_mmap_base(unsigned long *base, unsigned long *legacy_base, void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) { if (mmap_is_legacy()) - mm->get_unmapped_area = arch_get_unmapped_area; + clear_bit(MMF_TOPDOWN, &mm->flags); else - mm->get_unmapped_area = arch_get_unmapped_area_topdown; + set_bit(MMF_TOPDOWN, &mm->flags); arch_pick_mmap_base(&mm->mmap_base, &mm->mmap_legacy_base, arch_rnd(mmap64_rnd_bits), task_size_64bit(0), diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 3c6670cf905f1..9b80e622ae80a 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -544,7 +544,7 @@ static unsigned long get_unmapped_area_zero(struct file *file, } /* Otherwise flags & MAP_PRIVATE: with no shmem object beneath it */ - return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); + return mm_get_unmapped_area(current->mm, file, addr, len, pgoff, flags); #else return -ENOSYS; #endif diff --git a/drivers/dax/device.c b/drivers/dax/device.c index 93ebedc5ec8ca..47c126d37b59a 100644 --- a/drivers/dax/device.c +++ b/drivers/dax/device.c @@ -329,14 +329,14 @@ static unsigned long dax_get_unmapped_area(struct file *filp, if ((off + len_align) < off) goto out; - addr_align = current->mm->get_unmapped_area(filp, addr, len_align, - pgoff, flags); + addr_align = mm_get_unmapped_area(current->mm, filp, addr, len_align, + pgoff, flags); if (!IS_ERR_VALUE(addr_align)) { addr_align += (off - addr_align) & (align - 1); return addr_align; } out: - return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); + return mm_get_unmapped_area(current->mm, filp, addr, len, pgoff, flags); } static const struct address_space_operations dev_dax_aops = { diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 6502c7e776d19..3dee18bf47ed3 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -249,11 +249,11 @@ generic_hugetlb_get_unmapped_area(struct file *file, unsigned long addr, } /* - * Use mm->get_unmapped_area value as a hint to use topdown routine. + * Use MMF_TOPDOWN flag as a hint to use topdown routine. * If architectures have special needs, they should define their own * version of hugetlb_get_unmapped_area. */ - if (mm->get_unmapped_area == arch_get_unmapped_area_topdown) + if (test_bit(MMF_TOPDOWN, &mm->flags)) return hugetlb_get_unmapped_area_topdown(file, addr, len, pgoff, flags); return hugetlb_get_unmapped_area_bottomup(file, addr, len, diff --git a/fs/proc/inode.c b/fs/proc/inode.c index 75396a24fd8cd..d19434e2a58e7 100644 --- a/fs/proc/inode.c +++ b/fs/proc/inode.c @@ -455,8 +455,9 @@ pde_get_unmapped_area(struct proc_dir_entry *pde, struct file *file, unsigned lo return pde->proc_ops->proc_get_unmapped_area(file, orig_addr, len, pgoff, flags); #ifdef CONFIG_MMU - return current->mm->get_unmapped_area(file, orig_addr, len, pgoff, flags); + return mm_get_unmapped_area(current->mm, file, orig_addr, len, pgoff, flags); #endif + return orig_addr; } diff --git a/fs/ramfs/file-mmu.c b/fs/ramfs/file-mmu.c index c7a1aa3c882b0..b45c7edc3225e 100644 --- a/fs/ramfs/file-mmu.c +++ b/fs/ramfs/file-mmu.c @@ -35,7 +35,7 @@ static unsigned long ramfs_mmu_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); + return mm_get_unmapped_area(current->mm, file, addr, len, pgoff, flags); } const struct file_operations ramfs_file_operations = { diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index fee6561feb7eb..fa0d6995706f4 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -777,11 +777,7 @@ struct mm_struct { } ____cacheline_aligned_in_smp; struct maple_tree mm_mt; -#ifdef CONFIG_MMU - unsigned long (*get_unmapped_area) (struct file *filp, - unsigned long addr, unsigned long len, - unsigned long pgoff, unsigned long flags); -#endif + unsigned long mmap_base; /* base of mmap area */ unsigned long mmap_legacy_base; /* base of mmap area in bottom-up allocations */ #ifdef CONFIG_HAVE_ARCH_COMPAT_MMAP_BASES diff --git a/include/linux/sched/coredump.h b/include/linux/sched/coredump.h index 02f5090ffea29..e62ff805cfc95 100644 --- a/include/linux/sched/coredump.h +++ b/include/linux/sched/coredump.h @@ -92,9 +92,12 @@ static inline int get_dumpable(struct mm_struct *mm) #define MMF_VM_MERGE_ANY 30 #define MMF_VM_MERGE_ANY_MASK (1 << MMF_VM_MERGE_ANY) +#define MMF_TOPDOWN 31 /* mm searches top down by default */ +#define MMF_TOPDOWN_MASK (1 << MMF_TOPDOWN) + #define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\ MMF_DISABLE_THP_MASK | MMF_HAS_MDWE_MASK |\ - MMF_VM_MERGE_ANY_MASK) + MMF_VM_MERGE_ANY_MASK | MMF_TOPDOWN_MASK) static inline unsigned long mmf_init_flags(unsigned long flags) { diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index b6543f9d78d6b..ed1caa26c8be9 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -8,6 +8,7 @@ #include #include #include +#include /* * Routines for handling mm_structs @@ -186,6 +187,10 @@ arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); +unsigned long mm_get_unmapped_area(struct mm_struct *mm, struct file *filp, + unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags); + unsigned long generic_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index c170a2b8d2cf2..d5edfb8444d78 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3525,7 +3525,7 @@ static unsigned long io_uring_mmu_get_unmapped_area(struct file *filp, #else addr = 0UL; #endif - return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); + return mm_get_unmapped_area(current->mm, filp, addr, len, pgoff, flags); } #else /* !CONFIG_MMU */ diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 343c3456c8ddf..16dbf4f6b77fa 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -314,7 +314,7 @@ static unsigned long arena_get_unmapped_area(struct file *filp, unsigned long ad return -EINVAL; } - ret = current->mm->get_unmapped_area(filp, addr, len * 2, 0, flags); + ret = mm_get_unmapped_area(current->mm, filp, addr, len * 2, 0, flags); if (IS_ERR_VALUE(ret)) return ret; if ((ret >> 32) == ((ret + len - 1) >> 32)) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index c287925471f68..9cb89e875f0d3 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -980,7 +980,7 @@ static unsigned long bpf_get_unmapped_area(struct file *filp, unsigned long addr if (map->ops->map_get_unmapped_area) return map->ops->map_get_unmapped_area(filp, addr, len, pgoff, flags); #ifdef CONFIG_MMU - return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); + return mm_get_unmapped_area(current->mm, filp, addr, len, pgoff, flags); #else return addr; #endif diff --git a/mm/debug.c b/mm/debug.c index e8a96b8b71973..b71186f1fb0b1 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -177,9 +177,6 @@ EXPORT_SYMBOL(dump_vma); void dump_mm(const struct mm_struct *mm) { pr_emerg("mm %px task_size %lu\n" -#ifdef CONFIG_MMU - "get_unmapped_area %px\n" -#endif "mmap_base %lu mmap_legacy_base %lu\n" "pgd %px mm_users %d mm_count %d pgtables_bytes %lu map_count %d\n" "hiwater_rss %lx hiwater_vm %lx total_vm %lx locked_vm %lx\n" @@ -205,9 +202,6 @@ void dump_mm(const struct mm_struct *mm) "def_flags: %#lx(%pGv)\n", mm, mm->task_size, -#ifdef CONFIG_MMU - mm->get_unmapped_area, -#endif mm->mmap_base, mm->mmap_legacy_base, mm->pgd, atomic_read(&mm->mm_users), atomic_read(&mm->mm_count), diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 157cee64850c8..7fdb2275a1e04 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -816,8 +816,8 @@ static unsigned long __thp_get_unmapped_area(struct file *filp, if (len_pad < len || (off + len_pad) < off) return 0; - ret = current->mm->get_unmapped_area(filp, addr, len_pad, - off >> PAGE_SHIFT, flags); + ret = mm_get_unmapped_area(current->mm, filp, addr, len_pad, + off >> PAGE_SHIFT, flags); /* * The failure might be due to length padding. The caller will retry @@ -835,8 +835,7 @@ static unsigned long __thp_get_unmapped_area(struct file *filp, off_sub = (off - ret) & (size - 1); - if (current->mm->get_unmapped_area == arch_get_unmapped_area_topdown && - !off_sub) + if (test_bit(MMF_TOPDOWN, ¤t->mm->flags) && !off_sub) return ret + size; ret += off_sub; @@ -853,7 +852,7 @@ unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, if (ret) return ret; - return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); + return mm_get_unmapped_area(current->mm, filp, addr, len, pgoff, flags); } EXPORT_SYMBOL_GPL(thp_get_unmapped_area); diff --git a/mm/mmap.c b/mm/mmap.c index 77a625e13ec12..dfb8a518e8c96 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1812,7 +1812,8 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { unsigned long (*get_area)(struct file *, unsigned long, - unsigned long, unsigned long, unsigned long); + unsigned long, unsigned long, unsigned long) + = NULL; unsigned long error = arch_mmap_check(addr, len, flags); if (error) @@ -1822,7 +1823,6 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, if (len > TASK_SIZE) return -ENOMEM; - get_area = current->mm->get_unmapped_area; if (file) { if (file->f_op->get_unmapped_area) get_area = file->f_op->get_unmapped_area; @@ -1841,7 +1841,11 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, if (!file) pgoff = 0; - addr = get_area(file, addr, len, pgoff, flags); + if (get_area) + addr = get_area(file, addr, len, pgoff, flags); + else + addr = mm_get_unmapped_area(current->mm, file, addr, len, + pgoff, flags); if (IS_ERR_VALUE(addr)) return addr; @@ -1856,6 +1860,17 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, EXPORT_SYMBOL(get_unmapped_area); +unsigned long +mm_get_unmapped_area(struct mm_struct *mm, struct file *file, + unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags) +{ + if (test_bit(MMF_TOPDOWN, &mm->flags)) + return arch_get_unmapped_area_topdown(file, addr, len, pgoff, flags); + return arch_get_unmapped_area(file, addr, len, pgoff, flags); +} +EXPORT_SYMBOL(mm_get_unmapped_area); + /** * find_vma_intersection() - Look up the first VMA which intersects the interval * @mm: The process address space. diff --git a/mm/shmem.c b/mm/shmem.c index 98985179f495d..fa2a0ed97507d 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -2267,8 +2267,6 @@ unsigned long shmem_get_unmapped_area(struct file *file, unsigned long uaddr, unsigned long len, unsigned long pgoff, unsigned long flags) { - unsigned long (*get_area)(struct file *, - unsigned long, unsigned long, unsigned long, unsigned long); unsigned long addr; unsigned long offset; unsigned long inflated_len; @@ -2278,8 +2276,8 @@ unsigned long shmem_get_unmapped_area(struct file *file, if (len > TASK_SIZE) return -ENOMEM; - get_area = current->mm->get_unmapped_area; - addr = get_area(file, uaddr, len, pgoff, flags); + addr = mm_get_unmapped_area(current->mm, file, uaddr, len, pgoff, + flags); if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) return addr; @@ -2336,7 +2334,8 @@ unsigned long shmem_get_unmapped_area(struct file *file, if (inflated_len < len) return addr; - inflated_addr = get_area(NULL, uaddr, inflated_len, 0, flags); + inflated_addr = mm_get_unmapped_area(current->mm, NULL, uaddr, + inflated_len, 0, flags); if (IS_ERR_VALUE(inflated_addr)) return addr; if (inflated_addr & ~PAGE_MASK) @@ -4801,7 +4800,7 @@ unsigned long shmem_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); + return mm_get_unmapped_area(current->mm, file, addr, len, pgoff, flags); } #endif diff --git a/mm/util.c b/mm/util.c index a9e911b22b992..c9e519e6811f5 100644 --- a/mm/util.c +++ b/mm/util.c @@ -469,17 +469,17 @@ void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) if (mmap_is_legacy(rlim_stack)) { mm->mmap_base = TASK_UNMAPPED_BASE + random_factor; - mm->get_unmapped_area = arch_get_unmapped_area; + clear_bit(MMF_TOPDOWN, &mm->flags); } else { mm->mmap_base = mmap_base(random_factor, rlim_stack); - mm->get_unmapped_area = arch_get_unmapped_area_topdown; + set_bit(MMF_TOPDOWN, &mm->flags); } } #elif defined(CONFIG_MMU) && !defined(HAVE_ARCH_PICK_MMAP_LAYOUT) void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) { mm->mmap_base = TASK_UNMAPPED_BASE; - mm->get_unmapped_area = arch_get_unmapped_area; + clear_bit(MMF_TOPDOWN, &mm->flags); } #endif From 961148704acd814a4a5acaa4687a45e2315dd857 Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Mon, 25 Mar 2024 19:16:45 -0700 Subject: [PATCH 0700/1702] mm: introduce arch_get_unmapped_area_vmflags() When memory is being placed, mmap() will take care to respect the guard gaps of certain types of memory (VM_SHADOWSTACK, VM_GROWSUP and VM_GROWSDOWN). In order to ensure guard gaps between mappings, mmap() needs to consider two things: 1. That the new mapping isn't placed in an any existing mappings guard gaps. 2. That the new mapping isn't placed such that any existing mappings are not in *its* guard gaps. The longstanding behavior of mmap() is to ensure 1, but not take any care around 2. So for example, if there is a PAGE_SIZE free area, and a mmap() with a PAGE_SIZE size, and a type that has a guard gap is being placed, mmap() may place the shadow stack in the PAGE_SIZE free area. Then the mapping that is supposed to have a guard gap will not have a gap to the adjacent VMA. In order to take the start gap into account, the maple tree search needs to know the size of start gap the new mapping will need. The call chain from do_mmap() to the actual maple tree search looks like this: do_mmap(size, vm_flags, map_flags, ..) mm/mmap.c:get_unmapped_area(size, map_flags, ...) arch_get_unmapped_area(size, map_flags, ...) vm_unmapped_area(struct vm_unmapped_area_info) One option would be to add another MAP_ flag to mean a one page start gap (as is for shadow stack), but this consumes a flag unnecessarily. Another option could be to simply increase the size passed in do_mmap() by the start gap size, and adjust after the fact, but this will interfere with the alignment requirements passed in struct vm_unmapped_area_info, and unknown to mmap.c. Instead, introduce variants of arch_get_unmapped_area/_topdown() that take vm_flags. In future changes, these variants can be used in mmap.c:get_unmapped_area() to allow the vm_flags to be passed through to vm_unmapped_area(), while preserving the normal arch_get_unmapped_area/_topdown() for the existing callers. Link: https://lkml.kernel.org/r/20240326021656.202649-4-rick.p.edgecombe@intel.com Signed-off-by: Rick Edgecombe Cc: Alexei Starovoitov Cc: Andy Lutomirski Cc: Aneesh Kumar K.V Cc: Borislav Petkov (AMD) Cc: Christophe Leroy Cc: Dan Williams Cc: Dave Hansen Cc: Deepak Gupta Cc: Guo Ren Cc: Helge Deller Cc: H. Peter Anvin (Intel) Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Kees Cook Cc: Kirill A. Shutemov Cc: Liam R. Howlett Cc: Mark Brown Cc: Michael Ellerman Cc: Naveen N. Rao Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- include/linux/sched/mm.h | 17 +++++++++++++++++ mm/mmap.c | 28 ++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index ed1caa26c8be9..91546493c43da 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -191,6 +191,23 @@ unsigned long mm_get_unmapped_area(struct mm_struct *mm, struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); +unsigned long +arch_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags, vm_flags_t vm_flags); +unsigned long +arch_get_unmapped_area_topdown_vmflags(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags, vm_flags_t); + +unsigned long mm_get_unmapped_area_vmflags(struct mm_struct *mm, + struct file *filp, + unsigned long addr, + unsigned long len, + unsigned long pgoff, + unsigned long flags, + vm_flags_t vm_flags); + unsigned long generic_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, diff --git a/mm/mmap.c b/mm/mmap.c index dfb8a518e8c96..b33055d1561b9 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1807,6 +1807,34 @@ arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, } #endif +#ifndef HAVE_ARCH_UNMAPPED_AREA_VMFLAGS +unsigned long +arch_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags) +{ + return arch_get_unmapped_area(filp, addr, len, pgoff, flags); +} + +unsigned long +arch_get_unmapped_area_topdown_vmflags(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags, vm_flags_t vm_flags) +{ + return arch_get_unmapped_area_topdown(filp, addr, len, pgoff, flags); +} +#endif + +unsigned long mm_get_unmapped_area_vmflags(struct mm_struct *mm, struct file *filp, + unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags, + vm_flags_t vm_flags) +{ + if (test_bit(MMF_TOPDOWN, &mm->flags)) + return arch_get_unmapped_area_topdown_vmflags(filp, addr, len, pgoff, + flags, vm_flags); + return arch_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, vm_flags); +} + unsigned long get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) From 529781b24b73b2b29d7f1a88799e99952749c96b Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Mon, 25 Mar 2024 19:16:46 -0700 Subject: [PATCH 0701/1702] mm: remove export for get_unmapped_area() The mm/mmap.c function get_unmapped_area() is not used from any modules, so it doesn't need to be exported. Remove the export. Link: https://lkml.kernel.org/r/20240326021656.202649-5-rick.p.edgecombe@intel.com Signed-off-by: Rick Edgecombe Cc: Alexei Starovoitov Cc: Andy Lutomirski Cc: Aneesh Kumar K.V Cc: Borislav Petkov (AMD) Cc: Christophe Leroy Cc: Dan Williams Cc: Dave Hansen Cc: Deepak Gupta Cc: Guo Ren Cc: Helge Deller Cc: H. Peter Anvin (Intel) Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Kees Cook Cc: Kirill A. Shutemov Cc: Liam R. Howlett Cc: Mark Brown Cc: Michael Ellerman Cc: Naveen N. Rao Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- mm/mmap.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/mm/mmap.c b/mm/mmap.c index b33055d1561b9..30baba087c65e 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1886,8 +1886,6 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, return error ? error : addr; } -EXPORT_SYMBOL(get_unmapped_area); - unsigned long mm_get_unmapped_area(struct mm_struct *mm, struct file *file, unsigned long addr, unsigned long len, From 8a0fe564bb2e744f21ca94e3eabe96d232cf5f41 Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Mon, 25 Mar 2024 19:16:47 -0700 Subject: [PATCH 0702/1702] mm: use get_unmapped_area_vmflags() When memory is being placed, mmap() will take care to respect the guard gaps of certain types of memory (VM_SHADOWSTACK, VM_GROWSUP and VM_GROWSDOWN). In order to ensure guard gaps between mappings, mmap() needs to consider two things: 1. That the new mapping isn't placed in an any existing mappings guard gaps. 2. That the new mapping isn't placed such that any existing mappings are not in *its* guard gaps. The long standing behavior of mmap() is to ensure 1, but not take any care around 2. So for example, if there is a PAGE_SIZE free area, and a mmap() with a PAGE_SIZE size, and a type that has a guard gap is being placed, mmap() may place the shadow stack in the PAGE_SIZE free area. Then the mapping that is supposed to have a guard gap will not have a gap to the adjacent VMA. Use mm_get_unmapped_area_vmflags() in the do_mmap() so future changes can cause shadow stack mappings to be placed with a guard gap. Also use the THP variant that takes vm_flags, such that THP shadow stack can get the same treatment. Adjust the vm_flags calculation to happen earlier so that the vm_flags can be passed into __get_unmapped_area(). Link: https://lkml.kernel.org/r/20240326021656.202649-6-rick.p.edgecombe@intel.com Signed-off-by: Rick Edgecombe Reviewed-by: Christophe Leroy Cc: Alexei Starovoitov Cc: Andy Lutomirski Cc: Aneesh Kumar K.V Cc: Borislav Petkov (AMD) Cc: Dan Williams Cc: Dave Hansen Cc: Deepak Gupta Cc: Guo Ren Cc: Helge Deller Cc: H. Peter Anvin (Intel) Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Kees Cook Cc: Kirill A. Shutemov Cc: Liam R. Howlett Cc: Mark Brown Cc: Michael Ellerman Cc: Naveen N. Rao Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- include/linux/mm.h | 11 ++++++++++- mm/mmap.c | 32 ++++++++++++++++---------------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 3dda0dbcae144..1b6903f4c57b4 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3413,7 +3413,16 @@ extern int install_special_mapping(struct mm_struct *mm, unsigned long randomize_stack_top(unsigned long stack_top); unsigned long randomize_page(unsigned long start, unsigned long range); -extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); +unsigned long +__get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags); + +static inline unsigned long +get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags) +{ + return __get_unmapped_area(file, addr, len, pgoff, flags, 0); +} extern unsigned long mmap_region(struct file *file, unsigned long addr, unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, diff --git a/mm/mmap.c b/mm/mmap.c index 30baba087c65e..a37c8dd29d64f 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1255,18 +1255,6 @@ unsigned long do_mmap(struct file *file, unsigned long addr, if (mm->map_count > sysctl_max_map_count) return -ENOMEM; - /* Obtain the address to map to. we verify (or select) it and ensure - * that it represents a valid section of the address space. - */ - addr = get_unmapped_area(file, addr, len, pgoff, flags); - if (IS_ERR_VALUE(addr)) - return addr; - - if (flags & MAP_FIXED_NOREPLACE) { - if (find_vma_intersection(mm, addr, addr + len)) - return -EEXIST; - } - if (prot == PROT_EXEC) { pkey = execute_only_pkey(mm); if (pkey < 0) @@ -1280,6 +1268,18 @@ unsigned long do_mmap(struct file *file, unsigned long addr, vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) | mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; + /* Obtain the address to map to. we verify (or select) it and ensure + * that it represents a valid section of the address space. + */ + addr = __get_unmapped_area(file, addr, len, pgoff, flags, vm_flags); + if (IS_ERR_VALUE(addr)) + return addr; + + if (flags & MAP_FIXED_NOREPLACE) { + if (find_vma_intersection(mm, addr, addr + len)) + return -EEXIST; + } + if (flags & MAP_LOCKED) if (!can_do_mlock()) return -EPERM; @@ -1836,8 +1836,8 @@ unsigned long mm_get_unmapped_area_vmflags(struct mm_struct *mm, struct file *fi } unsigned long -get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, - unsigned long pgoff, unsigned long flags) +__get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags) { unsigned long (*get_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long) @@ -1872,8 +1872,8 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, if (get_area) addr = get_area(file, addr, len, pgoff, flags); else - addr = mm_get_unmapped_area(current->mm, file, addr, len, - pgoff, flags); + addr = mm_get_unmapped_area_vmflags(current->mm, file, addr, len, + pgoff, flags, vm_flags); if (IS_ERR_VALUE(addr)) return addr; From ed48e87c7df3651accfa3088076830727b070f54 Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Mon, 25 Mar 2024 19:16:48 -0700 Subject: [PATCH 0703/1702] thp: add thp_get_unmapped_area_vmflags() When memory is being placed, mmap() will take care to respect the guard gaps of certain types of memory (VM_SHADOWSTACK, VM_GROWSUP and VM_GROWSDOWN). In order to ensure guard gaps between mappings, mmap() needs to consider two things: 1. That the new mapping isn't placed in an any existing mappings guard gaps. 2. That the new mapping isn't placed such that any existing mappings are not in *its* guard gaps. The longstanding behavior of mmap() is to ensure 1, but not take any care around 2. So for example, if there is a PAGE_SIZE free area, and a mmap() with a PAGE_SIZE size, and a type that has a guard gap is being placed, mmap() may place the shadow stack in the PAGE_SIZE free area. Then the mapping that is supposed to have a guard gap will not have a gap to the adjacent VMA. Add a THP implementations of the vm_flags variant of get_unmapped_area(). Future changes will call this from mmap.c in the do_mmap() path to allow shadow stacks to be placed with consideration taken for the start guard gap. Shadow stack memory is always private and anonymous and so special guard gap logic is not needed in a lot of caseis, but it can be mapped by THP, so needs to be handled. Link: https://lkml.kernel.org/r/20240326021656.202649-7-rick.p.edgecombe@intel.com Signed-off-by: Rick Edgecombe Reviewed-by: Christophe Leroy Cc: Alexei Starovoitov Cc: Andy Lutomirski Cc: Aneesh Kumar K.V Cc: Borislav Petkov (AMD) Cc: Dan Williams Cc: Dave Hansen Cc: Deepak Gupta Cc: Guo Ren Cc: Helge Deller Cc: H. Peter Anvin (Intel) Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Kees Cook Cc: Kirill A. Shutemov Cc: Liam R. Howlett Cc: Mark Brown Cc: Michael Ellerman Cc: Naveen N. Rao Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 11 +++++++++++ mm/huge_memory.c | 23 ++++++++++++++++------- mm/mmap.c | 12 +++++++----- 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index ab26d9e65ec3b..e896ca4760f67 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -270,6 +270,9 @@ unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); +unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags, + vm_flags_t vm_flags); bool can_split_folio(struct folio *folio, int *pextra_pins); int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, @@ -413,6 +416,14 @@ static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, #define thp_get_unmapped_area NULL +static inline unsigned long +thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags, vm_flags_t vm_flags) +{ + return 0; +} + static inline bool can_split_folio(struct folio *folio, int *pextra_pins) { diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 7fdb2275a1e04..b757d9be310c8 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -800,7 +800,8 @@ static inline bool is_transparent_hugepage(const struct folio *folio) static unsigned long __thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, - loff_t off, unsigned long flags, unsigned long size) + loff_t off, unsigned long flags, unsigned long size, + vm_flags_t vm_flags) { loff_t off_end = off + len; loff_t off_align = round_up(off, size); @@ -816,8 +817,8 @@ static unsigned long __thp_get_unmapped_area(struct file *filp, if (len_pad < len || (off + len_pad) < off) return 0; - ret = mm_get_unmapped_area(current->mm, filp, addr, len_pad, - off >> PAGE_SHIFT, flags); + ret = mm_get_unmapped_area_vmflags(current->mm, filp, addr, len_pad, + off >> PAGE_SHIFT, flags, vm_flags); /* * The failure might be due to length padding. The caller will retry @@ -842,17 +843,25 @@ static unsigned long __thp_get_unmapped_area(struct file *filp, return ret; } -unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, - unsigned long len, unsigned long pgoff, unsigned long flags) +unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags, + vm_flags_t vm_flags) { unsigned long ret; loff_t off = (loff_t)pgoff << PAGE_SHIFT; - ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE); + ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags); if (ret) return ret; - return mm_get_unmapped_area(current->mm, filp, addr, len, pgoff, flags); + return mm_get_unmapped_area_vmflags(current->mm, filp, addr, len, pgoff, flags, + vm_flags); +} + +unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags) +{ + return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0); } EXPORT_SYMBOL_GPL(thp_get_unmapped_area); diff --git a/mm/mmap.c b/mm/mmap.c index a37c8dd29d64f..0ef1191e3be5b 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1860,20 +1860,22 @@ __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, * so use shmem's get_unmapped_area in case it can be huge. */ get_area = shmem_get_unmapped_area; - } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { - /* Ensures that larger anonymous mappings are THP aligned. */ - get_area = thp_get_unmapped_area; } /* Always treat pgoff as zero for anonymous memory. */ if (!file) pgoff = 0; - if (get_area) + if (get_area) { addr = get_area(file, addr, len, pgoff, flags); - else + } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + /* Ensures that larger anonymous mappings are THP aligned. */ + addr = thp_get_unmapped_area_vmflags(file, addr, len, + pgoff, flags, vm_flags); + } else { addr = mm_get_unmapped_area_vmflags(current->mm, file, addr, len, pgoff, flags, vm_flags); + } if (IS_ERR_VALUE(addr)) return addr; From bf6f3c18400c469d03baffd22ddcb7f132845630 Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Mon, 25 Mar 2024 19:16:49 -0700 Subject: [PATCH 0704/1702] csky: use initializer for struct vm_unmapped_area_info Future changes will need to add a new member to struct vm_unmapped_area_info. This would cause trouble for any call site that doesn't initialize the struct. Currently every caller sets each member manually, so if new members are added they will be uninitialized and the core code parsing the struct will see garbage in the new member. It could be possible to initialize the new member manually to 0 at each call site. This and a couple other options were discussed, and a working consensus (see links) was that in general the best way to accomplish this would be via static initialization with designated member initiators. Having some struct vm_unmapped_area_info instances not zero initialized will put those sites at risk of feeding garbage into vm_unmapped_area() if the convention is to zero initialize the struct and any new member addition misses a call site that initializes each member manually. It could be possible to leave the code mostly untouched, and just change the line: struct vm_unmapped_area_info info to: struct vm_unmapped_area_info info = {}; However, that would leave cleanup for the members that are manually set to zero, as it would no longer be required. So to be reduce the chance of bugs via uninitialized members, instead simply continue the process to initialize the struct this way tree wide. This will zero any unspecified members. Move the member initializers to the struct declaration when they are known at that time. Leave the members out that were manually initialized to zero, as this would be redundant for designated initializers. Link: https://lkml.kernel.org/r/20240326021656.202649-8-rick.p.edgecombe@intel.com Link: https://lore.kernel.org/lkml/202402280912.33AEE7A9CF@keescook/#t Link: https://lore.kernel.org/lkml/j7bfvig3gew3qruouxrh7z7ehjjafrgkbcmg6tcghhfh3rhmzi@wzlcoecgy5rs/ Signed-off-by: Rick Edgecombe Reviewed-by: Guo Ren Reviewed-by: Christophe Leroy Cc: Guo Ren Cc: Alexei Starovoitov Cc: Andy Lutomirski Cc: Aneesh Kumar K.V Cc: Borislav Petkov (AMD) Cc: Dan Williams Cc: Dave Hansen Cc: Deepak Gupta Cc: Helge Deller Cc: H. Peter Anvin (Intel) Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Kees Cook Cc: Kirill A. Shutemov Cc: Liam R. Howlett Cc: Mark Brown Cc: Michael Ellerman Cc: Naveen N. Rao Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- arch/csky/abiv1/mmap.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/csky/abiv1/mmap.c b/arch/csky/abiv1/mmap.c index 6792aca499991..7f826331d409b 100644 --- a/arch/csky/abiv1/mmap.c +++ b/arch/csky/abiv1/mmap.c @@ -28,7 +28,12 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, struct mm_struct *mm = current->mm; struct vm_area_struct *vma; int do_align = 0; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = { + .length = len, + .low_limit = mm->mmap_base, + .high_limit = TASK_SIZE, + .align_offset = pgoff << PAGE_SHIFT + }; /* * We only need to do colour alignment if either the I or D @@ -61,11 +66,6 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, return addr; } - info.flags = 0; - info.length = len; - info.low_limit = mm->mmap_base; - info.high_limit = TASK_SIZE; info.align_mask = do_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; - info.align_offset = pgoff << PAGE_SHIFT; return vm_unmapped_area(&info); } From 5e14522843f59e45950677c787b08d813ab6eb65 Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Mon, 25 Mar 2024 19:16:50 -0700 Subject: [PATCH 0705/1702] parisc: use initializer for struct vm_unmapped_area_info Future changes will need to add a new member to struct vm_unmapped_area_info. This would cause trouble for any call site that doesn't initialize the struct. Currently every caller sets each member manually, so if new members are added they will be uninitialized and the core code parsing the struct will see garbage in the new member. It could be possible to initialize the new member manually to 0 at each call site. This and a couple other options were discussed, and a working consensus (see links) was that in general the best way to accomplish this would be via static initialization with designated member initiators. Having some struct vm_unmapped_area_info instances not zero initialized will put those sites at risk of feeding garbage into vm_unmapped_area() if the convention is to zero initialize the struct and any new member addition misses a call site that initializes each member manually. It could be possible to leave the code mostly untouched, and just change the line: struct vm_unmapped_area_info info to: struct vm_unmapped_area_info info = {}; However, that would leave cleanup for the members that are manually set to zero, as it would no longer be required. So to be reduce the chance of bugs via uninitialized members, instead simply continue the process to initialize the struct this way tree wide. This will zero any unspecified members. Move the member initializers to the struct declaration when they are known at that time. Leave the members out that were manually initialized to zero, as this would be redundant for designated initializers. Link: https://lkml.kernel.org/r/20240326021656.202649-9-rick.p.edgecombe@intel.com Link: https://lore.kernel.org/lkml/202402280912.33AEE7A9CF@keescook/#t Link: https://lore.kernel.org/lkml/j7bfvig3gew3qruouxrh7z7ehjjafrgkbcmg6tcghhfh3rhmzi@wzlcoecgy5rs/ Signed-off-by: Rick Edgecombe Reviewed-by: Christophe Leroy Acked-by: Helge Deller Cc: "James E.J. Bottomley" Cc: Helge Deller Cc: Alexei Starovoitov Cc: Andy Lutomirski Cc: Aneesh Kumar K.V Cc: Borislav Petkov (AMD) Cc: Dan Williams Cc: Dave Hansen Cc: Deepak Gupta Cc: Guo Ren Cc: H. Peter Anvin (Intel) Cc: Ingo Molnar Cc: Kees Cook Cc: Kirill A. Shutemov Cc: Liam R. Howlett Cc: Mark Brown Cc: Michael Ellerman Cc: Naveen N. Rao Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- arch/parisc/kernel/sys_parisc.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/parisc/kernel/sys_parisc.c b/arch/parisc/kernel/sys_parisc.c index 98af719d5f85b..f7722451276ef 100644 --- a/arch/parisc/kernel/sys_parisc.c +++ b/arch/parisc/kernel/sys_parisc.c @@ -104,7 +104,9 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, struct vm_area_struct *vma, *prev; unsigned long filp_pgoff; int do_color_align; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = { + .length = len + }; if (unlikely(len > TASK_SIZE)) return -ENOMEM; @@ -139,7 +141,6 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, return addr; } - info.length = len; info.align_mask = do_color_align ? (PAGE_MASK & (SHM_COLOUR - 1)) : 0; info.align_offset = shared_align_offset(filp_pgoff, pgoff); @@ -160,7 +161,6 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, */ } - info.flags = 0; info.low_limit = mm->mmap_base; info.high_limit = mmap_upper_limit(NULL); return vm_unmapped_area(&info); From 9d8187b94b3e818e981ae2af637b8e4ac67b837c Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Mon, 25 Mar 2024 19:16:51 -0700 Subject: [PATCH 0706/1702] powerpc: use initializer for struct vm_unmapped_area_info Future changes will need to add a new member to struct vm_unmapped_area_info. This would cause trouble for any call site that doesn't initialize the struct. Currently every caller sets each member manually, so if new members are added they will be uninitialized and the core code parsing the struct will see garbage in the new member. It could be possible to initialize the new member manually to 0 at each call site. This and a couple other options were discussed, and a working consensus (see links) was that in general the best way to accomplish this would be via static initialization with designated member initiators. Having some struct vm_unmapped_area_info instances not zero initialized will put those sites at risk of feeding garbage into vm_unmapped_area() if the convention is to zero initialize the struct and any new member addition misses a call site that initializes each member manually. It could be possible to leave the code mostly untouched, and just change the line: struct vm_unmapped_area_info info to: struct vm_unmapped_area_info info = {}; However, that would leave cleanup for the members that are manually set to zero, as it would no longer be required. So to be reduce the chance of bugs via uninitialized members, instead simply continue the process to initialize the struct this way tree wide. This will zero any unspecified members. Move the member initializers to the struct declaration when they are known at that time. Leave the members out that were manually initialized to zero, as this would be redundant for designated initializers. Link: https://lkml.kernel.org/r/20240326021656.202649-10-rick.p.edgecombe@intel.com Link: https://lore.kernel.org/lkml/202402280912.33AEE7A9CF@keescook/#t Link: https://lore.kernel.org/lkml/j7bfvig3gew3qruouxrh7z7ehjjafrgkbcmg6tcghhfh3rhmzi@wzlcoecgy5rs/ Signed-off-by: Rick Edgecombe Acked-by: Michael Ellerman Cc: Nicholas Piggin Cc: Christophe Leroy Cc: Aneesh Kumar K.V Cc: Naveen N. Rao Cc: Alexei Starovoitov Cc: Andy Lutomirski Cc: Borislav Petkov (AMD) Cc: Dan Williams Cc: Dave Hansen Cc: Deepak Gupta Cc: Guo Ren Cc: Helge Deller Cc: H. Peter Anvin (Intel) Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Kees Cook Cc: Kirill A. Shutemov Cc: Liam R. Howlett Cc: Mark Brown Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- arch/powerpc/mm/book3s64/slice.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/arch/powerpc/mm/book3s64/slice.c b/arch/powerpc/mm/book3s64/slice.c index c0b58afb9a476..ef3ce37f1bb3e 100644 --- a/arch/powerpc/mm/book3s64/slice.c +++ b/arch/powerpc/mm/book3s64/slice.c @@ -282,12 +282,10 @@ static unsigned long slice_find_area_bottomup(struct mm_struct *mm, { int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT); unsigned long found, next_end; - struct vm_unmapped_area_info info; - - info.flags = 0; - info.length = len; - info.align_mask = PAGE_MASK & ((1ul << pshift) - 1); - info.align_offset = 0; + struct vm_unmapped_area_info info = { + .length = len, + .align_mask = PAGE_MASK & ((1ul << pshift) - 1), + }; /* * Check till the allow max value for this mmap request */ @@ -326,13 +324,13 @@ static unsigned long slice_find_area_topdown(struct mm_struct *mm, { int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT); unsigned long found, prev; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = { + .flags = VM_UNMAPPED_AREA_TOPDOWN, + .length = len, + .align_mask = PAGE_MASK & ((1ul << pshift) - 1), + }; unsigned long min_addr = max(PAGE_SIZE, mmap_min_addr); - info.flags = VM_UNMAPPED_AREA_TOPDOWN; - info.length = len; - info.align_mask = PAGE_MASK & ((1ul << pshift) - 1); - info.align_offset = 0; /* * If we are trying to allocate above DEFAULT_MAP_WINDOW * Add the different to the mmap_base. From b80fa3cbb78c0fbe5039682919d97a0dbe05ae7c Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Mon, 25 Mar 2024 19:16:52 -0700 Subject: [PATCH 0707/1702] treewide: use initializer for struct vm_unmapped_area_info Future changes will need to add a new member to struct vm_unmapped_area_info. This would cause trouble for any call site that doesn't initialize the struct. Currently every caller sets each member manually, so if new ones are added they will be uninitialized and the core code parsing the struct will see garbage in the new member. It could be possible to initialize the new member manually to 0 at each call site. This and a couple other options were discussed. Having some struct vm_unmapped_area_info instances not zero initialized will put those sites at risk of feeding garbage into vm_unmapped_area(), if the convention is to zero initialize the struct and any new field addition missed a call site that initializes each field manually. So it is useful to do things similar across the kernel. The consensus (see links) was that in general the best way to accomplish taking into account both code cleanliness and minimizing the chance of introducing bugs, was to do C99 static initialization. As in: struct vm_unmapped_area_info info = {}; With this method of initialization, the whole struct will be zero initialized, and any statements setting fields to zero will be unneeded. The change should not leave cleanup at the call sides. While iterating though the possible solutions a few archs kindly acked other variations that still zero initialized the struct. These sites have been modified in previous changes using the pattern acked by the respective arch. So to be reduce the chance of bugs via uninitialized fields, perform a tree wide change using the consensus for the best general way to do this change. Use C99 static initializing to zero the struct and remove and statements that simply set members to zero. Link: https://lkml.kernel.org/r/20240326021656.202649-11-rick.p.edgecombe@intel.com Link: https://lore.kernel.org/lkml/202402280912.33AEE7A9CF@keescook/#t Link: https://lore.kernel.org/lkml/j7bfvig3gew3qruouxrh7z7ehjjafrgkbcmg6tcghhfh3rhmzi@wzlcoecgy5rs/ Link: https://lore.kernel.org/lkml/ec3e377a-c0a0-4dd3-9cb9-96517e54d17e@csgroup.eu/ Signed-off-by: Rick Edgecombe Reviewed-by: Kees Cook Cc: Alexei Starovoitov Cc: Andy Lutomirski Cc: Aneesh Kumar K.V Cc: Borislav Petkov (AMD) Cc: Christophe Leroy Cc: Dan Williams Cc: Dave Hansen Cc: Deepak Gupta Cc: Guo Ren Cc: Helge Deller Cc: H. Peter Anvin (Intel) Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Kirill A. Shutemov Cc: Liam R. Howlett Cc: Mark Brown Cc: Michael Ellerman Cc: Naveen N. Rao Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- arch/alpha/kernel/osf_sys.c | 5 +---- arch/arc/mm/mmap.c | 4 +--- arch/arm/mm/mmap.c | 5 ++--- arch/loongarch/mm/mmap.c | 3 +-- arch/mips/mm/mmap.c | 3 +-- arch/s390/mm/hugetlbpage.c | 7 ++----- arch/s390/mm/mmap.c | 5 ++--- arch/sh/mm/mmap.c | 5 ++--- arch/sparc/kernel/sys_sparc_32.c | 3 +-- arch/sparc/kernel/sys_sparc_64.c | 5 ++--- arch/sparc/mm/hugetlbpage.c | 7 ++----- arch/x86/kernel/sys_x86_64.c | 7 ++----- arch/x86/mm/hugetlbpage.c | 7 ++----- fs/hugetlbfs/inode.c | 7 ++----- mm/mmap.c | 9 ++------- 15 files changed, 25 insertions(+), 57 deletions(-) diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c index 5db88b6274396..e5f881bc82881 100644 --- a/arch/alpha/kernel/osf_sys.c +++ b/arch/alpha/kernel/osf_sys.c @@ -1218,14 +1218,11 @@ static unsigned long arch_get_unmapped_area_1(unsigned long addr, unsigned long len, unsigned long limit) { - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; - info.flags = 0; info.length = len; info.low_limit = addr; info.high_limit = limit; - info.align_mask = 0; - info.align_offset = 0; return vm_unmapped_area(&info); } diff --git a/arch/arc/mm/mmap.c b/arch/arc/mm/mmap.c index 3c1c7ae732925..69a9152971559 100644 --- a/arch/arc/mm/mmap.c +++ b/arch/arc/mm/mmap.c @@ -27,7 +27,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; /* * We enforce the MAP_FIXED case. @@ -51,11 +51,9 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, return addr; } - info.flags = 0; info.length = len; info.low_limit = mm->mmap_base; info.high_limit = TASK_SIZE; - info.align_mask = 0; info.align_offset = pgoff << PAGE_SHIFT; return vm_unmapped_area(&info); } diff --git a/arch/arm/mm/mmap.c b/arch/arm/mm/mmap.c index a0f8a0ca0788a..d65d0e6ed10ab 100644 --- a/arch/arm/mm/mmap.c +++ b/arch/arm/mm/mmap.c @@ -34,7 +34,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, struct vm_area_struct *vma; int do_align = 0; int aliasing = cache_is_vipt_aliasing(); - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; /* * We only need to do colour alignment if either the I or D @@ -68,7 +68,6 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, return addr; } - info.flags = 0; info.length = len; info.low_limit = mm->mmap_base; info.high_limit = TASK_SIZE; @@ -87,7 +86,7 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, unsigned long addr = addr0; int do_align = 0; int aliasing = cache_is_vipt_aliasing(); - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; /* * We only need to do colour alignment if either the I or D diff --git a/arch/loongarch/mm/mmap.c b/arch/loongarch/mm/mmap.c index 89af7c12e8c08..8890309851351 100644 --- a/arch/loongarch/mm/mmap.c +++ b/arch/loongarch/mm/mmap.c @@ -25,7 +25,7 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, struct vm_area_struct *vma; unsigned long addr = addr0; int do_color_align; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; if (unlikely(len > TASK_SIZE)) return -ENOMEM; @@ -83,7 +83,6 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, */ } - info.flags = 0; info.low_limit = mm->mmap_base; info.high_limit = TASK_SIZE; return vm_unmapped_area(&info); diff --git a/arch/mips/mm/mmap.c b/arch/mips/mm/mmap.c index 00fe90c6db3e8..7e11d7b587610 100644 --- a/arch/mips/mm/mmap.c +++ b/arch/mips/mm/mmap.c @@ -34,7 +34,7 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, struct vm_area_struct *vma; unsigned long addr = addr0; int do_color_align; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; if (unlikely(len > TASK_SIZE)) return -ENOMEM; @@ -92,7 +92,6 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, */ } - info.flags = 0; info.low_limit = mm->mmap_base; info.high_limit = TASK_SIZE; return vm_unmapped_area(&info); diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c index 7d948e243f4b2..e1e63dc1b23d6 100644 --- a/arch/s390/mm/hugetlbpage.c +++ b/arch/s390/mm/hugetlbpage.c @@ -248,14 +248,12 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file, unsigned long pgoff, unsigned long flags) { struct hstate *h = hstate_file(file); - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; - info.flags = 0; info.length = len; info.low_limit = current->mm->mmap_base; info.high_limit = TASK_SIZE; info.align_mask = PAGE_MASK & ~huge_page_mask(h); - info.align_offset = 0; return vm_unmapped_area(&info); } @@ -264,7 +262,7 @@ static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file, unsigned long pgoff, unsigned long flags) { struct hstate *h = hstate_file(file); - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; unsigned long addr; info.flags = VM_UNMAPPED_AREA_TOPDOWN; @@ -272,7 +270,6 @@ static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file, info.low_limit = PAGE_SIZE; info.high_limit = current->mm->mmap_base; info.align_mask = PAGE_MASK & ~huge_page_mask(h); - info.align_offset = 0; addr = vm_unmapped_area(&info); /* diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c index 6b2e4436ad4a8..2067569465897 100644 --- a/arch/s390/mm/mmap.c +++ b/arch/s390/mm/mmap.c @@ -86,7 +86,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; if (len > TASK_SIZE - mmap_min_addr) return -ENOMEM; @@ -102,7 +102,6 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, goto check_asce_limit; } - info.flags = 0; info.length = len; info.low_limit = mm->mmap_base; info.high_limit = TASK_SIZE; @@ -122,7 +121,7 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp, unsigned long ad { struct vm_area_struct *vma; struct mm_struct *mm = current->mm; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; /* requested length too big for entire address space */ if (len > TASK_SIZE - mmap_min_addr) diff --git a/arch/sh/mm/mmap.c b/arch/sh/mm/mmap.c index b82199878b459..bee329d4149aa 100644 --- a/arch/sh/mm/mmap.c +++ b/arch/sh/mm/mmap.c @@ -57,7 +57,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, struct mm_struct *mm = current->mm; struct vm_area_struct *vma; int do_colour_align; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; if (flags & MAP_FIXED) { /* We do not accept a shared mapping if it would violate @@ -88,7 +88,6 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, return addr; } - info.flags = 0; info.length = len; info.low_limit = TASK_UNMAPPED_BASE; info.high_limit = TASK_SIZE; @@ -106,7 +105,7 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, struct mm_struct *mm = current->mm; unsigned long addr = addr0; int do_colour_align; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; if (flags & MAP_FIXED) { /* We do not accept a shared mapping if it would violate diff --git a/arch/sparc/kernel/sys_sparc_32.c b/arch/sparc/kernel/sys_sparc_32.c index 082a551897ed8..08a19727795ca 100644 --- a/arch/sparc/kernel/sys_sparc_32.c +++ b/arch/sparc/kernel/sys_sparc_32.c @@ -41,7 +41,7 @@ SYSCALL_DEFINE0(getpagesize) unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; if (flags & MAP_FIXED) { /* We do not accept a shared mapping if it would violate @@ -59,7 +59,6 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi if (!addr) addr = TASK_UNMAPPED_BASE; - info.flags = 0; info.length = len; info.low_limit = addr; info.high_limit = TASK_SIZE; diff --git a/arch/sparc/kernel/sys_sparc_64.c b/arch/sparc/kernel/sys_sparc_64.c index 1dbf7211666ea..d9c3b34ca7447 100644 --- a/arch/sparc/kernel/sys_sparc_64.c +++ b/arch/sparc/kernel/sys_sparc_64.c @@ -93,7 +93,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi struct vm_area_struct * vma; unsigned long task_size = TASK_SIZE; int do_color_align; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; if (flags & MAP_FIXED) { /* We do not accept a shared mapping if it would violate @@ -126,7 +126,6 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi return addr; } - info.flags = 0; info.length = len; info.low_limit = TASK_UNMAPPED_BASE; info.high_limit = min(task_size, VA_EXCLUDE_START); @@ -154,7 +153,7 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, unsigned long task_size = STACK_TOP32; unsigned long addr = addr0; int do_color_align; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; /* This should only ever run for 32-bit processes. */ BUG_ON(!test_thread_flag(TIF_32BIT)); diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c index c23012e3a3537..cc91ca7a1e182 100644 --- a/arch/sparc/mm/hugetlbpage.c +++ b/arch/sparc/mm/hugetlbpage.c @@ -31,17 +31,15 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp, { struct hstate *h = hstate_file(filp); unsigned long task_size = TASK_SIZE; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; if (test_thread_flag(TIF_32BIT)) task_size = STACK_TOP32; - info.flags = 0; info.length = len; info.low_limit = TASK_UNMAPPED_BASE; info.high_limit = min(task_size, VA_EXCLUDE_START); info.align_mask = PAGE_MASK & ~huge_page_mask(h); - info.align_offset = 0; addr = vm_unmapped_area(&info); if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) { @@ -63,7 +61,7 @@ hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, struct hstate *h = hstate_file(filp); struct mm_struct *mm = current->mm; unsigned long addr = addr0; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; /* This should only ever run for 32-bit processes. */ BUG_ON(!test_thread_flag(TIF_32BIT)); @@ -73,7 +71,6 @@ hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, info.low_limit = PAGE_SIZE; info.high_limit = mm->mmap_base; info.align_mask = PAGE_MASK & ~huge_page_mask(h); - info.align_offset = 0; addr = vm_unmapped_area(&info); /* diff --git a/arch/x86/kernel/sys_x86_64.c b/arch/x86/kernel/sys_x86_64.c index cb9fa1d5c66f7..96b9d29aead08 100644 --- a/arch/x86/kernel/sys_x86_64.c +++ b/arch/x86/kernel/sys_x86_64.c @@ -118,7 +118,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; unsigned long begin, end; if (flags & MAP_FIXED) @@ -137,11 +137,9 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, return addr; } - info.flags = 0; info.length = len; info.low_limit = begin; info.high_limit = end; - info.align_mask = 0; info.align_offset = pgoff << PAGE_SHIFT; if (filp) { info.align_mask = get_align_mask(); @@ -158,7 +156,7 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, struct vm_area_struct *vma; struct mm_struct *mm = current->mm; unsigned long addr = addr0; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; /* requested length too big for entire address space */ if (len > TASK_SIZE) @@ -203,7 +201,6 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, if (addr > DEFAULT_MAP_WINDOW && !in_32bit_syscall()) info.high_limit += TASK_SIZE_MAX - DEFAULT_MAP_WINDOW; - info.align_mask = 0; info.align_offset = pgoff << PAGE_SHIFT; if (filp) { info.align_mask = get_align_mask(); diff --git a/arch/x86/mm/hugetlbpage.c b/arch/x86/mm/hugetlbpage.c index 06ca9a60bac2a..807a5859a3c4b 100644 --- a/arch/x86/mm/hugetlbpage.c +++ b/arch/x86/mm/hugetlbpage.c @@ -25,9 +25,8 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file, unsigned long pgoff, unsigned long flags) { struct hstate *h = hstate_file(file); - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; - info.flags = 0; info.length = len; info.low_limit = get_mmap_base(1); @@ -39,7 +38,6 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file, task_size_32bit() : task_size_64bit(addr > DEFAULT_MAP_WINDOW); info.align_mask = PAGE_MASK & ~huge_page_mask(h); - info.align_offset = 0; return vm_unmapped_area(&info); } @@ -48,7 +46,7 @@ static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file, unsigned long pgoff, unsigned long flags) { struct hstate *h = hstate_file(file); - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.length = len; @@ -63,7 +61,6 @@ static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file, info.high_limit += TASK_SIZE_MAX - DEFAULT_MAP_WINDOW; info.align_mask = PAGE_MASK & ~huge_page_mask(h); - info.align_offset = 0; addr = vm_unmapped_area(&info); /* diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 3dee18bf47ed3..2f4e88552d3f2 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -176,14 +176,12 @@ hugetlb_get_unmapped_area_bottomup(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { struct hstate *h = hstate_file(file); - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; - info.flags = 0; info.length = len; info.low_limit = current->mm->mmap_base; info.high_limit = arch_get_mmap_end(addr, len, flags); info.align_mask = PAGE_MASK & ~huge_page_mask(h); - info.align_offset = 0; return vm_unmapped_area(&info); } @@ -192,14 +190,13 @@ hugetlb_get_unmapped_area_topdown(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { struct hstate *h = hstate_file(file); - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.length = len; info.low_limit = PAGE_SIZE; info.high_limit = arch_get_mmap_base(addr, current->mm->mmap_base); info.align_mask = PAGE_MASK & ~huge_page_mask(h); - info.align_offset = 0; addr = vm_unmapped_area(&info); /* diff --git a/mm/mmap.c b/mm/mmap.c index 0ef1191e3be5b..4ad386f3f63fa 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1704,7 +1704,7 @@ generic_get_unmapped_area(struct file *filp, unsigned long addr, { struct mm_struct *mm = current->mm; struct vm_area_struct *vma, *prev; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); if (len > mmap_end - mmap_min_addr) @@ -1722,12 +1722,9 @@ generic_get_unmapped_area(struct file *filp, unsigned long addr, return addr; } - info.flags = 0; info.length = len; info.low_limit = mm->mmap_base; info.high_limit = mmap_end; - info.align_mask = 0; - info.align_offset = 0; return vm_unmapped_area(&info); } @@ -1752,7 +1749,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, { struct vm_area_struct *vma, *prev; struct mm_struct *mm = current->mm; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); /* requested length too big for entire address space */ @@ -1776,8 +1773,6 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, info.length = len; info.low_limit = PAGE_SIZE; info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); - info.align_mask = 0; - info.align_offset = 0; addr = vm_unmapped_area(&info); /* From 44bd7ace9fd6490dce3b001b21d5c6bac869171b Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Mon, 25 Mar 2024 19:16:53 -0700 Subject: [PATCH 0708/1702] mm: take placement mappings gap into account When memory is being placed, mmap() will take care to respect the guard gaps of certain types of memory (VM_SHADOWSTACK, VM_GROWSUP and VM_GROWSDOWN). In order to ensure guard gaps between mappings, mmap() needs to consider two things: 1. That the new mapping isn't placed in an any existing mappings guard gaps. 2. That the new mapping isn't placed such that any existing mappings are not in *its* guard gaps. The longstanding behavior of mmap() is to ensure 1, but not take any care around 2. So for example, if there is a PAGE_SIZE free area, and a mmap() with a PAGE_SIZE size, and a type that has a guard gap is being placed, mmap() may place the shadow stack in the PAGE_SIZE free area. Then the mapping that is supposed to have a guard gap will not have a gap to the adjacent VMA. For MAP_GROWSDOWN/VM_GROWSDOWN and MAP_GROWSUP/VM_GROWSUP this has not been a problem in practice because applications place these kinds of mappings very early, when there is not many mappings to find a space between. But for shadow stacks, they may be placed throughout the lifetime of the application. Use the start_gap field to find a space that includes the guard gap for the new mapping. Take care to not interfere with the alignment. Link: https://lkml.kernel.org/r/20240326021656.202649-12-rick.p.edgecombe@intel.com Signed-off-by: Rick Edgecombe Reviewed-by: Christophe Leroy Cc: Alexei Starovoitov Cc: Andy Lutomirski Cc: Aneesh Kumar K.V Cc: Borislav Petkov (AMD) Cc: Dan Williams Cc: Dave Hansen Cc: Deepak Gupta Cc: Guo Ren Cc: Helge Deller Cc: H. Peter Anvin (Intel) Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Kees Cook Cc: Kirill A. Shutemov Cc: Liam R. Howlett Cc: Mark Brown Cc: Michael Ellerman Cc: Naveen N. Rao Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- include/linux/mm.h | 1 + mm/mmap.c | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 1b6903f4c57b4..2d5e492ef57ff 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3468,6 +3468,7 @@ struct vm_unmapped_area_info { unsigned long high_limit; unsigned long align_mask; unsigned long align_offset; + unsigned long start_gap; }; extern unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info); diff --git a/mm/mmap.c b/mm/mmap.c index 4ad386f3f63fa..d3c2ca8efa534 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1579,7 +1579,7 @@ static unsigned long unmapped_area(struct vm_unmapped_area_info *info) VMA_ITERATOR(vmi, current->mm, 0); /* Adjust search length to account for worst case alignment overhead */ - length = info->length + info->align_mask; + length = info->length + info->align_mask + info->start_gap; if (length < info->length) return -ENOMEM; @@ -1591,7 +1591,13 @@ static unsigned long unmapped_area(struct vm_unmapped_area_info *info) if (vma_iter_area_lowest(&vmi, low_limit, high_limit, length)) return -ENOMEM; - gap = vma_iter_addr(&vmi); + /* + * Adjust for the gap first so it doesn't interfere with the + * later alignment. The first step is the minimum needed to + * fulill the start gap, the next steps is the minimum to align + * that. It is the minimum needed to fulill both. + */ + gap = vma_iter_addr(&vmi) + info->start_gap; gap += (info->align_offset - gap) & info->align_mask; tmp = vma_next(&vmi); if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */ @@ -1630,7 +1636,7 @@ static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) VMA_ITERATOR(vmi, current->mm, 0); /* Adjust search length to account for worst case alignment overhead */ - length = info->length + info->align_mask; + length = info->length + info->align_mask + info->start_gap; if (length < info->length) return -ENOMEM; From c5ecd8eb8ca2f2f6746e61a83f6f16eeccb74d21 Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Mon, 25 Mar 2024 19:16:54 -0700 Subject: [PATCH 0709/1702] x86/mm: implement HAVE_ARCH_UNMAPPED_AREA_VMFLAGS When memory is being placed, mmap() will take care to respect the guard gaps of certain types of memory (VM_SHADOWSTACK, VM_GROWSUP and VM_GROWSDOWN). In order to ensure guard gaps between mappings, mmap() needs to consider two things: 1. That the new mapping isn't placed in an any existing mappings guard gaps. 2. That the new mapping isn't placed such that any existing mappings are not in *its* guard gaps. The longstanding behavior of mmap() is to ensure 1, but not take any care around 2. So for example, if there is a PAGE_SIZE free area, and a mmap() with a PAGE_SIZE size, and a type that has a guard gap is being placed, mmap() may place the shadow stack in the PAGE_SIZE free area. Then the mapping that is supposed to have a guard gap will not have a gap to the adjacent VMA. Add x86 arch implementations of arch_get_unmapped_area_vmflags/_topdown() so future changes can allow the guard gap of type of vma being placed to be taken into account. This will be used for shadow stack memory. Link: https://lkml.kernel.org/r/20240326021656.202649-13-rick.p.edgecombe@intel.com Signed-off-by: Rick Edgecombe Cc: Alexei Starovoitov Cc: Andy Lutomirski Cc: Aneesh Kumar K.V Cc: Borislav Petkov (AMD) Cc: Christophe Leroy Cc: Dan Williams Cc: Dave Hansen Cc: Deepak Gupta Cc: Guo Ren Cc: Helge Deller Cc: H. Peter Anvin (Intel) Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Kees Cook Cc: Kirill A. Shutemov Cc: Liam R. Howlett Cc: Mark Brown Cc: Michael Ellerman Cc: Naveen N. Rao Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- arch/x86/include/asm/pgtable_64.h | 1 + arch/x86/kernel/sys_x86_64.c | 25 ++++++++++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h index 7e9db77231ac7..3c4407271d083 100644 --- a/arch/x86/include/asm/pgtable_64.h +++ b/arch/x86/include/asm/pgtable_64.h @@ -245,6 +245,7 @@ extern void cleanup_highmap(void); #define HAVE_ARCH_UNMAPPED_AREA #define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN +#define HAVE_ARCH_UNMAPPED_AREA_VMFLAGS #define PAGE_AGP PAGE_KERNEL_NOCACHE #define HAVE_PAGE_AGP 1 diff --git a/arch/x86/kernel/sys_x86_64.c b/arch/x86/kernel/sys_x86_64.c index 96b9d29aead08..75966afb6251b 100644 --- a/arch/x86/kernel/sys_x86_64.c +++ b/arch/x86/kernel/sys_x86_64.c @@ -113,8 +113,8 @@ static void find_start_end(unsigned long addr, unsigned long flags, } unsigned long -arch_get_unmapped_area(struct file *filp, unsigned long addr, - unsigned long len, unsigned long pgoff, unsigned long flags) +arch_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; @@ -149,9 +149,9 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, } unsigned long -arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, - const unsigned long len, const unsigned long pgoff, - const unsigned long flags) +arch_get_unmapped_area_topdown_vmflags(struct file *filp, unsigned long addr0, + unsigned long len, unsigned long pgoff, + unsigned long flags, vm_flags_t vm_flags) { struct vm_area_struct *vma; struct mm_struct *mm = current->mm; @@ -220,3 +220,18 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, */ return arch_get_unmapped_area(filp, addr0, len, pgoff, flags); } + +unsigned long +arch_get_unmapped_area(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags) +{ + return arch_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0); +} + +unsigned long +arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr, + const unsigned long len, const unsigned long pgoff, + const unsigned long flags) +{ + return arch_get_unmapped_area_topdown_vmflags(filp, addr, len, pgoff, flags, 0); +} From c44357c2e76bd14e8af0a2074703d765b772671d Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Mon, 25 Mar 2024 19:16:55 -0700 Subject: [PATCH 0710/1702] x86/mm: care about shadow stack guard gap during placement When memory is being placed, mmap() will take care to respect the guard gaps of certain types of memory (VM_SHADOWSTACK, VM_GROWSUP and VM_GROWSDOWN). In order to ensure guard gaps between mappings, mmap() needs to consider two things: 1. That the new mapping isn't placed in an any existing mappings guard gaps. 2. That the new mapping isn't placed such that any existing mappings are not in *its* guard gaps. The longstanding behavior of mmap() is to ensure 1, but not take any care around 2. So for example, if there is a PAGE_SIZE free area, and a mmap() with a PAGE_SIZE size, and a type that has a guard gap is being placed, mmap() may place the shadow stack in the PAGE_SIZE free area. Then the mapping that is supposed to have a guard gap will not have a gap to the adjacent VMA. Now that the vm_flags is passed into the arch get_unmapped_area()'s, and vm_unmapped_area() is ready to consider it, have VM_SHADOW_STACK's get guard gap consideration for scenario 2. Link: https://lkml.kernel.org/r/20240326021656.202649-14-rick.p.edgecombe@intel.com Signed-off-by: Rick Edgecombe Cc: Alexei Starovoitov Cc: Andy Lutomirski Cc: Aneesh Kumar K.V Cc: Borislav Petkov (AMD) Cc: Christophe Leroy Cc: Dan Williams Cc: Dave Hansen Cc: Deepak Gupta Cc: Guo Ren Cc: Helge Deller Cc: H. Peter Anvin (Intel) Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Kees Cook Cc: Kirill A. Shutemov Cc: Liam R. Howlett Cc: Mark Brown Cc: Michael Ellerman Cc: Naveen N. Rao Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- arch/x86/kernel/sys_x86_64.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/x86/kernel/sys_x86_64.c b/arch/x86/kernel/sys_x86_64.c index 75966afb6251b..01d7cd85ef978 100644 --- a/arch/x86/kernel/sys_x86_64.c +++ b/arch/x86/kernel/sys_x86_64.c @@ -112,6 +112,14 @@ static void find_start_end(unsigned long addr, unsigned long flags, *end = task_size_64bit(addr > DEFAULT_MAP_WINDOW); } +static inline unsigned long stack_guard_placement(vm_flags_t vm_flags) +{ + if (vm_flags & VM_SHADOW_STACK) + return PAGE_SIZE; + + return 0; +} + unsigned long arch_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags) @@ -141,6 +149,7 @@ arch_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, unsigned l info.low_limit = begin; info.high_limit = end; info.align_offset = pgoff << PAGE_SHIFT; + info.start_gap = stack_guard_placement(vm_flags); if (filp) { info.align_mask = get_align_mask(); info.align_offset += get_align_bits(); @@ -190,6 +199,7 @@ arch_get_unmapped_area_topdown_vmflags(struct file *filp, unsigned long addr0, info.low_limit = PAGE_SIZE; info.high_limit = get_mmap_base(0); + info.start_gap = stack_guard_placement(vm_flags); /* * If hint address is above DEFAULT_MAP_WINDOW, look for unmapped area From a9bc15cb1cbd62b498b55958e92a90d0ea52a4b8 Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Mon, 25 Mar 2024 19:16:56 -0700 Subject: [PATCH 0711/1702] selftests/x86: add placement guard gap test for shstk The existing shadow stack test for guard gaps just checks that new mappings are not placed in an existing mapping's guard gap. Add one that checks that new mappings are not placed such that preexisting mappings are in the new mappings guard gap. Link: https://lkml.kernel.org/r/20240326021656.202649-15-rick.p.edgecombe@intel.com Signed-off-by: Rick Edgecombe Cc: Alexei Starovoitov Cc: Andy Lutomirski Cc: Aneesh Kumar K.V Cc: Borislav Petkov (AMD) Cc: Christophe Leroy Cc: Dan Williams Cc: Dave Hansen Cc: Deepak Gupta Cc: Guo Ren Cc: Helge Deller Cc: H. Peter Anvin (Intel) Cc: Ingo Molnar Cc: "James E.J. Bottomley" Cc: Kees Cook Cc: Kirill A. Shutemov Cc: Liam R. Howlett Cc: Mark Brown Cc: Michael Ellerman Cc: Naveen N. Rao Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- .../testing/selftests/x86/test_shadow_stack.c | 67 +++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/x86/test_shadow_stack.c b/tools/testing/selftests/x86/test_shadow_stack.c index 757e6527f67e2..ee909a7927f9d 100644 --- a/tools/testing/selftests/x86/test_shadow_stack.c +++ b/tools/testing/selftests/x86/test_shadow_stack.c @@ -556,7 +556,7 @@ struct node { * looked at the shadow stack gaps. * 5. See if it landed in the gap. */ -int test_guard_gap(void) +int test_guard_gap_other_gaps(void) { void *free_area, *shstk, *test_map = (void *)0xFFFFFFFFFFFFFFFF; struct node *head = NULL, *cur; @@ -593,11 +593,64 @@ int test_guard_gap(void) if (shstk - test_map - PAGE_SIZE != PAGE_SIZE) return 1; - printf("[OK]\tGuard gap test\n"); + printf("[OK]\tGuard gap test, other mapping's gaps\n"); return 0; } +/* Tests respecting the guard gap of the mapping getting placed */ +int test_guard_gap_new_mappings_gaps(void) +{ + void *free_area, *shstk_start, *test_map = (void *)0xFFFFFFFFFFFFFFFF; + struct node *head = NULL, *cur; + int ret = 0; + + free_area = mmap(0, PAGE_SIZE * 4, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + munmap(free_area, PAGE_SIZE * 4); + + /* Test letting map_shadow_stack find a free space */ + shstk_start = mmap(free_area, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (shstk_start == MAP_FAILED || shstk_start != free_area) + return 1; + + while (test_map > shstk_start) { + test_map = (void *)syscall(__NR_map_shadow_stack, 0, PAGE_SIZE, 0); + if (test_map == MAP_FAILED) { + printf("[INFO]\tmap_shadow_stack MAP_FAILED\n"); + ret = 1; + break; + } + + cur = malloc(sizeof(*cur)); + cur->mapping = test_map; + + cur->next = head; + head = cur; + + if (test_map == free_area + PAGE_SIZE) { + printf("[INFO]\tNew mapping has other mapping in guard gap!\n"); + ret = 1; + break; + } + } + + while (head) { + cur = head; + head = cur->next; + munmap(cur->mapping, PAGE_SIZE); + free(cur); + } + + munmap(shstk_start, PAGE_SIZE); + + if (!ret) + printf("[OK]\tGuard gap test, placement mapping's gaps\n"); + + return ret; +} + /* * Too complicated to pull it out of the 32 bit header, but also get the * 64 bit one needed above. Just define a copy here. @@ -850,9 +903,15 @@ int main(int argc, char *argv[]) goto out; } - if (test_guard_gap()) { + if (test_guard_gap_other_gaps()) { + ret = 1; + printf("[FAIL]\tGuard gap test, other mappings' gaps\n"); + goto out; + } + + if (test_guard_gap_new_mappings_gaps()) { ret = 1; - printf("[FAIL]\tGuard gap test\n"); + printf("[FAIL]\tGuard gap test, placement mapping's gaps\n"); goto out; } From 3a9e567ca45fb5280065283d10d9a11f0db61d2b Mon Sep 17 00:00:00 2001 From: Jinjiang Tu Date: Thu, 28 Mar 2024 19:10:08 +0800 Subject: [PATCH 0712/1702] mm/ksm: fix ksm exec support for prctl Patch series "mm/ksm: fix ksm exec support for prctl", v4. commit 3c6f33b7273a ("mm/ksm: support fork/exec for prctl") inherits MMF_VM_MERGE_ANY flag when a task calls execve(). However, it doesn't create the mm_slot, so ksmd will not try to scan this task. The first patch fixes the issue. The second patch refactors to prepare for the third patch. The third patch extends the selftests of ksm to verfity the deduplication really happens after fork/exec inherits ths KSM setting. This patch (of 3): commit 3c6f33b7273a ("mm/ksm: support fork/exec for prctl") inherits MMF_VM_MERGE_ANY flag when a task calls execve(). Howerver, it doesn't create the mm_slot, so ksmd will not try to scan this task. To fix it, allocate and add the mm_slot to ksm_mm_head in __bprm_mm_init() when the mm has MMF_VM_MERGE_ANY flag. Link: https://lkml.kernel.org/r/20240328111010.1502191-1-tujinjiang@huawei.com Link: https://lkml.kernel.org/r/20240328111010.1502191-2-tujinjiang@huawei.com Fixes: 3c6f33b7273a ("mm/ksm: support fork/exec for prctl") Signed-off-by: Jinjiang Tu Reviewed-by: David Hildenbrand Cc: Johannes Weiner Cc: Kefeng Wang Cc: Nanyong Sun Cc: Rik van Riel Cc: Stefan Roesch Signed-off-by: Andrew Morton --- fs/exec.c | 11 +++++++++++ include/linux/ksm.h | 13 +++++++++++++ 2 files changed, 24 insertions(+) diff --git a/fs/exec.c b/fs/exec.c index cf1df7f16e55c..0c5f06d08c355 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -67,6 +67,7 @@ #include #include #include +#include #include #include @@ -267,6 +268,14 @@ static int __bprm_mm_init(struct linux_binprm *bprm) goto err_free; } + /* + * Need to be called with mmap write lock + * held, to avoid race with ksmd. + */ + err = ksm_execve(mm); + if (err) + goto err_ksm; + /* * Place the stack at the largest stack address the architecture * supports. Later, we'll move this to an appropriate place. We don't @@ -288,6 +297,8 @@ static int __bprm_mm_init(struct linux_binprm *bprm) bprm->p = vma->vm_end - sizeof(void *); return 0; err: + ksm_exit(mm); +err_ksm: mmap_write_unlock(mm); err_free: bprm->vma = NULL; diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 401348e9f92b4..7e2b1de3996ac 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -59,6 +59,14 @@ static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm) return 0; } +static inline int ksm_execve(struct mm_struct *mm) +{ + if (test_bit(MMF_VM_MERGE_ANY, &mm->flags)) + return __ksm_enter(mm); + + return 0; +} + static inline void ksm_exit(struct mm_struct *mm) { if (test_bit(MMF_VM_MERGEABLE, &mm->flags)) @@ -107,6 +115,11 @@ static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm) return 0; } +static inline int ksm_execve(struct mm_struct *mm) +{ + return 0; +} + static inline void ksm_exit(struct mm_struct *mm) { } From 7abaacb8e5f55c6adaabc1b6bcbc1915e159fd1d Mon Sep 17 00:00:00 2001 From: Jinjiang Tu Date: Thu, 28 Mar 2024 19:10:09 +0800 Subject: [PATCH 0713/1702] selftest/mm: ksm_functional_tests: refactor mmap_and_merge_range() In order to extend test_prctl_fork() and test_prctl_fork_exec() to make sure that deduplication really happens, mmap_and_merge_range() needs to be refactored. Firstly, mmap_and_merge_range() will be called with no need to call enable KSM by madvise or prctl. So, switch the 'bool use_prctl' parameter to enum ksm_merge_mode. Secondly, mmap_and_merge_range() will be called in child process in the two testcases, it isn't appropriate to call ksft_test_result_{fail, skip}, because the global variables ksft_{fail, skip} aren't consistent with the parent process. Thus, convert calls of ksft_test_result_{fail, skip} to ksft_print_msg(), return differrent error according to the two cases, and rename mmap_and_merge_range() to __mmap_and_merge_range(). For existing callers, introduce new mmap_and_merge_range() to handle different return values of __mmap_and_merge_range(). Link: https://lkml.kernel.org/r/20240328111010.1502191-3-tujinjiang@huawei.com Signed-off-by: Jinjiang Tu Suggested-by: David Hildenbrand Reviewed-by: David Hildenbrand Cc: Johannes Weiner Cc: Kefeng Wang Cc: Nanyong Sun Cc: Rik van Riel Cc: Stefan Roesch Signed-off-by: Andrew Morton --- .../selftests/mm/ksm_functional_tests.c | 86 +++++++++++++------ 1 file changed, 61 insertions(+), 25 deletions(-) diff --git a/tools/testing/selftests/mm/ksm_functional_tests.c b/tools/testing/selftests/mm/ksm_functional_tests.c index d615767e396be..5462873df84c0 100644 --- a/tools/testing/selftests/mm/ksm_functional_tests.c +++ b/tools/testing/selftests/mm/ksm_functional_tests.c @@ -28,6 +28,15 @@ #define MiB (1024 * KiB) #define FORK_EXEC_CHILD_PRG_NAME "ksm_fork_exec_child" +#define MAP_MERGE_FAIL ((void *)-1) +#define MAP_MERGE_SKIP ((void *)-2) + +enum ksm_merge_mode { + KSM_MERGE_PRCTL, + KSM_MERGE_MADVISE, + KSM_MERGE_NONE, /* PRCTL already set */ +}; + static int mem_fd; static int ksm_fd; static int ksm_full_scans_fd; @@ -146,33 +155,34 @@ static int ksm_unmerge(void) return 0; } -static char *mmap_and_merge_range(char val, unsigned long size, int prot, - bool use_prctl) +static char *__mmap_and_merge_range(char val, unsigned long size, int prot, + enum ksm_merge_mode mode) { char *map; + char *err_map = MAP_MERGE_FAIL; int ret; /* Stabilize accounting by disabling KSM completely. */ if (ksm_unmerge()) { - ksft_test_result_fail("Disabling (unmerging) KSM failed\n"); - return MAP_FAILED; + ksft_print_msg("Disabling (unmerging) KSM failed\n"); + return err_map; } if (get_my_merging_pages() > 0) { - ksft_test_result_fail("Still pages merged\n"); - return MAP_FAILED; + ksft_print_msg("Still pages merged\n"); + return err_map; } map = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0); if (map == MAP_FAILED) { - ksft_test_result_fail("mmap() failed\n"); - return MAP_FAILED; + ksft_print_msg("mmap() failed\n"); + return err_map; } /* Don't use THP. Ignore if THP are not around on a kernel. */ if (madvise(map, size, MADV_NOHUGEPAGE) && errno != EINVAL) { - ksft_test_result_fail("MADV_NOHUGEPAGE failed\n"); + ksft_print_msg("MADV_NOHUGEPAGE failed\n"); goto unmap; } @@ -180,27 +190,36 @@ static char *mmap_and_merge_range(char val, unsigned long size, int prot, memset(map, val, size); if (mprotect(map, size, prot)) { - ksft_test_result_skip("mprotect() failed\n"); + ksft_print_msg("mprotect() failed\n"); + err_map = MAP_MERGE_SKIP; goto unmap; } - if (use_prctl) { + switch (mode) { + case KSM_MERGE_PRCTL: ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0); if (ret < 0 && errno == EINVAL) { - ksft_test_result_skip("PR_SET_MEMORY_MERGE not supported\n"); + ksft_print_msg("PR_SET_MEMORY_MERGE not supported\n"); + err_map = MAP_MERGE_SKIP; goto unmap; } else if (ret) { - ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 failed\n"); + ksft_print_msg("PR_SET_MEMORY_MERGE=1 failed\n"); goto unmap; } - } else if (madvise(map, size, MADV_MERGEABLE)) { - ksft_test_result_fail("MADV_MERGEABLE failed\n"); - goto unmap; + break; + case KSM_MERGE_MADVISE: + if (madvise(map, size, MADV_MERGEABLE)) { + ksft_print_msg("MADV_MERGEABLE failed\n"); + goto unmap; + } + break; + case KSM_MERGE_NONE: + break; } /* Run KSM to trigger merging and wait. */ if (ksm_merge()) { - ksft_test_result_fail("Running KSM failed\n"); + ksft_print_msg("Running KSM failed\n"); goto unmap; } @@ -209,14 +228,31 @@ static char *mmap_and_merge_range(char val, unsigned long size, int prot, * accounted differently (depending on kernel support). */ if (val && !get_my_merging_pages()) { - ksft_test_result_fail("No pages got merged\n"); + ksft_print_msg("No pages got merged\n"); goto unmap; } return map; unmap: munmap(map, size); - return MAP_FAILED; + return err_map; +} + +static char *mmap_and_merge_range(char val, unsigned long size, int prot, + enum ksm_merge_mode mode) +{ + char *map; + char *ret = MAP_FAILED; + + map = __mmap_and_merge_range(val, size, prot, mode); + if (map == MAP_MERGE_FAIL) + ksft_test_result_fail("Merging memory failed"); + else if (map == MAP_MERGE_SKIP) + ksft_test_result_skip("Merging memory skipped"); + else + ret = map; + + return ret; } static void test_unmerge(void) @@ -226,7 +262,7 @@ static void test_unmerge(void) ksft_print_msg("[RUN] %s\n", __func__); - map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, false); + map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, KSM_MERGE_MADVISE); if (map == MAP_FAILED) return; @@ -264,7 +300,7 @@ static void test_unmerge_zero_pages(void) } /* Let KSM deduplicate zero pages. */ - map = mmap_and_merge_range(0x00, size, PROT_READ | PROT_WRITE, false); + map = mmap_and_merge_range(0x00, size, PROT_READ | PROT_WRITE, KSM_MERGE_MADVISE); if (map == MAP_FAILED) return; @@ -312,7 +348,7 @@ static void test_unmerge_discarded(void) ksft_print_msg("[RUN] %s\n", __func__); - map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, false); + map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, KSM_MERGE_MADVISE); if (map == MAP_FAILED) return; @@ -344,7 +380,7 @@ static void test_unmerge_uffd_wp(void) ksft_print_msg("[RUN] %s\n", __func__); - map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, false); + map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, KSM_MERGE_MADVISE); if (map == MAP_FAILED) return; @@ -545,7 +581,7 @@ static void test_prctl_unmerge(void) ksft_print_msg("[RUN] %s\n", __func__); - map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, true); + map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, KSM_MERGE_PRCTL); if (map == MAP_FAILED) return; @@ -568,7 +604,7 @@ static void test_prot_none(void) ksft_print_msg("[RUN] %s\n", __func__); - map = mmap_and_merge_range(0x11, size, PROT_NONE, false); + map = mmap_and_merge_range(0x11, size, PROT_NONE, KSM_MERGE_MADVISE); if (map == MAP_FAILED) goto unmap; From 6c47de3be3a021d8b28d127802d590a49598a514 Mon Sep 17 00:00:00 2001 From: Jinjiang Tu Date: Thu, 28 Mar 2024 19:10:10 +0800 Subject: [PATCH 0714/1702] selftest/mm: ksm_functional_tests: extend test case for ksm fork/exec This extends test_prctl_fork() and test_prctl_fork_exec() to make sure that deduplication really happens, instead of only testing the MMF_VM_MERGE_ANY flag is set. [colin.i.king@gmail.com: fix spelling mistake in ksft_test_result_skip message] Link: https://lkml.kernel.org/r/20240402081537.1365939-1-colin.i.king@gmail.com Link: https://lkml.kernel.org/r/20240328111010.1502191-4-tujinjiang@huawei.com Signed-off-by: Jinjiang Tu Signed-off-by: Colin Ian King Suggested-by: David Hildenbrand Reviewed-by: David Hildenbrand Cc: Johannes Weiner Cc: Kefeng Wang Cc: Nanyong Sun Cc: Rik van Riel Cc: Stefan Roesch Signed-off-by: Andrew Morton --- .../selftests/mm/ksm_functional_tests.c | 49 ++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/mm/ksm_functional_tests.c b/tools/testing/selftests/mm/ksm_functional_tests.c index 5462873df84c0..db845dca8d195 100644 --- a/tools/testing/selftests/mm/ksm_functional_tests.c +++ b/tools/testing/selftests/mm/ksm_functional_tests.c @@ -475,6 +475,36 @@ static void test_prctl(void) ksft_test_result_pass("Setting/clearing PR_SET_MEMORY_MERGE works\n"); } +static int test_child_ksm(void) +{ + const unsigned int size = 2 * MiB; + char *map; + + /* Test if KSM is enabled for the process. */ + if (prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0) != 1) + return -1; + + /* Test if merge could really happen. */ + map = __mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, KSM_MERGE_NONE); + if (map == MAP_MERGE_FAIL) + return -2; + else if (map == MAP_MERGE_SKIP) + return -3; + + munmap(map, size); + return 0; +} + +static void test_child_ksm_err(int status) +{ + if (status == -1) + ksft_test_result_fail("unexpected PR_GET_MEMORY_MERGE result in child\n"); + else if (status == -2) + ksft_test_result_fail("Merge in child failed\n"); + else if (status == -3) + ksft_test_result_skip("Merge in child skipped\n"); +} + /* Verify that prctl ksm flag is inherited. */ static void test_prctl_fork(void) { @@ -494,7 +524,7 @@ static void test_prctl_fork(void) child_pid = fork(); if (!child_pid) { - exit(prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0)); + exit(test_child_ksm()); } else if (child_pid < 0) { ksft_test_result_fail("fork() failed\n"); return; @@ -503,8 +533,11 @@ static void test_prctl_fork(void) if (waitpid(child_pid, &status, 0) < 0) { ksft_test_result_fail("waitpid() failed\n"); return; - } else if (WEXITSTATUS(status) != 1) { - ksft_test_result_fail("unexpected PR_GET_MEMORY_MERGE result in child\n"); + } + + status = WEXITSTATUS(status); + if (status) { + test_child_ksm_err(status); return; } @@ -516,12 +549,6 @@ static void test_prctl_fork(void) ksft_test_result_pass("PR_SET_MEMORY_MERGE value is inherited\n"); } -static int ksm_fork_exec_child(void) -{ - /* Test if KSM is enabled for the process. */ - return prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0) == 1; -} - static void test_prctl_fork_exec(void) { int ret, status; @@ -554,7 +581,7 @@ static void test_prctl_fork_exec(void) if (WIFEXITED(status)) { status = WEXITSTATUS(status); if (status) { - ksft_test_result_fail("KSM not enabled\n"); + test_child_ksm_err(status); return; } } else { @@ -635,7 +662,7 @@ int main(int argc, char **argv) int err; if (argc > 1 && !strcmp(argv[1], FORK_EXEC_CHILD_PRG_NAME)) { - exit(ksm_fork_exec_child() == 1 ? 0 : 1); + exit(test_child_ksm()); } #ifdef __NR_userfaultfd From ba42b524a0408b5f92bd41edaee1ea84309ab9ae Mon Sep 17 00:00:00 2001 From: York Jasper Niebuhr Date: Fri, 29 Mar 2024 15:56:05 +0100 Subject: [PATCH 0715/1702] mm: init_mlocked_on_free_v3 Implements the "init_mlocked_on_free" boot option. When this boot option is enabled, any mlock'ed pages are zeroed on free. If the pages are munlock'ed beforehand, no initialization takes place. This boot option is meant to combat the performance hit of "init_on_free" as reported in commit 6471384af2a6 ("mm: security: introduce init_on_alloc=1 and init_on_free=1 boot options"). With "init_mlocked_on_free=1" only relevant data is freed while everything else is left untouched by the kernel. Correspondingly, this patch introduces no performance hit for unmapping non-mlock'ed memory. The unmapping overhead for purely mlocked memory was measured to be approximately 13%. Realistically, most systems mlock only a fraction of the total memory so the real-world system overhead should be close to zero. Optimally, userspace programs clear any key material or other confidential memory before exit and munlock the according memory regions. If a program crashes, userspace key managers fail to do this job. Accordingly, no munlock operations are performed so the data is caught and zeroed by the kernel. Should the program not crash, all memory will ideally be munlocked so no overhead is caused. CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON can be set to enable "init_mlocked_on_free" by default. Link: https://lkml.kernel.org/r/20240329145605.149917-1-yjnworkstation@gmail.com Signed-off-by: York Jasper Niebuhr Cc: Matthew Wilcox (Oracle) Cc: York Jasper Niebuhr Cc: Kees Cook Signed-off-by: Andrew Morton --- .../admin-guide/kernel-parameters.txt | 6 +++ include/linux/mm.h | 9 +++- mm/internal.h | 1 + mm/memory.c | 6 +++ mm/mm_init.c | 43 ++++++++++++++++--- mm/page_alloc.c | 2 +- security/Kconfig.hardening | 15 +++++++ 7 files changed, 73 insertions(+), 9 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 902ecd92a29fb..3ff97de349daf 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2148,6 +2148,12 @@ Format: 0 | 1 Default set by CONFIG_INIT_ON_FREE_DEFAULT_ON. + init_mlocked_on_free= [MM] Fill freed userspace memory with zeroes if + it was mlock'ed and not explicitly munlock'ed + afterwards. + Format: 0 | 1 + Default set by CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON + init_pkru= [X86] Specify the default memory protection keys rights register contents for all processes. 0x55555554 by default (disallow access to all but pkey 0). Can diff --git a/include/linux/mm.h b/include/linux/mm.h index 2d5e492ef57ff..4f4e460d78531 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3762,7 +3762,14 @@ DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); static inline bool want_init_on_free(void) { return static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON, - &init_on_free); + &init_on_free); +} + +DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON, init_mlocked_on_free); +static inline bool want_init_mlocked_on_free(void) +{ + return static_branch_maybe(CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON, + &init_mlocked_on_free); } extern bool _debug_pagealloc_enabled_early; diff --git a/mm/internal.h b/mm/internal.h index 6614ba4ca9dec..cf7799e293916 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -506,6 +506,7 @@ extern void __putback_isolated_page(struct page *page, unsigned int order, extern void memblock_free_pages(struct page *page, unsigned long pfn, unsigned int order); extern void __free_pages_core(struct page *page, unsigned int order); +extern void kernel_init_pages(struct page *page, int numpages); /* * This will have no effect, other than possibly generating a warning, if the diff --git a/mm/memory.c b/mm/memory.c index 0b92336bcebde..80944acb5b4e4 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1506,6 +1506,12 @@ static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, if (unlikely(page_mapcount(page) < 0)) print_bad_pte(vma, addr, ptent, page); } + + if (want_init_mlocked_on_free() && folio_test_mlocked(folio) && + !delay_rmap && folio_test_anon(folio)) { + kernel_init_pages(page, folio_nr_pages(folio)); + } + if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) { *force_flush = true; *force_break = true; diff --git a/mm/mm_init.c b/mm/mm_init.c index d01912b8a5973..2c8f3af4430f5 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2522,6 +2522,9 @@ EXPORT_SYMBOL(init_on_alloc); DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); EXPORT_SYMBOL(init_on_free); +DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON, init_mlocked_on_free); +EXPORT_SYMBOL(init_mlocked_on_free); + static bool _init_on_alloc_enabled_early __read_mostly = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON); static int __init early_init_on_alloc(char *buf) @@ -2539,6 +2542,14 @@ static int __init early_init_on_free(char *buf) } early_param("init_on_free", early_init_on_free); +static bool _init_mlocked_on_free_enabled_early __read_mostly + = IS_ENABLED(CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON); +static int __init early_init_mlocked_on_free(char *buf) +{ + return kstrtobool(buf, &_init_mlocked_on_free_enabled_early); +} +early_param("init_mlocked_on_free", early_init_mlocked_on_free); + DEFINE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); /* @@ -2566,12 +2577,21 @@ static void __init mem_debugging_and_hardening_init(void) } #endif - if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) && + if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early || + _init_mlocked_on_free_enabled_early) && page_poisoning_requested) { pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, " - "will take precedence over init_on_alloc and init_on_free\n"); + "will take precedence over init_on_alloc, init_on_free " + "and init_mlocked_on_free\n"); _init_on_alloc_enabled_early = false; _init_on_free_enabled_early = false; + _init_mlocked_on_free_enabled_early = false; + } + + if (_init_mlocked_on_free_enabled_early && _init_on_free_enabled_early) { + pr_info("mem auto-init: init_on_free is on, " + "will take precedence over init_mlocked_on_free\n"); + _init_mlocked_on_free_enabled_early = false; } if (_init_on_alloc_enabled_early) { @@ -2588,9 +2608,17 @@ static void __init mem_debugging_and_hardening_init(void) static_branch_disable(&init_on_free); } - if (IS_ENABLED(CONFIG_KMSAN) && - (_init_on_alloc_enabled_early || _init_on_free_enabled_early)) - pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n"); + if (_init_mlocked_on_free_enabled_early) { + want_check_pages = true; + static_branch_enable(&init_mlocked_on_free); + } else { + static_branch_disable(&init_mlocked_on_free); + } + + if (IS_ENABLED(CONFIG_KMSAN) && (_init_on_alloc_enabled_early || + _init_on_free_enabled_early || _init_mlocked_on_free_enabled_early)) + pr_info("mem auto-init: please make sure init_on_alloc, init_on_free and " + "init_mlocked_on_free are disabled when running KMSAN\n"); #ifdef CONFIG_DEBUG_PAGEALLOC if (debug_pagealloc_enabled()) { @@ -2629,9 +2657,10 @@ static void __init report_meminit(void) else stack = "off"; - pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n", + pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s, mlocked free:%s\n", stack, want_init_on_alloc(GFP_KERNEL) ? "on" : "off", - want_init_on_free() ? "on" : "off"); + want_init_on_free() ? "on" : "off", + want_init_mlocked_on_free() ? "on" : "off"); if (want_init_on_free()) pr_info("mem auto-init: clearing system memory may take some time...\n"); } diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 47ab0297838a6..e030ccf9d5bcd 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1032,7 +1032,7 @@ static inline bool should_skip_kasan_poison(struct page *page) return page_kasan_tag(page) == KASAN_TAG_KERNEL; } -static void kernel_init_pages(struct page *page, int numpages) +void kernel_init_pages(struct page *page, int numpages) { int i; diff --git a/security/Kconfig.hardening b/security/Kconfig.hardening index 2cff851ebfd7e..effbf5982be10 100644 --- a/security/Kconfig.hardening +++ b/security/Kconfig.hardening @@ -255,6 +255,21 @@ config INIT_ON_FREE_DEFAULT_ON touching "cold" memory areas. Most cases see 3-5% impact. Some synthetic workloads have measured as high as 8%. +config INIT_MLOCKED_ON_FREE_DEFAULT_ON + bool "Enable mlocked memory zeroing on free" + depends on !KMSAN + help + This config has the effect of setting "init_mlocked_on_free=1" + on the kernel command line. If it is enabled, all mlocked process + memory is zeroed when freed. This restriction to mlocked memory + improves performance over "init_on_free" but can still be used to + protect confidential data like key material from content exposures + to other processes, as well as live forensics and cold boot attacks. + Any non-mlocked memory is not cleared before it is reassigned. This + configuration can be overwritten by setting "init_mlocked_on_free=0" + on the command line. The "init_on_free" boot option takes + precedence over "init_mlocked_on_free". + config CC_HAS_ZERO_CALL_USED_REGS def_bool $(cc-option,-fzero-call-used-regs=used-gpr) # https://github.com/ClangBuiltLinux/linux/issues/1766 From 34efe1c3b688944d9817a5faaab7aad870182c59 Mon Sep 17 00:00:00 2001 From: Sergey Senozhatsky Date: Fri, 29 Mar 2024 18:39:41 +0900 Subject: [PATCH 0716/1702] zram: add max_pages param to recompression Introduce "max_pages" param to recompress device attribute which sets an upper limit on the number of entries (pages) zram attempts to recompress (in this particular recompression call). S/W recompression can be quite expensive so limiting the number of pages recompress touches can be quite helpful. Link: https://lkml.kernel.org/r/20240329094050.2815699-1-senozhatsky@chromium.org Signed-off-by: Sergey Senozhatsky Acked-by: Brian Geffon Cc: Minchan Kim Signed-off-by: Andrew Morton --- Documentation/admin-guide/blockdev/zram.rst | 5 ++++ drivers/block/zram/zram_drv.c | 31 +++++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/blockdev/zram.rst b/Documentation/admin-guide/blockdev/zram.rst index ee2b0030d4168..091e8bb388875 100644 --- a/Documentation/admin-guide/blockdev/zram.rst +++ b/Documentation/admin-guide/blockdev/zram.rst @@ -466,6 +466,11 @@ of equal or greater size::: #recompress idle pages larger than 2000 bytes echo "type=idle threshold=2000" > /sys/block/zramX/recompress +It is also possible to limit the number of pages zram re-compression will +attempt to recompress::: + + echo "type=huge_idle max_pages=42" > /sys/block/zramX/recompress + Recompression of idle pages requires memory tracking. During re-compression for every page, that matches re-compression criteria, diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index f0639df6cd184..4cf38f7d3e0a3 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1568,7 +1568,8 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, * Corresponding ZRAM slot should be locked. */ static int zram_recompress(struct zram *zram, u32 index, struct page *page, - u32 threshold, u32 prio, u32 prio_max) + u64 *num_recomp_pages, u32 threshold, u32 prio, + u32 prio_max) { struct zcomp_strm *zstrm = NULL; unsigned long handle_old; @@ -1645,6 +1646,15 @@ static int zram_recompress(struct zram *zram, u32 index, struct page *page, if (!zstrm) return 0; + /* + * Decrement the limit (if set) on pages we can recompress, even + * when current recompression was unsuccessful or did not compress + * the page below the threshold, because we still spent resources + * on it. + */ + if (*num_recomp_pages) + *num_recomp_pages -= 1; + if (class_index_new >= class_index_old) { /* * Secondary algorithms failed to re-compress the page @@ -1710,6 +1720,7 @@ static ssize_t recompress_store(struct device *dev, struct zram *zram = dev_to_zram(dev); unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; char *args, *param, *val, *algo = NULL; + u64 num_recomp_pages = ULLONG_MAX; u32 mode = 0, threshold = 0; unsigned long index; struct page *page; @@ -1732,6 +1743,17 @@ static ssize_t recompress_store(struct device *dev, continue; } + if (!strcmp(param, "max_pages")) { + /* + * Limit the number of entries (pages) we attempt to + * recompress. + */ + ret = kstrtoull(val, 10, &num_recomp_pages); + if (ret) + return ret; + continue; + } + if (!strcmp(param, "threshold")) { /* * We will re-compress only idle objects equal or @@ -1788,6 +1810,9 @@ static ssize_t recompress_store(struct device *dev, for (index = 0; index < nr_pages; index++) { int err = 0; + if (!num_recomp_pages) + break; + zram_slot_lock(zram, index); if (!zram_allocated(zram, index)) @@ -1807,8 +1832,8 @@ static ssize_t recompress_store(struct device *dev, zram_test_flag(zram, index, ZRAM_INCOMPRESSIBLE)) goto next; - err = zram_recompress(zram, index, page, threshold, - prio, prio_max); + err = zram_recompress(zram, index, page, &num_recomp_pages, + threshold, prio, prio_max); next: zram_slot_unlock(zram, index); if (err) { From 68dbcf4899f357c707c8bd42326533d729e8e8bb Mon Sep 17 00:00:00 2001 From: Barry Song Date: Fri, 29 Mar 2024 20:37:50 +1300 Subject: [PATCH 0717/1702] mm: alloc_anon_folio: avoid doing vma_thp_gfp_mask in fallback cases Fallback rates surpassing 90% have been observed on phones utilizing 64KiB CONT-PTE mTHP. In these scenarios, when one out of every 16 PTEs fails to allocate large folios, the remaining 15 PTEs fallback. Consequently, invoking vma_thp_gfp_mask seems redundant in such cases. Furthermore, abstaining from its use can also contribute to improved code readability. Link: https://lkml.kernel.org/r/20240329073750.20012-1-21cnbao@gmail.com Signed-off-by: Barry Song Reviewed-by: Zi Yan Acked-by: Yu Zhao Reviewed-by: Ryan Roberts Cc: Kefeng Wang Cc: John Hubbard Cc: David Hildenbrand Cc: Alistair Popple Cc: Anshuman Khandual Cc: Catalin Marinas Cc: David Rientjes Cc: "Huang, Ying" Cc: Hugh Dickins Cc: Itaru Kitayama Cc: Kirill A. Shutemov Cc: Luis Chamberlain Cc: Matthew Wilcox (Oracle) Cc: Vlastimil Babka Cc: Yang Shi Cc: Yin Fengwei Signed-off-by: Andrew Morton --- mm/memory.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/memory.c b/mm/memory.c index 80944acb5b4e4..93eaeaf5ca9dd 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4358,6 +4358,9 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf) pte_unmap(pte); + if (!orders) + goto fallback; + /* Try allocating the highest of the remaining orders. */ gfp = vma_thp_gfp_mask(vma); while (orders) { From 6b0ed7b3c77547d2308983a26db11a0d14a60ace Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Fri, 29 Mar 2024 14:56:45 +0800 Subject: [PATCH 0718/1702] mm: factor out the numa mapping rebuilding into a new helper Patch series "support multi-size THP numa balancing", v2. This patchset tries to support mTHP numa balancing, as a simple solution to start, the NUMA balancing algorithm for mTHP will follow the THP strategy as the basic support. Please find details in each patch. This patch (of 2): To support large folio's numa balancing, factor out the numa mapping rebuilding into a new helper as a preparation. Link: https://lkml.kernel.org/r/cover.1712132950.git.baolin.wang@linux.alibaba.com Link: https://lkml.kernel.org/r/cover.1711683069.git.baolin.wang@linux.alibaba.com Link: https://lkml.kernel.org/r/8bc2586bdd8dbbe6d83c09b77b360ec8fcac3736.1711683069.git.baolin.wang@linux.alibaba.com Signed-off-by: Baolin Wang Reviewed-by: "Huang, Ying" Cc: David Hildenbrand Cc: John Hubbard Cc: Kefeng Wang Cc: Mel Gorman Cc: Ryan Roberts Signed-off-by: Andrew Morton --- mm/memory.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index 93eaeaf5ca9dd..77c69bf297eea 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5063,6 +5063,20 @@ int numa_migrate_prep(struct folio *folio, struct vm_fault *vmf, return mpol_misplaced(folio, vmf, addr); } +static void numa_rebuild_single_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, + bool writable) +{ + pte_t pte, old_pte; + + old_pte = ptep_modify_prot_start(vma, vmf->address, vmf->pte); + pte = pte_modify(old_pte, vma->vm_page_prot); + pte = pte_mkyoung(pte); + if (writable) + pte = pte_mkwrite(pte, vma); + ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); +} + static vm_fault_t do_numa_page(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -5168,13 +5182,7 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) * Make it present again, depending on how arch implements * non-accessible ptes, some can allow access by kernel mode. */ - old_pte = ptep_modify_prot_start(vma, vmf->address, vmf->pte); - pte = pte_modify(old_pte, vma->vm_page_prot); - pte = pte_mkyoung(pte); - if (writable) - pte = pte_mkwrite(pte, vma); - ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte); - update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + numa_rebuild_single_mapping(vmf, vma, writable); pte_unmap_unlock(vmf->pte, vmf->ptl); goto out; } From d2136d749d76af980b3accd72704eea4eab625bd Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Fri, 29 Mar 2024 14:56:46 +0800 Subject: [PATCH 0719/1702] mm: support multi-size THP numa balancing Now the anonymous page allocation already supports multi-size THP (mTHP), but the numa balancing still prohibits mTHP migration even though it is an exclusive mapping, which is unreasonable. Allow scanning mTHP: Commit 859d4adc3415 ("mm: numa: do not trap faults on shared data section pages") skips shared CoW pages' NUMA page migration to avoid shared data segment migration. In addition, commit 80d47f5de5e3 ("mm: don't try to NUMA-migrate COW pages that have other uses") change to use page_count() to avoid GUP pages migration, that will also skip the mTHP numa scanning. Theoretically, we can use folio_maybe_dma_pinned() to detect the GUP issue, although there is still a GUP race, the issue seems to have been resolved by commit 80d47f5de5e3. Meanwhile, use the folio_likely_mapped_shared() to skip shared CoW pages though this is not a precise sharers count. To check if the folio is shared, ideally we want to make sure every page is mapped to the same process, but doing that seems expensive and using the estimated mapcount seems can work when running autonuma benchmark. Allow migrating mTHP: As mentioned in the previous thread[1], large folios (including THP) are more susceptible to false sharing issues among threads than 4K base page, leading to pages ping-pong back and forth during numa balancing, which is currently not easy to resolve. Therefore, as a start to support mTHP numa balancing, we can follow the PMD mapped THP's strategy, that means we can reuse the 2-stage filter in should_numa_migrate_memory() to check if the mTHP is being heavily contended among threads (through checking the CPU id and pid of the last access) to avoid false sharing at some degree. Thus, we can restore all PTE maps upon the first hint page fault of a large folio to follow the PMD mapped THP's strategy. In the future, we can continue to optimize the NUMA balancing algorithm to avoid the false sharing issue with large folios as much as possible. Performance data: Machine environment: 2 nodes, 128 cores Intel(R) Xeon(R) Platinum Base: 2024-03-25 mm-unstable branch Enable mTHP to run autonuma-benchmark mTHP:16K Base Patched numa01 numa01 224.70 143.48 numa01_THREAD_ALLOC numa01_THREAD_ALLOC 118.05 47.43 numa02 numa02 13.45 9.29 numa02_SMT numa02_SMT 14.80 7.50 mTHP:64K Base Patched numa01 numa01 216.15 114.40 numa01_THREAD_ALLOC numa01_THREAD_ALLOC 115.35 47.41 numa02 numa02 13.24 9.25 numa02_SMT numa02_SMT 14.67 7.34 mTHP:128K Base Patched numa01 numa01 205.13 144.45 numa01_THREAD_ALLOC numa01_THREAD_ALLOC 112.93 41.88 numa02 numa02 13.16 9.18 numa02_SMT numa02_SMT 14.81 7.49 [1] https://lore.kernel.org/all/20231117100745.fnpijbk4xgmals3k@techsingularity.net/ [baolin.wang@linux.alibaba.com: v3] Link: https://lkml.kernel.org/r/c33a5c0b0a0323b1f8ed53772f50501f4b196e25.1712132950.git.baolin.wang@linux.alibaba.com Link: https://lkml.kernel.org/r/d28d276d599c26df7f38c9de8446f60e22dd1950.1711683069.git.baolin.wang@linux.alibaba.com Signed-off-by: Baolin Wang Reviewed-by: "Huang, Ying" Cc: David Hildenbrand Cc: John Hubbard Cc: Kefeng Wang Cc: Mel Gorman Cc: Ryan Roberts Signed-off-by: Andrew Morton --- mm/memory.c | 62 +++++++++++++++++++++++++++++++++++++++++---------- mm/mprotect.c | 3 ++- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index 77c69bf297eea..694e18837cd8c 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5064,17 +5064,51 @@ int numa_migrate_prep(struct folio *folio, struct vm_fault *vmf, } static void numa_rebuild_single_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, + unsigned long fault_addr, pte_t *fault_pte, bool writable) { pte_t pte, old_pte; - old_pte = ptep_modify_prot_start(vma, vmf->address, vmf->pte); + old_pte = ptep_modify_prot_start(vma, fault_addr, fault_pte); pte = pte_modify(old_pte, vma->vm_page_prot); pte = pte_mkyoung(pte); if (writable) pte = pte_mkwrite(pte, vma); - ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte); - update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + ptep_modify_prot_commit(vma, fault_addr, fault_pte, old_pte, pte); + update_mmu_cache_range(vmf, vma, fault_addr, fault_pte, 1); +} + +static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, + struct folio *folio, pte_t fault_pte, + bool ignore_writable, bool pte_write_upgrade) +{ + int nr = pte_pfn(fault_pte) - folio_pfn(folio); + unsigned long start = max(vmf->address - nr * PAGE_SIZE, vma->vm_start); + unsigned long end = min(vmf->address + (folio_nr_pages(folio) - nr) * PAGE_SIZE, vma->vm_end); + pte_t *start_ptep = vmf->pte - (vmf->address - start) / PAGE_SIZE; + unsigned long addr; + + /* Restore all PTEs' mapping of the large folio */ + for (addr = start; addr != end; start_ptep++, addr += PAGE_SIZE) { + pte_t ptent = ptep_get(start_ptep); + bool writable = false; + + if (!pte_present(ptent) || !pte_protnone(ptent)) + continue; + + if (pfn_folio(pte_pfn(ptent)) != folio) + continue; + + if (!ignore_writable) { + ptent = pte_modify(ptent, vma->vm_page_prot); + writable = pte_write(ptent); + if (!writable && pte_write_upgrade && + can_change_pte_writable(vma, addr, ptent)) + writable = true; + } + + numa_rebuild_single_mapping(vmf, vma, addr, start_ptep, writable); + } } static vm_fault_t do_numa_page(struct vm_fault *vmf) @@ -5082,11 +5116,12 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; struct folio *folio = NULL; int nid = NUMA_NO_NODE; - bool writable = false; + bool writable = false, ignore_writable = false; + bool pte_write_upgrade = vma_wants_manual_pte_write_upgrade(vma); int last_cpupid; int target_nid; pte_t pte, old_pte; - int flags = 0; + int flags = 0, nr_pages; /* * The pte cannot be used safely until we verify, while holding the page @@ -5108,7 +5143,7 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) * is only valid while holding the PT lock. */ writable = pte_write(pte); - if (!writable && vma_wants_manual_pte_write_upgrade(vma) && + if (!writable && pte_write_upgrade && can_change_pte_writable(vma, vmf->address, pte)) writable = true; @@ -5116,10 +5151,6 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) if (!folio || folio_is_zone_device(folio)) goto out_map; - /* TODO: handle PTE-mapped THP */ - if (folio_test_large(folio)) - goto out_map; - /* * Avoid grouping on RO pages in general. RO pages shouldn't hurt as * much anyway since they can be in shared cache state. This misses @@ -5139,6 +5170,7 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) flags |= TNF_SHARED; nid = folio_nid(folio); + nr_pages = folio_nr_pages(folio); /* * For memory tiering mode, cpupid of slow memory page is used * to record page access time. So use default value. @@ -5155,6 +5187,7 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) } pte_unmap_unlock(vmf->pte, vmf->ptl); writable = false; + ignore_writable = true; /* Migrate to the requested node */ if (migrate_misplaced_folio(folio, vma, target_nid)) { @@ -5175,14 +5208,19 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) out: if (nid != NUMA_NO_NODE) - task_numa_fault(last_cpupid, nid, 1, flags); + task_numa_fault(last_cpupid, nid, nr_pages, flags); return 0; out_map: /* * Make it present again, depending on how arch implements * non-accessible ptes, some can allow access by kernel mode. */ - numa_rebuild_single_mapping(vmf, vma, writable); + if (folio && folio_test_large(folio)) + numa_rebuild_large_mapping(vmf, vma, folio, pte, ignore_writable, + pte_write_upgrade); + else + numa_rebuild_single_mapping(vmf, vma, vmf->address, vmf->pte, + writable); pte_unmap_unlock(vmf->pte, vmf->ptl); goto out; } diff --git a/mm/mprotect.c b/mm/mprotect.c index f8a4544b4601d..94878c39ee32d 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -129,7 +129,8 @@ static long change_pte_range(struct mmu_gather *tlb, /* Also skip shared copy-on-write pages */ if (is_cow_mapping(vma->vm_flags) && - folio_ref_count(folio) != 1) + (folio_maybe_dma_pinned(folio) || + folio_likely_mapped_shared(folio))) continue; /* From 835c3a25aa373d486514e4e0f5a7450ea82ae489 Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Fri, 29 Mar 2024 14:59:33 +0800 Subject: [PATCH 0720/1702] mm: huge_memory: add the missing folio_test_pmd_mappable() for THP split statistics Now the mTHP can also be split or added into the deferred list, so add folio_test_pmd_mappable() validation for PMD mapped THP, to avoid confusion with PMD mapped THP related statistics. [baolin.wang@linux.alibaba.com: check THP earlier in case folio is split, per Lance] Link: https://lkml.kernel.org/r/b99f8cb14bc85fdb6ab43721d1331cb5ebed2581.1713771041.git.baolin.wang@linux.alibaba.com Link: https://lkml.kernel.org/r/a5341defeef27c9ac7b85c97f030f93e4368bbc1.1711694852.git.baolin.wang@linux.alibaba.com Signed-off-by: Baolin Wang Acked-by: David Hildenbrand Reviewed-by: Lance Yang Cc: Muchun Song Signed-off-by: Andrew Morton --- mm/huge_memory.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index b757d9be310c8..4065bf8bfcc43 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2934,6 +2934,7 @@ int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, XA_STATE_ORDER(xas, &folio->mapping->i_pages, folio->index, new_order); struct anon_vma *anon_vma = NULL; struct address_space *mapping = NULL; + bool is_thp = folio_test_pmd_mappable(folio); int extra_pins, ret; pgoff_t end; bool is_hzp; @@ -3112,7 +3113,8 @@ int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, i_mmap_unlock_read(mapping); out: xas_destroy(&xas); - count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); + if (is_thp) + count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); return ret; } @@ -3174,7 +3176,8 @@ void deferred_split_folio(struct folio *folio) spin_lock_irqsave(&ds_queue->split_queue_lock, flags); if (list_empty(&folio->_deferred_list)) { - count_vm_event(THP_DEFERRED_SPLIT_PAGE); + if (folio_test_pmd_mappable(folio)) + count_vm_event(THP_DEFERRED_SPLIT_PAGE); list_add_tail(&folio->_deferred_list, &ds_queue->split_queue); ds_queue->split_queue_len++; #ifdef CONFIG_MEMCG From 7e8347413e5bc4d54712942dad43bfcf2501ab3b Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 Mar 2024 22:58:27 +0000 Subject: [PATCH 0721/1702] mm: correct page_mapped_in_vma() for large folios Patch series "Unify vma_address and vma_pgoff_address". The current vma_address() pretends that the ambiguity between head & tail page is an advantage. If you pass a head page to vma_address(), it will operate on all pages in the folio, while if you pass a tail page, it will operate on a single page. That's not what any of the callers actually want, so first convert all callers to use vma_pgoff_address() and then rename vma_pgoff_address() to vma_address(). This patch (of 3): If 'page' is the first page of a large folio then vma_address() will scan for any page in the entire folio. This can lead to page_mapped_in_vma() returning true if some of the tail pages are mapped and the head page is not. This could lead to memory failure choosing to kill a task unnecessarily. Link: https://lkml.kernel.org/r/20240328225831.1765286-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240328225831.1765286-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- mm/page_vma_mapped.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index 74d2de15fb5e0..ac48d6284badc 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -325,6 +325,8 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) */ int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) { + struct folio *folio = page_folio(page); + pgoff_t pgoff = folio->index + folio_page_idx(folio, page); struct page_vma_mapped_walk pvmw = { .pfn = page_to_pfn(page), .nr_pages = 1, @@ -332,7 +334,7 @@ int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) .flags = PVMW_SYNC, }; - pvmw.address = vma_address(page, vma); + pvmw.address = vma_pgoff_address(pgoff, 1, vma); if (pvmw.address == -EFAULT) return 0; if (!page_vma_mapped_walk(&pvmw)) From 412ad5fbe9285fd8066d3b977db0cd7fb39f671d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 Mar 2024 22:58:28 +0000 Subject: [PATCH 0722/1702] mm: remove vma_address() Convert the three remaining callers to call vma_pgoff_address() directly. This removes an ambiguity where we'd check just one page if passed a tail page and all N pages if passed a head page. Also add better kernel-doc for vma_pgoff_address(). Link: https://lkml.kernel.org/r/20240328225831.1765286-3-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/internal.h | 23 ++++++++--------------- mm/rmap.c | 12 +++++++++--- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index cf7799e293916..f4ef48d57b1c4 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -804,9 +804,14 @@ void mlock_drain_remote(int cpu); extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); -/* - * Return the start of user virtual address at the specific offset within - * a vma. +/** + * vma_pgoff_address - Find the virtual address a page range is mapped at + * @pgoff: The page offset within its object. + * @nr_pages: The number of pages to consider. + * @vma: The vma which maps this object. + * + * If any page in this range is mapped by this VMA, return the first address + * where any of these pages appear. Otherwise, return -EFAULT. */ static inline unsigned long vma_pgoff_address(pgoff_t pgoff, unsigned long nr_pages, @@ -829,18 +834,6 @@ vma_pgoff_address(pgoff_t pgoff, unsigned long nr_pages, return address; } -/* - * Return the start of user virtual address of a page within a vma. - * Returns -EFAULT if all of the page is outside the range of vma. - * If page is a compound head, the entire compound page is considered. - */ -static inline unsigned long -vma_address(struct page *page, struct vm_area_struct *vma) -{ - VM_BUG_ON_PAGE(PageKsm(page), page); /* KSM page->index unusable */ - return vma_pgoff_address(page_to_pgoff(page), compound_nr(page), vma); -} - /* * Then at what user virtual address will none of the range be found in vma? * Assumes that vma_address() already returned a good starting address. diff --git a/mm/rmap.c b/mm/rmap.c index 5ee9e338d09b5..4b08b1a066888 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -775,6 +775,8 @@ static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) { struct folio *folio = page_folio(page); + pgoff_t pgoff; + if (folio_test_anon(folio)) { struct anon_vma *page__anon_vma = folio_anon_vma(folio); /* @@ -790,7 +792,9 @@ unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) return -EFAULT; } - return vma_address(page, vma); + /* The !page__anon_vma above handles KSM folios */ + pgoff = folio->index + folio_page_idx(folio, page); + return vma_pgoff_address(pgoff, 1, vma); } /* @@ -2588,7 +2592,8 @@ static void rmap_walk_anon(struct folio *folio, anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff_start, pgoff_end) { struct vm_area_struct *vma = avc->vma; - unsigned long address = vma_address(&folio->page, vma); + unsigned long address = vma_pgoff_address(pgoff_start, + folio_nr_pages(folio), vma); VM_BUG_ON_VMA(address == -EFAULT, vma); cond_resched(); @@ -2649,7 +2654,8 @@ static void rmap_walk_file(struct folio *folio, lookup: vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff_start, pgoff_end) { - unsigned long address = vma_address(&folio->page, vma); + unsigned long address = vma_pgoff_address(pgoff_start, + folio_nr_pages(folio), vma); VM_BUG_ON_VMA(address == -EFAULT, vma); cond_resched(); From e0abfbb67142448d57d7841b749d35981a0b92c7 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 Mar 2024 22:58:29 +0000 Subject: [PATCH 0723/1702] mm: rename vma_pgoff_address back to vma_address With all callers converted, we can use the nice shorter name. Take this opportunity to reorder the arguments to the logical order (larger object first). Link: https://lkml.kernel.org/r/20240328225831.1765286-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/internal.h | 9 ++++----- mm/memory-failure.c | 2 +- mm/page_vma_mapped.c | 2 +- mm/rmap.c | 12 ++++++------ 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index f4ef48d57b1c4..d567381b12cc8 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -805,17 +805,16 @@ void mlock_drain_remote(int cpu); extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); /** - * vma_pgoff_address - Find the virtual address a page range is mapped at + * vma_address - Find the virtual address a page range is mapped at + * @vma: The vma which maps this object. * @pgoff: The page offset within its object. * @nr_pages: The number of pages to consider. - * @vma: The vma which maps this object. * * If any page in this range is mapped by this VMA, return the first address * where any of these pages appear. Otherwise, return -EFAULT. */ -static inline unsigned long -vma_pgoff_address(pgoff_t pgoff, unsigned long nr_pages, - struct vm_area_struct *vma) +static inline unsigned long vma_address(struct vm_area_struct *vma, + pgoff_t pgoff, unsigned long nr_pages) { unsigned long address; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 9e50586f2e37e..0d863e9216aff 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -455,7 +455,7 @@ static void __add_to_kill(struct task_struct *tsk, struct page *p, tk->addr = ksm_addr ? ksm_addr : page_address_in_vma(p, vma); if (is_zone_device_page(p)) { if (fsdax_pgoff != FSDAX_INVALID_PGOFF) - tk->addr = vma_pgoff_address(fsdax_pgoff, 1, vma); + tk->addr = vma_address(vma, fsdax_pgoff, 1); tk->size_shift = dev_pagemap_mapping_shift(vma, tk->addr); } else tk->size_shift = page_shift(compound_head(p)); diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index ac48d6284badc..53b8868ede61c 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -334,7 +334,7 @@ int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) .flags = PVMW_SYNC, }; - pvmw.address = vma_pgoff_address(pgoff, 1, vma); + pvmw.address = vma_address(vma, pgoff, 1); if (pvmw.address == -EFAULT) return 0; if (!page_vma_mapped_walk(&pvmw)) diff --git a/mm/rmap.c b/mm/rmap.c index 4b08b1a066888..56b313aa2ebfb 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -794,7 +794,7 @@ unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) /* The !page__anon_vma above handles KSM folios */ pgoff = folio->index + folio_page_idx(folio, page); - return vma_pgoff_address(pgoff, 1, vma); + return vma_address(vma, pgoff, 1); } /* @@ -1132,7 +1132,7 @@ int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff, if (invalid_mkclean_vma(vma, NULL)) return 0; - pvmw.address = vma_pgoff_address(pgoff, nr_pages, vma); + pvmw.address = vma_address(vma, pgoff, nr_pages); VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma); return page_vma_mkclean_one(&pvmw); @@ -2592,8 +2592,8 @@ static void rmap_walk_anon(struct folio *folio, anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff_start, pgoff_end) { struct vm_area_struct *vma = avc->vma; - unsigned long address = vma_pgoff_address(pgoff_start, - folio_nr_pages(folio), vma); + unsigned long address = vma_address(vma, pgoff_start, + folio_nr_pages(folio)); VM_BUG_ON_VMA(address == -EFAULT, vma); cond_resched(); @@ -2654,8 +2654,8 @@ static void rmap_walk_file(struct folio *folio, lookup: vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff_start, pgoff_end) { - unsigned long address = vma_pgoff_address(pgoff_start, - folio_nr_pages(folio), vma); + unsigned long address = vma_address(vma, pgoff_start, + folio_nr_pages(folio)); VM_BUG_ON_VMA(address == -EFAULT, vma); cond_resched(); From 7998df0b6407da65a0fe8325d6ff239c4e14ff7d Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Thu, 28 Mar 2024 16:57:48 +0100 Subject: [PATCH 0724/1702] memory: remove the now superfluous sentinel element from ctl_table array This commit comes at the tail end of a greater effort to remove the empty elements at the end of the ctl_table arrays (sentinels) which will reduce the overall build time size of the kernel and run time memory bloat by ~64 bytes per sentinel (further information Link : https://lore.kernel.org/all/ZO5Yx5JFogGi%2FcBo@bombadil.infradead.org/) Remove sentinel from all files under mm/ that register a sysctl table. Link: https://lkml.kernel.org/r/20240328-jag-sysctl_remset_misc-v1-1-47c1463b3af2@samsung.com Signed-off-by: Joel Granados Reviewed-by: Muchun Song Reviewed-by: Miaohe Lin Signed-off-by: Andrew Morton --- mm/compaction.c | 1 - mm/hugetlb.c | 1 - mm/hugetlb_vmemmap.c | 1 - mm/memory-failure.c | 1 - mm/oom_kill.c | 1 - mm/page-writeback.c | 1 - mm/page_alloc.c | 1 - 7 files changed, 7 deletions(-) diff --git a/mm/compaction.c b/mm/compaction.c index 70b01190d2f3f..e731d45befc78 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -3350,7 +3350,6 @@ static struct ctl_table vm_compaction[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, - { } }; static int __init kcompactd_init(void) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 8af2fd48f785d..10b4f8bde16fd 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5029,7 +5029,6 @@ static struct ctl_table hugetlb_table[] = { .mode = 0644, .proc_handler = hugetlb_overcommit_handler, }, - { } }; static void hugetlb_sysctl_init(void) diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index da177e49d9564..b9a55322e52ce 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -679,7 +679,6 @@ static struct ctl_table hugetlb_vmemmap_sysctls[] = { .mode = 0644, .proc_handler = proc_dobool, }, - { } }; static int __init hugetlb_vmemmap_init(void) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 0d863e9216aff..4f22330600ffb 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -141,7 +141,6 @@ static struct ctl_table memory_failure_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, - { } }; /* diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 8d6a207c3c590..4d7a0004df2ca 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -724,7 +724,6 @@ static struct ctl_table vm_oom_kill_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, - {} }; #endif diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 3e19b87049db1..fba324e1a010d 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -2291,7 +2291,6 @@ static struct ctl_table vm_page_writeback_sysctls[] = { .mode = 0644, .proc_handler = proc_dointvec_jiffies, }, - {} }; #endif diff --git a/mm/page_alloc.c b/mm/page_alloc.c index e030ccf9d5bcd..07c21774a7221 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6288,7 +6288,6 @@ static struct ctl_table page_alloc_sysctl_table[] = { .extra2 = SYSCTL_ONE_HUNDRED, }, #endif - {} }; void __init page_alloc_sysctl_init(void) From c7876a0cc6a0a74b407b10f9afe5d77920d803ba Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Sat, 30 Mar 2024 23:05:55 +0530 Subject: [PATCH 0725/1702] selftests/mm: mremap_test: optimize using pre-filled random array and memcpy Patch series "selftests/mm: mremap_test: Optimizations and style fixes". The mremap_test, in a worst case controlled by the -t flag, does a for loop iteration in orders of GB. Without compromising on the stdout report, the aim is to reduce this time. A pre-filled random buffer is allocated based on the seed, replacing repetitive rand() calls. The byte pattern in the memory locations is set through memcpy() from the random buffer. Replacing the loop for printing the mismatch index to stdout, employ an efficient algorithm by breaking the comparison into chunks, use the highly optimized memcmp() library function, and when a mismatch does occur, only then do a brute force iteration. Also, use sscanf() to parse /proc/self/maps for consistency across files. Execution time results (x86 system): ./mremap_test Original: 3 seconds After change: 0.8 seconds ./mremap_test -t100 Original: 17 seconds After change: 2 seconds ./mremap_test -t0 (worst case): Original: 9:40 minutes After change: 45 seconds This patch (of 3): Allocate a pre-filled random buffer using the seed. Replace iterative copying of the random sequence to buffers using the highly optimized library function memcpy(). Link: https://lkml.kernel.org/r/20240330173557.2697684-1-dev.jain@arm.com Link: https://lkml.kernel.org/r/20240330173557.2697684-2-dev.jain@arm.com Signed-off-by: Dev Jain Cc: Anshuman Khandual Cc: John Hubbard Cc: Kalesh Singh Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/mremap_test.c | 78 ++++++++++++++++-------- 1 file changed, 53 insertions(+), 25 deletions(-) diff --git a/tools/testing/selftests/mm/mremap_test.c b/tools/testing/selftests/mm/mremap_test.c index 2f8b991f78cb4..7fed9cc3911eb 100644 --- a/tools/testing/selftests/mm/mremap_test.c +++ b/tools/testing/selftests/mm/mremap_test.c @@ -23,6 +23,7 @@ #define VALIDATION_NO_THRESHOLD 0 /* Verify the entire region */ #define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) +#define MAX(X, Y) ((X) > (Y) ? (X) : (Y)) #define SIZE_MB(m) ((size_t)m * (1024 * 1024)) #define SIZE_KB(k) ((size_t)k * 1024) @@ -296,7 +297,7 @@ static void mremap_expand_merge_offset(FILE *maps_fp, unsigned long page_size) * * |DDDDddddSSSSssss| */ -static void mremap_move_within_range(char pattern_seed) +static void mremap_move_within_range(unsigned int pattern_seed, char *rand_addr) { char *test_name = "mremap mremap move within range"; void *src, *dest; @@ -316,10 +317,7 @@ static void mremap_move_within_range(char pattern_seed) src = (void *)((unsigned long)src & ~(SIZE_MB(2) - 1)); /* Set byte pattern for source block. */ - srand(pattern_seed); - for (i = 0; i < SIZE_MB(2); i++) { - ((char *)src)[i] = (char) rand(); - } + memcpy(src, rand_addr, SIZE_MB(2)); dest = src - SIZE_MB(2); @@ -357,7 +355,7 @@ static void mremap_move_within_range(char pattern_seed) /* Returns the time taken for the remap on success else returns -1. */ static long long remap_region(struct config c, unsigned int threshold_mb, - char pattern_seed) + unsigned int pattern_seed, char *rand_addr) { void *addr, *src_addr, *dest_addr, *dest_preamble_addr; int d; @@ -378,9 +376,7 @@ static long long remap_region(struct config c, unsigned int threshold_mb, } /* Set byte pattern for source block. */ - srand(pattern_seed); - for (t = 0; t < threshold; t++) - memset((char *) src_addr + t, (char) rand(), 1); + memcpy(src_addr, rand_addr, threshold); /* Mask to zero out lower bits of address for alignment */ align_mask = ~(c.dest_alignment - 1); @@ -420,9 +416,7 @@ static long long remap_region(struct config c, unsigned int threshold_mb, } /* Set byte pattern for the dest preamble block. */ - srand(pattern_seed); - for (d = 0; d < c.dest_preamble_size; d++) - memset((char *) dest_preamble_addr + d, (char) rand(), 1); + memcpy(dest_preamble_addr, rand_addr, c.dest_preamble_size); } clock_gettime(CLOCK_MONOTONIC, &t_start); @@ -494,7 +488,8 @@ static long long remap_region(struct config c, unsigned int threshold_mb, * the beginning of the mapping just because the aligned * down address landed on a mapping that maybe does not exist. */ -static void mremap_move_1mb_from_start(char pattern_seed) +static void mremap_move_1mb_from_start(unsigned int pattern_seed, + char *rand_addr) { char *test_name = "mremap move 1mb from start at 1MB+256KB aligned src"; void *src = NULL, *dest = NULL; @@ -520,10 +515,7 @@ static void mremap_move_1mb_from_start(char pattern_seed) } /* Set byte pattern for source block. */ - srand(pattern_seed); - for (i = 0; i < SIZE_MB(2); i++) { - ((char *)src)[i] = (char) rand(); - } + memcpy(src, rand_addr, SIZE_MB(2)); /* * Unmap the beginning of dest so that the aligned address @@ -568,10 +560,10 @@ static void mremap_move_1mb_from_start(char pattern_seed) static void run_mremap_test_case(struct test test_case, int *failures, unsigned int threshold_mb, - unsigned int pattern_seed) + unsigned int pattern_seed, char *rand_addr) { long long remap_time = remap_region(test_case.config, threshold_mb, - pattern_seed); + pattern_seed, rand_addr); if (remap_time < 0) { if (test_case.expect_failure) @@ -642,7 +634,15 @@ int main(int argc, char **argv) int failures = 0; int i, run_perf_tests; unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD; + + /* hard-coded test configs */ + size_t max_test_variable_region_size = _2GB; + size_t max_test_constant_region_size = _2MB; + size_t dest_preamble_size = 10 * _4MB; + unsigned int pattern_seed; + char *rand_addr; + size_t rand_size; int num_expand_tests = 2; int num_misc_tests = 2; struct test test_cases[MAX_TEST] = {}; @@ -659,6 +659,31 @@ int main(int argc, char **argv) ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n", threshold_mb, pattern_seed); + /* + * set preallocated random array according to test configs; see the + * functions for the logic of setting the size + */ + if (!threshold_mb) + rand_size = MAX(max_test_variable_region_size, + max_test_constant_region_size); + else + rand_size = MAX(MIN(threshold_mb * _1MB, + max_test_variable_region_size), + max_test_constant_region_size); + rand_size = MAX(dest_preamble_size, rand_size); + + rand_addr = (char *)mmap(NULL, rand_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (rand_addr == MAP_FAILED) { + perror("mmap"); + ksft_exit_fail_msg("cannot mmap rand_addr\n"); + } + + /* fill stream of random bytes */ + srand(pattern_seed); + for (unsigned long i = 0; i < rand_size; ++i) + rand_addr[i] = (char) rand(); + page_size = sysconf(_SC_PAGESIZE); /* Expected mremap failures */ @@ -730,13 +755,13 @@ int main(int argc, char **argv) for (i = 0; i < ARRAY_SIZE(test_cases); i++) run_mremap_test_case(test_cases[i], &failures, threshold_mb, - pattern_seed); + pattern_seed, rand_addr); maps_fp = fopen("/proc/self/maps", "r"); if (maps_fp == NULL) { - ksft_print_msg("Failed to read /proc/self/maps: %s\n", strerror(errno)); - exit(KSFT_FAIL); + munmap(rand_addr, rand_size); + ksft_exit_fail_msg("Failed to read /proc/self/maps: %s\n", strerror(errno)); } mremap_expand_merge(maps_fp, page_size); @@ -744,17 +769,20 @@ int main(int argc, char **argv) fclose(maps_fp); - mremap_move_within_range(pattern_seed); - mremap_move_1mb_from_start(pattern_seed); + mremap_move_within_range(pattern_seed, rand_addr); + mremap_move_1mb_from_start(pattern_seed, rand_addr); if (run_perf_tests) { ksft_print_msg("\n%s\n", "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:"); for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++) run_mremap_test_case(perf_test_cases[i], &failures, - threshold_mb, pattern_seed); + threshold_mb, pattern_seed, + rand_addr); } + munmap(rand_addr, rand_size); + if (failures > 0) ksft_exit_fail(); else From 7033c6cc96203eec738ff192829a18a4e12308dc Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Sat, 30 Mar 2024 23:05:56 +0530 Subject: [PATCH 0726/1702] selftests/mm: mremap_test: optimize execution time from minutes to seconds using chunkwise memcmp Mismatch index is currently being checked by a brute force iteration over the buffer. Instead, break the comparison into O(sqrt(n)) number of chunks, with the chunk size of this order only, where n is the size of the buffer. Do a brute-force iteration to print to stdout only when the highly optimized memcmp() library function returns a mismatch in the chunk. The time complexity of this algorithm is O(sqrt(n)) * t, where t is the time taken by memcmp(); for our test conditions, it is safe to assume t to be small. Link: https://lkml.kernel.org/r/20240330173557.2697684-3-dev.jain@arm.com Signed-off-by: Dev Jain Cc: Anshuman Khandual Cc: John Hubbard Cc: Kalesh Singh Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/mremap_test.c | 112 ++++++++++++++++++----- 1 file changed, 91 insertions(+), 21 deletions(-) diff --git a/tools/testing/selftests/mm/mremap_test.c b/tools/testing/selftests/mm/mremap_test.c index 7fed9cc3911eb..678c79d5b8efb 100644 --- a/tools/testing/selftests/mm/mremap_test.c +++ b/tools/testing/selftests/mm/mremap_test.c @@ -70,6 +70,27 @@ enum { .expect_failure = should_fail \ } +/* compute square root using binary search */ +static unsigned long get_sqrt(unsigned long val) +{ + unsigned long low = 1; + + /* assuming rand_size is less than 1TB */ + unsigned long high = (1UL << 20); + + while (low <= high) { + unsigned long mid = low + (high - low) / 2; + unsigned long temp = mid * mid; + + if (temp == val) + return mid; + if (temp < val) + low = mid + 1; + high = mid - 1; + } + return low; +} + /* * Returns false if the requested remap region overlaps with an * existing mapping (e.g text, stack) else returns true. @@ -355,14 +376,14 @@ static void mremap_move_within_range(unsigned int pattern_seed, char *rand_addr) /* Returns the time taken for the remap on success else returns -1. */ static long long remap_region(struct config c, unsigned int threshold_mb, - unsigned int pattern_seed, char *rand_addr) + char *rand_addr) { void *addr, *src_addr, *dest_addr, *dest_preamble_addr; - int d; - unsigned long long t; + unsigned long long t, d; struct timespec t_start = {0, 0}, t_end = {0, 0}; long long start_ns, end_ns, align_mask, ret, offset; unsigned long long threshold; + unsigned long num_chunks; if (threshold_mb == VALIDATION_NO_THRESHOLD) threshold = c.region_size; @@ -430,15 +451,42 @@ static long long remap_region(struct config c, unsigned int threshold_mb, goto clean_up_dest_preamble; } - /* Verify byte pattern after remapping */ - srand(pattern_seed); - for (t = 0; t < threshold; t++) { - char c = (char) rand(); + /* + * Verify byte pattern after remapping. Employ an algorithm with a + * square root time complexity in threshold: divide the range into + * chunks, if memcmp() returns non-zero, only then perform an + * iteration in that chunk to find the mismatch index. + */ + num_chunks = get_sqrt(threshold); + for (unsigned long i = 0; i < num_chunks; ++i) { + size_t chunk_size = threshold / num_chunks; + unsigned long shift = i * chunk_size; + + if (!memcmp(dest_addr + shift, rand_addr + shift, chunk_size)) + continue; + + /* brute force iteration only over mismatch segment */ + for (t = shift; t < shift + chunk_size; ++t) { + if (((char *) dest_addr)[t] != rand_addr[t]) { + ksft_print_msg("Data after remap doesn't match at offset %llu\n", + t); + ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[t] & 0xff, + ((char *) dest_addr)[t] & 0xff); + ret = -1; + goto clean_up_dest; + } + } + } - if (((char *) dest_addr)[t] != c) { + /* + * if threshold is not divisible by num_chunks, then check the + * last chunk + */ + for (t = num_chunks * (threshold / num_chunks); t < threshold; ++t) { + if (((char *) dest_addr)[t] != rand_addr[t]) { ksft_print_msg("Data after remap doesn't match at offset %llu\n", - t); - ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, + t); + ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[t] & 0xff, ((char *) dest_addr)[t] & 0xff); ret = -1; goto clean_up_dest; @@ -446,22 +494,44 @@ static long long remap_region(struct config c, unsigned int threshold_mb, } /* Verify the dest preamble byte pattern after remapping */ - if (c.dest_preamble_size) { - srand(pattern_seed); - for (d = 0; d < c.dest_preamble_size; d++) { - char c = (char) rand(); - - if (((char *) dest_preamble_addr)[d] != c) { - ksft_print_msg("Preamble data after remap doesn't match at offset %d\n", - d); - ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, - ((char *) dest_preamble_addr)[d] & 0xff); + if (!c.dest_preamble_size) + goto no_preamble; + + num_chunks = get_sqrt(c.dest_preamble_size); + + for (unsigned long i = 0; i < num_chunks; ++i) { + size_t chunk_size = c.dest_preamble_size / num_chunks; + unsigned long shift = i * chunk_size; + + if (!memcmp(dest_preamble_addr + shift, rand_addr + shift, + chunk_size)) + continue; + + /* brute force iteration only over mismatched segment */ + for (d = shift; d < shift + chunk_size; ++d) { + if (((char *) dest_preamble_addr)[d] != rand_addr[d]) { + ksft_print_msg("Preamble data after remap doesn't match at offset %llu\n", + d); + ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[d] & 0xff, + ((char *) dest_preamble_addr)[d] & 0xff); ret = -1; goto clean_up_dest; } } } + for (d = num_chunks * (c.dest_preamble_size / num_chunks); d < c.dest_preamble_size; ++d) { + if (((char *) dest_preamble_addr)[d] != rand_addr[d]) { + ksft_print_msg("Preamble data after remap doesn't match at offset %llu\n", + d); + ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[d] & 0xff, + ((char *) dest_preamble_addr)[d] & 0xff); + ret = -1; + goto clean_up_dest; + } + } + +no_preamble: start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec; end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec; ret = end_ns - start_ns; @@ -563,7 +633,7 @@ static void run_mremap_test_case(struct test test_case, int *failures, unsigned int pattern_seed, char *rand_addr) { long long remap_time = remap_region(test_case.config, threshold_mb, - pattern_seed, rand_addr); + rand_addr); if (remap_time < 0) { if (test_case.expect_failure) From e1e13262f0d6e58a6439d437c386bb3392aaca0b Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Sat, 30 Mar 2024 23:05:57 +0530 Subject: [PATCH 0727/1702] selftests/mm: mremap_test: use sscanf to parse /proc/self/maps Enforce consistency across files by avoiding two separate functions to parse /proc/self/maps, replacing them with a simple sscanf(). Link: https://lkml.kernel.org/r/20240330173557.2697684-4-dev.jain@arm.com Signed-off-by: Dev Jain Cc: Anshuman Khandual Cc: John Hubbard Cc: Kalesh Singh Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/mremap_test.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/mm/mremap_test.c b/tools/testing/selftests/mm/mremap_test.c index 678c79d5b8efb..1b03bcfaefdfa 100644 --- a/tools/testing/selftests/mm/mremap_test.c +++ b/tools/testing/selftests/mm/mremap_test.c @@ -148,19 +148,21 @@ static unsigned long long get_mmap_min_addr(void) * Using /proc/self/maps, assert that the specified address range is contained * within a single mapping. */ -static bool is_range_mapped(FILE *maps_fp, void *start, void *end) +static bool is_range_mapped(FILE *maps_fp, unsigned long start, + unsigned long end) { char *line = NULL; size_t len = 0; bool success = false; + unsigned long first_val, second_val; rewind(maps_fp); while (getline(&line, &len, maps_fp) != -1) { - char *first = strtok(line, "- "); - void *first_val = (void *)strtol(first, NULL, 16); - char *second = strtok(NULL, "- "); - void *second_val = (void *) strtol(second, NULL, 16); + if (sscanf(line, "%lx-%lx", &first_val, &second_val) != 2) { + ksft_exit_fail_msg("cannot parse /proc/self/maps\n"); + break; + } if (first_val <= start && second_val >= end) { success = true; @@ -255,7 +257,8 @@ static void mremap_expand_merge(FILE *maps_fp, unsigned long page_size) goto out; } - success = is_range_mapped(maps_fp, start, start + 3 * page_size); + success = is_range_mapped(maps_fp, (unsigned long)start, + (unsigned long)(start + 3 * page_size)); munmap(start, 3 * page_size); out: @@ -294,7 +297,8 @@ static void mremap_expand_merge_offset(FILE *maps_fp, unsigned long page_size) goto out; } - success = is_range_mapped(maps_fp, start, start + 3 * page_size); + success = is_range_mapped(maps_fp, (unsigned long)start, + (unsigned long)(start + 3 * page_size)); munmap(start, 3 * page_size); out: From 4746f5ce0fa52e21b5fe432970fe9516d1a45ebc Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 3 Apr 2024 18:18:30 +0100 Subject: [PATCH 0728/1702] khugepaged: inline hpage_collapse_alloc_folio() Patch series "khugepaged folio conversions". We've been kind of hacking piecemeal at converting khugepaged to use folios instead of compound pages, and so this patchset is a little larger than it should be as I undo some of our wrong moves in the past. In particular, collapse_file() now consistently uses 'new_folio' for the freshly allocated folio and 'folio' for the one that's currently in use. This patch (of 7): This function has one caller, and the combined function is simpler to read, reason about and modify. Link: https://lkml.kernel.org/r/20240403171838.1445826-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240403171838.1445826-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Vishal Moola (Oracle) Signed-off-by: Andrew Morton --- mm/khugepaged.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 38830174608fb..ad16dd8b26a8a 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -891,20 +891,6 @@ static int hpage_collapse_find_target_node(struct collapse_control *cc) } #endif -static bool hpage_collapse_alloc_folio(struct folio **folio, gfp_t gfp, int node, - nodemask_t *nmask) -{ - *folio = __folio_alloc(gfp, HPAGE_PMD_ORDER, node, nmask); - - if (unlikely(!*folio)) { - count_vm_event(THP_COLLAPSE_ALLOC_FAILED); - return false; - } - - count_vm_event(THP_COLLAPSE_ALLOC); - return true; -} - /* * If mmap_lock temporarily dropped, revalidate vma * before taking mmap_lock. @@ -1067,11 +1053,14 @@ static int alloc_charge_hpage(struct page **hpage, struct mm_struct *mm, int node = hpage_collapse_find_target_node(cc); struct folio *folio; - if (!hpage_collapse_alloc_folio(&folio, gfp, node, &cc->alloc_nmask)) { + folio = __folio_alloc(gfp, HPAGE_PMD_ORDER, node, &cc->alloc_nmask); + if (!folio) { *hpage = NULL; + count_vm_event(THP_COLLAPSE_ALLOC_FAILED); return SCAN_ALLOC_HUGE_PAGE_FAIL; } + count_vm_event(THP_COLLAPSE_ALLOC); if (unlikely(mem_cgroup_charge(folio, mm, gfp))) { folio_put(folio); *hpage = NULL; From d5ab50b9412c0bba750eef5a34fd2937de1aee55 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 3 Apr 2024 18:18:31 +0100 Subject: [PATCH 0729/1702] khugepaged: convert alloc_charge_hpage to alloc_charge_folio Both callers want to deal with a folio, so return a folio from this function. Link: https://lkml.kernel.org/r/20240403171838.1445826-3-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/khugepaged.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index ad16dd8b26a8a..2f1dacd65d12b 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1045,7 +1045,7 @@ static int __collapse_huge_page_swapin(struct mm_struct *mm, return result; } -static int alloc_charge_hpage(struct page **hpage, struct mm_struct *mm, +static int alloc_charge_folio(struct folio **foliop, struct mm_struct *mm, struct collapse_control *cc) { gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() : @@ -1055,7 +1055,7 @@ static int alloc_charge_hpage(struct page **hpage, struct mm_struct *mm, folio = __folio_alloc(gfp, HPAGE_PMD_ORDER, node, &cc->alloc_nmask); if (!folio) { - *hpage = NULL; + *foliop = NULL; count_vm_event(THP_COLLAPSE_ALLOC_FAILED); return SCAN_ALLOC_HUGE_PAGE_FAIL; } @@ -1063,13 +1063,13 @@ static int alloc_charge_hpage(struct page **hpage, struct mm_struct *mm, count_vm_event(THP_COLLAPSE_ALLOC); if (unlikely(mem_cgroup_charge(folio, mm, gfp))) { folio_put(folio); - *hpage = NULL; + *foliop = NULL; return SCAN_CGROUP_CHARGE_FAIL; } count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1); - *hpage = folio_page(folio, 0); + *foliop = folio; return SCAN_SUCCEED; } @@ -1098,7 +1098,8 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address, */ mmap_read_unlock(mm); - result = alloc_charge_hpage(&hpage, mm, cc); + result = alloc_charge_folio(&folio, mm, cc); + hpage = &folio->page; if (result != SCAN_SUCCEED) goto out_nolock; @@ -1204,7 +1205,6 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address, if (unlikely(result != SCAN_SUCCEED)) goto out_up_write; - folio = page_folio(hpage); /* * The smp_wmb() inside __folio_mark_uptodate() ensures the * copy_huge_page writes become visible before the set_pmd_at() @@ -1789,7 +1789,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, struct page *hpage; struct page *page; struct page *tmp; - struct folio *folio; + struct folio *folio, *new_folio; pgoff_t index = 0, end = start + HPAGE_PMD_NR; LIST_HEAD(pagelist); XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); @@ -1800,7 +1800,8 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); - result = alloc_charge_hpage(&hpage, mm, cc); + result = alloc_charge_folio(&new_folio, mm, cc); + hpage = &new_folio->page; if (result != SCAN_SUCCEED) goto out; From 0234779276e56fb17677f3cf64d7cd501f8abe69 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 3 Apr 2024 18:18:32 +0100 Subject: [PATCH 0730/1702] khugepaged: remove hpage from collapse_huge_page() Work purely in terms of the folio. Removes a call to compound_head() in put_page(). Link: https://lkml.kernel.org/r/20240403171838.1445826-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Vishal Moola (Oracle) Signed-off-by: Andrew Morton --- mm/khugepaged.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 2f1dacd65d12b..1c99d18602e5c 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1082,7 +1082,6 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address, pte_t *pte; pgtable_t pgtable; struct folio *folio; - struct page *hpage; spinlock_t *pmd_ptl, *pte_ptl; int result = SCAN_FAIL; struct vm_area_struct *vma; @@ -1099,7 +1098,6 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address, mmap_read_unlock(mm); result = alloc_charge_folio(&folio, mm, cc); - hpage = &folio->page; if (result != SCAN_SUCCEED) goto out_nolock; @@ -1198,7 +1196,7 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address, */ anon_vma_unlock_write(vma->anon_vma); - result = __collapse_huge_page_copy(pte, hpage, pmd, _pmd, + result = __collapse_huge_page_copy(pte, &folio->page, pmd, _pmd, vma, address, pte_ptl, &compound_pagelist); pte_unmap(pte); @@ -1213,7 +1211,7 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address, __folio_mark_uptodate(folio); pgtable = pmd_pgtable(_pmd); - _pmd = mk_huge_pmd(hpage, vma->vm_page_prot); + _pmd = mk_huge_pmd(&folio->page, vma->vm_page_prot); _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma); spin_lock(pmd_ptl); @@ -1225,14 +1223,14 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address, update_mmu_cache_pmd(vma, address, pmd); spin_unlock(pmd_ptl); - hpage = NULL; + folio = NULL; result = SCAN_SUCCEED; out_up_write: mmap_write_unlock(mm); out_nolock: - if (hpage) - put_page(hpage); + if (folio) + folio_put(folio); trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result); return result; } From 8eca68e2cfdf863e98dc3c2cc8b2be9cac46b9d6 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 3 Apr 2024 18:18:33 +0100 Subject: [PATCH 0731/1702] khugepaged: pass a folio to __collapse_huge_page_copy() Simplify the body of __collapse_huge_page_copy() while I'm looking at it. Link: https://lkml.kernel.org/r/20240403171838.1445826-5-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Vishal Moola (Oracle) Signed-off-by: Andrew Morton --- mm/khugepaged.c | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 1c99d18602e5c..71a4119ce3a81 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -767,7 +767,7 @@ static void __collapse_huge_page_copy_failed(pte_t *pte, * Returns SCAN_SUCCEED if copying succeeds, otherwise returns SCAN_COPY_MC. * * @pte: starting of the PTEs to copy from - * @page: the new hugepage to copy contents to + * @folio: the new hugepage to copy contents to * @pmd: pointer to the new hugepage's PMD * @orig_pmd: the original raw pages' PMD * @vma: the original raw pages' virtual memory area @@ -775,33 +775,29 @@ static void __collapse_huge_page_copy_failed(pte_t *pte, * @ptl: lock on raw pages' PTEs * @compound_pagelist: list that stores compound pages */ -static int __collapse_huge_page_copy(pte_t *pte, - struct page *page, - pmd_t *pmd, - pmd_t orig_pmd, - struct vm_area_struct *vma, - unsigned long address, - spinlock_t *ptl, - struct list_head *compound_pagelist) +static int __collapse_huge_page_copy(pte_t *pte, struct folio *folio, + pmd_t *pmd, pmd_t orig_pmd, struct vm_area_struct *vma, + unsigned long address, spinlock_t *ptl, + struct list_head *compound_pagelist) { - struct page *src_page; - pte_t *_pte; - pte_t pteval; - unsigned long _address; + unsigned int i; int result = SCAN_SUCCEED; /* * Copying pages' contents is subject to memory poison at any iteration. */ - for (_pte = pte, _address = address; _pte < pte + HPAGE_PMD_NR; - _pte++, page++, _address += PAGE_SIZE) { - pteval = ptep_get(_pte); + for (i = 0; i < HPAGE_PMD_NR; i++) { + pte_t pteval = ptep_get(pte + i); + struct page *page = folio_page(folio, i); + unsigned long src_addr = address + i * PAGE_SIZE; + struct page *src_page; + if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { - clear_user_highpage(page, _address); + clear_user_highpage(page, src_addr); continue; } src_page = pte_page(pteval); - if (copy_mc_user_highpage(page, src_page, _address, vma) > 0) { + if (copy_mc_user_highpage(page, src_page, src_addr, vma) > 0) { result = SCAN_COPY_MC; break; } @@ -1196,7 +1192,7 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address, */ anon_vma_unlock_write(vma->anon_vma); - result = __collapse_huge_page_copy(pte, &folio->page, pmd, _pmd, + result = __collapse_huge_page_copy(pte, folio, pmd, _pmd, vma, address, pte_ptl, &compound_pagelist); pte_unmap(pte); From 610ff817b981921213ae51e5c5f38c76c6f0405e Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 3 Apr 2024 18:18:34 +0100 Subject: [PATCH 0732/1702] khugepaged: remove hpage from collapse_file() Use new_folio throughout where we had been using hpage. Link: https://lkml.kernel.org/r/20240403171838.1445826-6-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Vishal Moola (Oracle) Signed-off-by: Andrew Morton --- include/trace/events/huge_memory.h | 6 +-- mm/khugepaged.c | 77 +++++++++++++++--------------- 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h index 6e2ef1d4b0028..dc6eeef2d3dac 100644 --- a/include/trace/events/huge_memory.h +++ b/include/trace/events/huge_memory.h @@ -207,10 +207,10 @@ TRACE_EVENT(mm_khugepaged_scan_file, ); TRACE_EVENT(mm_khugepaged_collapse_file, - TP_PROTO(struct mm_struct *mm, struct page *hpage, pgoff_t index, + TP_PROTO(struct mm_struct *mm, struct folio *new_folio, pgoff_t index, bool is_shmem, unsigned long addr, struct file *file, int nr, int result), - TP_ARGS(mm, hpage, index, addr, is_shmem, file, nr, result), + TP_ARGS(mm, new_folio, index, addr, is_shmem, file, nr, result), TP_STRUCT__entry( __field(struct mm_struct *, mm) __field(unsigned long, hpfn) @@ -224,7 +224,7 @@ TRACE_EVENT(mm_khugepaged_collapse_file, TP_fast_assign( __entry->mm = mm; - __entry->hpfn = hpage ? page_to_pfn(hpage) : -1; + __entry->hpfn = new_folio ? folio_pfn(new_folio) : -1; __entry->index = index; __entry->addr = addr; __entry->is_shmem = is_shmem; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 71a4119ce3a81..d44584b5e0041 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1780,30 +1780,27 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, struct collapse_control *cc) { struct address_space *mapping = file->f_mapping; - struct page *hpage; struct page *page; - struct page *tmp; + struct page *tmp, *dst; struct folio *folio, *new_folio; pgoff_t index = 0, end = start + HPAGE_PMD_NR; LIST_HEAD(pagelist); XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); int nr_none = 0, result = SCAN_SUCCEED; bool is_shmem = shmem_file(file); - int nr = 0; VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); result = alloc_charge_folio(&new_folio, mm, cc); - hpage = &new_folio->page; if (result != SCAN_SUCCEED) goto out; - __SetPageLocked(hpage); + __folio_set_locked(new_folio); if (is_shmem) - __SetPageSwapBacked(hpage); - hpage->index = start; - hpage->mapping = mapping; + __folio_set_swapbacked(new_folio); + new_folio->index = start; + new_folio->mapping = mapping; /* * Ensure we have slots for all the pages in the range. This is @@ -2036,20 +2033,24 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, * The old pages are locked, so they won't change anymore. */ index = start; + dst = folio_page(new_folio, 0); list_for_each_entry(page, &pagelist, lru) { while (index < page->index) { - clear_highpage(hpage + (index % HPAGE_PMD_NR)); + clear_highpage(dst); index++; + dst++; } - if (copy_mc_highpage(hpage + (page->index % HPAGE_PMD_NR), page) > 0) { + if (copy_mc_highpage(dst, page) > 0) { result = SCAN_COPY_MC; goto rollback; } index++; + dst++; } while (index < end) { - clear_highpage(hpage + (index % HPAGE_PMD_NR)); + clear_highpage(dst); index++; + dst++; } if (nr_none) { @@ -2077,16 +2078,17 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, } /* - * If userspace observed a missing page in a VMA with a MODE_MISSING - * userfaultfd, then it might expect a UFFD_EVENT_PAGEFAULT for that - * page. If so, we need to roll back to avoid suppressing such an - * event. Since wp/minor userfaultfds don't give userspace any - * guarantees that the kernel doesn't fill a missing page with a zero - * page, so they don't matter here. + * If userspace observed a missing page in a VMA with + * a MODE_MISSING userfaultfd, then it might expect a + * UFFD_EVENT_PAGEFAULT for that page. If so, we need to + * roll back to avoid suppressing such an event. Since + * wp/minor userfaultfds don't give userspace any + * guarantees that the kernel doesn't fill a missing + * page with a zero page, so they don't matter here. * - * Any userfaultfds registered after this point will not be able to - * observe any missing pages due to the previously inserted retry - * entries. + * Any userfaultfds registered after this point will + * not be able to observe any missing pages due to the + * previously inserted retry entries. */ vma_interval_tree_foreach(vma, &mapping->i_mmap, start, end) { if (userfaultfd_missing(vma)) { @@ -2111,33 +2113,32 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, xas_lock_irq(&xas); } - folio = page_folio(hpage); - nr = folio_nr_pages(folio); if (is_shmem) - __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, nr); + __lruvec_stat_mod_folio(new_folio, NR_SHMEM_THPS, HPAGE_PMD_NR); else - __lruvec_stat_mod_folio(folio, NR_FILE_THPS, nr); + __lruvec_stat_mod_folio(new_folio, NR_FILE_THPS, HPAGE_PMD_NR); if (nr_none) { - __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr_none); + __lruvec_stat_mod_folio(new_folio, NR_FILE_PAGES, nr_none); /* nr_none is always 0 for non-shmem. */ - __lruvec_stat_mod_folio(folio, NR_SHMEM, nr_none); + __lruvec_stat_mod_folio(new_folio, NR_SHMEM, nr_none); } /* - * Mark hpage as uptodate before inserting it into the page cache so - * that it isn't mistaken for an fallocated but unwritten page. + * Mark new_folio as uptodate before inserting it into the + * page cache so that it isn't mistaken for an fallocated but + * unwritten page. */ - folio_mark_uptodate(folio); - folio_ref_add(folio, HPAGE_PMD_NR - 1); + folio_mark_uptodate(new_folio); + folio_ref_add(new_folio, HPAGE_PMD_NR - 1); if (is_shmem) - folio_mark_dirty(folio); - folio_add_lru(folio); + folio_mark_dirty(new_folio); + folio_add_lru(new_folio); /* Join all the small entries into a single multi-index entry. */ xas_set_order(&xas, start, HPAGE_PMD_ORDER); - xas_store(&xas, folio); + xas_store(&xas, new_folio); WARN_ON_ONCE(xas_error(&xas)); xas_unlock_irq(&xas); @@ -2148,7 +2149,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, retract_page_tables(mapping, start); if (cc && !cc->is_khugepaged) result = SCAN_PTE_MAPPED_HUGEPAGE; - folio_unlock(folio); + folio_unlock(new_folio); /* * The collapse has succeeded, so free the old pages. @@ -2193,13 +2194,13 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, smp_mb(); } - hpage->mapping = NULL; + new_folio->mapping = NULL; - unlock_page(hpage); - put_page(hpage); + folio_unlock(new_folio); + folio_put(new_folio); out: VM_BUG_ON(!list_empty(&pagelist)); - trace_mm_khugepaged_collapse_file(mm, hpage, index, is_shmem, addr, file, nr, result); + trace_mm_khugepaged_collapse_file(mm, new_folio, index, is_shmem, addr, file, HPAGE_PMD_NR, result); return result; } From 8d1e24c0b82d9730d05ee85eb7f4195df8cdf6a6 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 3 Apr 2024 18:18:35 +0100 Subject: [PATCH 0733/1702] khugepaged: use a folio throughout collapse_file() Pull folios from the page cache instead of pages. Half of this work had been done already, but we were still operating on pages for a large chunk of this function. There is no attempt in this patch to handle large folios that are smaller than a THP; that will have to wait for a future patch. [willy@infradead.org: the unlikely() is embedded in IS_ERR()] Link: https://lkml.kernel.org/r/ZhIWX8K0E2tSyMSr@casper.infradead.org Link: https://lkml.kernel.org/r/20240403171838.1445826-7-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/khugepaged.c | 113 +++++++++++++++++++++++------------------------- 1 file changed, 54 insertions(+), 59 deletions(-) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index d44584b5e0041..f3116eba4dfd9 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1780,9 +1780,8 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, struct collapse_control *cc) { struct address_space *mapping = file->f_mapping; - struct page *page; - struct page *tmp, *dst; - struct folio *folio, *new_folio; + struct page *dst; + struct folio *folio, *tmp, *new_folio; pgoff_t index = 0, end = start + HPAGE_PMD_NR; LIST_HEAD(pagelist); XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); @@ -1820,11 +1819,11 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, for (index = start; index < end; index++) { xas_set(&xas, index); - page = xas_load(&xas); + folio = xas_load(&xas); VM_BUG_ON(index != xas.xa_index); if (is_shmem) { - if (!page) { + if (!folio) { /* * Stop if extent has been truncated or * hole-punched, and is now completely @@ -1840,7 +1839,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, continue; } - if (xa_is_value(page) || !PageUptodate(page)) { + if (xa_is_value(folio) || !folio_test_uptodate(folio)) { xas_unlock_irq(&xas); /* swap in or instantiate fallocated page */ if (shmem_get_folio(mapping->host, index, @@ -1850,28 +1849,27 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, } /* drain lru cache to help isolate_lru_page() */ lru_add_drain(); - page = folio_file_page(folio, index); - } else if (trylock_page(page)) { - get_page(page); + } else if (folio_trylock(folio)) { + folio_get(folio); xas_unlock_irq(&xas); } else { result = SCAN_PAGE_LOCK; goto xa_locked; } } else { /* !is_shmem */ - if (!page || xa_is_value(page)) { + if (!folio || xa_is_value(folio)) { xas_unlock_irq(&xas); page_cache_sync_readahead(mapping, &file->f_ra, file, index, end - index); /* drain lru cache to help isolate_lru_page() */ lru_add_drain(); - page = find_lock_page(mapping, index); - if (unlikely(page == NULL)) { + folio = filemap_lock_folio(mapping, index); + if (IS_ERR(folio)) { result = SCAN_FAIL; goto xa_unlocked; } - } else if (PageDirty(page)) { + } else if (folio_test_dirty(folio)) { /* * khugepaged only works on read-only fd, * so this page is dirty because it hasn't @@ -1889,12 +1887,12 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, filemap_flush(mapping); result = SCAN_FAIL; goto xa_unlocked; - } else if (PageWriteback(page)) { + } else if (folio_test_writeback(folio)) { xas_unlock_irq(&xas); result = SCAN_FAIL; goto xa_unlocked; - } else if (trylock_page(page)) { - get_page(page); + } else if (folio_trylock(folio)) { + folio_get(folio); xas_unlock_irq(&xas); } else { result = SCAN_PAGE_LOCK; @@ -1903,35 +1901,31 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, } /* - * The page must be locked, so we can drop the i_pages lock + * The folio must be locked, so we can drop the i_pages lock * without racing with truncate. */ - VM_BUG_ON_PAGE(!PageLocked(page), page); + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); - /* make sure the page is up to date */ - if (unlikely(!PageUptodate(page))) { + /* make sure the folio is up to date */ + if (unlikely(!folio_test_uptodate(folio))) { result = SCAN_FAIL; goto out_unlock; } /* * If file was truncated then extended, or hole-punched, before - * we locked the first page, then a THP might be there already. + * we locked the first folio, then a THP might be there already. * This will be discovered on the first iteration. */ - if (PageTransCompound(page)) { - struct page *head = compound_head(page); - - result = compound_order(head) == HPAGE_PMD_ORDER && - head->index == start + if (folio_test_large(folio)) { + result = folio_order(folio) == HPAGE_PMD_ORDER && + folio->index == start /* Maybe PMD-mapped */ ? SCAN_PTE_MAPPED_HUGEPAGE : SCAN_PAGE_COMPOUND; goto out_unlock; } - folio = page_folio(page); - if (folio_mapping(folio) != mapping) { result = SCAN_TRUNCATED; goto out_unlock; @@ -1941,7 +1935,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, folio_test_writeback(folio))) { /* * khugepaged only works on read-only fd, so this - * page is dirty because it hasn't been flushed + * folio is dirty because it hasn't been flushed * since first write. */ result = SCAN_FAIL; @@ -1965,33 +1959,34 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, xas_lock_irq(&xas); - VM_BUG_ON_PAGE(page != xa_load(xas.xa, index), page); + VM_BUG_ON_FOLIO(folio != xa_load(xas.xa, index), folio); /* - * We control three references to the page: + * We control three references to the folio: * - we hold a pin on it; * - one reference from page cache; - * - one from isolate_lru_page; - * If those are the only references, then any new usage of the - * page will have to fetch it from the page cache. That requires - * locking the page to handle truncate, so any new usage will be - * blocked until we unlock page after collapse/during rollback. + * - one from lru_isolate_folio; + * If those are the only references, then any new usage + * of the folio will have to fetch it from the page + * cache. That requires locking the folio to handle + * truncate, so any new usage will be blocked until we + * unlock folio after collapse/during rollback. */ - if (page_count(page) != 3) { + if (folio_ref_count(folio) != 3) { result = SCAN_PAGE_COUNT; xas_unlock_irq(&xas); - putback_lru_page(page); + folio_putback_lru(folio); goto out_unlock; } /* - * Accumulate the pages that are being collapsed. + * Accumulate the folios that are being collapsed. */ - list_add_tail(&page->lru, &pagelist); + list_add_tail(&folio->lru, &pagelist); continue; out_unlock: - unlock_page(page); - put_page(page); + folio_unlock(folio); + folio_put(folio); goto xa_unlocked; } @@ -2030,17 +2025,17 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, } /* - * The old pages are locked, so they won't change anymore. + * The old folios are locked, so they won't change anymore. */ index = start; dst = folio_page(new_folio, 0); - list_for_each_entry(page, &pagelist, lru) { - while (index < page->index) { + list_for_each_entry(folio, &pagelist, lru) { + while (index < folio->index) { clear_highpage(dst); index++; dst++; } - if (copy_mc_highpage(dst, page) > 0) { + if (copy_mc_highpage(dst, folio_page(folio, 0)) > 0) { result = SCAN_COPY_MC; goto rollback; } @@ -2152,15 +2147,15 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, folio_unlock(new_folio); /* - * The collapse has succeeded, so free the old pages. + * The collapse has succeeded, so free the old folios. */ - list_for_each_entry_safe(page, tmp, &pagelist, lru) { - list_del(&page->lru); - page->mapping = NULL; - ClearPageActive(page); - ClearPageUnevictable(page); - unlock_page(page); - folio_put_refs(page_folio(page), 3); + list_for_each_entry_safe(folio, tmp, &pagelist, lru) { + list_del(&folio->lru); + folio->mapping = NULL; + folio_clear_active(folio); + folio_clear_unevictable(folio); + folio_unlock(folio); + folio_put_refs(folio, 3); } goto out; @@ -2174,11 +2169,11 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, shmem_uncharge(mapping->host, nr_none); } - list_for_each_entry_safe(page, tmp, &pagelist, lru) { - list_del(&page->lru); - unlock_page(page); - putback_lru_page(page); - put_page(page); + list_for_each_entry_safe(folio, tmp, &pagelist, lru) { + list_del(&folio->lru); + folio_unlock(folio); + folio_putback_lru(folio); + folio_put(folio); } /* * Undo the updates of filemap_nr_thps_inc for non-SHMEM From 43849758fdc976a6d6108ed6dfccdb136fdeec39 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 3 Apr 2024 18:18:36 +0100 Subject: [PATCH 0734/1702] khugepaged: use a folio throughout hpage_collapse_scan_file() Replace the use of pages with folios. Saves a few calls to compound_head() and removes some uses of obsolete functions. Link: https://lkml.kernel.org/r/20240403171838.1445826-8-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Reviewed-by: Vishal Moola (Oracle) Signed-off-by: Andrew Morton --- include/trace/events/huge_memory.h | 6 +++--- mm/khugepaged.c | 33 +++++++++++++++--------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h index dc6eeef2d3dac..ab576898a1264 100644 --- a/include/trace/events/huge_memory.h +++ b/include/trace/events/huge_memory.h @@ -174,10 +174,10 @@ TRACE_EVENT(mm_collapse_huge_page_swapin, TRACE_EVENT(mm_khugepaged_scan_file, - TP_PROTO(struct mm_struct *mm, struct page *page, struct file *file, + TP_PROTO(struct mm_struct *mm, struct folio *folio, struct file *file, int present, int swap, int result), - TP_ARGS(mm, page, file, present, swap, result), + TP_ARGS(mm, folio, file, present, swap, result), TP_STRUCT__entry( __field(struct mm_struct *, mm) @@ -190,7 +190,7 @@ TRACE_EVENT(mm_khugepaged_scan_file, TP_fast_assign( __entry->mm = mm; - __entry->pfn = page ? page_to_pfn(page) : -1; + __entry->pfn = folio ? folio_pfn(folio) : -1; __assign_str(filename, file->f_path.dentry->d_iname); __entry->present = present; __entry->swap = swap; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index f3116eba4dfd9..8380ee0d0410c 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -2203,7 +2203,7 @@ static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr, struct file *file, pgoff_t start, struct collapse_control *cc) { - struct page *page = NULL; + struct folio *folio = NULL; struct address_space *mapping = file->f_mapping; XA_STATE(xas, &mapping->i_pages, start); int present, swap; @@ -2215,11 +2215,11 @@ static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr, memset(cc->node_load, 0, sizeof(cc->node_load)); nodes_clear(cc->alloc_nmask); rcu_read_lock(); - xas_for_each(&xas, page, start + HPAGE_PMD_NR - 1) { - if (xas_retry(&xas, page)) + xas_for_each(&xas, folio, start + HPAGE_PMD_NR - 1) { + if (xas_retry(&xas, folio)) continue; - if (xa_is_value(page)) { + if (xa_is_value(folio)) { ++swap; if (cc->is_khugepaged && swap > khugepaged_max_ptes_swap) { @@ -2234,11 +2234,9 @@ static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr, * TODO: khugepaged should compact smaller compound pages * into a PMD sized page */ - if (PageTransCompound(page)) { - struct page *head = compound_head(page); - - result = compound_order(head) == HPAGE_PMD_ORDER && - head->index == start + if (folio_test_large(folio)) { + result = folio_order(folio) == HPAGE_PMD_ORDER && + folio->index == start /* Maybe PMD-mapped */ ? SCAN_PTE_MAPPED_HUGEPAGE : SCAN_PAGE_COMPOUND; @@ -2251,28 +2249,29 @@ static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr, break; } - node = page_to_nid(page); + node = folio_nid(folio); if (hpage_collapse_scan_abort(node, cc)) { result = SCAN_SCAN_ABORT; break; } cc->node_load[node]++; - if (!PageLRU(page)) { + if (!folio_test_lru(folio)) { result = SCAN_PAGE_LRU; break; } - if (page_count(page) != - 1 + page_mapcount(page) + page_has_private(page)) { + if (folio_ref_count(folio) != + 1 + folio_mapcount(folio) + folio_test_private(folio)) { result = SCAN_PAGE_COUNT; break; } /* - * We probably should check if the page is referenced here, but - * nobody would transfer pte_young() to PageReferenced() for us. - * And rmap walk here is just too costly... + * We probably should check if the folio is referenced + * here, but nobody would transfer pte_young() to + * folio_test_referenced() for us. And rmap walk here + * is just too costly... */ present++; @@ -2294,7 +2293,7 @@ static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr, } } - trace_mm_khugepaged_scan_file(mm, page, file, present, swap, result); + trace_mm_khugepaged_scan_file(mm, folio, file, present, swap, result); return result; } #else From 03aa577f3b2809bcaeb9a544307c2d997a972965 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 2 Apr 2024 21:12:48 +0100 Subject: [PATCH 0735/1702] proc: convert clear_refs_pte_range to use a folio Patch series "Remove page_idle and page_young wrappers". There are only a couple of places left using the page wrappers for idle & young tracking. Convert the two users in proc and then we can remove the wrappers. That enables the further simplification of autogenerating the definitions when CONFIG_PAGE_IDLE_FLAG is disabled. This patch (of 4): Replaces four calls to compound_head() with two. Link: https://lkml.kernel.org/r/20240402201252.917342-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240402201252.917342-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- fs/proc/task_mmu.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 23fbab954c20b..b94101cd27068 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1161,7 +1161,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, struct vm_area_struct *vma = walk->vma; pte_t *pte, ptent; spinlock_t *ptl; - struct page *page; + struct folio *folio; ptl = pmd_trans_huge_lock(pmd, vma); if (ptl) { @@ -1173,12 +1173,12 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, if (!pmd_present(*pmd)) goto out; - page = pmd_page(*pmd); + folio = pmd_folio(*pmd); /* Clear accessed and referenced bits. */ pmdp_test_and_clear_young(vma, addr, pmd); - test_and_clear_page_young(page); - ClearPageReferenced(page); + folio_test_clear_young(folio); + folio_clear_referenced(folio); out: spin_unlock(ptl); return 0; @@ -1200,14 +1200,14 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, if (!pte_present(ptent)) continue; - page = vm_normal_page(vma, addr, ptent); - if (!page) + folio = vm_normal_folio(vma, addr, ptent); + if (!folio) continue; /* Clear accessed and referenced bits. */ ptep_test_and_clear_young(vma, addr, pte); - test_and_clear_page_young(page); - ClearPageReferenced(page); + folio_test_clear_young(folio); + folio_clear_referenced(folio); } pte_unmap_unlock(pte - 1, ptl); cond_resched(); From 6c977f36dc36baec8ad15faccbc6a08315eaa195 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 2 Apr 2024 21:12:49 +0100 Subject: [PATCH 0736/1702] proc: convert smaps_account() to use a folio Replace seven calls to compound_head() with one. Link: https://lkml.kernel.org/r/20240402201252.917342-3-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- fs/proc/task_mmu.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index b94101cd27068..e8d1008a838d7 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -444,6 +444,7 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page, bool compound, bool young, bool dirty, bool locked, bool migration) { + struct folio *folio = page_folio(page); int i, nr = compound ? compound_nr(page) : 1; unsigned long size = nr * PAGE_SIZE; @@ -451,27 +452,28 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page, * First accumulate quantities that depend only on |size| and the type * of the compound page. */ - if (PageAnon(page)) { + if (folio_test_anon(folio)) { mss->anonymous += size; - if (!PageSwapBacked(page) && !dirty && !PageDirty(page)) + if (!folio_test_swapbacked(folio) && !dirty && + !folio_test_dirty(folio)) mss->lazyfree += size; } - if (PageKsm(page)) + if (folio_test_ksm(folio)) mss->ksm += size; mss->resident += size; /* Accumulate the size in pages that have been accessed. */ - if (young || page_is_young(page) || PageReferenced(page)) + if (young || folio_test_young(folio) || folio_test_referenced(folio)) mss->referenced += size; /* * Then accumulate quantities that may depend on sharing, or that may * differ page-by-page. * - * page_count(page) == 1 guarantees the page is mapped exactly once. + * refcount == 1 guarantees the page is mapped exactly once. * If any subpage of the compound page mapped with PTE it would elevate - * page_count(). + * the refcount. * * The page_mapcount() is called to get a snapshot of the mapcount. * Without holding the page lock this snapshot can be slightly wrong as @@ -480,7 +482,7 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page, * especially for migration entries. Treat regular migration entries * as mapcount == 1. */ - if ((page_count(page) == 1) || migration) { + if ((folio_ref_count(folio) == 1) || migration) { smaps_page_accumulate(mss, page, size, size << PSS_SHIFT, dirty, locked, true); return; From 1ade67cd22e4aeb74ed0b86bab6760bbcf9e2f06 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 2 Apr 2024 21:12:50 +0100 Subject: [PATCH 0737/1702] mm: remove page_idle and page_young wrappers All users have now been converted to the folio equivalents, so remove the page wrappers. Link: https://lkml.kernel.org/r/20240402201252.917342-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/page_idle.h | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/include/linux/page_idle.h b/include/linux/page_idle.h index 511e22ef459fc..6357f1e7918a2 100644 --- a/include/linux/page_idle.h +++ b/include/linux/page_idle.h @@ -119,29 +119,4 @@ static inline void folio_clear_idle(struct folio *folio) } #endif /* CONFIG_PAGE_IDLE_FLAG */ - -static inline bool page_is_young(struct page *page) -{ - return folio_test_young(page_folio(page)); -} - -static inline void set_page_young(struct page *page) -{ - folio_set_young(page_folio(page)); -} - -static inline bool test_and_clear_page_young(struct page *page) -{ - return folio_test_clear_young(page_folio(page)); -} - -static inline bool page_is_idle(struct page *page) -{ - return folio_test_idle(page_folio(page)); -} - -static inline void set_page_idle(struct page *page) -{ - folio_set_idle(page_folio(page)); -} #endif /* _LINUX_MM_PAGE_IDLE_H */ From 8c9e8381ea4bcca4305cf0d24f87f4d39514ab35 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 2 Apr 2024 21:12:51 +0100 Subject: [PATCH 0738/1702] mm: generate PAGE_IDLE_FLAG definitions If CONFIG_PAGE_IDLE_FLAG is not set, we can use FOLIO_FLAG_FALSE() to generate these definitions. Link: https://lkml.kernel.org/r/20240402201252.917342-5-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 9 ++++++++- include/linux/page_idle.h | 37 ++----------------------------------- 2 files changed, 10 insertions(+), 36 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 7577fe7debafc..f83331684e470 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -626,12 +626,19 @@ PAGEFLAG_FALSE(HWPoison, hwpoison) #define __PG_HWPOISON 0 #endif -#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT) +#ifdef CONFIG_PAGE_IDLE_FLAG +#ifdef CONFIG_64BIT FOLIO_TEST_FLAG(young, FOLIO_HEAD_PAGE) FOLIO_SET_FLAG(young, FOLIO_HEAD_PAGE) FOLIO_TEST_CLEAR_FLAG(young, FOLIO_HEAD_PAGE) FOLIO_FLAG(idle, FOLIO_HEAD_PAGE) #endif +/* See page_idle.h for !64BIT workaround */ +#else /* !CONFIG_PAGE_IDLE_FLAG */ +FOLIO_FLAG_FALSE(young) +FOLIO_TEST_CLEAR_FLAG_FALSE(young) +FOLIO_FLAG_FALSE(idle) +#endif /* * PageReported() is used to track reported free pages within the Buddy diff --git a/include/linux/page_idle.h b/include/linux/page_idle.h index 6357f1e7918a2..89ca0d5dc1e71 100644 --- a/include/linux/page_idle.h +++ b/include/linux/page_idle.h @@ -6,9 +6,7 @@ #include #include -#ifdef CONFIG_PAGE_IDLE_FLAG - -#ifndef CONFIG_64BIT +#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT) /* * If there is not enough space to store Idle and Young bits in page flags, use * page ext flags instead. @@ -87,36 +85,5 @@ static inline void folio_clear_idle(struct folio *folio) clear_bit(PAGE_EXT_IDLE, &page_ext->flags); page_ext_put(page_ext); } -#endif /* !CONFIG_64BIT */ - -#else /* !CONFIG_PAGE_IDLE_FLAG */ - -static inline bool folio_test_young(const struct folio *folio) -{ - return false; -} - -static inline void folio_set_young(struct folio *folio) -{ -} - -static inline bool folio_test_clear_young(struct folio *folio) -{ - return false; -} - -static inline bool folio_test_idle(const struct folio *folio) -{ - return false; -} - -static inline void folio_set_idle(struct folio *folio) -{ -} - -static inline void folio_clear_idle(struct folio *folio) -{ -} - -#endif /* CONFIG_PAGE_IDLE_FLAG */ +#endif /* CONFIG_PAGE_IDLE_FLAG && !64BIT */ #endif /* _LINUX_MM_PAGE_IDLE_H */ From f1dc623fa0d37eeb02b31264dd2ce2011fec0c9b Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 3 Apr 2024 18:14:52 +0100 Subject: [PATCH 0739/1702] proc: convert gather_stats to use a folio Patch series "Use folio APIs in procfs". We're down to very few users of the PageFoo macros, with proc being a major user. After this patchset and another patchset I have for khugepaged, we can get rid of PageActive, PageReadahead and PageSwapBacked. This patchset has the usual advantages in its own right of removing hidden calls to compound_head(). We have the page table lock, so the mapcount & refcount are stable and there can't be any races with folios suddenly becoming tail pages. This patch (of 4): Replaces six calls to compound_head() with one. Shrinks the function from 5054 bytes to 1756 bytes in an allmodconfig build. Link: https://lkml.kernel.org/r/20240403171456.1445117-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240403171456.1445117-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Cc: Christian Brauner Signed-off-by: Andrew Morton --- fs/proc/task_mmu.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index e8d1008a838d7..5260a2788f748 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -2549,28 +2549,29 @@ struct numa_maps_private { static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty, unsigned long nr_pages) { + struct folio *folio = page_folio(page); int count = page_mapcount(page); md->pages += nr_pages; - if (pte_dirty || PageDirty(page)) + if (pte_dirty || folio_test_dirty(folio)) md->dirty += nr_pages; - if (PageSwapCache(page)) + if (folio_test_swapcache(folio)) md->swapcache += nr_pages; - if (PageActive(page) || PageUnevictable(page)) + if (folio_test_active(folio) || folio_test_unevictable(folio)) md->active += nr_pages; - if (PageWriteback(page)) + if (folio_test_writeback(folio)) md->writeback += nr_pages; - if (PageAnon(page)) + if (folio_test_anon(folio)) md->anon += nr_pages; if (count > md->mapcount_max) md->mapcount_max = count; - md->node[page_to_nid(page)] += nr_pages; + md->node[folio_nid(folio)] += nr_pages; } static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma, From cfc96da432fe5c6b514bdbf60ee036cad39a65af Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 3 Apr 2024 18:14:53 +0100 Subject: [PATCH 0740/1702] proc: convert smaps_page_accumulate to use a folio Replaces three calls to compound_head() with one. Shrinks the function from 2614 bytes to 1112 bytes in an allmodconfig build. Link: https://lkml.kernel.org/r/20240403171456.1445117-3-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Cc: Christian Brauner Signed-off-by: Andrew Morton --- fs/proc/task_mmu.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 5260a2788f748..2a3133dd47b1d 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -414,11 +414,12 @@ static void smaps_page_accumulate(struct mem_size_stats *mss, struct page *page, unsigned long size, unsigned long pss, bool dirty, bool locked, bool private) { + struct folio *folio = page_folio(page); mss->pss += pss; - if (PageAnon(page)) + if (folio_test_anon(folio)) mss->pss_anon += pss; - else if (PageSwapBacked(page)) + else if (folio_test_swapbacked(folio)) mss->pss_shmem += pss; else mss->pss_file += pss; @@ -426,7 +427,7 @@ static void smaps_page_accumulate(struct mem_size_stats *mss, if (locked) mss->pss_locked += pss; - if (dirty || PageDirty(page)) { + if (dirty || folio_test_dirty(folio)) { mss->pss_dirty += pss; if (private) mss->private_dirty += size; From 27bb0a70e5248683fd940f738a7a53066fb470e5 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 3 Apr 2024 18:14:54 +0100 Subject: [PATCH 0741/1702] proc: pass a folio to smaps_page_accumulate() Both callers already have a folio; pass it in instead of doing the conversion each time. Link: https://lkml.kernel.org/r/20240403171456.1445117-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Cc: Christian Brauner Signed-off-by: Andrew Morton --- fs/proc/task_mmu.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 2a3133dd47b1d..6d4f60bc8824f 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -411,10 +411,9 @@ struct mem_size_stats { }; static void smaps_page_accumulate(struct mem_size_stats *mss, - struct page *page, unsigned long size, unsigned long pss, + struct folio *folio, unsigned long size, unsigned long pss, bool dirty, bool locked, bool private) { - struct folio *folio = page_folio(page); mss->pss += pss; if (folio_test_anon(folio)) @@ -484,8 +483,8 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page, * as mapcount == 1. */ if ((folio_ref_count(folio) == 1) || migration) { - smaps_page_accumulate(mss, page, size, size << PSS_SHIFT, dirty, - locked, true); + smaps_page_accumulate(mss, folio, size, size << PSS_SHIFT, + dirty, locked, true); return; } for (i = 0; i < nr; i++, page++) { @@ -493,8 +492,8 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page, unsigned long pss = PAGE_SIZE << PSS_SHIFT; if (mapcount >= 2) pss /= mapcount; - smaps_page_accumulate(mss, page, PAGE_SIZE, pss, dirty, locked, - mapcount < 2); + smaps_page_accumulate(mss, folio, PAGE_SIZE, pss, + dirty, locked, mapcount < 2); } } From 039d26d10d62b71e242cde1092b042ae1fee498d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 3 Apr 2024 18:14:55 +0100 Subject: [PATCH 0742/1702] proc: convert smaps_pmd_entry to use a folio Replace two calls to compound_head() with one. Link: https://lkml.kernel.org/r/20240403171456.1445117-5-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Cc: Christian Brauner Signed-off-by: Andrew Morton --- fs/proc/task_mmu.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 6d4f60bc8824f..8ff79bd427ec6 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -578,6 +578,7 @@ static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, struct vm_area_struct *vma = walk->vma; bool locked = !!(vma->vm_flags & VM_LOCKED); struct page *page = NULL; + struct folio *folio; bool migration = false; if (pmd_present(*pmd)) { @@ -592,11 +593,12 @@ static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, } if (IS_ERR_OR_NULL(page)) return; - if (PageAnon(page)) + folio = page_folio(page); + if (folio_test_anon(folio)) mss->anonymous_thp += HPAGE_PMD_SIZE; - else if (PageSwapBacked(page)) + else if (folio_test_swapbacked(folio)) mss->shmem_thp += HPAGE_PMD_SIZE; - else if (is_zone_device_page(page)) + else if (folio_is_zone_device(folio)) /* pass */; else mss->file_thp += HPAGE_PMD_SIZE; From 6303d1c553c8d758f068de70a41668622b7a917c Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Wed, 3 Apr 2024 21:47:21 +0800 Subject: [PATCH 0743/1702] mm: page_alloc: use the correct THP order for THP PCP Commit 44042b449872 ("mm/page_alloc: allow high-order pages to be stored on the per-cpu lists") extends the PCP allocator to store THP pages, and it determines whether to cache THP pages in PCP by comparing with pageblock_order. But the pageblock_order is not always equal to THP order. It might also be MAX_PAGE_ORDER, which could prevent PCP from caching THP pages. Therefore, using HPAGE_PMD_ORDER instead to determine the need for caching THP for PCP will fix this issue Link: https://lkml.kernel.org/r/a25c9e14cd03907d5978b60546a69e6aa3fc2a7d.1712151833.git.baolin.wang@linux.alibaba.com Fixes: 44042b449872 ("mm/page_alloc: allow high-order pages to be stored on the per-cpu lists") Signed-off-by: Baolin Wang Acked-by: Vlastimil Babka Cc: Mel Gorman Reviewed-by: Barry Song Signed-off-by: Andrew Morton --- mm/page_alloc.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 07c21774a7221..cbe3f695eda9b 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -506,7 +506,7 @@ static inline unsigned int order_to_pindex(int migratetype, int order) { #ifdef CONFIG_TRANSPARENT_HUGEPAGE if (order > PAGE_ALLOC_COSTLY_ORDER) { - VM_BUG_ON(order != pageblock_order); + VM_BUG_ON(order != HPAGE_PMD_ORDER); return NR_LOWORDER_PCP_LISTS; } #else @@ -522,7 +522,7 @@ static inline int pindex_to_order(unsigned int pindex) #ifdef CONFIG_TRANSPARENT_HUGEPAGE if (pindex == NR_LOWORDER_PCP_LISTS) - order = pageblock_order; + order = HPAGE_PMD_ORDER; #else VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); #endif @@ -535,7 +535,7 @@ static inline bool pcp_allowed_order(unsigned int order) if (order <= PAGE_ALLOC_COSTLY_ORDER) return true; #ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (order == pageblock_order) + if (order == HPAGE_PMD_ORDER) return true; #endif return false; From d7d0d389ff90644546ffcb8e15ea3ccaf6138958 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 8 Apr 2024 19:39:40 +0100 Subject: [PATCH 0744/1702] mm: swap: remove CLUSTER_FLAG_HUGE from swap_cluster_info:flags Patch series "Swap-out mTHP without splitting", v7. This series adds support for swapping out multi-size THP (mTHP) without needing to first split the large folio via split_huge_page_to_list_to_order(). It closely follows the approach already used to swap-out PMD-sized THP. There are a couple of reasons for swapping out mTHP without splitting: - Performance: It is expensive to split a large folio and under extreme memory pressure some workloads regressed performance when using 64K mTHP vs 4K small folios because of this extra cost in the swap-out path. This series not only eliminates the regression but makes it faster to swap out 64K mTHP vs 4K small folios. - Memory fragmentation avoidance: If we can avoid splitting a large folio memory is less likely to become fragmented, making it easier to re-allocate a large folio in future. - Performance: Enables a separate series [7] to swap-in whole mTHPs, which means we won't lose the TLB-efficiency benefits of mTHP once the memory has been through a swap cycle. I've done what I thought was the smallest change possible, and as a result, this approach is only employed when the swap is backed by a non-rotating block device (just as PMD-sized THP is supported today). Discussion against the RFC concluded that this is sufficient. Performance Testing =================== I've run some swap performance tests on Ampere Altra VM (arm64) with 8 CPUs. The VM is set up with a 35G block ram device as the swap device and the test is run from inside a memcg limited to 40G memory. I've then run `usemem` from vm-scalability with 70 processes, each allocating and writing 1G of memory. I've repeated everything 6 times and taken the mean performance improvement relative to 4K page baseline: | alloc size | baseline | + this series | | | mm-unstable (~v6.9-rc1) | | |:-----------|------------------------:|------------------------:| | 4K Page | 0.0% | 1.3% | | 64K THP | -13.6% | 46.3% | | 2M THP | 91.4% | 89.6% | So with this change, the 64K swap performance goes from a 14% regression to a 46% improvement. While 2M shows a small regression I'm confident that this is just noise. [1] https://lore.kernel.org/linux-mm/20231010142111.3997780-1-ryan.roberts@arm.com/ [2] https://lore.kernel.org/linux-mm/20231017161302.2518826-1-ryan.roberts@arm.com/ [3] https://lore.kernel.org/linux-mm/20231025144546.577640-1-ryan.roberts@arm.com/ [4] https://lore.kernel.org/linux-mm/20240311150058.1122862-1-ryan.roberts@arm.com/ [5] https://lore.kernel.org/linux-mm/20240327144537.4165578-1-ryan.roberts@arm.com/ [6] https://lore.kernel.org/linux-mm/20240403114032.1162100-1-ryan.roberts@arm.com/ [7] https://lore.kernel.org/linux-mm/20240304081348.197341-1-21cnbao@gmail.com/ [8] https://lore.kernel.org/linux-mm/CAGsJ_4yMOow27WDvN2q=E4HAtDd2PJ=OQ5Pj9DG+6FLWwNuXUw@mail.gmail.com/ [9] https://lore.kernel.org/linux-mm/579d5127-c763-4001-9625-4563a9316ac3@redhat.com/ This patch (of 7): As preparation for supporting small-sized THP in the swap-out path, without first needing to split to order-0, Remove the CLUSTER_FLAG_HUGE, which, when present, always implies PMD-sized THP, which is the same as the cluster size. The only use of the flag was to determine whether a swap entry refers to a single page or a PMD-sized THP in swap_page_trans_huge_swapped(). Instead of relying on the flag, we now pass in order, which originates from the folio's order. This allows the logic to work for folios of any order. The one snag is that one of the swap_page_trans_huge_swapped() call sites does not have the folio. But it was only being called there to shortcut a call __try_to_reclaim_swap() in some cases. __try_to_reclaim_swap() gets the folio and (via some other functions) calls swap_page_trans_huge_swapped(). So I've removed the problematic call site and believe the new logic should be functionally equivalent. That said, removing the fast path means that we will take a reference and trylock a large folio much more often, which we would like to avoid. The next patch will solve this. Removing CLUSTER_FLAG_HUGE also means we can remove split_swap_cluster() which used to be called during folio splitting, since split_swap_cluster()'s only job was to remove the flag. Link: https://lkml.kernel.org/r/20240408183946.2991168-1-ryan.roberts@arm.com Link: https://lkml.kernel.org/r/20240408183946.2991168-2-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Reviewed-by: "Huang, Ying" Acked-by: Chris Li Acked-by: David Hildenbrand Cc: Barry Song <21cnbao@gmail.com> Cc: Gao Xiang Cc: Kefeng Wang Cc: Lance Yang Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Yang Shi Cc: Yu Zhao Cc: Barry Song Signed-off-by: Andrew Morton --- include/linux/swap.h | 10 ---------- mm/huge_memory.c | 3 --- mm/swapfile.c | 47 ++++++++------------------------------------ 3 files changed, 8 insertions(+), 52 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index f53d608daa013..a803de0ac24f5 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -259,7 +259,6 @@ struct swap_cluster_info { }; #define CLUSTER_FLAG_FREE 1 /* This cluster is free */ #define CLUSTER_FLAG_NEXT_NULL 2 /* This cluster has no next cluster */ -#define CLUSTER_FLAG_HUGE 4 /* This cluster is backing a transparent huge page */ /* * We assign a cluster to each CPU, so each CPU can allocate swap entry from @@ -587,15 +586,6 @@ static inline int add_swap_extent(struct swap_info_struct *sis, } #endif /* CONFIG_SWAP */ -#ifdef CONFIG_THP_SWAP -extern int split_swap_cluster(swp_entry_t entry); -#else -static inline int split_swap_cluster(swp_entry_t entry) -{ - return 0; -} -#endif - #ifdef CONFIG_MEMCG static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) { diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 4065bf8bfcc43..14f04fbc22d9a 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2844,9 +2844,6 @@ static void __split_huge_page(struct page *page, struct list_head *list, shmem_uncharge(folio->mapping->host, nr_dropped); remap_page(folio, nr); - if (folio_test_swapcache(folio)) - split_swap_cluster(folio->swap); - /* * set page to its compound_head when split to non order-0 pages, so * we can skip unlocking it below, since PG_locked is transferred to diff --git a/mm/swapfile.c b/mm/swapfile.c index 5e6d2304a2a47..1ded6d1dcab42 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -343,18 +343,6 @@ static inline void cluster_set_null(struct swap_cluster_info *info) info->data = 0; } -static inline bool cluster_is_huge(struct swap_cluster_info *info) -{ - if (IS_ENABLED(CONFIG_THP_SWAP)) - return info->flags & CLUSTER_FLAG_HUGE; - return false; -} - -static inline void cluster_clear_huge(struct swap_cluster_info *info) -{ - info->flags &= ~CLUSTER_FLAG_HUGE; -} - static inline struct swap_cluster_info *lock_cluster(struct swap_info_struct *si, unsigned long offset) { @@ -1027,7 +1015,7 @@ static int swap_alloc_cluster(struct swap_info_struct *si, swp_entry_t *slot) offset = idx * SWAPFILE_CLUSTER; ci = lock_cluster(si, offset); alloc_cluster(si, idx); - cluster_set_count_flag(ci, SWAPFILE_CLUSTER, CLUSTER_FLAG_HUGE); + cluster_set_count(ci, SWAPFILE_CLUSTER); memset(si->swap_map + offset, SWAP_HAS_CACHE, SWAPFILE_CLUSTER); unlock_cluster(ci); @@ -1365,7 +1353,6 @@ void put_swap_folio(struct folio *folio, swp_entry_t entry) ci = lock_cluster_or_swap_info(si, offset); if (size == SWAPFILE_CLUSTER) { - VM_BUG_ON(!cluster_is_huge(ci)); map = si->swap_map + offset; for (i = 0; i < SWAPFILE_CLUSTER; i++) { val = map[i]; @@ -1373,7 +1360,6 @@ void put_swap_folio(struct folio *folio, swp_entry_t entry) if (val == SWAP_HAS_CACHE) free_entries++; } - cluster_clear_huge(ci); if (free_entries == SWAPFILE_CLUSTER) { unlock_cluster_or_swap_info(si, ci); spin_lock(&si->lock); @@ -1395,23 +1381,6 @@ void put_swap_folio(struct folio *folio, swp_entry_t entry) unlock_cluster_or_swap_info(si, ci); } -#ifdef CONFIG_THP_SWAP -int split_swap_cluster(swp_entry_t entry) -{ - struct swap_info_struct *si; - struct swap_cluster_info *ci; - unsigned long offset = swp_offset(entry); - - si = _swap_info_get(entry); - if (!si) - return -EBUSY; - ci = lock_cluster(si, offset); - cluster_clear_huge(ci); - unlock_cluster(ci); - return 0; -} -#endif - static int swp_entry_cmp(const void *ent1, const void *ent2) { const swp_entry_t *e1 = ent1, *e2 = ent2; @@ -1519,22 +1488,23 @@ int swp_swapcount(swp_entry_t entry) } static bool swap_page_trans_huge_swapped(struct swap_info_struct *si, - swp_entry_t entry) + swp_entry_t entry, int order) { struct swap_cluster_info *ci; unsigned char *map = si->swap_map; + unsigned int nr_pages = 1 << order; unsigned long roffset = swp_offset(entry); - unsigned long offset = round_down(roffset, SWAPFILE_CLUSTER); + unsigned long offset = round_down(roffset, nr_pages); int i; bool ret = false; ci = lock_cluster_or_swap_info(si, offset); - if (!ci || !cluster_is_huge(ci)) { + if (!ci || nr_pages == 1) { if (swap_count(map[roffset])) ret = true; goto unlock_out; } - for (i = 0; i < SWAPFILE_CLUSTER; i++) { + for (i = 0; i < nr_pages; i++) { if (swap_count(map[offset + i])) { ret = true; break; @@ -1556,7 +1526,7 @@ static bool folio_swapped(struct folio *folio) if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!folio_test_large(folio))) return swap_swapcount(si, entry) != 0; - return swap_page_trans_huge_swapped(si, entry); + return swap_page_trans_huge_swapped(si, entry, folio_order(folio)); } /** @@ -1622,8 +1592,7 @@ int free_swap_and_cache(swp_entry_t entry) } count = __swap_entry_free(p, entry); - if (count == SWAP_HAS_CACHE && - !swap_page_trans_huge_swapped(p, entry)) + if (count == SWAP_HAS_CACHE) __try_to_reclaim_swap(p, swp_offset(entry), TTRS_UNMAPPED | TTRS_FULL); put_swap_device(p); From a62fb92ac12ed39df4930dca599a3b427552882a Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 8 Apr 2024 19:39:41 +0100 Subject: [PATCH 0745/1702] mm: swap: free_swap_and_cache_nr() as batched free_swap_and_cache() Now that we no longer have a convenient flag in the cluster to determine if a folio is large, free_swap_and_cache() will take a reference and lock a large folio much more often, which could lead to contention and (e.g.) failure to split large folios, etc. Let's solve that problem by batch freeing swap and cache with a new function, free_swap_and_cache_nr(), to free a contiguous range of swap entries together. This allows us to first drop a reference to each swap slot before we try to release the cache folio. This means we only try to release the folio once, only taking the reference and lock once - much better than the previous 512 times for the 2M THP case. Contiguous swap entries are gathered in zap_pte_range() and madvise_free_pte_range() in a similar way to how present ptes are already gathered in zap_pte_range(). While we are at it, let's simplify by converting the return type of both functions to void. The return value was used only by zap_pte_range() to print a bad pte, and was ignored by everyone else, so the extra reporting wasn't exactly guaranteed. We will still get the warning with most of the information from get_swap_device(). With the batch version, we wouldn't know which pte was bad anyway so could print the wrong one. [ryan.roberts@arm.com: fix a build warning on parisc] Link: https://lkml.kernel.org/r/20240409111840.3173122-1-ryan.roberts@arm.com Link: https://lkml.kernel.org/r/20240408183946.2991168-3-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Acked-by: David Hildenbrand Cc: Barry Song <21cnbao@gmail.com> Cc: Barry Song Cc: Chris Li Cc: Gao Xiang Cc: "Huang, Ying" Cc: Kefeng Wang Cc: Lance Yang Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Yang Shi Cc: Yu Zhao Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 29 ++++++++++++ include/linux/swap.h | 12 +++-- mm/internal.h | 64 +++++++++++++++++++++++++++ mm/madvise.c | 12 +++-- mm/memory.c | 13 +++--- mm/swapfile.c | 97 +++++++++++++++++++++++++++++++++-------- 6 files changed, 196 insertions(+), 31 deletions(-) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index a3fc8150b047d..75096025fe528 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -708,6 +708,35 @@ static inline void pte_clear_not_present_full(struct mm_struct *mm, } #endif +#ifndef clear_not_present_full_ptes +/** + * clear_not_present_full_ptes - Clear multiple not present PTEs which are + * consecutive in the pgtable. + * @mm: Address space the ptes represent. + * @addr: Address of the first pte. + * @ptep: Page table pointer for the first entry. + * @nr: Number of entries to clear. + * @full: Whether we are clearing a full mm. + * + * May be overridden by the architecture; otherwise, implemented as a simple + * loop over pte_clear_not_present_full(). + * + * Context: The caller holds the page table lock. The PTEs are all not present. + * The PTEs are all in the same PMD. + */ +static inline void clear_not_present_full_ptes(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, unsigned int nr, int full) +{ + for (;;) { + pte_clear_not_present_full(mm, addr, ptep, full); + if (--nr == 0) + break; + ptep++; + addr += PAGE_SIZE; + } +} +#endif + #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH extern pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address, diff --git a/include/linux/swap.h b/include/linux/swap.h index a803de0ac24f5..2d8f2b950ddfa 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -468,7 +468,7 @@ extern int swap_duplicate(swp_entry_t); extern int swapcache_prepare(swp_entry_t); extern void swap_free(swp_entry_t); extern void swapcache_free_entries(swp_entry_t *entries, int n); -extern int free_swap_and_cache(swp_entry_t); +extern void free_swap_and_cache_nr(swp_entry_t entry, int nr); int swap_type_of(dev_t device, sector_t offset); int find_first_swap(dev_t *device); extern unsigned int count_swap_pages(int, int); @@ -517,8 +517,9 @@ static inline void put_swap_device(struct swap_info_struct *si) #define free_pages_and_swap_cache(pages, nr) \ release_pages((pages), (nr)); -/* used to sanity check ptes in zap_pte_range when CONFIG_SWAP=0 */ -#define free_swap_and_cache(e) is_pfn_swap_entry(e) +static inline void free_swap_and_cache_nr(swp_entry_t entry, int nr) +{ +} static inline void free_swap_cache(struct folio *folio) { @@ -586,6 +587,11 @@ static inline int add_swap_extent(struct swap_info_struct *sis, } #endif /* CONFIG_SWAP */ +static inline void free_swap_and_cache(swp_entry_t entry) +{ + free_swap_and_cache_nr(entry, 1); +} + #ifdef CONFIG_MEMCG static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) { diff --git a/mm/internal.h b/mm/internal.h index d567381b12cc8..d34df04b11f62 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include struct folio_batch; @@ -189,6 +191,68 @@ static inline int folio_pte_batch(struct folio *folio, unsigned long addr, return min(ptep - start_ptep, max_nr); } + +/** + * pte_next_swp_offset - Increment the swap entry offset field of a swap pte. + * @pte: The initial pte state; is_swap_pte(pte) must be true and + * non_swap_entry() must be false. + * + * Increments the swap offset, while maintaining all other fields, including + * swap type, and any swp pte bits. The resulting pte is returned. + */ +static inline pte_t pte_next_swp_offset(pte_t pte) +{ + swp_entry_t entry = pte_to_swp_entry(pte); + pte_t new = __swp_entry_to_pte(__swp_entry(swp_type(entry), + (swp_offset(entry) + 1))); + + if (pte_swp_soft_dirty(pte)) + new = pte_swp_mksoft_dirty(new); + if (pte_swp_exclusive(pte)) + new = pte_swp_mkexclusive(new); + if (pte_swp_uffd_wp(pte)) + new = pte_swp_mkuffd_wp(new); + + return new; +} + +/** + * swap_pte_batch - detect a PTE batch for a set of contiguous swap entries + * @start_ptep: Page table pointer for the first entry. + * @max_nr: The maximum number of table entries to consider. + * @pte: Page table entry for the first entry. + * + * Detect a batch of contiguous swap entries: consecutive (non-present) PTEs + * containing swap entries all with consecutive offsets and targeting the same + * swap type, all with matching swp pte bits. + * + * max_nr must be at least one and must be limited by the caller so scanning + * cannot exceed a single page table. + * + * Return: the number of table entries in the batch. + */ +static inline int swap_pte_batch(pte_t *start_ptep, int max_nr, pte_t pte) +{ + pte_t expected_pte = pte_next_swp_offset(pte); + const pte_t *end_ptep = start_ptep + max_nr; + pte_t *ptep = start_ptep + 1; + + VM_WARN_ON(max_nr < 1); + VM_WARN_ON(!is_swap_pte(pte)); + VM_WARN_ON(non_swap_entry(pte_to_swp_entry(pte))); + + while (ptep < end_ptep) { + pte = ptep_get(ptep); + + if (!pte_same(pte, expected_pte)) + break; + + expected_pte = pte_next_swp_offset(expected_pte); + ptep++; + } + + return ptep - start_ptep; +} #endif /* CONFIG_MMU */ void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio, diff --git a/mm/madvise.c b/mm/madvise.c index 1f77a51baaac2..5011ecb243441 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -628,6 +628,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, struct folio *folio; int nr_swap = 0; unsigned long next; + int nr, max_nr; next = pmd_addr_end(addr, end); if (pmd_trans_huge(*pmd)) @@ -640,7 +641,8 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, return 0; flush_tlb_batched_pending(mm); arch_enter_lazy_mmu_mode(); - for (; addr != end; pte++, addr += PAGE_SIZE) { + for (; addr != end; pte += nr, addr += PAGE_SIZE * nr) { + nr = 1; ptent = ptep_get(pte); if (pte_none(ptent)) @@ -655,9 +657,11 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, entry = pte_to_swp_entry(ptent); if (!non_swap_entry(entry)) { - nr_swap--; - free_swap_and_cache(entry); - pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); + max_nr = (end - addr) / PAGE_SIZE; + nr = swap_pte_batch(pte, max_nr, ptent); + nr_swap -= nr; + free_swap_and_cache_nr(entry, nr); + clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm); } else if (is_hwpoison_entry(entry) || is_poisoned_swp_entry(entry)) { pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); diff --git a/mm/memory.c b/mm/memory.c index 694e18837cd8c..7880400370c8b 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1637,12 +1637,13 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, folio_remove_rmap_pte(folio, page, vma); folio_put(folio); } else if (!non_swap_entry(entry)) { - /* Genuine swap entry, hence a private anon page */ + max_nr = (end - addr) / PAGE_SIZE; + nr = swap_pte_batch(pte, max_nr, ptent); + /* Genuine swap entries, hence a private anon pages */ if (!should_zap_cows(details)) continue; - rss[MM_SWAPENTS]--; - if (unlikely(!free_swap_and_cache(entry))) - print_bad_pte(vma, addr, ptent, NULL); + rss[MM_SWAPENTS] -= nr; + free_swap_and_cache_nr(entry, nr); } else if (is_migration_entry(entry)) { folio = pfn_swap_entry_folio(entry); if (!should_zap_folio(details, folio)) @@ -1665,8 +1666,8 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, pr_alert("unrecognized swap entry 0x%lx\n", entry.val); WARN_ON_ONCE(1); } - pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); - zap_install_uffd_wp_if_needed(vma, addr, pte, 1, details, ptent); + clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm); + zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent); } while (pte += nr, addr += PAGE_SIZE * nr, addr != end); add_mm_rss_vec(mm, rss); diff --git a/mm/swapfile.c b/mm/swapfile.c index 1ded6d1dcab42..20c45757f2b25 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -130,7 +130,11 @@ static inline unsigned char swap_count(unsigned char ent) /* Reclaim the swap entry if swap is getting full*/ #define TTRS_FULL 0x4 -/* returns 1 if swap entry is freed */ +/* + * returns number of pages in the folio that backs the swap entry. If positive, + * the folio was reclaimed. If negative, the folio was not reclaimed. If 0, no + * folio was associated with the swap entry. + */ static int __try_to_reclaim_swap(struct swap_info_struct *si, unsigned long offset, unsigned long flags) { @@ -155,6 +159,7 @@ static int __try_to_reclaim_swap(struct swap_info_struct *si, ret = folio_free_swap(folio); folio_unlock(folio); } + ret = ret ? folio_nr_pages(folio) : -folio_nr_pages(folio); folio_put(folio); return ret; } @@ -895,7 +900,7 @@ static int scan_swap_map_slots(struct swap_info_struct *si, swap_was_freed = __try_to_reclaim_swap(si, offset, TTRS_ANYWAY); spin_lock(&si->lock); /* entry was freed successfully, try to use this again */ - if (swap_was_freed) + if (swap_was_freed > 0) goto checks; goto scan; /* check next one */ } @@ -1572,32 +1577,88 @@ bool folio_free_swap(struct folio *folio) return true; } -/* - * Free the swap entry like above, but also try to - * free the page cache entry if it is the last user. +/** + * free_swap_and_cache_nr() - Release reference on range of swap entries and + * reclaim their cache if no more references remain. + * @entry: First entry of range. + * @nr: Number of entries in range. + * + * For each swap entry in the contiguous range, release a reference. If any swap + * entries become free, try to reclaim their underlying folios, if present. The + * offset range is defined by [entry.offset, entry.offset + nr). */ -int free_swap_and_cache(swp_entry_t entry) +void free_swap_and_cache_nr(swp_entry_t entry, int nr) { - struct swap_info_struct *p; + const unsigned long start_offset = swp_offset(entry); + const unsigned long end_offset = start_offset + nr; + unsigned int type = swp_type(entry); + struct swap_info_struct *si; + bool any_only_cache = false; + unsigned long offset; unsigned char count; if (non_swap_entry(entry)) - return 1; + return; - p = get_swap_device(entry); - if (p) { - if (WARN_ON(data_race(!p->swap_map[swp_offset(entry)]))) { - put_swap_device(p); - return 0; + si = get_swap_device(entry); + if (!si) + return; + + if (WARN_ON(end_offset > si->max)) + goto out; + + /* + * First free all entries in the range. + */ + for (offset = start_offset; offset < end_offset; offset++) { + if (data_race(si->swap_map[offset])) { + count = __swap_entry_free(si, swp_entry(type, offset)); + if (count == SWAP_HAS_CACHE) + any_only_cache = true; + } else { + WARN_ON_ONCE(1); } + } + + /* + * Short-circuit the below loop if none of the entries had their + * reference drop to zero. + */ + if (!any_only_cache) + goto out; - count = __swap_entry_free(p, entry); - if (count == SWAP_HAS_CACHE) - __try_to_reclaim_swap(p, swp_offset(entry), + /* + * Now go back over the range trying to reclaim the swap cache. This is + * more efficient for large folios because we will only try to reclaim + * the swap once per folio in the common case. If we do + * __swap_entry_free() and __try_to_reclaim_swap() in the same loop, the + * latter will get a reference and lock the folio for every individual + * page but will only succeed once the swap slot for every subpage is + * zero. + */ + for (offset = start_offset; offset < end_offset; offset += nr) { + nr = 1; + if (READ_ONCE(si->swap_map[offset]) == SWAP_HAS_CACHE) { + /* + * Folios are always naturally aligned in swap so + * advance forward to the next boundary. Zero means no + * folio was found for the swap entry, so advance by 1 + * in this case. Negative value means folio was found + * but could not be reclaimed. Here we can still advance + * to the next boundary. + */ + nr = __try_to_reclaim_swap(si, offset, TTRS_UNMAPPED | TTRS_FULL); - put_swap_device(p); + if (nr == 0) + nr = 1; + else if (nr < 0) + nr = -nr; + nr = ALIGN(offset + 1, nr) - offset; + } } - return p != NULL; + +out: + put_swap_device(si); } #ifdef CONFIG_HIBERNATION From 14c62da21b2b865f4fc0c49edd74ed7299927d35 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 8 Apr 2024 19:39:42 +0100 Subject: [PATCH 0746/1702] mm: swap: simplify struct percpu_cluster struct percpu_cluster stores the index of cpu's current cluster and the offset of the next entry that will be allocated for the cpu. These two pieces of information are redundant because the cluster index is just (offset / SWAPFILE_CLUSTER). The only reason for explicitly keeping the cluster index is because the structure used for it also has a flag to indicate "no cluster". However this data structure also contains a spin lock, which is never used in this context, as a side effect the code copies the spinlock_t structure, which is questionable coding practice in my view. So let's clean this up and store only the next offset, and use a sentinal value (SWAP_NEXT_INVALID) to indicate "no cluster". SWAP_NEXT_INVALID is chosen to be 0, because 0 will never be seen legitimately; The first page in the swap file is the swap header, which is always marked bad to prevent it from being allocated as an entry. This also prevents the cluster to which it belongs being marked free, so it will never appear on the free list. This change saves 16 bytes per cpu. And given we are shortly going to extend this mechanism to be per-cpu-AND-per-order, we will end up saving 16 * 9 = 144 bytes per cpu, which adds up if you have 256 cpus in the system. Link: https://lkml.kernel.org/r/20240408183946.2991168-4-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Reviewed-by: "Huang, Ying" Cc: Barry Song <21cnbao@gmail.com> Cc: Barry Song Cc: Chris Li Cc: David Hildenbrand Cc: Gao Xiang Cc: Kefeng Wang Cc: Lance Yang Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Yang Shi Cc: Yu Zhao Signed-off-by: Andrew Morton --- include/linux/swap.h | 9 ++++++++- mm/swapfile.c | 22 +++++++++++----------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index 2d8f2b950ddfa..ce35fd4abdf0f 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -260,13 +260,20 @@ struct swap_cluster_info { #define CLUSTER_FLAG_FREE 1 /* This cluster is free */ #define CLUSTER_FLAG_NEXT_NULL 2 /* This cluster has no next cluster */ +/* + * The first page in the swap file is the swap header, which is always marked + * bad to prevent it from being allocated as an entry. This also prevents the + * cluster to which it belongs being marked free. Therefore 0 is safe to use as + * a sentinel to indicate next is not valid in percpu_cluster. + */ +#define SWAP_NEXT_INVALID 0 + /* * We assign a cluster to each CPU, so each CPU can allocate swap entry from * its own cluster and swapout sequentially. The purpose is to optimize swapout * throughput. */ struct percpu_cluster { - struct swap_cluster_info index; /* Current cluster index */ unsigned int next; /* Likely next allocation offset */ }; diff --git a/mm/swapfile.c b/mm/swapfile.c index 20c45757f2b25..e3f8554752788 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -609,7 +609,7 @@ scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si, return false; percpu_cluster = this_cpu_ptr(si->percpu_cluster); - cluster_set_null(&percpu_cluster->index); + percpu_cluster->next = SWAP_NEXT_INVALID; return true; } @@ -622,14 +622,14 @@ static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si, { struct percpu_cluster *cluster; struct swap_cluster_info *ci; - unsigned long tmp, max; + unsigned int tmp, max; new_cluster: cluster = this_cpu_ptr(si->percpu_cluster); - if (cluster_is_null(&cluster->index)) { + tmp = cluster->next; + if (tmp == SWAP_NEXT_INVALID) { if (!cluster_list_empty(&si->free_clusters)) { - cluster->index = si->free_clusters.head; - cluster->next = cluster_next(&cluster->index) * + tmp = cluster_next(&si->free_clusters.head) * SWAPFILE_CLUSTER; } else if (!cluster_list_empty(&si->discard_clusters)) { /* @@ -649,9 +649,7 @@ static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si, * Other CPUs can use our cluster if they can't find a free cluster, * check if there is still free entry in the cluster */ - tmp = cluster->next; - max = min_t(unsigned long, si->max, - (cluster_next(&cluster->index) + 1) * SWAPFILE_CLUSTER); + max = min_t(unsigned long, si->max, ALIGN(tmp + 1, SWAPFILE_CLUSTER)); if (tmp < max) { ci = lock_cluster(si, tmp); while (tmp < max) { @@ -662,12 +660,13 @@ static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si, unlock_cluster(ci); } if (tmp >= max) { - cluster_set_null(&cluster->index); + cluster->next = SWAP_NEXT_INVALID; goto new_cluster; } - cluster->next = tmp + 1; *offset = tmp; *scan_base = tmp; + tmp += 1; + cluster->next = tmp < max ? tmp : SWAP_NEXT_INVALID; return true; } @@ -3163,8 +3162,9 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) } for_each_possible_cpu(cpu) { struct percpu_cluster *cluster; + cluster = per_cpu_ptr(p->percpu_cluster, cpu); - cluster_set_null(&cluster->index); + cluster->next = SWAP_NEXT_INVALID; } } else { atomic_inc(&nr_rotate_swap); From 9faaa0f8168bfcd81469b0724b25ba3093097a08 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 8 Apr 2024 19:39:43 +0100 Subject: [PATCH 0747/1702] mm: swap: update get_swap_pages() to take folio order We are about to allow swap storage of any mTHP size. To prepare for that, let's change get_swap_pages() to take a folio order parameter instead of nr_pages. This makes the interface self-documenting; a power-of-2 number of pages must be provided. We will also need the order internally so this simplifies accessing it. Link: https://lkml.kernel.org/r/20240408183946.2991168-5-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Reviewed-by: "Huang, Ying" Reviewed-by: David Hildenbrand Cc: Barry Song <21cnbao@gmail.com> Cc: Barry Song Cc: Chris Li Cc: Gao Xiang Cc: Kefeng Wang Cc: Lance Yang Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Yang Shi Cc: Yu Zhao Signed-off-by: Andrew Morton --- include/linux/swap.h | 2 +- mm/swap_slots.c | 6 +++--- mm/swapfile.c | 13 +++++++------ 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index ce35fd4abdf0f..b69b067332473 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -468,7 +468,7 @@ swp_entry_t folio_alloc_swap(struct folio *folio); bool folio_free_swap(struct folio *folio); void put_swap_folio(struct folio *folio, swp_entry_t entry); extern swp_entry_t get_swap_page_of_type(int); -extern int get_swap_pages(int n, swp_entry_t swp_entries[], int entry_size); +extern int get_swap_pages(int n, swp_entry_t swp_entries[], int order); extern int add_swap_count_continuation(swp_entry_t, gfp_t); extern void swap_shmem_alloc(swp_entry_t); extern int swap_duplicate(swp_entry_t); diff --git a/mm/swap_slots.c b/mm/swap_slots.c index 53abeaf1371d0..13ab3b771409c 100644 --- a/mm/swap_slots.c +++ b/mm/swap_slots.c @@ -264,7 +264,7 @@ static int refill_swap_slots_cache(struct swap_slots_cache *cache) cache->cur = 0; if (swap_slot_cache_active) cache->nr = get_swap_pages(SWAP_SLOTS_CACHE_SIZE, - cache->slots, 1); + cache->slots, 0); return cache->nr; } @@ -311,7 +311,7 @@ swp_entry_t folio_alloc_swap(struct folio *folio) if (folio_test_large(folio)) { if (IS_ENABLED(CONFIG_THP_SWAP)) - get_swap_pages(1, &entry, folio_nr_pages(folio)); + get_swap_pages(1, &entry, folio_order(folio)); goto out; } @@ -343,7 +343,7 @@ swp_entry_t folio_alloc_swap(struct folio *folio) goto out; } - get_swap_pages(1, &entry, 1); + get_swap_pages(1, &entry, 0); out: if (mem_cgroup_try_charge_swap(folio, entry)) { put_swap_folio(folio, entry); diff --git a/mm/swapfile.c b/mm/swapfile.c index e3f8554752788..d2e3d3cd439f5 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -278,15 +278,15 @@ static void discard_swap_cluster(struct swap_info_struct *si, #ifdef CONFIG_THP_SWAP #define SWAPFILE_CLUSTER HPAGE_PMD_NR -#define swap_entry_size(size) (size) +#define swap_entry_order(order) (order) #else #define SWAPFILE_CLUSTER 256 /* - * Define swap_entry_size() as constant to let compiler to optimize + * Define swap_entry_order() as constant to let compiler to optimize * out some code if !CONFIG_THP_SWAP */ -#define swap_entry_size(size) 1 +#define swap_entry_order(order) 0 #endif #define LATENCY_LIMIT 256 @@ -1042,9 +1042,10 @@ static void swap_free_cluster(struct swap_info_struct *si, unsigned long idx) swap_range_free(si, offset, SWAPFILE_CLUSTER); } -int get_swap_pages(int n_goal, swp_entry_t swp_entries[], int entry_size) +int get_swap_pages(int n_goal, swp_entry_t swp_entries[], int entry_order) { - unsigned long size = swap_entry_size(entry_size); + int order = swap_entry_order(entry_order); + unsigned long size = 1 << order; struct swap_info_struct *si, *next; long avail_pgs; int n_ret = 0; @@ -1349,7 +1350,7 @@ void put_swap_folio(struct folio *folio, swp_entry_t entry) unsigned char *map; unsigned int i, free_entries = 0; unsigned char val; - int size = swap_entry_size(folio_nr_pages(folio)); + int size = 1 << swap_entry_order(folio_order(folio)); si = _swap_info_get(entry); if (!si) From 845982eb264bc64b0c3242ace217fb574f56a299 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 8 Apr 2024 19:39:44 +0100 Subject: [PATCH 0748/1702] mm: swap: allow storage of all mTHP orders Multi-size THP enables performance improvements by allocating large, pte-mapped folios for anonymous memory. However I've observed that on an arm64 system running a parallel workload (e.g. kernel compilation) across many cores, under high memory pressure, the speed regresses. This is due to bottlenecking on the increased number of TLBIs added due to all the extra folio splitting when the large folios are swapped out. Therefore, solve this regression by adding support for swapping out mTHP without needing to split the folio, just like is already done for PMD-sized THP. This change only applies when CONFIG_THP_SWAP is enabled, and when the swap backing store is a non-rotating block device. These are the same constraints as for the existing PMD-sized THP swap-out support. Note that no attempt is made to swap-in (m)THP here - this is still done page-by-page, like for PMD-sized THP. But swapping-out mTHP is a prerequisite for swapping-in mTHP. The main change here is to improve the swap entry allocator so that it can allocate any power-of-2 number of contiguous entries between [1, (1 << PMD_ORDER)]. This is done by allocating a cluster for each distinct order and allocating sequentially from it until the cluster is full. This ensures that we don't need to search the map and we get no fragmentation due to alignment padding for different orders in the cluster. If there is no current cluster for a given order, we attempt to allocate a free cluster from the list. If there are no free clusters, we fail the allocation and the caller can fall back to splitting the folio and allocates individual entries (as per existing PMD-sized THP fallback). The per-order current clusters are maintained per-cpu using the existing infrastructure. This is done to avoid interleving pages from different tasks, which would prevent IO being batched. This is already done for the order-0 allocations so we follow the same pattern. As is done for order-0 per-cpu clusters, the scanner now can steal order-0 entries from any per-cpu-per-order reserved cluster. This ensures that when the swap file is getting full, space doesn't get tied up in the per-cpu reserves. This change only modifies swap to be able to accept any order mTHP. It doesn't change the callers to elide doing the actual split. That will be done in separate changes. Link: https://lkml.kernel.org/r/20240408183946.2991168-6-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Reviewed-by: "Huang, Ying" Cc: Barry Song <21cnbao@gmail.com> Cc: Barry Song Cc: Chris Li Cc: David Hildenbrand Cc: Gao Xiang Cc: Kefeng Wang Cc: Lance Yang Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Yang Shi Cc: Yu Zhao Signed-off-by: Andrew Morton --- include/linux/swap.h | 8 ++- mm/swapfile.c | 162 ++++++++++++++++++++++++------------------- 2 files changed, 98 insertions(+), 72 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index b69b067332473..6b672a67ae5d9 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -268,13 +268,19 @@ struct swap_cluster_info { */ #define SWAP_NEXT_INVALID 0 +#ifdef CONFIG_THP_SWAP +#define SWAP_NR_ORDERS (PMD_ORDER + 1) +#else +#define SWAP_NR_ORDERS 1 +#endif + /* * We assign a cluster to each CPU, so each CPU can allocate swap entry from * its own cluster and swapout sequentially. The purpose is to optimize swapout * throughput. */ struct percpu_cluster { - unsigned int next; /* Likely next allocation offset */ + unsigned int next[SWAP_NR_ORDERS]; /* Likely next allocation offset */ }; struct swap_cluster_list { diff --git a/mm/swapfile.c b/mm/swapfile.c index d2e3d3cd439f5..148ef08f19dd6 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -551,10 +551,12 @@ static void free_cluster(struct swap_info_struct *si, unsigned long idx) /* * The cluster corresponding to page_nr will be used. The cluster will be - * removed from free cluster list and its usage counter will be increased. + * removed from free cluster list and its usage counter will be increased by + * count. */ -static void inc_cluster_info_page(struct swap_info_struct *p, - struct swap_cluster_info *cluster_info, unsigned long page_nr) +static void add_cluster_info_page(struct swap_info_struct *p, + struct swap_cluster_info *cluster_info, unsigned long page_nr, + unsigned long count) { unsigned long idx = page_nr / SWAPFILE_CLUSTER; @@ -563,9 +565,19 @@ static void inc_cluster_info_page(struct swap_info_struct *p, if (cluster_is_free(&cluster_info[idx])) alloc_cluster(p, idx); - VM_BUG_ON(cluster_count(&cluster_info[idx]) >= SWAPFILE_CLUSTER); + VM_BUG_ON(cluster_count(&cluster_info[idx]) + count > SWAPFILE_CLUSTER); cluster_set_count(&cluster_info[idx], - cluster_count(&cluster_info[idx]) + 1); + cluster_count(&cluster_info[idx]) + count); +} + +/* + * The cluster corresponding to page_nr will be used. The cluster will be + * removed from free cluster list and its usage counter will be increased by 1. + */ +static void inc_cluster_info_page(struct swap_info_struct *p, + struct swap_cluster_info *cluster_info, unsigned long page_nr) +{ + add_cluster_info_page(p, cluster_info, page_nr, 1); } /* @@ -595,7 +607,7 @@ static void dec_cluster_info_page(struct swap_info_struct *p, */ static bool scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si, - unsigned long offset) + unsigned long offset, int order) { struct percpu_cluster *percpu_cluster; bool conflict; @@ -609,24 +621,39 @@ scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si, return false; percpu_cluster = this_cpu_ptr(si->percpu_cluster); - percpu_cluster->next = SWAP_NEXT_INVALID; + percpu_cluster->next[order] = SWAP_NEXT_INVALID; + return true; +} + +static inline bool swap_range_empty(char *swap_map, unsigned int start, + unsigned int nr_pages) +{ + unsigned int i; + + for (i = 0; i < nr_pages; i++) { + if (swap_map[start + i]) + return false; + } + return true; } /* - * Try to get a swap entry from current cpu's swap entry pool (a cluster). This - * might involve allocating a new cluster for current CPU too. + * Try to get swap entries with specified order from current cpu's swap entry + * pool (a cluster). This might involve allocating a new cluster for current CPU + * too. */ static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si, - unsigned long *offset, unsigned long *scan_base) + unsigned long *offset, unsigned long *scan_base, int order) { + unsigned int nr_pages = 1 << order; struct percpu_cluster *cluster; struct swap_cluster_info *ci; unsigned int tmp, max; new_cluster: cluster = this_cpu_ptr(si->percpu_cluster); - tmp = cluster->next; + tmp = cluster->next[order]; if (tmp == SWAP_NEXT_INVALID) { if (!cluster_list_empty(&si->free_clusters)) { tmp = cluster_next(&si->free_clusters.head) * @@ -647,26 +674,27 @@ static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si, /* * Other CPUs can use our cluster if they can't find a free cluster, - * check if there is still free entry in the cluster + * check if there is still free entry in the cluster, maintaining + * natural alignment. */ max = min_t(unsigned long, si->max, ALIGN(tmp + 1, SWAPFILE_CLUSTER)); if (tmp < max) { ci = lock_cluster(si, tmp); while (tmp < max) { - if (!si->swap_map[tmp]) + if (swap_range_empty(si->swap_map, tmp, nr_pages)) break; - tmp++; + tmp += nr_pages; } unlock_cluster(ci); } if (tmp >= max) { - cluster->next = SWAP_NEXT_INVALID; + cluster->next[order] = SWAP_NEXT_INVALID; goto new_cluster; } *offset = tmp; *scan_base = tmp; - tmp += 1; - cluster->next = tmp < max ? tmp : SWAP_NEXT_INVALID; + tmp += nr_pages; + cluster->next[order] = tmp < max ? tmp : SWAP_NEXT_INVALID; return true; } @@ -796,13 +824,14 @@ static bool swap_offset_available_and_locked(struct swap_info_struct *si, static int scan_swap_map_slots(struct swap_info_struct *si, unsigned char usage, int nr, - swp_entry_t slots[]) + swp_entry_t slots[], int order) { struct swap_cluster_info *ci; unsigned long offset; unsigned long scan_base; unsigned long last_in_cluster = 0; int latency_ration = LATENCY_LIMIT; + unsigned int nr_pages = 1 << order; int n_ret = 0; bool scanned_many = false; @@ -817,6 +846,25 @@ static int scan_swap_map_slots(struct swap_info_struct *si, * And we let swap pages go all over an SSD partition. Hugh */ + if (order > 0) { + /* + * Should not even be attempting large allocations when huge + * page swap is disabled. Warn and fail the allocation. + */ + if (!IS_ENABLED(CONFIG_THP_SWAP) || + nr_pages > SWAPFILE_CLUSTER) { + VM_WARN_ON_ONCE(1); + return 0; + } + + /* + * Swapfile is not block device or not using clusters so unable + * to allocate large entries. + */ + if (!(si->flags & SWP_BLKDEV) || !si->cluster_info) + return 0; + } + si->flags += SWP_SCANNING; /* * Use percpu scan base for SSD to reduce lock contention on @@ -831,8 +879,11 @@ static int scan_swap_map_slots(struct swap_info_struct *si, /* SSD algorithm */ if (si->cluster_info) { - if (!scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) + if (!scan_swap_map_try_ssd_cluster(si, &offset, &scan_base, order)) { + if (order > 0) + goto no_page; goto scan; + } } else if (unlikely(!si->cluster_nr--)) { if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) { si->cluster_nr = SWAPFILE_CLUSTER - 1; @@ -874,13 +925,16 @@ static int scan_swap_map_slots(struct swap_info_struct *si, checks: if (si->cluster_info) { - while (scan_swap_map_ssd_cluster_conflict(si, offset)) { + while (scan_swap_map_ssd_cluster_conflict(si, offset, order)) { /* take a break if we already got some slots */ if (n_ret) goto done; if (!scan_swap_map_try_ssd_cluster(si, &offset, - &scan_base)) + &scan_base, order)) { + if (order > 0) + goto no_page; goto scan; + } } } if (!(si->flags & SWP_WRITEOK)) @@ -911,11 +965,11 @@ static int scan_swap_map_slots(struct swap_info_struct *si, else goto done; } - WRITE_ONCE(si->swap_map[offset], usage); - inc_cluster_info_page(si, si->cluster_info, offset); + memset(si->swap_map + offset, usage, nr_pages); + add_cluster_info_page(si, si->cluster_info, offset, nr_pages); unlock_cluster(ci); - swap_range_alloc(si, offset, 1); + swap_range_alloc(si, offset, nr_pages); slots[n_ret++] = swp_entry(si->type, offset); /* got enough slots or reach max slots? */ @@ -936,8 +990,10 @@ static int scan_swap_map_slots(struct swap_info_struct *si, /* try to get more slots in cluster */ if (si->cluster_info) { - if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) + if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base, order)) goto checks; + if (order > 0) + goto done; } else if (si->cluster_nr && !si->swap_map[++offset]) { /* non-ssd case, still more slots in cluster? */ --si->cluster_nr; @@ -964,11 +1020,13 @@ static int scan_swap_map_slots(struct swap_info_struct *si, } done: - set_cluster_next(si, offset + 1); + if (order == 0) + set_cluster_next(si, offset + 1); si->flags -= SWP_SCANNING; return n_ret; scan: + VM_WARN_ON(order > 0); spin_unlock(&si->lock); while (++offset <= READ_ONCE(si->highest_bit)) { if (unlikely(--latency_ration < 0)) { @@ -997,38 +1055,6 @@ static int scan_swap_map_slots(struct swap_info_struct *si, return n_ret; } -static int swap_alloc_cluster(struct swap_info_struct *si, swp_entry_t *slot) -{ - unsigned long idx; - struct swap_cluster_info *ci; - unsigned long offset; - - /* - * Should not even be attempting cluster allocations when huge - * page swap is disabled. Warn and fail the allocation. - */ - if (!IS_ENABLED(CONFIG_THP_SWAP)) { - VM_WARN_ON_ONCE(1); - return 0; - } - - if (cluster_list_empty(&si->free_clusters)) - return 0; - - idx = cluster_list_first(&si->free_clusters); - offset = idx * SWAPFILE_CLUSTER; - ci = lock_cluster(si, offset); - alloc_cluster(si, idx); - cluster_set_count(ci, SWAPFILE_CLUSTER); - - memset(si->swap_map + offset, SWAP_HAS_CACHE, SWAPFILE_CLUSTER); - unlock_cluster(ci); - swap_range_alloc(si, offset, SWAPFILE_CLUSTER); - *slot = swp_entry(si->type, offset); - - return 1; -} - static void swap_free_cluster(struct swap_info_struct *si, unsigned long idx) { unsigned long offset = idx * SWAPFILE_CLUSTER; @@ -1051,9 +1077,6 @@ int get_swap_pages(int n_goal, swp_entry_t swp_entries[], int entry_order) int n_ret = 0; int node; - /* Only single cluster request supported */ - WARN_ON_ONCE(n_goal > 1 && size == SWAPFILE_CLUSTER); - spin_lock(&swap_avail_lock); avail_pgs = atomic_long_read(&nr_swap_pages) / size; @@ -1089,14 +1112,10 @@ int get_swap_pages(int n_goal, swp_entry_t swp_entries[], int entry_order) spin_unlock(&si->lock); goto nextsi; } - if (size == SWAPFILE_CLUSTER) { - if (si->flags & SWP_BLKDEV) - n_ret = swap_alloc_cluster(si, swp_entries); - } else - n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE, - n_goal, swp_entries); + n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE, + n_goal, swp_entries, order); spin_unlock(&si->lock); - if (n_ret || size == SWAPFILE_CLUSTER) + if (n_ret || size > 1) goto check_out; cond_resched(); @@ -1673,7 +1692,7 @@ swp_entry_t get_swap_page_of_type(int type) /* This is called for allocating swap entry, not cache */ spin_lock(&si->lock); - if ((si->flags & SWP_WRITEOK) && scan_swap_map_slots(si, 1, 1, &entry)) + if ((si->flags & SWP_WRITEOK) && scan_swap_map_slots(si, 1, 1, &entry, 0)) atomic_long_dec(&nr_swap_pages); spin_unlock(&si->lock); fail: @@ -3127,7 +3146,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) p->flags |= SWP_SYNCHRONOUS_IO; if (p->bdev && bdev_nonrot(p->bdev)) { - int cpu; + int cpu, i; unsigned long ci, nr_cluster; p->flags |= SWP_SOLIDSTATE; @@ -3165,7 +3184,8 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) struct percpu_cluster *cluster; cluster = per_cpu_ptr(p->percpu_cluster, cpu); - cluster->next = SWAP_NEXT_INVALID; + for (i = 0; i < SWAP_NR_ORDERS; i++) + cluster->next[i] = SWAP_NEXT_INVALID; } } else { atomic_inc(&nr_rotate_swap); From 5ed890ce5147855c5360affd5e5419ed68a54100 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 8 Apr 2024 19:39:45 +0100 Subject: [PATCH 0749/1702] mm: vmscan: avoid split during shrink_folio_list() Now that swap supports storing all mTHP sizes, avoid splitting large folios before swap-out. This benefits performance of the swap-out path by eliding split_folio_to_list(), which is expensive, and also sets us up for swapping in large folios in a future series. If the folio is partially mapped, we continue to split it since we want to avoid the extra IO overhead and storage of writing out pages uneccessarily. THP_SWPOUT and THP_SWPOUT_FALLBACK counters should continue to count events only for PMD-mappable folios to avoid user confusion. THP_SWPOUT already has the appropriate guard. Add a guard for THP_SWPOUT_FALLBACK. It may be appropriate to add per-size counters in future. Link: https://lkml.kernel.org/r/20240408183946.2991168-7-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Reviewed-by: David Hildenbrand Reviewed-by: Barry Song Cc: Barry Song <21cnbao@gmail.com> Cc: Chris Li Cc: Gao Xiang Cc: "Huang, Ying" Cc: Kefeng Wang Cc: Lance Yang Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Yang Shi Cc: Yu Zhao Signed-off-by: Andrew Morton --- mm/vmscan.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 7554adc9c824e..da93213af981e 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1206,25 +1206,25 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, if (!can_split_folio(folio, NULL)) goto activate_locked; /* - * Split folios without a PMD map right - * away. Chances are some or all of the - * tail pages can be freed without IO. + * Split partially mapped folios right away. + * We can free the unmapped pages without IO. */ - if (!folio_entire_mapcount(folio) && - split_folio_to_list(folio, - folio_list)) + if (data_race(!list_empty(&folio->_deferred_list)) && + split_folio_to_list(folio, folio_list)) goto activate_locked; } if (!add_to_swap(folio)) { if (!folio_test_large(folio)) goto activate_locked_split; /* Fallback to swap normal pages */ - if (split_folio_to_list(folio, - folio_list)) + if (split_folio_to_list(folio, folio_list)) goto activate_locked; #ifdef CONFIG_TRANSPARENT_HUGEPAGE - count_memcg_folio_events(folio, THP_SWPOUT_FALLBACK, 1); - count_vm_event(THP_SWPOUT_FALLBACK); + if (nr_pages >= HPAGE_PMD_NR) { + count_memcg_folio_events(folio, + THP_SWPOUT_FALLBACK, 1); + count_vm_event(THP_SWPOUT_FALLBACK); + } #endif if (!add_to_swap(folio)) goto activate_locked_split; From 3931b871c4936c00c4e27c469056d8da47a3493f Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 8 Apr 2024 19:39:46 +0100 Subject: [PATCH 0750/1702] mm: madvise: avoid split during MADV_PAGEOUT and MADV_COLD Rework madvise_cold_or_pageout_pte_range() to avoid splitting any large folio that is fully and contiguously mapped in the pageout/cold vm range. This change means that large folios will be maintained all the way to swap storage. This both improves performance during swap-out, by eliding the cost of splitting the folio, and sets us up nicely for maintaining the large folio when it is swapped back in (to be covered in a separate series). Folios that are not fully mapped in the target range are still split, but note that behavior is changed so that if the split fails for any reason (folio locked, shared, etc) we now leave it as is and move to the next pte in the range and continue work on the proceeding folios. Previously any failure of this sort would cause the entire operation to give up and no folios mapped at higher addresses were paged out or made cold. Given large folios are becoming more common, this old behavior would have likely lead to wasted opportunities. While we are at it, change the code that clears young from the ptes to use ptep_test_and_clear_young(), via the new mkold_ptes() batch helper function. This is more efficent than get_and_clear/modify/set, especially for contpte mappings on arm64, where the old approach would require unfolding/refolding and the new approach can be done in place. Link: https://lkml.kernel.org/r/20240408183946.2991168-8-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Reviewed-by: Barry Song Acked-by: David Hildenbrand Cc: Barry Song <21cnbao@gmail.com> Cc: Chris Li Cc: Gao Xiang Cc: "Huang, Ying" Cc: Kefeng Wang Cc: Lance Yang Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Yang Shi Cc: Yu Zhao Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 30 ++++++++++++++ mm/internal.h | 12 +++++- mm/madvise.c | 87 +++++++++++++++++++++++------------------ mm/memory.c | 4 +- 4 files changed, 92 insertions(+), 41 deletions(-) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 75096025fe528..e2f45e22a6d13 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -361,6 +361,36 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, } #endif +#ifndef mkold_ptes +/** + * mkold_ptes - Mark PTEs that map consecutive pages of the same folio as old. + * @vma: VMA the pages are mapped into. + * @addr: Address the first page is mapped at. + * @ptep: Page table pointer for the first entry. + * @nr: Number of entries to mark old. + * + * May be overridden by the architecture; otherwise, implemented as a simple + * loop over ptep_test_and_clear_young(). + * + * Note that PTE bits in the PTE range besides the PFN can differ. For example, + * some PTEs might be write-protected. + * + * Context: The caller holds the page table lock. The PTEs map consecutive + * pages that belong to the same folio. The PTEs are all in the same PMD. + */ +static inline void mkold_ptes(struct vm_area_struct *vma, unsigned long addr, + pte_t *ptep, unsigned int nr) +{ + for (;;) { + ptep_test_and_clear_young(vma, addr, ptep); + if (--nr == 0) + break; + ptep++; + addr += PAGE_SIZE; + } +} +#endif + #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, diff --git a/mm/internal.h b/mm/internal.h index d34df04b11f62..640298a98a3cd 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -130,6 +130,8 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) * @flags: Flags to modify the PTE batch semantics. * @any_writable: Optional pointer to indicate whether any entry except the * first one is writable. + * @any_young: Optional pointer to indicate whether any entry except the + * first one is young. * * Detect a PTE batch: consecutive (present) PTEs that map consecutive * pages of the same large folio. @@ -145,16 +147,18 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) */ static inline int folio_pte_batch(struct folio *folio, unsigned long addr, pte_t *start_ptep, pte_t pte, int max_nr, fpb_t flags, - bool *any_writable) + bool *any_writable, bool *any_young) { unsigned long folio_end_pfn = folio_pfn(folio) + folio_nr_pages(folio); const pte_t *end_ptep = start_ptep + max_nr; pte_t expected_pte, *ptep; - bool writable; + bool writable, young; int nr; if (any_writable) *any_writable = false; + if (any_young) + *any_young = false; VM_WARN_ON_FOLIO(!pte_present(pte), folio); VM_WARN_ON_FOLIO(!folio_test_large(folio) || max_nr < 1, folio); @@ -168,6 +172,8 @@ static inline int folio_pte_batch(struct folio *folio, unsigned long addr, pte = ptep_get(ptep); if (any_writable) writable = !!pte_write(pte); + if (any_young) + young = !!pte_young(pte); pte = __pte_batch_clear_ignored(pte, flags); if (!pte_same(pte, expected_pte)) @@ -183,6 +189,8 @@ static inline int folio_pte_batch(struct folio *folio, unsigned long addr, if (any_writable) *any_writable |= writable; + if (any_young) + *any_young |= young; nr = pte_batch_hint(ptep, pte); expected_pte = pte_advance_pfn(expected_pte, nr); diff --git a/mm/madvise.c b/mm/madvise.c index 5011ecb243441..f59169888b8ee 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -336,6 +336,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, LIST_HEAD(folio_list); bool pageout_anon_only_filter; unsigned int batch_count = 0; + int nr; if (fatal_signal_pending(current)) return -EINTR; @@ -423,7 +424,8 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, return 0; flush_tlb_batched_pending(mm); arch_enter_lazy_mmu_mode(); - for (; addr < end; pte++, addr += PAGE_SIZE) { + for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) { + nr = 1; ptent = ptep_get(pte); if (++batch_count == SWAP_CLUSTER_MAX) { @@ -447,55 +449,66 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, continue; /* - * Creating a THP page is expensive so split it only if we - * are sure it's worth. Split it if we are only owner. + * If we encounter a large folio, only split it if it is not + * fully mapped within the range we are operating on. Otherwise + * leave it as is so that it can be swapped out whole. If we + * fail to split a folio, leave it in place and advance to the + * next pte in the range. */ if (folio_test_large(folio)) { - int err; - - if (folio_likely_mapped_shared(folio)) - break; - if (pageout_anon_only_filter && !folio_test_anon(folio)) - break; - if (!folio_trylock(folio)) - break; - folio_get(folio); - arch_leave_lazy_mmu_mode(); - pte_unmap_unlock(start_pte, ptl); - start_pte = NULL; - err = split_folio(folio); - folio_unlock(folio); - folio_put(folio); - if (err) - break; - start_pte = pte = - pte_offset_map_lock(mm, pmd, addr, &ptl); - if (!start_pte) - break; - arch_enter_lazy_mmu_mode(); - pte--; - addr -= PAGE_SIZE; - continue; + const fpb_t fpb_flags = FPB_IGNORE_DIRTY | + FPB_IGNORE_SOFT_DIRTY; + int max_nr = (end - addr) / PAGE_SIZE; + bool any_young; + + nr = folio_pte_batch(folio, addr, pte, ptent, max_nr, + fpb_flags, NULL, &any_young); + if (any_young) + ptent = pte_mkyoung(ptent); + + if (nr < folio_nr_pages(folio)) { + int err; + + if (folio_likely_mapped_shared(folio)) + continue; + if (pageout_anon_only_filter && !folio_test_anon(folio)) + continue; + if (!folio_trylock(folio)) + continue; + folio_get(folio); + arch_leave_lazy_mmu_mode(); + pte_unmap_unlock(start_pte, ptl); + start_pte = NULL; + err = split_folio(folio); + folio_unlock(folio); + folio_put(folio); + start_pte = pte = + pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!start_pte) + break; + arch_enter_lazy_mmu_mode(); + if (!err) + nr = 0; + continue; + } } /* * Do not interfere with other mappings of this folio and - * non-LRU folio. + * non-LRU folio. If we have a large folio at this point, we + * know it is fully mapped so if its mapcount is the same as its + * number of pages, it must be exclusive. */ - if (!folio_test_lru(folio) || folio_mapcount(folio) != 1) + if (!folio_test_lru(folio) || + folio_mapcount(folio) != folio_nr_pages(folio)) continue; if (pageout_anon_only_filter && !folio_test_anon(folio)) continue; - VM_BUG_ON_FOLIO(folio_test_large(folio), folio); - if (!pageout && pte_young(ptent)) { - ptent = ptep_get_and_clear_full(mm, addr, pte, - tlb->fullmm); - ptent = pte_mkold(ptent); - set_pte_at(mm, addr, pte, ptent); - tlb_remove_tlb_entry(tlb, pte, addr); + mkold_ptes(vma, addr, pte, nr); + tlb_remove_tlb_entries(tlb, pte, nr, addr); } /* diff --git a/mm/memory.c b/mm/memory.c index 7880400370c8b..59c05dc8b18a7 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -989,7 +989,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma flags |= FPB_IGNORE_SOFT_DIRTY; nr = folio_pte_batch(folio, addr, src_pte, pte, max_nr, flags, - &any_writable); + &any_writable, NULL); folio_ref_add(folio, nr); if (folio_test_anon(folio)) { if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page, @@ -1559,7 +1559,7 @@ static inline int zap_present_ptes(struct mmu_gather *tlb, */ if (unlikely(folio_test_large(folio) && max_nr != 1)) { nr = folio_pte_batch(folio, addr, pte, ptent, max_nr, fpb_flags, - NULL); + NULL, NULL); zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr, addr, details, rss, force_flush, From 6ea02ee489799317c6640ac014c49b1d1b7124c5 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 3 Apr 2024 16:37:59 +0800 Subject: [PATCH 0751/1702] arm64: mm: cleanup __do_page_fault() Patch series "arch/mm/fault: accelerate pagefault when badaccess", v2. After VMA lock-based page fault handling enabled, if bad access met under per-vma lock, it will fallback to mmap_lock-based handling, so it leads to unnessary mmap lock and vma find again. A test from lmbench shows 34% improve after this changes on arm64, lat_sig -P 1 prot lat_sig 0.29194 -> 0.19198 This patch (of 7): The __do_page_fault() only calls handle_mm_fault() after vm_flags checked, and it is only called by do_page_fault(), let's squash it into do_page_fault() to cleanup code. Link: https://lkml.kernel.org/r/20240403083805.1818160-1-wangkefeng.wang@huawei.com Link: https://lkml.kernel.org/r/20240403083805.1818160-2-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviewed-by: Suren Baghdasaryan Reviewed-by: Catalin Marinas Cc: Albert Ou Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: Christophe Leroy Cc: Dave Hansen Cc: Gerald Schaefer Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Peter Zijlstra Cc: Russell King Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/arm64/mm/fault.c | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 8251e2fea9c75..9bb9f395351ad 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -486,25 +486,6 @@ static void do_bad_area(unsigned long far, unsigned long esr, } } -#define VM_FAULT_BADMAP ((__force vm_fault_t)0x010000) -#define VM_FAULT_BADACCESS ((__force vm_fault_t)0x020000) - -static vm_fault_t __do_page_fault(struct mm_struct *mm, - struct vm_area_struct *vma, unsigned long addr, - unsigned int mm_flags, unsigned long vm_flags, - struct pt_regs *regs) -{ - /* - * Ok, we have a good vm_area for this memory access, so we can handle - * it. - * Check that the permissions on the VMA allow for the fault which - * occurred. - */ - if (!(vma->vm_flags & vm_flags)) - return VM_FAULT_BADACCESS; - return handle_mm_fault(vma, addr, mm_flags, regs); -} - static bool is_el0_instruction_abort(unsigned long esr) { return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW; @@ -519,6 +500,9 @@ static bool is_write_abort(unsigned long esr) return (esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM); } +#define VM_FAULT_BADMAP ((__force vm_fault_t)0x010000) +#define VM_FAULT_BADACCESS ((__force vm_fault_t)0x020000) + static int __kprobes do_page_fault(unsigned long far, unsigned long esr, struct pt_regs *regs) { @@ -617,7 +601,10 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, goto done; } - fault = __do_page_fault(mm, vma, addr, mm_flags, vm_flags, regs); + if (!(vma->vm_flags & vm_flags)) + fault = VM_FAULT_BADACCESS; + else + fault = handle_mm_fault(vma, addr, mm_flags, regs); /* Quick path to respond to signals */ if (fault_signal_pending(fault, regs)) { From faab3d0f250aba863b19bb2d72daea0ae90a1d5d Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 3 Apr 2024 16:38:00 +0800 Subject: [PATCH 0752/1702] arm64: mm: accelerate pagefault when VM_FAULT_BADACCESS The vm_flags of vma already checked under per-VMA lock, if it is a bad access, directly set fault to VM_FAULT_BADACCESS and handle error, no need to retry with mmap_lock again, the latency time reduces 34% in 'lat_sig -P 1 prot lat_sig' from lmbench testcase. Since the page fault is handled under per-VMA lock, count it as a vma lock event with VMA_LOCK_SUCCESS. Link: https://lkml.kernel.org/r/20240403083805.1818160-3-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviewed-by: Suren Baghdasaryan Reviewed-by: Catalin Marinas Cc: Albert Ou Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: Christophe Leroy Cc: Dave Hansen Cc: Gerald Schaefer Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Peter Zijlstra Cc: Russell King Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/arm64/mm/fault.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 9bb9f395351ad..405f9aa831bd2 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -572,7 +572,9 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, if (!(vma->vm_flags & vm_flags)) { vma_end_read(vma); - goto lock_mmap; + fault = VM_FAULT_BADACCESS; + count_vm_vma_lock_event(VMA_LOCK_SUCCESS); + goto done; } fault = handle_mm_fault(vma, addr, mm_flags | FAULT_FLAG_VMA_LOCK, regs); if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED))) From 15e4a5f5d8c9438530574d10241b8ece9f5edb61 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 3 Apr 2024 16:38:01 +0800 Subject: [PATCH 0753/1702] arm: mm: accelerate pagefault when VM_FAULT_BADACCESS The vm_flags of vma already checked under per-VMA lock, if it is a bad access, directly set fault to VM_FAULT_BADACCESS and handle error, no need to retry with mmap_lock again. Since the page faut is handled under per-VMA lock, count it as a vma lock event with VMA_LOCK_SUCCESS. Link: https://lkml.kernel.org/r/20240403083805.1818160-4-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviewed-by: Suren Baghdasaryan Cc: Albert Ou Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: Gerald Schaefer Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Peter Zijlstra Cc: Russell King Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/arm/mm/fault.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index 439dc6a26bb98..5c4b417e24f9e 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -294,7 +294,9 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) if (!(vma->vm_flags & vm_flags)) { vma_end_read(vma); - goto lock_mmap; + count_vm_vma_lock_event(VMA_LOCK_SUCCESS); + fault = VM_FAULT_BADACCESS; + goto bad_area; } fault = handle_mm_fault(vma, addr, flags | FAULT_FLAG_VMA_LOCK, regs); if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED))) From 0cec9541dcc550ce4a710c9a789f5f24e7c1d66d Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 3 Apr 2024 16:38:02 +0800 Subject: [PATCH 0754/1702] powerpc: mm: accelerate pagefault when badaccess The access_[pkey]_error() of vma already checked under per-VMA lock, if it is a bad access, directly handle error, no need to retry with mmap_lock again. In order to release the correct lock, pass the mm_struct into bad_access_pkey()/bad_access(), if mm is NULL, release vma lock, or release mmap_lock. Since the page faut is handled under per-VMA lock, count it as a vma lock event with VMA_LOCK_SUCCESS. Link: https://lkml.kernel.org/r/20240403083805.1818160-5-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Acked-by: Michael Ellerman (powerpc) Cc: Albert Ou Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: Gerald Schaefer Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Peter Zijlstra Cc: Russell King Cc: Suren Baghdasaryan Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/powerpc/mm/fault.c | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index 53335ae21a40a..2156904524959 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -71,23 +71,26 @@ static noinline int bad_area_nosemaphore(struct pt_regs *regs, unsigned long add return __bad_area_nosemaphore(regs, address, SEGV_MAPERR); } -static int __bad_area(struct pt_regs *regs, unsigned long address, int si_code) +static int __bad_area(struct pt_regs *regs, unsigned long address, int si_code, + struct mm_struct *mm, struct vm_area_struct *vma) { - struct mm_struct *mm = current->mm; /* * Something tried to access memory that isn't in our memory map.. * Fix it, but check if it's kernel or user first.. */ - mmap_read_unlock(mm); + if (mm) + mmap_read_unlock(mm); + else + vma_end_read(vma); return __bad_area_nosemaphore(regs, address, si_code); } static noinline int bad_access_pkey(struct pt_regs *regs, unsigned long address, + struct mm_struct *mm, struct vm_area_struct *vma) { - struct mm_struct *mm = current->mm; int pkey; /* @@ -109,7 +112,10 @@ static noinline int bad_access_pkey(struct pt_regs *regs, unsigned long address, */ pkey = vma_pkey(vma); - mmap_read_unlock(mm); + if (mm) + mmap_read_unlock(mm); + else + vma_end_read(vma); /* * If we are in kernel mode, bail out with a SEGV, this will @@ -124,9 +130,10 @@ static noinline int bad_access_pkey(struct pt_regs *regs, unsigned long address, return 0; } -static noinline int bad_access(struct pt_regs *regs, unsigned long address) +static noinline int bad_access(struct pt_regs *regs, unsigned long address, + struct mm_struct *mm, struct vm_area_struct *vma) { - return __bad_area(regs, address, SEGV_ACCERR); + return __bad_area(regs, address, SEGV_ACCERR, mm, vma); } static int do_sigbus(struct pt_regs *regs, unsigned long address, @@ -479,13 +486,13 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, if (unlikely(access_pkey_error(is_write, is_exec, (error_code & DSISR_KEYFAULT), vma))) { - vma_end_read(vma); - goto lock_mmap; + count_vm_vma_lock_event(VMA_LOCK_SUCCESS); + return bad_access_pkey(regs, address, NULL, vma); } if (unlikely(access_error(is_write, is_exec, vma))) { - vma_end_read(vma); - goto lock_mmap; + count_vm_vma_lock_event(VMA_LOCK_SUCCESS); + return bad_access(regs, address, NULL, vma); } fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs); @@ -521,10 +528,10 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, if (unlikely(access_pkey_error(is_write, is_exec, (error_code & DSISR_KEYFAULT), vma))) - return bad_access_pkey(regs, address, vma); + return bad_access_pkey(regs, address, mm, vma); if (unlikely(access_error(is_write, is_exec, vma))) - return bad_access(regs, address); + return bad_access(regs, address, mm, vma); /* * If for any reason at all we couldn't handle the fault, From cd1c91b8548372ced64a01eb6eef67b539506238 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 3 Apr 2024 16:38:03 +0800 Subject: [PATCH 0755/1702] riscv: mm: accelerate pagefault when badaccess The access_error() of vma already checked under per-VMA lock, if it is a bad access, directly handle error, no need to retry with mmap_lock again. Since the page faut is handled under per-VMA lock, count it as a vma lock event with VMA_LOCK_SUCCESS. [wangkefeng.wang@huawei.com: use `cause' rather than SIGSEGV, per Alexandre] Link: https://lkml.kernel.org/r/ac978061-ce1a-40a4-8b0a-61883b42bea7@huawei.com Link: https://lkml.kernel.org/r/20240403083805.1818160-6-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviewed-by: Suren Baghdasaryan Reviewed-by: Alexandre Ghiti Tested-by: Alexandre Ghiti Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: Gerald Schaefer Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Peter Zijlstra Cc: Russell King Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/riscv/mm/fault.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index 3ba1d4dde5dd1..5224f37338022 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -292,7 +292,10 @@ void handle_page_fault(struct pt_regs *regs) if (unlikely(access_error(cause, vma))) { vma_end_read(vma); - goto lock_mmap; + count_vm_vma_lock_event(VMA_LOCK_SUCCESS); + tsk->thread.bad_cause = cause; + bad_area_nosemaphore(regs, SEGV_ACCERR, addr); + return; } fault = handle_mm_fault(vma, addr, flags | FAULT_FLAG_VMA_LOCK, regs); From 82b7a618397c80736506c05d9add4aaea91297e8 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 3 Apr 2024 16:38:04 +0800 Subject: [PATCH 0756/1702] s390: mm: accelerate pagefault when badaccess The vm_flags of vma already checked under per-VMA lock, if it is a bad access, directly handle error, no need to retry with mmap_lock again. Since the page faut is handled under per-VMA lock, count it as a vma lock event with VMA_LOCK_SUCCESS. Link: https://lkml.kernel.org/r/20240403083805.1818160-7-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviewed-by: Heiko Carstens Cc: Albert Ou Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: Gerald Schaefer Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Peter Zijlstra Cc: Russell King Cc: Suren Baghdasaryan Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/s390/mm/fault.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 0c66b32e0f9f1..65747f15dbec4 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -325,7 +325,8 @@ static void do_exception(struct pt_regs *regs, int access) goto lock_mmap; if (!(vma->vm_flags & access)) { vma_end_read(vma); - goto lock_mmap; + count_vm_vma_lock_event(VMA_LOCK_SUCCESS); + return handle_fault_error_nolock(regs, SEGV_ACCERR); } fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs); if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED))) From bc7996c864bf58102f640474e04ec5ab04911ac1 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 3 Apr 2024 16:38:05 +0800 Subject: [PATCH 0757/1702] x86: mm: accelerate pagefault when badaccess The access_error() of vma is already checked under per-VMA lock, if it is a bad access, directly handle error, no need to retry with mmap_lock again. In order to release the correct lock, pass the mm_struct into bad_area_access_error(). If mm is NULL, release vma lock, or release mmap_lock. Since the page faut is handled under per-VMA lock, count it as a vma lock event with VMA_LOCK_SUCCESS. Link: https://lkml.kernel.org/r/20240403083805.1818160-8-wangkefeng.wang@huawei.com Reviewed-by: Suren Baghdasaryan Signed-off-by: Kefeng Wang Cc: Albert Ou Cc: Alexander Gordeev Cc: Andy Lutomirski Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: Gerald Schaefer Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Peter Zijlstra Cc: Russell King Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/x86/mm/fault.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index a4cc20d0036d4..67b18adc75ddc 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -866,14 +866,17 @@ bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code, static void __bad_area(struct pt_regs *regs, unsigned long error_code, - unsigned long address, u32 pkey, int si_code) + unsigned long address, struct mm_struct *mm, + struct vm_area_struct *vma, u32 pkey, int si_code) { - struct mm_struct *mm = current->mm; /* * Something tried to access memory that isn't in our memory map.. * Fix it, but check if it's kernel or user first.. */ - mmap_read_unlock(mm); + if (mm) + mmap_read_unlock(mm); + else + vma_end_read(vma); __bad_area_nosemaphore(regs, error_code, address, pkey, si_code); } @@ -897,7 +900,8 @@ static inline bool bad_area_access_from_pkeys(unsigned long error_code, static noinline void bad_area_access_error(struct pt_regs *regs, unsigned long error_code, - unsigned long address, struct vm_area_struct *vma) + unsigned long address, struct mm_struct *mm, + struct vm_area_struct *vma) { /* * This OSPKE check is not strictly necessary at runtime. @@ -927,9 +931,9 @@ bad_area_access_error(struct pt_regs *regs, unsigned long error_code, */ u32 pkey = vma_pkey(vma); - __bad_area(regs, error_code, address, pkey, SEGV_PKUERR); + __bad_area(regs, error_code, address, mm, vma, pkey, SEGV_PKUERR); } else { - __bad_area(regs, error_code, address, 0, SEGV_ACCERR); + __bad_area(regs, error_code, address, mm, vma, 0, SEGV_ACCERR); } } @@ -1357,8 +1361,9 @@ void do_user_addr_fault(struct pt_regs *regs, goto lock_mmap; if (unlikely(access_error(error_code, vma))) { - vma_end_read(vma); - goto lock_mmap; + bad_area_access_error(regs, error_code, address, NULL, vma); + count_vm_vma_lock_event(VMA_LOCK_SUCCESS); + return; } fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs); if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED))) @@ -1394,7 +1399,7 @@ void do_user_addr_fault(struct pt_regs *regs, * we can handle it.. */ if (unlikely(access_error(error_code, vma))) { - bad_area_access_error(regs, error_code, address, vma); + bad_area_access_error(regs, error_code, address, mm, vma); return; } From 4c773a44257f1f8a88ebc2c1bf67799bb4d6f409 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 2 Apr 2024 21:16:57 +0100 Subject: [PATCH 0758/1702] mm: remove struct page from get_shadow_from_swap_cache We don't actually use any parts of struct page; all we do is check the value of the pointer. So give the pointer the appropriate name & type. Link: https://lkml.kernel.org/r/20240402201659.918308-1-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- mm/swap_state.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mm/swap_state.c b/mm/swap_state.c index 2deac23633cdd..642c30d8376c6 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -73,11 +73,11 @@ void *get_shadow_from_swap_cache(swp_entry_t entry) { struct address_space *address_space = swap_address_space(entry); pgoff_t idx = swp_offset(entry); - struct page *page; + void *shadow; - page = xa_load(&address_space->i_pages, idx); - if (xa_is_value(page)) - return page; + shadow = xa_load(&address_space->i_pages, idx); + if (xa_is_value(shadow)) + return shadow; return NULL; } From f6a8dd98a2ce7ace0be59d77868751131af6d2f0 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 2 Apr 2024 21:06:54 +0100 Subject: [PATCH 0759/1702] hugetlb: convert alloc_buddy_hugetlb_folio to use a folio While this function returned a folio, it was still using __alloc_pages() and __free_pages(). Use __folio_alloc() and put_folio() instead. This actually removes a call to compound_head(), but more importantly, it prepares us for the move to memdescs. Link: https://lkml.kernel.org/r/20240402200656.913841-1-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Sidhartha Kumar Reviewed-by: Oscar Salvador Reviewed-by: Muchun Song Signed-off-by: Andrew Morton --- mm/hugetlb.c | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 10b4f8bde16fd..49d504af12013 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2177,13 +2177,13 @@ static struct folio *alloc_buddy_hugetlb_folio(struct hstate *h, nodemask_t *node_alloc_noretry) { int order = huge_page_order(h); - struct page *page; + struct folio *folio; bool alloc_try_hard = true; bool retry = true; /* - * By default we always try hard to allocate the page with - * __GFP_RETRY_MAYFAIL flag. However, if we are allocating pages in + * By default we always try hard to allocate the folio with + * __GFP_RETRY_MAYFAIL flag. However, if we are allocating folios in * a loop (to adjust global huge page counts) and previous allocation * failed, do not continue to try hard on the same node. Use the * node_alloc_noretry bitmap to manage this state information. @@ -2196,43 +2196,42 @@ static struct folio *alloc_buddy_hugetlb_folio(struct hstate *h, if (nid == NUMA_NO_NODE) nid = numa_mem_id(); retry: - page = __alloc_pages(gfp_mask, order, nid, nmask); + folio = __folio_alloc(gfp_mask, order, nid, nmask); - /* Freeze head page */ - if (page && !page_ref_freeze(page, 1)) { - __free_pages(page, order); + if (folio && !folio_ref_freeze(folio, 1)) { + folio_put(folio); if (retry) { /* retry once */ retry = false; goto retry; } /* WOW! twice in a row. */ - pr_warn("HugeTLB head page unexpected inflated ref count\n"); - page = NULL; + pr_warn("HugeTLB unexpected inflated folio ref count\n"); + folio = NULL; } /* - * If we did not specify __GFP_RETRY_MAYFAIL, but still got a page this - * indicates an overall state change. Clear bit so that we resume - * normal 'try hard' allocations. + * If we did not specify __GFP_RETRY_MAYFAIL, but still got a + * folio this indicates an overall state change. Clear bit so + * that we resume normal 'try hard' allocations. */ - if (node_alloc_noretry && page && !alloc_try_hard) + if (node_alloc_noretry && folio && !alloc_try_hard) node_clear(nid, *node_alloc_noretry); /* - * If we tried hard to get a page but failed, set bit so that + * If we tried hard to get a folio but failed, set bit so that * subsequent attempts will not try as hard until there is an * overall state change. */ - if (node_alloc_noretry && !page && alloc_try_hard) + if (node_alloc_noretry && !folio && alloc_try_hard) node_set(nid, *node_alloc_noretry); - if (!page) { + if (!folio) { __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL); return NULL; } __count_vm_event(HTLB_BUDDY_PGALLOC); - return page_folio(page); + return folio; } static struct folio *__alloc_fresh_hugetlb_folio(struct hstate *h, From 23babe1934d7637b598e4c9d9f3876e318fa63a4 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 2 Apr 2024 14:55:14 +0200 Subject: [PATCH 0760/1702] mm/gup: consistently name GUP-fast functions Patch series "mm/gup: consistently call it GUP-fast". Some cleanups around function names, comments and the config option of "GUP-fast" -- GUP without "lock" safety belts on. With this cleanup it's easy to judge which functions are GUP-fast specific. We now consistently call it "GUP-fast", avoiding mixing it with "fast GUP", "lockless", or simply "gup" (which I always considered confusing in the ode). So the magic now happens in functions that contain "gup_fast", whereby gup_fast() is the entry point into that magic. Comments consistently reference either "GUP-fast" or "gup_fast()". This patch (of 3): Let's consistently call the "fast-only" part of GUP "GUP-fast" and rename all relevant internal functions to start with "gup_fast", to make it clearer that this is not ordinary GUP. The current mixture of "lockless", "gup" and "gup_fast" is confusing. Further, avoid the term "huge" when talking about a "leaf" -- for example, we nowadays check pmd_leaf() because pmd_huge() is gone. For the "hugepd"/"hugepte" stuff, it's part of the name ("is_hugepd"), so that stays. What remains is the "external" interface: * get_user_pages_fast_only() * get_user_pages_fast() * pin_user_pages_fast() The high-level internal functions for GUP-fast (+slow fallback) are now: * internal_get_user_pages_fast() -> gup_fast_fallback() * lockless_pages_from_mm() -> gup_fast() The basic GUP-fast walker functions: * gup_pgd_range() -> gup_fast_pgd_range() * gup_p4d_range() -> gup_fast_p4d_range() * gup_pud_range() -> gup_fast_pud_range() * gup_pmd_range() -> gup_fast_pmd_range() * gup_pte_range() -> gup_fast_pte_range() * gup_huge_pgd() -> gup_fast_pgd_leaf() * gup_huge_pud() -> gup_fast_pud_leaf() * gup_huge_pmd() -> gup_fast_pmd_leaf() The weird hugepd stuff: * gup_huge_pd() -> gup_fast_hugepd() * gup_hugepte() -> gup_fast_hugepte() The weird devmap stuff: * __gup_device_huge_pud() -> gup_fast_devmap_pud_leaf() * __gup_device_huge_pmd -> gup_fast_devmap_pmd_leaf() * __gup_device_huge() -> gup_fast_devmap_leaf() * undo_dev_pagemap() -> gup_fast_undo_dev_pagemap() Helper functions: * unpin_user_pages_lockless() -> gup_fast_unpin_user_pages() * gup_fast_folio_allowed() is already properly named * gup_fast_permitted() is already properly named With "gup_fast()", we now even have a function that is referred to in comment in mm/mmu_gather.c. Link: https://lkml.kernel.org/r/20240402125516.223131-1-david@redhat.com Link: https://lkml.kernel.org/r/20240402125516.223131-2-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Jason Gunthorpe Reviewed-by: Mike Rapoport (IBM) Reviewed-by: John Hubbard Cc: Peter Xu Signed-off-by: Andrew Morton --- mm/gup.c | 205 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 103 insertions(+), 102 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index fbf80e16bc949..7609efc55f1c3 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -440,7 +440,7 @@ void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages, } EXPORT_SYMBOL(unpin_user_page_range_dirty_lock); -static void unpin_user_pages_lockless(struct page **pages, unsigned long npages) +static void gup_fast_unpin_user_pages(struct page **pages, unsigned long npages) { unsigned long i; struct folio *folio; @@ -525,9 +525,9 @@ static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end, return (__boundary - 1 < end - 1) ? __boundary : end; } -static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, - unsigned long end, unsigned int flags, - struct page **pages, int *nr) +static int gup_fast_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages, + int *nr) { unsigned long pte_end; struct page *page; @@ -577,7 +577,7 @@ static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, * of the other folios. See writable_file_mapping_allowed() and * gup_fast_folio_allowed() for more information. */ -static int gup_huge_pd(hugepd_t hugepd, unsigned long addr, +static int gup_fast_hugepd(hugepd_t hugepd, unsigned long addr, unsigned int pdshift, unsigned long end, unsigned int flags, struct page **pages, int *nr) { @@ -588,7 +588,7 @@ static int gup_huge_pd(hugepd_t hugepd, unsigned long addr, ptep = hugepte_offset(hugepd, addr, pdshift); do { next = hugepte_addr_end(addr, end, sz); - if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr)) + if (!gup_fast_hugepte(ptep, sz, addr, end, flags, pages, nr)) return 0; } while (ptep++, addr = next, addr != end); @@ -613,8 +613,8 @@ static struct page *follow_hugepd(struct vm_area_struct *vma, hugepd_t hugepd, h = hstate_vma(vma); ptep = hugepte_offset(hugepd, addr, pdshift); ptl = huge_pte_lock(h, vma->vm_mm, ptep); - ret = gup_huge_pd(hugepd, addr, pdshift, addr + PAGE_SIZE, - flags, &page, &nr); + ret = gup_fast_hugepd(hugepd, addr, pdshift, addr + PAGE_SIZE, + flags, &page, &nr); spin_unlock(ptl); if (ret) { @@ -626,7 +626,7 @@ static struct page *follow_hugepd(struct vm_area_struct *vma, hugepd_t hugepd, return NULL; } #else /* CONFIG_ARCH_HAS_HUGEPD */ -static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr, +static inline int gup_fast_hugepd(hugepd_t hugepd, unsigned long addr, unsigned int pdshift, unsigned long end, unsigned int flags, struct page **pages, int *nr) { @@ -2750,7 +2750,7 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, EXPORT_SYMBOL(get_user_pages_unlocked); /* - * Fast GUP + * GUP-fast * * get_user_pages_fast attempts to pin user pages by walking the page * tables directly and avoids taking locks. Thus the walker needs to be @@ -2764,7 +2764,7 @@ EXPORT_SYMBOL(get_user_pages_unlocked); * * Another way to achieve this is to batch up page table containing pages * belonging to more than one mm_user, then rcu_sched a callback to free those - * pages. Disabling interrupts will allow the fast_gup walker to both block + * pages. Disabling interrupts will allow the gup_fast() walker to both block * the rcu_sched callback, and an IPI that we broadcast for splitting THPs * (which is a relatively rare event). The code below adopts this strategy. * @@ -2873,9 +2873,8 @@ static bool gup_fast_folio_allowed(struct folio *folio, unsigned int flags) return !reject_file_backed || shmem_mapping(mapping); } -static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start, - unsigned int flags, - struct page **pages) +static void __maybe_unused gup_fast_undo_dev_pagemap(int *nr, int nr_start, + unsigned int flags, struct page **pages) { while ((*nr) - nr_start) { struct page *page = pages[--(*nr)]; @@ -2890,27 +2889,27 @@ static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start, #ifdef CONFIG_ARCH_HAS_PTE_SPECIAL /* - * Fast-gup relies on pte change detection to avoid concurrent pgtable + * GUP-fast relies on pte change detection to avoid concurrent pgtable * operations. * - * To pin the page, fast-gup needs to do below in order: + * To pin the page, GUP-fast needs to do below in order: * (1) pin the page (by prefetching pte), then (2) check pte not changed. * * For the rest of pgtable operations where pgtable updates can be racy - * with fast-gup, we need to do (1) clear pte, then (2) check whether page + * with GUP-fast, we need to do (1) clear pte, then (2) check whether page * is pinned. * * Above will work for all pte-level operations, including THP split. * - * For THP collapse, it's a bit more complicated because fast-gup may be + * For THP collapse, it's a bit more complicated because GUP-fast may be * walking a pgtable page that is being freed (pte is still valid but pmd * can be cleared already). To avoid race in such condition, we need to * also check pmd here to make sure pmd doesn't change (corresponds to * pmdp_collapse_flush() in the THP collapse code path). */ -static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, - unsigned long end, unsigned int flags, - struct page **pages, int *nr) +static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages, + int *nr) { struct dev_pagemap *pgmap = NULL; int nr_start = *nr, ret = 0; @@ -2943,7 +2942,7 @@ static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, pgmap = get_dev_pagemap(pte_pfn(pte), pgmap); if (unlikely(!pgmap)) { - undo_dev_pagemap(nr, nr_start, flags, pages); + gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages); goto pte_unmap; } } else if (pte_special(pte)) @@ -3007,20 +3006,19 @@ static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, * * For a futex to be placed on a THP tail page, get_futex_key requires a * get_user_pages_fast_only implementation that can pin pages. Thus it's still - * useful to have gup_huge_pmd even if we can't operate on ptes. + * useful to have gup_fast_pmd_leaf even if we can't operate on ptes. */ -static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, - unsigned long end, unsigned int flags, - struct page **pages, int *nr) +static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages, + int *nr) { return 0; } #endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */ #if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE) -static int __gup_device_huge(unsigned long pfn, unsigned long addr, - unsigned long end, unsigned int flags, - struct page **pages, int *nr) +static int gup_fast_devmap_leaf(unsigned long pfn, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages, int *nr) { int nr_start = *nr; struct dev_pagemap *pgmap = NULL; @@ -3030,19 +3028,19 @@ static int __gup_device_huge(unsigned long pfn, unsigned long addr, pgmap = get_dev_pagemap(pfn, pgmap); if (unlikely(!pgmap)) { - undo_dev_pagemap(nr, nr_start, flags, pages); + gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages); break; } if (!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(page)) { - undo_dev_pagemap(nr, nr_start, flags, pages); + gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages); break; } SetPageReferenced(page); pages[*nr] = page; if (unlikely(try_grab_page(page, flags))) { - undo_dev_pagemap(nr, nr_start, flags, pages); + gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages); break; } (*nr)++; @@ -3053,62 +3051,62 @@ static int __gup_device_huge(unsigned long pfn, unsigned long addr, return addr == end; } -static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, - unsigned long end, unsigned int flags, - struct page **pages, int *nr) +static int gup_fast_devmap_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages, + int *nr) { unsigned long fault_pfn; int nr_start = *nr; fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); - if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr)) + if (!gup_fast_devmap_leaf(fault_pfn, addr, end, flags, pages, nr)) return 0; if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) { - undo_dev_pagemap(nr, nr_start, flags, pages); + gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages); return 0; } return 1; } -static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, - unsigned long end, unsigned int flags, - struct page **pages, int *nr) +static int gup_fast_devmap_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages, + int *nr) { unsigned long fault_pfn; int nr_start = *nr; fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); - if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr)) + if (!gup_fast_devmap_leaf(fault_pfn, addr, end, flags, pages, nr)) return 0; if (unlikely(pud_val(orig) != pud_val(*pudp))) { - undo_dev_pagemap(nr, nr_start, flags, pages); + gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages); return 0; } return 1; } #else -static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, - unsigned long end, unsigned int flags, - struct page **pages, int *nr) +static int gup_fast_devmap_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages, + int *nr) { BUILD_BUG(); return 0; } -static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr, - unsigned long end, unsigned int flags, - struct page **pages, int *nr) +static int gup_fast_devmap_pud_leaf(pud_t pud, pud_t *pudp, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages, + int *nr) { BUILD_BUG(); return 0; } #endif -static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, - unsigned long end, unsigned int flags, - struct page **pages, int *nr) +static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages, + int *nr) { struct page *page; struct folio *folio; @@ -3120,8 +3118,8 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, if (pmd_devmap(orig)) { if (unlikely(flags & FOLL_LONGTERM)) return 0; - return __gup_device_huge_pmd(orig, pmdp, addr, end, flags, - pages, nr); + return gup_fast_devmap_pmd_leaf(orig, pmdp, addr, end, flags, + pages, nr); } page = pmd_page(orig); @@ -3150,9 +3148,9 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, return 1; } -static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, - unsigned long end, unsigned int flags, - struct page **pages, int *nr) +static int gup_fast_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages, + int *nr) { struct page *page; struct folio *folio; @@ -3164,8 +3162,8 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, if (pud_devmap(orig)) { if (unlikely(flags & FOLL_LONGTERM)) return 0; - return __gup_device_huge_pud(orig, pudp, addr, end, flags, - pages, nr); + return gup_fast_devmap_pud_leaf(orig, pudp, addr, end, flags, + pages, nr); } page = pud_page(orig); @@ -3195,9 +3193,9 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, return 1; } -static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr, - unsigned long end, unsigned int flags, - struct page **pages, int *nr) +static int gup_fast_pgd_leaf(pgd_t orig, pgd_t *pgdp, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages, + int *nr) { int refs; struct page *page; @@ -3235,8 +3233,9 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr, return 1; } -static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned long end, - unsigned int flags, struct page **pages, int *nr) +static int gup_fast_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages, + int *nr) { unsigned long next; pmd_t *pmdp; @@ -3250,11 +3249,11 @@ static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned lo return 0; if (unlikely(pmd_leaf(pmd))) { - /* See gup_pte_range() */ + /* See gup_fast_pte_range() */ if (pmd_protnone(pmd)) return 0; - if (!gup_huge_pmd(pmd, pmdp, addr, next, flags, + if (!gup_fast_pmd_leaf(pmd, pmdp, addr, next, flags, pages, nr)) return 0; @@ -3263,18 +3262,20 @@ static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned lo * architecture have different format for hugetlbfs * pmd format and THP pmd format */ - if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr, - PMD_SHIFT, next, flags, pages, nr)) + if (!gup_fast_hugepd(__hugepd(pmd_val(pmd)), addr, + PMD_SHIFT, next, flags, pages, nr)) return 0; - } else if (!gup_pte_range(pmd, pmdp, addr, next, flags, pages, nr)) + } else if (!gup_fast_pte_range(pmd, pmdp, addr, next, flags, + pages, nr)) return 0; } while (pmdp++, addr = next, addr != end); return 1; } -static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned long end, - unsigned int flags, struct page **pages, int *nr) +static int gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages, + int *nr) { unsigned long next; pud_t *pudp; @@ -3287,22 +3288,24 @@ static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned lo if (unlikely(!pud_present(pud))) return 0; if (unlikely(pud_leaf(pud))) { - if (!gup_huge_pud(pud, pudp, addr, next, flags, - pages, nr)) + if (!gup_fast_pud_leaf(pud, pudp, addr, next, flags, + pages, nr)) return 0; } else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) { - if (!gup_huge_pd(__hugepd(pud_val(pud)), addr, - PUD_SHIFT, next, flags, pages, nr)) + if (!gup_fast_hugepd(__hugepd(pud_val(pud)), addr, + PUD_SHIFT, next, flags, pages, nr)) return 0; - } else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr)) + } else if (!gup_fast_pmd_range(pudp, pud, addr, next, flags, + pages, nr)) return 0; } while (pudp++, addr = next, addr != end); return 1; } -static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned long end, - unsigned int flags, struct page **pages, int *nr) +static int gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, + unsigned long end, unsigned int flags, struct page **pages, + int *nr) { unsigned long next; p4d_t *p4dp; @@ -3316,17 +3319,18 @@ static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned lo return 0; BUILD_BUG_ON(p4d_leaf(p4d)); if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) { - if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr, - P4D_SHIFT, next, flags, pages, nr)) + if (!gup_fast_hugepd(__hugepd(p4d_val(p4d)), addr, + P4D_SHIFT, next, flags, pages, nr)) return 0; - } else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr)) + } else if (!gup_fast_pud_range(p4dp, p4d, addr, next, flags, + pages, nr)) return 0; } while (p4dp++, addr = next, addr != end); return 1; } -static void gup_pgd_range(unsigned long addr, unsigned long end, +static void gup_fast_pgd_range(unsigned long addr, unsigned long end, unsigned int flags, struct page **pages, int *nr) { unsigned long next; @@ -3340,19 +3344,20 @@ static void gup_pgd_range(unsigned long addr, unsigned long end, if (pgd_none(pgd)) return; if (unlikely(pgd_leaf(pgd))) { - if (!gup_huge_pgd(pgd, pgdp, addr, next, flags, - pages, nr)) + if (!gup_fast_pgd_leaf(pgd, pgdp, addr, next, flags, + pages, nr)) return; } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) { - if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr, - PGDIR_SHIFT, next, flags, pages, nr)) + if (!gup_fast_hugepd(__hugepd(pgd_val(pgd)), addr, + PGDIR_SHIFT, next, flags, pages, nr)) return; - } else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr)) + } else if (!gup_fast_p4d_range(pgdp, pgd, addr, next, flags, + pages, nr)) return; } while (pgdp++, addr = next, addr != end); } #else -static inline void gup_pgd_range(unsigned long addr, unsigned long end, +static inline void gup_fast_pgd_range(unsigned long addr, unsigned long end, unsigned int flags, struct page **pages, int *nr) { } @@ -3369,10 +3374,8 @@ static bool gup_fast_permitted(unsigned long start, unsigned long end) } #endif -static unsigned long lockless_pages_from_mm(unsigned long start, - unsigned long end, - unsigned int gup_flags, - struct page **pages) +static unsigned long gup_fast(unsigned long start, unsigned long end, + unsigned int gup_flags, struct page **pages) { unsigned long flags; int nr_pinned = 0; @@ -3400,16 +3403,16 @@ static unsigned long lockless_pages_from_mm(unsigned long start, * that come from THPs splitting. */ local_irq_save(flags); - gup_pgd_range(start, end, gup_flags, pages, &nr_pinned); + gup_fast_pgd_range(start, end, gup_flags, pages, &nr_pinned); local_irq_restore(flags); /* * When pinning pages for DMA there could be a concurrent write protect - * from fork() via copy_page_range(), in this case always fail fast GUP. + * from fork() via copy_page_range(), in this case always fail GUP-fast. */ if (gup_flags & FOLL_PIN) { if (read_seqcount_retry(¤t->mm->write_protect_seq, seq)) { - unpin_user_pages_lockless(pages, nr_pinned); + gup_fast_unpin_user_pages(pages, nr_pinned); return 0; } else { sanity_check_pinned_pages(pages, nr_pinned); @@ -3418,10 +3421,8 @@ static unsigned long lockless_pages_from_mm(unsigned long start, return nr_pinned; } -static int internal_get_user_pages_fast(unsigned long start, - unsigned long nr_pages, - unsigned int gup_flags, - struct page **pages) +static int gup_fast_fallback(unsigned long start, unsigned long nr_pages, + unsigned int gup_flags, struct page **pages) { unsigned long len, end; unsigned long nr_pinned; @@ -3449,7 +3450,7 @@ static int internal_get_user_pages_fast(unsigned long start, if (unlikely(!access_ok((void __user *)start, len))) return -EFAULT; - nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages); + nr_pinned = gup_fast(start, end, gup_flags, pages); if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY) return nr_pinned; @@ -3503,7 +3504,7 @@ int get_user_pages_fast_only(unsigned long start, int nr_pages, FOLL_GET | FOLL_FAST_ONLY)) return -EINVAL; - return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); + return gup_fast_fallback(start, nr_pages, gup_flags, pages); } EXPORT_SYMBOL_GPL(get_user_pages_fast_only); @@ -3534,7 +3535,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, */ if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_GET)) return -EINVAL; - return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); + return gup_fast_fallback(start, nr_pages, gup_flags, pages); } EXPORT_SYMBOL_GPL(get_user_pages_fast); @@ -3562,7 +3563,7 @@ int pin_user_pages_fast(unsigned long start, int nr_pages, { if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN)) return -EINVAL; - return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); + return gup_fast_fallback(start, nr_pages, gup_flags, pages); } EXPORT_SYMBOL_GPL(pin_user_pages_fast); From 25176ad09ca395fcc83b1fc78adf25c8eb1bd964 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 2 Apr 2024 14:55:15 +0200 Subject: [PATCH 0761/1702] mm/treewide: rename CONFIG_HAVE_FAST_GUP to CONFIG_HAVE_GUP_FAST Nowadays, we call it "GUP-fast", the external interface includes functions like "get_user_pages_fast()", and we renamed all internal functions to reflect that as well. Let's make the config option reflect that. Link: https://lkml.kernel.org/r/20240402125516.223131-3-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Mike Rapoport (IBM) Reviewed-by: Jason Gunthorpe Reviewed-by: John Hubbard Cc: Peter Xu Signed-off-by: Andrew Morton --- arch/arm/Kconfig | 2 +- arch/arm64/Kconfig | 2 +- arch/loongarch/Kconfig | 2 +- arch/mips/Kconfig | 2 +- arch/powerpc/Kconfig | 2 +- arch/riscv/Kconfig | 2 +- arch/s390/Kconfig | 2 +- arch/sh/Kconfig | 2 +- arch/x86/Kconfig | 2 +- include/linux/rmap.h | 8 ++++---- kernel/events/core.c | 4 ++-- mm/Kconfig | 2 +- mm/gup.c | 10 +++++----- mm/internal.h | 2 +- 14 files changed, 22 insertions(+), 22 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index b14aed3a17abb..817918f6635a5 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -99,7 +99,7 @@ config ARM select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE select HAVE_EFFICIENT_UNALIGNED_ACCESS if (CPU_V6 || CPU_V6K || CPU_V7) && MMU select HAVE_EXIT_THREAD - select HAVE_FAST_GUP if ARM_LPAE + select HAVE_GUP_FAST if ARM_LPAE select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL select HAVE_FUNCTION_ERROR_INJECTION select HAVE_FUNCTION_GRAPH_TRACER diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 7b11c98b3e84b..de076a191e9fd 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -205,7 +205,7 @@ config ARM64 select HAVE_SAMPLE_FTRACE_DIRECT select HAVE_SAMPLE_FTRACE_DIRECT_MULTI select HAVE_EFFICIENT_UNALIGNED_ACCESS - select HAVE_FAST_GUP + select HAVE_GUP_FAST select HAVE_FTRACE_MCOUNT_RECORD select HAVE_FUNCTION_TRACER select HAVE_FUNCTION_ERROR_INJECTION diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index a5f300ec6f280..cd642eefd9e51 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -119,7 +119,7 @@ config LOONGARCH select HAVE_EBPF_JIT select HAVE_EFFICIENT_UNALIGNED_ACCESS if !ARCH_STRICT_ALIGN select HAVE_EXIT_THREAD - select HAVE_FAST_GUP + select HAVE_GUP_FAST select HAVE_FTRACE_MCOUNT_RECORD select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_FUNCTION_ERROR_INJECTION diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 516dc7022bd74..f1aa1bf11166b 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -68,7 +68,7 @@ config MIPS select HAVE_DYNAMIC_FTRACE select HAVE_EBPF_JIT if !CPU_MICROMIPS select HAVE_EXIT_THREAD - select HAVE_FAST_GUP + select HAVE_GUP_FAST select HAVE_FTRACE_MCOUNT_RECORD select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 1c4be33736860..e42cc8cd415ff 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -236,7 +236,7 @@ config PPC select HAVE_DYNAMIC_FTRACE_WITH_REGS if ARCH_USING_PATCHABLE_FUNCTION_ENTRY || MPROFILE_KERNEL || PPC32 select HAVE_EBPF_JIT select HAVE_EFFICIENT_UNALIGNED_ACCESS - select HAVE_FAST_GUP + select HAVE_GUP_FAST select HAVE_FTRACE_MCOUNT_RECORD select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_FUNCTION_DESCRIPTORS if PPC64_ELF_ABI_V1 diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index be09c8836d56b..3ee60ddef93eb 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -132,7 +132,7 @@ config RISCV select HAVE_FUNCTION_GRAPH_RETVAL if HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !PREEMPTION select HAVE_EBPF_JIT if MMU - select HAVE_FAST_GUP if MMU + select HAVE_GUP_FAST if MMU select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_FUNCTION_ERROR_INJECTION select HAVE_GCC_PLUGINS diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 8f01ada6845e3..d9aed4c93ee67 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -174,7 +174,7 @@ config S390 select HAVE_DYNAMIC_FTRACE_WITH_REGS select HAVE_EBPF_JIT if HAVE_MARCH_Z196_FEATURES select HAVE_EFFICIENT_UNALIGNED_ACCESS - select HAVE_FAST_GUP + select HAVE_GUP_FAST select HAVE_FENTRY select HAVE_FTRACE_MCOUNT_RECORD select HAVE_FUNCTION_ARG_ACCESS_API diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index 2ad3e29f0ebec..7292542f75e8b 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -38,7 +38,7 @@ config SUPERH select HAVE_DEBUG_BUGVERBOSE select HAVE_DEBUG_KMEMLEAK select HAVE_DYNAMIC_FTRACE - select HAVE_FAST_GUP if MMU + select HAVE_GUP_FAST if MMU select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER select HAVE_FTRACE_MCOUNT_RECORD diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 4474bf32d0a49..d95de5eab213c 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -221,7 +221,7 @@ config X86 select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_EISA select HAVE_EXIT_THREAD - select HAVE_FAST_GUP + select HAVE_GUP_FAST select HAVE_FENTRY if X86_64 || DYNAMIC_FTRACE select HAVE_FTRACE_MCOUNT_RECORD select HAVE_FUNCTION_GRAPH_RETVAL if HAVE_FUNCTION_GRAPH_TRACER diff --git a/include/linux/rmap.h b/include/linux/rmap.h index b7944a833668a..9bf9324214fc3 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -284,7 +284,7 @@ static inline int hugetlb_try_share_anon_rmap(struct folio *folio) VM_WARN_ON_FOLIO(!PageAnonExclusive(&folio->page), folio); /* Paired with the memory barrier in try_grab_folio(). */ - if (IS_ENABLED(CONFIG_HAVE_FAST_GUP)) + if (IS_ENABLED(CONFIG_HAVE_GUP_FAST)) smp_mb(); if (unlikely(folio_maybe_dma_pinned(folio))) @@ -295,7 +295,7 @@ static inline int hugetlb_try_share_anon_rmap(struct folio *folio) * This is conceptually a smp_wmb() paired with the smp_rmb() in * gup_must_unshare(). */ - if (IS_ENABLED(CONFIG_HAVE_FAST_GUP)) + if (IS_ENABLED(CONFIG_HAVE_GUP_FAST)) smp_mb__after_atomic(); return 0; } @@ -541,7 +541,7 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, */ /* Paired with the memory barrier in try_grab_folio(). */ - if (IS_ENABLED(CONFIG_HAVE_FAST_GUP)) + if (IS_ENABLED(CONFIG_HAVE_GUP_FAST)) smp_mb(); if (unlikely(folio_maybe_dma_pinned(folio))) @@ -552,7 +552,7 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, * This is conceptually a smp_wmb() paired with the smp_rmb() in * gup_must_unshare(). */ - if (IS_ENABLED(CONFIG_HAVE_FAST_GUP)) + if (IS_ENABLED(CONFIG_HAVE_GUP_FAST)) smp_mb__after_atomic(); return 0; } diff --git a/kernel/events/core.c b/kernel/events/core.c index 724e6d7e128f3..c5a0dc1f135f0 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7539,7 +7539,7 @@ static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) { u64 size = 0; -#ifdef CONFIG_HAVE_FAST_GUP +#ifdef CONFIG_HAVE_GUP_FAST pgd_t *pgdp, pgd; p4d_t *p4dp, p4d; pud_t *pudp, pud; @@ -7587,7 +7587,7 @@ static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) if (pte_present(pte)) size = pte_leaf_size(pte); pte_unmap(ptep); -#endif /* CONFIG_HAVE_FAST_GUP */ +#endif /* CONFIG_HAVE_GUP_FAST */ return size; } diff --git a/mm/Kconfig b/mm/Kconfig index f0ed3168db004..50df323eaecef 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -473,7 +473,7 @@ config ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP config HAVE_MEMBLOCK_PHYS_MAP bool -config HAVE_FAST_GUP +config HAVE_GUP_FAST depends on MMU bool diff --git a/mm/gup.c b/mm/gup.c index 7609efc55f1c3..8dcbeae714e26 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -501,7 +501,7 @@ static inline void mm_set_has_pinned_flag(unsigned long *mm_flags) #ifdef CONFIG_MMU -#if defined(CONFIG_ARCH_HAS_HUGEPD) || defined(CONFIG_HAVE_FAST_GUP) +#if defined(CONFIG_ARCH_HAS_HUGEPD) || defined(CONFIG_HAVE_GUP_FAST) static int record_subpages(struct page *page, unsigned long sz, unsigned long addr, unsigned long end, struct page **pages) @@ -515,7 +515,7 @@ static int record_subpages(struct page *page, unsigned long sz, return nr; } -#endif /* CONFIG_ARCH_HAS_HUGEPD || CONFIG_HAVE_FAST_GUP */ +#endif /* CONFIG_ARCH_HAS_HUGEPD || CONFIG_HAVE_GUP_FAST */ #ifdef CONFIG_ARCH_HAS_HUGEPD static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end, @@ -2782,7 +2782,7 @@ EXPORT_SYMBOL(get_user_pages_unlocked); * * This code is based heavily on the PowerPC implementation by Nick Piggin. */ -#ifdef CONFIG_HAVE_FAST_GUP +#ifdef CONFIG_HAVE_GUP_FAST /* * Used in the GUP-fast path to determine whether GUP is permitted to work on @@ -3361,7 +3361,7 @@ static inline void gup_fast_pgd_range(unsigned long addr, unsigned long end, unsigned int flags, struct page **pages, int *nr) { } -#endif /* CONFIG_HAVE_FAST_GUP */ +#endif /* CONFIG_HAVE_GUP_FAST */ #ifndef gup_fast_permitted /* @@ -3381,7 +3381,7 @@ static unsigned long gup_fast(unsigned long start, unsigned long end, int nr_pinned = 0; unsigned seq; - if (!IS_ENABLED(CONFIG_HAVE_FAST_GUP) || + if (!IS_ENABLED(CONFIG_HAVE_GUP_FAST) || !gup_fast_permitted(start, end)) return 0; diff --git a/mm/internal.h b/mm/internal.h index 640298a98a3cd..8fd41f889a958 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1265,7 +1265,7 @@ static inline bool gup_must_unshare(struct vm_area_struct *vma, } /* Paired with a memory barrier in folio_try_share_anon_rmap_*(). */ - if (IS_ENABLED(CONFIG_HAVE_FAST_GUP)) + if (IS_ENABLED(CONFIG_HAVE_GUP_FAST)) smp_rmb(); /* From 0ae0b2b3255339827d9e04017874dfc93f1491c9 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 2 Apr 2024 14:55:16 +0200 Subject: [PATCH 0762/1702] mm: use "GUP-fast" instead "fast GUP" in remaining comments Let's fixup the remaining comments to consistently call that thing "GUP-fast". With this change, we consistently call it "GUP-fast". Link: https://lkml.kernel.org/r/20240402125516.223131-4-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Mike Rapoport (IBM) Reviewed-by: Jason Gunthorpe Reviewed-by: John Hubbard Cc: Peter Xu Signed-off-by: Andrew Morton --- mm/filemap.c | 2 +- mm/khugepaged.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 7b0b2229d4ed2..18f4bce3741ec 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1810,7 +1810,7 @@ EXPORT_SYMBOL(page_cache_prev_miss); * C. Return the page to the page allocator * * This means that any page may have its reference count temporarily - * increased by a speculative page cache (or fast GUP) lookup as it can + * increased by a speculative page cache (or GUP-fast) lookup as it can * be allocated by another user before the RCU grace period expires. * Because the refcount temporarily acquired here may end up being the * last refcount on the page, any page allocation must be freeable by diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 8380ee0d0410c..89e2624fb3ff1 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1153,7 +1153,7 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address, * huge and small TLB entries for the same virtual address to * avoid the risk of CPU bugs in that area. * - * Parallel fast GUP is fine since fast GUP will back off when + * Parallel GUP-fast is fine since GUP-fast will back off when * it detects PMD is changed. */ _pmd = pmdp_collapse_flush(vma, address, pmd); From 7edea4c6fdf23754c77582a0377791e1aa9d2700 Mon Sep 17 00:00:00 2001 From: Jinjiang Tu Date: Tue, 2 Apr 2024 10:49:34 +0800 Subject: [PATCH 0763/1702] mm/ksm: remove redundant code in ksm_fork Since commit 3c6f33b7273a ("mm/ksm: support fork/exec for prctl"), when a child process is forked, the MMF_VM_MERGE_ANY flag will be inherited in mm_init(). So, it's unnecessary to set the flag in ksm_fork(). Link: https://lkml.kernel.org/r/20240402024934.1093361-1-tujinjiang@huawei.com Signed-off-by: Jinjiang Tu Reviewed-by: David Hildenbrand Cc: Johannes Weiner Cc: Kefeng Wang Cc: Nanyong Sun Cc: Rik van Riel Cc: Stefan Roesch Signed-off-by: Andrew Morton --- include/linux/ksm.h | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 7e2b1de3996ac..358803cfd4d5e 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -45,16 +45,8 @@ static inline void ksm_might_unmap_zero_page(struct mm_struct *mm, pte_t pte) static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm) { - int ret; - - if (test_bit(MMF_VM_MERGEABLE, &oldmm->flags)) { - ret = __ksm_enter(mm); - if (ret) - return ret; - } - - if (test_bit(MMF_VM_MERGE_ANY, &oldmm->flags)) - set_bit(MMF_VM_MERGE_ANY, &mm->flags); + if (test_bit(MMF_VM_MERGEABLE, &oldmm->flags)) + return __ksm_enter(mm); return 0; } From 9b42fa16195fb471bf3b7849803c0a7d3e7620f2 Mon Sep 17 00:00:00 2001 From: "Vishal Moola (Oracle)" Date: Mon, 1 Apr 2024 13:26:49 -0700 Subject: [PATCH 0764/1702] hugetlb: convert hugetlb_fault() to use struct vm_fault Patch series "Hugetlb fault path to use struct vm_fault", v2. This patchset converts the hugetlb fault path to use struct vm_fault. This helps make the code more readable, and alleviates the stack by allowing us to consolidate many fault-related variables into an individual pointer. This patch (of 3): Now that hugetlb_fault() has a vm_fault available for fault tracking, use it throughout. This cleans up the code by removing 2 variables, and prepares hugetlb_fault() to take in a struct vm_fault argument. Link: https://lkml.kernel.org/r/20240401202651.31440-1-vishal.moola@gmail.com Link: https://lkml.kernel.org/r/20240401202651.31440-2-vishal.moola@gmail.com Signed-off-by: Vishal Moola (Oracle) Reviewed-by: Oscar Salvador Reviewed-by: Muchun Song Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/hugetlb.c | 84 +++++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 49d504af12013..c51b8bcbeae17 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6427,8 +6427,6 @@ u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx) vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, unsigned int flags) { - pte_t *ptep, entry; - spinlock_t *ptl; vm_fault_t ret; u32 hash; struct folio *folio = NULL; @@ -6436,13 +6434,13 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, struct hstate *h = hstate_vma(vma); struct address_space *mapping; int need_wait_lock = 0; - unsigned long haddr = address & huge_page_mask(h); struct vm_fault vmf = { .vma = vma, - .address = haddr, + .address = address & huge_page_mask(h), .real_address = address, .flags = flags, - .pgoff = vma_hugecache_offset(h, vma, haddr), + .pgoff = vma_hugecache_offset(h, vma, + address & huge_page_mask(h)), /* TODO: Track hugetlb faults using vm_fault */ /* @@ -6462,22 +6460,22 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, /* * Acquire vma lock before calling huge_pte_alloc and hold - * until finished with ptep. This prevents huge_pmd_unshare from - * being called elsewhere and making the ptep no longer valid. + * until finished with vmf.pte. This prevents huge_pmd_unshare from + * being called elsewhere and making the vmf.pte no longer valid. */ hugetlb_vma_lock_read(vma); - ptep = huge_pte_alloc(mm, vma, haddr, huge_page_size(h)); - if (!ptep) { + vmf.pte = huge_pte_alloc(mm, vma, vmf.address, huge_page_size(h)); + if (!vmf.pte) { hugetlb_vma_unlock_read(vma); mutex_unlock(&hugetlb_fault_mutex_table[hash]); return VM_FAULT_OOM; } - entry = huge_ptep_get(ptep); - if (huge_pte_none_mostly(entry)) { - if (is_pte_marker(entry)) { + vmf.orig_pte = huge_ptep_get(vmf.pte); + if (huge_pte_none_mostly(vmf.orig_pte)) { + if (is_pte_marker(vmf.orig_pte)) { pte_marker marker = - pte_marker_get(pte_to_swp_entry(entry)); + pte_marker_get(pte_to_swp_entry(vmf.orig_pte)); if (marker & PTE_MARKER_POISONED) { ret = VM_FAULT_HWPOISON_LARGE; @@ -6492,20 +6490,20 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, * mutex internally, which make us return immediately. */ return hugetlb_no_page(mm, vma, mapping, vmf.pgoff, address, - ptep, entry, flags, &vmf); + vmf.pte, vmf.orig_pte, flags, &vmf); } ret = 0; /* - * entry could be a migration/hwpoison entry at this point, so this - * check prevents the kernel from going below assuming that we have - * an active hugepage in pagecache. This goto expects the 2nd page - * fault, and is_hugetlb_entry_(migration|hwpoisoned) check will - * properly handle it. + * vmf.orig_pte could be a migration/hwpoison vmf.orig_pte at this + * point, so this check prevents the kernel from going below assuming + * that we have an active hugepage in pagecache. This goto expects + * the 2nd page fault, and is_hugetlb_entry_(migration|hwpoisoned) + * check will properly handle it. */ - if (!pte_present(entry)) { - if (unlikely(is_hugetlb_entry_migration(entry))) { + if (!pte_present(vmf.orig_pte)) { + if (unlikely(is_hugetlb_entry_migration(vmf.orig_pte))) { /* * Release the hugetlb fault lock now, but retain * the vma lock, because it is needed to guard the @@ -6514,9 +6512,9 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, * be released there. */ mutex_unlock(&hugetlb_fault_mutex_table[hash]); - migration_entry_wait_huge(vma, ptep); + migration_entry_wait_huge(vma, vmf.pte); return 0; - } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) + } else if (unlikely(is_hugetlb_entry_hwpoisoned(vmf.orig_pte))) ret = VM_FAULT_HWPOISON_LARGE | VM_FAULT_SET_HINDEX(hstate_index(h)); goto out_mutex; @@ -6530,13 +6528,13 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, * determine if a reservation has been consumed. */ if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) && - !(vma->vm_flags & VM_MAYSHARE) && !huge_pte_write(entry)) { - if (vma_needs_reservation(h, vma, haddr) < 0) { + !(vma->vm_flags & VM_MAYSHARE) && !huge_pte_write(vmf.orig_pte)) { + if (vma_needs_reservation(h, vma, vmf.address) < 0) { ret = VM_FAULT_OOM; goto out_mutex; } /* Just decrements count, does not deallocate */ - vma_end_reservation(h, vma, haddr); + vma_end_reservation(h, vma, vmf.address); pagecache_folio = filemap_lock_hugetlb_folio(h, mapping, vmf.pgoff); @@ -6544,17 +6542,17 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, pagecache_folio = NULL; } - ptl = huge_pte_lock(h, mm, ptep); + vmf.ptl = huge_pte_lock(h, mm, vmf.pte); /* Check for a racing update before calling hugetlb_wp() */ - if (unlikely(!pte_same(entry, huge_ptep_get(ptep)))) + if (unlikely(!pte_same(vmf.orig_pte, huge_ptep_get(vmf.pte)))) goto out_ptl; /* Handle userfault-wp first, before trying to lock more pages */ - if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(ptep)) && - (flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) { + if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(vmf.pte)) && + (flags & FAULT_FLAG_WRITE) && !huge_pte_write(vmf.orig_pte)) { if (!userfaultfd_wp_async(vma)) { - spin_unlock(ptl); + spin_unlock(vmf.ptl); if (pagecache_folio) { folio_unlock(pagecache_folio); folio_put(pagecache_folio); @@ -6564,18 +6562,18 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, return handle_userfault(&vmf, VM_UFFD_WP); } - entry = huge_pte_clear_uffd_wp(entry); - set_huge_pte_at(mm, haddr, ptep, entry, + vmf.orig_pte = huge_pte_clear_uffd_wp(vmf.orig_pte); + set_huge_pte_at(mm, vmf.address, vmf.pte, vmf.orig_pte, huge_page_size(hstate_vma(vma))); /* Fallthrough to CoW */ } /* - * hugetlb_wp() requires page locks of pte_page(entry) and + * hugetlb_wp() requires page locks of pte_page(vmf.orig_pte) and * pagecache_folio, so here we need take the former one * when folio != pagecache_folio or !pagecache_folio. */ - folio = page_folio(pte_page(entry)); + folio = page_folio(pte_page(vmf.orig_pte)); if (folio != pagecache_folio) if (!folio_trylock(folio)) { need_wait_lock = 1; @@ -6585,24 +6583,24 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, folio_get(folio); if (flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) { - if (!huge_pte_write(entry)) { - ret = hugetlb_wp(mm, vma, address, ptep, flags, - pagecache_folio, ptl, &vmf); + if (!huge_pte_write(vmf.orig_pte)) { + ret = hugetlb_wp(mm, vma, address, vmf.pte, flags, + pagecache_folio, vmf.ptl, &vmf); goto out_put_page; } else if (likely(flags & FAULT_FLAG_WRITE)) { - entry = huge_pte_mkdirty(entry); + vmf.orig_pte = huge_pte_mkdirty(vmf.orig_pte); } } - entry = pte_mkyoung(entry); - if (huge_ptep_set_access_flags(vma, haddr, ptep, entry, + vmf.orig_pte = pte_mkyoung(vmf.orig_pte); + if (huge_ptep_set_access_flags(vma, vmf.address, vmf.pte, vmf.orig_pte, flags & FAULT_FLAG_WRITE)) - update_mmu_cache(vma, haddr, ptep); + update_mmu_cache(vma, vmf.address, vmf.pte); out_put_page: if (folio != pagecache_folio) folio_unlock(folio); folio_put(folio); out_ptl: - spin_unlock(ptl); + spin_unlock(vmf.ptl); if (pagecache_folio) { folio_unlock(pagecache_folio); From 7b6ec181de372a243e3ef285ae1a48f32d5c71ba Mon Sep 17 00:00:00 2001 From: "Vishal Moola (Oracle)" Date: Mon, 1 Apr 2024 13:26:50 -0700 Subject: [PATCH 0765/1702] hugetlb: convert hugetlb_no_page() to use struct vm_fault hugetlb_no_page() can use the struct vm_fault passed in from hugetlb_fault(). This alleviates the stack by consolidating 7 variables into a single struct. [vishal.moola@gmail.com: simplify hugetlb_no_page() arguments] Link: https://lkml.kernel.org/r/ZhQtN8y5zud8iI1u@fedora Link: https://lkml.kernel.org/r/20240401202651.31440-3-vishal.moola@gmail.com Signed-off-by: Vishal Moola (Oracle) Reviewed-by: Oscar Salvador Cc: Matthew Wilcox (Oracle) Cc: Muchun Song Signed-off-by: Andrew Morton --- mm/hugetlb.c | 63 ++++++++++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index c51b8bcbeae17..3e6e10f0bba05 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6188,23 +6188,19 @@ static bool hugetlb_pte_stable(struct hstate *h, struct mm_struct *mm, return same; } -static vm_fault_t hugetlb_no_page(struct mm_struct *mm, - struct vm_area_struct *vma, - struct address_space *mapping, pgoff_t idx, - unsigned long address, pte_t *ptep, - pte_t old_pte, unsigned int flags, +static vm_fault_t hugetlb_no_page(struct address_space *mapping, struct vm_fault *vmf) { + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; struct hstate *h = hstate_vma(vma); vm_fault_t ret = VM_FAULT_SIGBUS; int anon_rmap = 0; unsigned long size; struct folio *folio; pte_t new_pte; - spinlock_t *ptl; - unsigned long haddr = address & huge_page_mask(h); bool new_folio, new_pagecache_folio = false; - u32 hash = hugetlb_fault_mutex_hash(mapping, idx); + u32 hash = hugetlb_fault_mutex_hash(mapping, vmf->pgoff); /* * Currently, we are forced to kill the process in the event the @@ -6223,10 +6219,10 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, * before we get page_table_lock. */ new_folio = false; - folio = filemap_lock_hugetlb_folio(h, mapping, idx); + folio = filemap_lock_hugetlb_folio(h, mapping, vmf->pgoff); if (IS_ERR(folio)) { size = i_size_read(mapping->host) >> huge_page_shift(h); - if (idx >= size) + if (vmf->pgoff >= size) goto out; /* Check for page in userfault range */ if (userfaultfd_missing(vma)) { @@ -6247,7 +6243,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, * never happen on the page after UFFDIO_COPY has * correctly installed the page and returned. */ - if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) { + if (!hugetlb_pte_stable(h, mm, vmf->pte, vmf->orig_pte)) { ret = 0; goto out; } @@ -6262,7 +6258,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, goto out; } - folio = alloc_hugetlb_folio(vma, haddr, 0); + folio = alloc_hugetlb_folio(vma, vmf->address, 0); if (IS_ERR(folio)) { /* * Returning error will result in faulting task being @@ -6276,18 +6272,20 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, * here. Before returning error, get ptl and make * sure there really is no pte entry. */ - if (hugetlb_pte_stable(h, mm, ptep, old_pte)) + if (hugetlb_pte_stable(h, mm, vmf->pte, vmf->orig_pte)) ret = vmf_error(PTR_ERR(folio)); else ret = 0; goto out; } - clear_huge_page(&folio->page, address, pages_per_huge_page(h)); + clear_huge_page(&folio->page, vmf->real_address, + pages_per_huge_page(h)); __folio_mark_uptodate(folio); new_folio = true; if (vma->vm_flags & VM_MAYSHARE) { - int err = hugetlb_add_to_page_cache(folio, mapping, idx); + int err = hugetlb_add_to_page_cache(folio, mapping, + vmf->pgoff); if (err) { /* * err can't be -EEXIST which implies someone @@ -6296,7 +6294,8 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, * to the page cache. So it's safe to call * restore_reserve_on_error() here. */ - restore_reserve_on_error(h, vma, haddr, folio); + restore_reserve_on_error(h, vma, vmf->address, + folio); folio_put(folio); ret = VM_FAULT_SIGBUS; goto out; @@ -6323,7 +6322,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, folio_unlock(folio); folio_put(folio); /* See comment in userfaultfd_missing() block above */ - if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) { + if (!hugetlb_pte_stable(h, mm, vmf->pte, vmf->orig_pte)) { ret = 0; goto out; } @@ -6338,23 +6337,23 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, * any allocations necessary to record that reservation occur outside * the spinlock. */ - if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { - if (vma_needs_reservation(h, vma, haddr) < 0) { + if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { + if (vma_needs_reservation(h, vma, vmf->address) < 0) { ret = VM_FAULT_OOM; goto backout_unlocked; } /* Just decrements count, does not deallocate */ - vma_end_reservation(h, vma, haddr); + vma_end_reservation(h, vma, vmf->address); } - ptl = huge_pte_lock(h, mm, ptep); + vmf->ptl = huge_pte_lock(h, mm, vmf->pte); ret = 0; /* If pte changed from under us, retry */ - if (!pte_same(huge_ptep_get(ptep), old_pte)) + if (!pte_same(huge_ptep_get(vmf->pte), vmf->orig_pte)) goto backout; if (anon_rmap) - hugetlb_add_new_anon_rmap(folio, vma, haddr); + hugetlb_add_new_anon_rmap(folio, vma, vmf->address); else hugetlb_add_file_rmap(folio); new_pte = make_huge_pte(vma, &folio->page, ((vma->vm_flags & VM_WRITE) @@ -6363,17 +6362,18 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, * If this pte was previously wr-protected, keep it wr-protected even * if populated. */ - if (unlikely(pte_marker_uffd_wp(old_pte))) + if (unlikely(pte_marker_uffd_wp(vmf->orig_pte))) new_pte = huge_pte_mkuffd_wp(new_pte); - set_huge_pte_at(mm, haddr, ptep, new_pte, huge_page_size(h)); + set_huge_pte_at(mm, vmf->address, vmf->pte, new_pte, huge_page_size(h)); hugetlb_count_add(pages_per_huge_page(h), mm); - if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { + if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { /* Optimization, do the COW without a second fault */ - ret = hugetlb_wp(mm, vma, address, ptep, flags, folio, ptl, vmf); + ret = hugetlb_wp(mm, vma, vmf->real_address, vmf->pte, + vmf->flags, folio, vmf->ptl, vmf); } - spin_unlock(ptl); + spin_unlock(vmf->ptl); /* * Only set hugetlb_migratable in newly allocated pages. Existing pages @@ -6390,10 +6390,10 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, return ret; backout: - spin_unlock(ptl); + spin_unlock(vmf->ptl); backout_unlocked: if (new_folio && !new_pagecache_folio) - restore_reserve_on_error(h, vma, haddr, folio); + restore_reserve_on_error(h, vma, vmf->address, folio); folio_unlock(folio); folio_put(folio); @@ -6489,8 +6489,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, * hugetlb_no_page will drop vma lock and hugetlb fault * mutex internally, which make us return immediately. */ - return hugetlb_no_page(mm, vma, mapping, vmf.pgoff, address, - vmf.pte, vmf.orig_pte, flags, &vmf); + return hugetlb_no_page(mapping, &vmf); } ret = 0; From bd722058e34de4857bc554e786b7f41c747ad894 Mon Sep 17 00:00:00 2001 From: "Vishal Moola (Oracle)" Date: Mon, 1 Apr 2024 13:26:51 -0700 Subject: [PATCH 0766/1702] hugetlb: convert hugetlb_wp() to use struct vm_fault hugetlb_wp() can use the struct vm_fault passed in from hugetlb_fault(). This alleviates the stack by consolidating 5 variables into a single struct. [vishal.moola@gmail.com: simplify hugetlb_wp() arguments] Link: https://lkml.kernel.org/r/ZhQtoFNZBNwBCeXn@fedora Link: https://lkml.kernel.org/r/20240401202651.31440-4-vishal.moola@gmail.com Signed-off-by: Vishal Moola (Oracle) Reviewed-by: Oscar Salvador Cc: Matthew Wilcox (Oracle) Cc: Muchun Song Signed-off-by: Andrew Morton --- mm/hugetlb.c | 64 ++++++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 3e6e10f0bba05..228c886c46c19 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5918,19 +5918,18 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, * cannot race with other handlers or page migration. * Keep the pte_same checks anyway to make transition from the mutex easier. */ -static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, - unsigned long address, pte_t *ptep, unsigned int flags, - struct folio *pagecache_folio, spinlock_t *ptl, +static vm_fault_t hugetlb_wp(struct folio *pagecache_folio, struct vm_fault *vmf) { - const bool unshare = flags & FAULT_FLAG_UNSHARE; - pte_t pte = huge_ptep_get(ptep); + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; + pte_t pte = huge_ptep_get(vmf->pte); struct hstate *h = hstate_vma(vma); struct folio *old_folio; struct folio *new_folio; int outside_reserve = 0; vm_fault_t ret = 0; - unsigned long haddr = address & huge_page_mask(h); struct mmu_notifier_range range; /* @@ -5953,7 +5952,7 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, /* Let's take out MAP_SHARED mappings first. */ if (vma->vm_flags & VM_MAYSHARE) { - set_huge_ptep_writable(vma, haddr, ptep); + set_huge_ptep_writable(vma, vmf->address, vmf->pte); return 0; } @@ -5972,7 +5971,7 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, SetPageAnonExclusive(&old_folio->page); } if (likely(!unshare)) - set_huge_ptep_writable(vma, haddr, ptep); + set_huge_ptep_writable(vma, vmf->address, vmf->pte); delayacct_wpcopy_end(); return 0; @@ -5999,8 +5998,8 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, * Drop page table lock as buddy allocator may be called. It will * be acquired again before returning to the caller, as expected. */ - spin_unlock(ptl); - new_folio = alloc_hugetlb_folio(vma, haddr, outside_reserve); + spin_unlock(vmf->ptl); + new_folio = alloc_hugetlb_folio(vma, vmf->address, outside_reserve); if (IS_ERR(new_folio)) { /* @@ -6025,19 +6024,21 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, * * Reacquire both after unmap operation. */ - idx = vma_hugecache_offset(h, vma, haddr); + idx = vma_hugecache_offset(h, vma, vmf->address); hash = hugetlb_fault_mutex_hash(mapping, idx); hugetlb_vma_unlock_read(vma); mutex_unlock(&hugetlb_fault_mutex_table[hash]); - unmap_ref_private(mm, vma, &old_folio->page, haddr); + unmap_ref_private(mm, vma, &old_folio->page, + vmf->address); mutex_lock(&hugetlb_fault_mutex_table[hash]); hugetlb_vma_lock_read(vma); - spin_lock(ptl); - ptep = hugetlb_walk(vma, haddr, huge_page_size(h)); - if (likely(ptep && - pte_same(huge_ptep_get(ptep), pte))) + spin_lock(vmf->ptl); + vmf->pte = hugetlb_walk(vma, vmf->address, + huge_page_size(h)); + if (likely(vmf->pte && + pte_same(huge_ptep_get(vmf->pte), pte))) goto retry_avoidcopy; /* * race occurs while re-acquiring page table @@ -6059,37 +6060,38 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, if (unlikely(ret)) goto out_release_all; - if (copy_user_large_folio(new_folio, old_folio, address, vma)) { + if (copy_user_large_folio(new_folio, old_folio, vmf->real_address, vma)) { ret = VM_FAULT_HWPOISON_LARGE; goto out_release_all; } __folio_mark_uptodate(new_folio); - mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, haddr, - haddr + huge_page_size(h)); + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, vmf->address, + vmf->address + huge_page_size(h)); mmu_notifier_invalidate_range_start(&range); /* * Retake the page table lock to check for racing updates * before the page tables are altered */ - spin_lock(ptl); - ptep = hugetlb_walk(vma, haddr, huge_page_size(h)); - if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) { + spin_lock(vmf->ptl); + vmf->pte = hugetlb_walk(vma, vmf->address, huge_page_size(h)); + if (likely(vmf->pte && pte_same(huge_ptep_get(vmf->pte), pte))) { pte_t newpte = make_huge_pte(vma, &new_folio->page, !unshare); /* Break COW or unshare */ - huge_ptep_clear_flush(vma, haddr, ptep); + huge_ptep_clear_flush(vma, vmf->address, vmf->pte); hugetlb_remove_rmap(old_folio); - hugetlb_add_new_anon_rmap(new_folio, vma, haddr); + hugetlb_add_new_anon_rmap(new_folio, vma, vmf->address); if (huge_pte_uffd_wp(pte)) newpte = huge_pte_mkuffd_wp(newpte); - set_huge_pte_at(mm, haddr, ptep, newpte, huge_page_size(h)); + set_huge_pte_at(mm, vmf->address, vmf->pte, newpte, + huge_page_size(h)); folio_set_hugetlb_migratable(new_folio); /* Make the old page be freed below */ new_folio = old_folio; } - spin_unlock(ptl); + spin_unlock(vmf->ptl); mmu_notifier_invalidate_range_end(&range); out_release_all: /* @@ -6097,12 +6099,12 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, * unshare) */ if (new_folio != old_folio) - restore_reserve_on_error(h, vma, haddr, new_folio); + restore_reserve_on_error(h, vma, vmf->address, new_folio); folio_put(new_folio); out_release_old: folio_put(old_folio); - spin_lock(ptl); /* Caller expects lock to be held */ + spin_lock(vmf->ptl); /* Caller expects lock to be held */ delayacct_wpcopy_end(); return ret; @@ -6369,8 +6371,7 @@ static vm_fault_t hugetlb_no_page(struct address_space *mapping, hugetlb_count_add(pages_per_huge_page(h), mm); if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { /* Optimization, do the COW without a second fault */ - ret = hugetlb_wp(mm, vma, vmf->real_address, vmf->pte, - vmf->flags, folio, vmf->ptl, vmf); + ret = hugetlb_wp(folio, vmf); } spin_unlock(vmf->ptl); @@ -6583,8 +6584,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, if (flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) { if (!huge_pte_write(vmf.orig_pte)) { - ret = hugetlb_wp(mm, vma, address, vmf.pte, flags, - pagecache_folio, vmf.ptl, &vmf); + ret = hugetlb_wp(pagecache_folio, &vmf); goto out_put_page; } else if (likely(flags & FAULT_FLAG_WRITE)) { vmf.orig_pte = huge_pte_mkdirty(vmf.orig_pte); From e076eaca59065a48b02384346e7bc110ca37b90b Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 27 Mar 2024 20:34:17 -0700 Subject: [PATCH 0767/1702] selftests: break the dependency upon local header files Patch series "Fix selftests/mm build without requiring "make headers"". As mentioned in each patch, this implements the solution that we discussed in December 2023, in [1]. This turned out to be very clean and easy. It should also be quite easy to maintain. This should also make Peter Zijlstra happy, because it directly addresses the root cause of his "NAK NAK NAK" reply [2]. :) [1] https://lore.kernel.org/all/783a4178-1dec-4e30-989a-5174b8176b09@redhat.com/ [2] https://lore.kernel.org/lkml/20231103121652.GA6217@noisy.programming.kicks-ass.net/ This patch (of 2): Use tools/include/uapi/ files instead. These are obtained by taking a snapshot: run "make headers" at the top level, then copy the desired header file into the appropriate subdir in tools/uapi/. This was discussed and solved in [1]. However, even before copying any additional files there, there are already quite a few in tools/include/uapi already. And these will immediately fix a number of selftests/mm build failures. So this patch: a) Adds TOOLS_INCLUDES to selftests/lib.mk, so that all selftests can immediately and easily include the snapshotted header files. b) Uses $(TOOLS_INCLUDES) in the selftests/mm build. On today's Arch Linux, this already fixes all build errors except for a few userfaultfd.h (those will be addressed in a subsequent patch). [1] https://lore.kernel.org/all/783a4178-1dec-4e30-989a-5174b8176b09@redhat.com/ Link: https://lkml.kernel.org/r/20240328033418.203790-1-jhubbard@nvidia.com Link: https://lkml.kernel.org/r/20240328033418.203790-2-jhubbard@nvidia.com Signed-off-by: John Hubbard Acked-by: David Hildenbrand Cc: Mark Brown Cc: Muhammad Usama Anjum Cc: Suren Baghdasaryan Cc: Peter Zijlstra Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Peter Xu Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/lib.mk | 9 +++++++++ tools/testing/selftests/mm/Makefile | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk index da2cade3bab0e..1dae4a02957f9 100644 --- a/tools/testing/selftests/lib.mk +++ b/tools/testing/selftests/lib.mk @@ -48,6 +48,15 @@ ifeq ($(KHDR_INCLUDES),) KHDR_INCLUDES := -isystem $(top_srcdir)/usr/include endif +# In order to use newer items that haven't yet been added to the user's system +# header files, add $(TOOLS_INCLUDES) to the compiler invocation in each +# each selftest. +# You may need to add files to that location, or to refresh an existing file. In +# order to do that, run "make headers" from $(top_srcdir), then copy the +# header file that you want from $(top_srcdir)/usr/include/... , to the matching +# subdir in $(TOOLS_INCLUDE). +TOOLS_INCLUDES := -isystem $(top_srcdir)/tools/include/uapi + # The following are built by lib.mk common compile rules. # TEST_CUSTOM_PROGS should be used by tests that require # custom build rule and prevent common build rule use. diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index eb5f39a2668b4..7ca9186a06394 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -32,7 +32,7 @@ endif # LDLIBS. MAKEFLAGS += --no-builtin-rules -CFLAGS = -Wall -I $(top_srcdir) $(EXTRA_CFLAGS) $(KHDR_INCLUDES) +CFLAGS = -Wall -I $(top_srcdir) $(EXTRA_CFLAGS) $(KHDR_INCLUDES) $(TOOLS_INCLUDES) LDLIBS = -lrt -lpthread -lm TEST_GEN_FILES = cow From 580ea358af0ade1176326e76edcf60dd50938808 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 27 Mar 2024 20:34:18 -0700 Subject: [PATCH 0768/1702] selftests/mm: fix additional build errors for selftests These build errors only occur if one fails to first run "make headers". However, that is a non-obvious and instrusive requirement, and so there was a discussion on how to get rid of it [1]. This uses that solution. These two files were created by taking a snapshot of the generated header files that are created via "make headers". These two files were copied from ./usr/include/linux/ to ./tools/include/uapi/linux/ . That fixes the selftests/mm build on today's Arch Linux (which required the userfaultfd.h) and Ubuntu 23.04 (which additionally required memfd.h). [1] https://lore.kernel.org/all/783a4178-1dec-4e30-989a-5174b8176b09@redhat.com/ Link: https://lkml.kernel.org/r/20240328033418.203790-3-jhubbard@nvidia.com Signed-off-by: John Hubbard Acked-by: David Hildenbrand Cc: Mark Brown Cc: Muhammad Usama Anjum Cc: Suren Baghdasaryan Cc: Peter Zijlstra Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Peter Xu Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/include/uapi/linux/memfd.h | 39 +++ tools/include/uapi/linux/userfaultfd.h | 386 +++++++++++++++++++++++++ 2 files changed, 425 insertions(+) create mode 100644 tools/include/uapi/linux/memfd.h create mode 100644 tools/include/uapi/linux/userfaultfd.h diff --git a/tools/include/uapi/linux/memfd.h b/tools/include/uapi/linux/memfd.h new file mode 100644 index 0000000000000..01c0324e77331 --- /dev/null +++ b/tools/include/uapi/linux/memfd.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _LINUX_MEMFD_H +#define _LINUX_MEMFD_H + +#include + +/* flags for memfd_create(2) (unsigned int) */ +#define MFD_CLOEXEC 0x0001U +#define MFD_ALLOW_SEALING 0x0002U +#define MFD_HUGETLB 0x0004U +/* not executable and sealed to prevent changing to executable. */ +#define MFD_NOEXEC_SEAL 0x0008U +/* executable */ +#define MFD_EXEC 0x0010U + +/* + * Huge page size encoding when MFD_HUGETLB is specified, and a huge page + * size other than the default is desired. See hugetlb_encode.h. + * All known huge page size encodings are provided here. It is the + * responsibility of the application to know which sizes are supported on + * the running system. See mmap(2) man page for details. + */ +#define MFD_HUGE_SHIFT HUGETLB_FLAG_ENCODE_SHIFT +#define MFD_HUGE_MASK HUGETLB_FLAG_ENCODE_MASK + +#define MFD_HUGE_64KB HUGETLB_FLAG_ENCODE_64KB +#define MFD_HUGE_512KB HUGETLB_FLAG_ENCODE_512KB +#define MFD_HUGE_1MB HUGETLB_FLAG_ENCODE_1MB +#define MFD_HUGE_2MB HUGETLB_FLAG_ENCODE_2MB +#define MFD_HUGE_8MB HUGETLB_FLAG_ENCODE_8MB +#define MFD_HUGE_16MB HUGETLB_FLAG_ENCODE_16MB +#define MFD_HUGE_32MB HUGETLB_FLAG_ENCODE_32MB +#define MFD_HUGE_256MB HUGETLB_FLAG_ENCODE_256MB +#define MFD_HUGE_512MB HUGETLB_FLAG_ENCODE_512MB +#define MFD_HUGE_1GB HUGETLB_FLAG_ENCODE_1GB +#define MFD_HUGE_2GB HUGETLB_FLAG_ENCODE_2GB +#define MFD_HUGE_16GB HUGETLB_FLAG_ENCODE_16GB + +#endif /* _LINUX_MEMFD_H */ diff --git a/tools/include/uapi/linux/userfaultfd.h b/tools/include/uapi/linux/userfaultfd.h new file mode 100644 index 0000000000000..4283de22d5b65 --- /dev/null +++ b/tools/include/uapi/linux/userfaultfd.h @@ -0,0 +1,386 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * include/linux/userfaultfd.h + * + * Copyright (C) 2007 Davide Libenzi + * Copyright (C) 2015 Red Hat, Inc. + * + */ + +#ifndef _LINUX_USERFAULTFD_H +#define _LINUX_USERFAULTFD_H + +#include + +/* ioctls for /dev/userfaultfd */ +#define USERFAULTFD_IOC 0xAA +#define USERFAULTFD_IOC_NEW _IO(USERFAULTFD_IOC, 0x00) + +/* + * If the UFFDIO_API is upgraded someday, the UFFDIO_UNREGISTER and + * UFFDIO_WAKE ioctls should be defined as _IOW and not as _IOR. In + * userfaultfd.h we assumed the kernel was reading (instead _IOC_READ + * means the userland is reading). + */ +#define UFFD_API ((__u64)0xAA) +#define UFFD_API_REGISTER_MODES (UFFDIO_REGISTER_MODE_MISSING | \ + UFFDIO_REGISTER_MODE_WP | \ + UFFDIO_REGISTER_MODE_MINOR) +#define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP | \ + UFFD_FEATURE_EVENT_FORK | \ + UFFD_FEATURE_EVENT_REMAP | \ + UFFD_FEATURE_EVENT_REMOVE | \ + UFFD_FEATURE_EVENT_UNMAP | \ + UFFD_FEATURE_MISSING_HUGETLBFS | \ + UFFD_FEATURE_MISSING_SHMEM | \ + UFFD_FEATURE_SIGBUS | \ + UFFD_FEATURE_THREAD_ID | \ + UFFD_FEATURE_MINOR_HUGETLBFS | \ + UFFD_FEATURE_MINOR_SHMEM | \ + UFFD_FEATURE_EXACT_ADDRESS | \ + UFFD_FEATURE_WP_HUGETLBFS_SHMEM | \ + UFFD_FEATURE_WP_UNPOPULATED | \ + UFFD_FEATURE_POISON | \ + UFFD_FEATURE_WP_ASYNC | \ + UFFD_FEATURE_MOVE) +#define UFFD_API_IOCTLS \ + ((__u64)1 << _UFFDIO_REGISTER | \ + (__u64)1 << _UFFDIO_UNREGISTER | \ + (__u64)1 << _UFFDIO_API) +#define UFFD_API_RANGE_IOCTLS \ + ((__u64)1 << _UFFDIO_WAKE | \ + (__u64)1 << _UFFDIO_COPY | \ + (__u64)1 << _UFFDIO_ZEROPAGE | \ + (__u64)1 << _UFFDIO_MOVE | \ + (__u64)1 << _UFFDIO_WRITEPROTECT | \ + (__u64)1 << _UFFDIO_CONTINUE | \ + (__u64)1 << _UFFDIO_POISON) +#define UFFD_API_RANGE_IOCTLS_BASIC \ + ((__u64)1 << _UFFDIO_WAKE | \ + (__u64)1 << _UFFDIO_COPY | \ + (__u64)1 << _UFFDIO_WRITEPROTECT | \ + (__u64)1 << _UFFDIO_CONTINUE | \ + (__u64)1 << _UFFDIO_POISON) + +/* + * Valid ioctl command number range with this API is from 0x00 to + * 0x3F. UFFDIO_API is the fixed number, everything else can be + * changed by implementing a different UFFD_API. If sticking to the + * same UFFD_API more ioctl can be added and userland will be aware of + * which ioctl the running kernel implements through the ioctl command + * bitmask written by the UFFDIO_API. + */ +#define _UFFDIO_REGISTER (0x00) +#define _UFFDIO_UNREGISTER (0x01) +#define _UFFDIO_WAKE (0x02) +#define _UFFDIO_COPY (0x03) +#define _UFFDIO_ZEROPAGE (0x04) +#define _UFFDIO_MOVE (0x05) +#define _UFFDIO_WRITEPROTECT (0x06) +#define _UFFDIO_CONTINUE (0x07) +#define _UFFDIO_POISON (0x08) +#define _UFFDIO_API (0x3F) + +/* userfaultfd ioctl ids */ +#define UFFDIO 0xAA +#define UFFDIO_API _IOWR(UFFDIO, _UFFDIO_API, \ + struct uffdio_api) +#define UFFDIO_REGISTER _IOWR(UFFDIO, _UFFDIO_REGISTER, \ + struct uffdio_register) +#define UFFDIO_UNREGISTER _IOR(UFFDIO, _UFFDIO_UNREGISTER, \ + struct uffdio_range) +#define UFFDIO_WAKE _IOR(UFFDIO, _UFFDIO_WAKE, \ + struct uffdio_range) +#define UFFDIO_COPY _IOWR(UFFDIO, _UFFDIO_COPY, \ + struct uffdio_copy) +#define UFFDIO_ZEROPAGE _IOWR(UFFDIO, _UFFDIO_ZEROPAGE, \ + struct uffdio_zeropage) +#define UFFDIO_MOVE _IOWR(UFFDIO, _UFFDIO_MOVE, \ + struct uffdio_move) +#define UFFDIO_WRITEPROTECT _IOWR(UFFDIO, _UFFDIO_WRITEPROTECT, \ + struct uffdio_writeprotect) +#define UFFDIO_CONTINUE _IOWR(UFFDIO, _UFFDIO_CONTINUE, \ + struct uffdio_continue) +#define UFFDIO_POISON _IOWR(UFFDIO, _UFFDIO_POISON, \ + struct uffdio_poison) + +/* read() structure */ +struct uffd_msg { + __u8 event; + + __u8 reserved1; + __u16 reserved2; + __u32 reserved3; + + union { + struct { + __u64 flags; + __u64 address; + union { + __u32 ptid; + } feat; + } pagefault; + + struct { + __u32 ufd; + } fork; + + struct { + __u64 from; + __u64 to; + __u64 len; + } remap; + + struct { + __u64 start; + __u64 end; + } remove; + + struct { + /* unused reserved fields */ + __u64 reserved1; + __u64 reserved2; + __u64 reserved3; + } reserved; + } arg; +} __attribute__((packed)); + +/* + * Start at 0x12 and not at 0 to be more strict against bugs. + */ +#define UFFD_EVENT_PAGEFAULT 0x12 +#define UFFD_EVENT_FORK 0x13 +#define UFFD_EVENT_REMAP 0x14 +#define UFFD_EVENT_REMOVE 0x15 +#define UFFD_EVENT_UNMAP 0x16 + +/* flags for UFFD_EVENT_PAGEFAULT */ +#define UFFD_PAGEFAULT_FLAG_WRITE (1<<0) /* If this was a write fault */ +#define UFFD_PAGEFAULT_FLAG_WP (1<<1) /* If reason is VM_UFFD_WP */ +#define UFFD_PAGEFAULT_FLAG_MINOR (1<<2) /* If reason is VM_UFFD_MINOR */ + +struct uffdio_api { + /* userland asks for an API number and the features to enable */ + __u64 api; + /* + * Kernel answers below with the all available features for + * the API, this notifies userland of which events and/or + * which flags for each event are enabled in the current + * kernel. + * + * Note: UFFD_EVENT_PAGEFAULT and UFFD_PAGEFAULT_FLAG_WRITE + * are to be considered implicitly always enabled in all kernels as + * long as the uffdio_api.api requested matches UFFD_API. + * + * UFFD_FEATURE_MISSING_HUGETLBFS means an UFFDIO_REGISTER + * with UFFDIO_REGISTER_MODE_MISSING mode will succeed on + * hugetlbfs virtual memory ranges. Adding or not adding + * UFFD_FEATURE_MISSING_HUGETLBFS to uffdio_api.features has + * no real functional effect after UFFDIO_API returns, but + * it's only useful for an initial feature set probe at + * UFFDIO_API time. There are two ways to use it: + * + * 1) by adding UFFD_FEATURE_MISSING_HUGETLBFS to the + * uffdio_api.features before calling UFFDIO_API, an error + * will be returned by UFFDIO_API on a kernel without + * hugetlbfs missing support + * + * 2) the UFFD_FEATURE_MISSING_HUGETLBFS can not be added in + * uffdio_api.features and instead it will be set by the + * kernel in the uffdio_api.features if the kernel supports + * it, so userland can later check if the feature flag is + * present in uffdio_api.features after UFFDIO_API + * succeeded. + * + * UFFD_FEATURE_MISSING_SHMEM works the same as + * UFFD_FEATURE_MISSING_HUGETLBFS, but it applies to shmem + * (i.e. tmpfs and other shmem based APIs). + * + * UFFD_FEATURE_SIGBUS feature means no page-fault + * (UFFD_EVENT_PAGEFAULT) event will be delivered, instead + * a SIGBUS signal will be sent to the faulting process. + * + * UFFD_FEATURE_THREAD_ID pid of the page faulted task_struct will + * be returned, if feature is not requested 0 will be returned. + * + * UFFD_FEATURE_MINOR_HUGETLBFS indicates that minor faults + * can be intercepted (via REGISTER_MODE_MINOR) for + * hugetlbfs-backed pages. + * + * UFFD_FEATURE_MINOR_SHMEM indicates the same support as + * UFFD_FEATURE_MINOR_HUGETLBFS, but for shmem-backed pages instead. + * + * UFFD_FEATURE_EXACT_ADDRESS indicates that the exact address of page + * faults would be provided and the offset within the page would not be + * masked. + * + * UFFD_FEATURE_WP_HUGETLBFS_SHMEM indicates that userfaultfd + * write-protection mode is supported on both shmem and hugetlbfs. + * + * UFFD_FEATURE_WP_UNPOPULATED indicates that userfaultfd + * write-protection mode will always apply to unpopulated pages + * (i.e. empty ptes). This will be the default behavior for shmem + * & hugetlbfs, so this flag only affects anonymous memory behavior + * when userfault write-protection mode is registered. + * + * UFFD_FEATURE_WP_ASYNC indicates that userfaultfd write-protection + * asynchronous mode is supported in which the write fault is + * automatically resolved and write-protection is un-set. + * It implies UFFD_FEATURE_WP_UNPOPULATED. + * + * UFFD_FEATURE_MOVE indicates that the kernel supports moving an + * existing page contents from userspace. + */ +#define UFFD_FEATURE_PAGEFAULT_FLAG_WP (1<<0) +#define UFFD_FEATURE_EVENT_FORK (1<<1) +#define UFFD_FEATURE_EVENT_REMAP (1<<2) +#define UFFD_FEATURE_EVENT_REMOVE (1<<3) +#define UFFD_FEATURE_MISSING_HUGETLBFS (1<<4) +#define UFFD_FEATURE_MISSING_SHMEM (1<<5) +#define UFFD_FEATURE_EVENT_UNMAP (1<<6) +#define UFFD_FEATURE_SIGBUS (1<<7) +#define UFFD_FEATURE_THREAD_ID (1<<8) +#define UFFD_FEATURE_MINOR_HUGETLBFS (1<<9) +#define UFFD_FEATURE_MINOR_SHMEM (1<<10) +#define UFFD_FEATURE_EXACT_ADDRESS (1<<11) +#define UFFD_FEATURE_WP_HUGETLBFS_SHMEM (1<<12) +#define UFFD_FEATURE_WP_UNPOPULATED (1<<13) +#define UFFD_FEATURE_POISON (1<<14) +#define UFFD_FEATURE_WP_ASYNC (1<<15) +#define UFFD_FEATURE_MOVE (1<<16) + __u64 features; + + __u64 ioctls; +}; + +struct uffdio_range { + __u64 start; + __u64 len; +}; + +struct uffdio_register { + struct uffdio_range range; +#define UFFDIO_REGISTER_MODE_MISSING ((__u64)1<<0) +#define UFFDIO_REGISTER_MODE_WP ((__u64)1<<1) +#define UFFDIO_REGISTER_MODE_MINOR ((__u64)1<<2) + __u64 mode; + + /* + * kernel answers which ioctl commands are available for the + * range, keep at the end as the last 8 bytes aren't read. + */ + __u64 ioctls; +}; + +struct uffdio_copy { + __u64 dst; + __u64 src; + __u64 len; +#define UFFDIO_COPY_MODE_DONTWAKE ((__u64)1<<0) + /* + * UFFDIO_COPY_MODE_WP will map the page write protected on + * the fly. UFFDIO_COPY_MODE_WP is available only if the + * write protected ioctl is implemented for the range + * according to the uffdio_register.ioctls. + */ +#define UFFDIO_COPY_MODE_WP ((__u64)1<<1) + __u64 mode; + + /* + * "copy" is written by the ioctl and must be at the end: the + * copy_from_user will not read the last 8 bytes. + */ + __s64 copy; +}; + +struct uffdio_zeropage { + struct uffdio_range range; +#define UFFDIO_ZEROPAGE_MODE_DONTWAKE ((__u64)1<<0) + __u64 mode; + + /* + * "zeropage" is written by the ioctl and must be at the end: + * the copy_from_user will not read the last 8 bytes. + */ + __s64 zeropage; +}; + +struct uffdio_writeprotect { + struct uffdio_range range; +/* + * UFFDIO_WRITEPROTECT_MODE_WP: set the flag to write protect a range, + * unset the flag to undo protection of a range which was previously + * write protected. + * + * UFFDIO_WRITEPROTECT_MODE_DONTWAKE: set the flag to avoid waking up + * any wait thread after the operation succeeds. + * + * NOTE: Write protecting a region (WP=1) is unrelated to page faults, + * therefore DONTWAKE flag is meaningless with WP=1. Removing write + * protection (WP=0) in response to a page fault wakes the faulting + * task unless DONTWAKE is set. + */ +#define UFFDIO_WRITEPROTECT_MODE_WP ((__u64)1<<0) +#define UFFDIO_WRITEPROTECT_MODE_DONTWAKE ((__u64)1<<1) + __u64 mode; +}; + +struct uffdio_continue { + struct uffdio_range range; +#define UFFDIO_CONTINUE_MODE_DONTWAKE ((__u64)1<<0) + /* + * UFFDIO_CONTINUE_MODE_WP will map the page write protected on + * the fly. UFFDIO_CONTINUE_MODE_WP is available only if the + * write protected ioctl is implemented for the range + * according to the uffdio_register.ioctls. + */ +#define UFFDIO_CONTINUE_MODE_WP ((__u64)1<<1) + __u64 mode; + + /* + * Fields below here are written by the ioctl and must be at the end: + * the copy_from_user will not read past here. + */ + __s64 mapped; +}; + +struct uffdio_poison { + struct uffdio_range range; +#define UFFDIO_POISON_MODE_DONTWAKE ((__u64)1<<0) + __u64 mode; + + /* + * Fields below here are written by the ioctl and must be at the end: + * the copy_from_user will not read past here. + */ + __s64 updated; +}; + +struct uffdio_move { + __u64 dst; + __u64 src; + __u64 len; + /* + * Especially if used to atomically remove memory from the + * address space the wake on the dst range is not needed. + */ +#define UFFDIO_MOVE_MODE_DONTWAKE ((__u64)1<<0) +#define UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES ((__u64)1<<1) + __u64 mode; + /* + * "move" is written by the ioctl and must be at the end: the + * copy_from_user will not read the last 8 bytes. + */ + __s64 move; +}; + +/* + * Flags for the userfaultfd(2) system call itself. + */ + +/* + * Create a userfaultfd that can handle page faults only in user mode. + */ +#define UFFD_USER_MODE_ONLY 1 + +#endif /* _LINUX_USERFAULTFD_H */ From b174f139bdc8aaaf72f5b67ad1bd512c4868a87e Mon Sep 17 00:00:00 2001 From: Frank van der Linden Date: Thu, 4 Apr 2024 16:25:14 +0000 Subject: [PATCH 0769/1702] mm/cma: drop incorrect alignment check in cma_init_reserved_mem cma_init_reserved_mem uses IS_ALIGNED to check if the size represented by one bit in the cma allocation bitmask is aligned with CMA_MIN_ALIGNMENT_BYTES (pageblock size). However, this is too strict, as this will fail if order_per_bit > pageblock_order, which is a valid configuration. We could check IS_ALIGNED both ways, but since both numbers are powers of two, no check is needed at all. Link: https://lkml.kernel.org/r/20240404162515.527802-1-fvdl@google.com Fixes: de9e14eebf33 ("drivers: dma-contiguous: add initialization from device tree") Signed-off-by: Frank van der Linden Acked-by: David Hildenbrand Cc: Marek Szyprowski Cc: Muchun Song Cc: Roman Gushchin Cc: Signed-off-by: Andrew Morton --- mm/cma.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mm/cma.c b/mm/cma.c index 01f5a8f71ddfa..3e9724716bad8 100644 --- a/mm/cma.c +++ b/mm/cma.c @@ -182,10 +182,6 @@ int __init cma_init_reserved_mem(phys_addr_t base, phys_addr_t size, if (!size || !memblock_is_region_reserved(base, size)) return -EINVAL; - /* alignment should be aligned with order_per_bit */ - if (!IS_ALIGNED(CMA_MIN_ALIGNMENT_PAGES, 1 << order_per_bit)) - return -EINVAL; - /* ensure minimal alignment required by mm core */ if (!IS_ALIGNED(base | size, CMA_MIN_ALIGNMENT_BYTES)) return -EINVAL; From 55d134a7b499c77e7cfd0ee41046f3c376e791e5 Mon Sep 17 00:00:00 2001 From: Frank van der Linden Date: Thu, 4 Apr 2024 16:25:15 +0000 Subject: [PATCH 0770/1702] mm/hugetlb: pass correct order_per_bit to cma_declare_contiguous_nid The hugetlb_cma code passes 0 in the order_per_bit argument to cma_declare_contiguous_nid (the alignment, computed using the page order, is correctly passed in). This causes a bit in the cma allocation bitmap to always represent a 4k page, making the bitmaps potentially very large, and slower. It would create bitmaps that would be pretty big. E.g. for a 4k page size on x86, hugetlb_cma=64G would mean a bitmap size of (64G / 4k) / 8 == 2M. With HUGETLB_PAGE_ORDER as order_per_bit, as intended, this would be (64G / 2M) / 8 == 4k. So, that's quite a difference. Also, this restricted the hugetlb_cma area to ((PAGE_SIZE << MAX_PAGE_ORDER) * 8) * PAGE_SIZE (e.g. 128G on x86) , since bitmap_alloc uses normal page allocation, and is thus restricted by MAX_PAGE_ORDER. Specifying anything about that would fail the CMA initialization. So, correctly pass in the order instead. Link: https://lkml.kernel.org/r/20240404162515.527802-2-fvdl@google.com Fixes: cf11e85fc08c ("mm: hugetlb: optionally allocate gigantic hugepages using cma") Signed-off-by: Frank van der Linden Acked-by: Roman Gushchin Acked-by: David Hildenbrand Cc: Marek Szyprowski Cc: Muchun Song Cc: Signed-off-by: Andrew Morton --- mm/hugetlb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 228c886c46c19..5dc3f5ea3a2ef 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -7794,9 +7794,9 @@ void __init hugetlb_cma_reserve(int order) * huge page demotion. */ res = cma_declare_contiguous_nid(0, size, 0, - PAGE_SIZE << HUGETLB_PAGE_ORDER, - 0, false, name, - &hugetlb_cma[nid], nid); + PAGE_SIZE << HUGETLB_PAGE_ORDER, + HUGETLB_PAGE_ORDER, false, name, + &hugetlb_cma[nid], nid); if (res) { pr_warn("hugetlb_cma: reservation failed: err %d, node %d", res, nid); From 2ebe90dab9808a15e5d1c973e7d3ddaee05ddbd3 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 5 Apr 2024 19:00:36 +0100 Subject: [PATCH 0771/1702] mm: convert pagecache_isize_extended to use a folio Remove four hidden calls to compound_head(). Also exit early if the filesystem block size is >= PAGE_SIZE instead of just equal to PAGE_SIZE. Link: https://lkml.kernel.org/r/20240405180038.2618624-1-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Pankaj Raghav Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- mm/truncate.c | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/mm/truncate.c b/mm/truncate.c index 725b150e47ac4..e99085bf3d34d 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -764,15 +764,15 @@ EXPORT_SYMBOL(truncate_setsize); * @from: original inode size * @to: new inode size * - * Handle extension of inode size either caused by extending truncate or by - * write starting after current i_size. We mark the page straddling current - * i_size RO so that page_mkwrite() is called on the nearest write access to - * the page. This way filesystem can be sure that page_mkwrite() is called on - * the page before user writes to the page via mmap after the i_size has been - * changed. + * Handle extension of inode size either caused by extending truncate or + * by write starting after current i_size. We mark the page straddling + * current i_size RO so that page_mkwrite() is called on the first + * write access to the page. The filesystem will update its per-block + * information before user writes to the page via mmap after the i_size + * has been changed. * * The function must be called after i_size is updated so that page fault - * coming after we unlock the page will already see the new i_size. + * coming after we unlock the folio will already see the new i_size. * The function must be called while we still hold i_rwsem - this not only * makes sure i_size is stable but also that userspace cannot observe new * i_size value before we are prepared to store mmap writes at new inode size. @@ -781,31 +781,29 @@ void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to) { int bsize = i_blocksize(inode); loff_t rounded_from; - struct page *page; - pgoff_t index; + struct folio *folio; WARN_ON(to > inode->i_size); - if (from >= to || bsize == PAGE_SIZE) + if (from >= to || bsize >= PAGE_SIZE) return; /* Page straddling @from will not have any hole block created? */ rounded_from = round_up(from, bsize); if (to <= rounded_from || !(rounded_from & (PAGE_SIZE - 1))) return; - index = from >> PAGE_SHIFT; - page = find_lock_page(inode->i_mapping, index); - /* Page not cached? Nothing to do */ - if (!page) + folio = filemap_lock_folio(inode->i_mapping, from / PAGE_SIZE); + /* Folio not cached? Nothing to do */ + if (IS_ERR(folio)) return; /* - * See clear_page_dirty_for_io() for details why set_page_dirty() + * See folio_clear_dirty_for_io() for details why folio_mark_dirty() * is needed. */ - if (page_mkclean(page)) - set_page_dirty(page); - unlock_page(page); - put_page(page); + if (folio_mkclean(folio)) + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); } EXPORT_SYMBOL(pagecache_isize_extended); From 2f166704290eadec480209cf28060f154184afe9 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 5 Apr 2024 16:32:23 +0100 Subject: [PATCH 0772/1702] mm: free non-hugetlb large folios in a batch Patch series "Clean up __folio_put()". With all the changes over the last few years, __folio_put_small and __folio_put_large have become almost identical to each other ... except you can't tell because they're spread over two files. Rearrange it all so that you can tell, and then inline them both into __folio_put(). This patch (of 5): free_unref_folios() can now handle non-hugetlb large folios, so keep normal large folios in the batch. hugetlb folios still need to be handled specially. [peterx@redhat.com: fix panic] Link: https://lkml.kernel.org/r/ZikjPB0Dt5HA8-uL@x1n Link: https://lkml.kernel.org/r/20240405153228.2563754-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240405153228.2563754-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Peter Xu Reviewed-by: Zi Yan Signed-off-by: Andrew Morton --- mm/swap.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/swap.c b/mm/swap.c index f72364e92d5f4..d7db3cd4e80a6 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -158,8 +158,8 @@ void put_pages_list(struct list_head *pages) list_for_each_entry_safe(folio, next, pages, lru) { if (!folio_put_testzero(folio)) continue; - if (folio_test_large(folio)) { - __folio_put_large(folio); + if (folio_test_hugetlb(folio)) { + free_huge_folio(folio); continue; } /* LRU flag must be clear because it's passed using the lru */ From 5b8d75913a0ed9deb16140c0aa880c4d6db2dc62 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 5 Apr 2024 16:32:24 +0100 Subject: [PATCH 0773/1702] mm: combine free_the_page() and free_unref_page() The pcp_allowed_order() check in free_the_page() was only being skipped by __folio_put_small() which is about to be rearranged. Link: https://lkml.kernel.org/r/20240405153228.2563754-3-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Zi Yan Signed-off-by: Andrew Morton --- mm/page_alloc.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index cbe3f695eda9b..c01757251de29 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -541,14 +541,6 @@ static inline bool pcp_allowed_order(unsigned int order) return false; } -static inline void free_the_page(struct page *page, unsigned int order) -{ - if (pcp_allowed_order(order)) /* Via pcp? */ - free_unref_page(page, order); - else - __free_pages_ok(page, order, FPI_NONE); -} - /* * Higher-order pages are called "compound pages". They are structured thusly: * @@ -584,7 +576,7 @@ void destroy_large_folio(struct folio *folio) folio_undo_large_rmappable(folio); mem_cgroup_uncharge(folio); - free_the_page(&folio->page, folio_order(folio)); + free_unref_page(&folio->page, folio_order(folio)); } static inline void set_buddy_order(struct page *page, unsigned int order) @@ -2573,6 +2565,11 @@ void free_unref_page(struct page *page, unsigned int order) unsigned long pfn = page_to_pfn(page); int migratetype; + if (!pcp_allowed_order(order)) { + __free_pages_ok(page, order, FPI_NONE); + return; + } + if (!free_pages_prepare(page, order)) return; @@ -4755,11 +4752,11 @@ void __free_pages(struct page *page, unsigned int order) struct alloc_tag *tag = pgalloc_tag_get(page); if (put_page_testzero(page)) - free_the_page(page, order); + free_unref_page(page, order); else if (!head) { pgalloc_tag_sub_pages(tag, (1 << order) - 1); while (order-- > 0) - free_the_page(page + (1 << order), order); + free_unref_page(page + (1 << order), order); } } EXPORT_SYMBOL(__free_pages); @@ -4821,7 +4818,7 @@ void __page_frag_cache_drain(struct page *page, unsigned int count) VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); if (page_ref_sub_and_test(page, count)) - free_the_page(page, compound_order(page)); + free_unref_page(page, compound_order(page)); } EXPORT_SYMBOL(__page_frag_cache_drain); @@ -4862,7 +4859,7 @@ void *__page_frag_alloc_align(struct page_frag_cache *nc, goto refill; if (unlikely(nc->pfmemalloc)) { - free_the_page(page, compound_order(page)); + free_unref_page(page, compound_order(page)); goto refill; } @@ -4906,7 +4903,7 @@ void page_frag_free(void *addr) struct page *page = virt_to_head_page(addr); if (unlikely(put_page_testzero(page))) - free_the_page(page, compound_order(page)); + free_unref_page(page, compound_order(page)); } EXPORT_SYMBOL(page_frag_free); From 2542b1ac9a46ac58f9565de0048457956898d481 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 5 Apr 2024 16:32:25 +0100 Subject: [PATCH 0774/1702] mm: inline destroy_large_folio() into __folio_put_large() destroy_large_folio() has only one caller, move its contents there. Link: https://lkml.kernel.org/r/20240405153228.2563754-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Zi Yan Signed-off-by: Andrew Morton --- include/linux/mm.h | 2 -- mm/page_alloc.c | 14 -------------- mm/swap.c | 13 ++++++++++--- 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 4f4e460d78531..cc9410d772ddf 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1318,8 +1318,6 @@ void folio_copy(struct folio *dst, struct folio *src); unsigned long nr_free_buffer_pages(void); -void destroy_large_folio(struct folio *folio); - /* Returns the number of bytes in this potentially compound page. */ static inline unsigned long page_size(struct page *page) { diff --git a/mm/page_alloc.c b/mm/page_alloc.c index c01757251de29..22e8b9f1d710b 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -565,20 +565,6 @@ void prep_compound_page(struct page *page, unsigned int order) prep_compound_head(page, order); } -void destroy_large_folio(struct folio *folio) -{ - if (folio_test_hugetlb(folio)) { - free_huge_folio(folio); - return; - } - - if (folio_test_large_rmappable(folio)) - folio_undo_large_rmappable(folio); - - mem_cgroup_uncharge(folio); - free_unref_page(&folio->page, folio_order(folio)); -} - static inline void set_buddy_order(struct page *page, unsigned int order) { set_page_private(page, order); diff --git a/mm/swap.c b/mm/swap.c index d7db3cd4e80a6..6e3bd03673eab 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -127,9 +127,16 @@ static void __folio_put_large(struct folio *folio) * (it's never listed to any LRU lists) and no memcg routines should * be called for hugetlb (it has a separate hugetlb_cgroup.) */ - if (!folio_test_hugetlb(folio)) - page_cache_release(folio); - destroy_large_folio(folio); + if (folio_test_hugetlb(folio)) { + free_huge_folio(folio); + return; + } + + page_cache_release(folio); + if (folio_test_large_rmappable(folio)) + folio_undo_large_rmappable(folio); + mem_cgroup_uncharge(folio); + free_unref_page(&folio->page, folio_order(folio)); } void __folio_put(struct folio *folio) From 79a48287515848c18a49d75c1fdf176c82bb13cf Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 5 Apr 2024 16:32:26 +0100 Subject: [PATCH 0775/1702] mm: combine __folio_put_small, __folio_put_large and __folio_put It's now obvious that __folio_put_small() and __folio_put_large() do almost exactly the same thing. Inline them both into __folio_put(). Link: https://lkml.kernel.org/r/20240405153228.2563754-5-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Zi Yan Signed-off-by: Andrew Morton --- mm/swap.c | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/mm/swap.c b/mm/swap.c index 6e3bd03673eab..4f3964c983d8d 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -112,42 +112,22 @@ static void page_cache_release(struct folio *folio) unlock_page_lruvec_irqrestore(lruvec, flags); } -static void __folio_put_small(struct folio *folio) -{ - page_cache_release(folio); - mem_cgroup_uncharge(folio); - free_unref_page(&folio->page, 0); -} - -static void __folio_put_large(struct folio *folio) +void __folio_put(struct folio *folio) { - /* - * __page_cache_release() is supposed to be called for thp, not for - * hugetlb. This is because hugetlb page does never have PageLRU set - * (it's never listed to any LRU lists) and no memcg routines should - * be called for hugetlb (it has a separate hugetlb_cgroup.) - */ - if (folio_test_hugetlb(folio)) { + if (unlikely(folio_is_zone_device(folio))) { + free_zone_device_page(&folio->page); + return; + } else if (folio_test_hugetlb(folio)) { free_huge_folio(folio); return; } page_cache_release(folio); - if (folio_test_large_rmappable(folio)) + if (folio_test_large(folio) && folio_test_large_rmappable(folio)) folio_undo_large_rmappable(folio); mem_cgroup_uncharge(folio); free_unref_page(&folio->page, folio_order(folio)); } - -void __folio_put(struct folio *folio) -{ - if (unlikely(folio_is_zone_device(folio))) - free_zone_device_page(&folio->page); - else if (unlikely(folio_test_large(folio))) - __folio_put_large(folio); - else - __folio_put_small(folio); -} EXPORT_SYMBOL(__folio_put); /** From 9f100e3b37590828ae23b0210ee634d14b28b8e8 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 5 Apr 2024 16:32:27 +0100 Subject: [PATCH 0776/1702] mm: convert free_zone_device_page to free_zone_device_folio Both callers already have a folio; pass it in and save a few calls to compound_head(). Link: https://lkml.kernel.org/r/20240405153228.2563754-6-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Zi Yan Signed-off-by: Andrew Morton --- mm/internal.h | 2 +- mm/memremap.c | 30 ++++++++++++++++-------------- mm/swap.c | 4 ++-- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 8fd41f889a958..22152e0c8494a 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1165,7 +1165,7 @@ void __vunmap_range_noflush(unsigned long start, unsigned long end); int numa_migrate_prep(struct folio *folio, struct vm_fault *vmf, unsigned long addr, int page_nid, int *flags); -void free_zone_device_page(struct page *page); +void free_zone_device_folio(struct folio *folio); int migrate_device_coherent_page(struct page *page); /* diff --git a/mm/memremap.c b/mm/memremap.c index 9e9fb1972fff0..e1776693e2eae 100644 --- a/mm/memremap.c +++ b/mm/memremap.c @@ -456,21 +456,23 @@ struct dev_pagemap *get_dev_pagemap(unsigned long pfn, } EXPORT_SYMBOL_GPL(get_dev_pagemap); -void free_zone_device_page(struct page *page) +void free_zone_device_folio(struct folio *folio) { - if (WARN_ON_ONCE(!page->pgmap->ops || !page->pgmap->ops->page_free)) + if (WARN_ON_ONCE(!folio->page.pgmap->ops || + !folio->page.pgmap->ops->page_free)) return; - mem_cgroup_uncharge(page_folio(page)); + mem_cgroup_uncharge(folio); /* * Note: we don't expect anonymous compound pages yet. Once supported * and we could PTE-map them similar to THP, we'd have to clear * PG_anon_exclusive on all tail pages. */ - VM_BUG_ON_PAGE(PageAnon(page) && PageCompound(page), page); - if (PageAnon(page)) - __ClearPageAnonExclusive(page); + if (folio_test_anon(folio)) { + VM_BUG_ON_FOLIO(folio_test_large(folio), folio); + __ClearPageAnonExclusive(folio_page(folio, 0)); + } /* * When a device managed page is freed, the folio->mapping field @@ -481,20 +483,20 @@ void free_zone_device_page(struct page *page) * * For other types of ZONE_DEVICE pages, migration is either * handled differently or not done at all, so there is no need - * to clear page->mapping. + * to clear folio->mapping. */ - page->mapping = NULL; - page->pgmap->ops->page_free(page); + folio->mapping = NULL; + folio->page.pgmap->ops->page_free(folio_page(folio, 0)); - if (page->pgmap->type != MEMORY_DEVICE_PRIVATE && - page->pgmap->type != MEMORY_DEVICE_COHERENT) + if (folio->page.pgmap->type != MEMORY_DEVICE_PRIVATE && + folio->page.pgmap->type != MEMORY_DEVICE_COHERENT) /* - * Reset the page count to 1 to prepare for handing out the page + * Reset the refcount to 1 to prepare for handing out the page * again. */ - set_page_count(page, 1); + folio_set_count(folio, 1); else - put_dev_pagemap(page->pgmap); + put_dev_pagemap(folio->page.pgmap); } void zone_device_page_init(struct page *page) diff --git a/mm/swap.c b/mm/swap.c index 4f3964c983d8d..8ae5cd4ed180b 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -115,7 +115,7 @@ static void page_cache_release(struct folio *folio) void __folio_put(struct folio *folio) { if (unlikely(folio_is_zone_device(folio))) { - free_zone_device_page(&folio->page); + free_zone_device_folio(folio); return; } else if (folio_test_hugetlb(folio)) { free_huge_folio(folio); @@ -983,7 +983,7 @@ void folios_put_refs(struct folio_batch *folios, unsigned int *refs) if (put_devmap_managed_page_refs(&folio->page, nr_refs)) continue; if (folio_ref_sub_and_test(folio, nr_refs)) - free_zone_device_page(&folio->page); + free_zone_device_folio(folio); continue; } From 72801513b2bfb6bf571956a604d38f98ce9aacd9 Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Fri, 5 Apr 2024 20:24:16 +0800 Subject: [PATCH 0777/1702] mm: set pageblock_order to HPAGE_PMD_ORDER in case with !CONFIG_HUGETLB_PAGE but THP enabled As Vlastimil suggested in previous discussion[1], it doesn't make sense to set pageblock_order as MAX_PAGE_ORDER when hugetlbfs is not enabled and THP is enabled. Instead, it should be set to HPAGE_PMD_ORDER. [1] https://lore.kernel.org/all/76457ec5-d789-449b-b8ca-dcb6ceb12445@suse.cz/ Link: https://lkml.kernel.org/r/3d57d253070035bdc0f6d6e5681ce1ed0e1934f7.1712286863.git.baolin.wang@linux.alibaba.com Signed-off-by: Baolin Wang Suggested-by: Vlastimil Babka Acked-by: Vlastimil Babka Reviewed-by: Zi Yan Acked-by: David Hildenbrand Cc: Mel Gorman Signed-off-by: Andrew Morton --- include/linux/pageblock-flags.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h index 3f2409b968ec6..547e82cdc89a4 100644 --- a/include/linux/pageblock-flags.h +++ b/include/linux/pageblock-flags.h @@ -28,7 +28,7 @@ enum pageblock_bits { NR_PAGEBLOCK_BITS }; -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE @@ -45,7 +45,11 @@ extern unsigned int pageblock_order; #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ -#else /* CONFIG_HUGETLB_PAGE */ +#elif defined(CONFIG_TRANSPARENT_HUGEPAGE) + +#define pageblock_order min_t(unsigned int, HPAGE_PMD_ORDER, MAX_PAGE_ORDER) + +#else /* CONFIG_TRANSPARENT_HUGEPAGE */ /* If huge pages are not used, group by MAX_ORDER_NR_PAGES */ #define pageblock_order MAX_PAGE_ORDER From 30dd3478c3cd7d01cc5afc4952e885ba4eefb730 Mon Sep 17 00:00:00 2001 From: Joseph Qi Date: Thu, 14 Mar 2024 10:17:13 +0800 Subject: [PATCH 0778/1702] ocfs2: correctly use ocfs2_find_next_zero_bit() If no bits are zero, ocfs2_find_next_zero_bit() will return max size, so check the return value with -1 is meaningless. Correct this usage and cleanup the code. Link: https://lkml.kernel.org/r/20240314021713.240796-1-joseph.qi@linux.alibaba.com Signed-off-by: Joseph Qi Reviewed-by: Heming Zhao Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Signed-off-by: Andrew Morton --- fs/ocfs2/localalloc.c | 19 ++++++------------- fs/ocfs2/reservations.c | 2 +- fs/ocfs2/suballoc.c | 6 ++---- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/fs/ocfs2/localalloc.c b/fs/ocfs2/localalloc.c index c803c10dd97ef..33aeaaa056d70 100644 --- a/fs/ocfs2/localalloc.c +++ b/fs/ocfs2/localalloc.c @@ -863,14 +863,8 @@ static int ocfs2_local_alloc_find_clear_bits(struct ocfs2_super *osb, numfound = bitoff = startoff = 0; left = le32_to_cpu(alloc->id1.bitmap1.i_total); - while ((bitoff = ocfs2_find_next_zero_bit(bitmap, left, startoff)) != -1) { - if (bitoff == left) { - /* mlog(0, "bitoff (%d) == left", bitoff); */ - break; - } - /* mlog(0, "Found a zero: bitoff = %d, startoff = %d, " - "numfound = %d\n", bitoff, startoff, numfound);*/ - + while ((bitoff = ocfs2_find_next_zero_bit(bitmap, left, startoff)) < + left) { /* Ok, we found a zero bit... is it contig. or do we * start over?*/ if (bitoff == startoff) { @@ -976,9 +970,9 @@ static int ocfs2_sync_local_to_main(struct ocfs2_super *osb, start = count = 0; left = le32_to_cpu(alloc->id1.bitmap1.i_total); - while ((bit_off = ocfs2_find_next_zero_bit(bitmap, left, start)) - != -1) { - if ((bit_off < left) && (bit_off == start)) { + while ((bit_off = ocfs2_find_next_zero_bit(bitmap, left, start)) < + left) { + if (bit_off == start) { count++; start++; continue; @@ -1002,8 +996,7 @@ static int ocfs2_sync_local_to_main(struct ocfs2_super *osb, goto bail; } } - if (bit_off >= left) - break; + count = 1; start = bit_off + 1; } diff --git a/fs/ocfs2/reservations.c b/fs/ocfs2/reservations.c index a9d1296d736dc..1fe61974d9f02 100644 --- a/fs/ocfs2/reservations.c +++ b/fs/ocfs2/reservations.c @@ -414,7 +414,7 @@ static int ocfs2_resmap_find_free_bits(struct ocfs2_reservation_map *resmap, start = search_start; while ((offset = ocfs2_find_next_zero_bit(bitmap, resmap->m_bitmap_len, - start)) != -1) { + start)) < resmap->m_bitmap_len) { /* Search reached end of the region */ if (offset >= (search_start + search_len)) break; diff --git a/fs/ocfs2/suballoc.c b/fs/ocfs2/suballoc.c index 166c8918c825a..961998415308d 100644 --- a/fs/ocfs2/suballoc.c +++ b/fs/ocfs2/suballoc.c @@ -1290,10 +1290,8 @@ static int ocfs2_block_group_find_clear_bits(struct ocfs2_super *osb, found = start = best_offset = best_size = 0; bitmap = bg->bg_bitmap; - while((offset = ocfs2_find_next_zero_bit(bitmap, total_bits, start)) != -1) { - if (offset == total_bits) - break; - + while ((offset = ocfs2_find_next_zero_bit(bitmap, total_bits, start)) < + total_bits) { if (!ocfs2_test_bg_bit_allocatable(bg_bh, offset)) { /* We found a zero, but we can't use it as it * hasn't been put to disk yet! */ From c9abe099865b2d0df66346657b76922d8efd43e4 Mon Sep 17 00:00:00 2001 From: Su Yue Date: Mon, 18 Mar 2024 19:56:09 +0800 Subject: [PATCH 0779/1702] ocfs2: update inode ctime in ocfs2_fileattr_set inode ctime should be updated if ocfs2_fileattr_set is called. Link: https://lkml.kernel.org/r/20240318115609.3194-1-l@damenly.org Signed-off-by: Su Yue Reviewed-by: Joseph Qi Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Signed-off-by: Andrew Morton --- fs/ocfs2/ioctl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/ocfs2/ioctl.c b/fs/ocfs2/ioctl.c index b1550ba73f963..71beef7f8a60b 100644 --- a/fs/ocfs2/ioctl.c +++ b/fs/ocfs2/ioctl.c @@ -125,6 +125,7 @@ int ocfs2_fileattr_set(struct mnt_idmap *idmap, ocfs2_inode->ip_attr = flags; ocfs2_set_inode_flags(inode); + inode_set_ctime_current(inode); status = ocfs2_mark_inode_dirty(handle, inode, bh); if (status < 0) From 5ef6dc08cfde240b8c748733759185646e654570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 13 Mar 2024 22:19:56 +0100 Subject: [PATCH 0780/1702] lib/build_OID_registry: don't mention the full path of the script in output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change strips the full path of the script generating lib/oid_registry_data.c to just lib/build_OID_registry. The motivation for this change is Yocto emitting a build warning File /usr/src/debug/linux-lxatac/6.7-r0/lib/oid_registry_data.c in package linux-lxatac-src contains reference to TMPDIR [buildpaths] So this change brings us one step closer to make the build result reproducible independent of the build path. Link: https://lkml.kernel.org/r/20240313211957.884561-2-u.kleine-koenig@pengutronix.de Signed-off-by: Uwe Kleine-König Cc: Masahiro Yamada Reviewed-by: Nicolas Schier Signed-off-by: Andrew Morton --- lib/build_OID_registry | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/build_OID_registry b/lib/build_OID_registry index d7fc32ea8ac22..56d8bafeb848b 100755 --- a/lib/build_OID_registry +++ b/lib/build_OID_registry @@ -8,6 +8,7 @@ # use strict; +use Cwd qw(abs_path); my @names = (); my @oids = (); @@ -17,6 +18,8 @@ if ($#ARGV != 1) { exit(2); } +my $abs_srctree = abs_path($ENV{'srctree'}); + # # Open the file to read from # @@ -35,7 +38,7 @@ close IN_FILE || die; # open C_FILE, ">$ARGV[1]" or die; print C_FILE "/*\n"; -print C_FILE " * Automatically generated by ", $0, ". Do not edit\n"; +print C_FILE " * Automatically generated by ", $0 =~ s#^\Q$abs_srctree/\E##r, ". Do not edit\n"; print C_FILE " */\n"; # From 212f863fa8811c780abacc1d0404c573fdc0a2de Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Wed, 20 Mar 2024 11:19:52 +0100 Subject: [PATCH 0781/1702] bootconfig: do not put quotes on cmdline items unless necessary When trying to migrate to using bootconfig to embed the kernel's and PID1's command line with the kernel image itself, and so allowing changing that without modifying the bootloader, I noticed that /proc/cmdline changed from e.g. console=ttymxc0,115200n8 cma=128M quiet -- --log-level=notice to console="ttymxc0,115200n8" cma="128M" quiet -- --log-level="notice" The kernel parameters are parsed just fine, and the quotes are indeed stripped from the actual argv[] given to PID1. However, the quoting doesn't really serve any purpose and looks excessive, and might confuse some (naive) userspace tool trying to parse /proc/cmdline. So do not quote the value unless it contains whitespace. Link: https://lkml.kernel.org/r/20240320101952.62135-1-linux@rasmusvillemoes.dk Signed-off-by: Rasmus Villemoes Cc: Masami Hiramatsu Signed-off-by: Andrew Morton --- init/main.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/init/main.c b/init/main.c index 881f6230ee59e..6b4b656cef1d9 100644 --- a/init/main.c +++ b/init/main.c @@ -327,7 +327,7 @@ static int __init xbc_snprint_cmdline(char *buf, size_t size, { struct xbc_node *knode, *vnode; char *end = buf + size; - const char *val; + const char *val, *q; int ret; xbc_node_for_each_key_value(root, knode, val) { @@ -345,8 +345,14 @@ static int __init xbc_snprint_cmdline(char *buf, size_t size, continue; } xbc_array_for_each_value(vnode, val) { - ret = snprintf(buf, rest(buf, end), "%s=\"%s\" ", - xbc_namebuf, val); + /* + * For prettier and more readable /proc/cmdline, only + * quote the value when necessary, i.e. when it contains + * whitespace. + */ + q = strpbrk(val, " \t\r\n") ? "\"" : ""; + ret = snprintf(buf, rest(buf, end), "%s=%s%s%s ", + xbc_namebuf, q, val, q); if (ret < 0) return ret; buf += ret; From 3429055f0451cd3a281f8ed6691335ead626b136 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Wed, 20 Mar 2024 11:18:49 +0100 Subject: [PATCH 0782/1702] mm: kmsan: implement kmsan_memmove() Provide a hook that can be used by custom memcpy implementations to tell KMSAN that the metadata needs to be copied. Without that, false positive reports are possible in the cases where KMSAN fails to intercept memory initialization. Link: https://lore.kernel.org/all/3b7dbd88-0861-4638-b2d2-911c97a4cadf@I-love.SAKURA.ne.jp/ Link: https://lkml.kernel.org/r/20240320101851.2589698-1-glider@google.com Signed-off-by: Alexander Potapenko Suggested-by: Tetsuo Handa Reviewed-by: Marco Elver Cc: Dmitry Vyukov Cc: Linus Torvalds Cc: Tetsuo Handa Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- include/linux/kmsan-checks.h | 15 +++++++++++++++ mm/kmsan/hooks.c | 11 +++++++++++ 2 files changed, 26 insertions(+) diff --git a/include/linux/kmsan-checks.h b/include/linux/kmsan-checks.h index c4cae333deec5..e1082dc40abc2 100644 --- a/include/linux/kmsan-checks.h +++ b/include/linux/kmsan-checks.h @@ -61,6 +61,17 @@ void kmsan_check_memory(const void *address, size_t size); void kmsan_copy_to_user(void __user *to, const void *from, size_t to_copy, size_t left); +/** + * kmsan_memmove() - Notify KMSAN about a data copy within kernel. + * @to: destination address in the kernel. + * @from: source address in the kernel. + * @size: number of bytes to copy. + * + * Invoked after non-instrumented version (e.g. implemented using assembly + * code) of memmove()/memcpy() is called, in order to copy KMSAN's metadata. + */ +void kmsan_memmove(void *to, const void *from, size_t to_copy); + #else static inline void kmsan_poison_memory(const void *address, size_t size, @@ -78,6 +89,10 @@ static inline void kmsan_copy_to_user(void __user *to, const void *from, { } +static inline void kmsan_memmove(void *to, const void *from, size_t to_copy) +{ +} + #endif #endif /* _LINUX_KMSAN_CHECKS_H */ diff --git a/mm/kmsan/hooks.c b/mm/kmsan/hooks.c index 0b09daa188ef6..22e8657800eff 100644 --- a/mm/kmsan/hooks.c +++ b/mm/kmsan/hooks.c @@ -285,6 +285,17 @@ void kmsan_copy_to_user(void __user *to, const void *from, size_t to_copy, } EXPORT_SYMBOL(kmsan_copy_to_user); +void kmsan_memmove(void *to, const void *from, size_t size) +{ + if (!kmsan_enabled || kmsan_in_runtime()) + return; + + kmsan_enter_runtime(); + kmsan_internal_memmove_metadata(to, (void *)from, size); + kmsan_leave_runtime(); +} +EXPORT_SYMBOL(kmsan_memmove); + /* Helper function to check an URB. */ void kmsan_handle_urb(const struct urb *urb, bool is_out) { From 922621a6828430ea3119b869336157d253489334 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Wed, 20 Mar 2024 11:18:50 +0100 Subject: [PATCH 0783/1702] instrumented.h: add instrument_memcpy_before, instrument_memcpy_after Bug detection tools based on compiler instrumentation may miss memory accesses in custom memcpy implementations (such as copy_mc_to_kernel). Provide instrumentation hooks that tell KASAN, KCSAN, and KMSAN about such accesses. Link: https://lore.kernel.org/all/3b7dbd88-0861-4638-b2d2-911c97a4cadf@I-love.SAKURA.ne.jp/ Link: https://lkml.kernel.org/r/20240320101851.2589698-2-glider@google.com Signed-off-by: Alexander Potapenko Reviewed-by: Marco Elver Cc: Dmitry Vyukov Cc: Tetsuo Handa Cc: Linus Torvalds Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- include/linux/instrumented.h | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/include/linux/instrumented.h b/include/linux/instrumented.h index 1b608e00290aa..711a1f0d1a735 100644 --- a/include/linux/instrumented.h +++ b/include/linux/instrumented.h @@ -147,6 +147,41 @@ instrument_copy_from_user_after(const void *to, const void __user *from, kmsan_unpoison_memory(to, n - left); } +/** + * instrument_memcpy_before - add instrumentation before non-instrumented memcpy + * @to: destination address + * @from: source address + * @n: number of bytes to copy + * + * Instrument memory accesses that happen in custom memcpy implementations. The + * instrumentation should be inserted before the memcpy call. + */ +static __always_inline void instrument_memcpy_before(void *to, const void *from, + unsigned long n) +{ + kasan_check_write(to, n); + kasan_check_read(from, n); + kcsan_check_write(to, n); + kcsan_check_read(from, n); +} + +/** + * instrument_memcpy_after - add instrumentation after non-instrumented memcpy + * @to: destination address + * @from: source address + * @n: number of bytes to copy + * @left: number of bytes not copied (if known) + * + * Instrument memory accesses that happen in custom memcpy implementations. The + * instrumentation should be inserted after the memcpy call. + */ +static __always_inline void instrument_memcpy_after(void *to, const void *from, + unsigned long n, + unsigned long left) +{ + kmsan_memmove(to, from, n - left); +} + /** * instrument_get_user() - add instrumentation to get_user()-like macros * @to: destination variable, may not be address-taken From 61b258b0d2f60858888e248da9744d63e0fa6856 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Wed, 20 Mar 2024 11:18:51 +0100 Subject: [PATCH 0784/1702] x86: call instrumentation hooks from copy_mc.c Memory accesses in copy_mc_to_kernel() and copy_mc_to_user() are performed by assembly routines and are invisible to KASAN, KCSAN, and KMSAN. Add hooks from instrumentation.h to tell the tools these functions have memcpy/copy_from_user semantics. The call to copy_mc_fragile() in copy_mc_fragile_handle_tail() is left intact, because the latter is only called from the assembly implementation of copy_mc_fragile(), so the memory accesses in it are covered by the instrumentation in copy_mc_to_kernel() and copy_mc_to_user(). Link: https://lore.kernel.org/all/3b7dbd88-0861-4638-b2d2-911c97a4cadf@I-love.SAKURA.ne.jp/ Link: https://lkml.kernel.org/r/20240320101851.2589698-3-glider@google.com Signed-off-by: Alexander Potapenko Suggested-by: Linus Torvalds Cc: Dmitry Vyukov Cc: Marco Elver Cc: Tetsuo Handa Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- arch/x86/lib/copy_mc.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/arch/x86/lib/copy_mc.c b/arch/x86/lib/copy_mc.c index 6e8b7e600def5..97e88e58567bf 100644 --- a/arch/x86/lib/copy_mc.c +++ b/arch/x86/lib/copy_mc.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -61,10 +62,20 @@ unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned */ unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len) { - if (copy_mc_fragile_enabled) - return copy_mc_fragile(dst, src, len); - if (static_cpu_has(X86_FEATURE_ERMS)) - return copy_mc_enhanced_fast_string(dst, src, len); + unsigned long ret; + + if (copy_mc_fragile_enabled) { + instrument_memcpy_before(dst, src, len); + ret = copy_mc_fragile(dst, src, len); + instrument_memcpy_after(dst, src, len, ret); + return ret; + } + if (static_cpu_has(X86_FEATURE_ERMS)) { + instrument_memcpy_before(dst, src, len); + ret = copy_mc_enhanced_fast_string(dst, src, len); + instrument_memcpy_after(dst, src, len, ret); + return ret; + } memcpy(dst, src, len); return 0; } @@ -75,6 +86,7 @@ unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, un unsigned long ret; if (copy_mc_fragile_enabled) { + instrument_copy_to_user(dst, src, len); __uaccess_begin(); ret = copy_mc_fragile((__force void *)dst, src, len); __uaccess_end(); @@ -82,6 +94,7 @@ unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, un } if (static_cpu_has(X86_FEATURE_ERMS)) { + instrument_copy_to_user(dst, src, len); __uaccess_begin(); ret = copy_mc_enhanced_fast_string((__force void *)dst, src, len); __uaccess_end(); From 4d9784c00a15631b68a5d95be907aa308f269551 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Fri, 22 Mar 2024 15:37:24 +0800 Subject: [PATCH 0785/1702] fs: add kernel-doc comments to fat_parse_long() Add kernel-doc style comments with complete parameter descriptions for fat_parse_long(). Link: https://lkml.kernel.org/r/20240322073724.102332-1-yang.lee@linux.alibaba.com Signed-off-by: Yang Li Acked-by: OGAWA Hirofumi Signed-off-by: Andrew Morton --- fs/fat/dir.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/fs/fat/dir.c b/fs/fat/dir.c index 00235b8a1823a..acbec5bdd5210 100644 --- a/fs/fat/dir.c +++ b/fs/fat/dir.c @@ -269,6 +269,18 @@ enum { PARSE_INVALID = 1, PARSE_NOT_LONGNAME, PARSE_EOF, }; /** * fat_parse_long - Parse extended directory entry. * + * @dir: Pointer to the inode that represents the directory. + * @pos: On input, contains the starting position to read from. + * On output, updated with the new position. + * @bh: Pointer to the buffer head that may be used for reading directory + * entries. May be updated. + * @de: On input, points to the current directory entry. + * On output, points to the next directory entry. + * @unicode: Pointer to a buffer where the parsed Unicode long filename will be + * stored. + * @nr_slots: Pointer to a variable that will store the number of longname + * slots found. + * * This function returns zero on success, negative value on error, or one of * the following: * From f9899c028151468d8c4af0bcbb3d5e87619b0973 Mon Sep 17 00:00:00 2001 From: Huang Shijie Date: Fri, 26 Jan 2024 14:44:51 +0800 Subject: [PATCH 0786/1702] NUMA: early use of cpu_to_node() returns 0 instead of the correct node id During the kernel booting, the generic cpu_to_node() is called too early in arm64, powerpc and riscv when CONFIG_NUMA is enabled. There are at least four places in the common code where the generic cpu_to_node() is called before it is initialized: 1.) early_trace_init() in kernel/trace/trace.c 2.) sched_init() in kernel/sched/core.c 3.) init_sched_fair_class() in kernel/sched/fair.c 4.) workqueue_init_early() in kernel/workqueue.c This will harm performance since there is an increase in off node accesses. In order to fix the bug, the patch introduces early_numa_node_init() which is called after smp_prepare_boot_cpu() in start_kernel. early_numa_node_init will initialize the "numa_node" as soon as the early_cpu_to_node() is ready, before the cpu_to_node() is called at the first time. Link: https://lkml.kernel.org/r/20240126064451.5465-1-shijie@os.amperecomputing.com Signed-off-by: Huang Shijie Acked-by: Palmer Dabbelt [RISC-V] Cc: Albert Ou Cc: Arnd Bergmann Cc: Catalin Marinas Cc: Greg Kroah-Hartman Cc: Huacai Chen Cc: Ingo Molnar Cc: Jakub Kicinski Cc: Jiaxun Yang Cc: Josh Poimboeuf Cc: Mark Rutland Cc: Masami Hiramatsu Cc: Michael Ellerman Cc: Michael Kelley (LINUX) Cc: "Mike Rapoport (IBM)" Cc: Nick Desaulniers Cc: Paul Walmsley Cc: Rafael J. Wysocki Cc: Thomas Gleixner Cc: Valentin Schneider Cc: Vlastimil Babka Cc: Will Deacon Cc: Yury Norov Signed-off-by: Andrew Morton --- init/main.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/init/main.c b/init/main.c index 6b4b656cef1d9..0aecd2839c1fd 100644 --- a/init/main.c +++ b/init/main.c @@ -882,6 +882,19 @@ static void __init print_unknown_bootoptions(void) memblock_free(unknown_options, len); } +static void __init early_numa_node_init(void) +{ +#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID +#ifndef cpu_to_node + int cpu; + + /* The early_cpu_to_node() should be ready here. */ + for_each_possible_cpu(cpu) + set_cpu_numa_node(cpu, early_cpu_to_node(cpu)); +#endif +#endif +} + asmlinkage __visible __init __no_sanitize_address __noreturn __no_stack_protector void start_kernel(void) { @@ -912,6 +925,7 @@ void start_kernel(void) setup_nr_cpu_ids(); setup_per_cpu_areas(); smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */ + early_numa_node_init(); boot_cpu_hotplug_init(); pr_notice("Kernel command line: %s\n", saved_command_line); From 6b839b3b76cf17296ebd4a893841f32cae08229c Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Mon, 5 Feb 2024 09:26:30 -0800 Subject: [PATCH 0787/1702] regset: use kvzalloc() for regset_get_alloc() While browsing through ChromeOS crash reports, I found one with an allocation failure that looked like this: chrome: page allocation failure: order:7, mode:0x40dc0(GFP_KERNEL|__GFP_COMP|__GFP_ZERO), nodemask=(null),cpuset=urgent,mems_allowed=0 CPU: 7 PID: 3295 Comm: chrome Not tainted 5.15.133-20574-g8044615ac35c #1 (HASH:1162 1) Hardware name: Google Lazor (rev3 - 8) with KB Backlight (DT) Call trace: ... warn_alloc+0x104/0x174 __alloc_pages+0x5f0/0x6e4 kmalloc_order+0x44/0x98 kmalloc_order_trace+0x34/0x124 __kmalloc+0x228/0x36c __regset_get+0x68/0xcc regset_get_alloc+0x1c/0x28 elf_core_dump+0x3d8/0xd8c do_coredump+0xeb8/0x1378 get_signal+0x14c/0x804 ... An order 7 allocation is (1 << 7) contiguous pages, or 512K. It's not a surprise that this allocation failed on a system that's been running for a while. More digging showed that it was fairly easy to see the order 7 allocation by just sending a SIGQUIT to chrome (or other processes) to generate a core dump. The actual amount being allocated was 279,584 bytes and it was for "core_note_type" NT_ARM_SVE. There was quite a bit of discussion [1] on the mailing lists in response to my v1 patch attempting to switch to vmalloc. The overall conclusion was that we could likely reduce the 279,584 byte allocation by quite a bit and Mark Brown has sent a patch to that effect [2]. However even with the 279,584 byte allocation gone there are still 65,552 byte allocations. These are just barely more than the 65,536 bytes and thus would require an order 5 allocation. An order 5 allocation is still something to avoid unless necessary and nothing needs the memory here to be contiguous. Change the allocation to kvzalloc() which should still be efficient for small allocations but doesn't force the memory subsystem to work hard (and maybe fail) at getting a large contiguous chunk. [1] https://lore.kernel.org/r/20240201171159.1.Id9ad163b60d21c9e56c2d686b0cc9083a8ba7924@changeid [2] https://lore.kernel.org/r/20240203-arm64-sve-ptrace-regset-size-v1-1-2c3ba1386b9e@kernel.org Link: https://lkml.kernel.org/r/20240205092626.v2.1.Id9ad163b60d21c9e56c2d686b0cc9083a8ba7924@changeid Signed-off-by: Douglas Anderson Reviewed-by: Catalin Marinas Cc: Al Viro Cc: Christian Brauner Cc: Dave Martin Cc: Eric Biederman Cc: Jan Kara Cc: Kees Cook Cc: Mark Brown Cc: Matthew Wilcox (Oracle) Cc: Oleg Nesterov Cc: Will Deacon Signed-off-by: Andrew Morton --- fs/binfmt_elf.c | 2 +- kernel/regset.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 5397b552fbeb5..ac178ad388239 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1928,7 +1928,7 @@ static void free_note_info(struct elf_note_info *info) threads = t->next; WARN_ON(t->notes[0].data && t->notes[0].data != &t->prstatus); for (i = 1; i < info->thread_notes; ++i) - kfree(t->notes[i].data); + kvfree(t->notes[i].data); kfree(t); } kfree(info->psinfo.data); diff --git a/kernel/regset.c b/kernel/regset.c index 586823786f397..b2871fa68b2a7 100644 --- a/kernel/regset.c +++ b/kernel/regset.c @@ -16,14 +16,14 @@ static int __regset_get(struct task_struct *target, if (size > regset->n * regset->size) size = regset->n * regset->size; if (!p) { - to_free = p = kzalloc(size, GFP_KERNEL); + to_free = p = kvzalloc(size, GFP_KERNEL); if (!p) return -ENOMEM; } res = regset->regset_get(target, regset, (struct membuf){.p = p, .left = size}); if (res < 0) { - kfree(to_free); + kvfree(to_free); return res; } *data = p; @@ -71,6 +71,6 @@ int copy_regset_to_user(struct task_struct *target, ret = regset_get_alloc(target, regset, size, &buf); if (ret > 0) ret = copy_to_user(data, buf, ret) ? -EFAULT : 0; - kfree(buf); + kvfree(buf); return ret; } From 4eb7b93e03101fd3f35e69affe566e4b1e3e3dca Mon Sep 17 00:00:00 2001 From: Heming Zhao Date: Thu, 28 Mar 2024 20:52:00 +0800 Subject: [PATCH 0788/1702] ocfs2: improve write IO performance when fragmentation is high The group_search function ocfs2_cluster_group_search() should bypass groups with insufficient space to avoid unnecessary searches. This patch is particularly useful when ocfs2 is handling huge number small files, and volume fragmentation is very high. In this case, ocfs2 is busy with looking up available la window from //global_bitmap. This patch introduces a new member in the Group Description (gd) struct called 'bg_contig_free_bits', representing the max contigous free bits in this gd. When ocfs2 allocates a new la window from //global_bitmap, 'bg_contig_free_bits' helps expedite the search process. Let's image below path. 1. la state (->local_alloc_state) is set THROTTLED or DISABLED. 2. when user delete a large file and trigger ocfs2_local_alloc_seen_free_bits set osb->local_alloc_state unconditionally. 3. a write IOs thread run and trigger the worst performance path ``` ocfs2_reserve_clusters_with_limit ocfs2_reserve_local_alloc_bits ocfs2_local_alloc_slide_window //[1] + ocfs2_local_alloc_reserve_for_window //[2] + ocfs2_local_alloc_new_window //[3] ocfs2_recalc_la_window ``` [1]: will be called when la window bits used up. [2]: under la state is ENABLED, and this func only check global_bitmap free bits, it will succeed in general. [3]: will use the default la window size to search clusters then fail. ocfs2_recalc_la_window attempts other la window sizes. the timing complexity is O(n^4), resulting in a significant time cost for scanning global bitmap. This leads to a dramatic slowdown in write I/Os (e.g., user space 'dd'). i.e. an ocfs2 partition size: 1.45TB, cluster size: 4KB, la window default size: 106MB. The partition is fragmentation by creating & deleting huge mount of small files. before this patch, the timing of [3] should be (the number got from real world): - la window size change order (size: MB): 106, 53, 26.5, 13, 6.5, 3.25, 1.6, 0.8 only 0.8MB succeed, 0.8MB also triggers la window to disable. ocfs2_local_alloc_new_window retries 8 times, first 7 times totally runs in worst case. - group chain number: 242 ocfs2_claim_suballoc_bits calls for-loop 242 times - each chain has 49 block group ocfs2_search_chain calls while-loop 49 times - each bg has 32256 blocks ocfs2_block_group_find_clear_bits calls while-loop for 32256 bits. for ocfs2_find_next_zero_bit uses ffz() to find zero bit, let's use (32256/64) (this is not worst value) for timing calucation. the loop times: 7*242*49*(32256/64) = 41835024 (~42 million times) In the worst case, user space writes 1MB data will trigger 42M scanning times. under this patch, the timing is '7*242*49 = 83006', reduced by three orders of magnitude. Link: https://lkml.kernel.org/r/20240328125203.20892-2-heming.zhao@suse.com Signed-off-by: Heming Zhao Reviewed-by: Joseph Qi Cc: Changwei Ge Cc: Gang He Cc: Joel Becker Cc: Jun Piao Cc: Junxiao Bi Cc: Mark Fasheh Signed-off-by: Andrew Morton --- fs/ocfs2/move_extents.c | 2 +- fs/ocfs2/ocfs2_fs.h | 3 +- fs/ocfs2/resize.c | 8 ++++ fs/ocfs2/suballoc.c | 100 ++++++++++++++++++++++++++++++++++++---- fs/ocfs2/suballoc.h | 6 ++- 5 files changed, 108 insertions(+), 11 deletions(-) diff --git a/fs/ocfs2/move_extents.c b/fs/ocfs2/move_extents.c index 1f9ed117e78b6..f9d6a4f9ca921 100644 --- a/fs/ocfs2/move_extents.c +++ b/fs/ocfs2/move_extents.c @@ -685,7 +685,7 @@ static int ocfs2_move_extent(struct ocfs2_move_extents_context *context, } ret = ocfs2_block_group_set_bits(handle, gb_inode, gd, gd_bh, - goal_bit, len); + goal_bit, len, 0, 0); if (ret) { ocfs2_rollback_alloc_dinode_counts(gb_inode, gb_bh, len, le16_to_cpu(gd->bg_chain)); diff --git a/fs/ocfs2/ocfs2_fs.h b/fs/ocfs2/ocfs2_fs.h index 7aebdbf5cc0a5..c93689b568fe0 100644 --- a/fs/ocfs2/ocfs2_fs.h +++ b/fs/ocfs2/ocfs2_fs.h @@ -883,7 +883,8 @@ struct ocfs2_group_desc __le16 bg_free_bits_count; /* Free bits count */ __le16 bg_chain; /* What chain I am in. */ /*10*/ __le32 bg_generation; - __le32 bg_reserved1; + __le16 bg_contig_free_bits; /* max contig free bits length */ + __le16 bg_reserved1; __le64 bg_next_group; /* Next group in my list, in blocks */ /*20*/ __le64 bg_parent_dinode; /* dinode which owns me, in diff --git a/fs/ocfs2/resize.c b/fs/ocfs2/resize.c index d65d43c61857a..c4a4016d3866a 100644 --- a/fs/ocfs2/resize.c +++ b/fs/ocfs2/resize.c @@ -91,6 +91,8 @@ static int ocfs2_update_last_group_and_inode(handle_t *handle, u16 cl_bpc = le16_to_cpu(cl->cl_bpc); u16 cl_cpg = le16_to_cpu(cl->cl_cpg); u16 old_bg_clusters; + u16 contig_bits; + __le16 old_bg_contig_free_bits; trace_ocfs2_update_last_group_and_inode(new_clusters, first_new_cluster); @@ -122,6 +124,11 @@ static int ocfs2_update_last_group_and_inode(handle_t *handle, le16_add_cpu(&group->bg_free_bits_count, -1 * backups); } + contig_bits = ocfs2_find_max_contig_free_bits(group->bg_bitmap, + le16_to_cpu(group->bg_bits), 0); + old_bg_contig_free_bits = group->bg_contig_free_bits; + group->bg_contig_free_bits = cpu_to_le16(contig_bits); + ocfs2_journal_dirty(handle, group_bh); /* update the inode accordingly. */ @@ -160,6 +167,7 @@ static int ocfs2_update_last_group_and_inode(handle_t *handle, le16_add_cpu(&group->bg_free_bits_count, backups); le16_add_cpu(&group->bg_bits, -1 * num_bits); le16_add_cpu(&group->bg_free_bits_count, -1 * num_bits); + group->bg_contig_free_bits = old_bg_contig_free_bits; } out: if (ret) diff --git a/fs/ocfs2/suballoc.c b/fs/ocfs2/suballoc.c index 961998415308d..7e30e9d798906 100644 --- a/fs/ocfs2/suballoc.c +++ b/fs/ocfs2/suballoc.c @@ -50,6 +50,10 @@ struct ocfs2_suballoc_result { u64 sr_blkno; /* The first allocated block */ unsigned int sr_bit_offset; /* The bit in the bg */ unsigned int sr_bits; /* How many bits we claimed */ + unsigned int sr_max_contig_bits; /* The length for contiguous + * free bits, only available + * for cluster group + */ }; static u64 ocfs2_group_from_res(struct ocfs2_suballoc_result *res) @@ -1272,6 +1276,26 @@ static int ocfs2_test_bg_bit_allocatable(struct buffer_head *bg_bh, return ret; } +u16 ocfs2_find_max_contig_free_bits(void *bitmap, + u16 total_bits, u16 start) +{ + u16 offset, free_bits; + u16 contig_bits = 0; + + while (start < total_bits) { + offset = ocfs2_find_next_zero_bit(bitmap, total_bits, start); + if (offset == total_bits) + break; + + start = ocfs2_find_next_bit(bitmap, total_bits, offset); + free_bits = start - offset; + if (contig_bits < free_bits) + contig_bits = free_bits; + } + + return contig_bits; +} + static int ocfs2_block_group_find_clear_bits(struct ocfs2_super *osb, struct buffer_head *bg_bh, unsigned int bits_wanted, @@ -1280,6 +1304,7 @@ static int ocfs2_block_group_find_clear_bits(struct ocfs2_super *osb, { void *bitmap; u16 best_offset, best_size; + u16 prev_best_size = 0; int offset, start, found, status = 0; struct ocfs2_group_desc *bg = (struct ocfs2_group_desc *) bg_bh->b_data; @@ -1306,6 +1331,7 @@ static int ocfs2_block_group_find_clear_bits(struct ocfs2_super *osb, /* got a zero after some ones */ found = 1; start = offset + 1; + prev_best_size = best_size; } if (found > best_size) { best_size = found; @@ -1318,6 +1344,8 @@ static int ocfs2_block_group_find_clear_bits(struct ocfs2_super *osb, } } + /* best_size will be allocated, we save prev_best_size */ + res->sr_max_contig_bits = prev_best_size; if (best_size) { res->sr_bit_offset = best_offset; res->sr_bits = best_size; @@ -1335,11 +1363,15 @@ int ocfs2_block_group_set_bits(handle_t *handle, struct ocfs2_group_desc *bg, struct buffer_head *group_bh, unsigned int bit_off, - unsigned int num_bits) + unsigned int num_bits, + unsigned int max_contig_bits, + int fastpath) { int status; void *bitmap = bg->bg_bitmap; int journal_type = OCFS2_JOURNAL_ACCESS_WRITE; + unsigned int start = bit_off + num_bits; + u16 contig_bits; /* All callers get the descriptor via * ocfs2_read_group_descriptor(). Any corruption is a code bug. */ @@ -1371,6 +1403,28 @@ int ocfs2_block_group_set_bits(handle_t *handle, while(num_bits--) ocfs2_set_bit(bit_off++, bitmap); + /* + * this is optimize path, caller set old contig value + * in max_contig_bits to bypass finding action. + */ + if (fastpath) { + bg->bg_contig_free_bits = cpu_to_le16(max_contig_bits); + } else if (ocfs2_is_cluster_bitmap(alloc_inode)) { + /* + * Usually, the block group bitmap allocates only 1 bit + * at a time, while the cluster group allocates n bits + * each time. Therefore, we only save the contig bits for + * the cluster group. + */ + contig_bits = ocfs2_find_max_contig_free_bits(bitmap, + le16_to_cpu(bg->bg_bits), start); + if (contig_bits > max_contig_bits) + max_contig_bits = contig_bits; + bg->bg_contig_free_bits = cpu_to_le16(max_contig_bits); + } else { + bg->bg_contig_free_bits = 0; + } + ocfs2_journal_dirty(handle, group_bh); bail: @@ -1484,7 +1538,12 @@ static int ocfs2_cluster_group_search(struct inode *inode, BUG_ON(!ocfs2_is_cluster_bitmap(inode)); - if (gd->bg_free_bits_count) { + if (le16_to_cpu(gd->bg_contig_free_bits) && + le16_to_cpu(gd->bg_contig_free_bits) < bits_wanted) + return -ENOSPC; + + /* ->bg_contig_free_bits may un-initialized, so compare again */ + if (le16_to_cpu(gd->bg_free_bits_count) >= bits_wanted) { max_bits = le16_to_cpu(gd->bg_bits); /* Tail groups in cluster bitmaps which aren't cpg @@ -1553,7 +1612,7 @@ static int ocfs2_block_group_search(struct inode *inode, BUG_ON(min_bits != 1); BUG_ON(ocfs2_is_cluster_bitmap(inode)); - if (bg->bg_free_bits_count) { + if (le16_to_cpu(bg->bg_free_bits_count) >= bits_wanted) { ret = ocfs2_block_group_find_clear_bits(OCFS2_SB(inode->i_sb), group_bh, bits_wanted, le16_to_cpu(bg->bg_bits), @@ -1713,7 +1772,8 @@ static int ocfs2_search_one_group(struct ocfs2_alloc_context *ac, } ret = ocfs2_block_group_set_bits(handle, alloc_inode, gd, group_bh, - res->sr_bit_offset, res->sr_bits); + res->sr_bit_offset, res->sr_bits, + res->sr_max_contig_bits, 0); if (ret < 0) { ocfs2_rollback_alloc_dinode_counts(alloc_inode, ac->ac_bh, res->sr_bits, @@ -1847,7 +1907,9 @@ static int ocfs2_search_chain(struct ocfs2_alloc_context *ac, bg, group_bh, res->sr_bit_offset, - res->sr_bits); + res->sr_bits, + res->sr_max_contig_bits, + 0); if (status < 0) { ocfs2_rollback_alloc_dinode_counts(alloc_inode, ac->ac_bh, res->sr_bits, chain); @@ -2161,7 +2223,9 @@ int ocfs2_claim_new_inode_at_loc(handle_t *handle, bg, bg_bh, res->sr_bit_offset, - res->sr_bits); + res->sr_bits, + res->sr_max_contig_bits, + 0); if (ret < 0) { ocfs2_rollback_alloc_dinode_counts(ac->ac_inode, ac->ac_bh, res->sr_bits, chain); @@ -2380,11 +2444,13 @@ static int ocfs2_block_group_clear_bits(handle_t *handle, struct buffer_head *group_bh, unsigned int bit_off, unsigned int num_bits, + unsigned int max_contig_bits, void (*undo_fn)(unsigned int bit, unsigned long *bmap)) { int status; unsigned int tmp; + u16 contig_bits; struct ocfs2_group_desc *undo_bg = NULL; struct journal_head *jh; @@ -2431,6 +2497,20 @@ static int ocfs2_block_group_clear_bits(handle_t *handle, num_bits); } + /* + * TODO: even 'num_bits == 1' (the worst case, release 1 cluster), + * we still need to rescan whole bitmap. + */ + if (ocfs2_is_cluster_bitmap(alloc_inode)) { + contig_bits = ocfs2_find_max_contig_free_bits(bg->bg_bitmap, + le16_to_cpu(bg->bg_bits), 0); + if (contig_bits > max_contig_bits) + max_contig_bits = contig_bits; + bg->bg_contig_free_bits = cpu_to_le16(max_contig_bits); + } else { + bg->bg_contig_free_bits = 0; + } + if (undo_fn) spin_unlock(&jh->b_state_lock); @@ -2457,6 +2537,7 @@ static int _ocfs2_free_suballoc_bits(handle_t *handle, struct ocfs2_chain_list *cl = &fe->id2.i_chain; struct buffer_head *group_bh = NULL; struct ocfs2_group_desc *group; + __le16 old_bg_contig_free_bits = 0; /* The alloc_bh comes from ocfs2_free_dinode() or * ocfs2_free_clusters(). The callers have all locked the @@ -2481,9 +2562,11 @@ static int _ocfs2_free_suballoc_bits(handle_t *handle, BUG_ON((count + start_bit) > le16_to_cpu(group->bg_bits)); + if (ocfs2_is_cluster_bitmap(alloc_inode)) + old_bg_contig_free_bits = group->bg_contig_free_bits; status = ocfs2_block_group_clear_bits(handle, alloc_inode, group, group_bh, - start_bit, count, undo_fn); + start_bit, count, 0, undo_fn); if (status < 0) { mlog_errno(status); goto bail; @@ -2494,7 +2577,8 @@ static int _ocfs2_free_suballoc_bits(handle_t *handle, if (status < 0) { mlog_errno(status); ocfs2_block_group_set_bits(handle, alloc_inode, group, group_bh, - start_bit, count); + start_bit, count, + le16_to_cpu(old_bg_contig_free_bits), 1); goto bail; } diff --git a/fs/ocfs2/suballoc.h b/fs/ocfs2/suballoc.h index 9c74eace3adc1..b481b834857d3 100644 --- a/fs/ocfs2/suballoc.h +++ b/fs/ocfs2/suballoc.h @@ -79,12 +79,16 @@ void ocfs2_rollback_alloc_dinode_counts(struct inode *inode, struct buffer_head *di_bh, u32 num_bits, u16 chain); +u16 ocfs2_find_max_contig_free_bits(void *bitmap, + u16 total_bits, u16 start); int ocfs2_block_group_set_bits(handle_t *handle, struct inode *alloc_inode, struct ocfs2_group_desc *bg, struct buffer_head *group_bh, unsigned int bit_off, - unsigned int num_bits); + unsigned int num_bits, + unsigned int max_contig_bits, + int fastpath); int ocfs2_claim_metadata(handle_t *handle, struct ocfs2_alloc_context *ac, From f51dac026f75863004ebfb7885cec98e6d3172bb Mon Sep 17 00:00:00 2001 From: Heming Zhao Date: Thu, 28 Mar 2024 20:52:01 +0800 Subject: [PATCH 0789/1702] ocfs2: adjust enabling place for la window Patch series "improve write IO performance when fragmentation is high", v6. This patch (of 4): After introducing gd->bg_contig_free_bits, the code path 'ocfs2_cluster_group_search() => ocfs2_local_alloc_seen_free_bits()' becomes death when all the gd->bg_contig_free_bits are set to the correct value. This patch relocates ocfs2_local_alloc_seen_free_bits() to a more appropriate location. (The new place being ocfs2_block_group_set_bits().) In ocfs2_local_alloc_seen_free_bits(), the scope of the spin-lock has been adjusted to reduce meaningless lock races. e.g: when userspace creates & deletes 1 cluster_size files in parallel, acquiring the spin-lock in ocfs2_local_alloc_seen_free_bits() is totally pointless and impedes IO performance. Link: https://lkml.kernel.org/r/20240328125203.20892-3-heming.zhao@suse.com Signed-off-by: Heming Zhao Reviewed-by: Joseph Qi Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Signed-off-by: Andrew Morton --- fs/ocfs2/localalloc.c | 11 ++++++----- fs/ocfs2/suballoc.c | 9 ++------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/fs/ocfs2/localalloc.c b/fs/ocfs2/localalloc.c index 33aeaaa056d70..c84ce53cdec03 100644 --- a/fs/ocfs2/localalloc.c +++ b/fs/ocfs2/localalloc.c @@ -212,14 +212,15 @@ static inline int ocfs2_la_state_enabled(struct ocfs2_super *osb) void ocfs2_local_alloc_seen_free_bits(struct ocfs2_super *osb, unsigned int num_clusters) { - spin_lock(&osb->osb_lock); - if (osb->local_alloc_state == OCFS2_LA_DISABLED || - osb->local_alloc_state == OCFS2_LA_THROTTLED) - if (num_clusters >= osb->local_alloc_default_bits) { + if (num_clusters >= osb->local_alloc_default_bits) { + spin_lock(&osb->osb_lock); + if (osb->local_alloc_state == OCFS2_LA_DISABLED || + osb->local_alloc_state == OCFS2_LA_THROTTLED) { cancel_delayed_work(&osb->la_enable_wq); osb->local_alloc_state = OCFS2_LA_ENABLED; } - spin_unlock(&osb->osb_lock); + spin_unlock(&osb->osb_lock); + } } void ocfs2_la_enable_worker(struct work_struct *work) diff --git a/fs/ocfs2/suballoc.c b/fs/ocfs2/suballoc.c index 7e30e9d798906..8314ec487cfb7 100644 --- a/fs/ocfs2/suballoc.c +++ b/fs/ocfs2/suballoc.c @@ -1372,6 +1372,7 @@ int ocfs2_block_group_set_bits(handle_t *handle, int journal_type = OCFS2_JOURNAL_ACCESS_WRITE; unsigned int start = bit_off + num_bits; u16 contig_bits; + struct ocfs2_super *osb = OCFS2_SB(alloc_inode->i_sb); /* All callers get the descriptor via * ocfs2_read_group_descriptor(). Any corruption is a code bug. */ @@ -1421,6 +1422,7 @@ int ocfs2_block_group_set_bits(handle_t *handle, if (contig_bits > max_contig_bits) max_contig_bits = contig_bits; bg->bg_contig_free_bits = cpu_to_le16(max_contig_bits); + ocfs2_local_alloc_seen_free_bits(osb, max_contig_bits); } else { bg->bg_contig_free_bits = 0; } @@ -1587,13 +1589,6 @@ static int ocfs2_cluster_group_search(struct inode *inode, * of bits. */ if (min_bits <= res->sr_bits) search = 0; /* success */ - else if (res->sr_bits) { - /* - * Don't show bits which we'll be returning - * for allocation to the local alloc bitmap. - */ - ocfs2_local_alloc_seen_free_bits(osb, res->sr_bits); - } } return search; From 525350221beb55bc6795595443d4cdeecb68ebec Mon Sep 17 00:00:00 2001 From: Heming Zhao Date: Thu, 28 Mar 2024 20:52:02 +0800 Subject: [PATCH 0790/1702] ocfs2: speed up chain-list searching Add short-circuit code to speed up searching Link: https://lkml.kernel.org/r/20240328125203.20892-4-heming.zhao@suse.com Signed-off-by: Heming Zhao Reviewed-by: Joseph Qi Cc: Changwei Ge Cc: Gang He Cc: Joel Becker Cc: Jun Piao Cc: Junxiao Bi Cc: Mark Fasheh Signed-off-by: Andrew Morton --- fs/ocfs2/suballoc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ocfs2/suballoc.c b/fs/ocfs2/suballoc.c index 8314ec487cfb7..f7b483f0de2ad 100644 --- a/fs/ocfs2/suballoc.c +++ b/fs/ocfs2/suballoc.c @@ -2006,7 +2006,7 @@ static int ocfs2_claim_suballoc_bits(struct ocfs2_alloc_context *ac, for (i = 0; i < le16_to_cpu(cl->cl_next_free_rec); i ++) { if (i == victim) continue; - if (!cl->cl_recs[i].c_free) + if (le32_to_cpu(cl->cl_recs[i].c_free) < bits_wanted) continue; ac->ac_chain = i; From fc07d2a2118aa83dca9bdc8fda6ab9fe2f28a8d6 Mon Sep 17 00:00:00 2001 From: Heming Zhao Date: Thu, 28 Mar 2024 20:52:03 +0800 Subject: [PATCH 0791/1702] ocfs2: fix sparse warnings 1. fs/ocfs2/localalloc.c:1224:41: warning: incorrect type in argument 1 (different base types) fs/ocfs2/localalloc.c:1224:41: expected unsigned long long val1 fs/ocfs2/localalloc.c:1224:41: got restricted __le32 [usertype] la_bm_off 2. fs/ocfs2/export.c:258:32: warning: cast to restricted __le32 fs/ocfs2/export.c:259:33: warning: cast to restricted __le32 fs/ocfs2/export.c:260:32: warning: cast to restricted __le32 fs/ocfs2/export.c:272:32: warning: cast to restricted __le32 fs/ocfs2/export.c:273:33: warning: cast to restricted __le32 fs/ocfs2/export.c:274:32: warning: cast to restricted __le32 3. fs/ocfs2/inode.c:1623:13: warning: context imbalance in 'ocfs2_inode_cache_lock' - wrong count at exit fs/ocfs2/inode.c:1630:13: warning: context imbalance in 'ocfs2_inode_cache_unlock' - unexpected unlock 4. fs/ocfs2/refcounttree.c:633:27: warning: incorrect type in assignment (different base types) fs/ocfs2/refcounttree.c:633:27: expected restricted __le32 [usertype] rf_generation fs/ocfs2/refcounttree.c:633:27: got unsigned int 5. fs/ocfs2/dlm/dlmdomain.c:1316:20: warning: context imbalance in 'dlm_query_nodeinfo_handler' - different lock contexts for basic block Link: https://lkml.kernel.org/r/20240328125203.20892-5-heming.zhao@suse.com Signed-off-by: Heming Zhao Reviewed-by: Joseph Qi Cc: Changwei Ge Cc: Gang He Cc: Joel Becker Cc: Jun Piao Cc: Junxiao Bi Cc: Mark Fasheh Signed-off-by: Andrew Morton --- fs/ocfs2/dlm/dlmdomain.c | 11 +++++------ fs/ocfs2/export.c | 12 ++++++------ fs/ocfs2/inode.c | 2 ++ fs/ocfs2/localalloc.c | 4 ++-- fs/ocfs2/refcounttree.c | 2 +- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/fs/ocfs2/dlm/dlmdomain.c b/fs/ocfs2/dlm/dlmdomain.c index 5c04dde99981c..2e0a2f3382829 100644 --- a/fs/ocfs2/dlm/dlmdomain.c +++ b/fs/ocfs2/dlm/dlmdomain.c @@ -1274,7 +1274,7 @@ static int dlm_query_nodeinfo_handler(struct o2net_msg *msg, u32 len, { struct dlm_query_nodeinfo *qn; struct dlm_ctxt *dlm = NULL; - int locked = 0, status = -EINVAL; + int status = -EINVAL; qn = (struct dlm_query_nodeinfo *) msg->buf; @@ -1290,12 +1290,11 @@ static int dlm_query_nodeinfo_handler(struct o2net_msg *msg, u32 len, } spin_lock(&dlm->spinlock); - locked = 1; if (dlm->joining_node != qn->qn_nodenum) { mlog(ML_ERROR, "Node %d queried nodes on domain %s but " "joining node is %d\n", qn->qn_nodenum, qn->qn_domain, dlm->joining_node); - goto bail; + goto unlock; } /* Support for node query was added in 1.1 */ @@ -1305,14 +1304,14 @@ static int dlm_query_nodeinfo_handler(struct o2net_msg *msg, u32 len, "but active dlm protocol is %d.%d\n", qn->qn_nodenum, qn->qn_domain, dlm->dlm_locking_proto.pv_major, dlm->dlm_locking_proto.pv_minor); - goto bail; + goto unlock; } status = dlm_match_nodes(dlm, qn); +unlock: + spin_unlock(&dlm->spinlock); bail: - if (locked) - spin_unlock(&dlm->spinlock); spin_unlock(&dlm_domain_lock); return status; diff --git a/fs/ocfs2/export.c b/fs/ocfs2/export.c index b8b6a191b5cb5..96b684763b390 100644 --- a/fs/ocfs2/export.c +++ b/fs/ocfs2/export.c @@ -255,9 +255,9 @@ static struct dentry *ocfs2_fh_to_dentry(struct super_block *sb, if (fh_len < 3 || fh_type > 2) return NULL; - handle.ih_blkno = (u64)le32_to_cpu(fid->raw[0]) << 32; - handle.ih_blkno |= (u64)le32_to_cpu(fid->raw[1]); - handle.ih_generation = le32_to_cpu(fid->raw[2]); + handle.ih_blkno = (u64)le32_to_cpu((__force __le32)fid->raw[0]) << 32; + handle.ih_blkno |= (u64)le32_to_cpu((__force __le32)fid->raw[1]); + handle.ih_generation = le32_to_cpu((__force __le32)fid->raw[2]); return ocfs2_get_dentry(sb, &handle); } @@ -269,9 +269,9 @@ static struct dentry *ocfs2_fh_to_parent(struct super_block *sb, if (fh_type != 2 || fh_len < 6) return NULL; - parent.ih_blkno = (u64)le32_to_cpu(fid->raw[3]) << 32; - parent.ih_blkno |= (u64)le32_to_cpu(fid->raw[4]); - parent.ih_generation = le32_to_cpu(fid->raw[5]); + parent.ih_blkno = (u64)le32_to_cpu((__force __le32)fid->raw[3]) << 32; + parent.ih_blkno |= (u64)le32_to_cpu((__force __le32)fid->raw[4]); + parent.ih_generation = le32_to_cpu((__force __le32)fid->raw[5]); return ocfs2_get_dentry(sb, &parent); } diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c index 999111bfc2717..2cc5c99fe9416 100644 --- a/fs/ocfs2/inode.c +++ b/fs/ocfs2/inode.c @@ -1621,6 +1621,7 @@ static struct super_block *ocfs2_inode_cache_get_super(struct ocfs2_caching_info } static void ocfs2_inode_cache_lock(struct ocfs2_caching_info *ci) +__acquires(&oi->ip_lock) { struct ocfs2_inode_info *oi = cache_info_to_inode(ci); @@ -1628,6 +1629,7 @@ static void ocfs2_inode_cache_lock(struct ocfs2_caching_info *ci) } static void ocfs2_inode_cache_unlock(struct ocfs2_caching_info *ci) +__releases(&oi->ip_lock) { struct ocfs2_inode_info *oi = cache_info_to_inode(ci); diff --git a/fs/ocfs2/localalloc.c b/fs/ocfs2/localalloc.c index c84ce53cdec03..5df34561c551c 100644 --- a/fs/ocfs2/localalloc.c +++ b/fs/ocfs2/localalloc.c @@ -336,7 +336,7 @@ int ocfs2_load_local_alloc(struct ocfs2_super *osb) "found = %u, set = %u, taken = %u, off = %u\n", num_used, le32_to_cpu(alloc->id1.bitmap1.i_used), le32_to_cpu(alloc->id1.bitmap1.i_total), - OCFS2_LOCAL_ALLOC(alloc)->la_bm_off); + le32_to_cpu(OCFS2_LOCAL_ALLOC(alloc)->la_bm_off)); status = -EINVAL; goto bail; @@ -1214,7 +1214,7 @@ static int ocfs2_local_alloc_new_window(struct ocfs2_super *osb, OCFS2_LOCAL_ALLOC(alloc)->la_bitmap); trace_ocfs2_local_alloc_new_window_result( - OCFS2_LOCAL_ALLOC(alloc)->la_bm_off, + le32_to_cpu(OCFS2_LOCAL_ALLOC(alloc)->la_bm_off), le32_to_cpu(alloc->id1.bitmap1.i_total)); bail: diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c index 3f80a56d0d603..1f303b1adf1ab 100644 --- a/fs/ocfs2/refcounttree.c +++ b/fs/ocfs2/refcounttree.c @@ -630,7 +630,7 @@ static int ocfs2_create_refcount_tree(struct inode *inode, rb->rf_records.rl_count = cpu_to_le16(ocfs2_refcount_recs_per_rb(osb->sb)); spin_lock(&osb->osb_lock); - rb->rf_generation = osb->s_next_generation++; + rb->rf_generation = cpu_to_le32(osb->s_next_generation++); spin_unlock(&osb->osb_lock); ocfs2_journal_dirty(handle, new_bh); From b0f970c50d439df46ade159950201faba36da10b Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Fri, 29 Mar 2024 21:28:25 +0800 Subject: [PATCH 0792/1702] Documentation: kdump: clean up the outdated description After commit 443cbaf9e2fd ("crash: split vmcoreinfo exporting code out from crash_core.c"), Kconfig item CRASH_CORE has gone away in kernel. Items VMCORE_INFO and CRASH_RESERVE are used instead. So clean up the outdated description about CRASH_CORE and update it accordingly. Link: https://lkml.kernel.org/r/20240329132825.1102459-3-bhe@redhat.com Signed-off-by: Baoquan He Cc: Jonathan Corbet Cc: Geert Uytterhoeven Cc: Huacai Chen Cc: WANG Xuerui Signed-off-by: Andrew Morton --- Documentation/admin-guide/kdump/kdump.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/admin-guide/kdump/kdump.rst b/Documentation/admin-guide/kdump/kdump.rst index 0302a93b1d40b..5376890adbeb0 100644 --- a/Documentation/admin-guide/kdump/kdump.rst +++ b/Documentation/admin-guide/kdump/kdump.rst @@ -136,10 +136,6 @@ System kernel config options CONFIG_KEXEC_CORE=y - Subsequently, CRASH_CORE is selected by KEXEC_CORE:: - - CONFIG_CRASH_CORE=y - 2) Enable "sysfs file system support" in "Filesystem" -> "Pseudo filesystems." This is usually enabled by default:: @@ -168,6 +164,10 @@ Dump-capture kernel config options (Arch Independent) CONFIG_CRASH_DUMP=y + And this will select VMCORE_INFO and CRASH_RESERVE:: + CONFIG_VMCORE_INFO=y + CONFIG_CRASH_RESERVE=y + 2) Enable "/proc/vmcore support" under "Filesystems" -> "Pseudo filesystems":: CONFIG_PROC_VMCORE=y From 56fd61628b7a5c1ff474619580796f11d5c8973a Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 28 Mar 2024 15:30:42 +0100 Subject: [PATCH 0793/1702] kcov: avoid clang out-of-range warning The area_size is never larger than the maximum on 64-bit architectutes: kernel/kcov.c:634:29: error: result of comparison of constant 1152921504606846975 with expression of type '__u32' (aka 'unsigned int') is always false [-Werror,-Wtautological-constant-out-of-range-compare] if (remote_arg->area_size > LONG_MAX / sizeof(unsigned long)) ~~~~~~~~~~~~~~~~~~~~~ ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The compiler can correctly optimize the check away and the code appears correct to me, so just add a cast to avoid the warning. Link: https://lkml.kernel.org/r/20240328143051.1069575-5-arnd@kernel.org Signed-off-by: Arnd Bergmann Reviewed-by: Justin Stitt Cc: Andrey Konovalov Cc: Bill Wendling Cc: Dmitry Vyukov Cc: Nathan Chancellor Cc: Nick Desaulniers Signed-off-by: Andrew Morton --- kernel/kcov.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/kcov.c b/kernel/kcov.c index f9ac2e9e460fc..c3124f6d5536b 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -627,7 +627,8 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, mode = kcov_get_mode(remote_arg->trace_mode); if (mode < 0) return mode; - if (remote_arg->area_size > LONG_MAX / sizeof(unsigned long)) + if ((unsigned long)remote_arg->area_size > + LONG_MAX / sizeof(unsigned long)) return -EINVAL; kcov->mode = mode; t->kcov = kcov; From 5f08383c1558f30e3a52db3fd30110835d687b9b Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Thu, 28 Mar 2024 16:57:51 +0100 Subject: [PATCH 0794/1702] initrd: remove the now superfluous sentinel element from ctl_table array This commit comes at the tail end of a greater effort to remove the empty elements at the end of the ctl_table arrays (sentinels) which will reduce the overall build time size of the kernel and run time memory bloat by ~64 bytes per sentinel (further information Link : https://lore.kernel.org/all/ZO5Yx5JFogGi%2FcBo@bombadil.infradead.org/) Remove sentinel from kern_do_mounts_initrd_table. Link: https://lkml.kernel.org/r/20240328-jag-sysctl_remset_misc-v1-4-47c1463b3af2@samsung.com Signed-off-by: Joel Granados Signed-off-by: Andrew Morton --- init/do_mounts_initrd.c | 1 - 1 file changed, 1 deletion(-) diff --git a/init/do_mounts_initrd.c b/init/do_mounts_initrd.c index 425f4bcf4b77e..22c7f41ff6422 100644 --- a/init/do_mounts_initrd.c +++ b/init/do_mounts_initrd.c @@ -29,7 +29,6 @@ static struct ctl_table kern_do_mounts_initrd_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, - { } }; static __init int kernel_do_mounts_initrd_sysctls_init(void) From 029c45bb24d02a4de7b1e3866fe7592d59c10718 Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Thu, 28 Mar 2024 16:57:52 +0100 Subject: [PATCH 0795/1702] ipc: remove the now superfluous sentinel element from ctl_table array This commit comes at the tail end of a greater effort to remove the empty elements at the end of the ctl_table arrays (sentinels) which will reduce the overall build time size of the kernel and run time memory bloat by ~64 bytes per sentinel (further information Link : https://lore.kernel.org/all/ZO5Yx5JFogGi%2FcBo@bombadil.infradead.org/) Remove the sentinels from ipc_sysctls and mq_sysctls Link: https://lkml.kernel.org/r/20240328-jag-sysctl_remset_misc-v1-5-47c1463b3af2@samsung.com Signed-off-by: Joel Granados Signed-off-by: Andrew Morton --- ipc/ipc_sysctl.c | 1 - ipc/mq_sysctl.c | 1 - 2 files changed, 2 deletions(-) diff --git a/ipc/ipc_sysctl.c b/ipc/ipc_sysctl.c index 45cb1dabce29a..0867535af96fb 100644 --- a/ipc/ipc_sysctl.c +++ b/ipc/ipc_sysctl.c @@ -178,7 +178,6 @@ static struct ctl_table ipc_sysctls[] = { .extra2 = SYSCTL_INT_MAX, }, #endif - {} }; static struct ctl_table_set *set_lookup(struct ctl_table_root *root) diff --git a/ipc/mq_sysctl.c b/ipc/mq_sysctl.c index 21fba3a6edaf7..22ec532c7fa11 100644 --- a/ipc/mq_sysctl.c +++ b/ipc/mq_sysctl.c @@ -64,7 +64,6 @@ static struct ctl_table mq_sysctls[] = { .extra1 = &msg_maxsize_limit_min, .extra2 = &msg_maxsize_limit_max, }, - {} }; static struct ctl_table_set *set_lookup(struct ctl_table_root *root) From 040bf9a717885d3646dcd5b4062dc3a4877ec421 Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Wed, 3 Apr 2024 19:33:52 +0100 Subject: [PATCH 0796/1702] Squashfs: remove deprecated strncpy by not copying the string Squashfs copied the passed string (name) into a temporary buffer to ensure it was NUL-terminated. This however is completely unnecessary as the string is already NUL-terminated. So remove the deprecated strncpy() by completely removing the string copy. The background behind this unnecessary string copy is that it dates back to the days when Squashfs was an out of kernel patch. The code deliberately did not assume the string was NUL-terminated in case in future this changed (due to kernel changes). This would mean the out of tree patches would be broken but still compile OK. Link: https://lkml.kernel.org/r/20240403183352.391308-1-phillip@squashfs.org.uk Signed-off-by: Phillip Lougher Reviewed-by: Kees Cook Reviewed-by: Justin Stitt Signed-off-by: Andrew Morton --- fs/squashfs/namei.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/fs/squashfs/namei.c b/fs/squashfs/namei.c index 11e4539b9eae4..65aae7e2a859d 100644 --- a/fs/squashfs/namei.c +++ b/fs/squashfs/namei.c @@ -62,27 +62,21 @@ */ static int get_dir_index_using_name(struct super_block *sb, u64 *next_block, int *next_offset, u64 index_start, - int index_offset, int i_count, const char *name, - int len) + int index_offset, int i_count, const char *name) { struct squashfs_sb_info *msblk = sb->s_fs_info; int i, length = 0, err; unsigned int size; struct squashfs_dir_index *index; - char *str; TRACE("Entered get_dir_index_using_name, i_count %d\n", i_count); - index = kmalloc(sizeof(*index) + SQUASHFS_NAME_LEN * 2 + 2, GFP_KERNEL); + index = kmalloc(sizeof(*index) + SQUASHFS_NAME_LEN + 1, GFP_KERNEL); if (index == NULL) { ERROR("Failed to allocate squashfs_dir_index\n"); goto out; } - str = &index->name[SQUASHFS_NAME_LEN + 1]; - strncpy(str, name, len); - str[len] = '\0'; - for (i = 0; i < i_count; i++) { err = squashfs_read_metadata(sb, index, &index_start, &index_offset, sizeof(*index)); @@ -101,7 +95,7 @@ static int get_dir_index_using_name(struct super_block *sb, index->name[size] = '\0'; - if (strcmp(index->name, str) > 0) + if (strcmp(index->name, name) > 0) break; length = le32_to_cpu(index->index); @@ -153,7 +147,7 @@ static struct dentry *squashfs_lookup(struct inode *dir, struct dentry *dentry, length = get_dir_index_using_name(dir->i_sb, &block, &offset, squashfs_i(dir)->dir_idx_start, squashfs_i(dir)->dir_idx_offset, - squashfs_i(dir)->dir_idx_cnt, name, len); + squashfs_i(dir)->dir_idx_cnt, name); while (length < i_size_read(dir)) { /* From b157f0e97e3ed6158cde4f247da4c537c3f6a0f9 Mon Sep 17 00:00:00 2001 From: Niklas Schnelle Date: Wed, 3 Apr 2024 15:25:47 +0200 Subject: [PATCH 0797/1702] kgdb: add HAS_IOPORT dependency In a future patch HAS_IOPORT=n will disable inb()/outb() and friends at compile time. We thus need to add HAS_IOPORT as dependency for those drivers using them. Link: https://lkml.kernel.org/r/20240403132547.762429-2-schnelle@linux.ibm.com Co-developed-by: Arnd Bergmann Signed-off-by: Arnd Bergmann Signed-off-by: Niklas Schnelle Signed-off-by: Andrew Morton --- lib/Kconfig.kgdb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/Kconfig.kgdb b/lib/Kconfig.kgdb index b5c0e6576749d..537e1b3f57340 100644 --- a/lib/Kconfig.kgdb +++ b/lib/Kconfig.kgdb @@ -122,6 +122,7 @@ config KDB_DEFAULT_ENABLE config KDB_KEYBOARD bool "KGDB_KDB: keyboard as input device" depends on VT && KGDB_KDB && !PARISC + depends on HAS_IOPORT default n help KDB can use a PS/2 type keyboard for an input device From f36c54f3ce3d69aadccc56bc7dc2cf6c464f3522 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 3 Apr 2024 13:46:56 +0300 Subject: [PATCH 0798/1702] devres: switch to use dev_err_probe() for unification Patch series "devres: A couple of cleanups". A couple of ad-hoc cleanups. No functional changes intended. This patch (of 2): The devm_*() APIs are supposed to be called during the ->probe() stage. Many drivers (especially new ones) have switched to use dev_err_probe() for error messaging for the sake of unification. Let's do the same in the devres APIs. Link: https://lkml.kernel.org/r/20240403104820.557487-1-andriy.shevchenko@linux.intel.com Link: https://lkml.kernel.org/r/20240403104820.557487-2-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Reviewed-by: Philipp Stanner Signed-off-by: Andrew Morton --- lib/devres.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/devres.c b/lib/devres.c index fe0c63caeb689..27f280a39dca5 100644 --- a/lib/devres.c +++ b/lib/devres.c @@ -125,12 +125,13 @@ __devm_ioremap_resource(struct device *dev, const struct resource *res, resource_size_t size; void __iomem *dest_ptr; char *pretty_name; + int ret; BUG_ON(!dev); if (!res || resource_type(res) != IORESOURCE_MEM) { - dev_err(dev, "invalid resource %pR\n", res); - return IOMEM_ERR_PTR(-EINVAL); + ret = dev_err_probe(dev, -EINVAL, "invalid resource %pR\n", res); + return IOMEM_ERR_PTR(ret); } if (type == DEVM_IOREMAP && res->flags & IORESOURCE_MEM_NONPOSTED) @@ -144,20 +145,20 @@ __devm_ioremap_resource(struct device *dev, const struct resource *res, else pretty_name = devm_kstrdup(dev, dev_name(dev), GFP_KERNEL); if (!pretty_name) { - dev_err(dev, "can't generate pretty name for resource %pR\n", res); - return IOMEM_ERR_PTR(-ENOMEM); + ret = dev_err_probe(dev, -ENOMEM, "can't generate pretty name for resource %pR\n", res); + return IOMEM_ERR_PTR(ret); } if (!devm_request_mem_region(dev, res->start, size, pretty_name)) { - dev_err(dev, "can't request region for resource %pR\n", res); - return IOMEM_ERR_PTR(-EBUSY); + ret = dev_err_probe(dev, -EBUSY, "can't request region for resource %pR\n", res); + return IOMEM_ERR_PTR(ret); } dest_ptr = __devm_ioremap(dev, res->start, size, type); if (!dest_ptr) { - dev_err(dev, "ioremap failed for resource %pR\n", res); devm_release_mem_region(dev, res->start, size); - dest_ptr = IOMEM_ERR_PTR(-ENOMEM); + ret = dev_err_probe(dev, -ENOMEM, "ioremap failed for resource %pR\n", res); + return IOMEM_ERR_PTR(ret); } return dest_ptr; From 3cc98aa11ec4c3b9b55fa185c8882e4ae20cbd6a Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 3 Apr 2024 13:46:57 +0300 Subject: [PATCH 0799/1702] devres: don't use "proxy" headers Update header inclusions to follow IWYU (Include What You Use) principle. Link: https://lkml.kernel.org/r/20240403104820.557487-3-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Reviewed-by: Philipp Stanner Signed-off-by: Andrew Morton --- lib/devres.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/devres.c b/lib/devres.c index 27f280a39dca5..4fc152de6d8bb 100644 --- a/lib/devres.c +++ b/lib/devres.c @@ -1,10 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include -#include -#include -#include +#include #include +#include +#include +#include #include +#include enum devm_ioremap_type { DEVM_IOREMAP = 0, From ad5f0eb540d31d95e6eefe6e24af0fe1b1b393fa Mon Sep 17 00:00:00 2001 From: Justin Stitt Date: Mon, 1 Apr 2024 18:39:55 +0000 Subject: [PATCH 0800/1702] vmcore: replace strncpy with strscpy_pad strncpy() is in the process of being replaced as it is deprecated [1]. We should move towards safer and less ambiguous string interfaces. Looking at vmcoredd_header's definition: | struct vmcoredd_header { | __u32 n_namesz; /* Name size */ | __u32 n_descsz; /* Content size */ | __u32 n_type; /* NT_VMCOREDD */ | __u8 name[8]; /* LINUX\0\0\0 */ | __u8 dump_name[VMCOREDD_MAX_NAME_BYTES]; /* Device dump's name */ | }; .. we see that @name wants to be NUL-padded. We're copying data->dump_name which is defined as: | char dump_name[VMCOREDD_MAX_NAME_BYTES]; /* Unique name of the dump */ .. which shares the same size as vdd_hdr->dump_name. Let's make sure we NUL-pad this as well. Use strscpy_pad() which NUL-terminates and NUL-pads its destination buffers. Specifically, use the new 2-argument version of strscpy_pad introduced in Commit e6584c3964f2f ("string: Allow 2-argument strscpy()"). Link: https://www.kernel.org/doc/html/latest/process/deprecated.html#strncpy-on-nul-terminated-strings [1] Link: https://github.com/KSPP/linux/issues/90 Link: https://lkml.kernel.org/r/20240401-strncpy-fs-proc-vmcore-c-v2-1-dd0a73f42635@google.com Signed-off-by: Justin Stitt Acked-by: Baoquan He Cc: Dave Young Cc: Vivek Goyal Signed-off-by: Andrew Morton --- fs/proc/vmcore.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/proc/vmcore.c b/fs/proc/vmcore.c index 1fb213f379a5b..5d08d4d159d30 100644 --- a/fs/proc/vmcore.c +++ b/fs/proc/vmcore.c @@ -1370,9 +1370,8 @@ static void vmcoredd_write_header(void *buf, struct vmcoredd_data *data, vdd_hdr->n_descsz = size + sizeof(vdd_hdr->dump_name); vdd_hdr->n_type = NT_VMCOREDD; - strncpy((char *)vdd_hdr->name, VMCOREDD_NOTE_NAME, - sizeof(vdd_hdr->name)); - memcpy(vdd_hdr->dump_name, data->dump_name, sizeof(vdd_hdr->dump_name)); + strscpy_pad(vdd_hdr->name, VMCOREDD_NOTE_NAME); + strscpy_pad(vdd_hdr->dump_name, data->dump_name); } /** From d11547071a9d916604a65394b169c0d1acf0a6d6 Mon Sep 17 00:00:00 2001 From: Su Yue Date: Mon, 8 Apr 2024 16:20:38 +0800 Subject: [PATCH 0801/1702] ocfs2: return real error code in ocfs2_dio_wr_get_block Patch series "ocfs2 bugs fixes exposed by fstests", v3. The patchset is to fix some wrong behavior of ocfs2 exposed by fstests. Patch 1 makes userspace happy when some error happens when doing direct io. Before the patch, DIO always return -EIO in case of error. After the patch, it returns real error code such like -ENOSPC, EDQUOT... Patch 2 fixes an error case when doing AIO+DIO and hole punching at same file position in parallel. generic/300 Patch 3 fixes inode link count mismatch after power failure. Without the patch, inode link would be wrong even fync was called on the file. tests/generic/040,041,104,107,336 patch 4 fixes wrong atime with mount option realtime. Without the patch, atime of new created file won't be updated in right time. tests/generic/192 For stable kernels, I added fixes to patch 2,3,4. The patch 1 is not recommended to be backported since ocfs2_dio_wr_get_block calls too many functions. It's diffcult to check every git history of ocfs2 for every LTS kernel. This patch (of 4): ocfs2_dio_wr_get_block always returns -EIO in case of errors. However, some programs expect right exit codes while doing dio. For example, tools like fio treat -ENOSPC as expected code while doing stress jobs. And quota tools expect -EDQUOT when disk quota exceeds. -EIO is too strong return code in the dio path. The caller of ocfs2_dio_wr_get_block is __blockdev_direct_IO which is widely used and it handles error codes well. I have checked functions called by ocfs2_dio_wr_get_block and their return codes look good and clear. So I think it's safe to let ocfs2_dio_wr_get_block return real error code. Link: https://lkml.kernel.org/r/20240408082041.20925-1-glass.su@suse.com Link: https://lkml.kernel.org/r/20240408082041.20925-2-glass.su@suse.com Signed-off-by: Su Yue Reviewed-by: Joseph Qi Cc: Changwei Ge Cc: Gang He Cc: Joel Becker Cc: Jun Piao Cc: Junxiao Bi Cc: Mark Fasheh Signed-off-by: Andrew Morton --- fs/ocfs2/aops.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c index b82185075de7d..f0467d3b3c880 100644 --- a/fs/ocfs2/aops.c +++ b/fs/ocfs2/aops.c @@ -2283,8 +2283,6 @@ static int ocfs2_dio_wr_get_block(struct inode *inode, sector_t iblock, ocfs2_inode_unlock(inode, 1); brelse(di_bh); out: - if (ret < 0) - ret = -EIO; return ret; } From 952b023f06a24b2ad6ba67304c4c84d45bea2f18 Mon Sep 17 00:00:00 2001 From: Su Yue Date: Mon, 8 Apr 2024 16:20:39 +0800 Subject: [PATCH 0802/1702] ocfs2: fix races between hole punching and AIO+DIO After commit "ocfs2: return real error code in ocfs2_dio_wr_get_block", fstests/generic/300 become from always failed to sometimes failed: ======================================================================== [ 473.293420 ] run fstests generic/300 [ 475.296983 ] JBD2: Ignoring recovery information on journal [ 475.302473 ] ocfs2: Mounting device (253,1) on (node local, slot 0) with ordered data mode. [ 494.290998 ] OCFS2: ERROR (device dm-1): ocfs2_change_extent_flag: Owner 5668 has an extent at cpos 78723 which can no longer be found [ 494.291609 ] On-disk corruption discovered. Please run fsck.ocfs2 once the filesystem is unmounted. [ 494.292018 ] OCFS2: File system is now read-only. [ 494.292224 ] (kworker/19:11,2628,19):ocfs2_mark_extent_written:5272 ERROR: status = -30 [ 494.292602 ] (kworker/19:11,2628,19):ocfs2_dio_end_io_write:2374 ERROR: status = -3 fio: io_u error on file /mnt/scratch/racer: Read-only file system: write offset=460849152, buflen=131072 ========================================================================= In __blockdev_direct_IO, ocfs2_dio_wr_get_block is called to add unwritten extents to a list. extents are also inserted into extent tree in ocfs2_write_begin_nolock. Then another thread call fallocate to puch a hole at one of the unwritten extent. The extent at cpos was removed by ocfs2_remove_extent(). At end io worker thread, ocfs2_search_extent_list found there is no such extent at the cpos. T1 T2 T3 inode lock ... insert extents ... inode unlock ocfs2_fallocate __ocfs2_change_file_space inode lock lock ip_alloc_sem ocfs2_remove_inode_range inode ocfs2_remove_btree_range ocfs2_remove_extent ^---remove the extent at cpos 78723 ... unlock ip_alloc_sem inode unlock ocfs2_dio_end_io ocfs2_dio_end_io_write lock ip_alloc_sem ocfs2_mark_extent_written ocfs2_change_extent_flag ocfs2_search_extent_list ^---failed to find extent ... unlock ip_alloc_sem In most filesystems, fallocate is not compatible with racing with AIO+DIO, so fix it by adding to wait for all dio before fallocate/punch_hole like ext4. Link: https://lkml.kernel.org/r/20240408082041.20925-3-glass.su@suse.com Fixes: b25801038da5 ("ocfs2: Support xfs style space reservation ioctls") Signed-off-by: Su Yue Reviewed-by: Joseph Qi Cc: Changwei Ge Cc: Gang He Cc: Joel Becker Cc: Jun Piao Cc: Junxiao Bi Cc: Mark Fasheh Cc: Signed-off-by: Andrew Morton --- fs/ocfs2/file.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index 0da8e7bd32616..ccc57038a9779 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -1936,6 +1936,8 @@ static int __ocfs2_change_file_space(struct file *file, struct inode *inode, inode_lock(inode); + /* Wait all existing dio workers, newcomers will block on i_rwsem */ + inode_dio_wait(inode); /* * This prevents concurrent writes on other nodes */ From 8c40984eeb8804cffcd28640f427f4fe829243fc Mon Sep 17 00:00:00 2001 From: Su Yue Date: Mon, 8 Apr 2024 16:20:40 +0800 Subject: [PATCH 0803/1702] ocfs2: update inode fsync transaction id in ocfs2_unlink and ocfs2_link transaction id should be updated in ocfs2_unlink and ocfs2_link. Otherwise, inode link will be wrong after journal replay even fsync was called before power failure: ======================================================================= $ touch testdir/bar $ ln testdir/bar testdir/bar_link $ fsync testdir/bar $ stat -c %h $SCRATCH_MNT/testdir/bar 1 $ stat -c %h $SCRATCH_MNT/testdir/bar 1 ======================================================================= Link: https://lkml.kernel.org/r/20240408082041.20925-4-glass.su@suse.com Fixes: ccd979bdbce9 ("[PATCH] OCFS2: The Second Oracle Cluster Filesystem") Signed-off-by: Su Yue Reviewed-by: Joseph Qi Cc: Changwei Ge Cc: Gang He Cc: Joel Becker Cc: Jun Piao Cc: Junxiao Bi Cc: Mark Fasheh Cc: Signed-off-by: Andrew Morton --- fs/ocfs2/namei.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/ocfs2/namei.c b/fs/ocfs2/namei.c index 9221a33f917b8..55c9d90caaaff 100644 --- a/fs/ocfs2/namei.c +++ b/fs/ocfs2/namei.c @@ -797,6 +797,7 @@ static int ocfs2_link(struct dentry *old_dentry, ocfs2_set_links_count(fe, inode->i_nlink); fe->i_ctime = cpu_to_le64(inode_get_ctime_sec(inode)); fe->i_ctime_nsec = cpu_to_le32(inode_get_ctime_nsec(inode)); + ocfs2_update_inode_fsync_trans(handle, inode, 0); ocfs2_journal_dirty(handle, fe_bh); err = ocfs2_add_entry(handle, dentry, inode, @@ -993,6 +994,7 @@ static int ocfs2_unlink(struct inode *dir, drop_nlink(inode); drop_nlink(inode); ocfs2_set_links_count(fe, inode->i_nlink); + ocfs2_update_inode_fsync_trans(handle, inode, 0); ocfs2_journal_dirty(handle, fe_bh); inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); From b8cb324277ee16f3eca3055b96fce4735a5a41c6 Mon Sep 17 00:00:00 2001 From: Su Yue Date: Mon, 8 Apr 2024 16:20:41 +0800 Subject: [PATCH 0804/1702] ocfs2: use coarse time for new created files The default atime related mount option is '-o realtime' which means file atime should be updated if atime <= ctime or atime <= mtime. atime should be updated in the following scenario, but it is not: ========================================================== $ rm /mnt/testfile; $ echo test > /mnt/testfile $ stat -c "%X %Y %Z" /mnt/testfile 1711881646 1711881646 1711881646 $ sleep 5 $ cat /mnt/testfile > /dev/null $ stat -c "%X %Y %Z" /mnt/testfile 1711881646 1711881646 1711881646 ========================================================== And the reason the atime in the test is not updated is that ocfs2 calls ktime_get_real_ts64() in __ocfs2_mknod_locked during file creation. Then inode_set_ctime_current() is called in inode_set_ctime_current() calls ktime_get_coarse_real_ts64() to get current time. ktime_get_real_ts64() is more accurate than ktime_get_coarse_real_ts64(). In my test box, I saw ctime set by ktime_get_coarse_real_ts64() is less than ktime_get_real_ts64() even ctime is set later. The ctime of the new inode is smaller than atime. The call trace is like: ocfs2_create ocfs2_mknod __ocfs2_mknod_locked .... ktime_get_real_ts64 <------- set atime,ctime,mtime, more accurate ocfs2_populate_inode ... ocfs2_init_acl ocfs2_acl_set_mode inode_set_ctime_current current_time ktime_get_coarse_real_ts64 <-------less accurate ocfs2_file_read_iter ocfs2_inode_lock_atime ocfs2_should_update_atime atime <= ctime ? <-------- false, ctime < atime due to accuracy So here call ktime_get_coarse_real_ts64 to set inode time coarser while creating new files. It may lower the accuracy of file times. But it's not a big deal since we already use coarse time in other places like ocfs2_update_inode_atime and inode_set_ctime_current. Link: https://lkml.kernel.org/r/20240408082041.20925-5-glass.su@suse.com Fixes: c62c38f6b91b ("ocfs2: replace CURRENT_TIME macro") Signed-off-by: Su Yue Reviewed-by: Joseph Qi Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Cc: Signed-off-by: Andrew Morton --- fs/ocfs2/namei.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ocfs2/namei.c b/fs/ocfs2/namei.c index 55c9d90caaaff..4d1ea8703fcdf 100644 --- a/fs/ocfs2/namei.c +++ b/fs/ocfs2/namei.c @@ -566,7 +566,7 @@ static int __ocfs2_mknod_locked(struct inode *dir, fe->i_last_eb_blk = 0; strcpy(fe->i_signature, OCFS2_INODE_SIGNATURE); fe->i_flags |= cpu_to_le32(OCFS2_VALID_FL); - ktime_get_real_ts64(&ts); + ktime_get_coarse_real_ts64(&ts); fe->i_atime = fe->i_ctime = fe->i_mtime = cpu_to_le64(ts.tv_sec); fe->i_mtime_nsec = fe->i_ctime_nsec = fe->i_atime_nsec = From 3ef3a05ba6ac52998a52dc1eca90853dda771316 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 9 Apr 2024 16:00:54 +0200 Subject: [PATCH 0805/1702] test_hexdump: avoid string truncation warning gcc can warn when a string is too long to fit into the strncpy() destination buffer, as it is here depending on the function arguments: inlined from 'test_hexdump_prepare_test.constprop' at /home/arnd/arm-soc/lib/test_hexdump.c:116:3: include/linux/fortify-string.h:108:33: error: '__builtin_strncpy' output truncated copying between 0 and 32 bytes from a string of length 32 [-Werror=stringop-truncation] 108 | #define __underlying_strncpy __builtin_strncpy | ^ include/linux/fortify-string.h:187:16: note: in expansion of macro '__underlying_strncpy' 187 | return __underlying_strncpy(p, q, size); | ^~~~~~~~~~~~~~~~~~~~ The intention here is to copy exactly 'l' bytes without any padding or NUL-termination, so the most logical change is to use memcpy(), just as a previous change adapted the other output from strncpy() to memcpy(). Link: https://lkml.kernel.org/r/20240409140059.3806717-2-arnd@kernel.org Signed-off-by: Arnd Bergmann Acked-by: Justin Stitt Cc: Alexey Starikovskiy Cc: Bob Moore Cc: Jens Axboe Cc: Len Brown Cc: Lin Ming Cc: Masahiro Yamada Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Cc: Nathan Chancellor Cc: Nicolas Schier Cc: Rafael J. Wysocki Cc: "Richard Russon (FlatCap)" Cc: Steven Rostedt Signed-off-by: Andrew Morton --- lib/test_hexdump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/test_hexdump.c b/lib/test_hexdump.c index b916801f23a89..fe2682bb21e6d 100644 --- a/lib/test_hexdump.c +++ b/lib/test_hexdump.c @@ -113,7 +113,7 @@ static void __init test_hexdump_prepare_test(size_t len, int rowsize, *p++ = ' '; } while (p < test + rs * 2 + rs / gs + 1); - strncpy(p, data_a, l); + memcpy(p, data_a, l); p += l; } From 597bc741e5ac2608e3ec0afd4256163879ab506b Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 9 Apr 2024 16:00:56 +0200 Subject: [PATCH 0806/1702] block/partitions/ldm: convert strncpy() to strscpy() The strncpy() here can cause a non-terminated string, which older gcc versions such as gcc-9 warn about: In function 'ldm_parse_tocblock', inlined from 'ldm_validate_tocblocks' at block/partitions/ldm.c:386:7, inlined from 'ldm_partition' at block/partitions/ldm.c:1457:7: block/partitions/ldm.c:134:2: error: 'strncpy' specified bound 16 equals destination size [-Werror=stringop-truncation] 134 | strncpy (toc->bitmap1_name, data + 0x24, sizeof (toc->bitmap1_name)); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ block/partitions/ldm.c:145:2: error: 'strncpy' specified bound 16 equals destination size [-Werror=stringop-truncation] 145 | strncpy (toc->bitmap2_name, data + 0x46, sizeof (toc->bitmap2_name)); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ New versions notice that the code is correct after all because of the following termination, but replacing the strncpy() with strscpy_pad() or strcpy() avoids the warning and simplifies the code at the same time. Use the padding version here to keep the existing behavior, in case the code relies on not including uninitialized data. Link: https://lkml.kernel.org/r/20240409140059.3806717-4-arnd@kernel.org Reviewed-by: Justin Stitt Signed-off-by: Arnd Bergmann Cc: Alexey Starikovskiy Cc: Bob Moore Cc: Jens Axboe Cc: Len Brown Cc: Lin Ming Cc: Masahiro Yamada Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Cc: Nathan Chancellor Cc: Nicolas Schier Cc: Rafael J. Wysocki Cc: "Richard Russon (FlatCap)" Cc: Steven Rostedt Signed-off-by: Andrew Morton --- block/partitions/ldm.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/block/partitions/ldm.c b/block/partitions/ldm.c index 38e58960ae036..2bd42fedb907a 100644 --- a/block/partitions/ldm.c +++ b/block/partitions/ldm.c @@ -131,8 +131,7 @@ static bool ldm_parse_tocblock (const u8 *data, struct tocblock *toc) ldm_crit ("Cannot find TOCBLOCK, database may be corrupt."); return false; } - strncpy (toc->bitmap1_name, data + 0x24, sizeof (toc->bitmap1_name)); - toc->bitmap1_name[sizeof (toc->bitmap1_name) - 1] = 0; + strscpy_pad(toc->bitmap1_name, data + 0x24, sizeof(toc->bitmap1_name)); toc->bitmap1_start = get_unaligned_be64(data + 0x2E); toc->bitmap1_size = get_unaligned_be64(data + 0x36); @@ -142,8 +141,7 @@ static bool ldm_parse_tocblock (const u8 *data, struct tocblock *toc) TOC_BITMAP1, toc->bitmap1_name); return false; } - strncpy (toc->bitmap2_name, data + 0x46, sizeof (toc->bitmap2_name)); - toc->bitmap2_name[sizeof (toc->bitmap2_name) - 1] = 0; + strscpy_pad(toc->bitmap2_name, data + 0x46, sizeof(toc->bitmap2_name)); toc->bitmap2_start = get_unaligned_be64(data + 0x50); toc->bitmap2_size = get_unaligned_be64(data + 0x58); if (strncmp (toc->bitmap2_name, TOC_BITMAP2, From 051e7503070179f2278c0d4fa7dee441adb558ca Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 9 Apr 2024 16:00:57 +0200 Subject: [PATCH 0807/1702] blktrace: convert strncpy() to strscpy_pad() gcc-9 warns about a possibly non-terminated string copy: kernel/trace/blktrace.c: In function 'do_blk_trace_setup': kernel/trace/blktrace.c:527:2: error: 'strncpy' specified bound 32 equals destination size [-Werror=stringop-truncation] Newer versions are fine here because they see the following explicit nul-termination. Using strscpy_pad() avoids the warning and simplifies the code a little. The padding helps give a clean buffer to userspace. Link: https://lkml.kernel.org/r/20240409140059.3806717-5-arnd@kernel.org Signed-off-by: Arnd Bergmann Acked-by: Justin Stitt Cc: Alexey Starikovskiy Cc: Bob Moore Cc: Jens Axboe Cc: Len Brown Cc: Lin Ming Cc: Masahiro Yamada Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Cc: Nathan Chancellor Cc: Nicolas Schier Cc: Rafael J. Wysocki Cc: "Richard Russon (FlatCap)" Cc: Steven Rostedt Signed-off-by: Andrew Morton --- kernel/trace/blktrace.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index d5d94510afd3f..8fd292d34d898 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c @@ -524,8 +524,7 @@ static int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev, if (!buts->buf_size || !buts->buf_nr) return -EINVAL; - strncpy(buts->name, name, BLKTRACE_BDEV_SIZE); - buts->name[BLKTRACE_BDEV_SIZE - 1] = '\0'; + strscpy_pad(buts->name, name, BLKTRACE_BDEV_SIZE); /* * some device names have larger paths - convert the slashes From 2725844080d2609c48a51e08f9a0a3fbe7316665 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Wed, 10 Apr 2024 16:56:27 +0900 Subject: [PATCH 0808/1702] nilfs2: add kernel-doc comments to nilfs_do_roll_forward() Patch series "nilfs2: fix missing kernel-doc comments". This commit adds kernel-doc style comments with complete parameter descriptions for the function nilfs_do_roll_forward. Link: https://lkml.kernel.org/r/20240410075629.3441-1-konishi.ryusuke@gmail.com Link: https://lkml.kernel.org/r/20240410075629.3441-2-konishi.ryusuke@gmail.com Signed-off-by: Yang Li Signed-off-by: Ryusuke Konishi Signed-off-by: Andrew Morton --- fs/nilfs2/recovery.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/nilfs2/recovery.c b/fs/nilfs2/recovery.c index 49a70c68bf3c0..e48372618ac40 100644 --- a/fs/nilfs2/recovery.c +++ b/fs/nilfs2/recovery.c @@ -563,6 +563,7 @@ static int nilfs_recover_dsync_blocks(struct the_nilfs *nilfs, * checkpoint * @nilfs: nilfs object * @sb: super block instance + * @root: NILFS root instance * @ri: pointer to a nilfs_recovery_info */ static int nilfs_do_roll_forward(struct the_nilfs *nilfs, From 3da9b9650acc3a2a0c3d3f4542b93d4abe9da1de Mon Sep 17 00:00:00 2001 From: Yang Li Date: Wed, 10 Apr 2024 16:56:28 +0900 Subject: [PATCH 0809/1702] nilfs2: add kernel-doc comments to nilfs_btree_convert_and_insert() This commit adds kernel-doc style comments with complete parameter descriptions for the function nilfs_btree_convert_and_insert. Link: https://lkml.kernel.org/r/20240410075629.3441-3-konishi.ryusuke@gmail.com Signed-off-by: Yang Li Signed-off-by: Ryusuke Konishi Signed-off-by: Andrew Morton --- fs/nilfs2/btree.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c index 65659fa0372e6..a139970e48041 100644 --- a/fs/nilfs2/btree.c +++ b/fs/nilfs2/btree.c @@ -1857,13 +1857,22 @@ nilfs_btree_commit_convert_and_insert(struct nilfs_bmap *btree, } /** - * nilfs_btree_convert_and_insert - - * @bmap: - * @key: - * @ptr: - * @keys: - * @ptrs: - * @n: + * nilfs_btree_convert_and_insert - Convert and insert entries into a B-tree + * @btree: NILFS B-tree structure + * @key: Key of the new entry to be inserted + * @ptr: Pointer (block number) associated with the key to be inserted + * @keys: Array of keys to be inserted in addition to @key + * @ptrs: Array of pointers associated with @keys + * @n: Number of keys and pointers in @keys and @ptrs + * + * This function is used to insert a new entry specified by @key and @ptr, + * along with additional entries specified by @keys and @ptrs arrays, into a + * NILFS B-tree. + * It prepares the necessary changes by allocating the required blocks and any + * necessary intermediate nodes. It converts configurations from other forms of + * block mapping (the one that currently exists is direct mapping) to a B-tree. + * + * Return: 0 on success or a negative error code on failure. */ int nilfs_btree_convert_and_insert(struct nilfs_bmap *btree, __u64 key, __u64 ptr, From 4a458576941c78d905415a4a6edb48d4c04f7444 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Wed, 10 Apr 2024 16:56:29 +0900 Subject: [PATCH 0810/1702] nilfs2: add kernel-doc comments to nilfs_remove_all_gcinodes() This commit adds kernel-doc style comments with complete parameter descriptions for the function nilfs_remove_all_gcinodes. Link: https://lkml.kernel.org/r/20240410075629.3441-4-konishi.ryusuke@gmail.com Signed-off-by: Yang Li Signed-off-by: Ryusuke Konishi Signed-off-by: Andrew Morton --- fs/nilfs2/gcinode.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/nilfs2/gcinode.c b/fs/nilfs2/gcinode.c index bf9a11d588172..1c9ae36a03abe 100644 --- a/fs/nilfs2/gcinode.c +++ b/fs/nilfs2/gcinode.c @@ -175,6 +175,7 @@ int nilfs_init_gcinode(struct inode *inode) /** * nilfs_remove_all_gcinodes() - remove all unprocessed gc inodes + * @nilfs: NILFS filesystem instance */ void nilfs_remove_all_gcinodes(struct the_nilfs *nilfs) { From 14ee22fef420c864c0869419e54aa4e88f64b4e6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 25 Apr 2024 15:16:59 +0200 Subject: [PATCH 0811/1702] xfs: factor out a xfs_dir_lookup_args helper Add a helper to switch between the different directory formats for lookup and to handle the -EEXIST return for a successful lookup. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_dir2.c | 66 ++++++++++++++++++++++++---------------- fs/xfs/libxfs/xfs_dir2.h | 2 ++ fs/xfs/scrub/readdir.c | 35 +-------------------- 3 files changed, 43 insertions(+), 60 deletions(-) diff --git a/fs/xfs/libxfs/xfs_dir2.c b/fs/xfs/libxfs/xfs_dir2.c index 7634344dc5153..b4f9359089117 100644 --- a/fs/xfs/libxfs/xfs_dir2.c +++ b/fs/xfs/libxfs/xfs_dir2.c @@ -352,6 +352,45 @@ xfs_dir_cilookup_result( return -EEXIST; } +int +xfs_dir_lookup_args( + struct xfs_da_args *args) +{ + bool is_block, is_leaf; + int error; + + if (args->dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) { + error = xfs_dir2_sf_lookup(args); + goto out; + } + + /* dir2 functions require that the data fork is loaded */ + error = xfs_iread_extents(args->trans, args->dp, XFS_DATA_FORK); + if (error) + goto out; + + error = xfs_dir2_isblock(args, &is_block); + if (error) + goto out; + + if (is_block) { + error = xfs_dir2_block_lookup(args); + goto out; + } + + error = xfs_dir2_isleaf(args, &is_leaf); + if (error) + goto out; + if (is_leaf) + error = xfs_dir2_leaf_lookup(args); + else + error = xfs_dir2_node_lookup(args); +out: + if (error != -EEXIST) + return error; + return 0; +} + /* * Lookup a name in a directory, give back the inode number. * If ci_name is not NULL, returns the actual name in ci_name if it differs @@ -368,7 +407,6 @@ xfs_dir_lookup( { struct xfs_da_args *args; int rval; - bool v; int lock_mode; ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); @@ -390,30 +428,7 @@ xfs_dir_lookup( args->op_flags |= XFS_DA_OP_CILOOKUP; lock_mode = xfs_ilock_data_map_shared(dp); - if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) { - rval = xfs_dir2_sf_lookup(args); - goto out_check_rval; - } - - rval = xfs_dir2_isblock(args, &v); - if (rval) - goto out_free; - if (v) { - rval = xfs_dir2_block_lookup(args); - goto out_check_rval; - } - - rval = xfs_dir2_isleaf(args, &v); - if (rval) - goto out_free; - if (v) - rval = xfs_dir2_leaf_lookup(args); - else - rval = xfs_dir2_node_lookup(args); - -out_check_rval: - if (rval == -EEXIST) - rval = 0; + rval = xfs_dir_lookup_args(args); if (!rval) { *inum = args->inumber; if (ci_name) { @@ -421,7 +436,6 @@ xfs_dir_lookup( ci_name->len = args->valuelen; } } -out_free: xfs_iunlock(dp, lock_mode); kfree(args); return rval; diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h index b580a78bcf4fc..982c2249bfa30 100644 --- a/fs/xfs/libxfs/xfs_dir2.h +++ b/fs/xfs/libxfs/xfs_dir2.h @@ -66,6 +66,8 @@ extern int xfs_dir_replace(struct xfs_trans *tp, struct xfs_inode *dp, extern int xfs_dir_canenter(struct xfs_trans *tp, struct xfs_inode *dp, struct xfs_name *name); +int xfs_dir_lookup_args(struct xfs_da_args *args); + /* * Direct call from the bmap code, bypassing the generic directory layer. */ diff --git a/fs/xfs/scrub/readdir.c b/fs/xfs/scrub/readdir.c index 28a94c78b0b19..0ac77359d8e9f 100644 --- a/fs/xfs/scrub/readdir.c +++ b/fs/xfs/scrub/readdir.c @@ -328,7 +328,6 @@ xchk_dir_lookup( .op_flags = XFS_DA_OP_OKNOENT, .owner = dp->i_ino, }; - bool isblock, isleaf; int error; if (xfs_is_shutdown(dp->i_mount)) @@ -344,39 +343,7 @@ xchk_dir_lookup( ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); xfs_assert_ilocked(dp, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); - if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) { - error = xfs_dir2_sf_lookup(&args); - goto out_check_rval; - } - - /* dir2 functions require that the data fork is loaded */ - error = xfs_iread_extents(sc->tp, dp, XFS_DATA_FORK); - if (error) - return error; - - error = xfs_dir2_isblock(&args, &isblock); - if (error) - return error; - - if (isblock) { - error = xfs_dir2_block_lookup(&args); - goto out_check_rval; - } - - error = xfs_dir2_isleaf(&args, &isleaf); - if (error) - return error; - - if (isleaf) { - error = xfs_dir2_leaf_lookup(&args); - goto out_check_rval; - } - - error = xfs_dir2_node_lookup(&args); - -out_check_rval: - if (error == -EEXIST) - error = 0; + error = xfs_dir_lookup_args(&args); if (!error) *ino = args.inumber; return error; From 4d893a40514e9c4ee6e3be48ed1b77efb38c313b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 25 Apr 2024 15:17:00 +0200 Subject: [PATCH 0812/1702] xfs: factor out a xfs_dir_createname_args helper Add a helper to switch between the different directory formats for creating a directory entry and to handle the XFS_DA_OP_JUSTCHECK flag based on the passed in ino number field. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_dir2.c | 53 +++++++++++++++++++++------------------ fs/xfs/libxfs/xfs_dir2.h | 1 + fs/xfs/scrub/dir_repair.c | 19 +------------- 3 files changed, 30 insertions(+), 43 deletions(-) diff --git a/fs/xfs/libxfs/xfs_dir2.c b/fs/xfs/libxfs/xfs_dir2.c index b4f9359089117..e2727602d0479 100644 --- a/fs/xfs/libxfs/xfs_dir2.c +++ b/fs/xfs/libxfs/xfs_dir2.c @@ -256,6 +256,33 @@ xfs_dir_init( return error; } +int +xfs_dir_createname_args( + struct xfs_da_args *args) +{ + bool is_block, is_leaf; + int error; + + if (!args->inumber) + args->op_flags |= XFS_DA_OP_JUSTCHECK; + + if (args->dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) + return xfs_dir2_sf_addname(args); + + error = xfs_dir2_isblock(args, &is_block); + if (error) + return error; + if (is_block) + return xfs_dir2_block_addname(args); + + error = xfs_dir2_isleaf(args, &is_leaf); + if (error) + return error; + if (is_leaf) + return xfs_dir2_leaf_addname(args); + return xfs_dir2_node_addname(args); +} + /* * Enter a name in a directory, or check for available space. * If inum is 0, only the available space test is performed. @@ -270,7 +297,6 @@ xfs_dir_createname( { struct xfs_da_args *args; int rval; - bool v; ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); @@ -297,31 +323,8 @@ xfs_dir_createname( args->trans = tp; args->op_flags = XFS_DA_OP_ADDNAME | XFS_DA_OP_OKNOENT; args->owner = dp->i_ino; - if (!inum) - args->op_flags |= XFS_DA_OP_JUSTCHECK; - - if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) { - rval = xfs_dir2_sf_addname(args); - goto out_free; - } - rval = xfs_dir2_isblock(args, &v); - if (rval) - goto out_free; - if (v) { - rval = xfs_dir2_block_addname(args); - goto out_free; - } - - rval = xfs_dir2_isleaf(args, &v); - if (rval) - goto out_free; - if (v) - rval = xfs_dir2_leaf_addname(args); - else - rval = xfs_dir2_node_addname(args); - -out_free: + rval = xfs_dir_createname_args(args); kfree(args); return rval; } diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h index 982c2249bfa30..f5361dd7b90a9 100644 --- a/fs/xfs/libxfs/xfs_dir2.h +++ b/fs/xfs/libxfs/xfs_dir2.h @@ -67,6 +67,7 @@ extern int xfs_dir_canenter(struct xfs_trans *tp, struct xfs_inode *dp, struct xfs_name *name); int xfs_dir_lookup_args(struct xfs_da_args *args); +int xfs_dir_createname_args(struct xfs_da_args *args); /* * Direct call from the bmap code, bypassing the generic directory layer. diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index fa1a9954d48d9..a1e31b7827881 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -687,7 +687,6 @@ xrep_dir_replay_createname( { struct xfs_scrub *sc = rd->sc; struct xfs_inode *dp = rd->sc->tempip; - bool is_block, is_leaf; int error; ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); @@ -702,23 +701,7 @@ xrep_dir_replay_createname( rd->args.inumber = inum; rd->args.total = total; rd->args.op_flags = XFS_DA_OP_ADDNAME | XFS_DA_OP_OKNOENT; - - if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) - return xfs_dir2_sf_addname(&rd->args); - - error = xfs_dir2_isblock(&rd->args, &is_block); - if (error) - return error; - if (is_block) - return xfs_dir2_block_addname(&rd->args); - - error = xfs_dir2_isleaf(&rd->args, &is_leaf); - if (error) - return error; - if (is_leaf) - return xfs_dir2_leaf_addname(&rd->args); - - return xfs_dir2_node_addname(&rd->args); + return xfs_dir_createname_args(&rd->args); } /* Replay a stashed removename onto the temporary directory. */ From 3866e6e669e2c1b3eebf580b8779ea55838c3f5a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 25 Apr 2024 15:17:01 +0200 Subject: [PATCH 0813/1702] xfs: factor out a xfs_dir_removename_args helper Add a helper to switch between the different directory formats for removing a directory entry. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_dir2.c | 48 ++++++++++++++++++++------------------- fs/xfs/libxfs/xfs_dir2.h | 1 + fs/xfs/scrub/dir_repair.c | 20 +--------------- 3 files changed, 27 insertions(+), 42 deletions(-) diff --git a/fs/xfs/libxfs/xfs_dir2.c b/fs/xfs/libxfs/xfs_dir2.c index e2727602d0479..76aa11ade2e92 100644 --- a/fs/xfs/libxfs/xfs_dir2.c +++ b/fs/xfs/libxfs/xfs_dir2.c @@ -444,6 +444,30 @@ xfs_dir_lookup( return rval; } +int +xfs_dir_removename_args( + struct xfs_da_args *args) +{ + bool is_block, is_leaf; + int error; + + if (args->dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) + return xfs_dir2_sf_removename(args); + + error = xfs_dir2_isblock(args, &is_block); + if (error) + return error; + if (is_block) + return xfs_dir2_block_removename(args); + + error = xfs_dir2_isleaf(args, &is_leaf); + if (error) + return error; + if (is_leaf) + return xfs_dir2_leaf_removename(args); + return xfs_dir2_node_removename(args); +} + /* * Remove an entry from a directory. */ @@ -457,7 +481,6 @@ xfs_dir_removename( { struct xfs_da_args *args; int rval; - bool v; ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); XFS_STATS_INC(dp->i_mount, xs_dir_remove); @@ -477,28 +500,7 @@ xfs_dir_removename( args->whichfork = XFS_DATA_FORK; args->trans = tp; args->owner = dp->i_ino; - - if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) { - rval = xfs_dir2_sf_removename(args); - goto out_free; - } - - rval = xfs_dir2_isblock(args, &v); - if (rval) - goto out_free; - if (v) { - rval = xfs_dir2_block_removename(args); - goto out_free; - } - - rval = xfs_dir2_isleaf(args, &v); - if (rval) - goto out_free; - if (v) - rval = xfs_dir2_leaf_removename(args); - else - rval = xfs_dir2_node_removename(args); -out_free: + rval = xfs_dir_removename_args(args); kfree(args); return rval; } diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h index f5361dd7b90a9..3db54801d69ec 100644 --- a/fs/xfs/libxfs/xfs_dir2.h +++ b/fs/xfs/libxfs/xfs_dir2.h @@ -68,6 +68,7 @@ extern int xfs_dir_canenter(struct xfs_trans *tp, struct xfs_inode *dp, int xfs_dir_lookup_args(struct xfs_da_args *args); int xfs_dir_createname_args(struct xfs_da_args *args); +int xfs_dir_removename_args(struct xfs_da_args *args); /* * Direct call from the bmap code, bypassing the generic directory layer. diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index a1e31b7827881..98e4ed25cc230 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -712,8 +712,6 @@ xrep_dir_replay_removename( xfs_extlen_t total) { struct xfs_inode *dp = rd->args.dp; - bool is_block, is_leaf; - int error; ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); @@ -722,23 +720,7 @@ xrep_dir_replay_removename( rd->args.total = total; trace_xrep_dir_replay_removename(dp, name, 0); - - if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) - return xfs_dir2_sf_removename(&rd->args); - - error = xfs_dir2_isblock(&rd->args, &is_block); - if (error) - return error; - if (is_block) - return xfs_dir2_block_removename(&rd->args); - - error = xfs_dir2_isleaf(&rd->args, &is_leaf); - if (error) - return error; - if (is_leaf) - return xfs_dir2_leaf_removename(&rd->args); - - return xfs_dir2_node_removename(&rd->args); + return xfs_dir_removename_args(&rd->args); } /* From dfe5febe2b6a175d730861441bff4f726fc58a6c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 25 Apr 2024 15:17:02 +0200 Subject: [PATCH 0814/1702] xfs: factor out a xfs_dir_replace_args helper Add a helper to switch between the different directory formats for removing a directory entry. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_dir2.c | 49 +++++++++++++++++++++------------------ fs/xfs/libxfs/xfs_dir2.h | 1 + fs/xfs/scrub/dir_repair.c | 19 +-------------- 3 files changed, 28 insertions(+), 41 deletions(-) diff --git a/fs/xfs/libxfs/xfs_dir2.c b/fs/xfs/libxfs/xfs_dir2.c index 76aa11ade2e92..d3d4d80c2098d 100644 --- a/fs/xfs/libxfs/xfs_dir2.c +++ b/fs/xfs/libxfs/xfs_dir2.c @@ -505,6 +505,31 @@ xfs_dir_removename( return rval; } +int +xfs_dir_replace_args( + struct xfs_da_args *args) +{ + bool is_block, is_leaf; + int error; + + if (args->dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) + return xfs_dir2_sf_replace(args); + + error = xfs_dir2_isblock(args, &is_block); + if (error) + return error; + if (is_block) + return xfs_dir2_block_replace(args); + + error = xfs_dir2_isleaf(args, &is_leaf); + if (error) + return error; + if (is_leaf) + return xfs_dir2_leaf_replace(args); + + return xfs_dir2_node_replace(args); +} + /* * Replace the inode number of a directory entry. */ @@ -518,7 +543,6 @@ xfs_dir_replace( { struct xfs_da_args *args; int rval; - bool v; ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); @@ -541,28 +565,7 @@ xfs_dir_replace( args->whichfork = XFS_DATA_FORK; args->trans = tp; args->owner = dp->i_ino; - - if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) { - rval = xfs_dir2_sf_replace(args); - goto out_free; - } - - rval = xfs_dir2_isblock(args, &v); - if (rval) - goto out_free; - if (v) { - rval = xfs_dir2_block_replace(args); - goto out_free; - } - - rval = xfs_dir2_isleaf(args, &v); - if (rval) - goto out_free; - if (v) - rval = xfs_dir2_leaf_replace(args); - else - rval = xfs_dir2_node_replace(args); -out_free: + rval = xfs_dir_replace_args(args); kfree(args); return rval; } diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h index 3db54801d69ec..6c00fe24a8987 100644 --- a/fs/xfs/libxfs/xfs_dir2.h +++ b/fs/xfs/libxfs/xfs_dir2.h @@ -69,6 +69,7 @@ extern int xfs_dir_canenter(struct xfs_trans *tp, struct xfs_inode *dp, int xfs_dir_lookup_args(struct xfs_da_args *args); int xfs_dir_createname_args(struct xfs_da_args *args); int xfs_dir_removename_args(struct xfs_da_args *args); +int xfs_dir_replace_args(struct xfs_da_args *args); /* * Direct call from the bmap code, bypassing the generic directory layer. diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index 98e4ed25cc230..64679fe084465 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -1513,7 +1513,6 @@ xrep_dir_replace( xfs_extlen_t total) { struct xfs_scrub *sc = rd->sc; - bool is_block, is_leaf; int error; ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); @@ -1525,23 +1524,7 @@ xrep_dir_replace( xrep_dir_init_args(rd, dp, name); rd->args.inumber = inum; rd->args.total = total; - - if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) - return xfs_dir2_sf_replace(&rd->args); - - error = xfs_dir2_isblock(&rd->args, &is_block); - if (error) - return error; - if (is_block) - return xfs_dir2_block_replace(&rd->args); - - error = xfs_dir2_isleaf(&rd->args, &is_leaf); - if (error) - return error; - if (is_leaf) - return xfs_dir2_leaf_replace(&rd->args); - - return xfs_dir2_node_replace(&rd->args); + return xfs_dir_replace_args(&rd->args); } /* From e58ac1770ded2a316447ca7608bb7809af82eca6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 25 Apr 2024 15:17:03 +0200 Subject: [PATCH 0815/1702] xfs: refactor dir format helpers Add a new enum and a xfs_dir2_format helper that returns it to allow the code to switch on the format of a directory in a single operation and switch all helpers of xfs_dir2_isblock and xfs_dir2_isleaf to it. This also removes the explicit xfs_iread_extents call in a few of the call sites given that xfs_bmap_last_offset already takes care of it underneath. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_dir2.c | 188 ++++++++++++++--------------------- fs/xfs/libxfs/xfs_dir2.h | 12 ++- fs/xfs/libxfs/xfs_exchmaps.c | 9 +- fs/xfs/scrub/dir.c | 3 +- fs/xfs/scrub/readdir.c | 24 ++--- fs/xfs/xfs_dir2_readdir.c | 19 ++-- 6 files changed, 105 insertions(+), 150 deletions(-) diff --git a/fs/xfs/libxfs/xfs_dir2.c b/fs/xfs/libxfs/xfs_dir2.c index d3d4d80c2098d..457f9a38f8504 100644 --- a/fs/xfs/libxfs/xfs_dir2.c +++ b/fs/xfs/libxfs/xfs_dir2.c @@ -256,31 +256,60 @@ xfs_dir_init( return error; } +enum xfs_dir2_fmt +xfs_dir2_format( + struct xfs_da_args *args, + int *error) +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_da_geometry *geo = mp->m_dir_geo; + xfs_fileoff_t eof; + + xfs_assert_ilocked(dp, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); + + *error = 0; + if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) + return XFS_DIR2_FMT_SF; + + *error = xfs_bmap_last_offset(dp, &eof, XFS_DATA_FORK); + if (*error) + return XFS_DIR2_FMT_ERROR; + + if (eof == XFS_B_TO_FSB(mp, geo->blksize)) { + if (XFS_IS_CORRUPT(mp, dp->i_disk_size != geo->blksize)) { + xfs_da_mark_sick(args); + *error = -EFSCORRUPTED; + return XFS_DIR2_FMT_ERROR; + } + return XFS_DIR2_FMT_BLOCK; + } + if (eof == geo->leafblk + geo->fsbcount) + return XFS_DIR2_FMT_LEAF; + return XFS_DIR2_FMT_NODE; +} + int xfs_dir_createname_args( struct xfs_da_args *args) { - bool is_block, is_leaf; int error; if (!args->inumber) args->op_flags |= XFS_DA_OP_JUSTCHECK; - if (args->dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) + switch (xfs_dir2_format(args, &error)) { + case XFS_DIR2_FMT_SF: return xfs_dir2_sf_addname(args); - - error = xfs_dir2_isblock(args, &is_block); - if (error) - return error; - if (is_block) + case XFS_DIR2_FMT_BLOCK: return xfs_dir2_block_addname(args); - - error = xfs_dir2_isleaf(args, &is_leaf); - if (error) - return error; - if (is_leaf) + case XFS_DIR2_FMT_LEAF: return xfs_dir2_leaf_addname(args); - return xfs_dir2_node_addname(args); + case XFS_DIR2_FMT_NODE: + return xfs_dir2_node_addname(args); + default: + return error; + } } /* @@ -359,36 +388,25 @@ int xfs_dir_lookup_args( struct xfs_da_args *args) { - bool is_block, is_leaf; int error; - if (args->dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) { + switch (xfs_dir2_format(args, &error)) { + case XFS_DIR2_FMT_SF: error = xfs_dir2_sf_lookup(args); - goto out; - } - - /* dir2 functions require that the data fork is loaded */ - error = xfs_iread_extents(args->trans, args->dp, XFS_DATA_FORK); - if (error) - goto out; - - error = xfs_dir2_isblock(args, &is_block); - if (error) - goto out; - - if (is_block) { + break; + case XFS_DIR2_FMT_BLOCK: error = xfs_dir2_block_lookup(args); - goto out; - } - - error = xfs_dir2_isleaf(args, &is_leaf); - if (error) - goto out; - if (is_leaf) + break; + case XFS_DIR2_FMT_LEAF: error = xfs_dir2_leaf_lookup(args); - else + break; + case XFS_DIR2_FMT_NODE: error = xfs_dir2_node_lookup(args); -out: + break; + default: + break; + } + if (error != -EEXIST) return error; return 0; @@ -448,24 +466,20 @@ int xfs_dir_removename_args( struct xfs_da_args *args) { - bool is_block, is_leaf; int error; - if (args->dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) + switch (xfs_dir2_format(args, &error)) { + case XFS_DIR2_FMT_SF: return xfs_dir2_sf_removename(args); - - error = xfs_dir2_isblock(args, &is_block); - if (error) - return error; - if (is_block) + case XFS_DIR2_FMT_BLOCK: return xfs_dir2_block_removename(args); - - error = xfs_dir2_isleaf(args, &is_leaf); - if (error) - return error; - if (is_leaf) + case XFS_DIR2_FMT_LEAF: return xfs_dir2_leaf_removename(args); - return xfs_dir2_node_removename(args); + case XFS_DIR2_FMT_NODE: + return xfs_dir2_node_removename(args); + default: + return error; + } } /* @@ -509,25 +523,20 @@ int xfs_dir_replace_args( struct xfs_da_args *args) { - bool is_block, is_leaf; int error; - if (args->dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) + switch (xfs_dir2_format(args, &error)) { + case XFS_DIR2_FMT_SF: return xfs_dir2_sf_replace(args); - - error = xfs_dir2_isblock(args, &is_block); - if (error) - return error; - if (is_block) + case XFS_DIR2_FMT_BLOCK: return xfs_dir2_block_replace(args); - - error = xfs_dir2_isleaf(args, &is_leaf); - if (error) - return error; - if (is_leaf) + case XFS_DIR2_FMT_LEAF: return xfs_dir2_leaf_replace(args); - - return xfs_dir2_node_replace(args); + case XFS_DIR2_FMT_NODE: + return xfs_dir2_node_replace(args); + default: + return error; + } } /* @@ -633,57 +642,6 @@ xfs_dir2_grow_inode( return 0; } -/* - * See if the directory is a single-block form directory. - */ -int -xfs_dir2_isblock( - struct xfs_da_args *args, - bool *isblock) -{ - struct xfs_mount *mp = args->dp->i_mount; - xfs_fileoff_t eof; - int error; - - error = xfs_bmap_last_offset(args->dp, &eof, XFS_DATA_FORK); - if (error) - return error; - - *isblock = false; - if (XFS_FSB_TO_B(mp, eof) != args->geo->blksize) - return 0; - - *isblock = true; - if (XFS_IS_CORRUPT(mp, args->dp->i_disk_size != args->geo->blksize)) { - xfs_da_mark_sick(args); - return -EFSCORRUPTED; - } - return 0; -} - -/* - * See if the directory is a single-leaf form directory. - */ -int -xfs_dir2_isleaf( - struct xfs_da_args *args, - bool *isleaf) -{ - xfs_fileoff_t eof; - int error; - - error = xfs_bmap_last_offset(args->dp, &eof, XFS_DATA_FORK); - if (error) - return error; - - *isleaf = false; - if (eof != args->geo->leafblk + args->geo->fsbcount) - return 0; - - *isleaf = true; - return 0; -} - /* * Remove the given block from the directory. * This routine is used for data and free blocks, leaf/node are done diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h index 6c00fe24a8987..6dbe6e9ecb491 100644 --- a/fs/xfs/libxfs/xfs_dir2.h +++ b/fs/xfs/libxfs/xfs_dir2.h @@ -36,6 +36,16 @@ xfs_dir2_samename( return !memcmp(n1->name, n2->name, n1->len); } +enum xfs_dir2_fmt { + XFS_DIR2_FMT_SF, + XFS_DIR2_FMT_BLOCK, + XFS_DIR2_FMT_LEAF, + XFS_DIR2_FMT_NODE, + XFS_DIR2_FMT_ERROR, +}; + +enum xfs_dir2_fmt xfs_dir2_format(struct xfs_da_args *args, int *error); + /* * Convert inode mode to directory entry filetype */ @@ -79,8 +89,6 @@ extern int xfs_dir2_sf_to_block(struct xfs_da_args *args); /* * Interface routines used by userspace utilities */ -extern int xfs_dir2_isblock(struct xfs_da_args *args, bool *isblock); -extern int xfs_dir2_isleaf(struct xfs_da_args *args, bool *isleaf); extern int xfs_dir2_shrink_inode(struct xfs_da_args *args, xfs_dir2_db_t db, struct xfs_buf *bp); diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index 44ab6a9235c0b..2021396651de2 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -465,17 +465,12 @@ xfs_exchmaps_dir_to_sf( }; struct xfs_dir2_sf_hdr sfh; struct xfs_buf *bp; - bool isblock; int size; - int error; + int error = 0; - error = xfs_dir2_isblock(&args, &isblock); - if (error) + if (xfs_dir2_format(&args, &error) != XFS_DIR2_FMT_BLOCK) return error; - if (!isblock) - return 0; - error = xfs_dir3_block_read(tp, xmi->xmi_ip2, xmi->xmi_ip2->i_ino, &bp); if (error) return error; diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index 62474d0557c41..bf9199e8df633 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -808,7 +808,8 @@ xchk_directory_blocks( free_lblk = XFS_B_TO_FSB(mp, XFS_DIR2_FREE_OFFSET); /* Is this a block dir? */ - error = xfs_dir2_isblock(&args, &is_block); + if (xfs_dir2_format(&args, &error) == XFS_DIR2_FMT_BLOCK) + is_block = true; if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) goto out; diff --git a/fs/xfs/scrub/readdir.c b/fs/xfs/scrub/readdir.c index 0ac77359d8e9f..01c9a2dc0f2c4 100644 --- a/fs/xfs/scrub/readdir.c +++ b/fs/xfs/scrub/readdir.c @@ -276,7 +276,6 @@ xchk_dir_walk( .trans = sc->tp, .owner = dp->i_ino, }; - bool isblock; int error; if (xfs_is_shutdown(dp->i_mount)) @@ -285,22 +284,17 @@ xchk_dir_walk( ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); xfs_assert_ilocked(dp, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); - if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) + switch (xfs_dir2_format(&args, &error)) { + case XFS_DIR2_FMT_SF: return xchk_dir_walk_sf(sc, dp, dirent_fn, priv); - - /* dir2 functions require that the data fork is loaded */ - error = xfs_iread_extents(sc->tp, dp, XFS_DATA_FORK); - if (error) - return error; - - error = xfs_dir2_isblock(&args, &isblock); - if (error) - return error; - - if (isblock) + case XFS_DIR2_FMT_BLOCK: return xchk_dir_walk_block(sc, dp, dirent_fn, priv); - - return xchk_dir_walk_leaf(sc, dp, dirent_fn, priv); + case XFS_DIR2_FMT_LEAF: + case XFS_DIR2_FMT_NODE: + return xchk_dir_walk_leaf(sc, dp, dirent_fn, priv); + default: + return error; + } } /* diff --git a/fs/xfs/xfs_dir2_readdir.c b/fs/xfs/xfs_dir2_readdir.c index b3abad5a6cd80..06ac5a7de60a0 100644 --- a/fs/xfs/xfs_dir2_readdir.c +++ b/fs/xfs/xfs_dir2_readdir.c @@ -516,7 +516,6 @@ xfs_readdir( { struct xfs_da_args args = { NULL }; unsigned int lock_mode; - bool isblock; int error; trace_xfs_readdir(dp); @@ -539,18 +538,18 @@ xfs_readdir( return xfs_dir2_sf_getdents(&args, ctx); lock_mode = xfs_ilock_data_map_shared(dp); - error = xfs_dir2_isblock(&args, &isblock); - if (error) - goto out_unlock; - - if (isblock) { + switch (xfs_dir2_format(&args, &error)) { + case XFS_DIR2_FMT_BLOCK: error = xfs_dir2_block_getdents(&args, ctx, &lock_mode); - goto out_unlock; + break; + case XFS_DIR2_FMT_LEAF: + case XFS_DIR2_FMT_NODE: + error = xfs_dir2_leaf_getdents(&args, ctx, bufsize, &lock_mode); + break; + default: + break; } - error = xfs_dir2_leaf_getdents(&args, ctx, bufsize, &lock_mode); - -out_unlock: if (lock_mode) xfs_iunlock(dp, lock_mode); return error; From a770ccd91d99f997e023b554d9e2033ce79e019e Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 24 Apr 2024 15:16:27 +0800 Subject: [PATCH 0816/1702] iommu/vt-d: Remove redundant assignment to variable err Variable err is being assigned a value that is never read. It is either being re-assigned later on error exit paths, or never referenced on the non-error path. Cleans up clang scan build warning: drivers/iommu/intel/dmar.c:1070:2: warning: Value stored to 'err' is never read [deadcode.DeadStores]` Signed-off-by: Colin Ian King Link: https://lore.kernel.org/r/20240411090535.306326-1-colin.i.king@gmail.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/dmar.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 36d7427b12026..351be94552146 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1067,7 +1067,6 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) goto error_free_seq_id; } - err = -EINVAL; if (!cap_sagaw(iommu->cap) && (!ecap_smts(iommu->ecap) || ecap_slts(iommu->ecap))) { pr_info("%s: No supported address widths. Not attempting DMA translation.\n", From 9e7ee0f045395dc8aa55fbdc164c062484f4c88d Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Wed, 24 Apr 2024 15:16:28 +0800 Subject: [PATCH 0817/1702] iommu/vt-d: Use try_cmpxchg64{,_local}() in iommu.c Replace this pattern in iommu.c: cmpxchg64{,_local}(*ptr, 0, new) != 0 ... with the simpler and faster: !try_cmpxchg64{,_local}(*ptr, &tmp, new) The x86 CMPXCHG instruction returns success in the ZF flag, so this change saves a compare after the CMPXCHG. No functional change intended. Signed-off-by: Uros Bizjak Cc: David Woodhouse Cc: Lu Baolu Cc: Joerg Roedel Cc: Will Deacon Cc: Robin Murphy Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240414162454.49584-1-ubizjak@gmail.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index a7ecd90303dc4..4a2afe89b464d 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -865,7 +865,7 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, break; if (!dma_pte_present(pte)) { - uint64_t pteval; + uint64_t pteval, tmp; tmp_page = alloc_pgtable_page(domain->nid, gfp); @@ -877,7 +877,8 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, if (domain->use_first_level) pteval |= DMA_FL_PTE_XD | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS; - if (cmpxchg64(&pte->val, 0ULL, pteval)) + tmp = 0ULL; + if (!try_cmpxchg64(&pte->val, &tmp, pteval)) /* Someone else set it while we were thinking; use theirs. */ free_pgtable_page(tmp_page); else @@ -2128,8 +2129,8 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, /* We don't need lock here, nobody else * touches the iova range */ - tmp = cmpxchg64_local(&pte->val, 0ULL, pteval); - if (tmp) { + tmp = 0ULL; + if (!try_cmpxchg64_local(&pte->val, &tmp, pteval)) { static int dumps = 5; pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n", iov_pfn, tmp, (unsigned long long)pteval); From d74169ceb0d2e32438946a2f1f9fc8c803304bd6 Mon Sep 17 00:00:00 2001 From: Dimitri Sivanich Date: Wed, 24 Apr 2024 15:16:29 +0800 Subject: [PATCH 0818/1702] iommu/vt-d: Allocate DMAR fault interrupts locally The Intel IOMMU code currently tries to allocate all DMAR fault interrupt vectors on the boot cpu. On large systems with high DMAR counts this results in vector exhaustion, and most of the vectors are not initially allocated socket local. Instead, have a cpu on each node do the vector allocation for the DMARs on that node. The boot cpu still does the allocation for its node during its boot sequence. Signed-off-by: Dimitri Sivanich Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/Zfydpp2Hm+as16TY@hpe.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 2 +- drivers/iommu/amd/init.c | 2 +- drivers/iommu/intel/dmar.c | 9 +++++++-- drivers/iommu/irq_remapping.c | 5 ++++- drivers/iommu/irq_remapping.h | 2 +- include/linux/dmar.h | 2 +- 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index f482aab420f78..410c360e7e245 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -33,7 +33,7 @@ int amd_iommu_prepare(void); int amd_iommu_enable(void); void amd_iommu_disable(void); int amd_iommu_reenable(int mode); -int amd_iommu_enable_faulting(void); +int amd_iommu_enable_faulting(unsigned int cpu); extern int amd_iommu_guest_ir; extern enum io_pgtable_fmt amd_iommu_pgtable; extern int amd_iommu_gpt_level; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index ac6754a85f350..8085e13e0100d 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -3392,7 +3392,7 @@ int amd_iommu_reenable(int mode) return 0; } -int __init amd_iommu_enable_faulting(void) +int __init amd_iommu_enable_faulting(unsigned int cpu) { /* We enable MSI later when PCI is initialized */ return 0; diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 351be94552146..932e0c10c0fe1 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -2121,7 +2121,7 @@ int dmar_set_interrupt(struct intel_iommu *iommu) return ret; } -int __init enable_drhd_fault_handling(void) +int enable_drhd_fault_handling(unsigned int cpu) { struct dmar_drhd_unit *drhd; struct intel_iommu *iommu; @@ -2131,7 +2131,12 @@ int __init enable_drhd_fault_handling(void) */ for_each_iommu(iommu, drhd) { u32 fault_status; - int ret = dmar_set_interrupt(iommu); + int ret; + + if (iommu->irq || iommu->node != cpu_to_node(cpu)) + continue; + + ret = dmar_set_interrupt(iommu); if (ret) { pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n", diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c index ee59647c20501..2f7281ccc05f6 100644 --- a/drivers/iommu/irq_remapping.c +++ b/drivers/iommu/irq_remapping.c @@ -151,7 +151,10 @@ int __init irq_remap_enable_fault_handling(void) if (!remap_ops->enable_faulting) return -ENODEV; - return remap_ops->enable_faulting(); + cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "dmar:enable_fault_handling", + remap_ops->enable_faulting, NULL); + + return remap_ops->enable_faulting(smp_processor_id()); } void panic_if_irq_remap(const char *msg) diff --git a/drivers/iommu/irq_remapping.h b/drivers/iommu/irq_remapping.h index 8c89cb947cdb2..0d6f140b5e014 100644 --- a/drivers/iommu/irq_remapping.h +++ b/drivers/iommu/irq_remapping.h @@ -41,7 +41,7 @@ struct irq_remap_ops { int (*reenable)(int); /* Enable fault handling */ - int (*enable_faulting)(void); + int (*enable_faulting)(unsigned int); }; extern struct irq_remap_ops intel_irq_remap_ops; diff --git a/include/linux/dmar.h b/include/linux/dmar.h index e34b601b71fd2..499bb2c634832 100644 --- a/include/linux/dmar.h +++ b/include/linux/dmar.h @@ -117,7 +117,7 @@ extern int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, int count); /* Intel IOMMU detection */ void detect_intel_iommu(void); -extern int enable_drhd_fault_handling(void); +extern int enable_drhd_fault_handling(unsigned int cpu); extern int dmar_device_add(acpi_handle handle); extern int dmar_device_remove(acpi_handle handle); From cc9e49d35b4de47d6b656ac144cb22b11dc65c2e Mon Sep 17 00:00:00 2001 From: Jingqi Liu Date: Wed, 24 Apr 2024 15:16:30 +0800 Subject: [PATCH 0819/1702] iommu/vt-d: Remove debugfs use of private data field Since the page fault report and response have been tracked by ftrace, the users can easily calculate the time used for a page fault handling. There's no need to expose the similar functionality in debugfs. Hence, remove the corresponding operations in debugfs. Signed-off-by: Jingqi Liu Link: https://lore.kernel.org/r/20240308103811.76744-2-Jingqi.liu@intel.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/debugfs.c | 7 ------- drivers/iommu/intel/perf.h | 1 - drivers/iommu/intel/svm.c | 9 --------- 3 files changed, 17 deletions(-) diff --git a/drivers/iommu/intel/debugfs.c b/drivers/iommu/intel/debugfs.c index 86b506af7daa1..affbf4a1558de 100644 --- a/drivers/iommu/intel/debugfs.c +++ b/drivers/iommu/intel/debugfs.c @@ -706,7 +706,6 @@ static ssize_t dmar_perf_latency_write(struct file *filp, dmar_latency_disable(iommu, DMAR_LATENCY_INV_IOTLB); dmar_latency_disable(iommu, DMAR_LATENCY_INV_DEVTLB); dmar_latency_disable(iommu, DMAR_LATENCY_INV_IEC); - dmar_latency_disable(iommu, DMAR_LATENCY_PRQ); } rcu_read_unlock(); break; @@ -728,12 +727,6 @@ static ssize_t dmar_perf_latency_write(struct file *filp, dmar_latency_enable(iommu, DMAR_LATENCY_INV_IEC); rcu_read_unlock(); break; - case 4: - rcu_read_lock(); - for_each_active_iommu(iommu, drhd) - dmar_latency_enable(iommu, DMAR_LATENCY_PRQ); - rcu_read_unlock(); - break; default: return -EINVAL; } diff --git a/drivers/iommu/intel/perf.h b/drivers/iommu/intel/perf.h index fd6db8049d1a7..df9a36942d643 100644 --- a/drivers/iommu/intel/perf.h +++ b/drivers/iommu/intel/perf.h @@ -11,7 +11,6 @@ enum latency_type { DMAR_LATENCY_INV_IOTLB = 0, DMAR_LATENCY_INV_DEVTLB, DMAR_LATENCY_INV_IEC, - DMAR_LATENCY_PRQ, DMAR_LATENCY_NUM }; diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index ee3b469e2da15..e014350db3545 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -583,12 +583,6 @@ static void intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; event.fault.prm.private_data[0] = desc->priv_data[0]; event.fault.prm.private_data[1] = desc->priv_data[1]; - } else if (dmar_latency_enabled(iommu, DMAR_LATENCY_PRQ)) { - /* - * If the private data fields are not used by hardware, use it - * to monitor the prq handle latency. - */ - event.fault.prm.private_data[0] = ktime_to_ns(ktime_get()); } iommu_report_device_fault(dev, &event); @@ -768,9 +762,6 @@ void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, if (private_present) { desc.qw2 = prm->private_data[0]; desc.qw3 = prm->private_data[1]; - } else if (prm->private_data[0]) { - dmar_latency_update(iommu, DMAR_LATENCY_PRQ, - ktime_to_ns(ktime_get()) - prm->private_data[0]); } qi_submit_sync(iommu, &desc, 1, 0); From 621b7e54f288c5e4e32d1dd81a926b8ecb547c60 Mon Sep 17 00:00:00 2001 From: Jingqi Liu Date: Wed, 24 Apr 2024 15:16:31 +0800 Subject: [PATCH 0820/1702] iommu/vt-d: Remove private data use in fault message According to Intel VT-d specification revision 4.0, "Private Data" field has been removed from Page Request/Response. Since the private data field is not used in fault message, remove the related definitions in page request descriptor and remove the related code in page request/response handler, as Intel hasn't shipped any products which support private data in the page request message. Signed-off-by: Jingqi Liu Link: https://lore.kernel.org/r/20240308103811.76744-3-Jingqi.liu@intel.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.h | 1 - drivers/iommu/intel/svm.c | 75 ++++++++----------------------------- include/linux/iommu.h | 3 +- 3 files changed, 16 insertions(+), 63 deletions(-) diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 404d2476a8774..9ee326f7bf621 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -455,7 +455,6 @@ enum { /* Page group response descriptor QW0 */ #define QI_PGRP_PASID_P(p) (((u64)(p)) << 4) -#define QI_PGRP_PDP(p) (((u64)(p)) << 5) #define QI_PGRP_RESP_CODE(res) (((u64)(res)) << 12) #define QI_PGRP_DID(rid) (((u64)(rid)) << 16) #define QI_PGRP_PASID(pasid) (((u64)(pasid)) << 32) diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index e014350db3545..e05c6c4cb8c39 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -418,8 +418,7 @@ struct page_req_dsc { struct { u64 type:8; u64 pasid_present:1; - u64 priv_data_present:1; - u64 rsvd:6; + u64 rsvd:7; u64 rid:16; u64 pasid:20; u64 exe_req:1; @@ -438,7 +437,8 @@ struct page_req_dsc { }; u64 qw_1; }; - u64 priv_data[2]; + u64 qw_2; + u64 qw_3; }; static bool is_canonical_address(u64 addr) @@ -572,18 +572,6 @@ static void intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; } - if (desc->priv_data_present) { - /* - * Set last page in group bit if private data is present, - * page response is required as it does for LPIG. - * iommu_report_device_fault() doesn't understand this vendor - * specific requirement thus we set last_page as a workaround. - */ - event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; - event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; - event.fault.prm.private_data[0] = desc->priv_data[0]; - event.fault.prm.private_data[1] = desc->priv_data[1]; - } iommu_report_device_fault(dev, &event); } @@ -591,39 +579,23 @@ static void intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, static void handle_bad_prq_event(struct intel_iommu *iommu, struct page_req_dsc *req, int result) { - struct qi_desc desc; + struct qi_desc desc = { }; pr_err("%s: Invalid page request: %08llx %08llx\n", iommu->name, ((unsigned long long *)req)[0], ((unsigned long long *)req)[1]); - /* - * Per VT-d spec. v3.0 ch7.7, system software must - * respond with page group response if private data - * is present (PDP) or last page in group (LPIG) bit - * is set. This is an additional VT-d feature beyond - * PCI ATS spec. - */ - if (!req->lpig && !req->priv_data_present) + if (!req->lpig) return; desc.qw0 = QI_PGRP_PASID(req->pasid) | QI_PGRP_DID(req->rid) | QI_PGRP_PASID_P(req->pasid_present) | - QI_PGRP_PDP(req->priv_data_present) | QI_PGRP_RESP_CODE(result) | QI_PGRP_RESP_TYPE; desc.qw1 = QI_PGRP_IDX(req->prg_index) | QI_PGRP_LPIG(req->lpig); - if (req->priv_data_present) { - desc.qw2 = req->priv_data[0]; - desc.qw3 = req->priv_data[1]; - } else { - desc.qw2 = 0; - desc.qw3 = 0; - } - qi_submit_sync(iommu, &desc, 1, 0); } @@ -691,7 +663,7 @@ static irqreturn_t prq_event_thread(int irq, void *d) intel_svm_prq_report(iommu, dev, req); trace_prq_report(iommu, dev, req->qw_0, req->qw_1, - req->priv_data[0], req->priv_data[1], + req->qw_2, req->qw_3, iommu->prq_seq_number++); mutex_unlock(&iommu->iopf_lock); prq_advance: @@ -730,7 +702,7 @@ void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, struct intel_iommu *iommu = info->iommu; u8 bus = info->bus, devfn = info->devfn; struct iommu_fault_page_request *prm; - bool private_present; + struct qi_desc desc; bool pasid_present; bool last_page; u16 sid; @@ -738,34 +710,17 @@ void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, prm = &evt->fault.prm; sid = PCI_DEVID(bus, devfn); pasid_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; - private_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; last_page = prm->flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; - /* - * Per VT-d spec. v3.0 ch7.7, system software must respond - * with page group response if private data is present (PDP) - * or last page in group (LPIG) bit is set. This is an - * additional VT-d requirement beyond PCI ATS spec. - */ - if (last_page || private_present) { - struct qi_desc desc; - - desc.qw0 = QI_PGRP_PASID(prm->pasid) | QI_PGRP_DID(sid) | - QI_PGRP_PASID_P(pasid_present) | - QI_PGRP_PDP(private_present) | - QI_PGRP_RESP_CODE(msg->code) | - QI_PGRP_RESP_TYPE; - desc.qw1 = QI_PGRP_IDX(prm->grpid) | QI_PGRP_LPIG(last_page); - desc.qw2 = 0; - desc.qw3 = 0; - - if (private_present) { - desc.qw2 = prm->private_data[0]; - desc.qw3 = prm->private_data[1]; - } + desc.qw0 = QI_PGRP_PASID(prm->pasid) | QI_PGRP_DID(sid) | + QI_PGRP_PASID_P(pasid_present) | + QI_PGRP_RESP_CODE(msg->code) | + QI_PGRP_RESP_TYPE; + desc.qw1 = QI_PGRP_IDX(prm->grpid) | QI_PGRP_LPIG(last_page); + desc.qw2 = 0; + desc.qw3 = 0; - qi_submit_sync(iommu, &desc, 1, 0); - } + qi_submit_sync(iommu, &desc, 1, 0); } static void intel_svm_domain_free(struct iommu_domain *domain) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 2e925b5eba534..e6549bdfaed96 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -69,8 +69,7 @@ enum iommu_fault_type { struct iommu_fault_page_request { #define IOMMU_FAULT_PAGE_REQUEST_PASID_VALID (1 << 0) #define IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE (1 << 1) -#define IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA (1 << 2) -#define IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID (1 << 3) +#define IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID (1 << 2) u32 flags; u32 pasid; u32 grpid; From 304b3bde24b58515a75fd198beb52ca57df6275f Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 24 Apr 2024 15:16:32 +0800 Subject: [PATCH 0821/1702] iommu/vt-d: Remove caching mode check before device TLB flush The Caching Mode (CM) of the Intel IOMMU indicates if the hardware implementation caches not-present or erroneous translation-structure entries except for the first-stage translation. The caching mode is irrelevant to the device TLB, therefore there is no need to check it before a device TLB invalidation operation. Remove two caching mode checks before device TLB invalidation in the driver. The removal of these checks doesn't change the driver's behavior in critical map/unmap paths. Hence, there is no functionality or performance impact, especially since commit <29b32839725f> ("iommu/vt-d: Do not use flush-queue when caching-mode is on") has already disabled flush-queue for caching mode. Therefore, caching mode will never call intel_flush_iotlb_all(). Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Reviewed-by: Yi Liu Link: https://lore.kernel.org/r/20240415013835.9527-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 4a2afe89b464d..002fee5fcb80d 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1502,11 +1502,7 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, else __iommu_flush_iotlb_psi(iommu, did, pfn, pages, ih); - /* - * In caching mode, changes of pages from non-present to present require - * flush. However, device IOTLB doesn't need to be flushed in this case. - */ - if (!cap_caching_mode(iommu->cap) || !map) + if (!map) iommu_flush_dev_iotlb(domain, addr, mask); } @@ -1580,8 +1576,7 @@ static void intel_flush_iotlb_all(struct iommu_domain *domain) iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH); - if (!cap_caching_mode(iommu->cap)) - iommu_flush_dev_iotlb(dmar_domain, 0, MAX_AGAW_PFN_WIDTH); + iommu_flush_dev_iotlb(dmar_domain, 0, MAX_AGAW_PFN_WIDTH); } if (dmar_domain->nested_parent) From 3b1d9e2b2d6856eabf5faa12d20c97fef657999f Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 24 Apr 2024 15:16:33 +0800 Subject: [PATCH 0822/1702] iommu/vt-d: Add cache tag assignment interface Caching tag is a combination of tags used by the hardware to cache various translations. Whenever a mapping in a domain is changed, the IOMMU driver should invalidate the caches with the caching tags. The VT-d specification describes caching tags in section 6.2.1, Tagging of Cached Translations. Add interface to assign caching tags to an IOMMU domain when attached to a RID or PASID, and unassign caching tags when a domain is detached from a RID or PASID. All caching tags are listed in the per-domain tag list and are protected by a dedicated lock. In addition to the basic IOTLB and devTLB caching tag types, NESTING_IOTLB and NESTING_DEVTLB tag types are also introduced. These tags are used for caches that store translations for DMA accesses through a nested user domain. They are affected by changes to mappings in the parent domain. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240416080656.60968-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/Makefile | 2 +- drivers/iommu/intel/cache.c | 214 +++++++++++++++++++++++++++++++++++ drivers/iommu/intel/iommu.c | 28 ++++- drivers/iommu/intel/iommu.h | 31 +++++ drivers/iommu/intel/nested.c | 19 +++- drivers/iommu/intel/svm.c | 10 +- 6 files changed, 295 insertions(+), 9 deletions(-) create mode 100644 drivers/iommu/intel/cache.c diff --git a/drivers/iommu/intel/Makefile b/drivers/iommu/intel/Makefile index 5402b699a1229..c8beb0281559f 100644 --- a/drivers/iommu/intel/Makefile +++ b/drivers/iommu/intel/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DMAR_TABLE) += dmar.o -obj-$(CONFIG_INTEL_IOMMU) += iommu.o pasid.o nested.o +obj-$(CONFIG_INTEL_IOMMU) += iommu.o pasid.o nested.o cache.o obj-$(CONFIG_DMAR_TABLE) += trace.o cap_audit.o obj-$(CONFIG_DMAR_PERF) += perf.o obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += debugfs.o diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c new file mode 100644 index 0000000000000..296f1645a739c --- /dev/null +++ b/drivers/iommu/intel/cache.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * cache.c - Intel VT-d cache invalidation + * + * Copyright (C) 2024 Intel Corporation + * + * Author: Lu Baolu + */ + +#define pr_fmt(fmt) "DMAR: " fmt + +#include +#include +#include +#include + +#include "iommu.h" +#include "pasid.h" + +/* Check if an existing cache tag can be reused for a new association. */ +static bool cache_tage_match(struct cache_tag *tag, u16 domain_id, + struct intel_iommu *iommu, struct device *dev, + ioasid_t pasid, enum cache_tag_type type) +{ + if (tag->type != type) + return false; + + if (tag->domain_id != domain_id || tag->pasid != pasid) + return false; + + if (type == CACHE_TAG_IOTLB || type == CACHE_TAG_NESTING_IOTLB) + return tag->iommu == iommu; + + if (type == CACHE_TAG_DEVTLB || type == CACHE_TAG_NESTING_DEVTLB) + return tag->dev == dev; + + return false; +} + +/* Assign a cache tag with specified type to domain. */ +static int cache_tag_assign(struct dmar_domain *domain, u16 did, + struct device *dev, ioasid_t pasid, + enum cache_tag_type type) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + struct intel_iommu *iommu = info->iommu; + struct cache_tag *tag, *temp; + unsigned long flags; + + tag = kzalloc(sizeof(*tag), GFP_KERNEL); + if (!tag) + return -ENOMEM; + + tag->type = type; + tag->iommu = iommu; + tag->domain_id = did; + tag->pasid = pasid; + tag->users = 1; + + if (type == CACHE_TAG_DEVTLB || type == CACHE_TAG_NESTING_DEVTLB) + tag->dev = dev; + else + tag->dev = iommu->iommu.dev; + + spin_lock_irqsave(&domain->cache_lock, flags); + list_for_each_entry(temp, &domain->cache_tags, node) { + if (cache_tage_match(temp, did, iommu, dev, pasid, type)) { + temp->users++; + spin_unlock_irqrestore(&domain->cache_lock, flags); + kfree(tag); + return 0; + } + } + list_add_tail(&tag->node, &domain->cache_tags); + spin_unlock_irqrestore(&domain->cache_lock, flags); + + return 0; +} + +/* Unassign a cache tag with specified type from domain. */ +static void cache_tag_unassign(struct dmar_domain *domain, u16 did, + struct device *dev, ioasid_t pasid, + enum cache_tag_type type) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + struct intel_iommu *iommu = info->iommu; + struct cache_tag *tag; + unsigned long flags; + + spin_lock_irqsave(&domain->cache_lock, flags); + list_for_each_entry(tag, &domain->cache_tags, node) { + if (cache_tage_match(tag, did, iommu, dev, pasid, type)) { + if (--tag->users == 0) { + list_del(&tag->node); + kfree(tag); + } + break; + } + } + spin_unlock_irqrestore(&domain->cache_lock, flags); +} + +static int __cache_tag_assign_domain(struct dmar_domain *domain, u16 did, + struct device *dev, ioasid_t pasid) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + int ret; + + ret = cache_tag_assign(domain, did, dev, pasid, CACHE_TAG_IOTLB); + if (ret || !info->ats_enabled) + return ret; + + ret = cache_tag_assign(domain, did, dev, pasid, CACHE_TAG_DEVTLB); + if (ret) + cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_IOTLB); + + return ret; +} + +static void __cache_tag_unassign_domain(struct dmar_domain *domain, u16 did, + struct device *dev, ioasid_t pasid) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + + cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_IOTLB); + + if (info->ats_enabled) + cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_DEVTLB); +} + +static int __cache_tag_assign_parent_domain(struct dmar_domain *domain, u16 did, + struct device *dev, ioasid_t pasid) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + int ret; + + ret = cache_tag_assign(domain, did, dev, pasid, CACHE_TAG_NESTING_IOTLB); + if (ret || !info->ats_enabled) + return ret; + + ret = cache_tag_assign(domain, did, dev, pasid, CACHE_TAG_NESTING_DEVTLB); + if (ret) + cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_NESTING_IOTLB); + + return ret; +} + +static void __cache_tag_unassign_parent_domain(struct dmar_domain *domain, u16 did, + struct device *dev, ioasid_t pasid) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + + cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_NESTING_IOTLB); + + if (info->ats_enabled) + cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_NESTING_DEVTLB); +} + +static u16 domain_get_id_for_dev(struct dmar_domain *domain, struct device *dev) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + struct intel_iommu *iommu = info->iommu; + + /* + * The driver assigns different domain IDs for all domains except + * the SVA type. + */ + if (domain->domain.type == IOMMU_DOMAIN_SVA) + return FLPT_DEFAULT_DID; + + return domain_id_iommu(domain, iommu); +} + +/* + * Assign cache tags to a domain when it's associated with a device's + * PASID using a specific domain ID. + * + * On success (return value of 0), cache tags are created and added to the + * domain's cache tag list. On failure (negative return value), an error + * code is returned indicating the reason for the failure. + */ +int cache_tag_assign_domain(struct dmar_domain *domain, + struct device *dev, ioasid_t pasid) +{ + u16 did = domain_get_id_for_dev(domain, dev); + int ret; + + ret = __cache_tag_assign_domain(domain, did, dev, pasid); + if (ret || domain->domain.type != IOMMU_DOMAIN_NESTED) + return ret; + + ret = __cache_tag_assign_parent_domain(domain->s2_domain, did, dev, pasid); + if (ret) + __cache_tag_unassign_domain(domain, did, dev, pasid); + + return ret; +} + +/* + * Remove the cache tags associated with a device's PASID when the domain is + * detached from the device. + * + * The cache tags must be previously assigned to the domain by calling the + * assign interface. + */ +void cache_tag_unassign_domain(struct dmar_domain *domain, + struct device *dev, ioasid_t pasid) +{ + u16 did = domain_get_id_for_dev(domain, dev); + + __cache_tag_unassign_domain(domain, did, dev, pasid); + if (domain->domain.type == IOMMU_DOMAIN_NESTED) + __cache_tag_unassign_parent_domain(domain->s2_domain, did, dev, pasid); +} diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 002fee5fcb80d..8220bb36e4201 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1746,7 +1746,9 @@ static struct dmar_domain *alloc_domain(unsigned int type) domain->has_iotlb_device = false; INIT_LIST_HEAD(&domain->devices); INIT_LIST_HEAD(&domain->dev_pasids); + INIT_LIST_HEAD(&domain->cache_tags); spin_lock_init(&domain->lock); + spin_lock_init(&domain->cache_lock); xa_init(&domain->iommu_array); return domain; @@ -1758,6 +1760,9 @@ int domain_attach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu) unsigned long ndomains; int num, ret = -ENOSPC; + if (domain->domain.type == IOMMU_DOMAIN_SVA) + return 0; + info = kzalloc(sizeof(*info), GFP_KERNEL); if (!info) return -ENOMEM; @@ -1805,6 +1810,9 @@ void domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu) { struct iommu_domain_info *info; + if (domain->domain.type == IOMMU_DOMAIN_SVA) + return; + spin_lock(&iommu->lock); info = xa_load(&domain->iommu_array, iommu->seq_id); if (--info->refcnt == 0) { @@ -2323,6 +2331,13 @@ static int dmar_domain_attach_device(struct dmar_domain *domain, ret = domain_attach_iommu(domain, iommu); if (ret) return ret; + + ret = cache_tag_assign_domain(domain, dev, IOMMU_NO_PASID); + if (ret) { + domain_detach_iommu(domain, iommu); + return ret; + } + info->domain = domain; spin_lock_irqsave(&domain->lock, flags); list_add(&info->link, &domain->devices); @@ -3811,6 +3826,7 @@ void device_block_translation(struct device *dev) list_del(&info->link); spin_unlock_irqrestore(&info->domain->lock, flags); + cache_tag_unassign_domain(info->domain, dev, IOMMU_NO_PASID); domain_detach_iommu(info->domain, iommu); info->domain = NULL; } @@ -4598,6 +4614,7 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid) domain = iommu_get_domain_for_dev_pasid(dev, pasid, 0); if (WARN_ON_ONCE(!domain)) goto out_tear_down; + dmar_domain = to_dmar_domain(domain); /* * The SVA implementation needs to handle its own stuffs like the mm @@ -4606,10 +4623,10 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid) */ if (domain->type == IOMMU_DOMAIN_SVA) { intel_svm_remove_dev_pasid(dev, pasid); + cache_tag_unassign_domain(dmar_domain, dev, pasid); goto out_tear_down; } - dmar_domain = to_dmar_domain(domain); spin_lock_irqsave(&dmar_domain->lock, flags); list_for_each_entry(curr, &dmar_domain->dev_pasids, link_domain) { if (curr->dev == dev && curr->pasid == pasid) { @@ -4621,6 +4638,7 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid) WARN_ON_ONCE(!dev_pasid); spin_unlock_irqrestore(&dmar_domain->lock, flags); + cache_tag_unassign_domain(dmar_domain, dev, pasid); domain_detach_iommu(dmar_domain, iommu); intel_iommu_debugfs_remove_dev_pasid(dev_pasid); kfree(dev_pasid); @@ -4660,6 +4678,10 @@ static int intel_iommu_set_dev_pasid(struct iommu_domain *domain, if (ret) goto out_free; + ret = cache_tag_assign_domain(dmar_domain, dev, pasid); + if (ret) + goto out_detach_iommu; + if (domain_type_is_si(dmar_domain)) ret = intel_pasid_setup_pass_through(iommu, dev, pasid); else if (dmar_domain->use_first_level) @@ -4669,7 +4691,7 @@ static int intel_iommu_set_dev_pasid(struct iommu_domain *domain, ret = intel_pasid_setup_second_level(iommu, dmar_domain, dev, pasid); if (ret) - goto out_detach_iommu; + goto out_unassign_tag; dev_pasid->dev = dev; dev_pasid->pasid = pasid; @@ -4681,6 +4703,8 @@ static int intel_iommu_set_dev_pasid(struct iommu_domain *domain, intel_iommu_debugfs_create_dev_pasid(dev_pasid); return 0; +out_unassign_tag: + cache_tag_unassign_domain(dmar_domain, dev, pasid); out_detach_iommu: domain_detach_iommu(dmar_domain, iommu); out_free: diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 9ee326f7bf621..b7c79cc19681e 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -606,6 +606,9 @@ struct dmar_domain { struct list_head devices; /* all devices' list */ struct list_head dev_pasids; /* all attached pasids */ + spinlock_t cache_lock; /* Protect the cache tag list */ + struct list_head cache_tags; /* Cache tag list */ + int iommu_superpage;/* Level of superpages supported: 0 == 4KiB (no superpages), 1 == 2MiB, 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */ @@ -1091,6 +1094,34 @@ struct iommu_domain *intel_nested_domain_alloc(struct iommu_domain *parent, const struct iommu_user_data *user_data); struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid); +enum cache_tag_type { + CACHE_TAG_IOTLB, + CACHE_TAG_DEVTLB, + CACHE_TAG_NESTING_IOTLB, + CACHE_TAG_NESTING_DEVTLB, +}; + +struct cache_tag { + struct list_head node; + enum cache_tag_type type; + struct intel_iommu *iommu; + /* + * The @dev field represents the location of the cache. For IOTLB, it + * resides on the IOMMU hardware. @dev stores the device pointer to + * the IOMMU hardware. For DevTLB, it locates in the PCIe endpoint. + * @dev stores the device pointer to that endpoint. + */ + struct device *dev; + u16 domain_id; + ioasid_t pasid; + unsigned int users; +}; + +int cache_tag_assign_domain(struct dmar_domain *domain, + struct device *dev, ioasid_t pasid); +void cache_tag_unassign_domain(struct dmar_domain *domain, + struct device *dev, ioasid_t pasid); + #ifdef CONFIG_INTEL_IOMMU_SVM void intel_svm_check(struct intel_iommu *iommu); int intel_svm_enable_prq(struct intel_iommu *iommu); diff --git a/drivers/iommu/intel/nested.c b/drivers/iommu/intel/nested.c index a7d68f3d518ac..13406ee742bfa 100644 --- a/drivers/iommu/intel/nested.c +++ b/drivers/iommu/intel/nested.c @@ -52,13 +52,14 @@ static int intel_nested_attach_dev(struct iommu_domain *domain, return ret; } + ret = cache_tag_assign_domain(dmar_domain, dev, IOMMU_NO_PASID); + if (ret) + goto detach_iommu; + ret = intel_pasid_setup_nested(iommu, dev, IOMMU_NO_PASID, dmar_domain); - if (ret) { - domain_detach_iommu(dmar_domain, iommu); - dev_err_ratelimited(dev, "Failed to setup pasid entry\n"); - return ret; - } + if (ret) + goto unassign_tag; info->domain = dmar_domain; spin_lock_irqsave(&dmar_domain->lock, flags); @@ -68,6 +69,12 @@ static int intel_nested_attach_dev(struct iommu_domain *domain, domain_update_iotlb(dmar_domain); return 0; +unassign_tag: + cache_tag_unassign_domain(dmar_domain, dev, IOMMU_NO_PASID); +detach_iommu: + domain_detach_iommu(dmar_domain, iommu); + + return ret; } static void intel_nested_domain_free(struct iommu_domain *domain) @@ -206,7 +213,9 @@ struct iommu_domain *intel_nested_domain_alloc(struct iommu_domain *parent, domain->domain.type = IOMMU_DOMAIN_NESTED; INIT_LIST_HEAD(&domain->devices); INIT_LIST_HEAD(&domain->dev_pasids); + INIT_LIST_HEAD(&domain->cache_tags); spin_lock_init(&domain->lock); + spin_lock_init(&domain->cache_lock); xa_init(&domain->iommu_array); spin_lock(&s2_domain->s1_lock); diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index e05c6c4cb8c39..2e627fbd5adbf 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -366,17 +366,23 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, sdev->qdep = 0; } + ret = cache_tag_assign_domain(to_dmar_domain(domain), dev, pasid); + if (ret) + goto free_sdev; + /* Setup the pasid table: */ sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0; ret = intel_pasid_setup_first_level(iommu, dev, mm->pgd, pasid, FLPT_DEFAULT_DID, sflags); if (ret) - goto free_sdev; + goto unassign_tag; list_add_rcu(&sdev->list, &svm->devs); return 0; +unassign_tag: + cache_tag_unassign_domain(to_dmar_domain(domain), dev, pasid); free_sdev: kfree(sdev); free_svm: @@ -741,6 +747,8 @@ struct iommu_domain *intel_svm_domain_alloc(void) if (!domain) return NULL; domain->domain.ops = &intel_svm_domain_ops; + INIT_LIST_HEAD(&domain->cache_tags); + spin_lock_init(&domain->cache_lock); return &domain->domain; } From c4d27ffaa8eb034ec438a9aedfe202ce81e15312 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 24 Apr 2024 15:16:34 +0800 Subject: [PATCH 0823/1702] iommu/vt-d: Add cache tag invalidation helpers Add several helpers to invalidate the caches after mappings in the affected domain are changed. - cache_tag_flush_range() invalidates a range of caches after mappings within this range are changed. It uses the page-selective cache invalidation methods. - cache_tag_flush_all() invalidates all caches tagged by a domain ID. It uses the domain-selective cache invalidation methods. - cache_tag_flush_range_np() invalidates a range of caches when new mappings are created in the domain and the corresponding page table entries change from non-present to present. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240416080656.60968-3-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/cache.c | 195 ++++++++++++++++++++++++++++++++++++ drivers/iommu/intel/iommu.c | 12 --- drivers/iommu/intel/iommu.h | 14 +++ 3 files changed, 209 insertions(+), 12 deletions(-) diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c index 296f1645a739c..0539275a9d209 100644 --- a/drivers/iommu/intel/cache.c +++ b/drivers/iommu/intel/cache.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include "iommu.h" @@ -212,3 +213,197 @@ void cache_tag_unassign_domain(struct dmar_domain *domain, if (domain->domain.type == IOMMU_DOMAIN_NESTED) __cache_tag_unassign_parent_domain(domain->s2_domain, did, dev, pasid); } + +static unsigned long calculate_psi_aligned_address(unsigned long start, + unsigned long end, + unsigned long *_pages, + unsigned long *_mask) +{ + unsigned long pages = aligned_nrpages(start, end - start + 1); + unsigned long aligned_pages = __roundup_pow_of_two(pages); + unsigned long bitmask = aligned_pages - 1; + unsigned long mask = ilog2(aligned_pages); + unsigned long pfn = IOVA_PFN(start); + + /* + * PSI masks the low order bits of the base address. If the + * address isn't aligned to the mask, then compute a mask value + * needed to ensure the target range is flushed. + */ + if (unlikely(bitmask & pfn)) { + unsigned long end_pfn = pfn + pages - 1, shared_bits; + + /* + * Since end_pfn <= pfn + bitmask, the only way bits + * higher than bitmask can differ in pfn and end_pfn is + * by carrying. This means after masking out bitmask, + * high bits starting with the first set bit in + * shared_bits are all equal in both pfn and end_pfn. + */ + shared_bits = ~(pfn ^ end_pfn) & ~bitmask; + mask = shared_bits ? __ffs(shared_bits) : BITS_PER_LONG; + } + + *_pages = aligned_pages; + *_mask = mask; + + return ALIGN_DOWN(start, VTD_PAGE_SIZE << mask); +} + +/* + * Invalidates a range of IOVA from @start (inclusive) to @end (inclusive) + * when the memory mappings in the target domain have been modified. + */ +void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start, + unsigned long end, int ih) +{ + unsigned long pages, mask, addr; + struct cache_tag *tag; + unsigned long flags; + + addr = calculate_psi_aligned_address(start, end, &pages, &mask); + + spin_lock_irqsave(&domain->cache_lock, flags); + list_for_each_entry(tag, &domain->cache_tags, node) { + struct intel_iommu *iommu = tag->iommu; + struct device_domain_info *info; + u16 sid; + + switch (tag->type) { + case CACHE_TAG_IOTLB: + case CACHE_TAG_NESTING_IOTLB: + if (domain->use_first_level) { + qi_flush_piotlb(iommu, tag->domain_id, + tag->pasid, addr, pages, ih); + } else { + /* + * Fallback to domain selective flush if no + * PSI support or the size is too big. + */ + if (!cap_pgsel_inv(iommu->cap) || + mask > cap_max_amask_val(iommu->cap)) + iommu->flush.flush_iotlb(iommu, tag->domain_id, + 0, 0, DMA_TLB_DSI_FLUSH); + else + iommu->flush.flush_iotlb(iommu, tag->domain_id, + addr | ih, mask, + DMA_TLB_PSI_FLUSH); + } + break; + case CACHE_TAG_NESTING_DEVTLB: + /* + * Address translation cache in device side caches the + * result of nested translation. There is no easy way + * to identify the exact set of nested translations + * affected by a change in S2. So just flush the entire + * device cache. + */ + addr = 0; + mask = MAX_AGAW_PFN_WIDTH; + fallthrough; + case CACHE_TAG_DEVTLB: + info = dev_iommu_priv_get(tag->dev); + sid = PCI_DEVID(info->bus, info->devfn); + + if (tag->pasid == IOMMU_NO_PASID) + qi_flush_dev_iotlb(iommu, sid, info->pfsid, + info->ats_qdep, addr, mask); + else + qi_flush_dev_iotlb_pasid(iommu, sid, info->pfsid, + tag->pasid, info->ats_qdep, + addr, mask); + + quirk_extra_dev_tlb_flush(info, addr, mask, tag->pasid, info->ats_qdep); + break; + } + } + spin_unlock_irqrestore(&domain->cache_lock, flags); +} + +/* + * Invalidates all ranges of IOVA when the memory mappings in the target + * domain have been modified. + */ +void cache_tag_flush_all(struct dmar_domain *domain) +{ + struct cache_tag *tag; + unsigned long flags; + + spin_lock_irqsave(&domain->cache_lock, flags); + list_for_each_entry(tag, &domain->cache_tags, node) { + struct intel_iommu *iommu = tag->iommu; + struct device_domain_info *info; + u16 sid; + + switch (tag->type) { + case CACHE_TAG_IOTLB: + case CACHE_TAG_NESTING_IOTLB: + if (domain->use_first_level) + qi_flush_piotlb(iommu, tag->domain_id, + tag->pasid, 0, -1, 0); + else + iommu->flush.flush_iotlb(iommu, tag->domain_id, + 0, 0, DMA_TLB_DSI_FLUSH); + break; + case CACHE_TAG_DEVTLB: + case CACHE_TAG_NESTING_DEVTLB: + info = dev_iommu_priv_get(tag->dev); + sid = PCI_DEVID(info->bus, info->devfn); + + qi_flush_dev_iotlb(iommu, sid, info->pfsid, info->ats_qdep, + 0, MAX_AGAW_PFN_WIDTH); + quirk_extra_dev_tlb_flush(info, 0, MAX_AGAW_PFN_WIDTH, + IOMMU_NO_PASID, info->ats_qdep); + break; + } + } + spin_unlock_irqrestore(&domain->cache_lock, flags); +} + +/* + * Invalidate a range of IOVA when new mappings are created in the target + * domain. + * + * - VT-d spec, Section 6.1 Caching Mode: When the CM field is reported as + * Set, any software updates to remapping structures other than first- + * stage mapping requires explicit invalidation of the caches. + * - VT-d spec, Section 6.8 Write Buffer Flushing: For hardware that requires + * write buffer flushing, software must explicitly perform write-buffer + * flushing, if cache invalidation is not required. + */ +void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start, + unsigned long end) +{ + unsigned long pages, mask, addr; + struct cache_tag *tag; + unsigned long flags; + + addr = calculate_psi_aligned_address(start, end, &pages, &mask); + + spin_lock_irqsave(&domain->cache_lock, flags); + list_for_each_entry(tag, &domain->cache_tags, node) { + struct intel_iommu *iommu = tag->iommu; + + if (!cap_caching_mode(iommu->cap) || domain->use_first_level) { + iommu_flush_write_buffer(iommu); + continue; + } + + if (tag->type == CACHE_TAG_IOTLB || + tag->type == CACHE_TAG_NESTING_IOTLB) { + /* + * Fallback to domain selective flush if no + * PSI support or the size is too big. + */ + if (!cap_pgsel_inv(iommu->cap) || + mask > cap_max_amask_val(iommu->cap)) + iommu->flush.flush_iotlb(iommu, tag->domain_id, + 0, 0, DMA_TLB_DSI_FLUSH); + else + iommu->flush.flush_iotlb(iommu, tag->domain_id, + addr, mask, + DMA_TLB_PSI_FLUSH); + } + } + spin_unlock_irqrestore(&domain->cache_lock, flags); +} diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 8220bb36e4201..473df7cd16721 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -54,11 +54,6 @@ __DOMAIN_MAX_PFN(gaw), (unsigned long)-1)) #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT) -/* IO virtual address start page frame number */ -#define IOVA_START_PFN (1) - -#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) - static void __init check_tylersburg_isoch(void); static int rwbf_quirk; @@ -1992,13 +1987,6 @@ domain_context_mapping(struct dmar_domain *domain, struct device *dev) domain_context_mapping_cb, domain); } -/* Returns a number of VTD pages, but aligned to MM page size */ -static unsigned long aligned_nrpages(unsigned long host_addr, size_t size) -{ - host_addr &= ~PAGE_MASK; - return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT; -} - /* Return largest possible superpage level for a given mapping */ static int hardware_largepage_caps(struct dmar_domain *domain, unsigned long iov_pfn, unsigned long phy_pfn, unsigned long pages) diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index b7c79cc19681e..cb83b0995391d 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -35,6 +35,8 @@ #define VTD_PAGE_MASK (((u64)-1) << VTD_PAGE_SHIFT) #define VTD_PAGE_ALIGN(addr) (((addr) + VTD_PAGE_SIZE - 1) & VTD_PAGE_MASK) +#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) + #define VTD_STRIDE_SHIFT (9) #define VTD_STRIDE_MASK (((u64)-1) << VTD_STRIDE_SHIFT) @@ -1040,6 +1042,13 @@ static inline void context_set_sm_pre(struct context_entry *context) context->lo |= BIT_ULL(4); } +/* Returns a number of VTD pages, but aligned to MM page size */ +static inline unsigned long aligned_nrpages(unsigned long host_addr, size_t size) +{ + host_addr &= ~PAGE_MASK; + return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT; +} + /* Convert value to context PASID directory size field coding. */ #define context_pdts(pds) (((pds) & 0x7) << 9) @@ -1121,6 +1130,11 @@ int cache_tag_assign_domain(struct dmar_domain *domain, struct device *dev, ioasid_t pasid); void cache_tag_unassign_domain(struct dmar_domain *domain, struct device *dev, ioasid_t pasid); +void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start, + unsigned long end, int ih); +void cache_tag_flush_all(struct dmar_domain *domain); +void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start, + unsigned long end); #ifdef CONFIG_INTEL_IOMMU_SVM void intel_svm_check(struct intel_iommu *iommu); From 446a68c58d2e5b8140d474f1a74082aebeee9bb0 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 24 Apr 2024 15:16:35 +0800 Subject: [PATCH 0824/1702] iommu/vt-d: Add trace events for cache tag interface Add trace events for cache tag assign/unassign/flush operations and trace the events in the interfaces. These trace events will improve debugging capabilities by providing detailed information about cache tag activity. A sample of the traced messages looks like below [messages have been stripped and wrapped to make the line short]. cache_tag_assign: dmar9/0000:00:01.0 type iotlb did 1 pasid 9 ref 1 cache_tag_assign: dmar9/0000:00:01.0 type devtlb did 1 pasid 9 ref 1 cache_tag_flush_all: dmar6/0000:8a:00.0 type iotlb did 7 pasid 0 ref 1 cache_tag_flush_range: dmar1 0000:00:1b.0[0] type iotlb did 9 [0xeab00000-0xeab1afff] addr 0xeab00000 pages 0x20 mask 0x5 cache_tag_flush_range: dmar1 0000:00:1b.0[0] type iotlb did 9 [0xeab20000-0xeab31fff] addr 0xeab20000 pages 0x20 mask 0x5 cache_tag_flush_range: dmar1 0000:00:1b.0[0] type iotlb did 9 [0xeaa40000-0xeaa51fff] addr 0xeaa40000 pages 0x20 mask 0x5 cache_tag_flush_range: dmar1 0000:00:1b.0[0] type iotlb did 9 [0x98de0000-0x98de4fff] addr 0x98de0000 pages 0x8 mask 0x3 cache_tag_flush_range: dmar1 0000:00:1b.0[0] type iotlb did 9 [0xe9828000-0xe9828fff] addr 0xe9828000 pages 0x1 mask 0x0 cache_tag_unassign: dmar9/0000:00:01.0 type iotlb did 1 pasid 9 ref 1 cache_tag_unassign: dmar9/0000:00:01.0 type devtlb did 1 pasid 9 ref 1 Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240416080656.60968-4-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/cache.c | 10 ++++ drivers/iommu/intel/trace.h | 97 +++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c index 0539275a9d209..e8418cdd8331b 100644 --- a/drivers/iommu/intel/cache.c +++ b/drivers/iommu/intel/cache.c @@ -17,6 +17,7 @@ #include "iommu.h" #include "pasid.h" +#include "trace.h" /* Check if an existing cache tag can be reused for a new association. */ static bool cache_tage_match(struct cache_tag *tag, u16 domain_id, @@ -69,11 +70,13 @@ static int cache_tag_assign(struct dmar_domain *domain, u16 did, temp->users++; spin_unlock_irqrestore(&domain->cache_lock, flags); kfree(tag); + trace_cache_tag_assign(temp); return 0; } } list_add_tail(&tag->node, &domain->cache_tags); spin_unlock_irqrestore(&domain->cache_lock, flags); + trace_cache_tag_assign(tag); return 0; } @@ -91,6 +94,7 @@ static void cache_tag_unassign(struct dmar_domain *domain, u16 did, spin_lock_irqsave(&domain->cache_lock, flags); list_for_each_entry(tag, &domain->cache_tags, node) { if (cache_tage_match(tag, did, iommu, dev, pasid, type)) { + trace_cache_tag_unassign(tag); if (--tag->users == 0) { list_del(&tag->node); kfree(tag); @@ -316,6 +320,8 @@ void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start, quirk_extra_dev_tlb_flush(info, addr, mask, tag->pasid, info->ats_qdep); break; } + + trace_cache_tag_flush_range(tag, start, end, addr, pages, mask); } spin_unlock_irqrestore(&domain->cache_lock, flags); } @@ -356,6 +362,8 @@ void cache_tag_flush_all(struct dmar_domain *domain) IOMMU_NO_PASID, info->ats_qdep); break; } + + trace_cache_tag_flush_all(tag); } spin_unlock_irqrestore(&domain->cache_lock, flags); } @@ -404,6 +412,8 @@ void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start, addr, mask, DMA_TLB_PSI_FLUSH); } + + trace_cache_tag_flush_range_np(tag, start, end, addr, pages, mask); } spin_unlock_irqrestore(&domain->cache_lock, flags); } diff --git a/drivers/iommu/intel/trace.h b/drivers/iommu/intel/trace.h index 93d96f93a89b7..961ac1c1bc210 100644 --- a/drivers/iommu/intel/trace.h +++ b/drivers/iommu/intel/trace.h @@ -89,6 +89,103 @@ TRACE_EVENT(prq_report, __entry->dw1, __entry->dw2, __entry->dw3) ) ); + +DECLARE_EVENT_CLASS(cache_tag_log, + TP_PROTO(struct cache_tag *tag), + TP_ARGS(tag), + TP_STRUCT__entry( + __string(iommu, tag->iommu->name) + __string(dev, dev_name(tag->dev)) + __field(u16, type) + __field(u16, domain_id) + __field(u32, pasid) + __field(u32, users) + ), + TP_fast_assign( + __assign_str(iommu, tag->iommu->name); + __assign_str(dev, dev_name(tag->dev)); + __entry->type = tag->type; + __entry->domain_id = tag->domain_id; + __entry->pasid = tag->pasid; + __entry->users = tag->users; + ), + TP_printk("%s/%s type %s did %d pasid %d ref %d", + __get_str(iommu), __get_str(dev), + __print_symbolic(__entry->type, + { CACHE_TAG_IOTLB, "iotlb" }, + { CACHE_TAG_DEVTLB, "devtlb" }, + { CACHE_TAG_NESTING_IOTLB, "nesting_iotlb" }, + { CACHE_TAG_NESTING_DEVTLB, "nesting_devtlb" }), + __entry->domain_id, __entry->pasid, __entry->users + ) +); + +DEFINE_EVENT(cache_tag_log, cache_tag_assign, + TP_PROTO(struct cache_tag *tag), + TP_ARGS(tag) +); + +DEFINE_EVENT(cache_tag_log, cache_tag_unassign, + TP_PROTO(struct cache_tag *tag), + TP_ARGS(tag) +); + +DEFINE_EVENT(cache_tag_log, cache_tag_flush_all, + TP_PROTO(struct cache_tag *tag), + TP_ARGS(tag) +); + +DECLARE_EVENT_CLASS(cache_tag_flush, + TP_PROTO(struct cache_tag *tag, unsigned long start, unsigned long end, + unsigned long addr, unsigned long pages, unsigned long mask), + TP_ARGS(tag, start, end, addr, pages, mask), + TP_STRUCT__entry( + __string(iommu, tag->iommu->name) + __string(dev, dev_name(tag->dev)) + __field(u16, type) + __field(u16, domain_id) + __field(u32, pasid) + __field(unsigned long, start) + __field(unsigned long, end) + __field(unsigned long, addr) + __field(unsigned long, pages) + __field(unsigned long, mask) + ), + TP_fast_assign( + __assign_str(iommu, tag->iommu->name); + __assign_str(dev, dev_name(tag->dev)); + __entry->type = tag->type; + __entry->domain_id = tag->domain_id; + __entry->pasid = tag->pasid; + __entry->start = start; + __entry->end = end; + __entry->addr = addr; + __entry->pages = pages; + __entry->mask = mask; + ), + TP_printk("%s %s[%d] type %s did %d [0x%lx-0x%lx] addr 0x%lx pages 0x%lx mask 0x%lx", + __get_str(iommu), __get_str(dev), __entry->pasid, + __print_symbolic(__entry->type, + { CACHE_TAG_IOTLB, "iotlb" }, + { CACHE_TAG_DEVTLB, "devtlb" }, + { CACHE_TAG_NESTING_IOTLB, "nesting_iotlb" }, + { CACHE_TAG_NESTING_DEVTLB, "nesting_devtlb" }), + __entry->domain_id, __entry->start, __entry->end, + __entry->addr, __entry->pages, __entry->mask + ) +); + +DEFINE_EVENT(cache_tag_flush, cache_tag_flush_range, + TP_PROTO(struct cache_tag *tag, unsigned long start, unsigned long end, + unsigned long addr, unsigned long pages, unsigned long mask), + TP_ARGS(tag, start, end, addr, pages, mask) +); + +DEFINE_EVENT(cache_tag_flush, cache_tag_flush_range_np, + TP_PROTO(struct cache_tag *tag, unsigned long start, unsigned long end, + unsigned long addr, unsigned long pages, unsigned long mask), + TP_ARGS(tag, start, end, addr, pages, mask) +); #endif /* _TRACE_INTEL_IOMMU_H */ /* This part must be outside protection */ From 4e589a53685c7658f9055fcedbce15bd4cc41134 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 24 Apr 2024 15:16:36 +0800 Subject: [PATCH 0825/1702] iommu/vt-d: Use cache_tag_flush_all() in flush_iotlb_all The flush_iotlb_all callback is called by the iommu core to flush all caches for the affected domain. Use cache_tag_flush_all() in this callback. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240416080656.60968-5-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 473df7cd16721..a268e2a51f4de 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1557,25 +1557,7 @@ static void parent_domain_flush(struct dmar_domain *domain, static void intel_flush_iotlb_all(struct iommu_domain *domain) { - struct dmar_domain *dmar_domain = to_dmar_domain(domain); - struct iommu_domain_info *info; - unsigned long idx; - - xa_for_each(&dmar_domain->iommu_array, idx, info) { - struct intel_iommu *iommu = info->iommu; - u16 did = domain_id_iommu(dmar_domain, iommu); - - if (dmar_domain->use_first_level) - domain_flush_pasid_iotlb(iommu, dmar_domain, 0, -1, 0); - else - iommu->flush.flush_iotlb(iommu, did, 0, 0, - DMA_TLB_DSI_FLUSH); - - iommu_flush_dev_iotlb(dmar_domain, 0, MAX_AGAW_PFN_WIDTH); - } - - if (dmar_domain->nested_parent) - parent_domain_flush(dmar_domain, 0, -1, 0); + cache_tag_flush_all(to_dmar_domain(domain)); } static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu) From a600ccd0a347e8f9c9f35f229e1ccd0dca9374c4 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 24 Apr 2024 15:16:37 +0800 Subject: [PATCH 0826/1702] iommu/vt-d: Use cache_tag_flush_range() in tlb_sync The tlb_sync callback is called by the iommu core to flush a range of caches for the affected domain. Use cache_tag_flush_range() in this callback. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240416080656.60968-6-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index a268e2a51f4de..f3926ad7d737d 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4104,25 +4104,8 @@ static size_t intel_iommu_unmap_pages(struct iommu_domain *domain, static void intel_iommu_tlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { - struct dmar_domain *dmar_domain = to_dmar_domain(domain); - unsigned long iova_pfn = IOVA_PFN(gather->start); - size_t size = gather->end - gather->start; - struct iommu_domain_info *info; - unsigned long start_pfn; - unsigned long nrpages; - unsigned long i; - - nrpages = aligned_nrpages(gather->start, size); - start_pfn = mm_to_dma_pfn_start(iova_pfn); - - xa_for_each(&dmar_domain->iommu_array, i, info) - iommu_flush_iotlb_psi(info->iommu, dmar_domain, - start_pfn, nrpages, - list_empty(&gather->freelist), 0); - - if (dmar_domain->nested_parent) - parent_domain_flush(dmar_domain, start_pfn, nrpages, - list_empty(&gather->freelist)); + cache_tag_flush_range(to_dmar_domain(domain), gather->start, + gather->end, list_empty(&gather->freelist)); put_pages_list(&gather->freelist); } From 129dab6e1286525fe5baed860d3dfcd9c6b4b327 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 24 Apr 2024 15:16:38 +0800 Subject: [PATCH 0827/1702] iommu/vt-d: Use cache_tag_flush_range_np() in iotlb_sync_map The iotlb_sync_map callback is called by the iommu core after non-present to present mappings are created. The iommu driver uses this callback to invalidate caches if IOMMU is working in caching mode and second-only translation is used for the domain. Use cache_tag_flush_range_np() in this callback. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240416080656.60968-7-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index f3926ad7d737d..c9ac8a1b635fc 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1501,20 +1501,6 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, iommu_flush_dev_iotlb(domain, addr, mask); } -/* Notification for newly created mappings */ -static void __mapping_notify_one(struct intel_iommu *iommu, struct dmar_domain *domain, - unsigned long pfn, unsigned int pages) -{ - /* - * It's a non-present to present mapping. Only flush if caching mode - * and second level. - */ - if (cap_caching_mode(iommu->cap) && !domain->use_first_level) - iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1); - else - iommu_flush_write_buffer(iommu); -} - /* * Flush the relevant caches in nested translation if the domain * also serves as a parent @@ -4544,14 +4530,8 @@ static bool risky_device(struct pci_dev *pdev) static int intel_iommu_iotlb_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size) { - struct dmar_domain *dmar_domain = to_dmar_domain(domain); - unsigned long pages = aligned_nrpages(iova, size); - unsigned long pfn = iova >> VTD_PAGE_SHIFT; - struct iommu_domain_info *info; - unsigned long i; + cache_tag_flush_range_np(to_dmar_domain(domain), iova, iova + size - 1); - xa_for_each(&dmar_domain->iommu_array, i, info) - __mapping_notify_one(info->iommu, dmar_domain, pfn, pages); return 0; } From 06792d06798931696dd78f65024de4cc66f2fd5b Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 24 Apr 2024 15:16:39 +0800 Subject: [PATCH 0828/1702] iommu/vt-d: Cleanup use of iommu_flush_iotlb_psi() Use cache_tag_flush_range() in switch_to_super_page() to invalidate the necessary caches when switching mappings from normal to super pages. The iommu_flush_iotlb_psi() call in intel_iommu_memory_notifier() is unnecessary since there should be no cache invalidation for the identity domain. Clean up iommu_flush_iotlb_psi() after the last call site is removed. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240416080656.60968-8-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 171 +----------------------------------- 1 file changed, 2 insertions(+), 169 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index c9ac8a1b635fc..b2bd96e7f03d0 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1390,157 +1390,6 @@ static void __iommu_flush_dev_iotlb(struct device_domain_info *info, quirk_extra_dev_tlb_flush(info, addr, mask, IOMMU_NO_PASID, qdep); } -static void iommu_flush_dev_iotlb(struct dmar_domain *domain, - u64 addr, unsigned mask) -{ - struct dev_pasid_info *dev_pasid; - struct device_domain_info *info; - unsigned long flags; - - if (!domain->has_iotlb_device) - return; - - spin_lock_irqsave(&domain->lock, flags); - list_for_each_entry(info, &domain->devices, link) - __iommu_flush_dev_iotlb(info, addr, mask); - - list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain) { - info = dev_iommu_priv_get(dev_pasid->dev); - - if (!info->ats_enabled) - continue; - - qi_flush_dev_iotlb_pasid(info->iommu, - PCI_DEVID(info->bus, info->devfn), - info->pfsid, dev_pasid->pasid, - info->ats_qdep, addr, - mask); - } - spin_unlock_irqrestore(&domain->lock, flags); -} - -static void domain_flush_pasid_iotlb(struct intel_iommu *iommu, - struct dmar_domain *domain, u64 addr, - unsigned long npages, bool ih) -{ - u16 did = domain_id_iommu(domain, iommu); - struct dev_pasid_info *dev_pasid; - unsigned long flags; - - spin_lock_irqsave(&domain->lock, flags); - list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain) - qi_flush_piotlb(iommu, did, dev_pasid->pasid, addr, npages, ih); - - if (!list_empty(&domain->devices)) - qi_flush_piotlb(iommu, did, IOMMU_NO_PASID, addr, npages, ih); - spin_unlock_irqrestore(&domain->lock, flags); -} - -static void __iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did, - unsigned long pfn, unsigned int pages, - int ih) -{ - unsigned int aligned_pages = __roundup_pow_of_two(pages); - unsigned long bitmask = aligned_pages - 1; - unsigned int mask = ilog2(aligned_pages); - u64 addr = (u64)pfn << VTD_PAGE_SHIFT; - - /* - * PSI masks the low order bits of the base address. If the - * address isn't aligned to the mask, then compute a mask value - * needed to ensure the target range is flushed. - */ - if (unlikely(bitmask & pfn)) { - unsigned long end_pfn = pfn + pages - 1, shared_bits; - - /* - * Since end_pfn <= pfn + bitmask, the only way bits - * higher than bitmask can differ in pfn and end_pfn is - * by carrying. This means after masking out bitmask, - * high bits starting with the first set bit in - * shared_bits are all equal in both pfn and end_pfn. - */ - shared_bits = ~(pfn ^ end_pfn) & ~bitmask; - mask = shared_bits ? __ffs(shared_bits) : BITS_PER_LONG; - } - - /* - * Fallback to domain selective flush if no PSI support or - * the size is too big. - */ - if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap)) - iommu->flush.flush_iotlb(iommu, did, 0, 0, - DMA_TLB_DSI_FLUSH); - else - iommu->flush.flush_iotlb(iommu, did, addr | ih, mask, - DMA_TLB_PSI_FLUSH); -} - -static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, - struct dmar_domain *domain, - unsigned long pfn, unsigned int pages, - int ih, int map) -{ - unsigned int aligned_pages = __roundup_pow_of_two(pages); - unsigned int mask = ilog2(aligned_pages); - uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT; - u16 did = domain_id_iommu(domain, iommu); - - if (WARN_ON(!pages)) - return; - - if (ih) - ih = 1 << 6; - - if (domain->use_first_level) - domain_flush_pasid_iotlb(iommu, domain, addr, pages, ih); - else - __iommu_flush_iotlb_psi(iommu, did, pfn, pages, ih); - - if (!map) - iommu_flush_dev_iotlb(domain, addr, mask); -} - -/* - * Flush the relevant caches in nested translation if the domain - * also serves as a parent - */ -static void parent_domain_flush(struct dmar_domain *domain, - unsigned long pfn, - unsigned long pages, int ih) -{ - struct dmar_domain *s1_domain; - - spin_lock(&domain->s1_lock); - list_for_each_entry(s1_domain, &domain->s1_domains, s2_link) { - struct device_domain_info *device_info; - struct iommu_domain_info *info; - unsigned long flags; - unsigned long i; - - xa_for_each(&s1_domain->iommu_array, i, info) - __iommu_flush_iotlb_psi(info->iommu, info->did, - pfn, pages, ih); - - if (!s1_domain->has_iotlb_device) - continue; - - spin_lock_irqsave(&s1_domain->lock, flags); - list_for_each_entry(device_info, &s1_domain->devices, link) - /* - * Address translation cache in device side caches the - * result of nested translation. There is no easy way - * to identify the exact set of nested translations - * affected by a change in S2. So just flush the entire - * device cache. - */ - __iommu_flush_dev_iotlb(device_info, 0, - MAX_AGAW_PFN_WIDTH); - spin_unlock_irqrestore(&s1_domain->lock, flags); - } - spin_unlock(&domain->s1_lock); -} - static void intel_flush_iotlb_all(struct iommu_domain *domain) { cache_tag_flush_all(to_dmar_domain(domain)); @@ -1991,9 +1840,7 @@ static void switch_to_super_page(struct dmar_domain *domain, unsigned long end_pfn, int level) { unsigned long lvl_pages = lvl_to_nr_pages(level); - struct iommu_domain_info *info; struct dma_pte *pte = NULL; - unsigned long i; while (start_pfn <= end_pfn) { if (!pte) @@ -2005,13 +1852,8 @@ static void switch_to_super_page(struct dmar_domain *domain, start_pfn + lvl_pages - 1, level + 1); - xa_for_each(&domain->iommu_array, i, info) - iommu_flush_iotlb_psi(info->iommu, domain, - start_pfn, lvl_pages, - 0, 0); - if (domain->nested_parent) - parent_domain_flush(domain, start_pfn, - lvl_pages, 0); + cache_tag_flush_range(domain, start_pfn << VTD_PAGE_SHIFT, + end_pfn << VTD_PAGE_SHIFT, 0); } pte++; @@ -3381,18 +3223,9 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb, case MEM_OFFLINE: case MEM_CANCEL_ONLINE: { - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu; LIST_HEAD(freelist); domain_unmap(si_domain, start_vpfn, last_vpfn, &freelist); - - rcu_read_lock(); - for_each_active_iommu(iommu, drhd) - iommu_flush_iotlb_psi(iommu, si_domain, - start_vpfn, mhp->nr_pages, - list_empty(&freelist), 0); - rcu_read_unlock(); put_pages_list(&freelist); } break; From 8ebc22366ed8bb2cbcf215bfa3fabb4f36d10a91 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 24 Apr 2024 15:16:40 +0800 Subject: [PATCH 0829/1702] iommu/vt-d: Use cache_tag_flush_range() in cache_invalidate_user The cache_invalidate_user callback is called to invalidate a range of caches for the affected user domain. Use cache_tag_flush_range() in this callback. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240416080656.60968-9-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.h | 6 +++++ drivers/iommu/intel/nested.c | 50 +++--------------------------------- 2 files changed, 9 insertions(+), 47 deletions(-) diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index cb83b0995391d..1d705a983dd7b 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1049,6 +1049,12 @@ static inline unsigned long aligned_nrpages(unsigned long host_addr, size_t size return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT; } +/* Return a size from number of VTD pages. */ +static inline unsigned long nrpages_to_size(unsigned long npages) +{ + return npages << VTD_PAGE_SHIFT; +} + /* Convert value to context PASID directory size field coding. */ #define context_pdts(pds) (((pds) & 0x7) << 9) diff --git a/drivers/iommu/intel/nested.c b/drivers/iommu/intel/nested.c index 13406ee742bfa..16a2bcf5cfeb9 100644 --- a/drivers/iommu/intel/nested.c +++ b/drivers/iommu/intel/nested.c @@ -88,50 +88,6 @@ static void intel_nested_domain_free(struct iommu_domain *domain) kfree(dmar_domain); } -static void nested_flush_dev_iotlb(struct dmar_domain *domain, u64 addr, - unsigned int mask) -{ - struct device_domain_info *info; - unsigned long flags; - u16 sid, qdep; - - spin_lock_irqsave(&domain->lock, flags); - list_for_each_entry(info, &domain->devices, link) { - if (!info->ats_enabled) - continue; - sid = info->bus << 8 | info->devfn; - qdep = info->ats_qdep; - qi_flush_dev_iotlb(info->iommu, sid, info->pfsid, - qdep, addr, mask); - quirk_extra_dev_tlb_flush(info, addr, mask, - IOMMU_NO_PASID, qdep); - } - spin_unlock_irqrestore(&domain->lock, flags); -} - -static void intel_nested_flush_cache(struct dmar_domain *domain, u64 addr, - u64 npages, bool ih) -{ - struct iommu_domain_info *info; - unsigned int mask; - unsigned long i; - - xa_for_each(&domain->iommu_array, i, info) - qi_flush_piotlb(info->iommu, - domain_id_iommu(domain, info->iommu), - IOMMU_NO_PASID, addr, npages, ih); - - if (!domain->has_iotlb_device) - return; - - if (npages == U64_MAX) - mask = 64 - VTD_PAGE_SHIFT; - else - mask = ilog2(__roundup_pow_of_two(npages)); - - nested_flush_dev_iotlb(domain, addr, mask); -} - static int intel_nested_cache_invalidate_user(struct iommu_domain *domain, struct iommu_user_data_array *array) { @@ -164,9 +120,9 @@ static int intel_nested_cache_invalidate_user(struct iommu_domain *domain, break; } - intel_nested_flush_cache(dmar_domain, inv_entry.addr, - inv_entry.npages, - inv_entry.flags & IOMMU_VTD_INV_FLAGS_LEAF); + cache_tag_flush_range(dmar_domain, inv_entry.addr, + inv_entry.addr + nrpages_to_size(inv_entry.npages) - 1, + inv_entry.flags & IOMMU_VTD_INV_FLAGS_LEAF); processed++; } From 4f609dbff51b3c98d5fc841758a20be74ab3f132 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 24 Apr 2024 15:16:41 +0800 Subject: [PATCH 0830/1702] iommu/vt-d: Use cache helpers in arch_invalidate_secondary_tlbs The arch_invalidate_secondary_tlbs callback is called in the SVA mm notification path. It invalidates all or a range of caches after the CPU page table is modified. Use the cache tag helps in this path. The mm_types defines vm_end as the first byte after the end address which is different from the iommu gather API, hence convert the end parameter from mm_types to iommu gather scheme before calling the cache_tag helper. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240416080656.60968-10-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.h | 1 + drivers/iommu/intel/svm.c | 81 +++++-------------------------------- 2 files changed, 11 insertions(+), 71 deletions(-) diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 1d705a983dd7b..fc0b4b0486353 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1166,6 +1166,7 @@ struct intel_svm { struct mm_struct *mm; u32 pasid; struct list_head devs; + struct dmar_domain *domain; }; #else static inline void intel_svm_check(struct intel_iommu *iommu) {} diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 2e627fbd5adbf..5ba0a7baa455a 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -168,88 +168,25 @@ void intel_svm_check(struct intel_iommu *iommu) iommu->flags |= VTD_FLAG_SVM_CAPABLE; } -static void __flush_svm_range_dev(struct intel_svm *svm, - struct intel_svm_dev *sdev, - unsigned long address, - unsigned long pages, int ih) -{ - struct device_domain_info *info = dev_iommu_priv_get(sdev->dev); - - if (WARN_ON(!pages)) - return; - - qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, address, pages, ih); - if (info->ats_enabled) { - qi_flush_dev_iotlb_pasid(sdev->iommu, sdev->sid, info->pfsid, - svm->pasid, sdev->qdep, address, - order_base_2(pages)); - quirk_extra_dev_tlb_flush(info, address, order_base_2(pages), - svm->pasid, sdev->qdep); - } -} - -static void intel_flush_svm_range_dev(struct intel_svm *svm, - struct intel_svm_dev *sdev, - unsigned long address, - unsigned long pages, int ih) -{ - unsigned long shift = ilog2(__roundup_pow_of_two(pages)); - unsigned long align = (1ULL << (VTD_PAGE_SHIFT + shift)); - unsigned long start = ALIGN_DOWN(address, align); - unsigned long end = ALIGN(address + (pages << VTD_PAGE_SHIFT), align); - - while (start < end) { - __flush_svm_range_dev(svm, sdev, start, align >> VTD_PAGE_SHIFT, ih); - start += align; - } -} - -static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address, - unsigned long pages, int ih) -{ - struct intel_svm_dev *sdev; - - rcu_read_lock(); - list_for_each_entry_rcu(sdev, &svm->devs, list) - intel_flush_svm_range_dev(svm, sdev, address, pages, ih); - rcu_read_unlock(); -} - -static void intel_flush_svm_all(struct intel_svm *svm) -{ - struct device_domain_info *info; - struct intel_svm_dev *sdev; - - rcu_read_lock(); - list_for_each_entry_rcu(sdev, &svm->devs, list) { - info = dev_iommu_priv_get(sdev->dev); - - qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, 0, -1UL, 0); - if (info->ats_enabled) { - qi_flush_dev_iotlb_pasid(sdev->iommu, sdev->sid, info->pfsid, - svm->pasid, sdev->qdep, - 0, 64 - VTD_PAGE_SHIFT); - quirk_extra_dev_tlb_flush(info, 0, 64 - VTD_PAGE_SHIFT, - svm->pasid, sdev->qdep); - } - } - rcu_read_unlock(); -} - /* Pages have been freed at this point */ static void intel_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, struct mm_struct *mm, unsigned long start, unsigned long end) { struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); + struct dmar_domain *domain = svm->domain; if (start == 0 && end == -1UL) { - intel_flush_svm_all(svm); + cache_tag_flush_all(domain); return; } - intel_flush_svm_range(svm, start, - (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0); + /* + * The mm_types defines vm_end as the first byte after the end address, + * different from IOMMU subsystem using the last address of an address + * range. + */ + cache_tag_flush_range(domain, start, end - 1, 0); } static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) @@ -336,6 +273,7 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, INIT_LIST_HEAD_RCU(&svm->devs); svm->notifier.ops = &intel_mmuops; + svm->domain = to_dmar_domain(domain); ret = mmu_notifier_register(&svm->notifier, mm); if (ret) { kfree(svm); @@ -747,6 +685,7 @@ struct iommu_domain *intel_svm_domain_alloc(void) if (!domain) return NULL; domain->domain.ops = &intel_svm_domain_ops; + domain->use_first_level = true; INIT_LIST_HEAD(&domain->cache_tags); spin_lock_init(&domain->cache_lock); From deda9a7bf38fb9846ff3fb83179ca07437b1511c Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 24 Apr 2024 15:16:42 +0800 Subject: [PATCH 0831/1702] iommu/vt-d: Remove intel_svm_dev The intel_svm_dev data structure used in the sva implementation for the Intel IOMMU driver stores information about a device attached to an SVA domain. It is a duplicate of dev_pasid_info that serves the same purpose. Replace intel_svm_dev with dev_pasid_info and clean up the use of intel_svm_dev. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240416080656.60968-11-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 7 +- drivers/iommu/intel/iommu.h | 15 +---- drivers/iommu/intel/svm.c | 130 ++++++++++-------------------------- 3 files changed, 42 insertions(+), 110 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index b2bd96e7f03d0..7631d00cc8825 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4387,11 +4387,8 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid) * notification. Before consolidating that code into iommu core, let * the intel sva code handle it. */ - if (domain->type == IOMMU_DOMAIN_SVA) { - intel_svm_remove_dev_pasid(dev, pasid); - cache_tag_unassign_domain(dmar_domain, dev, pasid); - goto out_tear_down; - } + if (domain->type == IOMMU_DOMAIN_SVA) + intel_svm_remove_dev_pasid(domain); spin_lock_irqsave(&dmar_domain->lock, flags); list_for_each_entry(curr, &dmar_domain->dev_pasids, link_domain) { diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index fc0b4b0486353..f16b0d10543fb 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -649,6 +649,7 @@ struct dmar_domain { struct list_head s2_link; }; }; + struct intel_svm *svm; struct iommu_domain domain; /* generic domain data structure for iommu core */ @@ -1149,23 +1150,13 @@ int intel_svm_finish_prq(struct intel_iommu *iommu); void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, struct iommu_page_response *msg); struct iommu_domain *intel_svm_domain_alloc(void); -void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid); +void intel_svm_remove_dev_pasid(struct iommu_domain *domain); void intel_drain_pasid_prq(struct device *dev, u32 pasid); -struct intel_svm_dev { - struct list_head list; - struct rcu_head rcu; - struct device *dev; - struct intel_iommu *iommu; - u16 did; - u16 sid, qdep; -}; - struct intel_svm { struct mmu_notifier notifier; struct mm_struct *mm; u32 pasid; - struct list_head devs; struct dmar_domain *domain; }; #else @@ -1176,7 +1167,7 @@ static inline struct iommu_domain *intel_svm_domain_alloc(void) return NULL; } -static inline void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid) +static inline void intel_svm_remove_dev_pasid(struct iommu_domain *domain) { } #endif diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 5ba0a7baa455a..e6568897042f8 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -43,23 +43,6 @@ static void *pasid_private_find(ioasid_t pasid) return xa_load(&pasid_private_array, pasid); } -static struct intel_svm_dev * -svm_lookup_device_by_dev(struct intel_svm *svm, struct device *dev) -{ - struct intel_svm_dev *sdev = NULL, *t; - - rcu_read_lock(); - list_for_each_entry_rcu(t, &svm->devs, list) { - if (t->dev == dev) { - sdev = t; - break; - } - } - rcu_read_unlock(); - - return sdev; -} - int intel_svm_enable_prq(struct intel_iommu *iommu) { struct iopf_queue *iopfq; @@ -192,7 +175,10 @@ static void intel_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) { struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); - struct intel_svm_dev *sdev; + struct dmar_domain *domain = svm->domain; + struct dev_pasid_info *dev_pasid; + struct device_domain_info *info; + unsigned long flags; /* This might end up being called from exit_mmap(), *before* the page * tables are cleared. And __mmu_notifier_release() will delete us from @@ -206,11 +192,13 @@ static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) * page) so that we end up taking a fault that the hardware really * *has* to handle gracefully without affecting other processes. */ - rcu_read_lock(); - list_for_each_entry_rcu(sdev, &svm->devs, list) - intel_pasid_tear_down_entry(sdev->iommu, sdev->dev, - svm->pasid, true); - rcu_read_unlock(); + spin_lock_irqsave(&domain->lock, flags); + list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain) { + info = dev_iommu_priv_get(dev_pasid->dev); + intel_pasid_tear_down_entry(info->iommu, dev_pasid->dev, + dev_pasid->pasid, true); + } + spin_unlock_irqrestore(&domain->lock, flags); } @@ -219,47 +207,17 @@ static const struct mmu_notifier_ops intel_mmuops = { .arch_invalidate_secondary_tlbs = intel_arch_invalidate_secondary_tlbs, }; -static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid, - struct intel_svm **rsvm, - struct intel_svm_dev **rsdev) -{ - struct intel_svm_dev *sdev = NULL; - struct intel_svm *svm; - - if (pasid == IOMMU_PASID_INVALID || pasid >= PASID_MAX) - return -EINVAL; - - svm = pasid_private_find(pasid); - if (IS_ERR(svm)) - return PTR_ERR(svm); - - if (!svm) - goto out; - - /* - * If we found svm for the PASID, there must be at least one device - * bond. - */ - if (WARN_ON(list_empty(&svm->devs))) - return -EINVAL; - sdev = svm_lookup_device_by_dev(svm, dev); - -out: - *rsvm = svm; - *rsdev = sdev; - - return 0; -} - static int intel_svm_set_dev_pasid(struct iommu_domain *domain, struct device *dev, ioasid_t pasid) { struct device_domain_info *info = dev_iommu_priv_get(dev); + struct dmar_domain *dmar_domain = to_dmar_domain(domain); struct intel_iommu *iommu = info->iommu; struct mm_struct *mm = domain->mm; - struct intel_svm_dev *sdev; + struct dev_pasid_info *dev_pasid; struct intel_svm *svm; unsigned long sflags; + unsigned long flags; int ret = 0; svm = pasid_private_find(pasid); @@ -270,7 +228,6 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, svm->pasid = pasid; svm->mm = mm; - INIT_LIST_HEAD_RCU(&svm->devs); svm->notifier.ops = &intel_mmuops; svm->domain = to_dmar_domain(domain); @@ -288,25 +245,17 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, } } - sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); - if (!sdev) { - ret = -ENOMEM; + dmar_domain->svm = svm; + dev_pasid = kzalloc(sizeof(*dev_pasid), GFP_KERNEL); + if (!dev_pasid) goto free_svm; - } - sdev->dev = dev; - sdev->iommu = iommu; - sdev->did = FLPT_DEFAULT_DID; - sdev->sid = PCI_DEVID(info->bus, info->devfn); - if (info->ats_enabled) { - sdev->qdep = info->ats_qdep; - if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS) - sdev->qdep = 0; - } + dev_pasid->dev = dev; + dev_pasid->pasid = pasid; ret = cache_tag_assign_domain(to_dmar_domain(domain), dev, pasid); if (ret) - goto free_sdev; + goto free_dev_pasid; /* Setup the pasid table: */ sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0; @@ -315,16 +264,18 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, if (ret) goto unassign_tag; - list_add_rcu(&sdev->list, &svm->devs); + spin_lock_irqsave(&dmar_domain->lock, flags); + list_add(&dev_pasid->link_domain, &dmar_domain->dev_pasids); + spin_unlock_irqrestore(&dmar_domain->lock, flags); return 0; unassign_tag: cache_tag_unassign_domain(to_dmar_domain(domain), dev, pasid); -free_sdev: - kfree(sdev); +free_dev_pasid: + kfree(dev_pasid); free_svm: - if (list_empty(&svm->devs)) { + if (list_empty(&dmar_domain->dev_pasids)) { mmu_notifier_unregister(&svm->notifier, mm); pasid_private_remove(pasid); kfree(svm); @@ -333,26 +284,17 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, return ret; } -void intel_svm_remove_dev_pasid(struct device *dev, u32 pasid) +void intel_svm_remove_dev_pasid(struct iommu_domain *domain) { - struct intel_svm_dev *sdev; - struct intel_svm *svm; - struct mm_struct *mm; - - if (pasid_to_svm_sdev(dev, pasid, &svm, &sdev)) - return; - mm = svm->mm; - - if (sdev) { - list_del_rcu(&sdev->list); - kfree_rcu(sdev, rcu); + struct dmar_domain *dmar_domain = to_dmar_domain(domain); + struct intel_svm *svm = dmar_domain->svm; + struct mm_struct *mm = domain->mm; - if (list_empty(&svm->devs)) { - if (svm->notifier.ops) - mmu_notifier_unregister(&svm->notifier, mm); - pasid_private_remove(svm->pasid); - kfree(svm); - } + if (list_empty(&dmar_domain->dev_pasids)) { + if (svm->notifier.ops) + mmu_notifier_unregister(&svm->notifier, mm); + pasid_private_remove(svm->pasid); + kfree(svm); } } @@ -686,8 +628,10 @@ struct iommu_domain *intel_svm_domain_alloc(void) return NULL; domain->domain.ops = &intel_svm_domain_ops; domain->use_first_level = true; + INIT_LIST_HEAD(&domain->dev_pasids); INIT_LIST_HEAD(&domain->cache_tags); spin_lock_init(&domain->cache_lock); + spin_lock_init(&domain->lock); return &domain->domain; } From 65442507026a79e2de8014880e6ba376b8b44584 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 24 Apr 2024 15:16:43 +0800 Subject: [PATCH 0832/1702] iommu: Add ops->domain_alloc_sva() Make a new op that receives the device and the mm_struct that the SVA domain should be created for. Unlike domain_alloc_paging() the dev argument is never NULL here. This allows drivers to fully initialize the SVA domain and allocate the mmu_notifier during allocation. It allows the notifier lifetime to follow the lifetime of the iommu_domain. Since we have only one call site, upgrade the new op to return ERR_PTR instead of NULL. Signed-off-by: Jason Gunthorpe Signed-off-by: Vasant Hegde Reviewed-by: Tina Zhang Link: https://lore.kernel.org/r/20240311090843.133455-15-vasant.hegde@amd.com Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240416080656.60968-12-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu-sva.c | 16 +++++++++++----- include/linux/iommu.h | 3 +++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index 640acc804e8cd..18a35e798b729 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -108,8 +108,8 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm /* Allocate a new domain and set it on device pasid. */ domain = iommu_sva_domain_alloc(dev, mm); - if (!domain) { - ret = -ENOMEM; + if (IS_ERR(domain)) { + ret = PTR_ERR(domain); goto out_free_handle; } @@ -283,9 +283,15 @@ struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, const struct iommu_ops *ops = dev_iommu_ops(dev); struct iommu_domain *domain; - domain = ops->domain_alloc(IOMMU_DOMAIN_SVA); - if (!domain) - return NULL; + if (ops->domain_alloc_sva) { + domain = ops->domain_alloc_sva(dev, mm); + if (IS_ERR(domain)) + return domain; + } else { + domain = ops->domain_alloc(IOMMU_DOMAIN_SVA); + if (!domain) + return ERR_PTR(-ENOMEM); + } domain->type = IOMMU_DOMAIN_SVA; mmgrab(mm); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index e6549bdfaed96..f4cdffa985e4a 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -517,6 +517,7 @@ static inline int __iommu_copy_struct_from_user_array( * Upon failure, ERR_PTR must be returned. * @domain_alloc_paging: Allocate an iommu_domain that can be used for * UNMANAGED, DMA, and DMA_FQ domain types. + * @domain_alloc_sva: Allocate an iommu_domain for Shared Virtual Addressing. * @probe_device: Add device to iommu driver handling * @release_device: Remove device from iommu driver handling * @probe_finalize: Do final setup work after the device is added to an IOMMU @@ -557,6 +558,8 @@ struct iommu_ops { struct device *dev, u32 flags, struct iommu_domain *parent, const struct iommu_user_data *user_data); struct iommu_domain *(*domain_alloc_paging)(struct device *dev); + struct iommu_domain *(*domain_alloc_sva)(struct device *dev, + struct mm_struct *mm); struct iommu_device *(*probe_device)(struct device *dev); void (*release_device)(struct device *dev); From 886f816c2f01f768384e78f91b3c0ff66920b53f Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 24 Apr 2024 15:16:44 +0800 Subject: [PATCH 0833/1702] iommu/vt-d: Remove struct intel_svm The struct intel_svm was used for keeping attached devices info for sva domain. Since sva domain is a kind of iommu_domain, the struct dmar_domain should centralize all info of a sva domain, including the info of attached devices. Therefore, retire struct intel_svm and clean up the code. Besides, register mmu notifier callback in domain_alloc_sva() callback which allows the memory management notifier lifetime to follow the lifetime of the iommu_domain. Call mmu_notifier_put() in the domain free and defer the real free to the mmu free_notifier callback. Co-developed-by: Tina Zhang Signed-off-by: Tina Zhang Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240416080656.60968-13-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 11 +---- drivers/iommu/intel/iommu.h | 26 ++++------ drivers/iommu/intel/svm.c | 99 ++++++++++--------------------------- 3 files changed, 37 insertions(+), 99 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 7631d00cc8825..916cdb65d8494 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -3683,8 +3683,6 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) return domain; case IOMMU_DOMAIN_IDENTITY: return &si_domain->domain; - case IOMMU_DOMAIN_SVA: - return intel_svm_domain_alloc(); default: return NULL; } @@ -4382,14 +4380,6 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid) goto out_tear_down; dmar_domain = to_dmar_domain(domain); - /* - * The SVA implementation needs to handle its own stuffs like the mm - * notification. Before consolidating that code into iommu core, let - * the intel sva code handle it. - */ - if (domain->type == IOMMU_DOMAIN_SVA) - intel_svm_remove_dev_pasid(domain); - spin_lock_irqsave(&dmar_domain->lock, flags); list_for_each_entry(curr, &dmar_domain->dev_pasids, link_domain) { if (curr->dev == dev && curr->pasid == pasid) { @@ -4624,6 +4614,7 @@ const struct iommu_ops intel_iommu_ops = { .hw_info = intel_iommu_hw_info, .domain_alloc = intel_iommu_domain_alloc, .domain_alloc_user = intel_iommu_domain_alloc_user, + .domain_alloc_sva = intel_svm_domain_alloc, .probe_device = intel_iommu_probe_device, .probe_finalize = intel_iommu_probe_finalize, .release_device = intel_iommu_release_device, diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index f16b0d10543fb..c9eef464cf5ce 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -648,8 +648,12 @@ struct dmar_domain { /* link to parent domain siblings */ struct list_head s2_link; }; + + /* SVA domain */ + struct { + struct mmu_notifier notifier; + }; }; - struct intel_svm *svm; struct iommu_domain domain; /* generic domain data structure for iommu core */ @@ -1149,26 +1153,16 @@ int intel_svm_enable_prq(struct intel_iommu *iommu); int intel_svm_finish_prq(struct intel_iommu *iommu); void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, struct iommu_page_response *msg); -struct iommu_domain *intel_svm_domain_alloc(void); -void intel_svm_remove_dev_pasid(struct iommu_domain *domain); +struct iommu_domain *intel_svm_domain_alloc(struct device *dev, + struct mm_struct *mm); void intel_drain_pasid_prq(struct device *dev, u32 pasid); - -struct intel_svm { - struct mmu_notifier notifier; - struct mm_struct *mm; - u32 pasid; - struct dmar_domain *domain; -}; #else static inline void intel_svm_check(struct intel_iommu *iommu) {} static inline void intel_drain_pasid_prq(struct device *dev, u32 pasid) {} -static inline struct iommu_domain *intel_svm_domain_alloc(void) -{ - return NULL; -} - -static inline void intel_svm_remove_dev_pasid(struct iommu_domain *domain) +static inline struct iommu_domain *intel_svm_domain_alloc(struct device *dev, + struct mm_struct *mm) { + return ERR_PTR(-ENODEV); } #endif diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index e6568897042f8..268a0082d37f7 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -26,23 +26,6 @@ static irqreturn_t prq_event_thread(int irq, void *d); -static DEFINE_XARRAY_ALLOC(pasid_private_array); -static int pasid_private_add(ioasid_t pasid, void *priv) -{ - return xa_alloc(&pasid_private_array, &pasid, priv, - XA_LIMIT(pasid, pasid), GFP_ATOMIC); -} - -static void pasid_private_remove(ioasid_t pasid) -{ - xa_erase(&pasid_private_array, pasid); -} - -static void *pasid_private_find(ioasid_t pasid) -{ - return xa_load(&pasid_private_array, pasid); -} - int intel_svm_enable_prq(struct intel_iommu *iommu) { struct iopf_queue *iopfq; @@ -156,10 +139,9 @@ static void intel_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, struct mm_struct *mm, unsigned long start, unsigned long end) { - struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); - struct dmar_domain *domain = svm->domain; + struct dmar_domain *domain = container_of(mn, struct dmar_domain, notifier); - if (start == 0 && end == -1UL) { + if (start == 0 && end == ULONG_MAX) { cache_tag_flush_all(domain); return; } @@ -174,8 +156,7 @@ static void intel_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) { - struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); - struct dmar_domain *domain = svm->domain; + struct dmar_domain *domain = container_of(mn, struct dmar_domain, notifier); struct dev_pasid_info *dev_pasid; struct device_domain_info *info; unsigned long flags; @@ -202,9 +183,15 @@ static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) } +static void intel_mm_free_notifier(struct mmu_notifier *mn) +{ + kfree(container_of(mn, struct dmar_domain, notifier)); +} + static const struct mmu_notifier_ops intel_mmuops = { .release = intel_mm_release, .arch_invalidate_secondary_tlbs = intel_arch_invalidate_secondary_tlbs, + .free_notifier = intel_mm_free_notifier, }; static int intel_svm_set_dev_pasid(struct iommu_domain *domain, @@ -215,40 +202,13 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, struct intel_iommu *iommu = info->iommu; struct mm_struct *mm = domain->mm; struct dev_pasid_info *dev_pasid; - struct intel_svm *svm; unsigned long sflags; unsigned long flags; int ret = 0; - svm = pasid_private_find(pasid); - if (!svm) { - svm = kzalloc(sizeof(*svm), GFP_KERNEL); - if (!svm) - return -ENOMEM; - - svm->pasid = pasid; - svm->mm = mm; - - svm->notifier.ops = &intel_mmuops; - svm->domain = to_dmar_domain(domain); - ret = mmu_notifier_register(&svm->notifier, mm); - if (ret) { - kfree(svm); - return ret; - } - - ret = pasid_private_add(svm->pasid, svm); - if (ret) { - mmu_notifier_unregister(&svm->notifier, mm); - kfree(svm); - return ret; - } - } - - dmar_domain->svm = svm; dev_pasid = kzalloc(sizeof(*dev_pasid), GFP_KERNEL); if (!dev_pasid) - goto free_svm; + return -ENOMEM; dev_pasid->dev = dev; dev_pasid->pasid = pasid; @@ -274,30 +234,10 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, cache_tag_unassign_domain(to_dmar_domain(domain), dev, pasid); free_dev_pasid: kfree(dev_pasid); -free_svm: - if (list_empty(&dmar_domain->dev_pasids)) { - mmu_notifier_unregister(&svm->notifier, mm); - pasid_private_remove(pasid); - kfree(svm); - } return ret; } -void intel_svm_remove_dev_pasid(struct iommu_domain *domain) -{ - struct dmar_domain *dmar_domain = to_dmar_domain(domain); - struct intel_svm *svm = dmar_domain->svm; - struct mm_struct *mm = domain->mm; - - if (list_empty(&dmar_domain->dev_pasids)) { - if (svm->notifier.ops) - mmu_notifier_unregister(&svm->notifier, mm); - pasid_private_remove(svm->pasid); - kfree(svm); - } -} - /* Page request queue descriptor */ struct page_req_dsc { union { @@ -611,7 +551,10 @@ void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, static void intel_svm_domain_free(struct iommu_domain *domain) { - kfree(to_dmar_domain(domain)); + struct dmar_domain *dmar_domain = to_dmar_domain(domain); + + /* dmar_domain free is deferred to the mmu free_notifier callback. */ + mmu_notifier_put(&dmar_domain->notifier); } static const struct iommu_domain_ops intel_svm_domain_ops = { @@ -619,13 +562,16 @@ static const struct iommu_domain_ops intel_svm_domain_ops = { .free = intel_svm_domain_free }; -struct iommu_domain *intel_svm_domain_alloc(void) +struct iommu_domain *intel_svm_domain_alloc(struct device *dev, + struct mm_struct *mm) { struct dmar_domain *domain; + int ret; domain = kzalloc(sizeof(*domain), GFP_KERNEL); if (!domain) - return NULL; + return ERR_PTR(-ENOMEM); + domain->domain.ops = &intel_svm_domain_ops; domain->use_first_level = true; INIT_LIST_HEAD(&domain->dev_pasids); @@ -633,5 +579,12 @@ struct iommu_domain *intel_svm_domain_alloc(void) spin_lock_init(&domain->cache_lock); spin_lock_init(&domain->lock); + domain->notifier.ops = &intel_mmuops; + ret = mmu_notifier_register(&domain->notifier, mm); + if (ret) { + kfree(domain); + return ERR_PTR(ret); + } + return &domain->domain; } From 209516caff7b0dd715e551b21d7236f73427a936 Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Fri, 19 Apr 2024 13:42:11 +0200 Subject: [PATCH 0834/1702] dt-bindings: iommu: renesas,ipmmu-vmsa: add r8a779h0 support Document support for the I/O Memory Management Unit (IPMMU) on the Renesas R-Car V4M (R8A779H0) SoC. Signed-off-by: Thanh Le Signed-off-by: Geert Uytterhoeven Reviewed-by: Yoshihiro Shimoda Acked-by: Rob Herring (Arm) Link: https://lore.kernel.org/r/13643259be4e8a8e30632de622ad7c685dbb7c61.1713526852.git.geert+renesas@glider.be Signed-off-by: Joerg Roedel --- Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml index be90f68c11d18..0acaa2bcec089 100644 --- a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml +++ b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml @@ -50,6 +50,7 @@ properties: - renesas,ipmmu-r8a779a0 # R-Car V3U - renesas,ipmmu-r8a779f0 # R-Car S4-8 - renesas,ipmmu-r8a779g0 # R-Car V4H + - renesas,ipmmu-r8a779h0 # R-Car V4M - const: renesas,rcar-gen4-ipmmu-vmsa # R-Car Gen4 reg: From 0c3457926e7e65710f32e02920c7d423417c93a2 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 19 Apr 2024 17:54:40 +0100 Subject: [PATCH 0835/1702] OF: Retire dma-ranges mask workaround The fixup adding 1 to the dma-ranges size may have been for the benefit of some early AMD Seattle DTs, or may have merely been a just-in-case, but either way anyone who might have deserved to get the message has hopefully seen the warning in the 9 years we've had it there. The modern dma_range_map mechanism should happily handle odd-sized ranges with no ill effect, so there's little need to care anyway now. Clean it up. Acked-by: Rob Herring Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/26620039901fdae52079ec1c8a4b2b324964a13e.1713523152.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/of/device.c | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/drivers/of/device.c b/drivers/of/device.c index de89f99063758..a988bee2ee5ad 100644 --- a/drivers/of/device.c +++ b/drivers/of/device.c @@ -129,22 +129,6 @@ int of_dma_configure_id(struct device *dev, struct device_node *np, dma_end = r->dma_start + r->size; } size = dma_end - dma_start; - - /* - * Add a work around to treat the size as mask + 1 in case - * it is defined in DT as a mask. - */ - if (size & 1) { - dev_warn(dev, "Invalid size 0x%llx for dma-range(s)\n", - size); - size = size + 1; - } - - if (!size) { - dev_err(dev, "Adjusted size 0x%llx invalid\n", size); - kfree(map); - return -EINVAL; - } } /* From ba503cf41c90e173f9421f7882d84b228bada97a Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 19 Apr 2024 17:54:41 +0100 Subject: [PATCH 0836/1702] OF: Simplify DMA range calculations Juggling start, end, and size values for a range is somewhat redundant and a little hard to follow. Consolidate down to just using inclusive start and end, which saves us worrying about size overflows for full 64-bit ranges (note that passing a potentially-overflowed value through to arch_setup_dma_ops() is benign for all current implementations, and this is working towards removing that anyway). Acked-by: Rob Herring Reviewed-by: Jason Gunthorpe Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/3e0a72fe3d79eae660e4284bb32f2cb39868ccd7.1713523152.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/of/device.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/drivers/of/device.c b/drivers/of/device.c index a988bee2ee5ad..841ccd3a19d1e 100644 --- a/drivers/of/device.c +++ b/drivers/of/device.c @@ -96,7 +96,7 @@ int of_dma_configure_id(struct device *dev, struct device_node *np, const struct bus_dma_region *map = NULL; struct device_node *bus_np; u64 dma_start = 0; - u64 mask, end, size = 0; + u64 mask, end = 0; bool coherent; int iommu_ret; int ret; @@ -118,17 +118,15 @@ int of_dma_configure_id(struct device *dev, struct device_node *np, return ret == -ENODEV ? 0 : ret; } else { const struct bus_dma_region *r = map; - u64 dma_end = 0; /* Determine the overall bounds of all DMA regions */ for (dma_start = ~0; r->size; r++) { /* Take lower and upper limits */ if (r->dma_start < dma_start) dma_start = r->dma_start; - if (r->dma_start + r->size > dma_end) - dma_end = r->dma_start + r->size; + if (r->dma_start + r->size > end) + end = r->dma_start + r->size; } - size = dma_end - dma_start; } /* @@ -142,16 +140,15 @@ int of_dma_configure_id(struct device *dev, struct device_node *np, dev->dma_mask = &dev->coherent_dma_mask; } - if (!size && dev->coherent_dma_mask) - size = max(dev->coherent_dma_mask, dev->coherent_dma_mask + 1); - else if (!size) - size = 1ULL << 32; + if (!end && dev->coherent_dma_mask) + end = dev->coherent_dma_mask; + else if (!end) + end = (1ULL << 32) - 1; /* * Limit coherent and dma mask based on size and default mask * set by the driver. */ - end = dma_start + size - 1; mask = DMA_BIT_MASK(ilog2(end) + 1); dev->coherent_dma_mask &= mask; *dev->dma_mask &= mask; @@ -185,7 +182,7 @@ int of_dma_configure_id(struct device *dev, struct device_node *np, } else dev_dbg(dev, "device is behind an iommu\n"); - arch_setup_dma_ops(dev, dma_start, size, coherent); + arch_setup_dma_ops(dev, dma_start, end - dma_start + 1, coherent); if (iommu_ret) of_dma_set_restricted_buffer(dev, np); From 91cfd679f9e8b9a7bf2f26adf66eff99dbe2026b Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 19 Apr 2024 17:54:42 +0100 Subject: [PATCH 0837/1702] ACPI/IORT: Handle memory address size limits as limits Return the Root Complex/Named Component memory address size limit as an inclusive limit value, rather than an exclusive size. This saves having to fudge an off-by-one for the 64-bit case, and simplifies our caller. Acked-by: Hanjun Guo Reviewed-by: Jason Gunthorpe Tested-by: Hanjun Guo Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/284ae9fbadb12f2e3b5a30cd4d037d0e6843a8f4.1713523152.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/acpi/arm64/dma.c | 9 +++------ drivers/acpi/arm64/iort.c | 20 ++++++++++---------- include/linux/acpi_iort.h | 4 ++-- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/drivers/acpi/arm64/dma.c b/drivers/acpi/arm64/dma.c index 93d796531af30..b98a149f8d509 100644 --- a/drivers/acpi/arm64/dma.c +++ b/drivers/acpi/arm64/dma.c @@ -8,7 +8,6 @@ void acpi_arch_dma_setup(struct device *dev) { int ret; u64 end, mask; - u64 size = 0; const struct bus_dma_region *map = NULL; /* @@ -23,9 +22,9 @@ void acpi_arch_dma_setup(struct device *dev) } if (dev->coherent_dma_mask) - size = max(dev->coherent_dma_mask, dev->coherent_dma_mask + 1); + end = dev->coherent_dma_mask; else - size = 1ULL << 32; + end = (1ULL << 32) - 1; ret = acpi_dma_get_range(dev, &map); if (!ret && map) { @@ -36,18 +35,16 @@ void acpi_arch_dma_setup(struct device *dev) end = r->dma_start + r->size - 1; } - size = end + 1; dev->dma_range_map = map; } if (ret == -ENODEV) - ret = iort_dma_get_ranges(dev, &size); + ret = iort_dma_get_ranges(dev, &end); if (!ret) { /* * Limit coherent and dma mask based on size retrieved from * firmware. */ - end = size - 1; mask = DMA_BIT_MASK(ilog2(end) + 1); dev->bus_dma_limit = end; dev->coherent_dma_mask = min(dev->coherent_dma_mask, mask); diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index 6496ff5a6ba20..c0b1c2c194442 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -1367,7 +1367,7 @@ int iort_iommu_configure_id(struct device *dev, const u32 *input_id) { return -ENODEV; } #endif -static int nc_dma_get_range(struct device *dev, u64 *size) +static int nc_dma_get_range(struct device *dev, u64 *limit) { struct acpi_iort_node *node; struct acpi_iort_named_component *ncomp; @@ -1384,13 +1384,13 @@ static int nc_dma_get_range(struct device *dev, u64 *size) return -EINVAL; } - *size = ncomp->memory_address_limit >= 64 ? U64_MAX : - 1ULL<memory_address_limit; + *limit = ncomp->memory_address_limit >= 64 ? U64_MAX : + (1ULL << ncomp->memory_address_limit) - 1; return 0; } -static int rc_dma_get_range(struct device *dev, u64 *size) +static int rc_dma_get_range(struct device *dev, u64 *limit) { struct acpi_iort_node *node; struct acpi_iort_root_complex *rc; @@ -1408,8 +1408,8 @@ static int rc_dma_get_range(struct device *dev, u64 *size) return -EINVAL; } - *size = rc->memory_address_limit >= 64 ? U64_MAX : - 1ULL<memory_address_limit; + *limit = rc->memory_address_limit >= 64 ? U64_MAX : + (1ULL << rc->memory_address_limit) - 1; return 0; } @@ -1417,16 +1417,16 @@ static int rc_dma_get_range(struct device *dev, u64 *size) /** * iort_dma_get_ranges() - Look up DMA addressing limit for the device * @dev: device to lookup - * @size: DMA range size result pointer + * @limit: DMA limit result pointer * * Return: 0 on success, an error otherwise. */ -int iort_dma_get_ranges(struct device *dev, u64 *size) +int iort_dma_get_ranges(struct device *dev, u64 *limit) { if (dev_is_pci(dev)) - return rc_dma_get_range(dev, size); + return rc_dma_get_range(dev, limit); else - return nc_dma_get_range(dev, size); + return nc_dma_get_range(dev, limit); } static void __init acpi_iort_register_irq(int hwirq, const char *name, diff --git a/include/linux/acpi_iort.h b/include/linux/acpi_iort.h index 1cb65592c95dd..d4ed5622cf2b0 100644 --- a/include/linux/acpi_iort.h +++ b/include/linux/acpi_iort.h @@ -39,7 +39,7 @@ void iort_get_rmr_sids(struct fwnode_handle *iommu_fwnode, void iort_put_rmr_sids(struct fwnode_handle *iommu_fwnode, struct list_head *head); /* IOMMU interface */ -int iort_dma_get_ranges(struct device *dev, u64 *size); +int iort_dma_get_ranges(struct device *dev, u64 *limit); int iort_iommu_configure_id(struct device *dev, const u32 *id_in); void iort_iommu_get_resv_regions(struct device *dev, struct list_head *head); phys_addr_t acpi_iort_dma_get_max_cpu_address(void); @@ -55,7 +55,7 @@ void iort_get_rmr_sids(struct fwnode_handle *iommu_fwnode, struct list_head *hea static inline void iort_put_rmr_sids(struct fwnode_handle *iommu_fwnode, struct list_head *head) { } /* IOMMU interface */ -static inline int iort_dma_get_ranges(struct device *dev, u64 *size) +static inline int iort_dma_get_ranges(struct device *dev, u64 *limit) { return -ENODEV; } static inline int iort_iommu_configure_id(struct device *dev, const u32 *id_in) { return -ENODEV; } From fece6530bf4b59b01a476a12851e07751e73d69f Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 19 Apr 2024 17:54:43 +0100 Subject: [PATCH 0838/1702] dma-mapping: Add helpers for dma_range_map bounds Several places want to compute the lower and/or upper bounds of a dma_range_map, so let's factor that out into reusable helpers. Acked-by: Rob Herring Reviewed-by: Christoph Hellwig Reviewed-by: Hanjun Guo # For arm64 Reviewed-by: Jason Gunthorpe Tested-by: Hanjun Guo Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/45ec52f033ec4dfb364e23f48abaf787f612fa53.1713523152.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- arch/loongarch/kernel/dma.c | 9 ++------- drivers/acpi/arm64/dma.c | 8 +------- drivers/of/device.c | 11 ++--------- include/linux/dma-direct.h | 18 ++++++++++++++++++ 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/arch/loongarch/kernel/dma.c b/arch/loongarch/kernel/dma.c index 7a9c6a9dd2d01..429555fb4e138 100644 --- a/arch/loongarch/kernel/dma.c +++ b/arch/loongarch/kernel/dma.c @@ -8,17 +8,12 @@ void acpi_arch_dma_setup(struct device *dev) { int ret; - u64 mask, end = 0; + u64 mask, end; const struct bus_dma_region *map = NULL; ret = acpi_dma_get_range(dev, &map); if (!ret && map) { - const struct bus_dma_region *r = map; - - for (end = 0; r->size; r++) { - if (r->dma_start + r->size - 1 > end) - end = r->dma_start + r->size - 1; - } + end = dma_range_map_max(map); mask = DMA_BIT_MASK(ilog2(end) + 1); dev->bus_dma_limit = end; diff --git a/drivers/acpi/arm64/dma.c b/drivers/acpi/arm64/dma.c index b98a149f8d509..52b2abf886898 100644 --- a/drivers/acpi/arm64/dma.c +++ b/drivers/acpi/arm64/dma.c @@ -28,13 +28,7 @@ void acpi_arch_dma_setup(struct device *dev) ret = acpi_dma_get_range(dev, &map); if (!ret && map) { - const struct bus_dma_region *r = map; - - for (end = 0; r->size; r++) { - if (r->dma_start + r->size - 1 > end) - end = r->dma_start + r->size - 1; - } - + end = dma_range_map_max(map); dev->dma_range_map = map; } diff --git a/drivers/of/device.c b/drivers/of/device.c index 841ccd3a19d1e..9e7963972fa75 100644 --- a/drivers/of/device.c +++ b/drivers/of/device.c @@ -117,16 +117,9 @@ int of_dma_configure_id(struct device *dev, struct device_node *np, if (!force_dma) return ret == -ENODEV ? 0 : ret; } else { - const struct bus_dma_region *r = map; - /* Determine the overall bounds of all DMA regions */ - for (dma_start = ~0; r->size; r++) { - /* Take lower and upper limits */ - if (r->dma_start < dma_start) - dma_start = r->dma_start; - if (r->dma_start + r->size > end) - end = r->dma_start + r->size; - } + dma_start = dma_range_map_min(map); + end = dma_range_map_max(map); } /* diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index 3eb3589ff43e9..edbe13d007762 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -54,6 +54,24 @@ static inline phys_addr_t translate_dma_to_phys(struct device *dev, return (phys_addr_t)-1; } +static inline dma_addr_t dma_range_map_min(const struct bus_dma_region *map) +{ + dma_addr_t ret = (dma_addr_t)U64_MAX; + + for (; map->size; map++) + ret = min(ret, map->dma_start); + return ret; +} + +static inline dma_addr_t dma_range_map_max(const struct bus_dma_region *map) +{ + dma_addr_t ret = 0; + + for (; map->size; map++) + ret = max(ret, map->dma_start + map->size - 1); + return ret; +} + #ifdef CONFIG_ARCH_HAS_PHYS_TO_DMA #include #ifndef phys_to_dma_unencrypted From ad4750b07d3462ce29a0c9b1e88b2a1f9795290e Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 19 Apr 2024 17:54:44 +0100 Subject: [PATCH 0839/1702] iommu/dma: Make limit checks self-contained It's now easy to retrieve the device's DMA limits if we want to check them against the domain aperture, so do that ourselves instead of relying on them being passed through the callchain. Reviewed-by: Jason Gunthorpe Tested-by: Hanjun Guo Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/e28a114243d1e79eb3609aded034f8529521333f.1713523152.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index e4cb26f6a9434..c827ed690f1fb 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -660,19 +660,16 @@ static void iommu_dma_init_options(struct iommu_dma_options *options, /** * iommu_dma_init_domain - Initialise a DMA mapping domain * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() - * @base: IOVA at which the mappable address space starts - * @limit: Last address of the IOVA space * @dev: Device the domain is being initialised for * - * @base and @limit + 1 should be exact multiples of IOMMU page granularity to - * avoid rounding surprises. If necessary, we reserve the page at address 0 + * If the geometry and dma_range_map include address 0, we reserve that page * to ensure it is an invalid IOVA. It is safe to reinitialise a domain, but * any change which could make prior IOVAs invalid will fail. */ -static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, - dma_addr_t limit, struct device *dev) +static int iommu_dma_init_domain(struct iommu_domain *domain, struct device *dev) { struct iommu_dma_cookie *cookie = domain->iova_cookie; + const struct bus_dma_region *map = dev->dma_range_map; unsigned long order, base_pfn; struct iova_domain *iovad; int ret; @@ -684,18 +681,18 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, /* Use the smallest supported page size for IOVA granularity */ order = __ffs(domain->pgsize_bitmap); - base_pfn = max_t(unsigned long, 1, base >> order); + base_pfn = 1; /* Check the domain allows at least some access to the device... */ - if (domain->geometry.force_aperture) { + if (map) { + dma_addr_t base = dma_range_map_min(map); if (base > domain->geometry.aperture_end || - limit < domain->geometry.aperture_start) { + dma_range_map_max(map) < domain->geometry.aperture_start) { pr_warn("specified DMA range outside IOMMU capability\n"); return -EFAULT; } /* ...then finally give it a kicking to make sure it fits */ - base_pfn = max_t(unsigned long, base_pfn, - domain->geometry.aperture_start >> order); + base_pfn = max(base, domain->geometry.aperture_start) >> order; } /* start_pfn is always nonzero for an already-initialised domain */ @@ -1760,7 +1757,7 @@ void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit) * underlying IOMMU driver needs to support via the dma-iommu layer. */ if (iommu_is_dma_domain(domain)) { - if (iommu_dma_init_domain(domain, dma_base, dma_limit, dev)) + if (iommu_dma_init_domain(domain, dev)) goto out_err; dev->dma_ops = &iommu_dma_ops; } From b67483b3c44eaef2f771fa4c712e13f452675a67 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 19 Apr 2024 17:54:45 +0100 Subject: [PATCH 0840/1702] iommu/dma: Centralise iommu_setup_dma_ops() It's somewhat hard to see, but arm64's arch_setup_dma_ops() should only ever call iommu_setup_dma_ops() after a successful iommu_probe_device(), which means there should be no harm in achieving the same order of operations by running it off the back of iommu_probe_device() itself. This then puts it in line with the x86 and s390 .probe_finalize bodges, letting us pull it all into the main flow properly. As a bonus this lets us fold in and de-scope the PCI workaround setup as well. At this point we can also then pull the call up inside the group mutex, and avoid having to think about whether iommu_group_store_type() could theoretically race and free the domain if iommu_setup_dma_ops() ran just *before* iommu_device_use_default_domain() claims it... Furthermore we replace one .probe_finalize call completely, since the only remaining implementations are now one which only needs to run once for the initial boot-time probe, and two which themselves render that path unreachable. This leaves us a big step closer to realistically being able to unpick the variety of different things that iommu_setup_dma_ops() has been muddling together, and further streamline iommu-dma into core API flows in future. Reviewed-by: Lu Baolu # For Intel IOMMU Reviewed-by: Jason Gunthorpe Tested-by: Hanjun Guo Signed-off-by: Robin Murphy Acked-by: Catalin Marinas Link: https://lore.kernel.org/r/bebea331c1d688b34d9862eefd5ede47503961b8.1713523152.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- arch/arm64/mm/dma-mapping.c | 2 -- drivers/iommu/amd/iommu.c | 8 -------- drivers/iommu/dma-iommu.c | 18 ++++++------------ drivers/iommu/dma-iommu.h | 14 ++++++-------- drivers/iommu/intel/iommu.c | 7 ------- drivers/iommu/iommu.c | 20 +++++++------------- drivers/iommu/s390-iommu.c | 6 ------ drivers/iommu/virtio-iommu.c | 10 ---------- include/linux/iommu.h | 7 ------- 9 files changed, 19 insertions(+), 73 deletions(-) diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 61886e43e3a10..313d8938a2f03 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -58,8 +58,6 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, ARCH_DMA_MINALIGN, cls); dev->dma_coherent = coherent; - if (device_iommu_mapped(dev)) - iommu_setup_dma_ops(dev, dma_base, dma_base + size - 1); xen_setup_dma_ops(dev); } diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index d35c1b8c8e65c..085abf098fa9a 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2175,13 +2175,6 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) return iommu_dev; } -static void amd_iommu_probe_finalize(struct device *dev) -{ - /* Domains are initialized for this device - have a look what we ended up with */ - set_dma_ops(dev, NULL); - iommu_setup_dma_ops(dev, 0, U64_MAX); -} - static void amd_iommu_release_device(struct device *dev) { struct amd_iommu *iommu; @@ -2784,7 +2777,6 @@ const struct iommu_ops amd_iommu_ops = { .domain_alloc_user = amd_iommu_domain_alloc_user, .probe_device = amd_iommu_probe_device, .release_device = amd_iommu_release_device, - .probe_finalize = amd_iommu_probe_finalize, .device_group = amd_iommu_device_group, .get_resv_regions = amd_iommu_get_resv_regions, .is_attach_deferred = amd_iommu_is_attach_deferred, diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index c827ed690f1fb..bc1490bd4f7a9 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1741,25 +1741,20 @@ static const struct dma_map_ops iommu_dma_ops = { .max_mapping_size = iommu_dma_max_mapping_size, }; -/* - * The IOMMU core code allocates the default DMA domain, which the underlying - * IOMMU driver needs to support via the dma-iommu layer. - */ -void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit) +void iommu_setup_dma_ops(struct device *dev) { struct iommu_domain *domain = iommu_get_domain_for_dev(dev); - if (!domain) - goto out_err; + if (dev_is_pci(dev)) + dev->iommu->pci_32bit_workaround = !iommu_dma_forcedac; - /* - * The IOMMU core code allocates the default DMA domain, which the - * underlying IOMMU driver needs to support via the dma-iommu layer. - */ if (iommu_is_dma_domain(domain)) { if (iommu_dma_init_domain(domain, dev)) goto out_err; dev->dma_ops = &iommu_dma_ops; + } else if (dev->dma_ops == &iommu_dma_ops) { + /* Clean up if we've switched *from* a DMA domain */ + dev->dma_ops = NULL; } return; @@ -1767,7 +1762,6 @@ void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit) pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n", dev_name(dev)); } -EXPORT_SYMBOL_GPL(iommu_setup_dma_ops); static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, phys_addr_t msi_addr, struct iommu_domain *domain) diff --git a/drivers/iommu/dma-iommu.h b/drivers/iommu/dma-iommu.h index c829f1f82a991..c12d63457c764 100644 --- a/drivers/iommu/dma-iommu.h +++ b/drivers/iommu/dma-iommu.h @@ -9,6 +9,8 @@ #ifdef CONFIG_IOMMU_DMA +void iommu_setup_dma_ops(struct device *dev); + int iommu_get_dma_cookie(struct iommu_domain *domain); void iommu_put_dma_cookie(struct iommu_domain *domain); @@ -17,13 +19,13 @@ int iommu_dma_init_fq(struct iommu_domain *domain); void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list); extern bool iommu_dma_forcedac; -static inline void iommu_dma_set_pci_32bit_workaround(struct device *dev) -{ - dev->iommu->pci_32bit_workaround = !iommu_dma_forcedac; -} #else /* CONFIG_IOMMU_DMA */ +static inline void iommu_setup_dma_ops(struct device *dev) +{ +} + static inline int iommu_dma_init_fq(struct iommu_domain *domain) { return -EINVAL; @@ -42,9 +44,5 @@ static inline void iommu_dma_get_resv_regions(struct device *dev, struct list_he { } -static inline void iommu_dma_set_pci_32bit_workaround(struct device *dev) -{ -} - #endif /* CONFIG_IOMMU_DMA */ #endif /* __DMA_IOMMU_H */ diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 45c75a8a0ef56..64adfdadeb496 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4349,12 +4349,6 @@ static void intel_iommu_release_device(struct device *dev) set_dma_ops(dev, NULL); } -static void intel_iommu_probe_finalize(struct device *dev) -{ - set_dma_ops(dev, NULL); - iommu_setup_dma_ops(dev, 0, U64_MAX); -} - static void intel_iommu_get_resv_regions(struct device *device, struct list_head *head) { @@ -4834,7 +4828,6 @@ const struct iommu_ops intel_iommu_ops = { .domain_alloc = intel_iommu_domain_alloc, .domain_alloc_user = intel_iommu_domain_alloc_user, .probe_device = intel_iommu_probe_device, - .probe_finalize = intel_iommu_probe_finalize, .release_device = intel_iommu_release_device, .get_resv_regions = intel_iommu_get_resv_regions, .device_group = intel_iommu_device_group, diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 3183b0ed4cdb9..9df7cc75c1bc5 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -581,10 +581,11 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list if (list_empty(&group->entry)) list_add_tail(&group->entry, group_list); } - mutex_unlock(&group->mutex); - if (dev_is_pci(dev)) - iommu_dma_set_pci_32bit_workaround(dev); + if (group->default_domain) + iommu_setup_dma_ops(dev); + + mutex_unlock(&group->mutex); return 0; @@ -1828,6 +1829,8 @@ int bus_iommu_probe(const struct bus_type *bus) mutex_unlock(&group->mutex); return ret; } + for_each_group_device(group, gdev) + iommu_setup_dma_ops(gdev->dev); mutex_unlock(&group->mutex); /* @@ -3066,18 +3069,9 @@ static ssize_t iommu_group_store_type(struct iommu_group *group, if (ret) goto out_unlock; - /* - * Release the mutex here because ops->probe_finalize() call-back of - * some vendor IOMMU drivers calls arm_iommu_attach_device() which - * in-turn might call back into IOMMU core code, where it tries to take - * group->mutex, resulting in a deadlock. - */ - mutex_unlock(&group->mutex); - /* Make sure dma_ops is appropriatley set */ for_each_group_device(group, gdev) - iommu_group_do_probe_finalize(gdev->dev); - return count; + iommu_setup_dma_ops(gdev->dev); out_unlock: mutex_unlock(&group->mutex); diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c index 9a5196f523de5..d8eaa7ea380bb 100644 --- a/drivers/iommu/s390-iommu.c +++ b/drivers/iommu/s390-iommu.c @@ -695,11 +695,6 @@ static size_t s390_iommu_unmap_pages(struct iommu_domain *domain, return size; } -static void s390_iommu_probe_finalize(struct device *dev) -{ - iommu_setup_dma_ops(dev, 0, U64_MAX); -} - struct zpci_iommu_ctrs *zpci_get_iommu_ctrs(struct zpci_dev *zdev) { if (!zdev || !zdev->s390_domain) @@ -785,7 +780,6 @@ static const struct iommu_ops s390_iommu_ops = { .capable = s390_iommu_capable, .domain_alloc_paging = s390_domain_alloc_paging, .probe_device = s390_iommu_probe_device, - .probe_finalize = s390_iommu_probe_finalize, .release_device = s390_iommu_release_device, .device_group = generic_device_group, .pgsize_bitmap = SZ_4K, diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 04048f64a2c0f..8e776f6c6e35b 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -1025,15 +1025,6 @@ static struct iommu_device *viommu_probe_device(struct device *dev) return ERR_PTR(ret); } -static void viommu_probe_finalize(struct device *dev) -{ -#ifndef CONFIG_ARCH_HAS_SETUP_DMA_OPS - /* First clear the DMA ops in case we're switching from a DMA domain */ - set_dma_ops(dev, NULL); - iommu_setup_dma_ops(dev, 0, U64_MAX); -#endif -} - static void viommu_release_device(struct device *dev) { struct viommu_endpoint *vdev = dev_iommu_priv_get(dev); @@ -1073,7 +1064,6 @@ static struct iommu_ops viommu_ops = { .capable = viommu_capable, .domain_alloc = viommu_domain_alloc, .probe_device = viommu_probe_device, - .probe_finalize = viommu_probe_finalize, .release_device = viommu_release_device, .device_group = viommu_device_group, .get_resv_regions = viommu_get_resv_regions, diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 40dd439307e83..86f0bff3cc95f 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -1446,9 +1446,6 @@ static inline void iommu_debugfs_setup(void) {} #ifdef CONFIG_IOMMU_DMA #include -/* Setup call for arch DMA mapping code */ -void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit); - int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base); int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr); @@ -1459,10 +1456,6 @@ void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg); struct msi_desc; struct msi_msg; -static inline void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit) -{ -} - static inline int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base) { return -ENODEV; From f091e93306e0429ebb7589b9874590b6a9705e64 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 19 Apr 2024 17:54:46 +0100 Subject: [PATCH 0841/1702] dma-mapping: Simplify arch_setup_dma_ops() The dma_base, size and iommu arguments are only used by ARM, and can now easily be deduced from the device itself, so there's no need to pass them through the callchain as well. Acked-by: Rob Herring Reviewed-by: Christoph Hellwig Reviewed-by: Michael Kelley # For Hyper-V Reviewed-by: Jason Gunthorpe Tested-by: Hanjun Guo Signed-off-by: Robin Murphy Acked-by: Catalin Marinas Link: https://lore.kernel.org/r/5291c2326eab405b1aa7693aa964e8d3cb7193de.1713523152.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- arch/arc/mm/dma.c | 3 +-- arch/arm/mm/dma-mapping-nommu.c | 3 +-- arch/arm/mm/dma-mapping.c | 16 +++++++++------- arch/arm64/mm/dma-mapping.c | 3 +-- arch/mips/mm/dma-noncoherent.c | 3 +-- arch/riscv/mm/dma-noncoherent.c | 3 +-- drivers/acpi/scan.c | 7 +------ drivers/hv/hv_common.c | 6 +----- drivers/of/device.c | 4 +--- include/linux/dma-map-ops.h | 6 ++---- 10 files changed, 19 insertions(+), 35 deletions(-) diff --git a/arch/arc/mm/dma.c b/arch/arc/mm/dma.c index 197707bc76588..6b85e94f32759 100644 --- a/arch/arc/mm/dma.c +++ b/arch/arc/mm/dma.c @@ -90,8 +90,7 @@ void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, /* * Plug in direct dma map ops. */ -void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - bool coherent) +void arch_setup_dma_ops(struct device *dev, bool coherent) { /* * IOC hardware snoops all DMA traffic keeping the caches consistent diff --git a/arch/arm/mm/dma-mapping-nommu.c b/arch/arm/mm/dma-mapping-nommu.c index b94850b579952..97db5397c320b 100644 --- a/arch/arm/mm/dma-mapping-nommu.c +++ b/arch/arm/mm/dma-mapping-nommu.c @@ -33,8 +33,7 @@ void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, } } -void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - bool coherent) +void arch_setup_dma_ops(struct device *dev, bool coherent) { if (IS_ENABLED(CONFIG_CPU_V7M)) { /* diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index f68db05eba29f..5adf1769eee42 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -1709,11 +1709,15 @@ void arm_iommu_detach_device(struct device *dev) } EXPORT_SYMBOL_GPL(arm_iommu_detach_device); -static void arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, - bool coherent) +static void arm_setup_iommu_dma_ops(struct device *dev) { struct dma_iommu_mapping *mapping; + u64 dma_base = 0, size = 1ULL << 32; + if (dev->dma_range_map) { + dma_base = dma_range_map_min(dev->dma_range_map); + size = dma_range_map_max(dev->dma_range_map) - dma_base; + } mapping = arm_iommu_create_mapping(dev->bus, dma_base, size); if (IS_ERR(mapping)) { pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n", @@ -1744,8 +1748,7 @@ static void arm_teardown_iommu_dma_ops(struct device *dev) #else -static void arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, - bool coherent) +static void arm_setup_iommu_dma_ops(struct device *dev) { } @@ -1753,8 +1756,7 @@ static void arm_teardown_iommu_dma_ops(struct device *dev) { } #endif /* CONFIG_ARM_DMA_USE_IOMMU */ -void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - bool coherent) +void arch_setup_dma_ops(struct device *dev, bool coherent) { /* * Due to legacy code that sets the ->dma_coherent flag from a bus @@ -1774,7 +1776,7 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, return; if (device_iommu_mapped(dev)) - arm_setup_iommu_dma_ops(dev, dma_base, size, coherent); + arm_setup_iommu_dma_ops(dev); xen_setup_dma_ops(dev); dev->archdata.dma_ops_setup = true; diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 313d8938a2f03..0b320a25a471e 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -46,8 +46,7 @@ void arch_teardown_dma_ops(struct device *dev) } #endif -void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - bool coherent) +void arch_setup_dma_ops(struct device *dev, bool coherent) { int cls = cache_line_size_of_cpu(); diff --git a/arch/mips/mm/dma-noncoherent.c b/arch/mips/mm/dma-noncoherent.c index 0f3cec663a12c..ab4f2a75a7d01 100644 --- a/arch/mips/mm/dma-noncoherent.c +++ b/arch/mips/mm/dma-noncoherent.c @@ -137,8 +137,7 @@ void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, #endif #ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS -void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - bool coherent) +void arch_setup_dma_ops(struct device *dev, bool coherent) { dev->dma_coherent = coherent; } diff --git a/arch/riscv/mm/dma-noncoherent.c b/arch/riscv/mm/dma-noncoherent.c index 843107f834b23..cb89d7e0ba88a 100644 --- a/arch/riscv/mm/dma-noncoherent.c +++ b/arch/riscv/mm/dma-noncoherent.c @@ -128,8 +128,7 @@ void arch_dma_prep_coherent(struct page *page, size_t size) ALT_CMO_OP(FLUSH, flush_addr, size, riscv_cbom_block_size); } -void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - bool coherent) +void arch_setup_dma_ops(struct device *dev, bool coherent) { WARN_TAINT(!coherent && riscv_cbom_block_size > ARCH_DMA_MINALIGN, TAINT_CPU_OUT_OF_SPEC, diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index 7c157bf926956..b1a88992c1a93 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -1675,12 +1675,7 @@ int acpi_dma_configure_id(struct device *dev, enum dev_dma_attr attr, if (ret == -EPROBE_DEFER) return -EPROBE_DEFER; - /* - * Historically this routine doesn't fail driver probing due to errors - * in acpi_iommu_configure_id() - */ - - arch_setup_dma_ops(dev, 0, U64_MAX, attr == DEV_DMA_COHERENT); + arch_setup_dma_ops(dev, attr == DEV_DMA_COHERENT); return 0; } diff --git a/drivers/hv/hv_common.c b/drivers/hv/hv_common.c index dde3f9b6871af..9c452bfbd5719 100644 --- a/drivers/hv/hv_common.c +++ b/drivers/hv/hv_common.c @@ -561,11 +561,7 @@ EXPORT_SYMBOL_GPL(hv_query_ext_cap); void hv_setup_dma_ops(struct device *dev, bool coherent) { - /* - * Hyper-V does not offer a vIOMMU in the guest - * VM, so pass 0/NULL for the IOMMU settings - */ - arch_setup_dma_ops(dev, 0, 0, coherent); + arch_setup_dma_ops(dev, coherent); } EXPORT_SYMBOL_GPL(hv_setup_dma_ops); diff --git a/drivers/of/device.c b/drivers/of/device.c index 9e7963972fa75..312c633612112 100644 --- a/drivers/of/device.c +++ b/drivers/of/device.c @@ -95,7 +95,6 @@ int of_dma_configure_id(struct device *dev, struct device_node *np, { const struct bus_dma_region *map = NULL; struct device_node *bus_np; - u64 dma_start = 0; u64 mask, end = 0; bool coherent; int iommu_ret; @@ -118,7 +117,6 @@ int of_dma_configure_id(struct device *dev, struct device_node *np, return ret == -ENODEV ? 0 : ret; } else { /* Determine the overall bounds of all DMA regions */ - dma_start = dma_range_map_min(map); end = dma_range_map_max(map); } @@ -175,7 +173,7 @@ int of_dma_configure_id(struct device *dev, struct device_node *np, } else dev_dbg(dev, "device is behind an iommu\n"); - arch_setup_dma_ops(dev, dma_start, end - dma_start + 1, coherent); + arch_setup_dma_ops(dev, coherent); if (iommu_ret) of_dma_set_restricted_buffer(dev, np); diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 4abc60f042092..ed89e1ce0114d 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -426,11 +426,9 @@ bool arch_dma_unmap_sg_direct(struct device *dev, struct scatterlist *sg, #endif #ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS -void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - bool coherent); +void arch_setup_dma_ops(struct device *dev, bool coherent); #else -static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, - u64 size, bool coherent) +static inline void arch_setup_dma_ops(struct device *dev, bool coherent) { } #endif /* CONFIG_ARCH_HAS_SETUP_DMA_OPS */ From 9433d5b2ace529d8a66863d4b5127c6c4ca6e4c2 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 18 Apr 2024 10:33:46 +0000 Subject: [PATCH 0842/1702] iommu/amd: Rename amd_iommu_v2_supported() as amd_iommu_pasid_supported() To reflect its usage. No functional changes intended. Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240418103400.6229-2-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 2 +- drivers/iommu/amd/init.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index f482aab420f78..d77660551f2a6 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -38,7 +38,7 @@ extern int amd_iommu_guest_ir; extern enum io_pgtable_fmt amd_iommu_pgtable; extern int amd_iommu_gpt_level; -bool amd_iommu_v2_supported(void); +bool amd_iommu_pasid_supported(void); /* Device capabilities */ int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev); diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index ac6754a85f350..5687c19825d72 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -3690,7 +3690,7 @@ __setup("ivrs_ioapic", parse_ivrs_ioapic); __setup("ivrs_hpet", parse_ivrs_hpet); __setup("ivrs_acpihid", parse_ivrs_acpihid); -bool amd_iommu_v2_supported(void) +bool amd_iommu_pasid_supported(void) { /* CPU page table size should match IOMMU guest page table size */ if (cpu_feature_enabled(X86_FEATURE_LA57) && From c5ebd09625391000026b0860952e05d0f7fc4519 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 18 Apr 2024 10:33:47 +0000 Subject: [PATCH 0843/1702] iommu/amd: Introduce per device DTE update function Consolidate per device update and flush logic into separate function. Also make it as global function as it will be used in subsequent series to update the DTE. Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240418103400.6229-3-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 1 + drivers/iommu/amd/iommu.c | 26 ++++++++++++++++++-------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index d77660551f2a6..98aa3ce8473f8 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -56,6 +56,7 @@ int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid); void amd_iommu_flush_all_caches(struct amd_iommu *iommu); void amd_iommu_update_and_flush_device_table(struct protection_domain *domain); void amd_iommu_domain_update(struct protection_domain *domain); +void amd_iommu_dev_update_dte(struct iommu_dev_data *dev_data, bool set); void amd_iommu_domain_flush_complete(struct protection_domain *domain); void amd_iommu_domain_flush_pages(struct protection_domain *domain, u64 address, size_t size); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index e692217fcb280..394623abbaa48 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2002,6 +2002,21 @@ static void clear_dte_entry(struct amd_iommu *iommu, u16 devid) amd_iommu_apply_erratum_63(iommu, devid); } +/* Update and flush DTE for the given device */ +void amd_iommu_dev_update_dte(struct iommu_dev_data *dev_data, bool set) +{ + struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); + + if (set) + set_dte_entry(iommu, dev_data); + else + clear_dte_entry(iommu, dev_data->devid); + + clone_aliases(iommu, dev_data->dev); + device_flush_dte(dev_data); + iommu_completion_wait(iommu); +} + static int do_attach(struct iommu_dev_data *dev_data, struct protection_domain *domain) { @@ -2036,10 +2051,7 @@ static int do_attach(struct iommu_dev_data *dev_data, } /* Update device table */ - set_dte_entry(iommu, dev_data); - clone_aliases(iommu, dev_data->dev); - - device_flush_dte(dev_data); + amd_iommu_dev_update_dte(dev_data, true); return ret; } @@ -2058,11 +2070,9 @@ static void do_detach(struct iommu_dev_data *dev_data) /* Update data structures */ dev_data->domain = NULL; list_del(&dev_data->list); - clear_dte_entry(iommu, dev_data->devid); - clone_aliases(iommu, dev_data->dev); - /* Flush the DTE entry */ - device_flush_dte(dev_data); + /* Clear DTE and flush the entry */ + amd_iommu_dev_update_dte(dev_data, false); /* Flush IOTLB and wait for the flushes to finish */ amd_iommu_domain_flush_all(domain); From db44bd517ff2de4224fdb0e9c0db426572f9fa11 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 18 Apr 2024 10:33:48 +0000 Subject: [PATCH 0844/1702] iommu/amd: Add support for enabling/disabling IOMMU features Add support for struct iommu_ops.dev_{enable/disable}_feat. Please note that the empty feature switches will be populated by subsequent patches. Signed-off-by: Wei Huang Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240418103400.6229-4-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 394623abbaa48..a22ecd8479dbe 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2791,6 +2791,32 @@ static const struct iommu_dirty_ops amd_dirty_ops = { .read_and_clear_dirty = amd_iommu_read_and_clear_dirty, }; +static int amd_iommu_dev_enable_feature(struct device *dev, + enum iommu_dev_features feat) +{ + int ret; + + switch (feat) { + default: + ret = -EINVAL; + break; + } + return ret; +} + +static int amd_iommu_dev_disable_feature(struct device *dev, + enum iommu_dev_features feat) +{ + int ret; + + switch (feat) { + default: + ret = -EINVAL; + break; + } + return ret; +} + const struct iommu_ops amd_iommu_ops = { .capable = amd_iommu_capable, .domain_alloc = amd_iommu_domain_alloc, @@ -2803,6 +2829,8 @@ const struct iommu_ops amd_iommu_ops = { .is_attach_deferred = amd_iommu_is_attach_deferred, .pgsize_bitmap = AMD_IOMMU_PGSIZES, .def_domain_type = amd_iommu_def_domain_type, + .dev_enable_feat = amd_iommu_dev_enable_feature, + .dev_disable_feat = amd_iommu_dev_disable_feature, .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = amd_iommu_attach_device, .map_pages = amd_iommu_map_pages, From e08fcd901c4301c150a8212df28df7f4d4811988 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Thu, 18 Apr 2024 10:33:49 +0000 Subject: [PATCH 0845/1702] iommu/amd: Move PPR-related functions into ppr.c In preparation to subsequent PPR-related patches, and also remove static declaration for certain helper functions so that it can be reused in other files. Also rename below functions: alloc_ppr_log -> amd_iommu_alloc_ppr_log iommu_enable_ppr_log -> amd_iommu_enable_ppr_log free_ppr_log -> amd_iommu_free_ppr_log iommu_poll_ppr_log -> amd_iommu_poll_ppr_log Signed-off-by: Suravee Suthikulpanit Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240418103400.6229-5-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/Makefile | 2 +- drivers/iommu/amd/amd_iommu.h | 17 ++++- drivers/iommu/amd/init.c | 65 +++---------------- drivers/iommu/amd/iommu.c | 55 +--------------- drivers/iommu/amd/ppr.c | 114 ++++++++++++++++++++++++++++++++++ 5 files changed, 139 insertions(+), 114 deletions(-) create mode 100644 drivers/iommu/amd/ppr.c diff --git a/drivers/iommu/amd/Makefile b/drivers/iommu/amd/Makefile index f454fbb1569eb..93b11b6d764fd 100644 --- a/drivers/iommu/amd/Makefile +++ b/drivers/iommu/amd/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o +obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o ppr.o obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 98aa3ce8473f8..159e9a43aa618 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -17,10 +17,16 @@ irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data); irqreturn_t amd_iommu_int_thread_galog(int irq, void *data); irqreturn_t amd_iommu_int_handler(int irq, void *data); void amd_iommu_apply_erratum_63(struct amd_iommu *iommu, u16 devid); +void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type, + u8 cntrl_intr, u8 cntrl_log, + u32 status_run_mask, u32 status_overflow_mask); void amd_iommu_restart_event_logging(struct amd_iommu *iommu); void amd_iommu_restart_ga_log(struct amd_iommu *iommu); void amd_iommu_restart_ppr_log(struct amd_iommu *iommu); void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid); +void iommu_feature_enable(struct amd_iommu *iommu, u8 bit); +void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, + gfp_t gfp, size_t size); #ifdef CONFIG_AMD_IOMMU_DEBUGFS void amd_iommu_debugfs_setup(struct amd_iommu *iommu); @@ -49,6 +55,14 @@ int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid, unsigned long gcr3); int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid); +/* PPR */ +int __init amd_iommu_alloc_ppr_log(struct amd_iommu *iommu); +void __init amd_iommu_free_ppr_log(struct amd_iommu *iommu); +void amd_iommu_enable_ppr_log(struct amd_iommu *iommu); +void amd_iommu_poll_ppr_log(struct amd_iommu *iommu); +int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, + int status, int tag); + /* * This function flushes all internal caches of * the IOMMU used by this driver. @@ -74,9 +88,6 @@ static inline int amd_iommu_create_irq_domain(struct amd_iommu *iommu) } #endif -int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, - int status, int tag); - static inline bool is_rd890_iommu(struct pci_dev *pdev) { return (pdev->vendor == PCI_VENDOR_ID_ATI) && diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 5687c19825d72..2694467178184 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -419,7 +419,7 @@ static void iommu_set_device_table(struct amd_iommu *iommu) } /* Generic functions to enable/disable certain features of the IOMMU. */ -static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit) +void iommu_feature_enable(struct amd_iommu *iommu, u8 bit) { u64 ctrl; @@ -746,9 +746,9 @@ static int __init alloc_command_buffer(struct amd_iommu *iommu) * Interrupt handler has processed all pending events and adjusted head * and tail pointer. Reset overflow mask and restart logging again. */ -static void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type, - u8 cntrl_intr, u8 cntrl_log, - u32 status_run_mask, u32 status_overflow_mask) +void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type, + u8 cntrl_intr, u8 cntrl_log, + u32 status_run_mask, u32 status_overflow_mask) { u32 status; @@ -789,17 +789,6 @@ void amd_iommu_restart_ga_log(struct amd_iommu *iommu) MMIO_STATUS_GALOG_OVERFLOW_MASK); } -/* - * This function restarts ppr logging in case the IOMMU experienced - * PPR log overflow. - */ -void amd_iommu_restart_ppr_log(struct amd_iommu *iommu) -{ - amd_iommu_restart_log(iommu, "PPR", CONTROL_PPRINT_EN, - CONTROL_PPRLOG_EN, MMIO_STATUS_PPR_RUN_MASK, - MMIO_STATUS_PPR_OVERFLOW_MASK); -} - /* * This function resets the command buffer if the IOMMU stopped fetching * commands from it. @@ -848,8 +837,8 @@ static void __init free_command_buffer(struct amd_iommu *iommu) free_pages((unsigned long)iommu->cmd_buf, get_order(CMD_BUFFER_SIZE)); } -static void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, - gfp_t gfp, size_t size) +void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp, + size_t size) { int order = get_order(size); void *buf = (void *)__get_free_pages(gfp, order); @@ -904,42 +893,6 @@ static void __init free_event_buffer(struct amd_iommu *iommu) free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE)); } -/* allocates the memory where the IOMMU will log its events to */ -static int __init alloc_ppr_log(struct amd_iommu *iommu) -{ - iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, - PPR_LOG_SIZE); - - return iommu->ppr_log ? 0 : -ENOMEM; -} - -static void iommu_enable_ppr_log(struct amd_iommu *iommu) -{ - u64 entry; - - if (iommu->ppr_log == NULL) - return; - - iommu_feature_enable(iommu, CONTROL_PPR_EN); - - entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512; - - memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET, - &entry, sizeof(entry)); - - /* set head and tail to zero manually */ - writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); - writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); - - iommu_feature_enable(iommu, CONTROL_PPRLOG_EN); - iommu_feature_enable(iommu, CONTROL_PPRINT_EN); -} - -static void __init free_ppr_log(struct amd_iommu *iommu) -{ - free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE)); -} - static void free_ga_log(struct amd_iommu *iommu) { #ifdef CONFIG_IRQ_REMAP @@ -1683,7 +1636,7 @@ static void __init free_iommu_one(struct amd_iommu *iommu) free_cwwb_sem(iommu); free_command_buffer(iommu); free_event_buffer(iommu); - free_ppr_log(iommu); + amd_iommu_free_ppr_log(iommu); free_ga_log(iommu); iommu_unmap_mmio_space(iommu); } @@ -2099,7 +2052,7 @@ static int __init iommu_init_pci(struct amd_iommu *iommu) amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval); } - if (check_feature(FEATURE_PPR) && alloc_ppr_log(iommu)) + if (check_feature(FEATURE_PPR) && amd_iommu_alloc_ppr_log(iommu)) return -ENOMEM; if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) { @@ -2845,7 +2798,7 @@ static void enable_iommus_v2(void) struct amd_iommu *iommu; for_each_iommu(iommu) - iommu_enable_ppr_log(iommu); + amd_iommu_enable_ppr_log(iommu); } static void enable_iommus_vapic(void) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index a22ecd8479dbe..0decab9581069 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -818,59 +818,6 @@ static void iommu_poll_events(struct amd_iommu *iommu) writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); } -static void iommu_poll_ppr_log(struct amd_iommu *iommu) -{ - u32 head, tail; - - if (iommu->ppr_log == NULL) - return; - - head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); - tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); - - while (head != tail) { - volatile u64 *raw; - u64 entry[2]; - int i; - - raw = (u64 *)(iommu->ppr_log + head); - - /* - * Hardware bug: Interrupt may arrive before the entry is - * written to memory. If this happens we need to wait for the - * entry to arrive. - */ - for (i = 0; i < LOOP_TIMEOUT; ++i) { - if (PPR_REQ_TYPE(raw[0]) != 0) - break; - udelay(1); - } - - /* Avoid memcpy function-call overhead */ - entry[0] = raw[0]; - entry[1] = raw[1]; - - /* - * To detect the hardware errata 733 we need to clear the - * entry back to zero. This issue does not exist on SNP - * enabled system. Also this buffer is not writeable on - * SNP enabled system. - */ - if (!amd_iommu_snp_en) - raw[0] = raw[1] = 0UL; - - /* Update head pointer of hardware ring-buffer */ - head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE; - writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); - - /* TODO: PPR Handler will be added when we add IOPF support */ - - /* Refresh ring-buffer information */ - head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); - tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); - } -} - #ifdef CONFIG_IRQ_REMAP static int (*iommu_ga_log_notifier)(u32); @@ -991,7 +938,7 @@ irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data) { amd_iommu_handle_irq(data, "PPR", MMIO_STATUS_PPR_INT_MASK, MMIO_STATUS_PPR_OVERFLOW_MASK, - iommu_poll_ppr_log, amd_iommu_restart_ppr_log); + amd_iommu_poll_ppr_log, amd_iommu_restart_ppr_log); return IRQ_HANDLED; } diff --git a/drivers/iommu/amd/ppr.c b/drivers/iommu/amd/ppr.c new file mode 100644 index 0000000000000..1f76ed549ec1c --- /dev/null +++ b/drivers/iommu/amd/ppr.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2023 Advanced Micro Devices, Inc. + */ + +#define pr_fmt(fmt) "AMD-Vi: " fmt +#define dev_fmt(fmt) pr_fmt(fmt) + +#include +#include +#include + +#include + +#include "amd_iommu.h" +#include "amd_iommu_types.h" + +int __init amd_iommu_alloc_ppr_log(struct amd_iommu *iommu) +{ + iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, + PPR_LOG_SIZE); + return iommu->ppr_log ? 0 : -ENOMEM; +} + +void amd_iommu_enable_ppr_log(struct amd_iommu *iommu) +{ + u64 entry; + + if (iommu->ppr_log == NULL) + return; + + iommu_feature_enable(iommu, CONTROL_PPR_EN); + + entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512; + + memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET, + &entry, sizeof(entry)); + + /* set head and tail to zero manually */ + writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); + writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); + + iommu_feature_enable(iommu, CONTROL_PPRINT_EN); + iommu_feature_enable(iommu, CONTROL_PPRLOG_EN); +} + +void __init amd_iommu_free_ppr_log(struct amd_iommu *iommu) +{ + free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE)); +} + +/* + * This function restarts ppr logging in case the IOMMU experienced + * PPR log overflow. + */ +void amd_iommu_restart_ppr_log(struct amd_iommu *iommu) +{ + amd_iommu_restart_log(iommu, "PPR", CONTROL_PPRINT_EN, + CONTROL_PPRLOG_EN, MMIO_STATUS_PPR_RUN_MASK, + MMIO_STATUS_PPR_OVERFLOW_MASK); +} + +void amd_iommu_poll_ppr_log(struct amd_iommu *iommu) +{ + u32 head, tail; + + if (iommu->ppr_log == NULL) + return; + + head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); + tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); + + while (head != tail) { + volatile u64 *raw; + u64 entry[2]; + int i; + + raw = (u64 *)(iommu->ppr_log + head); + + /* + * Hardware bug: Interrupt may arrive before the entry is + * written to memory. If this happens we need to wait for the + * entry to arrive. + */ + for (i = 0; i < LOOP_TIMEOUT; ++i) { + if (PPR_REQ_TYPE(raw[0]) != 0) + break; + udelay(1); + } + + /* Avoid memcpy function-call overhead */ + entry[0] = raw[0]; + entry[1] = raw[1]; + + /* + * To detect the hardware errata 733 we need to clear the + * entry back to zero. This issue does not exist on SNP + * enabled system. Also this buffer is not writeable on + * SNP enabled system. + */ + if (!amd_iommu_snp_en) + raw[0] = raw[1] = 0UL; + + /* Update head pointer of hardware ring-buffer */ + head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE; + writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); + + /* TODO: PPR Handler will be added when we add IOPF support */ + + /* Refresh ring-buffer information */ + head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); + tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); + } +} From 7c5b7176f0c3996adbe853adb1f857bd4f82b1e2 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 18 Apr 2024 10:33:50 +0000 Subject: [PATCH 0846/1702] iommu/amd: Fix PPR interrupt processing logic * Do not re-read ppr head pointer as its just updated by the driver. * Do not read PPR buffer tail pointer inside while loop. If IOMMU generates PPR events continuously then completing interrupt processing takes long time. In worst case it may cause infinite loop. Suggested-by: Tom Lendacky Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240418103400.6229-6-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/ppr.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/iommu/amd/ppr.c b/drivers/iommu/amd/ppr.c index 1f76ed549ec1c..65db1745f40c0 100644 --- a/drivers/iommu/amd/ppr.c +++ b/drivers/iommu/amd/ppr.c @@ -106,9 +106,5 @@ void amd_iommu_poll_ppr_log(struct amd_iommu *iommu) writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); /* TODO: PPR Handler will be added when we add IOPF support */ - - /* Refresh ring-buffer information */ - head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); - tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); } } From a0c47f233e683e6a81fced5c9c9cef6fa0da9446 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 18 Apr 2024 10:33:51 +0000 Subject: [PATCH 0847/1702] iommu/amd: Introduce iommu_dev_data.max_pasids This variable will track the number of PASIDs supported by the device. If IOMMU or device doesn't support PASID then it will be zero. This will be used while allocating GCR3 table to decide required number of PASID table levels. Also in PASID bind path it will use this variable to check whether device supports PASID or not. Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240418103400.6229-7-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 1 + drivers/iommu/amd/iommu.c | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index d1fed5fc219b3..03442e86ae448 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -813,6 +813,7 @@ struct iommu_dev_data { struct device *dev; u16 devid; /* PCI Device ID */ + u32 max_pasids; /* Max supported PASIDs */ u32 flags; /* Holds AMD_IOMMU_DEVICE_FLAG_<*> */ int ats_qdep; u8 ats_enabled :1; /* ATS state */ diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 0decab9581069..7daf6d75d964e 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2104,6 +2104,7 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) { struct iommu_device *iommu_dev; struct amd_iommu *iommu; + struct iommu_dev_data *dev_data; int ret; if (!check_device(dev)) @@ -2130,6 +2131,17 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) iommu_dev = &iommu->iommu; } + /* + * If IOMMU and device supports PASID then it will contain max + * supported PASIDs, else it will be zero. + */ + dev_data = dev_iommu_priv_get(dev); + if (amd_iommu_pasid_supported() && dev_is_pci(dev) && + pdev_pasid_supported(dev_data)) { + dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids, + pci_max_pasids(to_pci_dev(dev))); + } + iommu_completion_wait(iommu); return iommu_dev; From c9e8701132e6cc162d082e7dad8a2e9110f5f8fd Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 18 Apr 2024 10:33:52 +0000 Subject: [PATCH 0848/1702] iommu/amd: Setup GCR3 table in advance if domain is SVA capable SVA can be supported if domain is in passthrough mode or paging domain with v2 page table. Current code sets up GCR3 table for domain with v2 page table only. Setup GCR3 table for all SVA capable domains. - Move GCR3 init/destroy to separate function. - Change default GCR3 table to use MAX supported PASIDs. Ideally it should use 1 level PASID table as its using PASID zero only. But we don't have support to extend PASID table yet. We will fix this later. - When domain is configured with passthrough mode, allocate default GCR3 table only if device is SVA capable. Note that in attach_device() path it will not know whether device will use SVA or not. If device is attached to passthrough domain and if it doesn't use SVA then GCR3 table will never be used. We will endup wasting memory allocated for GCR3 table. This is done to avoid DTE update when attaching PASID to device. Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240418103400.6229-8-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 86 ++++++++++++++++++++++++++++++++------- 1 file changed, 71 insertions(+), 15 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 7daf6d75d964e..288cf7485306f 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -89,6 +89,21 @@ static inline bool pdom_is_v2_pgtbl_mode(struct protection_domain *pdom) return (pdom && (pdom->pd_mode == PD_MODE_V2)); } +static inline bool pdom_is_in_pt_mode(struct protection_domain *pdom) +{ + return (pdom->domain.type == IOMMU_DOMAIN_IDENTITY); +} + +/* + * We cannot support PASID w/ existing v1 page table in the same domain + * since it will be nested. However, existing domain w/ v2 page table + * or passthrough mode can be used for PASID. + */ +static inline bool pdom_is_sva_capable(struct protection_domain *pdom) +{ + return pdom_is_v2_pgtbl_mode(pdom) || pdom_is_in_pt_mode(pdom); +} + static inline int get_acpihid_device_id(struct device *dev, struct acpihid_map_entry **entry) { @@ -1964,6 +1979,58 @@ void amd_iommu_dev_update_dte(struct iommu_dev_data *dev_data, bool set) iommu_completion_wait(iommu); } +/* + * If domain is SVA capable then initialize GCR3 table. Also if domain is + * in v2 page table mode then update GCR3[0]. + */ +static int init_gcr3_table(struct iommu_dev_data *dev_data, + struct protection_domain *pdom) +{ + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + int max_pasids = dev_data->max_pasids; + int ret = 0; + + /* + * If domain is in pt mode then setup GCR3 table only if device + * is PASID capable + */ + if (pdom_is_in_pt_mode(pdom) && !pdev_pasid_supported(dev_data)) + return ret; + + /* + * By default, setup GCR3 table to support MAX PASIDs + * supported by the device/IOMMU. + */ + ret = setup_gcr3_table(&dev_data->gcr3_info, iommu, + max_pasids > 0 ? max_pasids : 1); + if (ret) + return ret; + + /* Setup GCR3[0] only if domain is setup with v2 page table mode */ + if (!pdom_is_v2_pgtbl_mode(pdom)) + return ret; + + ret = update_gcr3(dev_data, 0, iommu_virt_to_phys(pdom->iop.pgd), true); + if (ret) + free_gcr3_table(&dev_data->gcr3_info); + + return ret; +} + +static void destroy_gcr3_table(struct iommu_dev_data *dev_data, + struct protection_domain *pdom) +{ + struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; + + if (pdom_is_v2_pgtbl_mode(pdom)) + update_gcr3(dev_data, 0, 0, false); + + if (gcr3_info->gcr3_tbl == NULL) + return; + + free_gcr3_table(gcr3_info); +} + static int do_attach(struct iommu_dev_data *dev_data, struct protection_domain *domain) { @@ -1982,19 +2049,10 @@ static int do_attach(struct iommu_dev_data *dev_data, domain->dev_iommu[iommu->index] += 1; domain->dev_cnt += 1; - /* Init GCR3 table and update device table */ - if (domain->pd_mode == PD_MODE_V2) { - /* By default, setup GCR3 table to support single PASID */ - ret = setup_gcr3_table(&dev_data->gcr3_info, iommu, 1); + if (pdom_is_sva_capable(domain)) { + ret = init_gcr3_table(dev_data, domain); if (ret) return ret; - - ret = update_gcr3(dev_data, 0, - iommu_virt_to_phys(domain->iop.pgd), true); - if (ret) { - free_gcr3_table(&dev_data->gcr3_info); - return ret; - } } /* Update device table */ @@ -2009,10 +2067,8 @@ static void do_detach(struct iommu_dev_data *dev_data) struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); /* Clear GCR3 table */ - if (domain->pd_mode == PD_MODE_V2) { - update_gcr3(dev_data, 0, 0, false); - free_gcr3_table(&dev_data->gcr3_info); - } + if (pdom_is_sva_capable(domain)) + destroy_gcr3_table(dev_data, domain); /* Update data structures */ dev_data->domain = NULL; From 25efbb055863079bbd47b32fc6d3c4a6f082daf1 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 18 Apr 2024 10:33:53 +0000 Subject: [PATCH 0849/1702] iommu/amd: Enable PCI features based on attached domain capability Commit eda8c2860ab6 ("iommu/amd: Enable device ATS/PASID/PRI capabilities independently") changed the way it enables device capability while attaching devices. I missed to account the attached domain capability. Meaning if domain is not capable of handling PASID/PRI (ex: paging domain with v1 page table) then enabling device feature is not required. This patch enables PASID/PRI only if domain is capable of handling SVA. Also move pci feature enablement to do_attach() function so that we make SVA capability in one place. Finally make PRI enable/disable functions as static functions. Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240418103400.6229-9-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 4 ---- drivers/iommu/amd/iommu.c | 22 ++++++++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 159e9a43aa618..f8919c54d9aec 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -46,10 +46,6 @@ extern int amd_iommu_gpt_level; bool amd_iommu_pasid_supported(void); -/* Device capabilities */ -int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev); -void amd_iommu_pdev_disable_cap_pri(struct pci_dev *pdev); - /* GCR3 setup */ int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid, unsigned long gcr3); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 288cf7485306f..1319f93bb473d 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -399,7 +399,7 @@ static inline void pdev_disable_cap_ats(struct pci_dev *pdev) } } -int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev) +static inline int pdev_enable_cap_pri(struct pci_dev *pdev) { struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); int ret = -EINVAL; @@ -407,6 +407,9 @@ int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev) if (dev_data->pri_enabled) return 0; + if (!dev_data->ats_enabled) + return 0; + if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) { /* * First reset the PRI state of the device. @@ -423,7 +426,7 @@ int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev) return ret; } -void amd_iommu_pdev_disable_cap_pri(struct pci_dev *pdev) +static inline void pdev_disable_cap_pri(struct pci_dev *pdev) { struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); @@ -465,15 +468,14 @@ static void pdev_enable_caps(struct pci_dev *pdev) { pdev_enable_cap_ats(pdev); pdev_enable_cap_pasid(pdev); - amd_iommu_pdev_enable_cap_pri(pdev); - + pdev_enable_cap_pri(pdev); } static void pdev_disable_caps(struct pci_dev *pdev) { pdev_disable_cap_ats(pdev); pdev_disable_cap_pasid(pdev); - amd_iommu_pdev_disable_cap_pri(pdev); + pdev_disable_cap_pri(pdev); } /* @@ -2035,6 +2037,7 @@ static int do_attach(struct iommu_dev_data *dev_data, struct protection_domain *domain) { struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + struct pci_dev *pdev; int ret = 0; /* Update data structures */ @@ -2049,10 +2052,16 @@ static int do_attach(struct iommu_dev_data *dev_data, domain->dev_iommu[iommu->index] += 1; domain->dev_cnt += 1; + pdev = dev_is_pci(dev_data->dev) ? to_pci_dev(dev_data->dev) : NULL; if (pdom_is_sva_capable(domain)) { ret = init_gcr3_table(dev_data, domain); if (ret) return ret; + + if (pdev) + pdev_enable_caps(pdev); + } else if (pdev) { + pdev_enable_cap_ats(pdev); } /* Update device table */ @@ -2107,9 +2116,6 @@ static int attach_device(struct device *dev, goto out; } - if (dev_is_pci(dev)) - pdev_enable_caps(to_pci_dev(dev)); - ret = do_attach(dev_data, domain); out: From 61928bab9d26b147d41d9b9a4d2328d0e1d6c0c0 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Thu, 18 Apr 2024 10:33:54 +0000 Subject: [PATCH 0850/1702] iommu/amd: Define per-IOMMU iopf_queue AMD IOMMU hardware supports PCI Peripheral Paging Request (PPR) using a PPR log, which is a circular buffer containing requests from downstream end-point devices. There is one PPR log per IOMMU instance. Therefore, allocate an iopf_queue per IOMMU instance during driver initialization, and free the queue during driver deinitialization. Also rename enable_iommus_v2() -> enable_iommus_ppr() to reflect its usage. And add amd_iommu_gt_ppr_supported() check before enabling PPR log. Signed-off-by: Suravee Suthikulpanit Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240418103400.6229-10-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 4 ++++ drivers/iommu/amd/amd_iommu_types.h | 4 ++++ drivers/iommu/amd/init.c | 18 +++++++++++++++-- drivers/iommu/amd/ppr.c | 31 +++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index f8919c54d9aec..66bf20d4a91a6 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -46,6 +46,10 @@ extern int amd_iommu_gpt_level; bool amd_iommu_pasid_supported(void); +/* IOPF */ +int amd_iommu_iopf_init(struct amd_iommu *iommu); +void amd_iommu_iopf_uninit(struct amd_iommu *iommu); + /* GCR3 setup */ int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid, unsigned long gcr3); diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 03442e86ae448..1a014dca7c2eb 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -762,6 +762,10 @@ struct amd_iommu { /* DebugFS Info */ struct dentry *debugfs; #endif + + /* IOPF support */ + struct iopf_queue *iopf_queue; + unsigned char iopfq_name[32]; }; static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 2694467178184..7f9b5c710058e 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1639,6 +1639,7 @@ static void __init free_iommu_one(struct amd_iommu *iommu) amd_iommu_free_ppr_log(iommu); free_ga_log(iommu); iommu_unmap_mmio_space(iommu); + amd_iommu_iopf_uninit(iommu); } static void __init free_iommu_all(void) @@ -2108,6 +2109,16 @@ static int __init iommu_init_pci(struct amd_iommu *iommu) if (ret) return ret; + /* + * Allocate per IOMMU IOPF queue here so that in attach device path, + * PRI capable device can be added to IOPF queue + */ + if (amd_iommu_gt_ppr_supported()) { + ret = amd_iommu_iopf_init(iommu); + if (ret) + return ret; + } + iommu_device_register(&iommu->iommu, &amd_iommu_ops, NULL); return pci_enable_device(iommu->dev); @@ -2793,10 +2804,13 @@ static void early_enable_iommus(void) } } -static void enable_iommus_v2(void) +static void enable_iommus_ppr(void) { struct amd_iommu *iommu; + if (!amd_iommu_gt_ppr_supported()) + return; + for_each_iommu(iommu) amd_iommu_enable_ppr_log(iommu); } @@ -3134,7 +3148,7 @@ static int amd_iommu_enable_interrupts(void) * PPR and GA log interrupt for all IOMMUs. */ enable_iommus_vapic(); - enable_iommus_v2(); + enable_iommus_ppr(); out: return ret; diff --git a/drivers/iommu/amd/ppr.c b/drivers/iommu/amd/ppr.c index 65db1745f40c0..d4e08800beacd 100644 --- a/drivers/iommu/amd/ppr.c +++ b/drivers/iommu/amd/ppr.c @@ -108,3 +108,34 @@ void amd_iommu_poll_ppr_log(struct amd_iommu *iommu) /* TODO: PPR Handler will be added when we add IOPF support */ } } + +/************************************************************** + * + * IOPF handling stuff + */ + +/* Setup per-IOMMU IOPF queue if not exist. */ +int amd_iommu_iopf_init(struct amd_iommu *iommu) +{ + int ret = 0; + + if (iommu->iopf_queue) + return ret; + + snprintf(iommu->iopfq_name, sizeof(iommu->iopfq_name), + "amdiommu-%#x-iopfq", + PCI_SEG_DEVID_TO_SBDF(iommu->pci_seg->id, iommu->devid)); + + iommu->iopf_queue = iopf_queue_alloc(iommu->iopfq_name); + if (!iommu->iopf_queue) + ret = -ENOMEM; + + return ret; +} + +/* Destroy per-IOMMU IOPF queue if no longer needed. */ +void amd_iommu_iopf_uninit(struct amd_iommu *iommu) +{ + iopf_queue_free(iommu->iopf_queue); + iommu->iopf_queue = NULL; +} From 405e2f122b83363f6793c5cb086cede536495b4f Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Thu, 18 Apr 2024 10:33:55 +0000 Subject: [PATCH 0851/1702] iommu/amd: Add support for page response This generates AMD IOMMU COMPLETE_PPR_REQUEST for the specified device with the specified PRI Response Code. Also update amd_iommu_complete_ppr() to accept 'struct device' instead of pdev as it just need device reference. Signed-off-by: Suravee Suthikulpanit Signed-off-by: Wei Huang Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240418103400.6229-11-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 5 +++-- drivers/iommu/amd/iommu.c | 8 ++++---- drivers/iommu/amd/ppr.c | 6 ++++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 66bf20d4a91a6..bb9a4c2e40daa 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -49,6 +49,8 @@ bool amd_iommu_pasid_supported(void); /* IOPF */ int amd_iommu_iopf_init(struct amd_iommu *iommu); void amd_iommu_iopf_uninit(struct amd_iommu *iommu); +void amd_iommu_page_response(struct device *dev, struct iopf_fault *evt, + struct iommu_page_response *resp); /* GCR3 setup */ int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, @@ -60,8 +62,7 @@ int __init amd_iommu_alloc_ppr_log(struct amd_iommu *iommu); void __init amd_iommu_free_ppr_log(struct amd_iommu *iommu); void amd_iommu_enable_ppr_log(struct amd_iommu *iommu); void amd_iommu_poll_ppr_log(struct amd_iommu *iommu); -int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, - int status, int tag); +int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag); /* * This function flushes all internal caches of diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 1319f93bb473d..11b50a6c744ef 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1628,15 +1628,14 @@ void amd_iommu_domain_update(struct protection_domain *domain) amd_iommu_domain_flush_all(domain); } -int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, - int status, int tag) +int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag) { struct iommu_dev_data *dev_data; struct amd_iommu *iommu; struct iommu_cmd cmd; - dev_data = dev_iommu_priv_get(&pdev->dev); - iommu = get_amd_iommu_from_dev(&pdev->dev); + dev_data = dev_iommu_priv_get(dev); + iommu = get_amd_iommu_from_dev(dev); build_complete_ppr(&cmd, dev_data->devid, pasid, status, tag, dev_data->pri_tlp); @@ -2852,6 +2851,7 @@ const struct iommu_ops amd_iommu_ops = { .def_domain_type = amd_iommu_def_domain_type, .dev_enable_feat = amd_iommu_dev_enable_feature, .dev_disable_feat = amd_iommu_dev_disable_feature, + .page_response = amd_iommu_page_response, .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = amd_iommu_attach_device, .map_pages = amd_iommu_map_pages, diff --git a/drivers/iommu/amd/ppr.c b/drivers/iommu/amd/ppr.c index d4e08800beacd..a33ce537b76ee 100644 --- a/drivers/iommu/amd/ppr.c +++ b/drivers/iommu/amd/ppr.c @@ -139,3 +139,9 @@ void amd_iommu_iopf_uninit(struct amd_iommu *iommu) iopf_queue_free(iommu->iopf_queue); iommu->iopf_queue = NULL; } + +void amd_iommu_page_response(struct device *dev, struct iopf_fault *evt, + struct iommu_page_response *resp) +{ + amd_iommu_complete_ppr(dev, resp->pasid, resp->code, resp->grpid); +} From 978d626b8f1a239acc635323d731c77eae54eb61 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 18 Apr 2024 10:33:56 +0000 Subject: [PATCH 0852/1702] iommu/amd: Add IO page fault notifier handler Whenever there is a page fault IOMMU logs entry to ppr log and sends interrupt to host. We have to handle the page fault and respond to IOMMU. Add support to validate page fault request and hook it to core iommu page fault handler. Signed-off-by: Wei Huang Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240418103400.6229-12-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 8 +++ drivers/iommu/amd/ppr.c | 100 +++++++++++++++++++++++++++- 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 1a014dca7c2eb..61ca32b7bb07f 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -251,6 +251,14 @@ #define PPR_ENTRY_SIZE 16 #define PPR_LOG_SIZE (PPR_ENTRY_SIZE * PPR_LOG_ENTRIES) +/* PAGE_SERVICE_REQUEST PPR Log Buffer Entry flags */ +#define PPR_FLAG_EXEC 0x002 /* Execute permission requested */ +#define PPR_FLAG_READ 0x004 /* Read permission requested */ +#define PPR_FLAG_WRITE 0x020 /* Write permission requested */ +#define PPR_FLAG_US 0x040 /* 1: User, 0: Supervisor */ +#define PPR_FLAG_RVSD 0x080 /* Reserved bit not zero */ +#define PPR_FLAG_GN 0x100 /* GVA and PASID is valid */ + #define PPR_REQ_TYPE(x) (((x) >> 60) & 0xfULL) #define PPR_FLAGS(x) (((x) >> 48) & 0xfffULL) #define PPR_DEVID(x) ((x) & 0xffffULL) diff --git a/drivers/iommu/amd/ppr.c b/drivers/iommu/amd/ppr.c index a33ce537b76ee..9bdd1db5f60a8 100644 --- a/drivers/iommu/amd/ppr.c +++ b/drivers/iommu/amd/ppr.c @@ -60,6 +60,103 @@ void amd_iommu_restart_ppr_log(struct amd_iommu *iommu) MMIO_STATUS_PPR_OVERFLOW_MASK); } +static inline u32 ppr_flag_to_fault_perm(u16 flag) +{ + int perm = 0; + + if (flag & PPR_FLAG_READ) + perm |= IOMMU_FAULT_PERM_READ; + if (flag & PPR_FLAG_WRITE) + perm |= IOMMU_FAULT_PERM_WRITE; + if (flag & PPR_FLAG_EXEC) + perm |= IOMMU_FAULT_PERM_EXEC; + if (!(flag & PPR_FLAG_US)) + perm |= IOMMU_FAULT_PERM_PRIV; + + return perm; +} + +static bool ppr_is_valid(struct amd_iommu *iommu, u64 *raw) +{ + struct device *dev = iommu->iommu.dev; + u16 devid = PPR_DEVID(raw[0]); + + if (!(PPR_FLAGS(raw[0]) & PPR_FLAG_GN)) { + dev_dbg(dev, "PPR logged [Request ignored due to GN=0 (device=%04x:%02x:%02x.%x " + "pasid=0x%05llx address=0x%llx flags=0x%04llx tag=0x%03llx]\n", + iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + PPR_PASID(raw[0]), raw[1], PPR_FLAGS(raw[0]), PPR_TAG(raw[0])); + return false; + } + + if (PPR_FLAGS(raw[0]) & PPR_FLAG_RVSD) { + dev_dbg(dev, "PPR logged [Invalid request format (device=%04x:%02x:%02x.%x " + "pasid=0x%05llx address=0x%llx flags=0x%04llx tag=0x%03llx]\n", + iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + PPR_PASID(raw[0]), raw[1], PPR_FLAGS(raw[0]), PPR_TAG(raw[0])); + return false; + } + + return true; +} + +static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw) +{ + struct iommu_dev_data *dev_data; + struct iopf_fault event; + struct pci_dev *pdev; + u16 devid = PPR_DEVID(raw[0]); + + if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) { + pr_info_ratelimited("Unknown PPR request received\n"); + return; + } + + pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, + PCI_BUS_NUM(devid), devid & 0xff); + if (!pdev) + return; + + if (!ppr_is_valid(iommu, raw)) + goto out; + + memset(&event, 0, sizeof(struct iopf_fault)); + + event.fault.type = IOMMU_FAULT_PAGE_REQ; + event.fault.prm.perm = ppr_flag_to_fault_perm(PPR_FLAGS(raw[0])); + event.fault.prm.addr = (u64)(raw[1] & PAGE_MASK); + event.fault.prm.pasid = PPR_PASID(raw[0]); + event.fault.prm.grpid = PPR_TAG(raw[0]) & 0x1FF; + + /* + * PASID zero is used for requests from the I/O device without + * a PASID + */ + dev_data = dev_iommu_priv_get(&pdev->dev); + if (event.fault.prm.pasid == 0 || + event.fault.prm.pasid >= dev_data->max_pasids) { + pr_info_ratelimited("Invalid PASID : 0x%x, device : 0x%x\n", + event.fault.prm.pasid, pdev->dev.id); + goto out; + } + + event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; + event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; + if (PPR_TAG(raw[0]) & 0x200) + event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; + + /* Submit event */ + iommu_report_device_fault(&pdev->dev, &event); + + return; + +out: + /* Nobody cared, abort */ + amd_iommu_complete_ppr(&pdev->dev, PPR_PASID(raw[0]), + IOMMU_PAGE_RESP_FAILURE, + PPR_TAG(raw[0]) & 0x1FF); +} + void amd_iommu_poll_ppr_log(struct amd_iommu *iommu) { u32 head, tail; @@ -105,7 +202,8 @@ void amd_iommu_poll_ppr_log(struct amd_iommu *iommu) head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE; writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); - /* TODO: PPR Handler will be added when we add IOPF support */ + /* Handle PPR entry */ + iommu_call_iopf_notifier(iommu, entry); } } From c4cb23111103a841c2df30058597398443bcad5f Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 18 Apr 2024 10:33:57 +0000 Subject: [PATCH 0853/1702] iommu/amd: Add support for enable/disable IOPF Return success from enable_feature(IOPF) path as this interface is going away. Instead we will enable/disable IOPF support in attach/detach device path. In attach device path, if device is capable of PRI, then we will add it to per IOMMU IOPF queue and enable PPR support in IOMMU. Also it will attach device to domain even if it fails to enable PRI or add device to IOPF queue as device can continue to work without PRI support. In detach device patch it follows following sequence: - Flush the queue for the given device - Disable PPR support in DTE[devid] - Remove device from IOPF queue - Disable device PRI Also add IOMMU_IOPF as dependency to AMD_IOMMU driver. Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240418103400.6229-13-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/Kconfig | 1 + drivers/iommu/amd/amd_iommu.h | 4 ++++ drivers/iommu/amd/iommu.c | 39 ++++++++++++++++++++++++++------- drivers/iommu/amd/ppr.c | 41 +++++++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/amd/Kconfig b/drivers/iommu/amd/Kconfig index 443b2c13c37b5..d563f6d496ca0 100644 --- a/drivers/iommu/amd/Kconfig +++ b/drivers/iommu/amd/Kconfig @@ -10,6 +10,7 @@ config AMD_IOMMU select IOMMU_API select IOMMU_IOVA select IOMMU_IO_PGTABLE + select IOMMU_IOPF select IOMMUFD_DRIVER if IOMMUFD depends on X86_64 && PCI && ACPI && HAVE_CMPXCHG_DOUBLE help diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index bb9a4c2e40daa..dabbb85a71e9c 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -51,6 +51,10 @@ int amd_iommu_iopf_init(struct amd_iommu *iommu); void amd_iommu_iopf_uninit(struct amd_iommu *iommu); void amd_iommu_page_response(struct device *dev, struct iopf_fault *evt, struct iommu_page_response *resp); +int amd_iommu_iopf_add_device(struct amd_iommu *iommu, + struct iommu_dev_data *dev_data); +void amd_iommu_iopf_remove_device(struct amd_iommu *iommu, + struct iommu_dev_data *dev_data); /* GCR3 setup */ int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 11b50a6c744ef..9d5bffa16cb6d 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2057,8 +2057,17 @@ static int do_attach(struct iommu_dev_data *dev_data, if (ret) return ret; - if (pdev) + if (pdev) { pdev_enable_caps(pdev); + + /* + * Device can continue to function even if IOPF + * enablement failed. Hence in error path just + * disable device PRI support. + */ + if (amd_iommu_iopf_add_device(iommu, dev_data)) + pdev_disable_cap_pri(pdev); + } } else if (pdev) { pdev_enable_cap_ats(pdev); } @@ -2130,12 +2139,11 @@ static int attach_device(struct device *dev, */ static void detach_device(struct device *dev) { - struct protection_domain *domain; - struct iommu_dev_data *dev_data; + struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + struct protection_domain *domain = dev_data->domain; + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); unsigned long flags; - - dev_data = dev_iommu_priv_get(dev); - domain = dev_data->domain; + bool ppr = dev_data->ppr; spin_lock_irqsave(&domain->lock, flags); @@ -2150,8 +2158,19 @@ static void detach_device(struct device *dev) if (WARN_ON(!dev_data->domain)) goto out; + if (ppr) { + iopf_queue_flush_dev(dev); + + /* Updated here so that it gets reflected in DTE */ + dev_data->ppr = false; + } + do_detach(dev_data); + /* Remove IOPF handler */ + if (ppr) + amd_iommu_iopf_remove_device(iommu, dev_data); + if (dev_is_pci(dev)) pdev_disable_caps(to_pci_dev(dev)); @@ -2814,9 +2833,11 @@ static const struct iommu_dirty_ops amd_dirty_ops = { static int amd_iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat) { - int ret; + int ret = 0; switch (feat) { + case IOMMU_DEV_FEAT_IOPF: + break; default: ret = -EINVAL; break; @@ -2827,9 +2848,11 @@ static int amd_iommu_dev_enable_feature(struct device *dev, static int amd_iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) { - int ret; + int ret = 0; switch (feat) { + case IOMMU_DEV_FEAT_IOPF: + break; default: ret = -EINVAL; break; diff --git a/drivers/iommu/amd/ppr.c b/drivers/iommu/amd/ppr.c index 9bdd1db5f60a8..0463daa2d46b8 100644 --- a/drivers/iommu/amd/ppr.c +++ b/drivers/iommu/amd/ppr.c @@ -243,3 +243,44 @@ void amd_iommu_page_response(struct device *dev, struct iopf_fault *evt, { amd_iommu_complete_ppr(dev, resp->pasid, resp->code, resp->grpid); } + +int amd_iommu_iopf_add_device(struct amd_iommu *iommu, + struct iommu_dev_data *dev_data) +{ + unsigned long flags; + int ret = 0; + + if (!dev_data->pri_enabled) + return ret; + + raw_spin_lock_irqsave(&iommu->lock, flags); + + if (!iommu->iopf_queue) { + ret = -EINVAL; + goto out_unlock; + } + + ret = iopf_queue_add_device(iommu->iopf_queue, dev_data->dev); + if (ret) + goto out_unlock; + + dev_data->ppr = true; + +out_unlock: + raw_spin_unlock_irqrestore(&iommu->lock, flags); + return ret; +} + +/* Its assumed that caller has verified that device was added to iopf queue */ +void amd_iommu_iopf_remove_device(struct amd_iommu *iommu, + struct iommu_dev_data *dev_data) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&iommu->lock, flags); + + iopf_queue_remove_device(iommu->iopf_queue, dev_data->dev); + dev_data->ppr = false; + + raw_spin_unlock_irqrestore(&iommu->lock, flags); +} From 1af95763e0a33e854b4c0a961756719eb8a2b74d Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 18 Apr 2024 10:33:58 +0000 Subject: [PATCH 0854/1702] iommu/amd: Initial SVA support for AMD IOMMU This includes : - Add data structure to track per protection domain dev/pasid binding details protection_domain->dev_data_list will track attached list of dev_data/PASIDs. - Move 'to_pdomain()' to header file - Add iommu_sva_set_dev_pasid(). It will check whether PASID is supported or not. Also adds PASID to SVA protection domain list as well as to device GCR3 table. - Add iommu_ops.remove_dev_pasid support. It will unbind PASID from device. Also remove pasid data from protection domain device list. - Add IOMMU_SVA as dependency to AMD_IOMMU driver For a given PASID, iommu_set_dev_pasid() will bind all devices to same SVA protection domain (1 PASID : 1 SVA protection domain : N devices). This protection domain is different from device protection domain (one that's mapped in attach_device() path). IOMMU uses domain ID for caching, invalidation, etc. In SVA mode it will use per-device-domain-ID. Hence in invalidation path we retrieve domain ID from gcr3_info_table structure and use that for invalidation. Co-developed-by: Wei Huang Signed-off-by: Wei Huang Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240418103400.6229-14-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/Kconfig | 1 + drivers/iommu/amd/Makefile | 2 +- drivers/iommu/amd/amd_iommu.h | 11 +++ drivers/iommu/amd/amd_iommu_types.h | 19 +++++ drivers/iommu/amd/iommu.c | 7 +- drivers/iommu/amd/pasid.c | 122 ++++++++++++++++++++++++++++ 6 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 drivers/iommu/amd/pasid.c diff --git a/drivers/iommu/amd/Kconfig b/drivers/iommu/amd/Kconfig index d563f6d496ca0..68d8fc107cb9a 100644 --- a/drivers/iommu/amd/Kconfig +++ b/drivers/iommu/amd/Kconfig @@ -10,6 +10,7 @@ config AMD_IOMMU select IOMMU_API select IOMMU_IOVA select IOMMU_IO_PGTABLE + select IOMMU_SVA select IOMMU_IOPF select IOMMUFD_DRIVER if IOMMUFD depends on X86_64 && PCI && ACPI && HAVE_CMPXCHG_DOUBLE diff --git a/drivers/iommu/amd/Makefile b/drivers/iommu/amd/Makefile index 93b11b6d764fd..9de33b2d42f52 100644 --- a/drivers/iommu/amd/Makefile +++ b/drivers/iommu/amd/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o ppr.o +obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o ppr.o pasid.o obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index dabbb85a71e9c..49462d0731321 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -44,6 +44,12 @@ extern int amd_iommu_guest_ir; extern enum io_pgtable_fmt amd_iommu_pgtable; extern int amd_iommu_gpt_level; +/* Protection domain ops */ +int iommu_sva_set_dev_pasid(struct iommu_domain *domain, + struct device *dev, ioasid_t pasid); +void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid); + +/* SVA/PASID */ bool amd_iommu_pasid_supported(void); /* IOPF */ @@ -174,6 +180,11 @@ static inline struct amd_iommu *get_amd_iommu_from_dev_data(struct iommu_dev_dat return iommu_get_iommu_dev(dev_data->dev, struct amd_iommu, iommu); } +static inline struct protection_domain *to_pdomain(struct iommu_domain *dom) +{ + return container_of(dom, struct protection_domain, domain); +} + bool translation_pre_enabled(struct amd_iommu *iommu); bool amd_iommu_is_attach_deferred(struct device *dev); int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line); diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 61ca32b7bb07f..ed9971cd3f0e3 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -8,7 +8,9 @@ #ifndef _ASM_X86_AMD_IOMMU_TYPES_H #define _ASM_X86_AMD_IOMMU_TYPES_H +#include #include +#include #include #include #include @@ -511,6 +513,11 @@ extern struct kmem_cache *amd_iommu_irq_cache; list_for_each_entry((iommu), &amd_iommu_list, list) #define for_each_iommu_safe(iommu, next) \ list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list) +/* Making iterating over protection_domain->dev_data_list easier */ +#define for_each_pdom_dev_data(pdom_dev_data, pdom) \ + list_for_each_entry(pdom_dev_data, &pdom->dev_data_list, list) +#define for_each_pdom_dev_data_safe(pdom_dev_data, next, pdom) \ + list_for_each_entry_safe((pdom_dev_data), (next), &pdom->dev_data_list, list) struct amd_iommu; struct iommu_domain; @@ -552,6 +559,16 @@ enum protection_domain_mode { PD_MODE_V2, }; +/* Track dev_data/PASID list for the protection domain */ +struct pdom_dev_data { + /* Points to attached device data */ + struct iommu_dev_data *dev_data; + /* PASID attached to the protection domain */ + ioasid_t pasid; + /* For protection_domain->dev_data_list */ + struct list_head list; +}; + /* * This structure contains generic data for IOMMU protection domains * independent of their use. @@ -568,6 +585,8 @@ struct protection_domain { bool dirty_tracking; /* dirty tracking is enabled in the domain */ unsigned dev_cnt; /* devices assigned to this domain */ unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */ + + struct list_head dev_data_list; /* List of pdom_dev_data */ }; /* diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 9d5bffa16cb6d..e0f770c9c9a86 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -194,11 +194,6 @@ static struct amd_iommu *rlookup_amd_iommu(struct device *dev) return __rlookup_amd_iommu(seg, PCI_SBDF_TO_DEVID(devid)); } -static struct protection_domain *to_pdomain(struct iommu_domain *dom) -{ - return container_of(dom, struct protection_domain, domain); -} - static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid) { struct iommu_dev_data *dev_data; @@ -2345,6 +2340,7 @@ static struct protection_domain *protection_domain_alloc(unsigned int type) spin_lock_init(&domain->lock); INIT_LIST_HEAD(&domain->dev_list); + INIT_LIST_HEAD(&domain->dev_data_list); domain->nid = NUMA_NO_NODE; switch (type) { @@ -2874,6 +2870,7 @@ const struct iommu_ops amd_iommu_ops = { .def_domain_type = amd_iommu_def_domain_type, .dev_enable_feat = amd_iommu_dev_enable_feature, .dev_disable_feat = amd_iommu_dev_disable_feature, + .remove_dev_pasid = amd_iommu_remove_dev_pasid, .page_response = amd_iommu_page_response, .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = amd_iommu_attach_device, diff --git a/drivers/iommu/amd/pasid.c b/drivers/iommu/amd/pasid.c new file mode 100644 index 0000000000000..c96384ae34421 --- /dev/null +++ b/drivers/iommu/amd/pasid.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024 Advanced Micro Devices, Inc. + */ + +#define pr_fmt(fmt) "AMD-Vi: " fmt +#define dev_fmt(fmt) pr_fmt(fmt) + +#include +#include + +#include "amd_iommu.h" + +static inline bool is_pasid_enabled(struct iommu_dev_data *dev_data) +{ + if (dev_data->pasid_enabled && dev_data->max_pasids && + dev_data->gcr3_info.gcr3_tbl != NULL) + return true; + + return false; +} + +static inline bool is_pasid_valid(struct iommu_dev_data *dev_data, + ioasid_t pasid) +{ + if (pasid > 0 && pasid < dev_data->max_pasids) + return true; + + return false; +} + +static void remove_dev_pasid(struct pdom_dev_data *pdom_dev_data) +{ + /* Update GCR3 table and flush IOTLB */ + amd_iommu_clear_gcr3(pdom_dev_data->dev_data, pdom_dev_data->pasid); + + list_del(&pdom_dev_data->list); + kfree(pdom_dev_data); +} + +/* Clear PASID from device GCR3 table and remove pdom_dev_data from list */ +static void remove_pdom_dev_pasid(struct protection_domain *pdom, + struct device *dev, ioasid_t pasid) +{ + struct pdom_dev_data *pdom_dev_data; + struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + + lockdep_assert_held(&pdom->lock); + + for_each_pdom_dev_data(pdom_dev_data, pdom) { + if (pdom_dev_data->dev_data == dev_data && + pdom_dev_data->pasid == pasid) { + remove_dev_pasid(pdom_dev_data); + break; + } + } +} + +int iommu_sva_set_dev_pasid(struct iommu_domain *domain, + struct device *dev, ioasid_t pasid) +{ + struct pdom_dev_data *pdom_dev_data; + struct protection_domain *sva_pdom = to_pdomain(domain); + struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + unsigned long flags; + int ret = -EINVAL; + + /* PASID zero is used for requests from the I/O device without PASID */ + if (!is_pasid_valid(dev_data, pasid)) + return ret; + + /* Make sure PASID is enabled */ + if (!is_pasid_enabled(dev_data)) + return ret; + + /* Add PASID to protection domain pasid list */ + pdom_dev_data = kzalloc(sizeof(*pdom_dev_data), GFP_KERNEL); + if (pdom_dev_data == NULL) + return ret; + + pdom_dev_data->pasid = pasid; + pdom_dev_data->dev_data = dev_data; + + spin_lock_irqsave(&sva_pdom->lock, flags); + + /* Setup GCR3 table */ + ret = amd_iommu_set_gcr3(dev_data, pasid, + iommu_virt_to_phys(domain->mm->pgd)); + if (ret) { + kfree(pdom_dev_data); + goto out_unlock; + } + + list_add(&pdom_dev_data->list, &sva_pdom->dev_data_list); + +out_unlock: + spin_unlock_irqrestore(&sva_pdom->lock, flags); + return ret; +} + +void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid) +{ + struct protection_domain *sva_pdom; + struct iommu_domain *domain; + unsigned long flags; + + if (!is_pasid_valid(dev_iommu_priv_get(dev), pasid)) + return; + + /* Get protection domain */ + domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA); + if (!domain) + return; + sva_pdom = to_pdomain(domain); + + spin_lock_irqsave(&sva_pdom->lock, flags); + + /* Remove PASID from dev_data_list */ + remove_pdom_dev_pasid(sva_pdom, dev, pasid); + + spin_unlock_irqrestore(&sva_pdom->lock, flags); +} From 80af5a45202422db957549a241e00bf4d4e0ce89 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 18 Apr 2024 10:33:59 +0000 Subject: [PATCH 0855/1702] iommu: Add ops->domain_alloc_sva() Make a new op that receives the device and the mm_struct that the SVA domain should be created for. Unlike domain_alloc_paging() the dev argument is never NULL here. This allows drivers to fully initialize the SVA domain and allocate the mmu_notifier during allocation. It allows the notifier lifetime to follow the lifetime of the iommu_domain. Since we have only one call site, upgrade the new op to return ERR_PTR instead of NULL. Signed-off-by: Jason Gunthorpe [Removed smmu3 related changes - Vasant] Signed-off-by: Vasant Hegde Reviewed-by: Tina Zhang Link: https://lore.kernel.org/r/20240418103400.6229-15-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu-sva.c | 16 +++++++++++----- include/linux/iommu.h | 3 +++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index 640acc804e8cd..18a35e798b729 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -108,8 +108,8 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm /* Allocate a new domain and set it on device pasid. */ domain = iommu_sva_domain_alloc(dev, mm); - if (!domain) { - ret = -ENOMEM; + if (IS_ERR(domain)) { + ret = PTR_ERR(domain); goto out_free_handle; } @@ -283,9 +283,15 @@ struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, const struct iommu_ops *ops = dev_iommu_ops(dev); struct iommu_domain *domain; - domain = ops->domain_alloc(IOMMU_DOMAIN_SVA); - if (!domain) - return NULL; + if (ops->domain_alloc_sva) { + domain = ops->domain_alloc_sva(dev, mm); + if (IS_ERR(domain)) + return domain; + } else { + domain = ops->domain_alloc(IOMMU_DOMAIN_SVA); + if (!domain) + return ERR_PTR(-ENOMEM); + } domain->type = IOMMU_DOMAIN_SVA; mmgrab(mm); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 2e925b5eba534..8aabe83af8f26 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -518,6 +518,7 @@ static inline int __iommu_copy_struct_from_user_array( * Upon failure, ERR_PTR must be returned. * @domain_alloc_paging: Allocate an iommu_domain that can be used for * UNMANAGED, DMA, and DMA_FQ domain types. + * @domain_alloc_sva: Allocate an iommu_domain for Shared Virtual Addressing. * @probe_device: Add device to iommu driver handling * @release_device: Remove device from iommu driver handling * @probe_finalize: Do final setup work after the device is added to an IOMMU @@ -558,6 +559,8 @@ struct iommu_ops { struct device *dev, u32 flags, struct iommu_domain *parent, const struct iommu_user_data *user_data); struct iommu_domain *(*domain_alloc_paging)(struct device *dev); + struct iommu_domain *(*domain_alloc_sva)(struct device *dev, + struct mm_struct *mm); struct iommu_device *(*probe_device)(struct device *dev); void (*release_device)(struct device *dev); From a5a91e54846d35c234939fb9170e035805353049 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Thu, 18 Apr 2024 10:34:00 +0000 Subject: [PATCH 0856/1702] iommu/amd: Add SVA domain support - Allocate SVA domain and setup mmu notifier. In free path unregister mmu notifier and free protection domain. - Add mmu notifier callback function. It will retrieve SVA protection domain and invalidates IO/TLB. Signed-off-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240418103400.6229-16-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 5 ++ drivers/iommu/amd/amd_iommu_types.h | 1 + drivers/iommu/amd/iommu.c | 10 ++-- drivers/iommu/amd/pasid.c | 80 +++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 49462d0731321..0005da7af7394 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -45,6 +45,11 @@ extern enum io_pgtable_fmt amd_iommu_pgtable; extern int amd_iommu_gpt_level; /* Protection domain ops */ +struct protection_domain *protection_domain_alloc(unsigned int type); +void protection_domain_free(struct protection_domain *domain); +struct iommu_domain *amd_iommu_domain_alloc_sva(struct device *dev, + struct mm_struct *mm); +void amd_iommu_domain_free(struct iommu_domain *dom); int iommu_sva_set_dev_pasid(struct iommu_domain *domain, struct device *dev, ioasid_t pasid); void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid); diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index ed9971cd3f0e3..2b76b5dedc1d9 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -586,6 +586,7 @@ struct protection_domain { unsigned dev_cnt; /* devices assigned to this domain */ unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */ + struct mmu_notifier mn; /* mmu notifier for the SVA domain */ struct list_head dev_data_list; /* List of pdom_dev_data */ }; diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index e0f770c9c9a86..f7ba6552ccad8 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2280,7 +2280,7 @@ static void cleanup_domain(struct protection_domain *domain) WARN_ON(domain->dev_cnt != 0); } -static void protection_domain_free(struct protection_domain *domain) +void protection_domain_free(struct protection_domain *domain) { if (!domain) return; @@ -2323,7 +2323,7 @@ static int protection_domain_init_v2(struct protection_domain *pdom) return 0; } -static struct protection_domain *protection_domain_alloc(unsigned int type) +struct protection_domain *protection_domain_alloc(unsigned int type) { struct io_pgtable_ops *pgtbl_ops; struct protection_domain *domain; @@ -2346,6 +2346,7 @@ static struct protection_domain *protection_domain_alloc(unsigned int type) switch (type) { /* No need to allocate io pgtable ops in passthrough mode */ case IOMMU_DOMAIN_IDENTITY: + case IOMMU_DOMAIN_SVA: return domain; case IOMMU_DOMAIN_DMA: pgtable = amd_iommu_pgtable; @@ -2465,7 +2466,7 @@ amd_iommu_domain_alloc_user(struct device *dev, u32 flags, return do_iommu_domain_alloc(type, dev, flags); } -static void amd_iommu_domain_free(struct iommu_domain *dom) +void amd_iommu_domain_free(struct iommu_domain *dom) { struct protection_domain *domain; unsigned long flags; @@ -2833,6 +2834,7 @@ static int amd_iommu_dev_enable_feature(struct device *dev, switch (feat) { case IOMMU_DEV_FEAT_IOPF: + case IOMMU_DEV_FEAT_SVA: break; default: ret = -EINVAL; @@ -2848,6 +2850,7 @@ static int amd_iommu_dev_disable_feature(struct device *dev, switch (feat) { case IOMMU_DEV_FEAT_IOPF: + case IOMMU_DEV_FEAT_SVA: break; default: ret = -EINVAL; @@ -2860,6 +2863,7 @@ const struct iommu_ops amd_iommu_ops = { .capable = amd_iommu_capable, .domain_alloc = amd_iommu_domain_alloc, .domain_alloc_user = amd_iommu_domain_alloc_user, + .domain_alloc_sva = amd_iommu_domain_alloc_sva, .probe_device = amd_iommu_probe_device, .release_device = amd_iommu_release_device, .probe_finalize = amd_iommu_probe_finalize, diff --git a/drivers/iommu/amd/pasid.c b/drivers/iommu/amd/pasid.c index c96384ae34421..04beb2217d938 100644 --- a/drivers/iommu/amd/pasid.c +++ b/drivers/iommu/amd/pasid.c @@ -56,6 +56,49 @@ static void remove_pdom_dev_pasid(struct protection_domain *pdom, } } +static void sva_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + struct pdom_dev_data *pdom_dev_data; + struct protection_domain *sva_pdom; + unsigned long flags; + + sva_pdom = container_of(mn, struct protection_domain, mn); + + spin_lock_irqsave(&sva_pdom->lock, flags); + + for_each_pdom_dev_data(pdom_dev_data, sva_pdom) { + amd_iommu_dev_flush_pasid_pages(pdom_dev_data->dev_data, + pdom_dev_data->pasid, + start, end - start); + } + + spin_unlock_irqrestore(&sva_pdom->lock, flags); +} + +static void sva_mn_release(struct mmu_notifier *mn, struct mm_struct *mm) +{ + struct pdom_dev_data *pdom_dev_data, *next; + struct protection_domain *sva_pdom; + unsigned long flags; + + sva_pdom = container_of(mn, struct protection_domain, mn); + + spin_lock_irqsave(&sva_pdom->lock, flags); + + /* Assume dev_data_list contains same PASID with different devices */ + for_each_pdom_dev_data_safe(pdom_dev_data, next, sva_pdom) + remove_dev_pasid(pdom_dev_data); + + spin_unlock_irqrestore(&sva_pdom->lock, flags); +} + +static const struct mmu_notifier_ops sva_mn = { + .arch_invalidate_secondary_tlbs = sva_arch_invalidate_secondary_tlbs, + .release = sva_mn_release, +}; + int iommu_sva_set_dev_pasid(struct iommu_domain *domain, struct device *dev, ioasid_t pasid) { @@ -120,3 +163,40 @@ void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid) spin_unlock_irqrestore(&sva_pdom->lock, flags); } + +static void iommu_sva_domain_free(struct iommu_domain *domain) +{ + struct protection_domain *sva_pdom = to_pdomain(domain); + + if (sva_pdom->mn.ops) + mmu_notifier_unregister(&sva_pdom->mn, domain->mm); + + amd_iommu_domain_free(domain); +} + +static const struct iommu_domain_ops amd_sva_domain_ops = { + .set_dev_pasid = iommu_sva_set_dev_pasid, + .free = iommu_sva_domain_free +}; + +struct iommu_domain *amd_iommu_domain_alloc_sva(struct device *dev, + struct mm_struct *mm) +{ + struct protection_domain *pdom; + int ret; + + pdom = protection_domain_alloc(IOMMU_DOMAIN_SVA); + if (!pdom) + return ERR_PTR(-ENOMEM); + + pdom->domain.ops = &amd_sva_domain_ops; + pdom->mn.ops = &sva_mn; + + ret = mmu_notifier_register(&pdom->mn, mm); + if (ret) { + protection_domain_free(pdom); + return ERR_PTR(ret); + } + + return &pdom->domain; +} From e9730744bf3af04cda23799029342aa3cddbc454 Mon Sep 17 00:00:00 2001 From: Daniel Thompson Date: Wed, 24 Apr 2024 15:03:34 +0100 Subject: [PATCH 0857/1702] kdb: Fix buffer overflow during tab-complete Currently, when the user attempts symbol completion with the Tab key, kdb will use strncpy() to insert the completed symbol into the command buffer. Unfortunately it passes the size of the source buffer rather than the destination to strncpy() with predictably horrible results. Most obviously if the command buffer is already full but cp, the cursor position, is in the middle of the buffer, then we will write past the end of the supplied buffer. Fix this by replacing the dubious strncpy() calls with memmove()/memcpy() calls plus explicit boundary checks to make sure we have enough space before we start moving characters around. Reported-by: Justin Stitt Closes: https://lore.kernel.org/all/CAFhGd8qESuuifuHsNjFPR-Va3P80bxrw+LqvC8deA8GziUJLpw@mail.gmail.com/ Cc: stable@vger.kernel.org Reviewed-by: Douglas Anderson Reviewed-by: Justin Stitt Tested-by: Justin Stitt Link: https://lore.kernel.org/r/20240424-kgdb_read_refactor-v3-1-f236dbe9828d@linaro.org Signed-off-by: Daniel Thompson --- kernel/debug/kdb/kdb_io.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/kernel/debug/kdb/kdb_io.c b/kernel/debug/kdb/kdb_io.c index 9443bc63c5a24..06dfbccb10336 100644 --- a/kernel/debug/kdb/kdb_io.c +++ b/kernel/debug/kdb/kdb_io.c @@ -367,14 +367,19 @@ static char *kdb_read(char *buffer, size_t bufsize) kdb_printf(kdb_prompt_str); kdb_printf("%s", buffer); } else if (tab != 2 && count > 0) { - len_tmp = strlen(p_tmp); - strncpy(p_tmp+len_tmp, cp, lastchar-cp+1); - len_tmp = strlen(p_tmp); - strncpy(cp, p_tmp+len, len_tmp-len + 1); - len = len_tmp - len; - kdb_printf("%s", cp); - cp += len; - lastchar += len; + /* How many new characters do we want from tmpbuffer? */ + len_tmp = strlen(p_tmp) - len; + if (lastchar + len_tmp >= bufend) + len_tmp = bufend - lastchar; + + if (len_tmp) { + /* + 1 ensures the '\0' is memmove'd */ + memmove(cp+len_tmp, cp, (lastchar-cp) + 1); + memcpy(cp, p_tmp+len, len_tmp); + kdb_printf("%s", cp); + cp += len_tmp; + lastchar += len_tmp; + } } kdb_nextline = 1; /* reset output line number */ break; From 09b35989421dfd5573f0b4683c7700a7483c71f9 Mon Sep 17 00:00:00 2001 From: Daniel Thompson Date: Wed, 24 Apr 2024 15:03:35 +0100 Subject: [PATCH 0858/1702] kdb: Use format-strings rather than '\0' injection in kdb_read() Currently when kdb_read() needs to reposition the cursor it uses copy and paste code that works by injecting an '\0' at the cursor position before delivering a carriage-return and reprinting the line (which stops at the '\0'). Tidy up the code by hoisting the copy and paste code into an appropriately named function. Additionally let's replace the '\0' injection with a proper field width parameter so that the string will be abridged during formatting instead. Cc: stable@vger.kernel.org # Not a bug fix but it is needed for later bug fixes Tested-by: Justin Stitt Reviewed-by: Douglas Anderson Link: https://lore.kernel.org/r/20240424-kgdb_read_refactor-v3-2-f236dbe9828d@linaro.org Signed-off-by: Daniel Thompson --- kernel/debug/kdb/kdb_io.c | 55 ++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/kernel/debug/kdb/kdb_io.c b/kernel/debug/kdb/kdb_io.c index 06dfbccb10336..50789c99b3ba8 100644 --- a/kernel/debug/kdb/kdb_io.c +++ b/kernel/debug/kdb/kdb_io.c @@ -184,6 +184,33 @@ char kdb_getchar(void) unreachable(); } +/** + * kdb_position_cursor() - Place cursor in the correct horizontal position + * @prompt: Nil-terminated string containing the prompt string + * @buffer: Nil-terminated string containing the entire command line + * @cp: Cursor position, pointer the character in buffer where the cursor + * should be positioned. + * + * The cursor is positioned by sending a carriage-return and then printing + * the content of the line until we reach the correct cursor position. + * + * There is some additional fine detail here. + * + * Firstly, even though kdb_printf() will correctly format zero-width fields + * we want the second call to kdb_printf() to be conditional. That keeps things + * a little cleaner when LOGGING=1. + * + * Secondly, we can't combine everything into one call to kdb_printf() since + * that renders into a fixed length buffer and the combined print could result + * in unwanted truncation. + */ +static void kdb_position_cursor(char *prompt, char *buffer, char *cp) +{ + kdb_printf("\r%s", kdb_prompt_str); + if (cp > buffer) + kdb_printf("%.*s", (int)(cp - buffer), buffer); +} + /* * kdb_read * @@ -212,7 +239,6 @@ static char *kdb_read(char *buffer, size_t bufsize) * and null byte */ char *lastchar; char *p_tmp; - char tmp; static char tmpbuffer[CMD_BUFLEN]; int len = strlen(buffer); int len_tmp; @@ -249,12 +275,8 @@ static char *kdb_read(char *buffer, size_t bufsize) } *(--lastchar) = '\0'; --cp; - kdb_printf("\b%s \r", cp); - tmp = *cp; - *cp = '\0'; - kdb_printf(kdb_prompt_str); - kdb_printf("%s", buffer); - *cp = tmp; + kdb_printf("\b%s ", cp); + kdb_position_cursor(kdb_prompt_str, buffer, cp); } break; case 10: /* linefeed */ @@ -272,19 +294,14 @@ static char *kdb_read(char *buffer, size_t bufsize) memcpy(tmpbuffer, cp+1, lastchar - cp - 1); memcpy(cp, tmpbuffer, lastchar - cp - 1); *(--lastchar) = '\0'; - kdb_printf("%s \r", cp); - tmp = *cp; - *cp = '\0'; - kdb_printf(kdb_prompt_str); - kdb_printf("%s", buffer); - *cp = tmp; + kdb_printf("%s ", cp); + kdb_position_cursor(kdb_prompt_str, buffer, cp); } break; case 1: /* Home */ if (cp > buffer) { - kdb_printf("\r"); - kdb_printf(kdb_prompt_str); cp = buffer; + kdb_position_cursor(kdb_prompt_str, buffer, cp); } break; case 5: /* End */ @@ -390,13 +407,9 @@ static char *kdb_read(char *buffer, size_t bufsize) memcpy(cp+1, tmpbuffer, lastchar - cp); *++lastchar = '\0'; *cp = key; - kdb_printf("%s\r", cp); + kdb_printf("%s", cp); ++cp; - tmp = *cp; - *cp = '\0'; - kdb_printf(kdb_prompt_str); - kdb_printf("%s", buffer); - *cp = tmp; + kdb_position_cursor(kdb_prompt_str, buffer, cp); } else { *++lastchar = '\0'; *cp++ = key; From db2f9c7dc29114f531df4a425d0867d01e1f1e28 Mon Sep 17 00:00:00 2001 From: Daniel Thompson Date: Wed, 24 Apr 2024 15:03:36 +0100 Subject: [PATCH 0859/1702] kdb: Fix console handling when editing and tab-completing commands Currently, if the cursor position is not at the end of the command buffer and the user uses the Tab-complete functions, then the console does not leave the cursor in the correct position. For example consider the following buffer with the cursor positioned at the ^: md kdb_pro 10 ^ Pressing tab should result in: md kdb_prompt_str 10 ^ However this does not happen. Instead the cursor is placed at the end (after then 10) and further cursor movement redraws incorrectly. The same problem exists when we double-Tab but in a different part of the code. Fix this by sending a carriage return and then redisplaying the text to the left of the cursor. Cc: stable@vger.kernel.org Reviewed-by: Douglas Anderson Tested-by: Justin Stitt Link: https://lore.kernel.org/r/20240424-kgdb_read_refactor-v3-3-f236dbe9828d@linaro.org Signed-off-by: Daniel Thompson --- kernel/debug/kdb/kdb_io.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/debug/kdb/kdb_io.c b/kernel/debug/kdb/kdb_io.c index 50789c99b3ba8..5fccb46f399e5 100644 --- a/kernel/debug/kdb/kdb_io.c +++ b/kernel/debug/kdb/kdb_io.c @@ -383,6 +383,8 @@ static char *kdb_read(char *buffer, size_t bufsize) kdb_printf("\n"); kdb_printf(kdb_prompt_str); kdb_printf("%s", buffer); + if (cp != lastchar) + kdb_position_cursor(kdb_prompt_str, buffer, cp); } else if (tab != 2 && count > 0) { /* How many new characters do we want from tmpbuffer? */ len_tmp = strlen(p_tmp) - len; @@ -396,6 +398,9 @@ static char *kdb_read(char *buffer, size_t bufsize) kdb_printf("%s", cp); cp += len_tmp; lastchar += len_tmp; + if (cp != lastchar) + kdb_position_cursor(kdb_prompt_str, + buffer, cp); } } kdb_nextline = 1; /* reset output line number */ From 6244917f377bf64719551b58592a02a0336a7439 Mon Sep 17 00:00:00 2001 From: Daniel Thompson Date: Wed, 24 Apr 2024 15:03:37 +0100 Subject: [PATCH 0860/1702] kdb: Merge identical case statements in kdb_read() The code that handles case 14 (down) and case 16 (up) has been copy and pasted despite being byte-for-byte identical. Combine them. Cc: stable@vger.kernel.org # Not a bug fix but it is needed for later bug fixes Reviewed-by: Douglas Anderson Tested-by: Justin Stitt Link: https://lore.kernel.org/r/20240424-kgdb_read_refactor-v3-4-f236dbe9828d@linaro.org Signed-off-by: Daniel Thompson --- kernel/debug/kdb/kdb_io.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/kernel/debug/kdb/kdb_io.c b/kernel/debug/kdb/kdb_io.c index 5fccb46f399e5..a73779529803f 100644 --- a/kernel/debug/kdb/kdb_io.c +++ b/kernel/debug/kdb/kdb_io.c @@ -317,6 +317,7 @@ static char *kdb_read(char *buffer, size_t bufsize) } break; case 14: /* Down */ + case 16: /* Up */ memset(tmpbuffer, ' ', strlen(kdb_prompt_str) + (lastchar-buffer)); *(tmpbuffer+strlen(kdb_prompt_str) + @@ -331,15 +332,6 @@ static char *kdb_read(char *buffer, size_t bufsize) ++cp; } break; - case 16: /* Up */ - memset(tmpbuffer, ' ', - strlen(kdb_prompt_str) + (lastchar-buffer)); - *(tmpbuffer+strlen(kdb_prompt_str) + - (lastchar-buffer)) = '\0'; - kdb_printf("\r%s\r", tmpbuffer); - *lastchar = (char)key; - *(lastchar+1) = '\0'; - return lastchar; case 9: /* Tab */ if (tab < 2) ++tab; From c9b51ddb66b1d96e4d364c088da0f1dfb004c574 Mon Sep 17 00:00:00 2001 From: Daniel Thompson Date: Wed, 24 Apr 2024 15:03:38 +0100 Subject: [PATCH 0861/1702] kdb: Use format-specifiers rather than memset() for padding in kdb_read() Currently when the current line should be removed from the display kdb_read() uses memset() to fill a temporary buffer with spaces. The problem is not that this could be trivially implemented using a format string rather than open coding it. The real problem is that it is possible, on systems with a long kdb_prompt_str, to write past the end of the tmpbuffer. Happily, as mentioned above, this can be trivially implemented using a format string. Make it so! Cc: stable@vger.kernel.org Reviewed-by: Douglas Anderson Tested-by: Justin Stitt Link: https://lore.kernel.org/r/20240424-kgdb_read_refactor-v3-5-f236dbe9828d@linaro.org Signed-off-by: Daniel Thompson --- kernel/debug/kdb/kdb_io.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kernel/debug/kdb/kdb_io.c b/kernel/debug/kdb/kdb_io.c index a73779529803f..2aeaf9765b248 100644 --- a/kernel/debug/kdb/kdb_io.c +++ b/kernel/debug/kdb/kdb_io.c @@ -318,11 +318,9 @@ static char *kdb_read(char *buffer, size_t bufsize) break; case 14: /* Down */ case 16: /* Up */ - memset(tmpbuffer, ' ', - strlen(kdb_prompt_str) + (lastchar-buffer)); - *(tmpbuffer+strlen(kdb_prompt_str) + - (lastchar-buffer)) = '\0'; - kdb_printf("\r%s\r", tmpbuffer); + kdb_printf("\r%*c\r", + (int)(strlen(kdb_prompt_str) + (lastchar - buffer)), + ' '); *lastchar = (char)key; *(lastchar+1) = '\0'; return lastchar; From 80bd73c154e3063c4f9293163daf3262335f9f86 Mon Sep 17 00:00:00 2001 From: Daniel Thompson Date: Wed, 24 Apr 2024 15:03:39 +0100 Subject: [PATCH 0862/1702] kdb: Replace double memcpy() with memmove() in kdb_read() At several points in kdb_read() there are variants of the following code pattern (with offsets slightly altered): memcpy(tmpbuffer, cp, lastchar - cp); memcpy(cp-1, tmpbuffer, lastchar - cp); *(--lastchar) = '\0'; There is no need to use tmpbuffer here, since we can use memmove() instead so refactor in the obvious way. Additionally the strings that are being copied are already properly terminated so let's also change the code so that the library calls also move the terminator. Changing how the terminators are managed has no functional effect for now but might allow us to retire lastchar at a later point. lastchar, although stored as a pointer, is functionally equivalent to caching strlen(buffer). Reviewed-by: Douglas Anderson Tested-by: Justin Stitt Link: https://lore.kernel.org/r/20240424-kgdb_read_refactor-v3-6-f236dbe9828d@linaro.org Signed-off-by: Daniel Thompson --- kernel/debug/kdb/kdb_io.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/kernel/debug/kdb/kdb_io.c b/kernel/debug/kdb/kdb_io.c index 2aeaf9765b248..40617f36a6db4 100644 --- a/kernel/debug/kdb/kdb_io.c +++ b/kernel/debug/kdb/kdb_io.c @@ -269,12 +269,9 @@ static char *kdb_read(char *buffer, size_t bufsize) switch (key) { case 8: /* backspace */ if (cp > buffer) { - if (cp < lastchar) { - memcpy(tmpbuffer, cp, lastchar - cp); - memcpy(cp-1, tmpbuffer, lastchar - cp); - } - *(--lastchar) = '\0'; - --cp; + memmove(cp-1, cp, lastchar - cp + 1); + lastchar--; + cp--; kdb_printf("\b%s ", cp); kdb_position_cursor(kdb_prompt_str, buffer, cp); } @@ -291,9 +288,8 @@ static char *kdb_read(char *buffer, size_t bufsize) return buffer; case 4: /* Del */ if (cp < lastchar) { - memcpy(tmpbuffer, cp+1, lastchar - cp - 1); - memcpy(cp, tmpbuffer, lastchar - cp - 1); - *(--lastchar) = '\0'; + memmove(cp, cp+1, lastchar - cp); + lastchar--; kdb_printf("%s ", cp); kdb_position_cursor(kdb_prompt_str, buffer, cp); } @@ -398,9 +394,8 @@ static char *kdb_read(char *buffer, size_t bufsize) default: if (key >= 32 && lastchar < bufend) { if (cp < lastchar) { - memcpy(tmpbuffer, cp, lastchar - cp); - memcpy(cp+1, tmpbuffer, lastchar - cp); - *++lastchar = '\0'; + memmove(cp+1, cp, lastchar - cp + 1); + lastchar++; *cp = key; kdb_printf("%s", cp); ++cp; From 64d504cfcd514743aaed3a5b79c060f0143149e9 Mon Sep 17 00:00:00 2001 From: Daniel Thompson Date: Wed, 24 Apr 2024 15:03:40 +0100 Subject: [PATCH 0863/1702] kdb: Simplify management of tmpbuffer in kdb_read() The current approach to filling tmpbuffer with completion candidates is confusing, with the buffer management being especially hard to reason about. That's because it doesn't copy the completion canidate into tmpbuffer, instead of copies a whole bunch of other nonsense and then runs the completion search from the middle of tmpbuffer! Change this to copy nothing but the completion candidate into tmpbuffer. Pretty much everything else in this patch is renaming to reflect the above change: s/p_tmp/tmpbuffer/ s/buf_size/sizeof(tmpbuffer)/ Reviewed-by: Douglas Anderson Tested-by: Justin Stitt Link: https://lore.kernel.org/r/20240424-kgdb_read_refactor-v3-7-f236dbe9828d@linaro.org Signed-off-by: Daniel Thompson --- kernel/debug/kdb/kdb_io.c | 41 +++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/kernel/debug/kdb/kdb_io.c b/kernel/debug/kdb/kdb_io.c index 40617f36a6db4..3131334d7a81c 100644 --- a/kernel/debug/kdb/kdb_io.c +++ b/kernel/debug/kdb/kdb_io.c @@ -239,6 +239,7 @@ static char *kdb_read(char *buffer, size_t bufsize) * and null byte */ char *lastchar; char *p_tmp; + char tmp; static char tmpbuffer[CMD_BUFLEN]; int len = strlen(buffer); int len_tmp; @@ -246,8 +247,7 @@ static char *kdb_read(char *buffer, size_t bufsize) int count; int i; int diag, dtab_count; - int key, buf_size, ret; - + int key, ret; diag = kdbgetintenv("DTABCOUNT", &dtab_count); if (diag) @@ -329,21 +329,16 @@ static char *kdb_read(char *buffer, size_t bufsize) case 9: /* Tab */ if (tab < 2) ++tab; - p_tmp = buffer; - while (*p_tmp == ' ') - p_tmp++; - if (p_tmp > cp) - break; - memcpy(tmpbuffer, p_tmp, cp-p_tmp); - *(tmpbuffer + (cp-p_tmp)) = '\0'; - p_tmp = strrchr(tmpbuffer, ' '); - if (p_tmp) - ++p_tmp; - else - p_tmp = tmpbuffer; - len = strlen(p_tmp); - buf_size = sizeof(tmpbuffer) - (p_tmp - tmpbuffer); - count = kallsyms_symbol_complete(p_tmp, buf_size); + + tmp = *cp; + *cp = '\0'; + p_tmp = strrchr(buffer, ' '); + p_tmp = (p_tmp ? p_tmp + 1 : buffer); + strscpy(tmpbuffer, p_tmp, sizeof(tmpbuffer)); + *cp = tmp; + + len = strlen(tmpbuffer); + count = kallsyms_symbol_complete(tmpbuffer, sizeof(tmpbuffer)); if (tab == 2 && count > 0) { kdb_printf("\n%d symbols are found.", count); if (count > dtab_count) { @@ -355,14 +350,14 @@ static char *kdb_read(char *buffer, size_t bufsize) } kdb_printf("\n"); for (i = 0; i < count; i++) { - ret = kallsyms_symbol_next(p_tmp, i, buf_size); + ret = kallsyms_symbol_next(tmpbuffer, i, sizeof(tmpbuffer)); if (WARN_ON(!ret)) break; if (ret != -E2BIG) - kdb_printf("%s ", p_tmp); + kdb_printf("%s ", tmpbuffer); else - kdb_printf("%s... ", p_tmp); - *(p_tmp + len) = '\0'; + kdb_printf("%s... ", tmpbuffer); + tmpbuffer[len] = '\0'; } if (i >= dtab_count) kdb_printf("..."); @@ -373,14 +368,14 @@ static char *kdb_read(char *buffer, size_t bufsize) kdb_position_cursor(kdb_prompt_str, buffer, cp); } else if (tab != 2 && count > 0) { /* How many new characters do we want from tmpbuffer? */ - len_tmp = strlen(p_tmp) - len; + len_tmp = strlen(tmpbuffer) - len; if (lastchar + len_tmp >= bufend) len_tmp = bufend - lastchar; if (len_tmp) { /* + 1 ensures the '\0' is memmove'd */ memmove(cp+len_tmp, cp, (lastchar-cp) + 1); - memcpy(cp, p_tmp+len, len_tmp); + memcpy(cp, tmpbuffer+len, len_tmp); kdb_printf("%s", cp); cp += len_tmp; lastchar += len_tmp; From b2aba15ad6f908d1a620fd97f6af5620c3639742 Mon Sep 17 00:00:00 2001 From: Daniel Thompson Date: Wed, 24 Apr 2024 15:21:41 +0100 Subject: [PATCH 0864/1702] serial: kgdboc: Fix NMI-safety problems from keyboard reset code Currently, when kdb is compiled with keyboard support, then we will use schedule_work() to provoke reset of the keyboard status. Unfortunately schedule_work() gets called from the kgdboc post-debug-exception handler. That risks deadlock since schedule_work() is not NMI-safe and, even on platforms where the NMI is not directly used for debugging, the debug trap can have NMI-like behaviour depending on where breakpoints are placed. Fix this by using the irq work system, which is NMI-safe, to defer the call to schedule_work() to a point when it is safe to call. Reported-by: Liuye Closes: https://lore.kernel.org/all/20240228025602.3087748-1-liu.yeC@h3c.com/ Cc: stable@vger.kernel.org Reviewed-by: Douglas Anderson Acked-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240424-kgdboc_fix_schedule_work-v2-1-50f5a490aec5@linaro.org Signed-off-by: Daniel Thompson --- drivers/tty/serial/kgdboc.c | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/drivers/tty/serial/kgdboc.c b/drivers/tty/serial/kgdboc.c index 7ce7bb1640054..58ea1e1391cee 100644 --- a/drivers/tty/serial/kgdboc.c +++ b/drivers/tty/serial/kgdboc.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -48,6 +49,25 @@ static struct kgdb_io kgdboc_earlycon_io_ops; static int (*earlycon_orig_exit)(struct console *con); #endif /* IS_BUILTIN(CONFIG_KGDB_SERIAL_CONSOLE) */ +/* + * When we leave the debug trap handler we need to reset the keyboard status + * (since the original keyboard state gets partially clobbered by kdb use of + * the keyboard). + * + * The path to deliver the reset is somewhat circuitous. + * + * To deliver the reset we register an input handler, reset the keyboard and + * then deregister the input handler. However, to get this done right, we do + * have to carefully manage the calling context because we can only register + * input handlers from task context. + * + * In particular we need to trigger the action from the debug trap handler with + * all its NMI and/or NMI-like oddities. To solve this the kgdboc trap exit code + * (the "post_exception" callback) uses irq_work_queue(), which is NMI-safe, to + * schedule a callback from a hardirq context. From there we have to defer the + * work again, this time using schedule_work(), to get a callback using the + * system workqueue, which runs in task context. + */ #ifdef CONFIG_KDB_KEYBOARD static int kgdboc_reset_connect(struct input_handler *handler, struct input_dev *dev, @@ -99,10 +119,17 @@ static void kgdboc_restore_input_helper(struct work_struct *dummy) static DECLARE_WORK(kgdboc_restore_input_work, kgdboc_restore_input_helper); +static void kgdboc_queue_restore_input_helper(struct irq_work *unused) +{ + schedule_work(&kgdboc_restore_input_work); +} + +static DEFINE_IRQ_WORK(kgdboc_restore_input_irq_work, kgdboc_queue_restore_input_helper); + static void kgdboc_restore_input(void) { if (likely(system_state == SYSTEM_RUNNING)) - schedule_work(&kgdboc_restore_input_work); + irq_work_queue(&kgdboc_restore_input_irq_work); } static int kgdboc_register_kbd(char **cptr) @@ -133,6 +160,7 @@ static void kgdboc_unregister_kbd(void) i--; } } + irq_work_sync(&kgdboc_restore_input_irq_work); flush_work(&kgdboc_restore_input_work); } #else /* ! CONFIG_KDB_KEYBOARD */ From e07606713a908eee8099883c8c830ebe33aaf748 Mon Sep 17 00:00:00 2001 From: Sudan Landge Date: Wed, 17 Apr 2024 12:40:42 +0200 Subject: [PATCH 0865/1702] virt: vmgenid: change implementation to use a platform driver Re-implement vmgenid as a platform driver in preparation for adding devicetree bindings support in next commits. Signed-off-by: Sudan Landge Reviewed-by: Alexander Graf Tested-by: Babis Chalios [Jason: - Small style cleanups and refactoring.] Signed-off-by: Jason A. Donenfeld --- drivers/virt/vmgenid.c | 99 +++++++++++++++++++++++++++--------------- 1 file changed, 65 insertions(+), 34 deletions(-) diff --git a/drivers/virt/vmgenid.c b/drivers/virt/vmgenid.c index a1c467a0e9f71..0522107f9beeb 100644 --- a/drivers/virt/vmgenid.c +++ b/drivers/virt/vmgenid.c @@ -7,9 +7,10 @@ * information to random.c. */ +#include #include #include -#include +#include #include ACPI_MODULE_NAME("vmgenid"); @@ -21,19 +22,41 @@ struct vmgenid_state { u8 this_id[VMGENID_SIZE]; }; -static int vmgenid_add(struct acpi_device *device) +static void vmgenid_notify(struct device *device) +{ + struct vmgenid_state *state = device->driver_data; + u8 old_id[VMGENID_SIZE]; + + memcpy(old_id, state->this_id, sizeof(old_id)); + memcpy(state->this_id, state->next_id, sizeof(state->this_id)); + if (!memcmp(old_id, state->this_id, sizeof(old_id))) + return; + add_vmfork_randomness(state->this_id, sizeof(state->this_id)); +} + +static void setup_vmgenid_state(struct vmgenid_state *state, void *virt_addr) { + state->next_id = virt_addr; + memcpy(state->this_id, state->next_id, sizeof(state->this_id)); + add_device_randomness(state->this_id, sizeof(state->this_id)); +} + +static void vmgenid_acpi_handler(acpi_handle __always_unused handle, + u32 __always_unused event, void *dev) +{ + vmgenid_notify(dev); +} + +static int vmgenid_add_acpi(struct device *dev, struct vmgenid_state *state) +{ + struct acpi_device *device = ACPI_COMPANION(dev); struct acpi_buffer parsed = { ACPI_ALLOCATE_BUFFER }; - struct vmgenid_state *state; union acpi_object *obj; phys_addr_t phys_addr; acpi_status status; + void *virt_addr; int ret = 0; - state = devm_kmalloc(&device->dev, sizeof(*state), GFP_KERNEL); - if (!state) - return -ENOMEM; - status = acpi_evaluate_object(device->handle, "ADDR", NULL, &parsed); if (ACPI_FAILURE(status)) { ACPI_EXCEPTION((AE_INFO, status, "Evaluating ADDR")); @@ -49,53 +72,61 @@ static int vmgenid_add(struct acpi_device *device) phys_addr = (obj->package.elements[0].integer.value << 0) | (obj->package.elements[1].integer.value << 32); - state->next_id = devm_memremap(&device->dev, phys_addr, VMGENID_SIZE, MEMREMAP_WB); - if (IS_ERR(state->next_id)) { - ret = PTR_ERR(state->next_id); + + virt_addr = devm_memremap(&device->dev, phys_addr, VMGENID_SIZE, MEMREMAP_WB); + if (IS_ERR(virt_addr)) { + ret = PTR_ERR(virt_addr); goto out; } + setup_vmgenid_state(state, virt_addr); - memcpy(state->this_id, state->next_id, sizeof(state->this_id)); - add_device_randomness(state->this_id, sizeof(state->this_id)); - - device->driver_data = state; + status = acpi_install_notify_handler(device->handle, ACPI_DEVICE_NOTIFY, + vmgenid_acpi_handler, dev); + if (ACPI_FAILURE(status)) { + ret = -ENODEV; + goto out; + } + dev->driver_data = state; out: ACPI_FREE(parsed.pointer); return ret; } -static void vmgenid_notify(struct acpi_device *device, u32 event) +static int vmgenid_add(struct platform_device *pdev) { - struct vmgenid_state *state = acpi_driver_data(device); - u8 old_id[VMGENID_SIZE]; + struct device *dev = &pdev->dev; + struct vmgenid_state *state; + int ret; - memcpy(old_id, state->this_id, sizeof(old_id)); - memcpy(state->this_id, state->next_id, sizeof(state->this_id)); - if (!memcmp(old_id, state->this_id, sizeof(old_id))) - return; - add_vmfork_randomness(state->this_id, sizeof(state->this_id)); + state = devm_kmalloc(dev, sizeof(*state), GFP_KERNEL); + if (!state) + return -ENOMEM; + + ret = vmgenid_add_acpi(dev, state); + + if (ret < 0) + devm_kfree(dev, state); + return ret; } -static const struct acpi_device_id vmgenid_ids[] = { +static const struct acpi_device_id vmgenid_acpi_ids[] = { { "VMGENCTR", 0 }, { "VM_GEN_COUNTER", 0 }, { } }; - -static struct acpi_driver vmgenid_driver = { - .name = "vmgenid", - .ids = vmgenid_ids, - .owner = THIS_MODULE, - .ops = { - .add = vmgenid_add, - .notify = vmgenid_notify - } +MODULE_DEVICE_TABLE(acpi, vmgenid_acpi_ids); + +static struct platform_driver vmgenid_plaform_driver = { + .probe = vmgenid_add, + .driver = { + .name = "vmgenid", + .acpi_match_table = vmgenid_acpi_ids, + }, }; -module_acpi_driver(vmgenid_driver); +module_platform_driver(vmgenid_plaform_driver) -MODULE_DEVICE_TABLE(acpi, vmgenid_ids); MODULE_DESCRIPTION("Virtual Machine Generation ID"); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Jason A. Donenfeld "); From a4aded1ff5759a2ab754699c0408be79c9ebdda4 Mon Sep 17 00:00:00 2001 From: Sudan Landge Date: Wed, 17 Apr 2024 12:40:45 +0200 Subject: [PATCH 0866/1702] dt-bindings: rng: Add vmgenid support Virtual Machine Generation ID driver was introduced in commit af6b54e2b5ba ("virt: vmgenid: notify RNG of VM fork and supply generation ID"), as an ACPI only device. VMGenID specification http://go.microsoft.com/fwlink/?LinkId=260709 defines a mechanism for the BIOS/hypervisors to communicate to the virtual machine that it is executed with a different configuration (e.g. snapshot execution or creation from a template). The guest operating system can use the notification for various purposes such as re-initializing its random number generator etc. As per the specs, hypervisor should provide a globally unique identified, or GUID via ACPI. This patch tries to mimic the mechanism to provide the same functionality which is for a hypervisor/BIOS to notify the virtual machine when it is executed with a different configuration. As part of this support the devicetree bindings requires the hypervisors or BIOS to provide a memory address which holds the GUID and an IRQ which is used to notify when there is a change in the GUID. The memory exposed in the DT should follow the rules defined in the vmgenid spec mentioned above. Reason for this change: Chosing ACPI or devicetree is an intrinsic part of an hypervisor design. Without going into details of why a hypervisor would chose DT over ACPI, we would like to highlight that the hypervisors that have chose devicetree and now want to make use of the vmgenid functionality cannot do so today because vmgenid is an ACPI only device. This forces these hypervisors to change their design which could have undesirable impacts on their use-cases, test-scenarios etc. The point of vmgenid is to provide a mechanism to discover a GUID when the execution state of a virtual machine changes and the simplest way to do it is pass a memory location and an interrupt via devicetree. It would complicate things unnecessarily if instead of using devicetree, we try to implement a new protocol or modify other protocols to somehow provide the same functionility. We believe that adding a devicetree binding for vmgenid is a simpler, better alternative to provide the same functionality and will allow such hypervisors as mentioned above to continue using devicetree. More references to the vmgenid specs are found below. Signed-off-by: Sudan Landge Reviewed-by: Rob Herring Reviewed-by: Alexander Graf Link: https://www.qemu.org/docs/master/specs/vmgenid.html Link: https://learn.microsoft.com/en-us/windows/win32/hyperv_v2/virtual-machine-generation-identifier Signed-off-by: Jason A. Donenfeld --- .../bindings/rng/microsoft,vmgenid.yaml | 49 +++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 50 insertions(+) create mode 100644 Documentation/devicetree/bindings/rng/microsoft,vmgenid.yaml diff --git a/Documentation/devicetree/bindings/rng/microsoft,vmgenid.yaml b/Documentation/devicetree/bindings/rng/microsoft,vmgenid.yaml new file mode 100644 index 0000000000000..8f20dee93e7ea --- /dev/null +++ b/Documentation/devicetree/bindings/rng/microsoft,vmgenid.yaml @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rng/microsoft,vmgenid.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Virtual Machine Generation ID + +maintainers: + - Jason A. Donenfeld + +description: + Firmwares or hypervisors can use this devicetree to describe an + interrupt and a shared resource to inject a Virtual Machine Generation ID. + Virtual Machine Generation ID is a globally unique identifier (GUID) and + the devicetree binding follows VMGenID specification defined in + http://go.microsoft.com/fwlink/?LinkId=260709. + +properties: + compatible: + const: microsoft,vmgenid + + reg: + description: + Specifies a 16-byte VMGenID in endianness-agnostic hexadecimal format. + maxItems: 1 + + interrupts: + description: + Interrupt used to notify that a new VMGenID is available. + maxItems: 1 + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + #include + rng@80000000 { + compatible = "microsoft,vmgenid"; + reg = <0x80000000 0x1000>; + interrupts = ; + }; + +... diff --git a/MAINTAINERS b/MAINTAINERS index a86685c57e129..c70c14d6c635d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -18467,6 +18467,7 @@ M: "Theodore Ts'o" M: Jason A. Donenfeld S: Maintained T: git https://git.kernel.org/pub/scm/linux/kernel/git/crng/random.git +F: Documentation/devicetree/bindings/rng/microsoft,vmgenid.yaml F: drivers/char/random.c F: drivers/virt/vmgenid.c From 7b1bcd6b50a615671d4dcb0ad0378a2660a1a368 Mon Sep 17 00:00:00 2001 From: Sudan Landge Date: Wed, 17 Apr 2024 12:40:46 +0200 Subject: [PATCH 0867/1702] virt: vmgenid: add support for devicetree bindings Extend the vmgenid platform driver to support devicetree bindings. With this support, hypervisors can send vmgenid notifications to the virtual machine without the need to enable ACPI. The bindings are located at: Documentation/devicetree/bindings/rng/microsoft,vmgenid.yaml Since this is no longer ACPI-dependent, remove the dependency from Kconfig and protect the ACPI code with a single ifdef. Signed-off-by: Sudan Landge Reviewed-by: Alexander Graf Tested-by: Babis Chalios [Jason: - Small style cleanups and refactoring. - Re-work ACPI conditionalization. ] Signed-off-by: Jason A. Donenfeld --- drivers/virt/Kconfig | 1 - drivers/virt/vmgenid.c | 53 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/drivers/virt/Kconfig b/drivers/virt/Kconfig index 40129b6f0eca4..d8c848cf09a6a 100644 --- a/drivers/virt/Kconfig +++ b/drivers/virt/Kconfig @@ -16,7 +16,6 @@ if VIRT_DRIVERS config VMGENID tristate "Virtual Machine Generation ID driver" default y - depends on ACPI help Say Y here to use the hypervisor-provided Virtual Machine Generation ID to reseed the RNG when the VM is cloned. This is highly recommended if diff --git a/drivers/virt/vmgenid.c b/drivers/virt/vmgenid.c index 0522107f9beeb..66135eac3abff 100644 --- a/drivers/virt/vmgenid.c +++ b/drivers/virt/vmgenid.c @@ -2,12 +2,13 @@ /* * Copyright (C) 2022 Jason A. Donenfeld . All Rights Reserved. * - * The "Virtual Machine Generation ID" is exposed via ACPI and changes when a + * The "Virtual Machine Generation ID" is exposed via ACPI or DT and changes when a * virtual machine forks or is cloned. This driver exists for shepherding that * information to random.c. */ #include +#include #include #include #include @@ -41,6 +42,7 @@ static void setup_vmgenid_state(struct vmgenid_state *state, void *virt_addr) add_device_randomness(state->this_id, sizeof(state->this_id)); } +#ifdef CONFIG_ACPI static void vmgenid_acpi_handler(acpi_handle __always_unused handle, u32 __always_unused event, void *dev) { @@ -92,6 +94,43 @@ static int vmgenid_add_acpi(struct device *dev, struct vmgenid_state *state) ACPI_FREE(parsed.pointer); return ret; } +#else +static int vmgenid_add_acpi(struct device *dev, struct vmgenid_state *state) +{ + return -EINVAL; +} +#endif + +static irqreturn_t vmgenid_of_irq_handler(int __always_unused irq, void *dev) +{ + vmgenid_notify(dev); + return IRQ_HANDLED; +} + +static int vmgenid_add_of(struct platform_device *pdev, + struct vmgenid_state *state) +{ + void *virt_addr; + int ret; + + virt_addr = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); + if (IS_ERR(virt_addr)) + return PTR_ERR(virt_addr); + + setup_vmgenid_state(state, virt_addr); + + ret = platform_get_irq(pdev, 0); + if (ret < 0) + return ret; + + ret = devm_request_irq(&pdev->dev, ret, vmgenid_of_irq_handler, + IRQF_SHARED, "vmgenid", &pdev->dev); + if (ret < 0) + return ret; + + pdev->dev.driver_data = state; + return 0; +} static int vmgenid_add(struct platform_device *pdev) { @@ -103,13 +142,22 @@ static int vmgenid_add(struct platform_device *pdev) if (!state) return -ENOMEM; - ret = vmgenid_add_acpi(dev, state); + if (dev->of_node) + ret = vmgenid_add_of(pdev, state); + else + ret = vmgenid_add_acpi(dev, state); if (ret < 0) devm_kfree(dev, state); return ret; } +static const struct of_device_id vmgenid_of_ids[] = { + { .compatible = "microsoft,vmgenid", }, + { }, +}; +MODULE_DEVICE_TABLE(of, vmgenid_of_ids); + static const struct acpi_device_id vmgenid_acpi_ids[] = { { "VMGENCTR", 0 }, { "VM_GEN_COUNTER", 0 }, @@ -122,6 +170,7 @@ static struct platform_driver vmgenid_plaform_driver = { .driver = { .name = "vmgenid", .acpi_match_table = vmgenid_acpi_ids, + .of_match_table = vmgenid_of_ids, }, }; From d06b1043644a1831ab141bbee2669002bba15b0f Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Wed, 20 Dec 2023 23:17:22 +0100 Subject: [PATCH 0868/1702] clk: qcom: clk-rcg: introduce support for multiple conf for same freq Some RCG frequency can be reached by multiple configuration. We currently declare multiple configuration for the same frequency but that is not supported and always the first configuration will be taken. These multiple configuration are needed as based on the current parent configuration, it may be needed to use a different configuration to reach the same frequency. To handle this introduce 3 new macro, C, FM and FMS: - C is used to declare a freq_conf where src, pre_div, m and n are provided. - FM is used to declare a freq_multi_tbl with the frequency and an array of confs to insert all the config for the provided frequency. - FMS is used to declare a freq_multi_tbl with the frequency and an array of a single conf with the provided src, pre_div, m and n. Struct clk_rcg2 is changed to add a union type to reference a simple freq_tbl or a complex freq_multi_tbl. Signed-off-by: Christian Marangi Acked-by: Stephen Boyd Link: https://lore.kernel.org/r/20231220221724.3822-2-ansuelsmth@gmail.com Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/clk-rcg.h | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/drivers/clk/qcom/clk-rcg.h b/drivers/clk/qcom/clk-rcg.h index e6d84c8c7989c..c50e6616d02c6 100644 --- a/drivers/clk/qcom/clk-rcg.h +++ b/drivers/clk/qcom/clk-rcg.h @@ -17,6 +17,23 @@ struct freq_tbl { u16 n; }; +#define C(s, h, m, n) { (s), (2 * (h) - 1), (m), (n) } +#define FM(f, confs) { (f), ARRAY_SIZE(confs), (confs) } +#define FMS(f, s, h, m, n) { (f), 1, (const struct freq_conf []){ C(s, h, m, n) } } + +struct freq_conf { + u8 src; + u8 pre_div; + u16 m; + u16 n; +}; + +struct freq_multi_tbl { + unsigned long freq; + size_t num_confs; + const struct freq_conf *confs; +}; + /** * struct mn - M/N:D counter * @mnctr_en_bit: bit to enable mn counter @@ -138,6 +155,7 @@ extern const struct clk_ops clk_dyn_rcg_ops; * @safe_src_index: safe src index value * @parent_map: map from software's parent index to hardware's src_sel field * @freq_tbl: frequency table + * @freq_multi_tbl: frequency table for clocks reachable with multiple RCGs conf * @clkr: regmap clock handle * @cfg_off: defines the cfg register offset from the CMD_RCGR + CFG_REG * @parked_cfg: cached value of the CFG register for parked RCGs @@ -149,7 +167,10 @@ struct clk_rcg2 { u8 hid_width; u8 safe_src_index; const struct parent_map *parent_map; - const struct freq_tbl *freq_tbl; + union { + const struct freq_tbl *freq_tbl; + const struct freq_multi_tbl *freq_multi_tbl; + }; struct clk_regmap clkr; u8 cfg_off; u32 parked_cfg; From 89da22456af0762477d8c1345fdd17961b3ada80 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Wed, 20 Dec 2023 23:17:23 +0100 Subject: [PATCH 0869/1702] clk: qcom: clk-rcg2: add support for rcg2 freq multi ops Some RCG frequency can be reached by multiple configuration. Add clk_rcg2_fm_ops ops to support these special RCG configurations. These alternative ops will select the frequency using a CEIL policy. When the correct frequency is found, the correct config is selected by calculating the final rate (by checking the defined parent and values in the config that is being checked) and deciding based on the one that is less different than the requested one. These check are skipped if there is just one config for the requested freq. qcom_find_freq_multi is added to search the freq with the new struct freq_multi_tbl. __clk_rcg2_select_conf is used to select the correct conf by simulating the final clock. If a conf can't be found due to parent not reachable, a WARN is printed and -EINVAL is returned. Tested-by: Wei Lei Signed-off-by: Christian Marangi Acked-by: Stephen Boyd Link: https://lore.kernel.org/r/20231220221724.3822-3-ansuelsmth@gmail.com Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/clk-rcg.h | 1 + drivers/clk/qcom/clk-rcg2.c | 166 ++++++++++++++++++++++++++++++++++++ drivers/clk/qcom/common.c | 18 ++++ drivers/clk/qcom/common.h | 2 + 4 files changed, 187 insertions(+) diff --git a/drivers/clk/qcom/clk-rcg.h b/drivers/clk/qcom/clk-rcg.h index c50e6616d02c6..d7414361e432e 100644 --- a/drivers/clk/qcom/clk-rcg.h +++ b/drivers/clk/qcom/clk-rcg.h @@ -190,6 +190,7 @@ struct clk_rcg2_gfx3d { extern const struct clk_ops clk_rcg2_ops; extern const struct clk_ops clk_rcg2_floor_ops; +extern const struct clk_ops clk_rcg2_fm_ops; extern const struct clk_ops clk_rcg2_mux_closest_ops; extern const struct clk_ops clk_edp_pixel_ops; extern const struct clk_ops clk_byte_ops; diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c index 5183c74b074f8..9b3aaa7f20ac2 100644 --- a/drivers/clk/qcom/clk-rcg2.c +++ b/drivers/clk/qcom/clk-rcg2.c @@ -260,6 +260,115 @@ static int _freq_tbl_determine_rate(struct clk_hw *hw, const struct freq_tbl *f, return 0; } +static const struct freq_conf * +__clk_rcg2_select_conf(struct clk_hw *hw, const struct freq_multi_tbl *f, + unsigned long req_rate) +{ + unsigned long rate_diff, best_rate_diff = ULONG_MAX; + const struct freq_conf *conf, *best_conf = NULL; + struct clk_rcg2 *rcg = to_clk_rcg2(hw); + const char *name = clk_hw_get_name(hw); + unsigned long parent_rate, rate; + struct clk_hw *p; + int index, i; + + /* Exit early if only one config is defined */ + if (f->num_confs == 1) { + best_conf = f->confs; + goto exit; + } + + /* Search in each provided config the one that is near the wanted rate */ + for (i = 0, conf = f->confs; i < f->num_confs; i++, conf++) { + index = qcom_find_src_index(hw, rcg->parent_map, conf->src); + if (index < 0) + continue; + + p = clk_hw_get_parent_by_index(hw, index); + if (!p) + continue; + + parent_rate = clk_hw_get_rate(p); + rate = calc_rate(parent_rate, conf->n, conf->m, conf->n, conf->pre_div); + + if (rate == req_rate) { + best_conf = conf; + goto exit; + } + + rate_diff = abs_diff(req_rate, rate); + if (rate_diff < best_rate_diff) { + best_rate_diff = rate_diff; + best_conf = conf; + } + } + + /* + * Very unlikely. Warn if we couldn't find a correct config + * due to parent not found in every config. + */ + if (unlikely(!best_conf)) { + WARN(1, "%s: can't find a configuration for rate %lu\n", + name, req_rate); + return ERR_PTR(-EINVAL); + } + +exit: + return best_conf; +} + +static int _freq_tbl_fm_determine_rate(struct clk_hw *hw, const struct freq_multi_tbl *f, + struct clk_rate_request *req) +{ + unsigned long clk_flags, rate = req->rate; + struct clk_rcg2 *rcg = to_clk_rcg2(hw); + const struct freq_conf *conf; + struct clk_hw *p; + int index; + + f = qcom_find_freq_multi(f, rate); + if (!f || !f->confs) + return -EINVAL; + + conf = __clk_rcg2_select_conf(hw, f, rate); + if (IS_ERR(conf)) + return PTR_ERR(conf); + index = qcom_find_src_index(hw, rcg->parent_map, conf->src); + if (index < 0) + return index; + + clk_flags = clk_hw_get_flags(hw); + p = clk_hw_get_parent_by_index(hw, index); + if (!p) + return -EINVAL; + + if (clk_flags & CLK_SET_RATE_PARENT) { + rate = f->freq; + if (conf->pre_div) { + if (!rate) + rate = req->rate; + rate /= 2; + rate *= conf->pre_div + 1; + } + + if (conf->n) { + u64 tmp = rate; + + tmp = tmp * conf->n; + do_div(tmp, conf->m); + rate = tmp; + } + } else { + rate = clk_hw_get_rate(p); + } + + req->best_parent_hw = p; + req->best_parent_rate = rate; + req->rate = f->freq; + + return 0; +} + static int clk_rcg2_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { @@ -276,6 +385,14 @@ static int clk_rcg2_determine_floor_rate(struct clk_hw *hw, return _freq_tbl_determine_rate(hw, rcg->freq_tbl, req, FLOOR); } +static int clk_rcg2_fm_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct clk_rcg2 *rcg = to_clk_rcg2(hw); + + return _freq_tbl_fm_determine_rate(hw, rcg->freq_multi_tbl, req); +} + static int __clk_rcg2_configure(struct clk_rcg2 *rcg, const struct freq_tbl *f, u32 *_cfg) { @@ -371,6 +488,30 @@ static int __clk_rcg2_set_rate(struct clk_hw *hw, unsigned long rate, return clk_rcg2_configure(rcg, f); } +static int __clk_rcg2_fm_set_rate(struct clk_hw *hw, unsigned long rate) +{ + struct clk_rcg2 *rcg = to_clk_rcg2(hw); + const struct freq_multi_tbl *f; + const struct freq_conf *conf; + struct freq_tbl f_tbl = {}; + + f = qcom_find_freq_multi(rcg->freq_multi_tbl, rate); + if (!f || !f->confs) + return -EINVAL; + + conf = __clk_rcg2_select_conf(hw, f, rate); + if (IS_ERR(conf)) + return PTR_ERR(conf); + + f_tbl.freq = f->freq; + f_tbl.src = conf->src; + f_tbl.pre_div = conf->pre_div; + f_tbl.m = conf->m; + f_tbl.n = conf->n; + + return clk_rcg2_configure(rcg, &f_tbl); +} + static int clk_rcg2_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { @@ -383,6 +524,12 @@ static int clk_rcg2_set_floor_rate(struct clk_hw *hw, unsigned long rate, return __clk_rcg2_set_rate(hw, rate, FLOOR); } +static int clk_rcg2_fm_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + return __clk_rcg2_fm_set_rate(hw, rate); +} + static int clk_rcg2_set_rate_and_parent(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate, u8 index) { @@ -395,6 +542,12 @@ static int clk_rcg2_set_floor_rate_and_parent(struct clk_hw *hw, return __clk_rcg2_set_rate(hw, rate, FLOOR); } +static int clk_rcg2_fm_set_rate_and_parent(struct clk_hw *hw, + unsigned long rate, unsigned long parent_rate, u8 index) +{ + return __clk_rcg2_fm_set_rate(hw, rate); +} + static int clk_rcg2_get_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) { struct clk_rcg2 *rcg = to_clk_rcg2(hw); @@ -505,6 +658,19 @@ const struct clk_ops clk_rcg2_floor_ops = { }; EXPORT_SYMBOL_GPL(clk_rcg2_floor_ops); +const struct clk_ops clk_rcg2_fm_ops = { + .is_enabled = clk_rcg2_is_enabled, + .get_parent = clk_rcg2_get_parent, + .set_parent = clk_rcg2_set_parent, + .recalc_rate = clk_rcg2_recalc_rate, + .determine_rate = clk_rcg2_fm_determine_rate, + .set_rate = clk_rcg2_fm_set_rate, + .set_rate_and_parent = clk_rcg2_fm_set_rate_and_parent, + .get_duty_cycle = clk_rcg2_get_duty_cycle, + .set_duty_cycle = clk_rcg2_set_duty_cycle, +}; +EXPORT_SYMBOL_GPL(clk_rcg2_fm_ops); + const struct clk_ops clk_rcg2_mux_closest_ops = { .determine_rate = __clk_mux_determine_rate_closest, .get_parent = clk_rcg2_get_parent, diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c index 75f09e6e057e1..48f81e3a5e802 100644 --- a/drivers/clk/qcom/common.c +++ b/drivers/clk/qcom/common.c @@ -41,6 +41,24 @@ struct freq_tbl *qcom_find_freq(const struct freq_tbl *f, unsigned long rate) } EXPORT_SYMBOL_GPL(qcom_find_freq); +const struct freq_multi_tbl *qcom_find_freq_multi(const struct freq_multi_tbl *f, + unsigned long rate) +{ + if (!f) + return NULL; + + if (!f->freq) + return f; + + for (; f->freq; f++) + if (rate <= f->freq) + return f; + + /* Default to our fastest rate */ + return f - 1; +} +EXPORT_SYMBOL_GPL(qcom_find_freq_multi); + const struct freq_tbl *qcom_find_freq_floor(const struct freq_tbl *f, unsigned long rate) { diff --git a/drivers/clk/qcom/common.h b/drivers/clk/qcom/common.h index 9c8f7b798d9fc..2d4a8a837e6ce 100644 --- a/drivers/clk/qcom/common.h +++ b/drivers/clk/qcom/common.h @@ -45,6 +45,8 @@ extern const struct freq_tbl *qcom_find_freq(const struct freq_tbl *f, unsigned long rate); extern const struct freq_tbl *qcom_find_freq_floor(const struct freq_tbl *f, unsigned long rate); +extern const struct freq_multi_tbl *qcom_find_freq_multi(const struct freq_multi_tbl *f, + unsigned long rate); extern void qcom_pll_set_fsm_mode(struct regmap *m, u32 reg, u8 bias_count, u8 lock_count); extern int qcom_find_src_index(struct clk_hw *hw, const struct parent_map *map, From e88f03230dc07aa3293b6aeb078bd27370bb2594 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Wed, 20 Dec 2023 23:17:24 +0100 Subject: [PATCH 0870/1702] clk: qcom: gcc-ipq8074: rework nss_port5/6 clock to multiple conf Rework nss_port5/6 to use the new multiple configuration implementation and correctly fix the clocks for these port under some corner case. This is particularly relevant for device that have 2.5G or 10G port connected to port5 or port 6 on ipq8074. As the parent are shared across multiple port it may be required to select the correct configuration to accomplish the desired clock. Without this patch such port doesn't work in some specific ethernet speed as the clock will be set to the wrong frequency as we just select the first configuration for the related frequency instead of selecting the best one. Signed-off-by: Christian Marangi Acked-by: Stephen Boyd Link: https://lore.kernel.org/r/20231220221724.3822-4-ansuelsmth@gmail.com Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/gcc-ipq8074.c | 120 +++++++++++++++++++++------------ 1 file changed, 76 insertions(+), 44 deletions(-) diff --git a/drivers/clk/qcom/gcc-ipq8074.c b/drivers/clk/qcom/gcc-ipq8074.c index 7bc679871f324..d2be56c5892d0 100644 --- a/drivers/clk/qcom/gcc-ipq8074.c +++ b/drivers/clk/qcom/gcc-ipq8074.c @@ -1677,15 +1677,23 @@ static struct clk_regmap_div nss_port4_tx_div_clk_src = { }, }; -static const struct freq_tbl ftbl_nss_port5_rx_clk_src[] = { - F(19200000, P_XO, 1, 0, 0), - F(25000000, P_UNIPHY1_RX, 12.5, 0, 0), - F(25000000, P_UNIPHY0_RX, 5, 0, 0), - F(78125000, P_UNIPHY1_RX, 4, 0, 0), - F(125000000, P_UNIPHY1_RX, 2.5, 0, 0), - F(125000000, P_UNIPHY0_RX, 1, 0, 0), - F(156250000, P_UNIPHY1_RX, 2, 0, 0), - F(312500000, P_UNIPHY1_RX, 1, 0, 0), +static const struct freq_conf ftbl_nss_port5_rx_clk_src_25[] = { + C(P_UNIPHY1_RX, 12.5, 0, 0), + C(P_UNIPHY0_RX, 5, 0, 0), +}; + +static const struct freq_conf ftbl_nss_port5_rx_clk_src_125[] = { + C(P_UNIPHY1_RX, 2.5, 0, 0), + C(P_UNIPHY0_RX, 1, 0, 0), +}; + +static const struct freq_multi_tbl ftbl_nss_port5_rx_clk_src[] = { + FMS(19200000, P_XO, 1, 0, 0), + FM(25000000, ftbl_nss_port5_rx_clk_src_25), + FMS(78125000, P_UNIPHY1_RX, 4, 0, 0), + FM(125000000, ftbl_nss_port5_rx_clk_src_125), + FMS(156250000, P_UNIPHY1_RX, 2, 0, 0), + FMS(312500000, P_UNIPHY1_RX, 1, 0, 0), { } }; @@ -1712,14 +1720,14 @@ gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias_map[] = { static struct clk_rcg2 nss_port5_rx_clk_src = { .cmd_rcgr = 0x68060, - .freq_tbl = ftbl_nss_port5_rx_clk_src, + .freq_multi_tbl = ftbl_nss_port5_rx_clk_src, .hid_width = 5, .parent_map = gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias_map, .clkr.hw.init = &(struct clk_init_data){ .name = "nss_port5_rx_clk_src", .parent_data = gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias, .num_parents = ARRAY_SIZE(gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias), - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_fm_ops, }, }; @@ -1739,15 +1747,23 @@ static struct clk_regmap_div nss_port5_rx_div_clk_src = { }, }; -static const struct freq_tbl ftbl_nss_port5_tx_clk_src[] = { - F(19200000, P_XO, 1, 0, 0), - F(25000000, P_UNIPHY1_TX, 12.5, 0, 0), - F(25000000, P_UNIPHY0_TX, 5, 0, 0), - F(78125000, P_UNIPHY1_TX, 4, 0, 0), - F(125000000, P_UNIPHY1_TX, 2.5, 0, 0), - F(125000000, P_UNIPHY0_TX, 1, 0, 0), - F(156250000, P_UNIPHY1_TX, 2, 0, 0), - F(312500000, P_UNIPHY1_TX, 1, 0, 0), +static const struct freq_conf ftbl_nss_port5_tx_clk_src_25[] = { + C(P_UNIPHY1_TX, 12.5, 0, 0), + C(P_UNIPHY0_TX, 5, 0, 0), +}; + +static const struct freq_conf ftbl_nss_port5_tx_clk_src_125[] = { + C(P_UNIPHY1_TX, 2.5, 0, 0), + C(P_UNIPHY0_TX, 1, 0, 0), +}; + +static const struct freq_multi_tbl ftbl_nss_port5_tx_clk_src[] = { + FMS(19200000, P_XO, 1, 0, 0), + FM(25000000, ftbl_nss_port5_tx_clk_src_25), + FMS(78125000, P_UNIPHY1_TX, 4, 0, 0), + FM(125000000, ftbl_nss_port5_tx_clk_src_125), + FMS(156250000, P_UNIPHY1_TX, 2, 0, 0), + FMS(312500000, P_UNIPHY1_TX, 1, 0, 0), { } }; @@ -1774,14 +1790,14 @@ gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias_map[] = { static struct clk_rcg2 nss_port5_tx_clk_src = { .cmd_rcgr = 0x68068, - .freq_tbl = ftbl_nss_port5_tx_clk_src, + .freq_multi_tbl = ftbl_nss_port5_tx_clk_src, .hid_width = 5, .parent_map = gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias_map, .clkr.hw.init = &(struct clk_init_data){ .name = "nss_port5_tx_clk_src", .parent_data = gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias, .num_parents = ARRAY_SIZE(gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias), - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_fm_ops, }, }; @@ -1801,15 +1817,23 @@ static struct clk_regmap_div nss_port5_tx_div_clk_src = { }, }; -static const struct freq_tbl ftbl_nss_port6_rx_clk_src[] = { - F(19200000, P_XO, 1, 0, 0), - F(25000000, P_UNIPHY2_RX, 5, 0, 0), - F(25000000, P_UNIPHY2_RX, 12.5, 0, 0), - F(78125000, P_UNIPHY2_RX, 4, 0, 0), - F(125000000, P_UNIPHY2_RX, 1, 0, 0), - F(125000000, P_UNIPHY2_RX, 2.5, 0, 0), - F(156250000, P_UNIPHY2_RX, 2, 0, 0), - F(312500000, P_UNIPHY2_RX, 1, 0, 0), +static const struct freq_conf ftbl_nss_port6_rx_clk_src_25[] = { + C(P_UNIPHY2_RX, 5, 0, 0), + C(P_UNIPHY2_RX, 12.5, 0, 0), +}; + +static const struct freq_conf ftbl_nss_port6_rx_clk_src_125[] = { + C(P_UNIPHY2_RX, 1, 0, 0), + C(P_UNIPHY2_RX, 2.5, 0, 0), +}; + +static const struct freq_multi_tbl ftbl_nss_port6_rx_clk_src[] = { + FMS(19200000, P_XO, 1, 0, 0), + FM(25000000, ftbl_nss_port6_rx_clk_src_25), + FMS(78125000, P_UNIPHY2_RX, 4, 0, 0), + FM(125000000, ftbl_nss_port6_rx_clk_src_125), + FMS(156250000, P_UNIPHY2_RX, 2, 0, 0), + FMS(312500000, P_UNIPHY2_RX, 1, 0, 0), { } }; @@ -1831,14 +1855,14 @@ static const struct parent_map gcc_xo_uniphy2_rx_tx_ubi32_bias_map[] = { static struct clk_rcg2 nss_port6_rx_clk_src = { .cmd_rcgr = 0x68070, - .freq_tbl = ftbl_nss_port6_rx_clk_src, + .freq_multi_tbl = ftbl_nss_port6_rx_clk_src, .hid_width = 5, .parent_map = gcc_xo_uniphy2_rx_tx_ubi32_bias_map, .clkr.hw.init = &(struct clk_init_data){ .name = "nss_port6_rx_clk_src", .parent_data = gcc_xo_uniphy2_rx_tx_ubi32_bias, .num_parents = ARRAY_SIZE(gcc_xo_uniphy2_rx_tx_ubi32_bias), - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_fm_ops, }, }; @@ -1858,15 +1882,23 @@ static struct clk_regmap_div nss_port6_rx_div_clk_src = { }, }; -static const struct freq_tbl ftbl_nss_port6_tx_clk_src[] = { - F(19200000, P_XO, 1, 0, 0), - F(25000000, P_UNIPHY2_TX, 5, 0, 0), - F(25000000, P_UNIPHY2_TX, 12.5, 0, 0), - F(78125000, P_UNIPHY2_TX, 4, 0, 0), - F(125000000, P_UNIPHY2_TX, 1, 0, 0), - F(125000000, P_UNIPHY2_TX, 2.5, 0, 0), - F(156250000, P_UNIPHY2_TX, 2, 0, 0), - F(312500000, P_UNIPHY2_TX, 1, 0, 0), +static const struct freq_conf ftbl_nss_port6_tx_clk_src_25[] = { + C(P_UNIPHY2_TX, 5, 0, 0), + C(P_UNIPHY2_TX, 12.5, 0, 0), +}; + +static const struct freq_conf ftbl_nss_port6_tx_clk_src_125[] = { + C(P_UNIPHY2_TX, 1, 0, 0), + C(P_UNIPHY2_TX, 2.5, 0, 0), +}; + +static const struct freq_multi_tbl ftbl_nss_port6_tx_clk_src[] = { + FMS(19200000, P_XO, 1, 0, 0), + FM(25000000, ftbl_nss_port6_tx_clk_src_25), + FMS(78125000, P_UNIPHY1_RX, 4, 0, 0), + FM(125000000, ftbl_nss_port6_tx_clk_src_125), + FMS(156250000, P_UNIPHY1_RX, 2, 0, 0), + FMS(312500000, P_UNIPHY1_RX, 1, 0, 0), { } }; @@ -1888,14 +1920,14 @@ static const struct parent_map gcc_xo_uniphy2_tx_rx_ubi32_bias_map[] = { static struct clk_rcg2 nss_port6_tx_clk_src = { .cmd_rcgr = 0x68078, - .freq_tbl = ftbl_nss_port6_tx_clk_src, + .freq_multi_tbl = ftbl_nss_port6_tx_clk_src, .hid_width = 5, .parent_map = gcc_xo_uniphy2_tx_rx_ubi32_bias_map, .clkr.hw.init = &(struct clk_init_data){ .name = "nss_port6_tx_clk_src", .parent_data = gcc_xo_uniphy2_tx_rx_ubi32_bias, .num_parents = ARRAY_SIZE(gcc_xo_uniphy2_tx_rx_ubi32_bias), - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_fm_ops, }, }; From 5fce38e2a1a97900989d9fedebcf5a4dacdaee30 Mon Sep 17 00:00:00 2001 From: Gabor Juhos Date: Fri, 15 Mar 2024 17:16:41 +0100 Subject: [PATCH 0871/1702] clk: qcom: apss-ipq-pll: use stromer ops for IPQ5018 to fix boot failure Booting v6.8 results in a hang on various IPQ5018 based boards. Investigating the problem showed that the hang happens when the clk_alpha_pll_stromer_plus_set_rate() function tries to write into the PLL_MODE register of the APSS PLL. Checking the downstream code revealed that it uses [1] stromer specific operations for IPQ5018, whereas in the current code the stromer plus specific operations are used. The ops in the 'ipq_pll_stromer_plus' clock definition can't be changed since that is needed for IPQ5332, so add a new alpha pll clock declaration which uses the correct stromer ops and use this new clock for IPQ5018 to avoid the boot failure. Also, change pll_type in 'ipq5018_pll_data' to CLK_ALPHA_PLL_TYPE_STROMER to better reflect that it is a Stromer PLL and change the apss_ipq_pll_probe() function accordingly. 1. https://git.codelinaro.org/clo/qsdk/oss/kernel/linux-ipq-5.4/-/blob/NHSS.QSDK.12.4/drivers/clk/qcom/apss-ipq5018.c#L67 Cc: stable@vger.kernel.org Fixes: 50492f929486 ("clk: qcom: apss-ipq-pll: add support for IPQ5018") Signed-off-by: Gabor Juhos Tested-by: Kathiravan Thirumoorthy Reviewed-by: Kathiravan Thirumoorthy Link: https://lore.kernel.org/r/20240315-apss-ipq-pll-ipq5018-hang-v2-1-6fe30ada2009@gmail.com Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/apss-ipq-pll.c | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/drivers/clk/qcom/apss-ipq-pll.c b/drivers/clk/qcom/apss-ipq-pll.c index 678b805f13d45..dfffec2f06ae7 100644 --- a/drivers/clk/qcom/apss-ipq-pll.c +++ b/drivers/clk/qcom/apss-ipq-pll.c @@ -55,6 +55,29 @@ static struct clk_alpha_pll ipq_pll_huayra = { }, }; +static struct clk_alpha_pll ipq_pll_stromer = { + .offset = 0x0, + /* + * Reuse CLK_ALPHA_PLL_TYPE_STROMER_PLUS register offsets. + * Although this is a bit confusing, but the offset values + * are correct nevertheless. + */ + .regs = ipq_pll_offsets[CLK_ALPHA_PLL_TYPE_STROMER_PLUS], + .flags = SUPPORTS_DYNAMIC_UPDATE, + .clkr = { + .enable_reg = 0x0, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data) { + .name = "a53pll", + .parent_data = &(const struct clk_parent_data) { + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_stromer_ops, + }, + }, +}; + static struct clk_alpha_pll ipq_pll_stromer_plus = { .offset = 0x0, .regs = ipq_pll_offsets[CLK_ALPHA_PLL_TYPE_STROMER_PLUS], @@ -144,8 +167,8 @@ struct apss_pll_data { }; static const struct apss_pll_data ipq5018_pll_data = { - .pll_type = CLK_ALPHA_PLL_TYPE_STROMER_PLUS, - .pll = &ipq_pll_stromer_plus, + .pll_type = CLK_ALPHA_PLL_TYPE_STROMER, + .pll = &ipq_pll_stromer, .pll_config = &ipq5018_pll_config, }; @@ -203,7 +226,8 @@ static int apss_ipq_pll_probe(struct platform_device *pdev) if (data->pll_type == CLK_ALPHA_PLL_TYPE_HUAYRA) clk_alpha_pll_configure(data->pll, regmap, data->pll_config); - else if (data->pll_type == CLK_ALPHA_PLL_TYPE_STROMER_PLUS) + else if (data->pll_type == CLK_ALPHA_PLL_TYPE_STROMER || + data->pll_type == CLK_ALPHA_PLL_TYPE_STROMER_PLUS) clk_stromer_pll_configure(data->pll, regmap, data->pll_config); ret = devm_clk_register_regmap(dev, &data->pll->clkr); From ac3ee364a7312517edf71b1857c0f272919f2374 Mon Sep 17 00:00:00 2001 From: Gabor Juhos Date: Thu, 28 Mar 2024 10:23:10 +0100 Subject: [PATCH 0872/1702] clk: qcom: apss-ipq-pll: reuse Stromer reg offsets from 'clk_alpha_pll_regs' The register offset array defined locally for the CLK_ALPHA_PLL_TYPE_STROMER_PLUS is the same as the entry defined for CLK_ALPHA_PLL_TYPE_STROMER in the 'clk_alpha_pll_regs' array. To avoid code duplication, remove the local definition and use the global one instead. No functional changes. Reviewed-by: Dmitry Baryshkov Reviewed-by: Konrad Dybcio Signed-off-by: Gabor Juhos Link: https://lore.kernel.org/r/20240328-apss-ipq-pll-cleanup-v4-1-eddbf617f0c8@gmail.com Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/apss-ipq-pll.c | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/drivers/clk/qcom/apss-ipq-pll.c b/drivers/clk/qcom/apss-ipq-pll.c index dfffec2f06ae7..ed3e6405f99cb 100644 --- a/drivers/clk/qcom/apss-ipq-pll.c +++ b/drivers/clk/qcom/apss-ipq-pll.c @@ -24,17 +24,6 @@ static const u8 ipq_pll_offsets[][PLL_OFF_MAX_REGS] = { [PLL_OFF_TEST_CTL] = 0x30, [PLL_OFF_TEST_CTL_U] = 0x34, }, - [CLK_ALPHA_PLL_TYPE_STROMER_PLUS] = { - [PLL_OFF_L_VAL] = 0x08, - [PLL_OFF_ALPHA_VAL] = 0x10, - [PLL_OFF_ALPHA_VAL_U] = 0x14, - [PLL_OFF_USER_CTL] = 0x18, - [PLL_OFF_USER_CTL_U] = 0x1c, - [PLL_OFF_CONFIG_CTL] = 0x20, - [PLL_OFF_STATUS] = 0x28, - [PLL_OFF_TEST_CTL] = 0x30, - [PLL_OFF_TEST_CTL_U] = 0x34, - }, }; static struct clk_alpha_pll ipq_pll_huayra = { @@ -57,12 +46,7 @@ static struct clk_alpha_pll ipq_pll_huayra = { static struct clk_alpha_pll ipq_pll_stromer = { .offset = 0x0, - /* - * Reuse CLK_ALPHA_PLL_TYPE_STROMER_PLUS register offsets. - * Although this is a bit confusing, but the offset values - * are correct nevertheless. - */ - .regs = ipq_pll_offsets[CLK_ALPHA_PLL_TYPE_STROMER_PLUS], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_STROMER], .flags = SUPPORTS_DYNAMIC_UPDATE, .clkr = { .enable_reg = 0x0, @@ -80,7 +64,11 @@ static struct clk_alpha_pll ipq_pll_stromer = { static struct clk_alpha_pll ipq_pll_stromer_plus = { .offset = 0x0, - .regs = ipq_pll_offsets[CLK_ALPHA_PLL_TYPE_STROMER_PLUS], + /* + * The register offsets of the Stromer Plus PLL used in IPQ5332 + * are the same as the Stromer PLL's offsets. + */ + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_STROMER], .flags = SUPPORTS_DYNAMIC_UPDATE, .clkr = { .enable_reg = 0x0, From 35a99c6ffa1e9bd8d31ddd3098e6fad7ba884928 Mon Sep 17 00:00:00 2001 From: Gabor Juhos Date: Thu, 28 Mar 2024 10:23:11 +0100 Subject: [PATCH 0873/1702] clk: qcom: apss-ipq-pll: move Huayra register map to 'clk_alpha_pll_regs' Move the locally defined Huayra register map to 'clk_alpha_pll_regs' in order to allow using that by other drivers, like the clk-cbf-8996. No functional changes. Reviewed-by: Dmitry Baryshkov Reviewed-by: Konrad Dybcio Signed-off-by: Gabor Juhos Link: https://lore.kernel.org/r/20240328-apss-ipq-pll-cleanup-v4-2-eddbf617f0c8@gmail.com Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/apss-ipq-pll.c | 20 +------------------- drivers/clk/qcom/clk-alpha-pll.c | 10 ++++++++++ drivers/clk/qcom/clk-alpha-pll.h | 1 + 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/drivers/clk/qcom/apss-ipq-pll.c b/drivers/clk/qcom/apss-ipq-pll.c index ed3e6405f99cb..8cf17374a2e2a 100644 --- a/drivers/clk/qcom/apss-ipq-pll.c +++ b/drivers/clk/qcom/apss-ipq-pll.c @@ -8,27 +8,9 @@ #include "clk-alpha-pll.h" -/* - * Even though APSS PLL type is of existing one (like Huayra), its offsets - * are different from the one mentioned in the clk-alpha-pll.c, since the - * PLL is specific to APSS, so lets the define the same. - */ -static const u8 ipq_pll_offsets[][PLL_OFF_MAX_REGS] = { - [CLK_ALPHA_PLL_TYPE_HUAYRA] = { - [PLL_OFF_L_VAL] = 0x08, - [PLL_OFF_ALPHA_VAL] = 0x10, - [PLL_OFF_USER_CTL] = 0x18, - [PLL_OFF_CONFIG_CTL] = 0x20, - [PLL_OFF_CONFIG_CTL_U] = 0x24, - [PLL_OFF_STATUS] = 0x28, - [PLL_OFF_TEST_CTL] = 0x30, - [PLL_OFF_TEST_CTL_U] = 0x34, - }, -}; - static struct clk_alpha_pll ipq_pll_huayra = { .offset = 0x0, - .regs = ipq_pll_offsets[CLK_ALPHA_PLL_TYPE_HUAYRA], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_HUAYRA_APSS], .flags = SUPPORTS_DYNAMIC_UPDATE, .clkr = { .enable_reg = 0x0, diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index 7f0ed5bd51e3d..236794cb3e501 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -83,6 +83,16 @@ const u8 clk_alpha_pll_regs[][PLL_OFF_MAX_REGS] = { [PLL_OFF_TEST_CTL_U] = 0x20, [PLL_OFF_STATUS] = 0x24, }, + [CLK_ALPHA_PLL_TYPE_HUAYRA_APSS] = { + [PLL_OFF_L_VAL] = 0x08, + [PLL_OFF_ALPHA_VAL] = 0x10, + [PLL_OFF_USER_CTL] = 0x18, + [PLL_OFF_CONFIG_CTL] = 0x20, + [PLL_OFF_CONFIG_CTL_U] = 0x24, + [PLL_OFF_STATUS] = 0x28, + [PLL_OFF_TEST_CTL] = 0x30, + [PLL_OFF_TEST_CTL_U] = 0x34, + }, [CLK_ALPHA_PLL_TYPE_BRAMMO] = { [PLL_OFF_L_VAL] = 0x04, [PLL_OFF_ALPHA_VAL] = 0x08, diff --git a/drivers/clk/qcom/clk-alpha-pll.h b/drivers/clk/qcom/clk-alpha-pll.h index 1bb2d031dc9fa..c7055b6c42f1d 100644 --- a/drivers/clk/qcom/clk-alpha-pll.h +++ b/drivers/clk/qcom/clk-alpha-pll.h @@ -15,6 +15,7 @@ enum { CLK_ALPHA_PLL_TYPE_DEFAULT, CLK_ALPHA_PLL_TYPE_HUAYRA, + CLK_ALPHA_PLL_TYPE_HUAYRA_APSS, CLK_ALPHA_PLL_TYPE_BRAMMO, CLK_ALPHA_PLL_TYPE_FABIA, CLK_ALPHA_PLL_TYPE_TRION, From 07570342d305d90c1c5dba6d336fb4c8301aef45 Mon Sep 17 00:00:00 2001 From: Gabor Juhos Date: Thu, 28 Mar 2024 10:23:12 +0100 Subject: [PATCH 0874/1702] clk: qcom: apss-ipq-pll: constify match data structures The match data structures are used only by the apss_ipq_pll_probe() function and are never modified so mark those as constant. No functional changes. Reviewed-by: Dmitry Baryshkov Reviewed-by: Konrad Dybcio Signed-off-by: Gabor Juhos Link: https://lore.kernel.org/r/20240328-apss-ipq-pll-cleanup-v4-3-eddbf617f0c8@gmail.com Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/apss-ipq-pll.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/clk/qcom/apss-ipq-pll.c b/drivers/clk/qcom/apss-ipq-pll.c index 8cf17374a2e2a..191b87df51bba 100644 --- a/drivers/clk/qcom/apss-ipq-pll.c +++ b/drivers/clk/qcom/apss-ipq-pll.c @@ -142,25 +142,25 @@ static const struct apss_pll_data ipq5018_pll_data = { .pll_config = &ipq5018_pll_config, }; -static struct apss_pll_data ipq5332_pll_data = { +static const struct apss_pll_data ipq5332_pll_data = { .pll_type = CLK_ALPHA_PLL_TYPE_STROMER_PLUS, .pll = &ipq_pll_stromer_plus, .pll_config = &ipq5332_pll_config, }; -static struct apss_pll_data ipq8074_pll_data = { +static const struct apss_pll_data ipq8074_pll_data = { .pll_type = CLK_ALPHA_PLL_TYPE_HUAYRA, .pll = &ipq_pll_huayra, .pll_config = &ipq8074_pll_config, }; -static struct apss_pll_data ipq6018_pll_data = { +static const struct apss_pll_data ipq6018_pll_data = { .pll_type = CLK_ALPHA_PLL_TYPE_HUAYRA, .pll = &ipq_pll_huayra, .pll_config = &ipq6018_pll_config, }; -static struct apss_pll_data ipq9574_pll_data = { +static const struct apss_pll_data ipq9574_pll_data = { .pll_type = CLK_ALPHA_PLL_TYPE_HUAYRA, .pll = &ipq_pll_huayra, .pll_config = &ipq9574_pll_config, From 4a941e436432104ce5ed75bf8c5ba2c8463b089c Mon Sep 17 00:00:00 2001 From: Gabor Juhos Date: Thu, 28 Mar 2024 10:23:13 +0100 Subject: [PATCH 0875/1702] clk: qcom: apss-ipq-pll: constify clk_init_data structures The clk_init_data structures are never modified, so add const qualifier to the ones where it is missing. No functional changes. Reviewed-by: Dmitry Baryshkov Reviewed-by: Konrad Dybcio Signed-off-by: Gabor Juhos Link: https://lore.kernel.org/r/20240328-apss-ipq-pll-cleanup-v4-4-eddbf617f0c8@gmail.com Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/apss-ipq-pll.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/clk/qcom/apss-ipq-pll.c b/drivers/clk/qcom/apss-ipq-pll.c index 191b87df51bba..1a6f4db253796 100644 --- a/drivers/clk/qcom/apss-ipq-pll.c +++ b/drivers/clk/qcom/apss-ipq-pll.c @@ -15,7 +15,7 @@ static struct clk_alpha_pll ipq_pll_huayra = { .clkr = { .enable_reg = 0x0, .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ + .hw.init = &(const struct clk_init_data) { .name = "a53pll", .parent_data = &(const struct clk_parent_data) { .fw_name = "xo", @@ -55,7 +55,7 @@ static struct clk_alpha_pll ipq_pll_stromer_plus = { .clkr = { .enable_reg = 0x0, .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ + .hw.init = &(const struct clk_init_data) { .name = "a53pll", .parent_data = &(const struct clk_parent_data) { .fw_name = "xo", From f3574392290d37647821d40f89136177b28e5f34 Mon Sep 17 00:00:00 2001 From: Gabor Juhos Date: Thu, 28 Mar 2024 10:23:14 +0100 Subject: [PATCH 0876/1702] clk: qcom: clk-cbf-8996: use HUAYRA_APSS register map for cbf_pll The register map used for 'cbf_pll' is the same as the one defined for the CLK_ALPHA_PLL_TYPE_HUAYRA_APSS indice in the 'clk_alpha_pll_regs' array. Drop the local register map and use the global one instead to reduce code duplication. No functional changes intended. Compile tested only. Suggested-by: Dmitry Baryshkov Reviewed-by: Dmitry Baryshkov Reviewed-by: Konrad Dybcio Signed-off-by: Gabor Juhos Link: https://lore.kernel.org/r/20240328-apss-ipq-pll-cleanup-v4-5-eddbf617f0c8@gmail.com Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/clk-cbf-8996.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/drivers/clk/qcom/clk-cbf-8996.c b/drivers/clk/qcom/clk-cbf-8996.c index fe24b4abeab48..76bf523431b85 100644 --- a/drivers/clk/qcom/clk-cbf-8996.c +++ b/drivers/clk/qcom/clk-cbf-8996.c @@ -41,17 +41,6 @@ enum { #define CBF_PLL_OFFSET 0xf000 -static const u8 cbf_pll_regs[PLL_OFF_MAX_REGS] = { - [PLL_OFF_L_VAL] = 0x08, - [PLL_OFF_ALPHA_VAL] = 0x10, - [PLL_OFF_USER_CTL] = 0x18, - [PLL_OFF_CONFIG_CTL] = 0x20, - [PLL_OFF_CONFIG_CTL_U] = 0x24, - [PLL_OFF_TEST_CTL] = 0x30, - [PLL_OFF_TEST_CTL_U] = 0x34, - [PLL_OFF_STATUS] = 0x28, -}; - static struct alpha_pll_config cbfpll_config = { .l = 72, .config_ctl_val = 0x200d4828, @@ -67,7 +56,7 @@ static struct alpha_pll_config cbfpll_config = { static struct clk_alpha_pll cbf_pll = { .offset = CBF_PLL_OFFSET, - .regs = cbf_pll_regs, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_HUAYRA_APSS], .flags = SUPPORTS_DYNAMIC_UPDATE | SUPPORTS_FSM_MODE, .clkr.hw.init = &(struct clk_init_data){ .name = "cbf_pll", From e801038a02ce1e8c652a0b668dd233a4ee48aeb7 Mon Sep 17 00:00:00 2001 From: Dmitry Baryshkov Date: Wed, 24 Apr 2024 04:39:29 +0300 Subject: [PATCH 0877/1702] clk: qcom: dispcc-sm8450: fix DisplayPort clocks On SM8450 DisplayPort link clocks use frequency tables inherited from the vendor kernel, it is not applicable in the upstream kernel. Drop frequency tables and use clk_byte2_ops for those clocks. This fixes frequency selection in the OPP core (which otherwise attempts to use invalid 810 KHz as DP link rate), also fixing the following message: msm-dp-display ae90000.displayport-controller: _opp_config_clk_single: failed to set clock rate: -22 Fixes: 16fb89f92ec4 ("clk: qcom: Add support for Display Clock Controller on SM8450") Reviewed-by: Neil Armstrong Reviewed-by: Konrad Dybcio Signed-off-by: Dmitry Baryshkov Link: https://lore.kernel.org/r/20240424-dispcc-dp-clocks-v2-1-b44038f3fa96@linaro.org Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/dispcc-sm8450.c | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/drivers/clk/qcom/dispcc-sm8450.c b/drivers/clk/qcom/dispcc-sm8450.c index 92e9c4e7b13dc..49bb4f58c3915 100644 --- a/drivers/clk/qcom/dispcc-sm8450.c +++ b/drivers/clk/qcom/dispcc-sm8450.c @@ -309,26 +309,17 @@ static struct clk_rcg2 disp_cc_mdss_dptx0_aux_clk_src = { }, }; -static const struct freq_tbl ftbl_disp_cc_mdss_dptx0_link_clk_src[] = { - F(162000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(270000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(540000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(810000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - { } -}; - static struct clk_rcg2 disp_cc_mdss_dptx0_link_clk_src = { .cmd_rcgr = 0x819c, .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx0_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -382,13 +373,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx1_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx1_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -442,13 +432,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx2_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx2_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -502,13 +491,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx3_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx3_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; From 1113501cfb46d5c0eb960f0a8a9f6c0f91dc6fb6 Mon Sep 17 00:00:00 2001 From: Dmitry Baryshkov Date: Wed, 24 Apr 2024 04:39:30 +0300 Subject: [PATCH 0878/1702] clk: qcom: dispcc-sm6350: fix DisplayPort clocks On SM6350 DisplayPort link clocks use frequency tables inherited from the vendor kernel, it is not applicable in the upstream kernel. Drop frequency tables and use clk_byte2_ops for those clocks. This fixes frequency selection in the OPP core (which otherwise attempts to use invalid 810 KHz as DP link rate), also fixing the following message: msm-dp-display ae90000.displayport-controller: _opp_config_clk_single: failed to set clock rate: -22 Fixes: 837519775f1d ("clk: qcom: Add display clock controller driver for SM6350") Reviewed-by: Neil Armstrong Tested-by: Luca Weiss Reviewed-by: Konrad Dybcio Signed-off-by: Dmitry Baryshkov Link: https://lore.kernel.org/r/20240424-dispcc-dp-clocks-v2-2-b44038f3fa96@linaro.org Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/dispcc-sm6350.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/drivers/clk/qcom/dispcc-sm6350.c b/drivers/clk/qcom/dispcc-sm6350.c index 839435362010e..e4b7464c4d0e9 100644 --- a/drivers/clk/qcom/dispcc-sm6350.c +++ b/drivers/clk/qcom/dispcc-sm6350.c @@ -221,26 +221,17 @@ static struct clk_rcg2 disp_cc_mdss_dp_crypto_clk_src = { }, }; -static const struct freq_tbl ftbl_disp_cc_mdss_dp_link_clk_src[] = { - F(162000, P_DP_PHY_PLL_LINK_CLK, 1, 0, 0), - F(270000, P_DP_PHY_PLL_LINK_CLK, 1, 0, 0), - F(540000, P_DP_PHY_PLL_LINK_CLK, 1, 0, 0), - F(810000, P_DP_PHY_PLL_LINK_CLK, 1, 0, 0), - { } -}; - static struct clk_rcg2 disp_cc_mdss_dp_link_clk_src = { .cmd_rcgr = 0x10f8, .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_0, - .freq_tbl = ftbl_disp_cc_mdss_dp_link_clk_src, .clkr.hw.init = &(struct clk_init_data){ .name = "disp_cc_mdss_dp_link_clk_src", .parent_data = disp_cc_parent_data_0, .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; From e90b5139da8465a15c3820b4b67ca9468dce93b4 Mon Sep 17 00:00:00 2001 From: Dmitry Baryshkov Date: Wed, 24 Apr 2024 04:39:31 +0300 Subject: [PATCH 0879/1702] clk: qcom: dispcc-sm8550: fix DisplayPort clocks On SM8550 DisplayPort link clocks use frequency tables inherited from the vendor kernel, it is not applicable in the upstream kernel. Drop frequency tables and use clk_byte2_ops for those clocks. This fixes frequency selection in the OPP core (which otherwise attempts to use invalid 810 KHz as DP link rate), also fixing the following message: msm-dp-display ae90000.displayport-controller: _opp_config_clk_single: failed to set clock rate: -22 Fixes: 90114ca11476 ("clk: qcom: add SM8550 DISPCC driver") Reviewed-by: Neil Armstrong Tested-by: Neil Armstrong # on SM8550-HDK Reviewed-by: Konrad Dybcio Signed-off-by: Dmitry Baryshkov Link: https://lore.kernel.org/r/20240424-dispcc-dp-clocks-v2-3-b44038f3fa96@linaro.org Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/dispcc-sm8550.c | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/drivers/clk/qcom/dispcc-sm8550.c b/drivers/clk/qcom/dispcc-sm8550.c index 3672c73ac11c6..38ecea805503d 100644 --- a/drivers/clk/qcom/dispcc-sm8550.c +++ b/drivers/clk/qcom/dispcc-sm8550.c @@ -345,26 +345,17 @@ static struct clk_rcg2 disp_cc_mdss_dptx0_aux_clk_src = { }, }; -static const struct freq_tbl ftbl_disp_cc_mdss_dptx0_link_clk_src[] = { - F(162000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(270000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(540000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(810000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - { } -}; - static struct clk_rcg2 disp_cc_mdss_dptx0_link_clk_src = { .cmd_rcgr = 0x8170, .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_7, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx0_link_clk_src", .parent_data = disp_cc_parent_data_7, .num_parents = ARRAY_SIZE(disp_cc_parent_data_7), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -418,13 +409,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx1_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx1_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -478,13 +468,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx2_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx2_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -538,13 +527,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx3_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(struct clk_init_data) { .name = "disp_cc_mdss_dptx3_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; From 615a292ee4d51303246278f3fa33cc38700fe00e Mon Sep 17 00:00:00 2001 From: Dmitry Baryshkov Date: Wed, 24 Apr 2024 04:39:32 +0300 Subject: [PATCH 0880/1702] clk: qcom: dispcc-sm8650: fix DisplayPort clocks On SM8650 DisplayPort link clocks use frequency tables inherited from the vendor kernel, it is not applicable in the upstream kernel. Drop frequency tables and use clk_byte2_ops for those clocks. This fixes frequency selection in the OPP core (which otherwise attempts to use invalid 810 KHz as DP link rate), also fixing the following message: msm-dp-display af54000.displayport-controller: _opp_config_clk_single: failed to set clock rate: -22 Fixes: 9e939f008338 ("clk: qcom: add the SM8650 Display Clock Controller driver") Reviewed-by: Neil Armstrong Tested-by: Neil Armstrong # on SM8650-HDK Reviewed-by: Konrad Dybcio Signed-off-by: Dmitry Baryshkov Link: https://lore.kernel.org/r/20240424-dispcc-dp-clocks-v2-4-b44038f3fa96@linaro.org Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/dispcc-sm8650.c | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/drivers/clk/qcom/dispcc-sm8650.c b/drivers/clk/qcom/dispcc-sm8650.c index 9539db0d91145..3eb64bcad487e 100644 --- a/drivers/clk/qcom/dispcc-sm8650.c +++ b/drivers/clk/qcom/dispcc-sm8650.c @@ -343,26 +343,17 @@ static struct clk_rcg2 disp_cc_mdss_dptx0_aux_clk_src = { }, }; -static const struct freq_tbl ftbl_disp_cc_mdss_dptx0_link_clk_src[] = { - F(162000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(270000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(540000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - F(810000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), - { } -}; - static struct clk_rcg2 disp_cc_mdss_dptx0_link_clk_src = { .cmd_rcgr = 0x8170, .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_7, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(const struct clk_init_data) { .name = "disp_cc_mdss_dptx0_link_clk_src", .parent_data = disp_cc_parent_data_7, .num_parents = ARRAY_SIZE(disp_cc_parent_data_7), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -416,13 +407,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx1_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(const struct clk_init_data) { .name = "disp_cc_mdss_dptx1_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -476,13 +466,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx2_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(const struct clk_init_data) { .name = "disp_cc_mdss_dptx2_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; @@ -536,13 +525,12 @@ static struct clk_rcg2 disp_cc_mdss_dptx3_link_clk_src = { .mnd_width = 0, .hid_width = 5, .parent_map = disp_cc_parent_map_3, - .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, .clkr.hw.init = &(const struct clk_init_data) { .name = "disp_cc_mdss_dptx3_link_clk_src", .parent_data = disp_cc_parent_data_3, .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_byte2_ops, }, }; From e20ae5ae9f0c843aded4f06f3d1cab7384789e92 Mon Sep 17 00:00:00 2001 From: Marc Gonzalez Date: Thu, 25 Apr 2024 17:07:07 +0200 Subject: [PATCH 0881/1702] clk: qcom: mmcc-msm8998: fix venus clock issue Right now, msm8998 video decoder (venus) is non-functional: $ time mpv --hwdec=v4l2m2m-copy --vd-lavc-software-fallback=no --vo=null --no-audio --untimed --length=30 --quiet demo-480.webm (+) Video --vid=1 (*) (vp9 854x480 29.970fps) Audio --aid=1 --alang=eng (*) (opus 2ch 48000Hz) [ffmpeg/video] vp9_v4l2m2m: output VIDIOC_REQBUFS failed: Connection timed out [ffmpeg/video] vp9_v4l2m2m: no v4l2 output context's buffers [ffmpeg/video] vp9_v4l2m2m: can't configure decoder Could not open codec. Software decoding fallback is disabled. Exiting... (Quit) Bryan O'Donoghue suggested the proper fix: - Set required register offsets in venus GDSC structs. - Set HW_CTRL flag. $ time mpv --hwdec=v4l2m2m-copy --vd-lavc-software-fallback=no --vo=null --no-audio --untimed --length=30 --quiet demo-480.webm (+) Video --vid=1 (*) (vp9 854x480 29.970fps) Audio --aid=1 --alang=eng (*) (opus 2ch 48000Hz) [ffmpeg/video] vp9_v4l2m2m: VIDIOC_G_FMT ioctl [ffmpeg/video] vp9_v4l2m2m: VIDIOC_G_FMT ioctl ... Using hardware decoding (v4l2m2m-copy). VO: [null] 854x480 nv12 Exiting... (End of file) real 0m3.315s user 0m1.277s sys 0m0.453s NOTES: GDSC = Globally Distributed Switch Controller Use same code as mmcc-msm8996 with: s/venus_gdsc/video_top_gdsc/ s/venus_core0_gdsc/video_subcore0_gdsc/ s/venus_core1_gdsc/video_subcore1_gdsc/ https://git.codelinaro.org/clo/la/kernel/msm-4.4/-/blob/caf_migration/kernel.lnx.4.4.r38-rel/include/dt-bindings/clock/msm-clocks-hwio-8996.h https://git.codelinaro.org/clo/la/kernel/msm-4.4/-/blob/caf_migration/kernel.lnx.4.4.r38-rel/include/dt-bindings/clock/msm-clocks-hwio-8998.h 0x1024 = MMSS_VIDEO GDSCR (undocumented) 0x1028 = MMSS_VIDEO_CORE_CBCR 0x1030 = MMSS_VIDEO_AHB_CBCR 0x1034 = MMSS_VIDEO_AXI_CBCR 0x1038 = MMSS_VIDEO_MAXI_CBCR 0x1040 = MMSS_VIDEO_SUBCORE0 GDSCR (undocumented) 0x1044 = MMSS_VIDEO_SUBCORE1 GDSCR (undocumented) 0x1048 = MMSS_VIDEO_SUBCORE0_CBCR 0x104c = MMSS_VIDEO_SUBCORE1_CBCR Fixes: d14b15b5931c2b ("clk: qcom: Add MSM8998 Multimedia Clock Controller (MMCC) driver") Reviewed-by: Bryan O'Donoghue Signed-off-by: Marc Gonzalez Reviewed-by: Jeffrey Hugo Link: https://lore.kernel.org/r/ff4e2e34-a677-4c39-8c29-83655c5512ae@freebox.fr Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/mmcc-msm8998.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/clk/qcom/mmcc-msm8998.c b/drivers/clk/qcom/mmcc-msm8998.c index 1180e48c687ac..275fb3b71ede4 100644 --- a/drivers/clk/qcom/mmcc-msm8998.c +++ b/drivers/clk/qcom/mmcc-msm8998.c @@ -2535,6 +2535,8 @@ static struct clk_branch vmem_ahb_clk = { static struct gdsc video_top_gdsc = { .gdscr = 0x1024, + .cxcs = (unsigned int []){ 0x1028, 0x1034, 0x1038 }, + .cxc_count = 3, .pd = { .name = "video_top", }, @@ -2543,20 +2545,26 @@ static struct gdsc video_top_gdsc = { static struct gdsc video_subcore0_gdsc = { .gdscr = 0x1040, + .cxcs = (unsigned int []){ 0x1048 }, + .cxc_count = 1, .pd = { .name = "video_subcore0", }, .parent = &video_top_gdsc.pd, .pwrsts = PWRSTS_OFF_ON, + .flags = HW_CTRL, }; static struct gdsc video_subcore1_gdsc = { .gdscr = 0x1044, + .cxcs = (unsigned int []){ 0x104c }, + .cxc_count = 1, .pd = { .name = "video_subcore1", }, .parent = &video_top_gdsc.pd, .pwrsts = PWRSTS_OFF_ON, + .flags = HW_CTRL, }; static struct gdsc mdss_gdsc = { From 487fa28fa8b60417642ac58e8beda6e2509d18f9 Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Sat, 27 Apr 2024 19:43:51 +0200 Subject: [PATCH 0882/1702] parisc: Define sigset_t in parisc uapi header The util-linux debian package fails to build on parisc, because sigset_t isn't defined in asm/signal.h when included from userspace. Move the sigset_t type from internal header to the uapi header to fix the build. Link: https://buildd.debian.org/status/fetch.php?pkg=util-linux&arch=hppa&ver=2.40-7&stamp=1714163443&raw=0 Signed-off-by: Helge Deller Cc: stable@vger.kernel.org # v6.0+ --- arch/parisc/include/asm/signal.h | 12 ------------ arch/parisc/include/uapi/asm/signal.h | 10 ++++++++++ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/arch/parisc/include/asm/signal.h b/arch/parisc/include/asm/signal.h index 715c96ba2ec81..e84883c6b4c7a 100644 --- a/arch/parisc/include/asm/signal.h +++ b/arch/parisc/include/asm/signal.h @@ -4,23 +4,11 @@ #include -#define _NSIG 64 -/* bits-per-word, where word apparently means 'long' not 'int' */ -#define _NSIG_BPW BITS_PER_LONG -#define _NSIG_WORDS (_NSIG / _NSIG_BPW) - # ifndef __ASSEMBLY__ /* Most things should be clean enough to redefine this at will, if care is taken to make libc match. */ -typedef unsigned long old_sigset_t; /* at least 32 bits */ - -typedef struct { - /* next_signal() assumes this is a long - no choice */ - unsigned long sig[_NSIG_WORDS]; -} sigset_t; - #include #endif /* !__ASSEMBLY */ diff --git a/arch/parisc/include/uapi/asm/signal.h b/arch/parisc/include/uapi/asm/signal.h index 8e4895c5ea5d3..40d7a574c5dd1 100644 --- a/arch/parisc/include/uapi/asm/signal.h +++ b/arch/parisc/include/uapi/asm/signal.h @@ -57,10 +57,20 @@ #include +#define _NSIG 64 +#define _NSIG_BPW (sizeof(unsigned long) * 8) +#define _NSIG_WORDS (_NSIG / _NSIG_BPW) + # ifndef __ASSEMBLY__ # include +typedef unsigned long old_sigset_t; /* at least 32 bits */ + +typedef struct { + unsigned long sig[_NSIG_WORDS]; +} sigset_t; + /* Avoid too many header ordering problems. */ struct siginfo; From bb712842a85d595525e72f0e378c143e620b3ea2 Mon Sep 17 00:00:00 2001 From: Zhang Yi Date: Thu, 25 Apr 2024 21:13:27 +0800 Subject: [PATCH 0883/1702] xfs: match lock mode in xfs_buffered_write_iomap_begin() Commit 1aa91d9c9933 ("xfs: Add async buffered write support") replace xfs_ilock(XFS_ILOCK_EXCL) with xfs_ilock_for_iomap() when locking the writing inode, and a new variable lockmode is used to indicate the lock mode. Although the lockmode should always be XFS_ILOCK_EXCL, it's still better to use this variable instead of useing XFS_ILOCK_EXCL directly when unlocking the inode. Fixes: 1aa91d9c9933 ("xfs: Add async buffered write support") Signed-off-by: Zhang Yi Reviewed-by: "Darrick J. Wong" Reviewed-by: Christoph Hellwig Signed-off-by: Chandan Babu R --- fs/xfs/xfs_iomap.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 9ce0f6b9df93e..555603ef4234e 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -1175,13 +1175,13 @@ xfs_buffered_write_iomap_begin( * them out if the write happens to fail. */ seq = xfs_iomap_inode_sequence(ip, IOMAP_F_NEW); - xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_iunlock(ip, lockmode); trace_xfs_iomap_alloc(ip, offset, count, allocfork, &imap); return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, IOMAP_F_NEW, seq); found_imap: seq = xfs_iomap_inode_sequence(ip, 0); - xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_iunlock(ip, lockmode); return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, 0, seq); found_cow: @@ -1191,17 +1191,17 @@ xfs_buffered_write_iomap_begin( if (error) goto out_unlock; seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED); - xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_iunlock(ip, lockmode); return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, IOMAP_F_SHARED, seq); } xfs_trim_extent(&cmap, offset_fsb, imap.br_startoff - offset_fsb); - xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_iunlock(ip, lockmode); return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, 0, seq); out_unlock: - xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_iunlock(ip, lockmode); return error; } From fc8d0ba0ff5fe4700fa02008b7751ec6b84b7677 Mon Sep 17 00:00:00 2001 From: Zhang Yi Date: Thu, 25 Apr 2024 21:13:28 +0800 Subject: [PATCH 0884/1702] xfs: make the seq argument to xfs_bmapi_convert_delalloc() optional Allow callers to pass a NULLL seq argument if they don't care about the fork sequence number. Signed-off-by: Zhang Yi Reviewed-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 6053f5e5c71ee..c9b1dfe639444 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4648,7 +4648,8 @@ xfs_bmapi_convert_delalloc( if (!isnullstartblock(bma.got.br_startblock)) { xfs_bmbt_to_iomap(ip, iomap, &bma.got, 0, flags, xfs_iomap_inode_sequence(ip, flags)); - *seq = READ_ONCE(ifp->if_seq); + if (seq) + *seq = READ_ONCE(ifp->if_seq); goto out_trans_cancel; } @@ -4699,7 +4700,8 @@ xfs_bmapi_convert_delalloc( ASSERT(!isnullstartblock(bma.got.br_startblock)); xfs_bmbt_to_iomap(ip, iomap, &bma.got, 0, flags, xfs_iomap_inode_sequence(ip, flags)); - *seq = READ_ONCE(ifp->if_seq); + if (seq) + *seq = READ_ONCE(ifp->if_seq); if (whichfork == XFS_COW_FORK) xfs_refcount_alloc_cow_extent(tp, bma.blkno, bma.length); From 2e08371a83f1c06fd85eea8cd37c87a224cc4cc4 Mon Sep 17 00:00:00 2001 From: Zhang Yi Date: Thu, 25 Apr 2024 21:13:29 +0800 Subject: [PATCH 0885/1702] xfs: make xfs_bmapi_convert_delalloc() to allocate the target offset Since xfs_bmapi_convert_delalloc() only attempts to allocate the entire delalloc extent and require multiple invocations to allocate the target offset. So xfs_convert_blocks() add a loop to do this job and we call it in the write back path, but xfs_convert_blocks() isn't a common helper. Let's do it in xfs_bmapi_convert_delalloc() and drop xfs_convert_blocks(), preparing for the post EOF delalloc blocks converting in the buffered write begin path. Signed-off-by: Zhang Yi Reviewed-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 34 +++++++++++++++++++++++-- fs/xfs/xfs_aops.c | 54 +++++++++++----------------------------- 2 files changed, 46 insertions(+), 42 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index c9b1dfe639444..49e2fcad593df 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4590,8 +4590,8 @@ xfs_bmapi_write( * invocations to allocate the target offset if a large enough physical extent * is not available. */ -int -xfs_bmapi_convert_delalloc( +static int +xfs_bmapi_convert_one_delalloc( struct xfs_inode *ip, int whichfork, xfs_off_t offset, @@ -4724,6 +4724,36 @@ xfs_bmapi_convert_delalloc( return error; } +/* + * Pass in a dellalloc extent and convert it to real extents, return the real + * extent that maps offset_fsb in iomap. + */ +int +xfs_bmapi_convert_delalloc( + struct xfs_inode *ip, + int whichfork, + loff_t offset, + struct iomap *iomap, + unsigned int *seq) +{ + int error; + + /* + * Attempt to allocate whatever delalloc extent currently backs offset + * and put the result into iomap. Allocate in a loop because it may + * take several attempts to allocate real blocks for a contiguous + * delalloc extent if free space is sufficiently fragmented. + */ + do { + error = xfs_bmapi_convert_one_delalloc(ip, whichfork, offset, + iomap, seq); + if (error) + return error; + } while (iomap->offset + iomap->length <= offset); + + return 0; +} + int xfs_bmapi_remap( struct xfs_trans *tp, diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index 3f428620ebf2a..f7520f375ceee 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -233,45 +233,6 @@ xfs_imap_valid( return true; } -/* - * Pass in a dellalloc extent and convert it to real extents, return the real - * extent that maps offset_fsb in wpc->iomap. - * - * The current page is held locked so nothing could have removed the block - * backing offset_fsb, although it could have moved from the COW to the data - * fork by another thread. - */ -static int -xfs_convert_blocks( - struct iomap_writepage_ctx *wpc, - struct xfs_inode *ip, - int whichfork, - loff_t offset) -{ - int error; - unsigned *seq; - - if (whichfork == XFS_COW_FORK) - seq = &XFS_WPC(wpc)->cow_seq; - else - seq = &XFS_WPC(wpc)->data_seq; - - /* - * Attempt to allocate whatever delalloc extent currently backs offset - * and put the result into wpc->iomap. Allocate in a loop because it - * may take several attempts to allocate real blocks for a contiguous - * delalloc extent if free space is sufficiently fragmented. - */ - do { - error = xfs_bmapi_convert_delalloc(ip, whichfork, offset, - &wpc->iomap, seq); - if (error) - return error; - } while (wpc->iomap.offset + wpc->iomap.length <= offset); - - return 0; -} - static int xfs_map_blocks( struct iomap_writepage_ctx *wpc, @@ -290,6 +251,7 @@ xfs_map_blocks( struct xfs_iext_cursor icur; int retries = 0; int error = 0; + unsigned int *seq; if (xfs_is_shutdown(mp)) return -EIO; @@ -387,7 +349,19 @@ xfs_map_blocks( trace_xfs_map_blocks_found(ip, offset, count, whichfork, &imap); return 0; allocate_blocks: - error = xfs_convert_blocks(wpc, ip, whichfork, offset); + /* + * Convert a dellalloc extent to a real one. The current page is held + * locked so nothing could have removed the block backing offset_fsb, + * although it could have moved from the COW to the data fork by another + * thread. + */ + if (whichfork == XFS_COW_FORK) + seq = &XFS_WPC(wpc)->cow_seq; + else + seq = &XFS_WPC(wpc)->data_seq; + + error = xfs_bmapi_convert_delalloc(ip, whichfork, offset, + &wpc->iomap, seq); if (error) { /* * If we failed to find the extent in the COW fork we might have From 5ce5674187c345dc31534d2024c09ad8ef29b7ba Mon Sep 17 00:00:00 2001 From: Zhang Yi Date: Thu, 25 Apr 2024 21:13:30 +0800 Subject: [PATCH 0886/1702] xfs: convert delayed extents to unwritten when zeroing post eof blocks Current clone operation could be non-atomic if the destination of a file is beyond EOF, user could get a file with corrupted (zeroed) data on crash. The problem is about preallocations. If you write some data into a file: [A...B) and XFS decides to preallocate some post-eof blocks, then it can create a delayed allocation reservation: [A.........D) The writeback path tries to convert delayed extents to real ones by allocating blocks. If there aren't enough contiguous free space, we can end up with two extents, the first real and the second still delalloc: [A....C)[C.D) After that, both the in-memory and the on-disk file sizes are still B. If we clone into the range [E...F) from another file: [A....C)[C.D) [E...F) then xfs_reflink_zero_posteof() calls iomap_zero_range() to zero out the range [B, E) beyond EOF and flush it. Since [C, D) is still a delalloc extent, its pagecache will be zeroed and both the in-memory and on-disk size will be updated to D after flushing but before cloning. This is wrong, because the user can see the size change and read the zeroes while the clone operation is ongoing. We need to keep the in-memory and on-disk size before the clone operation starts, so instead of writing zeroes through the page cache for delayed ranges beyond EOF, we convert these ranges to unwritten and invalidate any cached data over that range beyond EOF. Suggested-by: Dave Chinner Signed-off-by: Zhang Yi Reviewed-by: "Darrick J. Wong" Reviewed-by: Christoph Hellwig Signed-off-by: Chandan Babu R --- fs/xfs/xfs_iomap.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 555603ef4234e..ec43aa7cec603 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -1039,6 +1039,24 @@ xfs_buffered_write_iomap_begin( goto out_unlock; } + /* + * For zeroing, trim a delalloc extent that extends beyond the EOF + * block. If it starts beyond the EOF block, convert it to an + * unwritten extent. + */ + if ((flags & IOMAP_ZERO) && imap.br_startoff <= offset_fsb && + isnullstartblock(imap.br_startblock)) { + xfs_fileoff_t eof_fsb = XFS_B_TO_FSB(mp, XFS_ISIZE(ip)); + + if (offset_fsb >= eof_fsb) + goto convert_delay; + if (end_fsb > eof_fsb) { + end_fsb = eof_fsb; + xfs_trim_extent(&imap, offset_fsb, + end_fsb - offset_fsb); + } + } + /* * Search the COW fork extent list even if we did not find a data fork * extent. This serves two purposes: first this implements the @@ -1184,6 +1202,17 @@ xfs_buffered_write_iomap_begin( xfs_iunlock(ip, lockmode); return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, 0, seq); +convert_delay: + xfs_iunlock(ip, lockmode); + truncate_pagecache(inode, offset); + error = xfs_bmapi_convert_delalloc(ip, XFS_DATA_FORK, offset, + iomap, NULL); + if (error) + return error; + + trace_xfs_iomap_alloc(ip, offset, count, XFS_DATA_FORK, &imap); + return 0; + found_cow: seq = xfs_iomap_inode_sequence(ip, 0); if (imap.br_startoff <= offset_fsb) { From f847e840157b91a490a13df78c4a6d4e5700ba0a Mon Sep 17 00:00:00 2001 From: Michael Margolin Date: Thu, 25 Apr 2024 17:18:14 +0000 Subject: [PATCH 0887/1702] RDMA/efa: Add shutdown notifier Add driver function to stop the device and release any active IRQs as preparation for shutdown. This should fix issues caused by unexpected AQ interrupts when booting kernel using kexec and possible data integrity issues when the system is being shutdown during traffic. Link: https://lore.kernel.org/r/20240425171814.25216-1-mrgolin@amazon.com Reviewed-by: Firas Jahjah Reviewed-by: Yonatan Nachum Signed-off-by: Michael Margolin Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/efa/efa_main.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index 5fa3603c80d82..d1a48f988f6c0 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -671,11 +671,22 @@ static void efa_remove(struct pci_dev *pdev) efa_remove_device(pdev); } +static void efa_shutdown(struct pci_dev *pdev) +{ + struct efa_dev *dev = pci_get_drvdata(pdev); + + efa_destroy_eqs(dev); + efa_com_dev_reset(&dev->edev, EFA_REGS_RESET_SHUTDOWN); + efa_free_irq(dev, &dev->admin_irq); + efa_disable_msix(dev); +} + static struct pci_driver efa_pci_driver = { .name = DRV_MODULE_NAME, .id_table = efa_pci_tbl, .probe = efa_probe, .remove = efa_remove, + .shutdown = efa_shutdown, }; module_pci_driver(efa_pci_driver); From 0f373e6d91b9dd75317d05a21abd999783cf70da Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sun, 14 Apr 2024 11:28:13 +0200 Subject: [PATCH 0888/1702] intel_th: remove usage of the deprecated ida_simple_xx() API ida_alloc() and ida_free() should be preferred to the deprecated ida_simple_get() and ida_simple_remove(). This is less verbose. Link: https://lkml.kernel.org/r/2aca50a9d061faecfd4ded80b5874cd3be9b855d.1713086613.git.christophe.jaillet@wanadoo.fr Signed-off-by: Christophe JAILLET Cc: Alexander Shishkin Cc: Greg Kroah-Hartman Signed-off-by: Andrew Morton --- drivers/hwtracing/intel_th/core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/hwtracing/intel_th/core.c b/drivers/hwtracing/intel_th/core.c index cc7f879bb175d..86c8efecd7c26 100644 --- a/drivers/hwtracing/intel_th/core.c +++ b/drivers/hwtracing/intel_th/core.c @@ -871,7 +871,7 @@ intel_th_alloc(struct device *dev, const struct intel_th_drvdata *drvdata, if (!th) return ERR_PTR(-ENOMEM); - th->id = ida_simple_get(&intel_th_ida, 0, 0, GFP_KERNEL); + th->id = ida_alloc(&intel_th_ida, GFP_KERNEL); if (th->id < 0) { err = th->id; goto err_alloc; @@ -931,7 +931,7 @@ intel_th_alloc(struct device *dev, const struct intel_th_drvdata *drvdata, "intel_th/output"); err_ida: - ida_simple_remove(&intel_th_ida, th->id); + ida_free(&intel_th_ida, th->id); err_alloc: kfree(th); @@ -964,7 +964,7 @@ void intel_th_free(struct intel_th *th) __unregister_chrdev(th->major, 0, TH_POSSIBLE_OUTPUTS, "intel_th/output"); - ida_simple_remove(&intel_th_ida, th->id); + ida_free(&intel_th_ida, th->id); kfree(th); } From 55dbc5b5174d0e7d1fa397d05aa4cb145e8b887e Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sun, 14 Apr 2024 12:10:17 +0200 Subject: [PATCH 0889/1702] pps: remove usage of the deprecated ida_simple_xx() API ida_alloc() and ida_free() should be preferred to the deprecated ida_simple_get() and ida_simple_remove(). This is less verbose. Link: https://lkml.kernel.org/r/9f681747d446b874952a892491387d79ffe565a9.1713089394.git.christophe.jaillet@wanadoo.fr Signed-off-by: Christophe JAILLET Cc: Rodolfo Giometti Cc: Greg Kroah-Hartman Signed-off-by: Andrew Morton --- drivers/pps/clients/pps_parport.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/pps/clients/pps_parport.c b/drivers/pps/clients/pps_parport.c index 42f93d4c6ee32..af972cdc04b53 100644 --- a/drivers/pps/clients/pps_parport.c +++ b/drivers/pps/clients/pps_parport.c @@ -148,7 +148,7 @@ static void parport_attach(struct parport *port) return; } - index = ida_simple_get(&pps_client_index, 0, 0, GFP_KERNEL); + index = ida_alloc(&pps_client_index, GFP_KERNEL); memset(&pps_client_cb, 0, sizeof(pps_client_cb)); pps_client_cb.private = device; pps_client_cb.irq_func = parport_irq; @@ -188,7 +188,7 @@ static void parport_attach(struct parport *port) err_unregister_dev: parport_unregister_device(device->pardev); err_free: - ida_simple_remove(&pps_client_index, index); + ida_free(&pps_client_index, index); kfree(device); } @@ -208,7 +208,7 @@ static void parport_detach(struct parport *port) pps_unregister_source(device->pps); parport_release(pardev); parport_unregister_device(pardev); - ida_simple_remove(&pps_client_index, device->index); + ida_free(&pps_client_index, device->index); kfree(device); } From 200a289b342bae6cbeb0b7406b2af55feef0de94 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sun, 14 Apr 2024 12:12:52 +0200 Subject: [PATCH 0890/1702] mux: remove usage of the deprecated ida_simple_xx() API ida_alloc() and ida_free() should be preferred to the deprecated ida_simple_get() and ida_simple_remove(). This is less verbose. Link: https://lkml.kernel.org/r/f82e013abe4c71f1c7d06819f96472f298acdcf3.1713089554.git.christophe.jaillet@wanadoo.fr Signed-off-by: Christophe JAILLET Cc: Peter Rosin Cc: Greg Kroah-Hartman Signed-off-by: Andrew Morton --- drivers/mux/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/mux/core.c b/drivers/mux/core.c index 7758161129320..78c0022697ec7 100644 --- a/drivers/mux/core.c +++ b/drivers/mux/core.c @@ -64,7 +64,7 @@ static void mux_chip_release(struct device *dev) { struct mux_chip *mux_chip = to_mux_chip(dev); - ida_simple_remove(&mux_ida, mux_chip->id); + ida_free(&mux_ida, mux_chip->id); kfree(mux_chip); } @@ -111,7 +111,7 @@ struct mux_chip *mux_chip_alloc(struct device *dev, mux_chip->dev.of_node = dev->of_node; dev_set_drvdata(&mux_chip->dev, mux_chip); - mux_chip->id = ida_simple_get(&mux_ida, 0, 0, GFP_KERNEL); + mux_chip->id = ida_alloc(&mux_ida, GFP_KERNEL); if (mux_chip->id < 0) { int err = mux_chip->id; From 1891e4d487554ef66350308c3603ed07b6557f17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Draszik?= Date: Fri, 26 Apr 2024 11:03:07 +0100 Subject: [PATCH 0891/1702] clk: samsung: gs101: add support for cmu_hsi0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CMU_HSI0 is the clock management unit for one of the high speed interfaces, which is used (amongst others) for USB Some notes about the clocks marked as CLK_IGNORE_UNUSED: * CLK_GOUT_HSI0_PCLK CLK_GOUT_HSI0_LHM_AXI_P_HSI0_I_CLK CLK_GOUT_HSI0_XIU_P_HSI0_ACLK need to be kept running as otherwise the system becomes unresponsive and it doesn't complete booting. * CLK_GOUT_HSI0_LHS_ACEL_D_HSI0_I_CLK CLK_GOUT_HSI0_SSMT_USB_ACLK CLK_GOUT_HSI0_SSMT_USB_PCLK CLK_GOUT_HSI0_SYSMMU_USB_CLK_S2 CLK_GOUT_HSI0_XIU_D0_HSI0_ACLK CLK_GOUT_HSI0_XIU_D1_HSI0_ACLK are needed for USB to come up properly (SSMT is for (secure) memory tagging). While at the moment we only support booting with the clk_ignore_unused kernel command line paramenter, it's still worthwhile to explicitly mark those clocks. While the usual (sed) script has been used to derive the linux clock names from the data sheet, one manual tweak was applied to fix a typo coming from the data sheet which we don't want to carry: hsi0_uspdpdbg_user -> hsi0_usbdpdbg_user (note usb vs usp). Signed-off-by: André Draszik Link: https://lore.kernel.org/r/20240426-hsi0-gs101-v2-4-2157da8b63e3@linaro.org Signed-off-by: Krzysztof Kozlowski --- drivers/clk/samsung/clk-gs101.c | 469 ++++++++++++++++++++++++++++++++ 1 file changed, 469 insertions(+) diff --git a/drivers/clk/samsung/clk-gs101.c b/drivers/clk/samsung/clk-gs101.c index bd3c1b02715b5..ce0ff35f45480 100644 --- a/drivers/clk/samsung/clk-gs101.c +++ b/drivers/clk/samsung/clk-gs101.c @@ -15,10 +15,12 @@ #include "clk.h" #include "clk-exynos-arm64.h" +#include "clk-pll.h" /* NOTE: Must be equal to the last clock ID increased by one */ #define CLKS_NR_TOP (CLK_GOUT_CMU_TPU_UART + 1) #define CLKS_NR_APM (CLK_APM_PLL_DIV16_APM + 1) +#define CLKS_NR_HSI0 (CLK_GOUT_HSI0_XIU_P_HSI0_ACLK + 1) #define CLKS_NR_MISC (CLK_GOUT_MISC_XIU_D_MISC_ACLK + 1) #define CLKS_NR_PERIC0 (CLK_GOUT_PERIC0_SYSREG_PERIC0_PCLK + 1) #define CLKS_NR_PERIC1 (CLK_GOUT_PERIC1_SYSREG_PERIC1_PCLK + 1) @@ -1919,6 +1921,470 @@ static const struct samsung_cmu_info apm_cmu_info __initconst = { .nr_clk_regs = ARRAY_SIZE(apm_clk_regs), }; +/* ---- CMU_HSI0 ------------------------------------------------------------ */ + +/* Register Offset definitions for CMU_HSI0 (0x11000000) */ +#define PLL_LOCKTIME_PLL_USB 0x0004 +#define PLL_CON0_PLL_USB 0x0140 +#define PLL_CON1_PLL_USB 0x0144 +#define PLL_CON2_PLL_USB 0x0148 +#define PLL_CON3_PLL_USB 0x014c +#define PLL_CON4_PLL_USB 0x0150 +#define PLL_CON0_MUX_CLKCMU_HSI0_ALT_USER 0x0600 +#define PLL_CON1_MUX_CLKCMU_HSI0_ALT_USER 0x0604 +#define PLL_CON0_MUX_CLKCMU_HSI0_BUS_USER 0x0610 +#define PLL_CON1_MUX_CLKCMU_HSI0_BUS_USER 0x0614 +#define PLL_CON0_MUX_CLKCMU_HSI0_DPGTC_USER 0x0620 +#define PLL_CON1_MUX_CLKCMU_HSI0_DPGTC_USER 0x0624 +#define PLL_CON0_MUX_CLKCMU_HSI0_TCXO_USER 0x0630 +#define PLL_CON1_MUX_CLKCMU_HSI0_TCXO_USER 0x0634 +#define PLL_CON0_MUX_CLKCMU_HSI0_USB20_USER 0x0640 +#define PLL_CON1_MUX_CLKCMU_HSI0_USB20_USER 0x0644 +#define PLL_CON0_MUX_CLKCMU_HSI0_USB31DRD_USER 0x0650 +#define PLL_CON1_MUX_CLKCMU_HSI0_USB31DRD_USER 0x0654 +#define PLL_CON0_MUX_CLKCMU_HSI0_USPDPDBG_USER 0x0660 +#define PLL_CON1_MUX_CLKCMU_HSI0_USPDPDBG_USER 0x0664 +#define HSI0_CMU_HSI0_CONTROLLER_OPTION 0x0800 +#define CLKOUT_CON_BLK_HSI0_CMU_HSI0_CLKOUT0 0x0810 +#define CLK_CON_MUX_MUX_CLK_HSI0_BUS 0x1000 +#define CLK_CON_MUX_MUX_CLK_HSI0_USB20_REF 0x1004 +#define CLK_CON_MUX_MUX_CLK_HSI0_USB31DRD 0x1008 +#define CLK_CON_DIV_DIV_CLK_HSI0_USB31DRD 0x1800 +#define CLK_CON_GAT_CLK_BLK_HSI0_UID_HSI0_CMU_HSI0_IPCLKPORT_PCLK 0x2000 +#define CLK_CON_GAT_CLK_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB31DRD_SUSPEND_CLK_26 0x2004 +#define CLK_CON_GAT_CLK_HSI0_ALT 0x2008 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_DP_LINK_IPCLKPORT_I_DP_GTC_CLK 0x200c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_DP_LINK_IPCLKPORT_I_PCLK 0x2010 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_D_TZPC_HSI0_IPCLKPORT_PCLK 0x2014 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_ETR_MIU_IPCLKPORT_I_ACLK 0x2018 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_ETR_MIU_IPCLKPORT_I_PCLK 0x201c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_GPC_HSI0_IPCLKPORT_PCLK 0x2020 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_G_ETR_HSI0_IPCLKPORT_I_CLK 0x2024 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_P_AOCHSI0_IPCLKPORT_I_CLK 0x2028 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_P_HSI0_IPCLKPORT_I_CLK 0x202c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHS_ACEL_D_HSI0_IPCLKPORT_I_CLK 0x2030 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHS_AXI_D_HSI0AOC_IPCLKPORT_I_CLK 0x2034 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_AOC_IPCLKPORT_ACLK 0x2038 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_AOC_IPCLKPORT_PCLK 0x203c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_BUS0_IPCLKPORT_ACLK 0x2040 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_BUS0_IPCLKPORT_PCLK 0x2044 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_RSTNSYNC_CLK_HSI0_BUS_IPCLKPORT_CLK 0x2048 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_SSMT_USB_IPCLKPORT_ACLK 0x204c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_SSMT_USB_IPCLKPORT_PCLK 0x2050 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_SYSMMU_USB_IPCLKPORT_CLK_S2 0x2054 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_SYSREG_HSI0_IPCLKPORT_PCLK 0x2058 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_CTRL_IPCLKPORT_ACLK 0x205c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_CTRL_IPCLKPORT_PCLK 0x2060 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_LINK_IPCLKPORT_ACLK 0x2064 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_LINK_IPCLKPORT_PCLK 0x2068 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_ACLK_PHYCTRL 0x206c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_BUS_CLK_EARLY 0x2070 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB20_PHY_REFCLK_26 0x2074 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB31DRD_REF_CLK_40 0x2078 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBDPPHY_REF_SOC_PLL 0x207c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBDPPHY_SCL_APB_PCLK 0x2080 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBPCS_APB_CLK 0x2084 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_USBDPPHY_I_ACLK 0x2088 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_USBDPPHY_UDBG_I_APB_PCLK 0x208c +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_D0_HSI0_IPCLKPORT_ACLK 0x2090 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_D1_HSI0_IPCLKPORT_ACLK 0x2094 +#define CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_P_HSI0_IPCLKPORT_ACLK 0x2098 +#define DMYQCH_CON_USB31DRD_QCH 0x3000 +#define DMYQCH_CON_USB31DRD_QCH_REF 0x3004 +#define PCH_CON_LHM_AXI_G_ETR_HSI0_PCH 0x3008 +#define PCH_CON_LHM_AXI_P_AOCHSI0_PCH 0x300c +#define PCH_CON_LHM_AXI_P_HSI0_PCH 0x3010 +#define PCH_CON_LHS_ACEL_D_HSI0_PCH 0x3014 +#define PCH_CON_LHS_AXI_D_HSI0AOC_PCH 0x3018 +#define QCH_CON_DP_LINK_QCH_GTC_CLK 0x301c +#define QCH_CON_DP_LINK_QCH_PCLK 0x3020 +#define QCH_CON_D_TZPC_HSI0_QCH 0x3024 +#define QCH_CON_ETR_MIU_QCH_ACLK 0x3028 +#define QCH_CON_ETR_MIU_QCH_PCLK 0x302c +#define QCH_CON_GPC_HSI0_QCH 0x3030 +#define QCH_CON_HSI0_CMU_HSI0_QCH 0x3034 +#define QCH_CON_LHM_AXI_G_ETR_HSI0_QCH 0x3038 +#define QCH_CON_LHM_AXI_P_AOCHSI0_QCH 0x303c +#define QCH_CON_LHM_AXI_P_HSI0_QCH 0x3040 +#define QCH_CON_LHS_ACEL_D_HSI0_QCH 0x3044 +#define QCH_CON_LHS_AXI_D_HSI0AOC_QCH 0x3048 +#define QCH_CON_PPMU_HSI0_AOC_QCH 0x304c +#define QCH_CON_PPMU_HSI0_BUS0_QCH 0x3050 +#define QCH_CON_SSMT_USB_QCH 0x3054 +#define QCH_CON_SYSMMU_USB_QCH 0x3058 +#define QCH_CON_SYSREG_HSI0_QCH 0x305c +#define QCH_CON_UASC_HSI0_CTRL_QCH 0x3060 +#define QCH_CON_UASC_HSI0_LINK_QCH 0x3064 +#define QCH_CON_USB31DRD_QCH_APB 0x3068 +#define QCH_CON_USB31DRD_QCH_DBG 0x306c +#define QCH_CON_USB31DRD_QCH_PCS 0x3070 +#define QCH_CON_USB31DRD_QCH_SLV_CTRL 0x3074 +#define QCH_CON_USB31DRD_QCH_SLV_LINK 0x3078 +#define QUEUE_CTRL_REG_BLK_HSI0_CMU_HSI0 0x3c00 + +static const unsigned long hsi0_clk_regs[] __initconst = { + PLL_LOCKTIME_PLL_USB, + PLL_CON0_PLL_USB, + PLL_CON1_PLL_USB, + PLL_CON2_PLL_USB, + PLL_CON3_PLL_USB, + PLL_CON4_PLL_USB, + PLL_CON0_MUX_CLKCMU_HSI0_ALT_USER, + PLL_CON1_MUX_CLKCMU_HSI0_ALT_USER, + PLL_CON0_MUX_CLKCMU_HSI0_BUS_USER, + PLL_CON1_MUX_CLKCMU_HSI0_BUS_USER, + PLL_CON0_MUX_CLKCMU_HSI0_DPGTC_USER, + PLL_CON1_MUX_CLKCMU_HSI0_DPGTC_USER, + PLL_CON0_MUX_CLKCMU_HSI0_TCXO_USER, + PLL_CON1_MUX_CLKCMU_HSI0_TCXO_USER, + PLL_CON0_MUX_CLKCMU_HSI0_USB20_USER, + PLL_CON1_MUX_CLKCMU_HSI0_USB20_USER, + PLL_CON0_MUX_CLKCMU_HSI0_USB31DRD_USER, + PLL_CON1_MUX_CLKCMU_HSI0_USB31DRD_USER, + PLL_CON0_MUX_CLKCMU_HSI0_USPDPDBG_USER, + PLL_CON1_MUX_CLKCMU_HSI0_USPDPDBG_USER, + HSI0_CMU_HSI0_CONTROLLER_OPTION, + CLKOUT_CON_BLK_HSI0_CMU_HSI0_CLKOUT0, + CLK_CON_MUX_MUX_CLK_HSI0_BUS, + CLK_CON_MUX_MUX_CLK_HSI0_USB20_REF, + CLK_CON_MUX_MUX_CLK_HSI0_USB31DRD, + CLK_CON_DIV_DIV_CLK_HSI0_USB31DRD, + CLK_CON_GAT_CLK_BLK_HSI0_UID_HSI0_CMU_HSI0_IPCLKPORT_PCLK, + CLK_CON_GAT_CLK_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB31DRD_SUSPEND_CLK_26, + CLK_CON_GAT_CLK_HSI0_ALT, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_DP_LINK_IPCLKPORT_I_DP_GTC_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_DP_LINK_IPCLKPORT_I_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_D_TZPC_HSI0_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_ETR_MIU_IPCLKPORT_I_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_ETR_MIU_IPCLKPORT_I_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_GPC_HSI0_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_G_ETR_HSI0_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_P_AOCHSI0_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_P_HSI0_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHS_ACEL_D_HSI0_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHS_AXI_D_HSI0AOC_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_AOC_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_AOC_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_BUS0_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_BUS0_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_RSTNSYNC_CLK_HSI0_BUS_IPCLKPORT_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SSMT_USB_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SSMT_USB_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SYSMMU_USB_IPCLKPORT_CLK_S2, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SYSREG_HSI0_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_CTRL_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_CTRL_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_LINK_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_LINK_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_ACLK_PHYCTRL, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_BUS_CLK_EARLY, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB20_PHY_REFCLK_26, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB31DRD_REF_CLK_40, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBDPPHY_REF_SOC_PLL, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBDPPHY_SCL_APB_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBPCS_APB_CLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_USBDPPHY_I_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_USBDPPHY_UDBG_I_APB_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_D0_HSI0_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_D1_HSI0_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_P_HSI0_IPCLKPORT_ACLK, + DMYQCH_CON_USB31DRD_QCH, + DMYQCH_CON_USB31DRD_QCH_REF, + PCH_CON_LHM_AXI_G_ETR_HSI0_PCH, + PCH_CON_LHM_AXI_P_AOCHSI0_PCH, + PCH_CON_LHM_AXI_P_HSI0_PCH, + PCH_CON_LHS_ACEL_D_HSI0_PCH, + PCH_CON_LHS_AXI_D_HSI0AOC_PCH, + QCH_CON_DP_LINK_QCH_GTC_CLK, + QCH_CON_DP_LINK_QCH_PCLK, + QCH_CON_D_TZPC_HSI0_QCH, + QCH_CON_ETR_MIU_QCH_ACLK, + QCH_CON_ETR_MIU_QCH_PCLK, + QCH_CON_GPC_HSI0_QCH, + QCH_CON_HSI0_CMU_HSI0_QCH, + QCH_CON_LHM_AXI_G_ETR_HSI0_QCH, + QCH_CON_LHM_AXI_P_AOCHSI0_QCH, + QCH_CON_LHM_AXI_P_HSI0_QCH, + QCH_CON_LHS_ACEL_D_HSI0_QCH, + QCH_CON_LHS_AXI_D_HSI0AOC_QCH, + QCH_CON_PPMU_HSI0_AOC_QCH, + QCH_CON_PPMU_HSI0_BUS0_QCH, + QCH_CON_SSMT_USB_QCH, + QCH_CON_SYSMMU_USB_QCH, + QCH_CON_SYSREG_HSI0_QCH, + QCH_CON_UASC_HSI0_CTRL_QCH, + QCH_CON_UASC_HSI0_LINK_QCH, + QCH_CON_USB31DRD_QCH_APB, + QCH_CON_USB31DRD_QCH_DBG, + QCH_CON_USB31DRD_QCH_PCS, + QCH_CON_USB31DRD_QCH_SLV_CTRL, + QCH_CON_USB31DRD_QCH_SLV_LINK, + QUEUE_CTRL_REG_BLK_HSI0_CMU_HSI0, +}; + +/* List of parent clocks for Muxes in CMU_HSI0 */ +PNAME(mout_pll_usb_p) = { "oscclk", "fout_usb_pll" }; +PNAME(mout_hsi0_alt_user_p) = { "oscclk", + "gout_hsi0_clk_hsi0_alt" }; +PNAME(mout_hsi0_bus_user_p) = { "oscclk", "dout_cmu_hsi0_bus" }; +PNAME(mout_hsi0_dpgtc_user_p) = { "oscclk", "dout_cmu_hsi0_dpgtc" }; +PNAME(mout_hsi0_tcxo_user_p) = { "oscclk", "tcxo_hsi1_hsi0" }; +PNAME(mout_hsi0_usb20_user_p) = { "oscclk", "usb20phy_phy_clock" }; +PNAME(mout_hsi0_usb31drd_user_p) = { "oscclk", + "dout_cmu_hsi0_usb31drd" }; +PNAME(mout_hsi0_usbdpdbg_user_p) = { "oscclk", + "dout_cmu_hsi0_usbdpdbg" }; +PNAME(mout_hsi0_bus_p) = { "mout_hsi0_bus_user", + "mout_hsi0_alt_user" }; +PNAME(mout_hsi0_usb20_ref_p) = { "fout_usb_pll", + "mout_hsi0_tcxo_user" }; +PNAME(mout_hsi0_usb31drd_p) = { "fout_usb_pll", + "mout_hsi0_usb31drd_user", + "dout_hsi0_usb31drd", + "fout_usb_pll" }; + +static const struct samsung_pll_rate_table cmu_hsi0_usb_pll_rates[] __initconst = { + PLL_35XX_RATE(24576000, 19200000, 150, 6, 5), + { /* sentinel */ } +}; + +static const struct samsung_pll_clock cmu_hsi0_pll_clks[] __initconst = { + PLL(pll_0518x, CLK_FOUT_USB_PLL, "fout_usb_pll", "oscclk", + PLL_LOCKTIME_PLL_USB, PLL_CON3_PLL_USB, + cmu_hsi0_usb_pll_rates), +}; + +static const struct samsung_mux_clock hsi0_mux_clks[] __initconst = { + MUX(CLK_MOUT_PLL_USB, + "mout_pll_usb", mout_pll_usb_p, + PLL_CON0_PLL_USB, 4, 1), + MUX(CLK_MOUT_HSI0_ALT_USER, + "mout_hsi0_alt_user", mout_hsi0_alt_user_p, + PLL_CON0_MUX_CLKCMU_HSI0_ALT_USER, 4, 1), + MUX(CLK_MOUT_HSI0_BUS_USER, + "mout_hsi0_bus_user", mout_hsi0_bus_user_p, + PLL_CON0_MUX_CLKCMU_HSI0_BUS_USER, 4, 1), + MUX(CLK_MOUT_HSI0_DPGTC_USER, + "mout_hsi0_dpgtc_user", mout_hsi0_dpgtc_user_p, + PLL_CON0_MUX_CLKCMU_HSI0_DPGTC_USER, 4, 1), + MUX(CLK_MOUT_HSI0_TCXO_USER, + "mout_hsi0_tcxo_user", mout_hsi0_tcxo_user_p, + PLL_CON0_MUX_CLKCMU_HSI0_TCXO_USER, 4, 1), + MUX(CLK_MOUT_HSI0_USB20_USER, + "mout_hsi0_usb20_user", mout_hsi0_usb20_user_p, + PLL_CON0_MUX_CLKCMU_HSI0_USB20_USER, 4, 1), + MUX(CLK_MOUT_HSI0_USB31DRD_USER, + "mout_hsi0_usb31drd_user", mout_hsi0_usb31drd_user_p, + PLL_CON0_MUX_CLKCMU_HSI0_USB31DRD_USER, 4, 1), + MUX(CLK_MOUT_HSI0_USBDPDBG_USER, + "mout_hsi0_usbdpdbg_user", mout_hsi0_usbdpdbg_user_p, + PLL_CON0_MUX_CLKCMU_HSI0_USPDPDBG_USER, 4, 1), + MUX(CLK_MOUT_HSI0_BUS, + "mout_hsi0_bus", mout_hsi0_bus_p, + CLK_CON_MUX_MUX_CLK_HSI0_BUS, 0, 1), + MUX(CLK_MOUT_HSI0_USB20_REF, + "mout_hsi0_usb20_ref", mout_hsi0_usb20_ref_p, + CLK_CON_MUX_MUX_CLK_HSI0_USB20_REF, 0, 1), + MUX(CLK_MOUT_HSI0_USB31DRD, + "mout_hsi0_usb31drd", mout_hsi0_usb31drd_p, + CLK_CON_MUX_MUX_CLK_HSI0_USB31DRD, 0, 2), +}; + +static const struct samsung_div_clock hsi0_div_clks[] __initconst = { + DIV(CLK_DOUT_HSI0_USB31DRD, + "dout_hsi0_usb31drd", "mout_hsi0_usb20_user", + CLK_CON_DIV_DIV_CLK_HSI0_USB31DRD, 0, 3), +}; + +static const struct samsung_gate_clock hsi0_gate_clks[] __initconst = { + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_PCLK, + "gout_hsi0_hsi0_pclk", "mout_hsi0_bus", + CLK_CON_GAT_CLK_BLK_HSI0_UID_HSI0_CMU_HSI0_IPCLKPORT_PCLK, + 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_I_USB31DRD_SUSPEND_CLK_26, + "gout_hsi0_usb31drd_i_usb31drd_suspend_clk_26", + "mout_hsi0_usb20_ref", + CLK_CON_GAT_CLK_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB31DRD_SUSPEND_CLK_26, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_CLK_HSI0_ALT, + "gout_hsi0_clk_hsi0_alt", "ioclk_clk_hsi0_alt", + CLK_CON_GAT_CLK_HSI0_ALT, 21, 0, 0), + GATE(CLK_GOUT_HSI0_DP_LINK_I_DP_GTC_CLK, + "gout_hsi0_dp_link_i_dp_gtc_clk", "mout_hsi0_dpgtc_user", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_DP_LINK_IPCLKPORT_I_DP_GTC_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_DP_LINK_I_PCLK, + "gout_hsi0_dp_link_i_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_DP_LINK_IPCLKPORT_I_PCLK, 21, 0, 0), + GATE(CLK_GOUT_HSI0_D_TZPC_HSI0_PCLK, + "gout_hsi0_d_tzpc_hsi0_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_D_TZPC_HSI0_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_ETR_MIU_I_ACLK, + "gout_hsi0_etr_miu_i_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_ETR_MIU_IPCLKPORT_I_ACLK, 21, 0, 0), + GATE(CLK_GOUT_HSI0_ETR_MIU_I_PCLK, + "gout_hsi0_etr_miu_i_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_ETR_MIU_IPCLKPORT_I_PCLK, 21, 0, 0), + GATE(CLK_GOUT_HSI0_GPC_HSI0_PCLK, + "gout_hsi0_gpc_hsi0_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_GPC_HSI0_IPCLKPORT_PCLK, 21, 0, 0), + GATE(CLK_GOUT_HSI0_LHM_AXI_G_ETR_HSI0_I_CLK, + "gout_hsi0_lhm_axi_g_etr_hsi0_i_clk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_G_ETR_HSI0_IPCLKPORT_I_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_LHM_AXI_P_AOCHSI0_I_CLK, + "gout_hsi0_lhm_axi_p_aochsi0_i_clk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_P_AOCHSI0_IPCLKPORT_I_CLK, + 21, 0, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_LHM_AXI_P_HSI0_I_CLK, + "gout_hsi0_lhm_axi_p_hsi0_i_clk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHM_AXI_P_HSI0_IPCLKPORT_I_CLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_LHS_ACEL_D_HSI0_I_CLK, + "gout_hsi0_lhs_acel_d_hsi0_i_clk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHS_ACEL_D_HSI0_IPCLKPORT_I_CLK, + 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_HSI0_LHS_AXI_D_HSI0AOC_I_CLK, + "gout_hsi0_lhs_axi_d_hsi0aoc_i_clk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_LHS_AXI_D_HSI0AOC_IPCLKPORT_I_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_PPMU_HSI0_AOC_ACLK, + "gout_hsi0_ppmu_hsi0_aoc_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_AOC_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_PPMU_HSI0_AOC_PCLK, + "gout_hsi0_ppmu_hsi0_aoc_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_AOC_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_PPMU_HSI0_BUS0_ACLK, + "gout_hsi0_ppmu_hsi0_bus0_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_BUS0_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_PPMU_HSI0_BUS0_PCLK, + "gout_hsi0_ppmu_hsi0_bus0_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_PPMU_HSI0_BUS0_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_CLK_HSI0_BUS_CLK, + "gout_hsi0_clk_hsi0_bus_clk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_RSTNSYNC_CLK_HSI0_BUS_IPCLKPORT_CLK, + 21, 0, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_SSMT_USB_ACLK, + "gout_hsi0_ssmt_usb_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SSMT_USB_IPCLKPORT_ACLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_SSMT_USB_PCLK, + "gout_hsi0_ssmt_usb_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SSMT_USB_IPCLKPORT_PCLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_SYSMMU_USB_CLK_S2, + "gout_hsi0_sysmmu_usb_clk_s2", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SYSMMU_USB_IPCLKPORT_CLK_S2, + 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_HSI0_SYSREG_HSI0_PCLK, + "gout_hsi0_sysreg_hsi0_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_SYSREG_HSI0_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_UASC_HSI0_CTRL_ACLK, + "gout_hsi0_uasc_hsi0_ctrl_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_CTRL_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_UASC_HSI0_CTRL_PCLK, + "gout_hsi0_uasc_hsi0_ctrl_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_CTRL_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_UASC_HSI0_LINK_ACLK, + "gout_hsi0_uasc_hsi0_link_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_LINK_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_UASC_HSI0_LINK_PCLK, + "gout_hsi0_uasc_hsi0_link_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_UASC_HSI0_LINK_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_ACLK_PHYCTRL, + "gout_hsi0_usb31drd_aclk_phyctrl", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_ACLK_PHYCTRL, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_BUS_CLK_EARLY, + "gout_hsi0_usb31drd_bus_clk_early", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_BUS_CLK_EARLY, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_I_USB20_PHY_REFCLK_26, + "gout_hsi0_usb31drd_i_usb20_phy_refclk_26", "mout_hsi0_usb20_ref", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB20_PHY_REFCLK_26, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_I_USB31DRD_REF_CLK_40, + "gout_hsi0_usb31drd_i_usb31drd_ref_clk_40", "mout_hsi0_usb31drd", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USB31DRD_REF_CLK_40, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_I_USBDPPHY_REF_SOC_PLL, + "gout_hsi0_usb31drd_i_usbdpphy_ref_soc_pll", + "mout_hsi0_usbdpdbg_user", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBDPPHY_REF_SOC_PLL, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_I_USBDPPHY_SCL_APB_PCLK, + "gout_hsi0_usb31drd_i_usbdpphy_scl_apb_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBDPPHY_SCL_APB_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_I_USBPCS_APB_CLK, + "gout_hsi0_usb31drd_i_usbpcs_apb_clk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_I_USBPCS_APB_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_USBDPPHY_I_ACLK, + "gout_hsi0_usb31drd_usbdpphy_i_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_USBDPPHY_I_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI0_USB31DRD_USBDPPHY_UDBG_I_APB_PCLK, + "gout_hsi0_usb31drd_usbdpphy_udbg_i_apb_pclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_USB31DRD_IPCLKPORT_USBDPPHY_UDBG_I_APB_PCLK, + 21, 0, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_XIU_D0_HSI0_ACLK, + "gout_hsi0_xiu_d0_hsi0_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_D0_HSI0_IPCLKPORT_ACLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_XIU_D1_HSI0_ACLK, + "gout_hsi0_xiu_d1_hsi0_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_D1_HSI0_IPCLKPORT_ACLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI0_XIU_P_HSI0_ACLK, + "gout_hsi0_xiu_p_hsi0_aclk", "mout_hsi0_bus", + CLK_CON_GAT_GOUT_BLK_HSI0_UID_XIU_P_HSI0_IPCLKPORT_ACLK, + 21, CLK_IGNORE_UNUSED, 0), +}; + +static const struct samsung_fixed_rate_clock hsi0_fixed_clks[] __initconst = { + FRATE(0, "tcxo_hsi1_hsi0", NULL, 0, 26000000), + FRATE(0, "usb20phy_phy_clock", NULL, 0, 120000000), + /* until we implement APMGSA */ + FRATE(0, "ioclk_clk_hsi0_alt", NULL, 0, 213000000), +}; + +static const struct samsung_cmu_info hsi0_cmu_info __initconst = { + .pll_clks = cmu_hsi0_pll_clks, + .nr_pll_clks = ARRAY_SIZE(cmu_hsi0_pll_clks), + .mux_clks = hsi0_mux_clks, + .nr_mux_clks = ARRAY_SIZE(hsi0_mux_clks), + .div_clks = hsi0_div_clks, + .nr_div_clks = ARRAY_SIZE(hsi0_div_clks), + .gate_clks = hsi0_gate_clks, + .nr_gate_clks = ARRAY_SIZE(hsi0_gate_clks), + .fixed_clks = hsi0_fixed_clks, + .nr_fixed_clks = ARRAY_SIZE(hsi0_fixed_clks), + .nr_clk_ids = CLKS_NR_HSI0, + .clk_regs = hsi0_clk_regs, + .nr_clk_regs = ARRAY_SIZE(hsi0_clk_regs), + .clk_name = "bus", +}; + /* ---- CMU_MISC ------------------------------------------------------------ */ /* Register Offset definitions for CMU_MISC (0x10010000) */ @@ -3441,6 +3907,9 @@ static const struct of_device_id gs101_cmu_of_match[] = { { .compatible = "google,gs101-cmu-apm", .data = &apm_cmu_info, + }, { + .compatible = "google,gs101-cmu-hsi0", + .data = &hsi0_cmu_info, }, { .compatible = "google,gs101-cmu-peric0", .data = &peric0_cmu_info, From 093c290084a494844f1650e70755b8912292ee14 Mon Sep 17 00:00:00 2001 From: Peter Griffin Date: Mon, 29 Apr 2024 14:02:19 +0100 Subject: [PATCH 0892/1702] clk: samsung: gs101: add support for cmu_hsi2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CMU_HSI2 is the clock management unit used for the hsi2 block. HSI stands for High Speed Interface and as such it generates clocks for PCIe, UFS and MMC card. This patch adds support for the muxes, dividers, and gates in cmu_hsi2. The following clocks are marked CLK_IS_CRITICAL as disabling them results in an immediate system hang. CLK_GOUT_HSI2_HSI2_CMU_HSI2_PCLK CLK_GOUT_HSI2_LHM_AXI_P_HSI2_I_CLK The following clocks are marked CLK_IGNORE_UNUSED as they are needed for UFS to be functional. CLK_GOUT_HSI2_SSMT_HSI2_ACLK CLK_GOUT_HSI2_SSMT_HSI2_PCLK CLK_GOUT_HSI2_LHS_ACEL_D_HSI2_I_CLK CLK_GOUT_HSI2_SYSMMU_HSI2_CLK_S2 CLK_GOUT_HSI2_XIU_D_HSI2_ACLK CLK_GOUT_HSI2_XIU_P_HSI2_ACLK CLK_GOUT_HSI2_GPIO_HSI2_PCLK is marked CLK_IGNORE_UNUSED until the exynos pinctrl clock patches land then it can be removed. Some clocks in this unit have very long names. To help with this the clock name mangling strategy was updated to include removing the following sub-strings. - G4X2_DWC_PCIE_CTL_ - G4X1_DWC_PCIE_CTL_ - PCIE_SUB_CTRL_ - INST_0_ - LN05LPE_ - TM_WRAPPER_ - SF_ Signed-off-by: Peter Griffin [AD: resolve merge conflicts] Signed-off-by: André Draszik Reviewed-by: André Draszik Link: https://lore.kernel.org/r/20240429-hsi0-gs101-v3-3-f233be0a2455@linaro.org Signed-off-by: Krzysztof Kozlowski --- drivers/clk/samsung/clk-gs101.c | 507 ++++++++++++++++++++++++++++++++ 1 file changed, 507 insertions(+) diff --git a/drivers/clk/samsung/clk-gs101.c b/drivers/clk/samsung/clk-gs101.c index ce0ff35f45480..05129c3b2f680 100644 --- a/drivers/clk/samsung/clk-gs101.c +++ b/drivers/clk/samsung/clk-gs101.c @@ -21,6 +21,7 @@ #define CLKS_NR_TOP (CLK_GOUT_CMU_TPU_UART + 1) #define CLKS_NR_APM (CLK_APM_PLL_DIV16_APM + 1) #define CLKS_NR_HSI0 (CLK_GOUT_HSI0_XIU_P_HSI0_ACLK + 1) +#define CLKS_NR_HSI2 (CLK_GOUT_HSI2_XIU_P_HSI2_ACLK + 1) #define CLKS_NR_MISC (CLK_GOUT_MISC_XIU_D_MISC_ACLK + 1) #define CLKS_NR_PERIC0 (CLK_GOUT_PERIC0_SYSREG_PERIC0_PCLK + 1) #define CLKS_NR_PERIC1 (CLK_GOUT_PERIC1_SYSREG_PERIC1_PCLK + 1) @@ -2385,6 +2386,509 @@ static const struct samsung_cmu_info hsi0_cmu_info __initconst = { .clk_name = "bus", }; +/* ---- CMU_HSI2 ------------------------------------------------------------ */ + +/* Register Offset definitions for CMU_HSI2 (0x14400000) */ +#define PLL_CON0_MUX_CLKCMU_HSI2_BUS_USER 0x0600 +#define PLL_CON1_MUX_CLKCMU_HSI2_BUS_USER 0x0604 +#define PLL_CON0_MUX_CLKCMU_HSI2_MMC_CARD_USER 0x0610 +#define PLL_CON1_MUX_CLKCMU_HSI2_MMC_CARD_USER 0x0614 +#define PLL_CON0_MUX_CLKCMU_HSI2_PCIE_USER 0x0620 +#define PLL_CON1_MUX_CLKCMU_HSI2_PCIE_USER 0x0624 +#define PLL_CON0_MUX_CLKCMU_HSI2_UFS_EMBD_USER 0x0630 +#define PLL_CON1_MUX_CLKCMU_HSI2_UFS_EMBD_USER 0x0634 +#define HSI2_CMU_HSI2_CONTROLLER_OPTION 0x0800 +#define CLKOUT_CON_BLK_HSI2_CMU_HSI2_CLKOUT0 0x0810 +#define CLK_CON_GAT_CLK_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_PCIE_SUB_CTRL_INST_0_PHY_REFCLK_IN 0x2000 +#define CLK_CON_GAT_CLK_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_PCIE_SUB_CTRL_INST_0_PHY_REFCLK_IN 0x2004 +#define CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4A_1_IPCLKPORT_ACLK 0x2008 +#define CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4A_1_IPCLKPORT_PCLK 0x200c +#define CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4B_1_IPCLKPORT_ACLK 0x2010 +#define CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4B_1_IPCLKPORT_PCLK 0x2014 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_D_TZPC_HSI2_IPCLKPORT_PCLK 0x201c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_GPC_HSI2_IPCLKPORT_PCLK 0x2020 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_GPIO_HSI2_IPCLKPORT_PCLK 0x2024 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_HSI2_CMU_HSI2_IPCLKPORT_PCLK 0x2028 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_LHM_AXI_P_HSI2_IPCLKPORT_I_CLK 0x202c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_LHS_ACEL_D_HSI2_IPCLKPORT_I_CLK 0x2030 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_MMC_CARD_IPCLKPORT_I_ACLK 0x2034 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_MMC_CARD_IPCLKPORT_SDCLKIN 0x2038 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_DBI_ACLK_UG 0x203c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_MSTR_ACLK_UG 0x2040 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_SLV_ACLK_UG 0x2044 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_PCIE_SUB_CTRL_INST_0_I_DRIVER_APB_CLK 0x2048 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_DBI_ACLK_UG 0x204c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_MSTR_ACLK_UG 0x2050 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_SLV_ACLK_UG 0x2054 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_PCIE_SUB_CTRL_INST_0_I_DRIVER_APB_CLK 0x2058 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_PHY_UDBG_I_APB_PCLK 0x205c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_PIPE_PAL_PCIE_INST_0_I_APB_PCLK 0x2060 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_SF_PCIEPHY210X2_LN05LPE_QCH_TM_WRAPPER_INST_0_I_APB_PCLK 0x2064 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_IA_GEN4A_1_IPCLKPORT_I_CLK 0x2068 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_IA_GEN4B_1_IPCLKPORT_I_CLK 0x206c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PPMU_HSI2_IPCLKPORT_ACLK 0x2070 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_PPMU_HSI2_IPCLKPORT_PCLK 0x2074 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_MMC_CARD_HSI2_IPCLKPORT_ACLK 0x2078 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_MMC_CARD_HSI2_IPCLKPORT_PCLK 0x207c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4A_HSI2_IPCLKPORT_ACLK 0x2080 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4A_HSI2_IPCLKPORT_PCLK 0x2084 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4B_HSI2_IPCLKPORT_ACLK 0x2088 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4B_HSI2_IPCLKPORT_PCLK 0x208c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_UFS_EMBD_HSI2_IPCLKPORT_ACLK 0x2090 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_UFS_EMBD_HSI2_IPCLKPORT_PCLK 0x2094 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_RSTNSYNC_CLK_HSI2_BUS_IPCLKPORT_CLK 0x2098 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_RSTNSYNC_CLK_HSI2_OSCCLK_IPCLKPORT_CLK 0x209c +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_SSMT_HSI2_IPCLKPORT_ACLK 0x20a0 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_SSMT_HSI2_IPCLKPORT_PCLK 0x20a4 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_SYSMMU_HSI2_IPCLKPORT_CLK_S2 0x20a8 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_SYSREG_HSI2_IPCLKPORT_PCLK 0x20ac +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_DBI_1_IPCLKPORT_ACLK 0x20b0 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_DBI_1_IPCLKPORT_PCLK 0x20b4 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_SLV_1_IPCLKPORT_ACLK 0x20b8 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_SLV_1_IPCLKPORT_PCLK 0x20bc +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_DBI_1_IPCLKPORT_ACLK 0x20c0 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_DBI_1_IPCLKPORT_PCLK 0x20c4 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_SLV_1_IPCLKPORT_ACLK 0x20c8 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_SLV_1_IPCLKPORT_PCLK 0x20cc +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_ACLK 0x20d0 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_CLK_UNIPRO 0x20d4 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_FMP_CLK 0x20d8 +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_XIU_D_HSI2_IPCLKPORT_ACLK 0x20dc +#define CLK_CON_GAT_GOUT_BLK_HSI2_UID_XIU_P_HSI2_IPCLKPORT_ACLK 0x20e0 +#define DMYQCH_CON_PCIE_GEN4_1_QCH_SCLK_1 0x3000 +#define PCH_CON_LHM_AXI_P_HSI2_PCH 0x3008 +#define PCH_CON_LHS_ACEL_D_HSI2_PCH 0x300c +#define QCH_CON_D_TZPC_HSI2_QCH 0x3010 +#define QCH_CON_GPC_HSI2_QCH 0x3014 +#define QCH_CON_GPIO_HSI2_QCH 0x3018 +#define QCH_CON_HSI2_CMU_HSI2_QCH 0x301c +#define QCH_CON_LHM_AXI_P_HSI2_QCH 0x3020 +#define QCH_CON_LHS_ACEL_D_HSI2_QCH 0x3024 +#define QCH_CON_MMC_CARD_QCH 0x3028 +#define QCH_CON_PCIE_GEN4_1_QCH_APB_1 0x302c +#define QCH_CON_PCIE_GEN4_1_QCH_APB_2 0x3030 +#define QCH_CON_PCIE_GEN4_1_QCH_AXI_1 0x3034 +#define QCH_CON_PCIE_GEN4_1_QCH_AXI_2 0x3038 +#define QCH_CON_PCIE_GEN4_1_QCH_DBG_1 0x303c +#define QCH_CON_PCIE_GEN4_1_QCH_DBG_2 0x3040 +#define QCH_CON_PCIE_GEN4_1_QCH_PCS_APB 0x3044 +#define QCH_CON_PCIE_GEN4_1_QCH_PMA_APB 0x3048 +#define QCH_CON_PCIE_GEN4_1_QCH_UDBG 0x304c +#define QCH_CON_PCIE_IA_GEN4A_1_QCH 0x3050 +#define QCH_CON_PCIE_IA_GEN4B_1_QCH 0x3054 +#define QCH_CON_PPMU_HSI2_QCH 0x3058 +#define QCH_CON_QE_MMC_CARD_HSI2_QCH 0x305c +#define QCH_CON_QE_PCIE_GEN4A_HSI2_QCH 0x3060 +#define QCH_CON_QE_PCIE_GEN4B_HSI2_QCH 0x3064 +#define QCH_CON_QE_UFS_EMBD_HSI2_QCH 0x3068 +#define QCH_CON_SSMT_HSI2_QCH 0x306c +#define QCH_CON_SSMT_PCIE_IA_GEN4A_1_QCH 0x3070 +#define QCH_CON_SSMT_PCIE_IA_GEN4B_1_QCH 0x3074 +#define QCH_CON_SYSMMU_HSI2_QCH 0x3078 +#define QCH_CON_SYSREG_HSI2_QCH 0x307c +#define QCH_CON_UASC_PCIE_GEN4A_DBI_1_QCH 0x3080 +#define QCH_CON_UASC_PCIE_GEN4A_SLV_1_QCH 0x3084 +#define QCH_CON_UASC_PCIE_GEN4B_DBI_1_QCH 0x3088 +#define QCH_CON_UASC_PCIE_GEN4B_SLV_1_QCH 0x308c +#define QCH_CON_UFS_EMBD_QCH 0x3090 +#define QCH_CON_UFS_EMBD_QCH_FMP 0x3094 +#define QUEUE_CTRL_REG_BLK_HSI2_CMU_HSI2 0x3c00 + +static const unsigned long cmu_hsi2_clk_regs[] __initconst = { + PLL_CON0_MUX_CLKCMU_HSI2_BUS_USER, + PLL_CON1_MUX_CLKCMU_HSI2_BUS_USER, + PLL_CON0_MUX_CLKCMU_HSI2_MMC_CARD_USER, + PLL_CON1_MUX_CLKCMU_HSI2_MMC_CARD_USER, + PLL_CON0_MUX_CLKCMU_HSI2_PCIE_USER, + PLL_CON1_MUX_CLKCMU_HSI2_PCIE_USER, + PLL_CON0_MUX_CLKCMU_HSI2_UFS_EMBD_USER, + PLL_CON1_MUX_CLKCMU_HSI2_UFS_EMBD_USER, + HSI2_CMU_HSI2_CONTROLLER_OPTION, + CLKOUT_CON_BLK_HSI2_CMU_HSI2_CLKOUT0, + CLK_CON_GAT_CLK_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_PCIE_SUB_CTRL_INST_0_PHY_REFCLK_IN, + CLK_CON_GAT_CLK_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_PCIE_SUB_CTRL_INST_0_PHY_REFCLK_IN, + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4A_1_IPCLKPORT_ACLK, + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4A_1_IPCLKPORT_PCLK, + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4B_1_IPCLKPORT_ACLK, + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4B_1_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_D_TZPC_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_GPC_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_GPIO_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_HSI2_CMU_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_LHM_AXI_P_HSI2_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_LHS_ACEL_D_HSI2_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_MMC_CARD_IPCLKPORT_I_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_MMC_CARD_IPCLKPORT_SDCLKIN, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_DBI_ACLK_UG, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_MSTR_ACLK_UG, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_SLV_ACLK_UG, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_PCIE_SUB_CTRL_INST_0_I_DRIVER_APB_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_DBI_ACLK_UG, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_MSTR_ACLK_UG, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_SLV_ACLK_UG, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_PCIE_SUB_CTRL_INST_0_I_DRIVER_APB_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_PHY_UDBG_I_APB_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_PIPE_PAL_PCIE_INST_0_I_APB_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_SF_PCIEPHY210X2_LN05LPE_QCH_TM_WRAPPER_INST_0_I_APB_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_IA_GEN4A_1_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_IA_GEN4B_1_IPCLKPORT_I_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PPMU_HSI2_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PPMU_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_MMC_CARD_HSI2_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_MMC_CARD_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4A_HSI2_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4A_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4B_HSI2_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4B_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_UFS_EMBD_HSI2_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_UFS_EMBD_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_RSTNSYNC_CLK_HSI2_BUS_IPCLKPORT_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_RSTNSYNC_CLK_HSI2_OSCCLK_IPCLKPORT_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SSMT_HSI2_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SSMT_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SYSMMU_HSI2_IPCLKPORT_CLK_S2, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SYSREG_HSI2_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_DBI_1_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_DBI_1_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_SLV_1_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_SLV_1_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_DBI_1_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_DBI_1_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_SLV_1_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_SLV_1_IPCLKPORT_PCLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_CLK_UNIPRO, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_FMP_CLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_XIU_D_HSI2_IPCLKPORT_ACLK, + CLK_CON_GAT_GOUT_BLK_HSI2_UID_XIU_P_HSI2_IPCLKPORT_ACLK, + DMYQCH_CON_PCIE_GEN4_1_QCH_SCLK_1, + PCH_CON_LHM_AXI_P_HSI2_PCH, + PCH_CON_LHS_ACEL_D_HSI2_PCH, + QCH_CON_D_TZPC_HSI2_QCH, + QCH_CON_GPC_HSI2_QCH, + QCH_CON_GPIO_HSI2_QCH, + QCH_CON_HSI2_CMU_HSI2_QCH, + QCH_CON_LHM_AXI_P_HSI2_QCH, + QCH_CON_LHS_ACEL_D_HSI2_QCH, + QCH_CON_MMC_CARD_QCH, + QCH_CON_PCIE_GEN4_1_QCH_APB_1, + QCH_CON_PCIE_GEN4_1_QCH_APB_2, + QCH_CON_PCIE_GEN4_1_QCH_AXI_1, + QCH_CON_PCIE_GEN4_1_QCH_AXI_2, + QCH_CON_PCIE_GEN4_1_QCH_DBG_1, + QCH_CON_PCIE_GEN4_1_QCH_DBG_2, + QCH_CON_PCIE_GEN4_1_QCH_PCS_APB, + QCH_CON_PCIE_GEN4_1_QCH_PMA_APB, + QCH_CON_PCIE_GEN4_1_QCH_UDBG, + QCH_CON_PCIE_IA_GEN4A_1_QCH, + QCH_CON_PCIE_IA_GEN4B_1_QCH, + QCH_CON_PPMU_HSI2_QCH, + QCH_CON_QE_MMC_CARD_HSI2_QCH, + QCH_CON_QE_PCIE_GEN4A_HSI2_QCH, + QCH_CON_QE_PCIE_GEN4B_HSI2_QCH, + QCH_CON_QE_UFS_EMBD_HSI2_QCH, + QCH_CON_SSMT_HSI2_QCH, + QCH_CON_SSMT_PCIE_IA_GEN4A_1_QCH, + QCH_CON_SSMT_PCIE_IA_GEN4B_1_QCH, + QCH_CON_SYSMMU_HSI2_QCH, + QCH_CON_SYSREG_HSI2_QCH, + QCH_CON_UASC_PCIE_GEN4A_DBI_1_QCH, + QCH_CON_UASC_PCIE_GEN4A_SLV_1_QCH, + QCH_CON_UASC_PCIE_GEN4B_DBI_1_QCH, + QCH_CON_UASC_PCIE_GEN4B_SLV_1_QCH, + QCH_CON_UFS_EMBD_QCH, + QCH_CON_UFS_EMBD_QCH_FMP, + QUEUE_CTRL_REG_BLK_HSI2_CMU_HSI2, +}; + +PNAME(mout_hsi2_ufs_embd_p) = { "oscclk", "dout_cmu_shared0_div4", + "dout_cmu_shared2_div2", "fout_spare_pll" }; + +PNAME(mout_hsi2_pcie_p) = { "oscclk", "dout_cmu_shared2_div2" }; + +PNAME(mout_hsi2_bus_p) = { "dout_cmu_shared0_div4", + "dout_cmu_shared1_div4", + "dout_cmu_shared2_div2", + "dout_cmu_shared3_div2", + "fout_spare_pll", "oscclk", "oscclk", + "oscclk" }; + +PNAME(mout_hsi2_mmc_card_p) = { "fout_shared2_pll", "fout_shared3_pll", + "dout_cmu_shared0_div4", "fout_spare_pll" }; + +PNAME(mout_hsi2_bus_user_p) = { "oscclk", "dout_cmu_hsi2_bus" }; +PNAME(mout_hsi2_mmc_card_user_p) = { "oscclk", "dout_cmu_hsi2_mmc_card" }; +PNAME(mout_hsi2_pcie_user_p) = { "oscclk", "dout_cmu_hsi2_pcie" }; +PNAME(mout_hsi2_ufs_embd_user_p) = { "oscclk", "dout_cmu_hsi2_ufs_embd" }; + +static const struct samsung_mux_clock hsi2_mux_clks[] __initconst = { + MUX(CLK_MOUT_HSI2_BUS_USER, "mout_hsi2_bus_user", mout_hsi2_bus_user_p, + PLL_CON0_MUX_CLKCMU_HSI2_BUS_USER, 4, 1), + MUX(CLK_MOUT_HSI2_MMC_CARD_USER, "mout_hsi2_mmc_card_user", + mout_hsi2_mmc_card_user_p, PLL_CON0_MUX_CLKCMU_HSI2_MMC_CARD_USER, + 4, 1), + MUX(CLK_MOUT_HSI2_PCIE_USER, "mout_hsi2_pcie_user", + mout_hsi2_pcie_user_p, PLL_CON0_MUX_CLKCMU_HSI2_PCIE_USER, + 4, 1), + MUX(CLK_MOUT_HSI2_UFS_EMBD_USER, "mout_hsi2_ufs_embd_user", + mout_hsi2_ufs_embd_user_p, PLL_CON0_MUX_CLKCMU_HSI2_UFS_EMBD_USER, + 4, 1), +}; + +static const struct samsung_gate_clock hsi2_gate_clks[] __initconst = { + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_003_PHY_REFCLK_IN, + "gout_hsi2_pcie_gen4_1_pcie_003_phy_refclk_in", + "mout_hsi2_pcie_user", + CLK_CON_GAT_CLK_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_PCIE_SUB_CTRL_INST_0_PHY_REFCLK_IN, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_004_PHY_REFCLK_IN, + "gout_hsi2_pcie_gen4_1_pcie_004_phy_refclk_in", + "mout_hsi2_pcie_user", + CLK_CON_GAT_CLK_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_PCIE_SUB_CTRL_INST_0_PHY_REFCLK_IN, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_SSMT_PCIE_IA_GEN4A_1_ACLK, + "gout_hsi2_ssmt_pcie_ia_gen4a_1_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4A_1_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_SSMT_PCIE_IA_GEN4A_1_PCLK, + "gout_hsi2_ssmt_pcie_ia_gen4a_1_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4A_1_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_SSMT_PCIE_IA_GEN4B_1_ACLK, + "gout_hsi2_ssmt_pcie_ia_gen4b_1_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4B_1_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_SSMT_PCIE_IA_GEN4B_1_PCLK, + "gout_hsi2_ssmt_pcie_ia_gen4b_1_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_CLK_BLK_HSI2_UID_SSMT_PCIE_IA_GEN4B_1_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_D_TZPC_HSI2_PCLK, + "gout_hsi2_d_tzpc_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_D_TZPC_HSI2_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_GPC_HSI2_PCLK, + "gout_hsi2_gpc_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_GPC_HSI2_IPCLKPORT_PCLK, 21, 0, 0), + GATE(CLK_GOUT_HSI2_GPIO_HSI2_PCLK, + "gout_hsi2_gpio_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_GPIO_HSI2_IPCLKPORT_PCLK, 21, + CLK_IGNORE_UNUSED, 0), + /* Disabling this clock makes the system hang. Mark the clock as critical. */ + GATE(CLK_GOUT_HSI2_HSI2_CMU_HSI2_PCLK, + "gout_hsi2_hsi2_cmu_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_HSI2_CMU_HSI2_IPCLKPORT_PCLK, + 21, CLK_IS_CRITICAL, 0), + /* Disabling this clock makes the system hang. Mark the clock as critical. */ + GATE(CLK_GOUT_HSI2_LHM_AXI_P_HSI2_I_CLK, + "gout_hsi2_lhm_axi_p_hsi2_i_clk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_LHM_AXI_P_HSI2_IPCLKPORT_I_CLK, + 21, CLK_IS_CRITICAL, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI2_LHS_ACEL_D_HSI2_I_CLK, + "gout_hsi2_lhs_acel_d_hsi2_i_clk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_LHS_ACEL_D_HSI2_IPCLKPORT_I_CLK, + 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_HSI2_MMC_CARD_I_ACLK, + "gout_hsi2_mmc_card_i_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_MMC_CARD_IPCLKPORT_I_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_MMC_CARD_SDCLKIN, + "gout_hsi2_mmc_card_sdclkin", "mout_hsi2_mmc_card_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_MMC_CARD_IPCLKPORT_SDCLKIN, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_003_DBI_ACLK_UG, + "gout_hsi2_pcie_gen4_1_pcie_003_dbi_aclk_ug", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_DBI_ACLK_UG, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_003_MSTR_ACLK_UG, + "gout_hsi2_pcie_gen4_1_pcie_003_mstr_aclk_ug", + "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_MSTR_ACLK_UG, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_003_SLV_ACLK_UG, + "gout_hsi2_pcie_gen4_1_pcie_003_slv_aclk_ug", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_G4X2_DWC_PCIE_CTL_INST_0_SLV_ACLK_UG, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_003_I_DRIVER_APB_CLK, + "gout_hsi2_pcie_gen4_1_pcie_003_i_driver_apb_clk", + "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_003_PCIE_SUB_CTRL_INST_0_I_DRIVER_APB_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_004_DBI_ACLK_UG, + "gout_hsi2_pcie_gen4_1_pcie_004_dbi_aclk_ug", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_DBI_ACLK_UG, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_004_MSTR_ACLK_UG, + "gout_hsi2_pcie_gen4_1_pcie_004_mstr_aclk_ug", + "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_MSTR_ACLK_UG, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_004_SLV_ACLK_UG, + "gout_hsi2_pcie_gen4_1_pcie_004_slv_aclk_ug", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_G4X1_DWC_PCIE_CTL_INST_0_SLV_ACLK_UG, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCIE_004_I_DRIVER_APB_CLK, + "gout_hsi2_pcie_gen4_1_pcie_004_i_driver_apb_clk", + "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCIE_004_PCIE_SUB_CTRL_INST_0_I_DRIVER_APB_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCS_PMA_PHY_UDBG_I_APB_PCLK, + "gout_hsi2_pcie_gen4_1_pcs_pma_phy_udbg_i_apb_pclk", + "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_PHY_UDBG_I_APB_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCS_PMA_PIPE_PAL_PCIE_I_APB_PCLK, + "gout_hsi2_pcie_gen4_1_pcs_pma_pipe_pal_pcie_i_apb_pclk", + "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_PIPE_PAL_PCIE_INST_0_I_APB_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_GEN4_1_PCS_PMA_PCIEPHY210X2_QCH_I_APB_PCLK, + "gout_hsi2_pcie_gen4_1_pcs_pma_pciephy210x2_qch_i_apb_pclk", + "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_GEN4_1_IPCLKPORT_PCS_PMA_INST_0_SF_PCIEPHY210X2_LN05LPE_QCH_TM_WRAPPER_INST_0_I_APB_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_IA_GEN4A_1_I_CLK, + "gout_hsi2_pcie_ia_gen4a_1_i_clk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_IA_GEN4A_1_IPCLKPORT_I_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PCIE_IA_GEN4B_1_I_CLK, + "gout_hsi2_pcie_ia_gen4b_1_i_clk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PCIE_IA_GEN4B_1_IPCLKPORT_I_CLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PPMU_HSI2_ACLK, + "gout_hsi2_ppmu_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PPMU_HSI2_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_PPMU_HSI2_PCLK, + "gout_hsi2_ppmu_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_PPMU_HSI2_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_MMC_CARD_HSI2_ACLK, + "gout_hsi2_qe_mmc_card_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_MMC_CARD_HSI2_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_MMC_CARD_HSI2_PCLK, + "gout_hsi2_qe_mmc_card_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_MMC_CARD_HSI2_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_PCIE_GEN4A_HSI2_ACLK, + "gout_hsi2_qe_pcie_gen4a_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4A_HSI2_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_PCIE_GEN4A_HSI2_PCLK, + "gout_hsi2_qe_pcie_gen4a_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4A_HSI2_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_PCIE_GEN4B_HSI2_ACLK, + "gout_hsi2_qe_pcie_gen4b_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4B_HSI2_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_PCIE_GEN4B_HSI2_PCLK, + "gout_hsi2_qe_pcie_gen4b_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_PCIE_GEN4B_HSI2_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_UFS_EMBD_HSI2_ACLK, + "gout_hsi2_qe_ufs_embd_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_UFS_EMBD_HSI2_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_QE_UFS_EMBD_HSI2_PCLK, + "gout_hsi2_qe_ufs_embd_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_QE_UFS_EMBD_HSI2_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_CLK_HSI2_BUS_CLK, + "gout_hsi2_clk_hsi2_bus_clk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_RSTNSYNC_CLK_HSI2_BUS_IPCLKPORT_CLK, + 21, CLK_IS_CRITICAL, 0), + GATE(CLK_GOUT_HSI2_CLK_HSI2_OSCCLK_CLK, + "gout_hsi2_clk_hsi2_oscclk_clk", "oscclk", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_RSTNSYNC_CLK_HSI2_OSCCLK_IPCLKPORT_CLK, + 21, 0, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI2_SSMT_HSI2_ACLK, + "gout_hsi2_ssmt_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SSMT_HSI2_IPCLKPORT_ACLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI2_SSMT_HSI2_PCLK, + "gout_hsi2_ssmt_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SSMT_HSI2_IPCLKPORT_PCLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI2_SYSMMU_HSI2_CLK_S2, + "gout_hsi2_sysmmu_hsi2_clk_s2", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SYSMMU_HSI2_IPCLKPORT_CLK_S2, + 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_HSI2_SYSREG_HSI2_PCLK, + "gout_hsi2_sysreg_hsi2_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_SYSREG_HSI2_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4A_DBI_1_ACLK, + "gout_hsi2_uasc_pcie_gen4a_dbi_1_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_DBI_1_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4A_DBI_1_PCLK, + "gout_hsi2_uasc_pcie_gen4a_dbi_1_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_DBI_1_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4A_SLV_1_ACLK, + "gout_hsi2_uasc_pcie_gen4a_slv_1_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_SLV_1_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4A_SLV_1_PCLK, + "gout_hsi2_uasc_pcie_gen4a_slv_1_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4A_SLV_1_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4B_DBI_1_ACLK, + "gout_hsi2_uasc_pcie_gen4b_dbi_1_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_DBI_1_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4B_DBI_1_PCLK, + "gout_hsi2_uasc_pcie_gen4b_dbi_1_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_DBI_1_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4B_SLV_1_ACLK, + "gout_hsi2_uasc_pcie_gen4b_slv_1_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_SLV_1_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UASC_PCIE_GEN4B_SLV_1_PCLK, + "gout_hsi2_uasc_pcie_gen4b_slv_1_pclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UASC_PCIE_GEN4B_SLV_1_IPCLKPORT_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UFS_EMBD_I_ACLK, + "gout_hsi2_ufs_embd_i_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UFS_EMBD_I_CLK_UNIPRO, + "gout_hsi2_ufs_embd_i_clk_unipro", "mout_hsi2_ufs_embd_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_CLK_UNIPRO, + 21, 0, 0), + GATE(CLK_GOUT_HSI2_UFS_EMBD_I_FMP_CLK, + "gout_hsi2_ufs_embd_i_fmp_clk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_UFS_EMBD_IPCLKPORT_I_FMP_CLK, + 21, 0, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI2_XIU_D_HSI2_ACLK, + "gout_hsi2_xiu_d_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_XIU_D_HSI2_IPCLKPORT_ACLK, + 21, CLK_IGNORE_UNUSED, 0), + /* TODO: should have a driver for this */ + GATE(CLK_GOUT_HSI2_XIU_P_HSI2_ACLK, + "gout_hsi2_xiu_p_hsi2_aclk", "mout_hsi2_bus_user", + CLK_CON_GAT_GOUT_BLK_HSI2_UID_XIU_P_HSI2_IPCLKPORT_ACLK, + 21, CLK_IGNORE_UNUSED, 0), +}; + +static const struct samsung_cmu_info hsi2_cmu_info __initconst = { + .mux_clks = hsi2_mux_clks, + .nr_mux_clks = ARRAY_SIZE(hsi2_mux_clks), + .gate_clks = hsi2_gate_clks, + .nr_gate_clks = ARRAY_SIZE(hsi2_gate_clks), + .nr_clk_ids = CLKS_NR_HSI2, + .clk_regs = cmu_hsi2_clk_regs, + .nr_clk_regs = ARRAY_SIZE(cmu_hsi2_clk_regs), + .clk_name = "bus", +}; + /* ---- CMU_MISC ------------------------------------------------------------ */ /* Register Offset definitions for CMU_MISC (0x10010000) */ @@ -3910,6 +4414,9 @@ static const struct of_device_id gs101_cmu_of_match[] = { }, { .compatible = "google,gs101-cmu-hsi0", .data = &hsi0_cmu_info, + }, { + .compatible = "google,gs101-cmu-hsi2", + .data = &hsi2_cmu_info, }, { .compatible = "google,gs101-cmu-peric0", .data = &peric0_cmu_info, From dff9f3fb6ba4f74eb805bc172cc16ff2c91648bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Draszik?= Date: Fri, 26 Apr 2024 14:25:14 +0100 Subject: [PATCH 0893/1702] dt-bindings: pinctrl: samsung: google,gs101-pinctrl needs a clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pin controller on Google Tensor gs101 requires a bus clock for register access to work. Add it. Signed-off-by: André Draszik Link: https://lore.kernel.org/r/20240426-samsung-pinctrl-busclock-v3-1-adb8664b8a7e@linaro.org Signed-off-by: Krzysztof Kozlowski --- .../bindings/pinctrl/samsung,pinctrl.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml index 118549c259765..242dd13c276b5 100644 --- a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml @@ -73,6 +73,13 @@ properties: minItems: 1 maxItems: 2 + clocks: + maxItems: 1 + + clock-names: + items: + - const: pclk + wakeup-interrupt-controller: $ref: samsung,pinctrl-wakeup-interrupt.yaml @@ -120,6 +127,20 @@ required: allOf: - $ref: pinctrl.yaml# + - if: + properties: + compatible: + contains: + const: google,gs101-pinctrl + then: + required: + - clocks + - clock-names + else: + properties: + clocks: false + clock-names: false + - if: properties: compatible: From f9c74474797351c60e009ebc59a798fcfd93ee57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Draszik?= Date: Fri, 26 Apr 2024 14:25:15 +0100 Subject: [PATCH 0894/1702] pinctrl: samsung: support a bus clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On some Samsung-based SoCs there are separate bus clocks / gates each for each pinctrl instance. To be able to access each pinctrl instance's registers, this bus clock needs to be running, otherwise register access will hang. Google Tensor gs101 is one example for such an implementation. Update the driver to handle this optional bus clock: * handle an optional bus clock from DT * prepare it during driver probe * enclose all relevant register accesses with a clock enable & disable Signed-off-by: André Draszik Link: https://lore.kernel.org/r/20240426-samsung-pinctrl-busclock-v3-2-adb8664b8a7e@linaro.org Signed-off-by: Krzysztof Kozlowski --- drivers/pinctrl/samsung/pinctrl-exynos.c | 112 ++++++++++++++++++++++ drivers/pinctrl/samsung/pinctrl-samsung.c | 97 ++++++++++++++++++- drivers/pinctrl/samsung/pinctrl-samsung.h | 2 + 3 files changed, 207 insertions(+), 4 deletions(-) diff --git a/drivers/pinctrl/samsung/pinctrl-exynos.c b/drivers/pinctrl/samsung/pinctrl-exynos.c index 871c1eb46ddfd..ce5e6783b5b99 100644 --- a/drivers/pinctrl/samsung/pinctrl-exynos.c +++ b/drivers/pinctrl/samsung/pinctrl-exynos.c @@ -13,6 +13,7 @@ // the Samsung pinctrl/gpiolib driver. It also includes the implementation of // external gpio and wakeup interrupt support. +#include #include #include #include @@ -61,6 +62,12 @@ static void exynos_irq_mask(struct irq_data *irqd) else reg_mask = our_chip->eint_mask + bank->eint_offset; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for masking IRQ\n"); + return; + } + raw_spin_lock_irqsave(&bank->slock, flags); mask = readl(bank->eint_base + reg_mask); @@ -68,6 +75,8 @@ static void exynos_irq_mask(struct irq_data *irqd) writel(mask, bank->eint_base + reg_mask); raw_spin_unlock_irqrestore(&bank->slock, flags); + + clk_disable(bank->drvdata->pclk); } static void exynos_irq_ack(struct irq_data *irqd) @@ -82,7 +91,15 @@ static void exynos_irq_ack(struct irq_data *irqd) else reg_pend = our_chip->eint_pend + bank->eint_offset; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock to ack IRQ\n"); + return; + } + writel(1 << irqd->hwirq, bank->eint_base + reg_pend); + + clk_disable(bank->drvdata->pclk); } static void exynos_irq_unmask(struct irq_data *irqd) @@ -110,6 +127,12 @@ static void exynos_irq_unmask(struct irq_data *irqd) else reg_mask = our_chip->eint_mask + bank->eint_offset; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for unmasking IRQ\n"); + return; + } + raw_spin_lock_irqsave(&bank->slock, flags); mask = readl(bank->eint_base + reg_mask); @@ -117,6 +140,8 @@ static void exynos_irq_unmask(struct irq_data *irqd) writel(mask, bank->eint_base + reg_mask); raw_spin_unlock_irqrestore(&bank->slock, flags); + + clk_disable(bank->drvdata->pclk); } static int exynos_irq_set_type(struct irq_data *irqd, unsigned int type) @@ -127,6 +152,7 @@ static int exynos_irq_set_type(struct irq_data *irqd, unsigned int type) unsigned int shift = EXYNOS_EINT_CON_LEN * irqd->hwirq; unsigned int con, trig_type; unsigned long reg_con; + int ret; switch (type) { case IRQ_TYPE_EDGE_RISING: @@ -159,11 +185,20 @@ static int exynos_irq_set_type(struct irq_data *irqd, unsigned int type) else reg_con = our_chip->eint_con + bank->eint_offset; + ret = clk_enable(bank->drvdata->pclk); + if (ret) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for configuring IRQ type\n"); + return ret; + } + con = readl(bank->eint_base + reg_con); con &= ~(EXYNOS_EINT_CON_MASK << shift); con |= trig_type << shift; writel(con, bank->eint_base + reg_con); + clk_disable(bank->drvdata->pclk); + return 0; } @@ -200,6 +235,14 @@ static int exynos_irq_request_resources(struct irq_data *irqd) shift = irqd->hwirq * bank_type->fld_width[PINCFG_TYPE_FUNC]; mask = (1 << bank_type->fld_width[PINCFG_TYPE_FUNC]) - 1; + ret = clk_enable(bank->drvdata->pclk); + if (ret) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for configuring pin %s-%lu\n", + bank->name, irqd->hwirq); + return ret; + } + raw_spin_lock_irqsave(&bank->slock, flags); con = readl(bank->pctl_base + reg_con); @@ -209,6 +252,8 @@ static int exynos_irq_request_resources(struct irq_data *irqd) raw_spin_unlock_irqrestore(&bank->slock, flags); + clk_disable(bank->drvdata->pclk); + return 0; } @@ -223,6 +268,13 @@ static void exynos_irq_release_resources(struct irq_data *irqd) shift = irqd->hwirq * bank_type->fld_width[PINCFG_TYPE_FUNC]; mask = (1 << bank_type->fld_width[PINCFG_TYPE_FUNC]) - 1; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for deconfiguring pin %s-%lu\n", + bank->name, irqd->hwirq); + return; + } + raw_spin_lock_irqsave(&bank->slock, flags); con = readl(bank->pctl_base + reg_con); @@ -232,6 +284,8 @@ static void exynos_irq_release_resources(struct irq_data *irqd) raw_spin_unlock_irqrestore(&bank->slock, flags); + clk_disable(bank->drvdata->pclk); + gpiochip_unlock_as_irq(&bank->gpio_chip, irqd->hwirq); } @@ -281,10 +335,19 @@ static irqreturn_t exynos_eint_gpio_irq(int irq, void *data) unsigned int svc, group, pin; int ret; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for handling IRQ\n"); + return IRQ_NONE; + } + if (bank->eint_con_offset) svc = readl(bank->eint_base + EXYNOSAUTO_SVC_OFFSET); else svc = readl(bank->eint_base + EXYNOS_SVC_OFFSET); + + clk_disable(bank->drvdata->pclk); + group = EXYNOS_SVC_GROUP(svc); pin = svc & EXYNOS_SVC_NUM_MASK; @@ -563,6 +626,20 @@ static void exynos_irq_demux_eint16_31(struct irq_desc *desc) chained_irq_enter(chip, desc); + /* + * just enable the clock once here, to avoid an enable/disable dance for + * each bank. + */ + if (eintd->nr_banks) { + struct samsung_pin_bank *b = eintd->banks[0]; + + if (clk_enable(b->drvdata->pclk)) { + dev_err(b->gpio_chip.parent, + "unable to enable clock for pending IRQs\n"); + return; + } + } + for (i = 0; i < eintd->nr_banks; ++i) { struct samsung_pin_bank *b = eintd->banks[i]; pend = readl(b->eint_base + b->irq_chip->eint_pend @@ -572,6 +649,9 @@ static void exynos_irq_demux_eint16_31(struct irq_desc *desc) exynos_irq_demux_eint(pend & ~mask, b->irq_domain); } + if (eintd->nr_banks) + clk_disable(eintd->banks[0]->drvdata->pclk); + chained_irq_exit(chip, desc); } @@ -695,6 +775,12 @@ static void exynos_pinctrl_suspend_bank( struct exynos_eint_gpio_save *save = bank->soc_priv; const void __iomem *regs = bank->eint_base; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for saving state\n"); + return; + } + save->eint_con = readl(regs + EXYNOS_GPIO_ECON_OFFSET + bank->eint_offset); save->eint_fltcon0 = readl(regs + EXYNOS_GPIO_EFLTCON_OFFSET @@ -704,6 +790,8 @@ static void exynos_pinctrl_suspend_bank( save->eint_mask = readl(regs + bank->irq_chip->eint_mask + bank->eint_offset); + clk_disable(bank->drvdata->pclk); + pr_debug("%s: save con %#010x\n", bank->name, save->eint_con); pr_debug("%s: save fltcon0 %#010x\n", bank->name, save->eint_fltcon0); pr_debug("%s: save fltcon1 %#010x\n", bank->name, save->eint_fltcon1); @@ -716,9 +804,17 @@ static void exynosauto_pinctrl_suspend_bank(struct samsung_pinctrl_drv_data *drv struct exynos_eint_gpio_save *save = bank->soc_priv; const void __iomem *regs = bank->eint_base; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for saving state\n"); + return; + } + save->eint_con = readl(regs + bank->pctl_offset + bank->eint_con_offset); save->eint_mask = readl(regs + bank->pctl_offset + bank->eint_mask_offset); + clk_disable(bank->drvdata->pclk); + pr_debug("%s: save con %#010x\n", bank->name, save->eint_con); pr_debug("%s: save mask %#010x\n", bank->name, save->eint_mask); } @@ -753,6 +849,12 @@ static void exynos_pinctrl_resume_bank( struct exynos_eint_gpio_save *save = bank->soc_priv; void __iomem *regs = bank->eint_base; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for restoring state\n"); + return; + } + pr_debug("%s: con %#010x => %#010x\n", bank->name, readl(regs + EXYNOS_GPIO_ECON_OFFSET + bank->eint_offset), save->eint_con); @@ -774,6 +876,8 @@ static void exynos_pinctrl_resume_bank( + 2 * bank->eint_offset + 4); writel(save->eint_mask, regs + bank->irq_chip->eint_mask + bank->eint_offset); + + clk_disable(bank->drvdata->pclk); } static void exynosauto_pinctrl_resume_bank(struct samsung_pinctrl_drv_data *drvdata, @@ -782,6 +886,12 @@ static void exynosauto_pinctrl_resume_bank(struct samsung_pinctrl_drv_data *drvd struct exynos_eint_gpio_save *save = bank->soc_priv; void __iomem *regs = bank->eint_base; + if (clk_enable(bank->drvdata->pclk)) { + dev_err(bank->gpio_chip.parent, + "unable to enable clock for restoring state\n"); + return; + } + pr_debug("%s: con %#010x => %#010x\n", bank->name, readl(regs + bank->pctl_offset + bank->eint_con_offset), save->eint_con); pr_debug("%s: mask %#010x => %#010x\n", bank->name, @@ -789,6 +899,8 @@ static void exynosauto_pinctrl_resume_bank(struct samsung_pinctrl_drv_data *drvd writel(save->eint_con, regs + bank->pctl_offset + bank->eint_con_offset); writel(save->eint_mask, regs + bank->pctl_offset + bank->eint_mask_offset); + + clk_disable(bank->drvdata->pclk); } void exynos_pinctrl_resume(struct samsung_pinctrl_drv_data *drvdata) diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.c b/drivers/pinctrl/samsung/pinctrl-samsung.c index ed07e23e09122..f4607b8493fff 100644 --- a/drivers/pinctrl/samsung/pinctrl-samsung.c +++ b/drivers/pinctrl/samsung/pinctrl-samsung.c @@ -15,6 +15,7 @@ // but provides extensions to which platform specific implementation of the gpio // and wakeup interrupts can be hooked to. +#include #include #include #include @@ -371,8 +372,8 @@ static void pin_to_reg_bank(struct samsung_pinctrl_drv_data *drvdata, } /* enable or disable a pinmux function */ -static void samsung_pinmux_setup(struct pinctrl_dev *pctldev, unsigned selector, - unsigned group) +static int samsung_pinmux_setup(struct pinctrl_dev *pctldev, unsigned selector, + unsigned group) { struct samsung_pinctrl_drv_data *drvdata; const struct samsung_pin_bank_type *type; @@ -382,6 +383,7 @@ static void samsung_pinmux_setup(struct pinctrl_dev *pctldev, unsigned selector, unsigned long flags; const struct samsung_pmx_func *func; const struct samsung_pin_group *grp; + int ret; drvdata = pinctrl_dev_get_drvdata(pctldev); func = &drvdata->pmx_functions[selector]; @@ -397,6 +399,12 @@ static void samsung_pinmux_setup(struct pinctrl_dev *pctldev, unsigned selector, reg += 4; } + ret = clk_enable(drvdata->pclk); + if (ret) { + dev_err(pctldev->dev, "failed to enable clock for setup\n"); + return ret; + } + raw_spin_lock_irqsave(&bank->slock, flags); data = readl(reg + type->reg_offset[PINCFG_TYPE_FUNC]); @@ -405,6 +413,10 @@ static void samsung_pinmux_setup(struct pinctrl_dev *pctldev, unsigned selector, writel(data, reg + type->reg_offset[PINCFG_TYPE_FUNC]); raw_spin_unlock_irqrestore(&bank->slock, flags); + + clk_disable(drvdata->pclk); + + return 0; } /* enable a specified pinmux by writing to registers */ @@ -412,8 +424,7 @@ static int samsung_pinmux_set_mux(struct pinctrl_dev *pctldev, unsigned selector, unsigned group) { - samsung_pinmux_setup(pctldev, selector, group); - return 0; + return samsung_pinmux_setup(pctldev, selector, group); } /* list of pinmux callbacks for the pinmux vertical in pinctrl core */ @@ -436,6 +447,7 @@ static int samsung_pinconf_rw(struct pinctrl_dev *pctldev, unsigned int pin, u32 data, width, pin_offset, mask, shift; u32 cfg_value, cfg_reg; unsigned long flags; + int ret; drvdata = pinctrl_dev_get_drvdata(pctldev); pin_to_reg_bank(drvdata, pin, ®_base, &pin_offset, &bank); @@ -447,6 +459,12 @@ static int samsung_pinconf_rw(struct pinctrl_dev *pctldev, unsigned int pin, width = type->fld_width[cfg_type]; cfg_reg = type->reg_offset[cfg_type]; + ret = clk_enable(drvdata->pclk); + if (ret) { + dev_err(drvdata->dev, "failed to enable clock\n"); + return ret; + } + raw_spin_lock_irqsave(&bank->slock, flags); mask = (1 << width) - 1; @@ -466,6 +484,8 @@ static int samsung_pinconf_rw(struct pinctrl_dev *pctldev, unsigned int pin, raw_spin_unlock_irqrestore(&bank->slock, flags); + clk_disable(drvdata->pclk); + return 0; } @@ -555,11 +575,19 @@ static void samsung_gpio_set_value(struct gpio_chip *gc, static void samsung_gpio_set(struct gpio_chip *gc, unsigned offset, int value) { struct samsung_pin_bank *bank = gpiochip_get_data(gc); + struct samsung_pinctrl_drv_data *drvdata = bank->drvdata; unsigned long flags; + if (clk_enable(drvdata->pclk)) { + dev_err(drvdata->dev, "failed to enable clock\n"); + return; + } + raw_spin_lock_irqsave(&bank->slock, flags); samsung_gpio_set_value(gc, offset, value); raw_spin_unlock_irqrestore(&bank->slock, flags); + + clk_disable(drvdata->pclk); } /* gpiolib gpio_get callback function */ @@ -569,12 +597,23 @@ static int samsung_gpio_get(struct gpio_chip *gc, unsigned offset) u32 data; struct samsung_pin_bank *bank = gpiochip_get_data(gc); const struct samsung_pin_bank_type *type = bank->type; + struct samsung_pinctrl_drv_data *drvdata = bank->drvdata; + int ret; reg = bank->pctl_base + bank->pctl_offset; + ret = clk_enable(drvdata->pclk); + if (ret) { + dev_err(drvdata->dev, "failed to enable clock\n"); + return ret; + } + data = readl(reg + type->reg_offset[PINCFG_TYPE_DAT]); data >>= offset; data &= 1; + + clk_disable(drvdata->pclk); + return data; } @@ -591,9 +630,11 @@ static int samsung_gpio_set_direction(struct gpio_chip *gc, struct samsung_pin_bank *bank; void __iomem *reg; u32 data, mask, shift; + struct samsung_pinctrl_drv_data *drvdata; bank = gpiochip_get_data(gc); type = bank->type; + drvdata = bank->drvdata; reg = bank->pctl_base + bank->pctl_offset + type->reg_offset[PINCFG_TYPE_FUNC]; @@ -619,12 +660,22 @@ static int samsung_gpio_set_direction(struct gpio_chip *gc, static int samsung_gpio_direction_input(struct gpio_chip *gc, unsigned offset) { struct samsung_pin_bank *bank = gpiochip_get_data(gc); + struct samsung_pinctrl_drv_data *drvdata = bank->drvdata; unsigned long flags; int ret; + ret = clk_enable(drvdata->pclk); + if (ret) { + dev_err(drvdata->dev, "failed to enable clock\n"); + return ret; + } + raw_spin_lock_irqsave(&bank->slock, flags); ret = samsung_gpio_set_direction(gc, offset, true); raw_spin_unlock_irqrestore(&bank->slock, flags); + + clk_disable(drvdata->pclk); + return ret; } @@ -633,14 +684,23 @@ static int samsung_gpio_direction_output(struct gpio_chip *gc, unsigned offset, int value) { struct samsung_pin_bank *bank = gpiochip_get_data(gc); + struct samsung_pinctrl_drv_data *drvdata = bank->drvdata; unsigned long flags; int ret; + ret = clk_enable(drvdata->pclk); + if (ret) { + dev_err(drvdata->dev, "failed to enable clock\n"); + return ret; + } + raw_spin_lock_irqsave(&bank->slock, flags); samsung_gpio_set_value(gc, offset, value); ret = samsung_gpio_set_direction(gc, offset, false); raw_spin_unlock_irqrestore(&bank->slock, flags); + clk_disable(drvdata->pclk); + return ret; } @@ -1164,6 +1224,12 @@ static int samsung_pinctrl_probe(struct platform_device *pdev) } } + drvdata->pclk = devm_clk_get_optional_prepared(dev, "pclk"); + if (IS_ERR(drvdata->pclk)) { + ret = PTR_ERR(drvdata->pclk); + goto err_put_banks; + } + ret = samsung_pinctrl_register(pdev, drvdata); if (ret) goto err_put_banks; @@ -1202,6 +1268,13 @@ static int __maybe_unused samsung_pinctrl_suspend(struct device *dev) struct samsung_pinctrl_drv_data *drvdata = dev_get_drvdata(dev); int i; + i = clk_enable(drvdata->pclk); + if (i) { + dev_err(drvdata->dev, + "failed to enable clock for saving state\n"); + return i; + } + for (i = 0; i < drvdata->nr_banks; i++) { struct samsung_pin_bank *bank = &drvdata->pin_banks[i]; const void __iomem *reg = bank->pctl_base + bank->pctl_offset; @@ -1231,6 +1304,8 @@ static int __maybe_unused samsung_pinctrl_suspend(struct device *dev) } } + clk_disable(drvdata->pclk); + if (drvdata->suspend) drvdata->suspend(drvdata); if (drvdata->retention_ctrl && drvdata->retention_ctrl->enable) @@ -1250,8 +1325,20 @@ static int __maybe_unused samsung_pinctrl_suspend(struct device *dev) static int __maybe_unused samsung_pinctrl_resume(struct device *dev) { struct samsung_pinctrl_drv_data *drvdata = dev_get_drvdata(dev); + int ret; int i; + /* + * enable clock before the callback, as we don't want to have to deal + * with callback cleanup on clock failures. + */ + ret = clk_enable(drvdata->pclk); + if (ret) { + dev_err(drvdata->dev, + "failed to enable clock for restoring state\n"); + return ret; + } + if (drvdata->resume) drvdata->resume(drvdata); @@ -1286,6 +1373,8 @@ static int __maybe_unused samsung_pinctrl_resume(struct device *dev) writel(bank->pm_save[type], reg + offs[type]); } + clk_disable(drvdata->pclk); + if (drvdata->retention_ctrl && drvdata->retention_ctrl->disable) drvdata->retention_ctrl->disable(drvdata); diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.h b/drivers/pinctrl/samsung/pinctrl-samsung.h index ab791afaabf55..d50ba6f07d5df 100644 --- a/drivers/pinctrl/samsung/pinctrl-samsung.h +++ b/drivers/pinctrl/samsung/pinctrl-samsung.h @@ -274,6 +274,7 @@ struct samsung_pin_ctrl { * through samsung_pinctrl_drv_data, not samsung_pin_bank). * @dev: device instance representing the controller. * @irq: interrpt number used by the controller to notify gpio interrupts. + * @pclk: optional bus clock if required for accessing registers * @ctrl: pin controller instance managed by the driver. * @pctl: pin controller descriptor registered with the pinctrl subsystem. * @pctl_dev: cookie representing pinctrl device instance. @@ -293,6 +294,7 @@ struct samsung_pinctrl_drv_data { void __iomem *virt_base; struct device *dev; int irq; + struct clk *pclk; struct pinctrl_desc pctl; struct pinctrl_dev *pctl_dev; From ecd69be71aad3b9299b37b3d8cc3f0fb6016286f Mon Sep 17 00:00:00 2001 From: Yifan Zhao Date: Thu, 25 Apr 2024 22:55:28 +0800 Subject: [PATCH 0895/1702] f2fs: remove redundant parameter in is_next_segment_free() is_next_segment_free() takes a redundant `type` parameter. Remove it. Signed-off-by: Yifan Zhao Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/segment.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 5d6dae4861f38..4396df2bceab9 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -2645,7 +2645,7 @@ static void write_current_sum_page(struct f2fs_sb_info *sbi, } static int is_next_segment_free(struct f2fs_sb_info *sbi, - struct curseg_info *curseg, int type) + struct curseg_info *curseg) { unsigned int segno = curseg->segno + 1; struct free_segmap_info *free_i = FREE_I(sbi); @@ -3073,8 +3073,7 @@ static bool need_new_seg(struct f2fs_sb_info *sbi, int type) if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) && curseg->seg_type == CURSEG_WARM_NODE) return true; - if (curseg->alloc_type == LFS && - is_next_segment_free(sbi, curseg, type) && + if (curseg->alloc_type == LFS && is_next_segment_free(sbi, curseg) && likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED))) return true; if (!f2fs_need_SSR(sbi) || !get_ssr_segment(sbi, type, SSR, 0)) From a320b2f08b3b26fff0a71777f018553fc1e6873e Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Fri, 26 Apr 2024 17:33:48 +0800 Subject: [PATCH 0896/1702] f2fs: fix to avoid allocating WARM_DATA segment for direct IO If active_log is not 6, we never use WARM_DATA segment, let's avoid allocating WARM_DATA segment for direct IO. Signed-off-by: Yunlei He Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/data.c | 3 ++- fs/f2fs/f2fs.h | 2 +- fs/f2fs/file.c | 5 +++-- fs/f2fs/segment.c | 11 +++++++++-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index bee1e45f76b8a..0c516c653f050 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -4179,7 +4179,8 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, map.m_lblk = bytes_to_blks(inode, offset); map.m_len = bytes_to_blks(inode, offset + length - 1) - map.m_lblk + 1; map.m_next_pgofs = &next_pgofs; - map.m_seg_type = f2fs_rw_hint_to_seg_type(inode->i_write_hint); + map.m_seg_type = f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode), + inode->i_write_hint); if (flags & IOMAP_WRITE) map.m_may_create = true; diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index a0ae99bcca39b..3a95c8ccb9e0b 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -3745,7 +3745,7 @@ int f2fs_build_segment_manager(struct f2fs_sb_info *sbi); void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi); int __init f2fs_create_segment_manager_caches(void); void f2fs_destroy_segment_manager_caches(void); -int f2fs_rw_hint_to_seg_type(enum rw_hint hint); +int f2fs_rw_hint_to_seg_type(struct f2fs_sb_info *sbi, enum rw_hint hint); enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi, enum page_type type, enum temp_type temp); unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi, diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index d382f8bc2fbe6..f1a6b4da57466 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -4637,7 +4637,8 @@ static int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *iter, map.m_may_create = true; if (dio) { - map.m_seg_type = f2fs_rw_hint_to_seg_type(inode->i_write_hint); + map.m_seg_type = f2fs_rw_hint_to_seg_type(sbi, + inode->i_write_hint); flag = F2FS_GET_BLOCK_PRE_DIO; } else { map.m_seg_type = NO_CHECK_TYPE; @@ -4690,7 +4691,7 @@ static void f2fs_dio_write_submit_io(const struct iomap_iter *iter, { struct inode *inode = iter->inode; struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - int seg_type = f2fs_rw_hint_to_seg_type(inode->i_write_hint); + int seg_type = f2fs_rw_hint_to_seg_type(sbi, inode->i_write_hint); enum temp_type temp = f2fs_get_segment_temp(seg_type); bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, DATA, temp); diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 4396df2bceab9..8ddd4f1c665a0 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -3351,8 +3351,14 @@ int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) return err; } -int f2fs_rw_hint_to_seg_type(enum rw_hint hint) +int f2fs_rw_hint_to_seg_type(struct f2fs_sb_info *sbi, enum rw_hint hint) { + if (F2FS_OPTION(sbi).active_logs == 2) + return CURSEG_HOT_DATA; + else if (F2FS_OPTION(sbi).active_logs == 4) + return CURSEG_COLD_DATA; + + /* active_log == 6 */ switch (hint) { case WRITE_LIFE_SHORT: return CURSEG_HOT_DATA; @@ -3492,7 +3498,8 @@ static int __get_segment_type_6(struct f2fs_io_info *fio) is_inode_flag_set(inode, FI_HOT_DATA) || f2fs_is_cow_file(inode)) return CURSEG_HOT_DATA; - return f2fs_rw_hint_to_seg_type(inode->i_write_hint); + return f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode), + inode->i_write_hint); } else { if (IS_DNODE(fio->page)) return is_cold_node(fio->page) ? CURSEG_WARM_NODE : From 20faaf30e55522bba2b56d9c46689233205d7717 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Thu, 25 Apr 2024 16:58:38 +0800 Subject: [PATCH 0897/1702] f2fs: fix to do sanity check on i_xattr_nid in sanity_check_inode() syzbot reports a kernel bug as below: F2FS-fs (loop0): Mounted with checkpoint version = 48b305e4 ================================================================== BUG: KASAN: slab-out-of-bounds in f2fs_test_bit fs/f2fs/f2fs.h:2933 [inline] BUG: KASAN: slab-out-of-bounds in current_nat_addr fs/f2fs/node.h:213 [inline] BUG: KASAN: slab-out-of-bounds in f2fs_get_node_info+0xece/0x1200 fs/f2fs/node.c:600 Read of size 1 at addr ffff88807a58c76c by task syz-executor280/5076 CPU: 1 PID: 5076 Comm: syz-executor280 Not tainted 6.9.0-rc5-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 03/27/2024 Call Trace: __dump_stack lib/dump_stack.c:88 [inline] dump_stack_lvl+0x241/0x360 lib/dump_stack.c:114 print_address_description mm/kasan/report.c:377 [inline] print_report+0x169/0x550 mm/kasan/report.c:488 kasan_report+0x143/0x180 mm/kasan/report.c:601 f2fs_test_bit fs/f2fs/f2fs.h:2933 [inline] current_nat_addr fs/f2fs/node.h:213 [inline] f2fs_get_node_info+0xece/0x1200 fs/f2fs/node.c:600 f2fs_xattr_fiemap fs/f2fs/data.c:1848 [inline] f2fs_fiemap+0x55d/0x1ee0 fs/f2fs/data.c:1925 ioctl_fiemap fs/ioctl.c:220 [inline] do_vfs_ioctl+0x1c07/0x2e50 fs/ioctl.c:838 __do_sys_ioctl fs/ioctl.c:902 [inline] __se_sys_ioctl+0x81/0x170 fs/ioctl.c:890 do_syscall_x64 arch/x86/entry/common.c:52 [inline] do_syscall_64+0xf5/0x240 arch/x86/entry/common.c:83 entry_SYSCALL_64_after_hwframe+0x77/0x7f The root cause is we missed to do sanity check on i_xattr_nid during f2fs_iget(), so that in fiemap() path, current_nat_addr() will access nat_bitmap w/ offset from invalid i_xattr_nid, result in triggering kasan bug report, fix it. Reported-and-tested-by: syzbot+3694e283cf5c40df6d14@syzkaller.appspotmail.com Closes: https://lore.kernel.org/linux-f2fs-devel/00000000000094036c0616e72a1d@google.com Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/inode.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index d7a5a88a1a5eb..7968b14d49f40 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -362,6 +362,12 @@ static bool sanity_check_inode(struct inode *inode, struct page *node_page) return false; } + if (fi->i_xattr_nid && f2fs_check_nid_range(sbi, fi->i_xattr_nid)) { + f2fs_warn(sbi, "%s: inode (ino=%lx) has corrupted i_xattr_nid: %u, run fsck to fix.", + __func__, inode->i_ino, fi->i_xattr_nid); + return false; + } + return true; } From 48d180e2bf5a527ba6513a8abddf8b3beb6b7674 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Sun, 28 Apr 2024 09:12:36 +0800 Subject: [PATCH 0898/1702] f2fs: zone: fix to don't trigger OPU on pinfile for direct IO Otherwise, it breaks pinfile's sematics. Cc: Daeho Jeong Signed-off-by: Chao Yu Reviewed-by: Daeho Jeong Signed-off-by: Jaegeuk Kim --- fs/f2fs/data.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 0c516c653f050..a990236f137ea 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -1595,8 +1595,9 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) } /* use out-place-update for direct IO under LFS mode */ - if (map->m_may_create && - (is_hole || (f2fs_lfs_mode(sbi) && flag == F2FS_GET_BLOCK_DIO))) { + if (map->m_may_create && (is_hole || + (flag == F2FS_GET_BLOCK_DIO && f2fs_lfs_mode(sbi) && + !f2fs_is_pinned_file(inode)))) { if (unlikely(f2fs_cp_error(sbi))) { err = -EIO; goto sync_out; From 9368cdf90f52a68120d039887ccff74ff33b4444 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Thu, 25 Apr 2024 09:55:51 -0700 Subject: [PATCH 0899/1702] clk: bcm: dvp: Assign ->num before accessing ->hws Commit f316cdff8d67 ("clk: Annotate struct clk_hw_onecell_data with __counted_by") annotated the hws member of 'struct clk_hw_onecell_data' with __counted_by, which informs the bounds sanitizer about the number of elements in hws, so that it can warn when hws is accessed out of bounds. As noted in that change, the __counted_by member must be initialized with the number of elements before the first array access happens, otherwise there will be a warning from each access prior to the initialization because the number of elements is zero. This occurs in clk_dvp_probe() due to ->num being assigned after ->hws has been accessed: UBSAN: array-index-out-of-bounds in drivers/clk/bcm/clk-bcm2711-dvp.c:59:2 index 0 is out of range for type 'struct clk_hw *[] __counted_by(num)' (aka 'struct clk_hw *[]') Move the ->num initialization to before the first access of ->hws, which clears up the warning. Cc: stable@vger.kernel.org Fixes: f316cdff8d67 ("clk: Annotate struct clk_hw_onecell_data with __counted_by") Signed-off-by: Nathan Chancellor Link: https://lore.kernel.org/r/20240425-cbl-bcm-assign-counted-by-val-before-access-v1-1-e2db3b82d5ef@kernel.org Reviewed-by: Kees Cook Reviewed-by: Florian Fainelli Signed-off-by: Stephen Boyd --- drivers/clk/bcm/clk-bcm2711-dvp.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/clk/bcm/clk-bcm2711-dvp.c b/drivers/clk/bcm/clk-bcm2711-dvp.c index e4fbbf3c40fe2..3cb235df9d379 100644 --- a/drivers/clk/bcm/clk-bcm2711-dvp.c +++ b/drivers/clk/bcm/clk-bcm2711-dvp.c @@ -56,6 +56,8 @@ static int clk_dvp_probe(struct platform_device *pdev) if (ret) return ret; + data->num = NR_CLOCKS; + data->hws[0] = clk_hw_register_gate_parent_data(&pdev->dev, "hdmi0-108MHz", &clk_dvp_parent, 0, @@ -76,7 +78,6 @@ static int clk_dvp_probe(struct platform_device *pdev) goto unregister_clk0; } - data->num = NR_CLOCKS; ret = of_clk_add_hw_provider(pdev->dev.of_node, of_clk_hw_onecell_get, data); if (ret) From 6dc445c1905096b2ed4db1a84570375b4e00cc0f Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Thu, 25 Apr 2024 09:55:52 -0700 Subject: [PATCH 0900/1702] clk: bcm: rpi: Assign ->num before accessing ->hws Commit f316cdff8d67 ("clk: Annotate struct clk_hw_onecell_data with __counted_by") annotated the hws member of 'struct clk_hw_onecell_data' with __counted_by, which informs the bounds sanitizer about the number of elements in hws, so that it can warn when hws is accessed out of bounds. As noted in that change, the __counted_by member must be initialized with the number of elements before the first array access happens, otherwise there will be a warning from each access prior to the initialization because the number of elements is zero. This occurs in raspberrypi_discover_clocks() due to ->num being assigned after ->hws has been accessed: UBSAN: array-index-out-of-bounds in drivers/clk/bcm/clk-raspberrypi.c:374:4 index 3 is out of range for type 'struct clk_hw *[] __counted_by(num)' (aka 'struct clk_hw *[]') Move the ->num initialization to before the first access of ->hws, which clears up the warning. Cc: stable@vger.kernel.org Fixes: f316cdff8d67 ("clk: Annotate struct clk_hw_onecell_data with __counted_by") Signed-off-by: Nathan Chancellor Link: https://lore.kernel.org/r/20240425-cbl-bcm-assign-counted-by-val-before-access-v1-2-e2db3b82d5ef@kernel.org Reviewed-by: Kees Cook Reviewed-by: Florian Fainelli Signed-off-by: Stephen Boyd --- drivers/clk/bcm/clk-raspberrypi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clk/bcm/clk-raspberrypi.c b/drivers/clk/bcm/clk-raspberrypi.c index 829406dc44a20..4d411408e4afe 100644 --- a/drivers/clk/bcm/clk-raspberrypi.c +++ b/drivers/clk/bcm/clk-raspberrypi.c @@ -371,8 +371,8 @@ static int raspberrypi_discover_clocks(struct raspberrypi_clk *rpi, if (IS_ERR(hw)) return PTR_ERR(hw); - data->hws[clks->id] = hw; data->num = clks->id + 1; + data->hws[clks->id] = hw; } clks++; From 6773da870ab89123d1b513da63ed59e32a29cb77 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Apr 2024 08:15:21 +0200 Subject: [PATCH 0901/1702] xfs: fix error returns from xfs_bmapi_write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit xfs_bmapi_write can return 0 without actually returning a mapping in mval in two different cases: 1) when there is absolutely no space available to do an allocation 2) when converting delalloc space, and the allocation is so small that it only covers parts of the delalloc extent before the range requested by the caller Callers at best can handle one of these cases, but in many cases can't cope with either one. Switch xfs_bmapi_write to always return a mapping or return an error code instead. For case 1) above ENOSPC is the obvious choice which is very much what the callers expect anyway. For case 2) there is no really good error code, so pick a funky one from the SysV streams portfolio. This fixes the reproducer here: https://lore.kernel.org/linux-xfs/CAEJPjCvT3Uag-pMTYuigEjWZHn1sGMZ0GCjVVCv29tNHK76Cgg@mail.gmail.com0/ which uses reserved blocks to create file systems that are gravely out of space and thus cause at least xfs_file_alloc_space to hang and trigger the lack of ENOSPC handling in xfs_dquot_disk_alloc. Note that this patch does not actually make any caller but xfs_alloc_file_space deal intelligently with case 2) above. Signed-off-by: Christoph Hellwig Reported-by: 刘通 Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_attr_remote.c | 1 - fs/xfs/libxfs/xfs_bmap.c | 46 ++++++++++++++++++++++++++------- fs/xfs/libxfs/xfs_da_btree.c | 20 ++++---------- fs/xfs/scrub/quota_repair.c | 6 ----- fs/xfs/scrub/rtbitmap_repair.c | 2 -- fs/xfs/xfs_bmap_util.c | 31 +++++++++++----------- fs/xfs/xfs_dquot.c | 1 - fs/xfs/xfs_iomap.c | 8 ------ fs/xfs/xfs_reflink.c | 14 ---------- fs/xfs/xfs_rtalloc.c | 2 -- 10 files changed, 57 insertions(+), 74 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr_remote.c b/fs/xfs/libxfs/xfs_attr_remote.c index a8de9dc1e998a..beb0efdd8f6b8 100644 --- a/fs/xfs/libxfs/xfs_attr_remote.c +++ b/fs/xfs/libxfs/xfs_attr_remote.c @@ -625,7 +625,6 @@ xfs_attr_rmtval_set_blk( if (error) return error; - ASSERT(nmap == 1); ASSERT((map->br_startblock != DELAYSTARTBLOCK) && (map->br_startblock != HOLESTARTBLOCK)); diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 49e2fcad593df..14c9781ec0ce7 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4217,8 +4217,10 @@ xfs_bmapi_allocate( } else { error = xfs_bmap_alloc_userdata(bma); } - if (error || bma->blkno == NULLFSBLOCK) + if (error) return error; + if (bma->blkno == NULLFSBLOCK) + return -ENOSPC; if (bma->flags & XFS_BMAPI_ZERO) { error = xfs_zero_extent(bma->ip, bma->blkno, bma->length); @@ -4397,6 +4399,15 @@ xfs_bmapi_finish( * extent state if necessary. Details behaviour is controlled by the flags * parameter. Only allocates blocks from a single allocation group, to avoid * locking problems. + * + * Returns 0 on success and places the extent mappings in mval. nmaps is used + * as an input/output parameter where the caller specifies the maximum number + * of mappings that may be returned and xfs_bmapi_write passes back the number + * of mappings (including existing mappings) it found. + * + * Returns a negative error code on failure, including -ENOSPC when it could not + * allocate any blocks and -ENOSR when it did allocate blocks to convert a + * delalloc range, but those blocks were before the passed in range. */ int xfs_bmapi_write( @@ -4525,10 +4536,16 @@ xfs_bmapi_write( ASSERT(len > 0); ASSERT(bma.length > 0); error = xfs_bmapi_allocate(&bma); - if (error) + if (error) { + /* + * If we already allocated space in a previous + * iteration return what we go so far when + * running out of space. + */ + if (error == -ENOSPC && bma.nallocs) + break; goto error0; - if (bma.blkno == NULLFSBLOCK) - break; + } /* * If this is a CoW allocation, record the data in @@ -4566,7 +4583,6 @@ xfs_bmapi_write( if (!xfs_iext_next_extent(ifp, &bma.icur, &bma.got)) eof = true; } - *nmap = n; error = xfs_bmap_btree_to_extents(tp, ip, bma.cur, &bma.logflags, whichfork); @@ -4577,7 +4593,22 @@ xfs_bmapi_write( ifp->if_nextents > XFS_IFORK_MAXEXT(ip, whichfork)); xfs_bmapi_finish(&bma, whichfork, 0); xfs_bmap_validate_ret(orig_bno, orig_len, orig_flags, orig_mval, - orig_nmap, *nmap); + orig_nmap, n); + + /* + * When converting delayed allocations, xfs_bmapi_allocate ignores + * the passed in bno and always converts from the start of the found + * delalloc extent. + * + * To avoid a successful return with *nmap set to 0, return the magic + * -ENOSR error code for this particular case so that the caller can + * handle it. + */ + if (!n) { + ASSERT(bma.nallocs >= *nmap); + return -ENOSR; + } + *nmap = n; return 0; error0: xfs_bmapi_finish(&bma, whichfork, error); @@ -4685,9 +4716,6 @@ xfs_bmapi_convert_one_delalloc( if (error) goto out_finish; - error = -ENOSPC; - if (WARN_ON_ONCE(bma.blkno == NULLFSBLOCK)) - goto out_finish; if (WARN_ON_ONCE(!xfs_valid_startblock(ip, bma.got.br_startblock))) { xfs_bmap_mark_sick(ip, whichfork); error = -EFSCORRUPTED; diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c index b13796629e221..16a529a887808 100644 --- a/fs/xfs/libxfs/xfs_da_btree.c +++ b/fs/xfs/libxfs/xfs_da_btree.c @@ -2297,8 +2297,8 @@ xfs_da_grow_inode_int( struct xfs_inode *dp = args->dp; int w = args->whichfork; xfs_rfsblock_t nblks = dp->i_nblocks; - struct xfs_bmbt_irec map, *mapp; - int nmap, error, got, i, mapi; + struct xfs_bmbt_irec map, *mapp = ↦ + int nmap, error, got, i, mapi = 1; /* * Find a spot in the file space to put the new block. @@ -2314,14 +2314,7 @@ xfs_da_grow_inode_int( error = xfs_bmapi_write(tp, dp, *bno, count, xfs_bmapi_aflag(w)|XFS_BMAPI_METADATA|XFS_BMAPI_CONTIG, args->total, &map, &nmap); - if (error) - return error; - - ASSERT(nmap <= 1); - if (nmap == 1) { - mapp = ↦ - mapi = 1; - } else if (nmap == 0 && count > 1) { + if (error == -ENOSPC && count > 1) { xfs_fileoff_t b; int c; @@ -2339,16 +2332,13 @@ xfs_da_grow_inode_int( args->total, &mapp[mapi], &nmap); if (error) goto out_free_map; - if (nmap < 1) - break; mapi += nmap; b = mapp[mapi - 1].br_startoff + mapp[mapi - 1].br_blockcount; } - } else { - mapi = 0; - mapp = NULL; } + if (error) + goto out_free_map; /* * Count the blocks we got, make sure it matches the total. diff --git a/fs/xfs/scrub/quota_repair.c b/fs/xfs/scrub/quota_repair.c index 0bab4c30cb85a..90cd1512bba96 100644 --- a/fs/xfs/scrub/quota_repair.c +++ b/fs/xfs/scrub/quota_repair.c @@ -77,8 +77,6 @@ xrep_quota_item_fill_bmap_hole( irec, &nmaps); if (error) return error; - if (nmaps != 1) - return -ENOSPC; dq->q_blkno = XFS_FSB_TO_DADDR(mp, irec->br_startblock); @@ -444,10 +442,6 @@ xrep_quota_data_fork( XFS_BMAPI_CONVERT, 0, &nrec, &nmap); if (error) goto out; - if (nmap != 1) { - error = -ENOSPC; - goto out; - } ASSERT(nrec.br_startoff == irec.br_startoff); ASSERT(nrec.br_blockcount == irec.br_blockcount); diff --git a/fs/xfs/scrub/rtbitmap_repair.c b/fs/xfs/scrub/rtbitmap_repair.c index 46f5d5f605c91..0fef98e9f8340 100644 --- a/fs/xfs/scrub/rtbitmap_repair.c +++ b/fs/xfs/scrub/rtbitmap_repair.c @@ -108,8 +108,6 @@ xrep_rtbitmap_data_mappings( 0, &map, &nmaps); if (error) return error; - if (nmaps != 1) - return -EFSCORRUPTED; /* Commit new extent and all deferred work. */ error = xrep_defer_finish(sc); diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index 53aa90a0ee3a8..2e6f08198c071 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -721,33 +721,32 @@ xfs_alloc_file_space( if (error) goto error; - error = xfs_bmapi_write(tp, ip, startoffset_fsb, - allocatesize_fsb, XFS_BMAPI_PREALLOC, 0, imapp, - &nimaps); - if (error) - goto error; - - ip->i_diflags |= XFS_DIFLAG_PREALLOC; - xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); - - error = xfs_trans_commit(tp); - xfs_iunlock(ip, XFS_ILOCK_EXCL); - if (error) - break; - /* * If the allocator cannot find a single free extent large * enough to cover the start block of the requested range, - * xfs_bmapi_write will return 0 but leave *nimaps set to 0. + * xfs_bmapi_write will return -ENOSR. * * In that case we simply need to keep looping with the same * startoffset_fsb so that one of the following allocations * will eventually reach the requested range. */ - if (nimaps) { + error = xfs_bmapi_write(tp, ip, startoffset_fsb, + allocatesize_fsb, XFS_BMAPI_PREALLOC, 0, imapp, + &nimaps); + if (error) { + if (error != -ENOSR) + goto error; + error = 0; + } else { startoffset_fsb += imapp->br_blockcount; allocatesize_fsb -= imapp->br_blockcount; } + + ip->i_diflags |= XFS_DIFLAG_PREALLOC; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + error = xfs_trans_commit(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); } return error; diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 13aba84bd64af..43acb4f0d1743 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -357,7 +357,6 @@ xfs_dquot_disk_alloc( goto err_cancel; ASSERT(map.br_blockcount == XFS_DQUOT_CLUSTER_SIZE_FSB); - ASSERT(nmaps == 1); ASSERT((map.br_startblock != DELAYSTARTBLOCK) && (map.br_startblock != HOLESTARTBLOCK)); diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index ec43aa7cec603..f5d0ed45721b5 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -322,14 +322,6 @@ xfs_iomap_write_direct( if (error) goto out_unlock; - /* - * Copy any maps to caller's array and return any error. - */ - if (nimaps == 0) { - error = -ENOSPC; - goto out_unlock; - } - if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock))) { xfs_bmap_mark_sick(ip, XFS_DATA_FORK); error = xfs_alert_fsblock_zero(ip, imap); diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 7da0e8f961d35..5ecb52a234bec 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -430,13 +430,6 @@ xfs_reflink_fill_cow_hole( if (error) return error; - /* - * Allocation succeeded but the requested range was not even partially - * satisfied? Bail out! - */ - if (nimaps == 0) - return -ENOSPC; - convert: return xfs_reflink_convert_unwritten(ip, imap, cmap, convert_now); @@ -499,13 +492,6 @@ xfs_reflink_fill_delalloc( error = xfs_trans_commit(tp); if (error) return error; - - /* - * Allocation succeeded but the requested range was not even - * partially satisfied? Bail out! - */ - if (nimaps == 0) - return -ENOSPC; } while (cmap->br_startoff + cmap->br_blockcount <= imap->br_startoff); return xfs_reflink_convert_unwritten(ip, imap, cmap, convert_now); diff --git a/fs/xfs/xfs_rtalloc.c b/fs/xfs/xfs_rtalloc.c index b476a876478d9..150f544445ca8 100644 --- a/fs/xfs/xfs_rtalloc.c +++ b/fs/xfs/xfs_rtalloc.c @@ -709,8 +709,6 @@ xfs_growfs_rt_alloc( nmap = 1; error = xfs_bmapi_write(tp, ip, oblocks, nblocks - oblocks, XFS_BMAPI_METADATA, 0, &map, &nmap); - if (!error && nmap < 1) - error = -ENOSPC; if (error) goto out_trans_cancel; /* From b11ed354c9f725ece2b9440dd6343b42cd5d031c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Apr 2024 08:15:22 +0200 Subject: [PATCH 0902/1702] xfs: remove the unusued tmp_logflags variable in xfs_bmapi_allocate tmp_logflags is initialized to 0 and then ORed into bma->logflags, which isn't actually doing anything. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 14c9781ec0ce7..d59fc8107d695 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4182,7 +4182,6 @@ xfs_bmapi_allocate( struct xfs_mount *mp = bma->ip->i_mount; int whichfork = xfs_bmapi_whichfork(bma->flags); struct xfs_ifork *ifp = xfs_ifork_ptr(bma->ip, whichfork); - int tmp_logflags = 0; int error; ASSERT(bma->length > 0); @@ -4253,8 +4252,6 @@ xfs_bmapi_allocate( error = xfs_bmap_add_extent_hole_real(bma->tp, bma->ip, whichfork, &bma->icur, &bma->cur, &bma->got, &bma->logflags, bma->flags); - - bma->logflags |= tmp_logflags; if (error) return error; From 04c609e6e5066294b60329420d3711e990c47abf Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Apr 2024 08:15:23 +0200 Subject: [PATCH 0903/1702] xfs: lift a xfs_valid_startblock into xfs_bmapi_allocate xfs_bmapi_convert_delalloc has a xfs_valid_startblock check on the block allocated by xfs_bmapi_allocate. Lift it into xfs_bmapi_allocate as we should assert the same for xfs_bmapi_write. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index d59fc8107d695..eade9689599f2 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4221,6 +4221,11 @@ xfs_bmapi_allocate( if (bma->blkno == NULLFSBLOCK) return -ENOSPC; + if (WARN_ON_ONCE(!xfs_valid_startblock(bma->ip, bma->blkno))) { + xfs_bmap_mark_sick(bma->ip, whichfork); + return -EFSCORRUPTED; + } + if (bma->flags & XFS_BMAPI_ZERO) { error = xfs_zero_extent(bma->ip, bma->blkno, bma->length); if (error) @@ -4713,12 +4718,6 @@ xfs_bmapi_convert_one_delalloc( if (error) goto out_finish; - if (WARN_ON_ONCE(!xfs_valid_startblock(ip, bma.got.br_startblock))) { - xfs_bmap_mark_sick(ip, whichfork); - error = -EFSCORRUPTED; - goto out_finish; - } - XFS_STATS_ADD(mp, xs_xstrat_bytes, XFS_FSB_TO_B(mp, bma.length)); XFS_STATS_INC(mp, xs_xstrat_quick); From 9d06960341ec5f45d3d65bdead3fbce753455e8a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Apr 2024 08:15:24 +0200 Subject: [PATCH 0904/1702] xfs: don't open code XFS_FILBLKS_MIN in xfs_bmapi_write XFS_FILBLKS_MIN uses min_t and thus does the comparison using the correct xfs_filblks_t type. Use it in xfs_bmapi_write and slightly adjust the comment document th potential pitfall to take account of this Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index eade9689599f2..6507dcaac438e 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4528,14 +4528,11 @@ xfs_bmapi_write( * allocation length request (which can be 64 bits in * length) and the bma length request, which is * xfs_extlen_t and therefore 32 bits. Hence we have to - * check for 32-bit overflows and handle them here. + * be careful and do the min() using the larger type to + * avoid overflows. */ - if (len > (xfs_filblks_t)XFS_MAX_BMBT_EXTLEN) - bma.length = XFS_MAX_BMBT_EXTLEN; - else - bma.length = len; + bma.length = XFS_FILBLKS_MIN(len, XFS_MAX_BMBT_EXTLEN); - ASSERT(len > 0); ASSERT(bma.length > 0); error = xfs_bmapi_allocate(&bma); if (error) { From 2a9b99d45be0981536f6d3faf40ae3f58febdd49 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Apr 2024 08:15:25 +0200 Subject: [PATCH 0905/1702] xfs: pass the actual offset and len to allocate to xfs_bmapi_allocate xfs_bmapi_allocate currently overwrites offset and len when converting delayed allocations, and duplicates the length cap done for non-delalloc allocations. Move all that logic into the callers to avoid duplication and to make the calling conventions more obvious. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 6507dcaac438e..c4126f37dea3c 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4185,21 +4185,11 @@ xfs_bmapi_allocate( int error; ASSERT(bma->length > 0); + ASSERT(bma->length <= XFS_MAX_BMBT_EXTLEN); - /* - * For the wasdelay case, we could also just allocate the stuff asked - * for in this bmap call but that wouldn't be as good. - */ if (bma->wasdel) { - bma->length = (xfs_extlen_t)bma->got.br_blockcount; - bma->offset = bma->got.br_startoff; if (!xfs_iext_peek_prev_extent(ifp, &bma->icur, &bma->prev)) bma->prev.br_startoff = NULLFILEOFF; - } else { - bma->length = XFS_FILBLKS_MIN(bma->length, XFS_MAX_BMBT_EXTLEN); - if (!bma->eof) - bma->length = XFS_FILBLKS_MIN(bma->length, - bma->got.br_startoff - bma->offset); } if (bma->flags & XFS_BMAPI_CONTIG) @@ -4533,6 +4523,15 @@ xfs_bmapi_write( */ bma.length = XFS_FILBLKS_MIN(len, XFS_MAX_BMBT_EXTLEN); + if (wasdelay) { + bma.offset = bma.got.br_startoff; + bma.length = bma.got.br_blockcount; + } else { + if (!eof) + bma.length = XFS_FILBLKS_MIN(bma.length, + bma.got.br_startoff - bno); + } + ASSERT(bma.length > 0); error = xfs_bmapi_allocate(&bma); if (error) { @@ -4686,11 +4685,16 @@ xfs_bmapi_convert_one_delalloc( bma.tp = tp; bma.ip = ip; bma.wasdel = true; - bma.offset = bma.got.br_startoff; - bma.length = max_t(xfs_filblks_t, bma.got.br_blockcount, - XFS_MAX_BMBT_EXTLEN); bma.minleft = xfs_bmapi_minleft(tp, ip, whichfork); + /* + * Always allocate convert from the start of the delalloc extent even if + * that is outside the passed in range to create large contiguous + * extents on disk. + */ + bma.offset = bma.got.br_startoff; + bma.length = bma.got.br_blockcount; + /* * When we're converting the delalloc reservations backing dirty pages * in the page cache, we must be careful about how we create the new From a8bb258f703f42c322638022afa16808ca4a7d25 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Apr 2024 08:15:26 +0200 Subject: [PATCH 0906/1702] xfs: remove the xfs_iext_peek_prev_extent call in xfs_bmapi_allocate Both callers of xfs_bmapi_allocate already initialize bma->prev, don't redo that in xfs_bmapi_allocate. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index c4126f37dea3c..d0aa64b73c788 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4187,11 +4187,6 @@ xfs_bmapi_allocate( ASSERT(bma->length > 0); ASSERT(bma->length <= XFS_MAX_BMBT_EXTLEN); - if (bma->wasdel) { - if (!xfs_iext_peek_prev_extent(ifp, &bma->icur, &bma->prev)) - bma->prev.br_startoff = NULLFILEOFF; - } - if (bma->flags & XFS_BMAPI_CONTIG) bma->minlen = bma->length; else From d69bee6a35d3c5e4873b9e164dd1a9711351a97c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Apr 2024 08:15:27 +0200 Subject: [PATCH 0907/1702] xfs: fix xfs_bmap_add_extent_delay_real for partial conversions xfs_bmap_add_extent_delay_real takes parts or all of a delalloc extent and converts them to a real extent. It is written to deal with any potential overlap of the to be converted range with the delalloc extent, but it turns out that currently only converting the entire extents, or a part starting at the beginning is actually exercised, as the only caller always tries to convert the entire delalloc extent, and either succeeds or at least progresses partially from the start. If it only converts a tiny part of a delalloc extent, the indirect block calculation for the new delalloc extent (da_new) might be equivalent to that of the existing delalloc extent (da_old). If this extent conversion now requires allocating an indirect block that gets accounted into da_new, leading to the assert that da_new must be smaller or equal to da_new unless we split the extent to trigger. Except for the assert that case is actually handled by just trying to allocate more space, as that already handled for the split case (which currently can't be reached at all), so just reusing it should be fine. Except that without dipping into the reserved block pool that would make it a bit too easy to trigger a fs shutdown due to ENOSPC. So in addition to adjusting the assert, also dip into the reserved block pool. Note that I could only reproduce the assert with a change to only convert the actually asked range instead of the full delalloc extent from xfs_bmapi_write. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index d0aa64b73c788..ec204c430d536 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -1570,6 +1570,7 @@ xfs_bmap_add_extent_delay_real( if (error) goto done; } + ASSERT(da_new <= da_old); break; case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING | BMAP_LEFT_CONTIG: @@ -1600,6 +1601,7 @@ xfs_bmap_add_extent_delay_real( if (error) goto done; } + ASSERT(da_new <= da_old); break; case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG: @@ -1634,6 +1636,7 @@ xfs_bmap_add_extent_delay_real( if (error) goto done; } + ASSERT(da_new <= da_old); break; case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING: @@ -1668,6 +1671,7 @@ xfs_bmap_add_extent_delay_real( goto done; } } + ASSERT(da_new <= da_old); break; case BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG: @@ -1706,6 +1710,7 @@ xfs_bmap_add_extent_delay_real( if (error) goto done; } + ASSERT(da_new <= da_old); break; case BMAP_LEFT_FILLING: @@ -1796,6 +1801,7 @@ xfs_bmap_add_extent_delay_real( xfs_iext_update_extent(bma->ip, state, &bma->icur, &PREV); xfs_iext_next(ifp, &bma->icur); xfs_iext_update_extent(bma->ip, state, &bma->icur, &RIGHT); + ASSERT(da_new <= da_old); break; case BMAP_RIGHT_FILLING: @@ -1845,6 +1851,7 @@ xfs_bmap_add_extent_delay_real( PREV.br_blockcount = temp; xfs_iext_insert(bma->ip, &bma->icur, &PREV, state); xfs_iext_next(ifp, &bma->icur); + ASSERT(da_new <= da_old); break; case 0: @@ -1967,12 +1974,10 @@ xfs_bmap_add_extent_delay_real( } /* adjust for changes in reserved delayed indirect blocks */ - if (da_new < da_old) { + if (da_new < da_old) xfs_add_fdblocks(mp, da_old - da_new); - } else if (da_new > da_old) { - ASSERT(state == 0); - error = xfs_dec_fdblocks(mp, da_new - da_old, false); - } + else if (da_new > da_old) + error = xfs_dec_fdblocks(mp, da_new - da_old, true); xfs_bmap_check_leaf_extents(bma->cur, bma->ip, whichfork); done: From 21255afdd7296f57dd65f815301426bcf911c82d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Apr 2024 08:15:28 +0200 Subject: [PATCH 0908/1702] xfs: do not allocate the entire delalloc extent in xfs_bmapi_write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While trying to convert the entire delalloc extent is a good decision for regular writeback as it leads to larger contigous on-disk extents, but for other callers of xfs_bmapi_write is is rather questionable as it forced them to loop creating new transactions just in case there is no large enough contiguous extent to cover the whole delalloc reservation. Change xfs_bmapi_write to only allocate the passed in range instead, whіle the writeback path through xfs_bmapi_convert_delalloc and xfs_bmapi_allocate still always converts the full extents. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index ec204c430d536..be515bac8ea1e 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4524,8 +4524,9 @@ xfs_bmapi_write( bma.length = XFS_FILBLKS_MIN(len, XFS_MAX_BMBT_EXTLEN); if (wasdelay) { - bma.offset = bma.got.br_startoff; - bma.length = bma.got.br_blockcount; + bma.length = XFS_FILBLKS_MIN(bma.length, + bma.got.br_blockcount - + (bno - bma.got.br_startoff)); } else { if (!eof) bma.length = XFS_FILBLKS_MIN(bma.length, From e5b3732a9654f26d21647d9e7b4fec846f6d4810 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 30 Apr 2024 08:03:04 +0200 Subject: [PATCH 0909/1702] pinctrl: samsung: drop redundant drvdata assignment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix W=1 warning: drivers/pinctrl/samsung/pinctrl-samsung.c: In function ‘samsung_gpio_set_direction’: drivers/pinctrl/samsung/pinctrl-samsung.c:633:42: warning: variable ‘drvdata’ set but not used [-Wunused-but-set-variable] Fixes: f9c744747973 ("pinctrl: samsung: support a bus clock") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202404300825.6lxLwvUY-lkp@intel.com/ Reviewed-by: André Draszik Link: https://lore.kernel.org/r/20240430060304.12332-1-krzysztof.kozlowski@linaro.org Signed-off-by: Krzysztof Kozlowski --- drivers/pinctrl/samsung/pinctrl-samsung.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.c b/drivers/pinctrl/samsung/pinctrl-samsung.c index f4607b8493fff..623df65a5d6f9 100644 --- a/drivers/pinctrl/samsung/pinctrl-samsung.c +++ b/drivers/pinctrl/samsung/pinctrl-samsung.c @@ -630,11 +630,9 @@ static int samsung_gpio_set_direction(struct gpio_chip *gc, struct samsung_pin_bank *bank; void __iomem *reg; u32 data, mask, shift; - struct samsung_pinctrl_drv_data *drvdata; bank = gpiochip_get_data(gc); type = bank->type; - drvdata = bank->drvdata; reg = bank->pctl_base + bank->pctl_offset + type->reg_offset[PINCFG_TYPE_FUNC]; From e18fa0bbcedf82aaa1db27079ef6a43e11367592 Mon Sep 17 00:00:00 2001 From: Chiara Meiohas Date: Tue, 16 Apr 2024 15:03:50 +0300 Subject: [PATCH 0910/1702] RDMA/core: Add an option to display driver-specific QPs in the rdmatool Utilize the -dd flag (driver-specific details) in the rdmatool to view driver-specific QPs which are not exposed yet. Add the netlink attribute to mark request to convey driver details and use it to return QP subtype as a string. $ rdma resource show qp link ibp8s0f1 link ibp8s0f1/1 lqpn 360 type UD state RTS sq-psn 0 comm [mlx5_ib] link ibp8s0f1/1 lqpn 0 type SMI state RTS sq-psn 0 comm [ib_core] link ibp8s0f1/1 lqpn 1 type GSI state RTS sq-psn 0 comm [ib_core] $ rdma resource show qp link ibp8s0f1 -dd link ibp8s0f1/1 lqpn 360 type UD state RTS sq-psn 0 comm [mlx5_ib] link ibp8s0f1/1 lqpn 465 type DRIVER subtype REG_UMR state RTS sq-psn 0 comm [mlx5_ib] link ibp8s0f1/1 lqpn 0 type SMI state RTS sq-psn 0 comm [ib_core] link ibp8s0f1/1 lqpn 1 type GSI state RTS sq-psn 0 comm [ib_core] $ rdma resource show 0: ibp8s0f0: pd 3 cq 4 qp 3 cm_id 0 mr 0 ctx 0 srq 2 1: ibp8s0f1: pd 3 cq 4 qp 3 cm_id 0 mr 0 ctx 0 srq 2 $ rdma resource show -dd 0: ibp8s0f0: pd 3 cq 4 qp 4 cm_id 0 mr 0 ctx 0 srq 2 1: ibp8s0f1: pd 3 cq 4 qp 4 cm_id 0 mr 0 ctx 0 srq 2 Signed-off-by: Chiara Meiohas Link: https://lore.kernel.org/r/2607bb3ddec3cae3443c2ea19e9f700825d20a98.1713268997.git.leon@kernel.org Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/nldev.c | 23 +++++++++++++++++++---- drivers/infiniband/core/restrack.c | 12 ++++++++++-- include/rdma/restrack.h | 7 +++++-- include/uapi/rdma/rdma_netlink.h | 6 ++++++ 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index 4900a0848124d..bc79ee630d8db 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -137,6 +137,8 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = { [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME]= { .type = NLA_NUL_STRING, .len = RDMA_NLDEV_ATTR_EMPTY_STRING }, [RDMA_NLDEV_ATTR_RES_TYPE] = { .type = NLA_U8 }, + [RDMA_NLDEV_ATTR_RES_SUBTYPE] = { .type = NLA_NUL_STRING, + .len = RDMA_NLDEV_ATTR_EMPTY_STRING }, [RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY]= { .type = NLA_U32 }, [RDMA_NLDEV_ATTR_RES_USECNT] = { .type = NLA_U64 }, [RDMA_NLDEV_ATTR_RES_SRQ] = { .type = NLA_NESTED }, @@ -164,6 +166,7 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = { [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX] = { .type = NLA_U32 }, [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC] = { .type = NLA_U8 }, [RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE] = { .type = NLA_U8 }, + [RDMA_NLDEV_ATTR_DRIVER_DETAILS] = { .type = NLA_U8 }, }; static int put_driver_name_print_type(struct sk_buff *msg, const char *name, @@ -399,7 +402,8 @@ static int fill_res_info_entry(struct sk_buff *msg, return -EMSGSIZE; } -static int fill_res_info(struct sk_buff *msg, struct ib_device *device) +static int fill_res_info(struct sk_buff *msg, struct ib_device *device, + bool show_details) { static const char * const names[RDMA_RESTRACK_MAX] = { [RDMA_RESTRACK_PD] = "pd", @@ -424,7 +428,7 @@ static int fill_res_info(struct sk_buff *msg, struct ib_device *device) for (i = 0; i < RDMA_RESTRACK_MAX; i++) { if (!names[i]) continue; - curr = rdma_restrack_count(device, i); + curr = rdma_restrack_count(device, i, show_details); ret = fill_res_info_entry(msg, names[i], curr); if (ret) goto err; @@ -1305,6 +1309,7 @@ static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh, struct netlink_ext_ack *extack) { struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; + bool show_details = false; struct ib_device *device; struct sk_buff *msg; u32 index; @@ -1320,6 +1325,9 @@ static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh, if (!device) return -EINVAL; + if (tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]) + show_details = nla_get_u8(tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]); + msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); if (!msg) { ret = -ENOMEM; @@ -1334,7 +1342,7 @@ static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh, goto err_free; } - ret = fill_res_info(msg, device); + ret = fill_res_info(msg, device, show_details); if (ret) goto err_free; @@ -1364,7 +1372,7 @@ static int _nldev_res_get_dumpit(struct ib_device *device, RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET), 0, NLM_F_MULTI); - if (!nlh || fill_res_info(skb, device)) { + if (!nlh || fill_res_info(skb, device, false)) { nlmsg_cancel(skb, nlh); goto out; } @@ -1534,6 +1542,7 @@ static int res_get_common_dumpit(struct sk_buff *skb, struct rdma_restrack_entry *res; struct rdma_restrack_root *rt; int err, ret = 0, idx = 0; + bool show_details = false; struct nlattr *table_attr; struct nlattr *entry_attr; struct ib_device *device; @@ -1562,6 +1571,9 @@ static int res_get_common_dumpit(struct sk_buff *skb, if (!device) return -EINVAL; + if (tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]) + show_details = nla_get_u8(tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]); + /* * If no PORT_INDEX is supplied, we will return all QPs from that device */ @@ -1599,6 +1611,9 @@ static int res_get_common_dumpit(struct sk_buff *skb, * objects. */ xa_for_each(&rt->xa, id, res) { + if (xa_get_mark(&rt->xa, res->id, RESTRACK_DD) && !show_details) + goto next; + if (idx < start || !rdma_restrack_get(res)) goto next; diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c index 438ed35881752..3313410014cd5 100644 --- a/drivers/infiniband/core/restrack.c +++ b/drivers/infiniband/core/restrack.c @@ -59,8 +59,10 @@ void rdma_restrack_clean(struct ib_device *dev) * rdma_restrack_count() - the current usage of specific object * @dev: IB device * @type: actual type of object to operate + * @show_details: count driver specific objects */ -int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type) +int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type, + bool show_details) { struct rdma_restrack_root *rt = &dev->res[type]; struct rdma_restrack_entry *e; @@ -68,8 +70,11 @@ int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type) u32 cnt = 0; xa_lock(&rt->xa); - xas_for_each(&xas, e, U32_MAX) + xas_for_each(&xas, e, U32_MAX) { + if (xa_get_mark(&rt->xa, e->id, RESTRACK_DD) && !show_details) + continue; cnt++; + } xa_unlock(&rt->xa); return cnt; } @@ -198,6 +203,9 @@ void rdma_restrack_add(struct rdma_restrack_entry *res) ret = xa_insert(&rt->xa, res->id, res, GFP_KERNEL); if (ret) res->id = 0; + + if (qp->qp_type >= IB_QPT_DRIVER) + xa_set_mark(&rt->xa, res->id, RESTRACK_DD); } else if (res->type == RDMA_RESTRACK_COUNTER) { /* Special case to ensure that cntn points to right counter */ struct rdma_counter *counter; diff --git a/include/rdma/restrack.h b/include/rdma/restrack.h index 8b7c46daeb078..0d69ded73bf24 100644 --- a/include/rdma/restrack.h +++ b/include/rdma/restrack.h @@ -14,6 +14,9 @@ #include #include +/* Mark entry as containing driver specific details, it is used to provide QP subtype for now */ +#define RESTRACK_DD XA_MARK_1 + struct ib_device; struct sk_buff; @@ -116,8 +119,8 @@ struct rdma_restrack_entry { u32 id; }; -int rdma_restrack_count(struct ib_device *dev, - enum rdma_restrack_type type); +int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type, + bool show_details); /** * rdma_is_kernel_res() - check the owner of resource * @res: resource entry diff --git a/include/uapi/rdma/rdma_netlink.h b/include/uapi/rdma/rdma_netlink.h index 723bbb0f70421..a214fc259f28c 100644 --- a/include/uapi/rdma/rdma_netlink.h +++ b/include/uapi/rdma/rdma_netlink.h @@ -558,6 +558,12 @@ enum rdma_nldev_attr { RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE, /* u8 */ + RDMA_NLDEV_ATTR_DRIVER_DETAILS, /* u8 */ + /* + * QP subtype string, used for driver QPs + */ + RDMA_NLDEV_ATTR_RES_SUBTYPE, /* string */ + /* * Always the end */ From fd3af5e21866b776713b8c60556153d758995fb7 Mon Sep 17 00:00:00 2001 From: Chiara Meiohas Date: Tue, 16 Apr 2024 15:03:51 +0300 Subject: [PATCH 0911/1702] RDMA/mlx5: Track DCT, DCI and REG_UMR QPs as diver_detail resources. Allow user to see driver-specific QPs (the "driver_detail" QPs) through the rdmatool, when requested. When creating DCT, DCI and REG_UMR QPs, we designate them as driver_detail resources. When filling the QP info for the rdma tool, for the driver_detail QPs: -the QP type is IB_QPT_DRIVER -the subtype is a string with the QP name ("DCT", "DCI", "REG_UMR") Signed-off-by: Chiara Meiohas Link: https://lore.kernel.org/r/452432d7d0917f053a80a893a614169857fe3b10.1713268997.git.leon@kernel.org Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/qp.c | 3 +-- drivers/infiniband/hw/mlx5/restrack.c | 29 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 8115ab107149f..e2164f813607a 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -3097,7 +3097,6 @@ static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, switch (qp->type) { case MLX5_IB_QPT_DCT: err = create_dct(dev, pd, qp, params); - rdma_restrack_no_track(&qp->ibqp.res); break; case MLX5_IB_QPT_DCI: err = create_dci(dev, pd, qp, params); @@ -3109,9 +3108,9 @@ static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, err = mlx5_ib_create_gsi(pd, qp, params->attr); break; case MLX5_IB_QPT_HW_GSI: - case MLX5_IB_QPT_REG_UMR: rdma_restrack_no_track(&qp->ibqp.res); fallthrough; + case MLX5_IB_QPT_REG_UMR: default: if (params->udata) err = create_user_qp(dev, pd, qp, params); diff --git a/drivers/infiniband/hw/mlx5/restrack.c b/drivers/infiniband/hw/mlx5/restrack.c index 4ac429e72004c..affcf8fe943c1 100644 --- a/drivers/infiniband/hw/mlx5/restrack.c +++ b/drivers/infiniband/hw/mlx5/restrack.c @@ -156,6 +156,34 @@ static int fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ibcq) return fill_res_raw(msg, dev, MLX5_SGMT_TYPE_PRM_QUERY_CQ, cq->mcq.cqn); } +static int fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp) +{ + struct mlx5_ib_qp *qp = to_mqp(ibqp); + int ret; + + if (qp->type < IB_QPT_DRIVER) + return 0; + + switch (qp->type) { + case MLX5_IB_QPT_REG_UMR: + ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, + "REG_UMR"); + break; + case MLX5_IB_QPT_DCT: + ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "DCT"); + break; + case MLX5_IB_QPT_DCI: + ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "DCI"); + break; + default: + return 0; + } + if (ret) + return ret; + + return nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, IB_QPT_DRIVER); +} + static int fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ibqp) { struct mlx5_ib_dev *dev = to_mdev(ibqp->device); @@ -168,6 +196,7 @@ static const struct ib_device_ops restrack_ops = { .fill_res_cq_entry_raw = fill_res_cq_entry_raw, .fill_res_mr_entry = fill_res_mr_entry, .fill_res_mr_entry_raw = fill_res_mr_entry_raw, + .fill_res_qp_entry = fill_res_qp_entry, .fill_res_qp_entry_raw = fill_res_qp_entry_raw, .fill_stat_mr_entry = fill_stat_mr_entry, }; From 55f7073f6f59ef2c9e98b70f74118dba62e1aabc Mon Sep 17 00:00:00 2001 From: Herman van Hazendonk Date: Mon, 15 Apr 2024 17:16:45 +0200 Subject: [PATCH 0912/1702] dt-bindings: power: supply: max8903: specify flt-gpios as input The FLT pin was incorrected documented as an output. The MAX8903 uses it to signal to the processor that a charging fault has occurred. Signed-off-by: Herman van Hazendonk Link: https://lore.kernel.org/r/20240415151645.1986014-1-github.com@herrie.org Signed-off-by: Sebastian Reichel --- .../devicetree/bindings/power/supply/maxim,max8903.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/power/supply/maxim,max8903.yaml b/Documentation/devicetree/bindings/power/supply/maxim,max8903.yaml index a8d625f285f1f..86af383789992 100644 --- a/Documentation/devicetree/bindings/power/supply/maxim,max8903.yaml +++ b/Documentation/devicetree/bindings/power/supply/maxim,max8903.yaml @@ -34,7 +34,7 @@ properties: flt-gpios: maxItems: 1 - description: Fault pin (active low, output) + description: Fault pin (active low, input) dcm-gpios: maxItems: 1 From 0ef11f604503b1862a21597436283f158114d77e Mon Sep 17 00:00:00 2001 From: Jean Delvare Date: Tue, 30 Apr 2024 18:29:32 +0200 Subject: [PATCH 0913/1702] firmware: dmi: Stop decoding on broken entry If a DMI table entry is shorter than 4 bytes, it is invalid. Due to how DMI table parsing works, it is impossible to safely recover from such an error, so we have to stop decoding the table. Signed-off-by: Jean Delvare Link: https://lore.kernel.org/linux-kernel/Zh2K3-HLXOesT_vZ@liuwe-devbox-debian-v2/T/ Reviewed-by: Michael Kelley --- drivers/firmware/dmi_scan.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/firmware/dmi_scan.c b/drivers/firmware/dmi_scan.c index 015c95a825d31..ac2a5d2d47463 100644 --- a/drivers/firmware/dmi_scan.c +++ b/drivers/firmware/dmi_scan.c @@ -101,6 +101,17 @@ static void dmi_decode_table(u8 *buf, (data - buf + sizeof(struct dmi_header)) <= dmi_len) { const struct dmi_header *dm = (const struct dmi_header *)data; + /* + * If a short entry is found (less than 4 bytes), not only it + * is invalid, but we cannot reliably locate the next entry. + */ + if (dm->length < sizeof(struct dmi_header)) { + pr_warn(FW_BUG + "Corrupted DMI table, offset %zd (only %d entries processed)\n", + data - buf, i); + break; + } + /* * We want to know the total length (formatted area and * strings) before decoding to make sure we won't run off the From 196eca020600470ca44da94c65607e7a98aa9d3c Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 27 Mar 2024 22:35:03 +0800 Subject: [PATCH 0914/1702] tools/power turbostat: Enhance ARL/LNL support ARL/LNL don't have PC8, other than that, it behaves the same as CNL. Copy cnl_features for ARL/LNL, except that PC8 support is removed. Signed-off-by: Zhang Rui Signed-off-by: Len Brown --- tools/power/x86/turbostat/turbostat.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index 4b95fd90e16c3..672936015b55a 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -663,6 +663,23 @@ static const struct platform_features adl_features = { .enable_tsc_tweak = 1, }; +static const struct platform_features arl_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC10, + .cst_limit = CST_LIMIT_HSW, + .has_irtl_msrs = 1, + .has_msr_core_c1_res = 1, + .has_ext_cst_msrs = 1, + .trl_msrs = TRL_BASE, + .tcc_offset_bits = 6, + .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX, + .enable_tsc_tweak = 1, +}; + static const struct platform_features skx_features = { .has_msr_misc_feature_control = 1, .has_msr_misc_pwr_mgmt = 1, @@ -905,8 +922,8 @@ static const struct platform_data turbostat_pdata[] = { { INTEL_FAM6_RAPTORLAKE_S, &adl_features }, { INTEL_FAM6_METEORLAKE, &cnl_features }, { INTEL_FAM6_METEORLAKE_L, &cnl_features }, - { INTEL_FAM6_ARROWLAKE, &cnl_features }, - { INTEL_FAM6_LUNARLAKE_M, &cnl_features }, + { INTEL_FAM6_ARROWLAKE, &arl_features }, + { INTEL_FAM6_LUNARLAKE_M, &arl_features }, { INTEL_FAM6_ATOM_SILVERMONT, &slv_features }, { INTEL_FAM6_ATOM_SILVERMONT_D, &slvd_features }, { INTEL_FAM6_ATOM_AIRMONT, &amt_features }, From f04fcc7ac8ceb87933244cca28759d0fac6103ce Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 27 Mar 2024 22:36:38 +0800 Subject: [PATCH 0915/1702] tools/power turbostat: Add ARL-H support Add turbostat support for ARL-H, which behaves the same as ARL. [lenb: also add ARL-U] Signed-off-by: Zhang Rui Signed-off-by: Len Brown --- tools/power/x86/turbostat/turbostat.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index 672936015b55a..fadf96934f4e1 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -922,6 +922,8 @@ static const struct platform_data turbostat_pdata[] = { { INTEL_FAM6_RAPTORLAKE_S, &adl_features }, { INTEL_FAM6_METEORLAKE, &cnl_features }, { INTEL_FAM6_METEORLAKE_L, &cnl_features }, + { INTEL_FAM6_ARROWLAKE_H, &arl_features }, + { INTEL_FAM6_ARROWLAKE_U, &arl_features }, { INTEL_FAM6_ARROWLAKE, &arl_features }, { INTEL_FAM6_LUNARLAKE_M, &arl_features }, { INTEL_FAM6_ATOM_SILVERMONT, &slv_features }, From d3e6f6253895f499b63bac261b81732f9efc4902 Mon Sep 17 00:00:00 2001 From: Patryk Wlazlyn Date: Tue, 23 Apr 2024 11:59:49 +0200 Subject: [PATCH 0916/1702] tools/power turbostat: Replace _Static_assert with BUILD_BUG_ON So it compiles on GCC older than 9.0. Signed-off-by: Patryk Wlazlyn Signed-off-by: Len Brown --- tools/power/x86/turbostat/turbostat.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index fadf96934f4e1..bd6cb31b70996 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -38,6 +38,7 @@ #include #include #include +#include #define UNUSED(x) (void)(x) @@ -3467,7 +3468,7 @@ int get_rapl_counters(int cpu, int domain, struct core_data *c, struct pkg_data } } - _Static_assert(NUM_RAPL_COUNTERS == 7); + BUILD_BUG_ON(NUM_RAPL_COUNTERS != 7); write_rapl_counter(&p->energy_pkg, rci, RAPL_RCI_INDEX_ENERGY_PKG); write_rapl_counter(&p->energy_cores, rci, RAPL_RCI_INDEX_ENERGY_CORES); write_rapl_counter(&p->energy_dram, rci, RAPL_RCI_INDEX_DRAM); From 0e39702fbbcdb16ad349439065d24a3bb5e2f331 Mon Sep 17 00:00:00 2001 From: Patryk Wlazlyn Date: Thu, 25 Apr 2024 17:54:18 +0200 Subject: [PATCH 0917/1702] tools/power turbostat: Enable non-privileged users to read sysfs counters A group of counters called "sysfs" displays software C-state request counts and resulting perceived C-state residency. They are not built-in counters that turbostat knows about ahead of time, rather they are discovered in sysfs when turbostat starts. Thus, they are added dynamically, using the same interface as user-added MSR counters. When turbostat enters "no-msr" mode, such as when running as a non-privileged user, it clears all added counters. Updating that to clear only actual MSR added counters allows regular users to see the sysfs counters. [lenb: commit message] Signed-off-by: Patryk Wlazlyn Signed-off-by: Len Brown --- tools/power/x86/turbostat/turbostat.c | 56 +++++++++++++++------------ 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index bd6cb31b70996..f92b46cfda31b 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -1377,36 +1377,42 @@ struct sys_counters { struct msr_counter *pp; } sys; -void free_sys_counters(void) +static size_t free_msr_counters_(struct msr_counter **pp) { - struct msr_counter *p = sys.tp, *pnext = NULL; + struct msr_counter *p = NULL; + size_t num_freed = 0; - while (p) { - pnext = p->next; - free(p); - p = pnext; - } + while (*pp) { + p = *pp; - p = sys.cp, pnext = NULL; - while (p) { - pnext = p->next; - free(p); - p = pnext; - } + if (p->msr_num != 0) { + *pp = p->next; - p = sys.pp, pnext = NULL; - while (p) { - pnext = p->next; - free(p); - p = pnext; + free(p); + ++num_freed; + + continue; + } + + pp = &p->next; } - sys.added_thread_counters = 0; - sys.added_core_counters = 0; - sys.added_package_counters = 0; - sys.tp = NULL; - sys.cp = NULL; - sys.pp = NULL; + return num_freed; +} + +/* + * Free all added counters accessed via msr. + */ +static void free_sys_msr_counters(void) +{ + /* Thread counters */ + sys.added_thread_counters -= free_msr_counters_(&sys.tp); + + /* Core counters */ + sys.added_core_counters -= free_msr_counters_(&sys.cp); + + /* Package counters */ + sys.added_package_counters -= free_msr_counters_(&sys.pp); } struct system_summary { @@ -1566,7 +1572,7 @@ static void bic_disable_msr_access(void) bic_enabled &= ~bic_msrs; - free_sys_counters(); + free_sys_msr_counters(); } static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags) From 57939f392371fc93c203b781807c951018c29606 Mon Sep 17 00:00:00 2001 From: Fabio Estevam Date: Mon, 29 Apr 2024 18:45:02 -0300 Subject: [PATCH 0918/1702] clk: imx: imx8mp: Switch to RUNTIME_PM_OPS() Replace SET_RUNTIME_PM_OPS() with its modern alternative RUNTIME_PM_OPS(). The combined usage of pm_ptr() and RUNTIME_PM_OPS() allows the compiler to evaluate if the suspend/resume() functions are used at buid time or are simply dead code. This fixes the following s390 allmodconfig build errors: drivers/clk/imx/clk-imx8mp-audiomix.c:363:12: error: 'clk_imx8mp_audiomix_runtime_resume' defined but not used [-Werror=unused-function] 363 | static int clk_imx8mp_audiomix_runtime_resume(struct device *dev) | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ drivers/clk/imx/clk-imx8mp-audiomix.c:356:12: error: 'clk_imx8mp_audiomix_runtime_suspend' defined but not used [-Werror=unused-function] 356 | static int clk_imx8mp_audiomix_runtime_suspend(struct device *dev) | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ cc1: all warnings being treated as errors Reported-by: Naresh Kamboju Closes: https://lore.kernel.org/linux-clk/CA+G9fYuP7S+a89Ep5g5_Ad69EMwRkJ8nM+MMTzbEcP+6H2oMXQ@mail.gmail.com/T/#u Fixes: 1496dd413b2e ("clk: imx: imx8mp: Add pm_runtime support for power saving") Signed-off-by: Fabio Estevam Reviewed-by: Peng Fan Link: https://lore.kernel.org/r/20240429214502.1363592-1-festevam@gmail.com Signed-off-by: Abel Vesa --- drivers/clk/imx/clk-imx8mp-audiomix.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/clk/imx/clk-imx8mp-audiomix.c b/drivers/clk/imx/clk-imx8mp-audiomix.c index 574a032309c1b..6a9b48b20cd60 100644 --- a/drivers/clk/imx/clk-imx8mp-audiomix.c +++ b/drivers/clk/imx/clk-imx8mp-audiomix.c @@ -368,8 +368,8 @@ static int clk_imx8mp_audiomix_runtime_resume(struct device *dev) } static const struct dev_pm_ops clk_imx8mp_audiomix_pm_ops = { - SET_RUNTIME_PM_OPS(clk_imx8mp_audiomix_runtime_suspend, - clk_imx8mp_audiomix_runtime_resume, NULL) + RUNTIME_PM_OPS(clk_imx8mp_audiomix_runtime_suspend, + clk_imx8mp_audiomix_runtime_resume, NULL) SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) }; @@ -386,7 +386,7 @@ static struct platform_driver clk_imx8mp_audiomix_driver = { .driver = { .name = "imx8mp-audio-blk-ctrl", .of_match_table = clk_imx8mp_audiomix_of_match, - .pm = &clk_imx8mp_audiomix_pm_ops, + .pm = pm_ptr(&clk_imx8mp_audiomix_pm_ops), }, }; From f5072cffb35c122ec85d91ef327fa8814f04297b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 23 Apr 2024 09:12:31 +0200 Subject: [PATCH 0919/1702] clk: imx: imx8mp: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). Trivially convert this driver from always returning zero in the remove callback to the void returning variant. Fixes: 1496dd413b2e ("clk: imx: imx8mp: Add pm_runtime support for power saving") Signed-off-by: Uwe Kleine-König Reviewed-by: Abel Vesa Link: https://lore.kernel.org/r/20240423071232.463201-2-u.kleine-koenig@pengutronix.de Signed-off-by: Abel Vesa --- drivers/clk/imx/clk-imx8mp-audiomix.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/clk/imx/clk-imx8mp-audiomix.c b/drivers/clk/imx/clk-imx8mp-audiomix.c index 6a9b48b20cd60..b381d6f784c89 100644 --- a/drivers/clk/imx/clk-imx8mp-audiomix.c +++ b/drivers/clk/imx/clk-imx8mp-audiomix.c @@ -346,11 +346,9 @@ static int clk_imx8mp_audiomix_probe(struct platform_device *pdev) return ret; } -static int clk_imx8mp_audiomix_remove(struct platform_device *pdev) +static void clk_imx8mp_audiomix_remove(struct platform_device *pdev) { pm_runtime_disable(&pdev->dev); - - return 0; } static int clk_imx8mp_audiomix_runtime_suspend(struct device *dev) @@ -382,7 +380,7 @@ MODULE_DEVICE_TABLE(of, clk_imx8mp_audiomix_of_match); static struct platform_driver clk_imx8mp_audiomix_driver = { .probe = clk_imx8mp_audiomix_probe, - .remove = clk_imx8mp_audiomix_remove, + .remove_new = clk_imx8mp_audiomix_remove, .driver = { .name = "imx8mp-audio-blk-ctrl", .of_match_table = clk_imx8mp_audiomix_of_match, From 3eaea21b4d27cff0017c20549aeb53034c58fc23 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 18 Mar 2024 11:17:26 -0700 Subject: [PATCH 0920/1702] uprobes: encapsulate preparation of uprobe args buffer Move the logic of fetching temporary per-CPU uprobe buffer and storing uprobes args into it to a new helper function. Store data size as part of this buffer, simplifying interfaces a bit, as now we only pass single uprobe_cpu_buffer reference around, instead of pointer + dsize. This logic was duplicated across uprobe_dispatcher and uretprobe_dispatcher, and now will be centralized. All this is also in preparation to make this uprobe_cpu_buffer handling logic optional in the next patch. Link: https://lore.kernel.org/all/20240318181728.2795838-2-andrii@kernel.org/ [Masami: update for v6.9-rc3 kernel] Signed-off-by: Andrii Nakryiko Reviewed-by: Jiri Olsa Acked-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_uprobe.c | 78 +++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index 9e461362450ae..155cba8e6e7e9 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -854,6 +854,7 @@ static const struct file_operations uprobe_profile_ops = { struct uprobe_cpu_buffer { struct mutex mutex; void *buf; + int dsize; }; static struct uprobe_cpu_buffer __percpu *uprobe_cpu_buffer; static int uprobe_buffer_refcnt; @@ -943,9 +944,26 @@ static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb) mutex_unlock(&ucb->mutex); } +static struct uprobe_cpu_buffer *prepare_uprobe_buffer(struct trace_uprobe *tu, + struct pt_regs *regs) +{ + struct uprobe_cpu_buffer *ucb; + int dsize, esize; + + esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); + dsize = __get_data_size(&tu->tp, regs, NULL); + + ucb = uprobe_buffer_get(); + ucb->dsize = tu->tp.size + dsize; + + store_trace_args(ucb->buf, &tu->tp, regs, NULL, esize, dsize); + + return ucb; +} + static void __uprobe_trace_func(struct trace_uprobe *tu, unsigned long func, struct pt_regs *regs, - struct uprobe_cpu_buffer *ucb, int dsize, + struct uprobe_cpu_buffer *ucb, struct trace_event_file *trace_file) { struct uprobe_trace_entry_head *entry; @@ -956,14 +974,14 @@ static void __uprobe_trace_func(struct trace_uprobe *tu, WARN_ON(call != trace_file->event_call); - if (WARN_ON_ONCE(tu->tp.size + dsize > PAGE_SIZE)) + if (WARN_ON_ONCE(ucb->dsize > PAGE_SIZE)) return; if (trace_trigger_soft_disabled(trace_file)) return; esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); - size = esize + tu->tp.size + dsize; + size = esize + ucb->dsize; entry = trace_event_buffer_reserve(&fbuffer, trace_file, size); if (!entry) return; @@ -977,14 +995,14 @@ static void __uprobe_trace_func(struct trace_uprobe *tu, data = DATAOF_TRACE_ENTRY(entry, false); } - memcpy(data, ucb->buf, tu->tp.size + dsize); + memcpy(data, ucb->buf, ucb->dsize); trace_event_buffer_commit(&fbuffer); } /* uprobe handler */ static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs, - struct uprobe_cpu_buffer *ucb, int dsize) + struct uprobe_cpu_buffer *ucb) { struct event_file_link *link; @@ -993,7 +1011,7 @@ static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs, rcu_read_lock(); trace_probe_for_each_link_rcu(link, &tu->tp) - __uprobe_trace_func(tu, 0, regs, ucb, dsize, link->file); + __uprobe_trace_func(tu, 0, regs, ucb, link->file); rcu_read_unlock(); return 0; @@ -1001,13 +1019,13 @@ static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs, static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func, struct pt_regs *regs, - struct uprobe_cpu_buffer *ucb, int dsize) + struct uprobe_cpu_buffer *ucb) { struct event_file_link *link; rcu_read_lock(); trace_probe_for_each_link_rcu(link, &tu->tp) - __uprobe_trace_func(tu, func, regs, ucb, dsize, link->file); + __uprobe_trace_func(tu, func, regs, ucb, link->file); rcu_read_unlock(); } @@ -1335,7 +1353,7 @@ static bool uprobe_perf_filter(struct uprobe_consumer *uc, static void __uprobe_perf_func(struct trace_uprobe *tu, unsigned long func, struct pt_regs *regs, - struct uprobe_cpu_buffer *ucb, int dsize) + struct uprobe_cpu_buffer *ucb) { struct trace_event_call *call = trace_probe_event_call(&tu->tp); struct uprobe_trace_entry_head *entry; @@ -1356,7 +1374,7 @@ static void __uprobe_perf_func(struct trace_uprobe *tu, esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); - size = esize + tu->tp.size + dsize; + size = esize + ucb->dsize; size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32); if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough")) return; @@ -1379,13 +1397,10 @@ static void __uprobe_perf_func(struct trace_uprobe *tu, data = DATAOF_TRACE_ENTRY(entry, false); } - memcpy(data, ucb->buf, tu->tp.size + dsize); + memcpy(data, ucb->buf, ucb->dsize); - if (size - esize > tu->tp.size + dsize) { - int len = tu->tp.size + dsize; - - memset(data + len, 0, size - esize - len); - } + if (size - esize > ucb->dsize) + memset(data + ucb->dsize, 0, size - esize - ucb->dsize); perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, head, NULL); @@ -1395,21 +1410,21 @@ static void __uprobe_perf_func(struct trace_uprobe *tu, /* uprobe profile handler */ static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs, - struct uprobe_cpu_buffer *ucb, int dsize) + struct uprobe_cpu_buffer *ucb) { if (!uprobe_perf_filter(&tu->consumer, 0, current->mm)) return UPROBE_HANDLER_REMOVE; if (!is_ret_probe(tu)) - __uprobe_perf_func(tu, 0, regs, ucb, dsize); + __uprobe_perf_func(tu, 0, regs, ucb); return 0; } static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func, struct pt_regs *regs, - struct uprobe_cpu_buffer *ucb, int dsize) + struct uprobe_cpu_buffer *ucb) { - __uprobe_perf_func(tu, func, regs, ucb, dsize); + __uprobe_perf_func(tu, func, regs, ucb); } int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type, @@ -1475,10 +1490,8 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs) struct trace_uprobe *tu; struct uprobe_dispatch_data udd; struct uprobe_cpu_buffer *ucb; - int dsize, esize; int ret = 0; - tu = container_of(con, struct trace_uprobe, consumer); tu->nhit++; @@ -1490,18 +1503,14 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs) if (WARN_ON_ONCE(!uprobe_cpu_buffer)) return 0; - dsize = __get_data_size(&tu->tp, regs, NULL); - esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); - - ucb = uprobe_buffer_get(); - store_trace_args(ucb->buf, &tu->tp, regs, NULL, esize, dsize); + ucb = prepare_uprobe_buffer(tu, regs); if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE)) - ret |= uprobe_trace_func(tu, regs, ucb, dsize); + ret |= uprobe_trace_func(tu, regs, ucb); #ifdef CONFIG_PERF_EVENTS if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE)) - ret |= uprobe_perf_func(tu, regs, ucb, dsize); + ret |= uprobe_perf_func(tu, regs, ucb); #endif uprobe_buffer_put(ucb); return ret; @@ -1513,7 +1522,6 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con, struct trace_uprobe *tu; struct uprobe_dispatch_data udd; struct uprobe_cpu_buffer *ucb; - int dsize, esize; tu = container_of(con, struct trace_uprobe, consumer); @@ -1525,18 +1533,14 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con, if (WARN_ON_ONCE(!uprobe_cpu_buffer)) return 0; - dsize = __get_data_size(&tu->tp, regs, NULL); - esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); - - ucb = uprobe_buffer_get(); - store_trace_args(ucb->buf, &tu->tp, regs, NULL, esize, dsize); + ucb = prepare_uprobe_buffer(tu, regs); if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE)) - uretprobe_trace_func(tu, func, regs, ucb, dsize); + uretprobe_trace_func(tu, func, regs, ucb); #ifdef CONFIG_PERF_EVENTS if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE)) - uretprobe_perf_func(tu, func, regs, ucb, dsize); + uretprobe_perf_func(tu, func, regs, ucb); #endif uprobe_buffer_put(ucb); return 0; From 1b8f85defbc82e2eb8f27c5f6060ea507ad4d5a3 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 18 Mar 2024 11:17:27 -0700 Subject: [PATCH 0921/1702] uprobes: prepare uprobe args buffer lazily MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit uprobe_cpu_buffer and corresponding logic to store uprobe args into it are used for uprobes/uretprobes that are created through tracefs or perf events. BPF is yet another user of uprobe/uretprobe infrastructure, but doesn't need uprobe_cpu_buffer and associated data. For BPF-only use cases this buffer handling and preparation is a pure overhead. At the same time, BPF-only uprobe/uretprobe usage is very common in practice. Also, for a lot of cases applications are very senstivie to performance overheads, as they might be tracing a very high frequency functions like malloc()/free(), so every bit of performance improvement matters. All that is to say that this uprobe_cpu_buffer preparation is an unnecessary overhead that each BPF user of uprobes/uretprobe has to pay. This patch is changing this by making uprobe_cpu_buffer preparation optional. It will happen only if either tracefs-based or perf event-based uprobe/uretprobe consumer is registered for given uprobe/uretprobe. For BPF-only use cases this step will be skipped. We used uprobe/uretprobe benchmark which is part of BPF selftests (see [0]) to estimate the improvements. We have 3 uprobe and 3 uretprobe scenarios, which vary an instruction that is replaced by uprobe: nop (fastest uprobe case), `push rbp` (typical case), and non-simulated `ret` instruction (slowest case). Benchmark thread is constantly calling user space function in a tight loop. User space function has attached BPF uprobe or uretprobe program doing nothing but atomic counter increments to count number of triggering calls. Benchmark emits throughput in millions of executions per second. BEFORE these changes ==================== uprobe-nop : 2.657 ± 0.024M/s uprobe-push : 2.499 ± 0.018M/s uprobe-ret : 1.100 ± 0.006M/s uretprobe-nop : 1.356 ± 0.004M/s uretprobe-push : 1.317 ± 0.019M/s uretprobe-ret : 0.785 ± 0.007M/s AFTER these changes =================== uprobe-nop : 2.732 ± 0.022M/s (+2.8%) uprobe-push : 2.621 ± 0.016M/s (+4.9%) uprobe-ret : 1.105 ± 0.007M/s (+0.5%) uretprobe-nop : 1.396 ± 0.007M/s (+2.9%) uretprobe-push : 1.347 ± 0.008M/s (+2.3%) uretprobe-ret : 0.800 ± 0.006M/s (+1.9) So the improvements on this particular machine seems to be between 2% and 5%. [0] https://github.com/torvalds/linux/blob/master/tools/testing/selftests/bpf/benchs/bench_trigger.c Reviewed-by: Jiri Olsa Link: https://lore.kernel.org/all/20240318181728.2795838-3-andrii@kernel.org/ Signed-off-by: Andrii Nakryiko Acked-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_uprobe.c | 49 +++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index 155cba8e6e7e9..cbab487e85228 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -941,15 +941,21 @@ static struct uprobe_cpu_buffer *uprobe_buffer_get(void) static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb) { + if (!ucb) + return; mutex_unlock(&ucb->mutex); } static struct uprobe_cpu_buffer *prepare_uprobe_buffer(struct trace_uprobe *tu, - struct pt_regs *regs) + struct pt_regs *regs, + struct uprobe_cpu_buffer **ucbp) { struct uprobe_cpu_buffer *ucb; int dsize, esize; + if (*ucbp) + return *ucbp; + esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); dsize = __get_data_size(&tu->tp, regs, NULL); @@ -958,22 +964,25 @@ static struct uprobe_cpu_buffer *prepare_uprobe_buffer(struct trace_uprobe *tu, store_trace_args(ucb->buf, &tu->tp, regs, NULL, esize, dsize); + *ucbp = ucb; return ucb; } static void __uprobe_trace_func(struct trace_uprobe *tu, unsigned long func, struct pt_regs *regs, - struct uprobe_cpu_buffer *ucb, + struct uprobe_cpu_buffer **ucbp, struct trace_event_file *trace_file) { struct uprobe_trace_entry_head *entry; struct trace_event_buffer fbuffer; + struct uprobe_cpu_buffer *ucb; void *data; int size, esize; struct trace_event_call *call = trace_probe_event_call(&tu->tp); WARN_ON(call != trace_file->event_call); + ucb = prepare_uprobe_buffer(tu, regs, ucbp); if (WARN_ON_ONCE(ucb->dsize > PAGE_SIZE)) return; @@ -1002,7 +1011,7 @@ static void __uprobe_trace_func(struct trace_uprobe *tu, /* uprobe handler */ static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs, - struct uprobe_cpu_buffer *ucb) + struct uprobe_cpu_buffer **ucbp) { struct event_file_link *link; @@ -1011,7 +1020,7 @@ static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs, rcu_read_lock(); trace_probe_for_each_link_rcu(link, &tu->tp) - __uprobe_trace_func(tu, 0, regs, ucb, link->file); + __uprobe_trace_func(tu, 0, regs, ucbp, link->file); rcu_read_unlock(); return 0; @@ -1019,13 +1028,13 @@ static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs, static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func, struct pt_regs *regs, - struct uprobe_cpu_buffer *ucb) + struct uprobe_cpu_buffer **ucbp) { struct event_file_link *link; rcu_read_lock(); trace_probe_for_each_link_rcu(link, &tu->tp) - __uprobe_trace_func(tu, func, regs, ucb, link->file); + __uprobe_trace_func(tu, func, regs, ucbp, link->file); rcu_read_unlock(); } @@ -1353,10 +1362,11 @@ static bool uprobe_perf_filter(struct uprobe_consumer *uc, static void __uprobe_perf_func(struct trace_uprobe *tu, unsigned long func, struct pt_regs *regs, - struct uprobe_cpu_buffer *ucb) + struct uprobe_cpu_buffer **ucbp) { struct trace_event_call *call = trace_probe_event_call(&tu->tp); struct uprobe_trace_entry_head *entry; + struct uprobe_cpu_buffer *ucb; struct hlist_head *head; void *data; int size, esize; @@ -1374,6 +1384,7 @@ static void __uprobe_perf_func(struct trace_uprobe *tu, esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); + ucb = prepare_uprobe_buffer(tu, regs, ucbp); size = esize + ucb->dsize; size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32); if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough")) @@ -1410,21 +1421,21 @@ static void __uprobe_perf_func(struct trace_uprobe *tu, /* uprobe profile handler */ static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs, - struct uprobe_cpu_buffer *ucb) + struct uprobe_cpu_buffer **ucbp) { if (!uprobe_perf_filter(&tu->consumer, 0, current->mm)) return UPROBE_HANDLER_REMOVE; if (!is_ret_probe(tu)) - __uprobe_perf_func(tu, 0, regs, ucb); + __uprobe_perf_func(tu, 0, regs, ucbp); return 0; } static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func, struct pt_regs *regs, - struct uprobe_cpu_buffer *ucb) + struct uprobe_cpu_buffer **ucbp) { - __uprobe_perf_func(tu, func, regs, ucb); + __uprobe_perf_func(tu, func, regs, ucbp); } int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type, @@ -1489,7 +1500,7 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs) { struct trace_uprobe *tu; struct uprobe_dispatch_data udd; - struct uprobe_cpu_buffer *ucb; + struct uprobe_cpu_buffer *ucb = NULL; int ret = 0; tu = container_of(con, struct trace_uprobe, consumer); @@ -1503,14 +1514,12 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs) if (WARN_ON_ONCE(!uprobe_cpu_buffer)) return 0; - ucb = prepare_uprobe_buffer(tu, regs); - if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE)) - ret |= uprobe_trace_func(tu, regs, ucb); + ret |= uprobe_trace_func(tu, regs, &ucb); #ifdef CONFIG_PERF_EVENTS if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE)) - ret |= uprobe_perf_func(tu, regs, ucb); + ret |= uprobe_perf_func(tu, regs, &ucb); #endif uprobe_buffer_put(ucb); return ret; @@ -1521,7 +1530,7 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con, { struct trace_uprobe *tu; struct uprobe_dispatch_data udd; - struct uprobe_cpu_buffer *ucb; + struct uprobe_cpu_buffer *ucb = NULL; tu = container_of(con, struct trace_uprobe, consumer); @@ -1533,14 +1542,12 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con, if (WARN_ON_ONCE(!uprobe_cpu_buffer)) return 0; - ucb = prepare_uprobe_buffer(tu, regs); - if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE)) - uretprobe_trace_func(tu, func, regs, ucb); + uretprobe_trace_func(tu, func, regs, &ucb); #ifdef CONFIG_PERF_EVENTS if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE)) - uretprobe_perf_func(tu, func, regs, ucb); + uretprobe_perf_func(tu, func, regs, &ucb); #endif uprobe_buffer_put(ucb); return 0; From cdf355cc60e388d992bdd205b8ee70dc4d533461 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 18 Mar 2024 11:17:28 -0700 Subject: [PATCH 0922/1702] uprobes: add speculative lockless system-wide uprobe filter check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's very common with BPF-based uprobe/uretprobe use cases to have a system-wide (not PID specific) probes used. In this case uprobe's trace_uprobe_filter->nr_systemwide counter is bumped at registration time, and actual filtering is short circuited at the time when uprobe/uretprobe is triggered. This is a great optimization, and the only issue with it is that to even get to checking this counter uprobe subsystem is taking read-side trace_uprobe_filter->rwlock. This is actually noticeable in profiles and is just another point of contention when uprobe is triggered on multiple CPUs simultaneously. This patch moves this nr_systemwide check outside of filter list's rwlock scope, as rwlock is meant to protect list modification, while nr_systemwide-based check is speculative and racy already, despite the lock (as discussed in [0]). trace_uprobe_filter_remove() and trace_uprobe_filter_add() already check for filter->nr_systewide explicitly outside of __uprobe_perf_filter, so no modifications are required there. Confirming with BPF selftests's based benchmarks. BEFORE (based on changes in previous patch) =========================================== uprobe-nop : 2.732 ± 0.022M/s uprobe-push : 2.621 ± 0.016M/s uprobe-ret : 1.105 ± 0.007M/s uretprobe-nop : 1.396 ± 0.007M/s uretprobe-push : 1.347 ± 0.008M/s uretprobe-ret : 0.800 ± 0.006M/s AFTER ===== uprobe-nop : 2.878 ± 0.017M/s (+5.5%, total +8.3%) uprobe-push : 2.753 ± 0.013M/s (+5.3%, total +10.2%) uprobe-ret : 1.142 ± 0.010M/s (+3.8%, total +3.8%) uretprobe-nop : 1.444 ± 0.008M/s (+3.5%, total +6.5%) uretprobe-push : 1.410 ± 0.010M/s (+4.8%, total +7.1%) uretprobe-ret : 0.816 ± 0.002M/s (+2.0%, total +3.9%) In the above, first percentage value is based on top of previous patch (lazy uprobe buffer optimization), while the "total" percentage is based on kernel without any of the changes in this patch set. As can be seen, we get about 4% - 10% speed up, in total, with both lazy uprobe buffer and speculative filter check optimizations. [0] https://lore.kernel.org/bpf/20240313131926.GA19986@redhat.com/ Reviewed-by: Jiri Olsa Link: https://lore.kernel.org/all/20240318181728.2795838-4-andrii@kernel.org/ Signed-off-by: Andrii Nakryiko Acked-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_uprobe.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index cbab487e85228..8541fa1494ae3 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -1226,9 +1226,6 @@ __uprobe_perf_filter(struct trace_uprobe_filter *filter, struct mm_struct *mm) { struct perf_event *event; - if (filter->nr_systemwide) - return true; - list_for_each_entry(event, &filter->perf_events, hw.tp_list) { if (event->hw.target->mm == mm) return true; @@ -1353,6 +1350,13 @@ static bool uprobe_perf_filter(struct uprobe_consumer *uc, tu = container_of(uc, struct trace_uprobe, consumer); filter = tu->tp.event->filter; + /* + * speculative short-circuiting check to avoid unnecessarily taking + * filter->rwlock below, if the uprobe has system-wide consumer + */ + if (READ_ONCE(filter->nr_systemwide)) + return true; + read_lock(&filter->rwlock); ret = __uprobe_perf_filter(filter, mm); read_unlock(&filter->rwlock); From d9b15224dd8ff83b2aef87e4cd5ad10c875ef7d6 Mon Sep 17 00:00:00 2001 From: Ye Bin Date: Fri, 22 Mar 2024 14:43:04 +0800 Subject: [PATCH 0923/1702] tracing/probes: support '%pd' type for print struct dentry's name During fault locating, the file name needs to be printed based on the dentry address. The offset needs to be calculated each time, which is troublesome. Similar to printk, kprobe support print type '%pd' for print dentry's name. For example "name=$arg1:%pd" casts the `$arg1` as (struct dentry *), dereferences the "d_name.name" field and stores it to "name" argument as a kernel string. Here is an example: [tracing]# echo 'p:testprobe dput name=$arg1:%pd' > kprobe_events [tracing]# echo 1 > events/kprobes/testprobe/enable [tracing]# grep -q "1" events/kprobes/testprobe/enable [tracing]# echo 0 > events/kprobes/testprobe/enable [tracing]# cat trace | grep "enable" bash-14844 [002] ..... 16912.889543: testprobe: (dput+0x4/0x30) name="enable" grep-15389 [003] ..... 16922.834182: testprobe: (dput+0x4/0x30) name="enable" grep-15389 [003] ..... 16922.836103: testprobe: (dput+0x4/0x30) name="enable" bash-14844 [001] ..... 16931.820909: testprobe: (dput+0x4/0x30) name="enable" Note that this expects the given argument (e.g. $arg1) is an address of struct dentry. User must ensure it. Link: https://lore.kernel.org/all/20240322064308.284457-2-yebin10@huawei.com/ Signed-off-by: Ye Bin Acked-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace.c | 2 +- kernel/trace/trace_fprobe.c | 6 +++++ kernel/trace/trace_kprobe.c | 6 +++++ kernel/trace/trace_probe.c | 50 +++++++++++++++++++++++++++++++++++++ kernel/trace/trace_probe.h | 2 ++ 5 files changed, 65 insertions(+), 1 deletion(-) diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 233d1af39fffe..869978d846729 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -5540,7 +5540,7 @@ static const char readme_msg[] = "\t kernel return probes support: $retval, $arg, $comm\n" "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, char, string, symbol,\n" "\t b@/, ustring,\n" - "\t symstr, \\[\\]\n" + "\t symstr, %pd, \\[\\]\n" #ifdef CONFIG_HIST_TRIGGERS "\t field: ;\n" "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n" diff --git a/kernel/trace/trace_fprobe.c b/kernel/trace/trace_fprobe.c index 4f42808155225..62e6a8f4aae9b 100644 --- a/kernel/trace/trace_fprobe.c +++ b/kernel/trace/trace_fprobe.c @@ -994,6 +994,7 @@ static int __trace_fprobe_create(int argc, const char *argv[]) char gbuf[MAX_EVENT_NAME_LEN]; char sbuf[KSYM_NAME_LEN]; char abuf[MAX_BTF_ARGS_LEN]; + char *dbuf = NULL; bool is_tracepoint = false; struct tracepoint *tpoint = NULL; struct traceprobe_parse_context ctx = { @@ -1104,6 +1105,10 @@ static int __trace_fprobe_create(int argc, const char *argv[]) argv = new_argv; } + ret = traceprobe_expand_dentry_args(argc, argv, &dbuf); + if (ret) + goto out; + /* setup a probe */ tf = alloc_trace_fprobe(group, event, symbol, tpoint, maxactive, argc, is_return); @@ -1154,6 +1159,7 @@ static int __trace_fprobe_create(int argc, const char *argv[]) trace_probe_log_clear(); kfree(new_argv); kfree(symbol); + kfree(dbuf); return ret; parse_error: diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index 14099cc17fc9e..c68d4e830fbe8 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c @@ -782,6 +782,7 @@ static int __trace_kprobe_create(int argc, const char *argv[]) char buf[MAX_EVENT_NAME_LEN]; char gbuf[MAX_EVENT_NAME_LEN]; char abuf[MAX_BTF_ARGS_LEN]; + char *dbuf = NULL; struct traceprobe_parse_context ctx = { .flags = TPARG_FL_KERNEL }; switch (argv[0][0]) { @@ -933,6 +934,10 @@ static int __trace_kprobe_create(int argc, const char *argv[]) argv = new_argv; } + ret = traceprobe_expand_dentry_args(argc, argv, &dbuf); + if (ret) + goto out; + /* setup a probe */ tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive, argc, is_return); @@ -979,6 +984,7 @@ static int __trace_kprobe_create(int argc, const char *argv[]) trace_probe_log_clear(); kfree(new_argv); kfree(symbol); + kfree(dbuf); return ret; parse_error: diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index dfe3ee6035ecc..f7ee97e45d1f3 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -1739,6 +1739,56 @@ const char **traceprobe_expand_meta_args(int argc, const char *argv[], return ERR_PTR(ret); } +/* @buf: *buf must be equal to NULL. Caller must to free *buf */ +int traceprobe_expand_dentry_args(int argc, const char *argv[], char **buf) +{ + int i, used, ret; + const int bufsize = MAX_DENTRY_ARGS_LEN; + char *tmpbuf = NULL; + + if (*buf) + return -EINVAL; + + used = 0; + for (i = 0; i < argc; i++) { + if (glob_match("*:%pd", argv[i])) { + char *tmp; + char *equal; + + if (!tmpbuf) { + tmpbuf = kmalloc(bufsize, GFP_KERNEL); + if (!tmpbuf) + return -ENOMEM; + } + + tmp = kstrdup(argv[i], GFP_KERNEL); + if (!tmp) + goto nomem; + + equal = strchr(tmp, '='); + if (equal) + *equal = '\0'; + tmp[strlen(argv[i]) - 4] = '\0'; + ret = snprintf(tmpbuf + used, bufsize - used, + "%s%s+0x0(+0x%zx(%s)):string", + equal ? tmp : "", equal ? "=" : "", + offsetof(struct dentry, d_name.name), + equal ? equal + 1 : tmp); + kfree(tmp); + if (ret >= bufsize - used) + goto nomem; + argv[i] = tmpbuf + used; + used += ret + 1; + } + } + + *buf = tmpbuf; + return 0; +nomem: + kfree(tmpbuf); + return -ENOMEM; +} + void traceprobe_finish_parse(struct traceprobe_parse_context *ctx) { clear_btf_context(ctx); diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index cef3a50628a3e..5803e6a415705 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -34,6 +34,7 @@ #define MAX_ARRAY_LEN 64 #define MAX_ARG_NAME_LEN 32 #define MAX_BTF_ARGS_LEN 128 +#define MAX_DENTRY_ARGS_LEN 256 #define MAX_STRING_SIZE PATH_MAX #define MAX_ARG_BUF_LEN (MAX_TRACE_ARGS * MAX_ARG_NAME_LEN) @@ -428,6 +429,7 @@ extern int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, const char **traceprobe_expand_meta_args(int argc, const char *argv[], int *new_argc, char *buf, int bufsize, struct traceprobe_parse_context *ctx); +extern int traceprobe_expand_dentry_args(int argc, const char *argv[], char **buf); extern int traceprobe_update_arg(struct probe_arg *arg); extern void traceprobe_free_probe_arg(struct probe_arg *arg); From 20fe4d07bde67ec26716835b61be2c4eeca1c6bc Mon Sep 17 00:00:00 2001 From: Ye Bin Date: Fri, 22 Mar 2024 14:43:05 +0800 Subject: [PATCH 0924/1702] tracing/probes: support '%pD' type for print struct file's name As like '%pd' type, this patch supports print type '%pD' for print file's name. For example "name=$arg1:%pD" casts the `$arg1` as (struct file*), dereferences the "file.f_path.dentry.d_name.name" field and stores it to "name" argument as a kernel string. Here is an example: [tracing]# echo 'p:testprobe vfs_read name=$arg1:%pD' > kprobe_event [tracing]# echo 1 > events/kprobes/testprobe/enable [tracing]# grep -q "1" events/kprobes/testprobe/enable [tracing]# echo 0 > events/kprobes/testprobe/enable [tracing]# grep "vfs_read" trace | grep "enable" grep-15108 [003] ..... 5228.328609: testprobe: (vfs_read+0x4/0xbb0) name="enable" Note that this expects the given argument (e.g. $arg1) is an address of struct file. User must ensure it. Link: https://lore.kernel.org/all/20240322064308.284457-3-yebin10@huawei.com/ [Masami: replaced "previous patch" with '%pd' type] Signed-off-by: Ye Bin Acked-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace.c | 2 +- kernel/trace/trace_probe.c | 57 +++++++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 869978d846729..7fe4b539a4234 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -5540,7 +5540,7 @@ static const char readme_msg[] = "\t kernel return probes support: $retval, $arg, $comm\n" "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, char, string, symbol,\n" "\t b@/, ustring,\n" - "\t symstr, %pd, \\[\\]\n" + "\t symstr, %pd/%pD, \\[\\]\n" #ifdef CONFIG_HIST_TRIGGERS "\t field: ;\n" "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n" diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index f7ee97e45d1f3..d4a887dac821d 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -12,6 +12,7 @@ #define pr_fmt(fmt) "trace_probe: " fmt #include +#include #include "trace_btf.h" #include "trace_probe.h" @@ -1751,35 +1752,47 @@ int traceprobe_expand_dentry_args(int argc, const char *argv[], char **buf) used = 0; for (i = 0; i < argc; i++) { - if (glob_match("*:%pd", argv[i])) { - char *tmp; - char *equal; - - if (!tmpbuf) { - tmpbuf = kmalloc(bufsize, GFP_KERNEL); - if (!tmpbuf) - return -ENOMEM; - } + char *tmp; + char *equal; + size_t arg_len; - tmp = kstrdup(argv[i], GFP_KERNEL); - if (!tmp) - goto nomem; + if (!glob_match("*:%p[dD]", argv[i])) + continue; - equal = strchr(tmp, '='); - if (equal) - *equal = '\0'; - tmp[strlen(argv[i]) - 4] = '\0'; + if (!tmpbuf) { + tmpbuf = kmalloc(bufsize, GFP_KERNEL); + if (!tmpbuf) + return -ENOMEM; + } + + tmp = kstrdup(argv[i], GFP_KERNEL); + if (!tmp) + goto nomem; + + equal = strchr(tmp, '='); + if (equal) + *equal = '\0'; + arg_len = strlen(argv[i]); + tmp[arg_len - 4] = '\0'; + if (argv[i][arg_len - 1] == 'd') ret = snprintf(tmpbuf + used, bufsize - used, "%s%s+0x0(+0x%zx(%s)):string", equal ? tmp : "", equal ? "=" : "", offsetof(struct dentry, d_name.name), equal ? equal + 1 : tmp); - kfree(tmp); - if (ret >= bufsize - used) - goto nomem; - argv[i] = tmpbuf + used; - used += ret + 1; - } + else + ret = snprintf(tmpbuf + used, bufsize - used, + "%s%s+0x0(+0x%zx(+0x%zx(%s))):string", + equal ? tmp : "", equal ? "=" : "", + offsetof(struct dentry, d_name.name), + offsetof(struct file, f_path.dentry), + equal ? equal + 1 : tmp); + + kfree(tmp); + if (ret >= bufsize - used) + goto nomem; + argv[i] = tmpbuf + used; + used += ret + 1; } *buf = tmpbuf; From 5e37460f5f9266bcd037137f1bd9eb24b9940faf Mon Sep 17 00:00:00 2001 From: Ye Bin Date: Fri, 22 Mar 2024 14:43:06 +0800 Subject: [PATCH 0925/1702] Documentation: tracing: add new type '%pd' and '%pD' for kprobe Similar to printk() '%pd' is for fetch dentry's name from struct dentry's pointer, and '%pD' is for fetch file's name from struct file's pointer. Link: https://lore.kernel.org/all/20240322064308.284457-4-yebin10@huawei.com/ Signed-off-by: Ye Bin Acked-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) --- Documentation/trace/kprobetrace.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Documentation/trace/kprobetrace.rst b/Documentation/trace/kprobetrace.rst index a49662ccd53cd..195c942d9becd 100644 --- a/Documentation/trace/kprobetrace.rst +++ b/Documentation/trace/kprobetrace.rst @@ -58,8 +58,9 @@ Synopsis of kprobe_events NAME=FETCHARG : Set NAME as the argument name of FETCHARG. FETCHARG:TYPE : Set TYPE as the type of FETCHARG. Currently, basic types (u8/u16/u32/u64/s8/s16/s32/s64), hexadecimal types - (x8/x16/x32/x64), "char", "string", "ustring", "symbol", "symstr" - and bitfield are supported. + (x8/x16/x32/x64), VFS layer common type(%pd/%pD), "char", + "string", "ustring", "symbol", "symstr" and bitfield are + supported. (\*1) only for the probe on function entry (offs == 0). Note, this argument access is best effort, because depending on the argument type, it may be passed on @@ -122,6 +123,9 @@ With 'symstr' type, you can filter the event with wildcard pattern of the symbols, and you don't need to solve symbol name by yourself. For $comm, the default type is "string"; any other type is invalid. +VFS layer common type(%pd/%pD) is a special type, which fetches dentry's or +file's name from struct dentry's address or struct file's address. + .. _user_mem_access: User Memory Access From c01768b05e306d8c8d5997356ff427f5d53c7d20 Mon Sep 17 00:00:00 2001 From: Ye Bin Date: Fri, 22 Mar 2024 14:43:07 +0800 Subject: [PATCH 0926/1702] selftests/ftrace: add kprobe test cases for VFS type "%pd" and "%pD" This patch adds test cases for new print format type "%pd/%pD".The test cases test the following items: 1. Test README if add "%pd/%pD" type; 2. Test "%pd" type for dput(); 3. Test "%pD" type for vfs_read(); This test case require enable CONFIG_HAVE_FUNCTION_ARG_ACCESS_API configuration. Link: https://lore.kernel.org/all/20240322064308.284457-5-yebin10@huawei.com/ Signed-off-by: Ye Bin Acked-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) --- .../ftrace/test.d/kprobe/kprobe_args_vfs.tc | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_vfs.tc diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_vfs.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_vfs.tc new file mode 100644 index 0000000000000..21a54be6894c1 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_vfs.tc @@ -0,0 +1,40 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: Kprobe event VFS type argument +# requires: kprobe_events "%pd/%pD":README + +: "Test argument %pd with name" +echo 'p:testprobe dput name=$arg1:%pd' > kprobe_events +echo 1 > events/kprobes/testprobe/enable +grep -q "1" events/kprobes/testprobe/enable +echo 0 > events/kprobes/testprobe/enable +grep "dput" trace | grep -q "enable" +echo "" > kprobe_events +echo "" > trace + +: "Test argument %pd without name" +echo 'p:testprobe dput $arg1:%pd' > kprobe_events +echo 1 > events/kprobes/testprobe/enable +grep -q "1" events/kprobes/testprobe/enable +echo 0 > events/kprobes/testprobe/enable +grep "dput" trace | grep -q "enable" +echo "" > kprobe_events +echo "" > trace + +: "Test argument %pD with name" +echo 'p:testprobe vfs_read name=$arg1:%pD' > kprobe_events +echo 1 > events/kprobes/testprobe/enable +grep -q "1" events/kprobes/testprobe/enable +echo 0 > events/kprobes/testprobe/enable +grep "vfs_read" trace | grep -q "enable" +echo "" > kprobe_events +echo "" > trace + +: "Test argument %pD without name" +echo 'p:testprobe vfs_read $arg1:%pD' > kprobe_events +echo 1 > events/kprobes/testprobe/enable +grep -q "1" events/kprobes/testprobe/enable +echo 0 > events/kprobes/testprobe/enable +grep "vfs_read" trace | grep -q "enable" +echo "" > kprobe_events +echo "" > trace From ee97e5e135c6fc937c4c81e0341c6513e2e6d44c Mon Sep 17 00:00:00 2001 From: Ye Bin Date: Fri, 22 Mar 2024 14:43:08 +0800 Subject: [PATCH 0927/1702] selftests/ftrace: add fprobe test cases for VFS type "%pd" and "%pD" This patch adds fprobe test cases for new print format type "%pd/%pD".The test cases test the following items: 1. Test "%pd" type for dput(); 2. Test "%pD" type for vfs_read(); This test case require enable CONFIG_HAVE_FUNCTION_ARG_ACCESS_API configuration. Link: https://lore.kernel.org/all/20240322064308.284457-6-yebin10@huawei.com/ Signed-off-by: Ye Bin Acked-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) --- .../ftrace/test.d/dynevent/fprobe_args_vfs.tc | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/dynevent/fprobe_args_vfs.tc diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_args_vfs.tc b/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_args_vfs.tc new file mode 100644 index 0000000000000..49a833bf334ce --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_args_vfs.tc @@ -0,0 +1,40 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: Fprobe event VFS type argument +# requires: kprobe_events "%pd/%pD":README + +: "Test argument %pd with name for fprobe" +echo 'f:testprobe dput name=$arg1:%pd' > dynamic_events +echo 1 > events/fprobes/testprobe/enable +grep -q "1" events/fprobes/testprobe/enable +echo 0 > events/fprobes/testprobe/enable +grep "dput" trace | grep -q "enable" +echo "" > dynamic_events +echo "" > trace + +: "Test argument %pd without name for fprobe" +echo 'f:testprobe dput $arg1:%pd' > dynamic_events +echo 1 > events/fprobes/testprobe/enable +grep -q "1" events/fprobes/testprobe/enable +echo 0 > events/fprobes/testprobe/enable +grep "dput" trace | grep -q "enable" +echo "" > dynamic_events +echo "" > trace + +: "Test argument %pD with name for fprobe" +echo 'f:testprobe vfs_read name=$arg1:%pD' > dynamic_events +echo 1 > events/fprobes/testprobe/enable +grep -q "1" events/fprobes/testprobe/enable +echo 0 > events/fprobes/testprobe/enable +grep "vfs_read" trace | grep -q "enable" +echo "" > dynamic_events +echo "" > trace + +: "Test argument %pD without name for fprobe" +echo 'f:testprobe vfs_read $arg1:%pD' > dynamic_events +echo 1 > events/fprobes/testprobe/enable +grep -q "1" events/fprobes/testprobe/enable +echo 0 > events/fprobes/testprobe/enable +grep "vfs_read" trace | grep -q "enable" +echo "" > dynamic_events +echo "" > trace From 73142cab3af1b99157837297f437b306d7a70bff Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Wed, 7 Feb 2024 16:35:47 +0100 Subject: [PATCH 0928/1702] fprobe: Add entry/exit callbacks types We are going to store callbacks in following change, so this will ease up the code. Link: https://lore.kernel.org/all/20240207153550.856536-2-jolsa@kernel.org/ Signed-off-by: Jiri Olsa Acked-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) --- include/linux/fprobe.h | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/include/linux/fprobe.h b/include/linux/fprobe.h index 3e03758151f47..f398695881175 100644 --- a/include/linux/fprobe.h +++ b/include/linux/fprobe.h @@ -7,6 +7,16 @@ #include #include +struct fprobe; + +typedef int (*fprobe_entry_cb)(struct fprobe *fp, unsigned long entry_ip, + unsigned long ret_ip, struct pt_regs *regs, + void *entry_data); + +typedef void (*fprobe_exit_cb)(struct fprobe *fp, unsigned long entry_ip, + unsigned long ret_ip, struct pt_regs *regs, + void *entry_data); + /** * struct fprobe - ftrace based probe. * @ops: The ftrace_ops. @@ -34,12 +44,8 @@ struct fprobe { size_t entry_data_size; int nr_maxactive; - int (*entry_handler)(struct fprobe *fp, unsigned long entry_ip, - unsigned long ret_ip, struct pt_regs *regs, - void *entry_data); - void (*exit_handler)(struct fprobe *fp, unsigned long entry_ip, - unsigned long ret_ip, struct pt_regs *regs, - void *entry_data); + fprobe_entry_cb entry_handler; + fprobe_exit_cb exit_handler; }; /* This fprobe is soft-disabled. */ From 5120d167e21c674afd0630c65e7f6a00fa0667f1 Mon Sep 17 00:00:00 2001 From: Kui-Feng Lee Date: Mon, 8 Apr 2024 10:51:40 -0700 Subject: [PATCH 0929/1702] rethook: Remove warning messages printed for finding return address of a frame. The function rethook_find_ret_addr() prints a warning message and returns 0 when the target task is running and is not the "current" task in order to prevent the incorrect return address, although it still may return an incorrect address. However, the warning message turns into noise when BPF profiling programs call bpf_get_task_stack() on running tasks in a firm with a large number of hosts. The callers should be aware and willing to take the risk of receiving an incorrect return address from a task that is currently running other than the "current" one. A warning is not needed here as the callers are intent on it. Link: https://lore.kernel.org/all/20240408175140.60223-1-thinker.li@gmail.com/ Acked-by: Andrii Nakryiko Acked-by: John Fastabend Signed-off-by: Kui-Feng Lee Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/rethook.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/trace/rethook.c b/kernel/trace/rethook.c index fa03094e9e698..4297a132a7ae2 100644 --- a/kernel/trace/rethook.c +++ b/kernel/trace/rethook.c @@ -248,7 +248,7 @@ unsigned long rethook_find_ret_addr(struct task_struct *tsk, unsigned long frame if (WARN_ON_ONCE(!cur)) return 0; - if (WARN_ON_ONCE(tsk != current && task_is_running(tsk))) + if (tsk != current && task_is_running(tsk)) return 0; do { From 0dc715295d4143d1659879f7f50ad4e9a6f6a99c Mon Sep 17 00:00:00 2001 From: Jonathan Haslam Date: Mon, 22 Apr 2024 03:23:05 -0700 Subject: [PATCH 0930/1702] uprobes: reduce contention on uprobes_tree access MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Active uprobes are stored in an RB tree and accesses to this tree are dominated by read operations. Currently these accesses are serialized by a spinlock but this leads to enormous contention when large numbers of threads are executing active probes. This patch converts the spinlock used to serialize access to the uprobes_tree RB tree into a reader-writer spinlock. This lock type aligns naturally with the overwhelmingly read-only nature of the tree usage here. Although the addition of reader-writer spinlocks are discouraged [0], this fix is proposed as an interim solution while an RCU based approach is implemented (that work is in a nascent form). This fix also has the benefit of being trivial, self contained and therefore simple to backport. We have used a uprobe benchmark from the BPF selftests [1] to estimate the improvements. Each block of results below show 1 line per execution of the benchmark ("the "Summary" line) and each line is a run with one more thread added - a thread is a "producer". The lines are edited to remove extraneous output. The tests were executed with this driver script: for num_threads in {1..20} do sudo ./bench -a -p $num_threads trig-uprobe-nop | grep Summary done SPINLOCK (BEFORE) ================== Summary: hits 1.396 ± 0.007M/s ( 1.396M/prod) Summary: hits 1.656 ± 0.016M/s ( 0.828M/prod) Summary: hits 2.246 ± 0.008M/s ( 0.749M/prod) Summary: hits 2.114 ± 0.010M/s ( 0.529M/prod) Summary: hits 2.013 ± 0.009M/s ( 0.403M/prod) Summary: hits 1.753 ± 0.008M/s ( 0.292M/prod) Summary: hits 1.847 ± 0.001M/s ( 0.264M/prod) Summary: hits 1.889 ± 0.001M/s ( 0.236M/prod) Summary: hits 1.833 ± 0.006M/s ( 0.204M/prod) Summary: hits 1.900 ± 0.003M/s ( 0.190M/prod) Summary: hits 1.918 ± 0.006M/s ( 0.174M/prod) Summary: hits 1.925 ± 0.002M/s ( 0.160M/prod) Summary: hits 1.837 ± 0.001M/s ( 0.141M/prod) Summary: hits 1.898 ± 0.001M/s ( 0.136M/prod) Summary: hits 1.799 ± 0.016M/s ( 0.120M/prod) Summary: hits 1.850 ± 0.005M/s ( 0.109M/prod) Summary: hits 1.816 ± 0.002M/s ( 0.101M/prod) Summary: hits 1.787 ± 0.001M/s ( 0.094M/prod) Summary: hits 1.764 ± 0.002M/s ( 0.088M/prod) RW SPINLOCK (AFTER) =================== Summary: hits 1.444 ± 0.020M/s ( 1.444M/prod) Summary: hits 2.279 ± 0.011M/s ( 1.139M/prod) Summary: hits 3.422 ± 0.014M/s ( 1.141M/prod) Summary: hits 3.565 ± 0.017M/s ( 0.891M/prod) Summary: hits 2.671 ± 0.013M/s ( 0.534M/prod) Summary: hits 2.409 ± 0.005M/s ( 0.401M/prod) Summary: hits 2.485 ± 0.008M/s ( 0.355M/prod) Summary: hits 2.496 ± 0.003M/s ( 0.312M/prod) Summary: hits 2.585 ± 0.002M/s ( 0.287M/prod) Summary: hits 2.908 ± 0.011M/s ( 0.291M/prod) Summary: hits 2.346 ± 0.016M/s ( 0.213M/prod) Summary: hits 2.804 ± 0.004M/s ( 0.234M/prod) Summary: hits 2.556 ± 0.001M/s ( 0.197M/prod) Summary: hits 2.754 ± 0.004M/s ( 0.197M/prod) Summary: hits 2.482 ± 0.002M/s ( 0.165M/prod) Summary: hits 2.412 ± 0.005M/s ( 0.151M/prod) Summary: hits 2.710 ± 0.003M/s ( 0.159M/prod) Summary: hits 2.826 ± 0.005M/s ( 0.157M/prod) Summary: hits 2.718 ± 0.001M/s ( 0.143M/prod) Summary: hits 2.844 ± 0.006M/s ( 0.142M/prod) The numbers in parenthesis give averaged throughput per thread which is of greatest interest here as a measure of scalability. Improvements are in the order of 22 - 68% with this particular benchmark (mean = 43%). V2: - Updated commit message to include benchmark results. [0] https://docs.kernel.org/locking/spinlocks.html [1] https://github.com/torvalds/linux/blob/master/tools/testing/selftests/bpf/benchs/bench_trigger.c Link: https://lore.kernel.org/all/20240422102306.6026-1-jonathan.haslam@gmail.com/ Signed-off-by: Jonathan Haslam Acked-by: Jiri Olsa Signed-off-by: Masami Hiramatsu (Google) --- kernel/events/uprobes.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index e4834d23e1d1a..8ae0eefc3a341 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -39,7 +39,7 @@ static struct rb_root uprobes_tree = RB_ROOT; */ #define no_uprobe_events() RB_EMPTY_ROOT(&uprobes_tree) -static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */ +static DEFINE_RWLOCK(uprobes_treelock); /* serialize rbtree access */ #define UPROBES_HASH_SZ 13 /* serialize uprobe->pending_list */ @@ -669,9 +669,9 @@ static struct uprobe *find_uprobe(struct inode *inode, loff_t offset) { struct uprobe *uprobe; - spin_lock(&uprobes_treelock); + read_lock(&uprobes_treelock); uprobe = __find_uprobe(inode, offset); - spin_unlock(&uprobes_treelock); + read_unlock(&uprobes_treelock); return uprobe; } @@ -701,9 +701,9 @@ static struct uprobe *insert_uprobe(struct uprobe *uprobe) { struct uprobe *u; - spin_lock(&uprobes_treelock); + write_lock(&uprobes_treelock); u = __insert_uprobe(uprobe); - spin_unlock(&uprobes_treelock); + write_unlock(&uprobes_treelock); return u; } @@ -935,9 +935,9 @@ static void delete_uprobe(struct uprobe *uprobe) if (WARN_ON(!uprobe_is_active(uprobe))) return; - spin_lock(&uprobes_treelock); + write_lock(&uprobes_treelock); rb_erase(&uprobe->rb_node, &uprobes_tree); - spin_unlock(&uprobes_treelock); + write_unlock(&uprobes_treelock); RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */ put_uprobe(uprobe); } @@ -1298,7 +1298,7 @@ static void build_probe_list(struct inode *inode, min = vaddr_to_offset(vma, start); max = min + (end - start) - 1; - spin_lock(&uprobes_treelock); + read_lock(&uprobes_treelock); n = find_node_in_range(inode, min, max); if (n) { for (t = n; t; t = rb_prev(t)) { @@ -1316,7 +1316,7 @@ static void build_probe_list(struct inode *inode, get_uprobe(u); } } - spin_unlock(&uprobes_treelock); + read_unlock(&uprobes_treelock); } /* @vma contains reference counter, not the probed instruction. */ @@ -1407,9 +1407,9 @@ vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long e min = vaddr_to_offset(vma, start); max = min + (end - start) - 1; - spin_lock(&uprobes_treelock); + read_lock(&uprobes_treelock); n = find_node_in_range(inode, min, max); - spin_unlock(&uprobes_treelock); + read_unlock(&uprobes_treelock); return !!n; } From b0e28a4b5becea84ae6fca5cbd8a6b80a134e223 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 18 Apr 2024 12:09:08 -0700 Subject: [PATCH 0931/1702] ftrace: make extra rcu_is_watching() validation check optional Introduce CONFIG_FTRACE_VALIDATE_RCU_IS_WATCHING config option to control whether ftrace low-level code performs additional rcu_is_watching()-based validation logic in an attempt to catch noinstr violations. This check is expected to never be true and is mostly useful for low-level validation of ftrace subsystem invariants. For most users it should probably be kept disabled to eliminate unnecessary runtime overhead. This improves BPF multi-kretprobe (relying on ftrace and rethook infrastructure) runtime throughput by 2%, according to BPF benchmarks ([0]). [0] https://lore.kernel.org/bpf/CAEf4BzauQ2WKMjZdc9s0rBWa01BYbgwHN6aNDXQSHYia47pQ-w@mail.gmail.com/ Link: https://lore.kernel.org/all/20240418190909.704286-1-andrii@kernel.org/ Cc: Steven Rostedt Cc: Masami Hiramatsu Cc: Paul E. McKenney Acked-by: Masami Hiramatsu (Google) Signed-off-by: Andrii Nakryiko Signed-off-by: Masami Hiramatsu (Google) --- include/linux/trace_recursion.h | 2 +- kernel/trace/Kconfig | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/include/linux/trace_recursion.h b/include/linux/trace_recursion.h index d48cd92d2364a..24ea8ac049b40 100644 --- a/include/linux/trace_recursion.h +++ b/include/linux/trace_recursion.h @@ -135,7 +135,7 @@ extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip); # define do_ftrace_record_recursion(ip, pip) do { } while (0) #endif -#ifdef CONFIG_ARCH_WANTS_NO_INSTR +#ifdef CONFIG_FTRACE_VALIDATE_RCU_IS_WATCHING # define trace_warn_on_no_rcu(ip) \ ({ \ bool __ret = !rcu_is_watching(); \ diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 47345bf1d4a9f..d093e16d01c17 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -974,6 +974,19 @@ config FTRACE_RECORD_RECURSION_SIZE This file can be reset, but the limit can not change in size at runtime. +config FTRACE_VALIDATE_RCU_IS_WATCHING + bool "Validate RCU is on during ftrace execution" + depends on FUNCTION_TRACER + depends on ARCH_WANTS_NO_INSTR + help + All callbacks that attach to the function tracing have some sort of + protection against recursion. This option is only to verify that + ftrace (and other users of ftrace_test_recursion_trylock()) are not + called outside of RCU, as if they are, it can cause a race. But it + also has a noticeable overhead when enabled. + + If unsure, say N + config RING_BUFFER_RECORD_RECURSION bool "Record functions that recurse in the ring buffer" depends on FTRACE_RECORD_RECURSION From e03c05ac9813410d15c9c39ccf02c84efe563533 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 18 Apr 2024 12:09:09 -0700 Subject: [PATCH 0932/1702] rethook: honor CONFIG_FTRACE_VALIDATE_RCU_IS_WATCHING in rethook_try_get() Take into account CONFIG_FTRACE_VALIDATE_RCU_IS_WATCHING when validating that RCU is watching when trying to setup rethooko on a function entry. One notable exception when we force rcu_is_watching() check is CONFIG_KPROBE_EVENTS_ON_NOTRACE=y case, in which case kretprobes will use old-style int3-based workflow instead of relying on ftrace, making RCU watching check important to validate. This further (in addition to improvements in the previous patch) improves BPF multi-kretprobe (which rely on rethook) runtime throughput by 2.3%, according to BPF benchmarks ([0]). [0] https://lore.kernel.org/bpf/CAEf4BzauQ2WKMjZdc9s0rBWa01BYbgwHN6aNDXQSHYia47pQ-w@mail.gmail.com/ Link: https://lore.kernel.org/all/20240418190909.704286-2-andrii@kernel.org/ Signed-off-by: Andrii Nakryiko Acked-by: Paul E. McKenney Acked-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/rethook.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/trace/rethook.c b/kernel/trace/rethook.c index 4297a132a7ae2..30d2249468815 100644 --- a/kernel/trace/rethook.c +++ b/kernel/trace/rethook.c @@ -166,6 +166,7 @@ struct rethook_node *rethook_try_get(struct rethook *rh) if (unlikely(!handler)) return NULL; +#if defined(CONFIG_FTRACE_VALIDATE_RCU_IS_WATCHING) || defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE) /* * This expects the caller will set up a rethook on a function entry. * When the function returns, the rethook will eventually be reclaimed @@ -174,6 +175,7 @@ struct rethook_node *rethook_try_get(struct rethook *rh) */ if (unlikely(!rcu_is_watching())) return NULL; +#endif return (struct rethook_node *)objpool_pop(&rh->pool); } From a3b00f10da808bd4a354f890b551cba471082d0e Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Wed, 24 Apr 2024 14:52:13 -0700 Subject: [PATCH 0933/1702] objpool: enable inlining objpool_push() and objpool_pop() operations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit objpool_push() and objpool_pop() are very performance-critical functions and can be called very frequently in kretprobe triggering path. As such, it makes sense to allow compiler to inline them completely to eliminate function calls overhead. Luckily, their logic is quite well isolated and doesn't have any sprawling dependencies. This patch moves both objpool_push() and objpool_pop() into include/linux/objpool.h and marks them as static inline functions, enabling inlining. To avoid anyone using internal helpers (objpool_try_get_slot, objpool_try_add_slot), rename them to use leading underscores. We used kretprobe microbenchmark from BPF selftests (bench trig-kprobe and trig-kprobe-multi benchmarks) running no-op BPF kretprobe/kretprobe.multi programs in a tight loop to evaluate the effect. BPF own overhead in this case is minimal and it mostly stresses the rest of in-kernel kretprobe infrastructure overhead. Results are in millions of calls per second. This is not super scientific, but shows the trend nevertheless. BEFORE ====== kretprobe : 9.794 ± 0.086M/s kretprobe-multi: 10.219 ± 0.032M/s AFTER ===== kretprobe : 9.937 ± 0.174M/s (+1.5%) kretprobe-multi: 10.440 ± 0.108M/s (+2.2%) Link: https://lore.kernel.org/all/20240424215214.3956041-2-andrii@kernel.org/ Cc: Matt (Qiang) Wu Signed-off-by: Andrii Nakryiko Signed-off-by: Masami Hiramatsu (Google) --- include/linux/objpool.h | 101 +++++++++++++++++++++++++++++++++++++++- lib/objpool.c | 100 --------------------------------------- 2 files changed, 99 insertions(+), 102 deletions(-) diff --git a/include/linux/objpool.h b/include/linux/objpool.h index 15aff4a17f0cb..d8b1f7b91128e 100644 --- a/include/linux/objpool.h +++ b/include/linux/objpool.h @@ -5,6 +5,10 @@ #include #include +#include +#include +#include +#include /* * objpool: ring-array based lockless MPMC queue @@ -118,13 +122,94 @@ int objpool_init(struct objpool_head *pool, int nr_objs, int object_size, gfp_t gfp, void *context, objpool_init_obj_cb objinit, objpool_fini_cb release); +/* try to retrieve object from slot */ +static inline void *__objpool_try_get_slot(struct objpool_head *pool, int cpu) +{ + struct objpool_slot *slot = pool->cpu_slots[cpu]; + /* load head snapshot, other cpus may change it */ + uint32_t head = smp_load_acquire(&slot->head); + + while (head != READ_ONCE(slot->last)) { + void *obj; + + /* + * data visibility of 'last' and 'head' could be out of + * order since memory updating of 'last' and 'head' are + * performed in push() and pop() independently + * + * before any retrieving attempts, pop() must guarantee + * 'last' is behind 'head', that is to say, there must + * be available objects in slot, which could be ensured + * by condition 'last != head && last - head <= nr_objs' + * that is equivalent to 'last - head - 1 < nr_objs' as + * 'last' and 'head' are both unsigned int32 + */ + if (READ_ONCE(slot->last) - head - 1 >= pool->nr_objs) { + head = READ_ONCE(slot->head); + continue; + } + + /* obj must be retrieved before moving forward head */ + obj = READ_ONCE(slot->entries[head & slot->mask]); + + /* move head forward to mark it's consumption */ + if (try_cmpxchg_release(&slot->head, &head, head + 1)) + return obj; + } + + return NULL; +} + /** * objpool_pop() - allocate an object from objpool * @pool: object pool * * return value: object ptr or NULL if failed */ -void *objpool_pop(struct objpool_head *pool); +static inline void *objpool_pop(struct objpool_head *pool) +{ + void *obj = NULL; + unsigned long flags; + int i, cpu; + + /* disable local irq to avoid preemption & interruption */ + raw_local_irq_save(flags); + + cpu = raw_smp_processor_id(); + for (i = 0; i < num_possible_cpus(); i++) { + obj = __objpool_try_get_slot(pool, cpu); + if (obj) + break; + cpu = cpumask_next_wrap(cpu, cpu_possible_mask, -1, 1); + } + raw_local_irq_restore(flags); + + return obj; +} + +/* adding object to slot, abort if the slot was already full */ +static inline int +__objpool_try_add_slot(void *obj, struct objpool_head *pool, int cpu) +{ + struct objpool_slot *slot = pool->cpu_slots[cpu]; + uint32_t head, tail; + + /* loading tail and head as a local snapshot, tail first */ + tail = READ_ONCE(slot->tail); + + do { + head = READ_ONCE(slot->head); + /* fault caught: something must be wrong */ + WARN_ON_ONCE(tail - head > pool->nr_objs); + } while (!try_cmpxchg_acquire(&slot->tail, &tail, tail + 1)); + + /* now the tail position is reserved for the given obj */ + WRITE_ONCE(slot->entries[tail & slot->mask], obj); + /* update sequence to make this obj available for pop() */ + smp_store_release(&slot->last, tail + 1); + + return 0; +} /** * objpool_push() - reclaim the object and return back to objpool @@ -134,7 +219,19 @@ void *objpool_pop(struct objpool_head *pool); * return: 0 or error code (it fails only when user tries to push * the same object multiple times or wrong "objects" into objpool) */ -int objpool_push(void *obj, struct objpool_head *pool); +static inline int objpool_push(void *obj, struct objpool_head *pool) +{ + unsigned long flags; + int rc; + + /* disable local irq to avoid preemption & interruption */ + raw_local_irq_save(flags); + rc = __objpool_try_add_slot(obj, pool, raw_smp_processor_id()); + raw_local_irq_restore(flags); + + return rc; +} + /** * objpool_drop() - discard the object and deref objpool diff --git a/lib/objpool.c b/lib/objpool.c index cfdc024208849..f696308fc0264 100644 --- a/lib/objpool.c +++ b/lib/objpool.c @@ -152,106 +152,6 @@ int objpool_init(struct objpool_head *pool, int nr_objs, int object_size, } EXPORT_SYMBOL_GPL(objpool_init); -/* adding object to slot, abort if the slot was already full */ -static inline int -objpool_try_add_slot(void *obj, struct objpool_head *pool, int cpu) -{ - struct objpool_slot *slot = pool->cpu_slots[cpu]; - uint32_t head, tail; - - /* loading tail and head as a local snapshot, tail first */ - tail = READ_ONCE(slot->tail); - - do { - head = READ_ONCE(slot->head); - /* fault caught: something must be wrong */ - WARN_ON_ONCE(tail - head > pool->nr_objs); - } while (!try_cmpxchg_acquire(&slot->tail, &tail, tail + 1)); - - /* now the tail position is reserved for the given obj */ - WRITE_ONCE(slot->entries[tail & slot->mask], obj); - /* update sequence to make this obj available for pop() */ - smp_store_release(&slot->last, tail + 1); - - return 0; -} - -/* reclaim an object to object pool */ -int objpool_push(void *obj, struct objpool_head *pool) -{ - unsigned long flags; - int rc; - - /* disable local irq to avoid preemption & interruption */ - raw_local_irq_save(flags); - rc = objpool_try_add_slot(obj, pool, raw_smp_processor_id()); - raw_local_irq_restore(flags); - - return rc; -} -EXPORT_SYMBOL_GPL(objpool_push); - -/* try to retrieve object from slot */ -static inline void *objpool_try_get_slot(struct objpool_head *pool, int cpu) -{ - struct objpool_slot *slot = pool->cpu_slots[cpu]; - /* load head snapshot, other cpus may change it */ - uint32_t head = smp_load_acquire(&slot->head); - - while (head != READ_ONCE(slot->last)) { - void *obj; - - /* - * data visibility of 'last' and 'head' could be out of - * order since memory updating of 'last' and 'head' are - * performed in push() and pop() independently - * - * before any retrieving attempts, pop() must guarantee - * 'last' is behind 'head', that is to say, there must - * be available objects in slot, which could be ensured - * by condition 'last != head && last - head <= nr_objs' - * that is equivalent to 'last - head - 1 < nr_objs' as - * 'last' and 'head' are both unsigned int32 - */ - if (READ_ONCE(slot->last) - head - 1 >= pool->nr_objs) { - head = READ_ONCE(slot->head); - continue; - } - - /* obj must be retrieved before moving forward head */ - obj = READ_ONCE(slot->entries[head & slot->mask]); - - /* move head forward to mark it's consumption */ - if (try_cmpxchg_release(&slot->head, &head, head + 1)) - return obj; - } - - return NULL; -} - -/* allocate an object from object pool */ -void *objpool_pop(struct objpool_head *pool) -{ - void *obj = NULL; - unsigned long flags; - int i, cpu; - - /* disable local irq to avoid preemption & interruption */ - raw_local_irq_save(flags); - - cpu = raw_smp_processor_id(); - for (i = 0; i < num_possible_cpus(); i++) { - obj = objpool_try_get_slot(pool, cpu); - if (obj) - break; - cpu = cpumask_next_wrap(cpu, cpu_possible_mask, -1, 1); - } - raw_local_irq_restore(flags); - - return obj; -} -EXPORT_SYMBOL_GPL(objpool_pop); - /* release whole objpool forcely */ void objpool_free(struct objpool_head *pool) { From 78d0b16127daa26d016c215a089ae330878291f7 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Wed, 24 Apr 2024 14:52:14 -0700 Subject: [PATCH 0934/1702] objpool: cache nr_possible_cpus() and avoid caching nr_cpu_ids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Profiling shows that calling nr_possible_cpus() in objpool_pop() takes a noticeable amount of CPU (when profiled on 80-core machine), as we need to recalculate number of set bits in a CPU bit mask. This number can't change, so there is no point in paying the price for recalculating it. As such, cache this value in struct objpool_head and use it in objpool_pop(). On the other hand, cached pool->nr_cpus isn't necessary, as it's not used in hot path and is also a pretty trivial value to retrieve. So drop pool->nr_cpus in favor of using nr_cpu_ids everywhere. This way the size of struct objpool_head remains the same, which is a nice bonus. Same BPF selftests benchmarks were used to evaluate the effect. Using changes in previous patch (inlining of objpool_pop/objpool_push) as baseline, here are the differences: BASELINE ======== kretprobe : 9.937 ± 0.174M/s kretprobe-multi: 10.440 ± 0.108M/s AFTER ===== kretprobe : 10.106 ± 0.120M/s (+1.7%) kretprobe-multi: 10.515 ± 0.180M/s (+0.7%) Link: https://lore.kernel.org/all/20240424215214.3956041-3-andrii@kernel.org/ Cc: Matt (Qiang) Wu Signed-off-by: Andrii Nakryiko Signed-off-by: Masami Hiramatsu (Google) --- include/linux/objpool.h | 6 +++--- lib/objpool.c | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/include/linux/objpool.h b/include/linux/objpool.h index d8b1f7b91128e..cb1758eaa2d38 100644 --- a/include/linux/objpool.h +++ b/include/linux/objpool.h @@ -73,7 +73,7 @@ typedef int (*objpool_fini_cb)(struct objpool_head *head, void *context); * struct objpool_head - object pooling metadata * @obj_size: object size, aligned to sizeof(void *) * @nr_objs: total objs (to be pre-allocated with objpool) - * @nr_cpus: local copy of nr_cpu_ids + * @nr_possible_cpus: cached value of num_possible_cpus() * @capacity: max objs can be managed by one objpool_slot * @gfp: gfp flags for kmalloc & vmalloc * @ref: refcount of objpool @@ -85,7 +85,7 @@ typedef int (*objpool_fini_cb)(struct objpool_head *head, void *context); struct objpool_head { int obj_size; int nr_objs; - int nr_cpus; + int nr_possible_cpus; int capacity; gfp_t gfp; refcount_t ref; @@ -176,7 +176,7 @@ static inline void *objpool_pop(struct objpool_head *pool) raw_local_irq_save(flags); cpu = raw_smp_processor_id(); - for (i = 0; i < num_possible_cpus(); i++) { + for (i = 0; i < pool->nr_possible_cpus; i++) { obj = __objpool_try_get_slot(pool, cpu); if (obj) break; diff --git a/lib/objpool.c b/lib/objpool.c index f696308fc0264..234f9d0bd081a 100644 --- a/lib/objpool.c +++ b/lib/objpool.c @@ -50,7 +50,7 @@ objpool_init_percpu_slots(struct objpool_head *pool, int nr_objs, { int i, cpu_count = 0; - for (i = 0; i < pool->nr_cpus; i++) { + for (i = 0; i < nr_cpu_ids; i++) { struct objpool_slot *slot; int nodes, size, rc; @@ -60,8 +60,8 @@ objpool_init_percpu_slots(struct objpool_head *pool, int nr_objs, continue; /* compute how many objects to be allocated with this slot */ - nodes = nr_objs / num_possible_cpus(); - if (cpu_count < (nr_objs % num_possible_cpus())) + nodes = nr_objs / pool->nr_possible_cpus; + if (cpu_count < (nr_objs % pool->nr_possible_cpus)) nodes++; cpu_count++; @@ -103,7 +103,7 @@ static void objpool_fini_percpu_slots(struct objpool_head *pool) if (!pool->cpu_slots) return; - for (i = 0; i < pool->nr_cpus; i++) + for (i = 0; i < nr_cpu_ids; i++) kvfree(pool->cpu_slots[i]); kfree(pool->cpu_slots); } @@ -130,13 +130,13 @@ int objpool_init(struct objpool_head *pool, int nr_objs, int object_size, /* initialize objpool pool */ memset(pool, 0, sizeof(struct objpool_head)); - pool->nr_cpus = nr_cpu_ids; + pool->nr_possible_cpus = num_possible_cpus(); pool->obj_size = object_size; pool->capacity = capacity; pool->gfp = gfp & ~__GFP_ZERO; pool->context = context; pool->release = release; - slot_size = pool->nr_cpus * sizeof(struct objpool_slot); + slot_size = nr_cpu_ids * sizeof(struct objpool_slot); pool->cpu_slots = kzalloc(slot_size, pool->gfp); if (!pool->cpu_slots) return -ENOMEM; From 0928fc15f31553c7acb8117b0609799fc0f22fa5 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 1 May 2024 15:27:29 +0100 Subject: [PATCH 0935/1702] iommu/arm-smmu-qcom: Don't build debug features as a kernel module The Qualcomm TBU debug support introduced by 414ecb030870 ("iommu/arm-smmu-qcom-debug: Add support for TBUs") provides its own driver initialisation function, which breaks the link when the core SMMU driver is built as a module: ld.lld: error: duplicate symbol: init_module >>> defined at arm-smmu.c >>> drivers/iommu/arm/arm-smmu/arm-smmu.o:(init_module) >>> defined at arm-smmu-qcom-debug.c >>> drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.o:(.init.text+0x4) Since we're late in the cycle, just make the debug features depend on a non-modular SMMU driver for now while the initialisation is reworked to hang off qcom_smmu_impl_init(). Signed-off-by: Will Deacon --- drivers/iommu/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 0ac80e1094f3a..13d9cb4ba52c5 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -376,7 +376,7 @@ config ARM_SMMU_QCOM config ARM_SMMU_QCOM_DEBUG bool "ARM SMMU QCOM implementation defined debug support" - depends on ARM_SMMU_QCOM + depends on ARM_SMMU_QCOM=y help Support for implementation specific debug features in ARM SMMU hardware found in QTI platforms. This include support for From de31c355541286aa4c938c982dfcafbf062fcb93 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 30 Apr 2024 14:21:33 -0300 Subject: [PATCH 0936/1702] iommu/arm-smmu-v3: Add an ops indirection to the STE code Prepare to put the CD code into the same mechanism. Add an ops indirection around all the STE specific code and make the worker functions independent of the entry content being processed. get_used and sync ops are provided to hook the correct code. Signed-off-by: Michael Shavit Reviewed-by: Michael Shavit Reviewed-by: Nicolin Chen Tested-by: Nicolin Chen Tested-by: Shameer Kolothum Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/1-v9-5040dc602008+177d7-smmuv3_newapi_p2_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 176 ++++++++++++-------- 1 file changed, 104 insertions(+), 72 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index ac7164baa68c2..d0ad1ed62f2be 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -42,8 +42,19 @@ enum arm_smmu_msi_index { ARM_SMMU_MAX_MSIS, }; -static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, - ioasid_t sid); +struct arm_smmu_entry_writer_ops; +struct arm_smmu_entry_writer { + const struct arm_smmu_entry_writer_ops *ops; + struct arm_smmu_master *master; +}; + +struct arm_smmu_entry_writer_ops { + void (*get_used)(const __le64 *entry, __le64 *used); + void (*sync)(struct arm_smmu_entry_writer *writer); +}; + +#define NUM_ENTRY_QWORDS 8 +static_assert(sizeof(struct arm_smmu_ste) == NUM_ENTRY_QWORDS * sizeof(u64)); static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = { [EVTQ_MSI_INDEX] = { @@ -972,43 +983,42 @@ void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid) * would be nice if this was complete according to the spec, but minimally it * has to capture the bits this driver uses. */ -static void arm_smmu_get_ste_used(const struct arm_smmu_ste *ent, - struct arm_smmu_ste *used_bits) +static void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits) { - unsigned int cfg = FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(ent->data[0])); + unsigned int cfg = FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(ent[0])); - used_bits->data[0] = cpu_to_le64(STRTAB_STE_0_V); - if (!(ent->data[0] & cpu_to_le64(STRTAB_STE_0_V))) + used_bits[0] = cpu_to_le64(STRTAB_STE_0_V); + if (!(ent[0] & cpu_to_le64(STRTAB_STE_0_V))) return; - used_bits->data[0] |= cpu_to_le64(STRTAB_STE_0_CFG); + used_bits[0] |= cpu_to_le64(STRTAB_STE_0_CFG); /* S1 translates */ if (cfg & BIT(0)) { - used_bits->data[0] |= cpu_to_le64(STRTAB_STE_0_S1FMT | - STRTAB_STE_0_S1CTXPTR_MASK | - STRTAB_STE_0_S1CDMAX); - used_bits->data[1] |= + used_bits[0] |= cpu_to_le64(STRTAB_STE_0_S1FMT | + STRTAB_STE_0_S1CTXPTR_MASK | + STRTAB_STE_0_S1CDMAX); + used_bits[1] |= cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR | STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH | STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW | STRTAB_STE_1_EATS); - used_bits->data[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID); + used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID); } /* S2 translates */ if (cfg & BIT(1)) { - used_bits->data[1] |= + used_bits[1] |= cpu_to_le64(STRTAB_STE_1_EATS | STRTAB_STE_1_SHCFG); - used_bits->data[2] |= + used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID | STRTAB_STE_2_VTCR | STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2ENDI | STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2R); - used_bits->data[3] |= cpu_to_le64(STRTAB_STE_3_S2TTB_MASK); + used_bits[3] |= cpu_to_le64(STRTAB_STE_3_S2TTB_MASK); } if (cfg == STRTAB_STE_0_CFG_BYPASS) - used_bits->data[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG); + used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG); } /* @@ -1017,57 +1027,55 @@ static void arm_smmu_get_ste_used(const struct arm_smmu_ste *ent, * unused_update is an intermediate value of entry that has unused bits set to * their new values. */ -static u8 arm_smmu_entry_qword_diff(const struct arm_smmu_ste *entry, - const struct arm_smmu_ste *target, - struct arm_smmu_ste *unused_update) +static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer, + const __le64 *entry, const __le64 *target, + __le64 *unused_update) { - struct arm_smmu_ste target_used = {}; - struct arm_smmu_ste cur_used = {}; + __le64 target_used[NUM_ENTRY_QWORDS] = {}; + __le64 cur_used[NUM_ENTRY_QWORDS] = {}; u8 used_qword_diff = 0; unsigned int i; - arm_smmu_get_ste_used(entry, &cur_used); - arm_smmu_get_ste_used(target, &target_used); + writer->ops->get_used(entry, cur_used); + writer->ops->get_used(target, target_used); - for (i = 0; i != ARRAY_SIZE(target_used.data); i++) { + for (i = 0; i != NUM_ENTRY_QWORDS; i++) { /* * Check that masks are up to date, the make functions are not * allowed to set a bit to 1 if the used function doesn't say it * is used. */ - WARN_ON_ONCE(target->data[i] & ~target_used.data[i]); + WARN_ON_ONCE(target[i] & ~target_used[i]); /* Bits can change because they are not currently being used */ - unused_update->data[i] = (entry->data[i] & cur_used.data[i]) | - (target->data[i] & ~cur_used.data[i]); + unused_update[i] = (entry[i] & cur_used[i]) | + (target[i] & ~cur_used[i]); /* * Each bit indicates that a used bit in a qword needs to be * changed after unused_update is applied. */ - if ((unused_update->data[i] & target_used.data[i]) != - target->data[i]) + if ((unused_update[i] & target_used[i]) != target[i]) used_qword_diff |= 1 << i; } return used_qword_diff; } -static bool entry_set(struct arm_smmu_device *smmu, ioasid_t sid, - struct arm_smmu_ste *entry, - const struct arm_smmu_ste *target, unsigned int start, +static bool entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry, + const __le64 *target, unsigned int start, unsigned int len) { bool changed = false; unsigned int i; for (i = start; len != 0; len--, i++) { - if (entry->data[i] != target->data[i]) { - WRITE_ONCE(entry->data[i], target->data[i]); + if (entry[i] != target[i]) { + WRITE_ONCE(entry[i], target[i]); changed = true; } } if (changed) - arm_smmu_sync_ste_for_sid(smmu, sid); + writer->ops->sync(writer); return changed; } @@ -1097,24 +1105,21 @@ static bool entry_set(struct arm_smmu_device *smmu, ioasid_t sid, * V=0 process. This relies on the IGNORED behavior described in the * specification. */ -static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, - struct arm_smmu_ste *entry, - const struct arm_smmu_ste *target) +static void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, + __le64 *entry, const __le64 *target) { - unsigned int num_entry_qwords = ARRAY_SIZE(target->data); - struct arm_smmu_device *smmu = master->smmu; - struct arm_smmu_ste unused_update; + __le64 unused_update[NUM_ENTRY_QWORDS]; u8 used_qword_diff; used_qword_diff = - arm_smmu_entry_qword_diff(entry, target, &unused_update); + arm_smmu_entry_qword_diff(writer, entry, target, unused_update); if (hweight8(used_qword_diff) == 1) { /* * Only one qword needs its used bits to be changed. This is a - * hitless update, update all bits the current STE is ignoring - * to their new values, then update a single "critical qword" to - * change the STE and finally 0 out any bits that are now unused - * in the target configuration. + * hitless update, update all bits the current STE/CD is + * ignoring to their new values, then update a single "critical + * qword" to change the STE/CD and finally 0 out any bits that + * are now unused in the target configuration. */ unsigned int critical_qword_index = ffs(used_qword_diff) - 1; @@ -1123,22 +1128,21 @@ static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, * writing it in the next step anyways. This can save a sync * when the only change is in that qword. */ - unused_update.data[critical_qword_index] = - entry->data[critical_qword_index]; - entry_set(smmu, sid, entry, &unused_update, 0, num_entry_qwords); - entry_set(smmu, sid, entry, target, critical_qword_index, 1); - entry_set(smmu, sid, entry, target, 0, num_entry_qwords); + unused_update[critical_qword_index] = + entry[critical_qword_index]; + entry_set(writer, entry, unused_update, 0, NUM_ENTRY_QWORDS); + entry_set(writer, entry, target, critical_qword_index, 1); + entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS); } else if (used_qword_diff) { /* * At least two qwords need their inuse bits to be changed. This * requires a breaking update, zero the V bit, write all qwords * but 0, then set qword 0 */ - unused_update.data[0] = entry->data[0] & - cpu_to_le64(~STRTAB_STE_0_V); - entry_set(smmu, sid, entry, &unused_update, 0, 1); - entry_set(smmu, sid, entry, target, 1, num_entry_qwords - 1); - entry_set(smmu, sid, entry, target, 0, 1); + unused_update[0] = 0; + entry_set(writer, entry, unused_update, 0, 1); + entry_set(writer, entry, target, 1, NUM_ENTRY_QWORDS - 1); + entry_set(writer, entry, target, 0, 1); } else { /* * No inuse bit changed. Sanity check that all unused bits are 0 @@ -1146,18 +1150,7 @@ static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, * compute_qword_diff(). */ WARN_ON_ONCE( - entry_set(smmu, sid, entry, target, 0, num_entry_qwords)); - } - - /* It's likely that we'll want to use the new STE soon */ - if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) { - struct arm_smmu_cmdq_ent - prefetch_cmd = { .opcode = CMDQ_OP_PREFETCH_CFG, - .prefetch = { - .sid = sid, - } }; - - arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); + entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS)); } } @@ -1430,17 +1423,56 @@ arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc) WRITE_ONCE(*dst, cpu_to_le64(val)); } -static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid) +struct arm_smmu_ste_writer { + struct arm_smmu_entry_writer writer; + u32 sid; +}; + +static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer) { + struct arm_smmu_ste_writer *ste_writer = + container_of(writer, struct arm_smmu_ste_writer, writer); struct arm_smmu_cmdq_ent cmd = { .opcode = CMDQ_OP_CFGI_STE, .cfgi = { - .sid = sid, + .sid = ste_writer->sid, .leaf = true, }, }; - arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); + arm_smmu_cmdq_issue_cmd_with_sync(writer->master->smmu, &cmd); +} + +static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = { + .sync = arm_smmu_ste_writer_sync_entry, + .get_used = arm_smmu_get_ste_used, +}; + +static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, + struct arm_smmu_ste *ste, + const struct arm_smmu_ste *target) +{ + struct arm_smmu_device *smmu = master->smmu; + struct arm_smmu_ste_writer ste_writer = { + .writer = { + .ops = &arm_smmu_ste_writer_ops, + .master = master, + }, + .sid = sid, + }; + + arm_smmu_write_entry(&ste_writer.writer, ste->data, target->data); + + /* It's likely that we'll want to use the new STE soon */ + if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) { + struct arm_smmu_cmdq_ent + prefetch_cmd = { .opcode = CMDQ_OP_PREFETCH_CFG, + .prefetch = { + .sid = sid, + } }; + + arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); + } } static void arm_smmu_make_abort_ste(struct arm_smmu_ste *target) From 78a5fbe8395b365d58142ff9b7a6aeb556481a1f Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 30 Apr 2024 14:21:34 -0300 Subject: [PATCH 0937/1702] iommu/arm-smmu-v3: Make CD programming use arm_smmu_write_entry() CD table entries and STE's have the same essential programming sequence, just with different types. Use the new ops indirection to link CD programming to the common writer. In a few more patches all CD writers will call an appropriate make function and then directly call arm_smmu_write_cd_entry(). arm_smmu_write_ctx_desc() will be removed. Until then lightly tweak arm_smmu_write_ctx_desc() to also use the new programmer by using the same logic as right now to build the target CD on the stack, sanitizing it to meet the used rules, and then using the writer. Sanitizing is necessary because the writer expects that the currently programmed CD follows the used rules. Next patches add new make functions and new direct calls to arm_smmu_write_cd_entry() which will require this. Signed-off-by: Michael Shavit Tested-by: Nicolin Chen Tested-by: Shameer Kolothum Reviewed-by: Moritz Fischer Reviewed-by: Nicolin Chen Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/2-v9-5040dc602008+177d7-smmuv3_newapi_p2_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 89 ++++++++++++++++----- 1 file changed, 67 insertions(+), 22 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index d0ad1ed62f2be..af56c188a7e65 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -55,6 +55,7 @@ struct arm_smmu_entry_writer_ops { #define NUM_ENTRY_QWORDS 8 static_assert(sizeof(struct arm_smmu_ste) == NUM_ENTRY_QWORDS * sizeof(u64)); +static_assert(sizeof(struct arm_smmu_cd) == NUM_ENTRY_QWORDS * sizeof(u64)); static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = { [EVTQ_MSI_INDEX] = { @@ -1230,6 +1231,59 @@ static struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, return &l1_desc->l2ptr[idx]; } +struct arm_smmu_cd_writer { + struct arm_smmu_entry_writer writer; + unsigned int ssid; +}; + +static void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits) +{ + used_bits[0] = cpu_to_le64(CTXDESC_CD_0_V); + if (!(ent[0] & cpu_to_le64(CTXDESC_CD_0_V))) + return; + memset(used_bits, 0xFF, sizeof(struct arm_smmu_cd)); + + /* + * If EPD0 is set by the make function it means + * T0SZ/TG0/IR0/OR0/SH0/TTB0 are IGNORED + */ + if (ent[0] & cpu_to_le64(CTXDESC_CD_0_TCR_EPD0)) { + used_bits[0] &= ~cpu_to_le64( + CTXDESC_CD_0_TCR_T0SZ | CTXDESC_CD_0_TCR_TG0 | + CTXDESC_CD_0_TCR_IRGN0 | CTXDESC_CD_0_TCR_ORGN0 | + CTXDESC_CD_0_TCR_SH0); + used_bits[1] &= ~cpu_to_le64(CTXDESC_CD_1_TTB0_MASK); + } +} + +static void arm_smmu_cd_writer_sync_entry(struct arm_smmu_entry_writer *writer) +{ + struct arm_smmu_cd_writer *cd_writer = + container_of(writer, struct arm_smmu_cd_writer, writer); + + arm_smmu_sync_cd(writer->master, cd_writer->ssid, true); +} + +static const struct arm_smmu_entry_writer_ops arm_smmu_cd_writer_ops = { + .sync = arm_smmu_cd_writer_sync_entry, + .get_used = arm_smmu_get_cd_used, +}; + +static void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid, + struct arm_smmu_cd *cdptr, + const struct arm_smmu_cd *target) +{ + struct arm_smmu_cd_writer cd_writer = { + .writer = { + .ops = &arm_smmu_cd_writer_ops, + .master = master, + }, + .ssid = ssid, + }; + + arm_smmu_write_entry(&cd_writer.writer, cdptr->data, target->data); +} + int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, struct arm_smmu_ctx_desc *cd) { @@ -1246,26 +1300,34 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, */ u64 val; bool cd_live; - struct arm_smmu_cd *cdptr; + struct arm_smmu_cd target; + struct arm_smmu_cd *cdptr = ⌖ + struct arm_smmu_cd *cd_table_entry; struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; struct arm_smmu_device *smmu = master->smmu; if (WARN_ON(ssid >= (1 << cd_table->s1cdmax))) return -E2BIG; - cdptr = arm_smmu_get_cd_ptr(master, ssid); - if (!cdptr) + cd_table_entry = arm_smmu_get_cd_ptr(master, ssid); + if (!cd_table_entry) return -ENOMEM; + target = *cd_table_entry; val = le64_to_cpu(cdptr->data[0]); cd_live = !!(val & CTXDESC_CD_0_V); if (!cd) { /* (5) */ + memset(cdptr, 0, sizeof(*cdptr)); val = 0; } else if (cd == &quiet_cd) { /* (4) */ + val &= ~(CTXDESC_CD_0_TCR_T0SZ | CTXDESC_CD_0_TCR_TG0 | + CTXDESC_CD_0_TCR_IRGN0 | CTXDESC_CD_0_TCR_ORGN0 | + CTXDESC_CD_0_TCR_SH0); if (!(smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) val &= ~(CTXDESC_CD_0_S | CTXDESC_CD_0_R); val |= CTXDESC_CD_0_TCR_EPD0; + cdptr->data[1] &= ~cpu_to_le64(CTXDESC_CD_1_TTB0_MASK); } else if (cd_live) { /* (3) */ val &= ~CTXDESC_CD_0_ASID; val |= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid); @@ -1278,13 +1340,6 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, cdptr->data[2] = 0; cdptr->data[3] = cpu_to_le64(cd->mair); - /* - * STE may be live, and the SMMU might read dwords of this CD in any - * order. Ensure that it observes valid values before reading - * V=1. - */ - arm_smmu_sync_cd(master, ssid, true); - val = cd->tcr | #ifdef __BIG_ENDIAN CTXDESC_CD_0_ENDI | @@ -1298,18 +1353,8 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, if (cd_table->stall_enabled) val |= CTXDESC_CD_0_S; } - - /* - * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3 - * "Configuration structures and configuration invalidation completion" - * - * The size of single-copy atomic reads made by the SMMU is - * IMPLEMENTATION DEFINED but must be at least 64 bits. Any single - * field within an aligned 64-bit span of a structure can be altered - * without first making the structure invalid. - */ - WRITE_ONCE(cdptr->data[0], cpu_to_le64(val)); - arm_smmu_sync_cd(master, ssid, true); + cdptr->data[0] = cpu_to_le64(val); + arm_smmu_write_cd_entry(master, ssid, cd_table_entry, &target); return 0; } From e9d1e4ff74b96cf180d04be38541a245c8c574c1 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 30 Apr 2024 14:21:35 -0300 Subject: [PATCH 0938/1702] iommu/arm-smmu-v3: Move the CD generation for S1 domains into a function Introduce arm_smmu_make_s1_cd() to build the CD from the paging S1 domain, and reorganize all the places programming S1 domain CD table entries to call it. Split arm_smmu_update_s1_domain_cd_entry() from arm_smmu_update_ctx_desc_devices() so that the S1 path has its own call chain separate from the unrelated SVA path. arm_smmu_update_s1_domain_cd_entry() only works on S1 domains attached to RIDs and refreshes all their CDs. Remove case (3) from arm_smmu_write_ctx_desc() as it is now handled by directly calling arm_smmu_write_cd_entry(). Remove the forced clear of the CD during S1 domain attach, arm_smmu_write_cd_entry() will do this automatically if necessary. Tested-by: Nicolin Chen Tested-by: Shameer Kolothum Reviewed-by: Michael Shavit Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/3-v9-5040dc602008+177d7-smmuv3_newapi_p2_jgg@nvidia.com [will: Drop unused arm_smmu_clean_cd_entry() function] Signed-off-by: Will Deacon --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 25 ++++++- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 71 +++++++++++-------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 9 +++ 3 files changed, 76 insertions(+), 29 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 41b44baef15e8..d159f60480935 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -53,6 +53,29 @@ static void arm_smmu_update_ctx_desc_devices(struct arm_smmu_domain *smmu_domain spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); } +static void +arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain) +{ + struct arm_smmu_master *master; + struct arm_smmu_cd target_cd; + unsigned long flags; + + spin_lock_irqsave(&smmu_domain->devices_lock, flags); + list_for_each_entry(master, &smmu_domain->devices, domain_head) { + struct arm_smmu_cd *cdptr; + + /* S1 domains only support RID attachment right now */ + cdptr = arm_smmu_get_cd_ptr(master, IOMMU_NO_PASID); + if (WARN_ON(!cdptr)) + continue; + + arm_smmu_make_s1_cd(&target_cd, master, smmu_domain); + arm_smmu_write_cd_entry(master, IOMMU_NO_PASID, cdptr, + &target_cd); + } + spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); +} + /* * Check if the CPU ASID is available on the SMMU side. If a private context * descriptor is using it, try to replace it. @@ -96,7 +119,7 @@ arm_smmu_share_asid(struct mm_struct *mm, u16 asid) * be some overlap between use of both ASIDs, until we invalidate the * TLB. */ - arm_smmu_update_ctx_desc_devices(smmu_domain, IOMMU_NO_PASID, cd); + arm_smmu_update_s1_domain_cd_entry(smmu_domain); /* Invalidate TLB entries previously associated with that context */ arm_smmu_tlb_inv_asid(smmu, asid); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index af56c188a7e65..1710a45e46715 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1203,8 +1203,8 @@ static void arm_smmu_write_cd_l1_desc(__le64 *dst, WRITE_ONCE(*dst, cpu_to_le64(val)); } -static struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, - u32 ssid) +struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, + u32 ssid) { __le64 *l1ptr; unsigned int idx; @@ -1269,9 +1269,9 @@ static const struct arm_smmu_entry_writer_ops arm_smmu_cd_writer_ops = { .get_used = arm_smmu_get_cd_used, }; -static void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid, - struct arm_smmu_cd *cdptr, - const struct arm_smmu_cd *target) +void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid, + struct arm_smmu_cd *cdptr, + const struct arm_smmu_cd *target) { struct arm_smmu_cd_writer cd_writer = { .writer = { @@ -1284,6 +1284,32 @@ static void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid, arm_smmu_write_entry(&cd_writer.writer, cdptr->data, target->data); } +void arm_smmu_make_s1_cd(struct arm_smmu_cd *target, + struct arm_smmu_master *master, + struct arm_smmu_domain *smmu_domain) +{ + struct arm_smmu_ctx_desc *cd = &smmu_domain->cd; + + memset(target, 0, sizeof(*target)); + + target->data[0] = cpu_to_le64( + cd->tcr | +#ifdef __BIG_ENDIAN + CTXDESC_CD_0_ENDI | +#endif + CTXDESC_CD_0_V | + CTXDESC_CD_0_AA64 | + (master->stall_enabled ? CTXDESC_CD_0_S : 0) | + CTXDESC_CD_0_R | + CTXDESC_CD_0_A | + CTXDESC_CD_0_ASET | + FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) + ); + + target->data[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK); + target->data[3] = cpu_to_le64(cd->mair); +} + int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, struct arm_smmu_ctx_desc *cd) { @@ -1292,14 +1318,11 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, * * (1) Install primary CD, for normal DMA traffic (SSID = IOMMU_NO_PASID = 0). * (2) Install a secondary CD, for SID+SSID traffic. - * (3) Update ASID of a CD. Atomically write the first 64 bits of the - * CD, then invalidate the old entry and mappings. * (4) Quiesce the context without clearing the valid bit. Disable * translation, and ignore any translation fault. * (5) Remove a secondary CD. */ u64 val; - bool cd_live; struct arm_smmu_cd target; struct arm_smmu_cd *cdptr = ⌖ struct arm_smmu_cd *cd_table_entry; @@ -1315,7 +1338,6 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, target = *cd_table_entry; val = le64_to_cpu(cdptr->data[0]); - cd_live = !!(val & CTXDESC_CD_0_V); if (!cd) { /* (5) */ memset(cdptr, 0, sizeof(*cdptr)); @@ -1328,13 +1350,6 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, val &= ~(CTXDESC_CD_0_S | CTXDESC_CD_0_R); val |= CTXDESC_CD_0_TCR_EPD0; cdptr->data[1] &= ~cpu_to_le64(CTXDESC_CD_1_TTB0_MASK); - } else if (cd_live) { /* (3) */ - val &= ~CTXDESC_CD_0_ASID; - val |= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid); - /* - * Until CD+TLB invalidation, both ASIDs may be used for tagging - * this substream's traffic - */ } else { /* (1) and (2) */ cdptr->data[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK); cdptr->data[2] = 0; @@ -2633,29 +2648,29 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); switch (smmu_domain->stage) { - case ARM_SMMU_DOMAIN_S1: + case ARM_SMMU_DOMAIN_S1: { + struct arm_smmu_cd target_cd; + struct arm_smmu_cd *cdptr; + if (!master->cd_table.cdtab) { ret = arm_smmu_alloc_cd_tables(master); if (ret) goto out_list_del; - } else { - /* - * arm_smmu_write_ctx_desc() relies on the entry being - * invalid to work, clear any existing entry. - */ - ret = arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, - NULL); - if (ret) - goto out_list_del; } - ret = arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, &smmu_domain->cd); - if (ret) + cdptr = arm_smmu_get_cd_ptr(master, IOMMU_NO_PASID); + if (!cdptr) { + ret = -ENOMEM; goto out_list_del; + } + arm_smmu_make_s1_cd(&target_cd, master, smmu_domain); + arm_smmu_write_cd_entry(master, IOMMU_NO_PASID, cdptr, + &target_cd); arm_smmu_make_cdtable_ste(&target, master); arm_smmu_install_ste_for_dev(master, &target); break; + } case ARM_SMMU_DOMAIN_S2: arm_smmu_make_s2_domain_ste(&target, master, smmu_domain); arm_smmu_install_ste_for_dev(master, &target); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 4b767e0eeeb68..bb08f087ba39e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -751,6 +751,15 @@ extern struct xarray arm_smmu_asid_xa; extern struct mutex arm_smmu_asid_lock; extern struct arm_smmu_ctx_desc quiet_cd; +struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, + u32 ssid); +void arm_smmu_make_s1_cd(struct arm_smmu_cd *target, + struct arm_smmu_master *master, + struct arm_smmu_domain *smmu_domain); +void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid, + struct arm_smmu_cd *cdptr, + const struct arm_smmu_cd *target); + int arm_smmu_write_ctx_desc(struct arm_smmu_master *smmu_master, int ssid, struct arm_smmu_ctx_desc *cd); void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid); From af8f0b83ea2bcc7cd365c32044f31bdadc07c351 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 30 Apr 2024 14:21:36 -0300 Subject: [PATCH 0939/1702] iommu/arm-smmu-v3: Consolidate clearing a CD table entry A cleared entry is all 0's. Make arm_smmu_clear_cd() do this sequence. If we are clearing an entry and for some reason it is not already allocated in the CD table then something has gone wrong. Remove case (5) from arm_smmu_write_ctx_desc(). Tested-by: Nicolin Chen Tested-by: Shameer Kolothum Reviewed-by: Michael Shavit Reviewed-by: Nicolin Chen Reviewed-by: Moritz Fischer Reviewed-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/4-v9-5040dc602008+177d7-smmuv3_newapi_p2_jgg@nvidia.com Signed-off-by: Will Deacon --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 2 +- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 26 ++++++++++++------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 + 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index d159f60480935..7cf286f7a009f 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -569,7 +569,7 @@ void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain, mutex_lock(&sva_lock); - arm_smmu_write_ctx_desc(master, id, NULL); + arm_smmu_clear_cd(master, id); list_for_each_entry(t, &master->bonds, list) { if (t->mm == mm) { diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 1710a45e46715..802046fbc9ca5 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1310,6 +1310,19 @@ void arm_smmu_make_s1_cd(struct arm_smmu_cd *target, target->data[3] = cpu_to_le64(cd->mair); } +void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid) +{ + struct arm_smmu_cd target = {}; + struct arm_smmu_cd *cdptr; + + if (!master->cd_table.cdtab) + return; + cdptr = arm_smmu_get_cd_ptr(master, ssid); + if (WARN_ON(!cdptr)) + return; + arm_smmu_write_cd_entry(master, ssid, cdptr, &target); +} + int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, struct arm_smmu_ctx_desc *cd) { @@ -1320,7 +1333,6 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, * (2) Install a secondary CD, for SID+SSID traffic. * (4) Quiesce the context without clearing the valid bit. Disable * translation, and ignore any translation fault. - * (5) Remove a secondary CD. */ u64 val; struct arm_smmu_cd target; @@ -1339,10 +1351,7 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, target = *cd_table_entry; val = le64_to_cpu(cdptr->data[0]); - if (!cd) { /* (5) */ - memset(cdptr, 0, sizeof(*cdptr)); - val = 0; - } else if (cd == &quiet_cd) { /* (4) */ + if (cd == &quiet_cd) { /* (4) */ val &= ~(CTXDESC_CD_0_TCR_T0SZ | CTXDESC_CD_0_TCR_TG0 | CTXDESC_CD_0_TCR_IRGN0 | CTXDESC_CD_0_TCR_ORGN0 | CTXDESC_CD_0_TCR_SH0); @@ -2674,9 +2683,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) case ARM_SMMU_DOMAIN_S2: arm_smmu_make_s2_domain_ste(&target, master, smmu_domain); arm_smmu_install_ste_for_dev(master, &target); - if (master->cd_table.cdtab) - arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, - NULL); + arm_smmu_clear_cd(master, IOMMU_NO_PASID); break; } @@ -2724,8 +2731,7 @@ static int arm_smmu_attach_dev_ste(struct device *dev, * arm_smmu_domain->devices to avoid races updating the same context * descriptor from arm_smmu_share_asid(). */ - if (master->cd_table.cdtab) - arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, NULL); + arm_smmu_clear_cd(master, IOMMU_NO_PASID); return 0; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index bb08f087ba39e..99fd6f24caa81 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -751,6 +751,7 @@ extern struct xarray arm_smmu_asid_xa; extern struct mutex arm_smmu_asid_lock; extern struct arm_smmu_ctx_desc quiet_cd; +void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid); struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, u32 ssid); void arm_smmu_make_s1_cd(struct arm_smmu_cd *target, From b2f4c0fcf094dacd2d1fb96a6fd6598919501589 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 30 Apr 2024 14:21:37 -0300 Subject: [PATCH 0940/1702] iommu/arm-smmu-v3: Make arm_smmu_alloc_cd_ptr() Only the attach callers can perform an allocation for the CD table entry, the other callers must not do so, they do not have the correct locking and they cannot sleep. Split up the functions so this is clear. arm_smmu_get_cd_ptr() will return pointer to a CD table entry without doing any kind of allocation. arm_smmu_alloc_cd_ptr() will allocate the table and any required leaf. A following patch will add lockdep assertions to arm_smmu_alloc_cd_ptr() once the restructuring is completed and arm_smmu_alloc_cd_ptr() is never called in the wrong context. Tested-by: Nicolin Chen Reviewed-by: Nicolin Chen Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/5-v9-5040dc602008+177d7-smmuv3_newapi_p2_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 59 +++++++++++++-------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 3 +- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 802046fbc9ca5..6d04b742a0e05 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -97,6 +97,7 @@ static struct arm_smmu_option_prop arm_smmu_options[] = { static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, struct arm_smmu_device *smmu); +static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master); static void parse_driver_options(struct arm_smmu_device *smmu) { @@ -1206,29 +1207,51 @@ static void arm_smmu_write_cd_l1_desc(__le64 *dst, struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, u32 ssid) { - __le64 *l1ptr; - unsigned int idx; struct arm_smmu_l1_ctx_desc *l1_desc; - struct arm_smmu_device *smmu = master->smmu; struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; + if (!cd_table->cdtab) + return NULL; + if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_LINEAR) return (struct arm_smmu_cd *)(cd_table->cdtab + ssid * CTXDESC_CD_DWORDS); - idx = ssid >> CTXDESC_SPLIT; - l1_desc = &cd_table->l1_desc[idx]; - if (!l1_desc->l2ptr) { - if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc)) + l1_desc = &cd_table->l1_desc[ssid / CTXDESC_L2_ENTRIES]; + if (!l1_desc->l2ptr) + return NULL; + return &l1_desc->l2ptr[ssid % CTXDESC_L2_ENTRIES]; +} + +static struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master, + u32 ssid) +{ + struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; + struct arm_smmu_device *smmu = master->smmu; + + if (!cd_table->cdtab) { + if (arm_smmu_alloc_cd_tables(master)) return NULL; + } + + if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_64K_L2) { + unsigned int idx = ssid / CTXDESC_L2_ENTRIES; + struct arm_smmu_l1_ctx_desc *l1_desc; + + l1_desc = &cd_table->l1_desc[idx]; + if (!l1_desc->l2ptr) { + __le64 *l1ptr; - l1ptr = cd_table->cdtab + idx * CTXDESC_L1_DESC_DWORDS; - arm_smmu_write_cd_l1_desc(l1ptr, l1_desc); - /* An invalid L1CD can be cached */ - arm_smmu_sync_cd(master, ssid, false); + if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc)) + return NULL; + + l1ptr = cd_table->cdtab + idx * CTXDESC_L1_DESC_DWORDS; + arm_smmu_write_cd_l1_desc(l1ptr, l1_desc); + /* An invalid L1CD can be cached */ + arm_smmu_sync_cd(master, ssid, false); + } } - idx = ssid & (CTXDESC_L2_ENTRIES - 1); - return &l1_desc->l2ptr[idx]; + return arm_smmu_get_cd_ptr(master, ssid); } struct arm_smmu_cd_writer { @@ -1344,7 +1367,7 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, if (WARN_ON(ssid >= (1 << cd_table->s1cdmax))) return -E2BIG; - cd_table_entry = arm_smmu_get_cd_ptr(master, ssid); + cd_table_entry = arm_smmu_alloc_cd_ptr(master, ssid); if (!cd_table_entry) return -ENOMEM; @@ -2661,13 +2684,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) struct arm_smmu_cd target_cd; struct arm_smmu_cd *cdptr; - if (!master->cd_table.cdtab) { - ret = arm_smmu_alloc_cd_tables(master); - if (ret) - goto out_list_del; - } - - cdptr = arm_smmu_get_cd_ptr(master, IOMMU_NO_PASID); + cdptr = arm_smmu_alloc_cd_ptr(master, IOMMU_NO_PASID); if (!cdptr) { ret = -ENOMEM; goto out_list_del; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 99fd6f24caa81..c5c55d3e28186 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -275,8 +275,7 @@ struct arm_smmu_ste { * 2lvl: at most 1024 L1 entries, * 1024 lazy entries per table. */ -#define CTXDESC_SPLIT 10 -#define CTXDESC_L2_ENTRIES (1 << CTXDESC_SPLIT) +#define CTXDESC_L2_ENTRIES 1024 #define CTXDESC_L1_DESC_DWORDS 1 #define CTXDESC_L1_DESC_V (1UL << 0) From 13abe4faac4348da0cf1c4eeb2b1b39fcfdb4b8f Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 30 Apr 2024 14:21:38 -0300 Subject: [PATCH 0941/1702] iommu/arm-smmu-v3: Allocate the CD table entry in advance Avoid arm_smmu_attach_dev() having to undo the changes to the smmu_domain->devices list, acquire the cdptr earlier so we don't need to handle that error. Now there is a clear break in arm_smmu_attach_dev() where all the prep-work has been done non-disruptively and we commit to making the HW change, which cannot fail. This completes transforming arm_smmu_attach_dev() so that it does not disturb the HW if it fails. Tested-by: Nicolin Chen Tested-by: Shameer Kolothum Reviewed-by: Michael Shavit Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/6-v9-5040dc602008+177d7-smmuv3_newapi_p2_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 24 +++++++-------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 6d04b742a0e05..d7d532c571add 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2635,6 +2635,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) struct arm_smmu_device *smmu; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_master *master; + struct arm_smmu_cd *cdptr; if (!fwspec) return -ENOENT; @@ -2663,6 +2664,12 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) if (ret) return ret; + if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { + cdptr = arm_smmu_alloc_cd_ptr(master, IOMMU_NO_PASID); + if (!cdptr) + return -ENOMEM; + } + /* * Prevent arm_smmu_share_asid() from trying to change the ASID * of either the old or new domain while we are working on it. @@ -2682,13 +2689,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) switch (smmu_domain->stage) { case ARM_SMMU_DOMAIN_S1: { struct arm_smmu_cd target_cd; - struct arm_smmu_cd *cdptr; - - cdptr = arm_smmu_alloc_cd_ptr(master, IOMMU_NO_PASID); - if (!cdptr) { - ret = -ENOMEM; - goto out_list_del; - } arm_smmu_make_s1_cd(&target_cd, master, smmu_domain); arm_smmu_write_cd_entry(master, IOMMU_NO_PASID, cdptr, @@ -2705,16 +2705,8 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) } arm_smmu_enable_ats(master, smmu_domain); - goto out_unlock; - -out_list_del: - spin_lock_irqsave(&smmu_domain->devices_lock, flags); - list_del_init(&master->domain_head); - spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); - -out_unlock: mutex_unlock(&arm_smmu_asid_lock); - return ret; + return 0; } static int arm_smmu_attach_dev_ste(struct device *dev, From 7b87c93c8b86d9d9b9567d83f0ca3d3046fdfc5a Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 30 Apr 2024 14:21:39 -0300 Subject: [PATCH 0942/1702] iommu/arm-smmu-v3: Move the CD generation for SVA into a function Pull all the calculations for building the CD table entry for a mmu_struct into arm_smmu_make_sva_cd(). Call it in the two places installing the SVA CD table entry. Open code the last caller of arm_smmu_update_ctx_desc_devices() and remove the function. Remove arm_smmu_write_ctx_desc() since all callers are gone. Add the locking assertions to arm_smmu_alloc_cd_ptr() since arm_smmu_update_ctx_desc_devices() was the last problematic caller. Remove quiet_cd since all users are gone, arm_smmu_make_sva_cd() creates the same value. The behavior of quiet_cd changes slightly, the old implementation edited the CD in place to set CTXDESC_CD_0_TCR_EPD0 assuming it was a SVA CD entry. This version generates a full CD entry with a 0 TTB0 and relies on arm_smmu_write_cd_entry() to install it hitlessly. Tested-by: Nicolin Chen Tested-by: Shameer Kolothum Reviewed-by: Nicolin Chen Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/7-v9-5040dc602008+177d7-smmuv3_newapi_p2_jgg@nvidia.com Signed-off-by: Will Deacon --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 155 +++++++++++------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 77 +-------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 7 +- 3 files changed, 107 insertions(+), 132 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 7cf286f7a009f..8730a7043909e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -34,25 +34,6 @@ struct arm_smmu_bond { static DEFINE_MUTEX(sva_lock); -/* - * Write the CD to the CD tables for all masters that this domain is attached - * to. Note that this is only used to update existing CD entries in the target - * CD table, for which it's assumed that arm_smmu_write_ctx_desc can't fail. - */ -static void arm_smmu_update_ctx_desc_devices(struct arm_smmu_domain *smmu_domain, - int ssid, - struct arm_smmu_ctx_desc *cd) -{ - struct arm_smmu_master *master; - unsigned long flags; - - spin_lock_irqsave(&smmu_domain->devices_lock, flags); - list_for_each_entry(master, &smmu_domain->devices, domain_head) { - arm_smmu_write_ctx_desc(master, ssid, cd); - } - spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); -} - static void arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain) { @@ -128,11 +109,85 @@ arm_smmu_share_asid(struct mm_struct *mm, u16 asid) return NULL; } +static u64 page_size_to_cd(void) +{ + static_assert(PAGE_SIZE == SZ_4K || PAGE_SIZE == SZ_16K || + PAGE_SIZE == SZ_64K); + if (PAGE_SIZE == SZ_64K) + return ARM_LPAE_TCR_TG0_64K; + if (PAGE_SIZE == SZ_16K) + return ARM_LPAE_TCR_TG0_16K; + return ARM_LPAE_TCR_TG0_4K; +} + +static void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, + struct arm_smmu_master *master, + struct mm_struct *mm, u16 asid) +{ + u64 par; + + memset(target, 0, sizeof(*target)); + + par = cpuid_feature_extract_unsigned_field( + read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1), + ID_AA64MMFR0_EL1_PARANGE_SHIFT); + + target->data[0] = cpu_to_le64( + CTXDESC_CD_0_TCR_EPD1 | +#ifdef __BIG_ENDIAN + CTXDESC_CD_0_ENDI | +#endif + CTXDESC_CD_0_V | + FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par) | + CTXDESC_CD_0_AA64 | + (master->stall_enabled ? CTXDESC_CD_0_S : 0) | + CTXDESC_CD_0_R | + CTXDESC_CD_0_A | + CTXDESC_CD_0_ASET | + FIELD_PREP(CTXDESC_CD_0_ASID, asid)); + + /* + * If no MM is passed then this creates a SVA entry that faults + * everything. arm_smmu_write_cd_entry() can hitlessly go between these + * two entries types since TTB0 is ignored by HW when EPD0 is set. + */ + if (mm) { + target->data[0] |= cpu_to_le64( + FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, + 64ULL - vabits_actual) | + FIELD_PREP(CTXDESC_CD_0_TCR_TG0, page_size_to_cd()) | + FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, + ARM_LPAE_TCR_RGN_WBWA) | + FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, + ARM_LPAE_TCR_RGN_WBWA) | + FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS)); + + target->data[1] = cpu_to_le64(virt_to_phys(mm->pgd) & + CTXDESC_CD_1_TTB0_MASK); + } else { + target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_EPD0); + + /* + * Disable stall and immediately generate an abort if stall + * disable is permitted. This speeds up cleanup for an unclean + * exit if the device is still doing a lot of DMA. + */ + if (!(master->smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) + target->data[0] &= + cpu_to_le64(~(CTXDESC_CD_0_S | CTXDESC_CD_0_R)); + } + + /* + * MAIR value is pretty much constant and global, so we can just get it + * from the current CPU register + */ + target->data[3] = cpu_to_le64(read_sysreg(mair_el1)); +} + static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm) { u16 asid; int err = 0; - u64 tcr, par, reg; struct arm_smmu_ctx_desc *cd; struct arm_smmu_ctx_desc *ret = NULL; @@ -166,39 +221,6 @@ static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm) if (err) goto out_free_asid; - tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 64ULL - vabits_actual) | - FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, ARM_LPAE_TCR_RGN_WBWA) | - FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, ARM_LPAE_TCR_RGN_WBWA) | - FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS) | - CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64; - - switch (PAGE_SIZE) { - case SZ_4K: - tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_4K); - break; - case SZ_16K: - tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_16K); - break; - case SZ_64K: - tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_64K); - break; - default: - WARN_ON(1); - err = -EINVAL; - goto out_free_asid; - } - - reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); - par = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT); - tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par); - - cd->ttbr = virt_to_phys(mm->pgd); - cd->tcr = tcr; - /* - * MAIR value is pretty much constant and global, so we can just get it - * from the current CPU register - */ - cd->mair = read_sysreg(mair_el1); cd->asid = asid; cd->mm = mm; @@ -276,6 +298,8 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) { struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn); struct arm_smmu_domain *smmu_domain = smmu_mn->domain; + struct arm_smmu_master *master; + unsigned long flags; mutex_lock(&sva_lock); if (smmu_mn->cleared) { @@ -287,8 +311,19 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events, * but disable translation. */ - arm_smmu_update_ctx_desc_devices(smmu_domain, mm_get_enqcmd_pasid(mm), - &quiet_cd); + spin_lock_irqsave(&smmu_domain->devices_lock, flags); + list_for_each_entry(master, &smmu_domain->devices, domain_head) { + struct arm_smmu_cd target; + struct arm_smmu_cd *cdptr; + + cdptr = arm_smmu_get_cd_ptr(master, mm_get_enqcmd_pasid(mm)); + if (WARN_ON(!cdptr)) + continue; + arm_smmu_make_sva_cd(&target, master, NULL, smmu_mn->cd->asid); + arm_smmu_write_cd_entry(master, mm_get_enqcmd_pasid(mm), cdptr, + &target); + } + spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_mn->cd->asid); arm_smmu_atc_inv_domain(smmu_domain, mm_get_enqcmd_pasid(mm), 0, 0); @@ -383,6 +418,8 @@ static int __arm_smmu_sva_bind(struct device *dev, ioasid_t pasid, struct mm_struct *mm) { int ret; + struct arm_smmu_cd target; + struct arm_smmu_cd *cdptr; struct arm_smmu_bond *bond; struct arm_smmu_master *master = dev_iommu_priv_get(dev); struct iommu_domain *domain = iommu_get_domain_for_dev(dev); @@ -409,9 +446,13 @@ static int __arm_smmu_sva_bind(struct device *dev, ioasid_t pasid, goto err_free_bond; } - ret = arm_smmu_write_ctx_desc(master, pasid, bond->smmu_mn->cd); - if (ret) + cdptr = arm_smmu_alloc_cd_ptr(master, mm_get_enqcmd_pasid(mm)); + if (!cdptr) { + ret = -ENOMEM; goto err_put_notifier; + } + arm_smmu_make_sva_cd(&target, master, mm, bond->smmu_mn->cd->asid); + arm_smmu_write_cd_entry(master, pasid, cdptr, &target); list_add(&bond->list, &master->bonds); return 0; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index d7d532c571add..f278513a229f0 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -83,12 +83,6 @@ struct arm_smmu_option_prop { DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa); DEFINE_MUTEX(arm_smmu_asid_lock); -/* - * Special value used by SVA when a process dies, to quiesce a CD without - * disabling it. - */ -struct arm_smmu_ctx_desc quiet_cd = { 0 }; - static struct arm_smmu_option_prop arm_smmu_options[] = { { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" }, { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"}, @@ -1200,7 +1194,7 @@ static void arm_smmu_write_cd_l1_desc(__le64 *dst, u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) | CTXDESC_L1_DESC_V; - /* See comment in arm_smmu_write_ctx_desc() */ + /* The HW has 64 bit atomicity with stores to the L2 CD table */ WRITE_ONCE(*dst, cpu_to_le64(val)); } @@ -1223,12 +1217,15 @@ struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, return &l1_desc->l2ptr[ssid % CTXDESC_L2_ENTRIES]; } -static struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master, - u32 ssid) +struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master, + u32 ssid) { struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; struct arm_smmu_device *smmu = master->smmu; + might_sleep(); + iommu_group_mutex_assert(master->dev); + if (!cd_table->cdtab) { if (arm_smmu_alloc_cd_tables(master)) return NULL; @@ -1346,65 +1343,6 @@ void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid) arm_smmu_write_cd_entry(master, ssid, cdptr, &target); } -int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid, - struct arm_smmu_ctx_desc *cd) -{ - /* - * This function handles the following cases: - * - * (1) Install primary CD, for normal DMA traffic (SSID = IOMMU_NO_PASID = 0). - * (2) Install a secondary CD, for SID+SSID traffic. - * (4) Quiesce the context without clearing the valid bit. Disable - * translation, and ignore any translation fault. - */ - u64 val; - struct arm_smmu_cd target; - struct arm_smmu_cd *cdptr = ⌖ - struct arm_smmu_cd *cd_table_entry; - struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; - struct arm_smmu_device *smmu = master->smmu; - - if (WARN_ON(ssid >= (1 << cd_table->s1cdmax))) - return -E2BIG; - - cd_table_entry = arm_smmu_alloc_cd_ptr(master, ssid); - if (!cd_table_entry) - return -ENOMEM; - - target = *cd_table_entry; - val = le64_to_cpu(cdptr->data[0]); - - if (cd == &quiet_cd) { /* (4) */ - val &= ~(CTXDESC_CD_0_TCR_T0SZ | CTXDESC_CD_0_TCR_TG0 | - CTXDESC_CD_0_TCR_IRGN0 | CTXDESC_CD_0_TCR_ORGN0 | - CTXDESC_CD_0_TCR_SH0); - if (!(smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) - val &= ~(CTXDESC_CD_0_S | CTXDESC_CD_0_R); - val |= CTXDESC_CD_0_TCR_EPD0; - cdptr->data[1] &= ~cpu_to_le64(CTXDESC_CD_1_TTB0_MASK); - } else { /* (1) and (2) */ - cdptr->data[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK); - cdptr->data[2] = 0; - cdptr->data[3] = cpu_to_le64(cd->mair); - - val = cd->tcr | -#ifdef __BIG_ENDIAN - CTXDESC_CD_0_ENDI | -#endif - CTXDESC_CD_0_R | CTXDESC_CD_0_A | - (cd->mm ? 0 : CTXDESC_CD_0_ASET) | - CTXDESC_CD_0_AA64 | - FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) | - CTXDESC_CD_0_V; - - if (cd_table->stall_enabled) - val |= CTXDESC_CD_0_S; - } - cdptr->data[0] = cpu_to_le64(val); - arm_smmu_write_cd_entry(master, ssid, cd_table_entry, &target); - return 0; -} - static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master) { int ret; @@ -1413,7 +1351,6 @@ static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master) struct arm_smmu_device *smmu = master->smmu; struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; - cd_table->stall_enabled = master->stall_enabled; cd_table->s1cdmax = master->ssid_bits; max_contexts = 1 << cd_table->s1cdmax; @@ -1511,7 +1448,7 @@ arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc) val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span); val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK; - /* See comment in arm_smmu_write_ctx_desc() */ + /* The HW has 64 bit atomicity with stores to the L2 STE table */ WRITE_ONCE(*dst, cpu_to_le64(val)); } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index c5c55d3e28186..5540609069fcd 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -608,8 +608,6 @@ struct arm_smmu_ctx_desc_cfg { u8 s1fmt; /* log2 of the maximum number of CDs supported by this table */ u8 s1cdmax; - /* Whether CD entries in this table have the stall bit set. */ - u8 stall_enabled:1; }; struct arm_smmu_s2_cfg { @@ -748,11 +746,12 @@ static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) extern struct xarray arm_smmu_asid_xa; extern struct mutex arm_smmu_asid_lock; -extern struct arm_smmu_ctx_desc quiet_cd; void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid); struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, u32 ssid); +struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master, + u32 ssid); void arm_smmu_make_s1_cd(struct arm_smmu_cd *target, struct arm_smmu_master *master, struct arm_smmu_domain *smmu_domain); @@ -760,8 +759,6 @@ void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid, struct arm_smmu_cd *cdptr, const struct arm_smmu_cd *target); -int arm_smmu_write_ctx_desc(struct arm_smmu_master *smmu_master, int ssid, - struct arm_smmu_ctx_desc *cd); void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid); void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, size_t granule, bool leaf, From 04905c17f64890311e6b5a5065d8c220602712e5 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 30 Apr 2024 14:21:40 -0300 Subject: [PATCH 0943/1702] iommu/arm-smmu-v3: Build the whole CD in arm_smmu_make_s1_cd() Half the code was living in arm_smmu_domain_finalise_s1(), just move it here and take the values directly from the pgtbl_ops instead of storing copies. Tested-by: Nicolin Chen Tested-by: Shameer Kolothum Reviewed-by: Michael Shavit Reviewed-by: Mostafa Saleh Reviewed-by: Nicolin Chen Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/8-v9-5040dc602008+177d7-smmuv3_newapi_p2_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 47 ++++++++------------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 3 -- 2 files changed, 18 insertions(+), 32 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index f278513a229f0..00be7392f9fb0 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1309,15 +1309,25 @@ void arm_smmu_make_s1_cd(struct arm_smmu_cd *target, struct arm_smmu_domain *smmu_domain) { struct arm_smmu_ctx_desc *cd = &smmu_domain->cd; + const struct io_pgtable_cfg *pgtbl_cfg = + &io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg; + typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = + &pgtbl_cfg->arm_lpae_s1_cfg.tcr; memset(target, 0, sizeof(*target)); target->data[0] = cpu_to_le64( - cd->tcr | + FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) | + FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) | + FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) | + FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) | + FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) | #ifdef __BIG_ENDIAN CTXDESC_CD_0_ENDI | #endif + CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_V | + FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) | CTXDESC_CD_0_AA64 | (master->stall_enabled ? CTXDESC_CD_0_S : 0) | CTXDESC_CD_0_R | @@ -1325,9 +1335,9 @@ void arm_smmu_make_s1_cd(struct arm_smmu_cd *target, CTXDESC_CD_0_ASET | FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) ); - - target->data[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK); - target->data[3] = cpu_to_le64(cd->mair); + target->data[1] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.ttbr & + CTXDESC_CD_1_TTB0_MASK); + target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.mair); } void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid) @@ -2284,13 +2294,11 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) } static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu, - struct arm_smmu_domain *smmu_domain, - struct io_pgtable_cfg *pgtbl_cfg) + struct arm_smmu_domain *smmu_domain) { int ret; u32 asid; struct arm_smmu_ctx_desc *cd = &smmu_domain->cd; - typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr; refcount_set(&cd->refs, 1); @@ -2298,31 +2306,13 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu, mutex_lock(&arm_smmu_asid_lock); ret = xa_alloc(&arm_smmu_asid_xa, &asid, cd, XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); - if (ret) - goto out_unlock; - cd->asid = (u16)asid; - cd->ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr; - cd->tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) | - FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) | - FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) | - FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) | - FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) | - FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) | - CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64; - cd->mair = pgtbl_cfg->arm_lpae_s1_cfg.mair; - - mutex_unlock(&arm_smmu_asid_lock); - return 0; - -out_unlock: mutex_unlock(&arm_smmu_asid_lock); return ret; } static int arm_smmu_domain_finalise_s2(struct arm_smmu_device *smmu, - struct arm_smmu_domain *smmu_domain, - struct io_pgtable_cfg *pgtbl_cfg) + struct arm_smmu_domain *smmu_domain) { int vmid; struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; @@ -2346,8 +2336,7 @@ static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, struct io_pgtable_cfg pgtbl_cfg; struct io_pgtable_ops *pgtbl_ops; int (*finalise_stage_fn)(struct arm_smmu_device *smmu, - struct arm_smmu_domain *smmu_domain, - struct io_pgtable_cfg *pgtbl_cfg); + struct arm_smmu_domain *smmu_domain); /* Restrict the stage to what we can actually support */ if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) @@ -2390,7 +2379,7 @@ static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, smmu_domain->domain.geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1; smmu_domain->domain.geometry.force_aperture = true; - ret = finalise_stage_fn(smmu, smmu_domain, &pgtbl_cfg); + ret = finalise_stage_fn(smmu, smmu_domain); if (ret < 0) { free_io_pgtable_ops(pgtbl_ops); return ret; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 5540609069fcd..392130b840d55 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -587,9 +587,6 @@ struct arm_smmu_strtab_l1_desc { struct arm_smmu_ctx_desc { u16 asid; - u64 ttbr; - u64 tcr; - u64 mair; refcount_t refs; struct mm_struct *mm; From 56e1a4cc2588a7cb9664457a62fd7a77e005aa01 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 30 Apr 2024 14:21:41 -0300 Subject: [PATCH 0944/1702] iommu/arm-smmu-v3: Add unit tests for arm_smmu_write_entry Add tests for some of the more common STE update operations that we expect to see, as well as some artificial STE updates to test the edges of arm_smmu_write_entry. These also serve as a record of which common operation is expected to be hitless, and how many syncs they require. arm_smmu_write_entry implements a generic algorithm that updates an STE/CD to any other abritrary STE/CD configuration. The update requires a sequence of write+sync operations with some invariants that must be held true after each sync. arm_smmu_write_entry lends itself well to unit-testing since the function's interaction with the STE/CD is already abstracted by input callbacks that we can hook to introspect into the sequence of operations. We can use these hooks to guarantee that invariants are held throughout the entire update operation. Link: https://lore.kernel.org/r/20240106083617.1173871-3-mshavit@google.com Tested-by: Nicolin Chen Signed-off-by: Michael Shavit Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/9-v9-5040dc602008+177d7-smmuv3_newapi_p2_jgg@nvidia.com Signed-off-by: Will Deacon --- drivers/iommu/Kconfig | 13 +- drivers/iommu/arm/arm-smmu-v3/Makefile | 1 + .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 8 +- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 465 ++++++++++++++++++ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 43 +- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 30 ++ 6 files changed, 533 insertions(+), 27 deletions(-) create mode 100644 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 13d9cb4ba52c5..66325210c8c98 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -401,9 +401,9 @@ config ARM_SMMU_V3 Say Y here if your system includes an IOMMU device implementing the ARM SMMUv3 architecture. +if ARM_SMMU_V3 config ARM_SMMU_V3_SVA bool "Shared Virtual Addressing support for the ARM SMMUv3" - depends on ARM_SMMU_V3 select IOMMU_SVA select IOMMU_IOPF select MMU_NOTIFIER @@ -414,6 +414,17 @@ config ARM_SMMU_V3_SVA Say Y here if your system supports SVA extensions such as PCIe PASID and PRI. +config ARM_SMMU_V3_KUNIT_TEST + bool "KUnit tests for arm-smmu-v3 driver" if !KUNIT_ALL_TESTS + depends on KUNIT + depends on ARM_SMMU_V3_SVA + default KUNIT_ALL_TESTS + help + Enable this option to unit-test arm-smmu-v3 driver functions. + + If unsure, say N. +endif + config S390_IOMMU def_bool y if S390 && PCI depends on S390 && PCI diff --git a/drivers/iommu/arm/arm-smmu-v3/Makefile b/drivers/iommu/arm/arm-smmu-v3/Makefile index 54feb1ecccad8..0b97054b3929b 100644 --- a/drivers/iommu/arm/arm-smmu-v3/Makefile +++ b/drivers/iommu/arm/arm-smmu-v3/Makefile @@ -2,4 +2,5 @@ obj-$(CONFIG_ARM_SMMU_V3) += arm_smmu_v3.o arm_smmu_v3-objs-y += arm-smmu-v3.o arm_smmu_v3-objs-$(CONFIG_ARM_SMMU_V3_SVA) += arm-smmu-v3-sva.o +arm_smmu_v3-objs-$(CONFIG_ARM_SMMU_V3_KUNIT_TEST) += arm-smmu-v3-test.o arm_smmu_v3-objs := $(arm_smmu_v3-objs-y) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 8730a7043909e..34a977a0767d4 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "arm-smmu-v3.h" #include "../../io-pgtable-arm.h" @@ -120,9 +121,10 @@ static u64 page_size_to_cd(void) return ARM_LPAE_TCR_TG0_4K; } -static void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, - struct arm_smmu_master *master, - struct mm_struct *mm, u16 asid) +VISIBLE_IF_KUNIT +void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, + struct arm_smmu_master *master, struct mm_struct *mm, + u16 asid) { u64 par; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c new file mode 100644 index 0000000000000..417804392ff08 --- /dev/null +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -0,0 +1,465 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2024 Google LLC. + */ +#include +#include + +#include "arm-smmu-v3.h" + +struct arm_smmu_test_writer { + struct arm_smmu_entry_writer writer; + struct kunit *test; + const __le64 *init_entry; + const __le64 *target_entry; + __le64 *entry; + + bool invalid_entry_written; + unsigned int num_syncs; +}; + +#define NUM_ENTRY_QWORDS 8 +#define NUM_EXPECTED_SYNCS(x) x + +static struct arm_smmu_ste bypass_ste; +static struct arm_smmu_ste abort_ste; +static struct arm_smmu_device smmu = { + .features = ARM_SMMU_FEAT_STALLS | ARM_SMMU_FEAT_ATTR_TYPES_OVR +}; +static struct mm_struct sva_mm = { + .pgd = (void *)0xdaedbeefdeadbeefULL, +}; + +static bool arm_smmu_entry_differs_in_used_bits(const __le64 *entry, + const __le64 *used_bits, + const __le64 *target, + unsigned int length) +{ + bool differs = false; + unsigned int i; + + for (i = 0; i < length; i++) { + if ((entry[i] & used_bits[i]) != target[i]) + differs = true; + } + return differs; +} + +static void +arm_smmu_test_writer_record_syncs(struct arm_smmu_entry_writer *writer) +{ + struct arm_smmu_test_writer *test_writer = + container_of(writer, struct arm_smmu_test_writer, writer); + __le64 *entry_used_bits; + + entry_used_bits = kunit_kzalloc( + test_writer->test, sizeof(*entry_used_bits) * NUM_ENTRY_QWORDS, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test_writer->test, entry_used_bits); + + pr_debug("STE value is now set to: "); + print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, + test_writer->entry, + NUM_ENTRY_QWORDS * sizeof(*test_writer->entry), + false); + + test_writer->num_syncs += 1; + if (!test_writer->entry[0]) { + test_writer->invalid_entry_written = true; + } else { + /* + * At any stage in a hitless transition, the entry must be + * equivalent to either the initial entry or the target entry + * when only considering the bits used by the current + * configuration. + */ + writer->ops->get_used(test_writer->entry, entry_used_bits); + KUNIT_EXPECT_FALSE( + test_writer->test, + arm_smmu_entry_differs_in_used_bits( + test_writer->entry, entry_used_bits, + test_writer->init_entry, NUM_ENTRY_QWORDS) && + arm_smmu_entry_differs_in_used_bits( + test_writer->entry, entry_used_bits, + test_writer->target_entry, + NUM_ENTRY_QWORDS)); + } +} + +static void +arm_smmu_v3_test_debug_print_used_bits(struct arm_smmu_entry_writer *writer, + const __le64 *ste) +{ + __le64 used_bits[NUM_ENTRY_QWORDS] = {}; + + arm_smmu_get_ste_used(ste, used_bits); + pr_debug("STE used bits: "); + print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, used_bits, + sizeof(used_bits), false); +} + +static const struct arm_smmu_entry_writer_ops test_ste_ops = { + .sync = arm_smmu_test_writer_record_syncs, + .get_used = arm_smmu_get_ste_used, +}; + +static const struct arm_smmu_entry_writer_ops test_cd_ops = { + .sync = arm_smmu_test_writer_record_syncs, + .get_used = arm_smmu_get_cd_used, +}; + +static void arm_smmu_v3_test_ste_expect_transition( + struct kunit *test, const struct arm_smmu_ste *cur, + const struct arm_smmu_ste *target, unsigned int num_syncs_expected, + bool hitless) +{ + struct arm_smmu_ste cur_copy = *cur; + struct arm_smmu_test_writer test_writer = { + .writer = { + .ops = &test_ste_ops, + }, + .test = test, + .init_entry = cur->data, + .target_entry = target->data, + .entry = cur_copy.data, + .num_syncs = 0, + .invalid_entry_written = false, + + }; + + pr_debug("STE initial value: "); + print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, cur_copy.data, + sizeof(cur_copy), false); + arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, cur->data); + pr_debug("STE target value: "); + print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, target->data, + sizeof(cur_copy), false); + arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, + target->data); + + arm_smmu_write_entry(&test_writer.writer, cur_copy.data, target->data); + + KUNIT_EXPECT_EQ(test, test_writer.invalid_entry_written, !hitless); + KUNIT_EXPECT_EQ(test, test_writer.num_syncs, num_syncs_expected); + KUNIT_EXPECT_MEMEQ(test, target->data, cur_copy.data, sizeof(cur_copy)); +} + +static void arm_smmu_v3_test_ste_expect_hitless_transition( + struct kunit *test, const struct arm_smmu_ste *cur, + const struct arm_smmu_ste *target, unsigned int num_syncs_expected) +{ + arm_smmu_v3_test_ste_expect_transition(test, cur, target, + num_syncs_expected, true); +} + +static const dma_addr_t fake_cdtab_dma_addr = 0xF0F0F0F0F0F0; + +static void arm_smmu_test_make_cdtable_ste(struct arm_smmu_ste *ste, + const dma_addr_t dma_addr) +{ + struct arm_smmu_master master = { + .cd_table.cdtab_dma = dma_addr, + .cd_table.s1cdmax = 0xFF, + .cd_table.s1fmt = STRTAB_STE_0_S1FMT_64K_L2, + .smmu = &smmu, + }; + + arm_smmu_make_cdtable_ste(ste, &master); +} + +static void arm_smmu_v3_write_ste_test_bypass_to_abort(struct kunit *test) +{ + /* + * Bypass STEs has used bits in the first two Qwords, while abort STEs + * only have used bits in the first QWord. Transitioning from bypass to + * abort requires two syncs: the first to set the first qword and make + * the STE into an abort, the second to clean up the second qword. + */ + arm_smmu_v3_test_ste_expect_hitless_transition( + test, &bypass_ste, &abort_ste, NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_ste_test_abort_to_bypass(struct kunit *test) +{ + /* + * Transitioning from abort to bypass also requires two syncs: the first + * to set the second qword data required by the bypass STE, and the + * second to set the first qword and switch to bypass. + */ + arm_smmu_v3_test_ste_expect_hitless_transition( + test, &abort_ste, &bypass_ste, NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_ste_test_cdtable_to_abort(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &abort_ste, + NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_ste_test_abort_to_cdtable(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &abort_ste, &ste, + NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_ste_test_cdtable_to_bypass(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &bypass_ste, + NUM_EXPECTED_SYNCS(3)); +} + +static void arm_smmu_v3_write_ste_test_bypass_to_cdtable(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &bypass_ste, &ste, + NUM_EXPECTED_SYNCS(3)); +} + +static void arm_smmu_test_make_s2_ste(struct arm_smmu_ste *ste, + bool ats_enabled) +{ + struct arm_smmu_master master = { + .smmu = &smmu, + .ats_enabled = ats_enabled, + }; + struct io_pgtable io_pgtable = {}; + struct arm_smmu_domain smmu_domain = { + .pgtbl_ops = &io_pgtable.ops, + }; + + io_pgtable.cfg.arm_lpae_s2_cfg.vttbr = 0xdaedbeefdeadbeefULL; + io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.ps = 1; + io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.tg = 2; + io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.sh = 3; + io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.orgn = 1; + io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.irgn = 2; + io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.sl = 3; + io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.tsz = 4; + + arm_smmu_make_s2_domain_ste(ste, &master, &smmu_domain); +} + +static void arm_smmu_v3_write_ste_test_s2_to_abort(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_s2_ste(&ste, true); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &abort_ste, + NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_ste_test_abort_to_s2(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_s2_ste(&ste, true); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &abort_ste, &ste, + NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_ste_test_s2_to_bypass(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_s2_ste(&ste, true); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &bypass_ste, + NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_ste_test_bypass_to_s2(struct kunit *test) +{ + struct arm_smmu_ste ste; + + arm_smmu_test_make_s2_ste(&ste, true); + arm_smmu_v3_test_ste_expect_hitless_transition(test, &bypass_ste, &ste, + NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_test_cd_expect_transition( + struct kunit *test, const struct arm_smmu_cd *cur, + const struct arm_smmu_cd *target, unsigned int num_syncs_expected, + bool hitless) +{ + struct arm_smmu_cd cur_copy = *cur; + struct arm_smmu_test_writer test_writer = { + .writer = { + .ops = &test_cd_ops, + }, + .test = test, + .init_entry = cur->data, + .target_entry = target->data, + .entry = cur_copy.data, + .num_syncs = 0, + .invalid_entry_written = false, + + }; + + pr_debug("CD initial value: "); + print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, cur_copy.data, + sizeof(cur_copy), false); + arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, cur->data); + pr_debug("CD target value: "); + print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, target->data, + sizeof(cur_copy), false); + arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, + target->data); + + arm_smmu_write_entry(&test_writer.writer, cur_copy.data, target->data); + + KUNIT_EXPECT_EQ(test, test_writer.invalid_entry_written, !hitless); + KUNIT_EXPECT_EQ(test, test_writer.num_syncs, num_syncs_expected); + KUNIT_EXPECT_MEMEQ(test, target->data, cur_copy.data, sizeof(cur_copy)); +} + +static void arm_smmu_v3_test_cd_expect_non_hitless_transition( + struct kunit *test, const struct arm_smmu_cd *cur, + const struct arm_smmu_cd *target, unsigned int num_syncs_expected) +{ + arm_smmu_v3_test_cd_expect_transition(test, cur, target, + num_syncs_expected, false); +} + +static void arm_smmu_v3_test_cd_expect_hitless_transition( + struct kunit *test, const struct arm_smmu_cd *cur, + const struct arm_smmu_cd *target, unsigned int num_syncs_expected) +{ + arm_smmu_v3_test_cd_expect_transition(test, cur, target, + num_syncs_expected, true); +} + +static void arm_smmu_test_make_s1_cd(struct arm_smmu_cd *cd, unsigned int asid) +{ + struct arm_smmu_master master = { + .smmu = &smmu, + }; + struct io_pgtable io_pgtable = {}; + struct arm_smmu_domain smmu_domain = { + .pgtbl_ops = &io_pgtable.ops, + .cd = { + .asid = asid, + }, + }; + + io_pgtable.cfg.arm_lpae_s1_cfg.ttbr = 0xdaedbeefdeadbeefULL; + io_pgtable.cfg.arm_lpae_s1_cfg.tcr.ips = 1; + io_pgtable.cfg.arm_lpae_s1_cfg.tcr.tg = 2; + io_pgtable.cfg.arm_lpae_s1_cfg.tcr.sh = 3; + io_pgtable.cfg.arm_lpae_s1_cfg.tcr.orgn = 1; + io_pgtable.cfg.arm_lpae_s1_cfg.tcr.irgn = 2; + io_pgtable.cfg.arm_lpae_s1_cfg.tcr.tsz = 4; + io_pgtable.cfg.arm_lpae_s1_cfg.mair = 0xabcdef012345678ULL; + + arm_smmu_make_s1_cd(cd, &master, &smmu_domain); +} + +static void arm_smmu_v3_write_cd_test_s1_clear(struct kunit *test) +{ + struct arm_smmu_cd cd = {}; + struct arm_smmu_cd cd_2; + + arm_smmu_test_make_s1_cd(&cd_2, 1997); + arm_smmu_v3_test_cd_expect_non_hitless_transition( + test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); + arm_smmu_v3_test_cd_expect_non_hitless_transition( + test, &cd_2, &cd, NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_cd_test_s1_change_asid(struct kunit *test) +{ + struct arm_smmu_cd cd = {}; + struct arm_smmu_cd cd_2; + + arm_smmu_test_make_s1_cd(&cd, 778); + arm_smmu_test_make_s1_cd(&cd_2, 1997); + arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd, &cd_2, + NUM_EXPECTED_SYNCS(1)); + arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd_2, &cd, + NUM_EXPECTED_SYNCS(1)); +} + +static void arm_smmu_test_make_sva_cd(struct arm_smmu_cd *cd, unsigned int asid) +{ + struct arm_smmu_master master = { + .smmu = &smmu, + }; + + arm_smmu_make_sva_cd(cd, &master, &sva_mm, asid); +} + +static void arm_smmu_test_make_sva_release_cd(struct arm_smmu_cd *cd, + unsigned int asid) +{ + struct arm_smmu_master master = { + .smmu = &smmu, + }; + + arm_smmu_make_sva_cd(cd, &master, NULL, asid); +} + +static void arm_smmu_v3_write_cd_test_sva_clear(struct kunit *test) +{ + struct arm_smmu_cd cd = {}; + struct arm_smmu_cd cd_2; + + arm_smmu_test_make_sva_cd(&cd_2, 1997); + arm_smmu_v3_test_cd_expect_non_hitless_transition( + test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); + arm_smmu_v3_test_cd_expect_non_hitless_transition( + test, &cd_2, &cd, NUM_EXPECTED_SYNCS(2)); +} + +static void arm_smmu_v3_write_cd_test_sva_release(struct kunit *test) +{ + struct arm_smmu_cd cd; + struct arm_smmu_cd cd_2; + + arm_smmu_test_make_sva_cd(&cd, 1997); + arm_smmu_test_make_sva_release_cd(&cd_2, 1997); + arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd, &cd_2, + NUM_EXPECTED_SYNCS(2)); + arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd_2, &cd, + NUM_EXPECTED_SYNCS(2)); +} + +static struct kunit_case arm_smmu_v3_test_cases[] = { + KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_abort), + KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_bypass), + KUNIT_CASE(arm_smmu_v3_write_ste_test_cdtable_to_abort), + KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_cdtable), + KUNIT_CASE(arm_smmu_v3_write_ste_test_cdtable_to_bypass), + KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_cdtable), + KUNIT_CASE(arm_smmu_v3_write_ste_test_s2_to_abort), + KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_s2), + KUNIT_CASE(arm_smmu_v3_write_ste_test_s2_to_bypass), + KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_s2), + KUNIT_CASE(arm_smmu_v3_write_cd_test_s1_clear), + KUNIT_CASE(arm_smmu_v3_write_cd_test_s1_change_asid), + KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_clear), + KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_release), + {}, +}; + +static int arm_smmu_v3_test_suite_init(struct kunit_suite *test) +{ + arm_smmu_make_bypass_ste(&smmu, &bypass_ste); + arm_smmu_make_abort_ste(&abort_ste); + return 0; +} + +static struct kunit_suite arm_smmu_v3_test_module = { + .name = "arm-smmu-v3-kunit-test", + .suite_init = arm_smmu_v3_test_suite_init, + .test_cases = arm_smmu_v3_test_cases, +}; +kunit_test_suites(&arm_smmu_v3_test_module); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 00be7392f9fb0..1ad0937760c69 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -26,6 +26,7 @@ #include #include #include +#include #include "arm-smmu-v3.h" #include "../../dma-iommu.h" @@ -42,17 +43,6 @@ enum arm_smmu_msi_index { ARM_SMMU_MAX_MSIS, }; -struct arm_smmu_entry_writer_ops; -struct arm_smmu_entry_writer { - const struct arm_smmu_entry_writer_ops *ops; - struct arm_smmu_master *master; -}; - -struct arm_smmu_entry_writer_ops { - void (*get_used)(const __le64 *entry, __le64 *used); - void (*sync)(struct arm_smmu_entry_writer *writer); -}; - #define NUM_ENTRY_QWORDS 8 static_assert(sizeof(struct arm_smmu_ste) == NUM_ENTRY_QWORDS * sizeof(u64)); static_assert(sizeof(struct arm_smmu_cd) == NUM_ENTRY_QWORDS * sizeof(u64)); @@ -979,7 +969,8 @@ void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid) * would be nice if this was complete according to the spec, but minimally it * has to capture the bits this driver uses. */ -static void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits) +VISIBLE_IF_KUNIT +void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits) { unsigned int cfg = FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(ent[0])); @@ -1101,8 +1092,9 @@ static bool entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry, * V=0 process. This relies on the IGNORED behavior described in the * specification. */ -static void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, - __le64 *entry, const __le64 *target) +VISIBLE_IF_KUNIT +void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry, + const __le64 *target) { __le64 unused_update[NUM_ENTRY_QWORDS]; u8 used_qword_diff; @@ -1256,7 +1248,8 @@ struct arm_smmu_cd_writer { unsigned int ssid; }; -static void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits) +VISIBLE_IF_KUNIT +void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits) { used_bits[0] = cpu_to_le64(CTXDESC_CD_0_V); if (!(ent[0] & cpu_to_le64(CTXDESC_CD_0_V))) @@ -1514,7 +1507,8 @@ static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, } } -static void arm_smmu_make_abort_ste(struct arm_smmu_ste *target) +VISIBLE_IF_KUNIT +void arm_smmu_make_abort_ste(struct arm_smmu_ste *target) { memset(target, 0, sizeof(*target)); target->data[0] = cpu_to_le64( @@ -1522,8 +1516,9 @@ static void arm_smmu_make_abort_ste(struct arm_smmu_ste *target) FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT)); } -static void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu, - struct arm_smmu_ste *target) +VISIBLE_IF_KUNIT +void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu, + struct arm_smmu_ste *target) { memset(target, 0, sizeof(*target)); target->data[0] = cpu_to_le64( @@ -1535,8 +1530,9 @@ static void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu, STRTAB_STE_1_SHCFG_INCOMING)); } -static void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, - struct arm_smmu_master *master) +VISIBLE_IF_KUNIT +void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, + struct arm_smmu_master *master) { struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; struct arm_smmu_device *smmu = master->smmu; @@ -1585,9 +1581,10 @@ static void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, } } -static void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, - struct arm_smmu_master *master, - struct arm_smmu_domain *smmu_domain) +VISIBLE_IF_KUNIT +void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, + struct arm_smmu_master *master, + struct arm_smmu_domain *smmu_domain) { struct arm_smmu_s2_cfg *s2_cfg = &smmu_domain->s2_cfg; const struct io_pgtable_cfg *pgtbl_cfg = diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 392130b840d55..1242a086c9f94 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -736,6 +736,36 @@ struct arm_smmu_domain { struct list_head mmu_notifiers; }; +/* The following are exposed for testing purposes. */ +struct arm_smmu_entry_writer_ops; +struct arm_smmu_entry_writer { + const struct arm_smmu_entry_writer_ops *ops; + struct arm_smmu_master *master; +}; + +struct arm_smmu_entry_writer_ops { + void (*get_used)(const __le64 *entry, __le64 *used); + void (*sync)(struct arm_smmu_entry_writer *writer); +}; + +#if IS_ENABLED(CONFIG_KUNIT) +void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits); +void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *cur, + const __le64 *target); +void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits); +void arm_smmu_make_abort_ste(struct arm_smmu_ste *target); +void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu, + struct arm_smmu_ste *target); +void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, + struct arm_smmu_master *master); +void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, + struct arm_smmu_master *master, + struct arm_smmu_domain *smmu_domain); +void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, + struct arm_smmu_master *master, struct mm_struct *mm, + u16 asid); +#endif + static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) { return container_of(dom, struct arm_smmu_domain, domain); From 604a57ba9781181b4b27443520b1b6c65398deae Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Fri, 5 Apr 2024 17:56:03 -0500 Subject: [PATCH 0945/1702] dt-bindings: kbuild: Add separate target/dependency for processed-schema.json Running dtbs_check and dt_compatible_check targets really only depend on processed-schema.json, but the dependency is 'dt_binding_check'. That was sort worked around with the CHECK_DT_BINDING variable in order to skip some of the work that 'dt_binding_check' does. It still runs the full checks of the schemas which is not necessary and adds 10s of seconds to the build time. That's significant when checking only a few DTBs and with recent changes that have improved the validation time by 6-7x. Add a new target, dt_binding_schema, which just builds processed-schema.json and can be used as the dependency for other targets. The scripts_dtc dependency isn't needed either as the examples aren't built for it. Signed-off-by: Rob Herring Tested-by: Conor Dooley Signed-off-by: Masahiro Yamada --- Documentation/devicetree/bindings/Makefile | 9 ++++++--- Makefile | 22 +++++++++++++--------- scripts/Makefile.lib | 2 +- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/Documentation/devicetree/bindings/Makefile b/Documentation/devicetree/bindings/Makefile index 3779405269ab9..8cdda477987fd 100644 --- a/Documentation/devicetree/bindings/Makefile +++ b/Documentation/devicetree/bindings/Makefile @@ -63,7 +63,7 @@ override DTC_FLAGS := \ $(obj)/processed-schema.json: $(DT_DOCS) check_dtschema_version FORCE $(call if_changed,mk_schema) -always-$(CHECK_DT_BINDING) += .dt-binding.checked .yamllint.checked +targets += .dt-binding.checked .yamllint.checked $(obj)/.yamllint.checked: $(DT_DOCS) $(src)/.yamllint FORCE $(if $(DT_SCHEMA_LINT),$(call if_changed,yamllint),) @@ -71,8 +71,8 @@ $(obj)/.dt-binding.checked: $(DT_DOCS) FORCE $(call if_changed,chk_bindings) always-y += processed-schema.json -always-$(CHECK_DT_BINDING) += $(patsubst $(obj)/%,%, $(CHK_DT_EXAMPLES)) -always-$(CHECK_DT_BINDING) += $(patsubst $(obj)/%.dtb,%.dts, $(CHK_DT_EXAMPLES)) +targets += $(patsubst $(obj)/%,%, $(CHK_DT_EXAMPLES)) +targets += $(patsubst $(obj)/%.dtb,%.dts, $(CHK_DT_EXAMPLES)) # Hack: avoid 'Argument list too long' error for 'make clean'. Remove most of # build artifacts here before they are processed by scripts/Makefile.clean @@ -81,3 +81,6 @@ clean-files = $(shell find $(obj) \( -name '*.example.dts' -o \ dt_compatible_check: $(obj)/processed-schema.json $(Q)$(srctree)/scripts/dtc/dt-extract-compatibles $(srctree) | xargs dt-check-compatible -v -s $< + +PHONY += dt_binding_check +dt_binding_check: $(obj)/.dt-binding.checked $(obj)/.yamllint.checked $(CHK_DT_EXAMPLES) diff --git a/Makefile b/Makefile index 43b10f3d438cf..026971f9f6bcc 100644 --- a/Makefile +++ b/Makefile @@ -1403,7 +1403,7 @@ export CHECK_DTBS=y endif ifneq ($(CHECK_DTBS),) -dtbs_prepare: dt_binding_check +dtbs_prepare: dt_binding_schemas endif dtbs_check: dtbs @@ -1422,15 +1422,18 @@ scripts_dtc: scripts_basic $(Q)$(MAKE) $(build)=scripts/dtc ifneq ($(filter dt_binding_check, $(MAKECMDGOALS)),) -export CHECK_DT_BINDING=y +export CHECK_DTBS=y endif -PHONY += dt_binding_check -dt_binding_check: scripts_dtc +PHONY += dt_binding_check dt_binding_schemas +dt_binding_check: dt_binding_schemas scripts_dtc + $(Q)$(MAKE) $(build)=Documentation/devicetree/bindings $@ + +dt_binding_schemas: $(Q)$(MAKE) $(build)=Documentation/devicetree/bindings PHONY += dt_compatible_check -dt_compatible_check: dt_binding_check +dt_compatible_check: dt_binding_schemas $(Q)$(MAKE) $(build)=Documentation/devicetree/bindings $@ # --------------------------------------------------------------------------- @@ -1626,10 +1629,11 @@ help: @echo '' @$(if $(dtstree), \ echo 'Devicetree:'; \ - echo '* dtbs - Build device tree blobs for enabled boards'; \ - echo ' dtbs_install - Install dtbs to $(INSTALL_DTBS_PATH)'; \ - echo ' dt_binding_check - Validate device tree binding documents'; \ - echo ' dtbs_check - Validate device tree source files';\ + echo '* dtbs - Build device tree blobs for enabled boards'; \ + echo ' dtbs_install - Install dtbs to $(INSTALL_DTBS_PATH)'; \ + echo ' dt_binding_check - Validate device tree binding documents and examples'; \ + echo ' dt_binding_schema - Build processed device tree binding schemas'; \ + echo ' dtbs_check - Validate device tree source files';\ echo '') @echo 'Userspace tools targets:' diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 3179747cbd2cc..d1d51e38b55d7 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -410,7 +410,7 @@ $(multi-dtb-y): FORCE $(call if_changed,fdtoverlay) $(call multi_depend, $(multi-dtb-y), .dtb, -dtbs) -ifneq ($(CHECK_DTBS)$(CHECK_DT_BINDING),) +ifneq ($(CHECK_DTBS),) DT_CHECKER ?= dt-validate DT_CHECKER_FLAGS ?= $(if $(DT_SCHEMA_FILES),-l $(DT_SCHEMA_FILES),-m) DT_BINDING_DIR := Documentation/devicetree/bindings From c3f7bed8fa141c02d715f856ba1a6a757b8c8db9 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 15 Apr 2024 02:41:39 +0900 Subject: [PATCH 0946/1702] kbuild: buildtar: add comments about inconsistent package generation scripts/package/buildtar checks some kernel packages, and copies the first image found. This may potentially produce an inconsistent (and possibly wrong) package. For instance, the for-loop for arm64 checks Image.{bz2,gz,lz4,lzma,lzo}, and vmlinuz.efi, then copies the first image found, which might be a stale image created in a previous build. When CONFIG_EFI_ZBOOT is enabled in the pristine source tree, 'make ARCH=arm64 tar-pkg' will build and copy vmlinuz.efi. This is the expected behavior. If you build the kernel with CONFIG_EFI_ZBOOT disabled, Image.gz will be created, which will remain in the tree until you run 'make clean'. Even if CONFIG_EFI_ZBOOT is turned on later, 'make ARCH=arm64 tar-pkg' will copy stale Image.gz instead of the latest vmlinuz.efi, as Image.gz takes precedence over vmlinuz.efi. In summary, the code "[ -f ... ] && cp" does not consistently produce the desired outcome. Other packaging targets are deterministic; deb-pkg and rpm-pkg copies ${KBUILD_IMAGE}, which is determined by CONFIG options. I removed [ -f ... ] checks from x86, alpha, parisc, and the default because they have a single kernel image to copy. If it is missing, it should be an error. I did not modify the code for mips, arm64, riscv. Instead, I left some comments. Eventually, someone may fix the code, or at the very least, it may discourage the copy-pasting of incorrect code to another architecture. Signed-off-by: Masahiro Yamada Reviewed-by: Emil Renner Berthing Reviewed-by: Nicolas Schier --- scripts/package/buildtar | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/package/buildtar b/scripts/package/buildtar index 72c91a1b832f9..ed8d9b4963055 100755 --- a/scripts/package/buildtar +++ b/scripts/package/buildtar @@ -53,18 +53,24 @@ cp -v -- "${objtree}/vmlinux" "${tmpdir}/boot/vmlinux-${KERNELRELEASE}" # # Install arch-specific kernel image(s) # +# Note: +# mips, arm64, and riscv copy the first image found. This may not produce +# the desired outcome because it may pick up a stale file remaining in the +# build tree. +# case "${ARCH}" in x86|i386|x86_64) - [ -f "${objtree}/arch/x86/boot/bzImage" ] && cp -v -- "${objtree}/arch/x86/boot/bzImage" "${tmpdir}/boot/vmlinuz-${KERNELRELEASE}" + cp -v -- "${objtree}/arch/x86/boot/bzImage" "${tmpdir}/boot/vmlinuz-${KERNELRELEASE}" ;; alpha) - [ -f "${objtree}/arch/alpha/boot/vmlinux.gz" ] && cp -v -- "${objtree}/arch/alpha/boot/vmlinux.gz" "${tmpdir}/boot/vmlinuz-${KERNELRELEASE}" + cp -v -- "${objtree}/arch/alpha/boot/vmlinux.gz" "${tmpdir}/boot/vmlinuz-${KERNELRELEASE}" ;; parisc*) - [ -f "${KBUILD_IMAGE}" ] && cp -v -- "${KBUILD_IMAGE}" "${tmpdir}/boot/vmlinux-${KERNELRELEASE}" + cp -v -- "${KBUILD_IMAGE}" "${tmpdir}/boot/vmlinux-${KERNELRELEASE}" [ -f "${objtree}/lifimage" ] && cp -v -- "${objtree}/lifimage" "${tmpdir}/boot/lifimage-${KERNELRELEASE}" ;; mips) + # Please note the following code may copy a stale file. if [ -f "${objtree}/arch/mips/boot/compressed/vmlinux.bin" ]; then cp -v -- "${objtree}/arch/mips/boot/compressed/vmlinux.bin" "${tmpdir}/boot/vmlinuz-${KERNELRELEASE}" elif [ -f "${objtree}/arch/mips/boot/compressed/vmlinux.ecoff" ]; then @@ -86,6 +92,7 @@ case "${ARCH}" in fi ;; arm64) + # Please note the following code may copy a stale file. for i in Image.bz2 Image.gz Image.lz4 Image.lzma Image.lzo vmlinuz.efi ; do if [ -f "${objtree}/arch/arm64/boot/${i}" ] ; then cp -v -- "${objtree}/arch/arm64/boot/${i}" "${tmpdir}/boot/vmlinuz-${KERNELRELEASE}" @@ -94,6 +101,7 @@ case "${ARCH}" in done ;; riscv) + # Please note the following code may copy a stale file. for i in Image.bz2 Image.gz Image; do if [ -f "${objtree}/arch/riscv/boot/${i}" ] ; then cp -v -- "${objtree}/arch/riscv/boot/${i}" "${tmpdir}/boot/vmlinux-${KERNELRELEASE}" @@ -102,7 +110,7 @@ case "${ARCH}" in done ;; *) - [ -f "${KBUILD_IMAGE}" ] && cp -v -- "${KBUILD_IMAGE}" "${tmpdir}/boot/vmlinux-kbuild-${KERNELRELEASE}" + cp -v -- "${KBUILD_IMAGE}" "${tmpdir}/boot/vmlinux-kbuild-${KERNELRELEASE}" echo "" >&2 echo '** ** ** WARNING ** ** **' >&2 echo "" >&2 From 951bcae6c5a0bfaa55b27c5f16178204988f0379 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 15 Apr 2024 18:20:43 +0200 Subject: [PATCH 0947/1702] kallsyms: Avoid weak references for kallsyms symbols kallsyms is a directory of all the symbols in the vmlinux binary, and so creating it is somewhat of a chicken-and-egg problem, as its non-zero size affects the layout of the binary, and therefore the values of the symbols. For this reason, the kernel is linked more than once, and the first pass does not include any kallsyms data at all. For the linker to accept this, the symbol declarations describing the kallsyms metadata are emitted as having weak linkage, so they can remain unsatisfied. During the subsequent passes, the weak references are satisfied by the kallsyms metadata that was constructed based on information gathered from the preceding passes. Weak references lead to somewhat worse codegen, because taking their address may need to produce NULL (if the reference was unsatisfied), and this is not usually supported by RIP or PC relative symbol references. Given that these references are ultimately always satisfied in the final link, let's drop the weak annotation, and instead, provide fallback definitions in the linker script that are only emitted if an unsatisfied reference exists. While at it, drop the FRV specific annotation that these symbols reside in .rodata - FRV is long gone. Tested-by: Nick Desaulniers # Boot Reviewed-by: Nick Desaulniers Reviewed-by: Kees Cook Acked-by: Arnd Bergmann Link: https://lkml.kernel.org/r/20230504174320.3930345-1-ardb%40kernel.org Signed-off-by: Ard Biesheuvel Signed-off-by: Masahiro Yamada --- include/asm-generic/vmlinux.lds.h | 19 +++++++++++++++++++ kernel/kallsyms.c | 6 ------ kernel/kallsyms_internal.h | 30 ++++++++++++------------------ 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index f7749d0f2562f..e8449be620582 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -448,11 +448,30 @@ #endif #endif +/* + * Some symbol definitions will not exist yet during the first pass of the + * link, but are guaranteed to exist in the final link. Provide preliminary + * definitions that will be superseded in the final link to avoid having to + * rely on weak external linkage, which requires a GOT when used in position + * independent code. + */ +#define PRELIMINARY_SYMBOL_DEFINITIONS \ + PROVIDE(kallsyms_addresses = .); \ + PROVIDE(kallsyms_offsets = .); \ + PROVIDE(kallsyms_names = .); \ + PROVIDE(kallsyms_num_syms = .); \ + PROVIDE(kallsyms_relative_base = .); \ + PROVIDE(kallsyms_token_table = .); \ + PROVIDE(kallsyms_token_index = .); \ + PROVIDE(kallsyms_markers = .); \ + PROVIDE(kallsyms_seqs_of_names = .); + /* * Read only Data */ #define RO_DATA(align) \ . = ALIGN((align)); \ + PRELIMINARY_SYMBOL_DEFINITIONS \ .rodata : AT(ADDR(.rodata) - LOAD_OFFSET) { \ __start_rodata = .; \ *(.rodata) *(.rodata.*) \ diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 18edd57b5fe81..22ea19a36e6e6 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -325,12 +325,6 @@ static unsigned long get_symbol_pos(unsigned long addr, unsigned long symbol_start = 0, symbol_end = 0; unsigned long i, low, high, mid; - /* This kernel should never had been booted. */ - if (!IS_ENABLED(CONFIG_KALLSYMS_BASE_RELATIVE)) - BUG_ON(!kallsyms_addresses); - else - BUG_ON(!kallsyms_offsets); - /* Do a binary search on the sorted kallsyms_addresses array. */ low = 0; high = kallsyms_num_syms; diff --git a/kernel/kallsyms_internal.h b/kernel/kallsyms_internal.h index 27fabdcc40f57..85480274fc8fb 100644 --- a/kernel/kallsyms_internal.h +++ b/kernel/kallsyms_internal.h @@ -5,27 +5,21 @@ #include /* - * These will be re-linked against their real values - * during the second link stage. + * These will be re-linked against their real values during the second link + * stage. Preliminary values must be provided in the linker script using the + * PROVIDE() directive so that the first link stage can complete successfully. */ -extern const unsigned long kallsyms_addresses[] __weak; -extern const int kallsyms_offsets[] __weak; -extern const u8 kallsyms_names[] __weak; +extern const unsigned long kallsyms_addresses[]; +extern const int kallsyms_offsets[]; +extern const u8 kallsyms_names[]; -/* - * Tell the compiler that the count isn't in the small data section if the arch - * has one (eg: FRV). - */ -extern const unsigned int kallsyms_num_syms -__section(".rodata") __attribute__((weak)); - -extern const unsigned long kallsyms_relative_base -__section(".rodata") __attribute__((weak)); +extern const unsigned int kallsyms_num_syms; +extern const unsigned long kallsyms_relative_base; -extern const char kallsyms_token_table[] __weak; -extern const u16 kallsyms_token_index[] __weak; +extern const char kallsyms_token_table[]; +extern const u16 kallsyms_token_index[]; -extern const unsigned int kallsyms_markers[] __weak; -extern const u8 kallsyms_seqs_of_names[] __weak; +extern const unsigned int kallsyms_markers[]; +extern const u8 kallsyms_seqs_of_names[]; #endif // LINUX_KALLSYMS_INTERNAL_H_ From 377d9095117c084b835e38c020faf5a78e386f01 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 15 Apr 2024 18:20:44 +0200 Subject: [PATCH 0948/1702] vmlinux: Avoid weak reference to notes section Weak references are references that are permitted to remain unsatisfied in the final link. This means they cannot be implemented using place relative relocations, resulting in GOT entries when using position independent code generation. The notes section should always exist, so the weak annotations can be omitted. Acked-by: Arnd Bergmann Signed-off-by: Ard Biesheuvel Signed-off-by: Masahiro Yamada --- kernel/ksysfs.c | 4 ++-- lib/buildid.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c index 495b69a71a5d7..07fb5987b42bd 100644 --- a/kernel/ksysfs.c +++ b/kernel/ksysfs.c @@ -228,8 +228,8 @@ KERNEL_ATTR_RW(rcu_normal); /* * Make /sys/kernel/notes give the raw contents of our kernel .notes section. */ -extern const void __start_notes __weak; -extern const void __stop_notes __weak; +extern const void __start_notes; +extern const void __stop_notes; #define notes_size (&__stop_notes - &__start_notes) static ssize_t notes_read(struct file *filp, struct kobject *kobj, diff --git a/lib/buildid.c b/lib/buildid.c index 898301b49eb64..7954dd92e36c0 100644 --- a/lib/buildid.c +++ b/lib/buildid.c @@ -182,8 +182,8 @@ unsigned char vmlinux_build_id[BUILD_ID_SIZE_MAX] __ro_after_init; */ void __init init_vmlinux_build_id(void) { - extern const void __start_notes __weak; - extern const void __stop_notes __weak; + extern const void __start_notes; + extern const void __stop_notes; unsigned int size = &__stop_notes - &__start_notes; build_id_parse_buf(&__start_notes, vmlinux_build_id, size); From 7284b4fbc8473965f9c053f3fa8b194af8ed4738 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 21 Apr 2024 18:02:51 +0900 Subject: [PATCH 0949/1702] kconfig: add menu_next() function and menu_for_each(_sub)_entry macros Several functions require traversing menu entries sequentially. This commit introduces some helpers to simplify such operations. The menu_next() function facilitates depth-first traversal: 1. Descend to the child level if the current menu has one 2. Move to the next sibling at the same level if available 3. Ascend to the parent level if there is no more child or sibling The menu_for_each_sub_entry() macro iterates over all submenu entries using depth-first traverse. The menu_for_each_entry() macro is the same, but over all menu entries. Signed-off-by: Masahiro Yamada --- scripts/kconfig/lkc.h | 5 +++++ scripts/kconfig/menu.c | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/scripts/kconfig/lkc.h b/scripts/kconfig/lkc.h index e7cc9e985c4f0..cfb7e9ac41a3c 100644 --- a/scripts/kconfig/lkc.h +++ b/scripts/kconfig/lkc.h @@ -79,6 +79,11 @@ void str_printf(struct gstr *gs, const char *fmt, ...); char *str_get(struct gstr *gs); /* menu.c */ +struct menu *menu_next(struct menu *menu, struct menu *root); +#define menu_for_each_sub_entry(menu, root) \ + for (menu = menu_next(root, root); menu; menu = menu_next(menu, root)) +#define menu_for_each_entry(menu) \ + menu_for_each_sub_entry(menu, &rootmenu) void _menu_init(void); void menu_warn(struct menu *menu, const char *fmt, ...); struct menu *menu_add_menu(void); diff --git a/scripts/kconfig/menu.c b/scripts/kconfig/menu.c index 3b822cd110f47..fe6af87006226 100644 --- a/scripts/kconfig/menu.c +++ b/scripts/kconfig/menu.c @@ -17,6 +17,27 @@ static const char nohelp_text[] = "There is no help available for this option."; struct menu rootmenu; static struct menu **last_entry_ptr; +/** + * menu_next - return the next menu entry with depth-first traversal + * @menu: pointer to the current menu + * @root: root of the sub-tree to traverse. If NULL is given, the traveral + * continues until it reaches the end of the entire menu tree. + * return: the menu to visit next, or NULL when it reaches the end. + */ +struct menu *menu_next(struct menu *menu, struct menu *root) +{ + if (menu->list) + return menu->list; + + while (menu != root && !menu->next) + menu = menu->parent; + + if (menu == root) + return NULL; + + return menu->next; +} + void menu_warn(struct menu *menu, const char *fmt, ...) { va_list ap; From 03c4ecaa5c76640f1993733be1bded18e0c074d5 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 21 Apr 2024 18:02:52 +0900 Subject: [PATCH 0950/1702] kconfig: use menu_for_each_entry() to traverse menu tree Use menu_for_each_entry() to traverse the menu tree instead of implementing similar logic in each function. Signed-off-by: Masahiro Yamada --- scripts/kconfig/confdata.c | 28 +++++----------------------- scripts/kconfig/parser.y | 13 +------------ 2 files changed, 6 insertions(+), 35 deletions(-) diff --git a/scripts/kconfig/confdata.c b/scripts/kconfig/confdata.c index 0e35c4819cf1d..ce0ef417b71b3 100644 --- a/scripts/kconfig/confdata.c +++ b/scripts/kconfig/confdata.c @@ -793,23 +793,19 @@ int conf_write_defconfig(const char *filename) sym_clear_all_valid(); - /* Traverse all menus to find all relevant symbols */ - menu = rootmenu.list; - - while (menu != NULL) - { + menu_for_each_entry(menu) { sym = menu->sym; if (sym && !sym_is_choice(sym)) { sym_calc_value(sym); if (!(sym->flags & SYMBOL_WRITE)) - goto next_menu; + continue; sym->flags &= ~SYMBOL_WRITE; /* If we cannot change the symbol - skip */ if (!sym_is_changeable(sym)) - goto next_menu; + continue; /* If symbol equals to default value - skip */ if (strcmp(sym_get_string_value(sym), sym_get_string_default(sym)) == 0) - goto next_menu; + continue; /* * If symbol is a choice value and equals to the @@ -827,25 +823,11 @@ int conf_write_defconfig(const char *filename) if (!sym_is_optional(cs) && sym == ds) { if ((sym->type == S_BOOLEAN) && sym_get_tristate_value(sym) == yes) - goto next_menu; + continue; } } print_symbol_for_dotconfig(out, sym); } -next_menu: - if (menu->list != NULL) { - menu = menu->list; - } - else if (menu->next != NULL) { - menu = menu->next; - } else { - while ((menu = menu->parent)) { - if (menu->next != NULL) { - menu = menu->next; - break; - } - } - } } fclose(out); return 0; diff --git a/scripts/kconfig/parser.y b/scripts/kconfig/parser.y index 7fb996612c966..8f339b47fe8da 100644 --- a/scripts/kconfig/parser.y +++ b/scripts/kconfig/parser.y @@ -517,20 +517,9 @@ void conf_parse(const char *name) menu_finalize(); - menu = &rootmenu; - while (menu) { + menu_for_each_entry(menu) { if (menu->sym && sym_check_deps(menu->sym)) yynerrs++; - - if (menu->list) { - menu = menu->list; - continue; - } - - while (!menu->next && menu->parent) - menu = menu->parent; - - menu = menu->next; } if (yynerrs) From c2af3d03c51289629edaadf118445fd29933c456 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 21 Apr 2024 20:13:02 +0900 Subject: [PATCH 0951/1702] kconfig: remove unneeded if-conditional in conf_choice() All symbols except choices have a name. child->sym->name never becomes NULL inside choice blocks. Signed-off-by: Masahiro Yamada --- scripts/kconfig/conf.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/kconfig/conf.c b/scripts/kconfig/conf.c index 965bb40c50e51..0156ca13f949c 100644 --- a/scripts/kconfig/conf.c +++ b/scripts/kconfig/conf.c @@ -497,9 +497,8 @@ static int conf_choice(struct menu *menu) printf("%*c", indent, '>'); } else printf("%*c", indent, ' '); - printf(" %d. %s", cnt, menu_get_prompt(child)); - if (child->sym->name) - printf(" (%s)", child->sym->name); + printf(" %d. %s (%s)", cnt, menu_get_prompt(child), + child->sym->name); if (!sym_has_value(child->sym)) printf(" (NEW)"); printf("\n"); From 2b1ab140506822dfc4711be08ea46fb4a83bf1ea Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Tue, 23 Apr 2024 00:46:10 +0900 Subject: [PATCH 0952/1702] kbuild: buildtar: remove warning for the default case Given KBUILD_IMAGE properly set in arch/*/Makefile, the default case should work in most scenarios. The only oddity is the naming of the copy destination, vmlinux-kbuild-${KERNELRELEASE}. Let's rename it to vmlinuz-${KERNELRELEASE} because the kernel is often compressed. Remove the warning to avoid unnecessary patch submissions when the default case suffices. Remove the x86 case, which is now equivalent to the default. Signed-off-by: Masahiro Yamada Reviewed-by: Nathan Chancellor --- scripts/package/buildtar | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/scripts/package/buildtar b/scripts/package/buildtar index ed8d9b4963055..fe816f62a290c 100755 --- a/scripts/package/buildtar +++ b/scripts/package/buildtar @@ -59,9 +59,6 @@ cp -v -- "${objtree}/vmlinux" "${tmpdir}/boot/vmlinux-${KERNELRELEASE}" # build tree. # case "${ARCH}" in - x86|i386|x86_64) - cp -v -- "${objtree}/arch/x86/boot/bzImage" "${tmpdir}/boot/vmlinuz-${KERNELRELEASE}" - ;; alpha) cp -v -- "${objtree}/arch/alpha/boot/vmlinux.gz" "${tmpdir}/boot/vmlinuz-${KERNELRELEASE}" ;; @@ -110,13 +107,6 @@ case "${ARCH}" in done ;; *) - cp -v -- "${KBUILD_IMAGE}" "${tmpdir}/boot/vmlinux-kbuild-${KERNELRELEASE}" - echo "" >&2 - echo '** ** ** WARNING ** ** **' >&2 - echo "" >&2 - echo "Your architecture did not define any architecture-dependent files" >&2 - echo "to be placed into the tarball. Please add those to ${0} ..." >&2 - echo "" >&2 - sleep 5 + cp -v -- "${KBUILD_IMAGE}" "${tmpdir}/boot/vmlinuz-${KERNELRELEASE}" ;; esac From 1da251c60def5df1475ed5e8670d7ba2b6a33983 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Tue, 23 Apr 2024 01:10:54 +0900 Subject: [PATCH 0953/1702] kconfig: remove SYMBOL_CHOICE flag All symbols except choices have a name. Previously, choices were allowed to have a name, but commit c83f020973bc ("kconfig: remove named choice support") eliminated that possibility. Now, it is easy to distinguish choices from normal symbols; if the name is NULL, it is a choice. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/kconfig/confdata.c | 2 +- scripts/kconfig/expr.h | 4 +--- scripts/kconfig/gconf.c | 2 -- scripts/kconfig/lkc.h | 3 ++- scripts/kconfig/parser.y | 2 +- scripts/kconfig/symbol.c | 2 +- 6 files changed, 6 insertions(+), 9 deletions(-) diff --git a/scripts/kconfig/confdata.c b/scripts/kconfig/confdata.c index ce0ef417b71b3..a86e71bab5fa7 100644 --- a/scripts/kconfig/confdata.c +++ b/scripts/kconfig/confdata.c @@ -888,7 +888,7 @@ int conf_write(const char *name) "# %s\n" "#\n", str); need_newline = false; - } else if (!(sym->flags & SYMBOL_CHOICE) && + } else if (!sym_is_choice(sym) && !(sym->flags & SYMBOL_WRITTEN)) { sym_calc_value(sym); if (!(sym->flags & SYMBOL_WRITE)) diff --git a/scripts/kconfig/expr.h b/scripts/kconfig/expr.h index 0158f5eac4542..68b3dd65cb086 100644 --- a/scripts/kconfig/expr.h +++ b/scripts/kconfig/expr.h @@ -72,8 +72,7 @@ enum { /* * Represents a configuration symbol. * - * Choices are represented as a special kind of symbol and have the - * SYMBOL_CHOICE bit set in 'flags'. + * Choices are represented as a special kind of symbol with null name. */ struct symbol { /* link node for the hash table */ @@ -131,7 +130,6 @@ struct symbol { #define SYMBOL_CONST 0x0001 /* symbol is const */ #define SYMBOL_CHECK 0x0008 /* used during dependency checking */ -#define SYMBOL_CHOICE 0x0010 /* start of a choice block (null name) */ #define SYMBOL_CHOICEVAL 0x0020 /* used as a value in a choice block */ #define SYMBOL_VALID 0x0080 /* set when symbol.curr is calculated */ #define SYMBOL_OPTIONAL 0x0100 /* choice is optional - values can be 'n' */ diff --git a/scripts/kconfig/gconf.c b/scripts/kconfig/gconf.c index 9709aca3a30fe..74f193272a00b 100644 --- a/scripts/kconfig/gconf.c +++ b/scripts/kconfig/gconf.c @@ -83,8 +83,6 @@ static const char *dbg_sym_flags(int val) strcat(buf, "const/"); if (val & SYMBOL_CHECK) strcat(buf, "check/"); - if (val & SYMBOL_CHOICE) - strcat(buf, "choice/"); if (val & SYMBOL_CHOICEVAL) strcat(buf, "choiceval/"); if (val & SYMBOL_VALID) diff --git a/scripts/kconfig/lkc.h b/scripts/kconfig/lkc.h index cfb7e9ac41a3c..5e27432e49398 100644 --- a/scripts/kconfig/lkc.h +++ b/scripts/kconfig/lkc.h @@ -129,7 +129,8 @@ static inline struct symbol *sym_get_choice_value(struct symbol *sym) static inline bool sym_is_choice(struct symbol *sym) { - return sym->flags & SYMBOL_CHOICE ? true : false; + /* A choice is a symbol with no name */ + return sym->name == NULL; } static inline bool sym_is_choice_value(struct symbol *sym) diff --git a/scripts/kconfig/parser.y b/scripts/kconfig/parser.y index 8f339b47fe8da..b95993ff38376 100644 --- a/scripts/kconfig/parser.y +++ b/scripts/kconfig/parser.y @@ -224,7 +224,7 @@ config_option: T_MODULES T_EOL choice: T_CHOICE T_EOL { - struct symbol *sym = sym_lookup(NULL, SYMBOL_CHOICE); + struct symbol *sym = sym_lookup(NULL, 0); sym->flags |= SYMBOL_NO_WRITE; menu_add_entry(sym); menu_add_expr(P_CHOICE, NULL, NULL); diff --git a/scripts/kconfig/symbol.c b/scripts/kconfig/symbol.c index 81fe1884ef8ae..8b34992ba5ed9 100644 --- a/scripts/kconfig/symbol.c +++ b/scripts/kconfig/symbol.c @@ -827,7 +827,7 @@ struct symbol *sym_lookup(const char *name, int flags) if (symbol->name && !strcmp(symbol->name, name) && (flags ? symbol->flags & flags - : !(symbol->flags & (SYMBOL_CONST|SYMBOL_CHOICE)))) + : !(symbol->flags & SYMBOL_CONST))) return symbol; } new_name = xstrdup(name); From d9a1dab65aa2d4de7244731c97c70491997225f2 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Tue, 23 Apr 2024 01:41:03 +0900 Subject: [PATCH 0954/1702] sh: Convert the last use of 'optional' property in Kconfig The 'choice' statement is primarily used to exclusively select one option, but the 'optional' property allows all entries to be disabled. This feature is rarely used. In fact, it is only used in arch/sh/Kconfig because the equivalent outcome can be achieved by inserting one more entry. The 'optional' property support will be removed from Kconfig. This commit replaces the 'optional' property with a dummy option, CMDLINE_FROM_BOOTLOADER, as seen in some other architectures. Note: The 'default CMDLINE_OVERWRITE' statement does not work as intended in combination with 'optional'. If neither CONFIG_CMDLINE_OVERWRITE nor CONFIG_CMDLINE_EXTEND is specified in a defconfig file, both of them are disabled. This is a bug. To maintain the current behavior, I added CONFIG_CMDLINE_FROM_BOOTLOADER=y to those defconfig files. Signed-off-by: Masahiro Yamada Acked-by: John Paul Adrian Glaubitz --- arch/sh/Kconfig | 6 +++++- arch/sh/configs/apsh4a3a_defconfig | 1 + arch/sh/configs/apsh4ad0a_defconfig | 1 + arch/sh/configs/edosk7705_defconfig | 1 + arch/sh/configs/hp6xx_defconfig | 1 + arch/sh/configs/landisk_defconfig | 1 + arch/sh/configs/magicpanelr2_defconfig | 1 + arch/sh/configs/rsk7264_defconfig | 1 + arch/sh/configs/rsk7269_defconfig | 1 + arch/sh/configs/se7619_defconfig | 1 + arch/sh/configs/se7705_defconfig | 1 + arch/sh/configs/se7722_defconfig | 1 + arch/sh/configs/se7750_defconfig | 1 + arch/sh/configs/secureedge5410_defconfig | 1 + arch/sh/configs/sh7710voipgw_defconfig | 1 + arch/sh/configs/sh7724_generic_defconfig | 1 + arch/sh/configs/sh7770_generic_defconfig | 1 + arch/sh/configs/sh7785lcr_32bit_defconfig | 1 + arch/sh/configs/sh7785lcr_defconfig | 1 + arch/sh/configs/urquell_defconfig | 1 + 20 files changed, 24 insertions(+), 1 deletion(-) diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index 2ad3e29f0ebec..8b64ca76aa4b4 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -709,7 +709,6 @@ config ROMIMAGE_MMCIF choice prompt "Kernel command line" - optional default CMDLINE_OVERWRITE help Setting this option allows the kernel command line arguments @@ -727,6 +726,11 @@ config CMDLINE_EXTEND Given string will be concatenated with arguments passed in by a bootloader. +config CMDLINE_FROM_BOOTLOADER + bool "Use bootloader kernel arguments" + help + Uses the command-line options passed by the boot loader. + endchoice config CMDLINE diff --git a/arch/sh/configs/apsh4a3a_defconfig b/arch/sh/configs/apsh4a3a_defconfig index cc909f3478776..9c2644443c4d5 100644 --- a/arch/sh/configs/apsh4a3a_defconfig +++ b/arch/sh/configs/apsh4a3a_defconfig @@ -15,6 +15,7 @@ CONFIG_MEMORY_START=0x0C000000 CONFIG_FLATMEM_MANUAL=y CONFIG_SH_STORE_QUEUES=y CONFIG_SH_APSH4A3A=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_HIGH_RES_TIMERS=y CONFIG_KEXEC=y CONFIG_PREEMPT=y diff --git a/arch/sh/configs/apsh4ad0a_defconfig b/arch/sh/configs/apsh4ad0a_defconfig index 64558bf60e106..05d21d91f41d6 100644 --- a/arch/sh/configs/apsh4ad0a_defconfig +++ b/arch/sh/configs/apsh4ad0a_defconfig @@ -42,6 +42,7 @@ CONFIG_SECCOMP=y CONFIG_PREEMPT=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_PM=y CONFIG_PM_DEBUG=y CONFIG_PM=y diff --git a/arch/sh/configs/edosk7705_defconfig b/arch/sh/configs/edosk7705_defconfig index 9ee35269bee26..57c79da1ff8ed 100644 --- a/arch/sh/configs/edosk7705_defconfig +++ b/arch/sh/configs/edosk7705_defconfig @@ -19,6 +19,7 @@ CONFIG_CPU_SUBTYPE_SH7705=y CONFIG_SH_EDOSK7705=y CONFIG_SH_PCLK_FREQ=31250000 +CONFIG_CMDLINE_FROM_BOOTLOADER=y # CONFIG_PREVENT_FIRMWARE_BUILD is not set # CONFIG_INPUT is not set # CONFIG_SERIO is not set diff --git a/arch/sh/configs/hp6xx_defconfig b/arch/sh/configs/hp6xx_defconfig index 0c45f2a0f9bd9..77e3185f63e47 100644 --- a/arch/sh/configs/hp6xx_defconfig +++ b/arch/sh/configs/hp6xx_defconfig @@ -15,6 +15,7 @@ CONFIG_SH_DMA_API=y CONFIG_HD64461_ENABLER=y CONFIG_PCCARD=y CONFIG_PM=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_APM_EMULATION=y # CONFIG_STANDALONE is not set CONFIG_BLK_DEV_SD=y diff --git a/arch/sh/configs/landisk_defconfig b/arch/sh/configs/landisk_defconfig index 5410820909187..0311380160f42 100644 --- a/arch/sh/configs/landisk_defconfig +++ b/arch/sh/configs/landisk_defconfig @@ -15,6 +15,7 @@ CONFIG_KEXEC=y CONFIG_PCI=y CONFIG_PCCARD=y CONFIG_YENTA=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/sh/configs/magicpanelr2_defconfig b/arch/sh/configs/magicpanelr2_defconfig index 52937f9cc2ab3..8d443749550ec 100644 --- a/arch/sh/configs/magicpanelr2_defconfig +++ b/arch/sh/configs/magicpanelr2_defconfig @@ -22,6 +22,7 @@ CONFIG_SH_PCLK_FREQ=24000000 CONFIG_SH_DMA=y CONFIG_SH_DMA_API=y CONFIG_HEARTBEAT=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/sh/configs/rsk7264_defconfig b/arch/sh/configs/rsk7264_defconfig index a88cb3b779577..e4ef259425c42 100644 --- a/arch/sh/configs/rsk7264_defconfig +++ b/arch/sh/configs/rsk7264_defconfig @@ -21,6 +21,7 @@ CONFIG_MEMORY_START=0x0c000000 CONFIG_FLATMEM_MANUAL=y CONFIG_CPU_BIG_ENDIAN=y CONFIG_SH_RSK=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y # CONFIG_SH_TIMER_MTU2 is not set CONFIG_BINFMT_FLAT=y CONFIG_NET=y diff --git a/arch/sh/configs/rsk7269_defconfig b/arch/sh/configs/rsk7269_defconfig index d9a7ce783c9bc..e0d1560b2bfd7 100644 --- a/arch/sh/configs/rsk7269_defconfig +++ b/arch/sh/configs/rsk7269_defconfig @@ -10,6 +10,7 @@ CONFIG_MEMORY_SIZE=0x02000000 CONFIG_FLATMEM_MANUAL=y CONFIG_CPU_BIG_ENDIAN=y CONFIG_SH_RSK=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y # CONFIG_SH_TIMER_MTU2 is not set CONFIG_SH_PCLK_FREQ=66700000 CONFIG_BINFMT_FLAT=y diff --git a/arch/sh/configs/se7619_defconfig b/arch/sh/configs/se7619_defconfig index 14d0f5ead502f..6b25e9713e777 100644 --- a/arch/sh/configs/se7619_defconfig +++ b/arch/sh/configs/se7619_defconfig @@ -14,6 +14,7 @@ CONFIG_FLATMEM_MANUAL=y CONFIG_CPU_BIG_ENDIAN=y CONFIG_SH_7619_SOLUTION_ENGINE=y CONFIG_HZ_100=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_BINFMT_FLAT=y CONFIG_BINFMT_ZFLAT=y # CONFIG_STANDALONE is not set diff --git a/arch/sh/configs/se7705_defconfig b/arch/sh/configs/se7705_defconfig index 16a0f72f08226..1752ddc2694a4 100644 --- a/arch/sh/configs/se7705_defconfig +++ b/arch/sh/configs/se7705_defconfig @@ -13,6 +13,7 @@ CONFIG_FLATMEM_MANUAL=y # CONFIG_SH_ADC is not set CONFIG_SH_SOLUTION_ENGINE=y CONFIG_HEARTBEAT=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_PREEMPT=y CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/sh/configs/se7722_defconfig b/arch/sh/configs/se7722_defconfig index 09e4558174474..5327a2f709805 100644 --- a/arch/sh/configs/se7722_defconfig +++ b/arch/sh/configs/se7722_defconfig @@ -17,6 +17,7 @@ CONFIG_SH_7722_SOLUTION_ENGINE=y CONFIG_NO_HZ=y CONFIG_HIGH_RES_TIMERS=y CONFIG_HEARTBEAT=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_KEXEC=y CONFIG_PREEMPT=y CONFIG_NET=y diff --git a/arch/sh/configs/se7750_defconfig b/arch/sh/configs/se7750_defconfig index 5fa6239ae4eae..a1e25d7de8a60 100644 --- a/arch/sh/configs/se7750_defconfig +++ b/arch/sh/configs/se7750_defconfig @@ -15,6 +15,7 @@ CONFIG_FLATMEM_MANUAL=y CONFIG_SH_SOLUTION_ENGINE=y CONFIG_SH_PCLK_FREQ=33333333 CONFIG_HEARTBEAT=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/sh/configs/secureedge5410_defconfig b/arch/sh/configs/secureedge5410_defconfig index 120176afe3f6f..2f77b60e9540b 100644 --- a/arch/sh/configs/secureedge5410_defconfig +++ b/arch/sh/configs/secureedge5410_defconfig @@ -10,6 +10,7 @@ CONFIG_SH_SECUREEDGE5410=y CONFIG_SH_DMA=y CONFIG_SH_DMA_API=y CONFIG_PCI=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_NET=y CONFIG_INET=y # CONFIG_INET_XFRM_MODE_TRANSPORT is not set diff --git a/arch/sh/configs/sh7710voipgw_defconfig b/arch/sh/configs/sh7710voipgw_defconfig index 7f742729df697..99a5d07605326 100644 --- a/arch/sh/configs/sh7710voipgw_defconfig +++ b/arch/sh/configs/sh7710voipgw_defconfig @@ -15,6 +15,7 @@ CONFIG_MEMORY_SIZE=0x00800000 CONFIG_FLATMEM_MANUAL=y # CONFIG_SH_ADC is not set CONFIG_SH_PCLK_FREQ=32768000 +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/sh/configs/sh7724_generic_defconfig b/arch/sh/configs/sh7724_generic_defconfig index cbc9389a89a8c..5440bd0ca4ed5 100644 --- a/arch/sh/configs/sh7724_generic_defconfig +++ b/arch/sh/configs/sh7724_generic_defconfig @@ -12,6 +12,7 @@ CONFIG_CPU_FREQ=y CONFIG_SH_CPU_FREQ=y CONFIG_KEXEC=y CONFIG_KEXEC_JUMP=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_HIBERNATION=y CONFIG_CPU_IDLE=y # CONFIG_PREVENT_FIRMWARE_BUILD is not set diff --git a/arch/sh/configs/sh7770_generic_defconfig b/arch/sh/configs/sh7770_generic_defconfig index ee2357deba0f1..4338af8d02d03 100644 --- a/arch/sh/configs/sh7770_generic_defconfig +++ b/arch/sh/configs/sh7770_generic_defconfig @@ -14,6 +14,7 @@ CONFIG_SH_CPU_FREQ=y CONFIG_KEXEC=y CONFIG_KEXEC_JUMP=y CONFIG_PM=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_HIBERNATION=y CONFIG_CPU_IDLE=y # CONFIG_PREVENT_FIRMWARE_BUILD is not set diff --git a/arch/sh/configs/sh7785lcr_32bit_defconfig b/arch/sh/configs/sh7785lcr_32bit_defconfig index 59262f42abe6f..44f9b2317f090 100644 --- a/arch/sh/configs/sh7785lcr_32bit_defconfig +++ b/arch/sh/configs/sh7785lcr_32bit_defconfig @@ -32,6 +32,7 @@ CONFIG_PREEMPT=y CONFIG_INTC_USERIMASK=y CONFIG_PCI=y CONFIG_PCI_DEBUG=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_PM=y CONFIG_CPU_IDLE=y CONFIG_NET=y diff --git a/arch/sh/configs/sh7785lcr_defconfig b/arch/sh/configs/sh7785lcr_defconfig index 94381f8268fff..aec74b0e70035 100644 --- a/arch/sh/configs/sh7785lcr_defconfig +++ b/arch/sh/configs/sh7785lcr_defconfig @@ -17,6 +17,7 @@ CONFIG_HEARTBEAT=y CONFIG_KEXEC=y CONFIG_PREEMPT=y CONFIG_PCI=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/sh/configs/urquell_defconfig b/arch/sh/configs/urquell_defconfig index 445bb451a5ec8..00ef62133b04d 100644 --- a/arch/sh/configs/urquell_defconfig +++ b/arch/sh/configs/urquell_defconfig @@ -34,6 +34,7 @@ CONFIG_PCIEPORTBUS=y CONFIG_PCIEASPM_DEBUG=y CONFIG_PCI_DEBUG=y CONFIG_BINFMT_MISC=y +CONFIG_CMDLINE_FROM_BOOTLOADER=y CONFIG_PM=y CONFIG_CPU_IDLE=y CONFIG_NET=y From 6a1215888e23aa9fbc514086402f04708c84f454 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Tue, 23 Apr 2024 01:41:04 +0900 Subject: [PATCH 0955/1702] kconfig: remove 'optional' property support The 'choice' statement is primarily used to exclusively select one option, but the 'optional' property allows all entries to be disabled. In the following example, both A and B can be disabled simultaneously: choice prompt "choose A, B, or nothing" optional config A bool "A" config B bool "B" endchoice You can achieve the equivalent outcome by other means. A common solution is to add another option to guard the choice block. In the following example, you can set ENABLE_A_B_CHOICE=n to disable the entire choice block: choice prompt "choose A or B" depends on ENABLE_A_B_CHOICE config A bool "A" config B bool "B" endchoice Another approach is to insert one more entry: choice prompt "choose A, B, or disable both" config A bool "A" config B bool "B" config DISABLE_A_AND_B bool "choose this to disable both A and B" endchoice Some real examples are DEBUG_INFO_NONE, INITRAMFS_COMPRESSION_NONE, LTO_NONE, etc. The 'optional' property is even more unnecessary for a tristate choice. Without the 'optional' property, you can disable A and B; you can set 'm' in the choice prompt, and disable A and B individually: choice prompt "choose one built-in or make them modular" config A tristate "A" config B tristate "B" endchoice In conclusion, the 'optional' property was unneeded. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- Documentation/kbuild/kconfig-language.rst | 3 --- scripts/kconfig/confdata.c | 5 +--- scripts/kconfig/expr.h | 1 - scripts/kconfig/gconf.c | 2 -- scripts/kconfig/lexer.l | 1 - scripts/kconfig/lkc.h | 5 ---- scripts/kconfig/menu.c | 12 +++------ scripts/kconfig/parser.y | 9 ------- scripts/kconfig/tests/choice/Kconfig | 26 ------------------- scripts/kconfig/tests/choice/__init__.py | 2 -- .../tests/choice/allmod_expected_config | 4 --- .../tests/choice/allyes_expected_config | 4 --- .../tests/choice/oldask0_expected_stdout | 2 -- scripts/kconfig/tests/choice/oldask1_config | 1 - .../tests/choice/oldask1_expected_stdout | 6 ----- 15 files changed, 5 insertions(+), 78 deletions(-) diff --git a/Documentation/kbuild/kconfig-language.rst b/Documentation/kbuild/kconfig-language.rst index 79ac2e8184f6f..555c2f8399697 100644 --- a/Documentation/kbuild/kconfig-language.rst +++ b/Documentation/kbuild/kconfig-language.rst @@ -410,9 +410,6 @@ to be set to 'm'. This can be used if multiple drivers for a single hardware exists and only a single driver can be compiled/loaded into the kernel, but all drivers can be compiled as modules. -A choice accepts another option "optional", which allows to set the -choice to 'n' and no entry needs to be selected. - comment:: "comment" diff --git a/scripts/kconfig/confdata.c b/scripts/kconfig/confdata.c index a86e71bab5fa7..bcce87658998c 100644 --- a/scripts/kconfig/confdata.c +++ b/scripts/kconfig/confdata.c @@ -810,9 +810,6 @@ int conf_write_defconfig(const char *filename) /* * If symbol is a choice value and equals to the * default for a choice - skip. - * But only if value is bool and equal to "y" and - * choice is not "optional". - * (If choice is "optional" then all values can be "n") */ if (sym_is_choice_value(sym)) { struct symbol *cs; @@ -820,7 +817,7 @@ int conf_write_defconfig(const char *filename) cs = prop_get_symbol(sym_get_choice_prop(sym)); ds = sym_choice_default(cs); - if (!sym_is_optional(cs) && sym == ds) { + if (sym == ds) { if ((sym->type == S_BOOLEAN) && sym_get_tristate_value(sym) == yes) continue; diff --git a/scripts/kconfig/expr.h b/scripts/kconfig/expr.h index 68b3dd65cb086..f646a98de0063 100644 --- a/scripts/kconfig/expr.h +++ b/scripts/kconfig/expr.h @@ -132,7 +132,6 @@ struct symbol { #define SYMBOL_CHECK 0x0008 /* used during dependency checking */ #define SYMBOL_CHOICEVAL 0x0020 /* used as a value in a choice block */ #define SYMBOL_VALID 0x0080 /* set when symbol.curr is calculated */ -#define SYMBOL_OPTIONAL 0x0100 /* choice is optional - values can be 'n' */ #define SYMBOL_WRITE 0x0200 /* write symbol to file (KCONFIG_CONFIG) */ #define SYMBOL_CHANGED 0x0400 /* ? */ #define SYMBOL_WRITTEN 0x0800 /* track info to avoid double-write to .config */ diff --git a/scripts/kconfig/gconf.c b/scripts/kconfig/gconf.c index 74f193272a00b..13e2449ac83fa 100644 --- a/scripts/kconfig/gconf.c +++ b/scripts/kconfig/gconf.c @@ -87,8 +87,6 @@ static const char *dbg_sym_flags(int val) strcat(buf, "choiceval/"); if (val & SYMBOL_VALID) strcat(buf, "valid/"); - if (val & SYMBOL_OPTIONAL) - strcat(buf, "optional/"); if (val & SYMBOL_WRITE) strcat(buf, "write/"); if (val & SYMBOL_CHANGED) diff --git a/scripts/kconfig/lexer.l b/scripts/kconfig/lexer.l index 89544c3a1a299..8dd597c4710dc 100644 --- a/scripts/kconfig/lexer.l +++ b/scripts/kconfig/lexer.l @@ -120,7 +120,6 @@ n [A-Za-z0-9_-] "menuconfig" return T_MENUCONFIG; "modules" return T_MODULES; "on" return T_ON; -"optional" return T_OPTIONAL; "prompt" return T_PROMPT; "range" return T_RANGE; "select" return T_SELECT; diff --git a/scripts/kconfig/lkc.h b/scripts/kconfig/lkc.h index 5e27432e49398..64dfc354dd5c1 100644 --- a/scripts/kconfig/lkc.h +++ b/scripts/kconfig/lkc.h @@ -138,11 +138,6 @@ static inline bool sym_is_choice_value(struct symbol *sym) return sym->flags & SYMBOL_CHOICEVAL ? true : false; } -static inline bool sym_is_optional(struct symbol *sym) -{ - return sym->flags & SYMBOL_OPTIONAL ? true : false; -} - static inline bool sym_has_value(struct symbol *sym) { return sym->flags & SYMBOL_DEF_USER ? true : false; diff --git a/scripts/kconfig/menu.c b/scripts/kconfig/menu.c index fe6af87006226..e01b9ee87c05b 100644 --- a/scripts/kconfig/menu.c +++ b/scripts/kconfig/menu.c @@ -593,15 +593,11 @@ static void _menu_finalize(struct menu *parent, bool inside_choice) } /* - * For non-optional choices, add a reverse dependency (corresponding to - * a select) of ' && m'. This prevents the user from - * setting the choice mode to 'n' when the choice is visible. - * - * This would also work for non-choice symbols, but only non-optional - * choices clear SYMBOL_OPTIONAL as of writing. Choices are implemented - * as a type of symbol. + * For choices, add a reverse dependency (corresponding to a select) of + * ' && m'. This prevents the user from setting the choice + * mode to 'n' when the choice is visible. */ - if (sym && !sym_is_optional(sym) && parent->prompt) { + if (sym && sym_is_choice(sym) && parent->prompt) { sym->rev_dep.expr = expr_alloc_or(sym->rev_dep.expr, expr_alloc_and(parent->prompt->visible.expr, expr_alloc_symbol(&symbol_mod))); diff --git a/scripts/kconfig/parser.y b/scripts/kconfig/parser.y index b95993ff38376..69dc0c098acbd 100644 --- a/scripts/kconfig/parser.y +++ b/scripts/kconfig/parser.y @@ -69,7 +69,6 @@ struct menu *current_menu, *current_entry; %token T_MODULES %token T_ON %token T_OPEN_PAREN -%token T_OPTIONAL %token T_PLUS_EQUAL %token T_PROMPT %token T_RANGE @@ -140,7 +139,6 @@ stmt_list_in_choice: config_entry_start: T_CONFIG nonconst_symbol T_EOL { - $2->flags |= SYMBOL_OPTIONAL; menu_add_entry($2); printd(DEBUG_PARSE, "%s:%d:config %s\n", cur_filename, cur_lineno, $2->name); }; @@ -152,7 +150,6 @@ config_stmt: config_entry_start config_option_list menuconfig_entry_start: T_MENUCONFIG nonconst_symbol T_EOL { - $2->flags |= SYMBOL_OPTIONAL; menu_add_entry($2); printd(DEBUG_PARSE, "%s:%d:menuconfig %s\n", cur_filename, cur_lineno, $2->name); }; @@ -272,12 +269,6 @@ choice_option: logic_type prompt_stmt_opt T_EOL printd(DEBUG_PARSE, "%s:%d:type(%u)\n", cur_filename, cur_lineno, $1); }; -choice_option: T_OPTIONAL T_EOL -{ - current_entry->sym->flags |= SYMBOL_OPTIONAL; - printd(DEBUG_PARSE, "%s:%d:optional\n", cur_filename, cur_lineno); -}; - choice_option: T_DEFAULT nonconst_symbol if_expr T_EOL { menu_add_symbol(P_DEFAULT, $2, $3); diff --git a/scripts/kconfig/tests/choice/Kconfig b/scripts/kconfig/tests/choice/Kconfig index 0930eb65e9328..8cdda40868a13 100644 --- a/scripts/kconfig/tests/choice/Kconfig +++ b/scripts/kconfig/tests/choice/Kconfig @@ -17,19 +17,6 @@ config BOOL_CHOICE1 endchoice -choice - prompt "optional boolean choice" - optional - default OPT_BOOL_CHOICE1 - -config OPT_BOOL_CHOICE0 - bool "choice 0" - -config OPT_BOOL_CHOICE1 - bool "choice 1" - -endchoice - choice prompt "tristate choice" default TRI_CHOICE1 @@ -41,16 +28,3 @@ config TRI_CHOICE1 tristate "choice 1" endchoice - -choice - prompt "optional tristate choice" - optional - default OPT_TRI_CHOICE1 - -config OPT_TRI_CHOICE0 - tristate "choice 0" - -config OPT_TRI_CHOICE1 - tristate "choice 1" - -endchoice diff --git a/scripts/kconfig/tests/choice/__init__.py b/scripts/kconfig/tests/choice/__init__.py index 4318fce05912f..05e162220085c 100644 --- a/scripts/kconfig/tests/choice/__init__.py +++ b/scripts/kconfig/tests/choice/__init__.py @@ -6,8 +6,6 @@ The behavior of 'y' choice is intuitive. If choice values are tristate, the choice can be 'm' where each value can be enabled independently. -Also, if a choice is marked as 'optional', the whole choice can be -invisible. """ diff --git a/scripts/kconfig/tests/choice/allmod_expected_config b/scripts/kconfig/tests/choice/allmod_expected_config index f1f5dcdb79230..d1f51651740c4 100644 --- a/scripts/kconfig/tests/choice/allmod_expected_config +++ b/scripts/kconfig/tests/choice/allmod_expected_config @@ -1,9 +1,5 @@ CONFIG_MODULES=y # CONFIG_BOOL_CHOICE0 is not set CONFIG_BOOL_CHOICE1=y -# CONFIG_OPT_BOOL_CHOICE0 is not set -CONFIG_OPT_BOOL_CHOICE1=y CONFIG_TRI_CHOICE0=m CONFIG_TRI_CHOICE1=m -CONFIG_OPT_TRI_CHOICE0=m -CONFIG_OPT_TRI_CHOICE1=m diff --git a/scripts/kconfig/tests/choice/allyes_expected_config b/scripts/kconfig/tests/choice/allyes_expected_config index e5a062a1157cd..8a76c1816893b 100644 --- a/scripts/kconfig/tests/choice/allyes_expected_config +++ b/scripts/kconfig/tests/choice/allyes_expected_config @@ -1,9 +1,5 @@ CONFIG_MODULES=y # CONFIG_BOOL_CHOICE0 is not set CONFIG_BOOL_CHOICE1=y -# CONFIG_OPT_BOOL_CHOICE0 is not set -CONFIG_OPT_BOOL_CHOICE1=y # CONFIG_TRI_CHOICE0 is not set CONFIG_TRI_CHOICE1=y -# CONFIG_OPT_TRI_CHOICE0 is not set -CONFIG_OPT_TRI_CHOICE1=y diff --git a/scripts/kconfig/tests/choice/oldask0_expected_stdout b/scripts/kconfig/tests/choice/oldask0_expected_stdout index b251bba9698b2..d2257db464230 100644 --- a/scripts/kconfig/tests/choice/oldask0_expected_stdout +++ b/scripts/kconfig/tests/choice/oldask0_expected_stdout @@ -3,8 +3,6 @@ boolean choice 1. choice 0 (BOOL_CHOICE0) (NEW) > 2. choice 1 (BOOL_CHOICE1) (NEW) choice[1-2?]: -optional boolean choice [N/y/?] (NEW) tristate choice [M/y/?] (NEW) choice 0 (TRI_CHOICE0) [N/m/?] (NEW) choice 1 (TRI_CHOICE1) [N/m/?] (NEW) -optional tristate choice [N/m/y/?] (NEW) diff --git a/scripts/kconfig/tests/choice/oldask1_config b/scripts/kconfig/tests/choice/oldask1_config index b67bfe3c641fa..0f417856c81c1 100644 --- a/scripts/kconfig/tests/choice/oldask1_config +++ b/scripts/kconfig/tests/choice/oldask1_config @@ -1,2 +1 @@ # CONFIG_MODULES is not set -CONFIG_OPT_BOOL_CHOICE0=y diff --git a/scripts/kconfig/tests/choice/oldask1_expected_stdout b/scripts/kconfig/tests/choice/oldask1_expected_stdout index c2125e9bf96a7..ffa20ad7f38eb 100644 --- a/scripts/kconfig/tests/choice/oldask1_expected_stdout +++ b/scripts/kconfig/tests/choice/oldask1_expected_stdout @@ -3,13 +3,7 @@ boolean choice 1. choice 0 (BOOL_CHOICE0) (NEW) > 2. choice 1 (BOOL_CHOICE1) (NEW) choice[1-2?]: -optional boolean choice [Y/n/?] (NEW) -optional boolean choice -> 1. choice 0 (OPT_BOOL_CHOICE0) - 2. choice 1 (OPT_BOOL_CHOICE1) (NEW) -choice[1-2?]: tristate choice 1. choice 0 (TRI_CHOICE0) (NEW) > 2. choice 1 (TRI_CHOICE1) (NEW) choice[1-2?]: -optional tristate choice [N/y/?] From aba091547ef6159d52471f42a3ef531b7b660ed8 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 1 May 2024 15:55:25 -0700 Subject: [PATCH 0956/1702] kbuild: Remove support for Clang's ThinLTO caching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is an issue in clang's ThinLTO caching (enabled for the kernel via '--thinlto-cache-dir') with .incbin, which the kernel occasionally uses to include data within the kernel, such as the .config file for /proc/config.gz. For example, when changing the .config and rebuilding vmlinux, the copy of .config in vmlinux does not match the copy of .config in the build folder: $ echo 'CONFIG_LTO_NONE=n CONFIG_LTO_CLANG_THIN=y CONFIG_IKCONFIG=y CONFIG_HEADERS_INSTALL=y' >kernel/configs/repro.config $ make -skj"$(nproc)" ARCH=x86_64 LLVM=1 clean defconfig repro.config vmlinux ... $ grep CONFIG_HEADERS_INSTALL .config CONFIG_HEADERS_INSTALL=y $ scripts/extract-ikconfig vmlinux | grep CONFIG_HEADERS_INSTALL CONFIG_HEADERS_INSTALL=y $ scripts/config -d HEADERS_INSTALL $ make -kj"$(nproc)" ARCH=x86_64 LLVM=1 vmlinux ... UPD kernel/config_data GZIP kernel/config_data.gz CC kernel/configs.o ... LD vmlinux ... $ grep CONFIG_HEADERS_INSTALL .config # CONFIG_HEADERS_INSTALL is not set $ scripts/extract-ikconfig vmlinux | grep CONFIG_HEADERS_INSTALL CONFIG_HEADERS_INSTALL=y Without '--thinlto-cache-dir' or when using full LTO, this issue does not occur. Benchmarking incremental builds on a few different machines with and without the cache shows a 20% increase in incremental build time without the cache when measured by touching init/main.c and running 'make all'. ARCH=arm64 defconfig + CONFIG_LTO_CLANG_THIN=y on an arm64 host: Benchmark 1: With ThinLTO cache Time (mean ± σ): 56.347 s ± 0.163 s [User: 83.768 s, System: 24.661 s] Range (min … max): 56.109 s … 56.594 s 10 runs Benchmark 2: Without ThinLTO cache Time (mean ± σ): 67.740 s ± 0.479 s [User: 718.458 s, System: 31.797 s] Range (min … max): 67.059 s … 68.556 s 10 runs Summary With ThinLTO cache ran 1.20 ± 0.01 times faster than Without ThinLTO cache ARCH=x86_64 defconfig + CONFIG_LTO_CLANG_THIN=y on an x86_64 host: Benchmark 1: With ThinLTO cache Time (mean ± σ): 85.772 s ± 0.252 s [User: 91.505 s, System: 8.408 s] Range (min … max): 85.447 s … 86.244 s 10 runs Benchmark 2: Without ThinLTO cache Time (mean ± σ): 103.833 s ± 0.288 s [User: 232.058 s, System: 8.569 s] Range (min … max): 103.286 s … 104.124 s 10 runs Summary With ThinLTO cache ran 1.21 ± 0.00 times faster than Without ThinLTO cache While it is unfortunate to take this performance improvement off the table, correctness is more important. If/when this is fixed in LLVM, it can potentially be brought back in a conditional manner. Alternatively, a developer can just disable LTO if doing incremental compiles quickly is important, as a full compile cycle can still take over a minute even with the cache and it is unlikely that LTO will result in functional differences for a kernel change. Cc: stable@vger.kernel.org Fixes: dc5723b02e52 ("kbuild: add support for Clang LTO") Reported-by: Yifan Hong Closes: https://github.com/ClangBuiltLinux/linux/issues/2021 Reported-by: Masami Hiramatsu Closes: https://lore.kernel.org/r/20220327115526.cc4b0ff55fc53c97683c3e4d@kernel.org/ Signed-off-by: Nathan Chancellor Signed-off-by: Masahiro Yamada --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 026971f9f6bcc..12e1a792b8de4 100644 --- a/Makefile +++ b/Makefile @@ -942,7 +942,6 @@ endif ifdef CONFIG_LTO_CLANG ifdef CONFIG_LTO_CLANG_THIN CC_FLAGS_LTO := -flto=thin -fsplit-lto-unit -KBUILD_LDFLAGS += --thinlto-cache-dir=$(extmod_prefix).thinlto-cache else CC_FLAGS_LTO := -flto endif @@ -1480,7 +1479,7 @@ endif # CONFIG_MODULES # Directories & files removed with 'make clean' CLEAN_FILES += vmlinux.symvers modules-only.symvers \ modules.builtin modules.builtin.modinfo modules.nsdeps \ - compile_commands.json .thinlto-cache rust/test \ + compile_commands.json rust/test \ rust-project.json .vmlinux.objs .vmlinux.export.c # Directories & files removed with 'make mrproper' @@ -1787,7 +1786,7 @@ PHONY += compile_commands.json clean-dirs := $(KBUILD_EXTMOD) clean: rm-files := $(KBUILD_EXTMOD)/Module.symvers $(KBUILD_EXTMOD)/modules.nsdeps \ - $(KBUILD_EXTMOD)/compile_commands.json $(KBUILD_EXTMOD)/.thinlto-cache + $(KBUILD_EXTMOD)/compile_commands.json PHONY += prepare # now expand this into a simple variable to reduce the cost of shell evaluations From a5dd673ab7d269e4a5b6565fc2b5c6295a079605 Mon Sep 17 00:00:00 2001 From: "Chang S. Bae" Date: Thu, 2 May 2024 13:58:44 +0300 Subject: [PATCH 0957/1702] x86/insn: Add Key Locker instructions to the opcode map The x86 instruction decoder needs to know these new instructions that are going to be used in the crypto library as well as the x86 core code. Add the following: LOADIWKEY: Load a CPU-internal wrapping key. ENCODEKEY128: Wrap a 128-bit AES key to a key handle. ENCODEKEY256: Wrap a 256-bit AES key to a key handle. AESENC128KL: Encrypt a 128-bit block of data using a 128-bit AES key indicated by a key handle. AESENC256KL: Encrypt a 128-bit block of data using a 256-bit AES key indicated by a key handle. AESDEC128KL: Decrypt a 128-bit block of data using a 128-bit AES key indicated by a key handle. AESDEC256KL: Decrypt a 128-bit block of data using a 256-bit AES key indicated by a key handle. AESENCWIDE128KL: Encrypt 8 128-bit blocks of data using a 128-bit AES key indicated by a key handle. AESENCWIDE256KL: Encrypt 8 128-bit blocks of data using a 256-bit AES key indicated by a key handle. AESDECWIDE128KL: Decrypt 8 128-bit blocks of data using a 128-bit AES key indicated by a key handle. AESDECWIDE256KL: Decrypt 8 128-bit blocks of data using a 256-bit AES key indicated by a key handle. The detail can be found in Intel Software Developer Manual. Signed-off-by: Chang S. Bae Signed-off-by: Adrian Hunter Signed-off-by: Ingo Molnar Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/20240502105853.5338-2-adrian.hunter@intel.com --- arch/x86/lib/x86-opcode-map.txt | 11 +++++++---- tools/arch/x86/lib/x86-opcode-map.txt | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/arch/x86/lib/x86-opcode-map.txt b/arch/x86/lib/x86-opcode-map.txt index 12af572201a29..c94988d5130d6 100644 --- a/arch/x86/lib/x86-opcode-map.txt +++ b/arch/x86/lib/x86-opcode-map.txt @@ -800,11 +800,12 @@ cb: sha256rnds2 Vdq,Wdq | vrcp28ss/d Vx,Hx,Wx (66),(ev) cc: sha256msg1 Vdq,Wdq | vrsqrt28ps/d Vx,Wx (66),(ev) cd: sha256msg2 Vdq,Wdq | vrsqrt28ss/d Vx,Hx,Wx (66),(ev) cf: vgf2p8mulb Vx,Wx (66) +d8: AESENCWIDE128KL Qpi (F3),(000),(00B) | AESENCWIDE256KL Qpi (F3),(000),(10B) | AESDECWIDE128KL Qpi (F3),(000),(01B) | AESDECWIDE256KL Qpi (F3),(000),(11B) db: VAESIMC Vdq,Wdq (66),(v1) -dc: vaesenc Vx,Hx,Wx (66) -dd: vaesenclast Vx,Hx,Wx (66) -de: vaesdec Vx,Hx,Wx (66) -df: vaesdeclast Vx,Hx,Wx (66) +dc: vaesenc Vx,Hx,Wx (66) | LOADIWKEY Vx,Hx (F3) | AESENC128KL Vpd,Qpi (F3) +dd: vaesenclast Vx,Hx,Wx (66) | AESDEC128KL Vpd,Qpi (F3) +de: vaesdec Vx,Hx,Wx (66) | AESENC256KL Vpd,Qpi (F3) +df: vaesdeclast Vx,Hx,Wx (66) | AESDEC256KL Vpd,Qpi (F3) f0: MOVBE Gy,My | MOVBE Gw,Mw (66) | CRC32 Gd,Eb (F2) | CRC32 Gd,Eb (66&F2) f1: MOVBE My,Gy | MOVBE Mw,Gw (66) | CRC32 Gd,Ey (F2) | CRC32 Gd,Ew (66&F2) f2: ANDN Gy,By,Ey (v) @@ -814,6 +815,8 @@ f6: ADCX Gy,Ey (66) | ADOX Gy,Ey (F3) | MULX By,Gy,rDX,Ey (F2),(v) | WRSSD/Q My, f7: BEXTR Gy,Ey,By (v) | SHLX Gy,Ey,By (66),(v) | SARX Gy,Ey,By (F3),(v) | SHRX Gy,Ey,By (F2),(v) f8: MOVDIR64B Gv,Mdqq (66) | ENQCMD Gv,Mdqq (F2) | ENQCMDS Gv,Mdqq (F3) f9: MOVDIRI My,Gy +fa: ENCODEKEY128 Ew,Ew (F3) +fb: ENCODEKEY256 Ew,Ew (F3) EndTable Table: 3-byte opcode 2 (0x0f 0x3a) diff --git a/tools/arch/x86/lib/x86-opcode-map.txt b/tools/arch/x86/lib/x86-opcode-map.txt index 12af572201a29..c94988d5130d6 100644 --- a/tools/arch/x86/lib/x86-opcode-map.txt +++ b/tools/arch/x86/lib/x86-opcode-map.txt @@ -800,11 +800,12 @@ cb: sha256rnds2 Vdq,Wdq | vrcp28ss/d Vx,Hx,Wx (66),(ev) cc: sha256msg1 Vdq,Wdq | vrsqrt28ps/d Vx,Wx (66),(ev) cd: sha256msg2 Vdq,Wdq | vrsqrt28ss/d Vx,Hx,Wx (66),(ev) cf: vgf2p8mulb Vx,Wx (66) +d8: AESENCWIDE128KL Qpi (F3),(000),(00B) | AESENCWIDE256KL Qpi (F3),(000),(10B) | AESDECWIDE128KL Qpi (F3),(000),(01B) | AESDECWIDE256KL Qpi (F3),(000),(11B) db: VAESIMC Vdq,Wdq (66),(v1) -dc: vaesenc Vx,Hx,Wx (66) -dd: vaesenclast Vx,Hx,Wx (66) -de: vaesdec Vx,Hx,Wx (66) -df: vaesdeclast Vx,Hx,Wx (66) +dc: vaesenc Vx,Hx,Wx (66) | LOADIWKEY Vx,Hx (F3) | AESENC128KL Vpd,Qpi (F3) +dd: vaesenclast Vx,Hx,Wx (66) | AESDEC128KL Vpd,Qpi (F3) +de: vaesdec Vx,Hx,Wx (66) | AESENC256KL Vpd,Qpi (F3) +df: vaesdeclast Vx,Hx,Wx (66) | AESDEC256KL Vpd,Qpi (F3) f0: MOVBE Gy,My | MOVBE Gw,Mw (66) | CRC32 Gd,Eb (F2) | CRC32 Gd,Eb (66&F2) f1: MOVBE My,Gy | MOVBE Mw,Gw (66) | CRC32 Gd,Ey (F2) | CRC32 Gd,Ew (66&F2) f2: ANDN Gy,By,Ey (v) @@ -814,6 +815,8 @@ f6: ADCX Gy,Ey (66) | ADOX Gy,Ey (F3) | MULX By,Gy,rDX,Ey (F2),(v) | WRSSD/Q My, f7: BEXTR Gy,Ey,By (v) | SHLX Gy,Ey,By (66),(v) | SARX Gy,Ey,By (F3),(v) | SHRX Gy,Ey,By (F2),(v) f8: MOVDIR64B Gv,Mdqq (66) | ENQCMD Gv,Mdqq (F2) | ENQCMDS Gv,Mdqq (F3) f9: MOVDIRI My,Gy +fa: ENCODEKEY128 Ew,Ew (F3) +fb: ENCODEKEY256 Ew,Ew (F3) EndTable Table: 3-byte opcode 2 (0x0f 0x3a) From 59162e0c11d7257cde15f907d19fefe26da66692 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 2 May 2024 13:58:45 +0300 Subject: [PATCH 0958/1702] x86/insn: Fix PUSH instruction in x86 instruction decoder opcode map The x86 instruction decoder is used not only for decoding kernel instructions. It is also used by perf uprobes (user space probes) and by perf tools Intel Processor Trace decoding. Consequently, it needs to support instructions executed by user space also. Opcode 0x68 PUSH instruction is currently defined as 64-bit operand size only i.e. (d64). That was based on Intel SDM Opcode Map. However that is contradicted by the Instruction Set Reference section for PUSH in the same manual. Remove 64-bit operand size only annotation from opcode 0x68 PUSH instruction. Example: $ cat pushw.s .global _start .text _start: pushw $0x1234 mov $0x1,%eax # system call number (sys_exit) int $0x80 $ as -o pushw.o pushw.s $ ld -s -o pushw pushw.o $ objdump -d pushw | tail -4 0000000000401000 <.text>: 401000: 66 68 34 12 pushw $0x1234 401004: b8 01 00 00 00 mov $0x1,%eax 401009: cd 80 int $0x80 $ perf record -e intel_pt//u ./pushw [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.014 MB perf.data ] Before: $ perf script --insn-trace=disasm Warning: 1 instruction trace errors pushw 10349 [000] 10586.869237014: 401000 [unknown] (/home/ahunter/git/misc/rtit-tests/pushw) pushw $0x1234 pushw 10349 [000] 10586.869237014: 401006 [unknown] (/home/ahunter/git/misc/rtit-tests/pushw) addb %al, (%rax) pushw 10349 [000] 10586.869237014: 401008 [unknown] (/home/ahunter/git/misc/rtit-tests/pushw) addb %cl, %ch pushw 10349 [000] 10586.869237014: 40100a [unknown] (/home/ahunter/git/misc/rtit-tests/pushw) addb $0x2e, (%rax) instruction trace error type 1 time 10586.869237224 cpu 0 pid 10349 tid 10349 ip 0x40100d code 6: Trace doesn't match instruction After: $ perf script --insn-trace=disasm pushw 10349 [000] 10586.869237014: 401000 [unknown] (./pushw) pushw $0x1234 pushw 10349 [000] 10586.869237014: 401004 [unknown] (./pushw) movl $1, %eax Fixes: eb13296cfaf6 ("x86: Instruction decoder API") Signed-off-by: Adrian Hunter Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20240502105853.5338-3-adrian.hunter@intel.com --- arch/x86/lib/x86-opcode-map.txt | 2 +- tools/arch/x86/lib/x86-opcode-map.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/lib/x86-opcode-map.txt b/arch/x86/lib/x86-opcode-map.txt index c94988d5130d6..4ea2e6adb4777 100644 --- a/arch/x86/lib/x86-opcode-map.txt +++ b/arch/x86/lib/x86-opcode-map.txt @@ -148,7 +148,7 @@ AVXcode: 65: SEG=GS (Prefix) 66: Operand-Size (Prefix) 67: Address-Size (Prefix) -68: PUSH Iz (d64) +68: PUSH Iz 69: IMUL Gv,Ev,Iz 6a: PUSH Ib (d64) 6b: IMUL Gv,Ev,Ib diff --git a/tools/arch/x86/lib/x86-opcode-map.txt b/tools/arch/x86/lib/x86-opcode-map.txt index c94988d5130d6..4ea2e6adb4777 100644 --- a/tools/arch/x86/lib/x86-opcode-map.txt +++ b/tools/arch/x86/lib/x86-opcode-map.txt @@ -148,7 +148,7 @@ AVXcode: 65: SEG=GS (Prefix) 66: Operand-Size (Prefix) 67: Address-Size (Prefix) -68: PUSH Iz (d64) +68: PUSH Iz 69: IMUL Gv,Ev,Iz 6a: PUSH Ib (d64) 6b: IMUL Gv,Ev,Ib From b8000264348979b60dbe479255570a40e1b3a097 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 2 May 2024 13:58:46 +0300 Subject: [PATCH 0959/1702] x86/insn: Add VEX versions of VPDPBUSD, VPDPBUSDS, VPDPWSSD and VPDPWSSDS The x86 instruction decoder is used not only for decoding kernel instructions. It is also used by perf uprobes (user space probes) and by perf tools Intel Processor Trace decoding. Consequently, it needs to support instructions executed by user space also. Intel Architecture Instruction Set Extensions and Future Features manual number 319433-044 of May 2021, documented VEX versions of instructions VPDPBUSD, VPDPBUSDS, VPDPWSSD and VPDPWSSDS, but the opcode map has them listed as EVEX only. Remove EVEX-only (ev) annotation from instructions VPDPBUSD, VPDPBUSDS, VPDPWSSD and VPDPWSSDS, which allows them to be decoded with either a VEX or EVEX prefix. Fixes: 0153d98f2dd6 ("x86/insn: Add misc instructions to x86 instruction decoder") Signed-off-by: Adrian Hunter Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20240502105853.5338-4-adrian.hunter@intel.com --- arch/x86/lib/x86-opcode-map.txt | 8 ++++---- tools/arch/x86/lib/x86-opcode-map.txt | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/x86/lib/x86-opcode-map.txt b/arch/x86/lib/x86-opcode-map.txt index 4ea2e6adb4777..24941b9d4e06f 100644 --- a/arch/x86/lib/x86-opcode-map.txt +++ b/arch/x86/lib/x86-opcode-map.txt @@ -698,10 +698,10 @@ AVXcode: 2 4d: vrcp14ss/d Vsd,Hpd,Wsd (66),(ev) 4e: vrsqrt14ps/d Vpd,Wpd (66),(ev) 4f: vrsqrt14ss/d Vsd,Hsd,Wsd (66),(ev) -50: vpdpbusd Vx,Hx,Wx (66),(ev) -51: vpdpbusds Vx,Hx,Wx (66),(ev) -52: vdpbf16ps Vx,Hx,Wx (F3),(ev) | vpdpwssd Vx,Hx,Wx (66),(ev) | vp4dpwssd Vdqq,Hdqq,Wdq (F2),(ev) -53: vpdpwssds Vx,Hx,Wx (66),(ev) | vp4dpwssds Vdqq,Hdqq,Wdq (F2),(ev) +50: vpdpbusd Vx,Hx,Wx (66) +51: vpdpbusds Vx,Hx,Wx (66) +52: vdpbf16ps Vx,Hx,Wx (F3),(ev) | vpdpwssd Vx,Hx,Wx (66) | vp4dpwssd Vdqq,Hdqq,Wdq (F2),(ev) +53: vpdpwssds Vx,Hx,Wx (66) | vp4dpwssds Vdqq,Hdqq,Wdq (F2),(ev) 54: vpopcntb/w Vx,Wx (66),(ev) 55: vpopcntd/q Vx,Wx (66),(ev) 58: vpbroadcastd Vx,Wx (66),(v) diff --git a/tools/arch/x86/lib/x86-opcode-map.txt b/tools/arch/x86/lib/x86-opcode-map.txt index 4ea2e6adb4777..24941b9d4e06f 100644 --- a/tools/arch/x86/lib/x86-opcode-map.txt +++ b/tools/arch/x86/lib/x86-opcode-map.txt @@ -698,10 +698,10 @@ AVXcode: 2 4d: vrcp14ss/d Vsd,Hpd,Wsd (66),(ev) 4e: vrsqrt14ps/d Vpd,Wpd (66),(ev) 4f: vrsqrt14ss/d Vsd,Hsd,Wsd (66),(ev) -50: vpdpbusd Vx,Hx,Wx (66),(ev) -51: vpdpbusds Vx,Hx,Wx (66),(ev) -52: vdpbf16ps Vx,Hx,Wx (F3),(ev) | vpdpwssd Vx,Hx,Wx (66),(ev) | vp4dpwssd Vdqq,Hdqq,Wdq (F2),(ev) -53: vpdpwssds Vx,Hx,Wx (66),(ev) | vp4dpwssds Vdqq,Hdqq,Wdq (F2),(ev) +50: vpdpbusd Vx,Hx,Wx (66) +51: vpdpbusds Vx,Hx,Wx (66) +52: vdpbf16ps Vx,Hx,Wx (F3),(ev) | vpdpwssd Vx,Hx,Wx (66) | vp4dpwssd Vdqq,Hdqq,Wdq (F2),(ev) +53: vpdpwssds Vx,Hx,Wx (66) | vp4dpwssds Vdqq,Hdqq,Wdq (F2),(ev) 54: vpopcntb/w Vx,Wx (66),(ev) 55: vpopcntd/q Vx,Wx (66),(ev) 58: vpbroadcastd Vx,Wx (66),(v) From 9dd3612895de3d967ebd607423571bd6b0854ccc Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 2 May 2024 13:58:47 +0300 Subject: [PATCH 0960/1702] x86/insn: Add misc new Intel instructions The x86 instruction decoder is used not only for decoding kernel instructions. It is also used by perf uprobes (user space probes) and by perf tools Intel Processor Trace decoding. Consequently, it needs to support instructions executed by user space also. Add instructions documented in Intel Architecture Instruction Set Extensions and Future Features Programming Reference March 2024 319433-052, that have not been added yet: AADD AAND AOR AXOR CMPccXADD PBNDKB RDMSRLIST URDMSR UWRMSR VBCSTNEBF162PS VBCSTNESH2PS VCVTNEEBF162PS VCVTNEEPH2PS VCVTNEOBF162PS VCVTNEOPH2PS VCVTNEPS2BF16 VPDPB[SU,UU,SS]D[,S] VPDPW[SU,US,UU]D[,S] VPMADD52HUQ VPMADD52LUQ VSHA512MSG1 VSHA512MSG2 VSHA512RNDS2 VSM3MSG1 VSM3MSG2 VSM3RNDS2 VSM4KEY4 VSM4RNDS4 WRMSRLIST TCMMIMFP16PS TCMMRLFP16PS TDPFP16PS PREFETCHIT1 PREFETCHIT0 Signed-off-by: Adrian Hunter Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20240502105853.5338-5-adrian.hunter@intel.com --- arch/x86/lib/x86-opcode-map.txt | 57 +++++++++++++++++++++------ tools/arch/x86/lib/x86-opcode-map.txt | 57 +++++++++++++++++++++------ 2 files changed, 90 insertions(+), 24 deletions(-) diff --git a/arch/x86/lib/x86-opcode-map.txt b/arch/x86/lib/x86-opcode-map.txt index 24941b9d4e06f..4e9f53581b587 100644 --- a/arch/x86/lib/x86-opcode-map.txt +++ b/arch/x86/lib/x86-opcode-map.txt @@ -698,8 +698,8 @@ AVXcode: 2 4d: vrcp14ss/d Vsd,Hpd,Wsd (66),(ev) 4e: vrsqrt14ps/d Vpd,Wpd (66),(ev) 4f: vrsqrt14ss/d Vsd,Hsd,Wsd (66),(ev) -50: vpdpbusd Vx,Hx,Wx (66) -51: vpdpbusds Vx,Hx,Wx (66) +50: vpdpbusd Vx,Hx,Wx (66) | vpdpbssd Vx,Hx,Wx (F2),(v) | vpdpbsud Vx,Hx,Wx (F3),(v) | vpdpbuud Vx,Hx,Wx (v) +51: vpdpbusds Vx,Hx,Wx (66) | vpdpbssds Vx,Hx,Wx (F2),(v) | vpdpbsuds Vx,Hx,Wx (F3),(v) | vpdpbuuds Vx,Hx,Wx (v) 52: vdpbf16ps Vx,Hx,Wx (F3),(ev) | vpdpwssd Vx,Hx,Wx (66) | vp4dpwssd Vdqq,Hdqq,Wdq (F2),(ev) 53: vpdpwssds Vx,Hx,Wx (66) | vp4dpwssds Vdqq,Hdqq,Wdq (F2),(ev) 54: vpopcntb/w Vx,Wx (66),(ev) @@ -708,7 +708,7 @@ AVXcode: 2 59: vpbroadcastq Vx,Wx (66),(v) | vbroadcasti32x2 Vx,Wx (66),(evo) 5a: vbroadcasti128 Vqq,Mdq (66),(v) | vbroadcasti32x4/64x2 Vx,Wx (66),(evo) 5b: vbroadcasti32x8/64x4 Vqq,Mdq (66),(ev) -5c: TDPBF16PS Vt,Wt,Ht (F3),(v1) +5c: TDPBF16PS Vt,Wt,Ht (F3),(v1) | TDPFP16PS Vt,Wt,Ht (F2),(v1),(o64) # Skip 0x5d 5e: TDPBSSD Vt,Wt,Ht (F2),(v1) | TDPBSUD Vt,Wt,Ht (F3),(v1) | TDPBUSD Vt,Wt,Ht (66),(v1) | TDPBUUD Vt,Wt,Ht (v1) # Skip 0x5f-0x61 @@ -718,10 +718,12 @@ AVXcode: 2 65: vblendmps/d Vx,Hx,Wx (66),(ev) 66: vpblendmb/w Vx,Hx,Wx (66),(ev) 68: vp2intersectd/q Kx,Hx,Wx (F2),(ev) -# Skip 0x69-0x6f +# Skip 0x69-0x6b +6c: TCMMIMFP16PS Vt,Wt,Ht (66),(v1),(o64) | TCMMRLFP16PS Vt,Wt,Ht (v1),(o64) +# Skip 0x6d-0x6f 70: vpshldvw Vx,Hx,Wx (66),(ev) 71: vpshldvd/q Vx,Hx,Wx (66),(ev) -72: vcvtne2ps2bf16 Vx,Hx,Wx (F2),(ev) | vcvtneps2bf16 Vx,Wx (F3),(ev) | vpshrdvw Vx,Hx,Wx (66),(ev) +72: vcvtne2ps2bf16 Vx,Hx,Wx (F2),(ev) | vcvtneps2bf16 Vx,Wx (F3) | vpshrdvw Vx,Hx,Wx (66),(ev) 73: vpshrdvd/q Vx,Hx,Wx (66),(ev) 75: vpermi2b/w Vx,Hx,Wx (66),(ev) 76: vpermi2d/q Vx,Hx,Wx (66),(ev) @@ -777,8 +779,10 @@ ac: vfnmadd213ps/d Vx,Hx,Wx (66),(v) ad: vfnmadd213ss/d Vx,Hx,Wx (66),(v),(v1) ae: vfnmsub213ps/d Vx,Hx,Wx (66),(v) af: vfnmsub213ss/d Vx,Hx,Wx (66),(v),(v1) -b4: vpmadd52luq Vx,Hx,Wx (66),(ev) -b5: vpmadd52huq Vx,Hx,Wx (66),(ev) +b0: vcvtneebf162ps Vx,Mx (F3),(!11B),(v) | vcvtneeph2ps Vx,Mx (66),(!11B),(v) | vcvtneobf162ps Vx,Mx (F2),(!11B),(v) | vcvtneoph2ps Vx,Mx (!11B),(v) +b1: vbcstnebf162ps Vx,Mw (F3),(!11B),(v) | vbcstnesh2ps Vx,Mw (66),(!11B),(v) +b4: vpmadd52luq Vx,Hx,Wx (66) +b5: vpmadd52huq Vx,Hx,Wx (66) b6: vfmaddsub231ps/d Vx,Hx,Wx (66),(v) b7: vfmsubadd231ps/d Vx,Hx,Wx (66),(v) b8: vfmadd231ps/d Vx,Hx,Wx (66),(v) @@ -796,16 +800,35 @@ c7: Grp19 (1A) c8: sha1nexte Vdq,Wdq | vexp2ps/d Vx,Wx (66),(ev) c9: sha1msg1 Vdq,Wdq ca: sha1msg2 Vdq,Wdq | vrcp28ps/d Vx,Wx (66),(ev) -cb: sha256rnds2 Vdq,Wdq | vrcp28ss/d Vx,Hx,Wx (66),(ev) -cc: sha256msg1 Vdq,Wdq | vrsqrt28ps/d Vx,Wx (66),(ev) -cd: sha256msg2 Vdq,Wdq | vrsqrt28ss/d Vx,Hx,Wx (66),(ev) +cb: sha256rnds2 Vdq,Wdq | vrcp28ss/d Vx,Hx,Wx (66),(ev) | vsha512rnds2 Vqq,Hqq,Udq (F2),(11B),(v) +cc: sha256msg1 Vdq,Wdq | vrsqrt28ps/d Vx,Wx (66),(ev) | vsha512msg1 Vqq,Udq (F2),(11B),(v) +cd: sha256msg2 Vdq,Wdq | vrsqrt28ss/d Vx,Hx,Wx (66),(ev) | vsha512msg2 Vqq,Uqq (F2),(11B),(v) cf: vgf2p8mulb Vx,Wx (66) +d2: vpdpwsud Vx,Hx,Wx (F3),(v) | vpdpwusd Vx,Hx,Wx (66),(v) | vpdpwuud Vx,Hx,Wx (v) +d3: vpdpwsuds Vx,Hx,Wx (F3),(v) | vpdpwusds Vx,Hx,Wx (66),(v) | vpdpwuuds Vx,Hx,Wx (v) d8: AESENCWIDE128KL Qpi (F3),(000),(00B) | AESENCWIDE256KL Qpi (F3),(000),(10B) | AESDECWIDE128KL Qpi (F3),(000),(01B) | AESDECWIDE256KL Qpi (F3),(000),(11B) +da: vsm3msg1 Vdq,Hdq,Udq (v1) | vsm3msg2 Vdq,Hdq,Udq (66),(v1) | vsm4key4 Vx,Hx,Wx (F3),(v) | vsm4rnds4 Vx,Hx,Wx (F2),(v) db: VAESIMC Vdq,Wdq (66),(v1) dc: vaesenc Vx,Hx,Wx (66) | LOADIWKEY Vx,Hx (F3) | AESENC128KL Vpd,Qpi (F3) dd: vaesenclast Vx,Hx,Wx (66) | AESDEC128KL Vpd,Qpi (F3) de: vaesdec Vx,Hx,Wx (66) | AESENC256KL Vpd,Qpi (F3) df: vaesdeclast Vx,Hx,Wx (66) | AESDEC256KL Vpd,Qpi (F3) +e0: CMPOXADD My,Gy,By (66),(v1),(o64) +e1: CMPNOXADD My,Gy,By (66),(v1),(o64) +e2: CMPBXADD My,Gy,By (66),(v1),(o64) +e3: CMPNBXADD My,Gy,By (66),(v1),(o64) +e4: CMPZXADD My,Gy,By (66),(v1),(o64) +e5: CMPNZXADD My,Gy,By (66),(v1),(o64) +e6: CMPBEXADD My,Gy,By (66),(v1),(o64) +e7: CMPNBEXADD My,Gy,By (66),(v1),(o64) +e8: CMPSXADD My,Gy,By (66),(v1),(o64) +e9: CMPNSXADD My,Gy,By (66),(v1),(o64) +ea: CMPPXADD My,Gy,By (66),(v1),(o64) +eb: CMPNPXADD My,Gy,By (66),(v1),(o64) +ec: CMPLXADD My,Gy,By (66),(v1),(o64) +ed: CMPNLXADD My,Gy,By (66),(v1),(o64) +ee: CMPLEXADD My,Gy,By (66),(v1),(o64) +ef: CMPNLEXADD My,Gy,By (66),(v1),(o64) f0: MOVBE Gy,My | MOVBE Gw,Mw (66) | CRC32 Gd,Eb (F2) | CRC32 Gd,Eb (66&F2) f1: MOVBE My,Gy | MOVBE Mw,Gw (66) | CRC32 Gd,Ey (F2) | CRC32 Gd,Ew (66&F2) f2: ANDN Gy,By,Ey (v) @@ -813,10 +836,11 @@ f3: Grp17 (1A) f5: BZHI Gy,Ey,By (v) | PEXT Gy,By,Ey (F3),(v) | PDEP Gy,By,Ey (F2),(v) | WRUSSD/Q My,Gy (66) f6: ADCX Gy,Ey (66) | ADOX Gy,Ey (F3) | MULX By,Gy,rDX,Ey (F2),(v) | WRSSD/Q My,Gy f7: BEXTR Gy,Ey,By (v) | SHLX Gy,Ey,By (66),(v) | SARX Gy,Ey,By (F3),(v) | SHRX Gy,Ey,By (F2),(v) -f8: MOVDIR64B Gv,Mdqq (66) | ENQCMD Gv,Mdqq (F2) | ENQCMDS Gv,Mdqq (F3) +f8: MOVDIR64B Gv,Mdqq (66) | ENQCMD Gv,Mdqq (F2) | ENQCMDS Gv,Mdqq (F3) | URDMSR Rq,Gq (F2),(11B) | UWRMSR Gq,Rq (F3),(11B) f9: MOVDIRI My,Gy fa: ENCODEKEY128 Ew,Ew (F3) fb: ENCODEKEY256 Ew,Ew (F3) +fc: AADD My,Gy | AAND My,Gy (66) | AOR My,Gy (F2) | AXOR My,Gy (F3) EndTable Table: 3-byte opcode 2 (0x0f 0x3a) @@ -896,6 +920,7 @@ c2: vcmpph Vx,Hx,Wx,Ib (ev) | vcmpsh Vx,Hx,Wx,Ib (F3),(ev) cc: sha1rnds4 Vdq,Wdq,Ib ce: vgf2p8affineqb Vx,Wx,Ib (66) cf: vgf2p8affineinvqb Vx,Wx,Ib (66) +de: vsm3rnds2 Vdq,Hdq,Wdq,Ib (66),(v1) df: VAESKEYGEN Vdq,Wdq,Ib (66),(v1) f0: RORX Gy,Ey,Ib (F2),(v) | HRESET Gv,Ib (F3),(000),(11B) EndTable @@ -978,6 +1003,12 @@ d6: vfcmulcph Vx,Hx,Wx (F2),(ev) | vfmulcph Vx,Hx,Wx (F3),(ev) d7: vfcmulcsh Vx,Hx,Wx (F2),(ev) | vfmulcsh Vx,Hx,Wx (F3),(ev) EndTable +Table: VEX map 7 +Referrer: +AVXcode: 7 +f8: URDMSR Rq,Id (F2),(v1),(11B) | UWRMSR Id,Rq (F3),(v1),(11B) +EndTable + GrpTable: Grp1 0: ADD 1: OR @@ -1054,7 +1085,7 @@ GrpTable: Grp6 EndTable GrpTable: Grp7 -0: SGDT Ms | VMCALL (001),(11B) | VMLAUNCH (010),(11B) | VMRESUME (011),(11B) | VMXOFF (100),(11B) | PCONFIG (101),(11B) | ENCLV (000),(11B) | WRMSRNS (110),(11B) +0: SGDT Ms | VMCALL (001),(11B) | VMLAUNCH (010),(11B) | VMRESUME (011),(11B) | VMXOFF (100),(11B) | PCONFIG (101),(11B) | ENCLV (000),(11B) | WRMSRNS (110),(11B) | RDMSRLIST (F2),(110),(11B) | WRMSRLIST (F3),(110),(11B) | PBNDKB (111),(11B) 1: SIDT Ms | MONITOR (000),(11B) | MWAIT (001),(11B) | CLAC (010),(11B) | STAC (011),(11B) | ENCLS (111),(11B) | ERETU (F3),(010),(11B) | ERETS (F2),(010),(11B) 2: LGDT Ms | XGETBV (000),(11B) | XSETBV (001),(11B) | VMFUNC (100),(11B) | XEND (101)(11B) | XTEST (110)(11B) | ENCLU (111),(11B) 3: LIDT Ms @@ -1140,6 +1171,8 @@ GrpTable: Grp16 1: prefetch T0 2: prefetch T1 3: prefetch T2 +6: prefetch IT1 +7: prefetch IT0 EndTable GrpTable: Grp17 diff --git a/tools/arch/x86/lib/x86-opcode-map.txt b/tools/arch/x86/lib/x86-opcode-map.txt index 24941b9d4e06f..4e9f53581b587 100644 --- a/tools/arch/x86/lib/x86-opcode-map.txt +++ b/tools/arch/x86/lib/x86-opcode-map.txt @@ -698,8 +698,8 @@ AVXcode: 2 4d: vrcp14ss/d Vsd,Hpd,Wsd (66),(ev) 4e: vrsqrt14ps/d Vpd,Wpd (66),(ev) 4f: vrsqrt14ss/d Vsd,Hsd,Wsd (66),(ev) -50: vpdpbusd Vx,Hx,Wx (66) -51: vpdpbusds Vx,Hx,Wx (66) +50: vpdpbusd Vx,Hx,Wx (66) | vpdpbssd Vx,Hx,Wx (F2),(v) | vpdpbsud Vx,Hx,Wx (F3),(v) | vpdpbuud Vx,Hx,Wx (v) +51: vpdpbusds Vx,Hx,Wx (66) | vpdpbssds Vx,Hx,Wx (F2),(v) | vpdpbsuds Vx,Hx,Wx (F3),(v) | vpdpbuuds Vx,Hx,Wx (v) 52: vdpbf16ps Vx,Hx,Wx (F3),(ev) | vpdpwssd Vx,Hx,Wx (66) | vp4dpwssd Vdqq,Hdqq,Wdq (F2),(ev) 53: vpdpwssds Vx,Hx,Wx (66) | vp4dpwssds Vdqq,Hdqq,Wdq (F2),(ev) 54: vpopcntb/w Vx,Wx (66),(ev) @@ -708,7 +708,7 @@ AVXcode: 2 59: vpbroadcastq Vx,Wx (66),(v) | vbroadcasti32x2 Vx,Wx (66),(evo) 5a: vbroadcasti128 Vqq,Mdq (66),(v) | vbroadcasti32x4/64x2 Vx,Wx (66),(evo) 5b: vbroadcasti32x8/64x4 Vqq,Mdq (66),(ev) -5c: TDPBF16PS Vt,Wt,Ht (F3),(v1) +5c: TDPBF16PS Vt,Wt,Ht (F3),(v1) | TDPFP16PS Vt,Wt,Ht (F2),(v1),(o64) # Skip 0x5d 5e: TDPBSSD Vt,Wt,Ht (F2),(v1) | TDPBSUD Vt,Wt,Ht (F3),(v1) | TDPBUSD Vt,Wt,Ht (66),(v1) | TDPBUUD Vt,Wt,Ht (v1) # Skip 0x5f-0x61 @@ -718,10 +718,12 @@ AVXcode: 2 65: vblendmps/d Vx,Hx,Wx (66),(ev) 66: vpblendmb/w Vx,Hx,Wx (66),(ev) 68: vp2intersectd/q Kx,Hx,Wx (F2),(ev) -# Skip 0x69-0x6f +# Skip 0x69-0x6b +6c: TCMMIMFP16PS Vt,Wt,Ht (66),(v1),(o64) | TCMMRLFP16PS Vt,Wt,Ht (v1),(o64) +# Skip 0x6d-0x6f 70: vpshldvw Vx,Hx,Wx (66),(ev) 71: vpshldvd/q Vx,Hx,Wx (66),(ev) -72: vcvtne2ps2bf16 Vx,Hx,Wx (F2),(ev) | vcvtneps2bf16 Vx,Wx (F3),(ev) | vpshrdvw Vx,Hx,Wx (66),(ev) +72: vcvtne2ps2bf16 Vx,Hx,Wx (F2),(ev) | vcvtneps2bf16 Vx,Wx (F3) | vpshrdvw Vx,Hx,Wx (66),(ev) 73: vpshrdvd/q Vx,Hx,Wx (66),(ev) 75: vpermi2b/w Vx,Hx,Wx (66),(ev) 76: vpermi2d/q Vx,Hx,Wx (66),(ev) @@ -777,8 +779,10 @@ ac: vfnmadd213ps/d Vx,Hx,Wx (66),(v) ad: vfnmadd213ss/d Vx,Hx,Wx (66),(v),(v1) ae: vfnmsub213ps/d Vx,Hx,Wx (66),(v) af: vfnmsub213ss/d Vx,Hx,Wx (66),(v),(v1) -b4: vpmadd52luq Vx,Hx,Wx (66),(ev) -b5: vpmadd52huq Vx,Hx,Wx (66),(ev) +b0: vcvtneebf162ps Vx,Mx (F3),(!11B),(v) | vcvtneeph2ps Vx,Mx (66),(!11B),(v) | vcvtneobf162ps Vx,Mx (F2),(!11B),(v) | vcvtneoph2ps Vx,Mx (!11B),(v) +b1: vbcstnebf162ps Vx,Mw (F3),(!11B),(v) | vbcstnesh2ps Vx,Mw (66),(!11B),(v) +b4: vpmadd52luq Vx,Hx,Wx (66) +b5: vpmadd52huq Vx,Hx,Wx (66) b6: vfmaddsub231ps/d Vx,Hx,Wx (66),(v) b7: vfmsubadd231ps/d Vx,Hx,Wx (66),(v) b8: vfmadd231ps/d Vx,Hx,Wx (66),(v) @@ -796,16 +800,35 @@ c7: Grp19 (1A) c8: sha1nexte Vdq,Wdq | vexp2ps/d Vx,Wx (66),(ev) c9: sha1msg1 Vdq,Wdq ca: sha1msg2 Vdq,Wdq | vrcp28ps/d Vx,Wx (66),(ev) -cb: sha256rnds2 Vdq,Wdq | vrcp28ss/d Vx,Hx,Wx (66),(ev) -cc: sha256msg1 Vdq,Wdq | vrsqrt28ps/d Vx,Wx (66),(ev) -cd: sha256msg2 Vdq,Wdq | vrsqrt28ss/d Vx,Hx,Wx (66),(ev) +cb: sha256rnds2 Vdq,Wdq | vrcp28ss/d Vx,Hx,Wx (66),(ev) | vsha512rnds2 Vqq,Hqq,Udq (F2),(11B),(v) +cc: sha256msg1 Vdq,Wdq | vrsqrt28ps/d Vx,Wx (66),(ev) | vsha512msg1 Vqq,Udq (F2),(11B),(v) +cd: sha256msg2 Vdq,Wdq | vrsqrt28ss/d Vx,Hx,Wx (66),(ev) | vsha512msg2 Vqq,Uqq (F2),(11B),(v) cf: vgf2p8mulb Vx,Wx (66) +d2: vpdpwsud Vx,Hx,Wx (F3),(v) | vpdpwusd Vx,Hx,Wx (66),(v) | vpdpwuud Vx,Hx,Wx (v) +d3: vpdpwsuds Vx,Hx,Wx (F3),(v) | vpdpwusds Vx,Hx,Wx (66),(v) | vpdpwuuds Vx,Hx,Wx (v) d8: AESENCWIDE128KL Qpi (F3),(000),(00B) | AESENCWIDE256KL Qpi (F3),(000),(10B) | AESDECWIDE128KL Qpi (F3),(000),(01B) | AESDECWIDE256KL Qpi (F3),(000),(11B) +da: vsm3msg1 Vdq,Hdq,Udq (v1) | vsm3msg2 Vdq,Hdq,Udq (66),(v1) | vsm4key4 Vx,Hx,Wx (F3),(v) | vsm4rnds4 Vx,Hx,Wx (F2),(v) db: VAESIMC Vdq,Wdq (66),(v1) dc: vaesenc Vx,Hx,Wx (66) | LOADIWKEY Vx,Hx (F3) | AESENC128KL Vpd,Qpi (F3) dd: vaesenclast Vx,Hx,Wx (66) | AESDEC128KL Vpd,Qpi (F3) de: vaesdec Vx,Hx,Wx (66) | AESENC256KL Vpd,Qpi (F3) df: vaesdeclast Vx,Hx,Wx (66) | AESDEC256KL Vpd,Qpi (F3) +e0: CMPOXADD My,Gy,By (66),(v1),(o64) +e1: CMPNOXADD My,Gy,By (66),(v1),(o64) +e2: CMPBXADD My,Gy,By (66),(v1),(o64) +e3: CMPNBXADD My,Gy,By (66),(v1),(o64) +e4: CMPZXADD My,Gy,By (66),(v1),(o64) +e5: CMPNZXADD My,Gy,By (66),(v1),(o64) +e6: CMPBEXADD My,Gy,By (66),(v1),(o64) +e7: CMPNBEXADD My,Gy,By (66),(v1),(o64) +e8: CMPSXADD My,Gy,By (66),(v1),(o64) +e9: CMPNSXADD My,Gy,By (66),(v1),(o64) +ea: CMPPXADD My,Gy,By (66),(v1),(o64) +eb: CMPNPXADD My,Gy,By (66),(v1),(o64) +ec: CMPLXADD My,Gy,By (66),(v1),(o64) +ed: CMPNLXADD My,Gy,By (66),(v1),(o64) +ee: CMPLEXADD My,Gy,By (66),(v1),(o64) +ef: CMPNLEXADD My,Gy,By (66),(v1),(o64) f0: MOVBE Gy,My | MOVBE Gw,Mw (66) | CRC32 Gd,Eb (F2) | CRC32 Gd,Eb (66&F2) f1: MOVBE My,Gy | MOVBE Mw,Gw (66) | CRC32 Gd,Ey (F2) | CRC32 Gd,Ew (66&F2) f2: ANDN Gy,By,Ey (v) @@ -813,10 +836,11 @@ f3: Grp17 (1A) f5: BZHI Gy,Ey,By (v) | PEXT Gy,By,Ey (F3),(v) | PDEP Gy,By,Ey (F2),(v) | WRUSSD/Q My,Gy (66) f6: ADCX Gy,Ey (66) | ADOX Gy,Ey (F3) | MULX By,Gy,rDX,Ey (F2),(v) | WRSSD/Q My,Gy f7: BEXTR Gy,Ey,By (v) | SHLX Gy,Ey,By (66),(v) | SARX Gy,Ey,By (F3),(v) | SHRX Gy,Ey,By (F2),(v) -f8: MOVDIR64B Gv,Mdqq (66) | ENQCMD Gv,Mdqq (F2) | ENQCMDS Gv,Mdqq (F3) +f8: MOVDIR64B Gv,Mdqq (66) | ENQCMD Gv,Mdqq (F2) | ENQCMDS Gv,Mdqq (F3) | URDMSR Rq,Gq (F2),(11B) | UWRMSR Gq,Rq (F3),(11B) f9: MOVDIRI My,Gy fa: ENCODEKEY128 Ew,Ew (F3) fb: ENCODEKEY256 Ew,Ew (F3) +fc: AADD My,Gy | AAND My,Gy (66) | AOR My,Gy (F2) | AXOR My,Gy (F3) EndTable Table: 3-byte opcode 2 (0x0f 0x3a) @@ -896,6 +920,7 @@ c2: vcmpph Vx,Hx,Wx,Ib (ev) | vcmpsh Vx,Hx,Wx,Ib (F3),(ev) cc: sha1rnds4 Vdq,Wdq,Ib ce: vgf2p8affineqb Vx,Wx,Ib (66) cf: vgf2p8affineinvqb Vx,Wx,Ib (66) +de: vsm3rnds2 Vdq,Hdq,Wdq,Ib (66),(v1) df: VAESKEYGEN Vdq,Wdq,Ib (66),(v1) f0: RORX Gy,Ey,Ib (F2),(v) | HRESET Gv,Ib (F3),(000),(11B) EndTable @@ -978,6 +1003,12 @@ d6: vfcmulcph Vx,Hx,Wx (F2),(ev) | vfmulcph Vx,Hx,Wx (F3),(ev) d7: vfcmulcsh Vx,Hx,Wx (F2),(ev) | vfmulcsh Vx,Hx,Wx (F3),(ev) EndTable +Table: VEX map 7 +Referrer: +AVXcode: 7 +f8: URDMSR Rq,Id (F2),(v1),(11B) | UWRMSR Id,Rq (F3),(v1),(11B) +EndTable + GrpTable: Grp1 0: ADD 1: OR @@ -1054,7 +1085,7 @@ GrpTable: Grp6 EndTable GrpTable: Grp7 -0: SGDT Ms | VMCALL (001),(11B) | VMLAUNCH (010),(11B) | VMRESUME (011),(11B) | VMXOFF (100),(11B) | PCONFIG (101),(11B) | ENCLV (000),(11B) | WRMSRNS (110),(11B) +0: SGDT Ms | VMCALL (001),(11B) | VMLAUNCH (010),(11B) | VMRESUME (011),(11B) | VMXOFF (100),(11B) | PCONFIG (101),(11B) | ENCLV (000),(11B) | WRMSRNS (110),(11B) | RDMSRLIST (F2),(110),(11B) | WRMSRLIST (F3),(110),(11B) | PBNDKB (111),(11B) 1: SIDT Ms | MONITOR (000),(11B) | MWAIT (001),(11B) | CLAC (010),(11B) | STAC (011),(11B) | ENCLS (111),(11B) | ERETU (F3),(010),(11B) | ERETS (F2),(010),(11B) 2: LGDT Ms | XGETBV (000),(11B) | XSETBV (001),(11B) | VMFUNC (100),(11B) | XEND (101)(11B) | XTEST (110)(11B) | ENCLU (111),(11B) 3: LIDT Ms @@ -1140,6 +1171,8 @@ GrpTable: Grp16 1: prefetch T0 2: prefetch T1 3: prefetch T2 +6: prefetch IT1 +7: prefetch IT0 EndTable GrpTable: Grp17 From eada38d575a2b947b3ffefd570fea90a5a17feb3 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 2 May 2024 13:58:48 +0300 Subject: [PATCH 0961/1702] x86/insn: Add support for REX2 prefix to the instruction decoder logic Intel Advanced Performance Extensions (APX) uses a new 2-byte prefix named REX2 to select extended general purpose registers (EGPRs) i.e. r16 to r31. The REX2 prefix is effectively an extended version of the REX prefix. REX2 and EVEX are also used with PUSH/POP instructions to provide a Push-Pop Acceleration (PPX) hint. With PPX hints, a CPU will attempt to fast-forward register data between matching PUSH and POP instructions. REX2 is valid only with opcodes in maps 0 and 1. Similar extension for other maps is provided by the EVEX prefix, covered in a separate patch. Some opcodes in maps 0 and 1 are reserved under REX2. One of these is used for a new 64-bit absolute direct jump instruction JMPABS. Refer to the Intel Advanced Performance Extensions (Intel APX) Architecture Specification for details. Define a code value for the REX2 prefix (INAT_PFX_REX2), and add attribute flags for opcodes reserved under REX2 (INAT_NO_REX2) and to identify opcodes (only JMPABS) that require a mandatory REX2 prefix (INAT_REX2_VARIANT). Amend logic to read the REX2 prefix and get the opcode attribute for the map number (0 or 1) encoded in the REX2 prefix. Amend the awk script that generates the attribute tables from the opcode map, to recognise "REX2" as attribute INAT_PFX_REX2, and "(!REX2)" as attribute INAT_NO_REX2, and "(REX2)" as attribute INAT_REX2_VARIANT. Signed-off-by: Adrian Hunter Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20240502105853.5338-6-adrian.hunter@intel.com --- arch/x86/include/asm/inat.h | 11 +++++++++- arch/x86/include/asm/insn.h | 25 ++++++++++++++++++---- arch/x86/lib/insn.c | 25 ++++++++++++++++++++++ arch/x86/tools/gen-insn-attr-x86.awk | 11 +++++++++- tools/arch/x86/include/asm/inat.h | 11 +++++++++- tools/arch/x86/include/asm/insn.h | 25 ++++++++++++++++++---- tools/arch/x86/lib/insn.c | 25 ++++++++++++++++++++++ tools/arch/x86/tools/gen-insn-attr-x86.awk | 11 +++++++++- 8 files changed, 132 insertions(+), 12 deletions(-) diff --git a/arch/x86/include/asm/inat.h b/arch/x86/include/asm/inat.h index b56c5741581a7..1331bdd39a23e 100644 --- a/arch/x86/include/asm/inat.h +++ b/arch/x86/include/asm/inat.h @@ -35,6 +35,8 @@ #define INAT_PFX_VEX2 13 /* 2-bytes VEX prefix */ #define INAT_PFX_VEX3 14 /* 3-bytes VEX prefix */ #define INAT_PFX_EVEX 15 /* EVEX prefix */ +/* x86-64 REX2 prefix */ +#define INAT_PFX_REX2 16 /* 0xD5 */ #define INAT_LSTPFX_MAX 3 #define INAT_LGCPFX_MAX 11 @@ -50,7 +52,7 @@ /* Legacy prefix */ #define INAT_PFX_OFFS 0 -#define INAT_PFX_BITS 4 +#define INAT_PFX_BITS 5 #define INAT_PFX_MAX ((1 << INAT_PFX_BITS) - 1) #define INAT_PFX_MASK (INAT_PFX_MAX << INAT_PFX_OFFS) /* Escape opcodes */ @@ -77,6 +79,8 @@ #define INAT_VEXOK (1 << (INAT_FLAG_OFFS + 5)) #define INAT_VEXONLY (1 << (INAT_FLAG_OFFS + 6)) #define INAT_EVEXONLY (1 << (INAT_FLAG_OFFS + 7)) +#define INAT_NO_REX2 (1 << (INAT_FLAG_OFFS + 8)) +#define INAT_REX2_VARIANT (1 << (INAT_FLAG_OFFS + 9)) /* Attribute making macros for attribute tables */ #define INAT_MAKE_PREFIX(pfx) (pfx << INAT_PFX_OFFS) #define INAT_MAKE_ESCAPE(esc) (esc << INAT_ESC_OFFS) @@ -128,6 +132,11 @@ static inline int inat_is_rex_prefix(insn_attr_t attr) return (attr & INAT_PFX_MASK) == INAT_PFX_REX; } +static inline int inat_is_rex2_prefix(insn_attr_t attr) +{ + return (attr & INAT_PFX_MASK) == INAT_PFX_REX2; +} + static inline int inat_last_prefix_id(insn_attr_t attr) { if ((attr & INAT_PFX_MASK) > INAT_LSTPFX_MAX) diff --git a/arch/x86/include/asm/insn.h b/arch/x86/include/asm/insn.h index 1b29f58f730fd..95249ec1f24ec 100644 --- a/arch/x86/include/asm/insn.h +++ b/arch/x86/include/asm/insn.h @@ -112,10 +112,15 @@ struct insn { #define X86_SIB_INDEX(sib) (((sib) & 0x38) >> 3) #define X86_SIB_BASE(sib) ((sib) & 0x07) -#define X86_REX_W(rex) ((rex) & 8) -#define X86_REX_R(rex) ((rex) & 4) -#define X86_REX_X(rex) ((rex) & 2) -#define X86_REX_B(rex) ((rex) & 1) +#define X86_REX2_M(rex) ((rex) & 0x80) /* REX2 M0 */ +#define X86_REX2_R(rex) ((rex) & 0x40) /* REX2 R4 */ +#define X86_REX2_X(rex) ((rex) & 0x20) /* REX2 X4 */ +#define X86_REX2_B(rex) ((rex) & 0x10) /* REX2 B4 */ + +#define X86_REX_W(rex) ((rex) & 8) /* REX or REX2 W */ +#define X86_REX_R(rex) ((rex) & 4) /* REX or REX2 R3 */ +#define X86_REX_X(rex) ((rex) & 2) /* REX or REX2 X3 */ +#define X86_REX_B(rex) ((rex) & 1) /* REX or REX2 B3 */ /* VEX bit flags */ #define X86_VEX_W(vex) ((vex) & 0x80) /* VEX3 Byte2 */ @@ -161,6 +166,18 @@ static inline void insn_get_attribute(struct insn *insn) /* Instruction uses RIP-relative addressing */ extern int insn_rip_relative(struct insn *insn); +static inline int insn_is_rex2(struct insn *insn) +{ + if (!insn->prefixes.got) + insn_get_prefixes(insn); + return insn->rex_prefix.nbytes == 2; +} + +static inline insn_byte_t insn_rex2_m_bit(struct insn *insn) +{ + return X86_REX2_M(insn->rex_prefix.bytes[1]); +} + static inline int insn_is_avx(struct insn *insn) { if (!insn->prefixes.got) diff --git a/arch/x86/lib/insn.c b/arch/x86/lib/insn.c index 1bb155a0955b3..6126ddc6e5f50 100644 --- a/arch/x86/lib/insn.c +++ b/arch/x86/lib/insn.c @@ -185,6 +185,17 @@ int insn_get_prefixes(struct insn *insn) if (X86_REX_W(b)) /* REX.W overrides opnd_size */ insn->opnd_bytes = 8; + } else if (inat_is_rex2_prefix(attr)) { + insn_set_byte(&insn->rex_prefix, 0, b); + b = peek_nbyte_next(insn_byte_t, insn, 1); + insn_set_byte(&insn->rex_prefix, 1, b); + insn->rex_prefix.nbytes = 2; + insn->next_byte += 2; + if (X86_REX_W(b)) + /* REX.W overrides opnd_size */ + insn->opnd_bytes = 8; + insn->rex_prefix.got = 1; + goto vex_end; } } insn->rex_prefix.got = 1; @@ -294,6 +305,20 @@ int insn_get_opcode(struct insn *insn) goto end; } + /* Check if there is REX2 prefix or not */ + if (insn_is_rex2(insn)) { + if (insn_rex2_m_bit(insn)) { + /* map 1 is escape 0x0f */ + insn_attr_t esc_attr = inat_get_opcode_attribute(0x0f); + + pfx_id = insn_last_prefix_id(insn); + insn->attr = inat_get_escape_attribute(op, pfx_id, esc_attr); + } else { + insn->attr = inat_get_opcode_attribute(op); + } + goto end; + } + insn->attr = inat_get_opcode_attribute(op); while (inat_is_escape(insn->attr)) { /* Get escaped opcode */ diff --git a/arch/x86/tools/gen-insn-attr-x86.awk b/arch/x86/tools/gen-insn-attr-x86.awk index af38469afd148..3f43aa7d8fefb 100644 --- a/arch/x86/tools/gen-insn-attr-x86.awk +++ b/arch/x86/tools/gen-insn-attr-x86.awk @@ -64,7 +64,9 @@ BEGIN { modrm_expr = "^([CDEGMNPQRSUVW/][a-z]+|NTA|T[012])" force64_expr = "\\([df]64\\)" - rex_expr = "^REX(\\.[XRWB]+)*" + rex_expr = "^((REX(\\.[XRWB]+)+)|(REX$))" + rex2_expr = "\\(REX2\\)" + no_rex2_expr = "\\(!REX2\\)" fpu_expr = "^ESC" # TODO lprefix1_expr = "\\((66|!F3)\\)" @@ -99,6 +101,7 @@ BEGIN { prefix_num["VEX+1byte"] = "INAT_PFX_VEX2" prefix_num["VEX+2byte"] = "INAT_PFX_VEX3" prefix_num["EVEX"] = "INAT_PFX_EVEX" + prefix_num["REX2"] = "INAT_PFX_REX2" clear_vars() } @@ -314,6 +317,10 @@ function convert_operands(count,opnd, i,j,imm,mod) if (match(ext, force64_expr)) flags = add_flags(flags, "INAT_FORCE64") + # check REX2 not allowed + if (match(ext, no_rex2_expr)) + flags = add_flags(flags, "INAT_NO_REX2") + # check REX prefix if (match(opcode, rex_expr)) flags = add_flags(flags, "INAT_MAKE_PREFIX(INAT_PFX_REX)") @@ -351,6 +358,8 @@ function convert_operands(count,opnd, i,j,imm,mod) lptable3[idx] = add_flags(lptable3[idx],flags) variant = "INAT_VARIANT" } + if (match(ext, rex2_expr)) + table[idx] = add_flags(table[idx], "INAT_REX2_VARIANT") if (!match(ext, lprefix_expr)){ table[idx] = add_flags(table[idx],flags) } diff --git a/tools/arch/x86/include/asm/inat.h b/tools/arch/x86/include/asm/inat.h index a610514003115..2e65312cae524 100644 --- a/tools/arch/x86/include/asm/inat.h +++ b/tools/arch/x86/include/asm/inat.h @@ -35,6 +35,8 @@ #define INAT_PFX_VEX2 13 /* 2-bytes VEX prefix */ #define INAT_PFX_VEX3 14 /* 3-bytes VEX prefix */ #define INAT_PFX_EVEX 15 /* EVEX prefix */ +/* x86-64 REX2 prefix */ +#define INAT_PFX_REX2 16 /* 0xD5 */ #define INAT_LSTPFX_MAX 3 #define INAT_LGCPFX_MAX 11 @@ -50,7 +52,7 @@ /* Legacy prefix */ #define INAT_PFX_OFFS 0 -#define INAT_PFX_BITS 4 +#define INAT_PFX_BITS 5 #define INAT_PFX_MAX ((1 << INAT_PFX_BITS) - 1) #define INAT_PFX_MASK (INAT_PFX_MAX << INAT_PFX_OFFS) /* Escape opcodes */ @@ -77,6 +79,8 @@ #define INAT_VEXOK (1 << (INAT_FLAG_OFFS + 5)) #define INAT_VEXONLY (1 << (INAT_FLAG_OFFS + 6)) #define INAT_EVEXONLY (1 << (INAT_FLAG_OFFS + 7)) +#define INAT_NO_REX2 (1 << (INAT_FLAG_OFFS + 8)) +#define INAT_REX2_VARIANT (1 << (INAT_FLAG_OFFS + 9)) /* Attribute making macros for attribute tables */ #define INAT_MAKE_PREFIX(pfx) (pfx << INAT_PFX_OFFS) #define INAT_MAKE_ESCAPE(esc) (esc << INAT_ESC_OFFS) @@ -128,6 +132,11 @@ static inline int inat_is_rex_prefix(insn_attr_t attr) return (attr & INAT_PFX_MASK) == INAT_PFX_REX; } +static inline int inat_is_rex2_prefix(insn_attr_t attr) +{ + return (attr & INAT_PFX_MASK) == INAT_PFX_REX2; +} + static inline int inat_last_prefix_id(insn_attr_t attr) { if ((attr & INAT_PFX_MASK) > INAT_LSTPFX_MAX) diff --git a/tools/arch/x86/include/asm/insn.h b/tools/arch/x86/include/asm/insn.h index 65c0d9ce1e295..1a7e8fc4d75a9 100644 --- a/tools/arch/x86/include/asm/insn.h +++ b/tools/arch/x86/include/asm/insn.h @@ -112,10 +112,15 @@ struct insn { #define X86_SIB_INDEX(sib) (((sib) & 0x38) >> 3) #define X86_SIB_BASE(sib) ((sib) & 0x07) -#define X86_REX_W(rex) ((rex) & 8) -#define X86_REX_R(rex) ((rex) & 4) -#define X86_REX_X(rex) ((rex) & 2) -#define X86_REX_B(rex) ((rex) & 1) +#define X86_REX2_M(rex) ((rex) & 0x80) /* REX2 M0 */ +#define X86_REX2_R(rex) ((rex) & 0x40) /* REX2 R4 */ +#define X86_REX2_X(rex) ((rex) & 0x20) /* REX2 X4 */ +#define X86_REX2_B(rex) ((rex) & 0x10) /* REX2 B4 */ + +#define X86_REX_W(rex) ((rex) & 8) /* REX or REX2 W */ +#define X86_REX_R(rex) ((rex) & 4) /* REX or REX2 R3 */ +#define X86_REX_X(rex) ((rex) & 2) /* REX or REX2 X3 */ +#define X86_REX_B(rex) ((rex) & 1) /* REX or REX2 B3 */ /* VEX bit flags */ #define X86_VEX_W(vex) ((vex) & 0x80) /* VEX3 Byte2 */ @@ -161,6 +166,18 @@ static inline void insn_get_attribute(struct insn *insn) /* Instruction uses RIP-relative addressing */ extern int insn_rip_relative(struct insn *insn); +static inline int insn_is_rex2(struct insn *insn) +{ + if (!insn->prefixes.got) + insn_get_prefixes(insn); + return insn->rex_prefix.nbytes == 2; +} + +static inline insn_byte_t insn_rex2_m_bit(struct insn *insn) +{ + return X86_REX2_M(insn->rex_prefix.bytes[1]); +} + static inline int insn_is_avx(struct insn *insn) { if (!insn->prefixes.got) diff --git a/tools/arch/x86/lib/insn.c b/tools/arch/x86/lib/insn.c index ada4b4a79dd48..f761adeb8e8c5 100644 --- a/tools/arch/x86/lib/insn.c +++ b/tools/arch/x86/lib/insn.c @@ -185,6 +185,17 @@ int insn_get_prefixes(struct insn *insn) if (X86_REX_W(b)) /* REX.W overrides opnd_size */ insn->opnd_bytes = 8; + } else if (inat_is_rex2_prefix(attr)) { + insn_set_byte(&insn->rex_prefix, 0, b); + b = peek_nbyte_next(insn_byte_t, insn, 1); + insn_set_byte(&insn->rex_prefix, 1, b); + insn->rex_prefix.nbytes = 2; + insn->next_byte += 2; + if (X86_REX_W(b)) + /* REX.W overrides opnd_size */ + insn->opnd_bytes = 8; + insn->rex_prefix.got = 1; + goto vex_end; } } insn->rex_prefix.got = 1; @@ -294,6 +305,20 @@ int insn_get_opcode(struct insn *insn) goto end; } + /* Check if there is REX2 prefix or not */ + if (insn_is_rex2(insn)) { + if (insn_rex2_m_bit(insn)) { + /* map 1 is escape 0x0f */ + insn_attr_t esc_attr = inat_get_opcode_attribute(0x0f); + + pfx_id = insn_last_prefix_id(insn); + insn->attr = inat_get_escape_attribute(op, pfx_id, esc_attr); + } else { + insn->attr = inat_get_opcode_attribute(op); + } + goto end; + } + insn->attr = inat_get_opcode_attribute(op); while (inat_is_escape(insn->attr)) { /* Get escaped opcode */ diff --git a/tools/arch/x86/tools/gen-insn-attr-x86.awk b/tools/arch/x86/tools/gen-insn-attr-x86.awk index af38469afd148..3f43aa7d8fefb 100644 --- a/tools/arch/x86/tools/gen-insn-attr-x86.awk +++ b/tools/arch/x86/tools/gen-insn-attr-x86.awk @@ -64,7 +64,9 @@ BEGIN { modrm_expr = "^([CDEGMNPQRSUVW/][a-z]+|NTA|T[012])" force64_expr = "\\([df]64\\)" - rex_expr = "^REX(\\.[XRWB]+)*" + rex_expr = "^((REX(\\.[XRWB]+)+)|(REX$))" + rex2_expr = "\\(REX2\\)" + no_rex2_expr = "\\(!REX2\\)" fpu_expr = "^ESC" # TODO lprefix1_expr = "\\((66|!F3)\\)" @@ -99,6 +101,7 @@ BEGIN { prefix_num["VEX+1byte"] = "INAT_PFX_VEX2" prefix_num["VEX+2byte"] = "INAT_PFX_VEX3" prefix_num["EVEX"] = "INAT_PFX_EVEX" + prefix_num["REX2"] = "INAT_PFX_REX2" clear_vars() } @@ -314,6 +317,10 @@ function convert_operands(count,opnd, i,j,imm,mod) if (match(ext, force64_expr)) flags = add_flags(flags, "INAT_FORCE64") + # check REX2 not allowed + if (match(ext, no_rex2_expr)) + flags = add_flags(flags, "INAT_NO_REX2") + # check REX prefix if (match(opcode, rex_expr)) flags = add_flags(flags, "INAT_MAKE_PREFIX(INAT_PFX_REX)") @@ -351,6 +358,8 @@ function convert_operands(count,opnd, i,j,imm,mod) lptable3[idx] = add_flags(lptable3[idx],flags) variant = "INAT_VARIANT" } + if (match(ext, rex2_expr)) + table[idx] = add_flags(table[idx], "INAT_REX2_VARIANT") if (!match(ext, lprefix_expr)){ table[idx] = add_flags(table[idx],flags) } From 159039af8c074a14545fd695c38d2772b3c98b12 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 2 May 2024 13:58:49 +0300 Subject: [PATCH 0962/1702] x86/insn: x86/insn: Add support for REX2 prefix to the instruction decoder opcode map Support for REX2 has been added to the instruction decoder logic and the awk script that generates the attribute tables from the opcode map. Add REX2 prefix byte (0xD5) to the opcode map. Add annotation (!REX2) for map 0/1 opcodes that are reserved under REX2. Add JMPABS to the opcode map and add annotation (REX2) to identify that it has a mandatory REX2 prefix. A separate opcode attribute table is not needed at this time because JMPABS has the same attribute encoding as the MOV instruction that it shares an opcode with i.e. INAT_MOFFSET. Signed-off-by: Adrian Hunter Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20240502105853.5338-7-adrian.hunter@intel.com --- arch/x86/lib/x86-opcode-map.txt | 148 +++++++++++++------------- tools/arch/x86/lib/x86-opcode-map.txt | 148 +++++++++++++------------- 2 files changed, 152 insertions(+), 144 deletions(-) diff --git a/arch/x86/lib/x86-opcode-map.txt b/arch/x86/lib/x86-opcode-map.txt index 4e9f53581b587..240ef714b64fa 100644 --- a/arch/x86/lib/x86-opcode-map.txt +++ b/arch/x86/lib/x86-opcode-map.txt @@ -33,6 +33,10 @@ # - (F2): the last prefix is 0xF2 # - (!F3) : the last prefix is not 0xF3 (including non-last prefix case) # - (66&F2): Both 0x66 and 0xF2 prefixes are specified. +# +# REX2 Prefix +# - (!REX2): REX2 is not allowed +# - (REX2): REX2 variant e.g. JMPABS Table: one byte opcode Referrer: @@ -157,22 +161,22 @@ AVXcode: 6e: OUTS/OUTSB DX,Xb 6f: OUTS/OUTSW/OUTSD DX,Xz # 0x70 - 0x7f -70: JO Jb -71: JNO Jb -72: JB/JNAE/JC Jb -73: JNB/JAE/JNC Jb -74: JZ/JE Jb -75: JNZ/JNE Jb -76: JBE/JNA Jb -77: JNBE/JA Jb -78: JS Jb -79: JNS Jb -7a: JP/JPE Jb -7b: JNP/JPO Jb -7c: JL/JNGE Jb -7d: JNL/JGE Jb -7e: JLE/JNG Jb -7f: JNLE/JG Jb +70: JO Jb (!REX2) +71: JNO Jb (!REX2) +72: JB/JNAE/JC Jb (!REX2) +73: JNB/JAE/JNC Jb (!REX2) +74: JZ/JE Jb (!REX2) +75: JNZ/JNE Jb (!REX2) +76: JBE/JNA Jb (!REX2) +77: JNBE/JA Jb (!REX2) +78: JS Jb (!REX2) +79: JNS Jb (!REX2) +7a: JP/JPE Jb (!REX2) +7b: JNP/JPO Jb (!REX2) +7c: JL/JNGE Jb (!REX2) +7d: JNL/JGE Jb (!REX2) +7e: JLE/JNG Jb (!REX2) +7f: JNLE/JG Jb (!REX2) # 0x80 - 0x8f 80: Grp1 Eb,Ib (1A) 81: Grp1 Ev,Iz (1A) @@ -208,24 +212,24 @@ AVXcode: 9e: SAHF 9f: LAHF # 0xa0 - 0xaf -a0: MOV AL,Ob -a1: MOV rAX,Ov -a2: MOV Ob,AL -a3: MOV Ov,rAX -a4: MOVS/B Yb,Xb -a5: MOVS/W/D/Q Yv,Xv -a6: CMPS/B Xb,Yb -a7: CMPS/W/D Xv,Yv -a8: TEST AL,Ib -a9: TEST rAX,Iz -aa: STOS/B Yb,AL -ab: STOS/W/D/Q Yv,rAX -ac: LODS/B AL,Xb -ad: LODS/W/D/Q rAX,Xv -ae: SCAS/B AL,Yb +a0: MOV AL,Ob (!REX2) +a1: MOV rAX,Ov (!REX2) | JMPABS O (REX2),(o64) +a2: MOV Ob,AL (!REX2) +a3: MOV Ov,rAX (!REX2) +a4: MOVS/B Yb,Xb (!REX2) +a5: MOVS/W/D/Q Yv,Xv (!REX2) +a6: CMPS/B Xb,Yb (!REX2) +a7: CMPS/W/D Xv,Yv (!REX2) +a8: TEST AL,Ib (!REX2) +a9: TEST rAX,Iz (!REX2) +aa: STOS/B Yb,AL (!REX2) +ab: STOS/W/D/Q Yv,rAX (!REX2) +ac: LODS/B AL,Xb (!REX2) +ad: LODS/W/D/Q rAX,Xv (!REX2) +ae: SCAS/B AL,Yb (!REX2) # Note: The May 2011 Intel manual shows Xv for the second parameter of the # next instruction but Yv is correct -af: SCAS/W/D/Q rAX,Yv +af: SCAS/W/D/Q rAX,Yv (!REX2) # 0xb0 - 0xbf b0: MOV AL/R8L,Ib b1: MOV CL/R9L,Ib @@ -266,7 +270,7 @@ d1: Grp2 Ev,1 (1A) d2: Grp2 Eb,CL (1A) d3: Grp2 Ev,CL (1A) d4: AAM Ib (i64) -d5: AAD Ib (i64) +d5: AAD Ib (i64) | REX2 (Prefix),(o64) d6: d7: XLAT/XLATB d8: ESC @@ -281,26 +285,26 @@ df: ESC # Note: "forced64" is Intel CPU behavior: they ignore 0x66 prefix # in 64-bit mode. AMD CPUs accept 0x66 prefix, it causes RIP truncation # to 16 bits. In 32-bit mode, 0x66 is accepted by both Intel and AMD. -e0: LOOPNE/LOOPNZ Jb (f64) -e1: LOOPE/LOOPZ Jb (f64) -e2: LOOP Jb (f64) -e3: JrCXZ Jb (f64) -e4: IN AL,Ib -e5: IN eAX,Ib -e6: OUT Ib,AL -e7: OUT Ib,eAX +e0: LOOPNE/LOOPNZ Jb (f64) (!REX2) +e1: LOOPE/LOOPZ Jb (f64) (!REX2) +e2: LOOP Jb (f64) (!REX2) +e3: JrCXZ Jb (f64) (!REX2) +e4: IN AL,Ib (!REX2) +e5: IN eAX,Ib (!REX2) +e6: OUT Ib,AL (!REX2) +e7: OUT Ib,eAX (!REX2) # With 0x66 prefix in 64-bit mode, for AMD CPUs immediate offset # in "near" jumps and calls is 16-bit. For CALL, # push of return address is 16-bit wide, RSP is decremented by 2 # but is not truncated to 16 bits, unlike RIP. -e8: CALL Jz (f64) -e9: JMP-near Jz (f64) -ea: JMP-far Ap (i64) -eb: JMP-short Jb (f64) -ec: IN AL,DX -ed: IN eAX,DX -ee: OUT DX,AL -ef: OUT DX,eAX +e8: CALL Jz (f64) (!REX2) +e9: JMP-near Jz (f64) (!REX2) +ea: JMP-far Ap (i64) (!REX2) +eb: JMP-short Jb (f64) (!REX2) +ec: IN AL,DX (!REX2) +ed: IN eAX,DX (!REX2) +ee: OUT DX,AL (!REX2) +ef: OUT DX,eAX (!REX2) # 0xf0 - 0xff f0: LOCK (Prefix) f1: @@ -386,14 +390,14 @@ AVXcode: 1 2e: vucomiss Vss,Wss (v1) | vucomisd Vsd,Wsd (66),(v1) 2f: vcomiss Vss,Wss (v1) | vcomisd Vsd,Wsd (66),(v1) # 0x0f 0x30-0x3f -30: WRMSR -31: RDTSC -32: RDMSR -33: RDPMC -34: SYSENTER -35: SYSEXIT +30: WRMSR (!REX2) +31: RDTSC (!REX2) +32: RDMSR (!REX2) +33: RDPMC (!REX2) +34: SYSENTER (!REX2) +35: SYSEXIT (!REX2) 36: -37: GETSEC +37: GETSEC (!REX2) 38: escape # 3-byte escape 1 39: 3a: escape # 3-byte escape 2 @@ -473,22 +477,22 @@ AVXcode: 1 7f: movq Qq,Pq | vmovdqa Wx,Vx (66) | vmovdqa32/64 Wx,Vx (66),(evo) | vmovdqu Wx,Vx (F3) | vmovdqu32/64 Wx,Vx (F3),(evo) | vmovdqu8/16 Wx,Vx (F2),(ev) # 0x0f 0x80-0x8f # Note: "forced64" is Intel CPU behavior (see comment about CALL insn). -80: JO Jz (f64) -81: JNO Jz (f64) -82: JB/JC/JNAE Jz (f64) -83: JAE/JNB/JNC Jz (f64) -84: JE/JZ Jz (f64) -85: JNE/JNZ Jz (f64) -86: JBE/JNA Jz (f64) -87: JA/JNBE Jz (f64) -88: JS Jz (f64) -89: JNS Jz (f64) -8a: JP/JPE Jz (f64) -8b: JNP/JPO Jz (f64) -8c: JL/JNGE Jz (f64) -8d: JNL/JGE Jz (f64) -8e: JLE/JNG Jz (f64) -8f: JNLE/JG Jz (f64) +80: JO Jz (f64) (!REX2) +81: JNO Jz (f64) (!REX2) +82: JB/JC/JNAE Jz (f64) (!REX2) +83: JAE/JNB/JNC Jz (f64) (!REX2) +84: JE/JZ Jz (f64) (!REX2) +85: JNE/JNZ Jz (f64) (!REX2) +86: JBE/JNA Jz (f64) (!REX2) +87: JA/JNBE Jz (f64) (!REX2) +88: JS Jz (f64) (!REX2) +89: JNS Jz (f64) (!REX2) +8a: JP/JPE Jz (f64) (!REX2) +8b: JNP/JPO Jz (f64) (!REX2) +8c: JL/JNGE Jz (f64) (!REX2) +8d: JNL/JGE Jz (f64) (!REX2) +8e: JLE/JNG Jz (f64) (!REX2) +8f: JNLE/JG Jz (f64) (!REX2) # 0x0f 0x90-0x9f 90: SETO Eb | kmovw/q Vk,Wk | kmovb/d Vk,Wk (66) 91: SETNO Eb | kmovw/q Mv,Vk | kmovb/d Mv,Vk (66) diff --git a/tools/arch/x86/lib/x86-opcode-map.txt b/tools/arch/x86/lib/x86-opcode-map.txt index 4e9f53581b587..240ef714b64fa 100644 --- a/tools/arch/x86/lib/x86-opcode-map.txt +++ b/tools/arch/x86/lib/x86-opcode-map.txt @@ -33,6 +33,10 @@ # - (F2): the last prefix is 0xF2 # - (!F3) : the last prefix is not 0xF3 (including non-last prefix case) # - (66&F2): Both 0x66 and 0xF2 prefixes are specified. +# +# REX2 Prefix +# - (!REX2): REX2 is not allowed +# - (REX2): REX2 variant e.g. JMPABS Table: one byte opcode Referrer: @@ -157,22 +161,22 @@ AVXcode: 6e: OUTS/OUTSB DX,Xb 6f: OUTS/OUTSW/OUTSD DX,Xz # 0x70 - 0x7f -70: JO Jb -71: JNO Jb -72: JB/JNAE/JC Jb -73: JNB/JAE/JNC Jb -74: JZ/JE Jb -75: JNZ/JNE Jb -76: JBE/JNA Jb -77: JNBE/JA Jb -78: JS Jb -79: JNS Jb -7a: JP/JPE Jb -7b: JNP/JPO Jb -7c: JL/JNGE Jb -7d: JNL/JGE Jb -7e: JLE/JNG Jb -7f: JNLE/JG Jb +70: JO Jb (!REX2) +71: JNO Jb (!REX2) +72: JB/JNAE/JC Jb (!REX2) +73: JNB/JAE/JNC Jb (!REX2) +74: JZ/JE Jb (!REX2) +75: JNZ/JNE Jb (!REX2) +76: JBE/JNA Jb (!REX2) +77: JNBE/JA Jb (!REX2) +78: JS Jb (!REX2) +79: JNS Jb (!REX2) +7a: JP/JPE Jb (!REX2) +7b: JNP/JPO Jb (!REX2) +7c: JL/JNGE Jb (!REX2) +7d: JNL/JGE Jb (!REX2) +7e: JLE/JNG Jb (!REX2) +7f: JNLE/JG Jb (!REX2) # 0x80 - 0x8f 80: Grp1 Eb,Ib (1A) 81: Grp1 Ev,Iz (1A) @@ -208,24 +212,24 @@ AVXcode: 9e: SAHF 9f: LAHF # 0xa0 - 0xaf -a0: MOV AL,Ob -a1: MOV rAX,Ov -a2: MOV Ob,AL -a3: MOV Ov,rAX -a4: MOVS/B Yb,Xb -a5: MOVS/W/D/Q Yv,Xv -a6: CMPS/B Xb,Yb -a7: CMPS/W/D Xv,Yv -a8: TEST AL,Ib -a9: TEST rAX,Iz -aa: STOS/B Yb,AL -ab: STOS/W/D/Q Yv,rAX -ac: LODS/B AL,Xb -ad: LODS/W/D/Q rAX,Xv -ae: SCAS/B AL,Yb +a0: MOV AL,Ob (!REX2) +a1: MOV rAX,Ov (!REX2) | JMPABS O (REX2),(o64) +a2: MOV Ob,AL (!REX2) +a3: MOV Ov,rAX (!REX2) +a4: MOVS/B Yb,Xb (!REX2) +a5: MOVS/W/D/Q Yv,Xv (!REX2) +a6: CMPS/B Xb,Yb (!REX2) +a7: CMPS/W/D Xv,Yv (!REX2) +a8: TEST AL,Ib (!REX2) +a9: TEST rAX,Iz (!REX2) +aa: STOS/B Yb,AL (!REX2) +ab: STOS/W/D/Q Yv,rAX (!REX2) +ac: LODS/B AL,Xb (!REX2) +ad: LODS/W/D/Q rAX,Xv (!REX2) +ae: SCAS/B AL,Yb (!REX2) # Note: The May 2011 Intel manual shows Xv for the second parameter of the # next instruction but Yv is correct -af: SCAS/W/D/Q rAX,Yv +af: SCAS/W/D/Q rAX,Yv (!REX2) # 0xb0 - 0xbf b0: MOV AL/R8L,Ib b1: MOV CL/R9L,Ib @@ -266,7 +270,7 @@ d1: Grp2 Ev,1 (1A) d2: Grp2 Eb,CL (1A) d3: Grp2 Ev,CL (1A) d4: AAM Ib (i64) -d5: AAD Ib (i64) +d5: AAD Ib (i64) | REX2 (Prefix),(o64) d6: d7: XLAT/XLATB d8: ESC @@ -281,26 +285,26 @@ df: ESC # Note: "forced64" is Intel CPU behavior: they ignore 0x66 prefix # in 64-bit mode. AMD CPUs accept 0x66 prefix, it causes RIP truncation # to 16 bits. In 32-bit mode, 0x66 is accepted by both Intel and AMD. -e0: LOOPNE/LOOPNZ Jb (f64) -e1: LOOPE/LOOPZ Jb (f64) -e2: LOOP Jb (f64) -e3: JrCXZ Jb (f64) -e4: IN AL,Ib -e5: IN eAX,Ib -e6: OUT Ib,AL -e7: OUT Ib,eAX +e0: LOOPNE/LOOPNZ Jb (f64) (!REX2) +e1: LOOPE/LOOPZ Jb (f64) (!REX2) +e2: LOOP Jb (f64) (!REX2) +e3: JrCXZ Jb (f64) (!REX2) +e4: IN AL,Ib (!REX2) +e5: IN eAX,Ib (!REX2) +e6: OUT Ib,AL (!REX2) +e7: OUT Ib,eAX (!REX2) # With 0x66 prefix in 64-bit mode, for AMD CPUs immediate offset # in "near" jumps and calls is 16-bit. For CALL, # push of return address is 16-bit wide, RSP is decremented by 2 # but is not truncated to 16 bits, unlike RIP. -e8: CALL Jz (f64) -e9: JMP-near Jz (f64) -ea: JMP-far Ap (i64) -eb: JMP-short Jb (f64) -ec: IN AL,DX -ed: IN eAX,DX -ee: OUT DX,AL -ef: OUT DX,eAX +e8: CALL Jz (f64) (!REX2) +e9: JMP-near Jz (f64) (!REX2) +ea: JMP-far Ap (i64) (!REX2) +eb: JMP-short Jb (f64) (!REX2) +ec: IN AL,DX (!REX2) +ed: IN eAX,DX (!REX2) +ee: OUT DX,AL (!REX2) +ef: OUT DX,eAX (!REX2) # 0xf0 - 0xff f0: LOCK (Prefix) f1: @@ -386,14 +390,14 @@ AVXcode: 1 2e: vucomiss Vss,Wss (v1) | vucomisd Vsd,Wsd (66),(v1) 2f: vcomiss Vss,Wss (v1) | vcomisd Vsd,Wsd (66),(v1) # 0x0f 0x30-0x3f -30: WRMSR -31: RDTSC -32: RDMSR -33: RDPMC -34: SYSENTER -35: SYSEXIT +30: WRMSR (!REX2) +31: RDTSC (!REX2) +32: RDMSR (!REX2) +33: RDPMC (!REX2) +34: SYSENTER (!REX2) +35: SYSEXIT (!REX2) 36: -37: GETSEC +37: GETSEC (!REX2) 38: escape # 3-byte escape 1 39: 3a: escape # 3-byte escape 2 @@ -473,22 +477,22 @@ AVXcode: 1 7f: movq Qq,Pq | vmovdqa Wx,Vx (66) | vmovdqa32/64 Wx,Vx (66),(evo) | vmovdqu Wx,Vx (F3) | vmovdqu32/64 Wx,Vx (F3),(evo) | vmovdqu8/16 Wx,Vx (F2),(ev) # 0x0f 0x80-0x8f # Note: "forced64" is Intel CPU behavior (see comment about CALL insn). -80: JO Jz (f64) -81: JNO Jz (f64) -82: JB/JC/JNAE Jz (f64) -83: JAE/JNB/JNC Jz (f64) -84: JE/JZ Jz (f64) -85: JNE/JNZ Jz (f64) -86: JBE/JNA Jz (f64) -87: JA/JNBE Jz (f64) -88: JS Jz (f64) -89: JNS Jz (f64) -8a: JP/JPE Jz (f64) -8b: JNP/JPO Jz (f64) -8c: JL/JNGE Jz (f64) -8d: JNL/JGE Jz (f64) -8e: JLE/JNG Jz (f64) -8f: JNLE/JG Jz (f64) +80: JO Jz (f64) (!REX2) +81: JNO Jz (f64) (!REX2) +82: JB/JC/JNAE Jz (f64) (!REX2) +83: JAE/JNB/JNC Jz (f64) (!REX2) +84: JE/JZ Jz (f64) (!REX2) +85: JNE/JNZ Jz (f64) (!REX2) +86: JBE/JNA Jz (f64) (!REX2) +87: JA/JNBE Jz (f64) (!REX2) +88: JS Jz (f64) (!REX2) +89: JNS Jz (f64) (!REX2) +8a: JP/JPE Jz (f64) (!REX2) +8b: JNP/JPO Jz (f64) (!REX2) +8c: JL/JNGE Jz (f64) (!REX2) +8d: JNL/JGE Jz (f64) (!REX2) +8e: JLE/JNG Jz (f64) (!REX2) +8f: JNLE/JG Jz (f64) (!REX2) # 0x0f 0x90-0x9f 90: SETO Eb | kmovw/q Vk,Wk | kmovb/d Vk,Wk (66) 91: SETNO Eb | kmovw/q Mv,Vk | kmovb/d Mv,Vk (66) From 87bbaf1a4be4904fcf04a024e7c1d9f9d1fa945b Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 2 May 2024 13:58:50 +0300 Subject: [PATCH 0963/1702] x86/insn: Add support for APX EVEX to the instruction decoder logic Intel Advanced Performance Extensions (APX) extends the EVEX prefix to support: - extended general purpose registers (EGPRs) i.e. r16 to r31 - Push-Pop Acceleration (PPX) hints - new data destination (NDD) register - suppress status flags writes (NF) of common instructions - new instructions Refer to the Intel Advanced Performance Extensions (Intel APX) Architecture Specification for details. The extended EVEX prefix does not need amended instruction decoder logic, except in one area. Some instructions are defined as SCALABLE which means the EVEX.W bit and EVEX.pp bits are used to determine operand size. Specifically, if an instruction is SCALABLE and EVEX.W is zero, then EVEX.pp value 0 (representing no prefix NP) means default operand size, whereas EVEX.pp value 1 (representing 66 prefix) means operand size override i.e. 16 bits Add an attribute (INAT_EVEX_SCALABLE) to identify such instructions, and amend the logic appropriately. Amend the awk script that generates the attribute tables from the opcode map, to recognise "(es)" as attribute INAT_EVEX_SCALABLE. Signed-off-by: Adrian Hunter Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20240502105853.5338-8-adrian.hunter@intel.com --- arch/x86/include/asm/inat.h | 6 ++++++ arch/x86/include/asm/insn.h | 7 +++++++ arch/x86/lib/insn.c | 4 ++++ arch/x86/tools/gen-insn-attr-x86.awk | 4 ++++ tools/arch/x86/include/asm/inat.h | 6 ++++++ tools/arch/x86/include/asm/insn.h | 7 +++++++ tools/arch/x86/lib/insn.c | 4 ++++ tools/arch/x86/tools/gen-insn-attr-x86.awk | 4 ++++ 8 files changed, 42 insertions(+) diff --git a/arch/x86/include/asm/inat.h b/arch/x86/include/asm/inat.h index 1331bdd39a23e..53e4015242b4f 100644 --- a/arch/x86/include/asm/inat.h +++ b/arch/x86/include/asm/inat.h @@ -81,6 +81,7 @@ #define INAT_EVEXONLY (1 << (INAT_FLAG_OFFS + 7)) #define INAT_NO_REX2 (1 << (INAT_FLAG_OFFS + 8)) #define INAT_REX2_VARIANT (1 << (INAT_FLAG_OFFS + 9)) +#define INAT_EVEX_SCALABLE (1 << (INAT_FLAG_OFFS + 10)) /* Attribute making macros for attribute tables */ #define INAT_MAKE_PREFIX(pfx) (pfx << INAT_PFX_OFFS) #define INAT_MAKE_ESCAPE(esc) (esc << INAT_ESC_OFFS) @@ -236,4 +237,9 @@ static inline int inat_must_evex(insn_attr_t attr) { return attr & INAT_EVEXONLY; } + +static inline int inat_evex_scalable(insn_attr_t attr) +{ + return attr & INAT_EVEX_SCALABLE; +} #endif diff --git a/arch/x86/include/asm/insn.h b/arch/x86/include/asm/insn.h index 95249ec1f24ec..7152ea809e6a5 100644 --- a/arch/x86/include/asm/insn.h +++ b/arch/x86/include/asm/insn.h @@ -215,6 +215,13 @@ static inline insn_byte_t insn_vex_p_bits(struct insn *insn) return X86_VEX_P(insn->vex_prefix.bytes[2]); } +static inline insn_byte_t insn_vex_w_bit(struct insn *insn) +{ + if (insn->vex_prefix.nbytes < 3) + return 0; + return X86_VEX_W(insn->vex_prefix.bytes[2]); +} + /* Get the last prefix id from last prefix or VEX prefix */ static inline int insn_last_prefix_id(struct insn *insn) { diff --git a/arch/x86/lib/insn.c b/arch/x86/lib/insn.c index 6126ddc6e5f50..5952ab41c60f4 100644 --- a/arch/x86/lib/insn.c +++ b/arch/x86/lib/insn.c @@ -294,6 +294,10 @@ int insn_get_opcode(struct insn *insn) m = insn_vex_m_bits(insn); p = insn_vex_p_bits(insn); insn->attr = inat_get_avx_attribute(op, m, p); + /* SCALABLE EVEX uses p bits to encode operand size */ + if (inat_evex_scalable(insn->attr) && !insn_vex_w_bit(insn) && + p == INAT_PFX_OPNDSZ) + insn->opnd_bytes = 2; if ((inat_must_evex(insn->attr) && !insn_is_evex(insn)) || (!inat_accept_vex(insn->attr) && !inat_is_group(insn->attr))) { diff --git a/arch/x86/tools/gen-insn-attr-x86.awk b/arch/x86/tools/gen-insn-attr-x86.awk index 3f43aa7d8fefb..5770c8097f320 100644 --- a/arch/x86/tools/gen-insn-attr-x86.awk +++ b/arch/x86/tools/gen-insn-attr-x86.awk @@ -83,6 +83,8 @@ BEGIN { vexonly_expr = "\\(v\\)" # All opcodes with (ev) superscript supports *only* EVEX prefix evexonly_expr = "\\(ev\\)" + # (es) is the same as (ev) but also "SCALABLE" i.e. W and pp determine operand size + evex_scalable_expr = "\\(es\\)" prefix_expr = "\\(Prefix\\)" prefix_num["Operand-Size"] = "INAT_PFX_OPNDSZ" @@ -332,6 +334,8 @@ function convert_operands(count,opnd, i,j,imm,mod) # check VEX codes if (match(ext, evexonly_expr)) flags = add_flags(flags, "INAT_VEXOK | INAT_EVEXONLY") + else if (match(ext, evex_scalable_expr)) + flags = add_flags(flags, "INAT_VEXOK | INAT_EVEXONLY | INAT_EVEX_SCALABLE") else if (match(ext, vexonly_expr)) flags = add_flags(flags, "INAT_VEXOK | INAT_VEXONLY") else if (match(ext, vexok_expr) || match(opcode, vexok_opcode_expr)) diff --git a/tools/arch/x86/include/asm/inat.h b/tools/arch/x86/include/asm/inat.h index 2e65312cae524..253690eb3c268 100644 --- a/tools/arch/x86/include/asm/inat.h +++ b/tools/arch/x86/include/asm/inat.h @@ -81,6 +81,7 @@ #define INAT_EVEXONLY (1 << (INAT_FLAG_OFFS + 7)) #define INAT_NO_REX2 (1 << (INAT_FLAG_OFFS + 8)) #define INAT_REX2_VARIANT (1 << (INAT_FLAG_OFFS + 9)) +#define INAT_EVEX_SCALABLE (1 << (INAT_FLAG_OFFS + 10)) /* Attribute making macros for attribute tables */ #define INAT_MAKE_PREFIX(pfx) (pfx << INAT_PFX_OFFS) #define INAT_MAKE_ESCAPE(esc) (esc << INAT_ESC_OFFS) @@ -236,4 +237,9 @@ static inline int inat_must_evex(insn_attr_t attr) { return attr & INAT_EVEXONLY; } + +static inline int inat_evex_scalable(insn_attr_t attr) +{ + return attr & INAT_EVEX_SCALABLE; +} #endif diff --git a/tools/arch/x86/include/asm/insn.h b/tools/arch/x86/include/asm/insn.h index 1a7e8fc4d75a9..0e5abd896ad42 100644 --- a/tools/arch/x86/include/asm/insn.h +++ b/tools/arch/x86/include/asm/insn.h @@ -215,6 +215,13 @@ static inline insn_byte_t insn_vex_p_bits(struct insn *insn) return X86_VEX_P(insn->vex_prefix.bytes[2]); } +static inline insn_byte_t insn_vex_w_bit(struct insn *insn) +{ + if (insn->vex_prefix.nbytes < 3) + return 0; + return X86_VEX_W(insn->vex_prefix.bytes[2]); +} + /* Get the last prefix id from last prefix or VEX prefix */ static inline int insn_last_prefix_id(struct insn *insn) { diff --git a/tools/arch/x86/lib/insn.c b/tools/arch/x86/lib/insn.c index f761adeb8e8c5..a43b37346a22d 100644 --- a/tools/arch/x86/lib/insn.c +++ b/tools/arch/x86/lib/insn.c @@ -294,6 +294,10 @@ int insn_get_opcode(struct insn *insn) m = insn_vex_m_bits(insn); p = insn_vex_p_bits(insn); insn->attr = inat_get_avx_attribute(op, m, p); + /* SCALABLE EVEX uses p bits to encode operand size */ + if (inat_evex_scalable(insn->attr) && !insn_vex_w_bit(insn) && + p == INAT_PFX_OPNDSZ) + insn->opnd_bytes = 2; if ((inat_must_evex(insn->attr) && !insn_is_evex(insn)) || (!inat_accept_vex(insn->attr) && !inat_is_group(insn->attr))) { diff --git a/tools/arch/x86/tools/gen-insn-attr-x86.awk b/tools/arch/x86/tools/gen-insn-attr-x86.awk index 3f43aa7d8fefb..5770c8097f320 100644 --- a/tools/arch/x86/tools/gen-insn-attr-x86.awk +++ b/tools/arch/x86/tools/gen-insn-attr-x86.awk @@ -83,6 +83,8 @@ BEGIN { vexonly_expr = "\\(v\\)" # All opcodes with (ev) superscript supports *only* EVEX prefix evexonly_expr = "\\(ev\\)" + # (es) is the same as (ev) but also "SCALABLE" i.e. W and pp determine operand size + evex_scalable_expr = "\\(es\\)" prefix_expr = "\\(Prefix\\)" prefix_num["Operand-Size"] = "INAT_PFX_OPNDSZ" @@ -332,6 +334,8 @@ function convert_operands(count,opnd, i,j,imm,mod) # check VEX codes if (match(ext, evexonly_expr)) flags = add_flags(flags, "INAT_VEXOK | INAT_EVEXONLY") + else if (match(ext, evex_scalable_expr)) + flags = add_flags(flags, "INAT_VEXOK | INAT_EVEXONLY | INAT_EVEX_SCALABLE") else if (match(ext, vexonly_expr)) flags = add_flags(flags, "INAT_VEXOK | INAT_VEXONLY") else if (match(ext, vexok_expr) || match(opcode, vexok_opcode_expr)) From 690ca3a3067f760bef92ca5db1c42490498ab5de Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 2 May 2024 13:58:51 +0300 Subject: [PATCH 0964/1702] x86/insn: Add support for APX EVEX instructions to the opcode map To support APX functionality, the EVEX prefix is used to: - promote legacy instructions - promote VEX instructions - add new instructions Promoted VEX instructions require no extra annotation because the opcodes do not change and the permissive nature of the instruction decoder already allows them to have an EVEX prefix. Promoted legacy instructions and new instructions are placed in map 4 which has not been used before. Create a new table for map 4 and add APX instructions. Annotate SCALABLE instructions with "(es)" - refer to patch "x86/insn: Add support for APX EVEX to the instruction decoder logic". SCALABLE instructions must be represented in both no-prefix (NP) and 66 prefix forms. Signed-off-by: Adrian Hunter Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20240502105853.5338-9-adrian.hunter@intel.com --- arch/x86/lib/x86-opcode-map.txt | 93 +++++++++++++++++++++++++++ tools/arch/x86/lib/x86-opcode-map.txt | 93 +++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) diff --git a/arch/x86/lib/x86-opcode-map.txt b/arch/x86/lib/x86-opcode-map.txt index 240ef714b64fa..caedb3ef6688f 100644 --- a/arch/x86/lib/x86-opcode-map.txt +++ b/arch/x86/lib/x86-opcode-map.txt @@ -23,6 +23,7 @@ # # AVX Superscripts # (ev): this opcode requires EVEX prefix. +# (es): this opcode requires EVEX prefix and is SCALABALE. # (evo): this opcode is changed by EVEX prefix (EVEX opcode) # (v): this opcode requires VEX prefix. # (v1): this opcode only supports 128bit VEX. @@ -929,6 +930,98 @@ df: VAESKEYGEN Vdq,Wdq,Ib (66),(v1) f0: RORX Gy,Ey,Ib (F2),(v) | HRESET Gv,Ib (F3),(000),(11B) EndTable +Table: EVEX map 4 +Referrer: +AVXcode: 4 +00: ADD Eb,Gb (ev) +01: ADD Ev,Gv (es) | ADD Ev,Gv (66),(es) +02: ADD Gb,Eb (ev) +03: ADD Gv,Ev (es) | ADD Gv,Ev (66),(es) +08: OR Eb,Gb (ev) +09: OR Ev,Gv (es) | OR Ev,Gv (66),(es) +0a: OR Gb,Eb (ev) +0b: OR Gv,Ev (es) | OR Gv,Ev (66),(es) +10: ADC Eb,Gb (ev) +11: ADC Ev,Gv (es) | ADC Ev,Gv (66),(es) +12: ADC Gb,Eb (ev) +13: ADC Gv,Ev (es) | ADC Gv,Ev (66),(es) +18: SBB Eb,Gb (ev) +19: SBB Ev,Gv (es) | SBB Ev,Gv (66),(es) +1a: SBB Gb,Eb (ev) +1b: SBB Gv,Ev (es) | SBB Gv,Ev (66),(es) +20: AND Eb,Gb (ev) +21: AND Ev,Gv (es) | AND Ev,Gv (66),(es) +22: AND Gb,Eb (ev) +23: AND Gv,Ev (es) | AND Gv,Ev (66),(es) +24: SHLD Ev,Gv,Ib (es) | SHLD Ev,Gv,Ib (66),(es) +28: SUB Eb,Gb (ev) +29: SUB Ev,Gv (es) | SUB Ev,Gv (66),(es) +2a: SUB Gb,Eb (ev) +2b: SUB Gv,Ev (es) | SUB Gv,Ev (66),(es) +2c: SHRD Ev,Gv,Ib (es) | SHRD Ev,Gv,Ib (66),(es) +30: XOR Eb,Gb (ev) +31: XOR Ev,Gv (es) | XOR Ev,Gv (66),(es) +32: XOR Gb,Eb (ev) +33: XOR Gv,Ev (es) | XOR Gv,Ev (66),(es) +# CCMPSCC instructions are: CCOMB, CCOMBE, CCOMF, CCOML, CCOMLE, CCOMNB, CCOMNBE, CCOMNL, CCOMNLE, +# CCOMNO, CCOMNS, CCOMNZ, CCOMO, CCOMS, CCOMT, CCOMZ +38: CCMPSCC Eb,Gb (ev) +39: CCMPSCC Ev,Gv (es) | CCMPSCC Ev,Gv (66),(es) +3a: CCMPSCC Gv,Ev (ev) +3b: CCMPSCC Gv,Ev (es) | CCMPSCC Gv,Ev (66),(es) +40: CMOVO Gv,Ev (es) | CMOVO Gv,Ev (66),(es) | CFCMOVO Ev,Ev (es) | CFCMOVO Ev,Ev (66),(es) | SETO Eb (F2),(ev) +41: CMOVNO Gv,Ev (es) | CMOVNO Gv,Ev (66),(es) | CFCMOVNO Ev,Ev (es) | CFCMOVNO Ev,Ev (66),(es) | SETNO Eb (F2),(ev) +42: CMOVB Gv,Ev (es) | CMOVB Gv,Ev (66),(es) | CFCMOVB Ev,Ev (es) | CFCMOVB Ev,Ev (66),(es) | SETB Eb (F2),(ev) +43: CMOVNB Gv,Ev (es) | CMOVNB Gv,Ev (66),(es) | CFCMOVNB Ev,Ev (es) | CFCMOVNB Ev,Ev (66),(es) | SETNB Eb (F2),(ev) +44: CMOVZ Gv,Ev (es) | CMOVZ Gv,Ev (66),(es) | CFCMOVZ Ev,Ev (es) | CFCMOVZ Ev,Ev (66),(es) | SETZ Eb (F2),(ev) +45: CMOVNZ Gv,Ev (es) | CMOVNZ Gv,Ev (66),(es) | CFCMOVNZ Ev,Ev (es) | CFCMOVNZ Ev,Ev (66),(es) | SETNZ Eb (F2),(ev) +46: CMOVBE Gv,Ev (es) | CMOVBE Gv,Ev (66),(es) | CFCMOVBE Ev,Ev (es) | CFCMOVBE Ev,Ev (66),(es) | SETBE Eb (F2),(ev) +47: CMOVNBE Gv,Ev (es) | CMOVNBE Gv,Ev (66),(es) | CFCMOVNBE Ev,Ev (es) | CFCMOVNBE Ev,Ev (66),(es) | SETNBE Eb (F2),(ev) +48: CMOVS Gv,Ev (es) | CMOVS Gv,Ev (66),(es) | CFCMOVS Ev,Ev (es) | CFCMOVS Ev,Ev (66),(es) | SETS Eb (F2),(ev) +49: CMOVNS Gv,Ev (es) | CMOVNS Gv,Ev (66),(es) | CFCMOVNS Ev,Ev (es) | CFCMOVNS Ev,Ev (66),(es) | SETNS Eb (F2),(ev) +4a: CMOVP Gv,Ev (es) | CMOVP Gv,Ev (66),(es) | CFCMOVP Ev,Ev (es) | CFCMOVP Ev,Ev (66),(es) | SETP Eb (F2),(ev) +4b: CMOVNP Gv,Ev (es) | CMOVNP Gv,Ev (66),(es) | CFCMOVNP Ev,Ev (es) | CFCMOVNP Ev,Ev (66),(es) | SETNP Eb (F2),(ev) +4c: CMOVL Gv,Ev (es) | CMOVL Gv,Ev (66),(es) | CFCMOVL Ev,Ev (es) | CFCMOVL Ev,Ev (66),(es) | SETL Eb (F2),(ev) +4d: CMOVNL Gv,Ev (es) | CMOVNL Gv,Ev (66),(es) | CFCMOVNL Ev,Ev (es) | CFCMOVNL Ev,Ev (66),(es) | SETNL Eb (F2),(ev) +4e: CMOVLE Gv,Ev (es) | CMOVLE Gv,Ev (66),(es) | CFCMOVLE Ev,Ev (es) | CFCMOVLE Ev,Ev (66),(es) | SETLE Eb (F2),(ev) +4f: CMOVNLE Gv,Ev (es) | CMOVNLE Gv,Ev (66),(es) | CFCMOVNLE Ev,Ev (es) | CFCMOVNLE Ev,Ev (66),(es) | SETNLE Eb (F2),(ev) +60: MOVBE Gv,Ev (es) | MOVBE Gv,Ev (66),(es) +61: MOVBE Ev,Gv (es) | MOVBE Ev,Gv (66),(es) +65: WRUSSD Md,Gd (66),(ev) | WRUSSQ Mq,Gq (66),(ev) +66: ADCX Gy,Ey (66),(ev) | ADOX Gy,Ey (F3),(ev) | WRSSD Md,Gd (ev) | WRSSQ Mq,Gq (66),(ev) +69: IMUL Gv,Ev,Iz (es) | IMUL Gv,Ev,Iz (66),(es) +6b: IMUL Gv,Ev,Ib (es) | IMUL Gv,Ev,Ib (66),(es) +80: Grp1 Eb,Ib (1A),(ev) +81: Grp1 Ev,Iz (1A),(es) +83: Grp1 Ev,Ib (1A),(es) +# CTESTSCC instructions are: CTESTB, CTESTBE, CTESTF, CTESTL, CTESTLE, CTESTNB, CTESTNBE, CTESTNL, +# CTESTNLE, CTESTNO, CTESTNS, CTESTNZ, CTESTO, CTESTS, CTESTT, CTESTZ +84: CTESTSCC (ev) +85: CTESTSCC (es) | CTESTSCC (66),(es) +88: POPCNT Gv,Ev (es) | POPCNT Gv,Ev (66),(es) +8f: POP2 Bq,Rq (000),(11B),(ev) +a5: SHLD Ev,Gv,CL (es) | SHLD Ev,Gv,CL (66),(es) +ad: SHRD Ev,Gv,CL (es) | SHRD Ev,Gv,CL (66),(es) +af: IMUL Gv,Ev (es) | IMUL Gv,Ev (66),(es) +c0: Grp2 Eb,Ib (1A),(ev) +c1: Grp2 Ev,Ib (1A),(es) +d0: Grp2 Eb,1 (1A),(ev) +d1: Grp2 Ev,1 (1A),(es) +d2: Grp2 Eb,CL (1A),(ev) +d3: Grp2 Ev,CL (1A),(es) +f0: CRC32 Gy,Eb (es) | INVEPT Gq,Mdq (F3),(ev) +f1: CRC32 Gy,Ey (es) | CRC32 Gy,Ey (66),(es) | INVVPID Gy,Mdq (F3),(ev) +f2: INVPCID Gy,Mdq (F3),(ev) +f4: TZCNT Gv,Ev (es) | TZCNT Gv,Ev (66),(es) +f5: LZCNT Gv,Ev (es) | LZCNT Gv,Ev (66),(es) +f6: Grp3_1 Eb (1A),(ev) +f7: Grp3_2 Ev (1A),(es) +f8: MOVDIR64B Gv,Mdqq (66),(ev) | ENQCMD Gv,Mdqq (F2),(ev) | ENQCMDS Gv,Mdqq (F3),(ev) | URDMSR Rq,Gq (F2),(11B),(ev) | UWRMSR Gq,Rq (F3),(11B),(ev) +f9: MOVDIRI My,Gy (ev) +fe: Grp4 (1A),(ev) +ff: Grp5 (1A),(es) | PUSH2 Bq,Rq (110),(11B),(ev) +EndTable + Table: EVEX map 5 Referrer: AVXcode: 5 diff --git a/tools/arch/x86/lib/x86-opcode-map.txt b/tools/arch/x86/lib/x86-opcode-map.txt index 240ef714b64fa..caedb3ef6688f 100644 --- a/tools/arch/x86/lib/x86-opcode-map.txt +++ b/tools/arch/x86/lib/x86-opcode-map.txt @@ -23,6 +23,7 @@ # # AVX Superscripts # (ev): this opcode requires EVEX prefix. +# (es): this opcode requires EVEX prefix and is SCALABALE. # (evo): this opcode is changed by EVEX prefix (EVEX opcode) # (v): this opcode requires VEX prefix. # (v1): this opcode only supports 128bit VEX. @@ -929,6 +930,98 @@ df: VAESKEYGEN Vdq,Wdq,Ib (66),(v1) f0: RORX Gy,Ey,Ib (F2),(v) | HRESET Gv,Ib (F3),(000),(11B) EndTable +Table: EVEX map 4 +Referrer: +AVXcode: 4 +00: ADD Eb,Gb (ev) +01: ADD Ev,Gv (es) | ADD Ev,Gv (66),(es) +02: ADD Gb,Eb (ev) +03: ADD Gv,Ev (es) | ADD Gv,Ev (66),(es) +08: OR Eb,Gb (ev) +09: OR Ev,Gv (es) | OR Ev,Gv (66),(es) +0a: OR Gb,Eb (ev) +0b: OR Gv,Ev (es) | OR Gv,Ev (66),(es) +10: ADC Eb,Gb (ev) +11: ADC Ev,Gv (es) | ADC Ev,Gv (66),(es) +12: ADC Gb,Eb (ev) +13: ADC Gv,Ev (es) | ADC Gv,Ev (66),(es) +18: SBB Eb,Gb (ev) +19: SBB Ev,Gv (es) | SBB Ev,Gv (66),(es) +1a: SBB Gb,Eb (ev) +1b: SBB Gv,Ev (es) | SBB Gv,Ev (66),(es) +20: AND Eb,Gb (ev) +21: AND Ev,Gv (es) | AND Ev,Gv (66),(es) +22: AND Gb,Eb (ev) +23: AND Gv,Ev (es) | AND Gv,Ev (66),(es) +24: SHLD Ev,Gv,Ib (es) | SHLD Ev,Gv,Ib (66),(es) +28: SUB Eb,Gb (ev) +29: SUB Ev,Gv (es) | SUB Ev,Gv (66),(es) +2a: SUB Gb,Eb (ev) +2b: SUB Gv,Ev (es) | SUB Gv,Ev (66),(es) +2c: SHRD Ev,Gv,Ib (es) | SHRD Ev,Gv,Ib (66),(es) +30: XOR Eb,Gb (ev) +31: XOR Ev,Gv (es) | XOR Ev,Gv (66),(es) +32: XOR Gb,Eb (ev) +33: XOR Gv,Ev (es) | XOR Gv,Ev (66),(es) +# CCMPSCC instructions are: CCOMB, CCOMBE, CCOMF, CCOML, CCOMLE, CCOMNB, CCOMNBE, CCOMNL, CCOMNLE, +# CCOMNO, CCOMNS, CCOMNZ, CCOMO, CCOMS, CCOMT, CCOMZ +38: CCMPSCC Eb,Gb (ev) +39: CCMPSCC Ev,Gv (es) | CCMPSCC Ev,Gv (66),(es) +3a: CCMPSCC Gv,Ev (ev) +3b: CCMPSCC Gv,Ev (es) | CCMPSCC Gv,Ev (66),(es) +40: CMOVO Gv,Ev (es) | CMOVO Gv,Ev (66),(es) | CFCMOVO Ev,Ev (es) | CFCMOVO Ev,Ev (66),(es) | SETO Eb (F2),(ev) +41: CMOVNO Gv,Ev (es) | CMOVNO Gv,Ev (66),(es) | CFCMOVNO Ev,Ev (es) | CFCMOVNO Ev,Ev (66),(es) | SETNO Eb (F2),(ev) +42: CMOVB Gv,Ev (es) | CMOVB Gv,Ev (66),(es) | CFCMOVB Ev,Ev (es) | CFCMOVB Ev,Ev (66),(es) | SETB Eb (F2),(ev) +43: CMOVNB Gv,Ev (es) | CMOVNB Gv,Ev (66),(es) | CFCMOVNB Ev,Ev (es) | CFCMOVNB Ev,Ev (66),(es) | SETNB Eb (F2),(ev) +44: CMOVZ Gv,Ev (es) | CMOVZ Gv,Ev (66),(es) | CFCMOVZ Ev,Ev (es) | CFCMOVZ Ev,Ev (66),(es) | SETZ Eb (F2),(ev) +45: CMOVNZ Gv,Ev (es) | CMOVNZ Gv,Ev (66),(es) | CFCMOVNZ Ev,Ev (es) | CFCMOVNZ Ev,Ev (66),(es) | SETNZ Eb (F2),(ev) +46: CMOVBE Gv,Ev (es) | CMOVBE Gv,Ev (66),(es) | CFCMOVBE Ev,Ev (es) | CFCMOVBE Ev,Ev (66),(es) | SETBE Eb (F2),(ev) +47: CMOVNBE Gv,Ev (es) | CMOVNBE Gv,Ev (66),(es) | CFCMOVNBE Ev,Ev (es) | CFCMOVNBE Ev,Ev (66),(es) | SETNBE Eb (F2),(ev) +48: CMOVS Gv,Ev (es) | CMOVS Gv,Ev (66),(es) | CFCMOVS Ev,Ev (es) | CFCMOVS Ev,Ev (66),(es) | SETS Eb (F2),(ev) +49: CMOVNS Gv,Ev (es) | CMOVNS Gv,Ev (66),(es) | CFCMOVNS Ev,Ev (es) | CFCMOVNS Ev,Ev (66),(es) | SETNS Eb (F2),(ev) +4a: CMOVP Gv,Ev (es) | CMOVP Gv,Ev (66),(es) | CFCMOVP Ev,Ev (es) | CFCMOVP Ev,Ev (66),(es) | SETP Eb (F2),(ev) +4b: CMOVNP Gv,Ev (es) | CMOVNP Gv,Ev (66),(es) | CFCMOVNP Ev,Ev (es) | CFCMOVNP Ev,Ev (66),(es) | SETNP Eb (F2),(ev) +4c: CMOVL Gv,Ev (es) | CMOVL Gv,Ev (66),(es) | CFCMOVL Ev,Ev (es) | CFCMOVL Ev,Ev (66),(es) | SETL Eb (F2),(ev) +4d: CMOVNL Gv,Ev (es) | CMOVNL Gv,Ev (66),(es) | CFCMOVNL Ev,Ev (es) | CFCMOVNL Ev,Ev (66),(es) | SETNL Eb (F2),(ev) +4e: CMOVLE Gv,Ev (es) | CMOVLE Gv,Ev (66),(es) | CFCMOVLE Ev,Ev (es) | CFCMOVLE Ev,Ev (66),(es) | SETLE Eb (F2),(ev) +4f: CMOVNLE Gv,Ev (es) | CMOVNLE Gv,Ev (66),(es) | CFCMOVNLE Ev,Ev (es) | CFCMOVNLE Ev,Ev (66),(es) | SETNLE Eb (F2),(ev) +60: MOVBE Gv,Ev (es) | MOVBE Gv,Ev (66),(es) +61: MOVBE Ev,Gv (es) | MOVBE Ev,Gv (66),(es) +65: WRUSSD Md,Gd (66),(ev) | WRUSSQ Mq,Gq (66),(ev) +66: ADCX Gy,Ey (66),(ev) | ADOX Gy,Ey (F3),(ev) | WRSSD Md,Gd (ev) | WRSSQ Mq,Gq (66),(ev) +69: IMUL Gv,Ev,Iz (es) | IMUL Gv,Ev,Iz (66),(es) +6b: IMUL Gv,Ev,Ib (es) | IMUL Gv,Ev,Ib (66),(es) +80: Grp1 Eb,Ib (1A),(ev) +81: Grp1 Ev,Iz (1A),(es) +83: Grp1 Ev,Ib (1A),(es) +# CTESTSCC instructions are: CTESTB, CTESTBE, CTESTF, CTESTL, CTESTLE, CTESTNB, CTESTNBE, CTESTNL, +# CTESTNLE, CTESTNO, CTESTNS, CTESTNZ, CTESTO, CTESTS, CTESTT, CTESTZ +84: CTESTSCC (ev) +85: CTESTSCC (es) | CTESTSCC (66),(es) +88: POPCNT Gv,Ev (es) | POPCNT Gv,Ev (66),(es) +8f: POP2 Bq,Rq (000),(11B),(ev) +a5: SHLD Ev,Gv,CL (es) | SHLD Ev,Gv,CL (66),(es) +ad: SHRD Ev,Gv,CL (es) | SHRD Ev,Gv,CL (66),(es) +af: IMUL Gv,Ev (es) | IMUL Gv,Ev (66),(es) +c0: Grp2 Eb,Ib (1A),(ev) +c1: Grp2 Ev,Ib (1A),(es) +d0: Grp2 Eb,1 (1A),(ev) +d1: Grp2 Ev,1 (1A),(es) +d2: Grp2 Eb,CL (1A),(ev) +d3: Grp2 Ev,CL (1A),(es) +f0: CRC32 Gy,Eb (es) | INVEPT Gq,Mdq (F3),(ev) +f1: CRC32 Gy,Ey (es) | CRC32 Gy,Ey (66),(es) | INVVPID Gy,Mdq (F3),(ev) +f2: INVPCID Gy,Mdq (F3),(ev) +f4: TZCNT Gv,Ev (es) | TZCNT Gv,Ev (66),(es) +f5: LZCNT Gv,Ev (es) | LZCNT Gv,Ev (66),(es) +f6: Grp3_1 Eb (1A),(ev) +f7: Grp3_2 Ev (1A),(es) +f8: MOVDIR64B Gv,Mdqq (66),(ev) | ENQCMD Gv,Mdqq (F2),(ev) | ENQCMDS Gv,Mdqq (F3),(ev) | URDMSR Rq,Gq (F2),(11B),(ev) | UWRMSR Gq,Rq (F3),(11B),(ev) +f9: MOVDIRI My,Gy (ev) +fe: Grp4 (1A),(ev) +ff: Grp5 (1A),(es) | PUSH2 Bq,Rq (110),(11B),(ev) +EndTable + Table: EVEX map 5 Referrer: AVXcode: 5 From a7c79cf3e4eb3c869148c81d2a7dc684bc8eeb07 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 27 Apr 2024 18:16:38 +0900 Subject: [PATCH 0965/1702] kconfig: remove SYMBOL_NO_WRITE flag This flag is set to symbols that are not intended to be written to the .config file. Since commit b75b0a819af9 ("kconfig: change defconfig_list option to environment variable"), SYMBOL_NO_WRITE is only set to choices. Therefore, (sym->flags & SYMBOL_NO_WRITE) is equivalent to sym_is_choice(sym). This flag is no longer necessary. Signed-off-by: Masahiro Yamada --- scripts/kconfig/confdata.c | 4 ++-- scripts/kconfig/expr.h | 1 - scripts/kconfig/gconf.c | 2 -- scripts/kconfig/parser.y | 2 +- scripts/kconfig/symbol.c | 3 +-- 5 files changed, 4 insertions(+), 8 deletions(-) diff --git a/scripts/kconfig/confdata.c b/scripts/kconfig/confdata.c index bcce87658998c..5caec434e6f4a 100644 --- a/scripts/kconfig/confdata.c +++ b/scripts/kconfig/confdata.c @@ -502,7 +502,7 @@ int conf_read(const char *name) for_all_symbols(sym) { sym_calc_value(sym); - if (sym_is_choice(sym) || (sym->flags & SYMBOL_NO_WRITE)) + if (sym_is_choice(sym)) continue; if (sym_has_value(sym) && (sym->flags & SYMBOL_WRITE)) { /* check that calculated value agrees with saved value */ @@ -1007,7 +1007,7 @@ static int conf_touch_deps(void) for_all_symbols(sym) { sym_calc_value(sym); - if ((sym->flags & SYMBOL_NO_WRITE) || !sym->name) + if (sym_is_choice(sym)) continue; if (sym->flags & SYMBOL_WRITE) { if (sym->flags & SYMBOL_DEF_AUTO) { diff --git a/scripts/kconfig/expr.h b/scripts/kconfig/expr.h index f646a98de0063..d965e427753eb 100644 --- a/scripts/kconfig/expr.h +++ b/scripts/kconfig/expr.h @@ -135,7 +135,6 @@ struct symbol { #define SYMBOL_WRITE 0x0200 /* write symbol to file (KCONFIG_CONFIG) */ #define SYMBOL_CHANGED 0x0400 /* ? */ #define SYMBOL_WRITTEN 0x0800 /* track info to avoid double-write to .config */ -#define SYMBOL_NO_WRITE 0x1000 /* Symbol for internal use only; it will not be written */ #define SYMBOL_CHECKED 0x2000 /* used during dependency checking */ #define SYMBOL_WARNED 0x8000 /* warning has been issued */ diff --git a/scripts/kconfig/gconf.c b/scripts/kconfig/gconf.c index 13e2449ac83fa..67a27c497c403 100644 --- a/scripts/kconfig/gconf.c +++ b/scripts/kconfig/gconf.c @@ -91,8 +91,6 @@ static const char *dbg_sym_flags(int val) strcat(buf, "write/"); if (val & SYMBOL_CHANGED) strcat(buf, "changed/"); - if (val & SYMBOL_NO_WRITE) - strcat(buf, "no_write/"); buf[strlen(buf) - 1] = '\0'; diff --git a/scripts/kconfig/parser.y b/scripts/kconfig/parser.y index 69dc0c098acbd..613fa8c9c2d04 100644 --- a/scripts/kconfig/parser.y +++ b/scripts/kconfig/parser.y @@ -222,7 +222,7 @@ config_option: T_MODULES T_EOL choice: T_CHOICE T_EOL { struct symbol *sym = sym_lookup(NULL, 0); - sym->flags |= SYMBOL_NO_WRITE; + menu_add_entry(sym); menu_add_expr(P_CHOICE, NULL, NULL); printd(DEBUG_PARSE, "%s:%d:choice\n", cur_filename, cur_lineno); diff --git a/scripts/kconfig/symbol.c b/scripts/kconfig/symbol.c index 8b34992ba5ed9..b909c64f3bac8 100644 --- a/scripts/kconfig/symbol.c +++ b/scripts/kconfig/symbol.c @@ -466,10 +466,9 @@ void sym_calc_value(struct symbol *sym) if (sym->flags & SYMBOL_CHANGED) sym_set_changed(choice_sym); } - } - if (sym->flags & SYMBOL_NO_WRITE) sym->flags &= ~SYMBOL_WRITE; + } if (sym->flags & SYMBOL_NEED_SET_CHOICE_VALUES) set_all_choice_values(sym); From b957df3b858d16ba3d4291233569bba09cfd08c7 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 27 Apr 2024 23:54:59 +0900 Subject: [PATCH 0966/1702] arch: use $(obj)/ instead of $(src)/ for preprocessed linker scripts These are generated files. Prefix them with $(obj)/ instead of $(src)/. Signed-off-by: Masahiro Yamada Acked-by: Helge Deller Reviewed-by: Nicolas Schier --- arch/arm64/kernel/vdso32/Makefile | 2 +- arch/csky/kernel/vdso/Makefile | 2 +- arch/parisc/kernel/vdso32/Makefile | 2 +- arch/parisc/kernel/vdso64/Makefile | 2 +- arch/powerpc/kernel/vdso/Makefile | 4 ++-- arch/s390/kernel/vdso32/Makefile | 2 +- arch/s390/kernel/vdso64/Makefile | 2 +- arch/sh/kernel/vsyscall/Makefile | 4 ++-- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/arm64/kernel/vdso32/Makefile b/arch/arm64/kernel/vdso32/Makefile index f5f80fdce0fe7..cc4508c604b28 100644 --- a/arch/arm64/kernel/vdso32/Makefile +++ b/arch/arm64/kernel/vdso32/Makefile @@ -136,7 +136,7 @@ $(obj)/vdso32.so.dbg: $(obj)/vdso.so.raw $(obj)/$(munge) FORCE $(call if_changed,vdsomunge) # Link rule for the .so file, .lds has to be first -$(obj)/vdso.so.raw: $(src)/vdso.lds $(obj-vdso) FORCE +$(obj)/vdso.so.raw: $(obj)/vdso.lds $(obj-vdso) FORCE $(call if_changed,vdsold_and_vdso_check) # Compilation rules for the vDSO sources diff --git a/arch/csky/kernel/vdso/Makefile b/arch/csky/kernel/vdso/Makefile index ddf784a62c113..24a01c22293ec 100644 --- a/arch/csky/kernel/vdso/Makefile +++ b/arch/csky/kernel/vdso/Makefile @@ -31,7 +31,7 @@ KCOV_INSTRUMENT := n $(obj)/vdso.o: $(obj)/vdso.so SYSCFLAGS_vdso.so.dbg = $(c_flags) -$(obj)/vdso.so.dbg: $(src)/vdso.lds $(obj-vdso) FORCE +$(obj)/vdso.so.dbg: $(obj)/vdso.lds $(obj-vdso) FORCE $(call if_changed,vdsold) SYSCFLAGS_vdso.so.dbg = -shared -s -Wl,-soname=linux-vdso.so.1 \ -Wl,--build-id=sha1 -Wl,--hash-style=both diff --git a/arch/parisc/kernel/vdso32/Makefile b/arch/parisc/kernel/vdso32/Makefile index e45d46bf46a23..60dc708a0f807 100644 --- a/arch/parisc/kernel/vdso32/Makefile +++ b/arch/parisc/kernel/vdso32/Makefile @@ -26,7 +26,7 @@ $(obj)/vdso32_wrapper.o : $(obj)/vdso32.so FORCE # Force dependency (incbin is bad) # link rule for the .so file, .lds has to be first -$(obj)/vdso32.so: $(src)/vdso32.lds $(obj-vdso32) $(VDSO_LIBGCC) FORCE +$(obj)/vdso32.so: $(obj)/vdso32.lds $(obj-vdso32) $(VDSO_LIBGCC) FORCE $(call if_changed,vdso32ld) # assembly rules for the .S files diff --git a/arch/parisc/kernel/vdso64/Makefile b/arch/parisc/kernel/vdso64/Makefile index f3d6045793f4c..0982f3099ae25 100644 --- a/arch/parisc/kernel/vdso64/Makefile +++ b/arch/parisc/kernel/vdso64/Makefile @@ -26,7 +26,7 @@ $(obj)/vdso64_wrapper.o : $(obj)/vdso64.so FORCE # Force dependency (incbin is bad) # link rule for the .so file, .lds has to be first -$(obj)/vdso64.so: $(src)/vdso64.lds $(obj-vdso64) $(VDSO_LIBGCC) FORCE +$(obj)/vdso64.so: $(obj)/vdso64.lds $(obj-vdso64) $(VDSO_LIBGCC) FORCE $(call if_changed,vdso64ld) # assembly rules for the .S files diff --git a/arch/powerpc/kernel/vdso/Makefile b/arch/powerpc/kernel/vdso/Makefile index 1b93655c2857e..a14eab3669939 100644 --- a/arch/powerpc/kernel/vdso/Makefile +++ b/arch/powerpc/kernel/vdso/Makefile @@ -74,9 +74,9 @@ targets += vdso64.lds CPPFLAGS_vdso64.lds += -P -C # link rule for the .so file, .lds has to be first -$(obj)/vdso32.so.dbg: $(src)/vdso32.lds $(obj-vdso32) $(obj)/vgettimeofday-32.o FORCE +$(obj)/vdso32.so.dbg: $(obj)/vdso32.lds $(obj-vdso32) $(obj)/vgettimeofday-32.o FORCE $(call if_changed,vdso32ld_and_check) -$(obj)/vdso64.so.dbg: $(src)/vdso64.lds $(obj-vdso64) $(obj)/vgettimeofday-64.o FORCE +$(obj)/vdso64.so.dbg: $(obj)/vdso64.lds $(obj-vdso64) $(obj)/vgettimeofday-64.o FORCE $(call if_changed,vdso64ld_and_check) # assembly rules for the .S files diff --git a/arch/s390/kernel/vdso32/Makefile b/arch/s390/kernel/vdso32/Makefile index b12a274cbb473..70e9949c26128 100644 --- a/arch/s390/kernel/vdso32/Makefile +++ b/arch/s390/kernel/vdso32/Makefile @@ -44,7 +44,7 @@ $(obj)/vdso32_wrapper.o : $(obj)/vdso32.so quiet_cmd_vdso_and_check = VDSO $@ cmd_vdso_and_check = $(cmd_ld); $(cmd_vdso_check) -$(obj)/vdso32.so.dbg: $(src)/vdso32.lds $(obj-vdso32) FORCE +$(obj)/vdso32.so.dbg: $(obj)/vdso32.lds $(obj-vdso32) FORCE $(call if_changed,vdso_and_check) # strip rule for the .so file diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile index ef98327260972..2b3617b6d1624 100644 --- a/arch/s390/kernel/vdso64/Makefile +++ b/arch/s390/kernel/vdso64/Makefile @@ -50,7 +50,7 @@ quiet_cmd_vdso_and_check = VDSO $@ cmd_vdso_and_check = $(cmd_ld); $(cmd_vdso_check) # link rule for the .so file, .lds has to be first -$(obj)/vdso64.so.dbg: $(src)/vdso64.lds $(obj-vdso64) $(obj-cvdso64) FORCE +$(obj)/vdso64.so.dbg: $(obj)/vdso64.lds $(obj-vdso64) $(obj-cvdso64) FORCE $(call if_changed,vdso_and_check) # strip rule for the .so file diff --git a/arch/sh/kernel/vsyscall/Makefile b/arch/sh/kernel/vsyscall/Makefile index 118744d349e21..cb4f0bb80c38e 100644 --- a/arch/sh/kernel/vsyscall/Makefile +++ b/arch/sh/kernel/vsyscall/Makefile @@ -19,14 +19,14 @@ vsyscall-flags = -shared -s -Wl,-soname=linux-gate.so.1 -Wl,--hash-style=sysv SYSCFLAGS_vsyscall-trapa.so = $(vsyscall-flags) $(obj)/vsyscall-trapa.so: \ -$(obj)/vsyscall-%.so: $(src)/vsyscall.lds $(obj)/vsyscall-%.o FORCE +$(obj)/vsyscall-%.so: $(obj)/vsyscall.lds $(obj)/vsyscall-%.o FORCE $(call if_changed,syscall) # We also create a special relocatable object that should mirror the symbol # table and layout of the linked DSO. With ld -R we can then refer to # these symbols in the kernel code rather than hand-coded addresses. SYSCFLAGS_vsyscall-dummy.o = -r -$(obj)/vsyscall-dummy.o: $(src)/vsyscall.lds \ +$(obj)/vsyscall-dummy.o: $(obj)/vsyscall.lds \ $(obj)/vsyscall-trapa.o $(obj)/vsyscall-note.o FORCE $(call if_changed,syscall) From 626c5acf39ba929a0d82d37e72072e21492a958e Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Thu, 2 May 2024 15:21:14 +0530 Subject: [PATCH 0967/1702] perf/x86/rapl: Rename 'maxdie' to nr_rapl_pmu and 'dieid' to rapl_pmu_idx AMD CPUs have the scope of RAPL energy-pkg event as package, whereas Intel Cascade Lake CPUs have the scope as die. To account for the difference in the energy-pkg event scope between AMD and Intel CPUs, give more generic and semantically correct names to the maxdie and dieid variables. No functional change. Signed-off-by: Dhananjay Ugwekar Signed-off-by: Ingo Molnar Tested-by: K Prateek Nayak Link: https://lore.kernel.org/r/20240502095115.177713-2-Dhananjay.Ugwekar@amd.com --- arch/x86/events/rapl.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/x86/events/rapl.c b/arch/x86/events/rapl.c index ca5f687fa4200..46e6735855603 100644 --- a/arch/x86/events/rapl.c +++ b/arch/x86/events/rapl.c @@ -114,8 +114,8 @@ struct rapl_pmu { struct rapl_pmus { struct pmu pmu; - unsigned int maxdie; - struct rapl_pmu *pmus[] __counted_by(maxdie); + unsigned int nr_rapl_pmu; + struct rapl_pmu *pmus[] __counted_by(nr_rapl_pmu); }; enum rapl_unit_quirk { @@ -141,13 +141,13 @@ static struct perf_msr *rapl_msrs; static inline struct rapl_pmu *cpu_to_rapl_pmu(unsigned int cpu) { - unsigned int dieid = topology_logical_die_id(cpu); + unsigned int rapl_pmu_idx = topology_logical_die_id(cpu); /* * The unsigned check also catches the '-1' return value for non * existent mappings in the topology map. */ - return dieid < rapl_pmus->maxdie ? rapl_pmus->pmus[dieid] : NULL; + return rapl_pmu_idx < rapl_pmus->nr_rapl_pmu ? rapl_pmus->pmus[rapl_pmu_idx] : NULL; } static inline u64 rapl_read_counter(struct perf_event *event) @@ -658,7 +658,7 @@ static void cleanup_rapl_pmus(void) { int i; - for (i = 0; i < rapl_pmus->maxdie; i++) + for (i = 0; i < rapl_pmus->nr_rapl_pmu; i++) kfree(rapl_pmus->pmus[i]); kfree(rapl_pmus); } @@ -674,13 +674,13 @@ static const struct attribute_group *rapl_attr_update[] = { static int __init init_rapl_pmus(void) { - int maxdie = topology_max_packages() * topology_max_dies_per_package(); + int nr_rapl_pmu = topology_max_packages() * topology_max_dies_per_package(); - rapl_pmus = kzalloc(struct_size(rapl_pmus, pmus, maxdie), GFP_KERNEL); + rapl_pmus = kzalloc(struct_size(rapl_pmus, pmus, nr_rapl_pmu), GFP_KERNEL); if (!rapl_pmus) return -ENOMEM; - rapl_pmus->maxdie = maxdie; + rapl_pmus->nr_rapl_pmu = nr_rapl_pmu; rapl_pmus->pmu.attr_groups = rapl_attr_groups; rapl_pmus->pmu.attr_update = rapl_attr_update; rapl_pmus->pmu.task_ctx_nr = perf_invalid_context; From c93f261dfc395b1386320b2f0b160a6f34ed9ea5 Mon Sep 17 00:00:00 2001 From: Michael Kelley Date: Wed, 1 May 2024 08:16:51 -0700 Subject: [PATCH 0968/1702] Documentation/core-api: add swiotlb documentation There's currently no documentation for the swiotlb. Add documentation describing usage scenarios, the key APIs, and implementation details. Group the new documentation with other DMA-related documentation. Signed-off-by: Michael Kelley Reviewed-by: Bagas Sanjaya Reviewed-by: Petr Tesarik Signed-off-by: Christoph Hellwig --- Documentation/core-api/index.rst | 1 + Documentation/core-api/swiotlb.rst | 321 +++++++++++++++++++++++++++++ 2 files changed, 322 insertions(+) create mode 100644 Documentation/core-api/swiotlb.rst diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst index 7a3a08d81f111..89c5176657631 100644 --- a/Documentation/core-api/index.rst +++ b/Documentation/core-api/index.rst @@ -102,6 +102,7 @@ more memory-management documentation in Documentation/mm/index.rst. dma-api-howto dma-attributes dma-isa-lpc + swiotlb mm-api genalloc pin_user_pages diff --git a/Documentation/core-api/swiotlb.rst b/Documentation/core-api/swiotlb.rst new file mode 100644 index 0000000000000..5ad2c9ca85bcb --- /dev/null +++ b/Documentation/core-api/swiotlb.rst @@ -0,0 +1,321 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=============== +DMA and swiotlb +=============== + +swiotlb is a memory buffer allocator used by the Linux kernel DMA layer. It is +typically used when a device doing DMA can't directly access the target memory +buffer because of hardware limitations or other requirements. In such a case, +the DMA layer calls swiotlb to allocate a temporary memory buffer that conforms +to the limitations. The DMA is done to/from this temporary memory buffer, and +the CPU copies the data between the temporary buffer and the original target +memory buffer. This approach is generically called "bounce buffering", and the +temporary memory buffer is called a "bounce buffer". + +Device drivers don't interact directly with swiotlb. Instead, drivers inform +the DMA layer of the DMA attributes of the devices they are managing, and use +the normal DMA map, unmap, and sync APIs when programming a device to do DMA. +These APIs use the device DMA attributes and kernel-wide settings to determine +if bounce buffering is necessary. If so, the DMA layer manages the allocation, +freeing, and sync'ing of bounce buffers. Since the DMA attributes are per +device, some devices in a system may use bounce buffering while others do not. + +Because the CPU copies data between the bounce buffer and the original target +memory buffer, doing bounce buffering is slower than doing DMA directly to the +original memory buffer, and it consumes more CPU resources. So it is used only +when necessary for providing DMA functionality. + +Usage Scenarios +--------------- +swiotlb was originally created to handle DMA for devices with addressing +limitations. As physical memory sizes grew beyond 4 GiB, some devices could +only provide 32-bit DMA addresses. By allocating bounce buffer memory below +the 4 GiB line, these devices with addressing limitations could still work and +do DMA. + +More recently, Confidential Computing (CoCo) VMs have the guest VM's memory +encrypted by default, and the memory is not accessible by the host hypervisor +and VMM. For the host to do I/O on behalf of the guest, the I/O must be +directed to guest memory that is unencrypted. CoCo VMs set a kernel-wide option +to force all DMA I/O to use bounce buffers, and the bounce buffer memory is set +up as unencrypted. The host does DMA I/O to/from the bounce buffer memory, and +the Linux kernel DMA layer does "sync" operations to cause the CPU to copy the +data to/from the original target memory buffer. The CPU copying bridges between +the unencrypted and the encrypted memory. This use of bounce buffers allows +device drivers to "just work" in a CoCo VM, with no modifications +needed to handle the memory encryption complexity. + +Other edge case scenarios arise for bounce buffers. For example, when IOMMU +mappings are set up for a DMA operation to/from a device that is considered +"untrusted", the device should be given access only to the memory containing +the data being transferred. But if that memory occupies only part of an IOMMU +granule, other parts of the granule may contain unrelated kernel data. Since +IOMMU access control is per-granule, the untrusted device can gain access to +the unrelated kernel data. This problem is solved by bounce buffering the DMA +operation and ensuring that unused portions of the bounce buffers do not +contain any unrelated kernel data. + +Core Functionality +------------------ +The primary swiotlb APIs are swiotlb_tbl_map_single() and +swiotlb_tbl_unmap_single(). The "map" API allocates a bounce buffer of a +specified size in bytes and returns the physical address of the buffer. The +buffer memory is physically contiguous. The expectation is that the DMA layer +maps the physical memory address to a DMA address, and returns the DMA address +to the driver for programming into the device. If a DMA operation specifies +multiple memory buffer segments, a separate bounce buffer must be allocated for +each segment. swiotlb_tbl_map_single() always does a "sync" operation (i.e., a +CPU copy) to initialize the bounce buffer to match the contents of the original +buffer. + +swiotlb_tbl_unmap_single() does the reverse. If the DMA operation might have +updated the bounce buffer memory and DMA_ATTR_SKIP_CPU_SYNC is not set, the +unmap does a "sync" operation to cause a CPU copy of the data from the bounce +buffer back to the original buffer. Then the bounce buffer memory is freed. + +swiotlb also provides "sync" APIs that correspond to the dma_sync_*() APIs that +a driver may use when control of a buffer transitions between the CPU and the +device. The swiotlb "sync" APIs cause a CPU copy of the data between the +original buffer and the bounce buffer. Like the dma_sync_*() APIs, the swiotlb +"sync" APIs support doing a partial sync, where only a subset of the bounce +buffer is copied to/from the original buffer. + +Core Functionality Constraints +------------------------------ +The swiotlb map/unmap/sync APIs must operate without blocking, as they are +called by the corresponding DMA APIs which may run in contexts that cannot +block. Hence the default memory pool for swiotlb allocations must be +pre-allocated at boot time (but see Dynamic swiotlb below). Because swiotlb +allocations must be physically contiguous, the entire default memory pool is +allocated as a single contiguous block. + +The need to pre-allocate the default swiotlb pool creates a boot-time tradeoff. +The pool should be large enough to ensure that bounce buffer requests can +always be satisfied, as the non-blocking requirement means requests can't wait +for space to become available. But a large pool potentially wastes memory, as +this pre-allocated memory is not available for other uses in the system. The +tradeoff is particularly acute in CoCo VMs that use bounce buffers for all DMA +I/O. These VMs use a heuristic to set the default pool size to ~6% of memory, +with a max of 1 GiB, which has the potential to be very wasteful of memory. +Conversely, the heuristic might produce a size that is insufficient, depending +on the I/O patterns of the workload in the VM. The dynamic swiotlb feature +described below can help, but has limitations. Better management of the swiotlb +default memory pool size remains an open issue. + +A single allocation from swiotlb is limited to IO_TLB_SIZE * IO_TLB_SEGSIZE +bytes, which is 256 KiB with current definitions. When a device's DMA settings +are such that the device might use swiotlb, the maximum size of a DMA segment +must be limited to that 256 KiB. This value is communicated to higher-level +kernel code via dma_map_mapping_size() and swiotlb_max_mapping_size(). If the +higher-level code fails to account for this limit, it may make requests that +are too large for swiotlb, and get a "swiotlb full" error. + +A key device DMA setting is "min_align_mask", which is a power of 2 minus 1 +so that some number of low order bits are set, or it may be zero. swiotlb +allocations ensure these min_align_mask bits of the physical address of the +bounce buffer match the same bits in the address of the original buffer. When +min_align_mask is non-zero, it may produce an "alignment offset" in the address +of the bounce buffer that slightly reduces the maximum size of an allocation. +This potential alignment offset is reflected in the value returned by +swiotlb_max_mapping_size(), which can show up in places like +/sys/block//queue/max_sectors_kb. For example, if a device does not use +swiotlb, max_sectors_kb might be 512 KiB or larger. If a device might use +swiotlb, max_sectors_kb will be 256 KiB. When min_align_mask is non-zero, +max_sectors_kb might be even smaller, such as 252 KiB. + +swiotlb_tbl_map_single() also takes an "alloc_align_mask" parameter. This +parameter specifies the allocation of bounce buffer space must start at a +physical address with the alloc_align_mask bits set to zero. But the actual +bounce buffer might start at a larger address if min_align_mask is non-zero. +Hence there may be pre-padding space that is allocated prior to the start of +the bounce buffer. Similarly, the end of the bounce buffer is rounded up to an +alloc_align_mask boundary, potentially resulting in post-padding space. Any +pre-padding or post-padding space is not initialized by swiotlb code. The +"alloc_align_mask" parameter is used by IOMMU code when mapping for untrusted +devices. It is set to the granule size - 1 so that the bounce buffer is +allocated entirely from granules that are not used for any other purpose. + +Data structures concepts +------------------------ +Memory used for swiotlb bounce buffers is allocated from overall system memory +as one or more "pools". The default pool is allocated during system boot with a +default size of 64 MiB. The default pool size may be modified with the +"swiotlb=" kernel boot line parameter. The default size may also be adjusted +due to other conditions, such as running in a CoCo VM, as described above. If +CONFIG_SWIOTLB_DYNAMIC is enabled, additional pools may be allocated later in +the life of the system. Each pool must be a contiguous range of physical +memory. The default pool is allocated below the 4 GiB physical address line so +it works for devices that can only address 32-bits of physical memory (unless +architecture-specific code provides the SWIOTLB_ANY flag). In a CoCo VM, the +pool memory must be decrypted before swiotlb is used. + +Each pool is divided into "slots" of size IO_TLB_SIZE, which is 2 KiB with +current definitions. IO_TLB_SEGSIZE contiguous slots (128 slots) constitute +what might be called a "slot set". When a bounce buffer is allocated, it +occupies one or more contiguous slots. A slot is never shared by multiple +bounce buffers. Furthermore, a bounce buffer must be allocated from a single +slot set, which leads to the maximum bounce buffer size being IO_TLB_SIZE * +IO_TLB_SEGSIZE. Multiple smaller bounce buffers may co-exist in a single slot +set if the alignment and size constraints can be met. + +Slots are also grouped into "areas", with the constraint that a slot set exists +entirely in a single area. Each area has its own spin lock that must be held to +manipulate the slots in that area. The division into areas avoids contending +for a single global spin lock when swiotlb is heavily used, such as in a CoCo +VM. The number of areas defaults to the number of CPUs in the system for +maximum parallelism, but since an area can't be smaller than IO_TLB_SEGSIZE +slots, it might be necessary to assign multiple CPUs to the same area. The +number of areas can also be set via the "swiotlb=" kernel boot parameter. + +When allocating a bounce buffer, if the area associated with the calling CPU +does not have enough free space, areas associated with other CPUs are tried +sequentially. For each area tried, the area's spin lock must be obtained before +trying an allocation, so contention may occur if swiotlb is relatively busy +overall. But an allocation request does not fail unless all areas do not have +enough free space. + +IO_TLB_SIZE, IO_TLB_SEGSIZE, and the number of areas must all be powers of 2 as +the code uses shifting and bit masking to do many of the calculations. The +number of areas is rounded up to a power of 2 if necessary to meet this +requirement. + +The default pool is allocated with PAGE_SIZE alignment. If an alloc_align_mask +argument to swiotlb_tbl_map_single() specifies a larger alignment, one or more +initial slots in each slot set might not meet the alloc_align_mask criterium. +Because a bounce buffer allocation can't cross a slot set boundary, eliminating +those initial slots effectively reduces the max size of a bounce buffer. +Currently, there's no problem because alloc_align_mask is set based on IOMMU +granule size, and granules cannot be larger than PAGE_SIZE. But if that were to +change in the future, the initial pool allocation might need to be done with +alignment larger than PAGE_SIZE. + +Dynamic swiotlb +--------------- +When CONFIG_DYNAMIC_SWIOTLB is enabled, swiotlb can do on-demand expansion of +the amount of memory available for allocation as bounce buffers. If a bounce +buffer request fails due to lack of available space, an asynchronous background +task is kicked off to allocate memory from general system memory and turn it +into an swiotlb pool. Creating an additional pool must be done asynchronously +because the memory allocation may block, and as noted above, swiotlb requests +are not allowed to block. Once the background task is kicked off, the bounce +buffer request creates a "transient pool" to avoid returning an "swiotlb full" +error. A transient pool has the size of the bounce buffer request, and is +deleted when the bounce buffer is freed. Memory for this transient pool comes +from the general system memory atomic pool so that creation does not block. +Creating a transient pool has relatively high cost, particularly in a CoCo VM +where the memory must be decrypted, so it is done only as a stopgap until the +background task can add another non-transient pool. + +Adding a dynamic pool has limitations. Like with the default pool, the memory +must be physically contiguous, so the size is limited to MAX_PAGE_ORDER pages +(e.g., 4 MiB on a typical x86 system). Due to memory fragmentation, a max size +allocation may not be available. The dynamic pool allocator tries smaller sizes +until it succeeds, but with a minimum size of 1 MiB. Given sufficient system +memory fragmentation, dynamically adding a pool might not succeed at all. + +The number of areas in a dynamic pool may be different from the number of areas +in the default pool. Because the new pool size is typically a few MiB at most, +the number of areas will likely be smaller. For example, with a new pool size +of 4 MiB and the 256 KiB minimum area size, only 16 areas can be created. If +the system has more than 16 CPUs, multiple CPUs must share an area, creating +more lock contention. + +New pools added via dynamic swiotlb are linked together in a linear list. +swiotlb code frequently must search for the pool containing a particular +swiotlb physical address, so that search is linear and not performant with a +large number of dynamic pools. The data structures could be improved for +faster searches. + +Overall, dynamic swiotlb works best for small configurations with relatively +few CPUs. It allows the default swiotlb pool to be smaller so that memory is +not wasted, with dynamic pools making more space available if needed (as long +as fragmentation isn't an obstacle). It is less useful for large CoCo VMs. + +Data Structure Details +---------------------- +swiotlb is managed with four primary data structures: io_tlb_mem, io_tlb_pool, +io_tlb_area, and io_tlb_slot. io_tlb_mem describes a swiotlb memory allocator, +which includes the default memory pool and any dynamic or transient pools +linked to it. Limited statistics on swiotlb usage are kept per memory allocator +and are stored in this data structure. These statistics are available under +/sys/kernel/debug/swiotlb when CONFIG_DEBUG_FS is set. + +io_tlb_pool describes a memory pool, either the default pool, a dynamic pool, +or a transient pool. The description includes the start and end addresses of +the memory in the pool, a pointer to an array of io_tlb_area structures, and a +pointer to an array of io_tlb_slot structures that are associated with the pool. + +io_tlb_area describes an area. The primary field is the spin lock used to +serialize access to slots in the area. The io_tlb_area array for a pool has an +entry for each area, and is accessed using a 0-based area index derived from the +calling processor ID. Areas exist solely to allow parallel access to swiotlb +from multiple CPUs. + +io_tlb_slot describes an individual memory slot in the pool, with size +IO_TLB_SIZE (2 KiB currently). The io_tlb_slot array is indexed by the slot +index computed from the bounce buffer address relative to the starting memory +address of the pool. The size of struct io_tlb_slot is 24 bytes, so the +overhead is about 1% of the slot size. + +The io_tlb_slot array is designed to meet several requirements. First, the DMA +APIs and the corresponding swiotlb APIs use the bounce buffer address as the +identifier for a bounce buffer. This address is returned by +swiotlb_tbl_map_single(), and then passed as an argument to +swiotlb_tbl_unmap_single() and the swiotlb_sync_*() functions. The original +memory buffer address obviously must be passed as an argument to +swiotlb_tbl_map_single(), but it is not passed to the other APIs. Consequently, +swiotlb data structures must save the original memory buffer address so that it +can be used when doing sync operations. This original address is saved in the +io_tlb_slot array. + +Second, the io_tlb_slot array must handle partial sync requests. In such cases, +the argument to swiotlb_sync_*() is not the address of the start of the bounce +buffer but an address somewhere in the middle of the bounce buffer, and the +address of the start of the bounce buffer isn't known to swiotlb code. But +swiotlb code must be able to calculate the corresponding original memory buffer +address to do the CPU copy dictated by the "sync". So an adjusted original +memory buffer address is populated into the struct io_tlb_slot for each slot +occupied by the bounce buffer. An adjusted "alloc_size" of the bounce buffer is +also recorded in each struct io_tlb_slot so a sanity check can be performed on +the size of the "sync" operation. The "alloc_size" field is not used except for +the sanity check. + +Third, the io_tlb_slot array is used to track available slots. The "list" field +in struct io_tlb_slot records how many contiguous available slots exist starting +at that slot. A "0" indicates that the slot is occupied. A value of "1" +indicates only the current slot is available. A value of "2" indicates the +current slot and the next slot are available, etc. The maximum value is +IO_TLB_SEGSIZE, which can appear in the first slot in a slot set, and indicates +that the entire slot set is available. These values are used when searching for +available slots to use for a new bounce buffer. They are updated when allocating +a new bounce buffer and when freeing a bounce buffer. At pool creation time, the +"list" field is initialized to IO_TLB_SEGSIZE down to 1 for the slots in every +slot set. + +Fourth, the io_tlb_slot array keeps track of any "padding slots" allocated to +meet alloc_align_mask requirements described above. When +swiotlb_tlb_map_single() allocates bounce buffer space to meet alloc_align_mask +requirements, it may allocate pre-padding space across zero or more slots. But +when swiotbl_tlb_unmap_single() is called with the bounce buffer address, the +alloc_align_mask value that governed the allocation, and therefore the +allocation of any padding slots, is not known. The "pad_slots" field records +the number of padding slots so that swiotlb_tbl_unmap_single() can free them. +The "pad_slots" value is recorded only in the first non-padding slot allocated +to the bounce buffer. + +Restricted pools +---------------- +The swiotlb machinery is also used for "restricted pools", which are pools of +memory separate from the default swiotlb pool, and that are dedicated for DMA +use by a particular device. Restricted pools provide a level of DMA memory +protection on systems with limited hardware protection capabilities, such as +those lacking an IOMMU. Such usage is specified by DeviceTree entries and +requires that CONFIG_DMA_RESTRICTED_POOL is set. Each restricted pool is based +on its own io_tlb_mem data structure that is independent of the main swiotlb +io_tlb_mem. + +Restricted pools add swiotlb_alloc() and swiotlb_free() APIs, which are called +from the dma_alloc_*() and dma_free_*() APIs. The swiotlb_alloc/free() APIs +allocate/free slots from/to the restricted pool directly and do not go through +swiotlb_tbl_map/unmap_single(). From 82e966130ddd67539ab904f2038e7bf5d4a66247 Mon Sep 17 00:00:00 2001 From: Jules Irenge Date: Wed, 1 May 2024 00:46:42 +0100 Subject: [PATCH 0969/1702] RDMA/mlx5: Remove NULL check before dev_{put, hold} Coccinelle reports a warning WARNING: NULL check before dev_{put, hold} functions is not needed The reason is the call netdev_{put, hold} of dev_{put,hold} will check NULL There is no need to check before using dev_{put, hold} Signed-off-by: Jules Irenge Link: https://lore.kernel.org/r/ZjGC4qXrOwZE0aHi@octinomon.home Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/main.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index c2b557e642906..2366c46eebc87 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -264,8 +264,7 @@ static struct net_device *mlx5_ib_get_netdev(struct ib_device *device, */ read_lock(&ibdev->port[port_num - 1].roce.netdev_lock); ndev = ibdev->port[port_num - 1].roce.netdev; - if (ndev) - dev_hold(ndev); + dev_hold(ndev); read_unlock(&ibdev->port[port_num - 1].roce.netdev_lock); out: From e4e40a87024c502dcca279504a4550e617eea037 Mon Sep 17 00:00:00 2001 From: Jules Irenge Date: Wed, 1 May 2024 00:47:33 +0100 Subject: [PATCH 0970/1702] RDMA/ipoib: Remove NULL check before dev_{put, hold} Coccinelle reports a warning WARNING: NULL check before dev_{put, hold} functions is not needed The reason is the call netdev_{put, hold} of dev_{put,hold} will check NULL There is no need to check before using dev_{put, hold} Signed-off-by: Jules Irenge Link: https://lore.kernel.org/r/ZjGDFatHRMI6Eg7M@octinomon.home Signed-off-by: Leon Romanovsky --- drivers/infiniband/ulp/ipoib/ipoib_main.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index 6f2a688fccbfb..4abec0124ea37 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -329,8 +329,7 @@ static struct net_device *ipoib_get_master_net_dev(struct net_device *dev) rcu_read_lock(); master = netdev_master_upper_dev_get_rcu(dev); - if (master) - dev_hold(master); + dev_hold(master); rcu_read_unlock(); if (master) From a86f8671d03e6eb31abaefdf6928b92df0a2368c Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 2 May 2024 07:48:35 -0700 Subject: [PATCH 0971/1702] xfs: use unsigned ints for non-negative quantities in xfs_attr_remote.c In the next few patches we're going to refactor the attr remote code so that we can support headerless remote xattr values for storing merkle tree blocks. For now, let's change the code to use unsigned int to describe quantities of bytes and blocks that cannot be negative. Signed-off-by: Darrick J. Wong Reviewed-by: Andrey Albershteyn Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr_remote.c | 61 ++++++++++++++++----------------- fs/xfs/libxfs/xfs_attr_remote.h | 2 +- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr_remote.c b/fs/xfs/libxfs/xfs_attr_remote.c index beb0efdd8f6b8..7c38c6feb8c9f 100644 --- a/fs/xfs/libxfs/xfs_attr_remote.c +++ b/fs/xfs/libxfs/xfs_attr_remote.c @@ -47,13 +47,13 @@ * Each contiguous block has a header, so it is not just a simple attribute * length to FSB conversion. */ -int +unsigned int xfs_attr3_rmt_blocks( - struct xfs_mount *mp, - int attrlen) + struct xfs_mount *mp, + unsigned int attrlen) { if (xfs_has_crc(mp)) { - int buflen = XFS_ATTR3_RMT_BUF_SPACE(mp, mp->m_sb.sb_blocksize); + unsigned int buflen = XFS_ATTR3_RMT_BUF_SPACE(mp, mp->m_sb.sb_blocksize); return (attrlen + buflen - 1) / buflen; } return XFS_B_TO_FSB(mp, attrlen); @@ -92,7 +92,6 @@ xfs_attr3_rmt_verify( struct xfs_mount *mp, struct xfs_buf *bp, void *ptr, - int fsbsize, xfs_daddr_t bno) { struct xfs_attr3_rmt_hdr *rmt = ptr; @@ -103,7 +102,7 @@ xfs_attr3_rmt_verify( return __this_address; if (be64_to_cpu(rmt->rm_blkno) != bno) return __this_address; - if (be32_to_cpu(rmt->rm_bytes) > fsbsize - sizeof(*rmt)) + if (be32_to_cpu(rmt->rm_bytes) > mp->m_attr_geo->blksize - sizeof(*rmt)) return __this_address; if (be32_to_cpu(rmt->rm_offset) + be32_to_cpu(rmt->rm_bytes) > XFS_XATTR_SIZE_MAX) @@ -122,9 +121,9 @@ __xfs_attr3_rmt_read_verify( { struct xfs_mount *mp = bp->b_mount; char *ptr; - int len; + unsigned int len; xfs_daddr_t bno; - int blksize = mp->m_attr_geo->blksize; + unsigned int blksize = mp->m_attr_geo->blksize; /* no verification of non-crc buffers */ if (!xfs_has_crc(mp)) @@ -141,7 +140,7 @@ __xfs_attr3_rmt_read_verify( *failaddr = __this_address; return -EFSBADCRC; } - *failaddr = xfs_attr3_rmt_verify(mp, bp, ptr, blksize, bno); + *failaddr = xfs_attr3_rmt_verify(mp, bp, ptr, bno); if (*failaddr) return -EFSCORRUPTED; len -= blksize; @@ -186,7 +185,7 @@ xfs_attr3_rmt_write_verify( { struct xfs_mount *mp = bp->b_mount; xfs_failaddr_t fa; - int blksize = mp->m_attr_geo->blksize; + unsigned int blksize = mp->m_attr_geo->blksize; char *ptr; int len; xfs_daddr_t bno; @@ -203,7 +202,7 @@ xfs_attr3_rmt_write_verify( while (len > 0) { struct xfs_attr3_rmt_hdr *rmt = (struct xfs_attr3_rmt_hdr *)ptr; - fa = xfs_attr3_rmt_verify(mp, bp, ptr, blksize, bno); + fa = xfs_attr3_rmt_verify(mp, bp, ptr, bno); if (fa) { xfs_verifier_error(bp, -EFSCORRUPTED, fa); return; @@ -281,20 +280,20 @@ xfs_attr_rmtval_copyout( struct xfs_buf *bp, struct xfs_inode *dp, xfs_ino_t owner, - int *offset, - int *valuelen, + unsigned int *offset, + unsigned int *valuelen, uint8_t **dst) { char *src = bp->b_addr; xfs_daddr_t bno = xfs_buf_daddr(bp); - int len = BBTOB(bp->b_length); - int blksize = mp->m_attr_geo->blksize; + unsigned int len = BBTOB(bp->b_length); + unsigned int blksize = mp->m_attr_geo->blksize; ASSERT(len >= blksize); while (len > 0 && *valuelen > 0) { - int hdr_size = 0; - int byte_cnt = XFS_ATTR3_RMT_BUF_SPACE(mp, blksize); + unsigned int hdr_size = 0; + unsigned int byte_cnt = XFS_ATTR3_RMT_BUF_SPACE(mp, blksize); byte_cnt = min(*valuelen, byte_cnt); @@ -330,20 +329,20 @@ xfs_attr_rmtval_copyin( struct xfs_mount *mp, struct xfs_buf *bp, xfs_ino_t ino, - int *offset, - int *valuelen, + unsigned int *offset, + unsigned int *valuelen, uint8_t **src) { char *dst = bp->b_addr; xfs_daddr_t bno = xfs_buf_daddr(bp); - int len = BBTOB(bp->b_length); - int blksize = mp->m_attr_geo->blksize; + unsigned int len = BBTOB(bp->b_length); + unsigned int blksize = mp->m_attr_geo->blksize; ASSERT(len >= blksize); while (len > 0 && *valuelen > 0) { - int hdr_size; - int byte_cnt = XFS_ATTR3_RMT_BUF_SPACE(mp, blksize); + unsigned int hdr_size; + unsigned int byte_cnt = XFS_ATTR3_RMT_BUF_SPACE(mp, blksize); byte_cnt = min(*valuelen, byte_cnt); hdr_size = xfs_attr3_rmt_hdr_set(mp, dst, ino, *offset, @@ -389,12 +388,12 @@ xfs_attr_rmtval_get( struct xfs_buf *bp; xfs_dablk_t lblkno = args->rmtblkno; uint8_t *dst = args->value; - int valuelen; + unsigned int valuelen; int nmap; int error; - int blkcnt = args->rmtblkcnt; + unsigned int blkcnt = args->rmtblkcnt; int i; - int offset = 0; + unsigned int offset = 0; trace_xfs_attr_rmtval_get(args); @@ -452,7 +451,7 @@ xfs_attr_rmt_find_hole( struct xfs_inode *dp = args->dp; struct xfs_mount *mp = dp->i_mount; int error; - int blkcnt; + unsigned int blkcnt; xfs_fileoff_t lfileoff = 0; /* @@ -481,11 +480,11 @@ xfs_attr_rmtval_set_value( struct xfs_bmbt_irec map; xfs_dablk_t lblkno; uint8_t *src = args->value; - int blkcnt; - int valuelen; + unsigned int blkcnt; + unsigned int valuelen; int nmap; int error; - int offset = 0; + unsigned int offset = 0; /* * Roll through the "value", copying the attribute value to the @@ -644,7 +643,7 @@ xfs_attr_rmtval_invalidate( struct xfs_da_args *args) { xfs_dablk_t lblkno; - int blkcnt; + unsigned int blkcnt; int error; /* diff --git a/fs/xfs/libxfs/xfs_attr_remote.h b/fs/xfs/libxfs/xfs_attr_remote.h index d097ec6c4dc35..c64b04f91cafd 100644 --- a/fs/xfs/libxfs/xfs_attr_remote.h +++ b/fs/xfs/libxfs/xfs_attr_remote.h @@ -6,7 +6,7 @@ #ifndef __XFS_ATTR_REMOTE_H__ #define __XFS_ATTR_REMOTE_H__ -int xfs_attr3_rmt_blocks(struct xfs_mount *mp, int attrlen); +unsigned int xfs_attr3_rmt_blocks(struct xfs_mount *mp, unsigned int attrlen); int xfs_attr_rmtval_get(struct xfs_da_args *args); int xfs_attr_rmtval_stale(struct xfs_inode *ip, struct xfs_bmbt_irec *map, From a5714b67cad586f44777ad834e577037ce6b64fd Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 2 May 2024 07:48:36 -0700 Subject: [PATCH 0972/1702] xfs: turn XFS_ATTR3_RMT_BUF_SPACE into a function Turn this into a properly typechecked function, and actually use the correct blocksize for extended attributes. The function cannot be static inline because xfsprogs userspace uses it. Signed-off-by: Darrick J. Wong Reviewed-by: Andrey Albershteyn Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr_remote.c | 19 ++++++++++++++++--- fs/xfs/libxfs/xfs_da_format.h | 4 +--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr_remote.c b/fs/xfs/libxfs/xfs_attr_remote.c index 7c38c6feb8c9f..043b837a3ef7c 100644 --- a/fs/xfs/libxfs/xfs_attr_remote.c +++ b/fs/xfs/libxfs/xfs_attr_remote.c @@ -43,6 +43,19 @@ * the logging system and therefore never have a log item. */ +/* How many bytes can be stored in a remote value buffer? */ +inline unsigned int +xfs_attr3_rmt_buf_space( + struct xfs_mount *mp) +{ + unsigned int blocksize = mp->m_attr_geo->blksize; + + if (xfs_has_crc(mp)) + return blocksize - sizeof(struct xfs_attr3_rmt_hdr); + + return blocksize; +} + /* * Each contiguous block has a header, so it is not just a simple attribute * length to FSB conversion. @@ -53,7 +66,7 @@ xfs_attr3_rmt_blocks( unsigned int attrlen) { if (xfs_has_crc(mp)) { - unsigned int buflen = XFS_ATTR3_RMT_BUF_SPACE(mp, mp->m_sb.sb_blocksize); + unsigned int buflen = xfs_attr3_rmt_buf_space(mp); return (attrlen + buflen - 1) / buflen; } return XFS_B_TO_FSB(mp, attrlen); @@ -293,7 +306,7 @@ xfs_attr_rmtval_copyout( while (len > 0 && *valuelen > 0) { unsigned int hdr_size = 0; - unsigned int byte_cnt = XFS_ATTR3_RMT_BUF_SPACE(mp, blksize); + unsigned int byte_cnt = xfs_attr3_rmt_buf_space(mp); byte_cnt = min(*valuelen, byte_cnt); @@ -342,7 +355,7 @@ xfs_attr_rmtval_copyin( while (len > 0 && *valuelen > 0) { unsigned int hdr_size; - unsigned int byte_cnt = XFS_ATTR3_RMT_BUF_SPACE(mp, blksize); + unsigned int byte_cnt = xfs_attr3_rmt_buf_space(mp); byte_cnt = min(*valuelen, byte_cnt); hdr_size = xfs_attr3_rmt_hdr_set(mp, dst, ino, *offset, diff --git a/fs/xfs/libxfs/xfs_da_format.h b/fs/xfs/libxfs/xfs_da_format.h index ebde6eb1da65d..86de99e2f7570 100644 --- a/fs/xfs/libxfs/xfs_da_format.h +++ b/fs/xfs/libxfs/xfs_da_format.h @@ -880,9 +880,7 @@ struct xfs_attr3_rmt_hdr { #define XFS_ATTR3_RMT_CRC_OFF offsetof(struct xfs_attr3_rmt_hdr, rm_crc) -#define XFS_ATTR3_RMT_BUF_SPACE(mp, bufsize) \ - ((bufsize) - (xfs_has_crc((mp)) ? \ - sizeof(struct xfs_attr3_rmt_hdr) : 0)) +unsigned int xfs_attr3_rmt_buf_space(struct xfs_mount *mp); /* Number of bytes in a directory block. */ static inline unsigned int xfs_dir2_dirblock_bytes(struct xfs_sb *sbp) From 204a26aa1d5a891154c9275fe4022e28793ca112 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 2 May 2024 07:48:36 -0700 Subject: [PATCH 0973/1702] xfs: create a helper to compute the blockcount of a max sized remote value Create a helper function to compute the number of fsblocks needed to store a maximally-sized extended attribute value. Signed-off-by: Darrick J. Wong Reviewed-by: Andrey Albershteyn Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr.c | 2 +- fs/xfs/libxfs/xfs_attr_remote.h | 6 ++++++ fs/xfs/scrub/reap.c | 4 ++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 1c2a27fce08a9..eb274d4d636d8 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -1036,7 +1036,7 @@ xfs_attr_set( break; case XFS_ATTRUPDATE_REMOVE: XFS_STATS_INC(mp, xs_attr_remove); - rmt_blks = xfs_attr3_rmt_blocks(mp, XFS_XATTR_SIZE_MAX); + rmt_blks = xfs_attr3_max_rmt_blocks(mp); break; } diff --git a/fs/xfs/libxfs/xfs_attr_remote.h b/fs/xfs/libxfs/xfs_attr_remote.h index c64b04f91cafd..e3c6c7d774bf9 100644 --- a/fs/xfs/libxfs/xfs_attr_remote.h +++ b/fs/xfs/libxfs/xfs_attr_remote.h @@ -8,6 +8,12 @@ unsigned int xfs_attr3_rmt_blocks(struct xfs_mount *mp, unsigned int attrlen); +/* Number of rmt blocks needed to store the maximally sized attr value */ +static inline unsigned int xfs_attr3_max_rmt_blocks(struct xfs_mount *mp) +{ + return xfs_attr3_rmt_blocks(mp, XFS_XATTR_SIZE_MAX); +} + int xfs_attr_rmtval_get(struct xfs_da_args *args); int xfs_attr_rmtval_stale(struct xfs_inode *ip, struct xfs_bmbt_irec *map, xfs_buf_flags_t incore_flags); diff --git a/fs/xfs/scrub/reap.c b/fs/xfs/scrub/reap.c index 01ceaa4efa16b..be283153c254e 100644 --- a/fs/xfs/scrub/reap.c +++ b/fs/xfs/scrub/reap.c @@ -223,7 +223,7 @@ xrep_bufscan_max_sectors( int max_fsbs; /* Remote xattr values are the largest buffers that we support. */ - max_fsbs = xfs_attr3_rmt_blocks(mp, XFS_XATTR_SIZE_MAX); + max_fsbs = xfs_attr3_max_rmt_blocks(mp); return XFS_FSB_TO_BB(mp, min_t(xfs_extlen_t, fsblocks, max_fsbs)); } @@ -806,7 +806,7 @@ xreap_bmapi_binval( * of the next hole. */ off = imap->br_startoff + imap->br_blockcount; - max_off = off + xfs_attr3_rmt_blocks(mp, XFS_XATTR_SIZE_MAX); + max_off = off + xfs_attr3_max_rmt_blocks(mp); while (off < max_off) { struct xfs_bmbt_irec hmap; int nhmaps = 1; From 3791a053294b037a6bf62df03480f5c5ddfd4d1b Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 2 May 2024 07:48:37 -0700 Subject: [PATCH 0974/1702] xfs: minor cleanups of xfs_attr3_rmt_blocks Clean up the type signature of this function since we don't have negative attr lengths or block counts. Signed-off-by: Darrick J. Wong Reviewed-by: Andrey Albershteyn Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr_remote.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr_remote.c b/fs/xfs/libxfs/xfs_attr_remote.c index 043b837a3ef7c..4c44ce1c8a644 100644 --- a/fs/xfs/libxfs/xfs_attr_remote.c +++ b/fs/xfs/libxfs/xfs_attr_remote.c @@ -56,19 +56,19 @@ xfs_attr3_rmt_buf_space( return blocksize; } -/* - * Each contiguous block has a header, so it is not just a simple attribute - * length to FSB conversion. - */ +/* Compute number of fsblocks needed to store a remote attr value */ unsigned int xfs_attr3_rmt_blocks( struct xfs_mount *mp, unsigned int attrlen) { - if (xfs_has_crc(mp)) { - unsigned int buflen = xfs_attr3_rmt_buf_space(mp); - return (attrlen + buflen - 1) / buflen; - } + /* + * Each contiguous block has a header, so it is not just a simple + * attribute length to FSB conversion. + */ + if (xfs_has_crc(mp)) + return howmany(attrlen, xfs_attr3_rmt_buf_space(mp)); + return XFS_B_TO_FSB(mp, attrlen); } From 1a3f1afb2532028c7dc5552b3aa39423c2621062 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 2 May 2024 07:48:37 -0700 Subject: [PATCH 0975/1702] xfs: widen flags argument to the xfs_iflags_* helpers xfs_inode.i_flags is an unsigned long, so make these helpers take that as the flags argument instead of unsigned short. This is needed for the next patch. While we're at it, remove the iflags variable from xfs_iget_cache_miss because we no longer need it. Signed-off-by: Darrick J. Wong Reviewed-by: Andrey Albershteyn --- fs/xfs/xfs_icache.c | 4 +--- fs/xfs/xfs_inode.h | 14 +++++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 74f1812b03cbd..0953163a2d849 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -613,7 +613,6 @@ xfs_iget_cache_miss( struct xfs_inode *ip; int error; xfs_agino_t agino = XFS_INO_TO_AGINO(mp, ino); - int iflags; ip = xfs_inode_alloc(mp, ino); if (!ip) @@ -693,13 +692,12 @@ xfs_iget_cache_miss( * memory barrier that ensures this detection works correctly at lookup * time. */ - iflags = XFS_INEW; if (flags & XFS_IGET_DONTCACHE) d_mark_dontcache(VFS_I(ip)); ip->i_udquot = NULL; ip->i_gdquot = NULL; ip->i_pdquot = NULL; - xfs_iflags_set(ip, iflags); + xfs_iflags_set(ip, XFS_INEW); /* insert the new inode */ spin_lock(&pag->pag_ici_lock); diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index 9fd4d29a57137..292b90b5f2ac8 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -207,13 +207,13 @@ xfs_new_eof(struct xfs_inode *ip, xfs_fsize_t new_size) * i_flags helper functions */ static inline void -__xfs_iflags_set(xfs_inode_t *ip, unsigned short flags) +__xfs_iflags_set(xfs_inode_t *ip, unsigned long flags) { ip->i_flags |= flags; } static inline void -xfs_iflags_set(xfs_inode_t *ip, unsigned short flags) +xfs_iflags_set(xfs_inode_t *ip, unsigned long flags) { spin_lock(&ip->i_flags_lock); __xfs_iflags_set(ip, flags); @@ -221,7 +221,7 @@ xfs_iflags_set(xfs_inode_t *ip, unsigned short flags) } static inline void -xfs_iflags_clear(xfs_inode_t *ip, unsigned short flags) +xfs_iflags_clear(xfs_inode_t *ip, unsigned long flags) { spin_lock(&ip->i_flags_lock); ip->i_flags &= ~flags; @@ -229,13 +229,13 @@ xfs_iflags_clear(xfs_inode_t *ip, unsigned short flags) } static inline int -__xfs_iflags_test(xfs_inode_t *ip, unsigned short flags) +__xfs_iflags_test(xfs_inode_t *ip, unsigned long flags) { return (ip->i_flags & flags); } static inline int -xfs_iflags_test(xfs_inode_t *ip, unsigned short flags) +xfs_iflags_test(xfs_inode_t *ip, unsigned long flags) { int ret; spin_lock(&ip->i_flags_lock); @@ -245,7 +245,7 @@ xfs_iflags_test(xfs_inode_t *ip, unsigned short flags) } static inline int -xfs_iflags_test_and_clear(xfs_inode_t *ip, unsigned short flags) +xfs_iflags_test_and_clear(xfs_inode_t *ip, unsigned long flags) { int ret; @@ -258,7 +258,7 @@ xfs_iflags_test_and_clear(xfs_inode_t *ip, unsigned short flags) } static inline int -xfs_iflags_test_and_set(xfs_inode_t *ip, unsigned short flags) +xfs_iflags_test_and_set(xfs_inode_t *ip, unsigned long flags) { int ret; From c2a09f3d782de952f09a3962d03b939e7fa7ffa4 Mon Sep 17 00:00:00 2001 From: "Ritesh Harjani (IBM)" Date: Thu, 29 Feb 2024 11:40:13 +0530 Subject: [PATCH 0976/1702] ext4: Fixes len calculation in mpage_journal_page_buffers Truncate operation can race with writeback, in which inode->i_size can get truncated and therefore size - folio_pos() can be negative. This fixes the len calculation. However this path doesn't get easily triggered even with data journaling. Cc: stable@kernel.org # v6.5 Fixes: 80be8c5cc925 ("Fixes: ext4: Make mpage_journal_page_buffers use folio") Signed-off-by: Ritesh Harjani (IBM) Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/cff4953b5c9306aba71e944ab176a5d396b9a1b7.1709182250.git.ritesh.list@gmail.com Signed-off-by: Theodore Ts'o --- fs/ext4/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 537803250ca9a..bab9223d94ac4 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -2334,7 +2334,7 @@ static int mpage_journal_page_buffers(handle_t *handle, if (folio_pos(folio) + len > size && !ext4_verity_in_progress(inode)) - len = size - folio_pos(folio); + len = size & (len - 1); return ext4_journal_folio_buffers(handle, folio, len); } From 53c17fe55a06cbb405b94d96759611d725d2c47a Mon Sep 17 00:00:00 2001 From: "Ritesh Harjani (IBM)" Date: Thu, 29 Feb 2024 11:40:14 +0530 Subject: [PATCH 0977/1702] ext4: Remove PAGE_MASK dependency on mpage_submit_folio This patch simply removes the PAGE_MASK dependency since mpage_submit_folio() is already converted to work with folio. Signed-off-by: Ritesh Harjani (IBM) Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/d6eadb090334ea49ceef4e643b371fabfcea328f.1709182251.git.ritesh.list@gmail.com Signed-off-by: Theodore Ts'o --- fs/ext4/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index bab9223d94ac4..e8b0773e5d2d4 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -1865,7 +1865,7 @@ static int mpage_submit_folio(struct mpage_da_data *mpd, struct folio *folio) len = folio_size(folio); if (folio_pos(folio) + len > size && !ext4_verity_in_progress(mpd->inode)) - len = size & ~PAGE_MASK; + len = size & (len - 1); err = ext4_bio_write_folio(&mpd->io_submit, folio, len); if (!err) mpd->wbc->nr_to_write--; From a0c7cce824a54dbb83bb722df19f1ddcfa5f8d25 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 29 Feb 2024 19:54:12 +0530 Subject: [PATCH 0978/1702] ext4: set FMODE_CAN_ODIRECT instead of a dummy direct_IO method Since commit a2ad63daa88b ("VFS: add FMODE_CAN_ODIRECT file flag") file systems can just set the FMODE_CAN_ODIRECT flag at open time instead of wiring up a dummy direct_IO method to indicate support for direct I/O. Signed-off-by: Christoph Hellwig [RH: Rebased to upstream] Signed-off-by: Ritesh Harjani (IBM) Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/e5797bb597219a49043e53e4e90aa494b97dc328.1709215665.git.ritesh.list@gmail.com Signed-off-by: Theodore Ts'o --- fs/ext4/file.c | 2 +- fs/ext4/inode.c | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/fs/ext4/file.c b/fs/ext4/file.c index 54d6ff22585cf..965febab1d048 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -886,7 +886,7 @@ static int ext4_file_open(struct inode *inode, struct file *filp) } filp->f_mode |= FMODE_NOWAIT | FMODE_BUF_RASYNC | - FMODE_DIO_PARALLEL_WRITE; + FMODE_DIO_PARALLEL_WRITE | FMODE_CAN_ODIRECT; return dquot_file_open(inode, filp); } diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index e8b0773e5d2d4..989e28c24a8cb 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -3530,7 +3530,6 @@ static const struct address_space_operations ext4_aops = { .bmap = ext4_bmap, .invalidate_folio = ext4_invalidate_folio, .release_folio = ext4_release_folio, - .direct_IO = noop_direct_IO, .migrate_folio = buffer_migrate_folio, .is_partially_uptodate = block_is_partially_uptodate, .error_remove_folio = generic_error_remove_folio, @@ -3547,7 +3546,6 @@ static const struct address_space_operations ext4_journalled_aops = { .bmap = ext4_bmap, .invalidate_folio = ext4_journalled_invalidate_folio, .release_folio = ext4_release_folio, - .direct_IO = noop_direct_IO, .migrate_folio = buffer_migrate_folio_norefs, .is_partially_uptodate = block_is_partially_uptodate, .error_remove_folio = generic_error_remove_folio, @@ -3564,7 +3562,6 @@ static const struct address_space_operations ext4_da_aops = { .bmap = ext4_bmap, .invalidate_folio = ext4_invalidate_folio, .release_folio = ext4_release_folio, - .direct_IO = noop_direct_IO, .migrate_folio = buffer_migrate_folio, .is_partially_uptodate = block_is_partially_uptodate, .error_remove_folio = generic_error_remove_folio, @@ -3573,7 +3570,6 @@ static const struct address_space_operations ext4_da_aops = { static const struct address_space_operations ext4_dax_aops = { .writepages = ext4_dax_writepages, - .direct_IO = noop_direct_IO, .dirty_folio = noop_dirty_folio, .bmap = ext4_bmap, .swap_activate = ext4_iomap_swap_activate, From ea7d09ad7c280122a322f408672ab8d75c1a0e30 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Sun, 17 Mar 2024 16:36:39 +0100 Subject: [PATCH 0979/1702] ext4: remove unneeded if checks before kfree kfree already checks if its argument is NULL. This fixes two Coccinelle/coccicheck warnings reported by ifnullfree.cocci. Signed-off-by: Thorsten Blum Reviewed-by: Ritesh Harjani (IBM) Link: https://lore.kernel.org/r/20240317153638.2136-2-thorsten.blum@toblux.com Signed-off-by: Theodore Ts'o --- fs/ext4/super.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 044135796f2b6..28cbe0f1996f8 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -2078,8 +2078,7 @@ static int unnote_qf_name(struct fs_context *fc, int qtype) { struct ext4_fs_context *ctx = fc->fs_private; - if (ctx->s_qf_names[qtype]) - kfree(ctx->s_qf_names[qtype]); + kfree(ctx->s_qf_names[qtype]); ctx->s_qf_names[qtype] = NULL; ctx->qname_spec |= 1 << qtype; @@ -2484,8 +2483,7 @@ static int parse_options(struct fs_context *fc, char *options) param.size = v_len; ret = ext4_parse_param(fc, ¶m); - if (param.string) - kfree(param.string); + kfree(param.string); if (ret < 0) return ret; } From 35a1f12f0ca857fee1d7a04ef52cbd5f1f84de13 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 7 Mar 2024 12:53:20 +0100 Subject: [PATCH 0980/1702] ext4: avoid excessive credit estimate in ext4_tmpfile() A user with minimum journal size (1024 blocks these days) complained about the following error triggered by generic/697 test in ext4_tmpfile(): run fstests generic/697 at 2024-02-28 05:34:46 JBD2: vfstest wants too many credits credits:260 rsv_credits:0 max:256 EXT4-fs error (device loop0) in __ext4_new_inode:1083: error 28 Indeed the credit estimate in ext4_tmpfile() is huge. EXT4_MAXQUOTAS_INIT_BLOCKS() is 219, then 10 credits from ext4_tmpfile() itself and then ext4_xattr_credits_for_new_inode() adds more credits needed for security attributes and ACLs. Now the EXT4_MAXQUOTAS_INIT_BLOCKS() is in fact unnecessary because we've already initialized quotas with dquot_init() shortly before and so EXT4_MAXQUOTAS_TRANS_BLOCKS() is enough (which boils down to 3 credits). Fixes: af51a2ac36d1 ("ext4: ->tmpfile() support") Signed-off-by: Jan Kara Tested-by: Luis Henriques Tested-by: Disha Goel Link: https://lore.kernel.org/r/20240307115320.28949-1-jack@suse.cz Signed-off-by: Theodore Ts'o --- fs/ext4/namei.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 5e4f65c14dfb9..a630b27a4cc6e 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -2897,7 +2897,7 @@ static int ext4_tmpfile(struct mnt_idmap *idmap, struct inode *dir, inode = ext4_new_inode_start_handle(idmap, dir, mode, NULL, 0, NULL, EXT4_HT_DIR, - EXT4_MAXQUOTAS_INIT_BLOCKS(dir->i_sb) + + EXT4_MAXQUOTAS_TRANS_BLOCKS(dir->i_sb) + 4 + EXT4_XATTR_TRANS_BLOCKS); handle = ext4_journal_current_handle(); err = PTR_ERR(inode); From fb092d407262eb4278f3d1ca24da54396a038c62 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 14 Mar 2024 23:53:02 -0400 Subject: [PATCH 0981/1702] ext4: add support for FS_IOC_GETFSSYSFSPATH The new sysfs path ioctl lets us get the /sys/fs/ path for a given filesystem in a fs agnostic way, potentially nudging us towards standarizing some of our reporting. Signed-off-by: Kent Overstreet Cc: Theodore Ts'o Cc: Andreas Dilger Cc: linux-ext4@vger.kernel.org Link: https://lore.kernel.org/r/20240315035308.3563511-4-kent.overstreet@linux.dev Signed-off-by: Theodore Ts'o --- fs/ext4/super.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 28cbe0f1996f8..9bde300b15513 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -5340,6 +5340,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) sb->s_quota_types = QTYPE_MASK_USR | QTYPE_MASK_GRP | QTYPE_MASK_PRJ; #endif super_set_uuid(sb, es->s_uuid, sizeof(es->s_uuid)); + super_set_sysfs_name_bdev(sb); INIT_LIST_HEAD(&sbi->s_orphan); /* unlinked but open files */ mutex_init(&sbi->s_orphan_lock); From c77194965dd0dcc26f9c1671d2e74e4eb1248af5 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Fri, 15 Mar 2024 15:29:56 +0100 Subject: [PATCH 0982/1702] Revert "ext4: apply umask if ACL support is disabled" This reverts commit 484fd6c1de13b336806a967908a927cc0356e312. The commit caused a regression because now the umask was applied to symlinks and the fix is unnecessary because the umask/O_TMPFILE bug has been fixed somewhere else already. Fixes: https://lore.kernel.org/lkml/28DSITL9912E1.2LSZUVTGTO52Q@mforney.org/ Signed-off-by: Max Kellermann Reviewed-by: Christian Brauner Tested-by: Michael Forney Link: https://lore.kernel.org/r/20240315142956.2420360-1-max.kellermann@ionos.com Signed-off-by: Theodore Ts'o --- fs/ext4/acl.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/fs/ext4/acl.h b/fs/ext4/acl.h index ef4c19e5f5706..0c5a79c3b5d48 100644 --- a/fs/ext4/acl.h +++ b/fs/ext4/acl.h @@ -68,11 +68,6 @@ extern int ext4_init_acl(handle_t *, struct inode *, struct inode *); static inline int ext4_init_acl(handle_t *handle, struct inode *inode, struct inode *dir) { - /* usually, the umask is applied by posix_acl_create(), but if - ext4 ACL support is disabled at compile time, we need to do - it here, because posix_acl_create() will never be called */ - inode->i_mode &= ~current_umask(); - return 0; } #endif /* CONFIG_EXT4_FS_POSIX_ACL */ From 9e8e819f8f272c4e5dcd0bd6c7450e36481ed139 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Tue, 19 Mar 2024 19:33:17 +0800 Subject: [PATCH 0983/1702] ext4: avoid overflow when setting values via sysfs When setting values of type unsigned int through sysfs, we use kstrtoul() to parse it and then truncate part of it as the final set value, when the set value is greater than UINT_MAX, the set value will not match what we see because of the truncation. As follows: $ echo 4294967296 > /sys/fs/ext4/sda/mb_max_linear_groups $ cat /sys/fs/ext4/sda/mb_max_linear_groups 0 So we use kstrtouint() to parse the attr_pointer_ui type to avoid the inconsistency described above. In addition, a judgment is added to avoid setting s_resv_clusters less than 0. Signed-off-by: Baokun Li Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240319113325.3110393-2-libaokun1@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/sysfs.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c index 6d332dff79ddc..ca820620b9742 100644 --- a/fs/ext4/sysfs.c +++ b/fs/ext4/sysfs.c @@ -104,7 +104,7 @@ static ssize_t reserved_clusters_store(struct ext4_sb_info *sbi, int ret; ret = kstrtoull(skip_spaces(buf), 0, &val); - if (ret || val >= clusters) + if (ret || val >= clusters || (s64)val < 0) return -EINVAL; atomic64_set(&sbi->s_resv_clusters, val); @@ -451,7 +451,8 @@ static ssize_t ext4_attr_store(struct kobject *kobj, s_kobj); struct ext4_attr *a = container_of(attr, struct ext4_attr, attr); void *ptr = calc_ptr(a, sbi); - unsigned long t; + unsigned int t; + unsigned long lt; int ret; switch (a->attr_id) { @@ -460,7 +461,7 @@ static ssize_t ext4_attr_store(struct kobject *kobj, case attr_pointer_ui: if (!ptr) return 0; - ret = kstrtoul(skip_spaces(buf), 0, &t); + ret = kstrtouint(skip_spaces(buf), 0, &t); if (ret) return ret; if (a->attr_ptr == ptr_ext4_super_block_offset) @@ -471,10 +472,10 @@ static ssize_t ext4_attr_store(struct kobject *kobj, case attr_pointer_ul: if (!ptr) return 0; - ret = kstrtoul(skip_spaces(buf), 0, &t); + ret = kstrtoul(skip_spaces(buf), 0, <); if (ret) return ret; - *((unsigned long *) ptr) = t; + *((unsigned long *) ptr) = lt; return len; case attr_inode_readahead: return inode_readahead_blks_store(sbi, buf, len); From f536808adcc37a546bf9cc819c349bd55a28159b Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Tue, 19 Mar 2024 19:33:18 +0800 Subject: [PATCH 0984/1702] ext4: refactor out ext4_generic_attr_store() Refactor out the function ext4_generic_attr_store() to handle the setting of values of various common types, with no functional changes. Signed-off-by: Baokun Li Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240319113325.3110393-3-libaokun1@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/sysfs.c | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c index ca820620b9742..2b1c529b7fdfb 100644 --- a/fs/ext4/sysfs.c +++ b/fs/ext4/sysfs.c @@ -443,24 +443,20 @@ static ssize_t ext4_attr_show(struct kobject *kobj, return 0; } -static ssize_t ext4_attr_store(struct kobject *kobj, - struct attribute *attr, - const char *buf, size_t len) +static ssize_t ext4_generic_attr_store(struct ext4_attr *a, + struct ext4_sb_info *sbi, + const char *buf, size_t len) { - struct ext4_sb_info *sbi = container_of(kobj, struct ext4_sb_info, - s_kobj); - struct ext4_attr *a = container_of(attr, struct ext4_attr, attr); - void *ptr = calc_ptr(a, sbi); + int ret; unsigned int t; unsigned long lt; - int ret; + void *ptr = calc_ptr(a, sbi); + + if (!ptr) + return 0; switch (a->attr_id) { - case attr_reserved_clusters: - return reserved_clusters_store(sbi, buf, len); case attr_pointer_ui: - if (!ptr) - return 0; ret = kstrtouint(skip_spaces(buf), 0, &t); if (ret) return ret; @@ -470,19 +466,33 @@ static ssize_t ext4_attr_store(struct kobject *kobj, *((unsigned int *) ptr) = t; return len; case attr_pointer_ul: - if (!ptr) - return 0; ret = kstrtoul(skip_spaces(buf), 0, <); if (ret) return ret; *((unsigned long *) ptr) = lt; return len; + } + return 0; +} + +static ssize_t ext4_attr_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, size_t len) +{ + struct ext4_sb_info *sbi = container_of(kobj, struct ext4_sb_info, + s_kobj); + struct ext4_attr *a = container_of(attr, struct ext4_attr, attr); + + switch (a->attr_id) { + case attr_reserved_clusters: + return reserved_clusters_store(sbi, buf, len); case attr_inode_readahead: return inode_readahead_blks_store(sbi, buf, len); case attr_trigger_test_error: return trigger_test_error(sbi, buf, len); + default: + return ext4_generic_attr_store(a, sbi, buf, len); } - return 0; } static void ext4_sb_release(struct kobject *kobj) From 57341fe3179c7694c92dcf99e7f836cee4c800dd Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Tue, 19 Mar 2024 19:33:19 +0800 Subject: [PATCH 0985/1702] ext4: refactor out ext4_generic_attr_show() Refactor out the function ext4_generic_attr_show() to handle the reading of values of various common types, with no functional changes. Signed-off-by: Baokun Li Reviewed-by: Zhang Yi Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240319113325.3110393-4-libaokun1@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/sysfs.c | 74 +++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 42 deletions(-) diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c index 2b1c529b7fdfb..7f455b5f22c0f 100644 --- a/fs/ext4/sysfs.c +++ b/fs/ext4/sysfs.c @@ -366,13 +366,42 @@ static ssize_t __print_tstamp(char *buf, __le32 lo, __u8 hi) #define print_tstamp(buf, es, tstamp) \ __print_tstamp(buf, (es)->tstamp, (es)->tstamp ## _hi) +static ssize_t ext4_generic_attr_show(struct ext4_attr *a, + struct ext4_sb_info *sbi, char *buf) +{ + void *ptr = calc_ptr(a, sbi); + + if (!ptr) + return 0; + + switch (a->attr_id) { + case attr_inode_readahead: + case attr_pointer_ui: + if (a->attr_ptr == ptr_ext4_super_block_offset) + return sysfs_emit(buf, "%u\n", le32_to_cpup(ptr)); + return sysfs_emit(buf, "%u\n", *((unsigned int *) ptr)); + case attr_pointer_ul: + return sysfs_emit(buf, "%lu\n", *((unsigned long *) ptr)); + case attr_pointer_u8: + return sysfs_emit(buf, "%u\n", *((unsigned char *) ptr)); + case attr_pointer_u64: + if (a->attr_ptr == ptr_ext4_super_block_offset) + return sysfs_emit(buf, "%llu\n", le64_to_cpup(ptr)); + return sysfs_emit(buf, "%llu\n", *((unsigned long long *) ptr)); + case attr_pointer_string: + return sysfs_emit(buf, "%.*s\n", a->attr_size, (char *) ptr); + case attr_pointer_atomic: + return sysfs_emit(buf, "%d\n", atomic_read((atomic_t *) ptr)); + } + return 0; +} + static ssize_t ext4_attr_show(struct kobject *kobj, struct attribute *attr, char *buf) { struct ext4_sb_info *sbi = container_of(kobj, struct ext4_sb_info, s_kobj); struct ext4_attr *a = container_of(attr, struct ext4_attr, attr); - void *ptr = calc_ptr(a, sbi); switch (a->attr_id) { case attr_delayed_allocation_blocks: @@ -391,45 +420,6 @@ static ssize_t ext4_attr_show(struct kobject *kobj, return sysfs_emit(buf, "%llu\n", (unsigned long long) percpu_counter_sum(&sbi->s_sra_exceeded_retry_limit)); - case attr_inode_readahead: - case attr_pointer_ui: - if (!ptr) - return 0; - if (a->attr_ptr == ptr_ext4_super_block_offset) - return sysfs_emit(buf, "%u\n", - le32_to_cpup(ptr)); - else - return sysfs_emit(buf, "%u\n", - *((unsigned int *) ptr)); - case attr_pointer_ul: - if (!ptr) - return 0; - return sysfs_emit(buf, "%lu\n", - *((unsigned long *) ptr)); - case attr_pointer_u8: - if (!ptr) - return 0; - return sysfs_emit(buf, "%u\n", - *((unsigned char *) ptr)); - case attr_pointer_u64: - if (!ptr) - return 0; - if (a->attr_ptr == ptr_ext4_super_block_offset) - return sysfs_emit(buf, "%llu\n", - le64_to_cpup(ptr)); - else - return sysfs_emit(buf, "%llu\n", - *((unsigned long long *) ptr)); - case attr_pointer_string: - if (!ptr) - return 0; - return sysfs_emit(buf, "%.*s\n", a->attr_size, - (char *) ptr); - case attr_pointer_atomic: - if (!ptr) - return 0; - return sysfs_emit(buf, "%d\n", - atomic_read((atomic_t *) ptr)); case attr_feature: return sysfs_emit(buf, "supported\n"); case attr_first_error_time: @@ -438,9 +428,9 @@ static ssize_t ext4_attr_show(struct kobject *kobj, return print_tstamp(buf, sbi->s_es, s_last_error_time); case attr_journal_task: return journal_task_show(sbi, buf); + default: + return ext4_generic_attr_show(a, sbi, buf); } - - return 0; } static ssize_t ext4_generic_attr_store(struct ext4_attr *a, From 13df4d44a3aaabe61cd01d277b6ee23ead2a5206 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Tue, 19 Mar 2024 19:33:20 +0800 Subject: [PATCH 0986/1702] ext4: fix slab-out-of-bounds in ext4_mb_find_good_group_avg_frag_lists() We can trigger a slab-out-of-bounds with the following commands: mkfs.ext4 -F /dev/$disk 10G mount /dev/$disk /tmp/test echo 2147483647 > /sys/fs/ext4/$disk/mb_group_prealloc echo test > /tmp/test/file && sync ================================================================== BUG: KASAN: slab-out-of-bounds in ext4_mb_find_good_group_avg_frag_lists+0x8a/0x200 [ext4] Read of size 8 at addr ffff888121b9d0f0 by task kworker/u2:0/11 CPU: 0 PID: 11 Comm: kworker/u2:0 Tainted: GL 6.7.0-next-20240118 #521 Call Trace: dump_stack_lvl+0x2c/0x50 kasan_report+0xb6/0xf0 ext4_mb_find_good_group_avg_frag_lists+0x8a/0x200 [ext4] ext4_mb_regular_allocator+0x19e9/0x2370 [ext4] ext4_mb_new_blocks+0x88a/0x1370 [ext4] ext4_ext_map_blocks+0x14f7/0x2390 [ext4] ext4_map_blocks+0x569/0xea0 [ext4] ext4_do_writepages+0x10f6/0x1bc0 [ext4] [...] ================================================================== The flow of issue triggering is as follows: // Set s_mb_group_prealloc to 2147483647 via sysfs ext4_mb_new_blocks ext4_mb_normalize_request ext4_mb_normalize_group_request ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc ext4_mb_regular_allocator ext4_mb_choose_next_group ext4_mb_choose_next_group_best_avail mb_avg_fragment_size_order order = fls(len) - 2 = 29 ext4_mb_find_good_group_avg_frag_lists frag_list = &sbi->s_mb_avg_fragment_size[order] if (list_empty(frag_list)) // Trigger SOOB! At 4k block size, the length of the s_mb_avg_fragment_size list is 14, but an oversized s_mb_group_prealloc is set, causing slab-out-of-bounds to be triggered by an attempt to access an element at index 29. Add a new attr_id attr_clusters_in_group with values in the range [0, sbi->s_clusters_per_group] and declare mb_group_prealloc as that type to fix the issue. In addition avoid returning an order from mb_avg_fragment_size_order() greater than MB_NUM_ORDERS(sb) and reduce some useless loops. Fixes: 7e170922f06b ("ext4: Add allocation criteria 1.5 (CR1_5)") CC: stable@vger.kernel.org Signed-off-by: Baokun Li Reviewed-by: Jan Kara Reviewed-by: Ojaswin Mujoo Link: https://lore.kernel.org/r/20240319113325.3110393-5-libaokun1@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc.c | 4 ++++ fs/ext4/sysfs.c | 13 ++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 12b3f196010b8..dbf04f91516cf 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -831,6 +831,8 @@ static int mb_avg_fragment_size_order(struct super_block *sb, ext4_grpblk_t len) return 0; if (order == MB_NUM_ORDERS(sb)) order--; + if (WARN_ON_ONCE(order > MB_NUM_ORDERS(sb))) + order = MB_NUM_ORDERS(sb) - 1; return order; } @@ -1008,6 +1010,8 @@ static void ext4_mb_choose_next_group_best_avail(struct ext4_allocation_context * goal length. */ order = fls(ac->ac_g_ex.fe_len) - 1; + if (WARN_ON_ONCE(order - 1 > MB_NUM_ORDERS(ac->ac_sb))) + order = MB_NUM_ORDERS(ac->ac_sb); min_order = order - sbi->s_mb_best_avail_max_trim_order; if (min_order < 0) min_order = 0; diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c index 7f455b5f22c0f..ddd71673176c3 100644 --- a/fs/ext4/sysfs.c +++ b/fs/ext4/sysfs.c @@ -29,6 +29,7 @@ typedef enum { attr_trigger_test_error, attr_first_error_time, attr_last_error_time, + attr_clusters_in_group, attr_feature, attr_pointer_ui, attr_pointer_ul, @@ -207,13 +208,14 @@ EXT4_ATTR_FUNC(sra_exceeded_retry_limit, 0444); EXT4_ATTR_OFFSET(inode_readahead_blks, 0644, inode_readahead, ext4_sb_info, s_inode_readahead_blks); +EXT4_ATTR_OFFSET(mb_group_prealloc, 0644, clusters_in_group, + ext4_sb_info, s_mb_group_prealloc); EXT4_RW_ATTR_SBI_UI(inode_goal, s_inode_goal); EXT4_RW_ATTR_SBI_UI(mb_stats, s_mb_stats); EXT4_RW_ATTR_SBI_UI(mb_max_to_scan, s_mb_max_to_scan); EXT4_RW_ATTR_SBI_UI(mb_min_to_scan, s_mb_min_to_scan); EXT4_RW_ATTR_SBI_UI(mb_order2_req, s_mb_order2_reqs); EXT4_RW_ATTR_SBI_UI(mb_stream_req, s_mb_stream_request); -EXT4_RW_ATTR_SBI_UI(mb_group_prealloc, s_mb_group_prealloc); EXT4_RW_ATTR_SBI_UI(mb_max_linear_groups, s_mb_max_linear_groups); EXT4_RW_ATTR_SBI_UI(extent_max_zeroout_kb, s_extent_max_zeroout_kb); EXT4_ATTR(trigger_fs_error, 0200, trigger_test_error); @@ -376,6 +378,7 @@ static ssize_t ext4_generic_attr_show(struct ext4_attr *a, switch (a->attr_id) { case attr_inode_readahead: + case attr_clusters_in_group: case attr_pointer_ui: if (a->attr_ptr == ptr_ext4_super_block_offset) return sysfs_emit(buf, "%u\n", le32_to_cpup(ptr)); @@ -455,6 +458,14 @@ static ssize_t ext4_generic_attr_store(struct ext4_attr *a, else *((unsigned int *) ptr) = t; return len; + case attr_clusters_in_group: + ret = kstrtouint(skip_spaces(buf), 0, &t); + if (ret) + return ret; + if (t > sbi->s_clusters_per_group) + return -EINVAL; + *((unsigned int *) ptr) = t; + return len; case attr_pointer_ul: ret = kstrtoul(skip_spaces(buf), 0, <); if (ret) From b7b2a5799b8fafe95fcd5455c32ba2c643c86f99 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Tue, 19 Mar 2024 19:33:21 +0800 Subject: [PATCH 0987/1702] ext4: add new attr pointer attr_mb_order The s_mb_best_avail_max_trim_order is of type unsigned int, and has a range of values well beyond the normal use of the mb_order. Although the mballoc code is careful enough that large numbers don't matter there, but this can mislead the sysadmin into thinking that it's normal to set such values. Hence add a new attr_id attr_mb_order with values in the range [0, 64] to avoid storing garbage values and make us more resilient to surprises in the future. Suggested-by: Jan Kara Signed-off-by: Baokun Li Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240319113325.3110393-6-libaokun1@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/sysfs.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c index ddd71673176c3..8e04731694587 100644 --- a/fs/ext4/sysfs.c +++ b/fs/ext4/sysfs.c @@ -30,6 +30,7 @@ typedef enum { attr_first_error_time, attr_last_error_time, attr_clusters_in_group, + attr_mb_order, attr_feature, attr_pointer_ui, attr_pointer_ul, @@ -210,6 +211,8 @@ EXT4_ATTR_OFFSET(inode_readahead_blks, 0644, inode_readahead, ext4_sb_info, s_inode_readahead_blks); EXT4_ATTR_OFFSET(mb_group_prealloc, 0644, clusters_in_group, ext4_sb_info, s_mb_group_prealloc); +EXT4_ATTR_OFFSET(mb_best_avail_max_trim_order, 0644, mb_order, + ext4_sb_info, s_mb_best_avail_max_trim_order); EXT4_RW_ATTR_SBI_UI(inode_goal, s_inode_goal); EXT4_RW_ATTR_SBI_UI(mb_stats, s_mb_stats); EXT4_RW_ATTR_SBI_UI(mb_max_to_scan, s_mb_max_to_scan); @@ -225,7 +228,6 @@ EXT4_RW_ATTR_SBI_UI(warning_ratelimit_interval_ms, s_warning_ratelimit_state.int EXT4_RW_ATTR_SBI_UI(warning_ratelimit_burst, s_warning_ratelimit_state.burst); EXT4_RW_ATTR_SBI_UI(msg_ratelimit_interval_ms, s_msg_ratelimit_state.interval); EXT4_RW_ATTR_SBI_UI(msg_ratelimit_burst, s_msg_ratelimit_state.burst); -EXT4_RW_ATTR_SBI_UI(mb_best_avail_max_trim_order, s_mb_best_avail_max_trim_order); #ifdef CONFIG_EXT4_DEBUG EXT4_RW_ATTR_SBI_UL(simulate_fail, s_simulate_fail); #endif @@ -379,6 +381,7 @@ static ssize_t ext4_generic_attr_show(struct ext4_attr *a, switch (a->attr_id) { case attr_inode_readahead: case attr_clusters_in_group: + case attr_mb_order: case attr_pointer_ui: if (a->attr_ptr == ptr_ext4_super_block_offset) return sysfs_emit(buf, "%u\n", le32_to_cpup(ptr)); @@ -458,6 +461,14 @@ static ssize_t ext4_generic_attr_store(struct ext4_attr *a, else *((unsigned int *) ptr) = t; return len; + case attr_mb_order: + ret = kstrtouint(skip_spaces(buf), 0, &t); + if (ret) + return ret; + if (t > 64) + return -EINVAL; + *((unsigned int *) ptr) = t; + return len; case attr_clusters_in_group: ret = kstrtouint(skip_spaces(buf), 0, &t); if (ret) From 63bfe841053f8dda09c9d059d543486d9dc16104 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Tue, 19 Mar 2024 19:33:22 +0800 Subject: [PATCH 0988/1702] ext4: add positive int attr pointer to avoid sysfs variables overflow The following variables controlled by the sysfs interface are of type int and are normally used in the range [0, INT_MAX], but are declared as attr_pointer_ui, and thus may be set to values that exceed INT_MAX and result in overflows to get negative values. err_ratelimit_burst msg_ratelimit_burst warning_ratelimit_burst err_ratelimit_interval_ms msg_ratelimit_interval_ms warning_ratelimit_interval_ms Therefore, we add attr_pointer_pi (aka positive int attr pointer) with a value range of 0-INT_MAX to avoid overflow. Signed-off-by: Baokun Li Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240319113325.3110393-7-libaokun1@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/sysfs.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c index 8e04731694587..ddb54608ca2ef 100644 --- a/fs/ext4/sysfs.c +++ b/fs/ext4/sysfs.c @@ -32,6 +32,7 @@ typedef enum { attr_clusters_in_group, attr_mb_order, attr_feature, + attr_pointer_pi, attr_pointer_ui, attr_pointer_ul, attr_pointer_u64, @@ -180,6 +181,9 @@ static struct ext4_attr ext4_attr_##_name = { \ #define EXT4_RO_ATTR_ES_STRING(_name,_elname,_size) \ EXT4_ATTR_STRING(_name, 0444, _size, ext4_super_block, _elname) +#define EXT4_RW_ATTR_SBI_PI(_name,_elname) \ + EXT4_ATTR_OFFSET(_name, 0644, pointer_pi, ext4_sb_info, _elname) + #define EXT4_RW_ATTR_SBI_UI(_name,_elname) \ EXT4_ATTR_OFFSET(_name, 0644, pointer_ui, ext4_sb_info, _elname) @@ -222,12 +226,12 @@ EXT4_RW_ATTR_SBI_UI(mb_stream_req, s_mb_stream_request); EXT4_RW_ATTR_SBI_UI(mb_max_linear_groups, s_mb_max_linear_groups); EXT4_RW_ATTR_SBI_UI(extent_max_zeroout_kb, s_extent_max_zeroout_kb); EXT4_ATTR(trigger_fs_error, 0200, trigger_test_error); -EXT4_RW_ATTR_SBI_UI(err_ratelimit_interval_ms, s_err_ratelimit_state.interval); -EXT4_RW_ATTR_SBI_UI(err_ratelimit_burst, s_err_ratelimit_state.burst); -EXT4_RW_ATTR_SBI_UI(warning_ratelimit_interval_ms, s_warning_ratelimit_state.interval); -EXT4_RW_ATTR_SBI_UI(warning_ratelimit_burst, s_warning_ratelimit_state.burst); -EXT4_RW_ATTR_SBI_UI(msg_ratelimit_interval_ms, s_msg_ratelimit_state.interval); -EXT4_RW_ATTR_SBI_UI(msg_ratelimit_burst, s_msg_ratelimit_state.burst); +EXT4_RW_ATTR_SBI_PI(err_ratelimit_interval_ms, s_err_ratelimit_state.interval); +EXT4_RW_ATTR_SBI_PI(err_ratelimit_burst, s_err_ratelimit_state.burst); +EXT4_RW_ATTR_SBI_PI(warning_ratelimit_interval_ms, s_warning_ratelimit_state.interval); +EXT4_RW_ATTR_SBI_PI(warning_ratelimit_burst, s_warning_ratelimit_state.burst); +EXT4_RW_ATTR_SBI_PI(msg_ratelimit_interval_ms, s_msg_ratelimit_state.interval); +EXT4_RW_ATTR_SBI_PI(msg_ratelimit_burst, s_msg_ratelimit_state.burst); #ifdef CONFIG_EXT4_DEBUG EXT4_RW_ATTR_SBI_UL(simulate_fail, s_simulate_fail); #endif @@ -382,6 +386,7 @@ static ssize_t ext4_generic_attr_show(struct ext4_attr *a, case attr_inode_readahead: case attr_clusters_in_group: case attr_mb_order: + case attr_pointer_pi: case attr_pointer_ui: if (a->attr_ptr == ptr_ext4_super_block_offset) return sysfs_emit(buf, "%u\n", le32_to_cpup(ptr)); @@ -452,6 +457,14 @@ static ssize_t ext4_generic_attr_store(struct ext4_attr *a, return 0; switch (a->attr_id) { + case attr_pointer_pi: + ret = kstrtouint(skip_spaces(buf), 0, &t); + if (ret) + return ret; + if ((int)t < 0) + return -EINVAL; + *((unsigned int *) ptr) = t; + return len; case attr_pointer_ui: ret = kstrtouint(skip_spaces(buf), 0, &t); if (ret) From 9a9f3a9842927e4af7ca10c19c94dad83bebd713 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Tue, 19 Mar 2024 19:33:23 +0800 Subject: [PATCH 0989/1702] ext4: set type of ac_groups_linear_remaining to __u32 to avoid overflow Now ac_groups_linear_remaining is of type __u16 and s_mb_max_linear_groups is of type unsigned int, so an overflow occurs when setting a value above 65535 through the mb_max_linear_groups sysfs interface. Therefore, the type of ac_groups_linear_remaining is set to __u32 to avoid overflow. Fixes: 196e402adf2e ("ext4: improve cr 0 / cr 1 group scanning") CC: stable@kernel.org Signed-off-by: Baokun Li Reviewed-by: Zhang Yi Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240319113325.3110393-8-libaokun1@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ext4/mballoc.h b/fs/ext4/mballoc.h index 56938532b4ce2..7bfc5fb5a1285 100644 --- a/fs/ext4/mballoc.h +++ b/fs/ext4/mballoc.h @@ -193,8 +193,8 @@ struct ext4_allocation_context { ext4_grpblk_t ac_orig_goal_len; __u32 ac_flags; /* allocation hints */ + __u32 ac_groups_linear_remaining; __u16 ac_groups_scanned; - __u16 ac_groups_linear_remaining; __u16 ac_found; __u16 ac_cX_found[EXT4_MB_NUM_CRS]; __u16 ac_tail; From 261341a932d9244cbcd372a3659428c8723e5a49 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Tue, 19 Mar 2024 19:33:24 +0800 Subject: [PATCH 0990/1702] ext4: set the type of max_zeroout to unsigned int to avoid overflow The max_zeroout is of type int and the s_extent_max_zeroout_kb is of type uint, and the s_extent_max_zeroout_kb can be freely modified via the sysfs interface. When the block size is 1024, max_zeroout may overflow, so declare it as unsigned int to avoid overflow. Signed-off-by: Baokun Li Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240319113325.3110393-9-libaokun1@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/extents.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index e57054bdc5fd8..e067f2dd0335c 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -3402,9 +3402,10 @@ static int ext4_ext_convert_to_initialized(handle_t *handle, struct ext4_extent *ex, *abut_ex; ext4_lblk_t ee_block, eof_block; unsigned int ee_len, depth, map_len = map->m_len; - int allocated = 0, max_zeroout = 0; int err = 0; int split_flag = EXT4_EXT_DATA_VALID2; + int allocated = 0; + unsigned int max_zeroout = 0; ext_debug(inode, "logical block %llu, max_blocks %u\n", (unsigned long long)map->m_lblk, map_len); From e19089dff547c9e1f09712acc3536d7b0aa9ce3d Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Tue, 19 Mar 2024 19:33:25 +0800 Subject: [PATCH 0991/1702] ext4: clean up s_mb_rb_lock to fix build warnings with C=1 Running sparse (make C=1) on mballoc.c we get the following warning: fs/ext4/mballoc.c:3194:13: warning: context imbalance in 'ext4_mb_seq_structs_summary_start' - wrong count at exit This is because __acquires(&EXT4_SB(sb)->s_mb_rb_lock) was called in ext4_mb_seq_structs_summary_start(), but s_mb_rb_lock was removed in commit 83e80a6e3543 ("ext4: use buckets for cr 1 block scan instead of rbtree"), so remove the __acquires to silence the warning. Signed-off-by: Baokun Li Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240319113325.3110393-10-libaokun1@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index dbf04f91516cf..c65fac9b8c72f 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -3190,7 +3190,6 @@ int ext4_seq_mb_stats_show(struct seq_file *seq, void *offset) } static void *ext4_mb_seq_structs_summary_start(struct seq_file *seq, loff_t *pos) -__acquires(&EXT4_SB(sb)->s_mb_rb_lock) { struct super_block *sb = pde_data(file_inode(seq->file)); unsigned long position; From 744a56389f7398f286231e062c2e63f0de01bcc6 Mon Sep 17 00:00:00 2001 From: Justin Stitt Date: Thu, 21 Mar 2024 01:03:10 +0000 Subject: [PATCH 0992/1702] ext4: replace deprecated strncpy with alternatives strncpy() is deprecated for use on NUL-terminated destination strings [1] and as such we should prefer more robust and less ambiguous string interfaces. in file.c: s_last_mounted is marked as __nonstring meaning it does not need to be NUL-terminated. Let's instead use strtomem_pad() to copy bytes from the string source to the byte array destination -- while also ensuring to pad with zeroes. in ioctl.c: We can drop the memset and size argument in favor of using the new 2-argument version of strscpy_pad() -- which was introduced with Commit e6584c3964f2f ("string: Allow 2-argument strscpy()"). This guarantees NUL-termination and NUL-padding on the destination buffer -- which seems to be a requirement judging from this comment: | static int ext4_ioctl_getlabel(struct ext4_sb_info *sbi, char __user *user_label) | { | char label[EXT4_LABEL_MAX + 1]; | | /* | * EXT4_LABEL_MAX must always be smaller than FSLABEL_MAX because | * FSLABEL_MAX must include terminating null byte, while s_volume_name | * does not have to. | */ in super.c: s_first_error_func is marked as __nonstring meaning we can take the same approach as in file.c; just use strtomem_pad() Link: https://www.kernel.org/doc/html/latest/process/deprecated.html#strncpy-on-nul-terminated-strings [1] Link: https://manpages.debian.org/testing/linux-manual-4.8/strscpy.9.en.html [2] Link: https://github.com/KSPP/linux/issues/90 Cc: linux-hardening@vger.kernel.org Signed-off-by: Justin Stitt Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20240321-strncpy-fs-ext4-file-c-v1-1-36a6a09fef0c@google.com Signed-off-by: Theodore Ts'o --- fs/ext4/file.c | 3 +-- fs/ext4/ioctl.c | 3 +-- fs/ext4/super.c | 7 +++---- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/fs/ext4/file.c b/fs/ext4/file.c index 965febab1d048..77529c655f958 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -844,8 +844,7 @@ static int ext4_sample_last_mounted(struct super_block *sb, if (err) goto out_journal; lock_buffer(sbi->s_sbh); - strncpy(sbi->s_es->s_last_mounted, cp, - sizeof(sbi->s_es->s_last_mounted)); + strtomem_pad(sbi->s_es->s_last_mounted, cp, 0); ext4_superblock_csum_set(sb); unlock_buffer(sbi->s_sbh); ext4_handle_dirty_metadata(handle, NULL, sbi->s_sbh); diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index 7160a71044c88..dab7acd497092 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -1150,9 +1150,8 @@ static int ext4_ioctl_getlabel(struct ext4_sb_info *sbi, char __user *user_label */ BUILD_BUG_ON(EXT4_LABEL_MAX >= FSLABEL_MAX); - memset(label, 0, sizeof(label)); lock_buffer(sbi->s_sbh); - strncpy(label, sbi->s_es->s_volume_name, EXT4_LABEL_MAX); + strscpy_pad(label, sbi->s_es->s_volume_name); unlock_buffer(sbi->s_sbh); if (copy_to_user(user_label, label, sizeof(label))) diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 9bde300b15513..f9b2b18374f3d 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -6129,8 +6129,8 @@ static void ext4_update_super(struct super_block *sb) __ext4_update_tstamp(&es->s_first_error_time, &es->s_first_error_time_hi, sbi->s_first_error_time); - strncpy(es->s_first_error_func, sbi->s_first_error_func, - sizeof(es->s_first_error_func)); + strtomem_pad(es->s_first_error_func, + sbi->s_first_error_func, 0); es->s_first_error_line = cpu_to_le32(sbi->s_first_error_line); es->s_first_error_ino = @@ -6143,8 +6143,7 @@ static void ext4_update_super(struct super_block *sb) __ext4_update_tstamp(&es->s_last_error_time, &es->s_last_error_time_hi, sbi->s_last_error_time); - strncpy(es->s_last_error_func, sbi->s_last_error_func, - sizeof(es->s_last_error_func)); + strtomem_pad(es->s_last_error_func, sbi->s_last_error_func, 0); es->s_last_error_line = cpu_to_le32(sbi->s_last_error_line); es->s_last_error_ino = cpu_to_le32(sbi->s_last_error_ino); es->s_last_error_block = cpu_to_le64(sbi->s_last_error_block); From 4f3e6db3c3719952cfef89340290e0b7b03f7cbc Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 21 Mar 2024 17:26:49 +0100 Subject: [PATCH 0993/1702] Revert "ext4: drop duplicate ea_inode handling in ext4_xattr_block_set()" This reverts commit 7f48212678e91a057259b3e281701f7feb1ee397. We will need the special cleanup handling once we move allocation of EA inode outside of the buffer lock in the following patch. Signed-off-by: Jan Kara Link: https://lore.kernel.org/r/20240321162657.27420-1-jack@suse.cz Signed-off-by: Theodore Ts'o --- fs/ext4/xattr.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index b67a176bfcf9f..146690c10c73f 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -2158,6 +2158,17 @@ ext4_xattr_block_set(handle_t *handle, struct inode *inode, ENTRY(header(s->base)+1)); if (error) goto getblk_failed; + if (ea_inode) { + /* Drop the extra ref on ea_inode. */ + error = ext4_xattr_inode_dec_ref(handle, + ea_inode); + if (error) + ext4_warning_inode(ea_inode, + "dec ref error=%d", + error); + iput(ea_inode); + ea_inode = NULL; + } lock_buffer(new_bh); error = ext4_journal_get_create_access(handle, sb, From 0a46ef234756dca04623b7591e8ebb3440622f0b Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 21 Mar 2024 17:26:50 +0100 Subject: [PATCH 0994/1702] ext4: do not create EA inode under buffer lock ext4_xattr_set_entry() creates new EA inodes while holding buffer lock on the external xattr block. This is problematic as it nests all the allocation locking (which acquires locks on other buffers) under the buffer lock. This can even deadlock when the filesystem is corrupted and e.g. quota file is setup to contain xattr block as data block. Move the allocation of EA inode out of ext4_xattr_set_entry() into the callers. Reported-by: syzbot+a43d4f48b8397d0e41a9@syzkaller.appspotmail.com Signed-off-by: Jan Kara Link: https://lore.kernel.org/r/20240321162657.27420-2-jack@suse.cz Signed-off-by: Theodore Ts'o --- fs/ext4/xattr.c | 113 +++++++++++++++++++++++------------------------- 1 file changed, 53 insertions(+), 60 deletions(-) diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index 146690c10c73f..04f90df8dbae9 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -1619,6 +1619,7 @@ static struct inode *ext4_xattr_inode_lookup_create(handle_t *handle, static int ext4_xattr_set_entry(struct ext4_xattr_info *i, struct ext4_xattr_search *s, handle_t *handle, struct inode *inode, + struct inode *new_ea_inode, bool is_block) { struct ext4_xattr_entry *last, *next; @@ -1626,7 +1627,6 @@ static int ext4_xattr_set_entry(struct ext4_xattr_info *i, size_t min_offs = s->end - s->base, name_len = strlen(i->name); int in_inode = i->in_inode; struct inode *old_ea_inode = NULL; - struct inode *new_ea_inode = NULL; size_t old_size, new_size; int ret; @@ -1711,38 +1711,11 @@ static int ext4_xattr_set_entry(struct ext4_xattr_info *i, old_ea_inode = NULL; goto out; } - } - if (i->value && in_inode) { - WARN_ON_ONCE(!i->value_len); - - new_ea_inode = ext4_xattr_inode_lookup_create(handle, inode, - i->value, i->value_len); - if (IS_ERR(new_ea_inode)) { - ret = PTR_ERR(new_ea_inode); - new_ea_inode = NULL; - goto out; - } - } - if (old_ea_inode) { /* We are ready to release ref count on the old_ea_inode. */ ret = ext4_xattr_inode_dec_ref(handle, old_ea_inode); - if (ret) { - /* Release newly required ref count on new_ea_inode. */ - if (new_ea_inode) { - int err; - - err = ext4_xattr_inode_dec_ref(handle, - new_ea_inode); - if (err) - ext4_warning_inode(new_ea_inode, - "dec ref new_ea_inode err=%d", - err); - ext4_xattr_inode_free_quota(inode, new_ea_inode, - i->value_len); - } + if (ret) goto out; - } ext4_xattr_inode_free_quota(inode, old_ea_inode, le32_to_cpu(here->e_value_size)); @@ -1866,7 +1839,6 @@ static int ext4_xattr_set_entry(struct ext4_xattr_info *i, ret = 0; out: iput(old_ea_inode); - iput(new_ea_inode); return ret; } @@ -1929,9 +1901,21 @@ ext4_xattr_block_set(handle_t *handle, struct inode *inode, size_t old_ea_inode_quota = 0; unsigned int ea_ino; - #define header(x) ((struct ext4_xattr_header *)(x)) + /* If we need EA inode, prepare it before locking the buffer */ + if (i->value && i->in_inode) { + WARN_ON_ONCE(!i->value_len); + + ea_inode = ext4_xattr_inode_lookup_create(handle, inode, + i->value, i->value_len); + if (IS_ERR(ea_inode)) { + error = PTR_ERR(ea_inode); + ea_inode = NULL; + goto cleanup; + } + } + if (s->base) { int offset = (char *)s->here - bs->bh->b_data; @@ -1940,6 +1924,7 @@ ext4_xattr_block_set(handle_t *handle, struct inode *inode, EXT4_JTR_NONE); if (error) goto cleanup; + lock_buffer(bs->bh); if (header(s->base)->h_refcount == cpu_to_le32(1)) { @@ -1966,7 +1951,7 @@ ext4_xattr_block_set(handle_t *handle, struct inode *inode, } ea_bdebug(bs->bh, "modifying in-place"); error = ext4_xattr_set_entry(i, s, handle, inode, - true /* is_block */); + ea_inode, true /* is_block */); ext4_xattr_block_csum_set(inode, bs->bh); unlock_buffer(bs->bh); if (error == -EFSCORRUPTED) @@ -2034,29 +2019,13 @@ ext4_xattr_block_set(handle_t *handle, struct inode *inode, s->end = s->base + sb->s_blocksize; } - error = ext4_xattr_set_entry(i, s, handle, inode, true /* is_block */); + error = ext4_xattr_set_entry(i, s, handle, inode, ea_inode, + true /* is_block */); if (error == -EFSCORRUPTED) goto bad_block; if (error) goto cleanup; - if (i->value && s->here->e_value_inum) { - /* - * A ref count on ea_inode has been taken as part of the call to - * ext4_xattr_set_entry() above. We would like to drop this - * extra ref but we have to wait until the xattr block is - * initialized and has its own ref count on the ea_inode. - */ - ea_ino = le32_to_cpu(s->here->e_value_inum); - error = ext4_xattr_inode_iget(inode, ea_ino, - le32_to_cpu(s->here->e_hash), - &ea_inode); - if (error) { - ea_inode = NULL; - goto cleanup; - } - } - inserted: if (!IS_LAST_ENTRY(s->first)) { new_bh = ext4_xattr_block_cache_find(inode, header(s->base), @@ -2209,17 +2178,16 @@ ext4_xattr_block_set(handle_t *handle, struct inode *inode, cleanup: if (ea_inode) { - int error2; - - error2 = ext4_xattr_inode_dec_ref(handle, ea_inode); - if (error2) - ext4_warning_inode(ea_inode, "dec ref error=%d", - error2); + if (error) { + int error2; - /* If there was an error, revert the quota charge. */ - if (error) + error2 = ext4_xattr_inode_dec_ref(handle, ea_inode); + if (error2) + ext4_warning_inode(ea_inode, "dec ref error=%d", + error2); ext4_xattr_inode_free_quota(inode, ea_inode, i_size_read(ea_inode)); + } iput(ea_inode); } if (ce) @@ -2277,14 +2245,38 @@ int ext4_xattr_ibody_set(handle_t *handle, struct inode *inode, { struct ext4_xattr_ibody_header *header; struct ext4_xattr_search *s = &is->s; + struct inode *ea_inode = NULL; int error; if (!EXT4_INODE_HAS_XATTR_SPACE(inode)) return -ENOSPC; - error = ext4_xattr_set_entry(i, s, handle, inode, false /* is_block */); - if (error) + /* If we need EA inode, prepare it before locking the buffer */ + if (i->value && i->in_inode) { + WARN_ON_ONCE(!i->value_len); + + ea_inode = ext4_xattr_inode_lookup_create(handle, inode, + i->value, i->value_len); + if (IS_ERR(ea_inode)) + return PTR_ERR(ea_inode); + } + error = ext4_xattr_set_entry(i, s, handle, inode, ea_inode, + false /* is_block */); + if (error) { + if (ea_inode) { + int error2; + + error2 = ext4_xattr_inode_dec_ref(handle, ea_inode); + if (error2) + ext4_warning_inode(ea_inode, "dec ref error=%d", + error2); + + ext4_xattr_inode_free_quota(inode, ea_inode, + i_size_read(ea_inode)); + iput(ea_inode); + } return error; + } header = IHDR(inode, ext4_raw_inode(&is->iloc)); if (!IS_LAST_ENTRY(s->first)) { header->h_magic = cpu_to_le32(EXT4_XATTR_MAGIC); @@ -2293,6 +2285,7 @@ int ext4_xattr_ibody_set(handle_t *handle, struct inode *inode, header->h_magic = cpu_to_le32(0); ext4_clear_inode_state(inode, EXT4_STATE_XATTR); } + iput(ea_inode); return 0; } From a11adf7be9d8baefe798eab49c356ab8e3924f0e Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Sat, 23 Mar 2024 00:55:18 +0800 Subject: [PATCH 0995/1702] ext4: implement filesystem specific alloc_inode in unit test We expect inode with ext4_info_info type as following: mbt_kunit_init mbt_mb_init ext4_mb_init ext4_mb_init_backend sbi->s_buddy_cache = new_inode(sb); EXT4_I(sbi->s_buddy_cache)->i_disksize = 0; Implement alloc_inode ionde with ext4_inode_info type to avoid out-of-bounds write. Signed-off-by: Kemeng Shi Reported-by: Guenter Roeck Tested-by: Guenter Roeck Link: https://lore.kernel.org/r/20240322165518.8147-1-shikemeng@huaweicloud.com Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc-test.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/fs/ext4/mballoc-test.c b/fs/ext4/mballoc-test.c index 044ca5238f419..49aabcfe6b46d 100644 --- a/fs/ext4/mballoc-test.c +++ b/fs/ext4/mballoc-test.c @@ -30,7 +30,31 @@ struct mbt_ext4_super_block { #define MBT_CTX(_sb) (&MBT_SB(_sb)->mbt_ctx) #define MBT_GRP_CTX(_sb, _group) (&MBT_CTX(_sb)->grp_ctx[_group]) +static struct inode *mbt_alloc_inode(struct super_block *sb) +{ + struct ext4_inode_info *ei; + + ei = kmalloc(sizeof(struct ext4_inode_info), GFP_KERNEL); + if (!ei) + return NULL; + + INIT_LIST_HEAD(&ei->i_orphan); + init_rwsem(&ei->xattr_sem); + init_rwsem(&ei->i_data_sem); + inode_init_once(&ei->vfs_inode); + ext4_fc_init_inode(&ei->vfs_inode); + + return &ei->vfs_inode; +} + +static void mbt_free_inode(struct inode *inode) +{ + kfree(EXT4_I(inode)); +} + static const struct super_operations mbt_sops = { + .alloc_inode = mbt_alloc_inode, + .free_inode = mbt_free_inode, }; static void mbt_kill_sb(struct super_block *sb) From 9c97c34a998a01bc0cf970b1685456bc2ea16b64 Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Wed, 24 Apr 2024 14:19:00 +0800 Subject: [PATCH 0996/1702] ext4: keep "prefetch_grp" and "nr" consistent Keep "prefetch_grp" and "nr" consistent to avoid to call ext4_mb_prefetch_fini with non-prefetched groups. When we step into next criteria, "prefetch_grp" is set to prefetch start of new criteria while "nr" is number of the prefetched group in previous criteria. If previous criteria and next criteria are both inexpensive (< CR_GOAL_LEN_SLOW) and prefetch_ios reachs sbi->s_mb_prefetch_limit in previous criteria, "prefetch_grp" and "nr" will be inconsistent and may introduce unexpected cost to do ext4_mb_init_group for non-prefetched groups. Reset "nr" to 0 when we reset "prefetch_grp" to goal group to keep them consistent. Signed-off-by: Kemeng Shi Reviewed-by: Ojaswin Mujoo Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240424061904.987525-2-shikemeng@huaweicloud.com Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index c65fac9b8c72f..33e6a3ebdb55f 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -2860,6 +2860,7 @@ ext4_mb_regular_allocator(struct ext4_allocation_context *ac) group = ac->ac_g_ex.fe_group; ac->ac_groups_linear_remaining = sbi->s_mb_max_linear_groups; prefetch_grp = group; + nr = 0; for (i = 0, new_cr = cr; i < ngroups; i++, ext4_mb_choose_next_group(ac, &new_cr, &group, ngroups)) { From d0b88624f81f272626c767f4b715f4b690fcbed2 Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Wed, 24 Apr 2024 14:19:01 +0800 Subject: [PATCH 0997/1702] ext4: add test_mb_mark_used_cost to estimate cost of mb_mark_used Add test_mb_mark_used_cost to estimate cost of mb_mark_used Signed-off-by: Kemeng Shi Link: https://lore.kernel.org/r/20240424061904.987525-3-shikemeng@huaweicloud.com Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc-test.c | 52 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/fs/ext4/mballoc-test.c b/fs/ext4/mballoc-test.c index 49aabcfe6b46d..bb2a223b207c1 100644 --- a/fs/ext4/mballoc-test.c +++ b/fs/ext4/mballoc-test.c @@ -883,6 +883,56 @@ static void test_mb_free_blocks(struct kunit *test) ext4_mb_unload_buddy(&e4b); } +#define COUNT_FOR_ESTIMATE 100000 +static void test_mb_mark_used_cost(struct kunit *test) +{ + struct ext4_buddy e4b; + struct super_block *sb = (struct super_block *)test->priv; + struct ext4_free_extent ex; + int ret; + struct test_range ranges[TEST_RANGE_COUNT]; + int i, j; + unsigned long start, end, all = 0; + + /* buddy cache assumes that each page contains at least one block */ + if (sb->s_blocksize > PAGE_SIZE) + kunit_skip(test, "blocksize exceeds pagesize"); + + ret = ext4_mb_load_buddy(sb, TEST_GOAL_GROUP, &e4b); + KUNIT_ASSERT_EQ(test, ret, 0); + + ex.fe_group = TEST_GOAL_GROUP; + for (j = 0; j < COUNT_FOR_ESTIMATE; j++) { + mbt_generate_test_ranges(sb, ranges, TEST_RANGE_COUNT); + start = jiffies; + for (i = 0; i < TEST_RANGE_COUNT; i++) { + if (ranges[i].len == 0) + continue; + + ex.fe_start = ranges[i].start; + ex.fe_len = ranges[i].len; + ext4_lock_group(sb, TEST_GOAL_GROUP); + mb_mark_used(&e4b, &ex); + ext4_unlock_group(sb, TEST_GOAL_GROUP); + } + end = jiffies; + all += (end - start); + + for (i = 0; i < TEST_RANGE_COUNT; i++) { + if (ranges[i].len == 0) + continue; + + ext4_lock_group(sb, TEST_GOAL_GROUP); + mb_free_blocks(NULL, &e4b, ranges[i].start, + ranges[i].len); + ext4_unlock_group(sb, TEST_GOAL_GROUP); + } + } + + kunit_info(test, "costed jiffies %lu\n", all); + ext4_mb_unload_buddy(&e4b); +} + static const struct mbt_ext4_block_layout mbt_test_layouts[] = { { .blocksize_bits = 10, @@ -925,6 +975,8 @@ static struct kunit_case mbt_test_cases[] = { KUNIT_CASE_PARAM(test_mb_mark_used, mbt_layouts_gen_params), KUNIT_CASE_PARAM(test_mb_free_blocks, mbt_layouts_gen_params), KUNIT_CASE_PARAM(test_mark_diskspace_used, mbt_layouts_gen_params), + KUNIT_CASE_PARAM_ATTR(test_mb_mark_used_cost, mbt_layouts_gen_params, + { .speed = KUNIT_SPEED_SLOW }), {} }; From d1a3924e43a35860ed7edaeec7f901a1ade2ac37 Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Wed, 24 Apr 2024 14:19:02 +0800 Subject: [PATCH 0998/1702] ext4: call ext4_mb_mark_free_simple to free continuous bits in found chunk In mb_mark_used, we will find free chunk and mark it inuse. For chunk in mid of passed range, we could simply mark whole chunk inuse. For chunk at end of range, we may need to mark a continuous bits at end of part of chunk inuse and keep rest part of chunk free. To only mark a part of chunk inuse, we firstly mark whole chunk inuse and then mark a continuous range at end of chunk free. Function mb_mark_used does several times of "mb_find_buddy; mb_clear_bit; ..." to mark a continuous range free which can be done by simply calling ext4_mb_mark_free_simple which free continuous bits in a more effective way. Just call ext4_mb_mark_free_simple in mb_mark_used to use existing and effective code to free continuous blocks in chunk at end of passed range. Signed-off-by: Kemeng Shi Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240424061904.987525-4-shikemeng@huaweicloud.com Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 33e6a3ebdb55f..9a2b0bd0dcbc6 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -2044,13 +2044,12 @@ static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex) int ord; int mlen = 0; int max = 0; - int cur; int start = ex->fe_start; int len = ex->fe_len; unsigned ret = 0; int len0 = len; void *buddy; - bool split = false; + int ord_start, ord_end; BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3)); BUG_ON(e4b->bd_group != ex->fe_group); @@ -2075,16 +2074,12 @@ static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex) /* let's maintain buddy itself */ while (len) { - if (!split) - ord = mb_find_order_for_block(e4b, start); + ord = mb_find_order_for_block(e4b, start); if (((start >> ord) << ord) == start && len >= (1 << ord)) { /* the whole chunk may be allocated at once! */ mlen = 1 << ord; - if (!split) - buddy = mb_find_buddy(e4b, ord, &max); - else - split = false; + buddy = mb_find_buddy(e4b, ord, &max); BUG_ON((start >> ord) >= max); mb_set_bit(start >> ord, buddy); e4b->bd_info->bb_counters[ord]--; @@ -2098,20 +2093,29 @@ static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex) if (ret == 0) ret = len | (ord << 16); - /* we have to split large buddy */ BUG_ON(ord <= 0); buddy = mb_find_buddy(e4b, ord, &max); mb_set_bit(start >> ord, buddy); e4b->bd_info->bb_counters[ord]--; - ord--; - cur = (start >> ord) & ~1U; - buddy = mb_find_buddy(e4b, ord, &max); - mb_clear_bit(cur, buddy); - mb_clear_bit(cur + 1, buddy); - e4b->bd_info->bb_counters[ord]++; - e4b->bd_info->bb_counters[ord]++; - split = true; + ord_start = (start >> ord) << ord; + ord_end = ord_start + (1 << ord); + /* first chunk */ + if (start > ord_start) + ext4_mb_mark_free_simple(e4b->bd_sb, e4b->bd_buddy, + ord_start, start - ord_start, + e4b->bd_info); + + /* last chunk */ + if (start + len < ord_end) { + ext4_mb_mark_free_simple(e4b->bd_sb, e4b->bd_buddy, + start + len, + ord_end - (start + len), + e4b->bd_info); + break; + } + len = start + len - ord_end; + start = ord_end; } mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info); From 2caffb6a277bb0f2a482a2eb824d012d5f45f4d0 Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Wed, 24 Apr 2024 14:19:03 +0800 Subject: [PATCH 0999/1702] ext4: use correct criteria name instead stale integer number in comment Use correct criteria name instead stale integer number in comment Signed-off-by: Kemeng Shi Reviewed-by: Ojaswin Mujoo Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240424061904.987525-5-shikemeng@huaweicloud.com Signed-off-by: Theodore Ts'o --- fs/ext4/ext4.h | 9 ++++++--- fs/ext4/mballoc.c | 16 +++++++++------- fs/ext4/mballoc.h | 4 ++-- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 8d126654019ef..983dad8c07ecd 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -213,11 +213,14 @@ enum criteria { #define EXT4_MB_USE_RESERVED 0x2000 /* Do strict check for free blocks while retrying block allocation */ #define EXT4_MB_STRICT_CHECK 0x4000 -/* Large fragment size list lookup succeeded at least once for cr = 0 */ +/* Large fragment size list lookup succeeded at least once for + * CR_POWER2_ALIGNED */ #define EXT4_MB_CR_POWER2_ALIGNED_OPTIMIZED 0x8000 -/* Avg fragment size rb tree lookup succeeded at least once for cr = 1 */ +/* Avg fragment size rb tree lookup succeeded at least once for + * CR_GOAL_LEN_FAST */ #define EXT4_MB_CR_GOAL_LEN_FAST_OPTIMIZED 0x00010000 -/* Avg fragment size rb tree lookup succeeded at least once for cr = 1.5 */ +/* Avg fragment size rb tree lookup succeeded at least once for + * CR_BEST_AVAIL_LEN */ #define EXT4_MB_CR_BEST_AVAIL_LEN_OPTIMIZED 0x00020000 struct ext4_allocation_request { diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 9a2b0bd0dcbc6..65ce2dd0cefb7 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -1135,8 +1135,9 @@ static void ext4_mb_choose_next_group(struct ext4_allocation_context *ac, ext4_mb_choose_next_group_best_avail(ac, new_cr, group); } else { /* - * TODO: For CR=2, we can arrange groups in an rb tree sorted by - * bb_free. But until that happens, we should never come here. + * TODO: For CR_GOAL_LEN_SLOW, we can arrange groups in an + * rb tree sorted by bb_free. But until that happens, we should + * never come here. */ WARN_ON(1); } @@ -2683,7 +2684,7 @@ static int ext4_mb_good_group_nolock(struct ext4_allocation_context *ac, int ret; /* - * cr=CR_POWER2_ALIGNED/CR_GOAL_LEN_FAST is a very optimistic + * CR_POWER2_ALIGNED/CR_GOAL_LEN_FAST is a very optimistic * search to find large good chunks almost for free. If buddy * data is not ready, then this optimization makes no sense. But * we never skip the first block group in a flex_bg, since this @@ -3448,10 +3449,11 @@ static int ext4_mb_init_backend(struct super_block *sb) } if (sbi->s_mb_prefetch > ext4_get_groups_count(sb)) sbi->s_mb_prefetch = ext4_get_groups_count(sb); - /* now many real IOs to prefetch within a single allocation at cr=0 - * given cr=0 is an CPU-related optimization we shouldn't try to - * load too many groups, at some point we should start to use what - * we've got in memory. + /* + * now many real IOs to prefetch within a single allocation at + * CR_POWER2_ALIGNED. Given CR_POWER2_ALIGNED is an CPU-related + * optimization we shouldn't try to load too many groups, at some point + * we should start to use what we've got in memory. * with an average random access time 5ms, it'd take a second to get * 200 groups (* N with flex_bg), so let's make this limit 4 */ diff --git a/fs/ext4/mballoc.h b/fs/ext4/mballoc.h index 7bfc5fb5a1285..7ee13d8eecd79 100644 --- a/fs/ext4/mballoc.h +++ b/fs/ext4/mballoc.h @@ -187,8 +187,8 @@ struct ext4_allocation_context { struct ext4_free_extent ac_f_ex; /* - * goal len can change in CR1.5, so save the original len. This is - * used while adjusting the PA window and for accounting. + * goal len can change in CR_BEST_AVAIL_LEN, so save the original len. + * This is used while adjusting the PA window and for accounting. */ ext4_grpblk_t ac_orig_goal_len; From da5704eef7037a5bc84a56519729d93d10a0e0a0 Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Wed, 24 Apr 2024 14:19:04 +0800 Subject: [PATCH 1000/1702] ext4: open coding repeated check in next_linear_group Open coding repeated check in next_linear_group. Signed-off-by: Kemeng Shi Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240424061904.987525-6-shikemeng@huaweicloud.com Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 65ce2dd0cefb7..e89c1bfb7ce31 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -1080,23 +1080,11 @@ static inline int should_optimize_scan(struct ext4_allocation_context *ac) } /* - * Return next linear group for allocation. If linear traversal should not be - * performed, this function just returns the same group + * Return next linear group for allocation. */ static ext4_group_t -next_linear_group(struct ext4_allocation_context *ac, ext4_group_t group, - ext4_group_t ngroups) +next_linear_group(ext4_group_t group, ext4_group_t ngroups) { - if (!should_optimize_scan(ac)) - goto inc_and_return; - - if (ac->ac_groups_linear_remaining) { - ac->ac_groups_linear_remaining--; - goto inc_and_return; - } - - return group; -inc_and_return: /* * Artificially restricted ngroups for non-extent * files makes group > ngroups possible on first loop. @@ -1122,8 +1110,19 @@ static void ext4_mb_choose_next_group(struct ext4_allocation_context *ac, { *new_cr = ac->ac_criteria; - if (!should_optimize_scan(ac) || ac->ac_groups_linear_remaining) { - *group = next_linear_group(ac, *group, ngroups); + if (!should_optimize_scan(ac)) { + *group = next_linear_group(*group, ngroups); + return; + } + + /* + * Optimized scanning can return non adjacent groups which can cause + * seek overhead for rotational disks. So try few linear groups before + * trying optimized scan. + */ + if (ac->ac_groups_linear_remaining) { + *group = next_linear_group(*group, ngroups); + ac->ac_groups_linear_remaining--; return; } From 45cf976008ddef4a9c9a30310c9b4fb2a9a6602a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 30 Apr 2024 06:07:55 +0200 Subject: [PATCH 1001/1702] xfs: fix log recovery buffer allocation for the legacy h_size fixup Commit a70f9fe52daa ("xfs: detect and handle invalid iclog size set by mkfs") added a fixup for incorrect h_size values used for the initial umount record in old xfsprogs versions. Later commit 0c771b99d6c9 ("xfs: clean up calculation of LR header blocks") cleaned up the log reover buffer calculation, but stoped using the fixed up h_size value to size the log recovery buffer, which can lead to an out of bounds access when the incorrect h_size does not come from the old mkfs tool, but a fuzzer. Fix this by open coding xlog_logrec_hblks and taking the fixed h_size into account for this calculation. Fixes: 0c771b99d6c9 ("xfs: clean up calculation of LR header blocks") Reported-by: Sam Sun Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/xfs_log_recover.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index b445e8ce4a7d2..bb8957927c3c2 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2999,7 +2999,7 @@ xlog_do_recovery_pass( int error = 0, h_size, h_len; int error2 = 0; int bblks, split_bblks; - int hblks, split_hblks, wrapped_hblks; + int hblks = 1, split_hblks, wrapped_hblks; int i; struct hlist_head rhash[XLOG_RHASH_SIZE]; LIST_HEAD (buffer_list); @@ -3055,14 +3055,22 @@ xlog_do_recovery_pass( if (error) goto bread_err1; - hblks = xlog_logrec_hblks(log, rhead); - if (hblks != 1) { - kvfree(hbp); - hbp = xlog_alloc_buffer(log, hblks); + /* + * This open codes xlog_logrec_hblks so that we can reuse the + * fixed up h_size value calculated above. Without that we'd + * still allocate the buffer based on the incorrect on-disk + * size. + */ + if (h_size > XLOG_HEADER_CYCLE_SIZE && + (rhead->h_version & cpu_to_be32(XLOG_VERSION_2))) { + hblks = DIV_ROUND_UP(h_size, XLOG_HEADER_CYCLE_SIZE); + if (hblks > 1) { + kvfree(hbp); + hbp = xlog_alloc_buffer(log, hblks); + } } } else { ASSERT(log->l_sectBBsize == 1); - hblks = 1; hbp = xlog_alloc_buffer(log, 1); h_size = XLOG_BIG_RECORD_BSIZE; } From 67a841f9d7246e45dd8953dc936776132d90a048 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 30 Apr 2024 06:07:56 +0200 Subject: [PATCH 1002/1702] xfs: clean up buffer allocation in xlog_do_recovery_pass Merge the initial xlog_alloc_buffer calls, and pass the variable designating the length that is initialized to 1 above instead of passing the open coded 1 directly. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/xfs_log_recover.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index bb8957927c3c2..4fe627991e865 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -3010,6 +3010,10 @@ xlog_do_recovery_pass( for (i = 0; i < XLOG_RHASH_SIZE; i++) INIT_HLIST_HEAD(&rhash[i]); + hbp = xlog_alloc_buffer(log, hblks); + if (!hbp) + return -ENOMEM; + /* * Read the header of the tail block and get the iclog buffer size from * h_size. Use this to tell how many sectors make up the log header. @@ -3020,10 +3024,6 @@ xlog_do_recovery_pass( * iclog header and extract the header size from it. Get a * new hbp that is the correct size. */ - hbp = xlog_alloc_buffer(log, 1); - if (!hbp) - return -ENOMEM; - error = xlog_bread(log, tail_blk, 1, hbp, &offset); if (error) goto bread_err1; @@ -3067,16 +3067,15 @@ xlog_do_recovery_pass( if (hblks > 1) { kvfree(hbp); hbp = xlog_alloc_buffer(log, hblks); + if (!hbp) + return -ENOMEM; } } } else { ASSERT(log->l_sectBBsize == 1); - hbp = xlog_alloc_buffer(log, 1); h_size = XLOG_BIG_RECORD_BSIZE; } - if (!hbp) - return -ENOMEM; dbp = xlog_alloc_buffer(log, BTOBB(h_size)); if (!dbp) { kvfree(hbp); From f7b9ee784511920545558db85da24d022fb2805f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 30 Apr 2024 14:58:21 +0200 Subject: [PATCH 1003/1702] xfs: consolidate the xfs_quota_reserve_blkres definitions xfs_trans_reserve_quota_nblks is already stubbed out if quota support is disabled, no need for an extra xfs_quota_reserve_blkres stub. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/xfs_quota.h | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/fs/xfs/xfs_quota.h b/fs/xfs/xfs_quota.h index 85a4ae1a17f67..621ea9d7cf06d 100644 --- a/fs/xfs/xfs_quota.h +++ b/fs/xfs/xfs_quota.h @@ -123,12 +123,6 @@ extern int xfs_qm_newmount(struct xfs_mount *, uint *, uint *); extern void xfs_qm_mount_quotas(struct xfs_mount *); extern void xfs_qm_unmount(struct xfs_mount *); extern void xfs_qm_unmount_quotas(struct xfs_mount *); - -static inline int -xfs_quota_reserve_blkres(struct xfs_inode *ip, int64_t blocks) -{ - return xfs_trans_reserve_quota_nblks(NULL, ip, blocks, 0, false); -} bool xfs_inode_near_dquot_enforcement(struct xfs_inode *ip, xfs_dqtype_t type); # ifdef CONFIG_XFS_LIVE_HOOKS @@ -187,12 +181,6 @@ static inline int xfs_trans_reserve_quota_bydquots(struct xfs_trans *tp, return 0; } -static inline int -xfs_quota_reserve_blkres(struct xfs_inode *ip, int64_t blocks) -{ - return 0; -} - static inline int xfs_trans_reserve_quota_icreate(struct xfs_trans *tp, struct xfs_dquot *udqp, struct xfs_dquot *gdqp, struct xfs_dquot *pdqp, int64_t dblocks) @@ -221,6 +209,12 @@ xfs_trans_reserve_quota_icreate(struct xfs_trans *tp, struct xfs_dquot *udqp, #endif /* CONFIG_XFS_QUOTA */ +static inline int +xfs_quota_reserve_blkres(struct xfs_inode *ip, int64_t blocks) +{ + return xfs_trans_reserve_quota_nblks(NULL, ip, blocks, 0, false); +} + static inline int xfs_quota_unreserve_blkres(struct xfs_inode *ip, int64_t blocks) { From cc3c92e7e79eb5f7f3ec4d5790ade384b7d294f7 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 30 Apr 2024 14:58:22 +0200 Subject: [PATCH 1004/1702] xfs: xfs_quota_unreserve_blkres can't fail Unreserving quotas can't fail due to quota limits, and we'll notice a shut down file system a bit later in all the callers anyway. Return void and remove the error checking and propagation in the callers. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_bmap.c | 16 +++++----------- fs/xfs/libxfs/xfs_bmap.h | 2 +- fs/xfs/xfs_aops.c | 6 +----- fs/xfs/xfs_bmap_util.c | 9 +++------ fs/xfs/xfs_bmap_util.h | 2 +- fs/xfs/xfs_iomap.c | 4 ++-- fs/xfs/xfs_quota.h | 7 ++++--- fs/xfs/xfs_reflink.c | 11 +++-------- 8 files changed, 20 insertions(+), 37 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index be515bac8ea1e..0633f285875de 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4928,7 +4928,7 @@ xfs_bmap_split_indlen( *indlen2 = len2; } -int +void xfs_bmap_del_extent_delay( struct xfs_inode *ip, int whichfork, @@ -4944,7 +4944,6 @@ xfs_bmap_del_extent_delay( xfs_filblks_t got_indlen, new_indlen, stolen = 0; uint32_t state = xfs_bmap_fork_to_state(whichfork); uint64_t fdblocks; - int error = 0; bool isrt; XFS_STATS_INC(mp, xs_del_exlist); @@ -4964,9 +4963,7 @@ xfs_bmap_del_extent_delay( * sb counters as we might have to borrow some blocks for the * indirect block accounting. */ - error = xfs_quota_unreserve_blkres(ip, del->br_blockcount); - if (error) - return error; + xfs_quota_unreserve_blkres(ip, del->br_blockcount); ip->i_delayed_blks -= del->br_blockcount; if (got->br_startoff == del->br_startoff) @@ -5064,7 +5061,6 @@ xfs_bmap_del_extent_delay( xfs_add_fdblocks(mp, fdblocks); xfs_mod_delalloc(ip, -(int64_t)del->br_blockcount, -da_diff); - return error; } void @@ -5622,18 +5618,16 @@ __xfs_bunmapi( delete: if (wasdel) { - error = xfs_bmap_del_extent_delay(ip, whichfork, &icur, - &got, &del); + xfs_bmap_del_extent_delay(ip, whichfork, &icur, &got, &del); } else { error = xfs_bmap_del_extent_real(ip, tp, &icur, cur, &del, &tmp_logflags, whichfork, flags); logflags |= tmp_logflags; + if (error) + goto error0; } - if (error) - goto error0; - end = del.br_startoff - 1; nodelete: /* diff --git a/fs/xfs/libxfs/xfs_bmap.h b/fs/xfs/libxfs/xfs_bmap.h index e98849eb9bbae..667b0c2b33d1d 100644 --- a/fs/xfs/libxfs/xfs_bmap.h +++ b/fs/xfs/libxfs/xfs_bmap.h @@ -202,7 +202,7 @@ int xfs_bmapi_write(struct xfs_trans *tp, struct xfs_inode *ip, int xfs_bunmapi(struct xfs_trans *tp, struct xfs_inode *ip, xfs_fileoff_t bno, xfs_filblks_t len, uint32_t flags, xfs_extnum_t nexts, int *done); -int xfs_bmap_del_extent_delay(struct xfs_inode *ip, int whichfork, +void xfs_bmap_del_extent_delay(struct xfs_inode *ip, int whichfork, struct xfs_iext_cursor *cur, struct xfs_bmbt_irec *got, struct xfs_bmbt_irec *del); void xfs_bmap_del_extent_cow(struct xfs_inode *ip, diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index f7520f375ceee..6dead20338e24 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -443,7 +443,6 @@ xfs_discard_folio( { struct xfs_inode *ip = XFS_I(folio->mapping->host); struct xfs_mount *mp = ip->i_mount; - int error; if (xfs_is_shutdown(mp)) return; @@ -457,11 +456,8 @@ xfs_discard_folio( * byte of the next folio. Hence the end offset is only dependent on the * folio itself and not the start offset that is passed in. */ - error = xfs_bmap_punch_delalloc_range(ip, pos, + xfs_bmap_punch_delalloc_range(ip, pos, folio_pos(folio) + folio_size(folio)); - - if (error && !xfs_is_shutdown(mp)) - xfs_alert(mp, "page discard unable to remove delalloc mapping."); } static const struct iomap_writeback_ops xfs_writeback_ops = { diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index 2e6f08198c071..14e0eb9b305fd 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -440,7 +440,7 @@ xfs_getbmap( * if the ranges only partially overlap them, so it is up to the caller to * ensure that partial blocks are not passed in. */ -int +void xfs_bmap_punch_delalloc_range( struct xfs_inode *ip, xfs_off_t start_byte, @@ -452,7 +452,6 @@ xfs_bmap_punch_delalloc_range( xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, end_byte); struct xfs_bmbt_irec got, del; struct xfs_iext_cursor icur; - int error = 0; ASSERT(!xfs_need_iread_extents(ifp)); @@ -476,15 +475,13 @@ xfs_bmap_punch_delalloc_range( continue; } - error = xfs_bmap_del_extent_delay(ip, XFS_DATA_FORK, &icur, - &got, &del); - if (error || !xfs_iext_get_extent(ifp, &icur, &got)) + xfs_bmap_del_extent_delay(ip, XFS_DATA_FORK, &icur, &got, &del); + if (!xfs_iext_get_extent(ifp, &icur, &got)) break; } out_unlock: xfs_iunlock(ip, XFS_ILOCK_EXCL); - return error; } /* diff --git a/fs/xfs/xfs_bmap_util.h b/fs/xfs/xfs_bmap_util.h index 77ecbb753ef20..51f84d8ff372f 100644 --- a/fs/xfs/xfs_bmap_util.h +++ b/fs/xfs/xfs_bmap_util.h @@ -30,7 +30,7 @@ xfs_bmap_rtalloc(struct xfs_bmalloca *ap) } #endif /* CONFIG_XFS_RT */ -int xfs_bmap_punch_delalloc_range(struct xfs_inode *ip, +void xfs_bmap_punch_delalloc_range(struct xfs_inode *ip, xfs_off_t start_byte, xfs_off_t end_byte); struct kgetbmap { diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index f5d0ed45721b5..3db93b876de34 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -1232,8 +1232,8 @@ xfs_buffered_write_delalloc_punch( loff_t offset, loff_t length) { - return xfs_bmap_punch_delalloc_range(XFS_I(inode), offset, - offset + length); + xfs_bmap_punch_delalloc_range(XFS_I(inode), offset, offset + length); + return 0; } static int diff --git a/fs/xfs/xfs_quota.h b/fs/xfs/xfs_quota.h index 621ea9d7cf06d..23d71a55bbc00 100644 --- a/fs/xfs/xfs_quota.h +++ b/fs/xfs/xfs_quota.h @@ -215,10 +215,11 @@ xfs_quota_reserve_blkres(struct xfs_inode *ip, int64_t blocks) return xfs_trans_reserve_quota_nblks(NULL, ip, blocks, 0, false); } -static inline int -xfs_quota_unreserve_blkres(struct xfs_inode *ip, int64_t blocks) +static inline void +xfs_quota_unreserve_blkres(struct xfs_inode *ip, uint64_t blocks) { - return xfs_quota_reserve_blkres(ip, -blocks); + /* don't return an error as unreserving quotas can't fail */ + xfs_quota_reserve_blkres(ip, -(int64_t)blocks); } extern int xfs_mount_reset_sbqflags(struct xfs_mount *); diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 5ecb52a234bec..24761e89ce9ae 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -592,10 +592,8 @@ xfs_reflink_cancel_cow_blocks( trace_xfs_reflink_cancel_cow(ip, &del); if (isnullstartblock(del.br_startblock)) { - error = xfs_bmap_del_extent_delay(ip, XFS_COW_FORK, - &icur, &got, &del); - if (error) - break; + xfs_bmap_del_extent_delay(ip, XFS_COW_FORK, &icur, &got, + &del); } else if (del.br_state == XFS_EXT_UNWRITTEN || cancel_real) { ASSERT((*tpp)->t_highest_agno == NULLAGNUMBER); @@ -618,10 +616,7 @@ xfs_reflink_cancel_cow_blocks( xfs_bmap_del_extent_cow(ip, &icur, &got, &del); /* Remove the quota reservation */ - error = xfs_quota_unreserve_blkres(ip, - del.br_blockcount); - if (error) - break; + xfs_quota_unreserve_blkres(ip, del.br_blockcount); } else { /* Didn't do anything, push cursor back. */ xfs_iext_prev(ifp, &icur); From 99fb6b7ad1f2fb83d8df2c1382be63a1f50b1ae0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 2 May 2024 09:33:53 +0200 Subject: [PATCH 1005/1702] xfs: upgrade the extent counters in xfs_reflink_end_cow_extent later Defer the extent counter size upgrade until we know we're going to modify the extent mapping. This also defers dirtying the transaction and will allow us safely back out later in the function in later changes. Fixes: 4f86bb4b66c9 ("xfs: Conditionally upgrade existing inodes to use large extent counters") Reviewed-by: "Darrick J. Wong" Signed-off-by: Christoph Hellwig Signed-off-by: Chandan Babu R --- fs/xfs/xfs_reflink.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 24761e89ce9ae..93f775c13b664 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -732,14 +732,6 @@ xfs_reflink_end_cow_extent( xfs_ilock(ip, XFS_ILOCK_EXCL); xfs_trans_ijoin(tp, ip, 0); - error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, - XFS_IEXT_REFLINK_END_COW_CNT); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, ip, - XFS_IEXT_REFLINK_END_COW_CNT); - if (error) - goto out_cancel; - /* * In case of racing, overlapping AIO writes no COW extents might be * left by the time I/O completes for the loser of the race. In that @@ -768,6 +760,14 @@ xfs_reflink_end_cow_extent( del = got; xfs_trim_extent(&del, *offset_fsb, end_fsb - *offset_fsb); + error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, + XFS_IEXT_REFLINK_END_COW_CNT); + if (error == -EFBIG) + error = xfs_iext_count_upgrade(tp, ip, + XFS_IEXT_REFLINK_END_COW_CNT); + if (error) + goto out_cancel; + /* Grab the corresponding mapping in the data fork. */ nmaps = 1; error = xfs_bmapi_read(ip, del.br_startoff, del.br_blockcount, &data, From 86de848403abda05bf9c16dcdb6bef65a8d88c41 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 2 May 2024 09:33:54 +0200 Subject: [PATCH 1006/1702] xfs: remove a racy if_bytes check in xfs_reflink_end_cow_extent Accessing if_bytes without the ilock is racy. Remove the initial if_bytes == 0 check in xfs_reflink_end_cow_extent and let ext_iext_lookup_extent fail for this case after we've taken the ilock. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Signed-off-by: Chandan Babu R --- fs/xfs/xfs_reflink.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 93f775c13b664..520f5ef3743c4 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -712,12 +712,6 @@ xfs_reflink_end_cow_extent( int nmaps; int error; - /* No COW extents? That's easy! */ - if (ifp->if_bytes == 0) { - *offset_fsb = end_fsb; - return 0; - } - resblks = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK); error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, XFS_TRANS_RESERVE, &tp); From 25576c5420e61dea4c2b52942460f2221b8e46e8 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 2 May 2024 09:33:55 +0200 Subject: [PATCH 1007/1702] xfs: simplify iext overflow checking and upgrade Currently the calls to xfs_iext_count_may_overflow and xfs_iext_count_upgrade are always paired. Merge them into a single function to simplify the callers and the actual check and upgrade logic itself. Signed-off-by: Christoph Hellwig Reviewed-by: "Darrick J. Wong" Reviewed-by: Dave Chinner Signed-off-by: Chandan Babu R --- fs/xfs/libxfs/xfs_attr.c | 5 +-- fs/xfs/libxfs/xfs_bmap.c | 5 +-- fs/xfs/libxfs/xfs_inode_fork.c | 57 +++++++++++++++------------------- fs/xfs/libxfs/xfs_inode_fork.h | 6 ++-- fs/xfs/xfs_bmap_item.c | 4 +-- fs/xfs/xfs_bmap_util.c | 23 +++----------- fs/xfs/xfs_dquot.c | 5 +-- fs/xfs/xfs_iomap.c | 9 ++---- fs/xfs/xfs_reflink.c | 9 ++---- fs/xfs/xfs_rtalloc.c | 5 +-- 10 files changed, 41 insertions(+), 87 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index eb274d4d636d8..430cd3244c143 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -1050,11 +1050,8 @@ xfs_attr_set( return error; if (op != XFS_ATTRUPDATE_REMOVE || xfs_inode_hasattr(dp)) { - error = xfs_iext_count_may_overflow(dp, XFS_ATTR_FORK, + error = xfs_iext_count_extend(args->trans, dp, XFS_ATTR_FORK, XFS_IEXT_ATTR_MANIP_CNT(rmt_blks)); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(args->trans, dp, - XFS_IEXT_ATTR_MANIP_CNT(rmt_blks)); if (error) goto out_trans_cancel; } diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 0633f285875de..3b3206d312d6f 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4651,11 +4651,8 @@ xfs_bmapi_convert_one_delalloc( xfs_ilock(ip, XFS_ILOCK_EXCL); xfs_trans_ijoin(tp, ip, 0); - error = xfs_iext_count_may_overflow(ip, whichfork, + error = xfs_iext_count_extend(tp, ip, whichfork, XFS_IEXT_ADD_NOSPLIT_CNT); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, ip, - XFS_IEXT_ADD_NOSPLIT_CNT); if (error) goto out_trans_cancel; diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index 7d660a9739090..9d11ae0159091 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -765,53 +765,46 @@ xfs_ifork_verify_local_attr( return 0; } +/* + * Check if the inode fork supports adding nr_to_add more extents. + * + * If it doesn't but we can upgrade it to large extent counters, do the upgrade. + * If we can't upgrade or are already using big counters but still can't fit the + * additional extents, return -EFBIG. + */ int -xfs_iext_count_may_overflow( +xfs_iext_count_extend( + struct xfs_trans *tp, struct xfs_inode *ip, int whichfork, - int nr_to_add) + uint nr_to_add) { + struct xfs_mount *mp = ip->i_mount; + bool has_large = + xfs_inode_has_large_extent_counts(ip); struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); - uint64_t max_exts; uint64_t nr_exts; + ASSERT(nr_to_add <= XFS_MAX_EXTCNT_UPGRADE_NR); + if (whichfork == XFS_COW_FORK) return 0; - max_exts = xfs_iext_max_nextents(xfs_inode_has_large_extent_counts(ip), - whichfork); - - if (XFS_TEST_ERROR(false, ip->i_mount, XFS_ERRTAG_REDUCE_MAX_IEXTENTS)) - max_exts = 10; - + /* no point in upgrading if if_nextents overflows */ nr_exts = ifp->if_nextents + nr_to_add; - if (nr_exts < ifp->if_nextents || nr_exts > max_exts) + if (nr_exts < ifp->if_nextents) return -EFBIG; - return 0; -} - -/* - * Upgrade this inode's extent counter fields to be able to handle a potential - * increase in the extent count by nr_to_add. Normally this is the same - * quantity that caused xfs_iext_count_may_overflow() to return -EFBIG. - */ -int -xfs_iext_count_upgrade( - struct xfs_trans *tp, - struct xfs_inode *ip, - uint nr_to_add) -{ - ASSERT(nr_to_add <= XFS_MAX_EXTCNT_UPGRADE_NR); - - if (!xfs_has_large_extent_counts(ip->i_mount) || - xfs_inode_has_large_extent_counts(ip) || - XFS_TEST_ERROR(false, ip->i_mount, XFS_ERRTAG_REDUCE_MAX_IEXTENTS)) + if (XFS_TEST_ERROR(false, mp, XFS_ERRTAG_REDUCE_MAX_IEXTENTS) && + nr_exts > 10) return -EFBIG; - ip->i_diflags2 |= XFS_DIFLAG2_NREXT64; - xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); - + if (nr_exts > xfs_iext_max_nextents(has_large, whichfork)) { + if (has_large || !xfs_has_large_extent_counts(mp)) + return -EFBIG; + ip->i_diflags2 |= XFS_DIFLAG2_NREXT64; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + } return 0; } diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index bd53eb951b651..2373d12fd474f 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -256,10 +256,8 @@ extern void xfs_ifork_init_cow(struct xfs_inode *ip); int xfs_ifork_verify_local_data(struct xfs_inode *ip); int xfs_ifork_verify_local_attr(struct xfs_inode *ip); -int xfs_iext_count_may_overflow(struct xfs_inode *ip, int whichfork, - int nr_to_add); -int xfs_iext_count_upgrade(struct xfs_trans *tp, struct xfs_inode *ip, - uint nr_to_add); +int xfs_iext_count_extend(struct xfs_trans *tp, struct xfs_inode *ip, + int whichfork, uint nr_to_add); bool xfs_ifork_is_realtime(struct xfs_inode *ip, int whichfork); /* returns true if the fork has extents but they are not read in yet. */ diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index d27859a684aa6..a19d62e78aa1e 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -524,9 +524,7 @@ xfs_bmap_recover_work( else iext_delta = XFS_IEXT_PUNCH_HOLE_CNT; - error = xfs_iext_count_may_overflow(ip, work->bi_whichfork, iext_delta); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, ip, iext_delta); + error = xfs_iext_count_extend(tp, ip, work->bi_whichfork, iext_delta); if (error) goto err_cancel; diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index 14e0eb9b305fd..ac2e77ebb54c7 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -710,11 +710,8 @@ xfs_alloc_file_space( if (error) break; - error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, XFS_IEXT_ADD_NOSPLIT_CNT); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, ip, - XFS_IEXT_ADD_NOSPLIT_CNT); if (error) goto error; @@ -771,10 +768,8 @@ xfs_unmap_extent( if (error) return error; - error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, XFS_IEXT_PUNCH_HOLE_CNT); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, ip, XFS_IEXT_PUNCH_HOLE_CNT); if (error) goto out_trans_cancel; @@ -1050,10 +1045,8 @@ xfs_insert_file_space( xfs_ilock(ip, XFS_ILOCK_EXCL); xfs_trans_ijoin(tp, ip, 0); - error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, XFS_IEXT_PUNCH_HOLE_CNT); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, ip, XFS_IEXT_PUNCH_HOLE_CNT); if (error) goto out_trans_cancel; @@ -1279,23 +1272,17 @@ xfs_swap_extent_rmap( trace_xfs_swap_extent_rmap_remap_piece(tip, &uirec); if (xfs_bmap_is_real_extent(&uirec)) { - error = xfs_iext_count_may_overflow(ip, + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, XFS_IEXT_SWAP_RMAP_CNT); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, ip, - XFS_IEXT_SWAP_RMAP_CNT); if (error) goto out; } if (xfs_bmap_is_real_extent(&irec)) { - error = xfs_iext_count_may_overflow(tip, + error = xfs_iext_count_extend(tp, tip, XFS_DATA_FORK, XFS_IEXT_SWAP_RMAP_CNT); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, ip, - XFS_IEXT_SWAP_RMAP_CNT); if (error) goto out; } diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 43acb4f0d1743..c1b211c260a9d 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -341,11 +341,8 @@ xfs_dquot_disk_alloc( goto err_cancel; } - error = xfs_iext_count_may_overflow(quotip, XFS_DATA_FORK, + error = xfs_iext_count_extend(tp, quotip, XFS_DATA_FORK, XFS_IEXT_ADD_NOSPLIT_CNT); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, quotip, - XFS_IEXT_ADD_NOSPLIT_CNT); if (error) goto err_cancel; diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 3db93b876de34..3783426739258 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -299,9 +299,7 @@ xfs_iomap_write_direct( if (error) return error; - error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, nr_exts); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, ip, nr_exts); + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, nr_exts); if (error) goto out_trans_cancel; @@ -617,11 +615,8 @@ xfs_iomap_write_unwritten( if (error) return error; - error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, XFS_IEXT_WRITE_UNWRITTEN_CNT); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, ip, - XFS_IEXT_WRITE_UNWRITTEN_CNT); if (error) goto error_on_bmapi_transaction; diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 520f5ef3743c4..063a2e00d1691 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -754,11 +754,8 @@ xfs_reflink_end_cow_extent( del = got; xfs_trim_extent(&del, *offset_fsb, end_fsb - *offset_fsb); - error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, XFS_IEXT_REFLINK_END_COW_CNT); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, ip, - XFS_IEXT_REFLINK_END_COW_CNT); if (error) goto out_cancel; @@ -1258,9 +1255,7 @@ xfs_reflink_remap_extent( if (dmap_written) ++iext_delta; - error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, iext_delta); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, ip, iext_delta); + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, iext_delta); if (error) goto out_cancel; diff --git a/fs/xfs/xfs_rtalloc.c b/fs/xfs/xfs_rtalloc.c index 150f544445ca8..5a7ddfed1bb85 100644 --- a/fs/xfs/xfs_rtalloc.c +++ b/fs/xfs/xfs_rtalloc.c @@ -695,11 +695,8 @@ xfs_growfs_rt_alloc( xfs_ilock(ip, XFS_ILOCK_EXCL); xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); - error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, XFS_IEXT_ADD_NOSPLIT_CNT); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, ip, - XFS_IEXT_ADD_NOSPLIT_CNT); if (error) goto out_trans_cancel; From 61fcbbf3ca038c048c942ce31bb3d3c846c87581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Tue, 23 Apr 2024 06:55:01 +0200 Subject: [PATCH 1008/1702] dt-bindings: pinctrl: mediatek: mt7622: fix array properties MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some properties (function groups & pins) are meant to be arrays and should allow multiple entries out of enum sets. Use "items" for those. Mistake was noticed during validation of in-kernel DTS files. Fixes: b9ffc18c6388 ("dt-bindings: mediatek: convert pinctrl to yaml") Signed-off-by: Rafał Miłecki Acked-by: Rob Herring Message-ID: <20240423045502.7778-1-zajec5@gmail.com> Signed-off-by: Linus Walleij --- .../pinctrl/mediatek,mt7622-pinctrl.yaml | 92 ++++++++++--------- 1 file changed, 49 insertions(+), 43 deletions(-) diff --git a/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml index 4637f3a40e83e..903d1f6979c59 100644 --- a/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml @@ -100,7 +100,8 @@ patternProperties: then: properties: groups: - enum: [emmc, emmc_rst] + items: + enum: [emmc, emmc_rst] - if: properties: function: @@ -108,8 +109,9 @@ patternProperties: then: properties: groups: - enum: [esw, esw_p0_p1, esw_p2_p3_p4, rgmii_via_esw, - rgmii_via_gmac1, rgmii_via_gmac2, mdc_mdio] + items: + enum: [esw, esw_p0_p1, esw_p2_p3_p4, rgmii_via_esw, + rgmii_via_gmac1, rgmii_via_gmac2, mdc_mdio] - if: properties: function: @@ -126,10 +128,11 @@ patternProperties: then: properties: groups: - enum: [i2s_in_mclk_bclk_ws, i2s1_in_data, i2s2_in_data, - i2s3_in_data, i2s4_in_data, i2s_out_mclk_bclk_ws, - i2s1_out_data, i2s2_out_data, i2s3_out_data, - i2s4_out_data] + items: + enum: [i2s_in_mclk_bclk_ws, i2s1_in_data, i2s2_in_data, + i2s3_in_data, i2s4_in_data, i2s_out_mclk_bclk_ws, + i2s1_out_data, i2s2_out_data, i2s3_out_data, + i2s4_out_data] - if: properties: function: @@ -162,10 +165,11 @@ patternProperties: then: properties: groups: - enum: [pcie0_0_waken, pcie0_1_waken, pcie1_0_waken, - pcie0_0_clkreq, pcie0_1_clkreq, pcie1_0_clkreq, - pcie0_pad_perst, pcie1_pad_perst, pcie_pereset, - pcie_wake, pcie_clkreq] + items: + enum: [pcie0_0_waken, pcie0_1_waken, pcie1_0_waken, + pcie0_0_clkreq, pcie0_1_clkreq, pcie1_0_clkreq, + pcie0_pad_perst, pcie1_pad_perst, pcie_pereset, + pcie_wake, pcie_clkreq] - if: properties: function: @@ -181,11 +185,12 @@ patternProperties: then: properties: groups: - enum: [pwm_ch1_0, pwm_ch1_1, pwm_ch1_2, pwm_ch2_0, pwm_ch2_1, - pwm_ch2_2, pwm_ch3_0, pwm_ch3_1, pwm_ch3_2, pwm_ch4_0, - pwm_ch4_1, pwm_ch4_2, pwm_ch4_3, pwm_ch5_0, pwm_ch5_1, - pwm_ch5_2, pwm_ch6_0, pwm_ch6_1, pwm_ch6_2, pwm_ch6_3, - pwm_ch7_0, pwm_0, pwm_1] + items: + enum: [pwm_ch1_0, pwm_ch1_1, pwm_ch1_2, pwm_ch2_0, pwm_ch2_1, + pwm_ch2_2, pwm_ch3_0, pwm_ch3_1, pwm_ch3_2, pwm_ch4_0, + pwm_ch4_1, pwm_ch4_2, pwm_ch4_3, pwm_ch5_0, pwm_ch5_1, + pwm_ch5_2, pwm_ch6_0, pwm_ch6_1, pwm_ch6_2, pwm_ch6_3, + pwm_ch7_0, pwm_0, pwm_1] - if: properties: function: @@ -263,33 +268,34 @@ patternProperties: pins: description: An array of strings. Each string contains the name of a pin. - enum: [GPIO_A, I2S1_IN, I2S1_OUT, I2S_BCLK, I2S_WS, I2S_MCLK, TXD0, - RXD0, SPI_WP, SPI_HOLD, SPI_CLK, SPI_MOSI, SPI_MISO, SPI_CS, - I2C_SDA, I2C_SCL, I2S2_IN, I2S3_IN, I2S4_IN, I2S2_OUT, - I2S3_OUT, I2S4_OUT, GPIO_B, MDC, MDIO, G2_TXD0, G2_TXD1, - G2_TXD2, G2_TXD3, G2_TXEN, G2_TXC, G2_RXD0, G2_RXD1, G2_RXD2, - G2_RXD3, G2_RXDV, G2_RXC, NCEB, NWEB, NREB, NDL4, NDL5, NDL6, - NDL7, NRB, NCLE, NALE, NDL0, NDL1, NDL2, NDL3, MDI_TP_P0, - MDI_TN_P0, MDI_RP_P0, MDI_RN_P0, MDI_TP_P1, MDI_TN_P1, - MDI_RP_P1, MDI_RN_P1, MDI_RP_P2, MDI_RN_P2, MDI_TP_P2, - MDI_TN_P2, MDI_TP_P3, MDI_TN_P3, MDI_RP_P3, MDI_RN_P3, - MDI_RP_P4, MDI_RN_P4, MDI_TP_P4, MDI_TN_P4, PMIC_SCL, - PMIC_SDA, SPIC1_CLK, SPIC1_MOSI, SPIC1_MISO, SPIC1_CS, - GPIO_D, WATCHDOG, RTS3_N, CTS3_N, TXD3, RXD3, PERST0_N, - PERST1_N, WLED_N, EPHY_LED0_N, AUXIN0, AUXIN1, AUXIN2, - AUXIN3, TXD4, RXD4, RTS4_N, CST4_N, PWM1, PWM2, PWM3, PWM4, - PWM5, PWM6, PWM7, GPIO_E, TOP_5G_CLK, TOP_5G_DATA, - WF0_5G_HB0, WF0_5G_HB1, WF0_5G_HB2, WF0_5G_HB3, WF0_5G_HB4, - WF0_5G_HB5, WF0_5G_HB6, XO_REQ, TOP_RST_N, SYS_WATCHDOG, - EPHY_LED0_N_JTDO, EPHY_LED1_N_JTDI, EPHY_LED2_N_JTMS, - EPHY_LED3_N_JTCLK, EPHY_LED4_N_JTRST_N, WF2G_LED_N, - WF5G_LED_N, GPIO_9, GPIO_10, GPIO_11, GPIO_12, UART1_TXD, - UART1_RXD, UART1_CTS, UART1_RTS, UART2_TXD, UART2_RXD, - UART2_CTS, UART2_RTS, SMI_MDC, SMI_MDIO, PCIE_PERESET_N, - PWM_0, GPIO_0, GPIO_1, GPIO_2, GPIO_3, GPIO_4, GPIO_5, - GPIO_6, GPIO_7, GPIO_8, UART0_TXD, UART0_RXD, TOP_2G_CLK, - TOP_2G_DATA, WF0_2G_HB0, WF0_2G_HB1, WF0_2G_HB2, WF0_2G_HB3, - WF0_2G_HB4, WF0_2G_HB5, WF0_2G_HB6] + items: + enum: [GPIO_A, I2S1_IN, I2S1_OUT, I2S_BCLK, I2S_WS, I2S_MCLK, TXD0, + RXD0, SPI_WP, SPI_HOLD, SPI_CLK, SPI_MOSI, SPI_MISO, SPI_CS, + I2C_SDA, I2C_SCL, I2S2_IN, I2S3_IN, I2S4_IN, I2S2_OUT, + I2S3_OUT, I2S4_OUT, GPIO_B, MDC, MDIO, G2_TXD0, G2_TXD1, + G2_TXD2, G2_TXD3, G2_TXEN, G2_TXC, G2_RXD0, G2_RXD1, G2_RXD2, + G2_RXD3, G2_RXDV, G2_RXC, NCEB, NWEB, NREB, NDL4, NDL5, NDL6, + NDL7, NRB, NCLE, NALE, NDL0, NDL1, NDL2, NDL3, MDI_TP_P0, + MDI_TN_P0, MDI_RP_P0, MDI_RN_P0, MDI_TP_P1, MDI_TN_P1, + MDI_RP_P1, MDI_RN_P1, MDI_RP_P2, MDI_RN_P2, MDI_TP_P2, + MDI_TN_P2, MDI_TP_P3, MDI_TN_P3, MDI_RP_P3, MDI_RN_P3, + MDI_RP_P4, MDI_RN_P4, MDI_TP_P4, MDI_TN_P4, PMIC_SCL, + PMIC_SDA, SPIC1_CLK, SPIC1_MOSI, SPIC1_MISO, SPIC1_CS, + GPIO_D, WATCHDOG, RTS3_N, CTS3_N, TXD3, RXD3, PERST0_N, + PERST1_N, WLED_N, EPHY_LED0_N, AUXIN0, AUXIN1, AUXIN2, + AUXIN3, TXD4, RXD4, RTS4_N, CST4_N, PWM1, PWM2, PWM3, PWM4, + PWM5, PWM6, PWM7, GPIO_E, TOP_5G_CLK, TOP_5G_DATA, + WF0_5G_HB0, WF0_5G_HB1, WF0_5G_HB2, WF0_5G_HB3, WF0_5G_HB4, + WF0_5G_HB5, WF0_5G_HB6, XO_REQ, TOP_RST_N, SYS_WATCHDOG, + EPHY_LED0_N_JTDO, EPHY_LED1_N_JTDI, EPHY_LED2_N_JTMS, + EPHY_LED3_N_JTCLK, EPHY_LED4_N_JTRST_N, WF2G_LED_N, + WF5G_LED_N, GPIO_9, GPIO_10, GPIO_11, GPIO_12, UART1_TXD, + UART1_RXD, UART1_CTS, UART1_RTS, UART2_TXD, UART2_RXD, + UART2_CTS, UART2_RTS, SMI_MDC, SMI_MDIO, PCIE_PERESET_N, + PWM_0, GPIO_0, GPIO_1, GPIO_2, GPIO_3, GPIO_4, GPIO_5, + GPIO_6, GPIO_7, GPIO_8, UART0_TXD, UART0_RXD, TOP_2G_CLK, + TOP_2G_DATA, WF0_2G_HB0, WF0_2G_HB1, WF0_2G_HB2, WF0_2G_HB3, + WF0_2G_HB4, WF0_2G_HB5, WF0_2G_HB6] bias-disable: true From 083f6675e194bca802c3121ccf59ca0a1cf81c09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Tue, 23 Apr 2024 06:55:02 +0200 Subject: [PATCH 1009/1702] dt-bindings: pinctrl: mediatek: mt7622: add "antsel" function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MT7622 pinctrl has ANTSEL* pins. Linux support for those was added in the commit 19f599e83ac5 ("pinctrl: mediatek: mt7622: add antsel pins/groups"). Include them in binding. Cc: Chuanhong Guo Signed-off-by: Rafał Miłecki Acked-by: Rob Herring Message-ID: <20240423045502.7778-2-zajec5@gmail.com> Signed-off-by: Linus Walleij --- .../pinctrl/mediatek,mt7622-pinctrl.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml index 903d1f6979c59..d74cae9d4d650 100644 --- a/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml @@ -78,8 +78,8 @@ patternProperties: function: description: A string containing the name of the function to mux to the group. - enum: [emmc, eth, i2c, i2s, ir, led, flash, pcie, pmic, pwm, sd, - spi, tdm, uart, watchdog, wifi] + enum: [antsel, emmc, eth, i2c, i2s, ir, led, flash, pcie, pmic, pwm, + sd, spi, tdm, uart, watchdog, wifi] groups: description: @@ -93,6 +93,20 @@ patternProperties: - function allOf: + - if: + properties: + function: + const: antsel + then: + properties: + groups: + items: + enum: [antsel0, antsel1, antsel2, antsel3, antsel4, antsel5, + antsel6, antsel7, antsel8, antsel9, antsel10, + antsel11, antsel12, antsel13, antsel14, antsel15, + antsel16, antsel17, antsel18, antsel19, antsel20, + antsel21, antsel22, antsel23, antsel24, antsel25, + antsel26, antsel27, antsel28, antsel29] - if: properties: function: From 5ed79863fae5c06eb33f5cd6b6bdf22dd7089392 Mon Sep 17 00:00:00 2001 From: Danila Tikhonov Date: Tue, 23 Apr 2024 23:32:45 +0300 Subject: [PATCH 1010/1702] pinctrl: qcom: pinctrl-sm7150: Fix sdc1 and ufs special pins regs SDC1 and UFS_RESET special pins are located in the west memory bank. SDC1 have address 0x359a000: 0x3500000 (TLMM BASE) + 0x0 (WEST) + 0x9a000 (SDC1_OFFSET) = 0x359a000 UFS_RESET have address 0x359f000: 0x3500000 (TLMM BASE) + 0x0 (WEST) + 0x9f000 (UFS_OFFSET) = 0x359a000 Fixes: b915395c9e04 ("pinctrl: qcom: Add SM7150 pinctrl driver") Signed-off-by: Danila Tikhonov Message-ID: <20240423203245.188480-1-danila@jiaxyga.com> Signed-off-by: Linus Walleij --- drivers/pinctrl/qcom/pinctrl-sm7150.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/pinctrl/qcom/pinctrl-sm7150.c b/drivers/pinctrl/qcom/pinctrl-sm7150.c index c542f9bc6bcd8..095a1ca758490 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm7150.c +++ b/drivers/pinctrl/qcom/pinctrl-sm7150.c @@ -65,7 +65,7 @@ enum { .intr_detection_width = 2, \ } -#define SDC_QDSD_PINGROUP(pg_name, ctl, pull, drv) \ +#define SDC_QDSD_PINGROUP(pg_name, _tile, ctl, pull, drv) \ { \ .grp = PINCTRL_PINGROUP(#pg_name, \ pg_name##_pins, \ @@ -75,7 +75,7 @@ enum { .intr_cfg_reg = 0, \ .intr_status_reg = 0, \ .intr_target_reg = 0, \ - .tile = SOUTH, \ + .tile = _tile, \ .mux_bit = -1, \ .pull_bit = pull, \ .drv_bit = drv, \ @@ -101,7 +101,7 @@ enum { .intr_cfg_reg = 0, \ .intr_status_reg = 0, \ .intr_target_reg = 0, \ - .tile = SOUTH, \ + .tile = WEST, \ .mux_bit = -1, \ .pull_bit = 3, \ .drv_bit = 0, \ @@ -1199,13 +1199,13 @@ static const struct msm_pingroup sm7150_groups[] = { [117] = PINGROUP(117, NORTH, _, _, _, _, _, _, _, _, _), [118] = PINGROUP(118, NORTH, _, _, _, _, _, _, _, _, _), [119] = UFS_RESET(ufs_reset, 0x9f000), - [120] = SDC_QDSD_PINGROUP(sdc1_rclk, 0x9a000, 15, 0), - [121] = SDC_QDSD_PINGROUP(sdc1_clk, 0x9a000, 13, 6), - [122] = SDC_QDSD_PINGROUP(sdc1_cmd, 0x9a000, 11, 3), - [123] = SDC_QDSD_PINGROUP(sdc1_data, 0x9a000, 9, 0), - [124] = SDC_QDSD_PINGROUP(sdc2_clk, 0x98000, 14, 6), - [125] = SDC_QDSD_PINGROUP(sdc2_cmd, 0x98000, 11, 3), - [126] = SDC_QDSD_PINGROUP(sdc2_data, 0x98000, 9, 0), + [120] = SDC_QDSD_PINGROUP(sdc1_rclk, WEST, 0x9a000, 15, 0), + [121] = SDC_QDSD_PINGROUP(sdc1_clk, WEST, 0x9a000, 13, 6), + [122] = SDC_QDSD_PINGROUP(sdc1_cmd, WEST, 0x9a000, 11, 3), + [123] = SDC_QDSD_PINGROUP(sdc1_data, WEST, 0x9a000, 9, 0), + [124] = SDC_QDSD_PINGROUP(sdc2_clk, SOUTH, 0x98000, 14, 6), + [125] = SDC_QDSD_PINGROUP(sdc2_cmd, SOUTH, 0x98000, 11, 3), + [126] = SDC_QDSD_PINGROUP(sdc2_data, SOUTH, 0x98000, 9, 0), }; static const struct msm_gpio_wakeirq_map sm7150_pdc_map[] = { From 4ad4f6e2fd0a44e0aa03c7f220ac79abfe866fbf Mon Sep 17 00:00:00 2001 From: Anjelique Melendez Date: Thu, 25 Apr 2024 11:56:03 -0700 Subject: [PATCH 1011/1702] dt-bindings: pinctrl: qcom,pmic-gpio: Fix "comptaible" typo for PMIH0108 Fix "comptaible" typo in if schema for qcom,pmih0108-gpio. Fixes: 6acc46f8c065 ("dt-bindings: pinctrl: qcom,pmic-gpio: Add PMIH0108 and PMD8028 support") Signed-off-by: Anjelique Melendez Reviewed-by: Krzysztof Kozlowski Acked-by: Rob Herring (Arm) Message-ID: <20240425185603.3295450-1-quic_amelende@quicinc.com> Signed-off-by: Linus Walleij --- Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml index bd9471de0c69f..50846a2d09c88 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml @@ -309,7 +309,7 @@ allOf: - if: properties: - comptaible: + compatible: contains: enum: - qcom,pmih0108-gpio From 6b32d7474e9b833dc7fadc1b4d4f08af9bd87fde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Draszik?= Date: Tue, 30 Apr 2024 10:54:59 +0100 Subject: [PATCH 1012/1702] clk: samsung: gs101: mark some apm UASC and XIU clocks critical MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The system hangs when any of these clocks are turned off. With the introduction of pinctrl clock support [1], the approach taken in this clock driver for the APM clocks to rely solely on the clk_ignore_unused kernel command line option does not work anymore and the system hangs during boot. gout_apm_func is a parent clock to the clocks that are going to be handled by the pinctrl driver [2], namely gout_apm_apbif_gpio_alive_pclk and gout_apm_apbif_gpio_far_alive_pclk. It also is the parent to the clocks marked as critical in this commit here (and some others that aren't relevant for this commit)). This means that once the pinctrl driver decides to turn off clocks, the clock framework will subsequently turn off parent clocks of those pinctrl clocks if they have no (apparent) user. Since gout_apm_func is the parent, and since no drivers are hooked up to it or any of its other children, gout_apm_func will be turned off. This will cause the system to hang, as the clocks marked as critical in this commit stop having an input. We might have to add a driver for these clocks, but in the meantime let's just ensure they stay on even if siblings are turned off. For the avoidance of doubt: This commit doesn't mean that we can boot without clk_ignore_unused. Link: https://lore.kernel.org/r/20240426-samsung-pinctrl-busclock-v3-0-adb8664b8a7e@linaro.org [1] Link: https://lore.kernel.org/r/20240429-samsung-pinctrl-busclock-dts-v1-0-5e935179f3ca@linaro.org [2] Signed-off-by: André Draszik Reviewed-by: Tudor Ambarus Link: https://lore.kernel.org/r/20240430-gs101-apm-clocks-v1-1-b2e2335e84f5@linaro.org Signed-off-by: Krzysztof Kozlowski --- drivers/clk/samsung/clk-gs101.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/clk/samsung/clk-gs101.c b/drivers/clk/samsung/clk-gs101.c index 05129c3b2f680..e2a6a19925057 100644 --- a/drivers/clk/samsung/clk-gs101.c +++ b/drivers/clk/samsung/clk-gs101.c @@ -1896,16 +1896,16 @@ static const struct samsung_gate_clock apm_gate_clks[] __initconst = { CLK_CON_GAT_GOUT_BLK_APM_UID_UASC_G_SWD_IPCLKPORT_PCLK, 21, 0, 0), GATE(CLK_GOUT_APM_UASC_P_APM_ACLK, "gout_apm_uasc_p_apm_aclk", "gout_apm_func", - CLK_CON_GAT_GOUT_BLK_APM_UID_UASC_P_APM_IPCLKPORT_ACLK, 21, 0, 0), + CLK_CON_GAT_GOUT_BLK_APM_UID_UASC_P_APM_IPCLKPORT_ACLK, 21, CLK_IS_CRITICAL, 0), GATE(CLK_GOUT_APM_UASC_P_APM_PCLK, "gout_apm_uasc_p_apm_pclk", "gout_apm_func", - CLK_CON_GAT_GOUT_BLK_APM_UID_UASC_P_APM_IPCLKPORT_PCLK, 21, 0, 0), + CLK_CON_GAT_GOUT_BLK_APM_UID_UASC_P_APM_IPCLKPORT_PCLK, 21, CLK_IS_CRITICAL, 0), GATE(CLK_GOUT_APM_WDT_APM_PCLK, "gout_apm_wdt_apm_pclk", "gout_apm_func", CLK_CON_GAT_GOUT_BLK_APM_UID_WDT_APM_IPCLKPORT_PCLK, 21, 0, 0), GATE(CLK_GOUT_APM_XIU_DP_APM_ACLK, "gout_apm_xiu_dp_apm_aclk", "gout_apm_func", - CLK_CON_GAT_GOUT_BLK_APM_UID_XIU_DP_APM_IPCLKPORT_ACLK, 21, 0, 0), + CLK_CON_GAT_GOUT_BLK_APM_UID_XIU_DP_APM_IPCLKPORT_ACLK, 21, CLK_IS_CRITICAL, 0), }; static const struct samsung_cmu_info apm_cmu_info __initconst = { From 290fa9471ee040734bbe873c3d7b2a20240b375a Mon Sep 17 00:00:00 2001 From: "Ritesh Harjani (IBM)" Date: Fri, 3 May 2024 14:58:52 +0530 Subject: [PATCH 1013/1702] ext2: Remove LEGACY_DIRECT_IO dependency commit fb5de4358e1a ("ext2: Move direct-io to use iomap"), converted ext2 direct-io to iomap which killed the call to blockdev_direct_IO(). So let's remove LEGACY_DIRECT_IO config dependency from ext2 Kconfig. Signed-off-by: Ritesh Harjani (IBM) Signed-off-by: Jan Kara Message-Id: --- fs/ext2/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/ext2/Kconfig b/fs/ext2/Kconfig index d6cfb18495801..d5bce83ad9054 100644 --- a/fs/ext2/Kconfig +++ b/fs/ext2/Kconfig @@ -3,7 +3,6 @@ config EXT2_FS tristate "Second extended fs support (DEPRECATED)" select BUFFER_HEAD select FS_IOMAP - select LEGACY_DIRECT_IO help Ext2 is a standard Linux file system for hard disks. From bbe1e78ae23e7e474401880d0d496acc8cec6863 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 29 Apr 2024 14:17:07 +0300 Subject: [PATCH 1014/1702] iommu/amd: Fix compilation error With WERROR=y, which is default, clang is not happy: .../amd/pasid.c:168:3: error: call to undeclared function 'mmu_notifier_unregister'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] .../amd/pasid.c:191:8: error: call to undeclared function 'mmu_notifier_register'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] 2 errors generated. Select missed dependency. Fixes: a5a91e54846d ("iommu/amd: Add SVA domain support") Signed-off-by: Andy Shevchenko Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20240429111707.2795194-1-andriy.shevchenko@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/iommu/amd/Kconfig b/drivers/iommu/amd/Kconfig index 68d8fc107cb9a..994063e5586f0 100644 --- a/drivers/iommu/amd/Kconfig +++ b/drivers/iommu/amd/Kconfig @@ -7,6 +7,7 @@ config AMD_IOMMU select PCI_ATS select PCI_PRI select PCI_PASID + select MMU_NOTIFIER select IOMMU_API select IOMMU_IOVA select IOMMU_IO_PGTABLE From 156e6498fc61157a8779a0920008cfdfa55c4a8b Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Mon, 22 Apr 2024 15:52:12 -0700 Subject: [PATCH 1015/1702] MIPS: SGI-IP27: fix -Wunused-variable in arch_init_irq() Commit 40e20fbccfb722f21 (MIPS: SGI-IP27: micro-optimize arch_init_irq()) replaced a for-loop iteration with bitmap_set() calls, but didn't remove an iteration variable. Fixes: 40e20fbccfb722f21 (MIPS: SGI-IP27: micro-optimize arch_init_irq()) Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202404161933.izfqZ32k-lkp@intel.com/ Signed-off-by: Yury Norov Signed-off-by: Thomas Bogendoerfer --- arch/mips/sgi-ip27/ip27-irq.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/mips/sgi-ip27/ip27-irq.c b/arch/mips/sgi-ip27/ip27-irq.c index dcb14a234b1c3..d8acdf0439d27 100644 --- a/arch/mips/sgi-ip27/ip27-irq.c +++ b/arch/mips/sgi-ip27/ip27-irq.c @@ -277,7 +277,6 @@ void __init arch_init_irq(void) { struct irq_domain *domain; struct fwnode_handle *fn; - int i; mips_cpu_irq_init(); From bfe4ab93c80cc5689ab5a891e61013dfec48f56e Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Tue, 16 Apr 2024 10:37:11 -0700 Subject: [PATCH 1016/1702] MIPS: SGI-IP27: use WARN_ON() output WARN_ON() propagates the result of conditional expression, and it can be used to return early in the following expression in the arch_init_irq(). This is a no-op cleanup, except that compiler may optimize the error paths better because WARN_ON() implies 'unlikely()'. Signed-off-by: Yury Norov Signed-off-by: Thomas Bogendoerfer --- arch/mips/sgi-ip27/ip27-irq.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/mips/sgi-ip27/ip27-irq.c b/arch/mips/sgi-ip27/ip27-irq.c index d8acdf0439d27..00e63e9ef61d8 100644 --- a/arch/mips/sgi-ip27/ip27-irq.c +++ b/arch/mips/sgi-ip27/ip27-irq.c @@ -289,13 +289,12 @@ void __init arch_init_irq(void) bitmap_set(hub_irq_map, NI_BRDCAST_ERR_A, MSC_PANIC_INTR - NI_BRDCAST_ERR_A + 1); fn = irq_domain_alloc_named_fwnode("HUB"); - WARN_ON(fn == NULL); - if (!fn) + if (WARN_ON(fn == NULL)) return; + domain = irq_domain_create_linear(fn, IP27_HUB_IRQ_COUNT, &hub_domain_ops, NULL); - WARN_ON(domain == NULL); - if (!domain) + if (WARN_ON(domain == NULL)) return; irq_set_default_host(domain); From 07e6a6d7f1d9fa4685003a195032698ba99577bb Mon Sep 17 00:00:00 2001 From: Siarhei Volkau Date: Tue, 30 Apr 2024 18:45:58 +0300 Subject: [PATCH 1017/1702] MIPS: Take in account load hazards for HI/LO restoring MIPS CPUs usually have 1 to 4 cycles load hazards, thus doing load and right after move to HI/LO will usually stall the pipeline for significant amount of time. Let's take it into account and separate loads and mthi/lo in instruction sequence. The patch uses t6 and t7 registers as temporaries in addition to t8. The patch tries to deal with SmartMIPS, but I know little about and haven't tested it. Changes in v2: - clear separation of actions for SmartMIPS and pre-MIPSR6. Signed-off-by: Siarhei Volkau Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/stackframe.h | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/arch/mips/include/asm/stackframe.h b/arch/mips/include/asm/stackframe.h index a8705aef47e12..a13431379073f 100644 --- a/arch/mips/include/asm/stackframe.h +++ b/arch/mips/include/asm/stackframe.h @@ -308,17 +308,12 @@ jal octeon_mult_restore #endif #ifdef CONFIG_CPU_HAS_SMARTMIPS - LONG_L $24, PT_ACX(sp) - mtlhx $24 - LONG_L $24, PT_HI(sp) - mtlhx $24 + LONG_L $14, PT_ACX(sp) LONG_L $24, PT_LO(sp) - mtlhx $24 + LONG_L $15, PT_HI(sp) #elif !defined(CONFIG_CPU_MIPSR6) LONG_L $24, PT_LO(sp) - mtlo $24 - LONG_L $24, PT_HI(sp) - mthi $24 + LONG_L $15, PT_HI(sp) #endif #ifdef CONFIG_32BIT cfi_ld $8, PT_R8, \docfi @@ -327,6 +322,14 @@ cfi_ld $10, PT_R10, \docfi cfi_ld $11, PT_R11, \docfi cfi_ld $12, PT_R12, \docfi +#ifdef CONFIG_CPU_HAS_SMARTMIPS + mtlhx $14 + mtlhx $15 + mtlhx $24 +#elif !defined(CONFIG_CPU_MIPSR6) + mtlo $24 + mthi $15 +#endif cfi_ld $13, PT_R13, \docfi cfi_ld $14, PT_R14, \docfi cfi_ld $15, PT_R15, \docfi From 11981485e27c7e5a630ee844a2eae1f1835ba807 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Wed, 10 Apr 2024 17:54:06 +0200 Subject: [PATCH 1018/1702] clk: meson: s4: fix module autoloading Add MODULE_DEVICE_TABLE(), so modules could be properly autoloaded based on the alias from of_device_id table. Clocks are considered core components, so usually they are built-in, however these can be built and used as modules on some generic kernel. Signed-off-by: Krzysztof Kozlowski Reviewed-by: Neil Armstrong Reviewed-by: Martin Blumenstingl Link: https://lore.kernel.org/r/20240410155406.224128-1-krzk@kernel.org Signed-off-by: Jerome Brunet --- drivers/clk/meson/s4-peripherals.c | 1 + drivers/clk/meson/s4-pll.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/clk/meson/s4-peripherals.c b/drivers/clk/meson/s4-peripherals.c index 1fceb93faf13d..5e17ca50ab091 100644 --- a/drivers/clk/meson/s4-peripherals.c +++ b/drivers/clk/meson/s4-peripherals.c @@ -3800,6 +3800,7 @@ static const struct of_device_id clkc_match_table[] = { }, {} }; +MODULE_DEVICE_TABLE(of, clkc_match_table); static struct platform_driver s4_driver = { .probe = meson_s4_periphs_probe, diff --git a/drivers/clk/meson/s4-pll.c b/drivers/clk/meson/s4-pll.c index c8a95842b14c6..d2650d96400cf 100644 --- a/drivers/clk/meson/s4-pll.c +++ b/drivers/clk/meson/s4-pll.c @@ -854,6 +854,7 @@ static const struct of_device_id clkc_match_table[] = { }, {} }; +MODULE_DEVICE_TABLE(of, clkc_match_table); static struct platform_driver s4_driver = { .probe = meson_s4_pll_probe, From c56436ef17520c5fb0f9c2fc47aa961a7946895f Mon Sep 17 00:00:00 2001 From: Oreoluwa Babatunde Date: Thu, 2 May 2024 12:24:03 -0700 Subject: [PATCH 1019/1702] of: reserved_mem: Remove the use of phandle from the reserved_mem APIs The __find_rmem() function is the only place that references the phandle field of the reserved_mem struct. __find_rmem() is used to match a device_node object to its corresponding entry in the reserved_mem array using its phandle value. But, there is already a function called of_reserved_mem_lookup() which carries out the same action using the name of the node. Using the of_reserved_mem_lookup() function is more reliable because every node is guaranteed to have a name, but not all nodes will have a phandle. Nodes are only assigned a phandle if they are explicitly defined in the DT using "phandle = ", or if they are referenced by another node in the DT. Hence, If the phandle field is empty, then __find_rmem() will return a false negative. Hence, delete the __find_rmem() function and switch to using the of_reserved_mem_lookup() function to find the corresponding entry of a device_node in the reserved_mem array. Since the phandle field of the reserved_mem struct is now unused, delete that as well. Signed-off-by: Oreoluwa Babatunde Link: https://lore.kernel.org/r/20240502192403.3307277-1-quic_obabatun@quicinc.com Signed-off-by: Rob Herring (Arm) --- drivers/of/of_reserved_mem.c | 22 +--------------------- include/linux/of_reserved_mem.h | 1 - 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c index 8236ecae29533..46e1c3fbc7692 100644 --- a/drivers/of/of_reserved_mem.c +++ b/drivers/of/of_reserved_mem.c @@ -437,17 +437,10 @@ void __init fdt_init_reserved_mem(void) for (i = 0; i < reserved_mem_count; i++) { struct reserved_mem *rmem = &reserved_mem[i]; unsigned long node = rmem->fdt_node; - int len; - const __be32 *prop; int err = 0; bool nomap; nomap = of_get_flat_dt_prop(node, "no-map", NULL) != NULL; - prop = of_get_flat_dt_prop(node, "phandle", &len); - if (!prop) - prop = of_get_flat_dt_prop(node, "linux,phandle", &len); - if (prop) - rmem->phandle = of_read_number(prop, len/4); if (rmem->size == 0) err = __reserved_mem_alloc_size(node, rmem->name, @@ -477,19 +470,6 @@ void __init fdt_init_reserved_mem(void) } } -static inline struct reserved_mem *__find_rmem(struct device_node *node) -{ - unsigned int i; - - if (!node->phandle) - return NULL; - - for (i = 0; i < reserved_mem_count; i++) - if (reserved_mem[i].phandle == node->phandle) - return &reserved_mem[i]; - return NULL; -} - struct rmem_assigned_device { struct device *dev; struct reserved_mem *rmem; @@ -534,7 +514,7 @@ int of_reserved_mem_device_init_by_idx(struct device *dev, return 0; } - rmem = __find_rmem(target); + rmem = of_reserved_mem_lookup(target); of_node_put(target); if (!rmem || !rmem->ops || !rmem->ops->device_init) diff --git a/include/linux/of_reserved_mem.h b/include/linux/of_reserved_mem.h index 4de2a24cadc9b..e338282da6520 100644 --- a/include/linux/of_reserved_mem.h +++ b/include/linux/of_reserved_mem.h @@ -11,7 +11,6 @@ struct reserved_mem_ops; struct reserved_mem { const char *name; unsigned long fdt_node; - unsigned long phandle; const struct reserved_mem_ops *ops; phys_addr_t base; phys_addr_t size; From 3fe5a2b9e7b4f2dad254b9c5a8e926cbcdebb49c Mon Sep 17 00:00:00 2001 From: Shresth Prasad Date: Sun, 28 Apr 2024 17:22:27 +0530 Subject: [PATCH 1020/1702] of: property: Use scope based cleanup on port_node Use __free cleanup handler which ensures that the resource is freed when it goes out of scope, thus removing the need to manually clean it up using of_node_put. Suggested-by: Julia Lawall Signed-off-by: Shresth Prasad Link: https://lore.kernel.org/r/20240428115226.41345-2-shresthprasad7@gmail.com Signed-off-by: Rob Herring (Arm) --- drivers/of/property.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/of/property.c b/drivers/of/property.c index 4ede45001025c..8d2477ff89303 100644 --- a/drivers/of/property.c +++ b/drivers/of/property.c @@ -576,7 +576,8 @@ EXPORT_SYMBOL_GPL(of_prop_next_string); int of_graph_parse_endpoint(const struct device_node *node, struct of_endpoint *endpoint) { - struct device_node *port_node = of_get_parent(node); + struct device_node *port_node __free(device_node) = + of_get_parent(node); WARN_ONCE(!port_node, "%s(): endpoint %pOF has no parent node\n", __func__, node); @@ -591,8 +592,6 @@ int of_graph_parse_endpoint(const struct device_node *node, of_property_read_u32(port_node, "reg", &endpoint->port); of_property_read_u32(node, "reg", &endpoint->id); - of_node_put(port_node); - return 0; } EXPORT_SYMBOL(of_graph_parse_endpoint); From 27db752673c8dfaa9295364ebae274d9393490fa Mon Sep 17 00:00:00 2001 From: "Rob Herring (Arm)" Date: Fri, 26 Apr 2024 14:54:04 -0500 Subject: [PATCH 1021/1702] dt-bindings: interrupt-controller: mediatek,mt6577-sysirq: Drop unnecessary quotes Drop unnecessary quotes which aren't needed in yaml. This is checked by yamllint, but this case was excluded due to the comma and yamllint's mishandling of some cases with commas. That's now fixed in yamllint 1.34. Reviewed-by: AngeloGioacchino Del Regno Link: https://lore.kernel.org/r/20240426195404.2771046-1-robh@kernel.org Signed-off-by: Rob Herring (Arm) --- .../bindings/interrupt-controller/mediatek,mt6577-sysirq.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/interrupt-controller/mediatek,mt6577-sysirq.yaml b/Documentation/devicetree/bindings/interrupt-controller/mediatek,mt6577-sysirq.yaml index e1a379c052e44..123d24b05556c 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/mediatek,mt6577-sysirq.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/mediatek,mt6577-sysirq.yaml @@ -48,7 +48,7 @@ properties: interrupt-controller: true "#interrupt-cells": - $ref: "arm,gic.yaml#/properties/#interrupt-cells" + $ref: arm,gic.yaml#/properties/#interrupt-cells required: - reg From 15be4f7ce5de825d1d940ba1bb8e02d09402e133 Mon Sep 17 00:00:00 2001 From: "Rob Herring (Arm)" Date: Fri, 26 Apr 2024 15:22:37 -0500 Subject: [PATCH 1022/1702] dt-bindings: Drop unnecessary quotes on keys The yamllint quoted-strings check wasn't checking keys for quotes, but support for checking keys was added in 1.34 release. Fix all the errors found when enabling the check. Clean-up the xilinx-versal-cpm formatting while we're here. Acked-by: Mark Brown Reviewed-by: Krzysztof Kozlowski Acked-by: Thierry Reding Link: https://lore.kernel.org/r/20240426202239.2837516-1-robh@kernel.org Signed-off-by: Rob Herring (Arm) --- Documentation/devicetree/bindings/net/sff,sfp.yaml | 12 ++++++------ .../devicetree/bindings/pci/xilinx-versal-cpm.yaml | 7 +++++-- .../devicetree/bindings/pci/xlnx,nwl-pcie.yaml | 2 +- .../devicetree/bindings/phy/brcm,sata-phy.yaml | 8 ++++---- .../devicetree/bindings/regulator/ti,tps62864.yaml | 2 +- .../bindings/soc/tegra/nvidia,tegra20-pmc.yaml | 6 +++--- Documentation/devicetree/bindings/tpm/ibm,vtpm.yaml | 4 ++-- 7 files changed, 22 insertions(+), 19 deletions(-) diff --git a/Documentation/devicetree/bindings/net/sff,sfp.yaml b/Documentation/devicetree/bindings/net/sff,sfp.yaml index bf6cbc7c2ba3b..90611b598d2bd 100644 --- a/Documentation/devicetree/bindings/net/sff,sfp.yaml +++ b/Documentation/devicetree/bindings/net/sff,sfp.yaml @@ -29,39 +29,39 @@ properties: allowable by a module in the slot, in milli-Watts. Presently, modules can be up to 1W, 1.5W or 2W. - "mod-def0-gpios": + mod-def0-gpios: maxItems: 1 description: GPIO phandle and a specifier of the MOD-DEF0 (AKA Mod_ABS) module presence input gpio signal, active (module absent) high. Must not be present for SFF modules - "los-gpios": + los-gpios: maxItems: 1 description: GPIO phandle and a specifier of the Receiver Loss of Signal Indication input gpio signal, active (signal lost) high - "tx-fault-gpios": + tx-fault-gpios: maxItems: 1 description: GPIO phandle and a specifier of the Module Transmitter Fault input gpio signal, active (fault condition) high - "tx-disable-gpios": + tx-disable-gpios: maxItems: 1 description: GPIO phandle and a specifier of the Transmitter Disable output gpio signal, active (Tx disable) high - "rate-select0-gpios": + rate-select0-gpios: maxItems: 1 description: GPIO phandle and a specifier of the Rx Signaling Rate Select (AKA RS0) output gpio signal, low - low Rx rate, high - high Rx rate Must not be present for SFF modules - "rate-select1-gpios": + rate-select1-gpios: maxItems: 1 description: GPIO phandle and a specifier of the Tx Signaling Rate Select (AKA RS1) diff --git a/Documentation/devicetree/bindings/pci/xilinx-versal-cpm.yaml b/Documentation/devicetree/bindings/pci/xilinx-versal-cpm.yaml index 4734be456bde6..c41344f8a242a 100644 --- a/Documentation/devicetree/bindings/pci/xilinx-versal-cpm.yaml +++ b/Documentation/devicetree/bindings/pci/xilinx-versal-cpm.yaml @@ -48,13 +48,16 @@ properties: interrupt-controller: description: Interrupt controller node for handling legacy PCI interrupts. type: object + additionalProperties: false + properties: "#address-cells": const: 0 + "#interrupt-cells": const: 1 - "interrupt-controller": true - additionalProperties: false + + interrupt-controller: true required: - reg diff --git a/Documentation/devicetree/bindings/pci/xlnx,nwl-pcie.yaml b/Documentation/devicetree/bindings/pci/xlnx,nwl-pcie.yaml index 426f90a47f355..cbe832c23daed 100644 --- a/Documentation/devicetree/bindings/pci/xlnx,nwl-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/xlnx,nwl-pcie.yaml @@ -84,7 +84,7 @@ properties: "#interrupt-cells": const: 1 - "interrupt-controller": true + interrupt-controller: true required: - "#address-cells" diff --git a/Documentation/devicetree/bindings/phy/brcm,sata-phy.yaml b/Documentation/devicetree/bindings/phy/brcm,sata-phy.yaml index 8467c8e6368cc..439bda1427646 100644 --- a/Documentation/devicetree/bindings/phy/brcm,sata-phy.yaml +++ b/Documentation/devicetree/bindings/phy/brcm,sata-phy.yaml @@ -59,14 +59,14 @@ patternProperties: "#phy-cells": const: 0 - "brcm,enable-ssc": + brcm,enable-ssc: $ref: /schemas/types.yaml#/definitions/flag description: | Use spread spectrum clocking (SSC) on this port This property is not applicable for "brcm,iproc-ns2-sata-phy", "brcm,iproc-nsp-sata-phy" and "brcm,iproc-sr-sata-phy". - "brcm,rxaeq-mode": + brcm,rxaeq-mode: $ref: /schemas/types.yaml#/definitions/string description: String that indicates the desired RX equalizer mode. @@ -75,7 +75,7 @@ patternProperties: - auto - manual - "brcm,rxaeq-value": + brcm,rxaeq-value: $ref: /schemas/types.yaml#/definitions/uint32 description: | When 'brcm,rxaeq-mode' is set to "manual", provides the RX @@ -83,7 +83,7 @@ patternProperties: minimum: 0 maximum: 63 - "brcm,tx-amplitude-millivolt": + brcm,tx-amplitude-millivolt: description: | Transmit amplitude voltage in millivolt. $ref: /schemas/types.yaml#/definitions/uint32 diff --git a/Documentation/devicetree/bindings/regulator/ti,tps62864.yaml b/Documentation/devicetree/bindings/regulator/ti,tps62864.yaml index 0f29c75f42ea6..dddea27596e99 100644 --- a/Documentation/devicetree/bindings/regulator/ti,tps62864.yaml +++ b/Documentation/devicetree/bindings/regulator/ti,tps62864.yaml @@ -24,7 +24,7 @@ properties: type: object properties: - "SW": + SW: type: object $ref: regulator.yaml# unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/soc/tegra/nvidia,tegra20-pmc.yaml b/Documentation/devicetree/bindings/soc/tegra/nvidia,tegra20-pmc.yaml index b86f6f53ca95d..7140c312d8986 100644 --- a/Documentation/devicetree/bindings/soc/tegra/nvidia,tegra20-pmc.yaml +++ b/Documentation/devicetree/bindings/soc/tegra/nvidia,tegra20-pmc.yaml @@ -365,9 +365,9 @@ allOf: additionalProperties: false dependencies: - "nvidia,suspend-mode": ["nvidia,core-pwr-off-time", "nvidia,cpu-pwr-off-time"] - "nvidia,core-pwr-off-time": ["nvidia,core-pwr-good-time"] - "nvidia,cpu-pwr-off-time": ["nvidia,cpu-pwr-good-time"] + nvidia,suspend-mode: ["nvidia,core-pwr-off-time", "nvidia,cpu-pwr-off-time"] + nvidia,core-pwr-off-time: ["nvidia,core-pwr-good-time"] + nvidia,cpu-pwr-off-time: ["nvidia,cpu-pwr-good-time"] examples: - | diff --git a/Documentation/devicetree/bindings/tpm/ibm,vtpm.yaml b/Documentation/devicetree/bindings/tpm/ibm,vtpm.yaml index 50a3fd31241cb..8b0d3d4be5d8c 100644 --- a/Documentation/devicetree/bindings/tpm/ibm,vtpm.yaml +++ b/Documentation/devicetree/bindings/tpm/ibm,vtpm.yaml @@ -33,13 +33,13 @@ properties: reg: maxItems: 1 - 'ibm,#dma-address-cells': + ibm,#dma-address-cells: description: number of cells that are used to encode the physical address field of dma-window properties $ref: /schemas/types.yaml#/definitions/uint32-array - 'ibm,#dma-size-cells': + ibm,#dma-size-cells: description: number of cells that are used to encode the size field of dma-window properties From 649bad67d4b118380bba1e1daa0db2978f7e20be Mon Sep 17 00:00:00 2001 From: Valentina Fernandez Date: Wed, 27 Mar 2024 12:24:39 +0000 Subject: [PATCH 1023/1702] dt-bindings: PCI: microchip: increase number of items in ranges property Increase the number of items in the ranges property to allow up to 3 ranges. For example a prefetchable range, a non-prefetchable range and an IO range, depending on configuration. Signed-off-by: Valentina Fernandez Signed-off-by: Conor Dooley Link: https://lore.kernel.org/r/20240327-debunk-perky-f5514ca332be@spud Signed-off-by: Rob Herring (Arm) --- Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml b/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml index f7a3c26363556..e8212a05b7b13 100644 --- a/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml +++ b/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml @@ -65,7 +65,8 @@ properties: - const: msi ranges: - maxItems: 1 + minItems: 1 + maxItems: 3 dma-ranges: minItems: 1 From b3e991240ed39f8fa4f74d3a876b48ef80726f84 Mon Sep 17 00:00:00 2001 From: "Rob Herring (Arm)" Date: Tue, 30 Apr 2024 13:04:14 -0500 Subject: [PATCH 1024/1702] dt-bindings: clock: fixed: Define a preferred node name Define "clock-" as the preferred node name for fixed-clock and fixed-factor-clock where is the output frequency of the clock. There isn't much of an existing pattern for names of these nodes. The most frequent patterns are a prefix or suffix of "clk", but there's a bunch that don't follow any sort of pattern. We could use "clock-controller-.*", but these nodes aren't really a controller in any way. So let's at least align with part of that and use 'clock-'. For now this only serves as documentation as the schema still allows anything to avoid lots of additional warnings for something low priority to fix. Once a "no deprecated" mode is added to the tools, warnings can be enabled selectively. Signed-off-by: Rob Herring (Arm) Link: https://lore.kernel.org/r/20240430180415.657067-1-robh@kernel.org Acked-by: Conor Dooley Signed-off-by: Stephen Boyd --- Documentation/devicetree/bindings/clock/fixed-clock.yaml | 9 +++++++++ .../devicetree/bindings/clock/fixed-factor-clock.yaml | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/Documentation/devicetree/bindings/clock/fixed-clock.yaml b/Documentation/devicetree/bindings/clock/fixed-clock.yaml index b0a4fb8256e2f..90fb106606847 100644 --- a/Documentation/devicetree/bindings/clock/fixed-clock.yaml +++ b/Documentation/devicetree/bindings/clock/fixed-clock.yaml @@ -11,6 +11,15 @@ maintainers: - Stephen Boyd properties: + $nodename: + anyOf: + - description: + Preferred name is 'clock-' with being the output + frequency as defined in the 'clock-frequency' property. + pattern: "^clock-([0-9]+|[a-z0-9-]+)$" + - description: Any name allowed + deprecated: true + compatible: const: fixed-clock diff --git a/Documentation/devicetree/bindings/clock/fixed-factor-clock.yaml b/Documentation/devicetree/bindings/clock/fixed-factor-clock.yaml index 8f71ab3004706..4afdb1c98f5fb 100644 --- a/Documentation/devicetree/bindings/clock/fixed-factor-clock.yaml +++ b/Documentation/devicetree/bindings/clock/fixed-factor-clock.yaml @@ -11,6 +11,15 @@ maintainers: - Stephen Boyd properties: + $nodename: + anyOf: + - description: + If the frequency is fixed, the preferred name is 'clock-' with + being the output frequency. + pattern: "^clock-([0-9]+|[0-9a-z-]+)$" + - description: Any name allowed + deprecated: true + compatible: enum: - fixed-factor-clock From 947b8f2a8b5155f6e9560af07ed65b3cc9aecd75 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Wed, 24 Apr 2024 16:48:29 +0200 Subject: [PATCH 1025/1702] clk: rockchip: Remove an unused field in struct rockchip_mmc_clock In "struct rockchip_mmc_clock", the 'id' field is unused. Remove it. Found with cppcheck, unusedStructMember. Signed-off-by: Christophe JAILLET Link: https://lore.kernel.org/r/410bc0f86c7b9f1c80f8a4e9a2a028a9a6ee1ec0.1713970085.git.christophe.jaillet@wanadoo.fr Signed-off-by: Heiko Stuebner --- drivers/clk/rockchip/clk-mmc-phase.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/clk/rockchip/clk-mmc-phase.c b/drivers/clk/rockchip/clk-mmc-phase.c index 975454a3dd72b..91012078681bf 100644 --- a/drivers/clk/rockchip/clk-mmc-phase.c +++ b/drivers/clk/rockchip/clk-mmc-phase.c @@ -14,7 +14,6 @@ struct rockchip_mmc_clock { struct clk_hw hw; void __iomem *reg; - int id; int shift; int cached_phase; struct notifier_block clk_rate_change_nb; From f513991b69885025995dcb4ca75d2ee7261e1273 Mon Sep 17 00:00:00 2001 From: Lucas Stach Date: Fri, 3 May 2024 17:33:29 +0200 Subject: [PATCH 1026/1702] clk: rockchip: rk3568: Add PLL rate for 724 MHz This rate allows to provide a low-jitter 72,4 MHz pixelclock for a custom eDP panel from the VPLL. Signed-off-by: Lucas Stach Link: https://lore.kernel.org/r/20240503153329.3906030-1-l.stach@pengutronix.de Signed-off-by: Heiko Stuebner --- drivers/clk/rockchip/clk-rk3568.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/clk/rockchip/clk-rk3568.c b/drivers/clk/rockchip/clk-rk3568.c index 2d44bcaef046b..53d10b1c627b5 100644 --- a/drivers/clk/rockchip/clk-rk3568.c +++ b/drivers/clk/rockchip/clk-rk3568.c @@ -64,6 +64,7 @@ static struct rockchip_pll_rate_table rk3568_pll_rates[] = { RK3036_PLL_RATE(912000000, 1, 76, 2, 1, 1, 0), RK3036_PLL_RATE(816000000, 1, 68, 2, 1, 1, 0), RK3036_PLL_RATE(800000000, 3, 200, 2, 1, 1, 0), + RK3036_PLL_RATE(724000000, 3, 181, 2, 1, 1, 0), RK3036_PLL_RATE(700000000, 3, 350, 4, 1, 1, 0), RK3036_PLL_RATE(696000000, 1, 116, 4, 1, 1, 0), RK3036_PLL_RATE(600000000, 1, 100, 4, 1, 1, 0), From 1f6602c8ed1eccac4fa83e338d02aafe3a0e2c0a Mon Sep 17 00:00:00 2001 From: Mark Pearson Date: Fri, 15 Mar 2024 15:52:12 -0400 Subject: [PATCH 1027/1702] watchdog: lenovo_se10_wdt: Watchdog driver for Lenovo SE10 platform Watchdog driver implementation for Lenovo SE10 platform. Signed-off-by: Mark Pearson Signed-off-by: David Ober Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20240315195227.91282-1-mpearson-lenovo@squebb.ca Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/Kconfig | 10 + drivers/watchdog/Makefile | 1 + drivers/watchdog/lenovo_se10_wdt.c | 308 +++++++++++++++++++++++++++++ 3 files changed, 319 insertions(+) create mode 100644 drivers/watchdog/lenovo_se10_wdt.c diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig index 6bee137cfbe00..419a2e3e0dbe9 100644 --- a/drivers/watchdog/Kconfig +++ b/drivers/watchdog/Kconfig @@ -254,6 +254,16 @@ config GPIO_WATCHDOG_ARCH_INITCALL arch_initcall. If in doubt, say N. +config LENOVO_SE10_WDT + tristate "Lenovo SE10 Watchdog" + select WATCHDOG_CORE + help + If you say yes here you get support for the watchdog + functionality for the Lenovo SE10 platform. + + This driver can also be built as a module. If so, the module + will be called lenovo-se10-wdt. + config MENF21BMC_WATCHDOG tristate "MEN 14F021P00 BMC Watchdog" depends on MFD_MENF21BMC || COMPILE_TEST diff --git a/drivers/watchdog/Makefile b/drivers/watchdog/Makefile index 3710c218f05e4..2d1117564f5b2 100644 --- a/drivers/watchdog/Makefile +++ b/drivers/watchdog/Makefile @@ -120,6 +120,7 @@ obj-$(CONFIG_WAFER_WDT) += wafer5823wdt.o obj-$(CONFIG_I6300ESB_WDT) += i6300esb.o obj-$(CONFIG_IE6XX_WDT) += ie6xx_wdt.o obj-$(CONFIG_ITCO_WDT) += iTCO_wdt.o +obj-$(CONFIG_LENOVO_SE10_WDT) += lenovo_se10_wdt.o ifeq ($(CONFIG_ITCO_VENDOR_SUPPORT),y) obj-$(CONFIG_ITCO_WDT) += iTCO_vendor_support.o endif diff --git a/drivers/watchdog/lenovo_se10_wdt.c b/drivers/watchdog/lenovo_se10_wdt.c new file mode 100644 index 0000000000000..139ff0e8220fb --- /dev/null +++ b/drivers/watchdog/lenovo_se10_wdt.c @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * WDT driver for Lenovo SE10. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define STATUS_PORT 0x6C +#define CMD_PORT 0x6C +#define DATA_PORT 0x68 +#define OUTBUF_FULL 0x01 +#define INBUF_EMPTY 0x02 +#define CFG_LDN 0x07 +#define CFG_BRAM_LDN 0x10 /* for BRAM Base */ +#define CFG_PORT 0x2E +#define CFG_SIZE 2 +#define CMD_SIZE 4 +#define BRAM_SIZE 2 + +#define UNLOCK_KEY 0x87 +#define LOCK_KEY 0xAA + +#define CUS_WDT_SWI 0x1A +#define CUS_WDT_CFG 0x1B +#define CUS_WDT_FEED 0xB0 +#define CUS_WDT_CNT 0xB1 + +#define DRVNAME "lenovo-se10-wdt" + +/*The timeout range is 1-255 seconds*/ +#define MIN_TIMEOUT 1 +#define MAX_TIMEOUT 255 +#define MAX_WAIT 10 + +#define WATCHDOG_TIMEOUT 60 /* 60 sec default timeout */ +static unsigned short bram_base; +static struct platform_device *se10_pdev; + +static int timeout; /* in seconds */ +module_param(timeout, int, 0); +MODULE_PARM_DESC(timeout, + "Watchdog timeout in seconds. 1 <= timeout <= 255, default=" + __MODULE_STRING(WATCHDOG_TIMEOUT) "."); + +static bool nowayout = WATCHDOG_NOWAYOUT; +module_param(nowayout, bool, 0); +MODULE_PARM_DESC(nowayout, + "Watchdog cannot be stopped once started (default=" + __MODULE_STRING(WATCHDOG_NOWAYOUT) ")"); + +struct se10_wdt { + struct watchdog_device wdd; +}; + +static int set_bram(unsigned char offset, unsigned char val) +{ + if (!request_muxed_region(bram_base, BRAM_SIZE, DRVNAME)) + return -EBUSY; + outb(offset, bram_base); + outb(val, bram_base + 1); + release_region(bram_base, BRAM_SIZE); + return 0; +} + +static void wait_for_buffer(int condition) +{ + int loop = 0; + + while (1) { + if (inb(STATUS_PORT) & condition || loop > MAX_WAIT) + break; + loop++; + usleep_range(10, 125); + } +} + +static void send_cmd(unsigned char cmd) +{ + wait_for_buffer(INBUF_EMPTY); + outb(cmd, CMD_PORT); + wait_for_buffer(INBUF_EMPTY); +} + +static void lpc_write(unsigned char index, unsigned char data) +{ + outb(index, CFG_PORT); + outb(data, CFG_PORT + 1); +} + +static unsigned char lpc_read(unsigned char index) +{ + outb(index, CFG_PORT); + return inb(CFG_PORT + 1); +} + +static int wdt_start(struct watchdog_device *wdog) +{ + return set_bram(CUS_WDT_SWI, 0x80); +} + +static int wdt_set_timeout(struct watchdog_device *wdog, unsigned int timeout) +{ + wdog->timeout = timeout; + return set_bram(CUS_WDT_CFG, wdog->timeout); +} + +static int wdt_stop(struct watchdog_device *wdog) +{ + return set_bram(CUS_WDT_SWI, 0); +} + +static unsigned int wdt_get_time(struct watchdog_device *wdog) +{ + unsigned char time; + + if (!request_muxed_region(CMD_PORT, CMD_SIZE, DRVNAME)) + return -EBUSY; + send_cmd(CUS_WDT_CNT); + wait_for_buffer(OUTBUF_FULL); + time = inb(DATA_PORT); + release_region(CMD_PORT, CMD_SIZE); + return time; +} + +static int wdt_ping(struct watchdog_device *wdog) +{ + if (!request_muxed_region(CMD_PORT, CMD_SIZE, DRVNAME)) + return -EBUSY; + send_cmd(CUS_WDT_FEED); + release_region(CMD_PORT, CMD_SIZE); + return 0; +} + +static const struct watchdog_info wdt_info = { + .options = WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING | WDIOF_MAGICCLOSE, + .identity = "Lenovo SE10 Watchdog", +}; + +static const struct watchdog_ops se10_wdt_ops = { + .owner = THIS_MODULE, + .start = wdt_start, + .stop = wdt_stop, + .ping = wdt_ping, + .set_timeout = wdt_set_timeout, + .get_timeleft = wdt_get_time, +}; + +static unsigned int get_chipID(void) +{ + unsigned char msb, lsb; + + outb(UNLOCK_KEY, CFG_PORT); + outb(0x01, CFG_PORT); + outb(0x55, CFG_PORT); + outb(0x55, CFG_PORT); + msb = lpc_read(0x20); + lsb = lpc_read(0x21); + outb(LOCK_KEY, CFG_PORT); + return (msb * 256 + lsb); +} + +static int se10_wdt_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct se10_wdt *priv; + unsigned int chip_id; + int ret; + + if (!request_muxed_region(CFG_PORT, CFG_SIZE, DRVNAME)) + return -EBUSY; + + chip_id = get_chipID(); + if (chip_id != 0x5632) { + release_region(CFG_PORT, CFG_SIZE); + return -ENODEV; + } + + lpc_write(CFG_LDN, CFG_BRAM_LDN); + bram_base = (lpc_read(0x60) << 8) | lpc_read(0x61); + release_region(CFG_PORT, CFG_SIZE); + + dev_info(dev, "Found Lenovo SE10 0x%x\n", chip_id); + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + watchdog_set_drvdata(&priv->wdd, priv); + + priv->wdd.parent = dev; + priv->wdd.info = &wdt_info, + priv->wdd.ops = &se10_wdt_ops, + priv->wdd.timeout = WATCHDOG_TIMEOUT; /* Set default timeout */ + priv->wdd.min_timeout = MIN_TIMEOUT; + priv->wdd.max_timeout = MAX_TIMEOUT; + + set_bram(CUS_WDT_CFG, WATCHDOG_TIMEOUT); /* Set time to default */ + + watchdog_init_timeout(&priv->wdd, timeout, dev); + watchdog_set_nowayout(&priv->wdd, nowayout); + watchdog_stop_on_reboot(&priv->wdd); + watchdog_stop_on_unregister(&priv->wdd); + + ret = devm_watchdog_register_device(dev, &priv->wdd); + + dev_dbg(&pdev->dev, "initialized. timeout=%d sec (nowayout=%d)\n", + priv->wdd.timeout, nowayout); + + return ret; +} + +static struct platform_driver se10_wdt_driver = { + .driver = { + .name = DRVNAME, + }, + .probe = se10_wdt_probe, +}; + +static int se10_create_platform_device(const struct dmi_system_id *id) +{ + int err; + + se10_pdev = platform_device_alloc("lenovo-se10-wdt", -1); + if (!se10_pdev) + return -ENOMEM; + + err = platform_device_add(se10_pdev); + if (err) + platform_device_put(se10_pdev); + + return err; +} + +static const struct dmi_system_id se10_dmi_table[] __initconst = { + { + .ident = "LENOVO-SE10", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "12NH"), + }, + .callback = se10_create_platform_device, + }, + { + .ident = "LENOVO-SE10", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "12NJ"), + }, + .callback = se10_create_platform_device, + }, + { + .ident = "LENOVO-SE10", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "12NK"), + }, + .callback = se10_create_platform_device, + }, + { + .ident = "LENOVO-SE10", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "12NL"), + }, + .callback = se10_create_platform_device, + }, + { + .ident = "LENOVO-SE10", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "12NM"), + }, + .callback = se10_create_platform_device, + }, + {} +}; +MODULE_DEVICE_TABLE(dmi, se10_dmi_table); + +static int __init se10_wdt_init(void) +{ + if (!dmi_check_system(se10_dmi_table)) + return -ENODEV; + + return platform_driver_register(&se10_wdt_driver); +} + +static void __exit se10_wdt_exit(void) +{ + if (se10_pdev) + platform_device_unregister(se10_pdev); + platform_driver_unregister(&se10_wdt_driver); +} + +module_init(se10_wdt_init); +module_exit(se10_wdt_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("David Ober"); +MODULE_AUTHOR("Mark Pearson "); +MODULE_DESCRIPTION("WDT driver for Lenovo SE10"); From 573601521277119f2e2ba5f28ae6e87fc594f4d4 Mon Sep 17 00:00:00 2001 From: Duoming Zhou Date: Sun, 24 Mar 2024 22:04:44 +0800 Subject: [PATCH 1028/1702] watchdog: cpu5wdt.c: Fix use-after-free bug caused by cpu5wdt_trigger When the cpu5wdt module is removing, the origin code uses del_timer() to de-activate the timer. If the timer handler is running, del_timer() could not stop it and will return directly. If the port region is released by release_region() and then the timer handler cpu5wdt_trigger() calls outb() to write into the region that is released, the use-after-free bug will happen. Change del_timer() to timer_shutdown_sync() in order that the timer handler could be finished before the port region is released. Fixes: e09d9c3e9f85 ("watchdog: cpu5wdt.c: add missing del_timer call") Signed-off-by: Duoming Zhou Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20240324140444.119584-1-duoming@zju.edu.cn Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/cpu5wdt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/watchdog/cpu5wdt.c b/drivers/watchdog/cpu5wdt.c index 688b112e712ba..9f279c0e13a66 100644 --- a/drivers/watchdog/cpu5wdt.c +++ b/drivers/watchdog/cpu5wdt.c @@ -252,7 +252,7 @@ static void cpu5wdt_exit(void) if (cpu5wdt_device.queue) { cpu5wdt_device.queue = 0; wait_for_completion(&cpu5wdt_device.stop); - del_timer(&cpu5wdt_device.timer); + timer_shutdown_sync(&cpu5wdt_device.timer); } misc_deregister(&cpu5wdt_misc); From 56e23c6d7ffbf0dfc6506552f1059b309a4d5dcd Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Wed, 27 Mar 2024 18:46:38 +0100 Subject: [PATCH 1029/1702] watchdog: mtx-1: drop driver owner assignment Core in platform_driver_register() already sets the .owner, so driver does not need to. Signed-off-by: Krzysztof Kozlowski Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20240327174638.519445-1-krzysztof.kozlowski@linaro.org Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/mtx-1_wdt.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/watchdog/mtx-1_wdt.c b/drivers/watchdog/mtx-1_wdt.c index 152e41ecbb142..06756135033d2 100644 --- a/drivers/watchdog/mtx-1_wdt.c +++ b/drivers/watchdog/mtx-1_wdt.c @@ -236,7 +236,6 @@ static struct platform_driver mtx1_wdt_driver = { .probe = mtx1_wdt_probe, .remove_new = mtx1_wdt_remove, .driver.name = "mtx1-wdt", - .driver.owner = THIS_MODULE, }; module_platform_driver(mtx1_wdt_driver); From e3b3afd34d84efcbe4543deb966b1990f43584b8 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Mon, 8 Apr 2024 13:02:31 +0300 Subject: [PATCH 1030/1702] watchdog: bd9576: Drop "always-running" property The always-running (from linux,wdt-gpio.yaml) is abused by the BD9576 watchdog driver. It's defined meaning is "the watchdog is always running and can not be stopped". The BD9576 watchdog driver has implemented it as "start watchdog when loading the module and prevent it from being stopped". Furthermore, the implementation does not set the WDOG_HW_RUNNING when enabling the watchdog due to the "always-running" at module loading. This will end up resulting a watchdog timeout if the device is not opened. The culprit was pointed out by Guenter, discussion can be found from https://lore.kernel.org/lkml/4fa3a64b-60fb-4e5e-8785-0f14da37eea2@roeck-us.net/ Drop the invalid "always-running" handling. Signed-off-by: Matti Vaittinen Reported-by: Guenter Roeck Fixes: b237bcac557a ("wdt: Support wdt on ROHM BD9576MUF and BD9573MUF") Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/ZhPAt76yaJMersXf@fedora Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/bd9576_wdt.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/drivers/watchdog/bd9576_wdt.c b/drivers/watchdog/bd9576_wdt.c index 4a20e07fbb699..f00ea1b4e40b6 100644 --- a/drivers/watchdog/bd9576_wdt.c +++ b/drivers/watchdog/bd9576_wdt.c @@ -29,7 +29,6 @@ struct bd9576_wdt_priv { struct gpio_desc *gpiod_en; struct device *dev; struct regmap *regmap; - bool always_running; struct watchdog_device wdd; }; @@ -62,10 +61,7 @@ static int bd9576_wdt_stop(struct watchdog_device *wdd) { struct bd9576_wdt_priv *priv = watchdog_get_drvdata(wdd); - if (!priv->always_running) - bd9576_wdt_disable(priv); - else - set_bit(WDOG_HW_RUNNING, &wdd->status); + bd9576_wdt_disable(priv); return 0; } @@ -264,9 +260,6 @@ static int bd9576_wdt_probe(struct platform_device *pdev) if (ret) return ret; - priv->always_running = device_property_read_bool(dev->parent, - "always-running"); - watchdog_set_drvdata(&priv->wdd, priv); priv->wdd.info = &bd957x_wdt_ident; @@ -281,9 +274,6 @@ static int bd9576_wdt_probe(struct platform_device *pdev) watchdog_stop_on_reboot(&priv->wdd); - if (priv->always_running) - bd9576_wdt_start(&priv->wdd); - return devm_watchdog_register_device(dev, &priv->wdd); } From 4c97f0433de08ab4b52d7a2747daf44430c6d489 Mon Sep 17 00:00:00 2001 From: Dawei Li Date: Wed, 10 Apr 2024 21:13:59 +0800 Subject: [PATCH 1031/1702] watchdog/wdt-main: Use cpumask_of() to avoid cpumask var on stack In general it's preferable to avoid placing cpumasks on the stack, as for large values of NR_CPUS these can consume significant amounts of stack space and make stack overflows more likely. Use cpumask_of() to avoid the need for a temporary cpumask on the stack Signed-off-by: Dawei Li Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/TYTP286MB3564B037A81DAAE1AC3A16F3CA062@TYTP286MB3564.JPNP286.PROD.OUTLOOK.COM Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/octeon-wdt-main.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/watchdog/octeon-wdt-main.c b/drivers/watchdog/octeon-wdt-main.c index 0fe71f7e66d5d..52d49e4e35a0c 100644 --- a/drivers/watchdog/octeon-wdt-main.c +++ b/drivers/watchdog/octeon-wdt-main.c @@ -381,11 +381,7 @@ static int octeon_wdt_cpu_online(unsigned int cpu) /* Must set the irq affinity here */ if (octeon_has_feature(OCTEON_FEATURE_CIU3)) { - cpumask_t mask; - - cpumask_clear(&mask); - cpumask_set_cpu(cpu, &mask); - irq_set_affinity(irq, &mask); + irq_set_affinity(irq, cpumask_of(cpu)); } cpumask_set_cpu(cpu, &irq_enabled_cpus); From 52df67b6b313984e1b58030fd1f6b8a873799e36 Mon Sep 17 00:00:00 2001 From: Niklas Schnelle Date: Wed, 10 Apr 2024 10:42:01 +0200 Subject: [PATCH 1032/1702] watchdog: add HAS_IOPORT dependencies In a future patch HAS_IOPORT=n will disable inb()/outb() and friends at compile time. We thus need to add HAS_IOPORT as dependency for those drivers using them. Co-developed-by: Arnd Bergmann Signed-off-by: Arnd Bergmann Signed-off-by: Niklas Schnelle Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20240410084201.1481930-2-schnelle@linux.ibm.com Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/Kconfig | 58 +++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig index 419a2e3e0dbe9..f99880a3b976a 100644 --- a/drivers/watchdog/Kconfig +++ b/drivers/watchdog/Kconfig @@ -492,6 +492,7 @@ config 21285_WATCHDOG config 977_WATCHDOG tristate "NetWinder WB83C977 watchdog" depends on (FOOTBRIDGE && ARCH_NETWINDER) || (ARM && COMPILE_TEST) + depends on HAS_IOPORT help Say Y here to include support for the WB977 watchdog included in NetWinder machines. Alternatively say M to compile the driver as @@ -1085,7 +1086,7 @@ config ACQUIRE_WDT config ADVANTECH_WDT tristate "Advantech SBC Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help If you are configuring a Linux kernel for the Advantech single-board computer, say `Y' here to support its built-in watchdog timer @@ -1094,7 +1095,7 @@ config ADVANTECH_WDT config ADVANTECH_EC_WDT tristate "Advantech Embedded Controller Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT select ISA_BUS_API select WATCHDOG_CORE help @@ -1127,7 +1128,7 @@ config ALIM7101_WDT config EBC_C384_WDT tristate "WinSystems EBC-C384 Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT select ISA_BUS_API select WATCHDOG_CORE help @@ -1137,7 +1138,7 @@ config EBC_C384_WDT config EXAR_WDT tristate "Exar Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT select WATCHDOG_CORE help Enables watchdog timer support for the watchdog timer present @@ -1148,7 +1149,7 @@ config EXAR_WDT config F71808E_WDT tristate "Fintek F718xx, F818xx Super I/O Watchdog" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT select WATCHDOG_CORE help This is the driver for the hardware watchdog on the Fintek F71808E, @@ -1160,7 +1161,7 @@ config F71808E_WDT config SP5100_TCO tristate "AMD/ATI SP5100 TCO Timer/Watchdog" - depends on (X86 || COMPILE_TEST) && PCI + depends on (X86 || COMPILE_TEST) && PCI && HAS_IOPORT select WATCHDOG_CORE help Hardware watchdog driver for the AMD/ATI SP5100 chipset. The TCO @@ -1199,7 +1200,7 @@ config SC520_WDT config SBC_FITPC2_WATCHDOG tristate "Compulab SBC-FITPC2 watchdog" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the built-in watchdog timer on the fit-PC2, fit-PC2i, CM-iAM single-board computers made by Compulab. @@ -1224,7 +1225,7 @@ config SBC_FITPC2_WATCHDOG config EUROTECH_WDT tristate "Eurotech CPU-1220/1410 Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help Enable support for the watchdog timer on the Eurotech CPU-1220 and CPU-1410 cards. These are PC/104 SBCs. Spec sheets and product @@ -1232,7 +1233,7 @@ config EUROTECH_WDT config IB700_WDT tristate "IB700 SBC Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the hardware watchdog on the IB700 Single Board Computer produced by TMC Technology (www.tmc-uk.com). This @@ -1249,7 +1250,7 @@ config IB700_WDT config IBMASR tristate "IBM Automatic Server Restart" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the IBM Automatic Server Restart watchdog timer built-in into some eServer xSeries machines. @@ -1259,7 +1260,7 @@ config IBMASR config WAFER_WDT tristate "ICP Single Board Computer Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is a driver for the hardware watchdog on the ICP Single Board Computer. This driver is working on (at least) the following @@ -1281,7 +1282,7 @@ config I6300ESB_WDT config IE6XX_WDT tristate "Intel Atom E6xx Watchdog" - depends on (X86 || COMPILE_TEST) && PCI + depends on (X86 || COMPILE_TEST) && PCI && HAS_IOPORT select WATCHDOG_CORE select MFD_CORE select LPC_SCH @@ -1311,6 +1312,7 @@ config ITCO_WDT select WATCHDOG_CORE depends on I2C || I2C=n depends on MFD_INTEL_PMC_BXT || !MFD_INTEL_PMC_BXT + depends on HAS_IOPORT # for I2C_I801 select LPC_ICH if !EXPERT select I2C_I801 if !EXPERT && I2C help @@ -1341,7 +1343,7 @@ config ITCO_VENDOR_SUPPORT config IT8712F_WDT tristate "IT8712F (Smart Guardian) Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the built-in watchdog timer on the IT8712F Super I/0 chipset used on many motherboards. @@ -1354,7 +1356,7 @@ config IT8712F_WDT config IT87_WDT tristate "IT87 Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT select WATCHDOG_CORE help This is the driver for the hardware watchdog on the ITE IT8607, @@ -1402,7 +1404,7 @@ config KEMPLD_WDT config SC1200_WDT tristate "National Semiconductor PC87307/PC97307 (ala SC1200) Watchdog" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is a driver for National Semiconductor PC87307/PC97307 hardware watchdog cards as found on the SC1200. This watchdog is mainly used @@ -1425,7 +1427,7 @@ config SCx200_WDT config PC87413_WDT tristate "NS PC87413 watchdog" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the hardware watchdog on the PC87413 chipset This watchdog simply watches your kernel to make sure it doesn't @@ -1439,7 +1441,7 @@ config PC87413_WDT config NV_TCO tristate "nVidia TCO Timer/Watchdog" - depends on (X86 || COMPILE_TEST) && PCI + depends on (X86 || COMPILE_TEST) && PCI && HAS_IOPORT help Hardware driver for the TCO timer built into the nVidia Hub family (such as the MCP51). The TCO (Total Cost of Ownership) timer is a @@ -1468,7 +1470,7 @@ config RDC321X_WDT config 60XX_WDT tristate "SBC-60XX Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This driver can be used with the watchdog timer found on some single board computers, namely the 6010 PII based computer. @@ -1508,7 +1510,7 @@ config SBC7240_WDT config CPU5_WDT tristate "SMA CPU5 Watchdog" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help TBD. To compile this driver as a module, choose M here: the @@ -1516,7 +1518,7 @@ config CPU5_WDT config SMSC_SCH311X_WDT tristate "SMSC SCH311X Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the hardware watchdog timer on the SMSC SCH3112, SCH3114 and SCH3116 Super IO chipset @@ -1528,7 +1530,7 @@ config SMSC_SCH311X_WDT config SMSC37B787_WDT tristate "Winbond SMsC37B787 Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the hardware watchdog component on the Winbond SMsC37B787 chipset as used on the NetRunner Mainboard @@ -1574,7 +1576,7 @@ config VIA_WDT config W83627HF_WDT tristate "Watchdog timer for W83627HF/W83627DHG and compatibles" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT select WATCHDOG_CORE help This is the driver for the hardware watchdog on the following @@ -1604,7 +1606,7 @@ config W83627HF_WDT config W83877F_WDT tristate "W83877F (EMACS) Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the hardware watchdog on the W83877F chipset as used in EMACS PC-104 motherboards (and likely others). This @@ -1619,7 +1621,7 @@ config W83877F_WDT config W83977F_WDT tristate "W83977F (PCM-5335) Watchdog Timer" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the hardware watchdog on the W83977F I/O chip as used in AAEON's PCM-5335 SBC (and likely others). This @@ -1632,7 +1634,7 @@ config W83977F_WDT config MACHZ_WDT tristate "ZF MachZ Watchdog" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help If you are using a ZF Micro MachZ processor, say Y here, otherwise N. This is the driver for the watchdog timer built-in on that @@ -1645,7 +1647,7 @@ config MACHZ_WDT config SBC_EPX_C3_WATCHDOG tristate "Winsystems SBC EPX-C3 watchdog" - depends on X86 || COMPILE_TEST + depends on (X86 || COMPILE_TEST) && HAS_IOPORT help This is the driver for the built-in watchdog timer on the EPX-C3 Single-board computer made by Winsystems, Inc. @@ -2207,7 +2209,7 @@ comment "PCI-based Watchdog Cards" config PCIPCWATCHDOG tristate "Berkshire Products PCI-PC Watchdog" - depends on PCI + depends on PCI && HAS_IOPORT help This is the driver for the Berkshire Products PCI-PC Watchdog card. This card simply watches your kernel to make sure it doesn't freeze, @@ -2222,7 +2224,7 @@ config PCIPCWATCHDOG config WDTPCI tristate "PCI-WDT500/501 Watchdog timer" - depends on PCI + depends on PCI && HAS_IOPORT help If you have a PCI-WDT500/501 watchdog board, say Y here, otherwise N. From cae58516534e110f4a8558d48aa4435e15519121 Mon Sep 17 00:00:00 2001 From: Judith Mendez Date: Wed, 17 Apr 2024 15:57:00 -0500 Subject: [PATCH 1033/1702] watchdog: rti_wdt: Set min_hw_heartbeat_ms to accommodate a safety margin On AM62x, the watchdog is pet before the valid window is open. Fix min_hw_heartbeat and accommodate a 2% + static offset safety margin. The static offset accounts for max hardware error. Remove the hack in the driver which shifts the open window boundary, since it is no longer necessary due to the fix mentioned above. cc: stable@vger.kernel.org Fixes: 5527483f8f7c ("watchdog: rti-wdt: attach to running watchdog during probe") Signed-off-by: Judith Mendez Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20240417205700.3947408-1-jm@ti.com Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/rti_wdt.c | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/drivers/watchdog/rti_wdt.c b/drivers/watchdog/rti_wdt.c index 9215793a1c814..4895a69015a8e 100644 --- a/drivers/watchdog/rti_wdt.c +++ b/drivers/watchdog/rti_wdt.c @@ -59,6 +59,8 @@ #define PON_REASON_EOF_NUM 0xCCCCBBBB #define RESERVED_MEM_MIN_SIZE 12 +#define MAX_HW_ERROR 250 + static int heartbeat = DEFAULT_HEARTBEAT; /* @@ -97,7 +99,7 @@ static int rti_wdt_start(struct watchdog_device *wdd) * to be 50% or less than that; we obviouly want to configure the open * window as large as possible so we select the 50% option. */ - wdd->min_hw_heartbeat_ms = 500 * wdd->timeout; + wdd->min_hw_heartbeat_ms = 520 * wdd->timeout + MAX_HW_ERROR; /* Generate NMI when wdt expires */ writel_relaxed(RTIWWDRX_NMI, wdt->base + RTIWWDRXCTRL); @@ -131,31 +133,33 @@ static int rti_wdt_setup_hw_hb(struct watchdog_device *wdd, u32 wsize) * be petted during the open window; not too early or not too late. * The HW configuration options only allow for the open window size * to be 50% or less than that. + * To avoid any glitches, we accommodate 2% + max hardware error + * safety margin. */ switch (wsize) { case RTIWWDSIZE_50P: - /* 50% open window => 50% min heartbeat */ - wdd->min_hw_heartbeat_ms = 500 * heartbeat; + /* 50% open window => 52% min heartbeat */ + wdd->min_hw_heartbeat_ms = 520 * heartbeat + MAX_HW_ERROR; break; case RTIWWDSIZE_25P: - /* 25% open window => 75% min heartbeat */ - wdd->min_hw_heartbeat_ms = 750 * heartbeat; + /* 25% open window => 77% min heartbeat */ + wdd->min_hw_heartbeat_ms = 770 * heartbeat + MAX_HW_ERROR; break; case RTIWWDSIZE_12P5: - /* 12.5% open window => 87.5% min heartbeat */ - wdd->min_hw_heartbeat_ms = 875 * heartbeat; + /* 12.5% open window => 89.5% min heartbeat */ + wdd->min_hw_heartbeat_ms = 895 * heartbeat + MAX_HW_ERROR; break; case RTIWWDSIZE_6P25: - /* 6.5% open window => 93.5% min heartbeat */ - wdd->min_hw_heartbeat_ms = 935 * heartbeat; + /* 6.5% open window => 95.5% min heartbeat */ + wdd->min_hw_heartbeat_ms = 955 * heartbeat + MAX_HW_ERROR; break; case RTIWWDSIZE_3P125: - /* 3.125% open window => 96.9% min heartbeat */ - wdd->min_hw_heartbeat_ms = 969 * heartbeat; + /* 3.125% open window => 98.9% min heartbeat */ + wdd->min_hw_heartbeat_ms = 989 * heartbeat + MAX_HW_ERROR; break; default: @@ -233,14 +237,6 @@ static int rti_wdt_probe(struct platform_device *pdev) return -EINVAL; } - /* - * If watchdog is running at 32k clock, it is not accurate. - * Adjust frequency down in this case so that we don't pet - * the watchdog too often. - */ - if (wdt->freq < 32768) - wdt->freq = wdt->freq * 9 / 10; - pm_runtime_enable(dev); ret = pm_runtime_resume_and_get(dev); if (ret < 0) { From 413bf4e857fd79617524d5dcd35f463e9aa2dd41 Mon Sep 17 00:00:00 2001 From: Chen Ni Date: Fri, 26 Apr 2024 15:58:08 +0800 Subject: [PATCH 1034/1702] watchdog: sa1100: Fix PTR_ERR_OR_ZERO() vs NULL check in sa1100dog_probe() devm_ioremap() doesn't return error pointers, it returns NULL on error. Update the check accordingly. Fixes: e86bd43bcfc5 ("watchdog: sa1100: use platform device registration") Signed-off-by: Chen Ni Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20240426075808.1582678-1-nichen@iscas.ac.cn Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/sa1100_wdt.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/watchdog/sa1100_wdt.c b/drivers/watchdog/sa1100_wdt.c index 5d2df008b92a5..34a917221e316 100644 --- a/drivers/watchdog/sa1100_wdt.c +++ b/drivers/watchdog/sa1100_wdt.c @@ -191,9 +191,8 @@ static int sa1100dog_probe(struct platform_device *pdev) if (!res) return -ENXIO; reg_base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); - ret = PTR_ERR_OR_ZERO(reg_base); - if (ret) - return ret; + if (!reg_base) + return -ENOMEM; clk = clk_get(NULL, "OSTIMER0"); if (IS_ERR(clk)) { From 48d80b484491f177c586874c480cf9ba3af82b4f Mon Sep 17 00:00:00 2001 From: Jules Irenge Date: Tue, 30 Apr 2024 23:47:45 +0100 Subject: [PATCH 1035/1702] RDMA/core: Remove NULL check before dev_{put, hold} Coccinelle reports a warning WARNING: NULL check before dev_{put, hold} functions is not needed The reason is the call netdev_{put, hold} of dev_{put,hold} will check NULL There is no need to check before using dev_{put, hold} Signed-off-by: Jules Irenge Link: https://lore.kernel.org/r/ZjF1Eedxwhn4JSkz@octinomon.home Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/device.c | 10 +++------- drivers/infiniband/core/lag.c | 3 +-- drivers/infiniband/core/roce_gid_mgmt.c | 3 +-- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index 07cb6c5ffda00..55aa7aa32d4ab 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -2174,8 +2174,7 @@ int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev, spin_unlock_irqrestore(&pdata->netdev_lock, flags); add_ndev_hash(pdata); - if (old_ndev) - __dev_put(old_ndev); + __dev_put(old_ndev); return 0; } @@ -2235,8 +2234,7 @@ struct net_device *ib_device_get_netdev(struct ib_device *ib_dev, spin_lock(&pdata->netdev_lock); res = rcu_dereference_protected( pdata->netdev, lockdep_is_held(&pdata->netdev_lock)); - if (res) - dev_hold(res); + dev_hold(res); spin_unlock(&pdata->netdev_lock); } @@ -2311,9 +2309,7 @@ void ib_enum_roce_netdev(struct ib_device *ib_dev, if (filter(ib_dev, port, idev, filter_cookie)) cb(ib_dev, port, idev, cookie); - - if (idev) - dev_put(idev); + dev_put(idev); } } diff --git a/drivers/infiniband/core/lag.c b/drivers/infiniband/core/lag.c index eca6e37c72ba5..8fd80adfe833e 100644 --- a/drivers/infiniband/core/lag.c +++ b/drivers/infiniband/core/lag.c @@ -93,8 +93,7 @@ static struct net_device *rdma_get_xmit_slave_udp(struct ib_device *device, slave = netdev_get_xmit_slave(master, skb, !!(device->lag_flags & RDMA_LAG_FLAGS_HASH_ALL_SLAVES)); - if (slave) - dev_hold(slave); + dev_hold(slave); rcu_read_unlock(); kfree_skb(skb); return slave; diff --git a/drivers/infiniband/core/roce_gid_mgmt.c b/drivers/infiniband/core/roce_gid_mgmt.c index e958c43dd28fd..d5131b3ba8ab0 100644 --- a/drivers/infiniband/core/roce_gid_mgmt.c +++ b/drivers/infiniband/core/roce_gid_mgmt.c @@ -601,8 +601,7 @@ static void del_netdev_default_ips_join(struct ib_device *ib_dev, u32 port, rcu_read_lock(); master_ndev = netdev_master_upper_dev_get_rcu(rdma_ndev); - if (master_ndev) - dev_hold(master_ndev); + dev_hold(master_ndev); rcu_read_unlock(); if (master_ndev) { From e73c882f0a0149d8cad79f87b28cbbc9b4ed9ebe Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Fri, 26 Apr 2024 06:12:36 -0700 Subject: [PATCH 1036/1702] RDMA/mana_ib: create EQs for RNIC CQs Create EQs within mana_ib device. Such EQs are required for creation of RNIC CQs. Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1714137160-5222-2-git-send-email-kotaranov@linux.microsoft.com Reviewed-by: Long Li Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/main.c | 34 ++++++++++++++++++++++++++-- drivers/infiniband/hw/mana/mana_ib.h | 1 + 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index f5401471bffe2..546d059470e5f 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -658,7 +658,7 @@ int mana_ib_create_eqs(struct mana_ib_dev *mdev) { struct gdma_context *gc = mdev_to_gc(mdev); struct gdma_queue_spec spec = {}; - int err; + int err, i; spec.type = GDMA_EQ; spec.monitor_avl_buf = false; @@ -672,12 +672,42 @@ int mana_ib_create_eqs(struct mana_ib_dev *mdev) if (err) return err; + mdev->eqs = kcalloc(mdev->ib_dev.num_comp_vectors, sizeof(struct gdma_queue *), + GFP_KERNEL); + if (!mdev->eqs) { + err = -ENOMEM; + goto destroy_fatal_eq; + } + + for (i = 0; i < mdev->ib_dev.num_comp_vectors; i++) { + spec.eq.msix_index = (i + 1) % gc->num_msix_usable; + err = mana_gd_create_mana_eq(mdev->gdma_dev, &spec, &mdev->eqs[i]); + if (err) + goto destroy_eqs; + } + return 0; + +destroy_eqs: + while (i-- > 0) + mana_gd_destroy_queue(gc, mdev->eqs[i]); + kfree(mdev->eqs); +destroy_fatal_eq: + mana_gd_destroy_queue(gc, mdev->fatal_err_eq); + return err; } void mana_ib_destroy_eqs(struct mana_ib_dev *mdev) { - mana_gd_destroy_queue(mdev_to_gc(mdev), mdev->fatal_err_eq); + struct gdma_context *gc = mdev_to_gc(mdev); + int i; + + mana_gd_destroy_queue(gc, mdev->fatal_err_eq); + + for (i = 0; i < mdev->ib_dev.num_comp_vectors; i++) + mana_gd_destroy_queue(gc, mdev->eqs[i]); + + kfree(mdev->eqs); } int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev) diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index 4c1240da0c5fc..bfcf6dfd221ef 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -56,6 +56,7 @@ struct mana_ib_dev { struct gdma_dev *gdma_dev; mana_handle_t adapter_handle; struct gdma_queue *fatal_err_eq; + struct gdma_queue **eqs; struct mana_ib_adapter_caps adapter_caps; }; From 5843415916852983d3aaddc87b57630af9b0adad Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Fri, 26 Apr 2024 06:12:37 -0700 Subject: [PATCH 1037/1702] RDMA/mana_ib: create and destroy RNIC cqs Implement RNIC requests for creation and destruction of RNIC CQs. Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1714137160-5222-3-git-send-email-kotaranov@linux.microsoft.com Reviewed-by: Long Li Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/main.c | 54 ++++++++++++++++++++++++++++ drivers/infiniband/hw/mana/mana_ib.h | 32 +++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index 546d059470e5f..2a411357640e6 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -834,3 +834,57 @@ int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 return 0; } + +int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq, u32 doorbell) +{ + struct gdma_context *gc = mdev_to_gc(mdev); + struct mana_rnic_create_cq_resp resp = {}; + struct mana_rnic_create_cq_req req = {}; + int err; + + mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_CQ, sizeof(req), sizeof(resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + req.gdma_region = cq->queue.gdma_region; + req.eq_id = mdev->eqs[cq->comp_vector]->id; + req.doorbell_page = doorbell; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + + if (err) { + ibdev_err(&mdev->ib_dev, "Failed to create cq err %d", err); + return err; + } + + cq->queue.id = resp.cq_id; + cq->cq_handle = resp.cq_handle; + /* The GDMA region is now owned by the CQ handle */ + cq->queue.gdma_region = GDMA_INVALID_DMA_REGION; + + return 0; +} + +int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) +{ + struct gdma_context *gc = mdev_to_gc(mdev); + struct mana_rnic_destroy_cq_resp resp = {}; + struct mana_rnic_destroy_cq_req req = {}; + int err; + + if (cq->cq_handle == INVALID_MANA_HANDLE) + return 0; + + mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_CQ, sizeof(req), sizeof(resp)); + req.hdr.dev_id = gc->mana_ib.dev_id; + req.adapter = mdev->adapter_handle; + req.cq_handle = cq->cq_handle; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + + if (err) { + ibdev_err(&mdev->ib_dev, "Failed to destroy cq err %d", err); + return err; + } + + return 0; +} diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index bfcf6dfd221ef..9162f29da02de 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -92,6 +92,7 @@ struct mana_ib_cq { struct mana_ib_queue queue; int cqe; u32 comp_vector; + mana_handle_t cq_handle; }; struct mana_ib_qp { @@ -119,6 +120,8 @@ enum mana_ib_command_code { MANA_IB_DESTROY_ADAPTER = 0x30003, MANA_IB_CONFIG_IP_ADDR = 0x30004, MANA_IB_CONFIG_MAC_ADDR = 0x30005, + MANA_IB_CREATE_CQ = 0x30008, + MANA_IB_DESTROY_CQ = 0x30009, }; struct mana_ib_query_adapter_caps_req { @@ -202,6 +205,31 @@ struct mana_rnic_config_mac_addr_resp { struct gdma_resp_hdr hdr; }; /* HW Data */ +struct mana_rnic_create_cq_req { + struct gdma_req_hdr hdr; + mana_handle_t adapter; + u64 gdma_region; + u32 eq_id; + u32 doorbell_page; +}; /* HW Data */ + +struct mana_rnic_create_cq_resp { + struct gdma_resp_hdr hdr; + mana_handle_t cq_handle; + u32 cq_id; + u32 reserved; +}; /* HW Data */ + +struct mana_rnic_destroy_cq_req { + struct gdma_req_hdr hdr; + mana_handle_t adapter; + mana_handle_t cq_handle; +}; /* HW Data */ + +struct mana_rnic_destroy_cq_resp { + struct gdma_resp_hdr hdr; +}; /* HW Data */ + static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev) { return mdev->gdma_dev->gdma_context; @@ -321,4 +349,8 @@ int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context); int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context); int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 *mac); + +int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq, u32 doorbell); + +int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq); #endif From 3e41105263d5d74840c0d117278894b428f02841 Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Fri, 26 Apr 2024 06:12:38 -0700 Subject: [PATCH 1038/1702] RDMA/mana_ib: introduce a helper to remove cq callbacks Intoduce the mana_ib_remove_cq_cb helper to remove cq callbacks. The helper removes code duplicates. Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1714137160-5222-4-git-send-email-kotaranov@linux.microsoft.com Reviewed-by: Long Li Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/cq.c | 19 ++++++++++++------- drivers/infiniband/hw/mana/mana_ib.h | 1 + drivers/infiniband/hw/mana/qp.c | 26 ++++---------------------- 3 files changed, 17 insertions(+), 29 deletions(-) diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index dc931b9c34919..298e8f15a6591 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -48,16 +48,10 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); struct ib_device *ibdev = ibcq->device; struct mana_ib_dev *mdev; - struct gdma_context *gc; mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); - gc = mdev_to_gc(mdev); - - if (cq->queue.id != INVALID_QUEUE_ID) { - kfree(gc->cq_table[cq->queue.id]); - gc->cq_table[cq->queue.id] = NULL; - } + mana_ib_remove_cq_cb(mdev, cq); mana_ib_destroy_queue(mdev, &cq->queue); return 0; @@ -89,3 +83,14 @@ int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) gc->cq_table[cq->queue.id] = gdma_cq; return 0; } + +void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) +{ + struct gdma_context *gc = mdev_to_gc(mdev); + + if (cq->queue.id >= gc->max_num_cqs || cq->queue.id == INVALID_QUEUE_ID) + return; + + kfree(gc->cq_table[cq->queue.id]); + gc->cq_table[cq->queue.id] = NULL; +} diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index 9162f29da02de..68c3b4f0faa44 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -255,6 +255,7 @@ static inline void copy_in_reverse(u8 *dst, const u8 *src, u32 size) } int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq); +void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq); int mana_ib_create_zero_offset_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem, mana_handle_t *gdma_region); diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 280e85a83f7eb..ba13c5abf8ef5 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -95,11 +95,9 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); struct mana_ib_dev *mdev = container_of(pd->device, struct mana_ib_dev, ib_dev); - struct gdma_context *gc = mdev_to_gc(mdev); struct ib_rwq_ind_table *ind_tbl = attr->rwq_ind_tbl; struct mana_ib_create_qp_rss_resp resp = {}; struct mana_ib_create_qp_rss ucmd = {}; - struct gdma_queue **gdma_cq_allocated; mana_handle_t *mana_ind_table; struct mana_port_context *mpc; unsigned int ind_tbl_size; @@ -173,13 +171,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, goto fail; } - gdma_cq_allocated = kcalloc(ind_tbl_size, sizeof(*gdma_cq_allocated), - GFP_KERNEL); - if (!gdma_cq_allocated) { - ret = -ENOMEM; - goto fail; - } - qp->port = port; for (i = 0; i < ind_tbl_size; i++) { @@ -229,8 +220,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, ret = mana_ib_install_cq_cb(mdev, cq); if (ret) goto fail; - - gdma_cq_allocated[i] = gc->cq_table[cq->queue.id]; } resp.num_entries = i; @@ -250,7 +239,6 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, goto fail; } - kfree(gdma_cq_allocated); kfree(mana_ind_table); return 0; @@ -262,13 +250,10 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, wq = container_of(ibwq, struct mana_ib_wq, ibwq); cq = container_of(ibcq, struct mana_ib_cq, ibcq); - gc->cq_table[cq->queue.id] = NULL; - kfree(gdma_cq_allocated[i]); - + mana_ib_remove_cq_cb(mdev, cq); mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object); } - kfree(gdma_cq_allocated); kfree(mana_ind_table); return ret; @@ -287,10 +272,8 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, struct mana_ib_ucontext *mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext); - struct gdma_context *gc = mdev_to_gc(mdev); struct mana_ib_create_qp_resp resp = {}; struct mana_ib_create_qp ucmd = {}; - struct gdma_queue *gdma_cq = NULL; struct mana_obj_spec wq_spec = {}; struct mana_obj_spec cq_spec = {}; struct mana_port_context *mpc; @@ -395,14 +378,13 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, ibdev_dbg(&mdev->ib_dev, "Failed copy udata for create qp-raw, %d\n", err); - goto err_release_gdma_cq; + goto err_remove_cq_cb; } return 0; -err_release_gdma_cq: - kfree(gdma_cq); - gc->cq_table[send_cq->queue.id] = NULL; +err_remove_cq_cb: + mana_ib_remove_cq_cb(mdev, send_cq); err_destroy_wq_obj: mana_destroy_wq_obj(mpc, GDMA_SQ, qp->qp_handle); From f79edef79b6a2161f4124112f9b0c46891bb0b74 Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Fri, 26 Apr 2024 06:12:39 -0700 Subject: [PATCH 1039/1702] RDMA/mana_ib: boundary check before installing cq callbacks Add a boundary check inside mana_ib_install_cq_cb to prevent index overflow. Fixes: 2a31c5a7e0d8 ("RDMA/mana_ib: Introduce mana_ib_install_cq_cb helper function") Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1714137160-5222-5-git-send-email-kotaranov@linux.microsoft.com Reviewed-by: Long Li Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/cq.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index 298e8f15a6591..688ffe61f6b22 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -70,6 +70,8 @@ int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) struct gdma_context *gc = mdev_to_gc(mdev); struct gdma_queue *gdma_cq; + if (cq->queue.id >= gc->max_num_cqs) + return -EINVAL; /* Create CQ table entry */ WARN_ON(gc->cq_table[cq->queue.id]); gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL); From 44b607ad4cdf23ae8f796b95bd14709fa06f7728 Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Fri, 26 Apr 2024 06:12:40 -0700 Subject: [PATCH 1040/1702] RDMA/mana_ib: implement uapi for creation of rnic cq Enable users to create RNIC CQs using a corresponding flag. With the previous request size, an ethernet CQ is created. As a response, return ID of the created CQ. Signed-off-by: Konstantin Taranov Link: https://lore.kernel.org/r/1714137160-5222-6-git-send-email-kotaranov@linux.microsoft.com Reviewed-by: Long Li Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/cq.c | 55 ++++++++++++++++++++++++++++++--- include/uapi/rdma/mana-abi.h | 12 +++++++ 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index 688ffe61f6b22..c6a3fd57a1969 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -9,17 +9,22 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata) { struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); + struct mana_ib_create_cq_resp resp = {}; + struct mana_ib_ucontext *mana_ucontext; struct ib_device *ibdev = ibcq->device; struct mana_ib_create_cq ucmd = {}; struct mana_ib_dev *mdev; + bool is_rnic_cq; + u32 doorbell; int err; mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); - if (udata->inlen < sizeof(ucmd)) - return -EINVAL; - cq->comp_vector = attr->comp_vector % ibdev->num_comp_vectors; + cq->cq_handle = INVALID_MANA_HANDLE; + + if (udata->inlen < offsetof(struct mana_ib_create_cq, flags)) + return -EINVAL; err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)); if (err) { @@ -28,7 +33,9 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, return err; } - if (attr->cqe > mdev->adapter_caps.max_qp_wr) { + is_rnic_cq = !!(ucmd.flags & MANA_IB_CREATE_RNIC_CQ); + + if (!is_rnic_cq && attr->cqe > mdev->adapter_caps.max_qp_wr) { ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe); return -EINVAL; } @@ -40,7 +47,41 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, return err; } + mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, + ibucontext); + doorbell = mana_ucontext->doorbell; + + if (is_rnic_cq) { + err = mana_ib_gd_create_cq(mdev, cq, doorbell); + if (err) { + ibdev_dbg(ibdev, "Failed to create RNIC cq, %d\n", err); + goto err_destroy_queue; + } + + err = mana_ib_install_cq_cb(mdev, cq); + if (err) { + ibdev_dbg(ibdev, "Failed to install cq callback, %d\n", err); + goto err_destroy_rnic_cq; + } + } + + resp.cqid = cq->queue.id; + err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); + if (err) { + ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err); + goto err_remove_cq_cb; + } + return 0; + +err_remove_cq_cb: + mana_ib_remove_cq_cb(mdev, cq); +err_destroy_rnic_cq: + mana_ib_gd_destroy_cq(mdev, cq); +err_destroy_queue: + mana_ib_destroy_queue(mdev, &cq->queue); + + return err; } int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) @@ -52,6 +93,12 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); mana_ib_remove_cq_cb(mdev, cq); + + /* Ignore return code as there is not much we can do about it. + * The error message is printed inside. + */ + mana_ib_gd_destroy_cq(mdev, cq); + mana_ib_destroy_queue(mdev, &cq->queue); return 0; diff --git a/include/uapi/rdma/mana-abi.h b/include/uapi/rdma/mana-abi.h index 5fcb31b37fb91..2c41cc315218f 100644 --- a/include/uapi/rdma/mana-abi.h +++ b/include/uapi/rdma/mana-abi.h @@ -16,8 +16,20 @@ #define MANA_IB_UVERBS_ABI_VERSION 1 +enum mana_ib_create_cq_flags { + MANA_IB_CREATE_RNIC_CQ = 1 << 0, +}; + struct mana_ib_create_cq { __aligned_u64 buf_addr; + __u16 flags; + __u16 reserved0; + __u32 reserved1; +}; + +struct mana_ib_create_cq_resp { + __u32 cqid; + __u32 reserved; }; struct mana_ib_create_qp { From 8f3b7103b41314d26e2653e9ccca29480123a204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ilpo=20J=C3=A4rvinen?= Date: Fri, 3 May 2024 16:36:40 +0300 Subject: [PATCH 1041/1702] RDMA/hfi1: Use RMW accessors for changing LNKCTL2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert open coded RMW accesses for LNKCTL2 to use pcie_capability_clear_and_set_word() which makes its easier to understand what the code tries to do. In addition, this futureproofs the code. LNKCTL2 is not really owned by any driver because it is a collection of control bits that PCI core might need to touch. RMW accessors already have support for proper locking for a selected set of registers to avoid losing concurrent updates (LNKCTL2 is not yet among the registers that need protection but likely will be in the future). Suggested-by: Lukas Wunner Signed-off-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20240503133640.15899-1-ilpo.jarvinen@linux.intel.com Reviewed-by: Dean Luick Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hfi1/pcie.c | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c index 119ec2f1382bf..7133964749f80 100644 --- a/drivers/infiniband/hw/hfi1/pcie.c +++ b/drivers/infiniband/hw/hfi1/pcie.c @@ -1207,14 +1207,11 @@ int do_pcie_gen3_transition(struct hfi1_devdata *dd) (u32)lnkctl2); /* only write to parent if target is not as high as ours */ if ((lnkctl2 & PCI_EXP_LNKCTL2_TLS) < target_vector) { - lnkctl2 &= ~PCI_EXP_LNKCTL2_TLS; - lnkctl2 |= target_vector; - dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__, - (u32)lnkctl2); - ret = pcie_capability_write_word(parent, - PCI_EXP_LNKCTL2, lnkctl2); + ret = pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL2, + PCI_EXP_LNKCTL2_TLS, + target_vector); if (ret) { - dd_dev_err(dd, "Unable to write to PCI config\n"); + dd_dev_err(dd, "Unable to change parent PCI target speed\n"); return_error = 1; goto done; } @@ -1223,22 +1220,11 @@ int do_pcie_gen3_transition(struct hfi1_devdata *dd) } dd_dev_info(dd, "%s: setting target link speed\n", __func__); - ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKCTL2, &lnkctl2); + ret = pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL2, + PCI_EXP_LNKCTL2_TLS, + target_vector); if (ret) { - dd_dev_err(dd, "Unable to read from PCI config\n"); - return_error = 1; - goto done; - } - - dd_dev_info(dd, "%s: ..old link control2: 0x%x\n", __func__, - (u32)lnkctl2); - lnkctl2 &= ~PCI_EXP_LNKCTL2_TLS; - lnkctl2 |= target_vector; - dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__, - (u32)lnkctl2); - ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_LNKCTL2, lnkctl2); - if (ret) { - dd_dev_err(dd, "Unable to write to PCI config\n"); + dd_dev_err(dd, "Unable to change device PCI target speed\n"); return_error = 1; goto done; } From 5194947e6a3966d50095c14c69edbec90ad191f9 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Fri, 3 May 2024 04:13:31 -0700 Subject: [PATCH 1042/1702] IB/hfi1: Do not use custom stat allocator With commit 34d21de99cea9 ("net: Move {l,t,d}stats allocation to core and convert veth & vrf"), stats allocation could be done on net core instead of in this driver. With this new approach, the driver doesn't have to bother with error handling (allocation failure checking, making sure free happens in the right spot, etc). This is core responsibility now. Remove the allocation in the hfi1 driver and leverage the network core allocation instead. Signed-off-by: Breno Leitao Link: https://lore.kernel.org/r/20240503111333.552360-1-leitao@debian.org Reviewed-by: Simon Horman Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hfi1/ipoib_main.c | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/drivers/infiniband/hw/hfi1/ipoib_main.c b/drivers/infiniband/hw/hfi1/ipoib_main.c index 5d814afdf7f39..59c6e55f4119e 100644 --- a/drivers/infiniband/hw/hfi1/ipoib_main.c +++ b/drivers/infiniband/hw/hfi1/ipoib_main.c @@ -21,36 +21,25 @@ static int hfi1_ipoib_dev_init(struct net_device *dev) struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); int ret; - dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); - if (!dev->tstats) - return -ENOMEM; - ret = priv->netdev_ops->ndo_init(dev); if (ret) - goto out_ret; + return ret; ret = hfi1_netdev_add_data(priv->dd, qpn_from_mac(priv->netdev->dev_addr), dev); if (ret < 0) { priv->netdev_ops->ndo_uninit(dev); - goto out_ret; + return ret; } return 0; -out_ret: - free_percpu(dev->tstats); - dev->tstats = NULL; - return ret; } static void hfi1_ipoib_dev_uninit(struct net_device *dev) { struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); - free_percpu(dev->tstats); - dev->tstats = NULL; - hfi1_netdev_remove_data(priv->dd, qpn_from_mac(priv->netdev->dev_addr)); priv->netdev_ops->ndo_uninit(dev); @@ -173,9 +162,6 @@ static void hfi1_ipoib_netdev_dtor(struct net_device *dev) hfi1_ipoib_txreq_deinit(priv); hfi1_ipoib_rxq_deinit(priv->netdev); - - free_percpu(dev->tstats); - dev->tstats = NULL; } static void hfi1_ipoib_set_id(struct net_device *dev, int id) @@ -234,6 +220,7 @@ static int hfi1_ipoib_setup_rn(struct ib_device *device, netdev->priv_destructor = hfi1_ipoib_netdev_dtor; netdev->needs_free_netdev = true; + netdev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; return 0; } From f483f6a29d4d701f1641898463e93d081bb03b52 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Fri, 3 May 2024 04:13:32 -0700 Subject: [PATCH 1043/1702] IB/hfi1: Remove generic .ndo_get_stats64 Commit 3e2f544dd8a33 ("net: get stats64 if device if driver is configured") moved the callback to dev_get_tstats64() to net core, so, unless the driver is doing some custom stats collection, it does not need to set .ndo_get_stats64. Since this driver is now relying in NETDEV_PCPU_STAT_TSTATS, then, it doesn't need to set the dev_get_tstats64() generic .ndo_get_stats64 function pointer. Signed-off-by: Breno Leitao Link: https://lore.kernel.org/r/20240503111333.552360-2-leitao@debian.org Reviewed-by: Simon Horman Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hfi1/ipoib_main.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/infiniband/hw/hfi1/ipoib_main.c b/drivers/infiniband/hw/hfi1/ipoib_main.c index 59c6e55f4119e..7c9d5203002bf 100644 --- a/drivers/infiniband/hw/hfi1/ipoib_main.c +++ b/drivers/infiniband/hw/hfi1/ipoib_main.c @@ -96,7 +96,6 @@ static const struct net_device_ops hfi1_ipoib_netdev_ops = { .ndo_uninit = hfi1_ipoib_dev_uninit, .ndo_open = hfi1_ipoib_dev_open, .ndo_stop = hfi1_ipoib_dev_stop, - .ndo_get_stats64 = dev_get_tstats64, }; static int hfi1_ipoib_mcast_attach(struct net_device *dev, From 51c87f0e6cca2b9762d10dfab52618318444d746 Mon Sep 17 00:00:00 2001 From: Animesh Agarwal Date: Wed, 20 Mar 2024 14:16:20 +0530 Subject: [PATCH 1044/1702] dt-bindings: i2c: nxp,pnx-i2c: Convert to dtschema Convert the NXP PNX I2C Controller bindings to DT schema. Keep only one example in DT schema to remove redundancy. Reviewed-by: Conor Dooley Signed-off-by: Animesh Agarwal Signed-off-by: Andi Shyti --- .../devicetree/bindings/i2c/i2c-pnx.txt | 34 -------------- .../devicetree/bindings/i2c/nxp,pnx-i2c.yaml | 46 +++++++++++++++++++ 2 files changed, 46 insertions(+), 34 deletions(-) delete mode 100644 Documentation/devicetree/bindings/i2c/i2c-pnx.txt create mode 100644 Documentation/devicetree/bindings/i2c/nxp,pnx-i2c.yaml diff --git a/Documentation/devicetree/bindings/i2c/i2c-pnx.txt b/Documentation/devicetree/bindings/i2c/i2c-pnx.txt deleted file mode 100644 index 2a59006cf79e5..0000000000000 --- a/Documentation/devicetree/bindings/i2c/i2c-pnx.txt +++ /dev/null @@ -1,34 +0,0 @@ -* NXP PNX I2C Controller - -Required properties: - - - reg: Offset and length of the register set for the device - - compatible: should be "nxp,pnx-i2c" - - interrupts: configure one interrupt line - - #address-cells: always 1 (for i2c addresses) - - #size-cells: always 0 - -Optional properties: - - - clock-frequency: desired I2C bus clock frequency in Hz, Default: 100000 Hz - -Examples: - - i2c1: i2c@400a0000 { - compatible = "nxp,pnx-i2c"; - reg = <0x400a0000 0x100>; - interrupt-parent = <&mic>; - interrupts = <51 0>; - #address-cells = <1>; - #size-cells = <0>; - }; - - i2c2: i2c@400a8000 { - compatible = "nxp,pnx-i2c"; - reg = <0x400a8000 0x100>; - interrupt-parent = <&mic>; - interrupts = <50 0>; - #address-cells = <1>; - #size-cells = <0>; - clock-frequency = <100000>; - }; diff --git a/Documentation/devicetree/bindings/i2c/nxp,pnx-i2c.yaml b/Documentation/devicetree/bindings/i2c/nxp,pnx-i2c.yaml new file mode 100644 index 0000000000000..798a6939b8948 --- /dev/null +++ b/Documentation/devicetree/bindings/i2c/nxp,pnx-i2c.yaml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/i2c/nxp,pnx-i2c.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: NXP PNX I2C Controller + +maintainers: + - Animesh Agarwal + +allOf: + - $ref: /schemas/i2c/i2c-controller.yaml# + +properties: + compatible: + const: nxp,pnx-i2c + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clock-frequency: + default: 100000 + +required: + - compatible + - reg + - interrupts + - "#address-cells" + - "#size-cells" + +unevaluatedProperties: false + +examples: + - | + i2c@400a0000 { + compatible = "nxp,pnx-i2c"; + reg = <0x400a0000 0x100>; + interrupt-parent = <&mic>; + interrupts = <51 0>; + #address-cells = <1>; + #size-cells = <0>; + }; From c1f39c62eb09fa2e692a9377295ecbd0740c0914 Mon Sep 17 00:00:00 2001 From: Lad Prabhakar Date: Tue, 19 Mar 2024 13:25:00 +0000 Subject: [PATCH 1045/1702] dt-bindings: i2c: renesas,riic: Document R9A09G057 support Document support for the I2C Bus Interface (RIIC) available in the Renesas RZ/V2H(P) (R9A09G057) SoC. The RIIC interface in the Renesas RZ/V2H(P) differs from RZ/A in a couple of ways: - Register offsets for the RZ/V2H(P) SoC differ from those of the RZ/A SoC. - RZ/V2H register access is limited to 8-bit, whereas RZ/A supports 8/16/32-bit. - RZ/V2H has bit differences in the slave address register. To accommodate these differences, a new compatible string "renesas,riic-r9a09g057" is added. Signed-off-by: Lad Prabhakar Reviewed-by: Geert Uytterhoeven Acked-by: Rob Herring Reviewed-by: Wolfram Sang Signed-off-by: Andi Shyti --- .../devicetree/bindings/i2c/renesas,riic.yaml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Documentation/devicetree/bindings/i2c/renesas,riic.yaml b/Documentation/devicetree/bindings/i2c/renesas,riic.yaml index 2291a7cd619be..91ecf17b7a81a 100644 --- a/Documentation/devicetree/bindings/i2c/renesas,riic.yaml +++ b/Documentation/devicetree/bindings/i2c/renesas,riic.yaml @@ -15,14 +15,17 @@ allOf: properties: compatible: - items: - - enum: - - renesas,riic-r7s72100 # RZ/A1H - - renesas,riic-r7s9210 # RZ/A2M - - renesas,riic-r9a07g043 # RZ/G2UL and RZ/Five - - renesas,riic-r9a07g044 # RZ/G2{L,LC} - - renesas,riic-r9a07g054 # RZ/V2L - - const: renesas,riic-rz # RZ/A or RZ/G2L + oneOf: + - items: + - enum: + - renesas,riic-r7s72100 # RZ/A1H + - renesas,riic-r7s9210 # RZ/A2M + - renesas,riic-r9a07g043 # RZ/G2UL and RZ/Five + - renesas,riic-r9a07g044 # RZ/G2{L,LC} + - renesas,riic-r9a07g054 # RZ/V2L + - const: renesas,riic-rz # RZ/A or RZ/G2L + + - const: renesas,riic-r9a09g057 # RZ/V2H(P) reg: maxItems: 1 From 26c7871100f2933a2827b217320366e89cef5a4c Mon Sep 17 00:00:00 2001 From: Lad Prabhakar Date: Tue, 19 Mar 2024 13:25:01 +0000 Subject: [PATCH 1046/1702] i2c: riic: Introduce helper functions for I2C read/write operations Introduce helper functions for performing I2C read and write operations in the RIIC driver. These helper functions lay the groundwork for adding support for the RZ/V2H SoC. This is essential because the register offsets for the RZ/V2H SoC differ from those of the RZ/A SoC. By abstracting the read and write operations, we can seamlessly adapt the driver to support different SoC variants without extensive modifications. This patch is part of the preparation process for integrating support for the RZ/V2H SoC into the RIIC driver. Signed-off-by: Lad Prabhakar Reviewed-by: Geert Uytterhoeven Reviewed-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-riic.c | 56 +++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/drivers/i2c/busses/i2c-riic.c b/drivers/i2c/busses/i2c-riic.c index e43ff483c56ec..ef35e67839fa8 100644 --- a/drivers/i2c/busses/i2c-riic.c +++ b/drivers/i2c/busses/i2c-riic.c @@ -105,9 +105,19 @@ struct riic_irq_desc { char *name; }; +static inline void riic_writeb(struct riic_dev *riic, u8 val, u8 offset) +{ + writeb(val, riic->base + offset); +} + +static inline u8 riic_readb(struct riic_dev *riic, u8 offset) +{ + return readb(riic->base + offset); +} + static inline void riic_clear_set_bit(struct riic_dev *riic, u8 clear, u8 set, u8 reg) { - writeb((readb(riic->base + reg) & ~clear) | set, riic->base + reg); + riic_writeb(riic, (riic_readb(riic, reg) & ~clear) | set, reg); } static int riic_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) @@ -119,7 +129,7 @@ static int riic_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) pm_runtime_get_sync(adap->dev.parent); - if (readb(riic->base + RIIC_ICCR2) & ICCR2_BBSY) { + if (riic_readb(riic, RIIC_ICCR2) & ICCR2_BBSY) { riic->err = -EBUSY; goto out; } @@ -127,7 +137,7 @@ static int riic_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) reinit_completion(&riic->msg_done); riic->err = 0; - writeb(0, riic->base + RIIC_ICSR2); + riic_writeb(riic, 0, RIIC_ICSR2); for (i = 0, start_bit = ICCR2_ST; i < num; i++) { riic->bytes_left = RIIC_INIT_MSG; @@ -135,9 +145,9 @@ static int riic_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) riic->msg = &msgs[i]; riic->is_last = (i == num - 1); - writeb(ICIER_NAKIE | ICIER_TIE, riic->base + RIIC_ICIER); + riic_writeb(riic, ICIER_NAKIE | ICIER_TIE, RIIC_ICIER); - writeb(start_bit, riic->base + RIIC_ICCR2); + riic_writeb(riic, start_bit, RIIC_ICCR2); time_left = wait_for_completion_timeout(&riic->msg_done, riic->adapter.timeout); if (time_left == 0) @@ -191,7 +201,7 @@ static irqreturn_t riic_tdre_isr(int irq, void *data) * value could be moved to the shadow shift register right away. So * this must be after updates to ICIER (where we want to disable TIE)! */ - writeb(val, riic->base + RIIC_ICDRT); + riic_writeb(riic, val, RIIC_ICDRT); return IRQ_HANDLED; } @@ -200,9 +210,9 @@ static irqreturn_t riic_tend_isr(int irq, void *data) { struct riic_dev *riic = data; - if (readb(riic->base + RIIC_ICSR2) & ICSR2_NACKF) { + if (riic_readb(riic, RIIC_ICSR2) & ICSR2_NACKF) { /* We got a NACKIE */ - readb(riic->base + RIIC_ICDRR); /* dummy read */ + riic_readb(riic, RIIC_ICDRR); /* dummy read */ riic_clear_set_bit(riic, ICSR2_NACKF, 0, RIIC_ICSR2); riic->err = -ENXIO; } else if (riic->bytes_left) { @@ -211,7 +221,7 @@ static irqreturn_t riic_tend_isr(int irq, void *data) if (riic->is_last || riic->err) { riic_clear_set_bit(riic, ICIER_TEIE, ICIER_SPIE, RIIC_ICIER); - writeb(ICCR2_SP, riic->base + RIIC_ICCR2); + riic_writeb(riic, ICCR2_SP, RIIC_ICCR2); } else { /* Transfer is complete, but do not send STOP */ riic_clear_set_bit(riic, ICIER_TEIE, 0, RIIC_ICIER); @@ -230,7 +240,7 @@ static irqreturn_t riic_rdrf_isr(int irq, void *data) if (riic->bytes_left == RIIC_INIT_MSG) { riic->bytes_left = riic->msg->len; - readb(riic->base + RIIC_ICDRR); /* dummy read */ + riic_readb(riic, RIIC_ICDRR); /* dummy read */ return IRQ_HANDLED; } @@ -238,7 +248,7 @@ static irqreturn_t riic_rdrf_isr(int irq, void *data) /* STOP must come before we set ACKBT! */ if (riic->is_last) { riic_clear_set_bit(riic, 0, ICIER_SPIE, RIIC_ICIER); - writeb(ICCR2_SP, riic->base + RIIC_ICCR2); + riic_writeb(riic, ICCR2_SP, RIIC_ICCR2); } riic_clear_set_bit(riic, 0, ICMR3_ACKBT, RIIC_ICMR3); @@ -248,7 +258,7 @@ static irqreturn_t riic_rdrf_isr(int irq, void *data) } /* Reading acks the RIE interrupt */ - *riic->buf = readb(riic->base + RIIC_ICDRR); + *riic->buf = riic_readb(riic, RIIC_ICDRR); riic->buf++; riic->bytes_left--; @@ -260,10 +270,10 @@ static irqreturn_t riic_stop_isr(int irq, void *data) struct riic_dev *riic = data; /* read back registers to confirm writes have fully propagated */ - writeb(0, riic->base + RIIC_ICSR2); - readb(riic->base + RIIC_ICSR2); - writeb(0, riic->base + RIIC_ICIER); - readb(riic->base + RIIC_ICIER); + riic_writeb(riic, 0, RIIC_ICSR2); + riic_readb(riic, RIIC_ICSR2); + riic_writeb(riic, 0, RIIC_ICIER); + riic_readb(riic, RIIC_ICIER); complete(&riic->msg_done); @@ -365,15 +375,15 @@ static int riic_init_hw(struct riic_dev *riic, struct i2c_timings *t) t->scl_rise_ns / (1000000000 / rate), cks, brl, brh); /* Changing the order of accessing IICRST and ICE may break things! */ - writeb(ICCR1_IICRST | ICCR1_SOWP, riic->base + RIIC_ICCR1); + riic_writeb(riic, ICCR1_IICRST | ICCR1_SOWP, RIIC_ICCR1); riic_clear_set_bit(riic, 0, ICCR1_ICE, RIIC_ICCR1); - writeb(ICMR1_CKS(cks), riic->base + RIIC_ICMR1); - writeb(brh | ICBR_RESERVED, riic->base + RIIC_ICBRH); - writeb(brl | ICBR_RESERVED, riic->base + RIIC_ICBRL); + riic_writeb(riic, ICMR1_CKS(cks), RIIC_ICMR1); + riic_writeb(riic, brh | ICBR_RESERVED, RIIC_ICBRH); + riic_writeb(riic, brl | ICBR_RESERVED, RIIC_ICBRL); - writeb(0, riic->base + RIIC_ICSER); - writeb(ICMR3_ACKWP | ICMR3_RDRFS, riic->base + RIIC_ICMR3); + riic_writeb(riic, 0, RIIC_ICSER); + riic_writeb(riic, ICMR3_ACKWP | ICMR3_RDRFS, RIIC_ICMR3); riic_clear_set_bit(riic, ICCR1_IICRST, 0, RIIC_ICCR1); @@ -481,7 +491,7 @@ static void riic_i2c_remove(struct platform_device *pdev) struct riic_dev *riic = platform_get_drvdata(pdev); pm_runtime_get_sync(&pdev->dev); - writeb(0, riic->base + RIIC_ICIER); + riic_writeb(riic, 0, RIIC_ICIER); pm_runtime_put(&pdev->dev); i2c_del_adapter(&riic->adapter); pm_runtime_disable(&pdev->dev); From 748ee3b2a477821957adfe0ee7d1fd11d0f9a512 Mon Sep 17 00:00:00 2001 From: Lad Prabhakar Date: Tue, 19 Mar 2024 13:25:02 +0000 Subject: [PATCH 1047/1702] i2c: riic: Pass register offsets and chip details as OF data With an increasing number of SoCs reusing this driver, each with slight variations in the RIIC IP, it becomes necessary to support passing these details as OF data. This approach simplifies the extension of the driver for other SoCs. This patch lays the groundwork for adding support for the Renesas RZ/V2H SoC. Signed-off-by: Lad Prabhakar Reviewed-by: Geert Uytterhoeven Reviewed-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-riic.c | 56 +++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/drivers/i2c/busses/i2c-riic.c b/drivers/i2c/busses/i2c-riic.c index ef35e67839fa8..3cd5033286ca1 100644 --- a/drivers/i2c/busses/i2c-riic.c +++ b/drivers/i2c/busses/i2c-riic.c @@ -46,18 +46,6 @@ #include #include -#define RIIC_ICCR1 0x00 -#define RIIC_ICCR2 0x04 -#define RIIC_ICMR1 0x08 -#define RIIC_ICMR3 0x10 -#define RIIC_ICSER 0x18 -#define RIIC_ICIER 0x1c -#define RIIC_ICSR2 0x24 -#define RIIC_ICBRL 0x34 -#define RIIC_ICBRH 0x38 -#define RIIC_ICDRT 0x3c -#define RIIC_ICDRR 0x40 - #define ICCR1_ICE 0x80 #define ICCR1_IICRST 0x40 #define ICCR1_SOWP 0x10 @@ -87,6 +75,25 @@ #define RIIC_INIT_MSG -1 +enum riic_reg_list { + RIIC_ICCR1 = 0, + RIIC_ICCR2, + RIIC_ICMR1, + RIIC_ICMR3, + RIIC_ICSER, + RIIC_ICIER, + RIIC_ICSR2, + RIIC_ICBRL, + RIIC_ICBRH, + RIIC_ICDRT, + RIIC_ICDRR, + RIIC_REG_END, +}; + +struct riic_of_data { + u8 regs[RIIC_REG_END]; +}; + struct riic_dev { void __iomem *base; u8 *buf; @@ -94,6 +101,7 @@ struct riic_dev { int bytes_left; int err; int is_last; + const struct riic_of_data *info; struct completion msg_done; struct i2c_adapter adapter; struct clk *clk; @@ -107,12 +115,12 @@ struct riic_irq_desc { static inline void riic_writeb(struct riic_dev *riic, u8 val, u8 offset) { - writeb(val, riic->base + offset); + writeb(val, riic->base + riic->info->regs[offset]); } static inline u8 riic_readb(struct riic_dev *riic, u8 offset) { - return readb(riic->base + offset); + return readb(riic->base + riic->info->regs[offset]); } static inline void riic_clear_set_bit(struct riic_dev *riic, u8 clear, u8 set, u8 reg) @@ -453,6 +461,8 @@ static int riic_i2c_probe(struct platform_device *pdev) } } + riic->info = of_device_get_match_data(&pdev->dev); + adap = &riic->adapter; i2c_set_adapdata(adap, riic); strscpy(adap->name, "Renesas RIIC adapter", sizeof(adap->name)); @@ -497,8 +507,24 @@ static void riic_i2c_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); } +static const struct riic_of_data riic_rz_a_info = { + .regs = { + [RIIC_ICCR1] = 0x00, + [RIIC_ICCR2] = 0x04, + [RIIC_ICMR1] = 0x08, + [RIIC_ICMR3] = 0x10, + [RIIC_ICSER] = 0x18, + [RIIC_ICIER] = 0x1c, + [RIIC_ICSR2] = 0x24, + [RIIC_ICBRL] = 0x34, + [RIIC_ICBRH] = 0x38, + [RIIC_ICDRT] = 0x3c, + [RIIC_ICDRR] = 0x40, + }, +}; + static const struct of_device_id riic_i2c_dt_ids[] = { - { .compatible = "renesas,riic-rz", }, + { .compatible = "renesas,riic-rz", .data = &riic_rz_a_info }, { /* Sentinel */ }, }; From a45f95d7480544fdaa9d566cb4956d0ca5d0dc33 Mon Sep 17 00:00:00 2001 From: Lad Prabhakar Date: Tue, 19 Mar 2024 13:25:03 +0000 Subject: [PATCH 1048/1702] i2c: riic: Add support for R9A09G057 SoC Extend the RIIC driver to support the RZ/V2H(P) ("R9A09G057") SoC. It accomplishes this by appending the compatible string list and passing the RZ/V2H-specific OF data. Signed-off-by: Lad Prabhakar Reviewed-by: Geert Uytterhoeven Reviewed-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-riic.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/drivers/i2c/busses/i2c-riic.c b/drivers/i2c/busses/i2c-riic.c index 3cd5033286ca1..f608b1838cade 100644 --- a/drivers/i2c/busses/i2c-riic.c +++ b/drivers/i2c/busses/i2c-riic.c @@ -523,8 +523,25 @@ static const struct riic_of_data riic_rz_a_info = { }, }; +static const struct riic_of_data riic_rz_v2h_info = { + .regs = { + [RIIC_ICCR1] = 0x00, + [RIIC_ICCR2] = 0x01, + [RIIC_ICMR1] = 0x02, + [RIIC_ICMR3] = 0x04, + [RIIC_ICSER] = 0x06, + [RIIC_ICIER] = 0x07, + [RIIC_ICSR2] = 0x09, + [RIIC_ICBRL] = 0x10, + [RIIC_ICBRH] = 0x11, + [RIIC_ICDRT] = 0x12, + [RIIC_ICDRR] = 0x13, + }, +}; + static const struct of_device_id riic_i2c_dt_ids[] = { { .compatible = "renesas,riic-rz", .data = &riic_rz_a_info }, + { .compatible = "renesas,riic-r9a09g057", .data = &riic_rz_v2h_info }, { /* Sentinel */ }, }; From 5bffbda7ad5465605d12ed7432dc844cfff625c9 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Wed, 27 Mar 2024 18:47:05 +0100 Subject: [PATCH 1049/1702] i2c: viperboard: drop driver owner assignment Core in platform_driver_register() already sets the .owner, so driver does not need to. Signed-off-by: Krzysztof Kozlowski Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-viperboard.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/i2c/busses/i2c-viperboard.c b/drivers/i2c/busses/i2c-viperboard.c index 9e153b5b0e8e4..3784b07f53716 100644 --- a/drivers/i2c/busses/i2c-viperboard.c +++ b/drivers/i2c/busses/i2c-viperboard.c @@ -416,7 +416,6 @@ static void vprbrd_i2c_remove(struct platform_device *pdev) static struct platform_driver vprbrd_i2c_driver = { .driver.name = "viperboard-i2c", - .driver.owner = THIS_MODULE, .probe = vprbrd_i2c_probe, .remove_new = vprbrd_i2c_remove, }; From 893fef0bc6aaedd482b7592d18d9b17682c1f94a Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Tue, 26 Mar 2024 21:42:44 +0100 Subject: [PATCH 1050/1702] i2c: i801: Call i2c_register_spd for muxed child segments Once the gpio mux driver binds to the "i2c-mux-gpio" platform device, this creates the i2c adapters for the muxed child segments. We can use the bus notifier mechanism to check for creation of the child i2c adapters, and call i2c_register_spd() for them. This allows to detect all DIMM's on systems with more than 8 memory slots. Reviewed-by: Wolfram Sang Signed-off-by: Heiner Kallweit Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-i801.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index 79870dd7a0146..4294c0c63cef0 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -105,6 +105,7 @@ #include #include #include +#include #include #include #include @@ -291,6 +292,7 @@ struct i801_priv { #if IS_ENABLED(CONFIG_I2C_MUX_GPIO) && defined CONFIG_DMI struct platform_device *mux_pdev; struct gpiod_lookup_table *lookup; + struct notifier_block mux_notifier_block; #endif struct platform_device *tco_pdev; @@ -1394,6 +1396,23 @@ static const struct dmi_system_id mux_dmi_table[] = { { } }; +static int i801_notifier_call(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct i801_priv *priv = container_of(nb, struct i801_priv, mux_notifier_block); + struct device *dev = data; + + if (action != BUS_NOTIFY_ADD_DEVICE || + dev->type != &i2c_adapter_type || + i2c_root_adapter(dev) != &priv->adapter) + return NOTIFY_DONE; + + /* Call i2c_register_spd for muxed child segments */ + i2c_register_spd(to_i2c_adapter(dev)); + + return NOTIFY_OK; +} + /* Setup multiplexing if needed */ static void i801_add_mux(struct i801_priv *priv) { @@ -1430,6 +1449,9 @@ static void i801_add_mux(struct i801_priv *priv) mux_config->gpios[i], "mux", 0); gpiod_add_lookup_table(lookup); + priv->mux_notifier_block.notifier_call = i801_notifier_call; + if (bus_register_notifier(&i2c_bus_type, &priv->mux_notifier_block)) + return; /* * Register the mux device, we use PLATFORM_DEVID_NONE here * because since we are referring to the GPIO chip by name we are @@ -1451,6 +1473,7 @@ static void i801_add_mux(struct i801_priv *priv) static void i801_del_mux(struct i801_priv *priv) { + bus_unregister_notifier(&i2c_bus_type, &priv->mux_notifier_block); platform_device_unregister(priv->mux_pdev); gpiod_remove_lookup_table(priv->lookup); } From 47c21d2d52e0da05abdae63faa3de65cfb3afb74 Mon Sep 17 00:00:00 2001 From: Niklas Schnelle Date: Fri, 5 Apr 2024 12:10:09 +0200 Subject: [PATCH 1051/1702] i2c: add HAS_IOPORT dependencies In a future patch HAS_IOPORT=n will disable inb()/outb() and friends at compile time. We thus need to add HAS_IOPORT as dependency for those drivers using them. Co-developed-by: Arnd Bergmann Signed-off-by: Arnd Bergmann Signed-off-by: Niklas Schnelle Reviewed-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/Kconfig | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index 97989c914260f..5f4dc31ed1426 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -18,7 +18,7 @@ config I2C_CCGX_UCSI config I2C_ALI1535 tristate "ALI 1535" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the SMB Host controller on Acer Labs Inc. (ALI) M1535 South Bridges. The SMB @@ -30,7 +30,7 @@ config I2C_ALI1535 config I2C_ALI1563 tristate "ALI 1563" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the SMB Host controller on Acer Labs Inc. (ALI) M1563 South Bridges. The SMB @@ -42,7 +42,7 @@ config I2C_ALI1563 config I2C_ALI15X3 tristate "ALI 15x3" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the Acer Labs Inc. (ALI) M1514 and M1543 motherboard I2C interfaces. @@ -52,7 +52,7 @@ config I2C_ALI15X3 config I2C_AMD756 tristate "AMD 756/766/768/8111 and nVidia nForce" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the AMD 756/766/768 mainboard I2C interfaces. The driver also includes @@ -77,7 +77,7 @@ config I2C_AMD756_S4882 config I2C_AMD8111 tristate "AMD 8111" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the second (SMBus 2.0) AMD 8111 mainboard I2C interface. @@ -107,7 +107,7 @@ config I2C_HIX5HD2 config I2C_I801 tristate "Intel 82801 (ICH/PCH)" - depends on PCI + depends on PCI && HAS_IOPORT select P2SB if X86 select CHECK_SIGNATURE if X86 && DMI select I2C_SMBUS @@ -165,7 +165,7 @@ config I2C_I801 config I2C_ISCH tristate "Intel SCH SMBus 1.0" - depends on PCI + depends on PCI && HAS_IOPORT select LPC_SCH help Say Y here if you want to use SMBus controller on the Intel SCH @@ -186,7 +186,7 @@ config I2C_ISMT config I2C_PIIX4 tristate "Intel PIIX4 and compatible (ATI/AMD/Serverworks/Broadcom/SMSC)" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the Intel PIIX4 family of mainboard I2C interfaces. Specifically, the following @@ -232,7 +232,7 @@ config I2C_CHT_WC config I2C_NFORCE2 tristate "Nvidia nForce2, nForce3 and nForce4" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the Nvidia nForce2, nForce3 and nForce4 families of mainboard I2C interfaces. @@ -265,7 +265,7 @@ config I2C_NVIDIA_GPU config I2C_SIS5595 tristate "SiS 5595" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the SiS5595 SMBus (a subset of I2C) interface. @@ -275,7 +275,7 @@ config I2C_SIS5595 config I2C_SIS630 tristate "SiS 630/730/964" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the SiS630, SiS730 and SiS964 SMBus (a subset of I2C) interface. @@ -285,7 +285,7 @@ config I2C_SIS630 config I2C_SIS96X tristate "SiS 96x" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the SiS 96x SMBus (a subset of I2C) interfaces. Specifically, the following @@ -303,7 +303,7 @@ config I2C_SIS96X config I2C_VIA tristate "VIA VT82C586B" - depends on PCI + depends on PCI && HAS_IOPORT select I2C_ALGOBIT help If you say yes to this option, support will be included for the VIA @@ -314,7 +314,7 @@ config I2C_VIA config I2C_VIAPRO tristate "VIA VT82C596/82C686/82xx and CX700/VX8xx/VX900" - depends on PCI + depends on PCI && HAS_IOPORT help If you say yes to this option, support will be included for the VIA VT82C596 and later SMBus interface. Specifically, the following @@ -885,6 +885,7 @@ config I2C_NPCM config I2C_OCORES tristate "OpenCores I2C Controller" + depends on HAS_IOPORT help If you say yes to this option, support will be included for the OpenCores I2C controller. For details see @@ -1397,6 +1398,7 @@ config I2C_ICY config I2C_MLXCPLD tristate "Mellanox I2C driver" depends on X86_64 || (ARM64 && ACPI) || COMPILE_TEST + depends on HAS_IOPORT help This exposes the Mellanox platform I2C busses to the linux I2C layer for X86 and ARM64/ACPI based systems. From c2e55b449de7298a751ed0256251019d302af453 Mon Sep 17 00:00:00 2001 From: Sai Pavan Boddu Date: Fri, 3 May 2024 15:12:08 +0530 Subject: [PATCH 1052/1702] i2c: cadence: Avoid fifo clear after start The Driver unintentionally programs ctrl reg to clear the fifo, which happens after the start of transaction. Previously, this was not an issue as it involved read-modified-write. However, this issue breaks i2c reads on QEMU, as i2c-read is executed before guest starts programming control register. Fixes: ff0cf7bca630 ("i2c: cadence: Remove unnecessary register reads") Signed-off-by: Sai Pavan Boddu Acked-by: Michal Simek Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-cadence.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/i2c/busses/i2c-cadence.c b/drivers/i2c/busses/i2c-cadence.c index 4bb7d6756947c..2fce3e84ba646 100644 --- a/drivers/i2c/busses/i2c-cadence.c +++ b/drivers/i2c/busses/i2c-cadence.c @@ -633,6 +633,7 @@ static void cdns_i2c_mrecv(struct cdns_i2c *id) if (hold_clear) { ctrl_reg &= ~CDNS_I2C_CR_HOLD; + ctrl_reg &= ~CDNS_I2C_CR_CLR_FIFO; /* * In case of Xilinx Zynq SOC, clear the HOLD bit before transfer size * register reaches '0'. This is an IP bug which causes transfer size From 48ace624878d01ed19df2191d5a5e902631e9dd3 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 8 Apr 2024 11:28:36 +0200 Subject: [PATCH 1053/1702] i2c: ocores: convert to ioport_map() for IORESOURCE_IO There is at least one machine that uses this driver but does not have support for inb()/outb() instructions. Convert this to using ioport_map() so it can build on architectures that don't provide these but work correctly on machines that require using port I/O. Fixes: 47c21d2d52e0 ("i2c: add HAS_IOPORT dependencies") Reported-by: Geert Uytterhoeven Link: https://lore.kernel.org/lkml/CAMuHMdVUQ2WgtpYPYfO2T=itMmZ7w=geREqDtsP8Q3ODh9rxdw@mail.gmail.com/ Signed-off-by: Arnd Bergmann Acked-by: Peter Korsgaard Signed-off-by: Andi Shyti --- drivers/i2c/busses/Kconfig | 1 - drivers/i2c/busses/i2c-ocores.c | 21 +++++++-------------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index 5f4dc31ed1426..760e14b04f959 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -885,7 +885,6 @@ config I2C_NPCM config I2C_OCORES tristate "OpenCores I2C Controller" - depends on HAS_IOPORT help If you say yes to this option, support will be included for the OpenCores I2C controller. For details see diff --git a/drivers/i2c/busses/i2c-ocores.c b/drivers/i2c/busses/i2c-ocores.c index e106af83cef4d..56a4dabf5a388 100644 --- a/drivers/i2c/busses/i2c-ocores.c +++ b/drivers/i2c/busses/i2c-ocores.c @@ -32,7 +32,6 @@ */ struct ocores_i2c { void __iomem *base; - int iobase; u32 reg_shift; u32 reg_io_width; unsigned long flags; @@ -136,16 +135,6 @@ static inline u8 oc_getreg_32be(struct ocores_i2c *i2c, int reg) return ioread32be(i2c->base + (reg << i2c->reg_shift)); } -static void oc_setreg_io_8(struct ocores_i2c *i2c, int reg, u8 value) -{ - outb(value, i2c->iobase + reg); -} - -static inline u8 oc_getreg_io_8(struct ocores_i2c *i2c, int reg) -{ - return inb(i2c->iobase + reg); -} - static inline void oc_setreg(struct ocores_i2c *i2c, int reg, u8 value) { i2c->setreg(i2c, reg, value); @@ -618,15 +607,19 @@ static int ocores_i2c_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_IO, 0); if (!res) return -EINVAL; - i2c->iobase = res->start; if (!devm_request_region(&pdev->dev, res->start, resource_size(res), pdev->name)) { dev_err(&pdev->dev, "Can't get I/O resource.\n"); return -EBUSY; } - i2c->setreg = oc_setreg_io_8; - i2c->getreg = oc_getreg_io_8; + i2c->base = devm_ioport_map(&pdev->dev, res->start, + resource_size(res)); + if (!i2c->base) { + dev_err(&pdev->dev, "Can't map I/O resource.\n"); + return -EBUSY; + } + i2c->reg_io_width = 1; } pdata = dev_get_platdata(&pdev->dev); From 9c535237245e4bf21758604277279b8ead58a724 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Thu, 4 Apr 2024 22:09:50 +0200 Subject: [PATCH 1054/1702] i2c: i801: Fix missing Kconfig dependency The original change adds usage of i2c_root_adapter(), which is implemented in i2c-mux.c. Therefore we can't use the multiplexing if I2C_I801=y and I2C_MUX=m. Handling the dependencies in the code would become unnecessarily complex, therefore create a new config symbol. Fixes: 893fef0bc6aa ("i2c: i801: Call i2c_register_spd for muxed child segments") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202404042206.MjAQC32x-lkp@intel.com/ Signed-off-by: Heiner Kallweit Reviewed-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/Kconfig | 8 ++++++++ drivers/i2c/busses/i2c-i801.c | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index 760e14b04f959..27d3ef565ca8b 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -163,6 +163,14 @@ config I2C_I801 This driver can also be built as a module. If so, the module will be called i2c-i801. +config I2C_I801_MUX + def_bool I2C_I801 + depends on DMI && I2C_MUX_GPIO + depends on !(I2C_I801=y && I2C_MUX=m) + help + Optional support for multiplexed SMBUS on certain systems with + more than 8 memory slots. + config I2C_ISCH tristate "Intel SCH SMBus 1.0" depends on PCI && HAS_IOPORT diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index 4294c0c63cef0..92b53b480c47f 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -120,7 +120,7 @@ #include #include -#if IS_ENABLED(CONFIG_I2C_MUX_GPIO) && defined CONFIG_DMI +#ifdef CONFIG_I2C_I801_MUX #include #include #endif @@ -289,7 +289,7 @@ struct i801_priv { int len; u8 *data; -#if IS_ENABLED(CONFIG_I2C_MUX_GPIO) && defined CONFIG_DMI +#ifdef CONFIG_I2C_I801_MUX struct platform_device *mux_pdev; struct gpiod_lookup_table *lookup; struct notifier_block mux_notifier_block; @@ -1300,7 +1300,7 @@ static void i801_probe_optional_slaves(struct i801_priv *priv) register_dell_lis3lv02d_i2c_device(priv); /* Instantiate SPD EEPROMs unless the SMBus is multiplexed */ -#if IS_ENABLED(CONFIG_I2C_MUX_GPIO) +#ifdef CONFIG_I2C_I801_MUX if (!priv->mux_pdev) #endif i2c_register_spd(&priv->adapter); @@ -1310,7 +1310,7 @@ static void __init input_apanel_init(void) {} static void i801_probe_optional_slaves(struct i801_priv *priv) {} #endif /* CONFIG_X86 && CONFIG_DMI */ -#if IS_ENABLED(CONFIG_I2C_MUX_GPIO) && defined CONFIG_DMI +#ifdef CONFIG_I2C_I801_MUX static struct i801_mux_config i801_mux_config_asus_z8_d12 = { .gpio_chip = "gpio_ich", .values = { 0x02, 0x03 }, From 31a18e4139e0ad6019aad740e1f84cd4b5079e15 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Thu, 11 Apr 2024 07:02:57 +0200 Subject: [PATCH 1055/1702] MAINTAINERS: adjust file entry in ARM/LPC32XX SOC SUPPORT Commit 51c87f0e6cca ("dt-bindings: i2c: nxp,pnx-i2c: Convert to dtschema") converts i2c-pnx.txt to nxp,pnx-i2c.yaml, but misses to adjust the file entry in ARM/LPC32XX SOC SUPPORT. Hence, ./scripts/get_maintainer.pl --self-test=patterns complains about a broken reference. Adjust the file entry in ARM/LPC32XX SOC SUPPORT after this conversion. Signed-off-by: Lukas Bulwahn Reviewed-by: Andi Shyti Signed-off-by: Andi Shyti --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index ec0284125e8f7..75554f93e555b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2320,7 +2320,7 @@ M: Vladimir Zapolskiy L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained T: git git://github.com/vzapolskiy/linux-lpc32xx.git -F: Documentation/devicetree/bindings/i2c/i2c-pnx.txt +F: Documentation/devicetree/bindings/i2c/nxp,pnx-i2c.yaml F: arch/arm/boot/dts/nxp/lpc/lpc32* F: arch/arm/mach-lpc32xx/ F: drivers/i2c/busses/i2c-pnx.c From f32a32ad5b5a7b5108d142f0a9430f9eac798f9e Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Fri, 12 Apr 2024 14:53:25 +0100 Subject: [PATCH 1056/1702] dt-bindings: i2c: qcom-cci: Document sc8280xp compatible Add sc8280xp compatible consistent with recent CAMSS CCI interfaces. sc8280xp has the following clock list and so requires its own compat string and sc8280xp specific clock definition in the yaml. - const: camnoc_axi - const: slow_ahb_src - const: cpas_ahb - const: cci Signed-off-by: Bryan O'Donoghue Reviewed-by: Krzysztof Kozlowski Reviewed-by: Vladimir Zapolskiy Signed-off-by: Andi Shyti --- .../devicetree/bindings/i2c/qcom,i2c-cci.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml b/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml index f0eabff863106..daf4e71b8e7f9 100644 --- a/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml +++ b/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml @@ -26,6 +26,7 @@ properties: - items: - enum: - qcom,sc7280-cci + - qcom,sc8280xp-cci - qcom,sdm845-cci - qcom,sm6350-cci - qcom,sm8250-cci @@ -176,6 +177,24 @@ allOf: - const: cci - const: cci_src + - if: + properties: + compatible: + contains: + enum: + - qcom,sc8280xp-cci + then: + properties: + clocks: + minItems: 4 + maxItems: 4 + clock-names: + items: + - const: camnoc_axi + - const: slow_ahb_src + - const: cpas_ahb + - const: cci + additionalProperties: false examples: From 6104b99b14d9cbc2dab48ca3de8e526ced987ddb Mon Sep 17 00:00:00 2001 From: Abhinav Jain Date: Mon, 15 Apr 2024 16:12:20 +0000 Subject: [PATCH 1057/1702] i2c: mpc: Removal of of_node_put with __free for auto cleanup Remove of_node_put from node_ctrl and node struct device_nodes. Move the declaration to initialization for ensuring scope sanity. Suggested-by: Julia Lawall Signed-off-by: Abhinav Jain Reviewed-by: Chris Packham Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-mpc.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/drivers/i2c/busses/i2c-mpc.c b/drivers/i2c/busses/i2c-mpc.c index 8d73c0f405ed5..c4223556b3b8b 100644 --- a/drivers/i2c/busses/i2c-mpc.c +++ b/drivers/i2c/busses/i2c-mpc.c @@ -304,13 +304,12 @@ static void mpc_i2c_setup_512x(struct device_node *node, struct mpc_i2c *i2c, u32 clock) { - struct device_node *node_ctrl; void __iomem *ctrl; u32 idx; /* Enable I2C interrupts for mpc5121 */ - node_ctrl = of_find_compatible_node(NULL, NULL, - "fsl,mpc5121-i2c-ctrl"); + struct device_node *node_ctrl __free(device_node) = + of_find_compatible_node(NULL, NULL, "fsl,mpc5121-i2c-ctrl"); if (node_ctrl) { ctrl = of_iomap(node_ctrl, 0); if (ctrl) { @@ -321,7 +320,6 @@ static void mpc_i2c_setup_512x(struct device_node *node, setbits32(ctrl, 1 << (24 + idx * 2)); iounmap(ctrl); } - of_node_put(node_ctrl); } /* The clock setup for the 52xx works also fine for the 512x */ @@ -358,11 +356,11 @@ static const struct mpc_i2c_divider mpc_i2c_dividers_8xxx[] = { static u32 mpc_i2c_get_sec_cfg_8xxx(void) { - struct device_node *node; u32 __iomem *reg; u32 val = 0; - node = of_find_node_by_name(NULL, "global-utilities"); + struct device_node *node __free(device_node) = + of_find_node_by_name(NULL, "global-utilities"); if (node) { const u32 *prop = of_get_property(node, "reg", NULL); if (prop) { @@ -383,7 +381,6 @@ static u32 mpc_i2c_get_sec_cfg_8xxx(void) iounmap(reg); } } - of_node_put(node); return val; } From 864d1d83879b238f0c89963bfe9e28594da214ec Mon Sep 17 00:00:00 2001 From: Shanth Murthy Date: Tue, 16 Apr 2024 09:31:25 +0300 Subject: [PATCH 1058/1702] i2c: designware: Add ACPI ID for Granite Rapids-D I2C controller Granite Rapids-D has additional I2C controller that is enumerated via ACPI. Add ACPI ID for it. Signed-off-by: Shanth Murthy Signed-off-by: Jarkko Nikula Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-designware-platdrv.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c index 4ab41ba39d55f..926fb74a85703 100644 --- a/drivers/i2c/busses/i2c-designware-platdrv.c +++ b/drivers/i2c/busses/i2c-designware-platdrv.c @@ -46,6 +46,7 @@ static const struct acpi_device_id dw_i2c_acpi_match[] = { { "INT33C3", 0 }, { "INT3432", 0 }, { "INT3433", 0 }, + { "INTC10EF", 0 }, { "80860F41", ACCESS_NO_IRQ_SUSPEND }, { "808622C1", ACCESS_NO_IRQ_SUSPEND }, { "AMD0010", ACCESS_INTR_MASK }, From 780868fc480ee8e89e8db89c7946716cc75a36e9 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Mon, 15 Apr 2024 22:47:55 +0200 Subject: [PATCH 1059/1702] i2c: i801: Remove usage of I2C_CLASS_SPD Only remaining client driver supporting I2C_CLASS_SPD is jc42. This type of thermal sensor can be found on several DDR3/DDR4 modules. i2c_register_spd() instantiates also such thermal sensor i2c devices. Since 893fef0bc6aa ("i2c: i801: Call i2c_register_spd for muxed child segments") i2c_register_spd() is called also for the remaining use case, systems with muxed SMBUS segments for SPD EEPROMs. Therefore I2C_CLASS_SPD class-based instantiation isn't needed any longer in this driver, so remove it. Signed-off-by: Heiner Kallweit Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-i801.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index 92b53b480c47f..6d027a77d1dd7 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -264,7 +264,6 @@ struct i801_mux_config { char *gpio_chip; unsigned values[3]; int n_values; - unsigned classes[3]; unsigned gpios[2]; /* Relative to gpio_chip->base */ int n_gpios; }; @@ -1315,7 +1314,6 @@ static struct i801_mux_config i801_mux_config_asus_z8_d12 = { .gpio_chip = "gpio_ich", .values = { 0x02, 0x03 }, .n_values = 2, - .classes = { I2C_CLASS_SPD, I2C_CLASS_SPD }, .gpios = { 52, 53 }, .n_gpios = 2, }; @@ -1324,7 +1322,6 @@ static struct i801_mux_config i801_mux_config_asus_z8_d18 = { .gpio_chip = "gpio_ich", .values = { 0x02, 0x03, 0x01 }, .n_values = 3, - .classes = { I2C_CLASS_SPD, I2C_CLASS_SPD, I2C_CLASS_SPD }, .gpios = { 52, 53 }, .n_gpios = 2, }; @@ -1434,7 +1431,6 @@ static void i801_add_mux(struct i801_priv *priv) gpio_data.parent = priv->adapter.nr; gpio_data.values = mux_config->values; gpio_data.n_values = mux_config->n_values; - gpio_data.classes = mux_config->classes; gpio_data.idle = I2C_MUX_GPIO_NO_IDLE; /* Register GPIO descriptor lookup table */ From 073e58bf6b1a689da2ffcfc7fc515005f8794018 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Mon, 15 Apr 2024 22:48:42 +0200 Subject: [PATCH 1060/1702] i2c: mux: gpio: remove support for class-based device instantiation i801 as only user of gpio i2c mux removed support for class-based device instantiation on muxed busses. Class-based device instantiation is a legacy mechanism and shouldn't be used in new code, therefore remove support also here. Signed-off-by: Heiner Kallweit Signed-off-by: Andi Shyti --- drivers/i2c/muxes/i2c-mux-gpio.c | 3 +-- include/linux/platform_data/i2c-mux-gpio.h | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/i2c/muxes/i2c-mux-gpio.c b/drivers/i2c/muxes/i2c-mux-gpio.c index 6b979a0a6ab86..0fbb33a3d518c 100644 --- a/drivers/i2c/muxes/i2c-mux-gpio.c +++ b/drivers/i2c/muxes/i2c-mux-gpio.c @@ -206,9 +206,8 @@ static int i2c_mux_gpio_probe(struct platform_device *pdev) for (i = 0; i < mux->data.n_values; i++) { u32 nr = mux->data.base_nr ? (mux->data.base_nr + i) : 0; - unsigned int class = mux->data.classes ? mux->data.classes[i] : 0; - ret = i2c_mux_add_adapter(muxc, nr, mux->data.values[i], class); + ret = i2c_mux_add_adapter(muxc, nr, mux->data.values[i], 0); if (ret) goto add_adapter_failed; } diff --git a/include/linux/platform_data/i2c-mux-gpio.h b/include/linux/platform_data/i2c-mux-gpio.h index 5e4c2c272a734..816a4cd3ccb55 100644 --- a/include/linux/platform_data/i2c-mux-gpio.h +++ b/include/linux/platform_data/i2c-mux-gpio.h @@ -18,7 +18,6 @@ * @values: Array of bitmasks of GPIO settings (low/high) for each * position * @n_values: Number of multiplexer positions (busses to instantiate) - * @classes: Optional I2C auto-detection classes * @idle: Bitmask to write to MUX when idle or GPIO_I2CMUX_NO_IDLE if not used */ struct i2c_mux_gpio_platform_data { @@ -26,7 +25,6 @@ struct i2c_mux_gpio_platform_data { int base_nr; const unsigned *values; int n_values; - const unsigned *classes; unsigned idle; }; From 355b1513b1e97b6cef84b786c6480325dfd3753d Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Fri, 12 Apr 2024 12:21:58 +0200 Subject: [PATCH 1061/1702] i2c: i801: Annotate apanel_addr as __ro_after_init Annotate this variable as __ro_after_init to protect it from being overwritten later. Signed-off-by: Heiner Kallweit Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-i801.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index 6d027a77d1dd7..e577abc776c1e 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -1060,7 +1060,7 @@ static const struct pci_device_id i801_ids[] = { MODULE_DEVICE_TABLE(pci, i801_ids); #if defined CONFIG_X86 && defined CONFIG_DMI -static unsigned char apanel_addr; +static unsigned char apanel_addr __ro_after_init; /* Scan the system ROM for the signature "FJKEYINF" */ static __init const void __iomem *bios_signature(const void __iomem *bios) From 4268254a39484fc11ba991ae148bacbe75d9cc0a Mon Sep 17 00:00:00 2001 From: Alexander Stein Date: Mon, 22 Apr 2024 13:36:29 +0200 Subject: [PATCH 1062/1702] i2c: lpi2c: Avoid calling clk_get_rate during transfer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of repeatedly calling clk_get_rate for each transfer, lock the clock rate and cache the value. A deadlock has been observed while adding tlv320aic32x4 audio codec to the system. When this clock provider adds its clock, the clk mutex is locked already, it needs to access i2c, which in return needs the mutex for clk_get_rate as well. Signed-off-by: Alexander Stein Reviewed-by: Uwe Kleine-König Reviewed-by: Andi Shyti Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-imx-lpi2c.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c index 6d72e4e126dde..36e8f6196a87b 100644 --- a/drivers/i2c/busses/i2c-imx-lpi2c.c +++ b/drivers/i2c/busses/i2c-imx-lpi2c.c @@ -99,6 +99,7 @@ struct lpi2c_imx_struct { __u8 *rx_buf; __u8 *tx_buf; struct completion complete; + unsigned long rate_per; unsigned int msglen; unsigned int delivered; unsigned int block_data; @@ -212,9 +213,7 @@ static int lpi2c_imx_config(struct lpi2c_imx_struct *lpi2c_imx) lpi2c_imx_set_mode(lpi2c_imx); - clk_rate = clk_get_rate(lpi2c_imx->clks[0].clk); - if (!clk_rate) - return -EINVAL; + clk_rate = lpi2c_imx->rate_per; if (lpi2c_imx->mode == HS || lpi2c_imx->mode == ULTRA_FAST) filt = 0; @@ -611,6 +610,20 @@ static int lpi2c_imx_probe(struct platform_device *pdev) if (ret) return ret; + /* + * Lock the parent clock rate to avoid getting parent clock upon + * each transfer + */ + ret = devm_clk_rate_exclusive_get(&pdev->dev, lpi2c_imx->clks[0].clk); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "can't lock I2C peripheral clock rate\n"); + + lpi2c_imx->rate_per = clk_get_rate(lpi2c_imx->clks[0].clk); + if (!lpi2c_imx->rate_per) + return dev_err_probe(&pdev->dev, -EINVAL, + "can't get I2C peripheral clock rate\n"); + pm_runtime_set_autosuspend_delay(&pdev->dev, I2C_PM_TIMEOUT); pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_get_noresume(&pdev->dev); From 375ee8d223b4a08e09041f7fdb20d8428ea1e918 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:15 +0200 Subject: [PATCH 1063/1702] i2c: at91-master: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-at91-master.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/i2c/busses/i2c-at91-master.c b/drivers/i2c/busses/i2c-at91-master.c index d311981d3e608..ee3b469ddfb98 100644 --- a/drivers/i2c/busses/i2c-at91-master.c +++ b/drivers/i2c/busses/i2c-at91-master.c @@ -591,7 +591,6 @@ static int at91_do_twi_transfer(struct at91_twi_dev *dev) dev->adapter.timeout); if (time_left == 0) { dev->transfer_status |= at91_twi_read(dev, AT91_TWI_SR); - dev_err(dev->dev, "controller timed out\n"); at91_init_twi_bus(dev); ret = -ETIMEDOUT; goto error; From 796e2c260187e32530cf343546ba1cdf2e2f5491 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:16 +0200 Subject: [PATCH 1064/1702] i2c: bcm-iproc: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-bcm-iproc.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/i2c/busses/i2c-bcm-iproc.c b/drivers/i2c/busses/i2c-bcm-iproc.c index e905734c26a04..133d02899c6bd 100644 --- a/drivers/i2c/busses/i2c-bcm-iproc.c +++ b/drivers/i2c/busses/i2c-bcm-iproc.c @@ -811,8 +811,6 @@ static int bcm_iproc_i2c_xfer_wait(struct bcm_iproc_i2c_dev *iproc_i2c, } if (!time_left && !iproc_i2c->xfer_is_done) { - dev_err(iproc_i2c->device, "transaction timed out\n"); - /* flush both TX/RX FIFOs */ val = BIT(M_FIFO_RX_FLUSH_SHIFT) | BIT(M_FIFO_TX_FLUSH_SHIFT); iproc_i2c_wr_reg(iproc_i2c, M_FIFO_CTRL_OFFSET, val); From 09a40cbc996e8cd6633930a58ccf4fe1fa12e60e Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:17 +0200 Subject: [PATCH 1065/1702] i2c: bcm2835: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-bcm2835.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/i2c/busses/i2c-bcm2835.c b/drivers/i2c/busses/i2c-bcm2835.c index b92de19442216..3045ba82380db 100644 --- a/drivers/i2c/busses/i2c-bcm2835.c +++ b/drivers/i2c/busses/i2c-bcm2835.c @@ -370,7 +370,6 @@ static int bcm2835_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], if (!time_left) { bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_C, BCM2835_I2C_C_CLEAR); - dev_err(i2c_dev->dev, "i2c transfer timed out\n"); return -ETIMEDOUT; } From 6808d67f512ab0db5d6f44c2f13e7d6a6fd5e4f5 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:18 +0200 Subject: [PATCH 1066/1702] i2c: cadence: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Acked-by: Michal Simek Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-cadence.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/i2c/busses/i2c-cadence.c b/drivers/i2c/busses/i2c-cadence.c index 4bb7d6756947c..1b0006e3b95cc 100644 --- a/drivers/i2c/busses/i2c-cadence.c +++ b/drivers/i2c/busses/i2c-cadence.c @@ -789,8 +789,6 @@ static int cdns_i2c_process_msg(struct cdns_i2c *id, struct i2c_msg *msg, time_left = wait_for_completion_timeout(&id->xfer_done, msg_timeout); if (time_left == 0) { cdns_i2c_master_reset(adap); - dev_err(id->adap.dev.parent, - "timeout waiting on completion\n"); return -ETIMEDOUT; } From fc93dcd7ebdd2543da7ebaaa8c2d51ce2ec689eb Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:19 +0200 Subject: [PATCH 1067/1702] i2c: davinci: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-davinci.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/i2c/busses/i2c-davinci.c b/drivers/i2c/busses/i2c-davinci.c index 02b3b1160fb06..7ae611120cfa8 100644 --- a/drivers/i2c/busses/i2c-davinci.c +++ b/drivers/i2c/busses/i2c-davinci.c @@ -489,7 +489,6 @@ i2c_davinci_xfer_msg(struct i2c_adapter *adap, struct i2c_msg *msg, int stop) time_left = wait_for_completion_timeout(&dev->cmd_complete, dev->adapter.timeout); if (!time_left) { - dev_err(dev->dev, "controller timed out\n"); i2c_recover_bus(adap); dev->buf_len = 0; return -ETIMEDOUT; From b752b21e6f84edb5b0e27f00e41d897bc671fe17 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:21 +0200 Subject: [PATCH 1068/1702] i2c: img-scb: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout and simplify the logic a little. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-img-scb.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/i2c/busses/i2c-img-scb.c b/drivers/i2c/busses/i2c-img-scb.c index f9d4bfef511c3..e0e87185f6bbf 100644 --- a/drivers/i2c/busses/i2c-img-scb.c +++ b/drivers/i2c/busses/i2c-img-scb.c @@ -1124,11 +1124,8 @@ static int img_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, IMG_I2C_TIMEOUT); del_timer_sync(&i2c->check_timer); - if (time_left == 0) { - dev_err(adap->dev.parent, "i2c transfer timed out\n"); + if (time_left == 0) i2c->msg_status = -ETIMEDOUT; - break; - } if (i2c->msg_status) break; From 4828450ad17d2966d2429156146797db3ec8182d Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:22 +0200 Subject: [PATCH 1069/1702] i2c: ismt: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-ismt.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/i2c/busses/i2c-ismt.c b/drivers/i2c/busses/i2c-ismt.c index c74985d77b0ec..655b5d851c48b 100644 --- a/drivers/i2c/busses/i2c-ismt.c +++ b/drivers/i2c/busses/i2c-ismt.c @@ -623,7 +623,6 @@ static int ismt_access(struct i2c_adapter *adap, u16 addr, dma_unmap_single(dev, dma_addr, dma_size, dma_direction); if (unlikely(!time_left)) { - dev_err(dev, "completion wait timed out\n"); ret = -ETIMEDOUT; goto out; } From 60b36100f815d0f75abecabffa31efe9caa16f04 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:23 +0200 Subject: [PATCH 1070/1702] i2c: nomadik: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Reviewed-by: Linus Walleij Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-nomadik.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/i2c/busses/i2c-nomadik.c b/drivers/i2c/busses/i2c-nomadik.c index 4f41a3c7824d0..45c6df26fcbf6 100644 --- a/drivers/i2c/busses/i2c-nomadik.c +++ b/drivers/i2c/busses/i2c-nomadik.c @@ -542,12 +542,9 @@ static int read_i2c(struct nmk_i2c_dev *priv, u16 flags) xfer_done = nmk_i2c_wait_xfer_done(priv); - if (!xfer_done) { - /* Controller timed out */ - dev_err(&priv->adev->dev, "read from slave 0x%x timed out\n", - priv->cli.slave_adr); + if (!xfer_done) status = -ETIMEDOUT; - } + return status; } From 760b37b373a8d1d90011265307225bb26be5cda1 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:24 +0200 Subject: [PATCH 1071/1702] i2c: omap: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-omap.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/i2c/busses/i2c-omap.c b/drivers/i2c/busses/i2c-omap.c index 42165ef57946c..36bebef36740d 100644 --- a/drivers/i2c/busses/i2c-omap.c +++ b/drivers/i2c/busses/i2c-omap.c @@ -752,7 +752,6 @@ static int omap_i2c_xfer_msg(struct i2c_adapter *adap, } if (timeout == 0) { - dev_err(omap->dev, "controller timed out\n"); omap_i2c_reset(omap); __omap_i2c_init(omap); return -ETIMEDOUT; From 6d128e73c7820356a3481024166b9f3dc3aee99c Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:25 +0200 Subject: [PATCH 1072/1702] i2c: qcom-geni: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Reviewed-by: Bjorn Andersson Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-qcom-geni.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/i2c/busses/i2c-qcom-geni.c b/drivers/i2c/busses/i2c-qcom-geni.c index da94df466e83c..090b4846ed62b 100644 --- a/drivers/i2c/busses/i2c-qcom-geni.c +++ b/drivers/i2c/busses/i2c-qcom-geni.c @@ -630,11 +630,8 @@ static int geni_i2c_gpi_xfer(struct geni_i2c_dev *gi2c, struct i2c_msg msgs[], i dma_async_issue_pending(gi2c->tx_c); timeout = wait_for_completion_timeout(&gi2c->done, XFER_TIMEOUT); - if (!timeout) { - dev_err(gi2c->se.dev, "I2C timeout gpi flags:%d addr:0x%x\n", - gi2c->cur->flags, gi2c->cur->addr); + if (!timeout) gi2c->err = -ETIMEDOUT; - } if (gi2c->err) { ret = gi2c->err; From 1667b35535bb2242b296513642fee1d3291a4503 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:26 +0200 Subject: [PATCH 1073/1702] i2c: qup: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Reviewed-by: Bjorn Andersson Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-qup.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/i2c/busses/i2c-qup.c b/drivers/i2c/busses/i2c-qup.c index 598102d16677a..c9b43a3c4bd3c 100644 --- a/drivers/i2c/busses/i2c-qup.c +++ b/drivers/i2c/busses/i2c-qup.c @@ -793,10 +793,8 @@ static int qup_i2c_bam_schedule_desc(struct qup_i2c_dev *qup) dma_async_issue_pending(qup->brx.dma); } - if (!wait_for_completion_timeout(&qup->xfer, qup->xfer_timeout)) { - dev_err(qup->dev, "normal trans timed out\n"); + if (!wait_for_completion_timeout(&qup->xfer, qup->xfer_timeout)) ret = -ETIMEDOUT; - } if (ret || qup->bus_err || qup->qup_err) { reinit_completion(&qup->xfer); From 0ee55dc9edd03a7d98e3ce8f8f03380f090d520b Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:27 +0200 Subject: [PATCH 1074/1702] i2c: rk3x: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Acked-by: Heiko Stuebner Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-rk3x.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c index 086fdf262e7b6..8c7367f289d36 100644 --- a/drivers/i2c/busses/i2c-rk3x.c +++ b/drivers/i2c/busses/i2c-rk3x.c @@ -1106,9 +1106,6 @@ static int rk3x_i2c_xfer_common(struct i2c_adapter *adap, spin_lock_irqsave(&i2c->lock, flags); if (timeout == 0) { - dev_err(i2c->dev, "timeout, ipd: 0x%02x, state: %d\n", - i2c_readl(i2c, REG_IPD), i2c->state); - /* Force a STOP condition without interrupt */ i2c_writel(i2c, 0, REG_IEN); val = i2c_readl(i2c, REG_CON) & REG_CON_TUNING_MASK; From 03bc6a248ab6e3d021b45bee3f324256b0008fbd Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:28 +0200 Subject: [PATCH 1075/1702] i2c: sh_mobile: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-sh_mobile.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/i2c/busses/i2c-sh_mobile.c b/drivers/i2c/busses/i2c-sh_mobile.c index c65ac3d7eadc5..f86c29737df12 100644 --- a/drivers/i2c/busses/i2c-sh_mobile.c +++ b/drivers/i2c/busses/i2c-sh_mobile.c @@ -688,7 +688,6 @@ static int sh_mobile_xfer(struct sh_mobile_i2c_data *pd, } if (!time_left) { - dev_err(pd->dev, "Transfer request timed out\n"); if (pd->dma_direction != DMA_NONE) sh_mobile_i2c_cleanup_dma(pd, true); From bc19b5fe972b39516d6eafd4b61830b58bcd5c54 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:29 +0200 Subject: [PATCH 1076/1702] i2c: st: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-st.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/i2c/busses/i2c-st.c b/drivers/i2c/busses/i2c-st.c index ce2333408904b..dbb93394ff946 100644 --- a/drivers/i2c/busses/i2c-st.c +++ b/drivers/i2c/busses/i2c-st.c @@ -689,11 +689,8 @@ static int st_i2c_xfer_msg(struct st_i2c_dev *i2c_dev, struct i2c_msg *msg, i2c_dev->adap.timeout); ret = c->result; - if (!timeout) { - dev_err(i2c_dev->dev, "Write to slave 0x%x timed out\n", - c->addr); + if (!timeout) ret = -ETIMEDOUT; - } i2c = SSC_I2C_STOPG | SSC_I2C_REPSTRTG; st_i2c_clr_bits(i2c_dev->base + SSC_I2C, i2c); From f757ec37efd2190e213f3fbcf17bdad8c10e9104 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:30 +0200 Subject: [PATCH 1077/1702] i2c: tegra: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-tegra.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c index 920d5a8cbf4c7..85b31edc558df 100644 --- a/drivers/i2c/busses/i2c-tegra.c +++ b/drivers/i2c/busses/i2c-tegra.c @@ -1331,7 +1331,6 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, dmaengine_terminate_sync(i2c_dev->dma_chan); if (!time_left && !completion_done(&i2c_dev->dma_complete)) { - dev_err(i2c_dev->dev, "DMA transfer timed out\n"); tegra_i2c_init(i2c_dev); return -ETIMEDOUT; } @@ -1351,7 +1350,6 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, tegra_i2c_mask_irq(i2c_dev, int_mask); if (time_left == 0) { - dev_err(i2c_dev->dev, "I2C transfer timed out\n"); tegra_i2c_init(i2c_dev); return -ETIMEDOUT; } From 58859d9b62d874a69ecb5243d347f1ce47286d49 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:31 +0200 Subject: [PATCH 1078/1702] i2c: uniphier-f: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-uniphier-f.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/i2c/busses/i2c-uniphier-f.c b/drivers/i2c/busses/i2c-uniphier-f.c index dbc91c7c3788f..6c3dac2cf568f 100644 --- a/drivers/i2c/busses/i2c-uniphier-f.c +++ b/drivers/i2c/busses/i2c-uniphier-f.c @@ -358,7 +358,6 @@ static int uniphier_fi2c_master_xfer_one(struct i2c_adapter *adap, spin_unlock_irqrestore(&priv->lock, flags); if (!time_left) { - dev_err(&adap->dev, "transaction timeout.\n"); uniphier_fi2c_recover(priv); return -ETIMEDOUT; } From 1d428f5d3827fd9826abe3b14609aaa2037bc00c Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Wed, 10 Apr 2024 13:24:32 +0200 Subject: [PATCH 1079/1702] i2c: uniphier: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-uniphier.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/i2c/busses/i2c-uniphier.c b/drivers/i2c/busses/i2c-uniphier.c index 854ac25b58628..e1b4c80e02854 100644 --- a/drivers/i2c/busses/i2c-uniphier.c +++ b/drivers/i2c/busses/i2c-uniphier.c @@ -71,10 +71,8 @@ static int uniphier_i2c_xfer_byte(struct i2c_adapter *adap, u32 txdata, writel(txdata, priv->membase + UNIPHIER_I2C_DTRM); time_left = wait_for_completion_timeout(&priv->comp, adap->timeout); - if (unlikely(!time_left)) { - dev_err(&adap->dev, "transaction timeout\n"); + if (unlikely(!time_left)) return -ETIMEDOUT; - } rxdata = readl(priv->membase + UNIPHIER_I2C_DREC); if (rxdatap) From 29914dac94d490001bdfc5b9fdb8a5c29a724c1b Mon Sep 17 00:00:00 2001 From: Ji Sheng Teoh Date: Tue, 23 Apr 2024 10:48:06 +0800 Subject: [PATCH 1080/1702] i2c: cadence: Add RISCV architecture support Add RISCV support to Cadence I2C Kconfig which is used in platform such as the StarFive JH8100. Signed-off-by: Eng Lee Teh Signed-off-by: Ji Sheng Teoh Signed-off-by: Andi Shyti --- drivers/i2c/busses/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index 27d3ef565ca8b..a5aff29c559d0 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -508,7 +508,7 @@ config I2C_BRCMSTB config I2C_CADENCE tristate "Cadence I2C Controller" - depends on ARCH_ZYNQ || ARM64 || XTENSA || COMPILE_TEST + depends on ARCH_ZYNQ || ARM64 || XTENSA || RISCV || COMPILE_TEST help Say yes here to select Cadence I2C Host Controller. This controller is e.g. used by Xilinx Zynq. From 114c69f4be3f3a55355b2f1db46a820bb0364457 Mon Sep 17 00:00:00 2001 From: Suneel Garapati Date: Tue, 23 Apr 2024 00:46:04 -0700 Subject: [PATCH 1081/1702] i2c: thunderx: Clock divisor logic changes Handle changes to clock divisor logic for OcteonTX2 SoC family using subsystem ID and using default reference clock source as 100MHz. Signed-off-by: Suneel Garapati Signed-off-by: Piyush Malgujar Acked-by: Andi Shyti Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-octeon-core.c | 39 +++++++++++++++++++++--- drivers/i2c/busses/i2c-octeon-core.h | 17 +++++++++++ drivers/i2c/busses/i2c-thunderx-pcidrv.c | 7 +++++ 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/drivers/i2c/busses/i2c-octeon-core.c b/drivers/i2c/busses/i2c-octeon-core.c index 845eda70b8cab..75efb375d8e49 100644 --- a/drivers/i2c/busses/i2c-octeon-core.c +++ b/drivers/i2c/busses/i2c-octeon-core.c @@ -17,9 +17,14 @@ #include #include #include +#include #include "i2c-octeon-core.h" +#define INITIAL_DELTA_HZ 1000000 +#define TWSI_MASTER_CLK_REG_DEF_VAL 0x18 +#define TWSI_MASTER_CLK_REG_OTX2_VAL 0x3 + /* interrupt service routine */ irqreturn_t octeon_i2c_isr(int irq, void *dev_id) { @@ -658,31 +663,57 @@ int octeon_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num) void octeon_i2c_set_clock(struct octeon_i2c *i2c) { int tclk, thp_base, inc, thp_idx, mdiv_idx, ndiv_idx, foscl, diff; - int thp = 0x18, mdiv = 2, ndiv = 0, delta_hz = 1000000; + bool is_plat_otx2; + unsigned int mdiv_min = 2; + /* + * Find divisors to produce target frequency, start with large delta + * to cover wider range of divisors, note thp = TCLK half period. + */ + unsigned int thp = TWSI_MASTER_CLK_REG_DEF_VAL, mdiv = 2, ndiv = 0; + unsigned int delta_hz = INITIAL_DELTA_HZ; + + is_plat_otx2 = octeon_i2c_is_otx2(to_pci_dev(i2c->dev)); + + if (is_plat_otx2) { + thp = TWSI_MASTER_CLK_REG_OTX2_VAL; + mdiv_min = 0; + } for (ndiv_idx = 0; ndiv_idx < 8 && delta_hz != 0; ndiv_idx++) { /* * An mdiv value of less than 2 seems to not work well * with ds1337 RTCs, so we constrain it to larger values. */ - for (mdiv_idx = 15; mdiv_idx >= 2 && delta_hz != 0; mdiv_idx--) { + for (mdiv_idx = 15; mdiv_idx >= mdiv_min && delta_hz != 0; mdiv_idx--) { /* * For given ndiv and mdiv values check the * two closest thp values. */ tclk = i2c->twsi_freq * (mdiv_idx + 1) * 10; tclk *= (1 << ndiv_idx); - thp_base = (i2c->sys_freq / (tclk * 2)) - 1; + if (is_plat_otx2) + thp_base = (i2c->sys_freq / tclk) - 2; + else + thp_base = (i2c->sys_freq / (tclk * 2)) - 1; for (inc = 0; inc <= 1; inc++) { thp_idx = thp_base + inc; if (thp_idx < 5 || thp_idx > 0xff) continue; - foscl = i2c->sys_freq / (2 * (thp_idx + 1)); + if (is_plat_otx2) + foscl = i2c->sys_freq / (thp_idx + 2); + else + foscl = i2c->sys_freq / + (2 * (thp_idx + 1)); foscl = foscl / (1 << ndiv_idx); foscl = foscl / (mdiv_idx + 1) / 10; diff = abs(foscl - i2c->twsi_freq); + /* + * Diff holds difference between calculated frequency + * value vs desired frequency. + * Delta_hz is updated with last minimum diff. + */ if (diff < delta_hz) { delta_hz = diff; thp = thp_idx; diff --git a/drivers/i2c/busses/i2c-octeon-core.h b/drivers/i2c/busses/i2c-octeon-core.h index 9bb9f64fdda03..69bd62940f99e 100644 --- a/drivers/i2c/busses/i2c-octeon-core.h +++ b/drivers/i2c/busses/i2c-octeon-core.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include +#include #include #include #include @@ -7,6 +8,7 @@ #include #include #include +#include /* Controller command patterns */ #define SW_TWSI_V BIT_ULL(63) /* Valid bit */ @@ -211,6 +213,21 @@ static inline void octeon_i2c_write_int(struct octeon_i2c *i2c, u64 data) octeon_i2c_writeq_flush(data, i2c->twsi_base + TWSI_INT(i2c)); } +#define PCI_SUBSYS_DEVID_9XXX 0xB +#define PCI_SUBSYS_MASK GENMASK(15, 12) +/** + * octeon_i2c_is_otx2 - check for chip ID + * @pdev: PCI dev structure + * + * Returns true if the device is an OcteonTX2, false otherwise. + */ +static inline bool octeon_i2c_is_otx2(struct pci_dev *pdev) +{ + u32 chip_id = FIELD_GET(PCI_SUBSYS_MASK, pdev->subsystem_device); + + return (chip_id == PCI_SUBSYS_DEVID_9XXX); +} + /* Prototypes */ irqreturn_t octeon_i2c_isr(int irq, void *dev_id); int octeon_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num); diff --git a/drivers/i2c/busses/i2c-thunderx-pcidrv.c b/drivers/i2c/busses/i2c-thunderx-pcidrv.c index a77cd86fe75ed..7556977400385 100644 --- a/drivers/i2c/busses/i2c-thunderx-pcidrv.c +++ b/drivers/i2c/busses/i2c-thunderx-pcidrv.c @@ -28,6 +28,7 @@ #define PCI_DEVICE_ID_THUNDER_TWSI 0xa012 #define SYS_FREQ_DEFAULT 700000000 +#define OTX2_REF_FREQ_DEFAULT 100000000 #define TWSI_INT_ENA_W1C 0x1028 #define TWSI_INT_ENA_W1S 0x1030 @@ -205,6 +206,12 @@ static int thunder_i2c_probe_pci(struct pci_dev *pdev, if (ret) goto error; + /* + * For OcteonTX2 chips, set reference frequency to 100MHz + * as refclk_src in TWSI_MODE register defaults to 100MHz. + */ + if (octeon_i2c_is_otx2(pdev)) + i2c->sys_freq = OTX2_REF_FREQ_DEFAULT; octeon_i2c_set_clock(i2c); i2c->adap = thunderx_i2c_ops; From 03240f826b02929476abd7c3a3cc132cc65c8614 Mon Sep 17 00:00:00 2001 From: Suneel Garapati Date: Tue, 23 Apr 2024 00:46:05 -0700 Subject: [PATCH 1082/1702] i2c: thunderx: Support for High speed mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To support bus operations for high speed bus frequencies greater than 400KHZ following control bits need to be setup accordingly - hs_mode (bit 0) field in Mode register to switch controller between low-speed and high-speed frequency operating mode. - Setup clock divisors for desired TWSI bus frequency using FOSCL output frequency divisor (D): 0 - sets the divisor to 10 for low speed mode 1 - sets the divisor to 15 for high speed mode. The TWSI bus output frequency, in master mode is based on: TCLK = 100MHz / (THP + 2) FOSCL = FSAMP / (M+1)×D = TCLK / (2 ^ N × (M + 1) × 15) FSAMP = TCLK / 2 ^ N where, N is <2:0> and M is <6:3> of TWSI Clock Control Register D is 10 for low speed or 15 for HS_MODE With high speed mode support, HLC mode usage is limited to low speed frequency (<=400KHz) bus transfers in hardware. Signed-off-by: Suneel Garapati Signed-off-by: Piyush Malgujar Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-octeon-core.c | 68 +++++++++++++++--------- drivers/i2c/busses/i2c-octeon-core.h | 8 +++ drivers/i2c/busses/i2c-thunderx-pcidrv.c | 3 +- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/drivers/i2c/busses/i2c-octeon-core.c b/drivers/i2c/busses/i2c-octeon-core.c index 75efb375d8e49..86be597866f81 100644 --- a/drivers/i2c/busses/i2c-octeon-core.c +++ b/drivers/i2c/busses/i2c-octeon-core.c @@ -612,25 +612,27 @@ int octeon_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num) struct octeon_i2c *i2c = i2c_get_adapdata(adap); int i, ret = 0; - if (num == 1) { - if (msgs[0].len > 0 && msgs[0].len <= 8) { - if (msgs[0].flags & I2C_M_RD) - ret = octeon_i2c_hlc_read(i2c, msgs); - else - ret = octeon_i2c_hlc_write(i2c, msgs); - goto out; - } - } else if (num == 2) { - if ((msgs[0].flags & I2C_M_RD) == 0 && - (msgs[1].flags & I2C_M_RECV_LEN) == 0 && - msgs[0].len > 0 && msgs[0].len <= 2 && - msgs[1].len > 0 && msgs[1].len <= 8 && - msgs[0].addr == msgs[1].addr) { - if (msgs[1].flags & I2C_M_RD) - ret = octeon_i2c_hlc_comp_read(i2c, msgs); - else - ret = octeon_i2c_hlc_comp_write(i2c, msgs); - goto out; + if (IS_LS_FREQ(i2c->twsi_freq)) { + if (num == 1) { + if (msgs[0].len > 0 && msgs[0].len <= 8) { + if (msgs[0].flags & I2C_M_RD) + ret = octeon_i2c_hlc_read(i2c, msgs); + else + ret = octeon_i2c_hlc_write(i2c, msgs); + goto out; + } + } else if (num == 2) { + if ((msgs[0].flags & I2C_M_RD) == 0 && + (msgs[1].flags & I2C_M_RECV_LEN) == 0 && + msgs[0].len > 0 && msgs[0].len <= 2 && + msgs[1].len > 0 && msgs[1].len <= 8 && + msgs[0].addr == msgs[1].addr) { + if (msgs[1].flags & I2C_M_RD) + ret = octeon_i2c_hlc_comp_read(i2c, msgs); + else + ret = octeon_i2c_hlc_comp_write(i2c, msgs); + goto out; + } } } @@ -664,12 +666,12 @@ void octeon_i2c_set_clock(struct octeon_i2c *i2c) { int tclk, thp_base, inc, thp_idx, mdiv_idx, ndiv_idx, foscl, diff; bool is_plat_otx2; - unsigned int mdiv_min = 2; /* * Find divisors to produce target frequency, start with large delta - * to cover wider range of divisors, note thp = TCLK half period. + * to cover wider range of divisors, note thp = TCLK half period and + * ds is OSCL output frequency divisor. */ - unsigned int thp = TWSI_MASTER_CLK_REG_DEF_VAL, mdiv = 2, ndiv = 0; + unsigned int thp, mdiv_min, mdiv = 2, ndiv = 0, ds = 10; unsigned int delta_hz = INITIAL_DELTA_HZ; is_plat_otx2 = octeon_i2c_is_otx2(to_pci_dev(i2c->dev)); @@ -677,6 +679,11 @@ void octeon_i2c_set_clock(struct octeon_i2c *i2c) if (is_plat_otx2) { thp = TWSI_MASTER_CLK_REG_OTX2_VAL; mdiv_min = 0; + if (!IS_LS_FREQ(i2c->twsi_freq)) + ds = 15; + } else { + thp = TWSI_MASTER_CLK_REG_DEF_VAL; + mdiv_min = 2; } for (ndiv_idx = 0; ndiv_idx < 8 && delta_hz != 0; ndiv_idx++) { @@ -689,7 +696,7 @@ void octeon_i2c_set_clock(struct octeon_i2c *i2c) * For given ndiv and mdiv values check the * two closest thp values. */ - tclk = i2c->twsi_freq * (mdiv_idx + 1) * 10; + tclk = i2c->twsi_freq * (mdiv_idx + 1) * ds; tclk *= (1 << ndiv_idx); if (is_plat_otx2) thp_base = (i2c->sys_freq / tclk) - 2; @@ -707,7 +714,9 @@ void octeon_i2c_set_clock(struct octeon_i2c *i2c) foscl = i2c->sys_freq / (2 * (thp_idx + 1)); foscl = foscl / (1 << ndiv_idx); - foscl = foscl / (mdiv_idx + 1) / 10; + foscl = foscl / (mdiv_idx + 1) / ds; + if (foscl > i2c->twsi_freq) + continue; diff = abs(foscl - i2c->twsi_freq); /* * Diff holds difference between calculated frequency @@ -725,6 +734,17 @@ void octeon_i2c_set_clock(struct octeon_i2c *i2c) } octeon_i2c_reg_write(i2c, SW_TWSI_OP_TWSI_CLK, thp); octeon_i2c_reg_write(i2c, SW_TWSI_EOP_TWSI_CLKCTL, (mdiv << 3) | ndiv); + if (is_plat_otx2) { + u64 mode; + + mode = __raw_readq(i2c->twsi_base + MODE(i2c)); + /* Set REFCLK_SRC and HS_MODE in TWSX_MODE register */ + if (!IS_LS_FREQ(i2c->twsi_freq)) + mode |= TWSX_MODE_HS_MASK; + else + mode &= ~TWSX_MODE_HS_MASK; + octeon_i2c_writeq_flush(mode, i2c->twsi_base + MODE(i2c)); + } } int octeon_i2c_init_lowlevel(struct octeon_i2c *i2c) diff --git a/drivers/i2c/busses/i2c-octeon-core.h b/drivers/i2c/busses/i2c-octeon-core.h index 69bd62940f99e..94c4401a4a567 100644 --- a/drivers/i2c/busses/i2c-octeon-core.h +++ b/drivers/i2c/busses/i2c-octeon-core.h @@ -94,11 +94,18 @@ struct octeon_i2c_reg_offset { unsigned int sw_twsi; unsigned int twsi_int; unsigned int sw_twsi_ext; + unsigned int mode; }; #define SW_TWSI(x) (x->roff.sw_twsi) #define TWSI_INT(x) (x->roff.twsi_int) #define SW_TWSI_EXT(x) (x->roff.sw_twsi_ext) +#define MODE(x) ((x)->roff.mode) + +/* Set REFCLK_SRC and HS_MODE in TWSX_MODE register */ +#define TWSX_MODE_REFCLK_SRC BIT(4) +#define TWSX_MODE_HS_MODE BIT(0) +#define TWSX_MODE_HS_MASK (TWSX_MODE_REFCLK_SRC | TWSX_MODE_HS_MODE) struct octeon_i2c { wait_queue_head_t queue; @@ -213,6 +220,7 @@ static inline void octeon_i2c_write_int(struct octeon_i2c *i2c, u64 data) octeon_i2c_writeq_flush(data, i2c->twsi_base + TWSI_INT(i2c)); } +#define IS_LS_FREQ(twsi_freq) ((twsi_freq) <= 400000) #define PCI_SUBSYS_DEVID_9XXX 0xB #define PCI_SUBSYS_MASK GENMASK(15, 12) /** diff --git a/drivers/i2c/busses/i2c-thunderx-pcidrv.c b/drivers/i2c/busses/i2c-thunderx-pcidrv.c index 7556977400385..31f11b77ab663 100644 --- a/drivers/i2c/busses/i2c-thunderx-pcidrv.c +++ b/drivers/i2c/busses/i2c-thunderx-pcidrv.c @@ -166,6 +166,7 @@ static int thunder_i2c_probe_pci(struct pci_dev *pdev, i2c->roff.sw_twsi = 0x1000; i2c->roff.twsi_int = 0x1010; i2c->roff.sw_twsi_ext = 0x1018; + i2c->roff.mode = 0x1038; i2c->dev = dev; pci_set_drvdata(pdev, i2c); @@ -210,7 +211,7 @@ static int thunder_i2c_probe_pci(struct pci_dev *pdev, * For OcteonTX2 chips, set reference frequency to 100MHz * as refclk_src in TWSI_MODE register defaults to 100MHz. */ - if (octeon_i2c_is_otx2(pdev)) + if (octeon_i2c_is_otx2(pdev) && IS_LS_FREQ(i2c->twsi_freq)) i2c->sys_freq = OTX2_REF_FREQ_DEFAULT; octeon_i2c_set_clock(i2c); From 0b042c72d90de2f53d6d6d768158614f4b717b16 Mon Sep 17 00:00:00 2001 From: Piyush Malgujar Date: Tue, 23 Apr 2024 00:46:06 -0700 Subject: [PATCH 1083/1702] i2c: octeon: Add platform prefix to macros The macros for TWSI register's offset are generically named, rename them to be platform specific macros by adding 'OCTEON_REG' as prefix. Signed-off-by: Piyush Malgujar Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-octeon-core.c | 36 ++++++++++++++-------------- drivers/i2c/busses/i2c-octeon-core.h | 26 ++++++++++---------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/drivers/i2c/busses/i2c-octeon-core.c b/drivers/i2c/busses/i2c-octeon-core.c index 86be597866f81..76a5ec100d303 100644 --- a/drivers/i2c/busses/i2c-octeon-core.c +++ b/drivers/i2c/busses/i2c-octeon-core.c @@ -85,7 +85,7 @@ static int octeon_i2c_wait(struct octeon_i2c *i2c) static bool octeon_i2c_hlc_test_valid(struct octeon_i2c *i2c) { - return (__raw_readq(i2c->twsi_base + SW_TWSI(i2c)) & SW_TWSI_V) == 0; + return (__raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)) & SW_TWSI_V) == 0; } static void octeon_i2c_hlc_int_clear(struct octeon_i2c *i2c) @@ -185,10 +185,10 @@ static int octeon_i2c_check_status(struct octeon_i2c *i2c, int final_read) /* * This is ugly... in HLC mode the status is not in the status register - * but in the lower 8 bits of SW_TWSI. + * but in the lower 8 bits of OCTEON_REG_SW_TWSI. */ if (i2c->hlc_enabled) - stat = __raw_readq(i2c->twsi_base + SW_TWSI(i2c)); + stat = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); else stat = octeon_i2c_stat_read(i2c); @@ -424,12 +424,12 @@ static int octeon_i2c_hlc_read(struct octeon_i2c *i2c, struct i2c_msg *msgs) else cmd |= SW_TWSI_OP_7; - octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI(i2c)); + octeon_i2c_writeq_flush(cmd, i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); ret = octeon_i2c_hlc_wait(i2c); if (ret) goto err; - cmd = __raw_readq(i2c->twsi_base + SW_TWSI(i2c)); + cmd = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); if ((cmd & SW_TWSI_R) == 0) return octeon_i2c_check_status(i2c, false); @@ -437,7 +437,7 @@ static int octeon_i2c_hlc_read(struct octeon_i2c *i2c, struct i2c_msg *msgs) msgs[0].buf[j] = (cmd >> (8 * i)) & 0xff; if (msgs[0].len > 4) { - cmd = __raw_readq(i2c->twsi_base + SW_TWSI_EXT(i2c)); + cmd = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI_EXT(i2c)); for (i = 0; i < msgs[0].len - 4 && i < 4; i++, j--) msgs[0].buf[j] = (cmd >> (8 * i)) & 0xff; } @@ -474,15 +474,15 @@ static int octeon_i2c_hlc_write(struct octeon_i2c *i2c, struct i2c_msg *msgs) for (i = 0; i < msgs[0].len - 4 && i < 4; i++, j--) ext |= (u64)msgs[0].buf[j] << (8 * i); - octeon_i2c_writeq_flush(ext, i2c->twsi_base + SW_TWSI_EXT(i2c)); + octeon_i2c_writeq_flush(ext, i2c->twsi_base + OCTEON_REG_SW_TWSI_EXT(i2c)); } - octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI(i2c)); + octeon_i2c_writeq_flush(cmd, i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); ret = octeon_i2c_hlc_wait(i2c); if (ret) goto err; - cmd = __raw_readq(i2c->twsi_base + SW_TWSI(i2c)); + cmd = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); if ((cmd & SW_TWSI_R) == 0) return octeon_i2c_check_status(i2c, false); @@ -515,19 +515,19 @@ static int octeon_i2c_hlc_comp_read(struct octeon_i2c *i2c, struct i2c_msg *msgs cmd |= SW_TWSI_EIA; ext = (u64)msgs[0].buf[0] << SW_TWSI_IA_SHIFT; cmd |= (u64)msgs[0].buf[1] << SW_TWSI_IA_SHIFT; - octeon_i2c_writeq_flush(ext, i2c->twsi_base + SW_TWSI_EXT(i2c)); + octeon_i2c_writeq_flush(ext, i2c->twsi_base + OCTEON_REG_SW_TWSI_EXT(i2c)); } else { cmd |= (u64)msgs[0].buf[0] << SW_TWSI_IA_SHIFT; } octeon_i2c_hlc_int_clear(i2c); - octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI(i2c)); + octeon_i2c_writeq_flush(cmd, i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); ret = octeon_i2c_hlc_wait(i2c); if (ret) goto err; - cmd = __raw_readq(i2c->twsi_base + SW_TWSI(i2c)); + cmd = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); if ((cmd & SW_TWSI_R) == 0) return octeon_i2c_check_status(i2c, false); @@ -535,7 +535,7 @@ static int octeon_i2c_hlc_comp_read(struct octeon_i2c *i2c, struct i2c_msg *msgs msgs[1].buf[j] = (cmd >> (8 * i)) & 0xff; if (msgs[1].len > 4) { - cmd = __raw_readq(i2c->twsi_base + SW_TWSI_EXT(i2c)); + cmd = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI_EXT(i2c)); for (i = 0; i < msgs[1].len - 4 && i < 4; i++, j--) msgs[1].buf[j] = (cmd >> (8 * i)) & 0xff; } @@ -582,16 +582,16 @@ static int octeon_i2c_hlc_comp_write(struct octeon_i2c *i2c, struct i2c_msg *msg set_ext = true; } if (set_ext) - octeon_i2c_writeq_flush(ext, i2c->twsi_base + SW_TWSI_EXT(i2c)); + octeon_i2c_writeq_flush(ext, i2c->twsi_base + OCTEON_REG_SW_TWSI_EXT(i2c)); octeon_i2c_hlc_int_clear(i2c); - octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI(i2c)); + octeon_i2c_writeq_flush(cmd, i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); ret = octeon_i2c_hlc_wait(i2c); if (ret) goto err; - cmd = __raw_readq(i2c->twsi_base + SW_TWSI(i2c)); + cmd = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); if ((cmd & SW_TWSI_R) == 0) return octeon_i2c_check_status(i2c, false); @@ -737,13 +737,13 @@ void octeon_i2c_set_clock(struct octeon_i2c *i2c) if (is_plat_otx2) { u64 mode; - mode = __raw_readq(i2c->twsi_base + MODE(i2c)); + mode = __raw_readq(i2c->twsi_base + OCTEON_REG_MODE(i2c)); /* Set REFCLK_SRC and HS_MODE in TWSX_MODE register */ if (!IS_LS_FREQ(i2c->twsi_freq)) mode |= TWSX_MODE_HS_MASK; else mode &= ~TWSX_MODE_HS_MASK; - octeon_i2c_writeq_flush(mode, i2c->twsi_base + MODE(i2c)); + octeon_i2c_writeq_flush(mode, i2c->twsi_base + OCTEON_REG_MODE(i2c)); } } diff --git a/drivers/i2c/busses/i2c-octeon-core.h b/drivers/i2c/busses/i2c-octeon-core.h index 94c4401a4a567..39481e23e36fa 100644 --- a/drivers/i2c/busses/i2c-octeon-core.h +++ b/drivers/i2c/busses/i2c-octeon-core.h @@ -97,10 +97,10 @@ struct octeon_i2c_reg_offset { unsigned int mode; }; -#define SW_TWSI(x) (x->roff.sw_twsi) -#define TWSI_INT(x) (x->roff.twsi_int) -#define SW_TWSI_EXT(x) (x->roff.sw_twsi_ext) -#define MODE(x) ((x)->roff.mode) +#define OCTEON_REG_SW_TWSI(x) ((x)->roff.sw_twsi) +#define OCTEON_REG_TWSI_INT(x) ((x)->roff.twsi_int) +#define OCTEON_REG_SW_TWSI_EXT(x) ((x)->roff.sw_twsi_ext) +#define OCTEON_REG_MODE(x) ((x)->roff.mode) /* Set REFCLK_SRC and HS_MODE in TWSX_MODE register */ #define TWSX_MODE_REFCLK_SRC BIT(4) @@ -143,16 +143,16 @@ static inline void octeon_i2c_writeq_flush(u64 val, void __iomem *addr) * @eop_reg: Register selector * @data: Value to be written * - * The I2C core registers are accessed indirectly via the SW_TWSI CSR. + * The I2C core registers are accessed indirectly via the OCTEON_REG_SW_TWSI CSR. */ static inline void octeon_i2c_reg_write(struct octeon_i2c *i2c, u64 eop_reg, u8 data) { int tries = 1000; u64 tmp; - __raw_writeq(SW_TWSI_V | eop_reg | data, i2c->twsi_base + SW_TWSI(i2c)); + __raw_writeq(SW_TWSI_V | eop_reg | data, i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); do { - tmp = __raw_readq(i2c->twsi_base + SW_TWSI(i2c)); + tmp = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); if (--tries < 0) return; } while ((tmp & SW_TWSI_V) != 0); @@ -178,9 +178,9 @@ static inline int octeon_i2c_reg_read(struct octeon_i2c *i2c, u64 eop_reg, int tries = 1000; u64 tmp; - __raw_writeq(SW_TWSI_V | eop_reg | SW_TWSI_R, i2c->twsi_base + SW_TWSI(i2c)); + __raw_writeq(SW_TWSI_V | eop_reg | SW_TWSI_R, i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); do { - tmp = __raw_readq(i2c->twsi_base + SW_TWSI(i2c)); + tmp = __raw_readq(i2c->twsi_base + OCTEON_REG_SW_TWSI(i2c)); if (--tries < 0) { /* signal that the returned data is invalid */ if (error) @@ -200,24 +200,24 @@ static inline int octeon_i2c_reg_read(struct octeon_i2c *i2c, u64 eop_reg, octeon_i2c_reg_read(i2c, SW_TWSI_EOP_TWSI_STAT, NULL) /** - * octeon_i2c_read_int - read the TWSI_INT register + * octeon_i2c_read_int - read the OCTEON_REG_TWSI_INT register * @i2c: The struct octeon_i2c * * Returns the value of the register. */ static inline u64 octeon_i2c_read_int(struct octeon_i2c *i2c) { - return __raw_readq(i2c->twsi_base + TWSI_INT(i2c)); + return __raw_readq(i2c->twsi_base + OCTEON_REG_TWSI_INT(i2c)); } /** - * octeon_i2c_write_int - write the TWSI_INT register + * octeon_i2c_write_int - write the OCTEON_REG_TWSI_INT register * @i2c: The struct octeon_i2c * @data: Value to be written */ static inline void octeon_i2c_write_int(struct octeon_i2c *i2c, u64 data) { - octeon_i2c_writeq_flush(data, i2c->twsi_base + TWSI_INT(i2c)); + octeon_i2c_writeq_flush(data, i2c->twsi_base + OCTEON_REG_TWSI_INT(i2c)); } #define IS_LS_FREQ(twsi_freq) ((twsi_freq) <= 400000) From b9960b902f42c80ef436bf172666f20acbda32ac Mon Sep 17 00:00:00 2001 From: Suneel Garapati Date: Tue, 23 Apr 2024 00:46:07 -0700 Subject: [PATCH 1084/1702] i2c: octeon: Handle watchdog timeout Add watchdog timeout handling to cater to the unhandled warnings seen during validation on boards with different I2C slaves. This status code reflects the state that controller couldn't receive any response from slave while being in non-idle state and HW recommends to reset before any further bus access. Signed-off-by: Suneel Garapati Signed-off-by: Piyush Malgujar Acked-by: Andi Shyti Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-octeon-core.c | 8 ++++++++ drivers/i2c/busses/i2c-octeon-core.h | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/drivers/i2c/busses/i2c-octeon-core.c b/drivers/i2c/busses/i2c-octeon-core.c index 76a5ec100d303..5b7b942141e72 100644 --- a/drivers/i2c/busses/i2c-octeon-core.c +++ b/drivers/i2c/busses/i2c-octeon-core.c @@ -182,6 +182,7 @@ static int octeon_i2c_hlc_wait(struct octeon_i2c *i2c) static int octeon_i2c_check_status(struct octeon_i2c *i2c, int final_read) { u8 stat; + u64 mode; /* * This is ugly... in HLC mode the status is not in the status register @@ -244,6 +245,13 @@ static int octeon_i2c_check_status(struct octeon_i2c *i2c, int final_read) case STAT_RXADDR_NAK: case STAT_AD2W_NAK: return -ENXIO; + + case STAT_WDOG_TOUT: + mode = __raw_readq(i2c->twsi_base + OCTEON_REG_MODE(i2c)); + /* Set BUS_MON_RST to reset bus monitor */ + mode |= BUS_MON_RST_MASK; + octeon_i2c_writeq_flush(mode, i2c->twsi_base + OCTEON_REG_MODE(i2c)); + return -EIO; default: dev_err(i2c->dev, "unhandled state: %d\n", stat); return -EIO; diff --git a/drivers/i2c/busses/i2c-octeon-core.h b/drivers/i2c/busses/i2c-octeon-core.h index 39481e23e36fa..7af01864da752 100644 --- a/drivers/i2c/busses/i2c-octeon-core.h +++ b/drivers/i2c/busses/i2c-octeon-core.h @@ -73,6 +73,7 @@ #define STAT_SLAVE_ACK 0xC8 #define STAT_AD2W_ACK 0xD0 #define STAT_AD2W_NAK 0xD8 +#define STAT_WDOG_TOUT 0xF0 #define STAT_IDLE 0xF8 /* TWSI_INT values */ @@ -107,6 +108,9 @@ struct octeon_i2c_reg_offset { #define TWSX_MODE_HS_MODE BIT(0) #define TWSX_MODE_HS_MASK (TWSX_MODE_REFCLK_SRC | TWSX_MODE_HS_MODE) +/* Set BUS_MON_RST to reset bus monitor */ +#define BUS_MON_RST_MASK BIT(3) + struct octeon_i2c { wait_queue_head_t queue; struct i2c_adapter adap; From 53e3d528ba723c8fd5bf968f517c9bc6a369f3ea Mon Sep 17 00:00:00 2001 From: Piyush Malgujar Date: Tue, 23 Apr 2024 00:46:08 -0700 Subject: [PATCH 1085/1702] i2c: thunderx: Adding ioclk support Read the ioclk property as reference clock if sclk not present in acpi table to make it SOC agnostic. In case, it's not populated from dts/acpi table, use the default clock of 800 MHz which is optimal in either case of sclk/ioclk. Signed-off-by: Piyush Malgujar Acked-by: Andi Shyti Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-thunderx-pcidrv.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/i2c/busses/i2c-thunderx-pcidrv.c b/drivers/i2c/busses/i2c-thunderx-pcidrv.c index 31f11b77ab663..32d0e3930b675 100644 --- a/drivers/i2c/busses/i2c-thunderx-pcidrv.c +++ b/drivers/i2c/busses/i2c-thunderx-pcidrv.c @@ -27,7 +27,7 @@ #define PCI_DEVICE_ID_THUNDER_TWSI 0xa012 -#define SYS_FREQ_DEFAULT 700000000 +#define SYS_FREQ_DEFAULT 800000000 #define OTX2_REF_FREQ_DEFAULT 100000000 #define TWSI_INT_ENA_W1C 0x1028 @@ -100,7 +100,8 @@ static void thunder_i2c_clock_enable(struct device *dev, struct octeon_i2c *i2c) i2c->sys_freq = clk_get_rate(i2c->clk); } else { /* ACPI */ - device_property_read_u32(dev, "sclk", &i2c->sys_freq); + if (device_property_read_u32(dev, "sclk", &i2c->sys_freq)) + device_property_read_u32(dev, "ioclk", &i2c->sys_freq); } skip: From 8525205fd51f543cdf92f6c9aa65a9720cda9513 Mon Sep 17 00:00:00 2001 From: Hans Hu Date: Mon, 8 Apr 2024 10:54:43 +0800 Subject: [PATCH 1086/1702] i2c: wmt: create wmt_i2c_init for general init Some common initialization actions are put in the function wmt_i2c_init(), which is convenient to share with zhaoxin. Reviewed-by: Andi Shyti Reviewed-by: Wolfram Sang Signed-off-by: Hans Hu Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-wmt.c | 65 ++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/drivers/i2c/busses/i2c-wmt.c b/drivers/i2c/busses/i2c-wmt.c index 198afee5233c3..76ede433f65c9 100644 --- a/drivers/i2c/busses/i2c-wmt.c +++ b/drivers/i2c/busses/i2c-wmt.c @@ -109,7 +109,7 @@ static int wmt_check_status(struct wmt_i2c_dev *i2c_dev) unsigned long wait_result; wait_result = wait_for_completion_timeout(&i2c_dev->complete, - msecs_to_jiffies(500)); + msecs_to_jiffies(500)); if (!wait_result) return -ETIMEDOUT; @@ -286,6 +286,38 @@ static irqreturn_t wmt_i2c_isr(int irq, void *data) return IRQ_HANDLED; } +static int wmt_i2c_init(struct platform_device *pdev, struct wmt_i2c_dev **pi2c_dev) +{ + int err; + struct wmt_i2c_dev *i2c_dev; + struct device_node *np = pdev->dev.of_node; + + i2c_dev = devm_kzalloc(&pdev->dev, sizeof(*i2c_dev), GFP_KERNEL); + if (!i2c_dev) + return -ENOMEM; + + i2c_dev->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); + if (IS_ERR(i2c_dev->base)) + return PTR_ERR(i2c_dev->base); + + i2c_dev->irq = irq_of_parse_and_map(np, 0); + if (!i2c_dev->irq) + return -EINVAL; + + err = devm_request_irq(&pdev->dev, i2c_dev->irq, wmt_i2c_isr, + 0, pdev->name, i2c_dev); + if (err) + return dev_err_probe(&pdev->dev, err, + "failed to request irq %i\n", i2c_dev->irq); + + i2c_dev->dev = &pdev->dev; + init_completion(&i2c_dev->complete); + platform_set_drvdata(pdev, i2c_dev); + + *pi2c_dev = i2c_dev; + return 0; +} + static int wmt_i2c_reset_hardware(struct wmt_i2c_dev *i2c_dev) { int err; @@ -327,19 +359,9 @@ static int wmt_i2c_probe(struct platform_device *pdev) int err; u32 clk_rate; - i2c_dev = devm_kzalloc(&pdev->dev, sizeof(*i2c_dev), GFP_KERNEL); - if (!i2c_dev) - return -ENOMEM; - - i2c_dev->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); - if (IS_ERR(i2c_dev->base)) - return PTR_ERR(i2c_dev->base); - - i2c_dev->irq = irq_of_parse_and_map(np, 0); - if (!i2c_dev->irq) { - dev_err(&pdev->dev, "irq missing or invalid\n"); - return -EINVAL; - } + err = wmt_i2c_init(pdev, &i2c_dev); + if (err) + return err; i2c_dev->clk = of_clk_get(np, 0); if (IS_ERR(i2c_dev->clk)) { @@ -348,18 +370,9 @@ static int wmt_i2c_probe(struct platform_device *pdev) } err = of_property_read_u32(np, "clock-frequency", &clk_rate); - if (!err && (clk_rate == I2C_MAX_FAST_MODE_FREQ)) + if (!err && clk_rate == I2C_MAX_FAST_MODE_FREQ) i2c_dev->tcr = TCR_FAST_MODE; - i2c_dev->dev = &pdev->dev; - - err = devm_request_irq(&pdev->dev, i2c_dev->irq, wmt_i2c_isr, 0, - "i2c", i2c_dev); - if (err) { - dev_err(&pdev->dev, "failed to request irq %i\n", i2c_dev->irq); - return err; - } - adap = &i2c_dev->adapter; i2c_set_adapdata(adap, i2c_dev); strscpy(adap->name, "WMT I2C adapter", sizeof(adap->name)); @@ -368,8 +381,6 @@ static int wmt_i2c_probe(struct platform_device *pdev) adap->dev.parent = &pdev->dev; adap->dev.of_node = pdev->dev.of_node; - init_completion(&i2c_dev->complete); - err = wmt_i2c_reset_hardware(i2c_dev); if (err) { dev_err(&pdev->dev, "error initializing hardware\n"); @@ -380,8 +391,6 @@ static int wmt_i2c_probe(struct platform_device *pdev) if (err) goto err_disable_clk; - platform_set_drvdata(pdev, i2c_dev); - return 0; err_disable_clk: From 5acd48fa72ad16b824a9117254ce48ed69c914e8 Mon Sep 17 00:00:00 2001 From: Hans Hu Date: Mon, 8 Apr 2024 10:54:44 +0800 Subject: [PATCH 1087/1702] i2c: wmt: split out common files Since the I2C IP of both wmt and zhaoxin originates from VIA, it is better to separate the common code first. The common driver is named as i2c-viai2c-common.c. Old i2c-wmt.c renamed to i2c-viai2c-wmt.c. The MAINTAINERS file will be updated accordingly in upcoming commits. Reviewed-by: Wolfram Sang Signed-off-by: Hans Hu Signed-off-by: Andi Shyti --- MAINTAINERS | 2 +- drivers/i2c/busses/Makefile | 1 + drivers/i2c/busses/i2c-viai2c-common.c | 221 +++++++++++++ drivers/i2c/busses/i2c-viai2c-common.h | 71 ++++ drivers/i2c/busses/i2c-viai2c-wmt.c | 148 +++++++++ drivers/i2c/busses/i2c-wmt.c | 430 ------------------------- 6 files changed, 442 insertions(+), 431 deletions(-) create mode 100644 drivers/i2c/busses/i2c-viai2c-common.c create mode 100644 drivers/i2c/busses/i2c-viai2c-common.h create mode 100644 drivers/i2c/busses/i2c-viai2c-wmt.c delete mode 100644 drivers/i2c/busses/i2c-wmt.c diff --git a/MAINTAINERS b/MAINTAINERS index 75554f93e555b..a3491d1aa7c37 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3017,7 +3017,7 @@ S: Orphan F: Documentation/devicetree/bindings/i2c/i2c-wmt.txt F: arch/arm/mach-vt8500/ F: drivers/clocksource/timer-vt8500.c -F: drivers/i2c/busses/i2c-wmt.c +F: drivers/i2c/busses/i2c-viai2c-wmt.c F: drivers/mmc/host/wmt-sdmmc.c F: drivers/pwm/pwm-vt8500.c F: drivers/rtc/rtc-vt8500.c diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile index aa0ee8ecd6f2f..63c7bbad8134f 100644 --- a/drivers/i2c/busses/Makefile +++ b/drivers/i2c/busses/Makefile @@ -118,6 +118,7 @@ obj-$(CONFIG_I2C_TEGRA_BPMP) += i2c-tegra-bpmp.o obj-$(CONFIG_I2C_UNIPHIER) += i2c-uniphier.o obj-$(CONFIG_I2C_UNIPHIER_F) += i2c-uniphier-f.o obj-$(CONFIG_I2C_VERSATILE) += i2c-versatile.o +i2c-wmt-objs := i2c-viai2c-wmt.o i2c-viai2c-common.o obj-$(CONFIG_I2C_WMT) += i2c-wmt.o i2c-octeon-objs := i2c-octeon-core.o i2c-octeon-platdrv.o obj-$(CONFIG_I2C_OCTEON) += i2c-octeon.o diff --git a/drivers/i2c/busses/i2c-viai2c-common.c b/drivers/i2c/busses/i2c-viai2c-common.c new file mode 100644 index 0000000000000..b36a5d902990e --- /dev/null +++ b/drivers/i2c/busses/i2c-viai2c-common.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include +#include "i2c-viai2c-common.h" + +int wmt_i2c_wait_bus_not_busy(struct wmt_i2c_dev *i2c_dev) +{ + unsigned long timeout; + + timeout = jiffies + WMT_I2C_TIMEOUT; + while (!(readw(i2c_dev->base + REG_CSR) & CSR_READY_MASK)) { + if (time_after(jiffies, timeout)) { + dev_warn(i2c_dev->dev, "timeout waiting for bus ready\n"); + return -EBUSY; + } + msleep(20); + } + + return 0; +} + +int wmt_check_status(struct wmt_i2c_dev *i2c_dev) +{ + int ret = 0; + unsigned long wait_result; + + wait_result = wait_for_completion_timeout(&i2c_dev->complete, + msecs_to_jiffies(500)); + if (!wait_result) + return -ETIMEDOUT; + + if (i2c_dev->cmd_status & ISR_NACK_ADDR) + ret = -EIO; + + if (i2c_dev->cmd_status & ISR_SCL_TIMEOUT) + ret = -ETIMEDOUT; + + return ret; +} + +static int wmt_i2c_write(struct wmt_i2c_dev *i2c_dev, struct i2c_msg *pmsg, int last) +{ + u16 val, tcr_val = i2c_dev->tcr; + int ret; + int xfer_len = 0; + + if (pmsg->len == 0) { + /* + * We still need to run through the while (..) once, so + * start at -1 and break out early from the loop + */ + xfer_len = -1; + writew(0, i2c_dev->base + REG_CDR); + } else { + writew(pmsg->buf[0] & 0xFF, i2c_dev->base + REG_CDR); + } + + if (!(pmsg->flags & I2C_M_NOSTART)) { + val = readw(i2c_dev->base + REG_CR); + val &= ~CR_TX_END; + val |= CR_CPU_RDY; + writew(val, i2c_dev->base + REG_CR); + } + + reinit_completion(&i2c_dev->complete); + + tcr_val |= (TCR_MASTER_WRITE | (pmsg->addr & TCR_SLAVE_ADDR_MASK)); + + writew(tcr_val, i2c_dev->base + REG_TCR); + + if (pmsg->flags & I2C_M_NOSTART) { + val = readw(i2c_dev->base + REG_CR); + val |= CR_CPU_RDY; + writew(val, i2c_dev->base + REG_CR); + } + + while (xfer_len < pmsg->len) { + ret = wmt_check_status(i2c_dev); + if (ret) + return ret; + + xfer_len++; + + val = readw(i2c_dev->base + REG_CSR); + if ((val & CSR_RCV_ACK_MASK) == CSR_RCV_NOT_ACK) { + dev_dbg(i2c_dev->dev, "write RCV NACK error\n"); + return -EIO; + } + + if (pmsg->len == 0) { + val = CR_TX_END | CR_CPU_RDY | CR_ENABLE; + writew(val, i2c_dev->base + REG_CR); + break; + } + + if (xfer_len == pmsg->len) { + if (last != 1) + writew(CR_ENABLE, i2c_dev->base + REG_CR); + } else { + writew(pmsg->buf[xfer_len] & 0xFF, i2c_dev->base + + REG_CDR); + writew(CR_CPU_RDY | CR_ENABLE, i2c_dev->base + REG_CR); + } + } + + return 0; +} + +static int wmt_i2c_read(struct wmt_i2c_dev *i2c_dev, struct i2c_msg *pmsg) +{ + u16 val, tcr_val = i2c_dev->tcr; + int ret; + u32 xfer_len = 0; + + val = readw(i2c_dev->base + REG_CR); + val &= ~(CR_TX_END | CR_TX_NEXT_NO_ACK); + + if (!(pmsg->flags & I2C_M_NOSTART)) + val |= CR_CPU_RDY; + + if (pmsg->len == 1) + val |= CR_TX_NEXT_NO_ACK; + + writew(val, i2c_dev->base + REG_CR); + + reinit_completion(&i2c_dev->complete); + + tcr_val |= TCR_MASTER_READ | (pmsg->addr & TCR_SLAVE_ADDR_MASK); + + writew(tcr_val, i2c_dev->base + REG_TCR); + + if (pmsg->flags & I2C_M_NOSTART) { + val = readw(i2c_dev->base + REG_CR); + val |= CR_CPU_RDY; + writew(val, i2c_dev->base + REG_CR); + } + + while (xfer_len < pmsg->len) { + ret = wmt_check_status(i2c_dev); + if (ret) + return ret; + + pmsg->buf[xfer_len] = readw(i2c_dev->base + REG_CDR) >> 8; + xfer_len++; + + val = readw(i2c_dev->base + REG_CR) | CR_CPU_RDY; + if (xfer_len == pmsg->len - 1) + val |= CR_TX_NEXT_NO_ACK; + writew(val, i2c_dev->base + REG_CR); + } + + return 0; +} + +int wmt_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) +{ + struct i2c_msg *pmsg; + int i; + int ret = 0; + struct wmt_i2c_dev *i2c_dev = i2c_get_adapdata(adap); + + for (i = 0; ret >= 0 && i < num; i++) { + pmsg = &msgs[i]; + if (!(pmsg->flags & I2C_M_NOSTART)) { + ret = wmt_i2c_wait_bus_not_busy(i2c_dev); + if (ret < 0) + return ret; + } + + if (pmsg->flags & I2C_M_RD) + ret = wmt_i2c_read(i2c_dev, pmsg); + else + ret = wmt_i2c_write(i2c_dev, pmsg, (i + 1) == num); + } + + return (ret < 0) ? ret : i; +} + +static irqreturn_t wmt_i2c_isr(int irq, void *data) +{ + struct wmt_i2c_dev *i2c_dev = data; + + /* save the status and write-clear it */ + i2c_dev->cmd_status = readw(i2c_dev->base + REG_ISR); + writew(i2c_dev->cmd_status, i2c_dev->base + REG_ISR); + + complete(&i2c_dev->complete); + + return IRQ_HANDLED; +} + +int wmt_i2c_init(struct platform_device *pdev, struct wmt_i2c_dev **pi2c_dev) +{ + int err; + struct wmt_i2c_dev *i2c_dev; + struct device_node *np = pdev->dev.of_node; + + i2c_dev = devm_kzalloc(&pdev->dev, sizeof(*i2c_dev), GFP_KERNEL); + if (!i2c_dev) + return -ENOMEM; + + i2c_dev->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); + if (IS_ERR(i2c_dev->base)) + return PTR_ERR(i2c_dev->base); + + i2c_dev->irq = irq_of_parse_and_map(np, 0); + if (!i2c_dev->irq) + return -EINVAL; + + err = devm_request_irq(&pdev->dev, i2c_dev->irq, wmt_i2c_isr, + 0, pdev->name, i2c_dev); + if (err) + return dev_err_probe(&pdev->dev, err, + "failed to request irq %i\n", i2c_dev->irq); + + i2c_dev->dev = &pdev->dev; + init_completion(&i2c_dev->complete); + platform_set_drvdata(pdev, i2c_dev); + + *pi2c_dev = i2c_dev; + return 0; +} diff --git a/drivers/i2c/busses/i2c-viai2c-common.h b/drivers/i2c/busses/i2c-viai2c-common.h new file mode 100644 index 0000000000000..fcff8e4456eb8 --- /dev/null +++ b/drivers/i2c/busses/i2c-viai2c-common.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __I2C_VIAI2C_COMMON_H_ +#define __I2C_VIAI2C_COMMON_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define REG_CR 0x00 +#define REG_TCR 0x02 +#define REG_CSR 0x04 +#define REG_ISR 0x06 +#define REG_IMR 0x08 +#define REG_CDR 0x0A +#define REG_TR 0x0C +#define REG_MCR 0x0E + +/* REG_CR Bit fields */ +#define CR_TX_NEXT_ACK 0x0000 +#define CR_ENABLE 0x0001 +#define CR_TX_NEXT_NO_ACK 0x0002 +#define CR_TX_END 0x0004 +#define CR_CPU_RDY 0x0008 +#define SLAV_MODE_SEL 0x8000 + +/* REG_TCR Bit fields */ +#define TCR_STANDARD_MODE 0x0000 +#define TCR_MASTER_WRITE 0x0000 +#define TCR_HS_MODE 0x2000 +#define TCR_MASTER_READ 0x4000 +#define TCR_FAST_MODE 0x8000 +#define TCR_SLAVE_ADDR_MASK 0x007F + +/* REG_ISR Bit fields */ +#define ISR_NACK_ADDR 0x0001 +#define ISR_BYTE_END 0x0002 +#define ISR_SCL_TIMEOUT 0x0004 +#define ISR_WRITE_ALL 0x0007 + +/* REG_IMR Bit fields */ +#define IMR_ENABLE_ALL 0x0007 + +/* REG_CSR Bit fields */ +#define CSR_RCV_NOT_ACK 0x0001 +#define CSR_RCV_ACK_MASK 0x0001 +#define CSR_READY_MASK 0x0002 + +#define WMT_I2C_TIMEOUT (msecs_to_jiffies(1000)) + +struct wmt_i2c_dev { + struct i2c_adapter adapter; + struct completion complete; + struct device *dev; + void __iomem *base; + struct clk *clk; + u16 tcr; + int irq; + u16 cmd_status; +}; + +int wmt_i2c_wait_bus_not_busy(struct wmt_i2c_dev *i2c_dev); +int wmt_check_status(struct wmt_i2c_dev *i2c_dev); +int wmt_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num); +int wmt_i2c_init(struct platform_device *pdev, struct wmt_i2c_dev **pi2c_dev); + +#endif diff --git a/drivers/i2c/busses/i2c-viai2c-wmt.c b/drivers/i2c/busses/i2c-viai2c-wmt.c new file mode 100644 index 0000000000000..3125374cc2c0d --- /dev/null +++ b/drivers/i2c/busses/i2c-viai2c-wmt.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Wondermedia I2C Master Mode Driver + * + * Copyright (C) 2012 Tony Prisk + * + * Derived from GPLv2+ licensed source: + * - Copyright (C) 2008 WonderMedia Technologies, Inc. + */ + +#include +#include +#include +#include "i2c-viai2c-common.h" + +#define REG_SLAVE_CR 0x10 +#define REG_SLAVE_SR 0x12 +#define REG_SLAVE_ISR 0x14 +#define REG_SLAVE_IMR 0x16 +#define REG_SLAVE_DR 0x18 +#define REG_SLAVE_TR 0x1A + +/* REG_TR */ +#define SCL_TIMEOUT(x) (((x) & 0xFF) << 8) +#define TR_STD 0x0064 +#define TR_HS 0x0019 + +/* REG_MCR */ +#define MCR_APB_96M 7 +#define MCR_APB_166M 12 + +static u32 wmt_i2c_func(struct i2c_adapter *adap) +{ + return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL | I2C_FUNC_NOSTART; +} + +static const struct i2c_algorithm wmt_i2c_algo = { + .master_xfer = wmt_i2c_xfer, + .functionality = wmt_i2c_func, +}; + +static int wmt_i2c_reset_hardware(struct wmt_i2c_dev *i2c_dev) +{ + int err; + + err = clk_prepare_enable(i2c_dev->clk); + if (err) { + dev_err(i2c_dev->dev, "failed to enable clock\n"); + return err; + } + + err = clk_set_rate(i2c_dev->clk, 20000000); + if (err) { + dev_err(i2c_dev->dev, "failed to set clock = 20Mhz\n"); + clk_disable_unprepare(i2c_dev->clk); + return err; + } + + writew(0, i2c_dev->base + REG_CR); + writew(MCR_APB_166M, i2c_dev->base + REG_MCR); + writew(ISR_WRITE_ALL, i2c_dev->base + REG_ISR); + writew(IMR_ENABLE_ALL, i2c_dev->base + REG_IMR); + writew(CR_ENABLE, i2c_dev->base + REG_CR); + readw(i2c_dev->base + REG_CSR); /* read clear */ + writew(ISR_WRITE_ALL, i2c_dev->base + REG_ISR); + + if (i2c_dev->tcr == TCR_FAST_MODE) + writew(SCL_TIMEOUT(128) | TR_HS, i2c_dev->base + REG_TR); + else + writew(SCL_TIMEOUT(128) | TR_STD, i2c_dev->base + REG_TR); + + return 0; +} + +static int wmt_i2c_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct wmt_i2c_dev *i2c_dev; + struct i2c_adapter *adap; + int err; + u32 clk_rate; + + err = wmt_i2c_init(pdev, &i2c_dev); + if (err) + return err; + + i2c_dev->clk = of_clk_get(np, 0); + if (IS_ERR(i2c_dev->clk)) { + dev_err(&pdev->dev, "unable to request clock\n"); + return PTR_ERR(i2c_dev->clk); + } + + err = of_property_read_u32(np, "clock-frequency", &clk_rate); + if (!err && clk_rate == I2C_MAX_FAST_MODE_FREQ) + i2c_dev->tcr = TCR_FAST_MODE; + + adap = &i2c_dev->adapter; + i2c_set_adapdata(adap, i2c_dev); + strscpy(adap->name, "WMT I2C adapter", sizeof(adap->name)); + adap->owner = THIS_MODULE; + adap->algo = &wmt_i2c_algo; + adap->dev.parent = &pdev->dev; + adap->dev.of_node = pdev->dev.of_node; + + err = wmt_i2c_reset_hardware(i2c_dev); + if (err) { + dev_err(&pdev->dev, "error initializing hardware\n"); + return err; + } + + err = i2c_add_adapter(adap); + if (err) + /* wmt_i2c_reset_hardware() enables i2c_dev->clk */ + clk_disable_unprepare(i2c_dev->clk); + + return err; +} + +static void wmt_i2c_remove(struct platform_device *pdev) +{ + struct wmt_i2c_dev *i2c_dev = platform_get_drvdata(pdev); + + /* Disable interrupts, clock and delete adapter */ + writew(0, i2c_dev->base + REG_IMR); + clk_disable_unprepare(i2c_dev->clk); + i2c_del_adapter(&i2c_dev->adapter); +} + +static const struct of_device_id wmt_i2c_dt_ids[] = { + { .compatible = "wm,wm8505-i2c" }, + { /* Sentinel */ }, +}; + +static struct platform_driver wmt_i2c_driver = { + .probe = wmt_i2c_probe, + .remove_new = wmt_i2c_remove, + .driver = { + .name = "wmt-i2c", + .of_match_table = wmt_i2c_dt_ids, + }, +}; + +module_platform_driver(wmt_i2c_driver); + +MODULE_DESCRIPTION("Wondermedia I2C master-mode bus adapter"); +MODULE_AUTHOR("Tony Prisk "); +MODULE_LICENSE("GPL"); +MODULE_DEVICE_TABLE(of, wmt_i2c_dt_ids); diff --git a/drivers/i2c/busses/i2c-wmt.c b/drivers/i2c/busses/i2c-wmt.c deleted file mode 100644 index 76ede433f65c9..0000000000000 --- a/drivers/i2c/busses/i2c-wmt.c +++ /dev/null @@ -1,430 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Wondermedia I2C Master Mode Driver - * - * Copyright (C) 2012 Tony Prisk - * - * Derived from GPLv2+ licensed source: - * - Copyright (C) 2008 WonderMedia Technologies, Inc. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define REG_CR 0x00 -#define REG_TCR 0x02 -#define REG_CSR 0x04 -#define REG_ISR 0x06 -#define REG_IMR 0x08 -#define REG_CDR 0x0A -#define REG_TR 0x0C -#define REG_MCR 0x0E -#define REG_SLAVE_CR 0x10 -#define REG_SLAVE_SR 0x12 -#define REG_SLAVE_ISR 0x14 -#define REG_SLAVE_IMR 0x16 -#define REG_SLAVE_DR 0x18 -#define REG_SLAVE_TR 0x1A - -/* REG_CR Bit fields */ -#define CR_TX_NEXT_ACK 0x0000 -#define CR_ENABLE 0x0001 -#define CR_TX_NEXT_NO_ACK 0x0002 -#define CR_TX_END 0x0004 -#define CR_CPU_RDY 0x0008 -#define SLAV_MODE_SEL 0x8000 - -/* REG_TCR Bit fields */ -#define TCR_STANDARD_MODE 0x0000 -#define TCR_MASTER_WRITE 0x0000 -#define TCR_HS_MODE 0x2000 -#define TCR_MASTER_READ 0x4000 -#define TCR_FAST_MODE 0x8000 -#define TCR_SLAVE_ADDR_MASK 0x007F - -/* REG_ISR Bit fields */ -#define ISR_NACK_ADDR 0x0001 -#define ISR_BYTE_END 0x0002 -#define ISR_SCL_TIMEOUT 0x0004 -#define ISR_WRITE_ALL 0x0007 - -/* REG_IMR Bit fields */ -#define IMR_ENABLE_ALL 0x0007 - -/* REG_CSR Bit fields */ -#define CSR_RCV_NOT_ACK 0x0001 -#define CSR_RCV_ACK_MASK 0x0001 -#define CSR_READY_MASK 0x0002 - -/* REG_TR */ -#define SCL_TIMEOUT(x) (((x) & 0xFF) << 8) -#define TR_STD 0x0064 -#define TR_HS 0x0019 - -/* REG_MCR */ -#define MCR_APB_96M 7 -#define MCR_APB_166M 12 - -#define WMT_I2C_TIMEOUT (msecs_to_jiffies(1000)) - -struct wmt_i2c_dev { - struct i2c_adapter adapter; - struct completion complete; - struct device *dev; - void __iomem *base; - struct clk *clk; - u16 tcr; - int irq; - u16 cmd_status; -}; - -static int wmt_i2c_wait_bus_not_busy(struct wmt_i2c_dev *i2c_dev) -{ - unsigned long timeout; - - timeout = jiffies + WMT_I2C_TIMEOUT; - while (!(readw(i2c_dev->base + REG_CSR) & CSR_READY_MASK)) { - if (time_after(jiffies, timeout)) { - dev_warn(i2c_dev->dev, "timeout waiting for bus ready\n"); - return -EBUSY; - } - msleep(20); - } - - return 0; -} - -static int wmt_check_status(struct wmt_i2c_dev *i2c_dev) -{ - int ret = 0; - unsigned long wait_result; - - wait_result = wait_for_completion_timeout(&i2c_dev->complete, - msecs_to_jiffies(500)); - if (!wait_result) - return -ETIMEDOUT; - - if (i2c_dev->cmd_status & ISR_NACK_ADDR) - ret = -EIO; - - if (i2c_dev->cmd_status & ISR_SCL_TIMEOUT) - ret = -ETIMEDOUT; - - return ret; -} - -static int wmt_i2c_write(struct wmt_i2c_dev *i2c_dev, struct i2c_msg *pmsg, - int last) -{ - u16 val, tcr_val = i2c_dev->tcr; - int ret; - int xfer_len = 0; - - if (pmsg->len == 0) { - /* - * We still need to run through the while (..) once, so - * start at -1 and break out early from the loop - */ - xfer_len = -1; - writew(0, i2c_dev->base + REG_CDR); - } else { - writew(pmsg->buf[0] & 0xFF, i2c_dev->base + REG_CDR); - } - - if (!(pmsg->flags & I2C_M_NOSTART)) { - val = readw(i2c_dev->base + REG_CR); - val &= ~CR_TX_END; - val |= CR_CPU_RDY; - writew(val, i2c_dev->base + REG_CR); - } - - reinit_completion(&i2c_dev->complete); - - tcr_val |= (TCR_MASTER_WRITE | (pmsg->addr & TCR_SLAVE_ADDR_MASK)); - - writew(tcr_val, i2c_dev->base + REG_TCR); - - if (pmsg->flags & I2C_M_NOSTART) { - val = readw(i2c_dev->base + REG_CR); - val |= CR_CPU_RDY; - writew(val, i2c_dev->base + REG_CR); - } - - while (xfer_len < pmsg->len) { - ret = wmt_check_status(i2c_dev); - if (ret) - return ret; - - xfer_len++; - - val = readw(i2c_dev->base + REG_CSR); - if ((val & CSR_RCV_ACK_MASK) == CSR_RCV_NOT_ACK) { - dev_dbg(i2c_dev->dev, "write RCV NACK error\n"); - return -EIO; - } - - if (pmsg->len == 0) { - val = CR_TX_END | CR_CPU_RDY | CR_ENABLE; - writew(val, i2c_dev->base + REG_CR); - break; - } - - if (xfer_len == pmsg->len) { - if (last != 1) - writew(CR_ENABLE, i2c_dev->base + REG_CR); - } else { - writew(pmsg->buf[xfer_len] & 0xFF, i2c_dev->base + - REG_CDR); - writew(CR_CPU_RDY | CR_ENABLE, i2c_dev->base + REG_CR); - } - } - - return 0; -} - -static int wmt_i2c_read(struct wmt_i2c_dev *i2c_dev, struct i2c_msg *pmsg) -{ - u16 val, tcr_val = i2c_dev->tcr; - int ret; - u32 xfer_len = 0; - - val = readw(i2c_dev->base + REG_CR); - val &= ~(CR_TX_END | CR_TX_NEXT_NO_ACK); - - if (!(pmsg->flags & I2C_M_NOSTART)) - val |= CR_CPU_RDY; - - if (pmsg->len == 1) - val |= CR_TX_NEXT_NO_ACK; - - writew(val, i2c_dev->base + REG_CR); - - reinit_completion(&i2c_dev->complete); - - tcr_val |= TCR_MASTER_READ | (pmsg->addr & TCR_SLAVE_ADDR_MASK); - - writew(tcr_val, i2c_dev->base + REG_TCR); - - if (pmsg->flags & I2C_M_NOSTART) { - val = readw(i2c_dev->base + REG_CR); - val |= CR_CPU_RDY; - writew(val, i2c_dev->base + REG_CR); - } - - while (xfer_len < pmsg->len) { - ret = wmt_check_status(i2c_dev); - if (ret) - return ret; - - pmsg->buf[xfer_len] = readw(i2c_dev->base + REG_CDR) >> 8; - xfer_len++; - - val = readw(i2c_dev->base + REG_CR) | CR_CPU_RDY; - if (xfer_len == pmsg->len - 1) - val |= CR_TX_NEXT_NO_ACK; - writew(val, i2c_dev->base + REG_CR); - } - - return 0; -} - -static int wmt_i2c_xfer(struct i2c_adapter *adap, - struct i2c_msg msgs[], - int num) -{ - struct i2c_msg *pmsg; - int i; - int ret = 0; - struct wmt_i2c_dev *i2c_dev = i2c_get_adapdata(adap); - - for (i = 0; ret >= 0 && i < num; i++) { - pmsg = &msgs[i]; - if (!(pmsg->flags & I2C_M_NOSTART)) { - ret = wmt_i2c_wait_bus_not_busy(i2c_dev); - if (ret < 0) - return ret; - } - - if (pmsg->flags & I2C_M_RD) - ret = wmt_i2c_read(i2c_dev, pmsg); - else - ret = wmt_i2c_write(i2c_dev, pmsg, (i + 1) == num); - } - - return (ret < 0) ? ret : i; -} - -static u32 wmt_i2c_func(struct i2c_adapter *adap) -{ - return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL | I2C_FUNC_NOSTART; -} - -static const struct i2c_algorithm wmt_i2c_algo = { - .master_xfer = wmt_i2c_xfer, - .functionality = wmt_i2c_func, -}; - -static irqreturn_t wmt_i2c_isr(int irq, void *data) -{ - struct wmt_i2c_dev *i2c_dev = data; - - /* save the status and write-clear it */ - i2c_dev->cmd_status = readw(i2c_dev->base + REG_ISR); - writew(i2c_dev->cmd_status, i2c_dev->base + REG_ISR); - - complete(&i2c_dev->complete); - - return IRQ_HANDLED; -} - -static int wmt_i2c_init(struct platform_device *pdev, struct wmt_i2c_dev **pi2c_dev) -{ - int err; - struct wmt_i2c_dev *i2c_dev; - struct device_node *np = pdev->dev.of_node; - - i2c_dev = devm_kzalloc(&pdev->dev, sizeof(*i2c_dev), GFP_KERNEL); - if (!i2c_dev) - return -ENOMEM; - - i2c_dev->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); - if (IS_ERR(i2c_dev->base)) - return PTR_ERR(i2c_dev->base); - - i2c_dev->irq = irq_of_parse_and_map(np, 0); - if (!i2c_dev->irq) - return -EINVAL; - - err = devm_request_irq(&pdev->dev, i2c_dev->irq, wmt_i2c_isr, - 0, pdev->name, i2c_dev); - if (err) - return dev_err_probe(&pdev->dev, err, - "failed to request irq %i\n", i2c_dev->irq); - - i2c_dev->dev = &pdev->dev; - init_completion(&i2c_dev->complete); - platform_set_drvdata(pdev, i2c_dev); - - *pi2c_dev = i2c_dev; - return 0; -} - -static int wmt_i2c_reset_hardware(struct wmt_i2c_dev *i2c_dev) -{ - int err; - - err = clk_prepare_enable(i2c_dev->clk); - if (err) { - dev_err(i2c_dev->dev, "failed to enable clock\n"); - return err; - } - - err = clk_set_rate(i2c_dev->clk, 20000000); - if (err) { - dev_err(i2c_dev->dev, "failed to set clock = 20Mhz\n"); - clk_disable_unprepare(i2c_dev->clk); - return err; - } - - writew(0, i2c_dev->base + REG_CR); - writew(MCR_APB_166M, i2c_dev->base + REG_MCR); - writew(ISR_WRITE_ALL, i2c_dev->base + REG_ISR); - writew(IMR_ENABLE_ALL, i2c_dev->base + REG_IMR); - writew(CR_ENABLE, i2c_dev->base + REG_CR); - readw(i2c_dev->base + REG_CSR); /* read clear */ - writew(ISR_WRITE_ALL, i2c_dev->base + REG_ISR); - - if (i2c_dev->tcr == TCR_FAST_MODE) - writew(SCL_TIMEOUT(128) | TR_HS, i2c_dev->base + REG_TR); - else - writew(SCL_TIMEOUT(128) | TR_STD, i2c_dev->base + REG_TR); - - return 0; -} - -static int wmt_i2c_probe(struct platform_device *pdev) -{ - struct device_node *np = pdev->dev.of_node; - struct wmt_i2c_dev *i2c_dev; - struct i2c_adapter *adap; - int err; - u32 clk_rate; - - err = wmt_i2c_init(pdev, &i2c_dev); - if (err) - return err; - - i2c_dev->clk = of_clk_get(np, 0); - if (IS_ERR(i2c_dev->clk)) { - dev_err(&pdev->dev, "unable to request clock\n"); - return PTR_ERR(i2c_dev->clk); - } - - err = of_property_read_u32(np, "clock-frequency", &clk_rate); - if (!err && clk_rate == I2C_MAX_FAST_MODE_FREQ) - i2c_dev->tcr = TCR_FAST_MODE; - - adap = &i2c_dev->adapter; - i2c_set_adapdata(adap, i2c_dev); - strscpy(adap->name, "WMT I2C adapter", sizeof(adap->name)); - adap->owner = THIS_MODULE; - adap->algo = &wmt_i2c_algo; - adap->dev.parent = &pdev->dev; - adap->dev.of_node = pdev->dev.of_node; - - err = wmt_i2c_reset_hardware(i2c_dev); - if (err) { - dev_err(&pdev->dev, "error initializing hardware\n"); - return err; - } - - err = i2c_add_adapter(adap); - if (err) - goto err_disable_clk; - - return 0; - -err_disable_clk: - clk_disable_unprepare(i2c_dev->clk); - return err; -} - -static void wmt_i2c_remove(struct platform_device *pdev) -{ - struct wmt_i2c_dev *i2c_dev = platform_get_drvdata(pdev); - - /* Disable interrupts, clock and delete adapter */ - writew(0, i2c_dev->base + REG_IMR); - clk_disable_unprepare(i2c_dev->clk); - i2c_del_adapter(&i2c_dev->adapter); -} - -static const struct of_device_id wmt_i2c_dt_ids[] = { - { .compatible = "wm,wm8505-i2c" }, - { /* Sentinel */ }, -}; - -static struct platform_driver wmt_i2c_driver = { - .probe = wmt_i2c_probe, - .remove_new = wmt_i2c_remove, - .driver = { - .name = "wmt-i2c", - .of_match_table = wmt_i2c_dt_ids, - }, -}; - -module_platform_driver(wmt_i2c_driver); - -MODULE_DESCRIPTION("Wondermedia I2C master-mode bus adapter"); -MODULE_AUTHOR("Tony Prisk "); -MODULE_LICENSE("GPL"); -MODULE_DEVICE_TABLE(of, wmt_i2c_dt_ids); From 013fa161a4b584e416b98a31c6fda67753e9d7b8 Mon Sep 17 00:00:00 2001 From: Hans Hu Date: Mon, 8 Apr 2024 10:54:45 +0800 Subject: [PATCH 1088/1702] i2c: wmt: rename something 1. The I2C IP for both wmt and zhaoxin originates from VIA. Rename common registers, functions, and variable names to follow the VIAI2C_ and viai2c_ naming conventions for consistency and clarity. 2. rename i2c_dev to i2c, to shorten the length of a line. 3. rename wait_result to time_left, make it better to reflect the meaning of the value returned by wait_for_completion_timeout(). 4. remove TCR_MASTER_WRITE, its value is 0. Reviewed-by: Wolfram Sang Signed-off-by: Hans Hu Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-viai2c-common.c | 157 ++++++++++++------------- drivers/i2c/busses/i2c-viai2c-common.h | 70 ++++++----- drivers/i2c/busses/i2c-viai2c-wmt.c | 62 +++++----- 3 files changed, 143 insertions(+), 146 deletions(-) diff --git a/drivers/i2c/busses/i2c-viai2c-common.c b/drivers/i2c/busses/i2c-viai2c-common.c index b36a5d902990e..59901c53ad48c 100644 --- a/drivers/i2c/busses/i2c-viai2c-common.c +++ b/drivers/i2c/busses/i2c-viai2c-common.c @@ -2,14 +2,14 @@ #include #include "i2c-viai2c-common.h" -int wmt_i2c_wait_bus_not_busy(struct wmt_i2c_dev *i2c_dev) +int viai2c_wait_bus_not_busy(struct viai2c *i2c) { unsigned long timeout; - timeout = jiffies + WMT_I2C_TIMEOUT; - while (!(readw(i2c_dev->base + REG_CSR) & CSR_READY_MASK)) { + timeout = jiffies + VIAI2C_TIMEOUT; + while (!(readw(i2c->base + VIAI2C_REG_CSR) & VIAI2C_CSR_READY_MASK)) { if (time_after(jiffies, timeout)) { - dev_warn(i2c_dev->dev, "timeout waiting for bus ready\n"); + dev_warn(i2c->dev, "timeout waiting for bus ready\n"); return -EBUSY; } msleep(20); @@ -18,28 +18,28 @@ int wmt_i2c_wait_bus_not_busy(struct wmt_i2c_dev *i2c_dev) return 0; } -int wmt_check_status(struct wmt_i2c_dev *i2c_dev) +int viai2c_check_status(struct viai2c *i2c) { int ret = 0; - unsigned long wait_result; + unsigned long time_left; - wait_result = wait_for_completion_timeout(&i2c_dev->complete, - msecs_to_jiffies(500)); - if (!wait_result) + time_left = wait_for_completion_timeout(&i2c->complete, + msecs_to_jiffies(500)); + if (!time_left) return -ETIMEDOUT; - if (i2c_dev->cmd_status & ISR_NACK_ADDR) + if (i2c->cmd_status & VIAI2C_ISR_NACK_ADDR) ret = -EIO; - if (i2c_dev->cmd_status & ISR_SCL_TIMEOUT) + if (i2c->cmd_status & VIAI2C_ISR_SCL_TIMEOUT) ret = -ETIMEDOUT; return ret; } -static int wmt_i2c_write(struct wmt_i2c_dev *i2c_dev, struct i2c_msg *pmsg, int last) +static int viai2c_write(struct viai2c *i2c, struct i2c_msg *pmsg, int last) { - u16 val, tcr_val = i2c_dev->tcr; + u16 val, tcr_val = i2c->tcr; int ret; int xfer_len = 0; @@ -49,173 +49,172 @@ static int wmt_i2c_write(struct wmt_i2c_dev *i2c_dev, struct i2c_msg *pmsg, int * start at -1 and break out early from the loop */ xfer_len = -1; - writew(0, i2c_dev->base + REG_CDR); + writew(0, i2c->base + VIAI2C_REG_CDR); } else { - writew(pmsg->buf[0] & 0xFF, i2c_dev->base + REG_CDR); + writew(pmsg->buf[0] & 0xFF, i2c->base + VIAI2C_REG_CDR); } if (!(pmsg->flags & I2C_M_NOSTART)) { - val = readw(i2c_dev->base + REG_CR); - val &= ~CR_TX_END; - val |= CR_CPU_RDY; - writew(val, i2c_dev->base + REG_CR); + val = readw(i2c->base + VIAI2C_REG_CR); + val &= ~VIAI2C_CR_TX_END; + val |= VIAI2C_CR_CPU_RDY; + writew(val, i2c->base + VIAI2C_REG_CR); } - reinit_completion(&i2c_dev->complete); + reinit_completion(&i2c->complete); - tcr_val |= (TCR_MASTER_WRITE | (pmsg->addr & TCR_SLAVE_ADDR_MASK)); + tcr_val |= pmsg->addr & VIAI2C_TCR_ADDR_MASK; - writew(tcr_val, i2c_dev->base + REG_TCR); + writew(tcr_val, i2c->base + VIAI2C_REG_TCR); if (pmsg->flags & I2C_M_NOSTART) { - val = readw(i2c_dev->base + REG_CR); - val |= CR_CPU_RDY; - writew(val, i2c_dev->base + REG_CR); + val = readw(i2c->base + VIAI2C_REG_CR); + val |= VIAI2C_CR_CPU_RDY; + writew(val, i2c->base + VIAI2C_REG_CR); } while (xfer_len < pmsg->len) { - ret = wmt_check_status(i2c_dev); + ret = viai2c_check_status(i2c); if (ret) return ret; xfer_len++; - val = readw(i2c_dev->base + REG_CSR); - if ((val & CSR_RCV_ACK_MASK) == CSR_RCV_NOT_ACK) { - dev_dbg(i2c_dev->dev, "write RCV NACK error\n"); + val = readw(i2c->base + VIAI2C_REG_CSR); + if (val & VIAI2C_CSR_RCV_NOT_ACK) { + dev_dbg(i2c->dev, "write RCV NACK error\n"); return -EIO; } if (pmsg->len == 0) { - val = CR_TX_END | CR_CPU_RDY | CR_ENABLE; - writew(val, i2c_dev->base + REG_CR); + val = VIAI2C_CR_TX_END | VIAI2C_CR_CPU_RDY | VIAI2C_CR_ENABLE; + writew(val, i2c->base + VIAI2C_REG_CR); break; } if (xfer_len == pmsg->len) { if (last != 1) - writew(CR_ENABLE, i2c_dev->base + REG_CR); + writew(VIAI2C_CR_ENABLE, i2c->base + VIAI2C_REG_CR); } else { - writew(pmsg->buf[xfer_len] & 0xFF, i2c_dev->base + - REG_CDR); - writew(CR_CPU_RDY | CR_ENABLE, i2c_dev->base + REG_CR); + writew(pmsg->buf[xfer_len] & 0xFF, i2c->base + VIAI2C_REG_CDR); + writew(VIAI2C_CR_CPU_RDY | VIAI2C_CR_ENABLE, i2c->base + VIAI2C_REG_CR); } } return 0; } -static int wmt_i2c_read(struct wmt_i2c_dev *i2c_dev, struct i2c_msg *pmsg) +static int viai2c_read(struct viai2c *i2c, struct i2c_msg *pmsg) { - u16 val, tcr_val = i2c_dev->tcr; + u16 val, tcr_val = i2c->tcr; int ret; u32 xfer_len = 0; - val = readw(i2c_dev->base + REG_CR); - val &= ~(CR_TX_END | CR_TX_NEXT_NO_ACK); + val = readw(i2c->base + VIAI2C_REG_CR); + val &= ~(VIAI2C_CR_TX_END | VIAI2C_CR_RX_END); if (!(pmsg->flags & I2C_M_NOSTART)) - val |= CR_CPU_RDY; + val |= VIAI2C_CR_CPU_RDY; if (pmsg->len == 1) - val |= CR_TX_NEXT_NO_ACK; + val |= VIAI2C_CR_RX_END; - writew(val, i2c_dev->base + REG_CR); + writew(val, i2c->base + VIAI2C_REG_CR); - reinit_completion(&i2c_dev->complete); + reinit_completion(&i2c->complete); - tcr_val |= TCR_MASTER_READ | (pmsg->addr & TCR_SLAVE_ADDR_MASK); + tcr_val |= VIAI2C_TCR_READ | (pmsg->addr & VIAI2C_TCR_ADDR_MASK); - writew(tcr_val, i2c_dev->base + REG_TCR); + writew(tcr_val, i2c->base + VIAI2C_REG_TCR); if (pmsg->flags & I2C_M_NOSTART) { - val = readw(i2c_dev->base + REG_CR); - val |= CR_CPU_RDY; - writew(val, i2c_dev->base + REG_CR); + val = readw(i2c->base + VIAI2C_REG_CR); + val |= VIAI2C_CR_CPU_RDY; + writew(val, i2c->base + VIAI2C_REG_CR); } while (xfer_len < pmsg->len) { - ret = wmt_check_status(i2c_dev); + ret = viai2c_check_status(i2c); if (ret) return ret; - pmsg->buf[xfer_len] = readw(i2c_dev->base + REG_CDR) >> 8; + pmsg->buf[xfer_len] = readw(i2c->base + VIAI2C_REG_CDR) >> 8; xfer_len++; - val = readw(i2c_dev->base + REG_CR) | CR_CPU_RDY; + val = readw(i2c->base + VIAI2C_REG_CR) | VIAI2C_CR_CPU_RDY; if (xfer_len == pmsg->len - 1) - val |= CR_TX_NEXT_NO_ACK; - writew(val, i2c_dev->base + REG_CR); + val |= VIAI2C_CR_RX_END; + writew(val, i2c->base + VIAI2C_REG_CR); } return 0; } -int wmt_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) +int viai2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) { struct i2c_msg *pmsg; int i; int ret = 0; - struct wmt_i2c_dev *i2c_dev = i2c_get_adapdata(adap); + struct viai2c *i2c = i2c_get_adapdata(adap); for (i = 0; ret >= 0 && i < num; i++) { pmsg = &msgs[i]; if (!(pmsg->flags & I2C_M_NOSTART)) { - ret = wmt_i2c_wait_bus_not_busy(i2c_dev); + ret = viai2c_wait_bus_not_busy(i2c); if (ret < 0) return ret; } if (pmsg->flags & I2C_M_RD) - ret = wmt_i2c_read(i2c_dev, pmsg); + ret = viai2c_read(i2c, pmsg); else - ret = wmt_i2c_write(i2c_dev, pmsg, (i + 1) == num); + ret = viai2c_write(i2c, pmsg, (i + 1) == num); } return (ret < 0) ? ret : i; } -static irqreturn_t wmt_i2c_isr(int irq, void *data) +static irqreturn_t viai2c_isr(int irq, void *data) { - struct wmt_i2c_dev *i2c_dev = data; + struct viai2c *i2c = data; /* save the status and write-clear it */ - i2c_dev->cmd_status = readw(i2c_dev->base + REG_ISR); - writew(i2c_dev->cmd_status, i2c_dev->base + REG_ISR); + i2c->cmd_status = readw(i2c->base + VIAI2C_REG_ISR); + writew(i2c->cmd_status, i2c->base + VIAI2C_REG_ISR); - complete(&i2c_dev->complete); + complete(&i2c->complete); return IRQ_HANDLED; } -int wmt_i2c_init(struct platform_device *pdev, struct wmt_i2c_dev **pi2c_dev) +int viai2c_init(struct platform_device *pdev, struct viai2c **pi2c) { int err; - struct wmt_i2c_dev *i2c_dev; + struct viai2c *i2c; struct device_node *np = pdev->dev.of_node; - i2c_dev = devm_kzalloc(&pdev->dev, sizeof(*i2c_dev), GFP_KERNEL); - if (!i2c_dev) + i2c = devm_kzalloc(&pdev->dev, sizeof(*i2c), GFP_KERNEL); + if (!i2c) return -ENOMEM; - i2c_dev->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); - if (IS_ERR(i2c_dev->base)) - return PTR_ERR(i2c_dev->base); + i2c->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); + if (IS_ERR(i2c->base)) + return PTR_ERR(i2c->base); - i2c_dev->irq = irq_of_parse_and_map(np, 0); - if (!i2c_dev->irq) + i2c->irq = irq_of_parse_and_map(np, 0); + if (!i2c->irq) return -EINVAL; - err = devm_request_irq(&pdev->dev, i2c_dev->irq, wmt_i2c_isr, - 0, pdev->name, i2c_dev); + err = devm_request_irq(&pdev->dev, i2c->irq, viai2c_isr, + 0, pdev->name, i2c); if (err) return dev_err_probe(&pdev->dev, err, - "failed to request irq %i\n", i2c_dev->irq); + "failed to request irq %i\n", i2c->irq); - i2c_dev->dev = &pdev->dev; - init_completion(&i2c_dev->complete); - platform_set_drvdata(pdev, i2c_dev); + i2c->dev = &pdev->dev; + init_completion(&i2c->complete); + platform_set_drvdata(pdev, i2c); - *pi2c_dev = i2c_dev; + *pi2c = i2c; return 0; } diff --git a/drivers/i2c/busses/i2c-viai2c-common.h b/drivers/i2c/busses/i2c-viai2c-common.h index fcff8e4456eb8..28799e7e97f0b 100644 --- a/drivers/i2c/busses/i2c-viai2c-common.h +++ b/drivers/i2c/busses/i2c-viai2c-common.h @@ -11,48 +11,46 @@ #include #include -#define REG_CR 0x00 -#define REG_TCR 0x02 -#define REG_CSR 0x04 -#define REG_ISR 0x06 -#define REG_IMR 0x08 -#define REG_CDR 0x0A -#define REG_TR 0x0C -#define REG_MCR 0x0E - /* REG_CR Bit fields */ -#define CR_TX_NEXT_ACK 0x0000 -#define CR_ENABLE 0x0001 -#define CR_TX_NEXT_NO_ACK 0x0002 -#define CR_TX_END 0x0004 -#define CR_CPU_RDY 0x0008 -#define SLAV_MODE_SEL 0x8000 +#define VIAI2C_REG_CR 0x00 +#define VIAI2C_CR_ENABLE BIT(0) +#define VIAI2C_CR_RX_END BIT(1) +#define VIAI2C_CR_TX_END BIT(2) +#define VIAI2C_CR_CPU_RDY BIT(3) +#define VIAI2C_CR_END_MASK GENMASK(2, 1) /* REG_TCR Bit fields */ -#define TCR_STANDARD_MODE 0x0000 -#define TCR_MASTER_WRITE 0x0000 -#define TCR_HS_MODE 0x2000 -#define TCR_MASTER_READ 0x4000 -#define TCR_FAST_MODE 0x8000 -#define TCR_SLAVE_ADDR_MASK 0x007F +#define VIAI2C_REG_TCR 0x02 +#define VIAI2C_TCR_HS_MODE BIT(13) +#define VIAI2C_TCR_READ BIT(14) +#define VIAI2C_TCR_FAST BIT(15) +#define VIAI2C_TCR_ADDR_MASK GENMASK(6, 0) + +/* REG_CSR Bit fields */ +#define VIAI2C_REG_CSR 0x04 +#define VIAI2C_CSR_RCV_NOT_ACK BIT(0) +#define VIAI2C_CSR_RCV_ACK_MASK BIT(0) +#define VIAI2C_CSR_READY_MASK BIT(1) /* REG_ISR Bit fields */ -#define ISR_NACK_ADDR 0x0001 -#define ISR_BYTE_END 0x0002 -#define ISR_SCL_TIMEOUT 0x0004 -#define ISR_WRITE_ALL 0x0007 +#define VIAI2C_REG_ISR 0x06 +#define VIAI2C_ISR_NACK_ADDR BIT(0) +#define VIAI2C_ISR_BYTE_END BIT(1) +#define VIAI2C_ISR_SCL_TIMEOUT BIT(2) +#define VIAI2C_ISR_MASK_ALL GENMASK(2, 0) /* REG_IMR Bit fields */ -#define IMR_ENABLE_ALL 0x0007 +#define VIAI2C_REG_IMR 0x08 +#define VIAI2C_IMR_BYTE BIT(1) +#define VIAI2C_IMR_ENABLE_ALL GENMASK(2, 0) -/* REG_CSR Bit fields */ -#define CSR_RCV_NOT_ACK 0x0001 -#define CSR_RCV_ACK_MASK 0x0001 -#define CSR_READY_MASK 0x0002 +#define VIAI2C_REG_CDR 0x0A +#define VIAI2C_REG_TR 0x0C +#define VIAI2C_REG_MCR 0x0E -#define WMT_I2C_TIMEOUT (msecs_to_jiffies(1000)) +#define VIAI2C_TIMEOUT (msecs_to_jiffies(1000)) -struct wmt_i2c_dev { +struct viai2c { struct i2c_adapter adapter; struct completion complete; struct device *dev; @@ -63,9 +61,9 @@ struct wmt_i2c_dev { u16 cmd_status; }; -int wmt_i2c_wait_bus_not_busy(struct wmt_i2c_dev *i2c_dev); -int wmt_check_status(struct wmt_i2c_dev *i2c_dev); -int wmt_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num); -int wmt_i2c_init(struct platform_device *pdev, struct wmt_i2c_dev **pi2c_dev); +int viai2c_wait_bus_not_busy(struct viai2c *i2c); +int viai2c_check_status(struct viai2c *i2c); +int viai2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num); +int viai2c_init(struct platform_device *pdev, struct viai2c **pi2c); #endif diff --git a/drivers/i2c/busses/i2c-viai2c-wmt.c b/drivers/i2c/busses/i2c-viai2c-wmt.c index 3125374cc2c0d..1e9db735e1fa4 100644 --- a/drivers/i2c/busses/i2c-viai2c-wmt.c +++ b/drivers/i2c/busses/i2c-viai2c-wmt.c @@ -35,39 +35,39 @@ static u32 wmt_i2c_func(struct i2c_adapter *adap) } static const struct i2c_algorithm wmt_i2c_algo = { - .master_xfer = wmt_i2c_xfer, + .master_xfer = viai2c_xfer, .functionality = wmt_i2c_func, }; -static int wmt_i2c_reset_hardware(struct wmt_i2c_dev *i2c_dev) +static int wmt_i2c_reset_hardware(struct viai2c *i2c) { int err; - err = clk_prepare_enable(i2c_dev->clk); + err = clk_prepare_enable(i2c->clk); if (err) { - dev_err(i2c_dev->dev, "failed to enable clock\n"); + dev_err(i2c->dev, "failed to enable clock\n"); return err; } - err = clk_set_rate(i2c_dev->clk, 20000000); + err = clk_set_rate(i2c->clk, 20000000); if (err) { - dev_err(i2c_dev->dev, "failed to set clock = 20Mhz\n"); - clk_disable_unprepare(i2c_dev->clk); + dev_err(i2c->dev, "failed to set clock = 20Mhz\n"); + clk_disable_unprepare(i2c->clk); return err; } - writew(0, i2c_dev->base + REG_CR); - writew(MCR_APB_166M, i2c_dev->base + REG_MCR); - writew(ISR_WRITE_ALL, i2c_dev->base + REG_ISR); - writew(IMR_ENABLE_ALL, i2c_dev->base + REG_IMR); - writew(CR_ENABLE, i2c_dev->base + REG_CR); - readw(i2c_dev->base + REG_CSR); /* read clear */ - writew(ISR_WRITE_ALL, i2c_dev->base + REG_ISR); + writew(0, i2c->base + VIAI2C_REG_CR); + writew(MCR_APB_166M, i2c->base + VIAI2C_REG_MCR); + writew(VIAI2C_ISR_MASK_ALL, i2c->base + VIAI2C_REG_ISR); + writew(VIAI2C_IMR_ENABLE_ALL, i2c->base + VIAI2C_REG_IMR); + writew(VIAI2C_CR_ENABLE, i2c->base + VIAI2C_REG_CR); + readw(i2c->base + VIAI2C_REG_CSR); /* read clear */ + writew(VIAI2C_ISR_MASK_ALL, i2c->base + VIAI2C_REG_ISR); - if (i2c_dev->tcr == TCR_FAST_MODE) - writew(SCL_TIMEOUT(128) | TR_HS, i2c_dev->base + REG_TR); + if (i2c->tcr == VIAI2C_TCR_FAST) + writew(SCL_TIMEOUT(128) | TR_HS, i2c->base + VIAI2C_REG_TR); else - writew(SCL_TIMEOUT(128) | TR_STD, i2c_dev->base + REG_TR); + writew(SCL_TIMEOUT(128) | TR_STD, i2c->base + VIAI2C_REG_TR); return 0; } @@ -75,34 +75,34 @@ static int wmt_i2c_reset_hardware(struct wmt_i2c_dev *i2c_dev) static int wmt_i2c_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; - struct wmt_i2c_dev *i2c_dev; + struct viai2c *i2c; struct i2c_adapter *adap; int err; u32 clk_rate; - err = wmt_i2c_init(pdev, &i2c_dev); + err = viai2c_init(pdev, &i2c); if (err) return err; - i2c_dev->clk = of_clk_get(np, 0); - if (IS_ERR(i2c_dev->clk)) { + i2c->clk = of_clk_get(np, 0); + if (IS_ERR(i2c->clk)) { dev_err(&pdev->dev, "unable to request clock\n"); - return PTR_ERR(i2c_dev->clk); + return PTR_ERR(i2c->clk); } err = of_property_read_u32(np, "clock-frequency", &clk_rate); if (!err && clk_rate == I2C_MAX_FAST_MODE_FREQ) - i2c_dev->tcr = TCR_FAST_MODE; + i2c->tcr = VIAI2C_TCR_FAST; - adap = &i2c_dev->adapter; - i2c_set_adapdata(adap, i2c_dev); + adap = &i2c->adapter; + i2c_set_adapdata(adap, i2c); strscpy(adap->name, "WMT I2C adapter", sizeof(adap->name)); adap->owner = THIS_MODULE; adap->algo = &wmt_i2c_algo; adap->dev.parent = &pdev->dev; adap->dev.of_node = pdev->dev.of_node; - err = wmt_i2c_reset_hardware(i2c_dev); + err = wmt_i2c_reset_hardware(i2c); if (err) { dev_err(&pdev->dev, "error initializing hardware\n"); return err; @@ -111,19 +111,19 @@ static int wmt_i2c_probe(struct platform_device *pdev) err = i2c_add_adapter(adap); if (err) /* wmt_i2c_reset_hardware() enables i2c_dev->clk */ - clk_disable_unprepare(i2c_dev->clk); + clk_disable_unprepare(i2c->clk); return err; } static void wmt_i2c_remove(struct platform_device *pdev) { - struct wmt_i2c_dev *i2c_dev = platform_get_drvdata(pdev); + struct viai2c *i2c = platform_get_drvdata(pdev); /* Disable interrupts, clock and delete adapter */ - writew(0, i2c_dev->base + REG_IMR); - clk_disable_unprepare(i2c_dev->clk); - i2c_del_adapter(&i2c_dev->adapter); + writew(0, i2c->base + VIAI2C_REG_IMR); + clk_disable_unprepare(i2c->clk); + i2c_del_adapter(&i2c->adapter); } static const struct of_device_id wmt_i2c_dt_ids[] = { From 2e829ccc2779d4ea47050a2f41e7531a247a46e5 Mon Sep 17 00:00:00 2001 From: Hans Hu Date: Mon, 8 Apr 2024 10:54:46 +0800 Subject: [PATCH 1089/1702] i2c: wmt: fix a bug when thread blocked During each byte access, the host performs clock stretching. To reduce the host performs clock stretching, move most of the per-msg processing to the interrupt context. Suggested-by: Wolfram Sang Reviewed-by: Wolfram Sang Signed-off-by: Hans Hu Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-viai2c-common.c | 145 +++++++++++++------------ drivers/i2c/busses/i2c-viai2c-common.h | 6 +- 2 files changed, 80 insertions(+), 71 deletions(-) diff --git a/drivers/i2c/busses/i2c-viai2c-common.c b/drivers/i2c/busses/i2c-viai2c-common.c index 59901c53ad48c..1b39cf4f0d6c8 100644 --- a/drivers/i2c/busses/i2c-viai2c-common.c +++ b/drivers/i2c/busses/i2c-viai2c-common.c @@ -18,37 +18,18 @@ int viai2c_wait_bus_not_busy(struct viai2c *i2c) return 0; } -int viai2c_check_status(struct viai2c *i2c) -{ - int ret = 0; - unsigned long time_left; - - time_left = wait_for_completion_timeout(&i2c->complete, - msecs_to_jiffies(500)); - if (!time_left) - return -ETIMEDOUT; - - if (i2c->cmd_status & VIAI2C_ISR_NACK_ADDR) - ret = -EIO; - - if (i2c->cmd_status & VIAI2C_ISR_SCL_TIMEOUT) - ret = -ETIMEDOUT; - - return ret; -} - static int viai2c_write(struct viai2c *i2c, struct i2c_msg *pmsg, int last) { u16 val, tcr_val = i2c->tcr; - int ret; - int xfer_len = 0; + + i2c->last = last; if (pmsg->len == 0) { /* * We still need to run through the while (..) once, so * start at -1 and break out early from the loop */ - xfer_len = -1; + i2c->xfered_len = -1; writew(0, i2c->base + VIAI2C_REG_CDR); } else { writew(pmsg->buf[0] & 0xFF, i2c->base + VIAI2C_REG_CDR); @@ -73,42 +54,15 @@ static int viai2c_write(struct viai2c *i2c, struct i2c_msg *pmsg, int last) writew(val, i2c->base + VIAI2C_REG_CR); } - while (xfer_len < pmsg->len) { - ret = viai2c_check_status(i2c); - if (ret) - return ret; - - xfer_len++; - - val = readw(i2c->base + VIAI2C_REG_CSR); - if (val & VIAI2C_CSR_RCV_NOT_ACK) { - dev_dbg(i2c->dev, "write RCV NACK error\n"); - return -EIO; - } - - if (pmsg->len == 0) { - val = VIAI2C_CR_TX_END | VIAI2C_CR_CPU_RDY | VIAI2C_CR_ENABLE; - writew(val, i2c->base + VIAI2C_REG_CR); - break; - } - - if (xfer_len == pmsg->len) { - if (last != 1) - writew(VIAI2C_CR_ENABLE, i2c->base + VIAI2C_REG_CR); - } else { - writew(pmsg->buf[xfer_len] & 0xFF, i2c->base + VIAI2C_REG_CDR); - writew(VIAI2C_CR_CPU_RDY | VIAI2C_CR_ENABLE, i2c->base + VIAI2C_REG_CR); - } - } + if (!wait_for_completion_timeout(&i2c->complete, VIAI2C_TIMEOUT)) + return -ETIMEDOUT; - return 0; + return i2c->ret; } static int viai2c_read(struct viai2c *i2c, struct i2c_msg *pmsg) { u16 val, tcr_val = i2c->tcr; - int ret; - u32 xfer_len = 0; val = readw(i2c->base + VIAI2C_REG_CR); val &= ~(VIAI2C_CR_TX_END | VIAI2C_CR_RX_END); @@ -133,21 +87,10 @@ static int viai2c_read(struct viai2c *i2c, struct i2c_msg *pmsg) writew(val, i2c->base + VIAI2C_REG_CR); } - while (xfer_len < pmsg->len) { - ret = viai2c_check_status(i2c); - if (ret) - return ret; - - pmsg->buf[xfer_len] = readw(i2c->base + VIAI2C_REG_CDR) >> 8; - xfer_len++; - - val = readw(i2c->base + VIAI2C_REG_CR) | VIAI2C_CR_CPU_RDY; - if (xfer_len == pmsg->len - 1) - val |= VIAI2C_CR_RX_END; - writew(val, i2c->base + VIAI2C_REG_CR); - } + if (!wait_for_completion_timeout(&i2c->complete, VIAI2C_TIMEOUT)) + return -ETIMEDOUT; - return 0; + return i2c->ret; } int viai2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) @@ -165,6 +108,9 @@ int viai2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) return ret; } + i2c->msg = pmsg; + i2c->xfered_len = 0; + if (pmsg->flags & I2C_M_RD) ret = viai2c_read(i2c, pmsg); else @@ -174,15 +120,76 @@ int viai2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) return (ret < 0) ? ret : i; } +/* + * Main process of the byte mode xfer + * + * Return value indicates whether the transfer is complete + * 1: all the data has been successfully transferred + * 0: there is still data that needs to be transferred + * -EIO: error occurred + */ +static int viai2c_irq_xfer(struct viai2c *i2c) +{ + u16 val; + struct i2c_msg *msg = i2c->msg; + u8 read = msg->flags & I2C_M_RD; + void __iomem *base = i2c->base; + + if (read) { + msg->buf[i2c->xfered_len] = readw(base + VIAI2C_REG_CDR) >> 8; + + val = readw(base + VIAI2C_REG_CR) | VIAI2C_CR_CPU_RDY; + if (i2c->xfered_len == msg->len - 2) + val |= VIAI2C_CR_RX_END; + writew(val, base + VIAI2C_REG_CR); + } else { + val = readw(base + VIAI2C_REG_CSR); + if (val & VIAI2C_CSR_RCV_NOT_ACK) + return -EIO; + + /* I2C_SMBUS_QUICK */ + if (msg->len == 0) { + val = VIAI2C_CR_TX_END | VIAI2C_CR_CPU_RDY | VIAI2C_CR_ENABLE; + writew(val, base + VIAI2C_REG_CR); + return 1; + } + + if ((i2c->xfered_len + 1) == msg->len) { + if (!i2c->last) + writew(VIAI2C_CR_ENABLE, base + VIAI2C_REG_CR); + } else { + writew(msg->buf[i2c->xfered_len + 1] & 0xFF, base + VIAI2C_REG_CDR); + writew(VIAI2C_CR_CPU_RDY | VIAI2C_CR_ENABLE, base + VIAI2C_REG_CR); + } + } + + i2c->xfered_len++; + + return i2c->xfered_len == msg->len; +} + static irqreturn_t viai2c_isr(int irq, void *data) { struct viai2c *i2c = data; + u8 status; /* save the status and write-clear it */ - i2c->cmd_status = readw(i2c->base + VIAI2C_REG_ISR); - writew(i2c->cmd_status, i2c->base + VIAI2C_REG_ISR); + status = readw(i2c->base + VIAI2C_REG_ISR); + writew(status, i2c->base + VIAI2C_REG_ISR); + + i2c->ret = 0; + if (status & VIAI2C_ISR_NACK_ADDR) + i2c->ret = -EIO; + + if (status & VIAI2C_ISR_SCL_TIMEOUT) + i2c->ret = -ETIMEDOUT; + + if (!i2c->ret) + i2c->ret = viai2c_irq_xfer(i2c); - complete(&i2c->complete); + /* All the data has been successfully transferred or error occurred */ + if (i2c->ret) + complete(&i2c->complete); return IRQ_HANDLED; } diff --git a/drivers/i2c/busses/i2c-viai2c-common.h b/drivers/i2c/busses/i2c-viai2c-common.h index 28799e7e97f0b..c92e054ac7e73 100644 --- a/drivers/i2c/busses/i2c-viai2c-common.h +++ b/drivers/i2c/busses/i2c-viai2c-common.h @@ -58,11 +58,13 @@ struct viai2c { struct clk *clk; u16 tcr; int irq; - u16 cmd_status; + u16 xfered_len; + struct i2c_msg *msg; + int ret; + bool last; }; int viai2c_wait_bus_not_busy(struct viai2c *i2c); -int viai2c_check_status(struct viai2c *i2c); int viai2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num); int viai2c_init(struct platform_device *pdev, struct viai2c **pi2c); From b06204c7d4f4a4059c16d20bd9eb618edad49aa1 Mon Sep 17 00:00:00 2001 From: Hans Hu Date: Mon, 8 Apr 2024 10:54:47 +0800 Subject: [PATCH 1090/1702] i2c: wmt: add platform type VIAI2C_PLAT_WMT Enumeration variables are added to differentiate between different platforms. Acked-by: Wolfram Sang Signed-off-by: Hans Hu Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-viai2c-common.c | 32 ++++++++++++++++---------- drivers/i2c/busses/i2c-viai2c-common.h | 7 +++++- drivers/i2c/busses/i2c-viai2c-wmt.c | 2 +- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/drivers/i2c/busses/i2c-viai2c-common.c b/drivers/i2c/busses/i2c-viai2c-common.c index 1b39cf4f0d6c8..22a2e31e9c14b 100644 --- a/drivers/i2c/busses/i2c-viai2c-common.c +++ b/drivers/i2c/busses/i2c-viai2c-common.c @@ -35,7 +35,7 @@ static int viai2c_write(struct viai2c *i2c, struct i2c_msg *pmsg, int last) writew(pmsg->buf[0] & 0xFF, i2c->base + VIAI2C_REG_CDR); } - if (!(pmsg->flags & I2C_M_NOSTART)) { + if (i2c->platform == VIAI2C_PLAT_WMT && !(pmsg->flags & I2C_M_NOSTART)) { val = readw(i2c->base + VIAI2C_REG_CR); val &= ~VIAI2C_CR_TX_END; val |= VIAI2C_CR_CPU_RDY; @@ -48,7 +48,7 @@ static int viai2c_write(struct viai2c *i2c, struct i2c_msg *pmsg, int last) writew(tcr_val, i2c->base + VIAI2C_REG_TCR); - if (pmsg->flags & I2C_M_NOSTART) { + if (i2c->platform == VIAI2C_PLAT_WMT && pmsg->flags & I2C_M_NOSTART) { val = readw(i2c->base + VIAI2C_REG_CR); val |= VIAI2C_CR_CPU_RDY; writew(val, i2c->base + VIAI2C_REG_CR); @@ -67,7 +67,7 @@ static int viai2c_read(struct viai2c *i2c, struct i2c_msg *pmsg) val = readw(i2c->base + VIAI2C_REG_CR); val &= ~(VIAI2C_CR_TX_END | VIAI2C_CR_RX_END); - if (!(pmsg->flags & I2C_M_NOSTART)) + if (i2c->platform == VIAI2C_PLAT_WMT && !(pmsg->flags & I2C_M_NOSTART)) val |= VIAI2C_CR_CPU_RDY; if (pmsg->len == 1) @@ -81,7 +81,7 @@ static int viai2c_read(struct viai2c *i2c, struct i2c_msg *pmsg) writew(tcr_val, i2c->base + VIAI2C_REG_TCR); - if (pmsg->flags & I2C_M_NOSTART) { + if (i2c->platform == VIAI2C_PLAT_WMT && (pmsg->flags & I2C_M_NOSTART)) { val = readw(i2c->base + VIAI2C_REG_CR); val |= VIAI2C_CR_CPU_RDY; writew(val, i2c->base + VIAI2C_REG_CR); @@ -102,7 +102,7 @@ int viai2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) for (i = 0; ret >= 0 && i < num; i++) { pmsg = &msgs[i]; - if (!(pmsg->flags & I2C_M_NOSTART)) { + if (i2c->platform == VIAI2C_PLAT_WMT && !(pmsg->flags & I2C_M_NOSTART)) { ret = viai2c_wait_bus_not_busy(i2c); if (ret < 0) return ret; @@ -155,7 +155,7 @@ static int viai2c_irq_xfer(struct viai2c *i2c) } if ((i2c->xfered_len + 1) == msg->len) { - if (!i2c->last) + if (i2c->platform == VIAI2C_PLAT_WMT && !i2c->last) writew(VIAI2C_CR_ENABLE, base + VIAI2C_REG_CR); } else { writew(msg->buf[i2c->xfered_len + 1] & 0xFF, base + VIAI2C_REG_CDR); @@ -181,7 +181,7 @@ static irqreturn_t viai2c_isr(int irq, void *data) if (status & VIAI2C_ISR_NACK_ADDR) i2c->ret = -EIO; - if (status & VIAI2C_ISR_SCL_TIMEOUT) + if (i2c->platform == VIAI2C_PLAT_WMT && (status & VIAI2C_ISR_SCL_TIMEOUT)) i2c->ret = -ETIMEDOUT; if (!i2c->ret) @@ -194,9 +194,10 @@ static irqreturn_t viai2c_isr(int irq, void *data) return IRQ_HANDLED; } -int viai2c_init(struct platform_device *pdev, struct viai2c **pi2c) +int viai2c_init(struct platform_device *pdev, struct viai2c **pi2c, int plat) { int err; + int irq_flags; struct viai2c *i2c; struct device_node *np = pdev->dev.of_node; @@ -208,12 +209,19 @@ int viai2c_init(struct platform_device *pdev, struct viai2c **pi2c) if (IS_ERR(i2c->base)) return PTR_ERR(i2c->base); - i2c->irq = irq_of_parse_and_map(np, 0); - if (!i2c->irq) - return -EINVAL; + if (plat == VIAI2C_PLAT_WMT) { + irq_flags = 0; + i2c->irq = irq_of_parse_and_map(np, 0); + if (!i2c->irq) + return -EINVAL; + } else { + return dev_err_probe(&pdev->dev, -EINVAL, "wrong platform type\n"); + } + + i2c->platform = plat; err = devm_request_irq(&pdev->dev, i2c->irq, viai2c_isr, - 0, pdev->name, i2c); + irq_flags, pdev->name, i2c); if (err) return dev_err_probe(&pdev->dev, err, "failed to request irq %i\n", i2c->irq); diff --git a/drivers/i2c/busses/i2c-viai2c-common.h b/drivers/i2c/busses/i2c-viai2c-common.h index c92e054ac7e73..cfe5ab2bd7792 100644 --- a/drivers/i2c/busses/i2c-viai2c-common.h +++ b/drivers/i2c/busses/i2c-viai2c-common.h @@ -50,6 +50,10 @@ #define VIAI2C_TIMEOUT (msecs_to_jiffies(1000)) +enum { + VIAI2C_PLAT_WMT, +}; + struct viai2c { struct i2c_adapter adapter; struct completion complete; @@ -62,10 +66,11 @@ struct viai2c { struct i2c_msg *msg; int ret; bool last; + unsigned int platform; }; int viai2c_wait_bus_not_busy(struct viai2c *i2c); int viai2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num); -int viai2c_init(struct platform_device *pdev, struct viai2c **pi2c); +int viai2c_init(struct platform_device *pdev, struct viai2c **pi2c, int plat); #endif diff --git a/drivers/i2c/busses/i2c-viai2c-wmt.c b/drivers/i2c/busses/i2c-viai2c-wmt.c index 1e9db735e1fa4..e1988f9460266 100644 --- a/drivers/i2c/busses/i2c-viai2c-wmt.c +++ b/drivers/i2c/busses/i2c-viai2c-wmt.c @@ -80,7 +80,7 @@ static int wmt_i2c_probe(struct platform_device *pdev) int err; u32 clk_rate; - err = viai2c_init(pdev, &i2c); + err = viai2c_init(pdev, &i2c, VIAI2C_PLAT_WMT); if (err) return err; From a06b80e83011c996147e94a3bcd9c7387c14abd9 Mon Sep 17 00:00:00 2001 From: Hans Hu Date: Mon, 8 Apr 2024 10:54:48 +0800 Subject: [PATCH 1091/1702] i2c: add zhaoxin i2c controller driver Add Zhaoxin I2C controller driver. It provides the access to the i2c busses, which connects to the touchpad, eeprom, I2S, etc. Zhaoxin I2C controller has two separate busses, so may accommodate up to two I2C adapters. Those adapters are listed in the ACPI namespace with the IIC1D17 HID, and probed by a platform driver. The driver works with IRQ mode, and supports basic I2C features. Flags I2C_AQ_NO_ZERO_LEN and I2C_AQ_COMB_WRITE_THEN_READ are used to limit the unsupported access. Acked-by: Wolfram Sang Signed-off-by: Hans Hu Signed-off-by: Andi Shyti --- MAINTAINERS | 8 + drivers/i2c/busses/Kconfig | 10 + drivers/i2c/busses/Makefile | 2 + drivers/i2c/busses/i2c-viai2c-common.c | 31 ++- drivers/i2c/busses/i2c-viai2c-common.h | 9 + drivers/i2c/busses/i2c-viai2c-zhaoxin.c | 298 ++++++++++++++++++++++++ 6 files changed, 353 insertions(+), 5 deletions(-) create mode 100644 drivers/i2c/busses/i2c-viai2c-zhaoxin.c diff --git a/MAINTAINERS b/MAINTAINERS index a3491d1aa7c37..0927ea8c36f13 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10258,6 +10258,14 @@ L: linux-i2c@vger.kernel.org F: Documentation/i2c/busses/i2c-ismt.rst F: drivers/i2c/busses/i2c-ismt.c +I2C/SMBUS ZHAOXIN DRIVER +M: Hans Hu +L: linux-i2c@vger.kernel.org +S: Maintained +W: https://www.zhaoxin.com +F: drivers/i2c/busses/i2c-viai2c-common.c +F: drivers/i2c/busses/i2c-viai2c-zhaoxin.c + I2C/SMBUS STUB DRIVER M: Jean Delvare L: linux-i2c@vger.kernel.org diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index a5aff29c559d0..fe6e8a1bb6072 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -344,6 +344,16 @@ config I2C_VIAPRO if ACPI +config I2C_ZHAOXIN + tristate "Zhaoxin I2C Interface" + depends on PCI || COMPILE_TEST + help + If you say yes to this option, support will be included for the + ZHAOXIN I2C interface + + This driver can also be built as a module. If so, the module + will be called i2c-zhaoxin. + comment "ACPI drivers" config I2C_SCMI diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile index 63c7bbad8134f..3d65934f5eb48 100644 --- a/drivers/i2c/busses/Makefile +++ b/drivers/i2c/busses/Makefile @@ -29,6 +29,8 @@ obj-$(CONFIG_I2C_SIS630) += i2c-sis630.o obj-$(CONFIG_I2C_SIS96X) += i2c-sis96x.o obj-$(CONFIG_I2C_VIA) += i2c-via.o obj-$(CONFIG_I2C_VIAPRO) += i2c-viapro.o +i2c-zhaoxin-objs := i2c-viai2c-zhaoxin.o i2c-viai2c-common.o +obj-$(CONFIG_I2C_ZHAOXIN) += i2c-zhaoxin.o # Mac SMBus host controller drivers obj-$(CONFIG_I2C_HYDRA) += i2c-hydra.o diff --git a/drivers/i2c/busses/i2c-viai2c-common.c b/drivers/i2c/busses/i2c-viai2c-common.c index 22a2e31e9c14b..1844d13f1f798 100644 --- a/drivers/i2c/busses/i2c-viai2c-common.c +++ b/drivers/i2c/busses/i2c-viai2c-common.c @@ -60,7 +60,7 @@ static int viai2c_write(struct viai2c *i2c, struct i2c_msg *pmsg, int last) return i2c->ret; } -static int viai2c_read(struct viai2c *i2c, struct i2c_msg *pmsg) +static int viai2c_read(struct viai2c *i2c, struct i2c_msg *pmsg, bool first) { u16 val, tcr_val = i2c->tcr; @@ -81,7 +81,8 @@ static int viai2c_read(struct viai2c *i2c, struct i2c_msg *pmsg) writew(tcr_val, i2c->base + VIAI2C_REG_TCR); - if (i2c->platform == VIAI2C_PLAT_WMT && (pmsg->flags & I2C_M_NOSTART)) { + if ((i2c->platform == VIAI2C_PLAT_WMT && (pmsg->flags & I2C_M_NOSTART)) || + (i2c->platform == VIAI2C_PLAT_ZHAOXIN && !first)) { val = readw(i2c->base + VIAI2C_REG_CR); val |= VIAI2C_CR_CPU_RDY; writew(val, i2c->base + VIAI2C_REG_CR); @@ -100,6 +101,7 @@ int viai2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) int ret = 0; struct viai2c *i2c = i2c_get_adapdata(adap); + i2c->mode = VIAI2C_BYTE_MODE; for (i = 0; ret >= 0 && i < num; i++) { pmsg = &msgs[i]; if (i2c->platform == VIAI2C_PLAT_WMT && !(pmsg->flags & I2C_M_NOSTART)) { @@ -112,7 +114,7 @@ int viai2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) i2c->xfered_len = 0; if (pmsg->flags & I2C_M_RD) - ret = viai2c_read(i2c, pmsg); + ret = viai2c_read(i2c, pmsg, i == 0); else ret = viai2c_write(i2c, pmsg, (i + 1) == num); } @@ -157,6 +159,8 @@ static int viai2c_irq_xfer(struct viai2c *i2c) if ((i2c->xfered_len + 1) == msg->len) { if (i2c->platform == VIAI2C_PLAT_WMT && !i2c->last) writew(VIAI2C_CR_ENABLE, base + VIAI2C_REG_CR); + else if (i2c->platform == VIAI2C_PLAT_ZHAOXIN && i2c->last) + writeb(VIAI2C_CR_TX_END, base + VIAI2C_REG_CR); } else { writew(msg->buf[i2c->xfered_len + 1] & 0xFF, base + VIAI2C_REG_CDR); writew(VIAI2C_CR_CPU_RDY | VIAI2C_CR_ENABLE, base + VIAI2C_REG_CR); @@ -168,6 +172,11 @@ static int viai2c_irq_xfer(struct viai2c *i2c) return i2c->xfered_len == msg->len; } +int __weak viai2c_fifo_irq_xfer(struct viai2c *i2c, bool irq) +{ + return 0; +} + static irqreturn_t viai2c_isr(int irq, void *data) { struct viai2c *i2c = data; @@ -175,6 +184,9 @@ static irqreturn_t viai2c_isr(int irq, void *data) /* save the status and write-clear it */ status = readw(i2c->base + VIAI2C_REG_ISR); + if (!status && i2c->platform == VIAI2C_PLAT_ZHAOXIN) + return IRQ_NONE; + writew(status, i2c->base + VIAI2C_REG_ISR); i2c->ret = 0; @@ -184,8 +196,12 @@ static irqreturn_t viai2c_isr(int irq, void *data) if (i2c->platform == VIAI2C_PLAT_WMT && (status & VIAI2C_ISR_SCL_TIMEOUT)) i2c->ret = -ETIMEDOUT; - if (!i2c->ret) - i2c->ret = viai2c_irq_xfer(i2c); + if (!i2c->ret) { + if (i2c->mode == VIAI2C_BYTE_MODE) + i2c->ret = viai2c_irq_xfer(i2c); + else + i2c->ret = viai2c_fifo_irq_xfer(i2c, true); + } /* All the data has been successfully transferred or error occurred */ if (i2c->ret) @@ -214,6 +230,11 @@ int viai2c_init(struct platform_device *pdev, struct viai2c **pi2c, int plat) i2c->irq = irq_of_parse_and_map(np, 0); if (!i2c->irq) return -EINVAL; + } else if (plat == VIAI2C_PLAT_ZHAOXIN) { + irq_flags = IRQF_SHARED; + i2c->irq = platform_get_irq(pdev, 0); + if (i2c->irq < 0) + return i2c->irq; } else { return dev_err_probe(&pdev->dev, -EINVAL, "wrong platform type\n"); } diff --git a/drivers/i2c/busses/i2c-viai2c-common.h b/drivers/i2c/busses/i2c-viai2c-common.h index cfe5ab2bd7792..81e827c544347 100644 --- a/drivers/i2c/busses/i2c-viai2c-common.h +++ b/drivers/i2c/busses/i2c-viai2c-common.h @@ -52,6 +52,12 @@ enum { VIAI2C_PLAT_WMT, + VIAI2C_PLAT_ZHAOXIN +}; + +enum { + VIAI2C_BYTE_MODE, + VIAI2C_FIFO_MODE }; struct viai2c { @@ -66,11 +72,14 @@ struct viai2c { struct i2c_msg *msg; int ret; bool last; + unsigned int mode; unsigned int platform; + void *pltfm_priv; }; int viai2c_wait_bus_not_busy(struct viai2c *i2c); int viai2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num); int viai2c_init(struct platform_device *pdev, struct viai2c **pi2c, int plat); +int viai2c_fifo_irq_xfer(struct viai2c *i2c, bool irq); #endif diff --git a/drivers/i2c/busses/i2c-viai2c-zhaoxin.c b/drivers/i2c/busses/i2c-viai2c-zhaoxin.c new file mode 100644 index 0000000000000..7e3ac2a3e1fd7 --- /dev/null +++ b/drivers/i2c/busses/i2c-viai2c-zhaoxin.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright(c) 2024 Shanghai Zhaoxin Semiconductor Corporation. + * All rights reserved. + */ + +#include +#include "i2c-viai2c-common.h" + +/* + * registers + */ +/* Zhaoxin specific register bit fields */ +/* REG_CR Bit fields */ +#define ZXI2C_CR_MST_RST BIT(7) +#define ZXI2C_CR_FIFO_MODE BIT(14) +/* REG_ISR/IMR Bit fields */ +#define ZXI2C_IRQ_FIFONACK BIT(4) +#define ZXI2C_IRQ_FIFOEND BIT(3) +#define ZXI2C_IRQ_MASK (VIAI2C_ISR_MASK_ALL \ + | ZXI2C_IRQ_FIFOEND \ + | ZXI2C_IRQ_FIFONACK) +/* Zhaoxin specific registers */ +#define ZXI2C_REG_CLK 0x10 +#define ZXI2C_CLK_50M BIT(0) +#define ZXI2C_REG_REV 0x11 +#define ZXI2C_REG_HCR 0x12 +#define ZXI2C_HCR_RST_FIFO GENMASK(1, 0) +#define ZXI2C_REG_HTDR 0x13 +#define ZXI2C_REG_HRDR 0x14 +#define ZXI2C_REG_HTLR 0x15 +#define ZXI2C_REG_HRLR 0x16 +#define ZXI2C_REG_HWCNTR 0x18 +#define ZXI2C_REG_HRCNTR 0x19 + +/* parameters Constants */ +#define ZXI2C_GOLD_FSTP_100K 0xF3 +#define ZXI2C_GOLD_FSTP_400K 0x38 +#define ZXI2C_GOLD_FSTP_1M 0x13 +#define ZXI2C_GOLD_FSTP_3400K 0x37 +#define ZXI2C_HS_MASTER_CODE (0x08 << 8) + +#define ZXI2C_FIFO_SIZE 32 + +struct viai2c_zhaoxin { + u8 hrv; + u16 tr; + u16 mcr; + u16 xfer_len; +}; + +/* 'irq == true' means in interrupt context */ +int viai2c_fifo_irq_xfer(struct viai2c *i2c, bool irq) +{ + u16 i; + u8 tmp; + struct i2c_msg *msg = i2c->msg; + void __iomem *base = i2c->base; + bool read = !!(msg->flags & I2C_M_RD); + struct viai2c_zhaoxin *priv = i2c->pltfm_priv; + + if (irq) { + /* get the received data */ + if (read) + for (i = 0; i < priv->xfer_len; i++) + msg->buf[i2c->xfered_len + i] = ioread8(base + ZXI2C_REG_HRDR); + + i2c->xfered_len += priv->xfer_len; + if (i2c->xfered_len == msg->len) + return 1; + } + + /* reset fifo buffer */ + tmp = ioread8(base + ZXI2C_REG_HCR); + iowrite8(tmp | ZXI2C_HCR_RST_FIFO, base + ZXI2C_REG_HCR); + + /* set xfer len */ + priv->xfer_len = min_t(u16, msg->len - i2c->xfered_len, ZXI2C_FIFO_SIZE); + if (read) { + iowrite8(priv->xfer_len - 1, base + ZXI2C_REG_HRLR); + } else { + iowrite8(priv->xfer_len - 1, base + ZXI2C_REG_HTLR); + /* set write data */ + for (i = 0; i < priv->xfer_len; i++) + iowrite8(msg->buf[i2c->xfered_len + i], base + ZXI2C_REG_HTDR); + } + + /* prepare to stop transmission */ + if (priv->hrv && msg->len == (i2c->xfered_len + priv->xfer_len)) { + tmp = ioread8(base + VIAI2C_REG_CR); + tmp |= read ? VIAI2C_CR_RX_END : VIAI2C_CR_TX_END; + iowrite8(tmp, base + VIAI2C_REG_CR); + } + + if (irq) { + /* continue transmission */ + tmp = ioread8(base + VIAI2C_REG_CR); + iowrite8(tmp |= VIAI2C_CR_CPU_RDY, base + VIAI2C_REG_CR); + } else { + u16 tcr_val = i2c->tcr; + + /* start transmission */ + tcr_val |= read ? VIAI2C_TCR_READ : 0; + writew(tcr_val | msg->addr, base + VIAI2C_REG_TCR); + } + + return 0; +} + +static int zxi2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num) +{ + u8 tmp; + int ret; + struct viai2c *i2c = (struct viai2c *)i2c_get_adapdata(adap); + struct viai2c_zhaoxin *priv = i2c->pltfm_priv; + void __iomem *base = i2c->base; + + ret = viai2c_wait_bus_not_busy(i2c); + if (ret) + return ret; + + tmp = ioread8(base + VIAI2C_REG_CR); + tmp &= ~(VIAI2C_CR_RX_END | VIAI2C_CR_TX_END); + + if (num == 1 && msgs->len >= 2 && (priv->hrv || msgs->len <= ZXI2C_FIFO_SIZE)) { + /* enable fifo mode */ + iowrite16(ZXI2C_CR_FIFO_MODE | tmp, base + VIAI2C_REG_CR); + /* clear irq status */ + iowrite8(ZXI2C_IRQ_MASK, base + VIAI2C_REG_ISR); + /* enable fifo irq */ + iowrite8(VIAI2C_ISR_NACK_ADDR | ZXI2C_IRQ_FIFOEND, base + VIAI2C_REG_IMR); + + i2c->msg = msgs; + i2c->mode = VIAI2C_FIFO_MODE; + priv->xfer_len = 0; + i2c->xfered_len = 0; + + viai2c_fifo_irq_xfer(i2c, 0); + + if (!wait_for_completion_timeout(&i2c->complete, VIAI2C_TIMEOUT)) + return -ETIMEDOUT; + + ret = i2c->ret; + } else { + /* enable byte mode */ + iowrite16(tmp, base + VIAI2C_REG_CR); + /* clear irq status */ + iowrite8(ZXI2C_IRQ_MASK, base + VIAI2C_REG_ISR); + /* enable byte irq */ + iowrite8(VIAI2C_ISR_NACK_ADDR | VIAI2C_IMR_BYTE, base + VIAI2C_REG_IMR); + + ret = viai2c_xfer(adap, msgs, num); + if (ret == -ETIMEDOUT) + iowrite16(tmp | VIAI2C_CR_END_MASK, base + VIAI2C_REG_CR); + } + /* dis interrupt */ + iowrite8(0, base + VIAI2C_REG_IMR); + + return ret; +} + +static u32 zxi2c_func(struct i2c_adapter *adap) +{ + return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; +} + +static const struct i2c_algorithm zxi2c_algorithm = { + .master_xfer = zxi2c_master_xfer, + .functionality = zxi2c_func, +}; + +static const struct i2c_adapter_quirks zxi2c_quirks = { + .flags = I2C_AQ_NO_ZERO_LEN | I2C_AQ_COMB_WRITE_THEN_READ, +}; + +static const u32 zxi2c_speed_params_table[][3] = { + /* speed, ZXI2C_TCR, ZXI2C_FSTP */ + { I2C_MAX_STANDARD_MODE_FREQ, 0, ZXI2C_GOLD_FSTP_100K }, + { I2C_MAX_FAST_MODE_FREQ, VIAI2C_TCR_FAST, ZXI2C_GOLD_FSTP_400K }, + { I2C_MAX_FAST_MODE_PLUS_FREQ, VIAI2C_TCR_FAST, ZXI2C_GOLD_FSTP_1M }, + { I2C_MAX_HIGH_SPEED_MODE_FREQ, VIAI2C_TCR_HS_MODE | VIAI2C_TCR_FAST, + ZXI2C_GOLD_FSTP_3400K }, +}; + +static void zxi2c_set_bus_speed(struct viai2c *i2c) +{ + struct viai2c_zhaoxin *priv = i2c->pltfm_priv; + + iowrite16(priv->tr, i2c->base + VIAI2C_REG_TR); + iowrite8(ZXI2C_CLK_50M, i2c->base + ZXI2C_REG_CLK); + iowrite16(priv->mcr, i2c->base + VIAI2C_REG_MCR); +} + +static void zxi2c_get_bus_speed(struct viai2c *i2c) +{ + u8 i, count; + u8 fstp; + const u32 *params; + struct viai2c_zhaoxin *priv = i2c->pltfm_priv; + u32 acpi_speed = i2c_acpi_find_bus_speed(i2c->dev); + + count = ARRAY_SIZE(zxi2c_speed_params_table); + for (i = 0; i < count; i++) + if (acpi_speed == zxi2c_speed_params_table[i][0]) + break; + /* if not found, use 400k as default */ + i = i < count ? i : 1; + + params = zxi2c_speed_params_table[i]; + fstp = ioread8(i2c->base + VIAI2C_REG_TR); + if (abs(fstp - params[2]) > 0x10) { + /* + * if BIOS setting value far from golden value, + * use golden value and warn user + */ + dev_warn(i2c->dev, "FW FSTP[%x] might cause wrong timings, dropped\n", fstp); + priv->tr = params[2] | 0xff00; + } else { + priv->tr = fstp | 0xff00; + } + + i2c->tcr = params[1]; + priv->mcr = ioread16(i2c->base + VIAI2C_REG_MCR); + /* for Hs-mode, use 0x80 as master code */ + if (params[0] == I2C_MAX_HIGH_SPEED_MODE_FREQ) + priv->mcr |= ZXI2C_HS_MASTER_CODE; + + dev_info(i2c->dev, "speed mode is %s\n", i2c_freq_mode_string(params[0])); +} + +static int zxi2c_probe(struct platform_device *pdev) +{ + int error; + struct viai2c *i2c; + struct i2c_adapter *adap; + struct viai2c_zhaoxin *priv; + + error = viai2c_init(pdev, &i2c, VIAI2C_PLAT_ZHAOXIN); + if (error) + return error; + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + i2c->pltfm_priv = priv; + + zxi2c_get_bus_speed(i2c); + zxi2c_set_bus_speed(i2c); + + priv->hrv = ioread8(i2c->base + ZXI2C_REG_REV); + + adap = &i2c->adapter; + adap->owner = THIS_MODULE; + adap->algo = &zxi2c_algorithm; + adap->quirks = &zxi2c_quirks; + adap->dev.parent = &pdev->dev; + ACPI_COMPANION_SET(&adap->dev, ACPI_COMPANION(&pdev->dev)); + snprintf(adap->name, sizeof(adap->name), "zhaoxin-%s-%s", + dev_name(pdev->dev.parent), dev_name(i2c->dev)); + i2c_set_adapdata(adap, i2c); + + return devm_i2c_add_adapter(&pdev->dev, adap); +} + +static int __maybe_unused zxi2c_resume(struct device *dev) +{ + struct viai2c *i2c = dev_get_drvdata(dev); + + iowrite8(ZXI2C_CR_MST_RST, i2c->base + VIAI2C_REG_CR); + zxi2c_set_bus_speed(i2c); + + return 0; +} + +static const struct dev_pm_ops zxi2c_pm = { + SET_SYSTEM_SLEEP_PM_OPS(NULL, zxi2c_resume) +}; + +static const struct acpi_device_id zxi2c_acpi_match[] = { + {"IIC1D17", 0 }, + { } +}; +MODULE_DEVICE_TABLE(acpi, zxi2c_acpi_match); + +static struct platform_driver zxi2c_driver = { + .probe = zxi2c_probe, + .driver = { + .name = "i2c_zhaoxin", + .acpi_match_table = zxi2c_acpi_match, + .pm = &zxi2c_pm, + }, +}; + +module_platform_driver(zxi2c_driver); + +MODULE_AUTHOR("HansHu@zhaoxin.com"); +MODULE_DESCRIPTION("Shanghai Zhaoxin IIC driver"); +MODULE_LICENSE("GPL"); From b757eb090fbdac31e2f144492c17202d3cf41d9b Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Tue, 23 Apr 2024 14:13:18 +0200 Subject: [PATCH 1092/1702] i2c: i801: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout and turn the SMBus-specific termination message to debug. Signed-off-by: Wolfram Sang Reviewed-by: Jean Delvare Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-i801.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index e577abc776c1e..d2d2a6dbe29f2 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -399,9 +399,7 @@ static int i801_check_post(struct i801_priv *priv, int status) * If the SMBus is still busy, we give up */ if (unlikely(status < 0)) { - dev_err(&priv->pci_dev->dev, "Transaction timeout\n"); /* try to stop the current command */ - dev_dbg(&priv->pci_dev->dev, "Terminating the current operation\n"); outb_p(SMBHSTCNT_KILL, SMBHSTCNT(priv)); usleep_range(1000, 2000); outb_p(0, SMBHSTCNT(priv)); @@ -410,7 +408,7 @@ static int i801_check_post(struct i801_priv *priv, int status) status = inb_p(SMBHSTSTS(priv)); if ((status & SMBHSTSTS_HOST_BUSY) || !(status & SMBHSTSTS_FAILED)) - dev_err(&priv->pci_dev->dev, + dev_dbg(&priv->pci_dev->dev, "Failed terminating the transaction\n"); return -ETIMEDOUT; } From aaf20dbceb063760eb1c8beb5e6a11c5ebacd3d5 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Tue, 23 Apr 2024 14:13:19 +0200 Subject: [PATCH 1093/1702] i2c: ali1535: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Reviewed-by: Jean Delvare Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-ali1535.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/i2c/busses/i2c-ali1535.c b/drivers/i2c/busses/i2c-ali1535.c index 461eb23f9d476..9d7b4efe26ad0 100644 --- a/drivers/i2c/busses/i2c-ali1535.c +++ b/drivers/i2c/busses/i2c-ali1535.c @@ -285,10 +285,8 @@ static int ali1535_transaction(struct i2c_adapter *adap) && (timeout++ < MAX_TIMEOUT)); /* If the SMBus is still busy, we give up */ - if (timeout > MAX_TIMEOUT) { + if (timeout > MAX_TIMEOUT) result = -ETIMEDOUT; - dev_err(&adap->dev, "SMBus Timeout!\n"); - } if (temp & ALI1535_STS_FAIL) { result = -EIO; @@ -313,10 +311,8 @@ static int ali1535_transaction(struct i2c_adapter *adap) } /* check to see if the "command complete" indication is set */ - if (!(temp & ALI1535_STS_DONE)) { + if (!(temp & ALI1535_STS_DONE)) result = -ETIMEDOUT; - dev_err(&adap->dev, "Error: command never completed\n"); - } dev_dbg(&adap->dev, "Transaction (post): STS=%02x, TYP=%02x, " "CMD=%02x, ADD=%02x, DAT0=%02x, DAT1=%02x\n", From d178a2fc252d4790841f6c841ad5b43940c71a8f Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Tue, 23 Apr 2024 14:13:20 +0200 Subject: [PATCH 1094/1702] i2c: ali1563: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Reviewed-by: Jean Delvare Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-ali1563.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/i2c/busses/i2c-ali1563.c b/drivers/i2c/busses/i2c-ali1563.c index 307fb0666ecb2..63897a89bb35f 100644 --- a/drivers/i2c/busses/i2c-ali1563.c +++ b/drivers/i2c/busses/i2c-ali1563.c @@ -99,7 +99,6 @@ static int ali1563_transaction(struct i2c_adapter *a, int size) return 0; if (!timeout) { - dev_err(&a->dev, "Timeout - Trying to KILL transaction!\n"); /* Issue 'kill' to host controller */ outb_p(HST_CNTL2_KILL, SMB_HST_CNTL2); data = inb_p(SMB_HST_STS); From 5895a867bbdd7fb8a678baaf0c8746f592727edd Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Tue, 23 Apr 2024 14:13:21 +0200 Subject: [PATCH 1095/1702] i2c: ali15x3: remove printout on handled timeouts I2C and SMBus timeouts are not something the user needs to be informed about on controller level. The client driver may know if that really is a problem and give more detailed information to the user. The controller should just pass this information upwards. Remove the printout. Signed-off-by: Wolfram Sang Reviewed-by: Jean Delvare Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-ali15x3.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/i2c/busses/i2c-ali15x3.c b/drivers/i2c/busses/i2c-ali15x3.c index d2fa30deb054c..956e5020d71e2 100644 --- a/drivers/i2c/busses/i2c-ali15x3.c +++ b/drivers/i2c/busses/i2c-ali15x3.c @@ -294,10 +294,8 @@ static int ali15x3_transaction(struct i2c_adapter *adap) && (timeout++ < MAX_TIMEOUT)); /* If the SMBus is still busy, we give up */ - if (timeout > MAX_TIMEOUT) { + if (timeout > MAX_TIMEOUT) result = -ETIMEDOUT; - dev_err(&adap->dev, "SMBus Timeout!\n"); - } if (temp & ALI15X3_STS_TERM) { result = -EIO; From 571b90f5d43c8846bc33dded9366d972f4290bae Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Sat, 27 Apr 2024 22:35:53 +0200 Subject: [PATCH 1096/1702] i2c: amd-mp2-plat: use 'time_left' variable with wait_for_completion_timeout() There is a confusing pattern in the kernel to use a variable named 'timeout' to store the result of wait_for_completion_timeout() causing patterns like: timeout = wait_for_completion_timeout(...) if (!timeout) return -ETIMEDOUT; with all kinds of permutations. Use 'time_left' as a variable to make the code self explaining. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-amd-mp2-plat.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/i2c/busses/i2c-amd-mp2-plat.c b/drivers/i2c/busses/i2c-amd-mp2-plat.c index 112fe2bc5662b..d3ac1c77a5098 100644 --- a/drivers/i2c/busses/i2c-amd-mp2-plat.c +++ b/drivers/i2c/busses/i2c-amd-mp2-plat.c @@ -97,17 +97,17 @@ static void i2c_amd_cmd_completion(struct amd_i2c_common *i2c_common) static int i2c_amd_check_cmd_completion(struct amd_i2c_dev *i2c_dev) { struct amd_i2c_common *i2c_common = &i2c_dev->common; - unsigned long timeout; + unsigned long time_left; - timeout = wait_for_completion_timeout(&i2c_dev->cmd_complete, - i2c_dev->adap.timeout); + time_left = wait_for_completion_timeout(&i2c_dev->cmd_complete, + i2c_dev->adap.timeout); if ((i2c_common->reqcmd == i2c_read || i2c_common->reqcmd == i2c_write) && i2c_common->msg->len > 32) i2c_amd_dma_unmap(i2c_common); - if (timeout == 0) { + if (time_left == 0) { amd_mp2_rw_timeout(i2c_common); return -ETIMEDOUT; } From 4804a8b786c81bb9c6d3e749ff0a841fa1d7dba5 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Sat, 27 Apr 2024 22:35:54 +0200 Subject: [PATCH 1097/1702] i2c: digicolor: use 'time_left' variable with wait_for_completion_timeout() There is a confusing pattern in the kernel to use a variable named 'timeout' to store the result of wait_for_completion_timeout() causing patterns like: timeout = wait_for_completion_timeout(...) if (!timeout) return -ETIMEDOUT; with all kinds of permutations. Use 'time_left' as a variable to make the code self explaining. Signed-off-by: Wolfram Sang Acked-by: Baruch Siach Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-digicolor.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/i2c/busses/i2c-digicolor.c b/drivers/i2c/busses/i2c-digicolor.c index 3462f2bc0fa87..737604ae11fc6 100644 --- a/drivers/i2c/busses/i2c-digicolor.c +++ b/drivers/i2c/busses/i2c-digicolor.c @@ -213,7 +213,7 @@ static irqreturn_t dc_i2c_irq(int irq, void *dev_id) static int dc_i2c_xfer_msg(struct dc_i2c *i2c, struct i2c_msg *msg, int first, int last) { - unsigned long timeout = msecs_to_jiffies(TIMEOUT_MS); + unsigned long time_left = msecs_to_jiffies(TIMEOUT_MS); unsigned long flags; spin_lock_irqsave(&i2c->lock, flags); @@ -227,9 +227,9 @@ static int dc_i2c_xfer_msg(struct dc_i2c *i2c, struct i2c_msg *msg, int first, dc_i2c_start_msg(i2c, first); spin_unlock_irqrestore(&i2c->lock, flags); - timeout = wait_for_completion_timeout(&i2c->done, timeout); + time_left = wait_for_completion_timeout(&i2c->done, time_left); dc_i2c_set_irq(i2c, 0); - if (timeout == 0) { + if (time_left == 0) { i2c->state = STATE_IDLE; return -ETIMEDOUT; } From 9b238f0d5612a2f48b99696c9d24c22e5c4e617e Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Sat, 27 Apr 2024 22:35:55 +0200 Subject: [PATCH 1098/1702] i2c: exynos5: use 'time_left' variable with wait_for_completion_timeout() There is a confusing pattern in the kernel to use a variable named 'timeout' to store the result of wait_for_completion_timeout() causing patterns like: timeout = wait_for_completion_timeout(...) if (!timeout) return -ETIMEDOUT; with all kinds of permutations. Use 'time_left' as a variable to make the code self explaining. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-exynos5.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/i2c/busses/i2c-exynos5.c b/drivers/i2c/busses/i2c-exynos5.c index 385ef9d9e4d4c..d8baca9b610c3 100644 --- a/drivers/i2c/busses/i2c-exynos5.c +++ b/drivers/i2c/busses/i2c-exynos5.c @@ -763,7 +763,7 @@ static bool exynos5_i2c_poll_irqs_timeout(struct exynos5_i2c *i2c, static int exynos5_i2c_xfer_msg(struct exynos5_i2c *i2c, struct i2c_msg *msgs, int stop) { - unsigned long timeout; + unsigned long time_left; int ret; i2c->msg = msgs; @@ -775,13 +775,13 @@ static int exynos5_i2c_xfer_msg(struct exynos5_i2c *i2c, exynos5_i2c_message_start(i2c, stop); if (!i2c->atomic) - timeout = wait_for_completion_timeout(&i2c->msg_complete, - EXYNOS5_I2C_TIMEOUT); - else - timeout = exynos5_i2c_poll_irqs_timeout(i2c, + time_left = wait_for_completion_timeout(&i2c->msg_complete, EXYNOS5_I2C_TIMEOUT); + else + time_left = exynos5_i2c_poll_irqs_timeout(i2c, + EXYNOS5_I2C_TIMEOUT); - if (timeout == 0) + if (time_left == 0) ret = -ETIMEDOUT; else ret = i2c->state; From b557e267c543b1a1732841fa276f4630a85470d8 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Sat, 27 Apr 2024 22:35:56 +0200 Subject: [PATCH 1099/1702] i2c: hix5hd2: use 'time_left' variable with wait_for_completion_timeout() There is a confusing pattern in the kernel to use a variable named 'timeout' to store the result of wait_for_completion_timeout() causing patterns like: timeout = wait_for_completion_timeout(...) if (!timeout) return -ETIMEDOUT; with all kinds of permutations. Use 'time_left' as a variable to make the code self explaining. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-hix5hd2.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/i2c/busses/i2c-hix5hd2.c b/drivers/i2c/busses/i2c-hix5hd2.c index 8e75515c3ca47..a47b9939fa2ce 100644 --- a/drivers/i2c/busses/i2c-hix5hd2.c +++ b/drivers/i2c/busses/i2c-hix5hd2.c @@ -314,7 +314,7 @@ static void hix5hd2_i2c_message_start(struct hix5hd2_i2c_priv *priv, int stop) static int hix5hd2_i2c_xfer_msg(struct hix5hd2_i2c_priv *priv, struct i2c_msg *msgs, int stop) { - unsigned long timeout; + unsigned long time_left; int ret; priv->msg = msgs; @@ -327,9 +327,9 @@ static int hix5hd2_i2c_xfer_msg(struct hix5hd2_i2c_priv *priv, reinit_completion(&priv->msg_complete); hix5hd2_i2c_message_start(priv, stop); - timeout = wait_for_completion_timeout(&priv->msg_complete, - priv->adap.timeout); - if (timeout == 0) { + time_left = wait_for_completion_timeout(&priv->msg_complete, + priv->adap.timeout); + if (time_left == 0) { priv->state = HIX5I2C_STAT_RW_ERR; priv->err = -ETIMEDOUT; dev_warn(priv->dev, "%s timeout=%d\n", From 8dacd79fec3cd86ddf6b8c87b0179c5b13250a9c Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Sat, 27 Apr 2024 22:35:57 +0200 Subject: [PATCH 1100/1702] i2c: imx-lpi2c: use 'time_left' variable with wait_for_completion_timeout() There is a confusing pattern in the kernel to use a variable named 'timeout' to store the result of wait_for_completion_timeout() causing patterns like: timeout = wait_for_completion_timeout(...) if (!timeout) return -ETIMEDOUT; with all kinds of permutations. Use 'time_left' as a variable to make the code self explaining. Signed-off-by: Wolfram Sang Reviewed-by: Peng Fan Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-imx-lpi2c.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c index 36e8f6196a87b..0197786892a24 100644 --- a/drivers/i2c/busses/i2c-imx-lpi2c.c +++ b/drivers/i2c/busses/i2c-imx-lpi2c.c @@ -307,11 +307,11 @@ static int lpi2c_imx_master_disable(struct lpi2c_imx_struct *lpi2c_imx) static int lpi2c_imx_msg_complete(struct lpi2c_imx_struct *lpi2c_imx) { - unsigned long timeout; + unsigned long time_left; - timeout = wait_for_completion_timeout(&lpi2c_imx->complete, HZ); + time_left = wait_for_completion_timeout(&lpi2c_imx->complete, HZ); - return timeout ? 0 : -ETIMEDOUT; + return time_left ? 0 : -ETIMEDOUT; } static int lpi2c_imx_txfifo_empty(struct lpi2c_imx_struct *lpi2c_imx) From c1f8f664678a50997dee8a92db4800f900711e73 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Sat, 27 Apr 2024 22:35:58 +0200 Subject: [PATCH 1101/1702] i2c: omap: use 'time_left' variable with wait_for_completion_timeout() There is a confusing pattern in the kernel to use a variable named 'timeout' to store the result of wait_for_completion_timeout() causing patterns like: timeout = wait_for_completion_timeout(...) if (!timeout) return -ETIMEDOUT; with all kinds of permutations. Use 'time_left' as a variable to make the code self explaining. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-omap.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/i2c/busses/i2c-omap.c b/drivers/i2c/busses/i2c-omap.c index 36bebef36740d..30a5ea282a8b0 100644 --- a/drivers/i2c/busses/i2c-omap.c +++ b/drivers/i2c/busses/i2c-omap.c @@ -660,7 +660,7 @@ static int omap_i2c_xfer_msg(struct i2c_adapter *adap, struct i2c_msg *msg, int stop, bool polling) { struct omap_i2c_dev *omap = i2c_get_adapdata(adap); - unsigned long timeout; + unsigned long time_left; u16 w; int ret; @@ -740,18 +740,18 @@ static int omap_i2c_xfer_msg(struct i2c_adapter *adap, * into arbitration and we're currently unable to recover from it. */ if (!polling) { - timeout = wait_for_completion_timeout(&omap->cmd_complete, - OMAP_I2C_TIMEOUT); + time_left = wait_for_completion_timeout(&omap->cmd_complete, + OMAP_I2C_TIMEOUT); } else { do { omap_i2c_wait(omap); ret = omap_i2c_xfer_data(omap); } while (ret == -EAGAIN); - timeout = !ret; + time_left = !ret; } - if (timeout == 0) { + if (time_left == 0) { omap_i2c_reset(omap); __omap_i2c_init(omap); return -ETIMEDOUT; From 7ae38232ebc4dd19627c5a5c2cc797aa487fd511 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Sat, 27 Apr 2024 22:35:59 +0200 Subject: [PATCH 1102/1702] i2c: st: use 'time_left' variable with wait_for_completion_timeout() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is a confusing pattern in the kernel to use a variable named 'timeout' to store the result of wait_for_completion_timeout() causing patterns like: timeout = wait_for_completion_timeout(...) if (!timeout) return -ETIMEDOUT; with all kinds of permutations. Use 'time_left' as a variable to make the code self explaining. Signed-off-by: Wolfram Sang Reviewed-by: Uwe Kleine-König Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-st.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/i2c/busses/i2c-st.c b/drivers/i2c/busses/i2c-st.c index dbb93394ff946..5e01fe3dbb63f 100644 --- a/drivers/i2c/busses/i2c-st.c +++ b/drivers/i2c/busses/i2c-st.c @@ -647,7 +647,7 @@ static int st_i2c_xfer_msg(struct st_i2c_dev *i2c_dev, struct i2c_msg *msg, { struct st_i2c_client *c = &i2c_dev->client; u32 ctl, i2c, it; - unsigned long timeout; + unsigned long time_left; int ret; c->addr = i2c_8bit_addr_from_msg(msg); @@ -685,11 +685,11 @@ static int st_i2c_xfer_msg(struct st_i2c_dev *i2c_dev, struct i2c_msg *msg, st_i2c_set_bits(i2c_dev->base + SSC_I2C, SSC_I2C_STRTG); } - timeout = wait_for_completion_timeout(&i2c_dev->complete, - i2c_dev->adap.timeout); + time_left = wait_for_completion_timeout(&i2c_dev->complete, + i2c_dev->adap.timeout); ret = c->result; - if (!timeout) + if (!time_left) ret = -ETIMEDOUT; i2c = SSC_I2C_STOPG | SSC_I2C_REPSTRTG; From 8839a8df93077bfb10710a1fd8ad8b7ac3ac2abe Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Sat, 27 Apr 2024 22:36:00 +0200 Subject: [PATCH 1103/1702] i2c: stm32f4: use 'time_left' variable with wait_for_completion_timeout() There is a confusing pattern in the kernel to use a variable named 'timeout' to store the result of wait_for_completion_timeout() causing patterns like: timeout = wait_for_completion_timeout(...) if (!timeout) return -ETIMEDOUT; with all kinds of permutations. Use 'time_left' as a variable to make the code self explaining. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-stm32f4.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/i2c/busses/i2c-stm32f4.c b/drivers/i2c/busses/i2c-stm32f4.c index 859ac0cf7f6cb..f8b12be6ef55b 100644 --- a/drivers/i2c/busses/i2c-stm32f4.c +++ b/drivers/i2c/busses/i2c-stm32f4.c @@ -681,7 +681,7 @@ static int stm32f4_i2c_xfer_msg(struct stm32f4_i2c_dev *i2c_dev, { struct stm32f4_i2c_msg *f4_msg = &i2c_dev->msg; void __iomem *reg = i2c_dev->base + STM32F4_I2C_CR1; - unsigned long timeout; + unsigned long time_left; u32 mask; int ret; @@ -706,11 +706,11 @@ static int stm32f4_i2c_xfer_msg(struct stm32f4_i2c_dev *i2c_dev, stm32f4_i2c_set_bits(reg, STM32F4_I2C_CR1_START); } - timeout = wait_for_completion_timeout(&i2c_dev->complete, - i2c_dev->adap.timeout); + time_left = wait_for_completion_timeout(&i2c_dev->complete, + i2c_dev->adap.timeout); ret = f4_msg->result; - if (!timeout) + if (!time_left) ret = -ETIMEDOUT; return ret; From 197264d377b875dfc5be3c12125bc35fc3643204 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Sat, 27 Apr 2024 22:36:01 +0200 Subject: [PATCH 1104/1702] i2c: stm32f7: use 'time_left' variable with wait_for_completion_timeout() There is a confusing pattern in the kernel to use a variable named 'timeout' to store the result of wait_for_completion_timeout() causing patterns like: timeout = wait_for_completion_timeout(...) if (!timeout) return -ETIMEDOUT; with all kinds of permutations. Use 'time_left' as a variable to make the code self explaining. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-stm32f7.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c index 01210452216b3..cfee2d9c09de3 100644 --- a/drivers/i2c/busses/i2c-stm32f7.c +++ b/drivers/i2c/busses/i2c-stm32f7.c @@ -1789,7 +1789,7 @@ static int stm32f7_i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr, struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg; struct stm32_i2c_dma *dma = i2c_dev->dma; struct device *dev = i2c_dev->dev; - unsigned long timeout; + unsigned long time_left; int i, ret; f7_msg->addr = addr; @@ -1809,8 +1809,8 @@ static int stm32f7_i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr, if (ret) goto pm_free; - timeout = wait_for_completion_timeout(&i2c_dev->complete, - i2c_dev->adap.timeout); + time_left = wait_for_completion_timeout(&i2c_dev->complete, + i2c_dev->adap.timeout); ret = f7_msg->result; if (ret) { if (i2c_dev->use_dma) @@ -1826,7 +1826,7 @@ static int stm32f7_i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr, goto pm_free; } - if (!timeout) { + if (!time_left) { dev_dbg(dev, "Access to slave 0x%x timed out\n", f7_msg->addr); if (i2c_dev->use_dma) dmaengine_terminate_sync(dma->chan_using); From 4259964b4a5299cb4fb9f5e06dccafb2ae05cb20 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Sat, 27 Apr 2024 22:36:02 +0200 Subject: [PATCH 1105/1702] i2c: synquacer: use 'time_left' variable with wait_for_completion_timeout() There is a confusing pattern in the kernel to use a variable named 'timeout' to store the result of wait_for_completion_timeout() causing patterns like: timeout = wait_for_completion_timeout(...) if (!timeout) return -ETIMEDOUT; with all kinds of permutations. Use 'time_left' as a variable to make the code self explaining. Signed-off-by: Wolfram Sang Acked-by: Ard Biesheuvel Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-synquacer.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/i2c/busses/i2c-synquacer.c b/drivers/i2c/busses/i2c-synquacer.c index bbea521b05dda..bf9bcfefa30e3 100644 --- a/drivers/i2c/busses/i2c-synquacer.c +++ b/drivers/i2c/busses/i2c-synquacer.c @@ -311,7 +311,7 @@ static int synquacer_i2c_doxfer(struct synquacer_i2c *i2c, struct i2c_msg *msgs, int num) { unsigned char bsr; - unsigned long timeout; + unsigned long time_left; int ret; synquacer_i2c_hw_init(i2c); @@ -335,9 +335,9 @@ static int synquacer_i2c_doxfer(struct synquacer_i2c *i2c, return ret; } - timeout = wait_for_completion_timeout(&i2c->completion, - msecs_to_jiffies(i2c->timeout_ms)); - if (timeout == 0) { + time_left = wait_for_completion_timeout(&i2c->completion, + msecs_to_jiffies(i2c->timeout_ms)); + if (time_left == 0) { dev_dbg(i2c->dev, "timeout\n"); return -EAGAIN; } From 66aa72ced659c5cf3e14674e04450f92e845a699 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Sat, 27 Apr 2024 22:36:03 +0200 Subject: [PATCH 1106/1702] i2c: jz4780: use 'time_left' variable with wait_for_completion_timeout() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is a confusing pattern in the kernel to use a variable named 'timeout' to store the result of wait_for_completion_timeout() causing patterns like: timeout = wait_for_completion_timeout(...) if (!timeout) return -ETIMEDOUT; with all kinds of permutations. Use 'time_left' as a variable to make the code self explaining. Fix to the proper variable type 'unsigned long' while here. Signed-off-by: Wolfram Sang Acked-by: Paul Cercueil Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-jz4780.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/i2c/busses/i2c-jz4780.c b/drivers/i2c/busses/i2c-jz4780.c index 55035cca0ae53..7951891d6b97c 100644 --- a/drivers/i2c/busses/i2c-jz4780.c +++ b/drivers/i2c/busses/i2c-jz4780.c @@ -565,7 +565,7 @@ static inline int jz4780_i2c_xfer_read(struct jz4780_i2c *i2c, int idx) { int ret = 0; - long timeout; + unsigned long time_left; int wait_time = JZ4780_I2C_TIMEOUT * (len + 5); unsigned short tmp; unsigned long flags; @@ -600,10 +600,10 @@ static inline int jz4780_i2c_xfer_read(struct jz4780_i2c *i2c, spin_unlock_irqrestore(&i2c->lock, flags); - timeout = wait_for_completion_timeout(&i2c->trans_waitq, - msecs_to_jiffies(wait_time)); + time_left = wait_for_completion_timeout(&i2c->trans_waitq, + msecs_to_jiffies(wait_time)); - if (!timeout) { + if (!time_left) { dev_err(&i2c->adap.dev, "irq read timeout\n"); dev_dbg(&i2c->adap.dev, "send cmd count:%d %d\n", i2c->cmd, i2c->cmd_buf[i2c->cmd]); @@ -627,7 +627,7 @@ static inline int jz4780_i2c_xfer_write(struct jz4780_i2c *i2c, { int ret = 0; int wait_time = JZ4780_I2C_TIMEOUT * (len + 5); - long timeout; + unsigned long time_left; unsigned short tmp; unsigned long flags; @@ -655,14 +655,14 @@ static inline int jz4780_i2c_xfer_write(struct jz4780_i2c *i2c, spin_unlock_irqrestore(&i2c->lock, flags); - timeout = wait_for_completion_timeout(&i2c->trans_waitq, - msecs_to_jiffies(wait_time)); - if (timeout && !i2c->stop_hold) { + time_left = wait_for_completion_timeout(&i2c->trans_waitq, + msecs_to_jiffies(wait_time)); + if (time_left && !i2c->stop_hold) { unsigned short i2c_sta; int write_in_process; - timeout = JZ4780_I2C_TIMEOUT * 100; - for (; timeout > 0; timeout--) { + time_left = JZ4780_I2C_TIMEOUT * 100; + for (; time_left > 0; time_left--) { i2c_sta = jz4780_i2c_readw(i2c, JZ4780_I2C_STA); write_in_process = (i2c_sta & JZ4780_I2C_STA_MSTACT) || @@ -673,7 +673,7 @@ static inline int jz4780_i2c_xfer_write(struct jz4780_i2c *i2c, } } - if (!timeout) { + if (!time_left) { dev_err(&i2c->adap.dev, "write wait timeout\n"); ret = -EIO; } From f9288ff67a8f4bc2645d32a512d4b30f73f956d5 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Sat, 27 Apr 2024 22:36:04 +0200 Subject: [PATCH 1107/1702] i2c: qcom-geni: use 'time_left' variable with wait_for_completion_timeout() There is a confusing pattern in the kernel to use a variable named 'timeout' to store the result of wait_for_completion_timeout() causing patterns like: timeout = wait_for_completion_timeout(...) if (!timeout) return -ETIMEDOUT; with all kinds of permutations. Use 'time_left' as a variable to make the code self explaining. Fix to the proper variable type 'unsigned long' while here. Signed-off-by: Wolfram Sang Reviewed-by: Bjorn Andersson Reviewed-by: Bryan O'Donoghue Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-qcom-geni.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/i2c/busses/i2c-qcom-geni.c b/drivers/i2c/busses/i2c-qcom-geni.c index 090b4846ed62b..0a8b95ce35f79 100644 --- a/drivers/i2c/busses/i2c-qcom-geni.c +++ b/drivers/i2c/busses/i2c-qcom-geni.c @@ -586,7 +586,8 @@ static int geni_i2c_gpi_xfer(struct geni_i2c_dev *gi2c, struct i2c_msg msgs[], i { struct dma_slave_config config = {}; struct gpi_i2c_config peripheral = {}; - int i, ret = 0, timeout; + int i, ret = 0; + unsigned long time_left; dma_addr_t tx_addr, rx_addr; void *tx_buf = NULL, *rx_buf = NULL; const struct geni_i2c_clk_fld *itr = gi2c->clk_fld; @@ -629,8 +630,8 @@ static int geni_i2c_gpi_xfer(struct geni_i2c_dev *gi2c, struct i2c_msg msgs[], i dma_async_issue_pending(gi2c->tx_c); - timeout = wait_for_completion_timeout(&gi2c->done, XFER_TIMEOUT); - if (!timeout) + time_left = wait_for_completion_timeout(&gi2c->done, XFER_TIMEOUT); + if (!time_left) gi2c->err = -ETIMEDOUT; if (gi2c->err) { From 749de720c7e568836e62c09508c92f00634c9bc3 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Sat, 27 Apr 2024 22:36:05 +0200 Subject: [PATCH 1108/1702] i2c: rk3x: use 'time_left' variable with wait_event_timeout() There is a confusing pattern in the kernel to use a variable named 'timeout' to store the result of wait_event_timeout() causing patterns like: timeout = wait_event_timeout(...) if (!timeout) return -ETIMEDOUT; with all kinds of permutations. Use 'time_left' as a variable to make the code self explaining. Fix to the proper variable type 'long' while here. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-rk3x.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c index 8c7367f289d36..beca61700c89c 100644 --- a/drivers/i2c/busses/i2c-rk3x.c +++ b/drivers/i2c/busses/i2c-rk3x.c @@ -1060,7 +1060,8 @@ static int rk3x_i2c_xfer_common(struct i2c_adapter *adap, struct i2c_msg *msgs, int num, bool polling) { struct rk3x_i2c *i2c = (struct rk3x_i2c *)adap->algo_data; - unsigned long timeout, flags; + unsigned long flags; + long time_left; u32 val; int ret = 0; int i; @@ -1092,20 +1093,20 @@ static int rk3x_i2c_xfer_common(struct i2c_adapter *adap, if (!polling) { rk3x_i2c_start(i2c); - timeout = wait_event_timeout(i2c->wait, !i2c->busy, - msecs_to_jiffies(WAIT_TIMEOUT)); + time_left = wait_event_timeout(i2c->wait, !i2c->busy, + msecs_to_jiffies(WAIT_TIMEOUT)); } else { disable_irq(i2c->irq); rk3x_i2c_start(i2c); - timeout = rk3x_i2c_wait_xfer_poll(i2c); + time_left = rk3x_i2c_wait_xfer_poll(i2c); enable_irq(i2c->irq); } spin_lock_irqsave(&i2c->lock, flags); - if (timeout == 0) { + if (time_left == 0) { /* Force a STOP condition without interrupt */ i2c_writel(i2c, 0, REG_IEN); val = i2c_readl(i2c, REG_CON) & REG_CON_TUNING_MASK; From bc0ff8022f26f8ee65dfea31fe590daf8adc8012 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Sat, 27 Apr 2024 22:36:06 +0200 Subject: [PATCH 1109/1702] i2c: s3c2410: use 'time_left' variable with wait_event_timeout() There is a confusing pattern in the kernel to use a variable named 'timeout' to store the result of wait_event_timeout() causing patterns like: timeout = wait_event_timeout(...) if (!timeout) return -ETIMEDOUT; with all kinds of permutations. Use 'time_left' as a variable to make the code self explaining. Fix to the proper variable type 'long' while here. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-s3c2410.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/i2c/busses/i2c-s3c2410.c b/drivers/i2c/busses/i2c-s3c2410.c index 275f7c42165cd..01419c738cfc4 100644 --- a/drivers/i2c/busses/i2c-s3c2410.c +++ b/drivers/i2c/busses/i2c-s3c2410.c @@ -685,7 +685,7 @@ static void s3c24xx_i2c_wait_idle(struct s3c24xx_i2c *i2c) static int s3c24xx_i2c_doxfer(struct s3c24xx_i2c *i2c, struct i2c_msg *msgs, int num) { - unsigned long timeout = 0; + long time_left = 0; int ret; ret = s3c24xx_i2c_set_master(i2c); @@ -715,7 +715,7 @@ static int s3c24xx_i2c_doxfer(struct s3c24xx_i2c *i2c, dev_err(i2c->dev, "deal with arbitration loss\n"); } } else { - timeout = wait_event_timeout(i2c->wait, i2c->msg_num == 0, HZ * 5); + time_left = wait_event_timeout(i2c->wait, i2c->msg_num == 0, HZ * 5); } ret = i2c->msg_idx; @@ -724,7 +724,7 @@ static int s3c24xx_i2c_doxfer(struct s3c24xx_i2c *i2c, * Having these next two as dev_err() makes life very * noisy when doing an i2cdetect */ - if (timeout == 0) + if (time_left == 0) dev_dbg(i2c->dev, "timeout\n"); else if (ret != num) dev_dbg(i2c->dev, "incomplete xfer (%d)\n", ret); From 1ce1ad6de56591b7e8ea7d056f0a179b47ed9480 Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Sat, 27 Apr 2024 22:36:07 +0200 Subject: [PATCH 1110/1702] i2c: pxa: use 'time_left' variable with wait_event_timeout() There is a confusing pattern in the kernel to use a variable named 'timeout' to store the result of wait_event_timeout() causing patterns like: timeout = wait_event_timeout(...) if (!timeout) return -ETIMEDOUT; with all kinds of permutations. Use 'time_left' as a variable to make the code self explaining. Signed-off-by: Wolfram Sang Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-pxa.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/i2c/busses/i2c-pxa.c b/drivers/i2c/busses/i2c-pxa.c index 888ca636f3f3b..f495560bd99c1 100644 --- a/drivers/i2c/busses/i2c-pxa.c +++ b/drivers/i2c/busses/i2c-pxa.c @@ -826,7 +826,7 @@ static inline void i2c_pxa_stop_message(struct pxa_i2c *i2c) static int i2c_pxa_send_mastercode(struct pxa_i2c *i2c) { u32 icr; - long timeout; + long time_left; spin_lock_irq(&i2c->lock); i2c->highmode_enter = true; @@ -837,12 +837,12 @@ static int i2c_pxa_send_mastercode(struct pxa_i2c *i2c) writel(icr, _ICR(i2c)); spin_unlock_irq(&i2c->lock); - timeout = wait_event_timeout(i2c->wait, - i2c->highmode_enter == false, HZ * 1); + time_left = wait_event_timeout(i2c->wait, + i2c->highmode_enter == false, HZ * 1); i2c->highmode_enter = false; - return (timeout == 0) ? I2C_RETRY : 0; + return (time_left == 0) ? I2C_RETRY : 0; } /* @@ -1050,7 +1050,7 @@ static irqreturn_t i2c_pxa_handler(int this_irq, void *dev_id) */ static int i2c_pxa_do_xfer(struct pxa_i2c *i2c, struct i2c_msg *msg, int num) { - long timeout; + long time_left; int ret; /* @@ -1095,7 +1095,7 @@ static int i2c_pxa_do_xfer(struct pxa_i2c *i2c, struct i2c_msg *msg, int num) /* * The rest of the processing occurs in the interrupt handler. */ - timeout = wait_event_timeout(i2c->wait, i2c->msg_num == 0, HZ * 5); + time_left = wait_event_timeout(i2c->wait, i2c->msg_num == 0, HZ * 5); i2c_pxa_stop_message(i2c); /* @@ -1103,7 +1103,7 @@ static int i2c_pxa_do_xfer(struct pxa_i2c *i2c, struct i2c_msg *msg, int num) */ ret = i2c->msg_idx; - if (!timeout && i2c->msg_num) { + if (!time_left && i2c->msg_num) { i2c_pxa_scream_blue_murder(i2c, "timeout with active message"); i2c_recover_bus(&i2c->adap); ret = I2C_RETRY; From 61e05bad821cb293418794738e7cc359949f44fe Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 25 Apr 2024 14:44:34 -0700 Subject: [PATCH 1111/1702] i2c: designware: Replace MODULE_ALIAS() with MODULE_DEVICE_TABLE() As Krzysztof Kozlowski pointed out the better is to use MODULE_DEVICE_TABLE() as it will be consistent with the content of the real ID table of the platform devices. While at it, drop unneeded and unused module alias in PCI glue driver as PCI already has its own ID table and automatic loading should just work. Reviewed-by: Andi Shyti Reviewed-by: Mario Limonciello Acked-by: Jarkko Nikula Tested-by: Jarkko Nikula Tested-by: Serge Semin Link: https://lore.kernel.org/r/20231120144641.1660574-9-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-designware-pcidrv.c | 2 -- drivers/i2c/busses/i2c-designware-platdrv.c | 8 ++++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/i2c/busses/i2c-designware-pcidrv.c b/drivers/i2c/busses/i2c-designware-pcidrv.c index 9be9a2658e1f6..a1b379a1e9040 100644 --- a/drivers/i2c/busses/i2c-designware-pcidrv.c +++ b/drivers/i2c/busses/i2c-designware-pcidrv.c @@ -424,8 +424,6 @@ static struct pci_driver dw_i2c_driver = { }; module_pci_driver(dw_i2c_driver); -/* Work with hotplug and coldplug */ -MODULE_ALIAS("i2c_designware-pci"); MODULE_AUTHOR("Baruch Siach "); MODULE_DESCRIPTION("Synopsys DesignWare PCI I2C bus adapter"); MODULE_LICENSE("GPL"); diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c index 926fb74a85703..29aac9c873684 100644 --- a/drivers/i2c/busses/i2c-designware-platdrv.c +++ b/drivers/i2c/busses/i2c-designware-platdrv.c @@ -480,8 +480,11 @@ static const struct dev_pm_ops dw_i2c_dev_pm_ops = { RUNTIME_PM_OPS(dw_i2c_plat_runtime_suspend, dw_i2c_plat_runtime_resume, NULL) }; -/* Work with hotplug and coldplug */ -MODULE_ALIAS("platform:i2c_designware"); +static const struct platform_device_id dw_i2c_platform_ids[] = { + { "i2c_designware" }, + {} +}; +MODULE_DEVICE_TABLE(platform, dw_i2c_platform_ids); static struct platform_driver dw_i2c_driver = { .probe = dw_i2c_plat_probe, @@ -492,6 +495,7 @@ static struct platform_driver dw_i2c_driver = { .acpi_match_table = ACPI_PTR(dw_i2c_acpi_match), .pm = pm_ptr(&dw_i2c_dev_pm_ops), }, + .id_table = dw_i2c_platform_ids, }; static int __init dw_i2c_init_driver(void) From a72a30af550c08423de1b9feecb6ceeddc434889 Mon Sep 17 00:00:00 2001 From: "Ho-Ren (Jack) Chuang" Date: Fri, 5 Apr 2024 00:07:05 +0000 Subject: [PATCH 1112/1702] memory tier: dax/kmem: introduce an abstract layer for finding, allocating, and putting memory types Patch series "Improved Memory Tier Creation for CPUless NUMA Nodes", v11. When a memory device, such as CXL1.1 type3 memory, is emulated as normal memory (E820_TYPE_RAM), the memory device is indistinguishable from normal DRAM in terms of memory tiering with the current implementation. The current memory tiering assigns all detected normal memory nodes to the same DRAM tier. This results in normal memory devices with different attributions being unable to be assigned to the correct memory tier, leading to the inability to migrate pages between different types of memory. https://lore.kernel.org/linux-mm/PH0PR08MB7955E9F08CCB64F23963B5C3A860A@PH0PR08MB7955.namprd08.prod.outlook.com/T/ This patchset automatically resolves the issues. It delays the initialization of memory tiers for CPUless NUMA nodes until they obtain HMAT information and after all devices are initialized at boot time, eliminating the need for user intervention. If no HMAT is specified, it falls back to using `default_dram_type`. Example usecase: We have CXL memory on the host, and we create VMs with a new system memory device backed by host CXL memory. We inject CXL memory performance attributes through QEMU, and the guest now sees memory nodes with performance attributes in HMAT. With this change, we enable the guest kernel to construct the correct memory tiering for the memory nodes. This patch (of 2): Since different memory devices require finding, allocating, and putting memory types, these common steps are abstracted in this patch, enhancing the scalability and conciseness of the code. Link: https://lkml.kernel.org/r/20240405000707.2670063-1-horenchuang@bytedance.com Link: https://lkml.kernel.org/r/20240405000707.2670063-2-horenchuang@bytedance.com Signed-off-by: Ho-Ren (Jack) Chuang Reviewed-by: "Huang, Ying" Reviewed-by: Jonathan Cameron Cc: Alistair Popple Cc: Aneesh Kumar K.V Cc: Dan Williams Cc: Dave Jiang Cc: Gregory Price Cc: Hao Xiang Cc: Jonathan Cameron Cc: Michal Hocko Cc: Ravi Jonnalagadda Cc: SeongJae Park Cc: Tejun Heo Cc: Vishal Verma Signed-off-by: Andrew Morton --- drivers/dax/kmem.c | 30 ++++-------------------------- include/linux/memory-tiers.h | 13 +++++++++++++ mm/memory-tiers.c | 29 +++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 26 deletions(-) diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c index 42ee360cf4e3d..4fe9d040e3754 100644 --- a/drivers/dax/kmem.c +++ b/drivers/dax/kmem.c @@ -55,36 +55,14 @@ static LIST_HEAD(kmem_memory_types); static struct memory_dev_type *kmem_find_alloc_memory_type(int adist) { - bool found = false; - struct memory_dev_type *mtype; - - mutex_lock(&kmem_memory_type_lock); - list_for_each_entry(mtype, &kmem_memory_types, list) { - if (mtype->adistance == adist) { - found = true; - break; - } - } - if (!found) { - mtype = alloc_memory_type(adist); - if (!IS_ERR(mtype)) - list_add(&mtype->list, &kmem_memory_types); - } - mutex_unlock(&kmem_memory_type_lock); - - return mtype; + guard(mutex)(&kmem_memory_type_lock); + return mt_find_alloc_memory_type(adist, &kmem_memory_types); } static void kmem_put_memory_types(void) { - struct memory_dev_type *mtype, *mtn; - - mutex_lock(&kmem_memory_type_lock); - list_for_each_entry_safe(mtype, mtn, &kmem_memory_types, list) { - list_del(&mtype->list); - put_memory_type(mtype); - } - mutex_unlock(&kmem_memory_type_lock); + guard(mutex)(&kmem_memory_type_lock); + mt_put_memory_types(&kmem_memory_types); } static int dev_dax_kmem_probe(struct dev_dax *dev_dax) diff --git a/include/linux/memory-tiers.h b/include/linux/memory-tiers.h index 69e7819000827..0d70788558f46 100644 --- a/include/linux/memory-tiers.h +++ b/include/linux/memory-tiers.h @@ -48,6 +48,9 @@ int mt_calc_adistance(int node, int *adist); int mt_set_default_dram_perf(int nid, struct access_coordinate *perf, const char *source); int mt_perf_to_adistance(struct access_coordinate *perf, int *adist); +struct memory_dev_type *mt_find_alloc_memory_type(int adist, + struct list_head *memory_types); +void mt_put_memory_types(struct list_head *memory_types); #ifdef CONFIG_MIGRATION int next_demotion_node(int node); void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets); @@ -136,5 +139,15 @@ static inline int mt_perf_to_adistance(struct access_coordinate *perf, int *adis { return -EIO; } + +static inline struct memory_dev_type *mt_find_alloc_memory_type(int adist, + struct list_head *memory_types) +{ + return NULL; +} + +static inline void mt_put_memory_types(struct list_head *memory_types) +{ +} #endif /* CONFIG_NUMA */ #endif /* _LINUX_MEMORY_TIERS_H */ diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c index 0537664620e5f..516b144fd45af 100644 --- a/mm/memory-tiers.c +++ b/mm/memory-tiers.c @@ -623,6 +623,35 @@ void clear_node_memory_type(int node, struct memory_dev_type *memtype) } EXPORT_SYMBOL_GPL(clear_node_memory_type); +struct memory_dev_type *mt_find_alloc_memory_type(int adist, struct list_head *memory_types) +{ + struct memory_dev_type *mtype; + + list_for_each_entry(mtype, memory_types, list) + if (mtype->adistance == adist) + return mtype; + + mtype = alloc_memory_type(adist); + if (IS_ERR(mtype)) + return mtype; + + list_add(&mtype->list, memory_types); + + return mtype; +} +EXPORT_SYMBOL_GPL(mt_find_alloc_memory_type); + +void mt_put_memory_types(struct list_head *memory_types) +{ + struct memory_dev_type *mtype, *mtn; + + list_for_each_entry_safe(mtype, mtn, memory_types, list) { + list_del(&mtype->list); + put_memory_type(mtype); + } +} +EXPORT_SYMBOL_GPL(mt_put_memory_types); + static void dump_hmem_attrs(struct access_coordinate *coord, const char *prefix) { pr_info( From cf93be18fa1bb837fa1e3015a9919953fff6ef22 Mon Sep 17 00:00:00 2001 From: "Ho-Ren (Jack) Chuang" Date: Fri, 5 Apr 2024 00:07:06 +0000 Subject: [PATCH 1113/1702] memory tier: create CPUless memory tiers after obtaining HMAT info The current implementation treats emulated memory devices, such as CXL1.1 type3 memory, as normal DRAM when they are emulated as normal memory (E820_TYPE_RAM). However, these emulated devices have different characteristics than traditional DRAM, making it important to distinguish them. Thus, we modify the tiered memory initialization process to introduce a delay specifically for CPUless NUMA nodes. This delay ensures that the memory tier initialization for these nodes is deferred until HMAT information is obtained during the boot process. Finally, demotion tables are recalculated at the end. * late_initcall(memory_tier_late_init); Some device drivers may have initialized memory tiers between `memory_tier_init()` and `memory_tier_late_init()`, potentially bringing online memory nodes and configuring memory tiers. They should be excluded in the late init. * Handle cases where there is no HMAT when creating memory tiers There is a scenario where a CPUless node does not provide HMAT information. If no HMAT is specified, it falls back to using the default DRAM tier. * Introduce another new lock `default_dram_perf_lock` for adist calculation In the current implementation, iterating through CPUlist nodes requires holding the `memory_tier_lock`. However, `mt_calc_adistance()` will end up trying to acquire the same lock, leading to a potential deadlock. Therefore, we propose introducing a standalone `default_dram_perf_lock` to protect `default_dram_perf_*`. This approach not only avoids deadlock but also prevents holding a large lock simultaneously. * Upgrade `set_node_memory_tier` to support additional cases, including default DRAM, late CPUless, and hot-plugged initializations. To cover hot-plugged memory nodes, `mt_calc_adistance()` and `mt_find_alloc_memory_type()` are moved into `set_node_memory_tier()` to handle cases where memtype is not initialized and where HMAT information is available. * Introduce `default_memory_types` for those memory types that are not initialized by device drivers. Because late initialized memory and default DRAM memory need to be managed, a default memory type is created for storing all memory types that are not initialized by device drivers and as a fallback. Link: https://lkml.kernel.org/r/20240405000707.2670063-3-horenchuang@bytedance.com Signed-off-by: Ho-Ren (Jack) Chuang Signed-off-by: Hao Xiang Reviewed-by: "Huang, Ying" Reviewed-by: Jonathan Cameron Cc: Alistair Popple Cc: Aneesh Kumar K.V Cc: Dan Williams Cc: Dave Jiang Cc: Gregory Price Cc: Michal Hocko Cc: Ravi Jonnalagadda Cc: SeongJae Park Cc: Tejun Heo Cc: Vishal Verma Cc: Jonathan Cameron Signed-off-by: Andrew Morton --- mm/memory-tiers.c | 94 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 70 insertions(+), 24 deletions(-) diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c index 516b144fd45af..6632102bd5c99 100644 --- a/mm/memory-tiers.c +++ b/mm/memory-tiers.c @@ -36,6 +36,11 @@ struct node_memory_type_map { static DEFINE_MUTEX(memory_tier_lock); static LIST_HEAD(memory_tiers); +/* + * The list is used to store all memory types that are not created + * by a device driver. + */ +static LIST_HEAD(default_memory_types); static struct node_memory_type_map node_memory_types[MAX_NUMNODES]; struct memory_dev_type *default_dram_type; @@ -108,6 +113,8 @@ static struct demotion_nodes *node_demotion __read_mostly; static BLOCKING_NOTIFIER_HEAD(mt_adistance_algorithms); +/* The lock is used to protect `default_dram_perf*` info and nid. */ +static DEFINE_MUTEX(default_dram_perf_lock); static bool default_dram_perf_error; static struct access_coordinate default_dram_perf; static int default_dram_perf_ref_nid = NUMA_NO_NODE; @@ -505,7 +512,8 @@ static inline void __init_node_memory_type(int node, struct memory_dev_type *mem static struct memory_tier *set_node_memory_tier(int node) { struct memory_tier *memtier; - struct memory_dev_type *memtype; + struct memory_dev_type *memtype = default_dram_type; + int adist = MEMTIER_ADISTANCE_DRAM; pg_data_t *pgdat = NODE_DATA(node); @@ -514,7 +522,16 @@ static struct memory_tier *set_node_memory_tier(int node) if (!node_state(node, N_MEMORY)) return ERR_PTR(-EINVAL); - __init_node_memory_type(node, default_dram_type); + mt_calc_adistance(node, &adist); + if (!node_memory_types[node].memtype) { + memtype = mt_find_alloc_memory_type(adist, &default_memory_types); + if (IS_ERR(memtype)) { + memtype = default_dram_type; + pr_info("Failed to allocate a memory type. Fall back.\n"); + } + } + + __init_node_memory_type(node, memtype); memtype = node_memory_types[node].memtype; node_set(node, memtype->nodes); @@ -652,6 +669,35 @@ void mt_put_memory_types(struct list_head *memory_types) } EXPORT_SYMBOL_GPL(mt_put_memory_types); +/* + * This is invoked via `late_initcall()` to initialize memory tiers for + * CPU-less memory nodes after driver initialization, which is + * expected to provide `adistance` algorithms. + */ +static int __init memory_tier_late_init(void) +{ + int nid; + + guard(mutex)(&memory_tier_lock); + for_each_node_state(nid, N_MEMORY) { + /* + * Some device drivers may have initialized memory tiers + * between `memory_tier_init()` and `memory_tier_late_init()`, + * potentially bringing online memory nodes and + * configuring memory tiers. Exclude them here. + */ + if (node_memory_types[nid].memtype) + continue; + + set_node_memory_tier(nid); + } + + establish_demotion_targets(); + + return 0; +} +late_initcall(memory_tier_late_init); + static void dump_hmem_attrs(struct access_coordinate *coord, const char *prefix) { pr_info( @@ -663,25 +709,19 @@ static void dump_hmem_attrs(struct access_coordinate *coord, const char *prefix) int mt_set_default_dram_perf(int nid, struct access_coordinate *perf, const char *source) { - int rc = 0; - - mutex_lock(&memory_tier_lock); - if (default_dram_perf_error) { - rc = -EIO; - goto out; - } + guard(mutex)(&default_dram_perf_lock); + if (default_dram_perf_error) + return -EIO; if (perf->read_latency + perf->write_latency == 0 || - perf->read_bandwidth + perf->write_bandwidth == 0) { - rc = -EINVAL; - goto out; - } + perf->read_bandwidth + perf->write_bandwidth == 0) + return -EINVAL; if (default_dram_perf_ref_nid == NUMA_NO_NODE) { default_dram_perf = *perf; default_dram_perf_ref_nid = nid; default_dram_perf_ref_source = kstrdup(source, GFP_KERNEL); - goto out; + return 0; } /* @@ -709,27 +749,25 @@ int mt_set_default_dram_perf(int nid, struct access_coordinate *perf, pr_info( " disable default DRAM node performance based abstract distance algorithm.\n"); default_dram_perf_error = true; - rc = -EINVAL; + return -EINVAL; } -out: - mutex_unlock(&memory_tier_lock); - return rc; + return 0; } int mt_perf_to_adistance(struct access_coordinate *perf, int *adist) { + guard(mutex)(&default_dram_perf_lock); if (default_dram_perf_error) return -EIO; - if (default_dram_perf_ref_nid == NUMA_NO_NODE) - return -ENOENT; - if (perf->read_latency + perf->write_latency == 0 || perf->read_bandwidth + perf->write_bandwidth == 0) return -EINVAL; - mutex_lock(&memory_tier_lock); + if (default_dram_perf_ref_nid == NUMA_NO_NODE) + return -ENOENT; + /* * The abstract distance of a memory node is in direct proportion to * its memory latency (read + write) and inversely proportional to its @@ -742,7 +780,6 @@ int mt_perf_to_adistance(struct access_coordinate *perf, int *adist) (default_dram_perf.read_latency + default_dram_perf.write_latency) * (default_dram_perf.read_bandwidth + default_dram_perf.write_bandwidth) / (perf->read_bandwidth + perf->write_bandwidth); - mutex_unlock(&memory_tier_lock); return 0; } @@ -855,7 +892,8 @@ static int __init memory_tier_init(void) * For now we can have 4 faster memory tiers with smaller adistance * than default DRAM tier. */ - default_dram_type = alloc_memory_type(MEMTIER_ADISTANCE_DRAM); + default_dram_type = mt_find_alloc_memory_type(MEMTIER_ADISTANCE_DRAM, + &default_memory_types); if (IS_ERR(default_dram_type)) panic("%s() failed to allocate default DRAM tier\n", __func__); @@ -865,6 +903,14 @@ static int __init memory_tier_init(void) * types assigned. */ for_each_node_state(node, N_MEMORY) { + if (!node_state(node, N_CPU)) + /* + * Defer memory tier initialization on + * CPUless numa nodes. These will be initialized + * after firmware and devices are initialized. + */ + continue; + memtier = set_node_memory_tier(node); if (IS_ERR(memtier)) /* From 38bc9c28c3780d8d0ca1ed1a1fbfe8c91e20f5f2 Mon Sep 17 00:00:00 2001 From: Hao Ge Date: Sun, 7 Apr 2024 14:26:53 +0800 Subject: [PATCH 1114/1702] mm/mmap: make vma_wants_writenotify return bool vma_wants_writenotify() should return bool, so change it. Link: https://lkml.kernel.org/r/20240407062653.803142-1-gehao@kylinos.cn Signed-off-by: Hao Ge Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/mm.h | 2 +- mm/mmap.c | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index cc9410d772ddf..0b933f9f48643 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2575,7 +2575,7 @@ extern unsigned long move_page_tables(struct vm_area_struct *vma, MM_CP_UFFD_WP_RESOLVE) bool vma_needs_dirty_tracking(struct vm_area_struct *vma); -int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot); +bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot); static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma) { /* diff --git a/mm/mmap.c b/mm/mmap.c index d3c2ca8efa534..c9367454e19f9 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1514,32 +1514,32 @@ bool vma_needs_dirty_tracking(struct vm_area_struct *vma) * to the private version (using protection_map[] without the * VM_SHARED bit). */ -int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) +bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) { /* If it was private or non-writable, the write bit is already clear */ if (!vma_is_shared_writable(vma)) - return 0; + return false; /* The backer wishes to know when pages are first written to? */ if (vm_ops_needs_writenotify(vma->vm_ops)) - return 1; + return true; /* The open routine did something to the protections that pgprot_modify * won't preserve? */ if (pgprot_val(vm_page_prot) != pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags))) - return 0; + return false; /* * Do we need to track softdirty? hugetlb does not support softdirty * tracking yet. */ if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma)) - return 1; + return true; /* Do we need write faults for uffd-wp tracking? */ if (userfaultfd_wp(vma)) - return 1; + return true; /* Can the mapping track the dirty pages? */ return vma_fs_can_writeback(vma); From 2bd9e6ee99cb249e4424ca5a4bf12d879ca9ce5d Mon Sep 17 00:00:00 2001 From: Hao Ge Date: Sun, 7 Apr 2024 14:38:43 +0800 Subject: [PATCH 1115/1702] mm/mmap: make accountable_mapping return bool accountable_mapping() can return bool, so change it. Link: https://lkml.kernel.org/r/20240407063843.804274-1-gehao@kylinos.cn Signed-off-by: Hao Ge Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/mmap.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/mmap.c b/mm/mmap.c index c9367454e19f9..2fc4601a6f850 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1549,14 +1549,14 @@ bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) * We account for memory if it's a private writeable mapping, * not hugepages and VM_NORESERVE wasn't set. */ -static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags) +static inline bool accountable_mapping(struct file *file, vm_flags_t vm_flags) { /* * hugetlb has its own accounting separate from the core VM * VM_HUGETLB may not be set yet so we cannot check for that flag. */ if (file && is_file_hugepages(file)) - return 0; + return false; return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE; } From d4a34d7fb440d070553743d9cadef9a076e809d2 Mon Sep 17 00:00:00 2001 From: Huang Ying Date: Sun, 7 Apr 2024 14:54:50 +0800 Subject: [PATCH 1116/1702] mm,swap: add document about RCU read lock and swapoff interaction During reviewing a patch to fix the race condition between free_swap_and_cache() and swapoff() [1], it was found that the document about how to prevent racing with swapoff isn't clear enough. Especially RCU read lock can prevent swapoff from freeing data structures. So, the document is added as comments. [1] https://lore.kernel.org/linux-mm/c8fe62d0-78b8-527a-5bef-ee663ccdc37a@huawei.com/ Link: https://lkml.kernel.org/r/20240407065450.498821-1-ying.huang@intel.com Signed-off-by: "Huang, Ying" Reviewed-by: Ryan Roberts Reviewed-by: David Hildenbrand Reviewed-by: Miaohe Lin Cc: Hugh Dickins Cc: Minchan Kim Signed-off-by: Andrew Morton --- mm/swapfile.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/mm/swapfile.c b/mm/swapfile.c index 148ef08f19dd6..28642c188c930 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1238,16 +1238,15 @@ static unsigned char __swap_entry_free_locked(struct swap_info_struct *p, /* * When we get a swap entry, if there aren't some other ways to - * prevent swapoff, such as the folio in swap cache is locked, page - * table lock is held, etc., the swap entry may become invalid because - * of swapoff. Then, we need to enclose all swap related functions - * with get_swap_device() and put_swap_device(), unless the swap - * functions call get/put_swap_device() by themselves. + * prevent swapoff, such as the folio in swap cache is locked, RCU + * reader side is locked, etc., the swap entry may become invalid + * because of swapoff. Then, we need to enclose all swap related + * functions with get_swap_device() and put_swap_device(), unless the + * swap functions call get/put_swap_device() by themselves. * - * Note that when only holding the PTL, swapoff might succeed immediately - * after freeing a swap entry. Therefore, immediately after - * __swap_entry_free(), the swap info might become stale and should not - * be touched without a prior get_swap_device(). + * RCU reader side lock (including any spinlock) is sufficient to + * prevent swapoff, because synchronize_rcu() is called in swapoff() + * before freeing data structures. * * Check whether swap entry is valid in the swap device. If so, * return pointer to swap_info_struct, and keep the swap entry valid @@ -2544,10 +2543,11 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) /* * Wait for swap operations protected by get/put_swap_device() - * to complete. - * - * We need synchronize_rcu() here to protect the accessing to - * the swap cache data structure. + * to complete. Because of synchronize_rcu() here, all swap + * operations protected by RCU reader side lock (including any + * spinlock) will be waited too. This makes it easy to + * prevent folio_test_swapcache() and the following swap cache + * operations from racing with swapoff. */ percpu_ref_kill(&p->users); synchronize_rcu(); From 3d6586008f7b638f91f3332602592caa8b00b559 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 10 Apr 2024 17:55:25 +0200 Subject: [PATCH 1117/1702] drivers/virt/acrn: fix PFNMAP PTE checks in acrn_vm_ram_map() Patch series "mm: follow_pte() improvements and acrn follow_pte() fixes". Patch #1 fixes a bunch of issues I spotted in the acrn driver. It compiles, that's all I know. I'll appreciate some review and testing from acrn folks. Patch #2+#3 improve follow_pte(), passing a VMA instead of the MM, adding more sanity checks, and improving the documentation. Gave it a quick test on x86-64 using VM_PAT that ends up using follow_pte(). This patch (of 3): We currently miss handling various cases, resulting in a dangerous follow_pte() (previously follow_pfn()) usage. (1) We're not checking PTE write permissions. Maybe we should simply always require pte_write() like we do for pin_user_pages_fast(FOLL_WRITE)? Hard to tell, so let's check for ACRN_MEM_ACCESS_WRITE for now. (2) We're not rejecting refcounted pages. As we are not using MMU notifiers, messing with refcounted pages is dangerous and can result in use-after-free. Let's make sure to reject them. (3) We are only looking at the first PTE of a bigger range. We only lookup a single PTE, but memmap->len may span a larger area. Let's loop over all involved PTEs and make sure the PFN range is actually contiguous. Reject everything else: it couldn't have worked either way, and rather made use access PFNs we shouldn't be accessing. Link: https://lkml.kernel.org/r/20240410155527.474777-1-david@redhat.com Link: https://lkml.kernel.org/r/20240410155527.474777-2-david@redhat.com Fixes: 8a6e85f75a83 ("virt: acrn: obtain pa from VMA with PFNMAP flag") Signed-off-by: David Hildenbrand Cc: Alex Williamson Cc: Christoph Hellwig Cc: Fei Li Cc: Gerald Schaefer Cc: Heiko Carstens Cc: Ingo Molnar Cc: Paolo Bonzini Cc: Yonghua Huang Cc: Sean Christopherson Signed-off-by: Andrew Morton --- drivers/virt/acrn/mm.c | 63 +++++++++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/drivers/virt/acrn/mm.c b/drivers/virt/acrn/mm.c index b30077baf352b..2d98e1e185c47 100644 --- a/drivers/virt/acrn/mm.c +++ b/drivers/virt/acrn/mm.c @@ -156,23 +156,29 @@ int acrn_vm_memseg_unmap(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) { struct vm_memory_region_batch *regions_info; - int nr_pages, i = 0, order, nr_regions = 0; + int nr_pages, i, order, nr_regions = 0; struct vm_memory_mapping *region_mapping; struct vm_memory_region_op *vm_region; struct page **pages = NULL, *page; void *remap_vaddr; int ret, pinned; u64 user_vm_pa; - unsigned long pfn; struct vm_area_struct *vma; if (!vm || !memmap) return -EINVAL; + /* Get the page number of the map region */ + nr_pages = memmap->len >> PAGE_SHIFT; + if (!nr_pages) + return -EINVAL; + mmap_read_lock(current->mm); vma = vma_lookup(current->mm, memmap->vma_base); if (vma && ((vma->vm_flags & VM_PFNMAP) != 0)) { + unsigned long start_pfn, cur_pfn; spinlock_t *ptl; + bool writable; pte_t *ptep; if ((memmap->vma_base + memmap->len) > vma->vm_end) { @@ -180,25 +186,53 @@ int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) return -EINVAL; } - ret = follow_pte(vma->vm_mm, memmap->vma_base, &ptep, &ptl); - if (ret < 0) { - mmap_read_unlock(current->mm); + for (i = 0; i < nr_pages; i++) { + ret = follow_pte(vma->vm_mm, + memmap->vma_base + i * PAGE_SIZE, + &ptep, &ptl); + if (ret) + break; + + cur_pfn = pte_pfn(ptep_get(ptep)); + if (i == 0) + start_pfn = cur_pfn; + writable = !!pte_write(ptep_get(ptep)); + pte_unmap_unlock(ptep, ptl); + + /* Disallow write access if the PTE is not writable. */ + if (!writable && + (memmap->attr & ACRN_MEM_ACCESS_WRITE)) { + ret = -EFAULT; + break; + } + + /* Disallow refcounted pages. */ + if (pfn_valid(cur_pfn) && + !PageReserved(pfn_to_page(cur_pfn))) { + ret = -EFAULT; + break; + } + + /* Disallow non-contiguous ranges. */ + if (cur_pfn != start_pfn + i) { + ret = -EINVAL; + break; + } + } + mmap_read_unlock(current->mm); + + if (ret) { dev_dbg(acrn_dev.this_device, "Failed to lookup PFN at VMA:%pK.\n", (void *)memmap->vma_base); return ret; } - pfn = pte_pfn(ptep_get(ptep)); - pte_unmap_unlock(ptep, ptl); - mmap_read_unlock(current->mm); return acrn_mm_region_add(vm, memmap->user_vm_pa, - PFN_PHYS(pfn), memmap->len, + PFN_PHYS(start_pfn), memmap->len, ACRN_MEM_TYPE_WB, memmap->attr); } mmap_read_unlock(current->mm); - /* Get the page number of the map region */ - nr_pages = memmap->len >> PAGE_SHIFT; pages = vzalloc(array_size(nr_pages, sizeof(*pages))); if (!pages) return -ENOMEM; @@ -242,12 +276,11 @@ int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) mutex_unlock(&vm->regions_mapping_lock); /* Calculate count of vm_memory_region_op */ - while (i < nr_pages) { + for (i = 0; i < nr_pages; i += 1 << order) { page = pages[i]; VM_BUG_ON_PAGE(PageTail(page), page); order = compound_order(page); nr_regions++; - i += 1 << order; } /* Prepare the vm_memory_region_batch */ @@ -264,8 +297,7 @@ int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) regions_info->vmid = vm->vmid; regions_info->regions_gpa = virt_to_phys(vm_region); user_vm_pa = memmap->user_vm_pa; - i = 0; - while (i < nr_pages) { + for (i = 0; i < nr_pages; i += 1 << order) { u32 region_size; page = pages[i]; @@ -281,7 +313,6 @@ int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) vm_region++; user_vm_pa += region_size; - i += 1 << order; } /* Inform the ACRN Hypervisor to set up EPT mappings */ From 29ae7d96d166fa08c7232daf8a314ef5ba1efd20 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 10 Apr 2024 17:55:26 +0200 Subject: [PATCH 1118/1702] mm: pass VMA instead of MM to follow_pte() ... and centralize the VM_IO/VM_PFNMAP sanity check in there. We'll now also perform these sanity checks for direct follow_pte() invocations. For generic_access_phys(), we might now check multiple times: nothing to worry about, really. Link: https://lkml.kernel.org/r/20240410155527.474777-3-david@redhat.com Signed-off-by: David Hildenbrand Acked-by: Sean Christopherson [KVM] Cc: Alex Williamson Cc: Christoph Hellwig Cc: Fei Li Cc: Gerald Schaefer Cc: Heiko Carstens Cc: Ingo Molnar Cc: Paolo Bonzini Cc: Yonghua Huang Signed-off-by: Andrew Morton --- arch/s390/pci/pci_mmio.c | 4 ++-- arch/x86/mm/pat/memtype.c | 5 +---- drivers/vfio/vfio_iommu_type1.c | 4 ++-- drivers/virt/acrn/mm.c | 3 +-- include/linux/mm.h | 2 +- mm/memory.c | 15 ++++++++------- virt/kvm/kvm_main.c | 4 ++-- 7 files changed, 17 insertions(+), 20 deletions(-) diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index a90499c087f0c..5398729bfe1b7 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -169,7 +169,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, if (!(vma->vm_flags & VM_WRITE)) goto out_unlock_mmap; - ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl); + ret = follow_pte(vma, mmio_addr, &ptep, &ptl); if (ret) goto out_unlock_mmap; @@ -308,7 +308,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, if (!(vma->vm_flags & VM_WRITE)) goto out_unlock_mmap; - ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl); + ret = follow_pte(vma, mmio_addr, &ptep, &ptl); if (ret) goto out_unlock_mmap; diff --git a/arch/x86/mm/pat/memtype.c b/arch/x86/mm/pat/memtype.c index d01c3b0bd6eb1..bdc2a240c2aae 100644 --- a/arch/x86/mm/pat/memtype.c +++ b/arch/x86/mm/pat/memtype.c @@ -954,10 +954,7 @@ static int follow_phys(struct vm_area_struct *vma, unsigned long *prot, pte_t *ptep, pte; spinlock_t *ptl; - if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) - return -EINVAL; - - if (follow_pte(vma->vm_mm, vma->vm_start, &ptep, &ptl)) + if (follow_pte(vma, vma->vm_start, &ptep, &ptl)) return -EINVAL; pte = ptep_get(ptep); diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index b5c15fe8f9fcf..3a0218171cfaa 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -518,7 +518,7 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, spinlock_t *ptl; int ret; - ret = follow_pte(vma->vm_mm, vaddr, &ptep, &ptl); + ret = follow_pte(vma, vaddr, &ptep, &ptl); if (ret) { bool unlocked = false; @@ -532,7 +532,7 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, if (ret) return ret; - ret = follow_pte(vma->vm_mm, vaddr, &ptep, &ptl); + ret = follow_pte(vma, vaddr, &ptep, &ptl); if (ret) return ret; } diff --git a/drivers/virt/acrn/mm.c b/drivers/virt/acrn/mm.c index 2d98e1e185c47..db8ff1d0ac23c 100644 --- a/drivers/virt/acrn/mm.c +++ b/drivers/virt/acrn/mm.c @@ -187,8 +187,7 @@ int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) } for (i = 0; i < nr_pages; i++) { - ret = follow_pte(vma->vm_mm, - memmap->vma_base + i * PAGE_SIZE, + ret = follow_pte(vma, memmap->vma_base + i * PAGE_SIZE, &ptep, &ptl); if (ret) break; diff --git a/include/linux/mm.h b/include/linux/mm.h index 0b933f9f48643..c25ce69929513 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2420,7 +2420,7 @@ void free_pgd_range(struct mmu_gather *tlb, unsigned long addr, unsigned long end, unsigned long floor, unsigned long ceiling); int copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma); -int follow_pte(struct mm_struct *mm, unsigned long address, +int follow_pte(struct vm_area_struct *vma, unsigned long address, pte_t **ptepp, spinlock_t **ptlp); int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, void *buf, int len, int write); diff --git a/mm/memory.c b/mm/memory.c index 59c05dc8b18a7..1d45d25c1bba1 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5926,7 +5926,7 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) /** * follow_pte - look up PTE at a user virtual address - * @mm: the mm_struct of the target address space + * @vma: the memory mapping * @address: user virtual address * @ptepp: location to store found PTE * @ptlp: location to store the lock for the PTE @@ -5945,15 +5945,19 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) * * Return: zero on success, -ve otherwise. */ -int follow_pte(struct mm_struct *mm, unsigned long address, +int follow_pte(struct vm_area_struct *vma, unsigned long address, pte_t **ptepp, spinlock_t **ptlp) { + struct mm_struct *mm = vma->vm_mm; pgd_t *pgd; p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *ptep; + if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) + goto out; + pgd = pgd_offset(mm, address); if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) goto out; @@ -6007,11 +6011,8 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, int offset = offset_in_page(addr); int ret = -EINVAL; - if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) - return -EINVAL; - retry: - if (follow_pte(vma->vm_mm, addr, &ptep, &ptl)) + if (follow_pte(vma, addr, &ptep, &ptl)) return -EINVAL; pte = ptep_get(ptep); pte_unmap_unlock(ptep, ptl); @@ -6026,7 +6027,7 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, if (!maddr) return -ENOMEM; - if (follow_pte(vma->vm_mm, addr, &ptep, &ptl)) + if (follow_pte(vma, addr, &ptep, &ptl)) goto out_unmap; if (!pte_same(pte, ptep_get(ptep))) { diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index fb49c2a602002..f57dbacb8689c 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2902,7 +2902,7 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, spinlock_t *ptl; int r; - r = follow_pte(vma->vm_mm, addr, &ptep, &ptl); + r = follow_pte(vma, addr, &ptep, &ptl); if (r) { /* * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does @@ -2917,7 +2917,7 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, if (r) return r; - r = follow_pte(vma->vm_mm, addr, &ptep, &ptl); + r = follow_pte(vma, addr, &ptep, &ptl); if (r) return r; } From c5541ba378e3d36ea88bf5839d5b23e33e7d1627 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 10 Apr 2024 17:55:27 +0200 Subject: [PATCH 1119/1702] mm: follow_pte() improvements follow_pte() is now our main function to lookup PTEs in VM_PFNMAP/VM_IO VMAs. Let's perform some more sanity checks to make this exported function harder to abuse. Further, extend the doc a bit, it still focuses on the KVM use case with MMU notifiers. Drop the KVM+follow_pfn() comment, follow_pfn() is no more, and we have other users nowadays. Also extend the doc regarding refcounted pages and the interaction with MMU notifiers. KVM is one example that uses MMU notifiers and can deal with refcounted pages properly. VFIO is one example that doesn't use MMU notifiers, and to prevent use-after-free, rejects refcounted pages: pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn)). Protection changes are less of a concern for users like VFIO: the behavior is similar to longterm-pinning a page, and getting the PTE protection changed afterwards. The primary concern with refcounted pages is use-after-free, which callers should be aware of. Link: https://lkml.kernel.org/r/20240410155527.474777-4-david@redhat.com Signed-off-by: David Hildenbrand Cc: Alex Williamson Cc: Christoph Hellwig Cc: Fei Li Cc: Gerald Schaefer Cc: Heiko Carstens Cc: Ingo Molnar Cc: Paolo Bonzini Cc: Sean Christopherson Cc: Yonghua Huang Signed-off-by: Andrew Morton --- mm/memory.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index 1d45d25c1bba1..36ba94eae853c 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5933,15 +5933,21 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) * * On a successful return, the pointer to the PTE is stored in @ptepp; * the corresponding lock is taken and its location is stored in @ptlp. - * The contents of the PTE are only stable until @ptlp is released; - * any further use, if any, must be protected against invalidation - * with MMU notifiers. + * + * The contents of the PTE are only stable until @ptlp is released using + * pte_unmap_unlock(). This function will fail if the PTE is non-present. + * Present PTEs may include PTEs that map refcounted pages, such as + * anonymous folios in COW mappings. + * + * Callers must be careful when relying on PTE content after + * pte_unmap_unlock(). Especially if the PTE maps a refcounted page, + * callers must protect against invalidation with MMU notifiers; otherwise + * access to the PFN at a later point in time can trigger use-after-free. * * Only IO mappings and raw PFN mappings are allowed. The mmap semaphore * should be taken for read. * - * KVM uses this function. While it is arguably less bad than the historic - * ``follow_pfn``, it is not a good general-purpose API. + * This function must not be used to modify PTE content. * * Return: zero on success, -ve otherwise. */ @@ -5955,6 +5961,10 @@ int follow_pte(struct vm_area_struct *vma, unsigned long address, pmd_t *pmd; pte_t *ptep; + mmap_assert_locked(mm); + if (unlikely(address < vma->vm_start || address >= vma->vm_end)) + goto out; + if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) goto out; From 02faa73f174c4d1e11cb9a421f9a8eac0dd881f1 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:44 +0200 Subject: [PATCH 1120/1702] mm: allow for detecting underflows with page_mapcount() again Patch series "mm: mapcount for large folios + page_mapcount() cleanups". This series tracks the mapcount of large folios in a single value, so it can be read efficiently and atomically, just like the mapcount of small folios. folio_mapcount() is then used in a couple more places, most notably to reduce false negatives in folio_likely_mapped_shared(), and many users of page_mapcount() are cleaned up (that's maybe why you got CCed on the full series, sorry sh+xtensa folks! :) ). The remaining s390x user and one KSM user of page_mapcount() are getting removed separately on the list right now. I have patches to handle the other KSM one, the khugepaged one and the kpagecount one; as they are not as "obvious", I will send them out separately in the future. Once that is all in place, I'm planning on moving page_mapcount() into fs/proc/task_mmu.c, the remaining user for the time being (and we can discuss at LSF/MM details on that :) ). I proposed the mapcount for large folios (previously called total mapcount) originally in part of [1] and I later included it in [2] where it is a requirement. In the meantime, I changed the patch a bit so I dropped all RB's. During the discussion of [1], Peter Xu correctly raised that this additional tracking might affect the performance when PMD->PTE remapping THPs. In the meantime. I addressed that by batching RMAP operations during fork(), unmap/zap and when PMD->PTE remapping THPs. Running some of my micro-benchmarks [3] (fork,munmap,cow-byte,remap) on 1 GiB of memory backed by folios with the same order, I observe the following on an Intel(R) Xeon(R) Silver 4210R CPU @ 2.40GHz tuned for reproducible results as much as possible: Standard deviation is mostly < 1%, except for order-9, where it's < 2% for fork() and munmap(). (1) Small folios are not affected (< 1%) in all 4 microbenchmarks. (2) Order-4 folios are not affected (< 1%) in all 4 microbenchmarks. A bit weird comapred to the other orders ... (3) PMD->PTE remapping of order-9 THPs is not affected (< 1%) (4) COW-byte (COWing a single page by writing a single byte) is not affected for any order (< 1 %). The page copy_fault overhead dominates everything. (5) fork() is mostly not affected (< 1%), except order-2, where we have a slowdown of ~4%. Already for order-3 folios, we're down to a slowdown of < 1%. (6) munmap() sees a slowdown by < 3% for some orders (order-5, order-6, order-9), but less for others (< 1% for order-4 and order-8, < 2% for order-2, order-3, order-7). Especially the fork() and munmap() benchmark are sensitive to each added instruction and other system noise, so I suspect some of the change and observed weirdness (order-4) is due to code layout changes and other factors, but not really due to the added atomics. So in the common case where we can batch, the added atomics don't really make a big difference, especially in light of the recent improvements for large folios that we recently gained due to batching. Surprisingly, for some cases where we cannot batch (e.g., COW), the added atomics don't seem to matter, because other overhead dominates. My fork and munmap micro-benchmarks don't cover cases where we cannot batch-process bigger parts of large folios. As this is not the common case, I'm not worrying about that right now. Future work is batching RMAP operations during swapout and folio migration. [1] https://lore.kernel.org/all/20230809083256.699513-1-david@redhat.com/ [2] https://lore.kernel.org/all/20231124132626.235350-1-david@redhat.com/ [3] https://gitlab.com/davidhildenbrand/scratchspace/-/raw/main/pte-mapped-folio-benchmarks.c?ref_type=heads This patch (of 18): Commit 53277bcf126d ("mm: support page_mapcount() on page_has_type() pages") made it impossible to detect mapcount underflows by treating any negative raw mapcount value as a mapcount of 0. We perform such underflow checks in zap_present_folio_ptes() and zap_huge_pmd(), which would currently no longer trigger. Let's check against PAGE_MAPCOUNT_RESERVE instead by using page_type_has_type(), like page_has_type() would, so we can still catch some underflows. [david@redhat.com: make page_mapcount() slighly more efficient] Link: https://lkml.kernel.org/r/1af4fd61-7926-47c8-be45-833c0dbec08b@redhat.com Link: https://lkml.kernel.org/r/20240409192301.907377-1-david@redhat.com Link: https://lkml.kernel.org/r/20240409192301.907377-2-david@redhat.com Fixes: 53277bcf126d ("mm: support page_mapcount() on page_has_type() pages") Signed-off-by: David Hildenbrand Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yin Fengwei Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/mm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index c25ce69929513..8fa4d78bcc09a 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1232,7 +1232,7 @@ static inline int page_mapcount(struct page *page) int mapcount = atomic_read(&page->_mapcount) + 1; /* Handle page_has_type() pages */ - if (mapcount < 0) + if (mapcount < PAGE_MAPCOUNT_RESERVE + 1) mapcount = 0; if (unlikely(PageCompound(page))) mapcount += folio_entire_mapcount(page_folio(page)); From c2e65ebc02fb60b64a1c5689fe2c6f60d0fc1626 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:45 +0200 Subject: [PATCH 1121/1702] mm/rmap: always inline anon/file rmap duplication of a single PTE As we grow the code, the compiler might make stupid decisions and unnecessarily degrade fork() performance. Let's make sure to always inline functions that operate on a single PTE so the compiler will always optimize out the loop and avoid a function call. This is a preparation for maintining a total mapcount for large folios. Link: https://lkml.kernel.org/r/20240409192301.907377-3-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Yin Fengwei Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/rmap.h | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 9bf9324214fc3..9549d78928bb1 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -347,8 +347,12 @@ static inline void folio_dup_file_rmap_ptes(struct folio *folio, { __folio_dup_file_rmap(folio, page, nr_pages, RMAP_LEVEL_PTE); } -#define folio_dup_file_rmap_pte(folio, page) \ - folio_dup_file_rmap_ptes(folio, page, 1) + +static __always_inline void folio_dup_file_rmap_pte(struct folio *folio, + struct page *page) +{ + __folio_dup_file_rmap(folio, page, 1, RMAP_LEVEL_PTE); +} /** * folio_dup_file_rmap_pmd - duplicate a PMD mapping of a page range of a folio @@ -448,8 +452,13 @@ static inline int folio_try_dup_anon_rmap_ptes(struct folio *folio, return __folio_try_dup_anon_rmap(folio, page, nr_pages, src_vma, RMAP_LEVEL_PTE); } -#define folio_try_dup_anon_rmap_pte(folio, page, vma) \ - folio_try_dup_anon_rmap_ptes(folio, page, 1, vma) + +static __always_inline int folio_try_dup_anon_rmap_pte(struct folio *folio, + struct page *page, struct vm_area_struct *src_vma) +{ + return __folio_try_dup_anon_rmap(folio, page, 1, src_vma, + RMAP_LEVEL_PTE); +} /** * folio_try_dup_anon_rmap_pmd - try duplicating a PMD mapping of a page range From 46d62de7ad1286854e0c2944ad26a1c1b1a5f191 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:46 +0200 Subject: [PATCH 1122/1702] mm/rmap: add fast-path for small folios when adding/removing/duplicating Let's add a fast-path for small folios to all relevant rmap functions. Note that only RMAP_LEVEL_PTE applies. This is a preparation for tracking the mapcount of large folios in a single value. Link: https://lkml.kernel.org/r/20240409192301.907377-4-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Yin Fengwei Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/rmap.h | 13 +++++++++++++ mm/rmap.c | 26 ++++++++++++++++---------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 9549d78928bb1..327f1ca5a487f 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -322,6 +322,11 @@ static __always_inline void __folio_dup_file_rmap(struct folio *folio, switch (level) { case RMAP_LEVEL_PTE: + if (!folio_test_large(folio)) { + atomic_inc(&page->_mapcount); + break; + } + do { atomic_inc(&page->_mapcount); } while (page++, --nr_pages > 0); @@ -405,6 +410,14 @@ static __always_inline int __folio_try_dup_anon_rmap(struct folio *folio, if (PageAnonExclusive(page + i)) return -EBUSY; } + + if (!folio_test_large(folio)) { + if (PageAnonExclusive(page)) + ClearPageAnonExclusive(page); + atomic_inc(&page->_mapcount); + break; + } + do { if (PageAnonExclusive(page)) ClearPageAnonExclusive(page); diff --git a/mm/rmap.c b/mm/rmap.c index 56b313aa2ebfb..4bde6d60db6ca 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1172,15 +1172,18 @@ static __always_inline unsigned int __folio_add_rmap(struct folio *folio, switch (level) { case RMAP_LEVEL_PTE: + if (!folio_test_large(folio)) { + nr = atomic_inc_and_test(&page->_mapcount); + break; + } + do { first = atomic_inc_and_test(&page->_mapcount); - if (first && folio_test_large(folio)) { + if (first) { first = atomic_inc_return_relaxed(mapped); - first = (first < ENTIRELY_MAPPED); + if (first < ENTIRELY_MAPPED) + nr++; } - - if (first) - nr++; } while (page++, --nr_pages > 0); break; case RMAP_LEVEL_PMD: @@ -1514,15 +1517,18 @@ static __always_inline void __folio_remove_rmap(struct folio *folio, switch (level) { case RMAP_LEVEL_PTE: + if (!folio_test_large(folio)) { + nr = atomic_add_negative(-1, &page->_mapcount); + break; + } + do { last = atomic_add_negative(-1, &page->_mapcount); - if (last && folio_test_large(folio)) { + if (last) { last = atomic_dec_return_relaxed(mapped); - last = (last < ENTIRELY_MAPPED); + if (last < ENTIRELY_MAPPED) + nr++; } - - if (last) - nr++; } while (page++, --nr_pages > 0); break; case RMAP_LEVEL_PMD: From 05c5323b2a344c19c51cd1b91a4ab9ae90853794 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:47 +0200 Subject: [PATCH 1123/1702] mm: track mapcount of large folios in single value Let's track the mapcount of large folios in a single value. The mapcount of a large folio currently corresponds to the sum of the entire mapcount and all page mapcounts. This sum is what we actually want to know in folio_mapcount() and it is also sufficient for implementing folio_mapped(). With PTE-mapped THP becoming more important and more widely used, we want to avoid looping over all pages of a folio just to obtain the mapcount of large folios. The comment "In the common case, avoid the loop when no pages mapped by PTE" in folio_total_mapcount() does no longer hold for mTHP that are always mapped by PTE. Further, we are planning on using folio_mapcount() more frequently, and might even want to remove page mapcounts for large folios in some kernel configs. Therefore, allow for reading the mapcount of large folios efficiently and atomically without looping over any pages. Maintain the mapcount also for hugetlb pages for simplicity. Use the new mapcount to implement folio_mapcount() and folio_mapped(). Make page_mapped() simply call folio_mapped(). We can now get rid of folio_large_is_mapped(). _nr_pages_mapped is now only used in rmap code and for debugging purposes. Keep folio_nr_pages_mapped() around, but document that its use should be limited to rmap internals and debugging purposes. This change implies one additional atomic add/sub whenever mapping/unmapping (parts of) a large folio. As we now batch RMAP operations for PTE-mapped THP during fork(), during unmap/zap, and when PTE-remapping a PMD-mapped THP, and we adjust the large mapcount for a PTE batch only once, the added overhead in the common case is small. Only when unmapping individual pages of a large folio (e.g., during COW), the overhead might be bigger in comparison, but it's essentially one additional atomic operation. Note that before the new mapcount would overflow, already our refcount would overflow: each mapping requires a folio reference. Extend the focumentation of folio_mapcount(). Link: https://lkml.kernel.org/r/20240409192301.907377-5-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Yin Fengwei Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- Documentation/mm/transhuge.rst | 12 +++++----- include/linux/mm.h | 44 ++++++++++++++++------------------ include/linux/mm_types.h | 5 ++-- include/linux/rmap.h | 10 ++++++++ mm/debug.c | 3 ++- mm/hugetlb.c | 4 ++-- mm/internal.h | 3 +++ mm/khugepaged.c | 2 +- mm/page_alloc.c | 4 ++++ mm/rmap.c | 34 +++++++++----------------- 10 files changed, 62 insertions(+), 59 deletions(-) diff --git a/Documentation/mm/transhuge.rst b/Documentation/mm/transhuge.rst index 93c9239b9ebe2..1ba0ad63246c5 100644 --- a/Documentation/mm/transhuge.rst +++ b/Documentation/mm/transhuge.rst @@ -116,14 +116,14 @@ pages: succeeds on tail pages. - map/unmap of a PMD entry for the whole THP increment/decrement - folio->_entire_mapcount and also increment/decrement - folio->_nr_pages_mapped by ENTIRELY_MAPPED when _entire_mapcount - goes from -1 to 0 or 0 to -1. + folio->_entire_mapcount, increment/decrement folio->_large_mapcount + and also increment/decrement folio->_nr_pages_mapped by ENTIRELY_MAPPED + when _entire_mapcount goes from -1 to 0 or 0 to -1. - map/unmap of individual pages with PTE entry increment/decrement - page->_mapcount and also increment/decrement folio->_nr_pages_mapped - when page->_mapcount goes from -1 to 0 or 0 to -1 as this counts - the number of pages mapped by PTE. + page->_mapcount, increment/decrement folio->_large_mapcount and also + increment/decrement folio->_nr_pages_mapped when page->_mapcount goes + from -1 to 0 or 0 to -1 as this counts the number of pages mapped by PTE. split_huge_page internally has to distribute the refcounts in the head page to the tail pages before clearing all PG_head/tail bits from the page diff --git a/include/linux/mm.h b/include/linux/mm.h index 8fa4d78bcc09a..0594778216251 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1240,16 +1240,26 @@ static inline int page_mapcount(struct page *page) return mapcount; } -int folio_total_mapcount(const struct folio *folio); +static inline int folio_large_mapcount(const struct folio *folio) +{ + VM_WARN_ON_FOLIO(!folio_test_large(folio), folio); + return atomic_read(&folio->_large_mapcount) + 1; +} /** - * folio_mapcount() - Calculate the number of mappings of this folio. + * folio_mapcount() - Number of mappings of this folio. * @folio: The folio. * - * A large folio tracks both how many times the entire folio is mapped, - * and how many times each individual page in the folio is mapped. - * This function calculates the total number of times the folio is - * mapped. + * The folio mapcount corresponds to the number of present user page table + * entries that reference any part of a folio. Each such present user page + * table entry must be paired with exactly on folio reference. + * + * For ordindary folios, each user page table entry (PTE/PMD/PUD/...) counts + * exactly once. + * + * For hugetlb folios, each abstracted "hugetlb" user page table entry that + * references the entire folio counts exactly once, even when such special + * page table entries are comprised of multiple ordinary page table entries. * * Return: The number of times this folio is mapped. */ @@ -1257,17 +1267,7 @@ static inline int folio_mapcount(const struct folio *folio) { if (likely(!folio_test_large(folio))) return atomic_read(&folio->_mapcount) + 1; - return folio_total_mapcount(folio); -} - -static inline bool folio_large_is_mapped(const struct folio *folio) -{ - /* - * Reading _entire_mapcount below could be omitted if hugetlb - * participated in incrementing nr_pages_mapped when compound mapped. - */ - return atomic_read(&folio->_nr_pages_mapped) > 0 || - atomic_read(&folio->_entire_mapcount) >= 0; + return folio_large_mapcount(folio); } /** @@ -1276,11 +1276,9 @@ static inline bool folio_large_is_mapped(const struct folio *folio) * * Return: True if any page in this folio is referenced by user page tables. */ -static inline bool folio_mapped(struct folio *folio) +static inline bool folio_mapped(const struct folio *folio) { - if (likely(!folio_test_large(folio))) - return atomic_read(&folio->_mapcount) >= 0; - return folio_large_is_mapped(folio); + return folio_mapcount(folio) >= 1; } /* @@ -1290,9 +1288,7 @@ static inline bool folio_mapped(struct folio *folio) */ static inline bool page_mapped(const struct page *page) { - if (likely(!PageCompound(page))) - return atomic_read(&page->_mapcount) >= 0; - return folio_large_is_mapped(page_folio(page)); + return folio_mapped(page_folio(page)); } static inline struct page *virt_to_head_page(const void *x) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index fa0d6995706f4..db0adf5721ccf 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -289,7 +289,8 @@ typedef struct { * @virtual: Virtual address in the kernel direct map. * @_last_cpupid: IDs of last CPU and last process that accessed the folio. * @_entire_mapcount: Do not use directly, call folio_entire_mapcount(). - * @_nr_pages_mapped: Do not use directly, call folio_mapcount(). + * @_large_mapcount: Do not use directly, call folio_mapcount(). + * @_nr_pages_mapped: Do not use outside of rmap and debug code. * @_pincount: Do not use directly, call folio_maybe_dma_pinned(). * @_folio_nr_pages: Do not use directly, call folio_nr_pages(). * @_hugetlb_subpool: Do not use directly, use accessor in hugetlb.h. @@ -348,8 +349,8 @@ struct folio { struct { unsigned long _flags_1; unsigned long _head_1; - unsigned long _folio_avail; /* public: */ + atomic_t _large_mapcount; atomic_t _entire_mapcount; atomic_t _nr_pages_mapped; atomic_t _pincount; diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 327f1ca5a487f..0f906dc6d2800 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -273,6 +273,7 @@ static inline int hugetlb_try_dup_anon_rmap(struct folio *folio, ClearPageAnonExclusive(&folio->page); } atomic_inc(&folio->_entire_mapcount); + atomic_inc(&folio->_large_mapcount); return 0; } @@ -306,6 +307,7 @@ static inline void hugetlb_add_file_rmap(struct folio *folio) VM_WARN_ON_FOLIO(folio_test_anon(folio), folio); atomic_inc(&folio->_entire_mapcount); + atomic_inc(&folio->_large_mapcount); } static inline void hugetlb_remove_rmap(struct folio *folio) @@ -313,11 +315,14 @@ static inline void hugetlb_remove_rmap(struct folio *folio) VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio); atomic_dec(&folio->_entire_mapcount); + atomic_dec(&folio->_large_mapcount); } static __always_inline void __folio_dup_file_rmap(struct folio *folio, struct page *page, int nr_pages, enum rmap_level level) { + const int orig_nr_pages = nr_pages; + __folio_rmap_sanity_checks(folio, page, nr_pages, level); switch (level) { @@ -330,9 +335,11 @@ static __always_inline void __folio_dup_file_rmap(struct folio *folio, do { atomic_inc(&page->_mapcount); } while (page++, --nr_pages > 0); + atomic_add(orig_nr_pages, &folio->_large_mapcount); break; case RMAP_LEVEL_PMD: atomic_inc(&folio->_entire_mapcount); + atomic_inc(&folio->_large_mapcount); break; } } @@ -382,6 +389,7 @@ static __always_inline int __folio_try_dup_anon_rmap(struct folio *folio, struct page *page, int nr_pages, struct vm_area_struct *src_vma, enum rmap_level level) { + const int orig_nr_pages = nr_pages; bool maybe_pinned; int i; @@ -423,6 +431,7 @@ static __always_inline int __folio_try_dup_anon_rmap(struct folio *folio, ClearPageAnonExclusive(page); atomic_inc(&page->_mapcount); } while (page++, --nr_pages > 0); + atomic_add(orig_nr_pages, &folio->_large_mapcount); break; case RMAP_LEVEL_PMD: if (PageAnonExclusive(page)) { @@ -431,6 +440,7 @@ static __always_inline int __folio_try_dup_anon_rmap(struct folio *folio, ClearPageAnonExclusive(page); } atomic_inc(&folio->_entire_mapcount); + atomic_inc(&folio->_large_mapcount); break; } return 0; diff --git a/mm/debug.c b/mm/debug.c index b71186f1fb0b1..d064db42af54e 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -68,8 +68,9 @@ static void __dump_folio(struct folio *folio, struct page *page, folio_ref_count(folio), mapcount, mapping, folio->index + idx, pfn); if (folio_test_large(folio)) { - pr_warn("head: order:%u entire_mapcount:%d nr_pages_mapped:%d pincount:%d\n", + pr_warn("head: order:%u mapcount:%d entire_mapcount:%d nr_pages_mapped:%d pincount:%d\n", folio_order(folio), + folio_mapcount(folio), folio_entire_mapcount(folio), folio_nr_pages_mapped(folio), atomic_read(&folio->_pincount)); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 5dc3f5ea3a2ef..d74289d3f30db 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1517,7 +1517,7 @@ static void __destroy_compound_gigantic_folio(struct folio *folio, struct page *p; atomic_set(&folio->_entire_mapcount, 0); - atomic_set(&folio->_nr_pages_mapped, 0); + atomic_set(&folio->_large_mapcount, 0); atomic_set(&folio->_pincount, 0); for (i = 1; i < nr_pages; i++) { @@ -2120,7 +2120,7 @@ static bool __prep_compound_gigantic_folio(struct folio *folio, /* we rely on prep_new_hugetlb_folio to set the hugetlb flag */ folio_set_order(folio, order); atomic_set(&folio->_entire_mapcount, -1); - atomic_set(&folio->_nr_pages_mapped, 0); + atomic_set(&folio->_large_mapcount, -1); atomic_set(&folio->_pincount, 0); return true; diff --git a/mm/internal.h b/mm/internal.h index 22152e0c8494a..2adc3f616b71f 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -72,6 +72,8 @@ void page_writeback_init(void); /* * How many individual pages have an elevated _mapcount. Excludes * the folio's entire_mapcount. + * + * Don't use this function outside of debugging code. */ static inline int folio_nr_pages_mapped(const struct folio *folio) { @@ -611,6 +613,7 @@ static inline void prep_compound_head(struct page *page, unsigned int order) struct folio *folio = (struct folio *)page; folio_set_order(folio, order); + atomic_set(&folio->_large_mapcount, -1); atomic_set(&folio->_entire_mapcount, -1); atomic_set(&folio->_nr_pages_mapped, 0); atomic_set(&folio->_pincount, 0); diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 89e2624fb3ff1..2f73d2aa9ae84 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1358,7 +1358,7 @@ static int hpage_collapse_scan_pmd(struct mm_struct *mm, * Check if the page has any GUP (or other external) pins. * * Here the check may be racy: - * it may see total_mapcount > refcount in some cases? + * it may see folio_mapcount() > folio_ref_count(). * But such case is ephemeral we could always retry collapse * later. However it may report false positive if the page * has excessive GUP pins (i.e. 512). Anyway the same check diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 22e8b9f1d710b..dd4265c760ff6 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -935,6 +935,10 @@ static int free_tail_page_prepare(struct page *head_page, struct page *page) bad_page(page, "nonzero entire_mapcount"); goto out; } + if (unlikely(folio_large_mapcount(folio))) { + bad_page(page, "nonzero large_mapcount"); + goto out; + } if (unlikely(atomic_read(&folio->_nr_pages_mapped))) { bad_page(page, "nonzero nr_pages_mapped"); goto out; diff --git a/mm/rmap.c b/mm/rmap.c index 4bde6d60db6ca..2608c40dffade 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1138,34 +1138,12 @@ int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff, return page_vma_mkclean_one(&pvmw); } -int folio_total_mapcount(const struct folio *folio) -{ - int mapcount = folio_entire_mapcount(folio); - int nr_pages; - int i; - - /* In the common case, avoid the loop when no pages mapped by PTE */ - if (folio_nr_pages_mapped(folio) == 0) - return mapcount; - /* - * Add all the PTE mappings of those pages mapped by PTE. - * Limit the loop to folio_nr_pages_mapped()? - * Perhaps: given all the raciness, that may be a good or a bad idea. - */ - nr_pages = folio_nr_pages(folio); - for (i = 0; i < nr_pages; i++) - mapcount += atomic_read(&folio_page(folio, i)->_mapcount); - - /* But each of those _mapcounts was based on -1 */ - mapcount += nr_pages; - return mapcount; -} - static __always_inline unsigned int __folio_add_rmap(struct folio *folio, struct page *page, int nr_pages, enum rmap_level level, int *nr_pmdmapped) { atomic_t *mapped = &folio->_nr_pages_mapped; + const int orig_nr_pages = nr_pages; int first, nr = 0; __folio_rmap_sanity_checks(folio, page, nr_pages, level); @@ -1185,6 +1163,7 @@ static __always_inline unsigned int __folio_add_rmap(struct folio *folio, nr++; } } while (page++, --nr_pages > 0); + atomic_add(orig_nr_pages, &folio->_large_mapcount); break; case RMAP_LEVEL_PMD: first = atomic_inc_and_test(&folio->_entire_mapcount); @@ -1201,6 +1180,7 @@ static __always_inline unsigned int __folio_add_rmap(struct folio *folio, nr = 0; } } + atomic_inc(&folio->_large_mapcount); break; } return nr; @@ -1436,10 +1416,14 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, SetPageAnonExclusive(page); } + /* increment count (starts at -1) */ + atomic_set(&folio->_large_mapcount, nr - 1); atomic_set(&folio->_nr_pages_mapped, nr); } else { /* increment count (starts at -1) */ atomic_set(&folio->_entire_mapcount, 0); + /* increment count (starts at -1) */ + atomic_set(&folio->_large_mapcount, 0); atomic_set(&folio->_nr_pages_mapped, ENTIRELY_MAPPED); SetPageAnonExclusive(&folio->page); __lruvec_stat_mod_folio(folio, NR_ANON_THPS, nr); @@ -1522,6 +1506,7 @@ static __always_inline void __folio_remove_rmap(struct folio *folio, break; } + atomic_sub(nr_pages, &folio->_large_mapcount); do { last = atomic_add_negative(-1, &page->_mapcount); if (last) { @@ -1532,6 +1517,7 @@ static __always_inline void __folio_remove_rmap(struct folio *folio, } while (page++, --nr_pages > 0); break; case RMAP_LEVEL_PMD: + atomic_dec(&folio->_large_mapcount); last = atomic_add_negative(-1, &folio->_entire_mapcount); if (last) { nr = atomic_sub_return_relaxed(ENTIRELY_MAPPED, mapped); @@ -2714,6 +2700,7 @@ void hugetlb_add_anon_rmap(struct folio *folio, struct vm_area_struct *vma, VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); atomic_inc(&folio->_entire_mapcount); + atomic_inc(&folio->_large_mapcount); if (flags & RMAP_EXCLUSIVE) SetPageAnonExclusive(&folio->page); VM_WARN_ON_FOLIO(folio_entire_mapcount(folio) > 1 && @@ -2728,6 +2715,7 @@ void hugetlb_add_new_anon_rmap(struct folio *folio, BUG_ON(address < vma->vm_start || address >= vma->vm_end); /* increment count (starts at -1) */ atomic_set(&folio->_entire_mapcount, 0); + atomic_set(&folio->_large_mapcount, 0); folio_clear_hugetlb_restore_reserve(folio); __folio_set_anon(folio, vma, address, true); SetPageAnonExclusive(&folio->page); From eefb9b2725e395d58119a92eb8eaea8f2504b5eb Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:48 +0200 Subject: [PATCH 1124/1702] mm: improve folio_likely_mapped_shared() using the mapcount of large folios We can now read the mapcount of large folios very efficiently. Use it to improve our handling of partially-mappable folios, falling back to making a guess only in case the folio is not "obviously mapped shared". We can now better detect partially-mappable folios where the first page is not mapped as "mapped shared", reducing "false negatives"; but false negatives are still possible. While at it, fixup a wrong comment (false positive vs. false negative) for KSM folios. Link: https://lkml.kernel.org/r/20240409192301.907377-6-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Yin Fengwei Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/mm.h | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 0594778216251..c7470da5797f1 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2184,7 +2184,7 @@ static inline size_t folio_size(struct folio *folio) * indicate "mapped shared" (false positive) when two VMAs in the same MM * cover the same file range. * #. For (small) KSM folios, the return value can wrongly indicate "mapped - * shared" (false negative), when the folio is mapped multiple times into + * shared" (false positive), when the folio is mapped multiple times into * the same MM. * * Further, this function only considers current page table mappings that @@ -2201,7 +2201,22 @@ static inline size_t folio_size(struct folio *folio) */ static inline bool folio_likely_mapped_shared(struct folio *folio) { - return page_mapcount(folio_page(folio, 0)) > 1; + int mapcount = folio_mapcount(folio); + + /* Only partially-mappable folios require more care. */ + if (!folio_test_large(folio) || unlikely(folio_test_hugetlb(folio))) + return mapcount > 1; + + /* A single mapping implies "mapped exclusively". */ + if (mapcount <= 1) + return false; + + /* If any page is mapped more than once we treat it "mapped shared". */ + if (folio_entire_mapcount(folio) || mapcount > folio_nr_pages(folio)) + return true; + + /* Let's guess based on the first subpage. */ + return atomic_read(&folio->_mapcount) > 0; } #ifndef HAVE_ARCH_MAKE_PAGE_ACCESSIBLE From 4103b93b07bceac30bc83cbce81693bb2ea93c22 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:49 +0200 Subject: [PATCH 1125/1702] mm: make folio_mapcount() return 0 for small typed folios We already handle it properly for large folios. Let's also return "0" for small typed folios, like page_mapcount() currently would. Consequently, folio_mapcount() will never return negative values for typed folios, but may return negative values for underflows. [david@redhat.com: make folio_mapcount() slightly more efficient] Link: https://lkml.kernel.org/r/c30fcda1-ed87-46f5-8297-cdedbddac009@redhat.com Link: https://lkml.kernel.org/r/20240409192301.907377-7-david@redhat.com Signed-off-by: David Hildenbrand Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yin Fengwei Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- include/linux/mm.h | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index c7470da5797f1..78e583b50e421 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1261,12 +1261,22 @@ static inline int folio_large_mapcount(const struct folio *folio) * references the entire folio counts exactly once, even when such special * page table entries are comprised of multiple ordinary page table entries. * + * Will report 0 for pages which cannot be mapped into userspace, such as + * slab, page tables and similar. + * * Return: The number of times this folio is mapped. */ static inline int folio_mapcount(const struct folio *folio) { - if (likely(!folio_test_large(folio))) - return atomic_read(&folio->_mapcount) + 1; + int mapcount; + + if (likely(!folio_test_large(folio))) { + mapcount = atomic_read(&folio->_mapcount) + 1; + /* Handle page_has_type() pages */ + if (mapcount < PAGE_MAPCOUNT_RESERVE + 1) + mapcount = 0; + return mapcount; + } return folio_large_mapcount(folio); } From 3aeea4fc835d31235947787c2b8dcbc255131106 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:50 +0200 Subject: [PATCH 1126/1702] mm/memory: use folio_mapcount() in zap_present_folio_ptes() We want to limit the use of page_mapcount() to the places where it is absolutely necessary. In zap_present_folio_ptes(), let's simply check the folio mapcount(). If there is some issue, it will underflow at some point either way when unmapping. As indicated already in commit 10ebac4f95e7 ("mm/memory: optimize unmap/zap with PTE-mapped THP"), we already documented "If we ever have a cheap folio_mapcount(), we might just want to check for underflows there.". There is no change for small folios. For large folios, we'll now catch more underflows when batch-unmapping, because instead of only testing the mapcount of the first subpage, we'll test if the folio mapcount underflows. Link: https://lkml.kernel.org/r/20240409192301.907377-8-david@redhat.com Signed-off-by: David Hildenbrand Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yin Fengwei Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/memory.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index 36ba94eae853c..d592128d7f7d5 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1502,8 +1502,7 @@ static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, if (!delay_rmap) { folio_remove_rmap_ptes(folio, page, nr, vma); - /* Only sanity-check the first page in a batch. */ - if (unlikely(page_mapcount(page) < 0)) + if (unlikely(folio_mapcount(folio) < 0)) print_bad_pte(vma, addr, ptent, page); } From 0a7bda48012b521579898fca1fc157c53014afee Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:51 +0200 Subject: [PATCH 1127/1702] mm/huge_memory: use folio_mapcount() in zap_huge_pmd() sanity check We want to limit the use of page_mapcount() to the places where it is absolutely necessary. Let's similarly check for folio_mapcount() underflows instead of page_mapcount() underflows like we do in zap_present_folio_ptes() now. Instead of the VM_BUG_ON(), we should actually be doing something like print_bad_pte(). For now, let's keep it simple and use WARN_ON_ONCE(), performing that check independently of DEBUG_VM. Link: https://lkml.kernel.org/r/20240409192301.907377-9-david@redhat.com Signed-off-by: David Hildenbrand Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yin Fengwei Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/huge_memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 14f04fbc22d9a..3ad76f23fadd0 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1791,7 +1791,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, folio = page_folio(page); folio_remove_rmap_pmd(folio, page, vma); - VM_BUG_ON_PAGE(page_mapcount(page) < 0, page); + WARN_ON_ONCE(folio_mapcount(folio) < 0); VM_BUG_ON_PAGE(!PageHead(page), page); } else if (thp_migration_supported()) { swp_entry_t entry; From 33d844bb8433d0474b11615c06feb3607f609821 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:52 +0200 Subject: [PATCH 1128/1702] mm/memory-failure: use folio_mapcount() in hwpoison_user_mappings() We want to limit the use of page_mapcount() to the places where it is absolutely necessary. We can only unmap full folios; page_mapped(), which we check here, is translated to folio_mapped() -- based on folio_mapcount(). So let's print the folio mapcount instead. Link: https://lkml.kernel.org/r/20240409192301.907377-10-david@redhat.com Signed-off-by: David Hildenbrand Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yin Fengwei Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/memory-failure.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 4f22330600ffb..db8d770ce8012 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1634,8 +1634,8 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, unmap_success = !page_mapped(p); if (!unmap_success) - pr_err("%#lx: failed to unmap page (mapcount=%d)\n", - pfn, page_mapcount(p)); + pr_err("%#lx: failed to unmap page (folio mapcount=%d)\n", + pfn, folio_mapcount(page_folio(p))); /* * try_to_unmap() might put mlocked page in lru cache, so call From 7115936ac10abd0997f2937e4305cb679d47b592 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:53 +0200 Subject: [PATCH 1129/1702] mm/page_alloc: use folio_mapped() in __alloc_contig_migrate_range() We want to limit the use of page_mapcount() to the places where it is absolutely necessary. For tracing purposes, we use page_mapcount() in __alloc_contig_migrate_range(). Adding that mapcount to total_mapped sounds strange: total_migrated and total_reclaimed would count each page only once, not multiple times. But then, isolate_migratepages_range() adds each folio only once to the list. So for large folios, we would query the mapcount of the first page of the folio, which doesn't make too much sense for large folios. Let's simply use folio_mapped() * folio_nr_pages(), which makes more sense as nr_migratepages is also incremented by the number of pages in the folio in case of successful migration. Link: https://lkml.kernel.org/r/20240409192301.907377-11-david@redhat.com Signed-off-by: David Hildenbrand Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yin Fengwei Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/page_alloc.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index dd4265c760ff6..45fccbcda763d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6347,8 +6347,12 @@ int __alloc_contig_migrate_range(struct compact_control *cc, if (trace_mm_alloc_contig_migrate_range_info_enabled()) { total_reclaimed += nr_reclaimed; - list_for_each_entry(page, &cc->migratepages, lru) - total_mapped += page_mapcount(page); + list_for_each_entry(page, &cc->migratepages, lru) { + struct folio *folio = page_folio(page); + + total_mapped += folio_mapped(folio) * + folio_nr_pages(folio); + } } ret = migrate_pages(&cc->migratepages, alloc_migration_target, From 31ce0d7ef8412e3f38cf90bb90b7436130c60614 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:54 +0200 Subject: [PATCH 1130/1702] mm/migrate: use folio_likely_mapped_shared() in add_page_for_migration() We want to limit the use of page_mapcount() to the places where it is absolutely necessary. In add_page_for_migration(), we actually want to check if the folio is mapped shared, to reject such folios. So let's use folio_likely_mapped_shared() instead. For small folios, fully mapped THP, and hugetlb folios, there is no change. For partially mapped, shared THP, we should now do a better job at rejecting such folios. Link: https://lkml.kernel.org/r/20240409192301.907377-12-david@redhat.com Signed-off-by: David Hildenbrand Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yin Fengwei Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/migrate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/migrate.c b/mm/migrate.c index 285072bca29ce..d87ce32645d4f 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2140,7 +2140,7 @@ static int add_page_for_migration(struct mm_struct *mm, const void __user *p, goto out_putfolio; err = -EACCES; - if (page_mapcount(page) > 1 && !migrate_all) + if (folio_likely_mapped_shared(folio) && !migrate_all) goto out_putfolio; err = -EBUSY; From 607065804b3b2943d8f99da88744b389a8226dc1 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:55 +0200 Subject: [PATCH 1131/1702] sh/mm/cache: use folio_mapped() in copy_from_user_page() We want to limit the use of page_mapcount() to the places where it is absolutely necessary. We're already using folio_mapped in copy_user_highpage() and copy_to_user_page() for a similar purpose so ... let's also simply use it for copy_from_user_page(). There is no change for small folios. Likely we won't stumble over many large folios on sh in that code either way. Link: https://lkml.kernel.org/r/20240409192301.907377-13-david@redhat.com Signed-off-by: David Hildenbrand Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yin Fengwei Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/sh/mm/cache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sh/mm/cache.c b/arch/sh/mm/cache.c index 9bcaa5619eabd..d8be352e14d20 100644 --- a/arch/sh/mm/cache.c +++ b/arch/sh/mm/cache.c @@ -84,7 +84,7 @@ void copy_from_user_page(struct vm_area_struct *vma, struct page *page, { struct folio *folio = page_folio(page); - if (boot_cpu_data.dcache.n_aliases && page_mapcount(page) && + if (boot_cpu_data.dcache.n_aliases && folio_mapped(folio) && test_bit(PG_dcache_clean, &folio->flags)) { void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK); memcpy(dst, vfrom, len); From f0376c71093577ab9ca30721f8a7f5795d1e88c7 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:56 +0200 Subject: [PATCH 1132/1702] mm/filemap: use folio_mapcount() in filemap_unaccount_folio() We want to limit the use of page_mapcount() to the places where it is absolutely necessary. Let's use folio_mapcount() instead of filemap_unaccount_folio(). No functional change intended, because we're only dealing with small folios. Link: https://lkml.kernel.org/r/20240409192301.907377-14-david@redhat.com Signed-off-by: David Hildenbrand Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yin Fengwei Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/filemap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/filemap.c b/mm/filemap.c index 18f4bce3741ec..21e70434d9318 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -168,7 +168,7 @@ static void filemap_unaccount_folio(struct address_space *mapping, add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); if (mapping_exiting(mapping) && !folio_test_large(folio)) { - int mapcount = page_mapcount(&folio->page); + int mapcount = folio_mapcount(folio); if (folio_ref_count(folio) >= mapcount + 2) { /* From f2f8a7a006dca086d942eec2e469f8d831b62443 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:57 +0200 Subject: [PATCH 1133/1702] mm/migrate_device: use folio_mapcount() in migrate_vma_check_page() We want to limit the use of page_mapcount() to the places where it is absolutely necessary. Let's convert migrate_vma_check_page() to work on a folio internally so we can remove the page_mapcount() usage. Note that we reject any large folios. There is a lot more folio conversion to be had, but that has to wait for another day. No functional change intended. Link: https://lkml.kernel.org/r/20240409192301.907377-15-david@redhat.com Signed-off-by: David Hildenbrand Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yin Fengwei Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/migrate_device.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mm/migrate_device.c b/mm/migrate_device.c index d40b46ae9d65b..b929b450b77c3 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -324,6 +324,8 @@ static void migrate_vma_collect(struct migrate_vma *migrate) */ static bool migrate_vma_check_page(struct page *page, struct page *fault_page) { + struct folio *folio = page_folio(page); + /* * One extra ref because caller holds an extra reference, either from * isolate_lru_page() for a regular page, or migrate_vma_collect() for @@ -336,18 +338,18 @@ static bool migrate_vma_check_page(struct page *page, struct page *fault_page) * check them than regular pages, because they can be mapped with a pmd * or with a pte (split pte mapping). */ - if (PageCompound(page)) + if (folio_test_large(folio)) return false; /* Page from ZONE_DEVICE have one extra reference */ - if (is_zone_device_page(page)) + if (folio_is_zone_device(folio)) extra++; /* For file back page */ - if (page_mapping(page)) - extra += 1 + page_has_private(page); + if (folio_mapping(folio)) + extra += 1 + folio_has_private(folio); - if ((page_count(page) - extra) > page_mapcount(page)) + if ((folio_ref_count(folio) - extra) > folio_mapcount(folio)) return false; return true; From 6eca32567455db2db38b1126e0d6ad8f0e5c3ed9 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:58 +0200 Subject: [PATCH 1134/1702] trace/events/page_ref: trace the raw page mapcount value We want to limit the use of page_mapcount() to the places where it is absolutely necessary. We already trace raw page->refcount, raw page->flags and raw page->mapping, and don't involve any folios. Let's also trace the raw mapcount value that does not consider the entire mapcount of large folios, and we don't add "1" to it. When dealing with typed folios, this makes a lot more sense. ... and it's for debugging purposes only either way. Link: https://lkml.kernel.org/r/20240409192301.907377-16-david@redhat.com Signed-off-by: David Hildenbrand Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yin Fengwei Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- include/trace/events/page_ref.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/trace/events/page_ref.h b/include/trace/events/page_ref.h index 8a99c1cd417b8..fe33a255b7d09 100644 --- a/include/trace/events/page_ref.h +++ b/include/trace/events/page_ref.h @@ -30,7 +30,7 @@ DECLARE_EVENT_CLASS(page_ref_mod_template, __entry->pfn = page_to_pfn(page); __entry->flags = page->flags; __entry->count = page_ref_count(page); - __entry->mapcount = page_mapcount(page); + __entry->mapcount = atomic_read(&page->_mapcount); __entry->mapping = page->mapping; __entry->mt = get_pageblock_migratetype(page); __entry->val = v; @@ -79,7 +79,7 @@ DECLARE_EVENT_CLASS(page_ref_mod_and_test_template, __entry->pfn = page_to_pfn(page); __entry->flags = page->flags; __entry->count = page_ref_count(page); - __entry->mapcount = page_mapcount(page); + __entry->mapcount = atomic_read(&page->_mapcount); __entry->mapping = page->mapping; __entry->mt = get_pageblock_migratetype(page); __entry->val = v; From 5f8856cdae5db200844630e660f1a64eb1560578 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:22:59 +0200 Subject: [PATCH 1135/1702] xtensa/mm: convert check_tlb_entry() to sanity check folios We want to limit the use of page_mapcount() to the places where it is absolutely necessary. So let's convert check_tlb_entry() to perform sanity checks on folios instead of pages. This essentially already happened: page_count() is mapped to folio_ref_count(), and page_mapped() to folio_mapped() internally. However, we would have printed the page_mapount(), which does not really match what page_mapped() would have checked. Let's simply print the folio mapcount to avoid using page_mapcount(). For small folios there is no change. Link: https://lkml.kernel.org/r/20240409192301.907377-17-david@redhat.com Signed-off-by: David Hildenbrand Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yin Fengwei Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- arch/xtensa/mm/tlb.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/xtensa/mm/tlb.c b/arch/xtensa/mm/tlb.c index 4f974b74883ca..d8b60d6e50a8e 100644 --- a/arch/xtensa/mm/tlb.c +++ b/arch/xtensa/mm/tlb.c @@ -256,12 +256,13 @@ static int check_tlb_entry(unsigned w, unsigned e, bool dtlb) dtlb ? 'D' : 'I', w, e, r0, r1, pte); if (pte == 0 || !pte_present(__pte(pte))) { struct page *p = pfn_to_page(r1 >> PAGE_SHIFT); - pr_err("page refcount: %d, mapcount: %d\n", - page_count(p), - page_mapcount(p)); - if (!page_count(p)) + struct folio *f = page_folio(p); + + pr_err("folio refcount: %d, mapcount: %d\n", + folio_ref_count(f), folio_mapcount(f)); + if (!folio_ref_count(f)) rc |= TLB_INSANE; - else if (page_mapcount(p)) + else if (folio_mapped(f)) rc |= TLB_SUSPICIOUS; } else { rc |= TLB_INSANE; From 7441d34922ba1aba7bbb069c27bcaabdc515f34e Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:23:00 +0200 Subject: [PATCH 1136/1702] mm/debug: print only page mapcount (excluding folio entire mapcount) in __dump_folio() Let's simplify and only print the page mapcount: we already print the large folio mapcount and the entire folio mapcount for large folios separately; that should be sufficient to figure out what's happening. While at it, print the page mapcount also if it had an underflow, filtering out only typed pages. Link: https://lkml.kernel.org/r/20240409192301.907377-18-david@redhat.com Signed-off-by: David Hildenbrand Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yin Fengwei Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- mm/debug.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/mm/debug.c b/mm/debug.c index d064db42af54e..69e524c3e6012 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -55,15 +55,10 @@ static void __dump_folio(struct folio *folio, struct page *page, unsigned long pfn, unsigned long idx) { struct address_space *mapping = folio_mapping(folio); - int mapcount = atomic_read(&page->_mapcount) + 1; + int mapcount = atomic_read(&page->_mapcount); char *type = ""; - /* Open-code page_mapcount() to avoid looking up a stale folio */ - if (mapcount < 0) - mapcount = 0; - if (folio_test_large(folio)) - mapcount += folio_entire_mapcount(folio); - + mapcount = page_type_has_type(mapcount) ? 0 : mapcount + 1; pr_warn("page: refcount:%d mapcount:%d mapping:%p index:%#lx pfn:%#lx\n", folio_ref_count(folio), mapcount, mapping, folio->index + idx, pfn); From 658670607fae51ed2f8a64fcbfcd407bf820dd4f Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 9 Apr 2024 21:23:01 +0200 Subject: [PATCH 1137/1702] Documentation/admin-guide/cgroup-v1/memory.rst: don't reference page_mapcount() Let's stop talking about page_mapcount(). Link: https://lkml.kernel.org/r/20240409192301.907377-19-david@redhat.com Signed-off-by: David Hildenbrand Cc: Chris Zankel Cc: Hugh Dickins Cc: John Paul Adrian Glaubitz Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Max Filippov Cc: Miaohe Lin Cc: Muchun Song Cc: Naoya Horiguchi Cc: Peter Xu Cc: Richard Chang Cc: Rich Felker Cc: Ryan Roberts Cc: Yang Shi Cc: Yin Fengwei Cc: Yoshinori Sato Cc: Zi Yan Signed-off-by: Andrew Morton --- Documentation/admin-guide/cgroup-v1/memory.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index 46110e6a31bbd..9cde26d338435 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -802,8 +802,8 @@ a page or a swap can be moved only when it is charged to the task's current | | anonymous pages, file pages (and swaps) in the range mmapped by the task | | | will be moved even if the task hasn't done page fault, i.e. they might | | | not be the task's "RSS", but other task's "RSS" that maps the same file. | -| | And mapcount of the page is ignored (the page can be moved even if | -| | page_mapcount(page) > 1). You must enable Swap Extension (see 2.4) to | +| | The mapcount of the page is ignored (the page can be moved independent | +| | of the mapcount). You must enable Swap Extension (see 2.4) to | | | enable move of swap charges. | +---+--------------------------------------------------------------------------+ From eebb5181a02f819cec5e324a9df1015934dc59f3 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Thu, 11 Apr 2024 21:09:24 +0800 Subject: [PATCH 1138/1702] arm64: mm: drop VM_FAULT_BADMAP/VM_FAULT_BADACCESS Patch series "mm: remove arch's private VM_FAULT_BADMAP/BADACCESS", v2. Directly set SEGV_MAPRR or SEGV_ACCERR for arm/arm64 to remove the last two arch's private vm_fault reasons. This patch (of 2): If bad map or access, directly set si_code to SEGV_MAPRR or SEGV_ACCERR, also set fault to 0 and goto error handling, which make us to drop the arch's special vm fault reason. Link: https://lkml.kernel.org/r/20240411130925.73281-1-wangkefeng.wang@huawei.com Link: https://lkml.kernel.org/r/20240411130925.73281-2-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviewed-by: Catalin Marinas Cc: Aishwarya TCV Cc: Cristian Marussi Cc: Mark Brown Cc: Russell King Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/arm64/mm/fault.c | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 405f9aa831bd2..451ba7cbd5adb 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -500,9 +500,6 @@ static bool is_write_abort(unsigned long esr) return (esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM); } -#define VM_FAULT_BADMAP ((__force vm_fault_t)0x010000) -#define VM_FAULT_BADACCESS ((__force vm_fault_t)0x020000) - static int __kprobes do_page_fault(unsigned long far, unsigned long esr, struct pt_regs *regs) { @@ -513,6 +510,7 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, unsigned int mm_flags = FAULT_FLAG_DEFAULT; unsigned long addr = untagged_addr(far); struct vm_area_struct *vma; + int si_code; if (kprobe_page_fault(regs, esr)) return 0; @@ -572,9 +570,10 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, if (!(vma->vm_flags & vm_flags)) { vma_end_read(vma); - fault = VM_FAULT_BADACCESS; + fault = 0; + si_code = SEGV_ACCERR; count_vm_vma_lock_event(VMA_LOCK_SUCCESS); - goto done; + goto bad_area; } fault = handle_mm_fault(vma, addr, mm_flags | FAULT_FLAG_VMA_LOCK, regs); if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED))) @@ -599,15 +598,19 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, retry: vma = lock_mm_and_find_vma(mm, addr, regs); if (unlikely(!vma)) { - fault = VM_FAULT_BADMAP; - goto done; + fault = 0; + si_code = SEGV_MAPERR; + goto bad_area; } - if (!(vma->vm_flags & vm_flags)) - fault = VM_FAULT_BADACCESS; - else - fault = handle_mm_fault(vma, addr, mm_flags, regs); + if (!(vma->vm_flags & vm_flags)) { + mmap_read_unlock(mm); + fault = 0; + si_code = SEGV_ACCERR; + goto bad_area; + } + fault = handle_mm_fault(vma, addr, mm_flags, regs); /* Quick path to respond to signals */ if (fault_signal_pending(fault, regs)) { if (!user_mode(regs)) @@ -626,13 +629,12 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, mmap_read_unlock(mm); done: - /* - * Handle the "normal" (no error) case first. - */ - if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | - VM_FAULT_BADACCESS)))) + /* Handle the "normal" (no error) case first. */ + if (likely(!(fault & VM_FAULT_ERROR))) return 0; + si_code = SEGV_MAPERR; +bad_area: /* * If we are in kernel mode at this point, we have no context to * handle this fault with. @@ -667,13 +669,8 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, arm64_force_sig_mceerr(BUS_MCEERR_AR, far, lsb, inf->name); } else { - /* - * Something tried to access memory that isn't in our memory - * map. - */ - arm64_force_sig_fault(SIGSEGV, - fault == VM_FAULT_BADACCESS ? SEGV_ACCERR : SEGV_MAPERR, - far, inf->name); + /* Something tried to access memory that out of memory map */ + arm64_force_sig_fault(SIGSEGV, si_code, far, inf->name); } return 0; From e9016174621112624f957c8138bd3c34692e2e93 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Thu, 11 Apr 2024 21:09:25 +0800 Subject: [PATCH 1139/1702] arm: mm: drop VM_FAULT_BADMAP/VM_FAULT_BADACCESS If bad map or access, directly set code to SEGV_MAPRR or SEGV_ACCERR, also set fault to 0 and goto error handling, which make us to drop the arch's special vm fault reason. [akpm@linux-foundation.org: coding-style cleanups] Link: https://lkml.kernel.org/r/20240411130925.73281-3-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Cc: Aishwarya TCV Cc: Catalin Marinas Cc: Cristian Marussi Cc: Mark Brown Cc: Russell King Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/arm/mm/fault.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index 5c4b417e24f9e..bf07afdb2dd9e 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -226,9 +226,6 @@ void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs) } #ifdef CONFIG_MMU -#define VM_FAULT_BADMAP ((__force vm_fault_t)0x010000) -#define VM_FAULT_BADACCESS ((__force vm_fault_t)0x020000) - static inline bool is_permission_fault(unsigned int fsr) { int fs = fsr_fs(fsr); @@ -295,7 +292,8 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) if (!(vma->vm_flags & vm_flags)) { vma_end_read(vma); count_vm_vma_lock_event(VMA_LOCK_SUCCESS); - fault = VM_FAULT_BADACCESS; + fault = 0; + code = SEGV_ACCERR; goto bad_area; } fault = handle_mm_fault(vma, addr, flags | FAULT_FLAG_VMA_LOCK, regs); @@ -321,7 +319,8 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) retry: vma = lock_mm_and_find_vma(mm, addr, regs); if (unlikely(!vma)) { - fault = VM_FAULT_BADMAP; + fault = 0; + code = SEGV_MAPERR; goto bad_area; } @@ -329,10 +328,14 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) * ok, we have a good vm_area for this memory access, check the * permissions on the VMA allow for the fault which occurred. */ - if (!(vma->vm_flags & vm_flags)) - fault = VM_FAULT_BADACCESS; - else - fault = handle_mm_fault(vma, addr & PAGE_MASK, flags, regs); + if (!(vma->vm_flags & vm_flags)) { + mmap_read_unlock(mm); + fault = 0; + code = SEGV_ACCERR; + goto bad_area; + } + + fault = handle_mm_fault(vma, addr & PAGE_MASK, flags, regs); /* If we need to retry but a fatal signal is pending, handle the * signal first. We do not need to release the mmap_lock because @@ -358,12 +361,11 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) mmap_read_unlock(mm); done: - /* - * Handle the "normal" case first - VM_FAULT_MAJOR - */ - if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | VM_FAULT_BADACCESS)))) + /* Handle the "normal" case first */ + if (likely(!(fault & VM_FAULT_ERROR))) return 0; + code = SEGV_MAPERR; bad_area: /* * If we are in kernel mode at this point, we @@ -395,8 +397,6 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) * isn't in our memory map.. */ sig = SIGSEGV; - code = fault == VM_FAULT_BADACCESS ? - SEGV_ACCERR : SEGV_MAPERR; } __do_user_fault(addr, fsr, sig, code, regs); From b91f94729d050eec86ae7ef084aa3805146c0a67 Mon Sep 17 00:00:00 2001 From: "Alex Shi (tencent)" Date: Thu, 11 Apr 2024 14:17:02 +0800 Subject: [PATCH 1140/1702] mm/ksm: add ksm_get_folio Patch series "transfer page to folio in KSM". This is the first part of page to folio transfer on KSM. Since only single page could be stored in KSM, we could safely transfer stable tree pages to folios. This patchset could reduce ksm.o 57kbytes from 2541776 bytes on latest akpm/mm-stable branch with CONFIG_DEBUG_VM enabled. It pass the KSM testing in LTP and kernel selftest. Thanks for Matthew Wilcox and David Hildenbrand's suggestions and comments! This patch (of 10): The ksm only contains single pages, so we could add a new func ksm_get_folio for get_ksm_page to use folio instead of pages to save a couple of compound_head calls. After all caller replaced, get_ksm_page will be removed. Link: https://lkml.kernel.org/r/20240411061713.1847574-1-alexs@kernel.org Link: https://lkml.kernel.org/r/20240411061713.1847574-2-alexs@kernel.org Signed-off-by: Alex Shi (tencent) Reviewed-by: David Hildenbrand Cc: Izik Eidus Cc: Matthew Wilcox Cc: Andrea Arcangeli Cc: Hugh Dickins Cc: Chris Wright Signed-off-by: Andrew Morton --- mm/ksm.c | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/mm/ksm.c b/mm/ksm.c index 8c001819cf10f..ac126a4c245c0 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -897,7 +897,7 @@ enum get_ksm_page_flags { }; /* - * get_ksm_page: checks if the page indicated by the stable node + * ksm_get_folio: checks if the page indicated by the stable node * is still its ksm page, despite having held no reference to it. * In which case we can trust the content of the page, and it * returns the gotten page; but if the page has now been zapped, @@ -915,10 +915,10 @@ enum get_ksm_page_flags { * a page to put something that might look like our key in page->mapping. * is on its way to being freed; but it is an anomaly to bear in mind. */ -static struct page *get_ksm_page(struct ksm_stable_node *stable_node, +static struct folio *ksm_get_folio(struct ksm_stable_node *stable_node, enum get_ksm_page_flags flags) { - struct page *page; + struct folio *folio; void *expected_mapping; unsigned long kpfn; @@ -926,8 +926,8 @@ static struct page *get_ksm_page(struct ksm_stable_node *stable_node, PAGE_MAPPING_KSM); again: kpfn = READ_ONCE(stable_node->kpfn); /* Address dependency. */ - page = pfn_to_page(kpfn); - if (READ_ONCE(page->mapping) != expected_mapping) + folio = pfn_folio(kpfn); + if (READ_ONCE(folio->mapping) != expected_mapping) goto stale; /* @@ -940,41 +940,41 @@ static struct page *get_ksm_page(struct ksm_stable_node *stable_node, * in folio_migrate_mapping(), it might still be our page, * in which case it's essential to keep the node. */ - while (!get_page_unless_zero(page)) { + while (!folio_try_get(folio)) { /* * Another check for page->mapping != expected_mapping would * work here too. We have chosen the !PageSwapCache test to * optimize the common case, when the page is or is about to * be freed: PageSwapCache is cleared (under spin_lock_irq) * in the ref_freeze section of __remove_mapping(); but Anon - * page->mapping reset to NULL later, in free_pages_prepare(). + * folio->mapping reset to NULL later, in free_pages_prepare(). */ - if (!PageSwapCache(page)) + if (!folio_test_swapcache(folio)) goto stale; cpu_relax(); } - if (READ_ONCE(page->mapping) != expected_mapping) { - put_page(page); + if (READ_ONCE(folio->mapping) != expected_mapping) { + folio_put(folio); goto stale; } if (flags == GET_KSM_PAGE_TRYLOCK) { - if (!trylock_page(page)) { - put_page(page); + if (!folio_trylock(folio)) { + folio_put(folio); return ERR_PTR(-EBUSY); } } else if (flags == GET_KSM_PAGE_LOCK) - lock_page(page); + folio_lock(folio); if (flags != GET_KSM_PAGE_NOLOCK) { - if (READ_ONCE(page->mapping) != expected_mapping) { - unlock_page(page); - put_page(page); + if (READ_ONCE(folio->mapping) != expected_mapping) { + folio_unlock(folio); + folio_put(folio); goto stale; } } - return page; + return folio; stale: /* @@ -990,6 +990,14 @@ static struct page *get_ksm_page(struct ksm_stable_node *stable_node, return NULL; } +static struct page *get_ksm_page(struct ksm_stable_node *stable_node, + enum get_ksm_page_flags flags) +{ + struct folio *folio = ksm_get_folio(stable_node, flags); + + return &folio->page; +} + /* * Removing rmap_item from stable or unstable tree. * This function will clean the information from the stable/unstable tree. From f39b6e2dc18efcc7f8d55cdf62c8c6cad47d1f06 Mon Sep 17 00:00:00 2001 From: "Alex Shi (tencent)" Date: Thu, 11 Apr 2024 14:17:03 +0800 Subject: [PATCH 1141/1702] mm/ksm: use folio in remove_rmap_item_from_tree To save 2 compound_head calls. Link: https://lkml.kernel.org/r/20240411061713.1847574-3-alexs@kernel.org Signed-off-by: Alex Shi (tencent) Reviewed-by: David Hildenbrand Cc: Izik Eidus Cc: Matthew Wilcox Cc: Andrea Arcangeli Cc: Hugh Dickins Cc: Chris Wright Signed-off-by: Andrew Morton --- mm/ksm.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mm/ksm.c b/mm/ksm.c index ac126a4c245c0..ef5c4b6d377c2 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -1006,16 +1006,16 @@ static void remove_rmap_item_from_tree(struct ksm_rmap_item *rmap_item) { if (rmap_item->address & STABLE_FLAG) { struct ksm_stable_node *stable_node; - struct page *page; + struct folio *folio; stable_node = rmap_item->head; - page = get_ksm_page(stable_node, GET_KSM_PAGE_LOCK); - if (!page) + folio = ksm_get_folio(stable_node, GET_KSM_PAGE_LOCK); + if (!folio) goto out; hlist_del(&rmap_item->hlist); - unlock_page(page); - put_page(page); + folio_unlock(folio); + folio_put(folio); if (!hlist_empty(&stable_node->hlist)) ksm_pages_sharing--; From b8b0ff244ddca0d475c91c9accf144e25311a951 Mon Sep 17 00:00:00 2001 From: "Alex Shi (tencent)" Date: Thu, 11 Apr 2024 14:17:04 +0800 Subject: [PATCH 1142/1702] mm/ksm: add folio_set_stable_node Turn set_page_stable_node() into a wrapper folio_set_stable_node, and then use it to replace the former. we will merge them together after all place converted to folio. Link: https://lkml.kernel.org/r/20240411061713.1847574-4-alexs@kernel.org Signed-off-by: Alex Shi (tencent) Reviewed-by: David Hildenbrand Cc: Izik Eidus Cc: Matthew Wilcox Cc: Andrea Arcangeli Cc: Hugh Dickins Cc: Chris Wright Signed-off-by: Andrew Morton --- mm/ksm.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mm/ksm.c b/mm/ksm.c index ef5c4b6d377c2..3c52bf9df84c8 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -1109,6 +1109,12 @@ static inline void set_page_stable_node(struct page *page, page->mapping = (void *)((unsigned long)stable_node | PAGE_MAPPING_KSM); } +static inline void folio_set_stable_node(struct folio *folio, + struct ksm_stable_node *stable_node) +{ + set_page_stable_node(&folio->page, stable_node); +} + #ifdef CONFIG_SYSFS /* * Only called through the sysfs control interface: @@ -3241,7 +3247,7 @@ void folio_migrate_ksm(struct folio *newfolio, struct folio *folio) * has gone stale (or that folio_test_swapcache has been cleared). */ smp_wmb(); - set_page_stable_node(&folio->page, NULL); + folio_set_stable_node(folio, NULL); } } #endif /* CONFIG_MIGRATION */ From 9d5cc14093594f0a5f8318cc70208bbbe71f5fce Mon Sep 17 00:00:00 2001 From: "Alex Shi (tencent)" Date: Thu, 11 Apr 2024 14:17:05 +0800 Subject: [PATCH 1143/1702] mm/ksm: use folio in remove_stable_node Pages in stable tree are all single normal page, so uses ksm_get_folio() and folio_set_stable_node(), also saves 3 calls to compound_head(). Link: https://lkml.kernel.org/r/20240411061713.1847574-5-alexs@kernel.org Signed-off-by: Alex Shi (tencent) Reviewed-by: David Hildenbrand Cc: Izik Eidus Cc: Matthew Wilcox Cc: Andrea Arcangeli Cc: Hugh Dickins Cc: Chris Wright Signed-off-by: Andrew Morton --- mm/ksm.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/mm/ksm.c b/mm/ksm.c index 3c52bf9df84c8..1a7b130045898 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -1121,13 +1121,13 @@ static inline void folio_set_stable_node(struct folio *folio, */ static int remove_stable_node(struct ksm_stable_node *stable_node) { - struct page *page; + struct folio *folio; int err; - page = get_ksm_page(stable_node, GET_KSM_PAGE_LOCK); - if (!page) { + folio = ksm_get_folio(stable_node, GET_KSM_PAGE_LOCK); + if (!folio) { /* - * get_ksm_page did remove_node_from_stable_tree itself. + * ksm_get_folio did remove_node_from_stable_tree itself. */ return 0; } @@ -1138,22 +1138,22 @@ static int remove_stable_node(struct ksm_stable_node *stable_node) * merge_across_nodes/max_page_sharing be switched. */ err = -EBUSY; - if (!page_mapped(page)) { + if (!folio_mapped(folio)) { /* - * The stable node did not yet appear stale to get_ksm_page(), - * since that allows for an unmapped ksm page to be recognized + * The stable node did not yet appear stale to ksm_get_folio(), + * since that allows for an unmapped ksm folio to be recognized * right up until it is freed; but the node is safe to remove. - * This page might be in an LRU cache waiting to be freed, - * or it might be PageSwapCache (perhaps under writeback), + * This folio might be in an LRU cache waiting to be freed, + * or it might be in the swapcache (perhaps under writeback), * or it might have been removed from swapcache a moment ago. */ - set_page_stable_node(page, NULL); + folio_set_stable_node(folio, NULL); remove_node_from_stable_tree(stable_node); err = 0; } - unlock_page(page); - put_page(page); + folio_unlock(folio); + folio_put(folio); return err; } From 6f528de2986e179bf4a100dcc8db2a6703aa7885 Mon Sep 17 00:00:00 2001 From: "Alex Shi (tencent)" Date: Thu, 11 Apr 2024 14:17:06 +0800 Subject: [PATCH 1144/1702] mm/ksm: use folio in stable_node_dup Use ksm_get_folio() and save 2 compound_head calls. Link: https://lkml.kernel.org/r/20240411061713.1847574-6-alexs@kernel.org Signed-off-by: Alex Shi (tencent) Reviewed-by: David Hildenbrand Cc: Izik Eidus Cc: Matthew Wilcox Cc: Andrea Arcangeli Cc: Hugh Dickins Cc: Chris Wright Signed-off-by: Andrew Morton --- mm/ksm.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mm/ksm.c b/mm/ksm.c index 1a7b130045898..654400f993fcc 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -1638,7 +1638,7 @@ static struct page *stable_node_dup(struct ksm_stable_node **_stable_node_dup, { struct ksm_stable_node *dup, *found = NULL, *stable_node = *_stable_node; struct hlist_node *hlist_safe; - struct page *_tree_page, *tree_page = NULL; + struct folio *folio, *tree_folio = NULL; int nr = 0; int found_rmap_hlist_len; @@ -1657,24 +1657,24 @@ static struct page *stable_node_dup(struct ksm_stable_node **_stable_node_dup, * We must walk all stable_node_dup to prune the stale * stable nodes during lookup. * - * get_ksm_page can drop the nodes from the + * ksm_get_folio can drop the nodes from the * stable_node->hlist if they point to freed pages * (that's why we do a _safe walk). The "dup" * stable_node parameter itself will be freed from * under us if it returns NULL. */ - _tree_page = get_ksm_page(dup, GET_KSM_PAGE_NOLOCK); - if (!_tree_page) + folio = ksm_get_folio(dup, GET_KSM_PAGE_NOLOCK); + if (!folio) continue; nr += 1; if (is_page_sharing_candidate(dup)) { if (!found || dup->rmap_hlist_len > found_rmap_hlist_len) { if (found) - put_page(tree_page); + folio_put(tree_folio); found = dup; found_rmap_hlist_len = found->rmap_hlist_len; - tree_page = _tree_page; + tree_folio = folio; /* skip put_page for found dup */ if (!prune_stale_stable_nodes) @@ -1682,7 +1682,7 @@ static struct page *stable_node_dup(struct ksm_stable_node **_stable_node_dup, continue; } } - put_page(_tree_page); + folio_put(folio); } if (found) { @@ -1747,7 +1747,7 @@ static struct page *stable_node_dup(struct ksm_stable_node **_stable_node_dup, } *_stable_node_dup = found; - return tree_page; + return &tree_folio->page; } static struct ksm_stable_node *stable_node_dup_any(struct ksm_stable_node *stable_node, From 72556a4c06646b7e314ee0920796699c9d4a8b47 Mon Sep 17 00:00:00 2001 From: "Alex Shi (tencent)" Date: Thu, 11 Apr 2024 14:17:07 +0800 Subject: [PATCH 1145/1702] mm/ksm: use ksm_get_folio in scan_get_next_rmap_item Save a compound_head call. Link: https://lkml.kernel.org/r/20240411061713.1847574-7-alexs@kernel.org Signed-off-by: Alex Shi (tencent) Reviewed-by: David Hildenbrand Cc: Izik Eidus Cc: Matthew Wilcox Cc: Andrea Arcangeli Cc: Hugh Dickins Cc: Chris Wright Signed-off-by: Andrew Morton --- mm/ksm.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mm/ksm.c b/mm/ksm.c index 654400f993fcc..b127d39c9af00 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -2611,14 +2611,14 @@ static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page) */ if (!ksm_merge_across_nodes) { struct ksm_stable_node *stable_node, *next; - struct page *page; + struct folio *folio; list_for_each_entry_safe(stable_node, next, &migrate_nodes, list) { - page = get_ksm_page(stable_node, - GET_KSM_PAGE_NOLOCK); - if (page) - put_page(page); + folio = ksm_get_folio(stable_node, + GET_KSM_PAGE_NOLOCK); + if (folio) + folio_put(folio); cond_resched(); } } From 40d707f33db5e6d7da37b701955a3f662a741b02 Mon Sep 17 00:00:00 2001 From: "Alex Shi (tencent)" Date: Thu, 11 Apr 2024 14:17:08 +0800 Subject: [PATCH 1146/1702] mm/ksm: use folio in write_protect_page Compound page is checked and skipped before write_protect_page() called, use folio to save a few compound_head checks. Link: https://lkml.kernel.org/r/20240411061713.1847574-8-alexs@kernel.org Signed-off-by: Alex Shi (tencent) Reviewed-by: David Hildenbrand Cc: Izik Eidus Cc: Matthew Wilcox Cc: Andrea Arcangeli Cc: Hugh Dickins Cc: Chris Wright Signed-off-by: Andrew Morton --- mm/ksm.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/mm/ksm.c b/mm/ksm.c index b127d39c9af00..2fdd6586a3a76 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -1289,23 +1289,24 @@ static u32 calc_checksum(struct page *page) return checksum; } -static int write_protect_page(struct vm_area_struct *vma, struct page *page, +static int write_protect_page(struct vm_area_struct *vma, struct folio *folio, pte_t *orig_pte) { struct mm_struct *mm = vma->vm_mm; - DEFINE_PAGE_VMA_WALK(pvmw, page, vma, 0, 0); + DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, 0, 0); int swapped; int err = -EFAULT; struct mmu_notifier_range range; bool anon_exclusive; pte_t entry; - pvmw.address = page_address_in_vma(page, vma); + if (WARN_ON_ONCE(folio_test_large(folio))) + return err; + + pvmw.address = page_address_in_vma(&folio->page, vma); if (pvmw.address == -EFAULT) goto out; - BUG_ON(PageTransCompound(page)); - mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, pvmw.address, pvmw.address + PAGE_SIZE); mmu_notifier_invalidate_range_start(&range); @@ -1315,12 +1316,12 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page, if (WARN_ONCE(!pvmw.pte, "Unexpected PMD mapping?")) goto out_unlock; - anon_exclusive = PageAnonExclusive(page); + anon_exclusive = PageAnonExclusive(&folio->page); entry = ptep_get(pvmw.pte); if (pte_write(entry) || pte_dirty(entry) || anon_exclusive || mm_tlb_flush_pending(mm)) { - swapped = PageSwapCache(page); - flush_cache_page(vma, pvmw.address, page_to_pfn(page)); + swapped = folio_test_swapcache(folio); + flush_cache_page(vma, pvmw.address, folio_pfn(folio)); /* * Ok this is tricky, when get_user_pages_fast() run it doesn't * take any lock, therefore the check that we are going to make @@ -1340,20 +1341,20 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page, * Check that no O_DIRECT or similar I/O is in progress on the * page */ - if (page_mapcount(page) + 1 + swapped != page_count(page)) { + if (folio_mapcount(folio) + 1 + swapped != folio_ref_count(folio)) { set_pte_at(mm, pvmw.address, pvmw.pte, entry); goto out_unlock; } /* See folio_try_share_anon_rmap_pte(): clear PTE first. */ if (anon_exclusive && - folio_try_share_anon_rmap_pte(page_folio(page), page)) { + folio_try_share_anon_rmap_pte(folio, &folio->page)) { set_pte_at(mm, pvmw.address, pvmw.pte, entry); goto out_unlock; } if (pte_dirty(entry)) - set_page_dirty(page); + folio_mark_dirty(folio); entry = pte_mkclean(entry); if (pte_write(entry)) @@ -1519,7 +1520,7 @@ static int try_to_merge_one_page(struct vm_area_struct *vma, * ptes are necessarily already write-protected. But in either * case, we need to lock and check page_count is not raised. */ - if (write_protect_page(vma, page, &orig_pte) == 0) { + if (write_protect_page(vma, page_folio(page), &orig_pte) == 0) { if (!kpage) { /* * While we hold page lock, upgrade page from From 79899cce33e0887c06d41e767aa543aaaaef48e2 Mon Sep 17 00:00:00 2001 From: "Alex Shi (tencent)" Date: Thu, 11 Apr 2024 14:17:09 +0800 Subject: [PATCH 1147/1702] mm/ksm: convert chain series funcs and replace get_ksm_page In ksm stable tree all page are single, let's convert them to use and folios as well as stable_tree_insert/stable_tree_search funcs. And replace get_ksm_page() by ksm_get_folio() since there is no more needs. It could save a few compound_head calls. Link: https://lkml.kernel.org/r/20240411061713.1847574-9-alexs@kernel.org Signed-off-by: Alex Shi (tencent) Cc: Izik Eidus Cc: Matthew Wilcox Cc: Andrea Arcangeli Cc: Hugh Dickins Cc: Chris Wright Cc: David Hildenbrand Signed-off-by: Andrew Morton --- mm/ksm.c | 136 ++++++++++++++++++++++++--------------------------- mm/migrate.c | 2 +- 2 files changed, 66 insertions(+), 72 deletions(-) diff --git a/mm/ksm.c b/mm/ksm.c index 2fdd6586a3a76..61a7b5b037a6b 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -990,14 +990,6 @@ static struct folio *ksm_get_folio(struct ksm_stable_node *stable_node, return NULL; } -static struct page *get_ksm_page(struct ksm_stable_node *stable_node, - enum get_ksm_page_flags flags) -{ - struct folio *folio = ksm_get_folio(stable_node, flags); - - return &folio->page; -} - /* * Removing rmap_item from stable or unstable tree. * This function will clean the information from the stable/unstable tree. @@ -1632,10 +1624,10 @@ bool is_page_sharing_candidate(struct ksm_stable_node *stable_node) return __is_page_sharing_candidate(stable_node, 0); } -static struct page *stable_node_dup(struct ksm_stable_node **_stable_node_dup, - struct ksm_stable_node **_stable_node, - struct rb_root *root, - bool prune_stale_stable_nodes) +static struct folio *stable_node_dup(struct ksm_stable_node **_stable_node_dup, + struct ksm_stable_node **_stable_node, + struct rb_root *root, + bool prune_stale_stable_nodes) { struct ksm_stable_node *dup, *found = NULL, *stable_node = *_stable_node; struct hlist_node *hlist_safe; @@ -1748,7 +1740,7 @@ static struct page *stable_node_dup(struct ksm_stable_node **_stable_node_dup, } *_stable_node_dup = found; - return &tree_folio->page; + return tree_folio; } static struct ksm_stable_node *stable_node_dup_any(struct ksm_stable_node *stable_node, @@ -1765,7 +1757,7 @@ static struct ksm_stable_node *stable_node_dup_any(struct ksm_stable_node *stabl } /* - * Like for get_ksm_page, this function can free the *_stable_node and + * Like for ksm_get_folio, this function can free the *_stable_node and * *_stable_node_dup if the returned tree_page is NULL. * * It can also free and overwrite *_stable_node with the found @@ -1778,16 +1770,16 @@ static struct ksm_stable_node *stable_node_dup_any(struct ksm_stable_node *stabl * function and will be overwritten in all cases, the caller doesn't * need to initialize it. */ -static struct page *__stable_node_chain(struct ksm_stable_node **_stable_node_dup, - struct ksm_stable_node **_stable_node, - struct rb_root *root, - bool prune_stale_stable_nodes) +static struct folio *__stable_node_chain(struct ksm_stable_node **_stable_node_dup, + struct ksm_stable_node **_stable_node, + struct rb_root *root, + bool prune_stale_stable_nodes) { struct ksm_stable_node *stable_node = *_stable_node; if (!is_stable_node_chain(stable_node)) { if (is_page_sharing_candidate(stable_node)) { *_stable_node_dup = stable_node; - return get_ksm_page(stable_node, GET_KSM_PAGE_NOLOCK); + return ksm_get_folio(stable_node, GET_KSM_PAGE_NOLOCK); } /* * _stable_node_dup set to NULL means the stable_node @@ -1800,24 +1792,24 @@ static struct page *__stable_node_chain(struct ksm_stable_node **_stable_node_du prune_stale_stable_nodes); } -static __always_inline struct page *chain_prune(struct ksm_stable_node **s_n_d, - struct ksm_stable_node **s_n, - struct rb_root *root) +static __always_inline struct folio *chain_prune(struct ksm_stable_node **s_n_d, + struct ksm_stable_node **s_n, + struct rb_root *root) { return __stable_node_chain(s_n_d, s_n, root, true); } -static __always_inline struct page *chain(struct ksm_stable_node **s_n_d, - struct ksm_stable_node *s_n, - struct rb_root *root) +static __always_inline struct folio *chain(struct ksm_stable_node **s_n_d, + struct ksm_stable_node *s_n, + struct rb_root *root) { struct ksm_stable_node *old_stable_node = s_n; - struct page *tree_page; + struct folio *tree_folio; - tree_page = __stable_node_chain(s_n_d, &s_n, root, false); + tree_folio = __stable_node_chain(s_n_d, &s_n, root, false); /* not pruning dups so s_n cannot have changed */ VM_BUG_ON(s_n != old_stable_node); - return tree_page; + return tree_folio; } /* @@ -1837,28 +1829,30 @@ static struct page *stable_tree_search(struct page *page) struct rb_node *parent; struct ksm_stable_node *stable_node, *stable_node_dup, *stable_node_any; struct ksm_stable_node *page_node; + struct folio *folio; - page_node = page_stable_node(page); + folio = page_folio(page); + page_node = folio_stable_node(folio); if (page_node && page_node->head != &migrate_nodes) { /* ksm page forked */ - get_page(page); - return page; + folio_get(folio); + return &folio->page; } - nid = get_kpfn_nid(page_to_pfn(page)); + nid = get_kpfn_nid(folio_pfn(folio)); root = root_stable_tree + nid; again: new = &root->rb_node; parent = NULL; while (*new) { - struct page *tree_page; + struct folio *tree_folio; int ret; cond_resched(); stable_node = rb_entry(*new, struct ksm_stable_node, node); stable_node_any = NULL; - tree_page = chain_prune(&stable_node_dup, &stable_node, root); + tree_folio = chain_prune(&stable_node_dup, &stable_node, root); /* * NOTE: stable_node may have been freed by * chain_prune() if the returned stable_node_dup is @@ -1892,14 +1886,14 @@ static struct page *stable_tree_search(struct page *page) * write protected at all times. Any will work * fine to continue the walk. */ - tree_page = get_ksm_page(stable_node_any, - GET_KSM_PAGE_NOLOCK); + tree_folio = ksm_get_folio(stable_node_any, + GET_KSM_PAGE_NOLOCK); } VM_BUG_ON(!stable_node_dup ^ !!stable_node_any); - if (!tree_page) { + if (!tree_folio) { /* * If we walked over a stale stable_node, - * get_ksm_page() will call rb_erase() and it + * ksm_get_folio() will call rb_erase() and it * may rebalance the tree from under us. So * restart the search from scratch. Returning * NULL would be safe too, but we'd generate @@ -1909,8 +1903,8 @@ static struct page *stable_tree_search(struct page *page) goto again; } - ret = memcmp_pages(page, tree_page); - put_page(tree_page); + ret = memcmp_pages(page, &tree_folio->page); + folio_put(tree_folio); parent = *new; if (ret < 0) @@ -1953,26 +1947,26 @@ static struct page *stable_tree_search(struct page *page) * It would be more elegant to return stable_node * than kpage, but that involves more changes. */ - tree_page = get_ksm_page(stable_node_dup, - GET_KSM_PAGE_TRYLOCK); + tree_folio = ksm_get_folio(stable_node_dup, + GET_KSM_PAGE_TRYLOCK); - if (PTR_ERR(tree_page) == -EBUSY) + if (PTR_ERR(tree_folio) == -EBUSY) return ERR_PTR(-EBUSY); - if (unlikely(!tree_page)) + if (unlikely(!tree_folio)) /* * The tree may have been rebalanced, * so re-evaluate parent and new. */ goto again; - unlock_page(tree_page); + folio_unlock(tree_folio); if (get_kpfn_nid(stable_node_dup->kpfn) != NUMA(stable_node_dup->nid)) { - put_page(tree_page); + folio_put(tree_folio); goto replace; } - return tree_page; + return &tree_folio->page; } } @@ -1985,8 +1979,8 @@ static struct page *stable_tree_search(struct page *page) rb_insert_color(&page_node->node, root); out: if (is_page_sharing_candidate(page_node)) { - get_page(page); - return page; + folio_get(folio); + return &folio->page; } else return NULL; @@ -2011,12 +2005,12 @@ static struct page *stable_tree_search(struct page *page) &page_node->node, root); if (is_page_sharing_candidate(page_node)) - get_page(page); + folio_get(folio); else - page = NULL; + folio = NULL; } else { rb_erase(&stable_node_dup->node, root); - page = NULL; + folio = NULL; } } else { VM_BUG_ON(!is_stable_node_chain(stable_node)); @@ -2027,16 +2021,16 @@ static struct page *stable_tree_search(struct page *page) DO_NUMA(page_node->nid = nid); stable_node_chain_add_dup(page_node, stable_node); if (is_page_sharing_candidate(page_node)) - get_page(page); + folio_get(folio); else - page = NULL; + folio = NULL; } else { - page = NULL; + folio = NULL; } } stable_node_dup->head = &migrate_nodes; list_add(&stable_node_dup->list, stable_node_dup->head); - return page; + return &folio->page; chain_append: /* stable_node_dup could be null if it reached the limit */ @@ -2079,7 +2073,7 @@ static struct page *stable_tree_search(struct page *page) * This function returns the stable tree node just allocated on success, * NULL otherwise. */ -static struct ksm_stable_node *stable_tree_insert(struct page *kpage) +static struct ksm_stable_node *stable_tree_insert(struct folio *kfolio) { int nid; unsigned long kpfn; @@ -2089,7 +2083,7 @@ static struct ksm_stable_node *stable_tree_insert(struct page *kpage) struct ksm_stable_node *stable_node, *stable_node_dup, *stable_node_any; bool need_chain = false; - kpfn = page_to_pfn(kpage); + kpfn = folio_pfn(kfolio); nid = get_kpfn_nid(kpfn); root = root_stable_tree + nid; again: @@ -2097,13 +2091,13 @@ static struct ksm_stable_node *stable_tree_insert(struct page *kpage) new = &root->rb_node; while (*new) { - struct page *tree_page; + struct folio *tree_folio; int ret; cond_resched(); stable_node = rb_entry(*new, struct ksm_stable_node, node); stable_node_any = NULL; - tree_page = chain(&stable_node_dup, stable_node, root); + tree_folio = chain(&stable_node_dup, stable_node, root); if (!stable_node_dup) { /* * Either all stable_node dups were full in @@ -2125,14 +2119,14 @@ static struct ksm_stable_node *stable_tree_insert(struct page *kpage) * write protected at all times. Any will work * fine to continue the walk. */ - tree_page = get_ksm_page(stable_node_any, - GET_KSM_PAGE_NOLOCK); + tree_folio = ksm_get_folio(stable_node_any, + GET_KSM_PAGE_NOLOCK); } VM_BUG_ON(!stable_node_dup ^ !!stable_node_any); - if (!tree_page) { + if (!tree_folio) { /* * If we walked over a stale stable_node, - * get_ksm_page() will call rb_erase() and it + * ksm_get_folio() will call rb_erase() and it * may rebalance the tree from under us. So * restart the search from scratch. Returning * NULL would be safe too, but we'd generate @@ -2142,8 +2136,8 @@ static struct ksm_stable_node *stable_tree_insert(struct page *kpage) goto again; } - ret = memcmp_pages(kpage, tree_page); - put_page(tree_page); + ret = memcmp_pages(&kfolio->page, &tree_folio->page); + folio_put(tree_folio); parent = *new; if (ret < 0) @@ -2162,7 +2156,7 @@ static struct ksm_stable_node *stable_tree_insert(struct page *kpage) INIT_HLIST_HEAD(&stable_node_dup->hlist); stable_node_dup->kpfn = kpfn; - set_page_stable_node(kpage, stable_node_dup); + folio_set_stable_node(kfolio, stable_node_dup); stable_node_dup->rmap_hlist_len = 0; DO_NUMA(stable_node_dup->nid = nid); if (!need_chain) { @@ -2440,7 +2434,7 @@ static void cmp_and_merge_page(struct page *page, struct ksm_rmap_item *rmap_ite * node in the stable tree and add both rmap_items. */ lock_page(kpage); - stable_node = stable_tree_insert(kpage); + stable_node = stable_tree_insert(page_folio(kpage)); if (stable_node) { stable_tree_append(tree_rmap_item, stable_node, false); @@ -3244,7 +3238,7 @@ void folio_migrate_ksm(struct folio *newfolio, struct folio *folio) /* * newfolio->mapping was set in advance; now we need smp_wmb() * to make sure that the new stable_node->kpfn is visible - * to get_ksm_page() before it can see that folio->mapping + * to ksm_get_folio() before it can see that folio->mapping * has gone stale (or that folio_test_swapcache has been cleared). */ smp_wmb(); @@ -3271,7 +3265,7 @@ static bool stable_node_dup_remove_range(struct ksm_stable_node *stable_node, if (stable_node->kpfn >= start_pfn && stable_node->kpfn < end_pfn) { /* - * Don't get_ksm_page, page has already gone: + * Don't ksm_get_folio, page has already gone: * which is why we keep kpfn instead of page* */ remove_node_from_stable_tree(stable_node); @@ -3359,7 +3353,7 @@ static int ksm_memory_callback(struct notifier_block *self, * Most of the work is done by page migration; but there might * be a few stable_nodes left over, still pointing to struct * pages which have been offlined: prune those from the tree, - * otherwise get_ksm_page() might later try to access a + * otherwise ksm_get_folio() might later try to access a * non-existent struct page. */ ksm_check_stable_tree(mn->start_pfn, diff --git a/mm/migrate.c b/mm/migrate.c index d87ce32645d4f..c7692f303fa73 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -616,7 +616,7 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio) folio_migrate_ksm(newfolio, folio); /* * Please do not reorder this without considering how mm/ksm.c's - * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache(). + * ksm_get_folio() depends upon ksm_migrate_page() and PageSwapCache(). */ if (folio_test_swapcache(folio)) folio_clear_swapcache(folio); From 85b67b01044e7cd52031d9b84745c816f831e68e Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 11 Apr 2024 14:17:10 +0800 Subject: [PATCH 1148/1702] mm/ksm: rename get_ksm_page_flags to ksm_get_folio_flags As we are removing get_ksm_page_flags(), make the flags match the new function name. Link: https://lkml.kernel.org/r/20240411061713.1847574-10-alexs@kernel.org Signed-off-by: David Hildenbrand Signed-off-by: Alex Shi Reviewed-by: Alex Shi Cc: Andrea Arcangeli Cc: Chris Wright Cc: Hugh Dickins Cc: Izik Eidus Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/ksm.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/mm/ksm.c b/mm/ksm.c index 61a7b5b037a6b..662fdaaf3ea33 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -890,10 +890,10 @@ static void remove_node_from_stable_tree(struct ksm_stable_node *stable_node) free_stable_node(stable_node); } -enum get_ksm_page_flags { - GET_KSM_PAGE_NOLOCK, - GET_KSM_PAGE_LOCK, - GET_KSM_PAGE_TRYLOCK +enum ksm_get_folio_flags { + KSM_GET_FOLIO_NOLOCK, + KSM_GET_FOLIO_LOCK, + KSM_GET_FOLIO_TRYLOCK }; /* @@ -916,7 +916,7 @@ enum get_ksm_page_flags { * is on its way to being freed; but it is an anomaly to bear in mind. */ static struct folio *ksm_get_folio(struct ksm_stable_node *stable_node, - enum get_ksm_page_flags flags) + enum ksm_get_folio_flags flags) { struct folio *folio; void *expected_mapping; @@ -959,15 +959,15 @@ static struct folio *ksm_get_folio(struct ksm_stable_node *stable_node, goto stale; } - if (flags == GET_KSM_PAGE_TRYLOCK) { + if (flags == KSM_GET_FOLIO_TRYLOCK) { if (!folio_trylock(folio)) { folio_put(folio); return ERR_PTR(-EBUSY); } - } else if (flags == GET_KSM_PAGE_LOCK) + } else if (flags == KSM_GET_FOLIO_LOCK) folio_lock(folio); - if (flags != GET_KSM_PAGE_NOLOCK) { + if (flags != KSM_GET_FOLIO_NOLOCK) { if (READ_ONCE(folio->mapping) != expected_mapping) { folio_unlock(folio); folio_put(folio); @@ -1001,7 +1001,7 @@ static void remove_rmap_item_from_tree(struct ksm_rmap_item *rmap_item) struct folio *folio; stable_node = rmap_item->head; - folio = ksm_get_folio(stable_node, GET_KSM_PAGE_LOCK); + folio = ksm_get_folio(stable_node, KSM_GET_FOLIO_LOCK); if (!folio) goto out; @@ -1116,7 +1116,7 @@ static int remove_stable_node(struct ksm_stable_node *stable_node) struct folio *folio; int err; - folio = ksm_get_folio(stable_node, GET_KSM_PAGE_LOCK); + folio = ksm_get_folio(stable_node, KSM_GET_FOLIO_LOCK); if (!folio) { /* * ksm_get_folio did remove_node_from_stable_tree itself. @@ -1656,7 +1656,7 @@ static struct folio *stable_node_dup(struct ksm_stable_node **_stable_node_dup, * stable_node parameter itself will be freed from * under us if it returns NULL. */ - folio = ksm_get_folio(dup, GET_KSM_PAGE_NOLOCK); + folio = ksm_get_folio(dup, KSM_GET_FOLIO_NOLOCK); if (!folio) continue; nr += 1; @@ -1779,7 +1779,7 @@ static struct folio *__stable_node_chain(struct ksm_stable_node **_stable_node_d if (!is_stable_node_chain(stable_node)) { if (is_page_sharing_candidate(stable_node)) { *_stable_node_dup = stable_node; - return ksm_get_folio(stable_node, GET_KSM_PAGE_NOLOCK); + return ksm_get_folio(stable_node, KSM_GET_FOLIO_NOLOCK); } /* * _stable_node_dup set to NULL means the stable_node @@ -1887,7 +1887,7 @@ static struct page *stable_tree_search(struct page *page) * fine to continue the walk. */ tree_folio = ksm_get_folio(stable_node_any, - GET_KSM_PAGE_NOLOCK); + KSM_GET_FOLIO_NOLOCK); } VM_BUG_ON(!stable_node_dup ^ !!stable_node_any); if (!tree_folio) { @@ -1948,7 +1948,7 @@ static struct page *stable_tree_search(struct page *page) * than kpage, but that involves more changes. */ tree_folio = ksm_get_folio(stable_node_dup, - GET_KSM_PAGE_TRYLOCK); + KSM_GET_FOLIO_TRYLOCK); if (PTR_ERR(tree_folio) == -EBUSY) return ERR_PTR(-EBUSY); @@ -2120,7 +2120,7 @@ static struct ksm_stable_node *stable_tree_insert(struct folio *kfolio) * fine to continue the walk. */ tree_folio = ksm_get_folio(stable_node_any, - GET_KSM_PAGE_NOLOCK); + KSM_GET_FOLIO_NOLOCK); } VM_BUG_ON(!stable_node_dup ^ !!stable_node_any); if (!tree_folio) { @@ -2611,7 +2611,7 @@ static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page) list_for_each_entry_safe(stable_node, next, &migrate_nodes, list) { folio = ksm_get_folio(stable_node, - GET_KSM_PAGE_NOLOCK); + KSM_GET_FOLIO_NOLOCK); if (folio) folio_put(folio); cond_resched(); From 452e862f4315d8e5e839fe1dc220fd2716be6d3a Mon Sep 17 00:00:00 2001 From: "Alex Shi (tencent)" Date: Thu, 11 Apr 2024 14:17:11 +0800 Subject: [PATCH 1149/1702] mm/ksm: replace set_page_stable_node by folio_set_stable_node Only single page could be reached where we set stable node after write protect, so use folio converted func to replace page's. And remove the unused func set_page_stable_node(). Link: https://lkml.kernel.org/r/20240411061713.1847574-11-alexs@kernel.org Signed-off-by: Alex Shi (tencent) Reviewed-by: David Hildenbrand Cc: Izik Eidus Cc: Matthew Wilcox Cc: Andrea Arcangeli Cc: Hugh Dickins Cc: Chris Wright Signed-off-by: Andrew Morton --- mm/ksm.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/mm/ksm.c b/mm/ksm.c index 662fdaaf3ea33..486c9974f8e20 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -1094,17 +1094,11 @@ static inline struct ksm_stable_node *page_stable_node(struct page *page) return folio_stable_node(page_folio(page)); } -static inline void set_page_stable_node(struct page *page, - struct ksm_stable_node *stable_node) -{ - VM_BUG_ON_PAGE(PageAnon(page) && PageAnonExclusive(page), page); - page->mapping = (void *)((unsigned long)stable_node | PAGE_MAPPING_KSM); -} - static inline void folio_set_stable_node(struct folio *folio, struct ksm_stable_node *stable_node) { - set_page_stable_node(&folio->page, stable_node); + VM_WARN_ON_FOLIO(folio_test_anon(folio) && PageAnonExclusive(&folio->page), folio); + folio->mapping = (void *)((unsigned long)stable_node | PAGE_MAPPING_KSM); } #ifdef CONFIG_SYSFS @@ -1519,7 +1513,7 @@ static int try_to_merge_one_page(struct vm_area_struct *vma, * PageAnon+anon_vma to PageKsm+NULL stable_node: * stable_tree_insert() will update stable_node. */ - set_page_stable_node(page, NULL); + folio_set_stable_node(page_folio(page), NULL); mark_page_accessed(page); /* * Page reclaim just frees a clean page with no dirty From 54fa49b2e0ef3af944bbeacb7ed2ba0b4f02facc Mon Sep 17 00:00:00 2001 From: Sidhartha Kumar Date: Thu, 11 Apr 2024 09:47:56 -0700 Subject: [PATCH 1150/1702] mm/hugetlb: convert dissolve_free_huge_pages() to folios Allows us to rename dissolve_free_huge_pages() to dissolve_free_hugetlb_folio(). Convert one caller to pass in a folio directly and use page_folio() to convert the caller in mm/memory-failure. [sidhartha.kumar@oracle.com: remove unneeded `extern'] Link: https://lkml.kernel.org/r/71760ed4-e80d-493a-95ea-2545414b1aba@oracle.com [sidhartha.kumar@oracle.com: v2] Link: https://lkml.kernel.org/r/20240412182139.120871-1-sidhartha.kumar@oracle.com Link: https://lkml.kernel.org/r/20240411164756.261178-1-sidhartha.kumar@oracle.com Signed-off-by: Sidhartha Kumar Reviewed-by: Oscar Salvador Reviewed-by: Vishal Moola (Oracle) Reviewed-by: Miaohe Lin Cc: Jane Chu Cc: Matthew Wilcox (Oracle) Cc: Muchun Song Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 4 ++-- mm/hugetlb.c | 17 ++++++++--------- mm/memory-failure.c | 8 ++++---- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 3f3e628802792..8968e8a3a205d 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -861,7 +861,7 @@ static inline int hstate_index(struct hstate *h) return h - hstates; } -extern int dissolve_free_huge_page(struct page *page); +int dissolve_free_hugetlb_folio(struct folio *folio); extern int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn); @@ -1148,7 +1148,7 @@ static inline int hstate_index(struct hstate *h) return 0; } -static inline int dissolve_free_huge_page(struct page *page) +static inline int dissolve_free_hugetlb_folio(struct folio *folio) { return 0; } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index d74289d3f30db..40bcaf2bad559 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2377,8 +2377,8 @@ static struct folio *remove_pool_hugetlb_folio(struct hstate *h, } /* - * Dissolve a given free hugepage into free buddy pages. This function does - * nothing for in-use hugepages and non-hugepages. + * Dissolve a given free hugetlb folio into free buddy pages. This function + * does nothing for in-use hugetlb folios and non-hugetlb folios. * This function returns values like below: * * -ENOMEM: failed to allocate vmemmap pages to free the freed hugepages @@ -2390,10 +2390,9 @@ static struct folio *remove_pool_hugetlb_folio(struct hstate *h, * 0: successfully dissolved free hugepages or the page is not a * hugepage (considered as already dissolved) */ -int dissolve_free_huge_page(struct page *page) +int dissolve_free_hugetlb_folio(struct folio *folio) { int rc = -EBUSY; - struct folio *folio = page_folio(page); retry: /* Not to disrupt normal path by vainly holding hugetlb_lock */ @@ -2470,13 +2469,13 @@ int dissolve_free_huge_page(struct page *page) * make specified memory blocks removable from the system. * Note that this will dissolve a free gigantic hugepage completely, if any * part of it lies within the given range. - * Also note that if dissolve_free_huge_page() returns with an error, all - * free hugepages that were dissolved before that error are lost. + * Also note that if dissolve_free_hugetlb_folio() returns with an error, all + * free hugetlb folios that were dissolved before that error are lost. */ int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn) { unsigned long pfn; - struct page *page; + struct folio *folio; int rc = 0; unsigned int order; struct hstate *h; @@ -2489,8 +2488,8 @@ int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn) order = min(order, huge_page_order(h)); for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order) { - page = pfn_to_page(pfn); - rc = dissolve_free_huge_page(page); + folio = pfn_folio(pfn); + rc = dissolve_free_hugetlb_folio(folio); if (rc) break; } diff --git a/mm/memory-failure.c b/mm/memory-failure.c index db8d770ce8012..68e1fe1c0b724 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -155,7 +155,7 @@ static int __page_handle_poison(struct page *page) /* * zone_pcp_disable() can't be used here. It will - * hold pcp_batch_high_lock and dissolve_free_huge_page() might hold + * hold pcp_batch_high_lock and dissolve_free_hugetlb_folio() might hold * cpu_hotplug_lock via static_key_slow_dec() when hugetlb vmemmap * optimization is enabled. This will break current lock dependency * chain and leads to deadlock. @@ -165,7 +165,7 @@ static int __page_handle_poison(struct page *page) * but nothing guarantees that those pages do not get back to a PCP * queue if we need to refill those. */ - ret = dissolve_free_huge_page(page); + ret = dissolve_free_hugetlb_folio(page_folio(page)); if (!ret) { drain_all_pages(page_zone(page)); ret = take_page_off_buddy(page); @@ -178,8 +178,8 @@ static bool page_handle_poison(struct page *page, bool hugepage_or_freepage, boo { if (hugepage_or_freepage) { /* - * Doing this check for free pages is also fine since dissolve_free_huge_page - * returns 0 for non-hugetlb pages as well. + * Doing this check for free pages is also fine since + * dissolve_free_hugetlb_folio() returns 0 for non-hugetlb folios as well. */ if (__page_handle_poison(page) <= 0) /* From d199483c2b972064be5fce846228ceb773cd9d17 Mon Sep 17 00:00:00 2001 From: Sidhartha Kumar Date: Fri, 12 Apr 2024 11:21:39 -0700 Subject: [PATCH 1151/1702] mm/hugetlb: rename dissolve_free_huge_pages() to dissolve_free_hugetlb_folios() dissolve_free_huge_pages() only uses folios internally, rename it to dissolve_free_hugetlb_folios() and change the comments which reference it. [akpm@linux-foundation.org: remove unneeded `extern'] Link: https://lkml.kernel.org/r/20240412182139.120871-2-sidhartha.kumar@oracle.com Signed-off-by: Sidhartha Kumar Reviewed-by: Vishal Moola (Oracle) Reviewed-by: Miaohe Lin Cc: Jane Chu Cc: Matthew Wilcox (Oracle) Cc: Muchun Song Cc: Oscar Salvador Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 4 ++-- mm/hugetlb.c | 2 +- mm/memory_hotplug.c | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 8968e8a3a205d..1bc93e7e315bb 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -862,7 +862,7 @@ static inline int hstate_index(struct hstate *h) } int dissolve_free_hugetlb_folio(struct folio *folio); -extern int dissolve_free_huge_pages(unsigned long start_pfn, +int dissolve_free_hugetlb_folios(unsigned long start_pfn, unsigned long end_pfn); #ifdef CONFIG_MEMORY_FAILURE @@ -1153,7 +1153,7 @@ static inline int dissolve_free_hugetlb_folio(struct folio *folio) return 0; } -static inline int dissolve_free_huge_pages(unsigned long start_pfn, +static inline int dissolve_free_hugetlb_folios(unsigned long start_pfn, unsigned long end_pfn) { return 0; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 40bcaf2bad559..3b7d5ddc32ad7 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2472,7 +2472,7 @@ int dissolve_free_hugetlb_folio(struct folio *folio) * Also note that if dissolve_free_hugetlb_folio() returns with an error, all * free hugetlb folios that were dissolved before that error are lost. */ -int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn) +int dissolve_free_hugetlb_folios(unsigned long start_pfn, unsigned long end_pfn) { unsigned long pfn; struct folio *folio; diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index b79ba36e09e03..431b1f6753c0b 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -2051,11 +2051,11 @@ int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages, } /* - * Dissolve free hugepages in the memory block before doing + * Dissolve free hugetlb folios in the memory block before doing * offlining actually in order to make hugetlbfs's object * counting consistent. */ - ret = dissolve_free_huge_pages(start_pfn, end_pfn); + ret = dissolve_free_hugetlb_folios(start_pfn, end_pfn); if (ret) { reason = "failure to dissolve huge pages"; goto failed_removal_isolated; From ec33687c674934dfefd782a8ffd58370b080b503 Mon Sep 17 00:00:00 2001 From: Barry Song Date: Fri, 12 Apr 2024 23:48:55 +1200 Subject: [PATCH 1152/1702] mm: add per-order mTHP anon_fault_alloc and anon_fault_fallback counters Patch series "mm: add per-order mTHP alloc and swpout counters", v6. The patchset introduces a framework to facilitate mTHP counters, starting with the allocation and swap-out counters. Currently, only four new nodes are appended to the stats directory for each mTHP size. /sys/kernel/mm/transparent_hugepage/hugepages-/stats anon_fault_alloc anon_fault_fallback anon_fault_fallback_charge anon_swpout anon_swpout_fallback These nodes are crucial for us to monitor the fragmentation levels of both the buddy system and the swap partitions. In the future, we may consider adding additional nodes for further insights. This patch (of 4): Profiling a system blindly with mTHP has become challenging due to the lack of visibility into its operations. Presenting the success rate of mTHP allocations appears to be pressing need. Recently, I've been experiencing significant difficulty debugging performance improvements and regressions without these figures. It's crucial for us to understand the true effectiveness of mTHP in real-world scenarios, especially in systems with fragmented memory. This patch establishes the framework for per-order mTHP counters. It begins by introducing the anon_fault_alloc and anon_fault_fallback counters. Additionally, to maintain consistency with thp_fault_fallback_charge in /proc/vmstat, this patch also tracks anon_fault_fallback_charge when mem_cgroup_charge fails for mTHP. Incorporating additional counters should now be straightforward as well. Link: https://lkml.kernel.org/r/20240412114858.407208-1-21cnbao@gmail.com Link: https://lkml.kernel.org/r/20240412114858.407208-2-21cnbao@gmail.com Signed-off-by: Barry Song Acked-by: David Hildenbrand Cc: Chris Li Cc: Domenico Cerasuolo Cc: Kairui Song Cc: Matthew Wilcox (Oracle) Cc: Peter Xu Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Yosry Ahmed Cc: Yu Zhao Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 21 +++++++++++++++++ mm/huge_memory.c | 52 +++++++++++++++++++++++++++++++++++++++++ mm/memory.c | 5 ++++ 3 files changed, 78 insertions(+) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index e896ca4760f67..d4fdb26410702 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -264,6 +264,27 @@ unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, enforce_sysfs, orders); } +enum mthp_stat_item { + MTHP_STAT_ANON_FAULT_ALLOC, + MTHP_STAT_ANON_FAULT_FALLBACK, + MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE, + __MTHP_STAT_COUNT +}; + +struct mthp_stat { + unsigned long stats[ilog2(MAX_PTRS_PER_PTE) + 1][__MTHP_STAT_COUNT]; +}; + +DECLARE_PER_CPU(struct mthp_stat, mthp_stats); + +static inline void count_mthp_stat(int order, enum mthp_stat_item item) +{ + if (order <= 0 || order > PMD_ORDER) + return; + + this_cpu_inc(mthp_stats.stats[order][item]); +} + #define transparent_hugepage_use_zero_page() \ (transparent_hugepage_flags & \ (1<stats[order][item]; + } + + return sum; +} + +#define DEFINE_MTHP_STAT_ATTR(_name, _index) \ +static ssize_t _name##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, char *buf) \ +{ \ + int order = to_thpsize(kobj)->order; \ + \ + return sysfs_emit(buf, "%lu\n", sum_mthp_stat(order, _index)); \ +} \ +static struct kobj_attribute _name##_attr = __ATTR_RO(_name) + +DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC); +DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK); +DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); + +static struct attribute *stats_attrs[] = { + &anon_fault_alloc_attr.attr, + &anon_fault_fallback_attr.attr, + &anon_fault_fallback_charge_attr.attr, + NULL, +}; + +static struct attribute_group stats_attr_group = { + .name = "stats", + .attrs = stats_attrs, +}; + static struct thpsize *thpsize_create(int order, struct kobject *parent) { unsigned long size = (PAGE_SIZE << order) / SZ_1K; @@ -549,6 +591,12 @@ static struct thpsize *thpsize_create(int order, struct kobject *parent) return ERR_PTR(ret); } + ret = sysfs_create_group(&thpsize->kobj, &stats_attr_group); + if (ret) { + kobject_put(&thpsize->kobj); + return ERR_PTR(ret); + } + thpsize->order = order; return thpsize; } @@ -880,6 +928,8 @@ static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, folio_put(folio); count_vm_event(THP_FAULT_FALLBACK); count_vm_event(THP_FAULT_FALLBACK_CHARGE); + count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_FALLBACK); + count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); return VM_FAULT_FALLBACK; } folio_throttle_swaprate(folio, gfp); @@ -929,6 +979,7 @@ static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, mm_inc_nr_ptes(vma->vm_mm); spin_unlock(vmf->ptl); count_vm_event(THP_FAULT_ALLOC); + count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_ALLOC); count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); } @@ -1050,6 +1101,7 @@ vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) folio = vma_alloc_folio(gfp, HPAGE_PMD_ORDER, vma, haddr, true); if (unlikely(!folio)) { count_vm_event(THP_FAULT_FALLBACK); + count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_FALLBACK); return VM_FAULT_FALLBACK; } return __do_huge_pmd_anonymous_page(vmf, &folio->page, gfp); diff --git a/mm/memory.c b/mm/memory.c index d592128d7f7d5..b71d011105962 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4368,6 +4368,7 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf) folio = vma_alloc_folio(gfp, order, vma, addr, true); if (folio) { if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) { + count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); folio_put(folio); goto next; } @@ -4376,6 +4377,7 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf) return folio; } next: + count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); order = next_order(&orders, order); } @@ -4485,6 +4487,9 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) folio_ref_add(folio, nr_pages - 1); add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr_pages); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + count_mthp_stat(folio_order(folio), MTHP_STAT_ANON_FAULT_ALLOC); +#endif folio_add_new_anon_rmap(folio, vma, addr); folio_add_lru_vma(folio, vma); setpte: From d0f048ac39f6a71566d3f49a5922dfd7fa0d585b Mon Sep 17 00:00:00 2001 From: Barry Song Date: Fri, 12 Apr 2024 23:48:56 +1200 Subject: [PATCH 1153/1702] mm: add per-order mTHP anon_swpout and anon_swpout_fallback counters This helps to display the fragmentation situation of the swapfile, knowing the proportion of how much we haven't split large folios. So far, we only support non-split swapout for anon memory, with the possibility of expanding to shmem in the future. So, we add the "anon" prefix to the counter names. Link: https://lkml.kernel.org/r/20240412114858.407208-3-21cnbao@gmail.com Signed-off-by: Barry Song Reviewed-by: Ryan Roberts Acked-by: David Hildenbrand Cc: Chris Li Cc: Domenico Cerasuolo Cc: Kairui Song Cc: Matthew Wilcox (Oracle) Cc: Peter Xu Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Yosry Ahmed Cc: Yu Zhao Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 2 ++ mm/huge_memory.c | 4 ++++ mm/page_io.c | 1 + mm/vmscan.c | 3 +++ 4 files changed, 10 insertions(+) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index d4fdb26410702..7cd07b83a3d0a 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -268,6 +268,8 @@ enum mthp_stat_item { MTHP_STAT_ANON_FAULT_ALLOC, MTHP_STAT_ANON_FAULT_FALLBACK, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE, + MTHP_STAT_ANON_SWPOUT, + MTHP_STAT_ANON_SWPOUT_FALLBACK, __MTHP_STAT_COUNT }; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 01f01d6a50a1d..264e09043f096 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -555,11 +555,15 @@ static struct kobj_attribute _name##_attr = __ATTR_RO(_name) DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC); DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK); DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); +DEFINE_MTHP_STAT_ATTR(anon_swpout, MTHP_STAT_ANON_SWPOUT); +DEFINE_MTHP_STAT_ATTR(anon_swpout_fallback, MTHP_STAT_ANON_SWPOUT_FALLBACK); static struct attribute *stats_attrs[] = { &anon_fault_alloc_attr.attr, &anon_fault_fallback_attr.attr, &anon_fault_fallback_charge_attr.attr, + &anon_swpout_attr.attr, + &anon_swpout_fallback_attr.attr, NULL, }; diff --git a/mm/page_io.c b/mm/page_io.c index a9a7c236aeccf..46c603dddf043 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -217,6 +217,7 @@ static inline void count_swpout_vm_event(struct folio *folio) count_memcg_folio_events(folio, THP_SWPOUT, 1); count_vm_event(THP_SWPOUT); } + count_mthp_stat(folio_order(folio), MTHP_STAT_ANON_SWPOUT); #endif count_vm_events(PSWPOUT, folio_nr_pages(folio)); } diff --git a/mm/vmscan.c b/mm/vmscan.c index da93213af981e..d194e7240df44 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1214,6 +1214,8 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, goto activate_locked; } if (!add_to_swap(folio)) { + int __maybe_unused order = folio_order(folio); + if (!folio_test_large(folio)) goto activate_locked_split; /* Fallback to swap normal pages */ @@ -1225,6 +1227,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, THP_SWPOUT_FALLBACK, 1); count_vm_event(THP_SWPOUT_FALLBACK); } + count_mthp_stat(order, MTHP_STAT_ANON_SWPOUT_FALLBACK); #endif if (!add_to_swap(folio)) goto activate_locked_split; From 42248b9d34ea1be1b959d343fff465906cb787fc Mon Sep 17 00:00:00 2001 From: Barry Song Date: Fri, 12 Apr 2024 23:48:57 +1200 Subject: [PATCH 1154/1702] mm: add docs for per-order mTHP counters and transhuge_page ABI This patch includes documentation for mTHP counters and an ABI file for sys-kernel-mm-transparent-hugepage, which appears to have been missing for some time. [v-songbaohua@oppo.com: fix the name and unexpected indentation] Link: https://lkml.kernel.org/r/20240415054538.17071-1-21cnbao@gmail.com Link: https://lkml.kernel.org/r/20240412114858.407208-4-21cnbao@gmail.com Signed-off-by: Barry Song Reviewed-by: Ryan Roberts Reviewed-by: David Hildenbrand Cc: Chris Li Cc: Domenico Cerasuolo Cc: Kairui Song Cc: Matthew Wilcox (Oracle) Cc: Peter Xu Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Yosry Ahmed Cc: Yu Zhao Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- .../sysfs-kernel-mm-transparent-hugepage | 18 ++++++++++++ Documentation/admin-guide/mm/transhuge.rst | 28 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 Documentation/ABI/testing/sysfs-kernel-mm-transparent-hugepage diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-transparent-hugepage b/Documentation/ABI/testing/sysfs-kernel-mm-transparent-hugepage new file mode 100644 index 0000000000000..7bfbb9cc2c113 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-kernel-mm-transparent-hugepage @@ -0,0 +1,18 @@ +What: /sys/kernel/mm/transparent_hugepage/ +Date: April 2024 +Contact: Linux memory management mailing list +Description: + /sys/kernel/mm/transparent_hugepage/ contains a number of files and + subdirectories, + + - defrag + - enabled + - hpage_pmd_size + - khugepaged + - shmem_enabled + - use_zero_page + - subdirectories of the form hugepages-kB, where + is the page size of the hugepages supported by the kernel/CPU + combination. + + See Documentation/admin-guide/mm/transhuge.rst for details. diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index 04eb45a2f9406..e0fe17affeb3f 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -447,6 +447,34 @@ thp_swpout_fallback Usually because failed to allocate some continuous swap space for the huge page. +In /sys/kernel/mm/transparent_hugepage/hugepages-kB/stats, There are +also individual counters for each huge page size, which can be utilized to +monitor the system's effectiveness in providing huge pages for usage. Each +counter has its own corresponding file. + +anon_fault_alloc + is incremented every time a huge page is successfully + allocated and charged to handle a page fault. + +anon_fault_fallback + is incremented if a page fault fails to allocate or charge + a huge page and instead falls back to using huge pages with + lower orders or small pages. + +anon_fault_fallback_charge + is incremented if a page fault fails to charge a huge page and + instead falls back to using huge pages with lower orders or + small pages even though the allocation was successful. + +anon_swpout + is incremented every time a huge page is swapped out in one + piece without splitting. + +anon_swpout_fallback + is incremented if a huge page has to be split before swapout. + Usually because failed to allocate some continuous swap space + for the huge page. + As the system ages, allocating huge pages may be expensive as the system uses memory compaction to copy data around memory to free a huge page for use. There are some counters in ``/proc/vmstat`` to help From a14421ae2a99378c4103bb03606465ab13e75509 Mon Sep 17 00:00:00 2001 From: Barry Song Date: Fri, 12 Apr 2024 23:48:58 +1200 Subject: [PATCH 1155/1702] mm: correct the docs for thp_fault_alloc and thp_fault_fallback The documentation does not align with the code. In __do_huge_pmd_anonymous_page(), THP_FAULT_FALLBACK is incremented when mem_cgroup_charge() fails, despite the allocation succeeding, whereas THP_FAULT_ALLOC is only incremented after a successful charge. Link: https://lkml.kernel.org/r/20240412114858.407208-5-21cnbao@gmail.com Signed-off-by: Barry Song Reviewed-by: Ryan Roberts Reviewed-by: David Hildenbrand Cc: Chris Li Cc: Domenico Cerasuolo Cc: Kairui Song Cc: Matthew Wilcox (Oracle) Cc: Peter Xu Cc: Ryan Roberts Cc: Suren Baghdasaryan Cc: Yosry Ahmed Cc: Yu Zhao Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- Documentation/admin-guide/mm/transhuge.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index e0fe17affeb3f..f82300b9193fe 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -369,7 +369,7 @@ monitor how successfully the system is providing huge pages for use. thp_fault_alloc is incremented every time a huge page is successfully - allocated to handle a page fault. + allocated and charged to handle a page fault. thp_collapse_alloc is incremented by khugepaged when it has found @@ -377,7 +377,7 @@ thp_collapse_alloc successfully allocated a new huge page to store the data. thp_fault_fallback - is incremented if a page fault fails to allocate + is incremented if a page fault fails to allocate or charge a huge page and instead falls back to using small pages. thp_fault_fallback_charge From 1f2d8b4421bd0da2c97fb8bad5cc85fc929fef64 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Fri, 12 Apr 2024 14:47:50 +0800 Subject: [PATCH 1156/1702] mm: move mm counter updating out of set_pte_range() Patch series "mm: batch mm counter updating in filemap_map_pages()", v3. Let's batch mm counter updating to accelerate filemap_map_pages(). This patch (of 2): In order to support batch mm counter updating in filemap_map_pages(), move mm counter updating out of set_pte_range(), the folios are file from filemap, and distinguish folios by vmf->flags and vma->vm_flags from another caller finish_fault(). Link: https://lkml.kernel.org/r/20240412064751.119015-1-wangkefeng.wang@huawei.com Link: https://lkml.kernel.org/r/20240412064751.119015-2-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/filemap.c | 4 ++++ mm/memory.c | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 21e70434d9318..5a518c5075fb3 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -3539,6 +3539,8 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf, skip: if (count) { set_pte_range(vmf, folio, page, count, addr); + add_mm_counter(vmf->vma->vm_mm, mm_counter_file(folio), + count); folio_ref_add(folio, count); if (in_range(vmf->address, addr, count * PAGE_SIZE)) ret = VM_FAULT_NOPAGE; @@ -3553,6 +3555,7 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf, if (count) { set_pte_range(vmf, folio, page, count, addr); + add_mm_counter(vmf->vma->vm_mm, mm_counter_file(folio), count); folio_ref_add(folio, count); if (in_range(vmf->address, addr, count * PAGE_SIZE)) ret = VM_FAULT_NOPAGE; @@ -3589,6 +3592,7 @@ static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, ret = VM_FAULT_NOPAGE; set_pte_range(vmf, folio, page, 1, addr); + add_mm_counter(vmf->vma->vm_mm, mm_counter_file(folio), 1); folio_ref_inc(folio); return ret; diff --git a/mm/memory.c b/mm/memory.c index b71d011105962..33d87b64d15db 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4687,12 +4687,10 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio, entry = pte_mkuffd_wp(entry); /* copy-on-write page */ if (write && !(vma->vm_flags & VM_SHARED)) { - add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr); VM_BUG_ON_FOLIO(nr != 1, folio); folio_add_new_anon_rmap(folio, vma, addr); folio_add_lru_vma(folio, vma); } else { - add_mm_counter(vma->vm_mm, mm_counter_file(folio), nr); folio_add_file_rmap_ptes(folio, page, nr, vma); } set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr); @@ -4729,9 +4727,11 @@ vm_fault_t finish_fault(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; struct page *page; vm_fault_t ret; + bool is_cow = (vmf->flags & FAULT_FLAG_WRITE) && + !(vma->vm_flags & VM_SHARED); /* Did we COW the page? */ - if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) + if (is_cow) page = vmf->cow_page; else page = vmf->page; @@ -4767,8 +4767,10 @@ vm_fault_t finish_fault(struct vm_fault *vmf) /* Re-check under ptl */ if (likely(!vmf_pte_changed(vmf))) { struct folio *folio = page_folio(page); + int type = is_cow ? MM_ANONPAGES : mm_counter_file(folio); set_pte_range(vmf, folio, page, 1, vmf->address); + add_mm_counter(vma->vm_mm, type, 1); ret = 0; } else { update_mmu_tlb(vma, vmf->address, vmf->pte); From ceca44991f3dd5a67b4e0ded6379c5e93e84cb31 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Fri, 12 Apr 2024 14:47:51 +0800 Subject: [PATCH 1157/1702] mm: filemap: batch mm counter updating in filemap_map_pages() Like copy_pte_range()/zap_pte_range(), make mm counter batch updating in filemap_map_pages(), since folios type are same(MM_SHMEMPAGES or MM_FILEPAGES) in filemap_map_pages(), only check the first folio type is enough, the 'lat_pagefault -P 1 file' test from lmbench shows 12% improvement, and the percpu_counter_add_batch() is gone from perf flame graph. Link: https://lkml.kernel.org/r/20240412064751.119015-3-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/filemap.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 5a518c5075fb3..787a2fc6f4aec 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -3505,7 +3505,7 @@ static struct folio *next_uptodate_folio(struct xa_state *xas, static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf, struct folio *folio, unsigned long start, unsigned long addr, unsigned int nr_pages, - unsigned int *mmap_miss) + unsigned long *rss, unsigned int *mmap_miss) { vm_fault_t ret = 0; struct page *page = folio_page(folio, start); @@ -3539,8 +3539,7 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf, skip: if (count) { set_pte_range(vmf, folio, page, count, addr); - add_mm_counter(vmf->vma->vm_mm, mm_counter_file(folio), - count); + *rss += count; folio_ref_add(folio, count); if (in_range(vmf->address, addr, count * PAGE_SIZE)) ret = VM_FAULT_NOPAGE; @@ -3555,7 +3554,7 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf, if (count) { set_pte_range(vmf, folio, page, count, addr); - add_mm_counter(vmf->vma->vm_mm, mm_counter_file(folio), count); + *rss += count; folio_ref_add(folio, count); if (in_range(vmf->address, addr, count * PAGE_SIZE)) ret = VM_FAULT_NOPAGE; @@ -3568,7 +3567,7 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf, static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, struct folio *folio, unsigned long addr, - unsigned int *mmap_miss) + unsigned long *rss, unsigned int *mmap_miss) { vm_fault_t ret = 0; struct page *page = &folio->page; @@ -3592,7 +3591,7 @@ static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, ret = VM_FAULT_NOPAGE; set_pte_range(vmf, folio, page, 1, addr); - add_mm_counter(vmf->vma->vm_mm, mm_counter_file(folio), 1); + (*rss)++; folio_ref_inc(folio); return ret; @@ -3609,7 +3608,8 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, XA_STATE(xas, &mapping->i_pages, start_pgoff); struct folio *folio; vm_fault_t ret = 0; - unsigned int nr_pages = 0, mmap_miss = 0, mmap_miss_saved; + unsigned long rss = 0; + unsigned int nr_pages = 0, mmap_miss = 0, mmap_miss_saved, folio_type; rcu_read_lock(); folio = next_uptodate_folio(&xas, mapping, end_pgoff); @@ -3628,6 +3628,8 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, folio_put(folio); goto out; } + + folio_type = mm_counter_file(folio); do { unsigned long end; @@ -3639,15 +3641,16 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, if (!folio_test_large(folio)) ret |= filemap_map_order0_folio(vmf, - folio, addr, &mmap_miss); + folio, addr, &rss, &mmap_miss); else ret |= filemap_map_folio_range(vmf, folio, xas.xa_index - folio->index, addr, - nr_pages, &mmap_miss); + nr_pages, &rss, &mmap_miss); folio_unlock(folio); folio_put(folio); } while ((folio = next_uptodate_folio(&xas, mapping, end_pgoff)) != NULL); + add_mm_counter(vma->vm_mm, folio_type, rss); pte_unmap_unlock(vmf->pte, vmf->ptl); out: rcu_read_unlock(); From 231f8c7127e37edcd4d9e3f87e0f9fcf0e90d902 Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Fri, 12 Apr 2024 11:27:04 +0800 Subject: [PATCH 1158/1702] mm: page_alloc: allowing mTHP compaction to capture the freed page directly Currently, compaction_capture() does not allow lower-order allocations to directly capture the movable free pages, even though lower-order allocations might also be requesting movable pages, that can lead to more compaction scanning. And, with the enablement of mTHP, such situations will become more common. Thus allowing lower-order (mTHP) allocations of movable page types directly capture the movable free pages can avoid unnecessary compaction scanning, meanwhile that won't pollute the movable pageblock. With testing 1M mTHP compaction, it can be seen that compaction scanning is significantly reduced. mm-unstable patched Ops Compaction pages isolated 116598741.00 120946702.00 Ops Compaction migrate scanned 1764870054.00 1488621550.00 Ops Compaction free scanned 7707879039.00 4986299318.00 Ops Compact scan efficiency 22.90 29.85 Ops Compaction cost 73797.69 72933.48 Link: https://lkml.kernel.org/r/8118a5d66a034736a48433beddaca60ed78577c4.1712892329.git.baolin.wang@linux.alibaba.com Signed-off-by: Baolin Wang Reviewed-by: Zi Yan Acked-by: Johannes Weiner Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Mel Gorman Cc: Ryan Roberts Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/page_alloc.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 45fccbcda763d..2e22ce5675ca1 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -595,12 +595,14 @@ compaction_capture(struct capture_control *capc, struct page *page, return false; /* - * Do not let lower order allocations pollute a movable pageblock. + * Do not let lower order allocations pollute a movable pageblock + * unless compaction is also requesting movable pages. * This might let an unmovable request use a reclaimable pageblock * and vice-versa but no more than normal fallback logic which can * have trouble finding a high-order free page. */ - if (order < pageblock_order && migratetype == MIGRATE_MOVABLE) + if (order < pageblock_order && migratetype == MIGRATE_MOVABLE && + capc->cc->migratetype != MIGRATE_MOVABLE) return false; capc->page = page; From b5ba3a64279355731252098d92550e12bf9649e4 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Sun, 14 Apr 2024 19:08:21 -0700 Subject: [PATCH 1159/1702] userfaultfd: remove WRITE_ONCE when setting folio->index during UFFDIO_MOVE When folio is moved with UFFDIO_MOVE it gets locked before the rmap and index are modified. Due to the folio lock being already held, WRITE_ONCE() is not needed when setting the folio index. Remove it. Link: https://lkml.kernel.org/r/20240415020821.1152951-1-surenb@google.com Reported-by: Matthew Wilcox Signed-off-by: Suren Baghdasaryan Reviewed-by: David Hildenbrand Reviewed-by: Peter Xu Cc: Lokesh Gidra Signed-off-by: Andrew Morton --- mm/huge_memory.c | 2 +- mm/userfaultfd.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 264e09043f096..31b6bbffea521 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2200,7 +2200,7 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm } folio_move_anon_rmap(src_folio, dst_vma); - WRITE_ONCE(src_folio->index, linear_page_index(dst_vma, dst_addr)); + src_folio->index = linear_page_index(dst_vma, dst_addr); _dst_pmd = mk_huge_pmd(&src_folio->page, dst_vma->vm_page_prot); /* Follow mremap() behavior and treat the entry dirty after the move */ diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index b70618e8dcd24..575ccf90325aa 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1026,7 +1026,7 @@ static int move_present_pte(struct mm_struct *mm, } folio_move_anon_rmap(src_folio, dst_vma); - WRITE_ONCE(src_folio->index, linear_page_index(dst_vma, dst_addr)); + src_folio->index = linear_page_index(dst_vma, dst_addr); orig_dst_pte = mk_pte(&src_folio->page, dst_vma->vm_page_prot); /* Follow mremap() behavior and treat the entry dirty after the move */ From 4ea3fa9dd2e9c58920d73b1b2cd6bcc6f14ae0e0 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Sat, 13 Apr 2024 02:24:04 +0000 Subject: [PATCH 1160/1702] mm: zswap: always shrink in zswap_store() if zswap_pool_reached_full Patch series "zswap same-filled and limit checking cleanups", v3. Miscellaneous cleanups for limit checking and same-filled handling in the store path. This series was broken out of the "zswap: store zero-filled pages more efficiently" series [1]. It contains the cleanups and drops the main functional changes. [1]https://lore.kernel.org/lkml/20240325235018.2028408-1-yosryahmed@google.com/ This patch (of 4): The cleanup code in zswap_store() is not pretty, particularly the 'shrink' label at the bottom that ends up jumping between cleanup labels. Instead of having a dedicated label to shrink the pool, just use zswap_pool_reached_full directly to figure out if the pool needs shrinking. zswap_pool_reached_full should be true if and only if the pool needs shrinking. The only caveat is that the value of zswap_pool_reached_full may be changed by concurrent zswap_store() calls between checking the limit and testing zswap_pool_reached_full in the cleanup code. This is fine because: - If zswap_pool_reached_full was true during limit checking then became false during the cleanup code, then someone else already took care of shrinking the pool and there is no need to queue the worker. That would be a good change. - If zswap_pool_reached_full was false during limit checking then became true during the cleanup code, then someone else hit the limit meanwhile. In this case, both threads will try to queue the worker, but it never gets queued more than once anyway. Also, calling queue_work() multiple times when the limit is hit could already happen today, so this isn't a significant change in any way. Link: https://lkml.kernel.org/r/20240413022407.785696-1-yosryahmed@google.com Link: https://lkml.kernel.org/r/20240413022407.785696-2-yosryahmed@google.com Signed-off-by: Yosry Ahmed Reviewed-by: Nhat Pham Reviewed-by: Chengming Zhou Acked-by: Johannes Weiner Cc: "Maciej S. Szmigiero" Signed-off-by: Andrew Morton --- mm/zswap.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index e6cf6282f5fd0..d753ad9d45918 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1436,12 +1436,12 @@ bool zswap_store(struct folio *folio) if (cur_pages >= max_pages) { zswap_pool_limit_hit++; zswap_pool_reached_full = true; - goto shrink; + goto reject; } if (zswap_pool_reached_full) { if (cur_pages > zswap_accept_thr_pages()) - goto shrink; + goto reject; else zswap_pool_reached_full = false; } @@ -1547,6 +1547,8 @@ bool zswap_store(struct folio *folio) zswap_entry_cache_free(entry); reject: obj_cgroup_put(objcg); + if (zswap_pool_reached_full) + queue_work(shrink_wq, &zswap_shrink_work); check_old: /* * If the zswap store fails or zswap is disabled, we must invalidate the @@ -1557,10 +1559,6 @@ bool zswap_store(struct folio *folio) if (entry) zswap_entry_free(entry); return false; - -shrink: - queue_work(shrink_wq, &zswap_shrink_work); - goto reject; } bool zswap_load(struct folio *folio) From 82e0f8e47b406bb5948c2300a02ac6ede21532c1 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Sat, 13 Apr 2024 02:24:05 +0000 Subject: [PATCH 1161/1702] mm: zswap: refactor limit checking from zswap_store() Refactor limit and acceptance threshold checking outside of zswap_store(). This code will be moved around in a following patch, so it would be cleaner to move a function call around. Link: https://lkml.kernel.org/r/20240413022407.785696-3-yosryahmed@google.com Signed-off-by: Yosry Ahmed Reviewed-by: Nhat Pham Cc: Chengming Zhou Cc: Johannes Weiner Cc: "Maciej S. Szmigiero" Signed-off-by: Andrew Morton --- mm/zswap.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index d753ad9d45918..1cbd3cf97eeff 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -517,6 +517,21 @@ unsigned long zswap_total_pages(void) return total; } +static bool zswap_check_limits(void) +{ + unsigned long cur_pages = zswap_total_pages(); + unsigned long max_pages = zswap_max_pages(); + + if (cur_pages >= max_pages) { + zswap_pool_limit_hit++; + zswap_pool_reached_full = true; + } else if (zswap_pool_reached_full && + cur_pages <= zswap_accept_thr_pages()) { + zswap_pool_reached_full = false; + } + return zswap_pool_reached_full; +} + /********************************* * param callbacks **********************************/ @@ -1406,7 +1421,6 @@ bool zswap_store(struct folio *folio) struct zswap_entry *entry, *old; struct obj_cgroup *objcg = NULL; struct mem_cgroup *memcg = NULL; - unsigned long max_pages, cur_pages; VM_WARN_ON_ONCE(!folio_test_locked(folio)); VM_WARN_ON_ONCE(!folio_test_swapcache(folio)); @@ -1429,22 +1443,8 @@ bool zswap_store(struct folio *folio) mem_cgroup_put(memcg); } - /* Check global limits */ - cur_pages = zswap_total_pages(); - max_pages = zswap_max_pages(); - - if (cur_pages >= max_pages) { - zswap_pool_limit_hit++; - zswap_pool_reached_full = true; + if (zswap_check_limits()) goto reject; - } - - if (zswap_pool_reached_full) { - if (cur_pages > zswap_accept_thr_pages()) - goto reject; - else - zswap_pool_reached_full = false; - } /* allocate entry */ entry = zswap_entry_cache_alloc(GFP_KERNEL, folio_nid(folio)); From e87b881489085b4832ad417da653739cbace49e7 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Sat, 13 Apr 2024 02:24:06 +0000 Subject: [PATCH 1162/1702] mm: zswap: move more same-filled pages checks outside of zswap_store() Currently, zswap_store() checks zswap_same_filled_pages_enabled, kmaps the folio, then calls zswap_is_page_same_filled() to check the folio contents. Move this logic into zswap_is_page_same_filled() as well (and rename it to use 'folio' while we are at it). This makes zswap_store() cleaner, and makes following changes to that logic contained within the helper. While we are at it: - Rename the insert_entry label to store_entry to match xa_store(). - Add comment headers for same-filled functions and the main API functions (load, store, invalidate, swapon, swapoff). No functional change intended. Link: https://lkml.kernel.org/r/20240413022407.785696-4-yosryahmed@google.com Signed-off-by: Yosry Ahmed Reviewed-by: Nhat Pham Reviewed-by: Chengming Zhou Acked-by: Johannes Weiner Cc: "Maciej S. Szmigiero" Signed-off-by: Andrew Morton --- mm/zswap.c | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index 1cbd3cf97eeff..18fd70795f5b6 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1383,26 +1383,35 @@ static void shrink_worker(struct work_struct *w) } while (zswap_total_pages() > thr); } -static int zswap_is_page_same_filled(void *ptr, unsigned long *value) +/********************************* +* same-filled functions +**********************************/ +static bool zswap_is_folio_same_filled(struct folio *folio, unsigned long *value) { unsigned long *page; unsigned long val; unsigned int pos, last_pos = PAGE_SIZE / sizeof(*page) - 1; + bool ret = false; - page = (unsigned long *)ptr; + if (!zswap_same_filled_pages_enabled) + return false; + + page = kmap_local_folio(folio, 0); val = page[0]; if (val != page[last_pos]) - return 0; + goto out; for (pos = 1; pos < last_pos; pos++) { if (val != page[pos]) - return 0; + goto out; } *value = val; - - return 1; + ret = true; +out: + kunmap_local(page); + return ret; } static void zswap_fill_page(void *ptr, unsigned long value) @@ -1413,6 +1422,9 @@ static void zswap_fill_page(void *ptr, unsigned long value) memset_l(page, value, PAGE_SIZE / sizeof(unsigned long)); } +/********************************* +* main API +**********************************/ bool zswap_store(struct folio *folio) { swp_entry_t swp = folio->swap; @@ -1421,6 +1433,7 @@ bool zswap_store(struct folio *folio) struct zswap_entry *entry, *old; struct obj_cgroup *objcg = NULL; struct mem_cgroup *memcg = NULL; + unsigned long value; VM_WARN_ON_ONCE(!folio_test_locked(folio)); VM_WARN_ON_ONCE(!folio_test_swapcache(folio)); @@ -1453,19 +1466,11 @@ bool zswap_store(struct folio *folio) goto reject; } - if (zswap_same_filled_pages_enabled) { - unsigned long value; - u8 *src; - - src = kmap_local_folio(folio, 0); - if (zswap_is_page_same_filled(src, &value)) { - kunmap_local(src); - entry->length = 0; - entry->value = value; - atomic_inc(&zswap_same_filled_pages); - goto insert_entry; - } - kunmap_local(src); + if (zswap_is_folio_same_filled(folio, &value)) { + entry->length = 0; + entry->value = value; + atomic_inc(&zswap_same_filled_pages); + goto store_entry; } if (!zswap_non_same_filled_pages_enabled) @@ -1488,7 +1493,7 @@ bool zswap_store(struct folio *folio) if (!zswap_compress(folio, entry)) goto put_pool; -insert_entry: +store_entry: entry->swpentry = swp; entry->objcg = objcg; From c074e1467f8546c8f8c9ea2128fb8bf8c4579418 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Sat, 13 Apr 2024 02:24:07 +0000 Subject: [PATCH 1163/1702] mm: zswap: remove same_filled module params These knobs offer more fine-grained control to userspace than needed and directly expose/influence kernel implementation; remove them. For disabling same_filled handling, there is no logical reason to refuse storing same-filled pages more efficiently and opt for compression. Scanning pages for patterns may be an argument, but the page contents will be read into the CPU cache anyway during compression. Also, removing the same_filled handling code does not move the needle significantly in terms of performance anyway [1]. For disabling non_same_filled handling, it was added when the compressed pages in zswap were not being properly charged to memcgs, as workloads could escape the accounting with compression [2]. This is no longer the case after commit f4840ccfca25 ("zswap: memcg accounting"), and using zswap without compression does not make much sense. [1]https://lore.kernel.org/lkml/CAJD7tkaySFP2hBQw4pnZHJJwe3bMdjJ1t9VC2VJd=khn1_TXvA@mail.gmail.com/ [2]https://lore.kernel.org/lkml/19d5cdee-2868-41bd-83d5-6da75d72e940@maciej.szmigiero.name/ [yosryahmed@google.com: remove same_filled_pages from docs] Link: https://lkml.kernel.org/r/ZhxFVggdyvCo79jc@google.com Link: https://lkml.kernel.org/r/20240413022407.785696-5-yosryahmed@google.com Signed-off-by: Yosry Ahmed Acked-by: Johannes Weiner Reviewed-by: Nhat Pham Reviewed-by: Chengming Zhou Cc: "Maciej S. Szmigiero" Signed-off-by: Andrew Morton --- Documentation/admin-guide/mm/zswap.rst | 29 ------------------- .../driver-api/crypto/iaa/iaa-crypto.rst | 2 -- mm/zswap.c | 19 ------------ 3 files changed, 50 deletions(-) diff --git a/Documentation/admin-guide/mm/zswap.rst b/Documentation/admin-guide/mm/zswap.rst index 13632671adaea..3598dcd7dbe7e 100644 --- a/Documentation/admin-guide/mm/zswap.rst +++ b/Documentation/admin-guide/mm/zswap.rst @@ -111,35 +111,6 @@ checked if it is a same-value filled page before compressing it. If true, the compressed length of the page is set to zero and the pattern or same-filled value is stored. -Same-value filled pages identification feature is enabled by default and can be -disabled at boot time by setting the ``same_filled_pages_enabled`` attribute -to 0, e.g. ``zswap.same_filled_pages_enabled=0``. It can also be enabled and -disabled at runtime using the sysfs ``same_filled_pages_enabled`` -attribute, e.g.:: - - echo 1 > /sys/module/zswap/parameters/same_filled_pages_enabled - -When zswap same-filled page identification is disabled at runtime, it will stop -checking for the same-value filled pages during store operation. -In other words, every page will be then considered non-same-value filled. -However, the existing pages which are marked as same-value filled pages remain -stored unchanged in zswap until they are either loaded or invalidated. - -In some circumstances it might be advantageous to make use of just the zswap -ability to efficiently store same-filled pages without enabling the whole -compressed page storage. -In this case the handling of non-same-value pages by zswap (enabled by default) -can be disabled by setting the ``non_same_filled_pages_enabled`` attribute -to 0, e.g. ``zswap.non_same_filled_pages_enabled=0``. -It can also be enabled and disabled at runtime using the sysfs -``non_same_filled_pages_enabled`` attribute, e.g.:: - - echo 1 > /sys/module/zswap/parameters/non_same_filled_pages_enabled - -Disabling both ``zswap.same_filled_pages_enabled`` and -``zswap.non_same_filled_pages_enabled`` effectively disables accepting any new -pages by zswap. - To prevent zswap from shrinking pool when zswap is full and there's a high pressure on swap (this will result in flipping pages in and out zswap pool without any real benefit but with a performance drop for the system), a diff --git a/Documentation/driver-api/crypto/iaa/iaa-crypto.rst b/Documentation/driver-api/crypto/iaa/iaa-crypto.rst index de587cf9cbed4..4cb1d52ea6dc4 100644 --- a/Documentation/driver-api/crypto/iaa/iaa-crypto.rst +++ b/Documentation/driver-api/crypto/iaa/iaa-crypto.rst @@ -457,7 +457,6 @@ Use the following commands to enable zswap:: # echo deflate-iaa > /sys/module/zswap/parameters/compressor # echo zsmalloc > /sys/module/zswap/parameters/zpool # echo 1 > /sys/module/zswap/parameters/enabled - # echo 0 > /sys/module/zswap/parameters/same_filled_pages_enabled # echo 100 > /proc/sys/vm/swappiness # echo never > /sys/kernel/mm/transparent_hugepage/enabled # echo 1 > /proc/sys/vm/overcommit_memory @@ -599,7 +598,6 @@ the 'fixed' compression mode:: echo deflate-iaa > /sys/module/zswap/parameters/compressor echo zsmalloc > /sys/module/zswap/parameters/zpool echo 1 > /sys/module/zswap/parameters/enabled - echo 0 > /sys/module/zswap/parameters/same_filled_pages_enabled echo 100 > /proc/sys/vm/swappiness echo never > /sys/kernel/mm/transparent_hugepage/enabled diff --git a/mm/zswap.c b/mm/zswap.c index 18fd70795f5b6..a50e2986cd2fa 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -123,19 +123,6 @@ static unsigned int zswap_accept_thr_percent = 90; /* of max pool size */ module_param_named(accept_threshold_percent, zswap_accept_thr_percent, uint, 0644); -/* - * Enable/disable handling same-value filled pages (enabled by default). - * If disabled every page is considered non-same-value filled. - */ -static bool zswap_same_filled_pages_enabled = true; -module_param_named(same_filled_pages_enabled, zswap_same_filled_pages_enabled, - bool, 0644); - -/* Enable/disable handling non-same-value filled pages (enabled by default) */ -static bool zswap_non_same_filled_pages_enabled = true; -module_param_named(non_same_filled_pages_enabled, zswap_non_same_filled_pages_enabled, - bool, 0644); - /* Number of zpools in zswap_pool (empirically determined for scalability) */ #define ZSWAP_NR_ZPOOLS 32 @@ -1393,9 +1380,6 @@ static bool zswap_is_folio_same_filled(struct folio *folio, unsigned long *value unsigned int pos, last_pos = PAGE_SIZE / sizeof(*page) - 1; bool ret = false; - if (!zswap_same_filled_pages_enabled) - return false; - page = kmap_local_folio(folio, 0); val = page[0]; @@ -1473,9 +1457,6 @@ bool zswap_store(struct folio *folio) goto store_entry; } - if (!zswap_non_same_filled_pages_enabled) - goto freepage; - /* if entry is successfully added, it keeps the reference */ entry->pool = zswap_pool_current_get(); if (!entry->pool) From 2aa339120c7dfe834297a77b13b1a98e12842932 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 16 Apr 2024 19:25:33 +0200 Subject: [PATCH 1164/1702] mm/ksm: remove page_mapcount() usage in stable_tree_search() We want to limit the use of page_mapcount() to the places where it is absolutely necessary. If our folio has a stable node, it is a (small) KSM folio -- see folio_stable_node(). Let's use folio_mapcount() in stable_tree_search() instead, which results in no functional change. The mapcount > 1 check is a bit confusing, because that's usually a check for page sharing. Looks like the reason is that we are guaranteed to not exceed ksm_max_page_sharing for the tree KSM folio when merging with that. Let's update the documentation to make that clearer. Link: https://lkml.kernel.org/r/20240416172533.663418-1-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Alex Shi Signed-off-by: Andrew Morton --- mm/ksm.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mm/ksm.c b/mm/ksm.c index 486c9974f8e20..159604ad47799 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -1909,12 +1909,15 @@ static struct page *stable_tree_search(struct page *page) if (page_node) { VM_BUG_ON(page_node->head != &migrate_nodes); /* - * Test if the migrated page should be merged - * into a stable node dup. If the mapcount is - * 1 we can migrate it with another KSM page - * without adding it to the chain. + * If the mapcount of our migrated KSM folio is + * at most 1, we can merge it with another + * KSM folio where we know that we have space + * for one more mapping without exceeding the + * ksm_max_page_sharing limit: see + * chain_prune(). This way, we can avoid adding + * this stable node to the chain. */ - if (page_mapcount(page) > 1) + if (folio_mapcount(folio) > 1) goto chain_append; } From ba591801a3df861b3b327f6122b9de4ef213aae6 Mon Sep 17 00:00:00 2001 From: Long Li Date: Tue, 16 Apr 2024 14:16:28 +0800 Subject: [PATCH 1165/1702] xarray: inline xas_descend to improve performance The commit 63b1898fffcd ("XArray: Disallow sibling entries of nodes") modified the xas_descend function in such a way that it was no longer being compiled as an inline function, because it increased the size of xas_descend(), and the compiler no longer optimizes it as inline. This had a negative impact on performance, xas_descend is called frequently to traverse downwards in the xarray tree, making it a hot function. Inlining xas_descend has been shown to significantly improve performance by approximately 4.95% in the iozone write test. Machine: Intel(R) Xeon(R) Gold 6240 CPU @ 2.60GHz #iozone i 0 -i 1 -s 64g -r 16m -f /test/tmptest Before this patch: kB reclen write rewrite read reread 67108864 16384 2230080 3637689 6315197 5496027 After this patch: kB reclen write rewrite read reread 67108864 16384 2340360 3666175 6272401 5460782 Percentage change: 4.95% 0.78% -0.68% -0.64% This patch introduces inlining to the xas_descend function. While this change increases the size of lib/xarray.o, the performance gains in critical workloads make this an acceptable trade-off. Size comparison before and after patch: .text .data .bss file 0x3502 0 0 lib/xarray.o.before 0x3602 0 0 lib/xarray.o.after Link: https://lkml.kernel.org/r/20240416061628.3768901-1-leo.lilong@huawei.com Signed-off-by: Long Li Cc: Hou Tao Cc: Matthew Wilcox (Oracle) Cc: yangerkun Cc: Zhang Yi Signed-off-by: Andrew Morton --- lib/xarray.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/xarray.c b/lib/xarray.c index da79128ad754f..1c87d871cacfa 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -200,7 +200,8 @@ static void *xas_start(struct xa_state *xas) return entry; } -static void *xas_descend(struct xa_state *xas, struct xa_node *node) +static __always_inline void *xas_descend(struct xa_state *xas, + struct xa_node *node) { unsigned int offset = get_offset(xas->xa_index, node); void *entry = xa_entry(xas->xa, node, offset); From 3d84d897920c75fc3aeeac452e8e8a4073398ce7 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 16 Apr 2024 04:17:45 +0100 Subject: [PATCH 1166/1702] doc: improve the description of __folio_mark_dirty Patch series "Improve buffer head documentation", v3. Turn buffer head documentation into its own document, and make many general improvements to the docs. Obviously there is much more that could be done. Tested with make htmldocs. This patch (of 8): I've learned why it's safe to call __folio_mark_dirty() from mark_buffer_dirty() without holding the folio lock, so update the description to explain why. Link: https://lkml.kernel.org/r/20240416031754.4076917-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240416031754.4076917-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Cc: Pankaj Raghav Signed-off-by: Andrew Morton --- mm/page-writeback.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mm/page-writeback.c b/mm/page-writeback.c index fba324e1a010d..2c1f595d1ddc4 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -2705,11 +2705,15 @@ void folio_account_cleaned(struct folio *folio, struct bdi_writeback *wb) * If warn is true, then emit a warning if the folio is not uptodate and has * not been truncated. * - * The caller must hold folio_memcg_lock(). Most callers have the folio - * locked. A few have the folio blocked from truncation through other - * means (eg zap_vma_pages() has it mapped and is holding the page table - * lock). This can also be called from mark_buffer_dirty(), which I - * cannot prove is always protected against truncate. + * The caller must hold folio_memcg_lock(). It is the caller's + * responsibility to prevent the folio from being truncated while + * this function is in progress, although it may have been truncated + * before this function is called. Most callers have the folio locked. + * A few have the folio blocked from truncation through other means (e.g. + * zap_vma_pages() has it mapped and is holding the page table lock). + * When called from mark_buffer_dirty(), the filesystem should hold a + * reference to the buffer_head that is being marked dirty, which causes + * try_to_free_buffers() to fail. */ void __folio_mark_dirty(struct folio *folio, struct address_space *mapping, int warn) From 3814ec89540d9ce1a92cb4c9a6f9f7a0a343d73d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 16 Apr 2024 04:17:46 +0100 Subject: [PATCH 1167/1702] buffer: add kernel-doc for block_dirty_folio() Turn the excellent documentation for this function into kernel-doc. Replace 'page' with 'folio' and make a few other minor updates. Link: https://lkml.kernel.org/r/20240416031754.4076917-3-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Pankaj Raghav Tested-by: Randy Dunlap Signed-off-by: Andrew Morton --- fs/buffer.c | 55 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index 4f73d23c2c469..b08526bdcb540 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -687,30 +687,37 @@ void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode) } EXPORT_SYMBOL(mark_buffer_dirty_inode); -/* - * Add a page to the dirty page list. - * - * It is a sad fact of life that this function is called from several places - * deeply under spinlocking. It may not sleep. - * - * If the page has buffers, the uptodate buffers are set dirty, to preserve - * dirty-state coherency between the page and the buffers. It the page does - * not have buffers then when they are later attached they will all be set - * dirty. - * - * The buffers are dirtied before the page is dirtied. There's a small race - * window in which a writepage caller may see the page cleanness but not the - * buffer dirtiness. That's fine. If this code were to set the page dirty - * before the buffers, a concurrent writepage caller could clear the page dirty - * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean - * page on the dirty page list. - * - * We use i_private_lock to lock against try_to_free_buffers while using the - * page's buffer list. Also use this to protect against clean buffers being - * added to the page after it was set dirty. - * - * FIXME: may need to call ->reservepage here as well. That's rather up to the - * address_space though. +/** + * block_dirty_folio - Mark a folio as dirty. + * @mapping: The address space containing this folio. + * @folio: The folio to mark dirty. + * + * Filesystems which use buffer_heads can use this function as their + * ->dirty_folio implementation. Some filesystems need to do a little + * work before calling this function. Filesystems which do not use + * buffer_heads should call filemap_dirty_folio() instead. + * + * If the folio has buffers, the uptodate buffers are set dirty, to + * preserve dirty-state coherency between the folio and the buffers. + * Buffers added to a dirty folio are created dirty. + * + * The buffers are dirtied before the folio is dirtied. There's a small + * race window in which writeback may see the folio cleanness but not the + * buffer dirtiness. That's fine. If this code were to set the folio + * dirty before the buffers, writeback could clear the folio dirty flag, + * see a bunch of clean buffers and we'd end up with dirty buffers/clean + * folio on the dirty folio list. + * + * We use i_private_lock to lock against try_to_free_buffers() while + * using the folio's buffer list. This also prevents clean buffers + * being added to the folio after it was set dirty. + * + * Context: May only be called from process context. Does not sleep. + * Caller must ensure that @folio cannot be truncated during this call, + * typically by holding the folio lock or having a page in the folio + * mapped and holding the page table lock. + * + * Return: True if the folio was dirtied; false if it was already dirtied. */ bool block_dirty_folio(struct address_space *mapping, struct folio *folio) { From b1888d143203589b71ab31b39d1070737287bc79 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 16 Apr 2024 04:17:47 +0100 Subject: [PATCH 1168/1702] buffer: add kernel-doc for try_to_free_buffers() The documentation for this function has become separated from it over time; move it to the right place and turn it into kernel-doc. Mild editing of the content to make it more about what the function does, and less about how it does it. Link: https://lkml.kernel.org/r/20240416031754.4076917-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Pankaj Raghav Tested-by: Randy Dunlap Signed-off-by: Andrew Morton --- fs/buffer.c | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index b08526bdcb540..0466ed7ed95a0 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -2868,26 +2868,6 @@ int sync_dirty_buffer(struct buffer_head *bh) } EXPORT_SYMBOL(sync_dirty_buffer); -/* - * try_to_free_buffers() checks if all the buffers on this particular folio - * are unused, and releases them if so. - * - * Exclusion against try_to_free_buffers may be obtained by either - * locking the folio or by holding its mapping's i_private_lock. - * - * If the folio is dirty but all the buffers are clean then we need to - * be sure to mark the folio clean as well. This is because the folio - * may be against a block device, and a later reattachment of buffers - * to a dirty folio will set *all* buffers dirty. Which would corrupt - * filesystem data on the same device. - * - * The same applies to regular filesystem folios: if all the buffers are - * clean then we set the folio clean and proceed. To do that, we require - * total exclusion from block_dirty_folio(). That is obtained with - * i_private_lock. - * - * try_to_free_buffers() is non-blocking. - */ static inline int buffer_busy(struct buffer_head *bh) { return atomic_read(&bh->b_count) | @@ -2921,6 +2901,30 @@ drop_buffers(struct folio *folio, struct buffer_head **buffers_to_free) return false; } +/** + * try_to_free_buffers - Release buffers attached to this folio. + * @folio: The folio. + * + * If any buffers are in use (dirty, under writeback, elevated refcount), + * no buffers will be freed. + * + * If the folio is dirty but all the buffers are clean then we need to + * be sure to mark the folio clean as well. This is because the folio + * may be against a block device, and a later reattachment of buffers + * to a dirty folio will set *all* buffers dirty. Which would corrupt + * filesystem data on the same device. + * + * The same applies to regular filesystem folios: if all the buffers are + * clean then we set the folio clean and proceed. To do that, we require + * total exclusion from block_dirty_folio(). That is obtained with + * i_private_lock. + * + * Exclusion against try_to_free_buffers may be obtained by either + * locking the folio or by holding its mapping's i_private_lock. + * + * Context: Process context. @folio must be locked. Will not sleep. + * Return: true if all buffers attached to this folio were freed. + */ bool try_to_free_buffers(struct folio *folio) { struct address_space * const mapping = folio->mapping; From 324ecaee46f86c1eaf083fd82eaf997335e70163 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 16 Apr 2024 04:17:48 +0100 Subject: [PATCH 1169/1702] buffer: fix __bread and __bread_gfp kernel-doc The extra indentation confused the kernel-doc parser, so remove it. Fix some other wording while I'm here, and advise the user they need to call brelse() on this buffer. __bread_gfp() isn't used directly by filesystems, but the other wrappers for it don't have documentation, so document it accordingly. Link: https://lkml.kernel.org/r/20240416031754.4076917-5-willy@infradead.org Co-developed-by: Pankaj Raghav Signed-off-by: Pankaj Raghav Signed-off-by: Matthew Wilcox (Oracle) Tested-by: Randy Dunlap Signed-off-by: Andrew Morton --- fs/buffer.c | 35 ++++++++++++++++++++++------------- include/linux/buffer_head.h | 22 +++++++++++++--------- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index 0466ed7ed95a0..32ab3eddc44fc 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -1453,20 +1453,29 @@ void __breadahead(struct block_device *bdev, sector_t block, unsigned size) EXPORT_SYMBOL(__breadahead); /** - * __bread_gfp() - reads a specified block and returns the bh - * @bdev: the block_device to read from - * @block: number of block - * @size: size (in bytes) to read - * @gfp: page allocation flag - * - * Reads a specified block, and returns buffer head that contains it. - * The page cache can be allocated from non-movable area - * not to prevent page migration if you set gfp to zero. - * It returns NULL if the block was unreadable. + * __bread_gfp() - Read a block. + * @bdev: The block device to read from. + * @block: Block number in units of block size. + * @size: The block size of this device in bytes. + * @gfp: Not page allocation flags; see below. + * + * You are not expected to call this function. You should use one of + * sb_bread(), sb_bread_unmovable() or __bread(). + * + * Read a specified block, and return the buffer head that refers to it. + * If @gfp is 0, the memory will be allocated using the block device's + * default GFP flags. If @gfp is __GFP_MOVABLE, the memory may be + * allocated from a movable area. Do not pass in a complete set of + * GFP flags. + * + * The returned buffer head has its refcount increased. The caller should + * call brelse() when it has finished with the buffer. + * + * Context: May sleep waiting for I/O. + * Return: NULL if the block was unreadable. */ -struct buffer_head * -__bread_gfp(struct block_device *bdev, sector_t block, - unsigned size, gfp_t gfp) +struct buffer_head *__bread_gfp(struct block_device *bdev, sector_t block, + unsigned size, gfp_t gfp) { struct buffer_head *bh; diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index d78454a4dd1f0..56a1e9c1e71eb 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -437,17 +437,21 @@ static inline void bh_readahead_batch(int nr, struct buffer_head *bhs[], } /** - * __bread() - reads a specified block and returns the bh - * @bdev: the block_device to read from - * @block: number of block - * @size: size (in bytes) to read + * __bread() - Read a block. + * @bdev: The block device to read from. + * @block: Block number in units of block size. + * @size: The block size of this device in bytes. * - * Reads a specified block, and returns buffer head that contains it. - * The page cache is allocated from movable area so that it can be migrated. - * It returns NULL if the block was unreadable. + * Read a specified block, and return the buffer head that refers + * to it. The memory is allocated from the movable area so that it can + * be migrated. The returned buffer head has its refcount increased. + * The caller should call brelse() when it has finished with the buffer. + * + * Context: May sleep waiting for I/O. + * Return: NULL if the block was unreadable. */ -static inline struct buffer_head * -__bread(struct block_device *bdev, sector_t block, unsigned size) +static inline struct buffer_head *__bread(struct block_device *bdev, + sector_t block, unsigned size) { return __bread_gfp(bdev, block, size, __GFP_MOVABLE); } From 66924fdaf835f5fc6fe78d92a79afcca7d4db7ec Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 16 Apr 2024 04:17:49 +0100 Subject: [PATCH 1170/1702] buffer: add kernel-doc for brelse() and __brelse() Move the documentation for __brelse() to brelse(), format it as kernel-doc and update it from talking about pages to folios. Link: https://lkml.kernel.org/r/20240416031754.4076917-6-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Tested-by: Randy Dunlap Cc: Pankaj Raghav Signed-off-by: Andrew Morton --- fs/buffer.c | 17 ++++++++--------- include/linux/buffer_head.h | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index 32ab3eddc44fc..e5beca3868a77 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -1226,17 +1226,16 @@ void mark_buffer_write_io_error(struct buffer_head *bh) } EXPORT_SYMBOL(mark_buffer_write_io_error); -/* - * Decrement a buffer_head's reference count. If all buffers against a page - * have zero reference count, are clean and unlocked, and if the page is clean - * and unlocked then try_to_free_buffers() may strip the buffers from the page - * in preparation for freeing it (sometimes, rarely, buffers are removed from - * a page but it ends up not being freed, and buffers may later be reattached). +/** + * __brelse - Release a buffer. + * @bh: The buffer to release. + * + * This variant of brelse() can be called if @bh is guaranteed to not be NULL. */ -void __brelse(struct buffer_head * buf) +void __brelse(struct buffer_head *bh) { - if (atomic_read(&buf->b_count)) { - put_bh(buf); + if (atomic_read(&bh->b_count)) { + put_bh(bh); return; } WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n"); diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index 56a1e9c1e71eb..c145817c6ca0b 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -303,6 +303,22 @@ static inline void put_bh(struct buffer_head *bh) atomic_dec(&bh->b_count); } +/** + * brelse - Release a buffer. + * @bh: The buffer to release. + * + * Decrement a buffer_head's reference count. If @bh is NULL, this + * function is a no-op. + * + * If all buffers on a folio have zero reference count, are clean + * and unlocked, and if the folio is unlocked and not under writeback + * then try_to_free_buffers() may strip the buffers from the folio in + * preparation for freeing it (sometimes, rarely, buffers are removed + * from a folio but it ends up not being freed, and buffers may later + * be reattached). + * + * Context: Any context. + */ static inline void brelse(struct buffer_head *bh) { if (bh) From b73a936f99914e23fbe236f75ecf257923cb06e7 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 16 Apr 2024 04:17:50 +0100 Subject: [PATCH 1171/1702] buffer: add kernel-doc for bforget() and __bforget() Distinguish these functions from brelse() and __brelse(). Link: https://lkml.kernel.org/r/20240416031754.4076917-7-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Tested-by: Randy Dunlap Cc: Pankaj Raghav Signed-off-by: Andrew Morton --- fs/buffer.c | 9 ++++++--- include/linux/buffer_head.h | 10 ++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index e5beca3868a77..60829312787ab 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -1242,9 +1242,12 @@ void __brelse(struct buffer_head *bh) } EXPORT_SYMBOL(__brelse); -/* - * bforget() is like brelse(), except it discards any - * potentially dirty data. +/** + * __bforget - Discard any dirty data in a buffer. + * @bh: The buffer to forget. + * + * This variant of bforget() can be called if @bh is guaranteed to not + * be NULL. */ void __bforget(struct buffer_head *bh) { diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index c145817c6ca0b..a1c0bdd0cca66 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -325,6 +325,16 @@ static inline void brelse(struct buffer_head *bh) __brelse(bh); } +/** + * bforget - Discard any dirty data in a buffer. + * @bh: The buffer to forget. + * + * Call this function instead of brelse() if the data written to a buffer + * no longer needs to be written back. It will clear the buffer's dirty + * flag so writeback of this buffer will be skipped. + * + * Context: Any context. + */ static inline void bforget(struct buffer_head *bh) { if (bh) From 0b116ff4dc40ec84ce4bfd451436e66ab2bbc86d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 16 Apr 2024 04:17:51 +0100 Subject: [PATCH 1172/1702] buffer: improve bdev_getblk documentation Add some more information about the state of the buffer_head returned. Link: https://lkml.kernel.org/r/20240416031754.4076917-8-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Cc: Pankaj Raghav Signed-off-by: Andrew Morton --- fs/buffer.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fs/buffer.c b/fs/buffer.c index 60829312787ab..ed698caa8834b 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -1424,6 +1424,11 @@ EXPORT_SYMBOL(__find_get_block); * @size: The size of buffer_heads for this @bdev. * @gfp: The memory allocation flags to use. * + * The returned buffer head has its reference count incremented, but is + * not locked. The caller should call brelse() when it has finished + * with the buffer. The buffer may not be uptodate. If needed, the + * caller can bring it uptodate either by reading it or overwriting it. + * * Return: The buffer head, or NULL if memory could not be allocated. */ struct buffer_head *bdev_getblk(struct block_device *bdev, sector_t block, From 5ec5aab7754e4ed5b91be103427e00ab9a35f67d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 16 Apr 2024 04:17:52 +0100 Subject: [PATCH 1173/1702] doc: split buffer.rst out of api-summary.rst Buffer heads are no longer a generic filesystem API but an optional filesystem support library. Make the documentation structure reflect that, and include the fine documentation kept in buffer_head.h. We could give a better overview of what buffer heads are all about, but my enthusiasm for documenting it is limited. [willy@infradead.org: fix kerneldoc warning] Link: https://lkml.kernel.org/r/20240417015933.453505-1-willy@infradead.org [akpm@linux-foundation.org: remove newline at EOF] Link: https://lkml.kernel.org/r/20240416031754.4076917-9-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Tested-by: Randy Dunlap Cc: Pankaj Raghav Signed-off-by: Andrew Morton --- Documentation/filesystems/api-summary.rst | 3 --- Documentation/filesystems/buffer.rst | 12 ++++++++++++ Documentation/filesystems/index.rst | 1 + 3 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 Documentation/filesystems/buffer.rst diff --git a/Documentation/filesystems/api-summary.rst b/Documentation/filesystems/api-summary.rst index 98db2ea5fa12c..cc5cc7f3fbd8e 100644 --- a/Documentation/filesystems/api-summary.rst +++ b/Documentation/filesystems/api-summary.rst @@ -56,9 +56,6 @@ Other Functions .. kernel-doc:: fs/namei.c :export: -.. kernel-doc:: fs/buffer.c - :export: - .. kernel-doc:: block/bio.c :export: diff --git a/Documentation/filesystems/buffer.rst b/Documentation/filesystems/buffer.rst new file mode 100644 index 0000000000000..ae24faf68efab --- /dev/null +++ b/Documentation/filesystems/buffer.rst @@ -0,0 +1,12 @@ +Buffer Heads +============ + +Linux uses buffer heads to maintain state about individual filesystem blocks. +Buffer heads are deprecated and new filesystems should use iomap instead. + +Functions +--------- + +.. kernel-doc:: include/linux/buffer_head.h +.. kernel-doc:: fs/buffer.c + :export: diff --git a/Documentation/filesystems/index.rst b/Documentation/filesystems/index.rst index 1f9b4c905a6a7..8f5c1ee02e2f9 100644 --- a/Documentation/filesystems/index.rst +++ b/Documentation/filesystems/index.rst @@ -50,6 +50,7 @@ filesystem implementations. .. toctree:: :maxdepth: 2 + buffer journalling fscrypt fsverity From 122ff80e12b3aa586d702b7fb651a99d443e7777 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Tue, 16 Apr 2024 01:25:59 +0000 Subject: [PATCH 1174/1702] mm/sparse: guard the size of mem_section is power of 2 We usually have this check, while commit 2a3cb8baef71 ("mm/sparse: delete old sparse_init and enable new one") missed to take it. Link: https://lkml.kernel.org/r/20240416012559.4536-1-richard.weiyang@gmail.com Signed-off-by: Wei Yang Acked-by: Oscar Salvador Reviewed-by: Pasha Tatashin Cc: "Mike Rapoport (IBM)" Signed-off-by: Andrew Morton --- mm/sparse.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/sparse.c b/mm/sparse.c index 46e88549d1a6a..de40b2c734067 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -560,6 +560,8 @@ void __init sparse_init(void) unsigned long pnum_end, pnum_begin, map_count = 1; int nid_begin; + /* see include/linux/mmzone.h 'struct mem_section' definition */ + BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section))); memblocks_present(); pnum_begin = first_present_section_nr(); From 88e4e47c12836cf6875f73fca87baaf20c6ca1a2 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 17 Apr 2024 11:23:12 +0200 Subject: [PATCH 1175/1702] fs/proc/task_mmu: convert pagemap_hugetlb_range() to work on folios Patch series "fs/proc/task_mmu: convert hugetlb functions to work on folis". Let's convert two more functions, getting rid of two more page_mapcount() calls. This patch (of 2): Let's get rid of another page_mapcount() check and simply use folio_likely_mapped_shared(), which is precise for hugetlb folios. While at it, also check for PMD table sharing, like we do in smaps_hugetlb_range(). No functional change intended, except that we would now detect hugetlb folios shared via PMD table sharing correctly. Link: https://lkml.kernel.org/r/20240417092313.753919-1-david@redhat.com Link: https://lkml.kernel.org/r/20240417092313.753919-2-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Oscar Salvador Cc: Muchun Song Signed-off-by: Andrew Morton --- fs/proc/task_mmu.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 8ff79bd427ec6..cd6e45e0cde8e 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1578,12 +1578,13 @@ static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask, pte = huge_ptep_get(ptep); if (pte_present(pte)) { - struct page *page = pte_page(pte); + struct folio *folio = page_folio(pte_page(pte)); - if (!PageAnon(page)) + if (!folio_test_anon(folio)) flags |= PM_FILE; - if (page_mapcount(page) == 1) + if (!folio_likely_mapped_shared(folio) && + !hugetlb_pmd_shared(ptep)) flags |= PM_MMAP_EXCLUSIVE; if (huge_pte_uffd_wp(pte)) From 6401a2e6900843a77a27873c0529dea68f61193d Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 17 Apr 2024 11:23:13 +0200 Subject: [PATCH 1176/1702] fs/proc/task_mmu: convert smaps_hugetlb_range() to work on folios Let's get rid of another page_mapcount() check and simply use folio_likely_mapped_shared(), which is precise for hugetlb folios. While at it, use huge_ptep_get() + pte_page() instead of ptep_get() + vm_normal_page(), just like we do in pagemap_hugetlb_range(). No functional change intended. Link: https://lkml.kernel.org/r/20240417092313.753919-3-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Oscar Salvador Cc: Muchun Song Signed-off-by: Andrew Morton --- fs/proc/task_mmu.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index cd6e45e0cde8e..f4259b7edfded 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -730,19 +730,20 @@ static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask, { struct mem_size_stats *mss = walk->private; struct vm_area_struct *vma = walk->vma; - struct page *page = NULL; - pte_t ptent = ptep_get(pte); + pte_t ptent = huge_ptep_get(pte); + struct folio *folio = NULL; if (pte_present(ptent)) { - page = vm_normal_page(vma, addr, ptent); + folio = page_folio(pte_page(ptent)); } else if (is_swap_pte(ptent)) { swp_entry_t swpent = pte_to_swp_entry(ptent); if (is_pfn_swap_entry(swpent)) - page = pfn_swap_entry_to_page(swpent); + folio = pfn_swap_entry_folio(swpent); } - if (page) { - if (page_mapcount(page) >= 2 || hugetlb_pmd_shared(pte)) + if (folio) { + if (folio_likely_mapped_shared(folio) || + hugetlb_pmd_shared(pte)) mss->shared_hugetlb += huge_page_size(hstate_vma(vma)); else mss->private_hugetlb += huge_page_size(hstate_vma(vma)); From 3ccae1dc84086721c300a1026d2c38574e433e69 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 17 Apr 2024 17:18:36 -0400 Subject: [PATCH 1177/1702] mm/hugetlb: assert hugetlb_lock in __hugetlb_cgroup_commit_charge This is similar to __hugetlb_cgroup_uncharge_folio() where it relies on holding hugetlb_lock. Add the similar assertion like the other one, since it looks like such things may help some day. Link: https://lkml.kernel.org/r/20240417211836.2742593-4-peterx@redhat.com Signed-off-by: Peter Xu Reviewed-by: Mina Almasry Cc: David Hildenbrand Cc: David Rientjes Cc: Muchun Song Signed-off-by: Andrew Morton --- mm/hugetlb_cgroup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c index aa4486bd39049..e20339a346b93 100644 --- a/mm/hugetlb_cgroup.c +++ b/mm/hugetlb_cgroup.c @@ -308,7 +308,7 @@ static void __hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, { if (hugetlb_cgroup_disabled() || !h_cg) return; - + lockdep_assert_held(&hugetlb_lock); __set_hugetlb_cgroup(folio, h_cg, rsvd); if (!rsvd) { unsigned long usage = From 8430557fc584657559bfbd5150b6ae1bb90f35a0 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 17 Apr 2024 17:25:49 -0400 Subject: [PATCH 1178/1702] mm/page_table_check: support userfault wr-protect entries Allow page_table_check hooks to check over userfaultfd wr-protect criteria upon pgtable updates. The rule is no co-existance allowed for any writable flag against userfault wr-protect flag. This should be better than c2da319c2e, where we used to only sanitize such issues during a pgtable walk, but when hitting such issue we don't have a good chance to know where does that writable bit came from [1], so that even the pgtable walk exposes a kernel bug (which is still helpful on triaging) but not easy to track and debug. Now we switch to track the source. It's much easier too with the recent introduction of page table check. There are some limitations with using the page table check here for userfaultfd wr-protect purpose: - It is only enabled with explicit enablement of page table check configs and/or boot parameters, but should be good enough to track at least syzbot issues, as syzbot should enable PAGE_TABLE_CHECK[_ENFORCED] for x86 [1]. We used to have DEBUG_VM but it's now off for most distros, while distros also normally not enable PAGE_TABLE_CHECK[_ENFORCED], which is similar. - It conditionally works with the ptep_modify_prot API. It will be bypassed when e.g. XEN PV is enabled, however still work for most of the rest scenarios, which should be the common cases so should be good enough. - Hugetlb check is a bit hairy, as the page table check cannot identify hugetlb pte or normal pte via trapping at set_pte_at(), because of the current design where hugetlb maps every layers to pte_t... For example, the default set_huge_pte_at() can invoke set_pte_at() directly and lose the hugetlb context, treating it the same as a normal pte_t. So far it's fine because we have huge_pte_uffd_wp() always equals to pte_uffd_wp() as long as supported (x86 only). It'll be a bigger problem when we'll define _PAGE_UFFD_WP differently at various pgtable levels, because then one huge_pte_uffd_wp() per-arch will stop making sense first.. as of now we can leave this for later too. This patch also removes commit c2da319c2e altogether, as we have something better now. [1] https://lore.kernel.org/all/000000000000dce0530615c89210@google.com/ Link: https://lkml.kernel.org/r/20240417212549.2766883-1-peterx@redhat.com Signed-off-by: Peter Xu Reviewed-by: Pasha Tatashin Cc: Axel Rasmussen Cc: David Hildenbrand Cc: Nadav Amit Signed-off-by: Andrew Morton --- Documentation/mm/page_table_check.rst | 9 +++++++- arch/x86/include/asm/pgtable.h | 18 +--------------- mm/page_table_check.c | 30 +++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 18 deletions(-) diff --git a/Documentation/mm/page_table_check.rst b/Documentation/mm/page_table_check.rst index c12838ce6b8de..c59f22eb6a0f9 100644 --- a/Documentation/mm/page_table_check.rst +++ b/Documentation/mm/page_table_check.rst @@ -14,7 +14,7 @@ Page table check performs extra verifications at the time when new pages become accessible from the userspace by getting their page table entries (PTEs PMDs etc.) added into the table. -In case of detected corruption, the kernel is crashed. There is a small +In case of most detected corruption, the kernel is crashed. There is a small performance and memory overhead associated with the page table check. Therefore, it is disabled by default, but can be optionally enabled on systems where the extra hardening outweighs the performance costs. Also, because page table check @@ -22,6 +22,13 @@ is synchronous, it can help with debugging double map memory corruption issues, by crashing kernel at the time wrong mapping occurs instead of later which is often the case with memory corruptions bugs. +It can also be used to do page table entry checks over various flags, dump +warnings when illegal combinations of entry flags are detected. Currently, +userfaultfd is the only user of such to sanity check wr-protect bit against +any writable flags. Illegal flag combinations will not directly cause data +corruption in this case immediately, but that will cause read-only data to +be writable, leading to corrupt when the page content is later modified. + Double mapping detection logic ============================== diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 273f7557218cd..65b8e5bb902cc 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -388,23 +388,7 @@ static inline pte_t pte_wrprotect(pte_t pte) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP static inline int pte_uffd_wp(pte_t pte) { - bool wp = pte_flags(pte) & _PAGE_UFFD_WP; - -#ifdef CONFIG_DEBUG_VM - /* - * Having write bit for wr-protect-marked present ptes is fatal, - * because it means the uffd-wp bit will be ignored and write will - * just go through. - * - * Use any chance of pgtable walking to verify this (e.g., when - * page swapped out or being migrated for all purposes). It means - * something is already wrong. Tell the admin even before the - * process crashes. We also nail it with wrong pgtable setup. - */ - WARN_ON_ONCE(wp && pte_write(pte)); -#endif - - return wp; + return pte_flags(pte) & _PAGE_UFFD_WP; } static inline pte_t pte_mkuffd_wp(pte_t pte) diff --git a/mm/page_table_check.c b/mm/page_table_check.c index af69c3c8f7c2d..4169576bed729 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #undef pr_fmt #define pr_fmt(fmt) "page_table_check: " fmt @@ -182,6 +184,22 @@ void __page_table_check_pud_clear(struct mm_struct *mm, pud_t pud) } EXPORT_SYMBOL(__page_table_check_pud_clear); +/* Whether the swap entry cached writable information */ +static inline bool swap_cached_writable(swp_entry_t entry) +{ + return is_writable_device_exclusive_entry(entry) || + is_writable_device_private_entry(entry) || + is_writable_migration_entry(entry); +} + +static inline void page_table_check_pte_flags(pte_t pte) +{ + if (pte_present(pte) && pte_uffd_wp(pte)) + WARN_ON_ONCE(pte_write(pte)); + else if (is_swap_pte(pte) && pte_swp_uffd_wp(pte)) + WARN_ON_ONCE(swap_cached_writable(pte_to_swp_entry(pte))); +} + void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte, unsigned int nr) { @@ -190,6 +208,8 @@ void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte, if (&init_mm == mm) return; + page_table_check_pte_flags(pte); + for (i = 0; i < nr; i++) __page_table_check_pte_clear(mm, ptep_get(ptep + i)); if (pte_user_accessible_page(pte)) @@ -197,11 +217,21 @@ void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte, } EXPORT_SYMBOL(__page_table_check_ptes_set); +static inline void page_table_check_pmd_flags(pmd_t pmd) +{ + if (pmd_present(pmd) && pmd_uffd_wp(pmd)) + WARN_ON_ONCE(pmd_write(pmd)); + else if (is_swap_pmd(pmd) && pmd_swp_uffd_wp(pmd)) + WARN_ON_ONCE(swap_cached_writable(pmd_to_swp_entry(pmd))); +} + void __page_table_check_pmd_set(struct mm_struct *mm, pmd_t *pmdp, pmd_t pmd) { if (&init_mm == mm) return; + page_table_check_pmd_flags(pmd); + __page_table_check_pmd_clear(mm, *pmdp); if (pmd_user_accessible_page(pmd)) { page_table_check_set(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT, From d21f996b02a027f8915e493ee01370c4610ed2e2 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 18 Apr 2024 17:18:34 +0200 Subject: [PATCH 1179/1702] mm/huge_memory: improve split_huge_page_to_list_to_order() return value documentation The documentation is wrong and relying on it almost resulted in BUGs in new callers: ever since fd4a7ac32918 ("mm: migrate: try again if THP split is failed due to page refcnt") we return -EAGAIN on unexpected folio references, not -EBUSY. Let's fix that and also document which other return values we can currently see and why they could happen. [david@redhat.com: v2] Link: https://lkml.kernel.org/r/20240422194217.442933-1-david@redhat.com Link: https://lkml.kernel.org/r/20240418151834.216557-1-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Zi Yan Reviewed-by: John Hubbard Reviewed-by: Baolin Wang Cc: Matthew Wilcox Signed-off-by: Andrew Morton --- mm/huge_memory.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 31b6bbffea521..eabf088cb444b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2956,7 +2956,7 @@ bool can_split_folio(struct folio *folio, int *pextra_pins) * * 3) The folio must not be pinned. Any unexpected folio references, including * GUP pins, will result in the folio not getting split; instead, the caller - * will receive an -EBUSY. + * will receive an -EAGAIN. * * 4) @new_order > 1, usually. Splitting to order-1 anonymous folios is not * supported for non-file-backed folios, because folio->_deferred_list, which @@ -2975,8 +2975,16 @@ bool can_split_folio(struct folio *folio, int *pextra_pins) * * Returns 0 if the huge page was split successfully. * - * Returns -EBUSY if @page's folio is pinned, or if the anon_vma disappeared - * from under us. + * Returns -EAGAIN if the folio has unexpected reference (e.g., GUP) or if + * the folio was concurrently removed from the page cache. + * + * Returns -EBUSY when trying to split the huge zeropage, if the folio is + * under writeback, if fs-specific folio metadata cannot currently be + * released, or if some unexpected race happened (e.g., anon VMA disappeared, + * truncation). + * + * Returns -EINVAL when trying to split to an order that is incompatible + * with the folio. Splitting to order 0 is compatible with all folios. */ int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, unsigned int new_order) From 80e75021486bd2f6463259c4569e2eb7668ae953 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Thu, 18 Apr 2024 21:56:44 +0800 Subject: [PATCH 1180/1702] mm: swapfile: check usable swap device in __folio_throttle_swaprate() Skip blk_cgroup_congested() if there is no usable swap device since no swapin/out will occur, Thereby avoid taking swap_lock. The difference is shown below from perf date of CoW pagefault, perf report -g -i perf.data.swapon | egrep "blk_cgroup_congested|__folio_throttle_swaprate" 1.01% 0.16% page_fault2_pro [kernel.kallsyms] [k] __folio_throttle_swaprate 0.83% 0.80% page_fault2_pro [kernel.kallsyms] [k] blk_cgroup_congested perf report -g -i perf.data.swapoff | egrep "blk_cgroup_congested|__folio_throttle_swaprate" 0.15% 0.15% page_fault2_pro [kernel.kallsyms] [k] __folio_throttle_swaprate Link: https://lkml.kernel.org/r/20240418135644.2736748-1-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Cc: Tejun Heo Signed-off-by: Andrew Morton --- mm/swapfile.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/mm/swapfile.c b/mm/swapfile.c index 28642c188c930..96606580ee091 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2444,13 +2444,17 @@ static void reinsert_swap_info(struct swap_info_struct *p) spin_unlock(&swap_lock); } +static bool __has_usable_swap(void) +{ + return !plist_head_empty(&swap_active_head); +} + bool has_usable_swap(void) { - bool ret = true; + bool ret; spin_lock(&swap_lock); - if (plist_head_empty(&swap_active_head)) - ret = false; + ret = __has_usable_swap(); spin_unlock(&swap_lock); return ret; } @@ -3710,6 +3714,9 @@ void __folio_throttle_swaprate(struct folio *folio, gfp_t gfp) if (!(gfp & __GFP_IO)) return; + if (!__has_usable_swap()) + return; + if (!blk_cgroup_congested()) return; From 1b68112c40395b3b0fed3c8bb648e2d9d0b37ec2 Mon Sep 17 00:00:00 2001 From: Lance Yang Date: Thu, 18 Apr 2024 21:44:32 +0800 Subject: [PATCH 1181/1702] mm/madvise: introduce clear_young_dirty_ptes() batch helper Patch series "mm/madvise: enhance lazyfreeing with mTHP in madvise_free", v10. This patchset adds support for lazyfreeing multi-size THP (mTHP) without needing to first split the large folio via split_folio(). However, we still need to split a large folio that is not fully mapped within the target range. If a large folio is locked or shared, or if we fail to split it, we just leave it in place and advance to the next PTE in the range. But note that the behavior is changed; previously, any failure of this sort would cause the entire operation to give up. As large folios become more common, sticking to the old way could result in wasted opportunities. Performance Testing =================== On an Intel I5 CPU, lazyfreeing a 1GiB VMA backed by PTE-mapped folios of the same size results in the following runtimes for madvise(MADV_FREE) in seconds (shorter is better): Folio Size | Old | New | Change ------------------------------------------ 4KiB | 0.590251 | 0.590259 | 0% 16KiB | 2.990447 | 0.185655 | -94% 32KiB | 2.547831 | 0.104870 | -95% 64KiB | 2.457796 | 0.052812 | -97% 128KiB | 2.281034 | 0.032777 | -99% 256KiB | 2.230387 | 0.017496 | -99% 512KiB | 2.189106 | 0.010781 | -99% 1024KiB | 2.183949 | 0.007753 | -99% 2048KiB | 0.002799 | 0.002804 | 0% This patch (of 4): This commit introduces clear_young_dirty_ptes() to replace mkold_ptes(). By doing so, we can use the same function for both use cases (madvise_pageout and madvise_free), and it also provides the flexibility to only clear the dirty flag in the future if needed. Link: https://lkml.kernel.org/r/20240418134435.6092-1-ioworker0@gmail.com Link: https://lkml.kernel.org/r/20240418134435.6092-2-ioworker0@gmail.com Signed-off-by: Lance Yang Suggested-by: Ryan Roberts Acked-by: David Hildenbrand Reviewed-by: Ryan Roberts Cc: Barry Song <21cnbao@gmail.com> Cc: Jeff Xie Cc: Kefeng Wang Cc: Michal Hocko Cc: Minchan Kim Cc: Muchun Song Cc: Peter Xu Cc: Yang Shi Cc: Yin Fengwei Cc: Zach O'Keefe Signed-off-by: Andrew Morton --- include/linux/mm_types.h | 9 +++++ include/linux/pgtable.h | 74 ++++++++++++++++++++++++---------------- mm/madvise.c | 3 +- 3 files changed, 55 insertions(+), 31 deletions(-) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index db0adf5721ccf..24323c7d0bd48 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1368,6 +1368,15 @@ enum fault_flag { typedef unsigned int __bitwise zap_flags_t; +/* Flags for clear_young_dirty_ptes(). */ +typedef int __bitwise cydp_t; + +/* Clear the access bit */ +#define CYDP_CLEAR_YOUNG ((__force cydp_t)BIT(0)) + +/* Clear the dirty bit */ +#define CYDP_CLEAR_DIRTY ((__force cydp_t)BIT(1)) + /* * FOLL_PIN and FOLL_LONGTERM may be used in various combinations with each * other. Here is what they mean, and how to use them: diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index e2f45e22a6d13..18019f037baec 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -361,36 +361,6 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, } #endif -#ifndef mkold_ptes -/** - * mkold_ptes - Mark PTEs that map consecutive pages of the same folio as old. - * @vma: VMA the pages are mapped into. - * @addr: Address the first page is mapped at. - * @ptep: Page table pointer for the first entry. - * @nr: Number of entries to mark old. - * - * May be overridden by the architecture; otherwise, implemented as a simple - * loop over ptep_test_and_clear_young(). - * - * Note that PTE bits in the PTE range besides the PFN can differ. For example, - * some PTEs might be write-protected. - * - * Context: The caller holds the page table lock. The PTEs map consecutive - * pages that belong to the same folio. The PTEs are all in the same PMD. - */ -static inline void mkold_ptes(struct vm_area_struct *vma, unsigned long addr, - pte_t *ptep, unsigned int nr) -{ - for (;;) { - ptep_test_and_clear_young(vma, addr, ptep); - if (--nr == 0) - break; - ptep++; - addr += PAGE_SIZE; - } -} -#endif - #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, @@ -489,6 +459,50 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, } #endif +#ifndef clear_young_dirty_ptes +/** + * clear_young_dirty_ptes - Mark PTEs that map consecutive pages of the + * same folio as old/clean. + * @mm: Address space the pages are mapped into. + * @addr: Address the first page is mapped at. + * @ptep: Page table pointer for the first entry. + * @nr: Number of entries to mark old/clean. + * @flags: Flags to modify the PTE batch semantics. + * + * May be overridden by the architecture; otherwise, implemented by + * get_and_clear/modify/set for each pte in the range. + * + * Note that PTE bits in the PTE range besides the PFN can differ. For example, + * some PTEs might be write-protected. + * + * Context: The caller holds the page table lock. The PTEs map consecutive + * pages that belong to the same folio. The PTEs are all in the same PMD. + */ +static inline void clear_young_dirty_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + unsigned int nr, cydp_t flags) +{ + pte_t pte; + + for (;;) { + if (flags == CYDP_CLEAR_YOUNG) + ptep_test_and_clear_young(vma, addr, ptep); + else { + pte = ptep_get_and_clear(vma->vm_mm, addr, ptep); + if (flags & CYDP_CLEAR_YOUNG) + pte = pte_mkold(pte); + if (flags & CYDP_CLEAR_DIRTY) + pte = pte_mkclean(pte); + set_pte_at(vma->vm_mm, addr, ptep, pte); + } + if (--nr == 0) + break; + ptep++; + addr += PAGE_SIZE; + } +} +#endif + static inline void ptep_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { diff --git a/mm/madvise.c b/mm/madvise.c index f59169888b8ee..edb592adb7495 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -507,7 +507,8 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, continue; if (!pageout && pte_young(ptent)) { - mkold_ptes(vma, addr, pte, nr); + clear_young_dirty_ptes(vma, addr, pte, nr, + CYDP_CLEAR_YOUNG); tlb_remove_tlb_entries(tlb, pte, nr, addr); } From 89e86854fb0aa6e20c0f3d88285fa9cedef4f4e0 Mon Sep 17 00:00:00 2001 From: Lance Yang Date: Thu, 18 Apr 2024 21:44:33 +0800 Subject: [PATCH 1182/1702] mm/arm64: override clear_young_dirty_ptes() batch helper The per-pte get_and_clear/modify/set approach would result in unfolding/refolding for contpte mappings on arm64. So we need to override clear_young_dirty_ptes() for arm64 to avoid it. Link: https://lkml.kernel.org/r/20240418134435.6092-3-ioworker0@gmail.com Signed-off-by: Lance Yang Suggested-by: Barry Song <21cnbao@gmail.com> Suggested-by: Ryan Roberts Reviewed-by: Ryan Roberts Cc: David Hildenbrand Cc: Jeff Xie Cc: Kefeng Wang Cc: Michal Hocko Cc: Minchan Kim Cc: Muchun Song Cc: Peter Xu Cc: Yang Shi Cc: Yin Fengwei Cc: Zach O'Keefe Signed-off-by: Andrew Morton --- arch/arm64/include/asm/pgtable.h | 55 ++++++++++++++++++++++++++++++++ arch/arm64/mm/contpte.c | 29 +++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 9fd8613b2db2a..1303d30287dce 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -1223,6 +1223,46 @@ static inline void __wrprotect_ptes(struct mm_struct *mm, unsigned long address, __ptep_set_wrprotect(mm, address, ptep); } +static inline void __clear_young_dirty_pte(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t pte, cydp_t flags) +{ + pte_t old_pte; + + do { + old_pte = pte; + + if (flags & CYDP_CLEAR_YOUNG) + pte = pte_mkold(pte); + if (flags & CYDP_CLEAR_DIRTY) + pte = pte_mkclean(pte); + + pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep), + pte_val(old_pte), pte_val(pte)); + } while (pte_val(pte) != pte_val(old_pte)); +} + +static inline void __clear_young_dirty_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + unsigned int nr, cydp_t flags) +{ + pte_t pte; + + for (;;) { + pte = __ptep_get(ptep); + + if (flags == (CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY)) + __set_pte(ptep, pte_mkclean(pte_mkold(pte))); + else + __clear_young_dirty_pte(vma, addr, ptep, pte, flags); + + if (--nr == 0) + break; + ptep++; + addr += PAGE_SIZE; + } +} + #ifdef CONFIG_TRANSPARENT_HUGEPAGE #define __HAVE_ARCH_PMDP_SET_WRPROTECT static inline void pmdp_set_wrprotect(struct mm_struct *mm, @@ -1379,6 +1419,9 @@ extern void contpte_wrprotect_ptes(struct mm_struct *mm, unsigned long addr, extern int contpte_ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t entry, int dirty); +extern void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + unsigned int nr, cydp_t flags); static __always_inline void contpte_try_fold(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) @@ -1603,6 +1646,17 @@ static inline int ptep_set_access_flags(struct vm_area_struct *vma, return contpte_ptep_set_access_flags(vma, addr, ptep, entry, dirty); } +#define clear_young_dirty_ptes clear_young_dirty_ptes +static inline void clear_young_dirty_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + unsigned int nr, cydp_t flags) +{ + if (likely(nr == 1 && !pte_cont(__ptep_get(ptep)))) + __clear_young_dirty_ptes(vma, addr, ptep, nr, flags); + else + contpte_clear_young_dirty_ptes(vma, addr, ptep, nr, flags); +} + #else /* CONFIG_ARM64_CONTPTE */ #define ptep_get __ptep_get @@ -1622,6 +1676,7 @@ static inline int ptep_set_access_flags(struct vm_area_struct *vma, #define wrprotect_ptes __wrprotect_ptes #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS #define ptep_set_access_flags __ptep_set_access_flags +#define clear_young_dirty_ptes __clear_young_dirty_ptes #endif /* CONFIG_ARM64_CONTPTE */ diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index 1b64b4c3f8bf8..9f9486de0004d 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -361,6 +361,35 @@ void contpte_wrprotect_ptes(struct mm_struct *mm, unsigned long addr, } EXPORT_SYMBOL_GPL(contpte_wrprotect_ptes); +void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + unsigned int nr, cydp_t flags) +{ + /* + * We can safely clear access/dirty without needing to unfold from + * the architectures perspective, even when contpte is set. If the + * range starts or ends midway through a contpte block, we can just + * expand to include the full contpte block. While this is not + * exactly what the core-mm asked for, it tracks access/dirty per + * folio, not per page. And since we only create a contpte block + * when it is covered by a single folio, we can get away with + * clearing access/dirty for the whole block. + */ + unsigned long start = addr; + unsigned long end = start + nr; + + if (pte_cont(__ptep_get(ptep + nr - 1))) + end = ALIGN(end, CONT_PTE_SIZE); + + if (pte_cont(__ptep_get(ptep))) { + start = ALIGN_DOWN(start, CONT_PTE_SIZE); + ptep = contpte_align_down(ptep); + } + + __clear_young_dirty_ptes(vma, start, ptep, end - start, flags); +} +EXPORT_SYMBOL_GPL(contpte_clear_young_dirty_ptes); + int contpte_ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t entry, int dirty) From 96ebdb032096f67e37b582cd2ea2558c402f878b Mon Sep 17 00:00:00 2001 From: Lance Yang Date: Thu, 18 Apr 2024 21:44:34 +0800 Subject: [PATCH 1183/1702] mm/memory: add any_dirty optional pointer to folio_pte_batch() This commit adds the any_dirty pointer as an optional parameter to folio_pte_batch() function. By using both the any_young and any_dirty pointers, madvise_free can make smarter decisions about whether to clear the PTEs when marking large folios as lazyfree. Link: https://lkml.kernel.org/r/20240418134435.6092-4-ioworker0@gmail.com Signed-off-by: Lance Yang Suggested-by: David Hildenbrand Acked-by: David Hildenbrand Cc: Barry Song <21cnbao@gmail.com> Cc: Jeff Xie Cc: Kefeng Wang Cc: Michal Hocko Cc: Minchan Kim Cc: Muchun Song Cc: Peter Xu Cc: Ryan Roberts Cc: Yang Shi Cc: Yin Fengwei Cc: Zach O'Keefe Signed-off-by: Andrew Morton --- mm/internal.h | 12 ++++++++++-- mm/madvise.c | 19 ++++++++++++++----- mm/memory.c | 4 ++-- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 2adc3f616b71f..5d5e49b86fe32 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -134,6 +134,8 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) * first one is writable. * @any_young: Optional pointer to indicate whether any entry except the * first one is young. + * @any_dirty: Optional pointer to indicate whether any entry except the + * first one is dirty. * * Detect a PTE batch: consecutive (present) PTEs that map consecutive * pages of the same large folio. @@ -149,18 +151,20 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) */ static inline int folio_pte_batch(struct folio *folio, unsigned long addr, pte_t *start_ptep, pte_t pte, int max_nr, fpb_t flags, - bool *any_writable, bool *any_young) + bool *any_writable, bool *any_young, bool *any_dirty) { unsigned long folio_end_pfn = folio_pfn(folio) + folio_nr_pages(folio); const pte_t *end_ptep = start_ptep + max_nr; pte_t expected_pte, *ptep; - bool writable, young; + bool writable, young, dirty; int nr; if (any_writable) *any_writable = false; if (any_young) *any_young = false; + if (any_dirty) + *any_dirty = false; VM_WARN_ON_FOLIO(!pte_present(pte), folio); VM_WARN_ON_FOLIO(!folio_test_large(folio) || max_nr < 1, folio); @@ -176,6 +180,8 @@ static inline int folio_pte_batch(struct folio *folio, unsigned long addr, writable = !!pte_write(pte); if (any_young) young = !!pte_young(pte); + if (any_dirty) + dirty = !!pte_dirty(pte); pte = __pte_batch_clear_ignored(pte, flags); if (!pte_same(pte, expected_pte)) @@ -193,6 +199,8 @@ static inline int folio_pte_batch(struct folio *folio, unsigned long addr, *any_writable |= writable; if (any_young) *any_young |= young; + if (any_dirty) + *any_dirty |= dirty; nr = pte_batch_hint(ptep, pte); expected_pte = pte_advance_pfn(expected_pte, nr); diff --git a/mm/madvise.c b/mm/madvise.c index edb592adb7495..5b5ba394992fd 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -321,6 +321,18 @@ static inline bool can_do_file_pageout(struct vm_area_struct *vma) file_permission(vma->vm_file, MAY_WRITE) == 0; } +static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end, + struct folio *folio, pte_t *ptep, + pte_t pte, bool *any_young, + bool *any_dirty) +{ + const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY; + int max_nr = (end - addr) / PAGE_SIZE; + + return folio_pte_batch(folio, addr, ptep, pte, max_nr, fpb_flags, NULL, + any_young, any_dirty); +} + static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct mm_walk *walk) @@ -456,13 +468,10 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, * next pte in the range. */ if (folio_test_large(folio)) { - const fpb_t fpb_flags = FPB_IGNORE_DIRTY | - FPB_IGNORE_SOFT_DIRTY; - int max_nr = (end - addr) / PAGE_SIZE; bool any_young; - nr = folio_pte_batch(folio, addr, pte, ptent, max_nr, - fpb_flags, NULL, &any_young); + nr = madvise_folio_pte_batch(addr, end, folio, pte, + ptent, &any_young, NULL); if (any_young) ptent = pte_mkyoung(ptent); diff --git a/mm/memory.c b/mm/memory.c index 33d87b64d15db..9e07d1b9020cb 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -989,7 +989,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma flags |= FPB_IGNORE_SOFT_DIRTY; nr = folio_pte_batch(folio, addr, src_pte, pte, max_nr, flags, - &any_writable, NULL); + &any_writable, NULL, NULL); folio_ref_add(folio, nr); if (folio_test_anon(folio)) { if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page, @@ -1558,7 +1558,7 @@ static inline int zap_present_ptes(struct mmu_gather *tlb, */ if (unlikely(folio_test_large(folio) && max_nr != 1)) { nr = folio_pte_batch(folio, addr, pte, ptent, max_nr, fpb_flags, - NULL, NULL); + NULL, NULL, NULL); zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr, addr, details, rss, force_flush, From dce7d10be4bbd31412c4bedd3a8bb2d25b96e025 Mon Sep 17 00:00:00 2001 From: Lance Yang Date: Thu, 18 Apr 2024 21:44:35 +0800 Subject: [PATCH 1184/1702] mm/madvise: optimize lazyfreeing with mTHP in madvise_free This patch optimizes lazyfreeing with PTE-mapped mTHP[1] (Inspired by David Hildenbrand[2]). We aim to avoid unnecessary folio splitting if the large folio is fully mapped within the target range. If a large folio is locked or shared, or if we fail to split it, we just leave it in place and advance to the next PTE in the range. But note that the behavior is changed; previously, any failure of this sort would cause the entire operation to give up. As large folios become more common, sticking to the old way could result in wasted opportunities. On an Intel I5 CPU, lazyfreeing a 1GiB VMA backed by PTE-mapped folios of the same size results in the following runtimes for madvise(MADV_FREE) in seconds (shorter is better): Folio Size | Old | New | Change ------------------------------------------ 4KiB | 0.590251 | 0.590259 | 0% 16KiB | 2.990447 | 0.185655 | -94% 32KiB | 2.547831 | 0.104870 | -95% 64KiB | 2.457796 | 0.052812 | -97% 128KiB | 2.281034 | 0.032777 | -99% 256KiB | 2.230387 | 0.017496 | -99% 512KiB | 2.189106 | 0.010781 | -99% 1024KiB | 2.183949 | 0.007753 | -99% 2048KiB | 0.002799 | 0.002804 | 0% [1] https://lkml.kernel.org/r/20231207161211.2374093-5-ryan.roberts@arm.com [2] https://lore.kernel.org/linux-mm/20240214204435.167852-1-david@redhat.com Link: https://lkml.kernel.org/r/20240418134435.6092-5-ioworker0@gmail.com Signed-off-by: Lance Yang Reviewed-by: Ryan Roberts Acked-by: David Hildenbrand Cc: Barry Song <21cnbao@gmail.com> Cc: Jeff Xie Cc: Kefeng Wang Cc: Michal Hocko Cc: Minchan Kim Cc: Muchun Song Cc: Peter Xu Cc: Yang Shi Cc: Yin Fengwei Cc: Zach O'Keefe Signed-off-by: Andrew Morton --- mm/madvise.c | 85 +++++++++++++++++++++++++++------------------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/mm/madvise.c b/mm/madvise.c index 5b5ba394992fd..56efea02e26c3 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -643,6 +643,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct mm_walk *walk) { + const cydp_t cydp_flags = CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY; struct mmu_gather *tlb = walk->private; struct mm_struct *mm = tlb->mm; struct vm_area_struct *vma = walk->vma; @@ -697,44 +698,57 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, continue; /* - * If pmd isn't transhuge but the folio is large and - * is owned by only this process, split it and - * deactivate all pages. + * If we encounter a large folio, only split it if it is not + * fully mapped within the range we are operating on. Otherwise + * leave it as is so that it can be marked as lazyfree. If we + * fail to split a folio, leave it in place and advance to the + * next pte in the range. */ if (folio_test_large(folio)) { - int err; + bool any_young, any_dirty; - if (folio_likely_mapped_shared(folio)) - break; - if (!folio_trylock(folio)) - break; - folio_get(folio); - arch_leave_lazy_mmu_mode(); - pte_unmap_unlock(start_pte, ptl); - start_pte = NULL; - err = split_folio(folio); - folio_unlock(folio); - folio_put(folio); - if (err) - break; - start_pte = pte = - pte_offset_map_lock(mm, pmd, addr, &ptl); - if (!start_pte) - break; - arch_enter_lazy_mmu_mode(); - pte--; - addr -= PAGE_SIZE; - continue; + nr = madvise_folio_pte_batch(addr, end, folio, pte, + ptent, &any_young, &any_dirty); + + if (nr < folio_nr_pages(folio)) { + int err; + + if (folio_likely_mapped_shared(folio)) + continue; + if (!folio_trylock(folio)) + continue; + folio_get(folio); + arch_leave_lazy_mmu_mode(); + pte_unmap_unlock(start_pte, ptl); + start_pte = NULL; + err = split_folio(folio); + folio_unlock(folio); + folio_put(folio); + pte = pte_offset_map_lock(mm, pmd, addr, &ptl); + start_pte = pte; + if (!start_pte) + break; + arch_enter_lazy_mmu_mode(); + if (!err) + nr = 0; + continue; + } + + if (any_young) + ptent = pte_mkyoung(ptent); + if (any_dirty) + ptent = pte_mkdirty(ptent); } if (folio_test_swapcache(folio) || folio_test_dirty(folio)) { if (!folio_trylock(folio)) continue; /* - * If folio is shared with others, we mustn't clear - * the folio's dirty flag. + * If we have a large folio at this point, we know it is + * fully mapped so if its mapcount is the same as its + * number of pages, it must be exclusive. */ - if (folio_mapcount(folio) != 1) { + if (folio_mapcount(folio) != folio_nr_pages(folio)) { folio_unlock(folio); continue; } @@ -750,19 +764,8 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, } if (pte_young(ptent) || pte_dirty(ptent)) { - /* - * Some of architecture(ex, PPC) don't update TLB - * with set_pte_at and tlb_remove_tlb_entry so for - * the portability, remap the pte with old|clean - * after pte clearing. - */ - ptent = ptep_get_and_clear_full(mm, addr, pte, - tlb->fullmm); - - ptent = pte_mkold(ptent); - ptent = pte_mkclean(ptent); - set_pte_at(mm, addr, pte, ptent); - tlb_remove_tlb_entry(tlb, pte, addr); + clear_young_dirty_ptes(vma, addr, pte, nr, cydp_flags); + tlb_remove_tlb_entries(tlb, pte, nr, addr); } folio_mark_lazyfree(folio); } From 2d8b272cdcad17d9b875c206fbcb0422b791ab3a Mon Sep 17 00:00:00 2001 From: Hao Ge Date: Mon, 22 Apr 2024 11:27:25 +0800 Subject: [PATCH 1185/1702] mm/page-flags: make PageUptodate return bool Make PageUptodate return bool to align the return values of folio_test_uptodate function Link: https://lkml.kernel.org/r/20240422032725.41452-1-gehao@kylinos.cn Signed-off-by: Hao Ge Cc: David Hildenbrand Cc: Josef Bacik Cc: Matthew Wilcox (Oracle) Cc: Ruihan Li Cc: Vishal Moola (Oracle) Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index f83331684e470..0af2cef02383b 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -789,7 +789,7 @@ static inline bool folio_test_uptodate(const struct folio *folio) return ret; } -static inline int PageUptodate(const struct page *page) +static inline bool PageUptodate(const struct page *page) { return folio_test_uptodate(page_folio(page)); } From 6ed31ba3921162ef5d4f2f99b91681e8fd24ff34 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Mon, 22 Apr 2024 11:00:39 +0800 Subject: [PATCH 1186/1702] mm: memory: check userfaultfd_wp() in vmf_orig_pte_uffd_wp() Add userfaultfd_wp() check in vmf_orig_pte_uffd_wp() to avoid the unnecessary FAULT_FLAG_ORIG_PTE_VALID check/pte_marker_entry_uffd_wp() in most pagefault, note, the function vmf_orig_pte_uffd_wp() is not inlined in the two kernel versions, the difference is shown below, perf date, perf report -i perf.data.before | grep vmf 0.17% 0.13% lat_pagefault [kernel.kallsyms] [k] vmf_orig_pte_uffd_wp.part.0.isra.0 perf report -i perf.data.after | grep vmf lat_pagefault -W 5 -N 5 /tmp/XXX latency before after diff average(8 tests) 0.262675 0.2600375 -0.0026375 Although it's a small, but the uffd_wp is a new feature than previous kernel, when the vma is not registered with UFFD_WP, let's avoid to execute the new logical, also adding __always_inline attribute to vmf_orig_pte_uffd_wp(), which make set_pte_range() only check VM_UFFD_WP flags without the function call. In addition, directly call the vmf_orig_pte_uffd_wp() in do_anonymous_page() and set_pte_range() to save an uffd_wp variable. Link: https://lkml.kernel.org/r/20240422030039.3293568-1-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviewed-by: Peter Xu Signed-off-by: Andrew Morton --- mm/memory.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index 9e07d1b9020cb..c0bd4e0d5e7a7 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -112,8 +112,10 @@ static bool vmf_pte_changed(struct vm_fault *vmf); * Return true if the original pte was a uffd-wp pte marker (so the pte was * wr-protected). */ -static bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf) +static __always_inline bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf) { + if (!userfaultfd_wp(vmf->vma)) + return false; if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID)) return false; @@ -4393,7 +4395,6 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf) */ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) { - bool uffd_wp = vmf_orig_pte_uffd_wp(vmf); struct vm_area_struct *vma = vmf->vma; unsigned long addr = vmf->address; struct folio *folio; @@ -4493,7 +4494,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) folio_add_new_anon_rmap(folio, vma, addr); folio_add_lru_vma(folio, vma); setpte: - if (uffd_wp) + if (vmf_orig_pte_uffd_wp(vmf)) entry = pte_mkuffd_wp(entry); set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr_pages); @@ -4668,7 +4669,6 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio, struct page *page, unsigned int nr, unsigned long addr) { struct vm_area_struct *vma = vmf->vma; - bool uffd_wp = vmf_orig_pte_uffd_wp(vmf); bool write = vmf->flags & FAULT_FLAG_WRITE; bool prefault = in_range(vmf->address, addr, nr * PAGE_SIZE); pte_t entry; @@ -4683,7 +4683,7 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio, if (write) entry = maybe_mkwrite(pte_mkdirty(entry), vma); - if (unlikely(uffd_wp)) + if (unlikely(vmf_orig_pte_uffd_wp(vmf))) entry = pte_mkuffd_wp(entry); /* copy-on-write page */ if (write && !(vma->vm_flags & VM_SHARED)) { From 91882c1617c15eb2a4e996b0822e0b9cbd854dea Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Sat, 20 Apr 2024 16:25:05 -0700 Subject: [PATCH 1187/1702] memcg: simple cleanup of stats update functions mod_memcg_lruvec_state() is never called from outside of memcontrol.c and with always irq disabled. So, replace it with the irq disabled version and add an assert that irq is disabled in the caller. Similarly mod_objcg_state() is not called from outside of memcontrol.c, so simply make it static and change it's name to __mod_objcg_state(). Link: https://lkml.kernel.org/r/20240420232505.2768428-1-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Acked-by: Johannes Weiner Reviewed-by: T.J. Mercier Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 17 ----------------- mm/memcontrol.c | 31 +++++++++++++++---------------- mm/slab.h | 2 -- 3 files changed, 15 insertions(+), 35 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 8f332b4ae84ca..9aba0d0462ca6 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1077,8 +1077,6 @@ static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec, void mem_cgroup_flush_stats(struct mem_cgroup *memcg); void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg); -void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, - int val); void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val); static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx, @@ -1091,16 +1089,6 @@ static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx, local_irq_restore(flags); } -static inline void mod_memcg_lruvec_state(struct lruvec *lruvec, - enum node_stat_item idx, int val) -{ - unsigned long flags; - - local_irq_save(flags); - __mod_memcg_lruvec_state(lruvec, idx, val); - local_irq_restore(flags); -} - void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, unsigned long count); @@ -1594,11 +1582,6 @@ static inline void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg) { } -static inline void __mod_memcg_lruvec_state(struct lruvec *lruvec, - enum node_stat_item idx, int val) -{ -} - static inline void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val) { diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 29fa5b17a1ef5..9095ab05d47a8 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -836,8 +836,9 @@ static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx) return x; } -void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, - int val) +static void __mod_memcg_lruvec_state(struct lruvec *lruvec, + enum node_stat_item idx, + int val) { struct mem_cgroup_per_node *pn; struct mem_cgroup *memcg; @@ -2978,21 +2979,19 @@ void mem_cgroup_commit_charge(struct folio *folio, struct mem_cgroup *memcg) #ifdef CONFIG_MEMCG_KMEM -/* - * mod_objcg_mlstate() may be called with irq enabled, so - * mod_memcg_lruvec_state() should be used. - */ -static inline void mod_objcg_mlstate(struct obj_cgroup *objcg, - struct pglist_data *pgdat, - enum node_stat_item idx, int nr) +static inline void __mod_objcg_mlstate(struct obj_cgroup *objcg, + struct pglist_data *pgdat, + enum node_stat_item idx, int nr) { struct mem_cgroup *memcg; struct lruvec *lruvec; + lockdep_assert_irqs_disabled(); + rcu_read_lock(); memcg = obj_cgroup_memcg(objcg); lruvec = mem_cgroup_lruvec(memcg, pgdat); - mod_memcg_lruvec_state(lruvec, idx, nr); + __mod_memcg_lruvec_state(lruvec, idx, nr); rcu_read_unlock(); } @@ -3312,7 +3311,7 @@ void __memcg_kmem_uncharge_page(struct page *page, int order) obj_cgroup_put(objcg); } -void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, +static void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, enum node_stat_item idx, int nr) { struct memcg_stock_pcp *stock; @@ -3340,12 +3339,12 @@ void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, struct pglist_data *oldpg = stock->cached_pgdat; if (stock->nr_slab_reclaimable_b) { - mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B, + __mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B, stock->nr_slab_reclaimable_b); stock->nr_slab_reclaimable_b = 0; } if (stock->nr_slab_unreclaimable_b) { - mod_objcg_mlstate(objcg, oldpg, NR_SLAB_UNRECLAIMABLE_B, + __mod_objcg_mlstate(objcg, oldpg, NR_SLAB_UNRECLAIMABLE_B, stock->nr_slab_unreclaimable_b); stock->nr_slab_unreclaimable_b = 0; } @@ -3371,7 +3370,7 @@ void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, } } if (nr) - mod_objcg_mlstate(objcg, pgdat, idx, nr); + __mod_objcg_mlstate(objcg, pgdat, idx, nr); local_unlock_irqrestore(&memcg_stock.stock_lock, flags); obj_cgroup_put(old); @@ -3437,13 +3436,13 @@ static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock) */ if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) { if (stock->nr_slab_reclaimable_b) { - mod_objcg_mlstate(old, stock->cached_pgdat, + __mod_objcg_mlstate(old, stock->cached_pgdat, NR_SLAB_RECLAIMABLE_B, stock->nr_slab_reclaimable_b); stock->nr_slab_reclaimable_b = 0; } if (stock->nr_slab_unreclaimable_b) { - mod_objcg_mlstate(old, stock->cached_pgdat, + __mod_objcg_mlstate(old, stock->cached_pgdat, NR_SLAB_UNRECLAIMABLE_B, stock->nr_slab_unreclaimable_b); stock->nr_slab_unreclaimable_b = 0; diff --git a/mm/slab.h b/mm/slab.h index 411251b9bdd15..d12fb4392e353 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -581,8 +581,6 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, gfp_t flags, size_t size, void **p); void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, int objects, struct slabobj_ext *obj_exts); -void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, - enum node_stat_item idx, int nr); #endif size_t __ksize(const void *objp); From ccde70f4d4baea4d2ebc3d646a0a4ab4cfdf683c Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 23 Apr 2024 17:20:24 +0300 Subject: [PATCH 1188/1702] xarray: use BITS_PER_LONGS() Patch series "xarray: Clean up xarray.h". Main portion of this change is to get rid of kernel.h included into other globally available headers. This decreases a dependency hell degree. The first patch makes it possible to avoid math.h to be included as bitops.h is implied by bitmap.h. This patch (of 2): Use BITS_PER_LONGS() instead of open coded variant. Link: https://lkml.kernel.org/r/20240423142204.2408923-2-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Matthew Wilcox (Oracle) Cc: Rasmus Villemoes Signed-off-by: Andrew Morton --- include/linux/xarray.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/xarray.h b/include/linux/xarray.h index d9d479334c9e6..d54a1e98b0ca7 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -1146,7 +1146,7 @@ static inline void xa_release(struct xarray *xa, unsigned long index) #define XA_CHUNK_SIZE (1UL << XA_CHUNK_SHIFT) #define XA_CHUNK_MASK (XA_CHUNK_SIZE - 1) #define XA_MAX_MARKS 3 -#define XA_MARK_LONGS DIV_ROUND_UP(XA_CHUNK_SIZE, BITS_PER_LONG) +#define XA_MARK_LONGS BITS_TO_LONGS(XA_CHUNK_SIZE) /* * @count is the count of every non-NULL element in the ->slots array From d0aea4dcd23ca8e3e28b54d81156fdbe017fd595 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 23 Apr 2024 17:20:25 +0300 Subject: [PATCH 1189/1702] xarray: don't use "proxy" headers Update header inclusions to follow IWYU (Include What You Use) principle. Link: https://lkml.kernel.org/r/20240423142204.2408923-3-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Matthew Wilcox (Oracle) Cc: Rasmus Villemoes Signed-off-by: Andrew Morton --- include/linux/xarray.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/linux/xarray.h b/include/linux/xarray.h index d54a1e98b0ca7..d2e4d36246890 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -12,14 +12,18 @@ #include #include #include +#include #include #include -#include +#include +#include #include #include #include #include +struct list_lru; + /* * The bottom two bits of the entry determine how the XArray interprets * the contents: From 1c0501e8315c0713c3fbc7a2df7fbbf151fb214b Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 12 Apr 2024 20:34:58 +0100 Subject: [PATCH 1190/1702] mm/memory-failure: remove fsdax_pgoff argument from __add_to_kill Patch series "Some cleanups for memory-failure", v3. A lot of folio conversions, plus some other simplifications. This patch (of 11): Unify the KSM and DAX codepaths by calculating the addr in add_to_kill_fsdax() instead of telling __add_to_kill() to calculate it. Link: https://lkml.kernel.org/r/20240412193510.2356957-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240412193510.2356957-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Acked-by: Miaohe Lin Reviewed-by: Jane Chu Reviewed-by: Dan Williams Reviewed-by: Oscar Salvador Signed-off-by: Andrew Morton --- mm/memory-failure.c | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 68e1fe1c0b724..396f939c0e1fe 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -427,21 +427,13 @@ static unsigned long dev_pagemap_mapping_shift(struct vm_area_struct *vma, * not much we can do. We just print a message and ignore otherwise. */ -#define FSDAX_INVALID_PGOFF ULONG_MAX - /* * Schedule a process for later kill. * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM. - * - * Note: @fsdax_pgoff is used only when @p is a fsdax page and a - * filesystem with a memory failure handler has claimed the - * memory_failure event. In all other cases, page->index and - * page->mapping are sufficient for mapping the page back to its - * corresponding user virtual address. */ static void __add_to_kill(struct task_struct *tsk, struct page *p, struct vm_area_struct *vma, struct list_head *to_kill, - unsigned long ksm_addr, pgoff_t fsdax_pgoff) + unsigned long addr) { struct to_kill *tk; @@ -451,12 +443,10 @@ static void __add_to_kill(struct task_struct *tsk, struct page *p, return; } - tk->addr = ksm_addr ? ksm_addr : page_address_in_vma(p, vma); - if (is_zone_device_page(p)) { - if (fsdax_pgoff != FSDAX_INVALID_PGOFF) - tk->addr = vma_address(vma, fsdax_pgoff, 1); + tk->addr = addr ? addr : page_address_in_vma(p, vma); + if (is_zone_device_page(p)) tk->size_shift = dev_pagemap_mapping_shift(vma, tk->addr); - } else + else tk->size_shift = page_shift(compound_head(p)); /* @@ -486,7 +476,7 @@ static void add_to_kill_anon_file(struct task_struct *tsk, struct page *p, struct vm_area_struct *vma, struct list_head *to_kill) { - __add_to_kill(tsk, p, vma, to_kill, 0, FSDAX_INVALID_PGOFF); + __add_to_kill(tsk, p, vma, to_kill, 0); } #ifdef CONFIG_KSM @@ -504,10 +494,10 @@ static bool task_in_to_kill_list(struct list_head *to_kill, } void add_to_kill_ksm(struct task_struct *tsk, struct page *p, struct vm_area_struct *vma, struct list_head *to_kill, - unsigned long ksm_addr) + unsigned long addr) { if (!task_in_to_kill_list(to_kill, tsk)) - __add_to_kill(tsk, p, vma, to_kill, ksm_addr, FSDAX_INVALID_PGOFF); + __add_to_kill(tsk, p, vma, to_kill, addr); } #endif /* @@ -681,7 +671,8 @@ static void add_to_kill_fsdax(struct task_struct *tsk, struct page *p, struct vm_area_struct *vma, struct list_head *to_kill, pgoff_t pgoff) { - __add_to_kill(tsk, p, vma, to_kill, 0, pgoff); + unsigned long addr = vma_address(vma, pgoff, 1); + __add_to_kill(tsk, p, vma, to_kill, addr); } /* From f2b37197c267206979213592923b418039d232dd Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 12 Apr 2024 20:34:59 +0100 Subject: [PATCH 1191/1702] mm/memory-failure: pass addr to __add_to_kill() Handle anon/file folios the same way as KSM & DAX folios by passing in the address. Link: https://lkml.kernel.org/r/20240412193510.2356957-3-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Acked-by: Miaohe Lin Reviewed-by: Jane Chu Reviewed-by: Oscar Salvador Cc: Dan Williams Signed-off-by: Andrew Morton --- mm/memory-failure.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 396f939c0e1fe..9e1a7d8ca7454 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -443,7 +443,7 @@ static void __add_to_kill(struct task_struct *tsk, struct page *p, return; } - tk->addr = addr ? addr : page_address_in_vma(p, vma); + tk->addr = addr; if (is_zone_device_page(p)) tk->size_shift = dev_pagemap_mapping_shift(vma, tk->addr); else @@ -476,7 +476,8 @@ static void add_to_kill_anon_file(struct task_struct *tsk, struct page *p, struct vm_area_struct *vma, struct list_head *to_kill) { - __add_to_kill(tsk, p, vma, to_kill, 0); + unsigned long addr = page_address_in_vma(p, vma); + __add_to_kill(tsk, p, vma, to_kill, addr); } #ifdef CONFIG_KSM @@ -492,6 +493,7 @@ static bool task_in_to_kill_list(struct list_head *to_kill, return false; } + void add_to_kill_ksm(struct task_struct *tsk, struct page *p, struct vm_area_struct *vma, struct list_head *to_kill, unsigned long addr) From 37bc2ff506b184411e4cc80f111c638b2b4c83d4 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 12 Apr 2024 20:35:00 +0100 Subject: [PATCH 1192/1702] mm: return the address from page_mapped_in_vma() The only user of this function calls page_address_in_vma() immediately after page_mapped_in_vma() calculates it and uses it to return true/false. Return the address instead, allowing memory-failure to skip the call to page_address_in_vma(). Link: https://lkml.kernel.org/r/20240412193510.2356957-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Acked-by: Miaohe Lin Reviewed-by: Jane Chu Cc: Dan Williams Cc: Oscar Salvador Signed-off-by: Andrew Morton --- include/linux/rmap.h | 2 +- mm/memory-failure.c | 22 +++++++++++++--------- mm/page_vma_mapped.c | 16 +++++++++------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 0f906dc6d2800..7229b9baf20d8 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -730,7 +730,7 @@ int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff, void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked); -int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma); +unsigned long page_mapped_in_vma(struct page *page, struct vm_area_struct *vma); /* * rmap_walk_control: To control rmap traversing for specific needs diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 9e1a7d8ca7454..12e5d2844cb15 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -473,10 +473,11 @@ static void __add_to_kill(struct task_struct *tsk, struct page *p, } static void add_to_kill_anon_file(struct task_struct *tsk, struct page *p, - struct vm_area_struct *vma, - struct list_head *to_kill) + struct vm_area_struct *vma, struct list_head *to_kill, + unsigned long addr) { - unsigned long addr = page_address_in_vma(p, vma); + if (addr == -EFAULT) + return; __add_to_kill(tsk, p, vma, to_kill, addr); } @@ -601,7 +602,6 @@ struct task_struct *task_early_kill(struct task_struct *tsk, int force_early) static void collect_procs_anon(struct folio *folio, struct page *page, struct list_head *to_kill, int force_early) { - struct vm_area_struct *vma; struct task_struct *tsk; struct anon_vma *av; pgoff_t pgoff; @@ -613,8 +613,10 @@ static void collect_procs_anon(struct folio *folio, struct page *page, pgoff = page_to_pgoff(page); rcu_read_lock(); for_each_process(tsk) { + struct vm_area_struct *vma; struct anon_vma_chain *vmac; struct task_struct *t = task_early_kill(tsk, force_early); + unsigned long addr; if (!t) continue; @@ -623,9 +625,8 @@ static void collect_procs_anon(struct folio *folio, struct page *page, vma = vmac->vma; if (vma->vm_mm != t->mm) continue; - if (!page_mapped_in_vma(page, vma)) - continue; - add_to_kill_anon_file(t, page, vma, to_kill); + addr = page_mapped_in_vma(page, vma); + add_to_kill_anon_file(t, page, vma, to_kill, addr); } } rcu_read_unlock(); @@ -648,6 +649,7 @@ static void collect_procs_file(struct folio *folio, struct page *page, pgoff = page_to_pgoff(page); for_each_process(tsk) { struct task_struct *t = task_early_kill(tsk, force_early); + unsigned long addr; if (!t) continue; @@ -660,8 +662,10 @@ static void collect_procs_file(struct folio *folio, struct page *page, * Assume applications who requested early kill want * to be informed of all such data corruptions. */ - if (vma->vm_mm == t->mm) - add_to_kill_anon_file(t, page, vma, to_kill); + if (vma->vm_mm != t->mm) + continue; + addr = page_address_in_vma(page, vma); + add_to_kill_anon_file(t, page, vma, to_kill, addr); } } rcu_read_unlock(); diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index 53b8868ede61c..c202eab84936d 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -319,11 +319,12 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) * @page: the page to test * @vma: the VMA to test * - * Returns 1 if the page is mapped into the page tables of the VMA, 0 - * if the page is not mapped into the page tables of this VMA. Only - * valid for normal file or anonymous VMAs. + * Return: The address the page is mapped at if the page is in the range + * covered by the VMA and present in the page table. If the page is + * outside the VMA or not present, returns -EFAULT. + * Only valid for normal file or anonymous VMAs. */ -int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) +unsigned long page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) { struct folio *folio = page_folio(page); pgoff_t pgoff = folio->index + folio_page_idx(folio, page); @@ -336,9 +337,10 @@ int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) pvmw.address = vma_address(vma, pgoff, 1); if (pvmw.address == -EFAULT) - return 0; + goto out; if (!page_vma_mapped_walk(&pvmw)) - return 0; + return -EFAULT; page_vma_mapped_walk_done(&pvmw); - return 1; +out: + return pvmw.address; } From b87f978dc77519943b659dc8b753f544772e7c85 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 12 Apr 2024 20:35:01 +0100 Subject: [PATCH 1193/1702] mm: make page_mapped_in_vma conditional on CONFIG_MEMORY_FAILURE This function is only currently used by the memory-failure code, so we can omit it if we're not compiling in the memory-failure code. Link: https://lkml.kernel.org/r/20240412193510.2356957-5-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Suggested-by: Miaohe Lin Reviewed-by: Jane Chu Reviewed-by: Oscar Salvador Acked-by: Miaohe Lin Cc: Dan Williams Signed-off-by: Andrew Morton --- mm/page_vma_mapped.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index c202eab84936d..ae5cc42aa2087 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -314,6 +314,7 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) return false; } +#ifdef CONFIG_MEMORY_FAILURE /** * page_mapped_in_vma - check whether a page is really mapped in a VMA * @page: the page to test @@ -344,3 +345,4 @@ unsigned long page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) out: return pvmw.address; } +#endif From fed5348ee2b136c84c5a27d6fceef14066beeb66 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 12 Apr 2024 20:35:02 +0100 Subject: [PATCH 1194/1702] mm/memory-failure: convert shake_page() to shake_folio() Removes two calls to compound_head(). Move the prototype to internal.h; we definitely don't want code outside mm using it. Link: https://lkml.kernel.org/r/20240412193510.2356957-6-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Jane Chu Acked-by: Miaohe Lin Cc: Dan Williams Cc: Miaohe Lin Cc: Oscar Salvador Signed-off-by: Andrew Morton --- include/linux/mm.h | 1 - mm/hwpoison-inject.c | 11 ++++++----- mm/internal.h | 1 + mm/memory-failure.c | 15 ++++++++++----- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 78e583b50e421..b9ac49c9eb008 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4033,7 +4033,6 @@ int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index, extern int memory_failure(unsigned long pfn, int flags); extern void memory_failure_queue_kick(int cpu); extern int unpoison_memory(unsigned long pfn); -extern void shake_page(struct page *p); extern atomic_long_t num_poisoned_pages __read_mostly; extern int soft_offline_page(unsigned long pfn, int flags); #ifdef CONFIG_MEMORY_FAILURE diff --git a/mm/hwpoison-inject.c b/mm/hwpoison-inject.c index d0548e382b6ba..c9d653f51e45b 100644 --- a/mm/hwpoison-inject.c +++ b/mm/hwpoison-inject.c @@ -15,7 +15,7 @@ static int hwpoison_inject(void *data, u64 val) { unsigned long pfn = val; struct page *p; - struct page *hpage; + struct folio *folio; int err; if (!capable(CAP_SYS_ADMIN)) @@ -25,16 +25,17 @@ static int hwpoison_inject(void *data, u64 val) return -ENXIO; p = pfn_to_page(pfn); - hpage = compound_head(p); + folio = page_folio(p); if (!hwpoison_filter_enable) goto inject; - shake_page(hpage); + shake_folio(folio); /* * This implies unable to support non-LRU pages except free page. */ - if (!PageLRU(hpage) && !PageHuge(p) && !is_free_buddy_page(p)) + if (!folio_test_lru(folio) && !folio_test_hugetlb(folio) && + !is_free_buddy_page(p)) return 0; /* @@ -42,7 +43,7 @@ static int hwpoison_inject(void *data, u64 val) * the targeted owner (or on a free page). * memory_failure() will redo the check reliably inside page lock. */ - err = hwpoison_filter(hpage); + err = hwpoison_filter(&folio->page); if (err) return 0; diff --git a/mm/internal.h b/mm/internal.h index 5d5e49b86fe32..6803c7b17c1f3 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1037,6 +1037,7 @@ static inline int find_next_best_node(int node, nodemask_t *used_node_mask) /* * mm/memory-failure.c */ +void shake_folio(struct folio *folio); extern int hwpoison_filter(struct page *p); extern u32 hwpoison_filter_dev_major; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 12e5d2844cb15..4daf581e38784 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -369,20 +369,25 @@ static int kill_proc(struct to_kill *tk, unsigned long pfn, int flags) * Unknown page type encountered. Try to check whether it can turn PageLRU by * lru_add_drain_all. */ -void shake_page(struct page *p) +void shake_folio(struct folio *folio) { - if (PageHuge(p)) + if (folio_test_hugetlb(folio)) return; /* * TODO: Could shrink slab caches here if a lightweight range-based * shrinker will be available. */ - if (PageSlab(p)) + if (folio_test_slab(folio)) return; lru_add_drain_all(); } -EXPORT_SYMBOL_GPL(shake_page); +EXPORT_SYMBOL_GPL(shake_folio); + +static void shake_page(struct page *page) +{ + shake_folio(page_folio(page)); +} static unsigned long dev_pagemap_mapping_shift(struct vm_area_struct *vma, unsigned long address) @@ -1639,7 +1644,7 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, * shake_page() again to ensure that it's flushed. */ if (mlocked) - shake_page(hpage); + shake_folio(folio); /* * Now that the dirty bit has been propagated to the From 6e8cda4c2c87b2a44828e651a10705647a6fd542 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 12 Apr 2024 20:35:03 +0100 Subject: [PATCH 1195/1702] mm: convert hugetlb_page_mapping_lock_write to folio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The page is only used to get the mapping, so the folio will do just as well. Both callers already have a folio available, so this saves a call to compound_head(). Link: https://lkml.kernel.org/r/20240412193510.2356957-7-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Jane Chu  Reviewed-by: Oscar Salvador Acked-by: Miaohe Lin Cc: Dan Williams Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 6 +++--- mm/hugetlb.c | 6 +++--- mm/memory-failure.c | 2 +- mm/migrate.c | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 1bc93e7e315bb..68244bb3637a8 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -178,7 +178,7 @@ bool hugetlbfs_pagecache_present(struct hstate *h, struct vm_area_struct *vma, unsigned long address); -struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage); +struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio); extern int sysctl_hugetlb_shm_group; extern struct list_head huge_boot_pages[MAX_NUMNODES]; @@ -297,8 +297,8 @@ static inline unsigned long hugetlb_total_pages(void) return 0; } -static inline struct address_space *hugetlb_page_mapping_lock_write( - struct page *hpage) +static inline struct address_space *hugetlb_folio_mapping_lock_write( + struct folio *folio) { return NULL; } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 3b7d5ddc32ad7..417fc5cdb6eeb 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2155,13 +2155,13 @@ static bool prep_compound_gigantic_folio_for_demote(struct folio *folio, /* * Find and lock address space (mapping) in write mode. * - * Upon entry, the page is locked which means that page_mapping() is + * Upon entry, the folio is locked which means that folio_mapping() is * stable. Due to locking order, we can only trylock_write. If we can * not get the lock, simply return NULL to caller. */ -struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage) +struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio) { - struct address_space *mapping = page_mapping(hpage); + struct address_space *mapping = folio_mapping(folio); if (!mapping) return mapping; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 4daf581e38784..1a5f3403dd2a9 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1624,7 +1624,7 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, * TTU_RMAP_LOCKED to indicate we have taken the lock * at this higher level. */ - mapping = hugetlb_page_mapping_lock_write(hpage); + mapping = hugetlb_folio_mapping_lock_write(folio); if (mapping) { try_to_unmap(folio, ttu|TTU_RMAP_LOCKED); i_mmap_unlock_write(mapping); diff --git a/mm/migrate.c b/mm/migrate.c index c7692f303fa73..dd04f578c19c3 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1425,7 +1425,7 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio, * semaphore in write mode here and set TTU_RMAP_LOCKED * to let lower levels know we have taken the lock. */ - mapping = hugetlb_page_mapping_lock_write(&src->page); + mapping = hugetlb_folio_mapping_lock_write(src); if (unlikely(!mapping)) goto unlock_put_anon; From 5dba5c356ab3bbf6b00a42632f3e14728f327553 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 12 Apr 2024 20:35:04 +0100 Subject: [PATCH 1196/1702] mm/memory-failure: convert memory_failure() to use a folio Saves dozens of calls to compound_head(). Link: https://lkml.kernel.org/r/20240412193510.2356957-8-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Acked-by: Miaohe Lin Cc: Dan Williams Cc: Jane Chu Cc: Miaohe Lin Cc: Oscar Salvador Signed-off-by: Andrew Morton --- mm/memory-failure.c | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 1a5f3403dd2a9..03ec7754787be 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -2189,7 +2189,7 @@ static int memory_failure_dev_pagemap(unsigned long pfn, int flags, int memory_failure(unsigned long pfn, int flags) { struct page *p; - struct page *hpage; + struct folio *folio; struct dev_pagemap *pgmap; int res = 0; unsigned long page_flags; @@ -2277,8 +2277,8 @@ int memory_failure(unsigned long pfn, int flags) } } - hpage = compound_head(p); - if (PageTransHuge(hpage)) { + folio = page_folio(p); + if (folio_test_large(folio)) { /* * The flag must be set after the refcount is bumped * otherwise it may race with THP split. @@ -2292,12 +2292,13 @@ int memory_failure(unsigned long pfn, int flags) * or unhandlable page. The refcount is bumped iff the * page is a valid handlable page. */ - SetPageHasHWPoisoned(hpage); + folio_set_has_hwpoisoned(folio); if (try_to_split_thp_page(p) < 0) { res = action_result(pfn, MF_MSG_UNSPLIT_THP, MF_IGNORED); goto unlock_mutex; } VM_BUG_ON_PAGE(!page_count(p), p); + folio = page_folio(p); } /* @@ -2308,9 +2309,9 @@ int memory_failure(unsigned long pfn, int flags) * The check (unnecessarily) ignores LRU pages being isolated and * walked by the page reclaim code, however that's not a big loss. */ - shake_page(p); + shake_folio(folio); - lock_page(p); + folio_lock(folio); /* * We're only intended to deal with the non-Compound page here. @@ -2318,11 +2319,11 @@ int memory_failure(unsigned long pfn, int flags) * race window. If this happens, we could try again to hopefully * handle the page next round. */ - if (PageCompound(p)) { + if (folio_test_large(folio)) { if (retry) { ClearPageHWPoison(p); - unlock_page(p); - put_page(p); + folio_unlock(folio); + folio_put(folio); flags &= ~MF_COUNT_INCREASED; retry = false; goto try_again; @@ -2338,29 +2339,29 @@ int memory_failure(unsigned long pfn, int flags) * folio_remove_rmap_*() in try_to_unmap_one(). So to determine page * status correctly, we save a copy of the page flags at this time. */ - page_flags = p->flags; + page_flags = folio->flags; if (hwpoison_filter(p)) { ClearPageHWPoison(p); - unlock_page(p); - put_page(p); + folio_unlock(folio); + folio_put(folio); res = -EOPNOTSUPP; goto unlock_mutex; } /* - * __munlock_folio() may clear a writeback page's LRU flag without - * page_lock. We need wait writeback completion for this page or it - * may trigger vfs BUG while evict inode. + * __munlock_folio() may clear a writeback folio's LRU flag without + * the folio lock. We need to wait for writeback completion for this + * folio or it may trigger a vfs BUG while evicting inode. */ - if (!PageLRU(p) && !PageWriteback(p)) + if (!folio_test_lru(folio) && !folio_test_writeback(folio)) goto identify_page_state; /* * It's very difficult to mess with pages currently under IO * and in many cases impossible, so we just avoid it here. */ - wait_on_page_writeback(p); + folio_wait_writeback(folio); /* * Now take care of user space mappings. @@ -2374,7 +2375,8 @@ int memory_failure(unsigned long pfn, int flags) /* * Torn down by someone else? */ - if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) { + if (folio_test_lru(folio) && !folio_test_swapcache(folio) && + folio->mapping == NULL) { res = action_result(pfn, MF_MSG_TRUNCATED_LRU, MF_IGNORED); goto unlock_page; } @@ -2384,7 +2386,7 @@ int memory_failure(unsigned long pfn, int flags) mutex_unlock(&mf_mutex); return res; unlock_page: - unlock_page(p); + folio_unlock(folio); unlock_mutex: mutex_unlock(&mf_mutex); return res; From 03468a0f52893b8dea4a96677ad9ff78bf55d765 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 12 Apr 2024 20:35:05 +0100 Subject: [PATCH 1197/1702] mm/memory-failure: convert hwpoison_user_mappings to take a folio Pass the folio from the callers, and use it throughout instead of hpage. Saves dozens of calls to compound_head(). Link: https://lkml.kernel.org/r/20240412193510.2356957-9-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Acked-by: Miaohe Lin Reviewed-by: Jane Chu Cc: Dan Williams Cc: Oscar Salvador Signed-off-by: Andrew Morton --- mm/memory-failure.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 03ec7754787be..130d132d075d7 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1559,24 +1559,24 @@ static int get_hwpoison_page(struct page *p, unsigned long flags) * Do all that is necessary to remove user space mappings. Unmap * the pages and send SIGBUS to the processes if the data was dirty. */ -static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, - int flags, struct page *hpage) +static bool hwpoison_user_mappings(struct folio *folio, struct page *p, + unsigned long pfn, int flags) { - struct folio *folio = page_folio(hpage); enum ttu_flags ttu = TTU_IGNORE_MLOCK | TTU_SYNC | TTU_HWPOISON; struct address_space *mapping; LIST_HEAD(tokill); bool unmap_success; int forcekill; - bool mlocked = PageMlocked(hpage); + bool mlocked = folio_test_mlocked(folio); /* * Here we are interested only in user-mapped pages, so skip any * other types of pages. */ - if (PageReserved(p) || PageSlab(p) || PageTable(p) || PageOffline(p)) + if (folio_test_reserved(folio) || folio_test_slab(folio) || + folio_test_pgtable(folio) || folio_test_offline(folio)) return true; - if (!(PageLRU(hpage) || PageHuge(p))) + if (!(folio_test_lru(folio) || folio_test_hugetlb(folio))) return true; /* @@ -1586,7 +1586,7 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, if (!page_mapped(p)) return true; - if (PageSwapCache(p)) { + if (folio_test_swapcache(folio)) { pr_err("%#lx: keeping poisoned page in swap cache\n", pfn); ttu &= ~TTU_HWPOISON; } @@ -1597,11 +1597,11 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, * XXX: the dirty test could be racy: set_page_dirty() may not always * be called inside page lock (it's recommended but not enforced). */ - mapping = page_mapping(hpage); - if (!(flags & MF_MUST_KILL) && !PageDirty(hpage) && mapping && + mapping = folio_mapping(folio); + if (!(flags & MF_MUST_KILL) && !folio_test_dirty(folio) && mapping && mapping_can_writeback(mapping)) { - if (page_mkclean(hpage)) { - SetPageDirty(hpage); + if (folio_mkclean(folio)) { + folio_set_dirty(folio); } else { ttu &= ~TTU_HWPOISON; pr_info("%#lx: corrupted page was clean: dropped without side effects\n", @@ -1616,7 +1616,7 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, */ collect_procs(folio, p, &tokill, flags & MF_ACTION_REQUIRED); - if (PageHuge(hpage) && !PageAnon(hpage)) { + if (folio_test_hugetlb(folio) && !folio_test_anon(folio)) { /* * For hugetlb pages in shared mappings, try_to_unmap * could potentially call huge_pmd_unshare. Because of @@ -1656,7 +1656,7 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, * use a more force-full uncatchable kill to prevent * any accesses to the poisoned memory. */ - forcekill = PageDirty(hpage) || (flags & MF_MUST_KILL) || + forcekill = folio_test_dirty(folio) || (flags & MF_MUST_KILL) || !unmap_success; kill_procs(&tokill, forcekill, !unmap_success, pfn, flags); @@ -2100,7 +2100,7 @@ static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb page_flags = folio->flags; - if (!hwpoison_user_mappings(p, pfn, flags, &folio->page)) { + if (!hwpoison_user_mappings(folio, p, pfn, flags)) { folio_unlock(folio); return action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED); } @@ -2367,7 +2367,7 @@ int memory_failure(unsigned long pfn, int flags) * Now take care of user space mappings. * Abort on fail: __filemap_remove_folio() assumes unmapped page. */ - if (!hwpoison_user_mappings(p, pfn, flags, p)) { + if (!hwpoison_user_mappings(folio, p, pfn, flags)) { res = action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED); goto unlock_page; } From ee299e9849736f60e6e01f7a5dcb258de7c7d1b9 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 12 Apr 2024 20:35:06 +0100 Subject: [PATCH 1198/1702] mm/memory-failure: add some folio conversions to unpoison_memory Some of these folio APIs didn't exist when the unpoison_memory() conversion was done originally. Link: https://lkml.kernel.org/r/20240412193510.2356957-10-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Acked-by: Miaohe Lin Reviewed-by: Jane Chu Cc: Dan Williams Cc: Oscar Salvador Signed-off-by: Andrew Morton --- mm/memory-failure.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 130d132d075d7..521e0efd08e7d 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -2556,8 +2556,8 @@ int unpoison_memory(unsigned long pfn) goto unlock_mutex; } - if (folio_test_slab(folio) || PageTable(&folio->page) || - folio_test_reserved(folio) || PageOffline(&folio->page)) + if (folio_test_slab(folio) || folio_test_pgtable(folio) || + folio_test_reserved(folio) || folio_test_offline(folio)) goto unlock_mutex; /* @@ -2578,7 +2578,7 @@ int unpoison_memory(unsigned long pfn) ghp = get_hwpoison_page(p, MF_UNPOISON); if (!ghp) { - if (PageHuge(p)) { + if (folio_test_hugetlb(folio)) { huge = true; count = folio_free_raw_hwp(folio, false); if (count == 0) @@ -2594,7 +2594,7 @@ int unpoison_memory(unsigned long pfn) pfn, &unpoison_rs); } } else { - if (PageHuge(p)) { + if (folio_test_hugetlb(folio)) { huge = true; count = folio_free_raw_hwp(folio, false); if (count == 0) { From 0edb5b282ac5a4f9b1bdc22120c9b145be315622 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 12 Apr 2024 20:35:07 +0100 Subject: [PATCH 1199/1702] mm/memory-failure: use folio functions throughout collect_procs() Saves a couple of calls to compound_head(). Link: https://lkml.kernel.org/r/20240412193510.2356957-11-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Jane Chu Acked-by: Miaohe Lin Cc: Dan Williams Cc: Miaohe Lin Cc: Oscar Salvador Signed-off-by: Andrew Morton --- mm/memory-failure.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 521e0efd08e7d..498564f23abbf 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -728,9 +728,9 @@ static void collect_procs(struct folio *folio, struct page *page, { if (!folio->mapping) return; - if (unlikely(PageKsm(page))) + if (unlikely(folio_test_ksm(folio))) collect_procs_ksm(page, tokill, force_early); - else if (PageAnon(page)) + else if (folio_test_anon(folio)) collect_procs_anon(folio, page, tokill, force_early); else collect_procs_file(folio, page, tokill, force_early); From b650e1d2aefbbb31e7578ad60a0a71bf5e5c5346 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 12 Apr 2024 20:35:08 +0100 Subject: [PATCH 1200/1702] mm/memory-failure: pass the folio to collect_procs_ksm() We've already calculated it, so pass it in instead of recalculating it in collect_procs_ksm(). Link: https://lkml.kernel.org/r/20240412193510.2356957-12-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Jane Chu Reviewed-by: Miaohe Lin Cc: Dan Williams Cc: Miaohe Lin Cc: Oscar Salvador Signed-off-by: Andrew Morton --- include/linux/ksm.h | 14 +++----------- mm/ksm.c | 5 ++--- mm/memory-failure.c | 2 +- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 358803cfd4d5e..52c63a9c5a9cd 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -81,15 +81,9 @@ struct folio *ksm_might_need_to_copy(struct folio *folio, void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc); void folio_migrate_ksm(struct folio *newfolio, struct folio *folio); - -#ifdef CONFIG_MEMORY_FAILURE -void collect_procs_ksm(struct page *page, struct list_head *to_kill, - int force_early); -#endif - -#ifdef CONFIG_PROC_FS +void collect_procs_ksm(struct folio *folio, struct page *page, + struct list_head *to_kill, int force_early); long ksm_process_profit(struct mm_struct *); -#endif /* CONFIG_PROC_FS */ #else /* !CONFIG_KSM */ @@ -120,12 +114,10 @@ static inline void ksm_might_unmap_zero_page(struct mm_struct *mm, pte_t pte) { } -#ifdef CONFIG_MEMORY_FAILURE -static inline void collect_procs_ksm(struct page *page, +static inline void collect_procs_ksm(struct folio *folio, struct page *page, struct list_head *to_kill, int force_early) { } -#endif #ifdef CONFIG_MMU static inline int ksm_madvise(struct vm_area_struct *vma, unsigned long start, diff --git a/mm/ksm.c b/mm/ksm.c index 159604ad47799..e1034bf1c937e 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -3178,12 +3178,11 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc) /* * Collect processes when the error hit an ksm page. */ -void collect_procs_ksm(struct page *page, struct list_head *to_kill, - int force_early) +void collect_procs_ksm(struct folio *folio, struct page *page, + struct list_head *to_kill, int force_early) { struct ksm_stable_node *stable_node; struct ksm_rmap_item *rmap_item; - struct folio *folio = page_folio(page); struct vm_area_struct *vma; struct task_struct *tsk; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 498564f23abbf..ec30611ec5ad0 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -729,7 +729,7 @@ static void collect_procs(struct folio *folio, struct page *page, if (!folio->mapping) return; if (unlikely(folio_test_ksm(folio))) - collect_procs_ksm(page, tokill, force_early); + collect_procs_ksm(folio, page, tokill, force_early); else if (folio_test_anon(folio)) collect_procs_anon(folio, page, tokill, force_early); else From 262f014dd7de3747f20ae7d1c3cb5cc68241e770 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 23 Apr 2024 23:55:32 +0100 Subject: [PATCH 1201/1702] fscrypt: convert bh_get_inode_and_lblk_num to use a folio Patch series "Remove page_mapping()". There are only a few users left. Convert them all to either call folio_mapping() or just use folio->mapping directly. This patch (of 6): Remove uses of page->index, page_mapping() and b_page. Saves a call to compound_head(). Link: https://lkml.kernel.org/r/20240423225552.4113447-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240423225552.4113447-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Eric Biggers Reviewed-by: David Hildenbrand Cc: Sidhartha Kumar Signed-off-by: Andrew Morton --- fs/crypto/inline_crypt.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/crypto/inline_crypt.c b/fs/crypto/inline_crypt.c index b4002aea7cdb9..40de69860dcf9 100644 --- a/fs/crypto/inline_crypt.c +++ b/fs/crypto/inline_crypt.c @@ -284,7 +284,7 @@ static bool bh_get_inode_and_lblk_num(const struct buffer_head *bh, const struct inode **inode_ret, u64 *lblk_num_ret) { - struct page *page = bh->b_page; + struct folio *folio = bh->b_folio; const struct address_space *mapping; const struct inode *inode; @@ -292,13 +292,13 @@ static bool bh_get_inode_and_lblk_num(const struct buffer_head *bh, * The ext4 journal (jbd2) can submit a buffer_head it directly created * for a non-pagecache page. fscrypt doesn't care about these. */ - mapping = page_mapping(page); + mapping = folio_mapping(folio); if (!mapping) return false; inode = mapping->host; *inode_ret = inode; - *lblk_num_ret = ((u64)page->index << (PAGE_SHIFT - inode->i_blkbits)) + + *lblk_num_ret = ((u64)folio->index << (PAGE_SHIFT - inode->i_blkbits)) + (bh_offset(bh) >> inode->i_blkbits); return true; } From 196ad49cd62614979903c3a6033afd2299b29441 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 23 Apr 2024 23:55:33 +0100 Subject: [PATCH 1202/1702] f2fs: convert f2fs_clear_page_cache_dirty_tag to use a folio Removes uses of page_mapping() and page_index(). Link: https://lkml.kernel.org/r/20240423225552.4113447-3-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Cc: Eric Biggers Cc: Sidhartha Kumar Signed-off-by: Andrew Morton --- fs/f2fs/data.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index d9494b5fc7c18..961e6ff77c723 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -4082,11 +4082,12 @@ const struct address_space_operations f2fs_dblock_aops = { void f2fs_clear_page_cache_dirty_tag(struct page *page) { - struct address_space *mapping = page_mapping(page); + struct folio *folio = page_folio(page); + struct address_space *mapping = folio->mapping; unsigned long flags; xa_lock_irqsave(&mapping->i_pages, flags); - __xa_clear_mark(&mapping->i_pages, page_index(page), + __xa_clear_mark(&mapping->i_pages, folio->index, PAGECACHE_TAG_DIRTY); xa_unlock_irqrestore(&mapping->i_pages, flags); } From 89f5c54b228181713f1c00b27b360b29643cdfa6 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 23 Apr 2024 23:55:34 +0100 Subject: [PATCH 1203/1702] memory-failure: remove calls to page_mapping() This is mostly just inlining page_mapping() into the two callers. Link: https://lkml.kernel.org/r/20240423225552.4113447-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Sidhartha Kumar Reviewed-by: David Hildenbrand Acked-by: Miaohe Lin Cc: Eric Biggers Signed-off-by: Andrew Morton --- mm/memory-failure.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index ec30611ec5ad0..16ada4fb02b79 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -216,6 +216,7 @@ EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value); static int hwpoison_filter_dev(struct page *p) { + struct folio *folio = page_folio(p); struct address_space *mapping; dev_t dev; @@ -223,7 +224,7 @@ static int hwpoison_filter_dev(struct page *p) hwpoison_filter_dev_minor == ~0U) return 0; - mapping = page_mapping(p); + mapping = folio_mapping(folio); if (mapping == NULL || mapping->host == NULL) return -EINVAL; @@ -1090,7 +1091,8 @@ static int me_pagecache_clean(struct page_state *ps, struct page *p) */ static int me_pagecache_dirty(struct page_state *ps, struct page *p) { - struct address_space *mapping = page_mapping(p); + struct folio *folio = page_folio(p); + struct address_space *mapping = folio_mapping(folio); SetPageError(p); /* TBD: print more information about the file. */ From e18a9faf06c2407916326bf5f1112f5eba859331 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 23 Apr 2024 23:55:35 +0100 Subject: [PATCH 1204/1702] migrate: expand the use of folio in __migrate_device_pages() Removes a few calls to compound_head() and a call to page_mapping(). Link: https://lkml.kernel.org/r/20240423225552.4113447-5-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Cc: Eric Biggers Cc: Sidhartha Kumar Signed-off-by: Andrew Morton --- mm/migrate_device.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/mm/migrate_device.c b/mm/migrate_device.c index b929b450b77c3..7e0712493d1f4 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -696,6 +696,7 @@ static void __migrate_device_pages(unsigned long *src_pfns, struct page *newpage = migrate_pfn_to_page(dst_pfns[i]); struct page *page = migrate_pfn_to_page(src_pfns[i]); struct address_space *mapping; + struct folio *folio; int r; if (!newpage) { @@ -730,15 +731,12 @@ static void __migrate_device_pages(unsigned long *src_pfns, continue; } - mapping = page_mapping(page); + folio = page_folio(page); + mapping = folio_mapping(folio); if (is_device_private_page(newpage) || is_device_coherent_page(newpage)) { if (mapping) { - struct folio *folio; - - folio = page_folio(page); - /* * For now only support anonymous memory migrating to * device private or coherent memory. @@ -761,11 +759,10 @@ static void __migrate_device_pages(unsigned long *src_pfns, if (migrate && migrate->fault_page == page) r = migrate_folio_extra(mapping, page_folio(newpage), - page_folio(page), - MIGRATE_SYNC_NO_COPY, 1); + folio, MIGRATE_SYNC_NO_COPY, 1); else r = migrate_folio(mapping, page_folio(newpage), - page_folio(page), MIGRATE_SYNC_NO_COPY); + folio, MIGRATE_SYNC_NO_COPY); if (r != MIGRATEPAGE_SUCCESS) src_pfns[i] &= ~MIGRATE_PFN_MIGRATE; } From a568b4126b20ebbc01914e12d083379720911799 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 23 Apr 2024 23:55:36 +0100 Subject: [PATCH 1205/1702] userfault; expand folio use in mfill_atomic_install_pte() Call page_folio() a little earlier so we can use folio_mapping() instead of page_mapping(), saving a call to compound_head(). Link: https://lkml.kernel.org/r/20240423225552.4113447-6-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: David Hildenbrand Cc: Eric Biggers Cc: Sidhartha Kumar Signed-off-by: Andrew Morton --- mm/userfaultfd.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 575ccf90325aa..8b1005ef9dfa9 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -180,9 +180,9 @@ int mfill_atomic_install_pte(pmd_t *dst_pmd, pte_t _dst_pte, *dst_pte; bool writable = dst_vma->vm_flags & VM_WRITE; bool vm_shared = dst_vma->vm_flags & VM_SHARED; - bool page_in_cache = page_mapping(page); spinlock_t *ptl; - struct folio *folio; + struct folio *folio = page_folio(page); + bool page_in_cache = folio_mapping(folio); _dst_pte = mk_pte(page, dst_vma->vm_page_prot); _dst_pte = pte_mkdirty(_dst_pte); @@ -212,7 +212,6 @@ int mfill_atomic_install_pte(pmd_t *dst_pmd, if (!pte_none_mostly(ptep_get(dst_pte))) goto out_unlock; - folio = page_folio(page); if (page_in_cache) { /* Usually, cache pages are already added to LRU */ if (newly_allocated) From 3f2ae4ebd53b6e555455f293364e1e0d4b7530e2 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Apr 2024 20:19:06 +0100 Subject: [PATCH 1206/1702] mm: remove page_cache_alloc() Patch series "More folio compat code removal". More code removal with bonus kernel-doc addition. This patch (of 7): All callers have now been converted to filemap_alloc_folio(). Link: https://lkml.kernel.org/r/20240424191914.361554-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240424191914.361554-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/pagemap.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 65e332df4d769..2a0bc2ff52583 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -558,11 +558,6 @@ static inline struct page *__page_cache_alloc(gfp_t gfp) return &filemap_alloc_folio(gfp, 0)->page; } -static inline struct page *page_cache_alloc(struct address_space *x) -{ - return __page_cache_alloc(mapping_gfp_mask(x)); -} - static inline gfp_t readahead_gfp_mask(struct address_space *x) { return mapping_gfp_mask(x) | __GFP_NORETRY | __GFP_NOWARN; From 6785c54a1b43c44ade9064ce46db4aa4d09e8cef Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Apr 2024 20:19:07 +0100 Subject: [PATCH 1207/1702] mm: remove put_devmap_managed_page() It only has one caller; convert that caller to use put_devmap_managed_page_refs() instead. Link: https://lkml.kernel.org/r/20240424191914.361554-3-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/mm.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index b9ac49c9eb008..deb51f8dc2834 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1457,11 +1457,6 @@ static inline bool put_devmap_managed_page_refs(struct page *page, int refs) } #endif /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */ -static inline bool put_devmap_managed_page(struct page *page) -{ - return put_devmap_managed_page_refs(page, 1); -} - /* 127: arbitrary random number, small enough to assemble well */ #define folio_ref_zero_or_close_to_overflow(folio) \ ((unsigned int) folio_ref_count(folio) + 127u <= 127u) @@ -1580,7 +1575,7 @@ static inline void put_page(struct page *page) * For some devmap managed pages we need to catch refcount transition * from 2 to 1: */ - if (put_devmap_managed_page(&folio->page)) + if (put_devmap_managed_page_refs(&folio->page, 1)) return; folio_put(folio); } From 53e45c4f6d4f6cd7e62f4cb016018ba31c2ac8b4 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Apr 2024 20:19:08 +0100 Subject: [PATCH 1208/1702] mm: convert put_devmap_managed_page_refs() to put_devmap_managed_folio_refs() All callers have a folio so we can remove this use of page_ref_sub_return(). Link: https://lkml.kernel.org/r/20240424191914.361554-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/mm.h | 12 ++++++------ mm/gup.c | 6 +++--- mm/memremap.c | 10 +++++----- mm/swap.c | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index deb51f8dc2834..9849dfda44d43 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1441,17 +1441,17 @@ vm_fault_t finish_fault(struct vm_fault *vmf); #if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_FS_DAX) DECLARE_STATIC_KEY_FALSE(devmap_managed_key); -bool __put_devmap_managed_page_refs(struct page *page, int refs); -static inline bool put_devmap_managed_page_refs(struct page *page, int refs) +bool __put_devmap_managed_folio_refs(struct folio *folio, int refs); +static inline bool put_devmap_managed_folio_refs(struct folio *folio, int refs) { if (!static_branch_unlikely(&devmap_managed_key)) return false; - if (!is_zone_device_page(page)) + if (!folio_is_zone_device(folio)) return false; - return __put_devmap_managed_page_refs(page, refs); + return __put_devmap_managed_folio_refs(folio, refs); } #else /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */ -static inline bool put_devmap_managed_page_refs(struct page *page, int refs) +static inline bool put_devmap_managed_folio_refs(struct folio *folio, int refs) { return false; } @@ -1575,7 +1575,7 @@ static inline void put_page(struct page *page) * For some devmap managed pages we need to catch refcount transition * from 2 to 1: */ - if (put_devmap_managed_page_refs(&folio->page, 1)) + if (put_devmap_managed_folio_refs(folio, 1)) return; folio_put(folio); } diff --git a/mm/gup.c b/mm/gup.c index 8dcbeae714e26..44c6ceee7e1f7 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -89,7 +89,7 @@ static inline struct folio *try_get_folio(struct page *page, int refs) * belongs to this folio. */ if (unlikely(page_folio(page) != folio)) { - if (!put_devmap_managed_page_refs(&folio->page, refs)) + if (!put_devmap_managed_folio_refs(folio, refs)) folio_put_refs(folio, refs); goto retry; } @@ -156,7 +156,7 @@ struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags) */ if (unlikely((flags & FOLL_LONGTERM) && !folio_is_longterm_pinnable(folio))) { - if (!put_devmap_managed_page_refs(&folio->page, refs)) + if (!put_devmap_managed_folio_refs(folio, refs)) folio_put_refs(folio, refs); return NULL; } @@ -198,7 +198,7 @@ static void gup_put_folio(struct folio *folio, int refs, unsigned int flags) refs *= GUP_PIN_COUNTING_BIAS; } - if (!put_devmap_managed_page_refs(&folio->page, refs)) + if (!put_devmap_managed_folio_refs(folio, refs)) folio_put_refs(folio, refs); } diff --git a/mm/memremap.c b/mm/memremap.c index e1776693e2eae..40d4547ce5144 100644 --- a/mm/memremap.c +++ b/mm/memremap.c @@ -512,9 +512,9 @@ void zone_device_page_init(struct page *page) EXPORT_SYMBOL_GPL(zone_device_page_init); #ifdef CONFIG_FS_DAX -bool __put_devmap_managed_page_refs(struct page *page, int refs) +bool __put_devmap_managed_folio_refs(struct folio *folio, int refs) { - if (page->pgmap->type != MEMORY_DEVICE_FS_DAX) + if (folio->page.pgmap->type != MEMORY_DEVICE_FS_DAX) return false; /* @@ -522,9 +522,9 @@ bool __put_devmap_managed_page_refs(struct page *page, int refs) * refcount is 1, then the page is free and the refcount is * stable because nobody holds a reference on the page. */ - if (page_ref_sub_return(page, refs) == 1) - wake_up_var(&page->_refcount); + if (folio_ref_sub_return(folio, refs) == 1) + wake_up_var(&folio->_refcount); return true; } -EXPORT_SYMBOL(__put_devmap_managed_page_refs); +EXPORT_SYMBOL(__put_devmap_managed_folio_refs); #endif /* CONFIG_FS_DAX */ diff --git a/mm/swap.c b/mm/swap.c index 8ae5cd4ed180b..bfce6d53b2378 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -980,7 +980,7 @@ void folios_put_refs(struct folio_batch *folios, unsigned int *refs) unlock_page_lruvec_irqrestore(lruvec, flags); lruvec = NULL; } - if (put_devmap_managed_page_refs(&folio->page, nr_refs)) + if (put_devmap_managed_folio_refs(folio, nr_refs)) continue; if (folio_ref_sub_and_test(folio, nr_refs)) free_zone_device_folio(folio); From 498aefbc69d5719d8e4713b568122c259167f7b9 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Apr 2024 20:19:09 +0100 Subject: [PATCH 1209/1702] mm: remove page_ref_sub_return() With all callers converted to folios, we can act directly on folio->_refcount. Link: https://lkml.kernel.org/r/20240424191914.361554-5-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/page_ref.h | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/include/linux/page_ref.h b/include/linux/page_ref.h index d7c2d33baa7f8..1acf5bac7f503 100644 --- a/include/linux/page_ref.h +++ b/include/linux/page_ref.h @@ -139,20 +139,15 @@ static inline void folio_ref_sub(struct folio *folio, int nr) page_ref_sub(&folio->page, nr); } -static inline int page_ref_sub_return(struct page *page, int nr) +static inline int folio_ref_sub_return(struct folio *folio, int nr) { - int ret = atomic_sub_return(nr, &page->_refcount); + int ret = atomic_sub_return(nr, &folio->_refcount); if (page_ref_tracepoint_active(page_ref_mod_and_return)) - __page_ref_mod_and_return(page, -nr, ret); + __page_ref_mod_and_return(&folio->page, -nr, ret); return ret; } -static inline int folio_ref_sub_return(struct folio *folio, int nr) -{ - return page_ref_sub_return(&folio->page, nr); -} - static inline void page_ref_inc(struct page *page) { atomic_inc(&page->_refcount); From 9cbe4954c6d93e6a72680fe634913c0d056fd932 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Apr 2024 20:19:10 +0100 Subject: [PATCH 1210/1702] gup: use folios for gup_devmap Use try_grab_folio() instead of try_grab_page() so we get the folio back that we calculated, and then use folio_set_referenced() instead of SetPageReferenced(). Correspondingly, use gup_put_folio() to put any unneeded references. Link: https://lkml.kernel.org/r/20240424191914.361554-6-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/gup.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index 44c6ceee7e1f7..2f7baf96f6559 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2877,13 +2877,10 @@ static void __maybe_unused gup_fast_undo_dev_pagemap(int *nr, int nr_start, unsigned int flags, struct page **pages) { while ((*nr) - nr_start) { - struct page *page = pages[--(*nr)]; + struct folio *folio = page_folio(pages[--(*nr)]); - ClearPageReferenced(page); - if (flags & FOLL_PIN) - unpin_user_page(page); - else - put_page(page); + folio_clear_referenced(folio); + gup_put_folio(folio, 1, flags); } } @@ -3024,6 +3021,7 @@ static int gup_fast_devmap_leaf(unsigned long pfn, unsigned long addr, struct dev_pagemap *pgmap = NULL; do { + struct folio *folio; struct page *page = pfn_to_page(pfn); pgmap = get_dev_pagemap(pfn, pgmap); @@ -3037,12 +3035,13 @@ static int gup_fast_devmap_leaf(unsigned long pfn, unsigned long addr, break; } - SetPageReferenced(page); - pages[*nr] = page; - if (unlikely(try_grab_page(page, flags))) { + folio = try_grab_folio(page, 1, flags); + if (!folio) { gup_fast_undo_dev_pagemap(nr, nr_start, flags, pages); break; } + folio_set_referenced(folio); + pages[*nr] = page; (*nr)++; pfn++; } while (addr += PAGE_SIZE, addr != end); From 21db296aaf5cb6d5477697088db025f8e42345c1 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Apr 2024 20:19:11 +0100 Subject: [PATCH 1211/1702] mm: add kernel-doc for folio_mark_accessed() Convert the existing documentation to kernel-doc and remove references to pages. Link: https://lkml.kernel.org/r/20240424191914.361554-7-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/swap.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/mm/swap.c b/mm/swap.c index bfce6d53b2378..67786cb771305 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -447,15 +447,18 @@ static void folio_inc_refs(struct folio *folio) } #endif /* CONFIG_LRU_GEN */ -/* - * Mark a page as having seen activity. +/** + * folio_mark_accessed - Mark a folio as having seen activity. + * @folio: The folio to mark. + * + * This function will perform one of the following transitions: * - * inactive,unreferenced -> inactive,referenced - * inactive,referenced -> active,unreferenced - * active,unreferenced -> active,referenced + * * inactive,unreferenced -> inactive,referenced + * * inactive,referenced -> active,unreferenced + * * active,unreferenced -> active,referenced * - * When a newly allocated page is not yet visible, so safe for non-atomic ops, - * __SetPageReferenced(page) may be substituted for mark_page_accessed(page). + * When a newly allocated folio is not yet visible, so safe for non-atomic ops, + * __folio_set_referenced() may be substituted for folio_mark_accessed(). */ void folio_mark_accessed(struct folio *folio) { From 093137ea97bdc6d3617157e42ea58f8d4f376460 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Apr 2024 20:19:12 +0100 Subject: [PATCH 1212/1702] mm: remove PageReferenced All callers now use folio_*_referenced() so we can remove the PageReferenced family of functions. Link: https://lkml.kernel.org/r/20240424191914.361554-8-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 0af2cef02383b..104078afe0b16 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -512,9 +512,9 @@ static inline int TestClearPage##uname(struct page *page) { return 0; } __PAGEFLAG(Locked, locked, PF_NO_TAIL) FOLIO_FLAG(waiters, FOLIO_HEAD_PAGE) PAGEFLAG(Error, error, PF_NO_TAIL) TESTCLEARFLAG(Error, error, PF_NO_TAIL) -PAGEFLAG(Referenced, referenced, PF_HEAD) - TESTCLEARFLAG(Referenced, referenced, PF_HEAD) - __SETPAGEFLAG(Referenced, referenced, PF_HEAD) +FOLIO_FLAG(referenced, FOLIO_HEAD_PAGE) + FOLIO_TEST_CLEAR_FLAG(referenced, FOLIO_HEAD_PAGE) + __FOLIO_SET_FLAG(referenced, FOLIO_HEAD_PAGE) PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD) __CLEARPAGEFLAG(Dirty, dirty, PF_HEAD) PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD) From 78ec6f9df6642418411c534683da6133e0962ec7 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 24 Apr 2024 05:59:39 -0700 Subject: [PATCH 1213/1702] memcg: fix data-race KCSAN bug in rstats A data-race issue in memcg rstat occurs when two distinct code paths access the same 4-byte region concurrently. KCSAN detection triggers the following BUG as a result. BUG: KCSAN: data-race in __count_memcg_events / mem_cgroup_css_rstat_flush write to 0xffffe8ffff98e300 of 4 bytes by task 5274 on cpu 17: mem_cgroup_css_rstat_flush (mm/memcontrol.c:5850) cgroup_rstat_flush_locked (kernel/cgroup/rstat.c:243 (discriminator 7)) cgroup_rstat_flush (./include/linux/spinlock.h:401 kernel/cgroup/rstat.c:278) mem_cgroup_flush_stats.part.0 (mm/memcontrol.c:767) memory_numa_stat_show (mm/memcontrol.c:6911) read to 0xffffe8ffff98e300 of 4 bytes by task 410848 on cpu 27: __count_memcg_events (mm/memcontrol.c:725 mm/memcontrol.c:962) count_memcg_event_mm.part.0 (./include/linux/memcontrol.h:1097 ./include/linux/memcontrol.h:1120) handle_mm_fault (mm/memory.c:5483 mm/memory.c:5622) value changed: 0x00000029 -> 0x00000000 The race occurs because two code paths access the same "stats_updates" location. Although "stats_updates" is a per-CPU variable, it is remotely accessed by another CPU at cgroup_rstat_flush_locked()->mem_cgroup_css_rstat_flush(), leading to the data race mentioned. Considering that memcg_rstat_updated() is in the hot code path, adding a lock to protect it may not be desirable, especially since this variable pertains solely to statistics. Therefore, annotating accesses to stats_updates with READ/WRITE_ONCE() can prevent KCSAN splats and potential partial reads/writes. Link: https://lkml.kernel.org/r/20240424125940.2410718-1-leitao@debian.org Fixes: 9cee7e8ef3e3 ("mm: memcg: optimize parent iteration in memcg_rstat_updated()") Signed-off-by: Breno Leitao Suggested-by: Shakeel Butt Acked-by: Johannes Weiner Acked-by: Shakeel Butt Reviewed-by: Yosry Ahmed Cc: Michal Hocko Cc: Roman Gushchin Cc: Muchun Song Signed-off-by: Andrew Morton --- mm/memcontrol.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 9095ab05d47a8..a111e0d981baa 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -715,6 +715,7 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) { struct memcg_vmstats_percpu *statc; int cpu = smp_processor_id(); + unsigned int stats_updates; if (!val) return; @@ -722,8 +723,9 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) cgroup_rstat_updated(memcg->css.cgroup, cpu); statc = this_cpu_ptr(memcg->vmstats_percpu); for (; statc; statc = statc->parent) { - statc->stats_updates += abs(val); - if (statc->stats_updates < MEMCG_CHARGE_BATCH) + stats_updates = READ_ONCE(statc->stats_updates) + abs(val); + WRITE_ONCE(statc->stats_updates, stats_updates); + if (stats_updates < MEMCG_CHARGE_BATCH) continue; /* @@ -731,9 +733,9 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) * redundant. Avoid the overhead of the atomic update. */ if (!memcg_vmstats_needs_flush(statc->vmstats)) - atomic64_add(statc->stats_updates, + atomic64_add(stats_updates, &statc->vmstats->stats_updates); - statc->stats_updates = 0; + WRITE_ONCE(statc->stats_updates, 0); } } @@ -5887,7 +5889,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) } } } - statc->stats_updates = 0; + WRITE_ONCE(statc->stats_updates, 0); /* We are in a per-cpu loop here, only do the atomic write once */ if (atomic64_read(&memcg->vmstats->stats_updates)) atomic64_set(&memcg->vmstats->stats_updates, 0); From 1bafe96e89f056cb6e25d47451fb16aee2c7c4d0 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 24 Apr 2024 14:26:30 +0200 Subject: [PATCH 1214/1702] mm/khugepaged: replace page_mapcount() check by folio_likely_mapped_shared() We want to limit the use of page_mapcount() to places where absolutely required, to prepare for kernel configs where we won't keep track of per-page mapcounts in large folios. khugepaged is one of the remaining "more challenging" page_mapcount() users, but we might be able to move away from page_mapcount() without resulting in a significant behavior change that would warrant special-casing based on kernel configs. In 2020, we first added support to khugepaged for collapsing COW-shared pages via commit 9445689f3b61 ("khugepaged: allow to collapse a page shared across fork"), followed by support for collapsing PTE-mapped THP in commit 5503fbf2b0b8 ("khugepaged: allow to collapse PTE-mapped compound pages") and limiting the memory waste via the "page_count() > 1" check in commit 71a2c112a0f6 ("khugepaged: introduce 'max_ptes_shared' tunable"). As a default, khugepaged will allow up to half of the PTEs to map shared pages: where page_mapcount() > 1. MADV_COLLAPSE ignores the khugepaged setting. khugepaged does currently not care about swapcache page references, and does not check under folio lock: so in some corner cases the "shared vs. exclusive" detection might be a bit off, making us detect "exclusive" when it's actually "shared". Most of our anonymous folios in the system are usually exclusive. We frequently see sharing of anonymous folios for a short period of time, after which our short-lived suprocesses either quit or exec(). There are some famous examples, though, where child processes exist for a long time, and where memory is COW-shared with a lot of processes (webservers, webbrowsers, sshd, ...) and COW-sharing is crucial for reducing the memory footprint. We don't want to suddenly change the behavior to result in a significant increase in memory waste. Interestingly, khugepaged will only collapse an anonymous THP if at least one PTE is writable. After fork(), that means that something (usually a page fault) populated at least a single exclusive anonymous THP in that PMD range. So ... what happens when we switch to "is this folio mapped shared" instead of "is this page mapped shared" by using folio_likely_mapped_shared()? For "not-COW-shared" folios, small folios and for THPs (large folios) that are completely mapped into at least one process, switching to folio_likely_mapped_shared() will not result in a change. We'll only see a change for COW-shared PTE-mapped THPs that are partially mapped into all involved processes. There are two cases to consider: (A) folio_likely_mapped_shared() returns "false" for a PTE-mapped THP If the folio is detected as exclusive, and it actually is exclusive, there is no change: page_mapcount() == 1. This is the common case without fork() or with short-lived child processes. folio_likely_mapped_shared() might currently still detect a folio as exclusive although it is shared (false negatives): if the first page is not mapped multiple times and if the average per-page mapcount is smaller than 1, implying that (1) the folio is partially mapped and (2) if we are responsible for many mapcounts by mapping many pages others can't ("mostly exclusive") (3) if we are not responsible for many mapcounts by mapping little pages ("mostly shared") it won't make a big impact on the end result. So while we might now detect a page as "exclusive" although it isn't, it's not expected to make a big difference in common cases. (B) folio_likely_mapped_shared() returns "true" for a PTE-mapped THP folio_likely_mapped_shared() will never detect a large anonymous folio as shared although it is exclusive: there are no false positives. If we detect a THP as shared, at least one page of the THP is mapped by another process. It could well be that some pages are actually exclusive. For example, our child processes could have unmapped/COW'ed some pages such that they would now be exclusive to out process, which we now would treat as still-shared. Examples: (1) Parent maps all pages of a THP, child maps some pages. We detect all pages in the parent as shared although some are actually exclusive. (2) Parent maps all but some page of a THP, child maps the remainder. We detect all pages of the THP that the parent maps as shared although they are all exclusive. In (1) we wouldn't collapse a THP right now already: no PTE is writable, because a write fault would have resulted in COW of a single page and the parent would no longer map all pages of that THP. For (2) we would have collapsed a THP in the parent so far, now we wouldn't as long as the child process is still alive: unless the child process unmaps the remaining THP pages or we decide to split that THP. Possibly, the child COW'ed many pages, meaning that it's likely that we can populate a THP for our child first, and then for our parent. For (2), we are making really bad use of the THP in the first place (not even mapped completely in at least one process). If the THP would be completely partially mapped, it would be on the deferred split queue where we would split it lazily later. For short-running child processes, we don't particularly care. For long-running processes, the expectation is that such scenarios are rather rare: further, a THP might be best placed if most data in the PMD range is actually written, implying that we'll have to COW more pages first before khugepaged would collapse it. To summarize, in the common case, this change is not expected to matter much. The more common application of khugepaged operates on exclusive pages, either before fork() or after a child quit. Can we improve (A)? Yes, if we implement more precise tracking of "mapped shared" vs. "mapped exclusively", we could get rid of the false negatives completely. Can we improve (B)? We could count how many pages of a large folio we map inside the current page table and detect that we are responsible for most of the folio mapcount and conclude "as good as exclusive", which might help in some cases. ... but likely, some other mechanism should detect that the THP is not a good use in the scenario (not even mapped completely in a single process) and try splitting that folio lazily etc. We'll move the folio_test_anon() check before our "shared" check, so we might get more expressive results for SCAN_EXCEED_SHARED_PTE: this order of checks now matches the one in __collapse_huge_page_isolate(). Extend documentation. Link: https://lkml.kernel.org/r/20240424122630.495788-1-david@redhat.com Signed-off-by: David Hildenbrand Cc: Jonathan Corbet Cc: Kirill A. Shutemov Cc: Zi Yan Cc: Yang Shi Cc: John Hubbard Cc: Ryan Roberts Signed-off-by: Andrew Morton --- Documentation/admin-guide/mm/transhuge.rst | 3 ++- mm/khugepaged.c | 22 +++++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index f82300b9193fe..076443cc10a6c 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -278,7 +278,8 @@ collapsed, resulting fewer pages being collapsed into THPs, and lower memory access performance. ``max_ptes_shared`` specifies how many pages can be shared across multiple -processes. Exceeding the number would block the collapse:: +processes. khugepaged might treat pages of THPs as shared if any page of +that THP is shared. Exceeding the number would block the collapse:: /sys/kernel/mm/transparent_hugepage/khugepaged/max_ptes_shared diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 2f73d2aa9ae84..cf518fc440982 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -583,7 +583,8 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, folio = page_folio(page); VM_BUG_ON_FOLIO(!folio_test_anon(folio), folio); - if (page_mapcount(page) > 1) { + /* See hpage_collapse_scan_pmd(). */ + if (folio_likely_mapped_shared(folio)) { ++shared; if (cc->is_khugepaged && shared > khugepaged_max_ptes_shared) { @@ -1317,8 +1318,20 @@ static int hpage_collapse_scan_pmd(struct mm_struct *mm, result = SCAN_PAGE_NULL; goto out_unmap; } + folio = page_folio(page); - if (page_mapcount(page) > 1) { + if (!folio_test_anon(folio)) { + result = SCAN_PAGE_ANON; + goto out_unmap; + } + + /* + * We treat a single page as shared if any part of the THP + * is shared. "False negatives" from + * folio_likely_mapped_shared() are not expected to matter + * much in practice. + */ + if (folio_likely_mapped_shared(folio)) { ++shared; if (cc->is_khugepaged && shared > khugepaged_max_ptes_shared) { @@ -1328,7 +1341,6 @@ static int hpage_collapse_scan_pmd(struct mm_struct *mm, } } - folio = page_folio(page); /* * Record which node the original page is from and save this * information to cc->node_load[]. @@ -1349,10 +1361,6 @@ static int hpage_collapse_scan_pmd(struct mm_struct *mm, result = SCAN_PAGE_LOCK; goto out_unmap; } - if (!folio_test_anon(folio)) { - result = SCAN_PAGE_ANON; - goto out_unmap; - } /* * Check if the page has any GUP (or other external) pins. From 21e516b913c137cf955e50359e499c0994d4b9a9 Mon Sep 17 00:00:00 2001 From: Hariom Panthi Date: Wed, 24 Apr 2024 16:48:38 +0530 Subject: [PATCH 1215/1702] mm: vmalloc: dump page owner info if page is already mapped In vmap_pte_range, BUG_ON is called when page is already mapped, It doesn't give enough information to debug further. Dumping page owner information alongwith BUG_ON will be more useful in case of multiple page mapping. Example: [ 14.552875] page: refcount:1 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x10b923 [ 14.553440] flags: 0xbffff0000000000(node=0|zone=2|lastcpupid=0x3ffff) [ 14.554001] page_type: 0xffffffff() [ 14.554783] raw: 0bffff0000000000 0000000000000000 dead000000000122 0000000000000000 [ 14.555230] raw: 0000000000000000 0000000000000000 00000001ffffffff 0000000000000000 [ 14.555768] page dumped because: remapping already mapped page [ 14.556172] page_owner tracks the page as allocated [ 14.556482] page last allocated via order 0, migratetype Unmovable, gfp_mask 0xcc0(GFP_KERNEL), pid 80, tgid 80 (insmod), ts 14552004992, free_ts 0 [ 14.557286] prep_new_page+0xa8/0x10c [ 14.558052] get_page_from_freelist+0x7f8/0x1248 [ 14.558298] __alloc_pages+0x164/0x2b4 [ 14.558514] alloc_pages_mpol+0x88/0x230 [ 14.558904] alloc_pages+0x4c/0x7c [ 14.559157] load_module+0x74/0x1af4 [ 14.559361] __do_sys_init_module+0x190/0x1fc [ 14.559615] __arm64_sys_init_module+0x1c/0x28 [ 14.559883] invoke_syscall+0x44/0x108 [ 14.560109] el0_svc_common.constprop.0+0x40/0xe0 [ 14.560371] do_el0_svc_compat+0x1c/0x34 [ 14.560600] el0_svc_compat+0x2c/0x80 [ 14.560820] el0t_32_sync_handler+0x90/0x140 [ 14.561040] el0t_32_sync+0x194/0x198 [ 14.561329] page_owner free stack trace missing [ 14.562049] ------------[ cut here ]------------ [ 14.562314] kernel BUG at mm/vmalloc.c:113! Link: https://lkml.kernel.org/r/20240424111838.3782931-2-hariom1.p@samsung.com Signed-off-by: Hariom Panthi Cc: Christoph Hellwig Cc: Lorenzo Stoakes Cc: Maninder Singh Cc: Oscar Salvador Cc: Rohit Thapliyal Cc: Uladzislau Rezki (Sony) Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/vmalloc.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 6856fad1a8f8c..b00c2f5f70ea7 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -42,6 +42,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -96,6 +97,7 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, { pte_t *pte; u64 pfn; + struct page *page; unsigned long size = PAGE_SIZE; pfn = phys_addr >> PAGE_SHIFT; @@ -103,7 +105,13 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, if (!pte) return -ENOMEM; do { - BUG_ON(!pte_none(ptep_get(pte))); + if (!pte_none(ptep_get(pte))) { + if (pfn_valid(pfn)) { + page = pfn_to_page(pfn); + dump_page(page, "remapping already mapped page"); + } + BUG(); + } #ifdef CONFIG_HUGETLB_PAGE size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift); From 4673ad3bdca2678a8970c0076aa07f5302d914fb Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Wed, 24 Apr 2024 11:53:01 +0100 Subject: [PATCH 1216/1702] selftests/mm: soft-dirty should fail if a testcase fails Previously soft-dirty was unconditionally exiting with success, even if one of its testcases failed. Let's fix that so that failure can be reported to automated systems properly. Link: https://lkml.kernel.org/r/20240424105301.3157695-1-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Reviewed-by: Muhammad Usama Anjum Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/soft-dirty.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/mm/soft-dirty.c b/tools/testing/selftests/mm/soft-dirty.c index 7dbfa53d93a05..bdfa5d085f009 100644 --- a/tools/testing/selftests/mm/soft-dirty.c +++ b/tools/testing/selftests/mm/soft-dirty.c @@ -209,5 +209,5 @@ int main(int argc, char **argv) close(pagemap_fd); - return ksft_exit_pass(); + ksft_finished(); } From e32e27009fb0d42c9ce9e01c5d96769897d1fa70 Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Tue, 23 Apr 2024 11:46:39 +0800 Subject: [PATCH 1217/1702] writeback: collect stats of all wb of bdi in bdi_debug_stats_show Patch series "Improve visibility of writeback", v5. This series tries to improve visilibity of writeback. Patch 1 make /sys/kernel/debug/bdi/xxx/stats show writeback info of whole bdi instead of only writeback info in root cgroup. Patch 2 add a new debug file /sys/kernel/debug/bdi/xxx/wb_stats to show per wb writeback info. Patch 3 add wb_monitor.py to monitor basic writeback info of running system, more info could be added on demand. Patch 4 is a random cleanup. More details can be found in respective patches. Following domain hierarchy is tested: global domain (320G) / \ cgroup domain1(10G) cgroup domain2(10G) | | bdi wb1 wb2 /* all writeback info of bdi is successfully collected */ cat stats BdiWriteback: 4704 kB BdiReclaimable: 1294496 kB BdiDirtyThresh: 204208088 kB DirtyThresh: 195259944 kB BackgroundThresh: 32503588 kB BdiDirtied: 48519296 kB BdiWritten: 47225696 kB BdiWriteBandwidth: 1173892 kBps b_dirty: 1 b_io: 0 b_more_io: 1 b_dirty_time: 0 bdi_list: 1 state: 1 /* per wb writeback info of bdi is collected */ cat /sys/kernel/debug/bdi/252:16/wb_stats WbCgIno: 1 WbWriteback: 0 kB WbReclaimable: 0 kB WbDirtyThresh: 0 kB WbDirtied: 0 kB WbWritten: 0 kB WbWriteBandwidth: 102400 kBps b_dirty: 0 b_io: 0 b_more_io: 0 b_dirty_time: 0 state: 1 WbCgIno: 4208 WbWriteback: 59808 kB WbReclaimable: 676480 kB WbDirtyThresh: 6004624 kB WbDirtied: 23348192 kB WbWritten: 22614592 kB WbWriteBandwidth: 593204 kBps b_dirty: 1 b_io: 1 b_more_io: 0 b_dirty_time: 0 state: 7 WbCgIno: 4249 WbWriteback: 144256 kB WbReclaimable: 432096 kB WbDirtyThresh: 6004344 kB WbDirtied: 25727744 kB WbWritten: 25154752 kB WbWriteBandwidth: 577904 kBps b_dirty: 0 b_io: 1 b_more_io: 0 b_dirty_time: 0 state: 7 The wb_monitor.py script output is as following: ./wb_monitor.py 252:16 -c writeback reclaimable dirtied written avg_bw 252:16_1 0 0 0 0 102400 252:16_4284 672 820064 9230368 8410304 685612 252:16_4325 896 819840 10491264 9671648 652348 252:16 1568 1639904 19721632 18081952 1440360 writeback reclaimable dirtied written avg_bw 252:16_1 0 0 0 0 102400 252:16_4284 672 820064 9230368 8410304 685612 252:16_4325 896 819840 10491264 9671648 652348 252:16 1568 1639904 19721632 18081952 1440360 ... This patch (of 5): /sys/kernel/debug/bdi/xxx/stats is supposed to show writeback information of whole bdi, but only writeback information of bdi in root cgroup is collected. So writeback information in non-root cgroup are missing now. To be more specific, considering following case: /* create writeback cgroup */ cd /sys/fs/cgroup echo "+memory +io" > cgroup.subtree_control mkdir group1 cd group1 echo $$ > cgroup.procs /* do writeback in cgroup */ fio -name test -filename=/dev/vdb ... /* get writeback info of bdi */ cat /sys/kernel/debug/bdi/xxx/stats The cat result unexpectedly implies that there is no writeback on target bdi. Fix this by collecting stats of all wb in bdi instead of only wb in root cgroup. Following domain hierarchy is tested: global domain (320G) / \ cgroup domain1(10G) cgroup domain2(10G) | | bdi wb1 wb2 /* all writeback info of bdi is successfully collected */ cat stats BdiWriteback: 2912 kB BdiReclaimable: 1598464 kB BdiDirtyThresh: 167479028 kB DirtyThresh: 195038532 kB BackgroundThresh: 32466728 kB BdiDirtied: 19141696 kB BdiWritten: 17543456 kB BdiWriteBandwidth: 1136172 kBps b_dirty: 2 b_io: 0 b_more_io: 1 b_dirty_time: 0 bdi_list: 1 state: 1 Link: https://lkml.kernel.org/r/20240423034643.141219-1-shikemeng@huaweicloud.com Link: https://lkml.kernel.org/r/20240423034643.141219-2-shikemeng@huaweicloud.com Signed-off-by: Kemeng Shi Acked-by: Tejun Heo Cc: Brian Foster Cc: David Howells Cc: David Sterba Cc: Jan Kara Cc: Mateusz Guzik Cc: Matthew Wilcox (Oracle) Cc: SeongJae Park Cc: Stephen Rothwell Signed-off-by: Andrew Morton --- mm/backing-dev.c | 96 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 73 insertions(+), 23 deletions(-) diff --git a/mm/backing-dev.c b/mm/backing-dev.c index 5fa3666356f95..089146feb8301 100644 --- a/mm/backing-dev.c +++ b/mm/backing-dev.c @@ -39,6 +39,19 @@ struct workqueue_struct *bdi_wq; #include #include +struct wb_stats { + unsigned long nr_dirty; + unsigned long nr_io; + unsigned long nr_more_io; + unsigned long nr_dirty_time; + unsigned long nr_writeback; + unsigned long nr_reclaimable; + unsigned long nr_dirtied; + unsigned long nr_written; + unsigned long dirty_thresh; + unsigned long wb_thresh; +}; + static struct dentry *bdi_debug_root; static void bdi_debug_init(void) @@ -46,31 +59,68 @@ static void bdi_debug_init(void) bdi_debug_root = debugfs_create_dir("bdi", NULL); } -static int bdi_debug_stats_show(struct seq_file *m, void *v) +static void collect_wb_stats(struct wb_stats *stats, + struct bdi_writeback *wb) { - struct backing_dev_info *bdi = m->private; - struct bdi_writeback *wb = &bdi->wb; - unsigned long background_thresh; - unsigned long dirty_thresh; - unsigned long wb_thresh; - unsigned long nr_dirty, nr_io, nr_more_io, nr_dirty_time; struct inode *inode; - nr_dirty = nr_io = nr_more_io = nr_dirty_time = 0; spin_lock(&wb->list_lock); list_for_each_entry(inode, &wb->b_dirty, i_io_list) - nr_dirty++; + stats->nr_dirty++; list_for_each_entry(inode, &wb->b_io, i_io_list) - nr_io++; + stats->nr_io++; list_for_each_entry(inode, &wb->b_more_io, i_io_list) - nr_more_io++; + stats->nr_more_io++; list_for_each_entry(inode, &wb->b_dirty_time, i_io_list) if (inode->i_state & I_DIRTY_TIME) - nr_dirty_time++; + stats->nr_dirty_time++; spin_unlock(&wb->list_lock); + stats->nr_writeback += wb_stat(wb, WB_WRITEBACK); + stats->nr_reclaimable += wb_stat(wb, WB_RECLAIMABLE); + stats->nr_dirtied += wb_stat(wb, WB_DIRTIED); + stats->nr_written += wb_stat(wb, WB_WRITTEN); + stats->wb_thresh += wb_calc_thresh(wb, stats->dirty_thresh); +} + +#ifdef CONFIG_CGROUP_WRITEBACK +static void bdi_collect_stats(struct backing_dev_info *bdi, + struct wb_stats *stats) +{ + struct bdi_writeback *wb; + + rcu_read_lock(); + list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node) { + if (!wb_tryget(wb)) + continue; + + collect_wb_stats(stats, wb); + wb_put(wb); + } + rcu_read_unlock(); +} +#else +static void bdi_collect_stats(struct backing_dev_info *bdi, + struct wb_stats *stats) +{ + collect_wb_stats(stats, &bdi->wb); +} +#endif + +static int bdi_debug_stats_show(struct seq_file *m, void *v) +{ + struct backing_dev_info *bdi = m->private; + unsigned long background_thresh; + unsigned long dirty_thresh; + struct wb_stats stats; + unsigned long tot_bw; + global_dirty_limits(&background_thresh, &dirty_thresh); - wb_thresh = wb_calc_thresh(wb, dirty_thresh); + + memset(&stats, 0, sizeof(stats)); + stats.dirty_thresh = dirty_thresh; + bdi_collect_stats(bdi, &stats); + tot_bw = atomic_long_read(&bdi->tot_write_bandwidth); seq_printf(m, "BdiWriteback: %10lu kB\n" @@ -87,18 +137,18 @@ static int bdi_debug_stats_show(struct seq_file *m, void *v) "b_dirty_time: %10lu\n" "bdi_list: %10u\n" "state: %10lx\n", - (unsigned long) K(wb_stat(wb, WB_WRITEBACK)), - (unsigned long) K(wb_stat(wb, WB_RECLAIMABLE)), - K(wb_thresh), + K(stats.nr_writeback), + K(stats.nr_reclaimable), + K(stats.wb_thresh), K(dirty_thresh), K(background_thresh), - (unsigned long) K(wb_stat(wb, WB_DIRTIED)), - (unsigned long) K(wb_stat(wb, WB_WRITTEN)), - (unsigned long) K(wb->write_bandwidth), - nr_dirty, - nr_io, - nr_more_io, - nr_dirty_time, + K(stats.nr_dirtied), + K(stats.nr_written), + K(tot_bw), + stats.nr_dirty, + stats.nr_io, + stats.nr_more_io, + stats.nr_dirty_time, !list_empty(&bdi->bdi_list), bdi->wb.state); return 0; From 4b5bbc39d7a637a135fd8b3ef86e60a6a7301e9d Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Tue, 23 Apr 2024 11:46:40 +0800 Subject: [PATCH 1218/1702] writeback: support retrieving per group debug writeback stats of bdi Add /sys/kernel/debug/bdi/xxx/wb_stats to show per group writeback stats of bdi. Following domain hierarchy is tested: global domain (320G) / \ cgroup domain1(10G) cgroup domain2(10G) | | bdi wb1 wb2 /* per wb writeback info of bdi is collected */ cat wb_stats WbCgIno: 1 WbWriteback: 0 kB WbReclaimable: 0 kB WbDirtyThresh: 0 kB WbDirtied: 0 kB WbWritten: 0 kB WbWriteBandwidth: 102400 kBps b_dirty: 0 b_io: 0 b_more_io: 0 b_dirty_time: 0 state: 1 WbCgIno: 4091 WbWriteback: 1792 kB WbReclaimable: 820512 kB WbDirtyThresh: 6004692 kB WbDirtied: 1820448 kB WbWritten: 999488 kB WbWriteBandwidth: 169020 kBps b_dirty: 0 b_io: 0 b_more_io: 1 b_dirty_time: 0 state: 5 WbCgIno: 4131 WbWriteback: 1120 kB WbReclaimable: 820064 kB WbDirtyThresh: 6004728 kB WbDirtied: 1822688 kB WbWritten: 1002400 kB WbWriteBandwidth: 153520 kBps b_dirty: 0 b_io: 0 b_more_io: 1 b_dirty_time: 0 state: 5 [shikemeng@huaweicloud.com: fix build problems] Link: https://lkml.kernel.org/r/20240423034643.141219-4-shikemeng@huaweicloud.com Link: https://lkml.kernel.org/r/20240423034643.141219-3-shikemeng@huaweicloud.com Signed-off-by: Kemeng Shi Cc: Brian Foster Cc: David Howells Cc: David Sterba Cc: Jan Kara Cc: Mateusz Guzik Cc: Matthew Wilcox (Oracle) Cc: SeongJae Park Cc: Stephen Rothwell Cc: Tejun Heo Signed-off-by: Andrew Morton --- include/linux/writeback.h | 1 + mm/backing-dev.c | 81 ++++++++++++++++++++++++++++++++++++++- mm/page-writeback.c | 19 +++++++++ 3 files changed, 99 insertions(+), 2 deletions(-) diff --git a/include/linux/writeback.h b/include/linux/writeback.h index 9845cb62e40b2..112d806ddbe49 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -355,6 +355,7 @@ int dirtytime_interval_handler(struct ctl_table *table, int write, void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty); unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh); +unsigned long cgwb_calc_thresh(struct bdi_writeback *wb); void wb_update_bandwidth(struct bdi_writeback *wb); diff --git a/mm/backing-dev.c b/mm/backing-dev.c index 089146feb8301..e61bbb1bd6221 100644 --- a/mm/backing-dev.c +++ b/mm/backing-dev.c @@ -155,19 +155,96 @@ static int bdi_debug_stats_show(struct seq_file *m, void *v) } DEFINE_SHOW_ATTRIBUTE(bdi_debug_stats); +static void wb_stats_show(struct seq_file *m, struct bdi_writeback *wb, + struct wb_stats *stats) +{ + + seq_printf(m, + "WbCgIno: %10lu\n" + "WbWriteback: %10lu kB\n" + "WbReclaimable: %10lu kB\n" + "WbDirtyThresh: %10lu kB\n" + "WbDirtied: %10lu kB\n" + "WbWritten: %10lu kB\n" + "WbWriteBandwidth: %10lu kBps\n" + "b_dirty: %10lu\n" + "b_io: %10lu\n" + "b_more_io: %10lu\n" + "b_dirty_time: %10lu\n" + "state: %10lx\n\n", +#ifdef CONFIG_CGROUP_WRITEBACK + cgroup_ino(wb->memcg_css->cgroup), +#else + 1ul, +#endif + K(stats->nr_writeback), + K(stats->nr_reclaimable), + K(stats->wb_thresh), + K(stats->nr_dirtied), + K(stats->nr_written), + K(wb->avg_write_bandwidth), + stats->nr_dirty, + stats->nr_io, + stats->nr_more_io, + stats->nr_dirty_time, + wb->state); +} + +static int cgwb_debug_stats_show(struct seq_file *m, void *v) +{ + struct backing_dev_info *bdi = m->private; + unsigned long background_thresh; + unsigned long dirty_thresh; + struct bdi_writeback *wb; + + global_dirty_limits(&background_thresh, &dirty_thresh); + + rcu_read_lock(); + list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node) { + struct wb_stats stats = { .dirty_thresh = dirty_thresh }; + + if (!wb_tryget(wb)) + continue; + + collect_wb_stats(&stats, wb); + + /* + * Calculate thresh of wb in writeback cgroup which is min of + * thresh in global domain and thresh in cgroup domain. Drop + * rcu lock because cgwb_calc_thresh may sleep in + * cgroup_rstat_flush. We can do so here because we have a ref. + */ + if (mem_cgroup_wb_domain(wb)) { + rcu_read_unlock(); + stats.wb_thresh = min(stats.wb_thresh, cgwb_calc_thresh(wb)); + rcu_read_lock(); + } + + wb_stats_show(m, wb, &stats); + + wb_put(wb); + } + rcu_read_unlock(); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(cgwb_debug_stats); + static void bdi_debug_register(struct backing_dev_info *bdi, const char *name) { bdi->debug_dir = debugfs_create_dir(name, bdi_debug_root); debugfs_create_file("stats", 0444, bdi->debug_dir, bdi, &bdi_debug_stats_fops); + debugfs_create_file("wb_stats", 0444, bdi->debug_dir, bdi, + &cgwb_debug_stats_fops); } static void bdi_debug_unregister(struct backing_dev_info *bdi) { debugfs_remove_recursive(bdi->debug_dir); } -#else +#else /* CONFIG_DEBUG_FS */ static inline void bdi_debug_init(void) { } @@ -178,7 +255,7 @@ static inline void bdi_debug_register(struct backing_dev_info *bdi, static inline void bdi_debug_unregister(struct backing_dev_info *bdi) { } -#endif +#endif /* CONFIG_DEBUG_FS */ static ssize_t read_ahead_kb_store(struct device *dev, struct device_attribute *attr, diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 2c1f595d1ddc4..c5fb3121469ad 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -892,6 +892,25 @@ unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh) return __wb_calc_thresh(&gdtc); } +unsigned long cgwb_calc_thresh(struct bdi_writeback *wb) +{ + struct dirty_throttle_control gdtc = { GDTC_INIT_NO_WB }; + struct dirty_throttle_control mdtc = { MDTC_INIT(wb, &gdtc) }; + unsigned long filepages = 0, headroom = 0, writeback = 0; + + gdtc.avail = global_dirtyable_memory(); + gdtc.dirty = global_node_page_state(NR_FILE_DIRTY) + + global_node_page_state(NR_WRITEBACK); + + mem_cgroup_wb_stats(wb, &filepages, &headroom, + &mdtc.dirty, &writeback); + mdtc.dirty += writeback; + mdtc_calc_avail(&mdtc, filepages, headroom); + domain_dirty_limits(&mdtc); + + return __wb_calc_thresh(&mdtc); +} + /* * setpoint - dirty 3 * f(dirty) := 1.0 + (----------------) From 881f1bb5e25c8982ed963b2d319fc0fc732e55db Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Tue, 23 Apr 2024 11:46:42 +0800 Subject: [PATCH 1219/1702] writeback: add wb_monitor.py script to monitor writeback info on bdi Add wb_monitor.py script to monitor writeback information on backing dev which makes it easier and more convenient to observe writeback behaviors of running system. The wb_monitor.py script is written based on wq_monitor.py. Following domain hierarchy is tested: global domain (320G) / \ cgroup domain1(10G) cgroup domain2(10G) | | bdi wb1 wb2 The wb_monitor.py script output is as following: ./wb_monitor.py 252:16 -c writeback reclaimable dirtied written avg_bw 252:16_1 0 0 0 0 102400 252:16_4284 672 820064 9230368 8410304 685612 252:16_4325 896 819840 10491264 9671648 652348 252:16 1568 1639904 19721632 18081952 1440360 writeback reclaimable dirtied written avg_bw 252:16_1 0 0 0 0 102400 252:16_4284 672 820064 9230368 8410304 685612 252:16_4325 896 819840 10491264 9671648 652348 252:16 1568 1639904 19721632 18081952 1440360 ... Link: https://lkml.kernel.org/r/20240423034643.141219-5-shikemeng@huaweicloud.com Signed-off-by: Kemeng Shi Suggested-by: Tejun Heo Cc: Brian Foster Cc: David Howells Cc: David Sterba Cc: Jan Kara Cc: Mateusz Guzik Cc: Matthew Wilcox (Oracle) Cc: SeongJae Park Cc: Stephen Rothwell Signed-off-by: Andrew Morton --- tools/writeback/wb_monitor.py | 172 ++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 tools/writeback/wb_monitor.py diff --git a/tools/writeback/wb_monitor.py b/tools/writeback/wb_monitor.py new file mode 100644 index 0000000000000..5e3591f1f9a9d --- /dev/null +++ b/tools/writeback/wb_monitor.py @@ -0,0 +1,172 @@ +#!/usr/bin/env drgn +# +# Copyright (C) 2024 Kemeng Shi +# Copyright (C) 2024 Huawei Inc + +desc = """ +This is a drgn script based on wq_monitor.py to monitor writeback info on +backing dev. For more info on drgn, visit https://github.com/osandov/drgn. + + writeback(kB) Amount of dirty pages are currently being written back to + disk. + + reclaimable(kB) Amount of pages are currently reclaimable. + + dirtied(kB) Amount of pages have been dirtied. + + wrttien(kB) Amount of dirty pages have been written back to disk. + + avg_wb(kBps) Smoothly estimated write bandwidth of writing dirty pages + back to disk. +""" + +import signal +import re +import time +import json + +import drgn +from drgn.helpers.linux.list import list_for_each_entry + +import argparse +parser = argparse.ArgumentParser(description=desc, + formatter_class=argparse.RawTextHelpFormatter) +parser.add_argument('bdi', metavar='REGEX', nargs='*', + help='Target backing device name patterns (all if empty)') +parser.add_argument('-i', '--interval', metavar='SECS', type=float, default=1, + help='Monitoring interval (0 to print once and exit)') +parser.add_argument('-j', '--json', action='store_true', + help='Output in json') +parser.add_argument('-c', '--cgroup', action='store_true', + help='show writeback of bdi in cgroup') +args = parser.parse_args() + +bdi_list = prog['bdi_list'] + +WB_RECLAIMABLE = prog['WB_RECLAIMABLE'] +WB_WRITEBACK = prog['WB_WRITEBACK'] +WB_DIRTIED = prog['WB_DIRTIED'] +WB_WRITTEN = prog['WB_WRITTEN'] +NR_WB_STAT_ITEMS = prog['NR_WB_STAT_ITEMS'] + +PAGE_SHIFT = prog['PAGE_SHIFT'] + +def K(x): + return x << (PAGE_SHIFT - 10) + +class Stats: + def dict(self, now): + return { 'timestamp' : now, + 'name' : self.name, + 'writeback' : self.stats[WB_WRITEBACK], + 'reclaimable' : self.stats[WB_RECLAIMABLE], + 'dirtied' : self.stats[WB_DIRTIED], + 'written' : self.stats[WB_WRITTEN], + 'avg_wb' : self.avg_bw, } + + def table_header_str(): + return f'{"":>16} {"writeback":>10} {"reclaimable":>12} ' \ + f'{"dirtied":>9} {"written":>9} {"avg_bw":>9}' + + def table_row_str(self): + out = f'{self.name[-16:]:16} ' \ + f'{self.stats[WB_WRITEBACK]:10} ' \ + f'{self.stats[WB_RECLAIMABLE]:12} ' \ + f'{self.stats[WB_DIRTIED]:9} ' \ + f'{self.stats[WB_WRITTEN]:9} ' \ + f'{self.avg_bw:9} ' + return out + + def show_header(): + if Stats.table_fmt: + print() + print(Stats.table_header_str()) + + def show_stats(self): + if Stats.table_fmt: + print(self.table_row_str()) + else: + print(self.dict(Stats.now)) + +class WbStats(Stats): + def __init__(self, wb): + bdi_name = wb.bdi.dev_name.string_().decode() + # avoid to use bdi.wb.memcg_css which is only defined when + # CONFIG_CGROUP_WRITEBACK is enabled + if wb == wb.bdi.wb.address_of_(): + ino = "1" + else: + ino = str(wb.memcg_css.cgroup.kn.id.value_()) + self.name = bdi_name + '_' + ino + + self.stats = [0] * NR_WB_STAT_ITEMS + for i in range(NR_WB_STAT_ITEMS): + if wb.stat[i].count >= 0: + self.stats[i] = int(K(wb.stat[i].count)) + else: + self.stats[i] = 0 + + self.avg_bw = int(K(wb.avg_write_bandwidth)) + +class BdiStats(Stats): + def __init__(self, bdi): + self.name = bdi.dev_name.string_().decode() + self.stats = [0] * NR_WB_STAT_ITEMS + self.avg_bw = 0 + + def collectStats(self, wb_stats): + for i in range(NR_WB_STAT_ITEMS): + self.stats[i] += wb_stats.stats[i] + + self.avg_bw += wb_stats.avg_bw + +exit_req = False + +def sigint_handler(signr, frame): + global exit_req + exit_req = True + +def main(): + # handle args + Stats.table_fmt = not args.json + interval = args.interval + cgroup = args.cgroup + + re_str = None + if args.bdi: + for r in args.bdi: + if re_str is None: + re_str = r + else: + re_str += '|' + r + + filter_re = re.compile(re_str) if re_str else None + + # monitoring loop + signal.signal(signal.SIGINT, sigint_handler) + + while not exit_req: + Stats.now = time.time() + + Stats.show_header() + for bdi in list_for_each_entry('struct backing_dev_info', bdi_list.address_of_(), 'bdi_list'): + bdi_stats = BdiStats(bdi) + if filter_re and not filter_re.search(bdi_stats.name): + continue + + for wb in list_for_each_entry('struct bdi_writeback', bdi.wb_list.address_of_(), 'bdi_node'): + wb_stats = WbStats(wb) + bdi_stats.collectStats(wb_stats) + if cgroup: + wb_stats.show_stats() + + bdi_stats.show_stats() + if cgroup and Stats.table_fmt: + print() + + if interval == 0: + break + time.sleep(interval) + +if __name__ == "__main__": + main() From 826881a7f66557143df5c7f9f401509aa1a7fa5c Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Tue, 23 Apr 2024 11:46:43 +0800 Subject: [PATCH 1220/1702] writeback: rename nr_reclaimable to nr_dirty in balance_dirty_pages Commit 8d92890bd6b85 ("mm/writeback: discard NR_UNSTABLE_NFS, use NR_WRITEBACK instead") removed NR_UNSTABLE_NFS and nr_reclaimable only contains dirty page now. Rename nr_reclaimable to nr_dirty properly. Link: https://lkml.kernel.org/r/20240423034643.141219-6-shikemeng@huaweicloud.com Signed-off-by: Kemeng Shi Reviewed-by: Jan Kara Cc: Brian Foster Cc: David Howells Cc: David Sterba Cc: Mateusz Guzik Cc: Matthew Wilcox (Oracle) Cc: SeongJae Park Cc: Stephen Rothwell Cc: Tejun Heo Signed-off-by: Andrew Morton --- mm/page-writeback.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mm/page-writeback.c b/mm/page-writeback.c index c5fb3121469ad..db5769cb12fd5 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -1694,7 +1694,7 @@ static int balance_dirty_pages(struct bdi_writeback *wb, struct dirty_throttle_control * const mdtc = mdtc_valid(&mdtc_stor) ? &mdtc_stor : NULL; struct dirty_throttle_control *sdtc; - unsigned long nr_reclaimable; /* = file_dirty */ + unsigned long nr_dirty; long period; long pause; long max_pause; @@ -1715,9 +1715,9 @@ static int balance_dirty_pages(struct bdi_writeback *wb, unsigned long m_thresh = 0; unsigned long m_bg_thresh = 0; - nr_reclaimable = global_node_page_state(NR_FILE_DIRTY); + nr_dirty = global_node_page_state(NR_FILE_DIRTY); gdtc->avail = global_dirtyable_memory(); - gdtc->dirty = nr_reclaimable + global_node_page_state(NR_WRITEBACK); + gdtc->dirty = nr_dirty + global_node_page_state(NR_WRITEBACK); domain_dirty_limits(gdtc); @@ -1768,7 +1768,7 @@ static int balance_dirty_pages(struct bdi_writeback *wb, * In normal mode, we start background writeout at the lower * background_thresh, to keep the amount of dirty memory low. */ - if (!laptop_mode && nr_reclaimable > gdtc->bg_thresh && + if (!laptop_mode && nr_dirty > gdtc->bg_thresh && !writeback_in_progress(wb)) wb_start_background_writeback(wb); From 13fc441284b39ae8fc394547c032c2dad2268406 Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Thu, 25 Apr 2024 21:17:21 +0800 Subject: [PATCH 1221/1702] mm: enable __wb_calc_thresh to calculate dirty background threshold Patch series "Fix and cleanups to page-writeback", v2. This series contains some random cleanups and a fix to correct calculation of wb's bg_thresh in cgroup domain. More details can be found respective patches. This patch (of 4): Originally, __wb_calc_thresh always calculate wb's share of dirty throttling threshold. By getting thresh of wb_domain from caller, __wb_calc_thresh could be used for both dirty throttling and dirty background threshold. This is a preparation to correct threshold calculation of wb in cgroup. Link: https://lkml.kernel.org/r/20240425131724.36778-1-shikemeng@huaweicloud.com Link: https://lkml.kernel.org/r/20240425131724.36778-2-shikemeng@huaweicloud.com Signed-off-by: Kemeng Shi Reviewed-by: Jan Kara Cc: Howard Cochran Cc: Jens Axboe Cc: Matthew Wilcox (Oracle) Cc: Miklos Szeredi Cc: Tejun Heo Signed-off-by: Andrew Morton --- mm/page-writeback.c | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/mm/page-writeback.c b/mm/page-writeback.c index db5769cb12fd5..2a3b68aae336d 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -838,13 +838,15 @@ static void mdtc_calc_avail(struct dirty_throttle_control *mdtc, } /** - * __wb_calc_thresh - @wb's share of dirty throttling threshold + * __wb_calc_thresh - @wb's share of dirty threshold * @dtc: dirty_throttle_context of interest + * @thresh: dirty throttling or dirty background threshold of wb_domain in @dtc * - * Note that balance_dirty_pages() will only seriously take it as a hard limit - * when sleeping max_pause per page is not enough to keep the dirty pages under - * control. For example, when the device is completely stalled due to some error - * conditions, or when there are 1000 dd tasks writing to a slow 10MB/s USB key. + * Note that balance_dirty_pages() will only seriously take dirty throttling + * threshold as a hard limit when sleeping max_pause per page is not enough + * to keep the dirty pages under control. For example, when the device is + * completely stalled due to some error conditions, or when there are 1000 + * dd tasks writing to a slow 10MB/s USB key. * In the other normal situations, it acts more gently by throttling the tasks * more (rather than completely block them) when the wb dirty pages go high. * @@ -855,19 +857,20 @@ static void mdtc_calc_avail(struct dirty_throttle_control *mdtc, * The wb's share of dirty limit will be adapting to its throughput and * bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set. * - * Return: @wb's dirty limit in pages. The term "dirty" in the context of - * dirty balancing includes all PG_dirty and PG_writeback pages. + * Return: @wb's dirty limit in pages. For dirty throttling limit, the term + * "dirty" in the context of dirty balancing includes all PG_dirty and + * PG_writeback pages. */ -static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc) +static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc, + unsigned long thresh) { struct wb_domain *dom = dtc_dom(dtc); - unsigned long thresh = dtc->thresh; u64 wb_thresh; unsigned long numerator, denominator; unsigned long wb_min_ratio, wb_max_ratio; /* - * Calculate this BDI's share of the thresh ratio. + * Calculate this wb's share of the thresh ratio. */ fprop_fraction_percpu(&dom->completions, dtc->wb_completions, &numerator, &denominator); @@ -887,9 +890,9 @@ static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc) unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh) { - struct dirty_throttle_control gdtc = { GDTC_INIT(wb), - .thresh = thresh }; - return __wb_calc_thresh(&gdtc); + struct dirty_throttle_control gdtc = { GDTC_INIT(wb) }; + + return __wb_calc_thresh(&gdtc, thresh); } unsigned long cgwb_calc_thresh(struct bdi_writeback *wb) @@ -908,7 +911,7 @@ unsigned long cgwb_calc_thresh(struct bdi_writeback *wb) mdtc_calc_avail(&mdtc, filepages, headroom); domain_dirty_limits(&mdtc); - return __wb_calc_thresh(&mdtc); + return __wb_calc_thresh(&mdtc, mdtc.thresh); } /* @@ -1655,7 +1658,7 @@ static inline void wb_dirty_limits(struct dirty_throttle_control *dtc) * wb_position_ratio() will let the dirtier task progress * at some rate <= (write_bw / 2) for bringing down wb_dirty. */ - dtc->wb_thresh = __wb_calc_thresh(dtc); + dtc->wb_thresh = __wb_calc_thresh(dtc, dtc->thresh); dtc->wb_bg_thresh = dtc->thresh ? div64_u64(dtc->wb_thresh * dtc->bg_thresh, dtc->thresh) : 0; From fabd2e42bc71d78d1e9dcd5af453d1ab86cc2a7e Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Thu, 25 Apr 2024 21:17:22 +0800 Subject: [PATCH 1222/1702] mm: correct calculation of wb's bg_thresh in cgroup domain wb_calc_thresh() is calculating wb's share of bg_thresh in the global domain. However in case of cgroup writeback this is not the right thing to do. Consider the following domain hierarchy: global domain (> 20G) / \ cgroup1 (10G) cgroup2 (10G) | | bdi wb1 wb2 and assume wb1 and wb2 have the same bandwidth and the background threshold is set at 10%. The bg_thresh of cgroup1 and cgroup2 is going to be 1G. Now because wb_calc_thresh(mdtc->wb, mdtc->bg_thresh) calculates per-wb threshold in the global domain as (wb bandwidth) / (domain bandwidth) it returns bg_thresh for wb1 as 0.5G although it has nobody to compete against in cgroup1. Fix the problem by calculating wb's share of bg_thresh in the cgroup domain. Test as following: /* make it easier to observe the issue */ echo 300000 > /proc/sys/vm/dirty_expire_centisecs echo 100 > /proc/sys/vm/dirty_writeback_centisecs /* run fio in wb1 */ cd /sys/fs/cgroup echo "+memory +io" > cgroup.subtree_control mkdir group1 cd group1 echo 10G > memory.high echo 10G > memory.max echo $$ > cgroup.procs mkfs.ext4 -F /dev/vdb mount /dev/vdb /bdi1/ fio -name test -filename=/bdi1/file -size=600M -ioengine=libaio -bs=4K \ -iodepth=1 -rw=write -direct=0 --time_based -runtime=600 -invalidate=0 /* run fio in wb2 with a new shell */ cd /sys/fs/cgroup mkdir group2 cd group2 echo 10G > memory.high echo 10G > memory.max echo $$ > cgroup.procs mkfs.ext4 -F /dev/vdc mount /dev/vdc /bdi2/ fio -name test -filename=/bdi2/file -size=600M -ioengine=libaio -bs=4K \ -iodepth=1 -rw=write -direct=0 --time_based -runtime=600 -invalidate=0 Before fix, the wrttien pages of wb1 and wb2 reported from toos/writeback/wb_monitor.py keep growing. After fix, rare written pages are accumulated. There is no obvious change in fio result. [jack@suse.cz: changelog rewording] Link: https://lkml.kernel.org/r/20240425131724.36778-3-shikemeng@huaweicloud.com Fixes: 74d369443325 ("writeback: Fix performance regression in wb_over_bg_thresh()") Signed-off-by: Kemeng Shi Reviewed-by: Jan Kara Cc: Howard Cochran Cc: Jens Axboe Cc: Matthew Wilcox (Oracle) Cc: Miklos Szeredi Cc: Tejun Heo Signed-off-by: Andrew Morton --- mm/page-writeback.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 2a3b68aae336d..14893b20d38cc 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -2137,7 +2137,7 @@ bool wb_over_bg_thresh(struct bdi_writeback *wb) if (mdtc->dirty > mdtc->bg_thresh) return true; - thresh = wb_calc_thresh(mdtc->wb, mdtc->bg_thresh); + thresh = __wb_calc_thresh(mdtc, mdtc->bg_thresh); if (thresh < 2 * wb_stat_error()) reclaimable = wb_stat_sum(wb, WB_RECLAIMABLE); else From 3b3e412e5f4800f3423f7eea6713254d990fe22d Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Thu, 25 Apr 2024 21:17:23 +0800 Subject: [PATCH 1223/1702] mm: call __wb_calc_thresh instead of wb_calc_thresh in wb_over_bg_thresh Call __wb_calc_thresh to calculate wb bg_thresh of gdtc in wb_over_bg_thresh to remove unnecessary wrap in wb_calc_thresh. Link: https://lkml.kernel.org/r/20240425131724.36778-4-shikemeng@huaweicloud.com Signed-off-by: Kemeng Shi Reviewed-by: Jan Kara Cc: Howard Cochran Cc: Jens Axboe Cc: Matthew Wilcox (Oracle) Cc: Miklos Szeredi Cc: Tejun Heo Signed-off-by: Andrew Morton --- mm/page-writeback.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 14893b20d38cc..22e1acec899ea 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -2117,7 +2117,7 @@ bool wb_over_bg_thresh(struct bdi_writeback *wb) if (gdtc->dirty > gdtc->bg_thresh) return true; - thresh = wb_calc_thresh(gdtc->wb, gdtc->bg_thresh); + thresh = __wb_calc_thresh(gdtc, gdtc->bg_thresh); if (thresh < 2 * wb_stat_error()) reclaimable = wb_stat_sum(wb, WB_RECLAIMABLE); else From dc6e0ae5b1700c54a9c34daf3913adb40b6ddbad Mon Sep 17 00:00:00 2001 From: Kemeng Shi Date: Thu, 25 Apr 2024 21:17:24 +0800 Subject: [PATCH 1224/1702] mm: remove stale comment __folio_mark_dirty The __folio_mark_dirty will not mark inode dirty any longer. Remove the stale comment of it. Link: https://lkml.kernel.org/r/20240425131724.36778-5-shikemeng@huaweicloud.com Signed-off-by: Kemeng Shi Reviewed-by: Matthew Wilcox (Oracle) Reviewed-by: Jan Kara Cc: Howard Cochran Cc: Jens Axboe Cc: Miklos Szeredi Cc: Tejun Heo Signed-off-by: Andrew Morton --- mm/page-writeback.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 22e1acec899ea..692c0da04cbd0 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -2721,8 +2721,7 @@ void folio_account_cleaned(struct folio *folio, struct bdi_writeback *wb) } /* - * Mark the folio dirty, and set it dirty in the page cache, and mark - * the inode dirty. + * Mark the folio dirty, and set it dirty in the page cache. * * If warn is true, then emit a warning if the folio is not uptodate and has * not been truncated. From e0ffb29bc54d86b9ab10ebafc66eb1b7229e0cd7 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 25 Apr 2024 05:00:55 +0100 Subject: [PATCH 1225/1702] mm: simplify thp_vma_allowable_order Combine the three boolean arguments into one flags argument for readability. Signed-off-by: Matthew Wilcox (Oracle) Cc: David Hildenbrand Cc: Kefeng Wang Cc: Ryan Roberts Signed-off-by: Andrew Morton --- fs/proc/task_mmu.c | 4 ++-- include/linux/huge_mm.h | 29 +++++++++++++++-------------- mm/huge_memory.c | 7 +++++-- mm/khugepaged.c | 16 +++++++--------- mm/memory.c | 10 ++++++---- 5 files changed, 35 insertions(+), 31 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index f4259b7edfded..81fbecfe5ff6c 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -871,8 +871,8 @@ static int show_smap(struct seq_file *m, void *v) __show_smap(m, &mss, false); seq_printf(m, "THPeligible: %8u\n", - !!thp_vma_allowable_orders(vma, vma->vm_flags, true, false, - true, THP_ORDERS_ALL)); + !!thp_vma_allowable_orders(vma, vma->vm_flags, + TVA_SMAPS | TVA_ENFORCE_SYSFS, THP_ORDERS_ALL)); if (arch_pkeys_enabled()) seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 7cd07b83a3d0a..c8d3ec116e291 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -81,8 +81,12 @@ extern struct kobj_attribute shmem_enabled_attr; */ #define THP_ORDERS_ALL (THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_FILE) -#define thp_vma_allowable_order(vma, vm_flags, smaps, in_pf, enforce_sysfs, order) \ - (!!thp_vma_allowable_orders(vma, vm_flags, smaps, in_pf, enforce_sysfs, BIT(order))) +#define TVA_SMAPS (1 << 0) /* Will be used for procfs */ +#define TVA_IN_PF (1 << 1) /* Page fault handler */ +#define TVA_ENFORCE_SYSFS (1 << 2) /* Obey sysfs configuration */ + +#define thp_vma_allowable_order(vma, vm_flags, tva_flags, order) \ + (!!thp_vma_allowable_orders(vma, vm_flags, tva_flags, BIT(order))) #ifdef CONFIG_PGTABLE_HAS_HUGE_LEAVES #define HPAGE_PMD_SHIFT PMD_SHIFT @@ -218,17 +222,15 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma) } unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, - unsigned long vm_flags, bool smaps, - bool in_pf, bool enforce_sysfs, + unsigned long vm_flags, + unsigned long tva_flags, unsigned long orders); /** * thp_vma_allowable_orders - determine hugepage orders that are allowed for vma * @vma: the vm area to check * @vm_flags: use these vm_flags instead of vma->vm_flags - * @smaps: whether answer will be used for smaps file - * @in_pf: whether answer will be used by page fault handler - * @enforce_sysfs: whether sysfs config should be taken into account + * @tva_flags: Which TVA flags to honour * @orders: bitfield of all orders to consider * * Calculates the intersection of the requested hugepage orders and the allowed @@ -241,12 +243,12 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, */ static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, - unsigned long vm_flags, bool smaps, - bool in_pf, bool enforce_sysfs, + unsigned long vm_flags, + unsigned long tva_flags, unsigned long orders) { /* Optimization to check if required orders are enabled early. */ - if (enforce_sysfs && vma_is_anonymous(vma)) { + if ((tva_flags & TVA_ENFORCE_SYSFS) && vma_is_anonymous(vma)) { unsigned long mask = READ_ONCE(huge_anon_orders_always); if (vm_flags & VM_HUGEPAGE) @@ -260,8 +262,7 @@ unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, return 0; } - return __thp_vma_allowable_orders(vma, vm_flags, smaps, in_pf, - enforce_sysfs, orders); + return __thp_vma_allowable_orders(vma, vm_flags, tva_flags, orders); } enum mthp_stat_item { @@ -428,8 +429,8 @@ static inline unsigned long thp_vma_suitable_orders(struct vm_area_struct *vma, } static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, - unsigned long vm_flags, bool smaps, - bool in_pf, bool enforce_sysfs, + unsigned long vm_flags, + unsigned long tva_flags, unsigned long orders) { return 0; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index eabf088cb444b..aaa327bd5155b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -81,10 +81,13 @@ unsigned long huge_anon_orders_madvise __read_mostly; unsigned long huge_anon_orders_inherit __read_mostly; unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, - unsigned long vm_flags, bool smaps, - bool in_pf, bool enforce_sysfs, + unsigned long vm_flags, + unsigned long tva_flags, unsigned long orders) { + bool smaps = tva_flags & TVA_SMAPS; + bool in_pf = tva_flags & TVA_IN_PF; + bool enforce_sysfs = tva_flags & TVA_ENFORCE_SYSFS; /* Check the intersection of requested and supported orders. */ orders &= vma_is_anonymous(vma) ? THP_ORDERS_ALL_ANON : THP_ORDERS_ALL_FILE; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index cf518fc440982..774a97e6e2da3 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -453,7 +453,7 @@ void khugepaged_enter_vma(struct vm_area_struct *vma, { if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && hugepage_flags_enabled()) { - if (thp_vma_allowable_order(vma, vm_flags, false, false, true, + if (thp_vma_allowable_order(vma, vm_flags, TVA_ENFORCE_SYSFS, PMD_ORDER)) __khugepaged_enter(vma->vm_mm); } @@ -900,6 +900,7 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, struct collapse_control *cc) { struct vm_area_struct *vma; + unsigned long tva_flags = cc->is_khugepaged ? TVA_ENFORCE_SYSFS : 0; if (unlikely(hpage_collapse_test_exit_or_disable(mm))) return SCAN_ANY_PROCESS; @@ -910,8 +911,7 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, if (!thp_vma_suitable_order(vma, address, PMD_ORDER)) return SCAN_ADDRESS_RANGE; - if (!thp_vma_allowable_order(vma, vma->vm_flags, false, false, - cc->is_khugepaged, PMD_ORDER)) + if (!thp_vma_allowable_order(vma, vma->vm_flags, tva_flags, PMD_ORDER)) return SCAN_VMA_CHECK; /* * Anon VMA expected, the address may be unmapped then @@ -1501,8 +1501,7 @@ int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr, * and map it by a PMD, regardless of sysfs THP settings. As such, let's * analogously elide sysfs THP settings here. */ - if (!thp_vma_allowable_order(vma, vma->vm_flags, false, false, false, - PMD_ORDER)) + if (!thp_vma_allowable_order(vma, vma->vm_flags, 0, PMD_ORDER)) return SCAN_VMA_CHECK; /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ @@ -2363,8 +2362,8 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, progress++; break; } - if (!thp_vma_allowable_order(vma, vma->vm_flags, false, false, - true, PMD_ORDER)) { + if (!thp_vma_allowable_order(vma, vma->vm_flags, + TVA_ENFORCE_SYSFS, PMD_ORDER)) { skip: progress++; continue; @@ -2701,8 +2700,7 @@ int madvise_collapse(struct vm_area_struct *vma, struct vm_area_struct **prev, *prev = vma; - if (!thp_vma_allowable_order(vma, vma->vm_flags, false, false, false, - PMD_ORDER)) + if (!thp_vma_allowable_order(vma, vma->vm_flags, 0, PMD_ORDER)) return -EINVAL; cc = kmalloc(sizeof(*cc), GFP_KERNEL); diff --git a/mm/memory.c b/mm/memory.c index c0bd4e0d5e7a7..79d851be8ab25 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4334,8 +4334,8 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf) * for this vma. Then filter out the orders that can't be allocated over * the faulting address and still be fully contained in the vma. */ - orders = thp_vma_allowable_orders(vma, vma->vm_flags, false, true, true, - BIT(PMD_ORDER) - 1); + orders = thp_vma_allowable_orders(vma, vma->vm_flags, + TVA_IN_PF | TVA_ENFORCE_SYSFS, BIT(PMD_ORDER) - 1); orders = thp_vma_suitable_orders(vma, vmf->address, orders); if (!orders) @@ -5438,7 +5438,8 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, return VM_FAULT_OOM; retry_pud: if (pud_none(*vmf.pud) && - thp_vma_allowable_order(vma, vm_flags, false, true, true, PUD_ORDER)) { + thp_vma_allowable_order(vma, vm_flags, + TVA_IN_PF | TVA_ENFORCE_SYSFS, PUD_ORDER)) { ret = create_huge_pud(&vmf); if (!(ret & VM_FAULT_FALLBACK)) return ret; @@ -5472,7 +5473,8 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, goto retry_pud; if (pmd_none(*vmf.pmd) && - thp_vma_allowable_order(vma, vm_flags, false, true, true, PMD_ORDER)) { + thp_vma_allowable_order(vma, vm_flags, + TVA_IN_PF | TVA_ENFORCE_SYSFS, PMD_ORDER)) { ret = create_huge_pmd(&vmf); if (!(ret & VM_FAULT_FALLBACK)) return ret; From 3be51060599ff01899b6d8c3f8aca456506cf5ea Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 26 Apr 2024 15:45:00 +0100 Subject: [PATCH 1226/1702] mm: assert the mmap_lock is held in __anon_vma_prepare() Patch series "Improve anon_vma scalability for anon VMAs". We have a 3x throughput improvement reported by Intel's kernel test robot: https://lore.kernel.org/all/202404261055.c5e24608-oliver.sang@intel.com/ This is from delaying taking the mmap_lock for page faults until we actually need the mmap_lock in order to assign an anon_vma to the vma. It cleans up the page fault path a little by making the anon fault handler more similar to the file fault handler. This patch (of 4): Convert the comment into an assertion. Link: https://lkml.kernel.org/r/20240426144506.1290619-1-willy@infradead.org Link: https://lkml.kernel.org/r/20240426144506.1290619-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Suren Baghdasaryan Reviewed-by: David Hildenbrand Cc: Jann Horn Signed-off-by: Andrew Morton --- mm/rmap.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index 2608c40dffade..619d4d65d99b2 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -182,8 +182,6 @@ static void anon_vma_chain_link(struct vm_area_struct *vma, * for the new allocation. At the same time, we do not want * to do any locking for the common case of already having * an anon_vma. - * - * This must be called with the mmap_lock held for reading. */ int __anon_vma_prepare(struct vm_area_struct *vma) { @@ -191,6 +189,7 @@ int __anon_vma_prepare(struct vm_area_struct *vma) struct anon_vma *anon_vma, *allocated; struct anon_vma_chain *avc; + mmap_assert_locked(mm); might_sleep(); avc = anon_vma_chain_alloc(GFP_KERNEL); From a373baed5a9dca65a4d9fa55e61800a18c9936f1 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 26 Apr 2024 15:45:01 +0100 Subject: [PATCH 1227/1702] mm: delay the check for a NULL anon_vma Instead of checking the anon_vma early in the fault path where all page faults pay the cost, delay it until we know we're going to need the anon_vma to be filled in. This will have a slight negative effect on the first fault in an anonymous VMA, but it shortens every other page fault. It also makes the code slightly cleaner as the anon and file backed fault handling look more similar. The Intel kernel test bot reports a 3x improvement in vm-scalability throughput with the small-allocs-mt test. This is clearly an extreme situation that won't be replicated in any real-world workload, but it's a nice win. https://lore.kernel.org/all/202404261055.c5e24608-oliver.sang@intel.com/ Link: https://lkml.kernel.org/r/20240426144506.1290619-3-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Suren Baghdasaryan Cc: David Hildenbrand Cc: Jann Horn Signed-off-by: Andrew Morton --- mm/huge_memory.c | 6 ++++-- mm/memory.c | 29 ++++++++++++++++++----------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index aaa327bd5155b..08e4f3343bcd0 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1057,11 +1057,13 @@ vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) gfp_t gfp; struct folio *folio; unsigned long haddr = vmf->address & HPAGE_PMD_MASK; + vm_fault_t ret; if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER)) return VM_FAULT_FALLBACK; - if (unlikely(anon_vma_prepare(vma))) - return VM_FAULT_OOM; + ret = vmf_anon_prepare(vmf); + if (ret) + return ret; khugepaged_enter_vma(vma, vma->vm_flags); if (!(vmf->flags & FAULT_FLAG_WRITE) && diff --git a/mm/memory.c b/mm/memory.c index 79d851be8ab25..b5f72ce569e65 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3214,6 +3214,21 @@ static inline vm_fault_t vmf_can_call_fault(const struct vm_fault *vmf) return VM_FAULT_RETRY; } +/** + * vmf_anon_prepare - Prepare to handle an anonymous fault. + * @vmf: The vm_fault descriptor passed from the fault handler. + * + * When preparing to insert an anonymous page into a VMA from a + * fault handler, call this function rather than anon_vma_prepare(). + * If this vma does not already have an associated anon_vma and we are + * only protected by the per-VMA lock, the caller must retry with the + * mmap_lock held. __anon_vma_prepare() will look at adjacent VMAs to + * determine if this VMA can share its anon_vma, and that's not safe to + * do with only the per-VMA lock held for this VMA. + * + * Return: 0 if fault handling can proceed. Any other value should be + * returned to the caller. + */ vm_fault_t vmf_anon_prepare(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -4439,8 +4454,9 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) } /* Allocate our own private page. */ - if (unlikely(anon_vma_prepare(vma))) - goto oom; + ret = vmf_anon_prepare(vmf); + if (ret) + return ret; /* Returns NULL on OOM or ERR_PTR(-EAGAIN) if we must retry the fault */ folio = alloc_anon_folio(vmf); if (IS_ERR(folio)) @@ -5828,15 +5844,6 @@ struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, if (!vma_start_read(vma)) goto inval; - /* - * find_mergeable_anon_vma uses adjacent vmas which are not locked. - * This check must happen after vma_start_read(); otherwise, a - * concurrent mremap() with MREMAP_DONTUNMAP could dissociate the VMA - * from its anon_vma. - */ - if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) - goto inval_end_read; - /* Check since vm_start/vm_end might change before we lock the VMA */ if (unlikely(address < vma->vm_start || address >= vma->vm_end)) goto inval_end_read; From 73b4a0cd8243709870701349611722ba3c351815 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 26 Apr 2024 15:45:02 +0100 Subject: [PATCH 1228/1702] mm: fix some minor per-VMA lock issues in userfaultfd Rename lock_vma() to uffd_lock_vma() because it really is uffd specific. Remove comment referencing unlock_vma() which doesn't exist. Fix the comment about lock_vma_under_rcu() which I just made incorrect. Link: https://lkml.kernel.org/r/20240426144506.1290619-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Suren Baghdasaryan Cc: David Hildenbrand Cc: Jann Horn Signed-off-by: Andrew Morton --- mm/userfaultfd.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 8b1005ef9dfa9..d9e82ae68244e 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -56,17 +56,16 @@ struct vm_area_struct *find_vma_and_prepare_anon(struct mm_struct *mm, #ifdef CONFIG_PER_VMA_LOCK /* - * lock_vma() - Lookup and lock vma corresponding to @address. + * uffd_lock_vma() - Lookup and lock vma corresponding to @address. * @mm: mm to search vma in. * @address: address that the vma should contain. * - * Should be called without holding mmap_lock. vma should be unlocked after use - * with unlock_vma(). + * Should be called without holding mmap_lock. * * Return: A locked vma containing @address, -ENOENT if no vma is found, or * -ENOMEM if anon_vma couldn't be allocated. */ -static struct vm_area_struct *lock_vma(struct mm_struct *mm, +static struct vm_area_struct *uffd_lock_vma(struct mm_struct *mm, unsigned long address) { struct vm_area_struct *vma; @@ -74,9 +73,8 @@ static struct vm_area_struct *lock_vma(struct mm_struct *mm, vma = lock_vma_under_rcu(mm, address); if (vma) { /* - * lock_vma_under_rcu() only checks anon_vma for private - * anonymous mappings. But we need to ensure it is assigned in - * private file-backed vmas as well. + * We know we're going to need to use anon_vma, so check + * that early. */ if (!(vma->vm_flags & VM_SHARED) && unlikely(!vma->anon_vma)) vma_end_read(vma); @@ -107,7 +105,7 @@ static struct vm_area_struct *uffd_mfill_lock(struct mm_struct *dst_mm, { struct vm_area_struct *dst_vma; - dst_vma = lock_vma(dst_mm, dst_start); + dst_vma = uffd_lock_vma(dst_mm, dst_start); if (IS_ERR(dst_vma) || validate_dst_vma(dst_vma, dst_start + len)) return dst_vma; @@ -1401,7 +1399,7 @@ static int uffd_move_lock(struct mm_struct *mm, struct vm_area_struct *vma; int err; - vma = lock_vma(mm, dst_start); + vma = uffd_lock_vma(mm, dst_start); if (IS_ERR(vma)) return PTR_ERR(vma); @@ -1416,7 +1414,7 @@ static int uffd_move_lock(struct mm_struct *mm, } /* - * Using lock_vma() to get src_vma can lead to following deadlock: + * Using uffd_lock_vma() to get src_vma can lead to following deadlock: * * Thread1 Thread2 * ------- ------- @@ -1438,7 +1436,7 @@ static int uffd_move_lock(struct mm_struct *mm, err = find_vmas_mm_locked(mm, dst_start, src_start, dst_vmap, src_vmap); if (!err) { /* - * See comment in lock_vma() as to why not using + * See comment in uffd_lock_vma() as to why not using * vma_start_read() here. */ down_read(&(*dst_vmap)->vm_lock->lock); From 737019cf6ac5babb75645ad324aeead7bc04749d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 26 Apr 2024 15:45:03 +0100 Subject: [PATCH 1229/1702] mm: optimise vmf_anon_prepare() for VMAs without an anon_vma If the mmap_lock can be taken for read, we can call __anon_vma_prepare() while holding it, saving ourselves a trip back through the fault handler. Link: https://lkml.kernel.org/r/20240426144506.1290619-5-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Jann Horn Reviewed-by: Suren Baghdasaryan Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- mm/memory.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index b5f72ce569e65..eea6e4984eaef 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3232,16 +3232,21 @@ static inline vm_fault_t vmf_can_call_fault(const struct vm_fault *vmf) vm_fault_t vmf_anon_prepare(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; + vm_fault_t ret = 0; if (likely(vma->anon_vma)) return 0; if (vmf->flags & FAULT_FLAG_VMA_LOCK) { - vma_end_read(vma); - return VM_FAULT_RETRY; + if (!mmap_read_trylock(vma->vm_mm)) { + vma_end_read(vma); + return VM_FAULT_RETRY; + } } if (__anon_vma_prepare(vma)) - return VM_FAULT_OOM; - return 0; + ret = VM_FAULT_OOM; + if (vmf->flags & FAULT_FLAG_VMA_LOCK) + mmap_read_unlock(vma->vm_mm); + return ret; } /* From 180d928e55a8160a421aa96b522dc317a6f52140 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 26 Apr 2024 12:52:40 -0700 Subject: [PATCH 1230/1702] mm/damon/paddr: implement damon_folio_young() Patch series "mm/damon: add a DAMOS filter type for page granularity access recheck". DAMON provides its best-effort accuracy-overhead tradeoff under the user-defined ranges of acceptable level of the monitoring accuracy and overhead. A recent discussion for tiered memory management support from DAMON[1] concluded that finding memory regions of specific access pattern with low overhead despite of low accuracy via DAMON first, and then double checking the access of the region again in a finer (e.g., page) granularity could be a useful strategy for some DAMOS schemes. Add a new type of DAMOS filter, namely 'young' for such a case. It checks each page of DAMOS target region is accessed since the last check, and filters it out or in if 'matching' parameter is 'true' or 'false', respectively. Because this is a filter type that applied in page granularity, the support depends on DAMON operations set, similar to 'anon' and 'memcg' DAMOS filter types. Implement the support on the DAMON operations set for the physical address space, 'paddr', since one of the expected usages[1] is based on the physical address space. [1] https://lore.kernel.org/r/20240227235121.153277-1-sj@kernel.org This patch (of 7): damon_pa_young() receives physical address, get the folio covering the address, and show if the folio is accessed since the last check. A following commit will reuse the internal logic for checking access to a given folio. To avoid duplication of the code, split the internal logic. Also, change the rmap walker function's name from __damon_pa_young() to damon_folio_young_one(), following the change of the caller's name and the naming rule that more commonly used by other rmap walkers. Link: https://lkml.kernel.org/r/20240426195247.100306-1-sj@kernel.org Link: https://lkml.kernel.org/r/20240426195247.100306-2-sj@kernel.org Signed-off-by: SeongJae Park Tested-by: Honggyu Kim Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- mm/damon/paddr.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 5e6dc312072cd..25c3ba2a9eaf4 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -79,8 +79,8 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) } } -static bool __damon_pa_young(struct folio *folio, struct vm_area_struct *vma, - unsigned long addr, void *arg) +static bool damon_folio_young_one(struct folio *folio, + struct vm_area_struct *vma, unsigned long addr, void *arg) { bool *accessed = arg; DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); @@ -111,38 +111,44 @@ static bool __damon_pa_young(struct folio *folio, struct vm_area_struct *vma, return *accessed == false; } -static bool damon_pa_young(unsigned long paddr, unsigned long *folio_sz) +static bool damon_folio_young(struct folio *folio) { - struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); bool accessed = false; struct rmap_walk_control rwc = { .arg = &accessed, - .rmap_one = __damon_pa_young, + .rmap_one = damon_folio_young_one, .anon_lock = folio_lock_anon_vma_read, }; bool need_lock; - if (!folio) - return false; - if (!folio_mapped(folio) || !folio_raw_mapping(folio)) { if (folio_test_idle(folio)) - accessed = false; + return false; else - accessed = true; - goto out; + return true; } need_lock = !folio_test_anon(folio) || folio_test_ksm(folio); if (need_lock && !folio_trylock(folio)) - goto out; + return false; rmap_walk(folio, &rwc); if (need_lock) folio_unlock(folio); -out: + return accessed; +} + +static bool damon_pa_young(unsigned long paddr, unsigned long *folio_sz) +{ + struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); + bool accessed; + + if (!folio) + return false; + + accessed = damon_folio_young(folio); *folio_sz = folio_size(folio); folio_put(folio); return accessed; From 6daea38215e6a046892f87a8ff2c61f791de4ddd Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 26 Apr 2024 12:52:41 -0700 Subject: [PATCH 1231/1702] mm/damon/paddr: implement damon_folio_mkold() damon_pa_mkold() receives physical address, get the folio covering the address, and makes the folio as old. A following commit will reuse the internal logic for marking a given folio as old. To avoid duplication of the code, split the internal logic. Also, change the rmap walker function's name from __damon_pa_mkold() to damon_folio_mkold_one(), following the change of the caller's name and the naming rule that more commonly used by other rmap walkers. Link: https://lkml.kernel.org/r/20240426195247.100306-3-sj@kernel.org Signed-off-by: SeongJae Park Tested-by: Honggyu Kim Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- mm/damon/paddr.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 25c3ba2a9eaf4..310b803c62779 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -16,8 +16,8 @@ #include "../internal.h" #include "ops-common.h" -static bool __damon_pa_mkold(struct folio *folio, struct vm_area_struct *vma, - unsigned long addr, void *arg) +static bool damon_folio_mkold_one(struct folio *folio, + struct vm_area_struct *vma, unsigned long addr, void *arg) { DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); @@ -31,33 +31,38 @@ static bool __damon_pa_mkold(struct folio *folio, struct vm_area_struct *vma, return true; } -static void damon_pa_mkold(unsigned long paddr) +static void damon_folio_mkold(struct folio *folio) { - struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); struct rmap_walk_control rwc = { - .rmap_one = __damon_pa_mkold, + .rmap_one = damon_folio_mkold_one, .anon_lock = folio_lock_anon_vma_read, }; bool need_lock; - if (!folio) - return; - if (!folio_mapped(folio) || !folio_raw_mapping(folio)) { folio_set_idle(folio); - goto out; + return; } need_lock = !folio_test_anon(folio) || folio_test_ksm(folio); if (need_lock && !folio_trylock(folio)) - goto out; + return; rmap_walk(folio, &rwc); if (need_lock) folio_unlock(folio); -out: +} + +static void damon_pa_mkold(unsigned long paddr) +{ + struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); + + if (!folio) + return; + + damon_folio_mkold(folio); folio_put(folio); } From 2d8b24654feabc62ed9e132911319f1c48ab0256 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 26 Apr 2024 12:52:42 -0700 Subject: [PATCH 1232/1702] mm/damon: add DAMOS filter type YOUNG Define yet another DAMOS filter type, YOUNG. Like anon and memcg, the type of filter will be applied to each page in the memory region, and see if the page is accessed since the last check. Based on the 'matching' parameter, the page is filtered out or in. Note that this commit is adding only the type definition. The implementation should be made by DAMON operations sets. A commit for the implementation on 'paddr' DAMON operations set will follow. Link: https://lkml.kernel.org/r/20240426195247.100306-4-sj@kernel.org Signed-off-by: SeongJae Park Tested-by: Honggyu Kim Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- include/linux/damon.h | 2 ++ mm/damon/sysfs-schemes.c | 1 + 2 files changed, 3 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index 886d07294f4e7..f7da65e1ac041 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -297,6 +297,7 @@ struct damos_stat { * enum damos_filter_type - Type of memory for &struct damos_filter * @DAMOS_FILTER_TYPE_ANON: Anonymous pages. * @DAMOS_FILTER_TYPE_MEMCG: Specific memcg's pages. + * @DAMOS_FILTER_TYPE_YOUNG: Recently accessed pages. * @DAMOS_FILTER_TYPE_ADDR: Address range. * @DAMOS_FILTER_TYPE_TARGET: Data Access Monitoring target. * @NR_DAMOS_FILTER_TYPES: Number of filter types. @@ -315,6 +316,7 @@ struct damos_stat { enum damos_filter_type { DAMOS_FILTER_TYPE_ANON, DAMOS_FILTER_TYPE_MEMCG, + DAMOS_FILTER_TYPE_YOUNG, DAMOS_FILTER_TYPE_ADDR, DAMOS_FILTER_TYPE_TARGET, NR_DAMOS_FILTER_TYPES, diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 53a90ac678fb9..bea5bc52846a6 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -343,6 +343,7 @@ static struct damon_sysfs_scheme_filter *damon_sysfs_scheme_filter_alloc(void) static const char * const damon_sysfs_scheme_filter_type_strs[] = { "anon", "memcg", + "young", "addr", "target", }; From ade414bdf6aeb039991cb04bbafb42385960db60 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 26 Apr 2024 12:52:43 -0700 Subject: [PATCH 1233/1702] mm/damon/paddr: implement DAMOS filter type YOUNG DAMOS filter of type YOUNG is defined, but not yet implemented by any DAMON operations set. Add the implementation on 'paddr', the DAMON operations set for the physical address space. Link: https://lkml.kernel.org/r/20240426195247.100306-5-sj@kernel.org Signed-off-by: SeongJae Park Tested-by: Honggyu Kim Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- mm/damon/paddr.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 310b803c62779..5685ba485097d 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -214,6 +214,11 @@ static bool __damos_pa_filter_out(struct damos_filter *filter, matched = filter->memcg_id == mem_cgroup_id(memcg); rcu_read_unlock(); break; + case DAMOS_FILTER_TYPE_YOUNG: + matched = damon_folio_young(folio); + if (matched) + damon_folio_mkold(folio); + break; default: break; } From 26dd7cc7bb90f12f674f1bf55883de2c216fc119 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 26 Apr 2024 12:52:44 -0700 Subject: [PATCH 1234/1702] Docs/mm/damon/design: document 'young page' type DAMOS filter Update DAMON design document for the newly added DAMOS filter type, 'young page'. Link: https://lkml.kernel.org/r/20240426195247.100306-6-sj@kernel.org Signed-off-by: SeongJae Park Cc: Honggyu Kim Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- Documentation/mm/damon/design.rst | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index 5620aab9b3850..f2baf617184d0 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -461,15 +461,17 @@ number of filters for each scheme. Each filter specifies the type of target memory, and whether it should exclude the memory of the type (filter-out), or all except the memory of the type (filter-in). -Currently, anonymous page, memory cgroup, address range, and DAMON monitoring -target type filters are supported by the feature. Some filter target types -require additional arguments. The memory cgroup filter type asks users to -specify the file path of the memory cgroup for the filter. The address range -type asks the start and end addresses of the range. The DAMON monitoring -target type asks the index of the target from the context's monitoring targets -list. Hence, users can apply specific schemes to only anonymous pages, -non-anonymous pages, pages of specific cgroups, all pages excluding those of -specific cgroups, pages in specific address range, pages in specific DAMON +Currently, anonymous page, memory cgroup, young page, address range, and DAMON +monitoring target type filters are supported by the feature. Some filter +target types require additional arguments. The memory cgroup filter type asks +users to specify the file path of the memory cgroup for the filter. The +address range type asks the start and end addresses of the range. The DAMON +monitoring target type asks the index of the target from the context's +monitoring targets list. Hence, users can apply specific schemes to only +anonymous pages, non-anonymous pages, pages of specific cgroups, all pages +excluding those of specific cgroups, pages that not accessed after the last +access check from the scheme, pages that accessed after the last access check +from the scheme, pages in specific address range, pages in specific DAMON monitoring targets, and any combination of those. To handle filters efficiently, the address range and DAMON monitoring target From ed13c93b9393c9b9e0fc1f25fc5da13715bfddbd Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 26 Apr 2024 12:52:45 -0700 Subject: [PATCH 1235/1702] Docs/admin-guide/mm/damon/usage: update for young page type DAMOS filter Update DAMON usage document for the newly added DAMOS filter type, 'young page'. Link: https://lkml.kernel.org/r/20240426195247.100306-7-sj@kernel.org Signed-off-by: SeongJae Park Cc: Honggyu Kim Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- Documentation/admin-guide/mm/damon/usage.rst | 26 ++++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst index 6fce035fdbf5c..69bc8fabf3781 100644 --- a/Documentation/admin-guide/mm/damon/usage.rst +++ b/Documentation/admin-guide/mm/damon/usage.rst @@ -410,19 +410,19 @@ in the numeric order. Each filter directory contains six files, namely ``type``, ``matcing``, ``memcg_path``, ``addr_start``, ``addr_end``, and ``target_idx``. To ``type`` -file, you can write one of four special keywords: ``anon`` for anonymous pages, -``memcg`` for specific memory cgroup, ``addr`` for specific address range (an -open-ended interval), or ``target`` for specific DAMON monitoring target -filtering. In case of the memory cgroup filtering, you can specify the memory -cgroup of the interest by writing the path of the memory cgroup from the -cgroups mount point to ``memcg_path`` file. In case of the address range -filtering, you can specify the start and end address of the range to -``addr_start`` and ``addr_end`` files, respectively. For the DAMON monitoring -target filtering, you can specify the index of the target between the list of -the DAMON context's monitoring targets list to ``target_idx`` file. You can -write ``Y`` or ``N`` to ``matching`` file to filter out pages that does or does -not match to the type, respectively. Then, the scheme's action will not be -applied to the pages that specified to be filtered out. +file, you can write one of five special keywords: ``anon`` for anonymous pages, +``memcg`` for specific memory cgroup, ``young`` for young pages, ``addr`` for +specific address range (an open-ended interval), or ``target`` for specific +DAMON monitoring target filtering. In case of the memory cgroup filtering, you +can specify the memory cgroup of the interest by writing the path of the memory +cgroup from the cgroups mount point to ``memcg_path`` file. In case of the +address range filtering, you can specify the start and end address of the range +to ``addr_start`` and ``addr_end`` files, respectively. For the DAMON +monitoring target filtering, you can specify the index of the target between +the list of the DAMON context's monitoring targets list to ``target_idx`` file. +You can write ``Y`` or ``N`` to ``matching`` file to filter out pages that does +or does not match to the type, respectively. Then, the scheme's action will +not be applied to the pages that specified to be filtered out. For example, below restricts a DAMOS action to be applied to only non-anonymous pages of all memory cgroups except ``/having_care_already``.:: From eedbd23dca88f9e0c80a9461a71f552c7c8e4a0e Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 26 Apr 2024 12:52:46 -0700 Subject: [PATCH 1236/1702] Docs/ABI/damon: update for 'youg page' type DAMOS filter Update DAMON ABI document for the newly added DAMO filter type, 'young page'. Link: https://lkml.kernel.org/r/20240426195247.100306-8-sj@kernel.org Signed-off-by: SeongJae Park Cc: Honggyu Kim Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- Documentation/ABI/testing/sysfs-kernel-mm-damon | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index dad4d5ffd7865..cef6e1d20b185 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -314,9 +314,9 @@ Date: Dec 2022 Contact: SeongJae Park Description: Writing to and reading from this file sets and gets the type of the memory of the interest. 'anon' for anonymous pages, - 'memcg' for specific memory cgroup, 'addr' for address range - (an open-ended interval), or 'target' for DAMON monitoring - target can be written and read. + 'memcg' for specific memory cgroup, 'young' for young pages, + 'addr' for address range (an open-ended interval), or 'target' + for DAMON monitoring target can be written and read. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//filters//memcg_path Date: Dec 2022 From 7491f3f34891ef8baf5418f6856af91b58f7d200 Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Thu, 2 May 2024 09:28:51 -0400 Subject: [PATCH 1237/1702] mm/rmap: do not add fully unmapped large folio to deferred split list In __folio_remove_rmap(), a large folio is added to deferred split list if any page in a folio loses its final mapping. But it is possible that the folio is fully unmapped and adding it to deferred split list is unnecessary. For PMD-mapped THPs, that was not really an issue, because removing the last PMD mapping in the absence of PTE mappings would not have added the folio to the deferred split queue. However, for PTE-mapped THPs, which are now more prominent due to mTHP, they are always added to the deferred split queue. One side effect is that the THP_DEFERRED_SPLIT_PAGE stat for a PTE-mapped folio can be unintentionally increased, making it look like there are many partially mapped folios -- although the whole folio is fully unmapped stepwise. Core-mm now tries batch-unmapping consecutive PTEs of PTE-mapped THPs where possible starting from commit b06dc281aa99 ("mm/rmap: introduce folio_remove_rmap_[pte|ptes|pmd]()"). When it happens, a whole PTE-mapped folio is unmapped in one go and can avoid being added to deferred split list, reducing the THP_DEFERRED_SPLIT_PAGE noise. But there will still be noise when we cannot batch-unmap a complete PTE-mapped folio in one go -- or where this type of batching is not implemented yet, e.g., migration. To avoid the unnecessary addition, folio->_nr_pages_mapped is checked to tell if the whole folio is unmapped. If the folio is already on deferred split list, it will be skipped, too. Note: commit 98046944a159 ("mm: huge_memory: add the missing folio_test_pmd_mappable() for THP split statistics") tried to exclude mTHP deferred split stats from THP_DEFERRED_SPLIT_PAGE, but it does not fix the above issue. A fully unmapped PTE-mapped order-9 THP was still added to deferred split list and counted as THP_DEFERRED_SPLIT_PAGE, since nr is 512 (non zero), level is RMAP_LEVEL_PTE, and inside deferred_split_folio() the order-9 folio is folio_test_pmd_mappable(). Link: https://lkml.kernel.org/r/20240502132852.862138-1-zi.yan@sent.com Signed-off-by: Zi Yan Suggested-by: David Hildenbrand Reviewed-by: Yang Shi Reviewed-by: David Hildenbrand Reviewed-by: Barry Song Reviewed-by: Lance Yang Cc: Alexander Gordeev Cc: Matthew Wilcox (Oracle) Cc: Ryan Roberts Signed-off-by: Andrew Morton --- mm/rmap.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index 619d4d65d99b2..c912cc1d7b44c 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1494,6 +1494,7 @@ static __always_inline void __folio_remove_rmap(struct folio *folio, { atomic_t *mapped = &folio->_nr_pages_mapped; int last, nr = 0, nr_pmdmapped = 0; + bool partially_mapped = false; enum node_stat_item idx; __folio_rmap_sanity_checks(folio, page, nr_pages, level); @@ -1514,6 +1515,8 @@ static __always_inline void __folio_remove_rmap(struct folio *folio, nr++; } } while (page++, --nr_pages > 0); + + partially_mapped = nr && atomic_read(mapped); break; case RMAP_LEVEL_PMD: atomic_dec(&folio->_large_mapcount); @@ -1531,6 +1534,8 @@ static __always_inline void __folio_remove_rmap(struct folio *folio, nr = 0; } } + + partially_mapped = nr < nr_pmdmapped; break; } @@ -1551,10 +1556,12 @@ static __always_inline void __folio_remove_rmap(struct folio *folio, * Queue anon large folio for deferred split if at least one * page of the folio is unmapped and at least one page * is still mapped. + * + * Check partially_mapped first to ensure it is a large folio. */ - if (folio_test_large(folio) && folio_test_anon(folio)) - if (level == RMAP_LEVEL_PTE || nr < nr_pmdmapped) - deferred_split_folio(folio); + if (folio_test_anon(folio) && partially_mapped && + list_empty(&folio->_deferred_list)) + deferred_split_folio(folio); } /* From 620875560bd65084a8bfcea6899003d6a29ddc57 Mon Sep 17 00:00:00 2001 From: Hao Ge Date: Sun, 28 Apr 2024 09:47:11 +0800 Subject: [PATCH 1238/1702] mm/pagemap: make trylock_page return bool Make trylock_page return bool to align the return values of folio_trylock function and it also corresponds to its comment. Link: https://lkml.kernel.org/r/20240428014711.11169-1-gehao@kylinos.cn Signed-off-by: Hao Ge Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/pagemap.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 2a0bc2ff52583..850d32057939d 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1012,7 +1012,7 @@ static inline bool folio_trylock(struct folio *folio) /* * Return true if the page was successfully locked */ -static inline int trylock_page(struct page *page) +static inline bool trylock_page(struct page *page) { return folio_trylock(page_folio(page)); } From 637a900b08c65387dcac1417a964399986b1b424 Mon Sep 17 00:00:00 2001 From: Hao Ge Date: Sun, 28 Apr 2024 09:20:48 +0800 Subject: [PATCH 1239/1702] mm/rmap: change the type of we_locked from int to bool Change the type of we_locked from int to bool because folio_trylock return bool Link: https://lkml.kernel.org/r/20240428012049.8182-1-gehao@kylinos.cn Signed-off-by: Hao Ge Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/rmap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/rmap.c b/mm/rmap.c index c912cc1d7b44c..fd021efe02a12 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -964,7 +964,7 @@ static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg) int folio_referenced(struct folio *folio, int is_locked, struct mem_cgroup *memcg, unsigned long *vm_flags) { - int we_locked = 0; + bool we_locked = false; struct folio_referenced_arg pra = { .mapcount = folio_mapcount(folio), .memcg = memcg, From 5ee9562c586cd4ca9402b3636157abdd58ab7978 Mon Sep 17 00:00:00 2001 From: linke li Date: Sat, 27 Apr 2024 14:29:56 +0800 Subject: [PATCH 1240/1702] mm/swapfile: mark racy access on si->highest_bit In scan_swap_map_slots(), si->highest_bit can by changed by swap_range_alloc() concurrently. All reads on si->highest_bit except one is either protected by lock or read using READ_ONCE. So mark the one racy read on si->highest_bit as benign using READ_ONCE. This patch is aimed at reducing the number of benign races reported by KCSAN in order to focus future debugging effort on harmful races. Link: https://lkml.kernel.org/r/tencent_912BC3E8B0291DA4A0028AB424076375DA07@qq.com Signed-off-by: linke li Signed-off-by: Andrew Morton --- mm/swapfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/swapfile.c b/mm/swapfile.c index 96606580ee091..f6ca215fb92f3 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -902,7 +902,7 @@ static int scan_swap_map_slots(struct swap_info_struct *si, last_in_cluster = offset + SWAPFILE_CLUSTER - 1; /* Locate the first empty (unaligned) cluster */ - for (; last_in_cluster <= si->highest_bit; offset++) { + for (; last_in_cluster <= READ_ONCE(si->highest_bit); offset++) { if (si->swap_map[offset]) last_in_cluster = offset + SWAPFILE_CLUSTER; else if (offset == last_in_cluster) { From 8ff05989b44e1a8f7d2bbe67320990ebc2fbb5e5 Mon Sep 17 00:00:00 2001 From: Stefan Wahren Date: Fri, 3 May 2024 08:27:45 +0200 Subject: [PATCH 1241/1702] pinctrl: bcm2835: Make pin freeing behavior configurable Until now after a bcm2835 pin was freed its pinmux was set to GPIO_IN. So in case it was configured as GPIO_OUT before the configured output level also get lost. As long as GPIO sysfs was used this wasn't actually a problem because the pins and their possible output level were kept by sysfs. Since more and more Raspberry Pi users start using libgpiod they are confused about this behavior. So make the pin freeing behavior of GPIO_OUT configurable via module parameter. In case pinctrl-bcm2835.persist_gpio_outputs is set to 1, the output level is kept. This patch based on the downstream work of Phil Elwell. Link: https://github.com/raspberrypi/linux/pull/6117 Signed-off-by: Stefan Wahren Message-ID: <20240503062745.11298-1-wahrenst@gmx.net> Signed-off-by: Linus Walleij --- drivers/pinctrl/bcm/pinctrl-bcm2835.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/drivers/pinctrl/bcm/pinctrl-bcm2835.c index f5a9372d43bde..7178a38475ccc 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm2835.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm2835.c @@ -244,6 +244,10 @@ static const char * const irq_type_names[] = { [IRQ_TYPE_LEVEL_LOW] = "level-low", }; +static bool persist_gpio_outputs; +module_param(persist_gpio_outputs, bool, 0644); +MODULE_PARM_DESC(persist_gpio_outputs, "Enable GPIO_OUT persistence when pin is freed"); + static inline u32 bcm2835_gpio_rd(struct bcm2835_pinctrl *pc, unsigned reg) { return readl(pc->base + reg); @@ -926,6 +930,13 @@ static int bcm2835_pmx_free(struct pinctrl_dev *pctldev, unsigned offset) { struct bcm2835_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev); + enum bcm2835_fsel fsel = bcm2835_pinctrl_fsel_get(pc, offset); + + if (fsel == BCM2835_FSEL_GPIO_IN) + return 0; + + if (persist_gpio_outputs && fsel == BCM2835_FSEL_GPIO_OUT) + return 0; /* disable by setting to GPIO_IN */ bcm2835_pinctrl_fsel_set(pc, offset, BCM2835_FSEL_GPIO_IN); @@ -970,10 +981,7 @@ static void bcm2835_pmx_gpio_disable_free(struct pinctrl_dev *pctldev, struct pinctrl_gpio_range *range, unsigned offset) { - struct bcm2835_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev); - - /* disable by setting to GPIO_IN */ - bcm2835_pinctrl_fsel_set(pc, offset, BCM2835_FSEL_GPIO_IN); + bcm2835_pmx_free(pctldev, offset); } static int bcm2835_pmx_gpio_set_direction(struct pinctrl_dev *pctldev, @@ -1419,6 +1427,9 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev) goto out_remove; } + dev_info(dev, "GPIO_OUT persistence: %s\n", + persist_gpio_outputs ? "yes" : "no"); + return 0; out_remove: From 3f858bbf04dbac934ac279aaee05d49eb9910051 Mon Sep 17 00:00:00 2001 From: Hamish Martin Date: Wed, 13 Mar 2024 11:16:32 +1300 Subject: [PATCH 1242/1702] i2c: acpi: Unbind mux adapters before delete There is an issue with ACPI overlay table removal specifically related to I2C multiplexers. Consider an ACPI SSDT Overlay that defines a PCA9548 I2C mux on an existing I2C bus. When this table is loaded we see the creation of a device for the overall PCA9548 chip and 8 further devices - one i2c_adapter each for the mux channels. These are all bound to their ACPI equivalents via an eventual invocation of acpi_bind_one(). When we unload the SSDT overlay we run into the problem. The ACPI devices are deleted as normal via acpi_device_del_work_fn() and the acpi_device_del_list. However, the following warning and stack trace is output as the deletion does not go smoothly: ------------[ cut here ]------------ kernfs: can not remove 'physical_node', no directory WARNING: CPU: 1 PID: 11 at fs/kernfs/dir.c:1674 kernfs_remove_by_name_ns+0xb9/0xc0 Modules linked in: CPU: 1 PID: 11 Comm: kworker/u128:0 Not tainted 6.8.0-rc6+ #1 Hardware name: congatec AG conga-B7E3/conga-B7E3, BIOS 5.13 05/16/2023 Workqueue: kacpi_hotplug acpi_device_del_work_fn RIP: 0010:kernfs_remove_by_name_ns+0xb9/0xc0 Code: e4 00 48 89 ef e8 07 71 db ff 5b b8 fe ff ff ff 5d 41 5c 41 5d e9 a7 55 e4 00 0f 0b eb a6 48 c7 c7 f0 38 0d 9d e8 97 0a d5 ff <0f> 0b eb dc 0f 1f 00 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 RSP: 0018:ffff9f864008fb28 EFLAGS: 00010286 RAX: 0000000000000000 RBX: ffff8ef90a8d4940 RCX: 0000000000000000 RDX: ffff8f000e267d10 RSI: ffff8f000e25c780 RDI: ffff8f000e25c780 RBP: ffff8ef9186f9870 R08: 0000000000013ffb R09: 00000000ffffbfff R10: 00000000ffffbfff R11: ffff8f000e0a0000 R12: ffff9f864008fb50 R13: ffff8ef90c93dd60 R14: ffff8ef9010d0958 R15: ffff8ef9186f98c8 FS: 0000000000000000(0000) GS:ffff8f000e240000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007f48f5253a08 CR3: 00000003cb82e000 CR4: 00000000003506f0 Call Trace: ? kernfs_remove_by_name_ns+0xb9/0xc0 ? __warn+0x7c/0x130 ? kernfs_remove_by_name_ns+0xb9/0xc0 ? report_bug+0x171/0x1a0 ? handle_bug+0x3c/0x70 ? exc_invalid_op+0x17/0x70 ? asm_exc_invalid_op+0x1a/0x20 ? kernfs_remove_by_name_ns+0xb9/0xc0 ? kernfs_remove_by_name_ns+0xb9/0xc0 acpi_unbind_one+0x108/0x180 device_del+0x18b/0x490 ? srso_return_thunk+0x5/0x5f ? srso_return_thunk+0x5/0x5f device_unregister+0xd/0x30 i2c_del_adapter.part.0+0x1bf/0x250 i2c_mux_del_adapters+0xa1/0xe0 i2c_device_remove+0x1e/0x80 device_release_driver_internal+0x19a/0x200 bus_remove_device+0xbf/0x100 device_del+0x157/0x490 ? __pfx_device_match_fwnode+0x10/0x10 ? srso_return_thunk+0x5/0x5f device_unregister+0xd/0x30 i2c_acpi_notify+0x10f/0x140 notifier_call_chain+0x58/0xd0 blocking_notifier_call_chain+0x3a/0x60 acpi_device_del_work_fn+0x85/0x1d0 process_one_work+0x134/0x2f0 worker_thread+0x2f0/0x410 ? __pfx_worker_thread+0x10/0x10 kthread+0xe3/0x110 ? __pfx_kthread+0x10/0x10 ret_from_fork+0x2f/0x50 ? __pfx_kthread+0x10/0x10 ret_from_fork_asm+0x1b/0x30 ---[ end trace 0000000000000000 ]--- ... repeated 7 more times, 1 for each channel of the mux ... The issue is that the binding of the ACPI devices to their peer I2C adapters is not correctly cleaned up. Digging deeper into the issue we see that the deletion order is such that the ACPI devices matching the mux channel i2c adapters are deleted first during the SSDT overlay removal. For each of the channels we see a call to i2c_acpi_notify() with ACPI_RECONFIG_DEVICE_REMOVE but, because these devices are not actually i2c_clients, nothing is done for them. Later on, after each of the mux channels has been dealt with, we come to delete the i2c_client representing the PCA9548 device. This is the call stack we see above, whereby the kernel cleans up the i2c_client including destruction of the mux and its channel adapters. At this point we do attempt to unbind from the ACPI peers but those peers no longer exist and so we hit the kernfs errors. The fix is to augment i2c_acpi_notify() to handle i2c_adapters. But, given that the life cycle of the adapters is linked to the i2c_client, instead of deleting the i2c_adapters during the i2c_acpi_notify(), we just trigger unbinding of the ACPI device from the adapter device, and allow the clean up of the adapter to continue in the way it always has. Signed-off-by: Hamish Martin Reviewed-by: Mika Westerberg Reviewed-by: Andi Shyti Fixes: 525e6fabeae2 ("i2c / ACPI: add support for ACPI reconfigure notifications") Cc: # v4.8+ Signed-off-by: Wolfram Sang --- drivers/i2c/i2c-core-acpi.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c index d6037a3286690..14ae0cfc325ef 100644 --- a/drivers/i2c/i2c-core-acpi.c +++ b/drivers/i2c/i2c-core-acpi.c @@ -445,6 +445,11 @@ static struct i2c_client *i2c_acpi_find_client_by_adev(struct acpi_device *adev) return i2c_find_device_by_fwnode(acpi_fwnode_handle(adev)); } +static struct i2c_adapter *i2c_acpi_find_adapter_by_adev(struct acpi_device *adev) +{ + return i2c_find_adapter_by_fwnode(acpi_fwnode_handle(adev)); +} + static int i2c_acpi_notify(struct notifier_block *nb, unsigned long value, void *arg) { @@ -471,11 +476,17 @@ static int i2c_acpi_notify(struct notifier_block *nb, unsigned long value, break; client = i2c_acpi_find_client_by_adev(adev); - if (!client) - break; + if (client) { + i2c_unregister_device(client); + put_device(&client->dev); + } + + adapter = i2c_acpi_find_adapter_by_adev(adev); + if (adapter) { + acpi_unbind_one(&adapter->dev); + put_device(&adapter->dev); + } - i2c_unregister_device(client); - put_device(&client->dev); break; } From ba00196ca41c4f6d0b0d3c4a6748a133577abe05 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Fri, 3 May 2024 21:36:02 +0800 Subject: [PATCH 1243/1702] iommu/vt-d: Decouple igfx_off from graphic identity mapping A kernel command called igfx_off was introduced in commit ("Intel IOMMU: Intel IOMMU driver"). This command allows the user to disable the IOMMU dedicated to SOC-integrated graphic devices. Commit <9452618e7462> ("iommu/intel: disable DMAR for g4x integrated gfx") used this mechanism to disable the graphic-dedicated IOMMU for some problematic devices. Later, more problematic graphic devices were added to the list by commit <1f76249cc3beb> ("iommu/vt-d: Declare Broadwell igfx dmar support snafu"). On the other hand, commit <19943b0e30b05> ("intel-iommu: Unify hardware and software passthrough support") uses the identity domain for graphic devices if CONFIG_DMAR_BROKEN_GFX_WA is selected. + if (iommu_pass_through) + iommu_identity_mapping = 1; +#ifdef CONFIG_DMAR_BROKEN_GFX_WA + else + iommu_identity_mapping = 2; +#endif ... static int iommu_should_identity_map(struct pci_dev *pdev, int startup) { + if (iommu_identity_mapping == 2) + return IS_GFX_DEVICE(pdev); ... In the following driver evolution, CONFIG_DMAR_BROKEN_GFX_WA and quirk_iommu_igfx() are mixed together, causing confusion in the driver's device_def_domain_type callback. On one hand, dmar_map_gfx is used to turn off the graphic-dedicated IOMMU as a workaround for some buggy hardware; on the other hand, for those graphic devices, IDENTITY mapping is required for the IOMMU core. Commit <4b8d18c0c986> "iommu/vt-d: Remove INTEL_IOMMU_BROKEN_GFX_WA" has removed the CONFIG_DMAR_BROKEN_GFX_WA option, so the IDENTITY_DOMAIN requirement for graphic devices is no longer needed. Therefore, this requirement can be removed from device_def_domain_type() and igfx_off can be made independent. Fixes: 4b8d18c0c986 ("iommu/vt-d: Remove INTEL_IOMMU_BROKEN_GFX_WA") Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240428032020.214616-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 916cdb65d8494..e8e62ff1a65eb 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -216,12 +216,11 @@ int intel_iommu_sm = IS_ENABLED(CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON); int intel_iommu_enabled = 0; EXPORT_SYMBOL_GPL(intel_iommu_enabled); -static int dmar_map_gfx = 1; static int intel_iommu_superpage = 1; static int iommu_identity_mapping; static int iommu_skip_te_disable; +static int disable_igfx_iommu; -#define IDENTMAP_GFX 2 #define IDENTMAP_AZALIA 4 const struct iommu_ops intel_iommu_ops; @@ -260,7 +259,7 @@ static int __init intel_iommu_setup(char *str) no_platform_optin = 1; pr_info("IOMMU disabled\n"); } else if (!strncmp(str, "igfx_off", 8)) { - dmar_map_gfx = 0; + disable_igfx_iommu = 1; pr_info("Disable GFX device mapping\n"); } else if (!strncmp(str, "forcedac", 8)) { pr_warn("intel_iommu=forcedac deprecated; use iommu.forcedac instead\n"); @@ -2211,9 +2210,6 @@ static int device_def_domain_type(struct device *dev) if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev)) return IOMMU_DOMAIN_IDENTITY; - - if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev)) - return IOMMU_DOMAIN_IDENTITY; } return 0; @@ -2514,9 +2510,6 @@ static int __init init_dmars(void) iommu_set_root_entry(iommu); } - if (!dmar_map_gfx) - iommu_identity_mapping |= IDENTMAP_GFX; - check_tylersburg_isoch(); ret = si_domain_init(hw_pass_through); @@ -2607,7 +2600,7 @@ static void __init init_no_remapping_devices(void) /* This IOMMU has *only* gfx devices. Either bypass it or set the gfx_mapped flag, as appropriate */ drhd->gfx_dedicated = 1; - if (!dmar_map_gfx) + if (disable_igfx_iommu) drhd->ignored = 1; } } @@ -4649,7 +4642,7 @@ static void quirk_iommu_igfx(struct pci_dev *dev) return; pci_info(dev, "Disabling IOMMU for graphics on this chipset\n"); - dmar_map_gfx = 0; + disable_igfx_iommu = 1; } /* G4x/GM45 integrated gfx dmar support is totally busted. */ @@ -4730,8 +4723,8 @@ static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev) if (!(ggc & GGC_MEMORY_VT_ENABLED)) { pci_info(dev, "BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n"); - dmar_map_gfx = 0; - } else if (dmar_map_gfx) { + disable_igfx_iommu = 1; + } else if (!disable_igfx_iommu) { /* we have to ensure the gfx device is idle before we flush */ pci_info(dev, "Disabling batched IOTLB flush on Ironlake\n"); iommu_set_dma_strict(); From de111f6b4f6a3010020825d22a068f416bc29c95 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Mon, 6 May 2024 08:20:39 +0000 Subject: [PATCH 1244/1702] iommu/amd: Enable Guest Translation after reading IOMMU feature register Commit 8e0179733172 ("iommu/amd: Enable Guest Translation before registering devices") moved IOMMU Guest Translation (GT) enablement to early init path. It does feature check based on Global EFR value (got from ACPI IVRS table). Later it adjusts EFR value based on IOMMU feature register (late_iommu_features_init()). It seems in some systems BIOS doesn't set gloabl EFR value properly. This is causing mismatch. Hence move IOMMU GT enablement after late_iommu_features_init() so that it does check based on IOMMU EFR value. Fixes: 8e0179733172 ("iommu/amd: Enable Guest Translation before registering devices") Reported-by: Klara Modin Closes: https://lore.kernel.org/linux-iommu/333e6eb6-361c-4afb-8107-2573324bf689@gmail.com/ Signed-off-by: Vasant Hegde Tested-by: Klara Modin Link: https://lore.kernel.org/r/20240506082039.7575-1-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index fd3e76e436992..b292181995b7d 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -2046,6 +2046,8 @@ static int __init iommu_init_pci(struct amd_iommu *iommu) amd_iommu_max_glx_val = glxval; else amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval); + + iommu_enable_gt(iommu); } if (check_feature(FEATURE_PPR) && amd_iommu_alloc_ppr_log(iommu)) @@ -2732,7 +2734,6 @@ static void early_enable_iommu(struct amd_iommu *iommu) iommu_enable_command_buffer(iommu); iommu_enable_event_buffer(iommu); iommu_set_exclusion_range(iommu); - iommu_enable_gt(iommu); iommu_enable_ga(iommu); iommu_enable_xt(iommu); iommu_enable_irtcachedis(iommu); @@ -2789,7 +2790,6 @@ static void early_enable_iommus(void) iommu_disable_irtcachedis(iommu); iommu_enable_command_buffer(iommu); iommu_enable_event_buffer(iommu); - iommu_enable_gt(iommu); iommu_enable_ga(iommu); iommu_enable_xt(iommu); iommu_enable_irtcachedis(iommu); From 55750148e5595bb85605e8fbb40b2759c2c4c2d7 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 6 Jan 2024 13:48:24 +0100 Subject: [PATCH 1245/1702] i2c: synquacer: Fix an error handling path in synquacer_i2c_probe() If an error occurs after the clk_prepare_enable() call, it should be undone by a corresponding clk_disable_unprepare() call, as already done in the remove() function. As devm_clk_get() is used, we can switch to devm_clk_get_enabled() to handle it automatically and fix the probe. Update the remove() function accordingly and remove the now useless clk_disable_unprepare() call. Fixes: 0d676a6c4390 ("i2c: add support for Socionext SynQuacer I2C controller") Signed-off-by: Christophe JAILLET Acked-by: Ard Biesheuvel Signed-off-by: Andi Shyti --- drivers/i2c/busses/i2c-synquacer.c | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/drivers/i2c/busses/i2c-synquacer.c b/drivers/i2c/busses/i2c-synquacer.c index bbea521b05dda..a73f5bb9a1645 100644 --- a/drivers/i2c/busses/i2c-synquacer.c +++ b/drivers/i2c/busses/i2c-synquacer.c @@ -550,17 +550,13 @@ static int synquacer_i2c_probe(struct platform_device *pdev) device_property_read_u32(&pdev->dev, "socionext,pclk-rate", &i2c->pclkrate); - i2c->pclk = devm_clk_get(&pdev->dev, "pclk"); - if (PTR_ERR(i2c->pclk) == -EPROBE_DEFER) - return -EPROBE_DEFER; - if (!IS_ERR_OR_NULL(i2c->pclk)) { - dev_dbg(&pdev->dev, "clock source %p\n", i2c->pclk); - - ret = clk_prepare_enable(i2c->pclk); - if (ret) - return dev_err_probe(&pdev->dev, ret, "failed to enable clock\n"); - i2c->pclkrate = clk_get_rate(i2c->pclk); - } + i2c->pclk = devm_clk_get_enabled(&pdev->dev, "pclk"); + if (IS_ERR(i2c->pclk)) + return dev_err_probe(&pdev->dev, PTR_ERR(i2c->pclk), + "failed to get and enable clock\n"); + + dev_dbg(&pdev->dev, "clock source %p\n", i2c->pclk); + i2c->pclkrate = clk_get_rate(i2c->pclk); if (i2c->pclkrate < SYNQUACER_I2C_MIN_CLK_RATE || i2c->pclkrate > SYNQUACER_I2C_MAX_CLK_RATE) @@ -615,8 +611,6 @@ static void synquacer_i2c_remove(struct platform_device *pdev) struct synquacer_i2c *i2c = platform_get_drvdata(pdev); i2c_del_adapter(&i2c->adapter); - if (!IS_ERR(i2c->pclk)) - clk_disable_unprepare(i2c->pclk); }; static const struct of_device_id synquacer_i2c_dt_ids[] __maybe_unused = { From 23df17788c6212809848f836b10c4f85b16843a5 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 8 Apr 2024 12:09:15 +1000 Subject: [PATCH 1246/1702] nfsd: perform all find_openstateowner_str calls in the one place. Currently find_openstateowner_str look ups are done both in nfsd4_process_open1() and alloc_init_open_stateowner() - the latter possibly being a surprise based on its name. It would be easier to follow, and more conformant to common patterns, if the lookup was all in the one place. So replace alloc_init_open_stateowner() with find_or_alloc_open_stateowner() and use the latter in nfsd4_process_open1() without any calls to find_openstateowner_str(). This means all finds are find_openstateowner_str_locked() and find_openstateowner_str() is no longer needed. So discard find_openstateowner_str() and rename find_openstateowner_str_locked() to find_openstateowner_str(). Reviewed-by: Jeff Layton Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 93 +++++++++++++++++++-------------------------- 1 file changed, 40 insertions(+), 53 deletions(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 84d4093ca7131..65225b3d1d93c 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -541,7 +541,7 @@ same_owner_str(struct nfs4_stateowner *sop, struct xdr_netobj *owner) } static struct nfs4_openowner * -find_openstateowner_str_locked(unsigned int hashval, struct nfsd4_open *open, +find_openstateowner_str(unsigned int hashval, struct nfsd4_open *open, struct nfs4_client *clp) { struct nfs4_stateowner *so; @@ -558,18 +558,6 @@ find_openstateowner_str_locked(unsigned int hashval, struct nfsd4_open *open, return NULL; } -static struct nfs4_openowner * -find_openstateowner_str(unsigned int hashval, struct nfsd4_open *open, - struct nfs4_client *clp) -{ - struct nfs4_openowner *oo; - - spin_lock(&clp->cl_lock); - oo = find_openstateowner_str_locked(hashval, open, clp); - spin_unlock(&clp->cl_lock); - return oo; -} - static inline u32 opaque_hashval(const void *ptr, int nbytes) { @@ -4866,34 +4854,46 @@ nfsd4_find_and_lock_existing_open(struct nfs4_file *fp, struct nfsd4_open *open) } static struct nfs4_openowner * -alloc_init_open_stateowner(unsigned int strhashval, struct nfsd4_open *open, - struct nfsd4_compound_state *cstate) +find_or_alloc_open_stateowner(unsigned int strhashval, struct nfsd4_open *open, + struct nfsd4_compound_state *cstate) { struct nfs4_client *clp = cstate->clp; - struct nfs4_openowner *oo, *ret; + struct nfs4_openowner *oo, *new = NULL; - oo = alloc_stateowner(openowner_slab, &open->op_owner, clp); - if (!oo) - return NULL; - oo->oo_owner.so_ops = &openowner_ops; - oo->oo_owner.so_is_open_owner = 1; - oo->oo_owner.so_seqid = open->op_seqid; - oo->oo_flags = 0; - if (nfsd4_has_session(cstate)) - oo->oo_flags |= NFS4_OO_CONFIRMED; - oo->oo_time = 0; - oo->oo_last_closed_stid = NULL; - INIT_LIST_HEAD(&oo->oo_close_lru); +retry: spin_lock(&clp->cl_lock); - ret = find_openstateowner_str_locked(strhashval, open, clp); - if (ret == NULL) { - hash_openowner(oo, clp, strhashval); - ret = oo; - } else - nfs4_free_stateowner(&oo->oo_owner); - + oo = find_openstateowner_str(strhashval, open, clp); + if (!oo && new) { + hash_openowner(new, clp, strhashval); + spin_unlock(&clp->cl_lock); + return new; + } spin_unlock(&clp->cl_lock); - return ret; + + if (oo && !(oo->oo_flags & NFS4_OO_CONFIRMED)) { + /* Replace unconfirmed owners without checking for replay. */ + release_openowner(oo); + oo = NULL; + } + if (oo) { + if (new) + nfs4_free_stateowner(&new->oo_owner); + return oo; + } + + new = alloc_stateowner(openowner_slab, &open->op_owner, clp); + if (!new) + return NULL; + new->oo_owner.so_ops = &openowner_ops; + new->oo_owner.so_is_open_owner = 1; + new->oo_owner.so_seqid = open->op_seqid; + new->oo_flags = 0; + if (nfsd4_has_session(cstate)) + new->oo_flags |= NFS4_OO_CONFIRMED; + new->oo_time = 0; + new->oo_last_closed_stid = NULL; + INIT_LIST_HEAD(&new->oo_close_lru); + goto retry; } static struct nfs4_ol_stateid * @@ -5342,27 +5342,14 @@ nfsd4_process_open1(struct nfsd4_compound_state *cstate, clp = cstate->clp; strhashval = ownerstr_hashval(&open->op_owner); - oo = find_openstateowner_str(strhashval, open, clp); + oo = find_or_alloc_open_stateowner(strhashval, open, cstate); open->op_openowner = oo; - if (!oo) { - goto new_owner; - } - if (!(oo->oo_flags & NFS4_OO_CONFIRMED)) { - /* Replace unconfirmed owners without checking for replay. */ - release_openowner(oo); - open->op_openowner = NULL; - goto new_owner; - } + if (!oo) + return nfserr_jukebox; status = nfsd4_check_seqid(cstate, &oo->oo_owner, open->op_seqid); if (status) return status; - goto alloc_stateid; -new_owner: - oo = alloc_init_open_stateowner(strhashval, open, cstate); - if (oo == NULL) - return nfserr_jukebox; - open->op_openowner = oo; -alloc_stateid: + open->op_stp = nfs4_alloc_open_stateid(clp); if (!open->op_stp) return nfserr_jukebox; From b3f03739ca8cd6058a5d8754ea1354bc21fa0f2f Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 8 Apr 2024 12:09:16 +1000 Subject: [PATCH 1247/1702] nfsd: move nfsd4_cstate_assign_replay() earlier in open handling. Rather than taking the rp_mutex (via nfsd4_cstate_assign_replay) in nfsd4_cleanup_open_state() (which seems counter-intuitive), take it and assign rp_owner as soon as possible - in nfsd4_process_open1(). This will support a future change when nfsd4_cstate_assign_replay() might fail. Reviewed-by: Jeff Layton Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 65225b3d1d93c..43bdff49aa03b 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -5346,6 +5346,7 @@ nfsd4_process_open1(struct nfsd4_compound_state *cstate, open->op_openowner = oo; if (!oo) return nfserr_jukebox; + nfsd4_cstate_assign_replay(cstate, &oo->oo_owner); status = nfsd4_check_seqid(cstate, &oo->oo_owner, open->op_seqid); if (status) return status; @@ -6120,12 +6121,8 @@ nfsd4_process_open2(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nf void nfsd4_cleanup_open_state(struct nfsd4_compound_state *cstate, struct nfsd4_open *open) { - if (open->op_openowner) { - struct nfs4_stateowner *so = &open->op_openowner->oo_owner; - - nfsd4_cstate_assign_replay(cstate, so); - nfs4_put_stateowner(so); - } + if (open->op_openowner) + nfs4_put_stateowner(&open->op_openowner->oo_owner); if (open->op_file) kmem_cache_free(file_slab, open->op_file); if (open->op_stp) From eec7620800081e27dbf8019ac2e66259f0d5bf6f Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 8 Apr 2024 12:09:17 +1000 Subject: [PATCH 1248/1702] nfsd: replace rp_mutex to avoid deadlock in move_to_close_lru() move_to_close_lru() waits for sc_count to become zero while holding rp_mutex. This can deadlock if another thread holds a reference and is waiting for rp_mutex. By the time we get to move_to_close_lru() the openowner is unhashed and cannot be found any more. So code waiting for the mutex can safely retry the lookup if move_to_close_lru() has started. So change rp_mutex to an atomic_t with three states: RP_UNLOCK - state is still hashed, not locked for reply RP_LOCKED - state is still hashed, is locked for reply RP_UNHASHED - state is not hashed, no code can get a lock. Use wait_var_event() to wait for either a lock, or for the owner to be unhashed. In the latter case, retry the lookup. Reviewed-by: Jeff Layton Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 38 +++++++++++++++++++++++++++++++------- fs/nfsd/state.h | 2 +- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 43bdff49aa03b..e91e741719a02 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -4650,21 +4650,32 @@ nfsd4_init_leases_net(struct nfsd_net *nn) atomic_set(&nn->nfsd_courtesy_clients, 0); } +enum rp_lock { + RP_UNLOCKED, + RP_LOCKED, + RP_UNHASHED, +}; + static void init_nfs4_replay(struct nfs4_replay *rp) { rp->rp_status = nfserr_serverfault; rp->rp_buflen = 0; rp->rp_buf = rp->rp_ibuf; - mutex_init(&rp->rp_mutex); + atomic_set(&rp->rp_locked, RP_UNLOCKED); } -static void nfsd4_cstate_assign_replay(struct nfsd4_compound_state *cstate, - struct nfs4_stateowner *so) +static int nfsd4_cstate_assign_replay(struct nfsd4_compound_state *cstate, + struct nfs4_stateowner *so) { if (!nfsd4_has_session(cstate)) { - mutex_lock(&so->so_replay.rp_mutex); + wait_var_event(&so->so_replay.rp_locked, + atomic_cmpxchg(&so->so_replay.rp_locked, + RP_UNLOCKED, RP_LOCKED) != RP_LOCKED); + if (atomic_read(&so->so_replay.rp_locked) == RP_UNHASHED) + return -EAGAIN; cstate->replay_owner = nfs4_get_stateowner(so); } + return 0; } void nfsd4_cstate_clear_replay(struct nfsd4_compound_state *cstate) @@ -4673,7 +4684,8 @@ void nfsd4_cstate_clear_replay(struct nfsd4_compound_state *cstate) if (so != NULL) { cstate->replay_owner = NULL; - mutex_unlock(&so->so_replay.rp_mutex); + atomic_set(&so->so_replay.rp_locked, RP_UNLOCKED); + wake_up_var(&so->so_replay.rp_locked); nfs4_put_stateowner(so); } } @@ -4969,7 +4981,11 @@ move_to_close_lru(struct nfs4_ol_stateid *s, struct net *net) * Wait for the refcount to drop to 2. Since it has been unhashed, * there should be no danger of the refcount going back up again at * this point. + * Some threads with a reference might be waiting for rp_locked, + * so tell them to stop waiting. */ + atomic_set(&oo->oo_owner.so_replay.rp_locked, RP_UNHASHED); + wake_up_var(&oo->oo_owner.so_replay.rp_locked); wait_event(close_wq, refcount_read(&s->st_stid.sc_count) == 2); release_all_access(s); @@ -5342,11 +5358,15 @@ nfsd4_process_open1(struct nfsd4_compound_state *cstate, clp = cstate->clp; strhashval = ownerstr_hashval(&open->op_owner); +retry: oo = find_or_alloc_open_stateowner(strhashval, open, cstate); open->op_openowner = oo; if (!oo) return nfserr_jukebox; - nfsd4_cstate_assign_replay(cstate, &oo->oo_owner); + if (nfsd4_cstate_assign_replay(cstate, &oo->oo_owner) == -EAGAIN) { + nfs4_put_stateowner(&oo->oo_owner); + goto retry; + } status = nfsd4_check_seqid(cstate, &oo->oo_owner, open->op_seqid); if (status) return status; @@ -7186,12 +7206,16 @@ nfs4_preprocess_seqid_op(struct nfsd4_compound_state *cstate, u32 seqid, trace_nfsd_preprocess(seqid, stateid); *stpp = NULL; +retry: status = nfsd4_lookup_stateid(cstate, stateid, typemask, statusmask, &s, nn); if (status) return status; stp = openlockstateid(s); - nfsd4_cstate_assign_replay(cstate, stp->st_stateowner); + if (nfsd4_cstate_assign_replay(cstate, stp->st_stateowner) == -EAGAIN) { + nfs4_put_stateowner(stp->st_stateowner); + goto retry; + } status = nfs4_seqid_op_checks(cstate, stateid, seqid, stp); if (!status) diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index 2ed0fcf879fd1..a6261754deede 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -486,7 +486,7 @@ struct nfs4_replay { unsigned int rp_buflen; char *rp_buf; struct knfsd_fh rp_openfh; - struct mutex rp_mutex; + atomic_t rp_locked; char rp_ibuf[NFSD4_REPLAY_ISIZE]; }; From 56c35f43eef013579c76c007ba1f386d8c2cac14 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 8 Apr 2024 12:09:18 +1000 Subject: [PATCH 1249/1702] nfsd: drop st_mutex before calling move_to_close_lru() move_to_close_lru() is currently called with ->st_mutex held. This can lead to a deadlock as move_to_close_lru() waits for sc_count to drop to 2, and some threads holding a reference might be waiting for the mutex. These references will never be dropped so sc_count will never reach 2. There can be no harm in dropping ->st_mutex before move_to_close_lru() because the only place that takes the mutex is nfsd4_lock_ol_stateid(), and it quickly aborts if sc_type is NFS4_CLOSED_STID, which it will be before move_to_close_lru() is called. See also https://lore.kernel.org/lkml/4dd1fe21e11344e5969bb112e954affb@jd.com/T/ where this problem was raised but not successfully resolved. Reviewed-by: Jeff Layton Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index e91e741719a02..df07202cc53eb 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -7357,7 +7357,7 @@ nfsd4_open_downgrade(struct svc_rqst *rqstp, return status; } -static void nfsd4_close_open_stateid(struct nfs4_ol_stateid *s) +static bool nfsd4_close_open_stateid(struct nfs4_ol_stateid *s) { struct nfs4_client *clp = s->st_stid.sc_client; bool unhashed; @@ -7374,11 +7374,11 @@ static void nfsd4_close_open_stateid(struct nfs4_ol_stateid *s) list_for_each_entry(stp, &reaplist, st_locks) nfs4_free_cpntf_statelist(clp->net, &stp->st_stid); free_ol_stateid_reaplist(&reaplist); + return false; } else { spin_unlock(&clp->cl_lock); free_ol_stateid_reaplist(&reaplist); - if (unhashed) - move_to_close_lru(s, clp->net); + return unhashed; } } @@ -7394,6 +7394,7 @@ nfsd4_close(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, struct nfs4_ol_stateid *stp; struct net *net = SVC_NET(rqstp); struct nfsd_net *nn = net_generic(net, nfsd_net_id); + bool need_move_to_close_list; dprintk("NFSD: nfsd4_close on file %pd\n", cstate->current_fh.fh_dentry); @@ -7418,8 +7419,10 @@ nfsd4_close(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, */ nfs4_inc_and_copy_stateid(&close->cl_stateid, &stp->st_stid); - nfsd4_close_open_stateid(stp); + need_move_to_close_list = nfsd4_close_open_stateid(stp); mutex_unlock(&stp->st_mutex); + if (need_move_to_close_list) + move_to_close_lru(stp, net); /* v4.1+ suggests that we send a special stateid in here, since the * clients should just ignore this anyway. Since this is not useful From 38f080f3cd19fbd87eb7f6c4f6c236c1c9df6fba Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Mon, 1 Apr 2024 16:05:39 -0400 Subject: [PATCH 1250/1702] NFSD: Move callback_wq into struct nfs4_client Commit 883820366747 ("nfsd: update workqueue creation") made the callback_wq single-threaded, presumably to protect modifications of cl_cb_client. See documenting comment for nfsd4_process_cb_update(). However, cl_cb_client is per-lease. There's no other reason that all callback operations need to be dispatched via a single thread. The single threading here means all client callbacks can be blocked by a problem with one client. Change the NFSv4 callback client so it serializes per-lease instead of serializing all NFSv4 callback operations on the server. Reported-by: Dai Ngo Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4callback.c | 31 +++++++++---------------------- fs/nfsd/nfs4state.c | 14 +++++++------- fs/nfsd/state.h | 4 ++-- 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index e88aca0c6e8ef..d756f443fc444 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -978,12 +978,12 @@ static int max_cb_time(struct net *net) return max(((u32)nn->nfsd4_lease)/10, 1u) * HZ; } -static struct workqueue_struct *callback_wq; - static bool nfsd4_queue_cb(struct nfsd4_callback *cb) { - trace_nfsd_cb_queue(cb->cb_clp, cb); - return queue_work(callback_wq, &cb->cb_work); + struct nfs4_client *clp = cb->cb_clp; + + trace_nfsd_cb_queue(clp, cb); + return queue_work(clp->cl_callback_wq, &cb->cb_work); } static void nfsd41_cb_inflight_begin(struct nfs4_client *clp) @@ -1153,7 +1153,7 @@ void nfsd4_probe_callback(struct nfs4_client *clp) void nfsd4_probe_callback_sync(struct nfs4_client *clp) { nfsd4_probe_callback(clp); - flush_workqueue(callback_wq); + flush_workqueue(clp->cl_callback_wq); } void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *conn) @@ -1372,19 +1372,6 @@ static const struct rpc_call_ops nfsd4_cb_ops = { .rpc_release = nfsd4_cb_release, }; -int nfsd4_create_callback_queue(void) -{ - callback_wq = alloc_ordered_workqueue("nfsd4_callbacks", 0); - if (!callback_wq) - return -ENOMEM; - return 0; -} - -void nfsd4_destroy_callback_queue(void) -{ - destroy_workqueue(callback_wq); -} - /* must be called under the state lock */ void nfsd4_shutdown_callback(struct nfs4_client *clp) { @@ -1398,7 +1385,7 @@ void nfsd4_shutdown_callback(struct nfs4_client *clp) * client, destroy the rpc client, and stop: */ nfsd4_run_cb(&clp->cl_cb_null); - flush_workqueue(callback_wq); + flush_workqueue(clp->cl_callback_wq); nfsd41_cb_inflight_wait_complete(clp); } @@ -1420,9 +1407,9 @@ static struct nfsd4_conn * __nfsd4_find_backchannel(struct nfs4_client *clp) /* * Note there isn't a lot of locking in this code; instead we depend on - * the fact that it is run from the callback_wq, which won't run two - * work items at once. So, for example, callback_wq handles all access - * of cl_cb_client and all calls to rpc_create or rpc_shutdown_client. + * the fact that it is run from clp->cl_callback_wq, which won't run two + * work items at once. So, for example, clp->cl_callback_wq handles all + * access of cl_cb_client and all calls to rpc_create or rpc_shutdown_client. */ static void nfsd4_process_cb_update(struct nfsd4_callback *cb) { diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index df07202cc53eb..5ae70fb1d3a28 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -2233,6 +2233,10 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name, GFP_KERNEL); if (!clp->cl_ownerstr_hashtbl) goto err_no_hashtbl; + clp->cl_callback_wq = alloc_ordered_workqueue("nfsd4_callbacks", 0); + if (!clp->cl_callback_wq) + goto err_no_callback_wq; + for (i = 0; i < OWNER_HASH_SIZE; i++) INIT_LIST_HEAD(&clp->cl_ownerstr_hashtbl[i]); INIT_LIST_HEAD(&clp->cl_sessions); @@ -2255,6 +2259,8 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name, spin_lock_init(&clp->cl_lock); rpc_init_wait_queue(&clp->cl_cb_waitq, "Backchannel slot table"); return clp; +err_no_callback_wq: + kfree(clp->cl_ownerstr_hashtbl); err_no_hashtbl: kfree(clp->cl_name.data); err_no_name: @@ -2268,6 +2274,7 @@ static void __free_client(struct kref *k) struct nfs4_client *clp = container_of(c, struct nfs4_client, cl_nfsdfs); free_svc_cred(&clp->cl_cred); + destroy_workqueue(clp->cl_callback_wq); kfree(clp->cl_ownerstr_hashtbl); kfree(clp->cl_name.data); kfree(clp->cl_nii_domain.data); @@ -8636,12 +8643,6 @@ nfs4_state_start(void) if (ret) return ret; - ret = nfsd4_create_callback_queue(); - if (ret) { - rhltable_destroy(&nfs4_file_rhltable); - return ret; - } - set_max_delegations(); return 0; } @@ -8682,7 +8683,6 @@ nfs4_state_shutdown_net(struct net *net) void nfs4_state_shutdown(void) { - nfsd4_destroy_callback_queue(); rhltable_destroy(&nfs4_file_rhltable); } diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index a6261754deede..ffc217099d191 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -408,6 +408,8 @@ struct nfs4_client { 1 << NFSD4_CLIENT_CB_KILL) #define NFSD4_CLIENT_CB_RECALL_ANY (6) unsigned long cl_flags; + + struct workqueue_struct *cl_callback_wq; const struct cred *cl_cb_cred; struct rpc_clnt *cl_cb_client; u32 cl_cb_ident; @@ -735,8 +737,6 @@ extern void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn * extern void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp, const struct nfsd4_callback_ops *ops, enum nfsd4_cb_op op); extern bool nfsd4_run_cb(struct nfsd4_callback *cb); -extern int nfsd4_create_callback_queue(void); -extern void nfsd4_destroy_callback_queue(void); extern void nfsd4_shutdown_callback(struct nfs4_client *); extern void nfsd4_shutdown_copy(struct nfs4_client *clp); extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(struct xdr_netobj name, From 33a1e6ea73e5f1defe6706f006c0930a82ebdaaa Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Sat, 23 Mar 2024 09:25:54 -0400 Subject: [PATCH 1251/1702] nfsd: trivial GET_DIR_DELEGATION support This adds basic infrastructure for handing GET_DIR_DELEGATION calls from clients, including the decoders and encoders. For now, it always just returns NFS4_OK + GDD4_UNAVAIL. Eventually clients may start sending this operation, and it's better if we can return GDD4_UNAVAIL instead of having to abort the whole compound. Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4proc.c | 41 ++++++++++++++++++++++++ fs/nfsd/nfs4xdr.c | 76 ++++++++++++++++++++++++++++++++++++++++++-- fs/nfsd/xdr4.h | 19 +++++++++++ include/linux/nfs4.h | 6 ++++ 4 files changed, 140 insertions(+), 2 deletions(-) diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 2927b1263f086..3dc173b29803f 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -2154,6 +2154,29 @@ nfsd4_verify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, return status == nfserr_same ? nfs_ok : status; } +static __be32 +nfsd4_get_dir_delegation(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_get_dir_delegation *gdd = &u->get_dir_delegation; + + /* + * RFC 8881, section 18.39.3 says: + * + * "The server may refuse to grant the delegation. In that case, the + * server will return NFS4ERR_DIRDELEG_UNAVAIL." + * + * This is sub-optimal, since it means that the server would need to + * abort compound processing just because the delegation wasn't + * available. RFC8881bis should change this to allow the server to + * return NFS4_OK with a non-fatal status of GDD4_UNAVAIL in this + * situation. + */ + gdd->gddrnf_status = GDD4_UNAVAIL; + return nfs_ok; +} + #ifdef CONFIG_NFSD_PNFS static const struct nfsd4_layout_ops * nfsd4_layout_verify(struct svc_export *exp, unsigned int layout_type) @@ -3082,6 +3105,18 @@ static u32 nfsd4_copy_notify_rsize(const struct svc_rqst *rqstp, * sizeof(__be32); } +static u32 nfsd4_get_dir_delegation_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + + 1 /* gddr_status */ + + op_encode_verifier_maxsz + + op_encode_stateid_maxsz + + 2 /* gddr_notification */ + + 2 /* gddr_child_attributes */ + + 2 /* gddr_dir_attributes */); +} + #ifdef CONFIG_NFSD_PNFS static u32 nfsd4_getdeviceinfo_rsize(const struct svc_rqst *rqstp, const struct nfsd4_op *op) @@ -3470,6 +3505,12 @@ static const struct nfsd4_operation nfsd4_ops[] = { .op_get_currentstateid = nfsd4_get_freestateid, .op_rsize_bop = nfsd4_only_status_rsize, }, + [OP_GET_DIR_DELEGATION] = { + .op_func = nfsd4_get_dir_delegation, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_GET_DIR_DELEGATION", + .op_rsize_bop = nfsd4_get_dir_delegation_rsize, + }, #ifdef CONFIG_NFSD_PNFS [OP_GETDEVICEINFO] = { .op_func = nfsd4_getdeviceinfo, diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index a644460f3a5e7..ea76100e50caa 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -1732,6 +1732,35 @@ nfsd4_decode_free_stateid(struct nfsd4_compoundargs *argp, return nfsd4_decode_stateid4(argp, &free_stateid->fr_stateid); } +static __be32 +nfsd4_decode_get_dir_delegation(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_get_dir_delegation *gdd = &u->get_dir_delegation; + __be32 status; + + memset(gdd, 0, sizeof(*gdd)); + + if (xdr_stream_decode_bool(argp->xdr, &gdd->gdda_signal_deleg_avail) < 0) + return nfserr_bad_xdr; + status = nfsd4_decode_bitmap4(argp, gdd->gdda_notification_types, + ARRAY_SIZE(gdd->gdda_notification_types)); + if (status) + return status; + status = nfsd4_decode_nfstime4(argp, &gdd->gdda_child_attr_delay); + if (status) + return status; + status = nfsd4_decode_nfstime4(argp, &gdd->gdda_dir_attr_delay); + if (status) + return status; + status = nfsd4_decode_bitmap4(argp, gdd->gdda_child_attributes, + ARRAY_SIZE(gdd->gdda_child_attributes)); + if (status) + return status; + return nfsd4_decode_bitmap4(argp, gdd->gdda_dir_attributes, + ARRAY_SIZE(gdd->gdda_dir_attributes)); +} + #ifdef CONFIG_NFSD_PNFS static __be32 nfsd4_decode_getdeviceinfo(struct nfsd4_compoundargs *argp, @@ -2370,7 +2399,7 @@ static const nfsd4_dec nfsd4_dec_ops[] = { [OP_CREATE_SESSION] = nfsd4_decode_create_session, [OP_DESTROY_SESSION] = nfsd4_decode_destroy_session, [OP_FREE_STATEID] = nfsd4_decode_free_stateid, - [OP_GET_DIR_DELEGATION] = nfsd4_decode_notsupp, + [OP_GET_DIR_DELEGATION] = nfsd4_decode_get_dir_delegation, #ifdef CONFIG_NFSD_PNFS [OP_GETDEVICEINFO] = nfsd4_decode_getdeviceinfo, [OP_GETDEVICELIST] = nfsd4_decode_notsupp, @@ -4963,6 +4992,49 @@ nfsd4_encode_test_stateid(struct nfsd4_compoundres *resp, __be32 nfserr, return nfs_ok; } +static __be32 +nfsd4_encode_get_dir_delegation(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_get_dir_delegation *gdd = &u->get_dir_delegation; + struct xdr_stream *xdr = resp->xdr; + __be32 status = nfserr_resource; + + switch(gdd->gddrnf_status) { + case GDD4_OK: + if (xdr_stream_encode_u32(xdr, GDD4_OK) != XDR_UNIT) + break; + status = nfsd4_encode_verifier4(xdr, &gdd->gddr_cookieverf); + if (status) + break; + status = nfsd4_encode_stateid4(xdr, &gdd->gddr_stateid); + if (status) + break; + status = nfsd4_encode_bitmap4(xdr, gdd->gddr_notification[0], 0, 0); + if (status) + break; + status = nfsd4_encode_bitmap4(xdr, gdd->gddr_child_attributes[0], + gdd->gddr_child_attributes[1], + gdd->gddr_child_attributes[2]); + if (status) + break; + status = nfsd4_encode_bitmap4(xdr, gdd->gddr_dir_attributes[0], + gdd->gddr_dir_attributes[1], + gdd->gddr_dir_attributes[2]); + break; + default: + pr_warn("nfsd: bad gddrnf_status (%u)\n", gdd->gddrnf_status); + gdd->gddrnf_will_signal_deleg_avail = 0; + fallthrough; + case GDD4_UNAVAIL: + if (xdr_stream_encode_u32(xdr, GDD4_UNAVAIL) != XDR_UNIT) + break; + status = nfsd4_encode_bool(xdr, gdd->gddrnf_will_signal_deleg_avail); + break; + } + return status; +} + #ifdef CONFIG_NFSD_PNFS static __be32 nfsd4_encode_device_addr4(struct xdr_stream *xdr, @@ -5579,7 +5651,7 @@ static const nfsd4_enc nfsd4_enc_ops[] = { [OP_CREATE_SESSION] = nfsd4_encode_create_session, [OP_DESTROY_SESSION] = nfsd4_encode_noop, [OP_FREE_STATEID] = nfsd4_encode_noop, - [OP_GET_DIR_DELEGATION] = nfsd4_encode_noop, + [OP_GET_DIR_DELEGATION] = nfsd4_encode_get_dir_delegation, #ifdef CONFIG_NFSD_PNFS [OP_GETDEVICEINFO] = nfsd4_encode_getdeviceinfo, [OP_GETDEVICELIST] = nfsd4_encode_noop, diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h index 415516c1b27e5..446e72b0385e7 100644 --- a/fs/nfsd/xdr4.h +++ b/fs/nfsd/xdr4.h @@ -518,6 +518,24 @@ struct nfsd4_free_stateid { stateid_t fr_stateid; /* request */ }; +struct nfsd4_get_dir_delegation { + /* request */ + u32 gdda_signal_deleg_avail; + u32 gdda_notification_types[1]; + struct timespec64 gdda_child_attr_delay; + struct timespec64 gdda_dir_attr_delay; + u32 gdda_child_attributes[3]; + u32 gdda_dir_attributes[3]; + /* response */ + u32 gddrnf_status; + nfs4_verifier gddr_cookieverf; + stateid_t gddr_stateid; + u32 gddr_notification[1]; + u32 gddr_child_attributes[3]; + u32 gddr_dir_attributes[3]; + bool gddrnf_will_signal_deleg_avail; +}; + /* also used for NVERIFY */ struct nfsd4_verify { u32 ve_bmval[3]; /* request */ @@ -797,6 +815,7 @@ struct nfsd4_op { struct nfsd4_reclaim_complete reclaim_complete; struct nfsd4_test_stateid test_stateid; struct nfsd4_free_stateid free_stateid; + struct nfsd4_get_dir_delegation get_dir_delegation; struct nfsd4_getdeviceinfo getdeviceinfo; struct nfsd4_layoutget layoutget; struct nfsd4_layoutcommit layoutcommit; diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h index ef8d2d618d5b3..0d896ce296cec 100644 --- a/include/linux/nfs4.h +++ b/include/linux/nfs4.h @@ -701,6 +701,12 @@ enum state_protect_how4 { SP4_SSV = 2 }; +/* GET_DIR_DELEGATION non-fatal status codes */ +enum gddrnf4_status { + GDD4_OK = 0, + GDD4_UNAVAIL = 1 +}; + enum pnfs_layouttype { LAYOUT_NFSV4_1_FILES = 1, LAYOUT_OSD2_OBJECTS = 2, From 7d12cce8784c66ad9351e91743cd68a573850732 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Mon, 25 Mar 2024 21:21:39 +0800 Subject: [PATCH 1252/1702] fs: nfsd: use group allocation/free of per-cpu counters API Use group allocation/free of per-cpu counters api to accelerate nfsd percpu_counters init/destroy(), and also squash the nfsd_percpu_counters_init/reset/destroy() and nfsd_counters_init/destroy() into callers to simplify code. Signed-off-by: Kefeng Wang Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/export.c | 16 ++++++++++------ fs/nfsd/nfsctl.c | 5 +++-- fs/nfsd/stats.c | 42 ------------------------------------------ fs/nfsd/stats.h | 5 ----- 4 files changed, 13 insertions(+), 55 deletions(-) diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index 7b641095a665f..50b3135d07ac7 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -334,21 +334,25 @@ static void nfsd4_fslocs_free(struct nfsd4_fs_locations *fsloc) static int export_stats_init(struct export_stats *stats) { stats->start_time = ktime_get_seconds(); - return nfsd_percpu_counters_init(stats->counter, EXP_STATS_COUNTERS_NUM); + return percpu_counter_init_many(stats->counter, 0, GFP_KERNEL, + EXP_STATS_COUNTERS_NUM); } static void export_stats_reset(struct export_stats *stats) { - if (stats) - nfsd_percpu_counters_reset(stats->counter, - EXP_STATS_COUNTERS_NUM); + if (stats) { + int i; + + for (i = 0; i < EXP_STATS_COUNTERS_NUM; i++) + percpu_counter_set(&stats->counter[i], 0); + } } static void export_stats_destroy(struct export_stats *stats) { if (stats) - nfsd_percpu_counters_destroy(stats->counter, - EXP_STATS_COUNTERS_NUM); + percpu_counter_destroy_many(stats->counter, + EXP_STATS_COUNTERS_NUM); } static void svc_export_put(struct kref *ref) diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index ecd18bffeebc7..93c87587e6461 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -1672,7 +1672,8 @@ static __net_init int nfsd_net_init(struct net *net) retval = nfsd_idmap_init(net); if (retval) goto out_idmap_error; - retval = nfsd_stat_counters_init(nn); + retval = percpu_counter_init_many(nn->counter, 0, GFP_KERNEL, + NFSD_STATS_COUNTERS_NUM); if (retval) goto out_repcache_error; memset(&nn->nfsd_svcstats, 0, sizeof(nn->nfsd_svcstats)); @@ -1704,7 +1705,7 @@ static __net_exit void nfsd_net_exit(struct net *net) struct nfsd_net *nn = net_generic(net, nfsd_net_id); nfsd_proc_stat_shutdown(net); - nfsd_stat_counters_destroy(nn); + percpu_counter_destroy_many(nn->counter, NFSD_STATS_COUNTERS_NUM); nfsd_idmap_shutdown(net); nfsd_export_shutdown(net); nfsd_netns_free_versions(nn); diff --git a/fs/nfsd/stats.c b/fs/nfsd/stats.c index be52fb1e928ed..bb22893f1157e 100644 --- a/fs/nfsd/stats.c +++ b/fs/nfsd/stats.c @@ -73,48 +73,6 @@ static int nfsd_show(struct seq_file *seq, void *v) DEFINE_PROC_SHOW_ATTRIBUTE(nfsd); -int nfsd_percpu_counters_init(struct percpu_counter *counters, int num) -{ - int i, err = 0; - - for (i = 0; !err && i < num; i++) - err = percpu_counter_init(&counters[i], 0, GFP_KERNEL); - - if (!err) - return 0; - - for (; i > 0; i--) - percpu_counter_destroy(&counters[i-1]); - - return err; -} - -void nfsd_percpu_counters_reset(struct percpu_counter counters[], int num) -{ - int i; - - for (i = 0; i < num; i++) - percpu_counter_set(&counters[i], 0); -} - -void nfsd_percpu_counters_destroy(struct percpu_counter counters[], int num) -{ - int i; - - for (i = 0; i < num; i++) - percpu_counter_destroy(&counters[i]); -} - -int nfsd_stat_counters_init(struct nfsd_net *nn) -{ - return nfsd_percpu_counters_init(nn->counter, NFSD_STATS_COUNTERS_NUM); -} - -void nfsd_stat_counters_destroy(struct nfsd_net *nn) -{ - nfsd_percpu_counters_destroy(nn->counter, NFSD_STATS_COUNTERS_NUM); -} - void nfsd_proc_stat_init(struct net *net) { struct nfsd_net *nn = net_generic(net, nfsd_net_id); diff --git a/fs/nfsd/stats.h b/fs/nfsd/stats.h index d2753e975dfd3..04aacb6c36e25 100644 --- a/fs/nfsd/stats.h +++ b/fs/nfsd/stats.h @@ -10,11 +10,6 @@ #include #include -int nfsd_percpu_counters_init(struct percpu_counter *counters, int num); -void nfsd_percpu_counters_reset(struct percpu_counter *counters, int num); -void nfsd_percpu_counters_destroy(struct percpu_counter *counters, int num); -int nfsd_stat_counters_init(struct nfsd_net *nn); -void nfsd_stat_counters_destroy(struct nfsd_net *nn); void nfsd_proc_stat_init(struct net *net); void nfsd_proc_stat_shutdown(struct net *net); From a576f36971ab4097b6aa76433532aa1fb5ee2d3b Mon Sep 17 00:00:00 2001 From: Aleksandr Aprelkov Date: Wed, 27 Mar 2024 14:10:44 +0700 Subject: [PATCH 1253/1702] sunrpc: removed redundant procp check since vs_proc pointer is dereferenced before getting it's address there's no need to check for NULL. Found by Linux Verification Center (linuxtesting.org) with SVACE. Fixes: 8e5b67731d08 ("SUNRPC: Add a callback to initialise server requests") Signed-off-by: Aleksandr Aprelkov Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- net/sunrpc/svc.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index b33e429336fb7..2b4b1276d4e86 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -1265,8 +1265,6 @@ svc_generic_init_request(struct svc_rqst *rqstp, if (rqstp->rq_proc >= versp->vs_nproc) goto err_bad_proc; rqstp->rq_procinfo = procp = &versp->vs_proc[rqstp->rq_proc]; - if (!procp) - goto err_bad_proc; /* Initialize storage for argp and resp */ memset(rqstp->rq_argp, 0, procp->pc_argzero); From db2acd6e8ab1606c47bcea3e269ac7cff897eb8d Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Fri, 5 Apr 2024 14:40:49 -0400 Subject: [PATCH 1254/1702] nfsd: drop extraneous newline from nfsd tracepoints We never want a newline in tracepoint output. Signed-off-by: Jeff Layton Tested-by: Vladimir Benes Signed-off-by: Chuck Lever --- fs/nfsd/trace.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 1cd2076210b14..c0cc345a576e0 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -1534,7 +1534,7 @@ TRACE_EVENT(nfsd_cb_seq_status, __entry->seq_status = cb->cb_seq_status; ), TP_printk(SUNRPC_TRACE_TASK_SPECIFIER - " sessionid=%08x:%08x:%08x:%08x tk_status=%d seq_status=%d\n", + " sessionid=%08x:%08x:%08x:%08x tk_status=%d seq_status=%d", __entry->task_id, __entry->client_id, __entry->cl_boot, __entry->cl_id, __entry->seqno, __entry->reserved, @@ -1573,7 +1573,7 @@ TRACE_EVENT(nfsd_cb_free_slot, __entry->slot_seqno = session->se_cb_seq_nr; ), TP_printk(SUNRPC_TRACE_TASK_SPECIFIER - " sessionid=%08x:%08x:%08x:%08x new slot seqno=%u\n", + " sessionid=%08x:%08x:%08x:%08x new slot seqno=%u", __entry->task_id, __entry->client_id, __entry->cl_boot, __entry->cl_id, __entry->seqno, __entry->reserved, @@ -1978,7 +1978,7 @@ TRACE_EVENT(nfsd_ctl_time, __entry->time = time; __assign_str(name, name); ), - TP_printk("file=%s time=%d\n", + TP_printk("file=%s time=%d", __get_str(name), __entry->time ) ); From 2d49901150ecf82f9a3d3d2dc30e38b17fef71a0 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 5 Apr 2024 14:40:50 -0400 Subject: [PATCH 1255/1702] nfsd: new tracepoint for check_slot_seqid Replace a dprintk in check_slot_seqid with tracepoints. These new tracepoints track slot sequence numbers during operation. Suggested-by: Jeffrey Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 14 ++++----- fs/nfsd/trace.h | 70 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 5ae70fb1d3a28..bd81217ada7f7 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -3636,12 +3636,8 @@ nfsd4_exchange_id(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, return status; } -static __be32 -check_slot_seqid(u32 seqid, u32 slot_seqid, int slot_inuse) +static __be32 check_slot_seqid(u32 seqid, u32 slot_seqid, bool slot_inuse) { - dprintk("%s enter. seqid %d slot_seqid %d\n", __func__, seqid, - slot_seqid); - /* The slot is in use, and no response has been sent. */ if (slot_inuse) { if (seqid == slot_seqid) @@ -3818,10 +3814,13 @@ nfsd4_create_session(struct svc_rqst *rqstp, } /* RFC 8881 Section 18.36.4 Phase 2: Sequence ID processing. */ - if (conf) + if (conf) { cs_slot = &conf->cl_cs_slot; - else + trace_nfsd_slot_seqid_conf(conf, cr_ses); + } else { cs_slot = &unconf->cl_cs_slot; + trace_nfsd_slot_seqid_unconf(unconf, cr_ses); + } status = check_slot_seqid(cr_ses->seqid, cs_slot->sl_seqid, 0); switch (status) { case nfs_ok: @@ -4216,6 +4215,7 @@ nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, * sr_highest_slotid and the sr_target_slot id to maxslots */ seq->maxslots = session->se_fchannel.maxreqs; + trace_nfsd_slot_seqid_sequence(clp, seq, slot); status = check_slot_seqid(seq->seqid, slot->sl_seqid, slot->sl_flags & NFSD4_SLOT_INUSE); if (status == nfserr_replay_cache) { diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index c0cc345a576e0..ac2124461a08f 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -749,6 +749,76 @@ TRACE_EVENT_CONDITION(nfsd_seq4_status, ) ); +DECLARE_EVENT_CLASS(nfsd_cs_slot_class, + TP_PROTO( + const struct nfs4_client *clp, + const struct nfsd4_create_session *cs + ), + TP_ARGS(clp, cs), + TP_STRUCT__entry( + __field(u32, seqid) + __field(u32, slot_seqid) + __field(u32, cl_boot) + __field(u32, cl_id) + __sockaddr(addr, clp->cl_cb_conn.cb_addrlen) + ), + TP_fast_assign( + const struct nfsd4_clid_slot *slot = &clp->cl_cs_slot; + + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __assign_sockaddr(addr, &clp->cl_cb_conn.cb_addr, + clp->cl_cb_conn.cb_addrlen); + __entry->seqid = cs->seqid; + __entry->slot_seqid = slot->sl_seqid; + ), + TP_printk("addr=%pISpc client %08x:%08x seqid=%u slot_seqid=%u", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, + __entry->seqid, __entry->slot_seqid + ) +); + +#define DEFINE_CS_SLOT_EVENT(name) \ +DEFINE_EVENT(nfsd_cs_slot_class, nfsd_##name, \ + TP_PROTO( \ + const struct nfs4_client *clp, \ + const struct nfsd4_create_session *cs \ + ), \ + TP_ARGS(clp, cs)) + +DEFINE_CS_SLOT_EVENT(slot_seqid_conf); +DEFINE_CS_SLOT_EVENT(slot_seqid_unconf); + +TRACE_EVENT(nfsd_slot_seqid_sequence, + TP_PROTO( + const struct nfs4_client *clp, + const struct nfsd4_sequence *seq, + const struct nfsd4_slot *slot + ), + TP_ARGS(clp, seq, slot), + TP_STRUCT__entry( + __field(u32, seqid) + __field(u32, slot_seqid) + __field(u32, cl_boot) + __field(u32, cl_id) + __sockaddr(addr, clp->cl_cb_conn.cb_addrlen) + __field(bool, in_use) + ), + TP_fast_assign( + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __assign_sockaddr(addr, &clp->cl_cb_conn.cb_addr, + clp->cl_cb_conn.cb_addrlen); + __entry->seqid = seq->seqid; + __entry->slot_seqid = slot->sl_seqid; + ), + TP_printk("addr=%pISpc client %08x:%08x seqid=%u slot_seqid=%u (%sin use)", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, + __entry->seqid, __entry->slot_seqid, + __entry->in_use ? "" : "not " + ) +); + DECLARE_EVENT_CLASS(nfsd_clientid_class, TP_PROTO(const clientid_t *clid), TP_ARGS(clid), From 9320f27fda5498c80f9e0c7d8c80d50e8297d89e Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Fri, 5 Apr 2024 14:40:51 -0400 Subject: [PATCH 1256/1702] nfsd: add tracepoint in mark_client_expired_locked Show client info alongside the number of cl_rpc_users. If that's elevated, then we can infer that this function returned nfserr_jukebox. [ cel: For additional debugging of RPC user refcounting ] Signed-off-by: Jeff Layton Tested-by: Vladimir Benes Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 6 +++++- fs/nfsd/trace.h | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index bd81217ada7f7..66f9a4f031a4d 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -2347,7 +2347,11 @@ unhash_client(struct nfs4_client *clp) static __be32 mark_client_expired_locked(struct nfs4_client *clp) { - if (atomic_read(&clp->cl_rpc_users)) + int users = atomic_read(&clp->cl_rpc_users); + + trace_nfsd_mark_client_expired(clp, users); + + if (users) return nfserr_jukebox; unhash_client_locked(clp); return nfs_ok; diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index ac2124461a08f..b5e48d504062d 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -848,6 +848,30 @@ DEFINE_CLIENTID_EVENT(purged); DEFINE_CLIENTID_EVENT(renew); DEFINE_CLIENTID_EVENT(stale); +TRACE_EVENT(nfsd_mark_client_expired, + TP_PROTO( + const struct nfs4_client *clp, + int cl_rpc_users + ), + TP_ARGS(clp, cl_rpc_users), + TP_STRUCT__entry( + __field(int, cl_rpc_users) + __field(u32, cl_boot) + __field(u32, cl_id) + __sockaddr(addr, clp->cl_cb_conn.cb_addrlen) + ), + TP_fast_assign( + __entry->cl_rpc_users = cl_rpc_users; + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __assign_sockaddr(addr, &clp->cl_cb_conn.cb_addr, + clp->cl_cb_conn.cb_addrlen) + ), + TP_printk("addr=%pISpc client %08x:%08x cl_rpc_users=%d", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, + __entry->cl_rpc_users) +); + DECLARE_EVENT_CLASS(nfsd_net_class, TP_PROTO(const struct nfsd_net *nn), TP_ARGS(nn), From d43113fbbf79c00fe3ecd637f8cadebc4df9a63c Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 8 Apr 2024 11:29:52 +1000 Subject: [PATCH 1257/1702] nfsd: optimise recalculate_deny_mode() for a common case recalculate_deny_mode() takes time that is linear in the number of stateids active on the file. When called from release_openowner -> free_ol_stateid_reaplist ->nfs4_free_ol_stateid -> release_all_access the number of times it is called is linear in the number of stateids. The net result is that time taken by release_openowner is quadratic in the number of stateids. When the nfsd server is shut down while there are many active stateids this can result in a soft lockup. ("CPU stuck for 302s" seen in one case). In many cases all the states have the same deny modes and there is no need to examine the entire list in recalculate_deny_mode(). In particular, recalculate_deny_mode() will only reduce the deny mode, never increase it. So if some prefix of the list causes the original deny mode to be required, there is no need to examine the remainder of the list. So we can improve recalculate_deny_mode() to usually run in constant time, so release_openowner will typically be only linear in the number of states. Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 66f9a4f031a4d..a20c2c9d7d457 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -1397,11 +1397,16 @@ static void recalculate_deny_mode(struct nfs4_file *fp) { struct nfs4_ol_stateid *stp; + u32 old_deny; spin_lock(&fp->fi_lock); + old_deny = fp->fi_share_deny; fp->fi_share_deny = 0; - list_for_each_entry(stp, &fp->fi_stateids, st_perfile) + list_for_each_entry(stp, &fp->fi_stateids, st_perfile) { fp->fi_share_deny |= bmap_to_share_mode(stp->st_deny_bmap); + if (fp->fi_share_deny == old_deny) + break; + } spin_unlock(&fp->fi_lock); } From 0770249b90f9d9f69714b76adc36cf6c895bc1f9 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Wed, 17 Apr 2024 07:23:02 +1000 Subject: [PATCH 1258/1702] nfsd: don't create nfsv4recoverydir in nfsdfs when not used. When CONFIG_NFSD_LEGACY_CLIENT_TRACKING is not set, the virtual file /proc/fs/nfsd/nfsv4recoverydir is created but responds EINVAL to any access. This is not useful, is somewhat surprising, and it causes ltp to complain. The only known user of this file is in nfs-utils, which handles non-existence and read-failure equally well. So there is nothing to gain from leaving the file present but inaccessible. So this patch removes the file when its content is not available - i.e. when that config option is not selected. Also remove the #ifdef which hides some of the enum values when CONFIG_NFSD_V$ not selection. simple_fill_super() quietly ignores array entries that are not present, so having slots in the array that don't get used is perfectly acceptable. So there is no value in this #ifdef. Reported-by: Petr Vorel Reviewed-by: Jeff Layton Fixes: 74fd48739d04 ("nfsd: new Kconfig option for legacy client tracking") Signed-off-by: NeilBrown Reviewed-by: Petr Vorel Signed-off-by: Chuck Lever --- fs/nfsd/nfsctl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 93c87587e6461..340c5d61f199d 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -48,12 +48,10 @@ enum { NFSD_MaxBlkSize, NFSD_MaxConnections, NFSD_Filecache, -#ifdef CONFIG_NFSD_V4 NFSD_Leasetime, NFSD_Gracetime, NFSD_RecoveryDir, NFSD_V4EndGrace, -#endif NFSD_MaxReserved }; @@ -1360,7 +1358,9 @@ static int nfsd_fill_super(struct super_block *sb, struct fs_context *fc) #ifdef CONFIG_NFSD_V4 [NFSD_Leasetime] = {"nfsv4leasetime", &transaction_ops, S_IWUSR|S_IRUSR}, [NFSD_Gracetime] = {"nfsv4gracetime", &transaction_ops, S_IWUSR|S_IRUSR}, +#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING [NFSD_RecoveryDir] = {"nfsv4recoverydir", &transaction_ops, S_IWUSR|S_IRUSR}, +#endif [NFSD_V4EndGrace] = {"v4_end_grace", &transaction_ops, S_IWUSR|S_IRUGO}, #endif /* last one */ {""} From 03b0036f452d244b6bfeab3973e9e57e8eac9c04 Mon Sep 17 00:00:00 2001 From: Li kunyu Date: Wed, 17 Apr 2024 16:28:07 +0800 Subject: [PATCH 1259/1702] =?UTF-8?q?lockd:=20host:=20Remove=20unnecessary?= =?UTF-8?q?=20statements=EF=BC=87host=20=3D=20NULL;=EF=BC=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In 'nlm_alloc_host', the host has already been assigned a value of NULL when defined, so 'host=NULL;' Can be deleted. Signed-off-by: Li kunyu Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/lockd/host.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/lockd/host.c b/fs/lockd/host.c index 127a728fcbc81..c115168017845 100644 --- a/fs/lockd/host.c +++ b/fs/lockd/host.c @@ -117,7 +117,6 @@ static struct nlm_host *nlm_alloc_host(struct nlm_lookup_host_info *ni, if (nsm != NULL) refcount_inc(&nsm->sm_count); else { - host = NULL; nsm = nsm_get_handle(ni->net, ni->sap, ni->salen, ni->hostname, ni->hostname_len); if (unlikely(nsm == NULL)) { From 0842b4c80bc7162a803b1558dc6d887e0e8feb39 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 23 Apr 2024 15:25:38 +0200 Subject: [PATCH 1260/1702] NFSD: move nfsd_mutex handling into nfsd_svc callers Currently nfsd_svc holds the nfsd_mutex over the whole function. For some of the later netlink patches though, we want to do some other things to the server before starting it. Move the mutex handling into the callers. Signed-off-by: Jeff Layton Signed-off-by: Lorenzo Bianconi Signed-off-by: Chuck Lever --- fs/nfsd/nfsctl.c | 2 ++ fs/nfsd/nfssvc.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 340c5d61f199d..2fe78b802f984 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -404,7 +404,9 @@ static ssize_t write_threads(struct file *file, char *buf, size_t size) if (newthreads < 0) return -EINVAL; trace_nfsd_ctl_threads(net, newthreads); + mutex_lock(&nfsd_mutex); rv = nfsd_svc(newthreads, net, file->f_cred); + mutex_unlock(&nfsd_mutex); if (rv < 0) return rv; } else diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index c0d17b92b249f..ca193f7ff0e15 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -775,7 +775,8 @@ nfsd_svc(int nrservs, struct net *net, const struct cred *cred) struct nfsd_net *nn = net_generic(net, nfsd_net_id); struct svc_serv *serv; - mutex_lock(&nfsd_mutex); + lockdep_assert_held(&nfsd_mutex); + dprintk("nfsd: creating service\n"); nrservs = max(nrservs, 0); @@ -804,7 +805,6 @@ nfsd_svc(int nrservs, struct net *net, const struct cred *cred) if (serv->sv_nrthreads == 0) nfsd_destroy_serv(net); out: - mutex_unlock(&nfsd_mutex); return error; } From 9077d59847896745712dff00839eac14f84b21f8 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 23 Apr 2024 15:25:39 +0200 Subject: [PATCH 1261/1702] NFSD: allow callers to pass in scope string to nfsd_svc Currently admins set this by using unshare to create a new uts namespace, and then resetting the hostname. With the new netlink interface we can just pass this in directly. Prepare nfsd_svc for this change. Signed-off-by: Jeff Layton Signed-off-by: Lorenzo Bianconi Signed-off-by: Chuck Lever --- fs/nfsd/nfsctl.c | 2 +- fs/nfsd/nfsd.h | 2 +- fs/nfsd/nfssvc.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 2fe78b802f984..34148d73b6c18 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -405,7 +405,7 @@ static ssize_t write_threads(struct file *file, char *buf, size_t size) return -EINVAL; trace_nfsd_ctl_threads(net, newthreads); mutex_lock(&nfsd_mutex); - rv = nfsd_svc(newthreads, net, file->f_cred); + rv = nfsd_svc(newthreads, net, file->f_cred, NULL); mutex_unlock(&nfsd_mutex); if (rv < 0) return rv; diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 16c5a05f340e5..2f6c6f3815b4d 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -103,7 +103,7 @@ bool nfssvc_encode_voidres(struct svc_rqst *rqstp, /* * Function prototypes. */ -int nfsd_svc(int nrservs, struct net *net, const struct cred *cred); +int nfsd_svc(int nrservs, struct net *net, const struct cred *cred, const char *scope); int nfsd_dispatch(struct svc_rqst *rqstp); int nfsd_nrthreads(struct net *); diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index ca193f7ff0e15..e25b9b829749d 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -769,7 +769,7 @@ int nfsd_set_nrthreads(int n, int *nthreads, struct net *net) * this is the first time nrservs is nonzero. */ int -nfsd_svc(int nrservs, struct net *net, const struct cred *cred) +nfsd_svc(int nrservs, struct net *net, const struct cred *cred, const char *scope) { int error; struct nfsd_net *nn = net_generic(net, nfsd_net_id); @@ -786,7 +786,7 @@ nfsd_svc(int nrservs, struct net *net, const struct cred *cred) if (nrservs == 0 && nn->nfsd_serv == NULL) goto out; - strscpy(nn->nfsd_name, utsname()->nodename, + strscpy(nn->nfsd_name, scope ? scope : utsname()->nodename, sizeof(nn->nfsd_name)); error = nfsd_create_serv(net); From 924f4fb003ba114c60b3c07a011dcd86a8956cd1 Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Tue, 23 Apr 2024 15:25:40 +0200 Subject: [PATCH 1262/1702] NFSD: convert write_threads to netlink command Introduce write_threads netlink command similar to the one available through the procfs. Tested-by: Jeff Layton Reviewed-by: Jeff Layton Co-developed-by: Jeff Layton Signed-off-by: Jeff Layton Signed-off-by: Lorenzo Bianconi Signed-off-by: Chuck Lever --- Documentation/netlink/specs/nfsd.yaml | 39 +++++++ fs/nfsd/netlink.c | 20 ++++ fs/nfsd/netlink.h | 2 + fs/nfsd/nfsctl.c | 143 ++++++++++++++++++++++++++ include/uapi/linux/nfsd_netlink.h | 12 +++ 5 files changed, 216 insertions(+) diff --git a/Documentation/netlink/specs/nfsd.yaml b/Documentation/netlink/specs/nfsd.yaml index 05acc73e2e338..703c2d11d0a9f 100644 --- a/Documentation/netlink/specs/nfsd.yaml +++ b/Documentation/netlink/specs/nfsd.yaml @@ -62,6 +62,22 @@ attribute-sets: name: compound-ops type: u32 multi-attr: true + - + name: server + attributes: + - + name: threads + type: u32 + multi-attr: true + - + name: gracetime + type: u32 + - + name: leasetime + type: u32 + - + name: scope + type: string operations: list: @@ -87,3 +103,26 @@ operations: - sport - dport - compound-ops + - + name: threads-set + doc: set the number of running threads + attribute-set: server + flags: [ admin-perm ] + do: + request: + attributes: + - threads + - gracetime + - leasetime + - scope + - + name: threads-get + doc: get the number of running threads + attribute-set: server + do: + reply: + attributes: + - threads + - gracetime + - leasetime + - scope diff --git a/fs/nfsd/netlink.c b/fs/nfsd/netlink.c index 0e1d635ec5f91..fe9eef5c7f273 100644 --- a/fs/nfsd/netlink.c +++ b/fs/nfsd/netlink.c @@ -10,6 +10,14 @@ #include +/* NFSD_CMD_THREADS_SET - do */ +static const struct nla_policy nfsd_threads_set_nl_policy[NFSD_A_SERVER_SCOPE + 1] = { + [NFSD_A_SERVER_THREADS] = { .type = NLA_U32, }, + [NFSD_A_SERVER_GRACETIME] = { .type = NLA_U32, }, + [NFSD_A_SERVER_LEASETIME] = { .type = NLA_U32, }, + [NFSD_A_SERVER_SCOPE] = { .type = NLA_NUL_STRING, }, +}; + /* Ops table for nfsd */ static const struct genl_split_ops nfsd_nl_ops[] = { { @@ -19,6 +27,18 @@ static const struct genl_split_ops nfsd_nl_ops[] = { .done = nfsd_nl_rpc_status_get_done, .flags = GENL_CMD_CAP_DUMP, }, + { + .cmd = NFSD_CMD_THREADS_SET, + .doit = nfsd_nl_threads_set_doit, + .policy = nfsd_threads_set_nl_policy, + .maxattr = NFSD_A_SERVER_SCOPE, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_THREADS_GET, + .doit = nfsd_nl_threads_get_doit, + .flags = GENL_CMD_CAP_DO, + }, }; struct genl_family nfsd_nl_family __ro_after_init = { diff --git a/fs/nfsd/netlink.h b/fs/nfsd/netlink.h index d83dd6bdee927..4137fac477e4f 100644 --- a/fs/nfsd/netlink.h +++ b/fs/nfsd/netlink.h @@ -16,6 +16,8 @@ int nfsd_nl_rpc_status_get_done(struct netlink_callback *cb); int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb); +int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_threads_get_doit(struct sk_buff *skb, struct genl_info *info); extern struct genl_family nfsd_nl_family; diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 34148d73b6c18..5bfdebb2e7e9d 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -1653,6 +1654,148 @@ int nfsd_nl_rpc_status_get_done(struct netlink_callback *cb) return 0; } +/** + * nfsd_nl_threads_set_doit - set the number of running threads + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info) +{ + int nthreads = 0, count = 0, nrpools, ret = -EOPNOTSUPP, rem; + struct net *net = genl_info_net(info); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + const struct nlattr *attr; + const char *scope = NULL; + + if (GENL_REQ_ATTR_CHECK(info, NFSD_A_SERVER_THREADS)) + return -EINVAL; + + /* count number of SERVER_THREADS values */ + nlmsg_for_each_attr(attr, info->nlhdr, GENL_HDRLEN, rem) { + if (nla_type(attr) == NFSD_A_SERVER_THREADS) + count++; + } + + mutex_lock(&nfsd_mutex); + + nrpools = nfsd_nrpools(net); + if (nrpools && count > nrpools) + count = nrpools; + + /* XXX: make this handle non-global pool-modes */ + if (count > 1) + goto out_unlock; + + nthreads = nla_get_u32(info->attrs[NFSD_A_SERVER_THREADS]); + if (info->attrs[NFSD_A_SERVER_GRACETIME] || + info->attrs[NFSD_A_SERVER_LEASETIME] || + info->attrs[NFSD_A_SERVER_SCOPE]) { + ret = -EBUSY; + if (nn->nfsd_serv && nn->nfsd_serv->sv_nrthreads) + goto out_unlock; + + ret = -EINVAL; + attr = info->attrs[NFSD_A_SERVER_GRACETIME]; + if (attr) { + u32 gracetime = nla_get_u32(attr); + + if (gracetime < 10 || gracetime > 3600) + goto out_unlock; + + nn->nfsd4_grace = gracetime; + } + + attr = info->attrs[NFSD_A_SERVER_LEASETIME]; + if (attr) { + u32 leasetime = nla_get_u32(attr); + + if (leasetime < 10 || leasetime > 3600) + goto out_unlock; + + nn->nfsd4_lease = leasetime; + } + + attr = info->attrs[NFSD_A_SERVER_SCOPE]; + if (attr) + scope = nla_data(attr); + } + + ret = nfsd_svc(nthreads, net, get_current_cred(), scope); + +out_unlock: + mutex_unlock(&nfsd_mutex); + + return ret == nthreads ? 0 : ret; +} + +/** + * nfsd_nl_threads_get_doit - get the number of running threads + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_threads_get_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct net *net = genl_info_net(info); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + void *hdr; + int err; + + skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + hdr = genlmsg_iput(skb, info); + if (!hdr) { + err = -EMSGSIZE; + goto err_free_msg; + } + + mutex_lock(&nfsd_mutex); + + err = nla_put_u32(skb, NFSD_A_SERVER_GRACETIME, + nn->nfsd4_grace) || + nla_put_u32(skb, NFSD_A_SERVER_LEASETIME, + nn->nfsd4_lease) || + nla_put_string(skb, NFSD_A_SERVER_SCOPE, + nn->nfsd_name); + if (err) + goto err_unlock; + + if (nn->nfsd_serv) { + int i; + + for (i = 0; i < nfsd_nrpools(net); ++i) { + struct svc_pool *sp = &nn->nfsd_serv->sv_pools[i]; + + err = nla_put_u32(skb, NFSD_A_SERVER_THREADS, + atomic_read(&sp->sp_nrthreads)); + if (err) + goto err_unlock; + } + } else { + err = nla_put_u32(skb, NFSD_A_SERVER_THREADS, 0); + if (err) + goto err_unlock; + } + + mutex_unlock(&nfsd_mutex); + + genlmsg_end(skb, hdr); + + return genlmsg_reply(skb, info); + +err_unlock: + mutex_unlock(&nfsd_mutex); +err_free_msg: + nlmsg_free(skb); + + return err; +} + /** * nfsd_net_init - Prepare the nfsd_net portion of a new net namespace * @net: a freshly-created network namespace diff --git a/include/uapi/linux/nfsd_netlink.h b/include/uapi/linux/nfsd_netlink.h index 3cd044edee5d8..4bbccd5db7cdc 100644 --- a/include/uapi/linux/nfsd_netlink.h +++ b/include/uapi/linux/nfsd_netlink.h @@ -29,8 +29,20 @@ enum { NFSD_A_RPC_STATUS_MAX = (__NFSD_A_RPC_STATUS_MAX - 1) }; +enum { + NFSD_A_SERVER_THREADS = 1, + NFSD_A_SERVER_GRACETIME, + NFSD_A_SERVER_LEASETIME, + NFSD_A_SERVER_SCOPE, + + __NFSD_A_SERVER_MAX, + NFSD_A_SERVER_MAX = (__NFSD_A_SERVER_MAX - 1) +}; + enum { NFSD_CMD_RPC_STATUS_GET = 1, + NFSD_CMD_THREADS_SET, + NFSD_CMD_THREADS_GET, __NFSD_CMD_MAX, NFSD_CMD_MAX = (__NFSD_CMD_MAX - 1) From 5a939bea25be9793d9aa5d8494df667dfe625e6b Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Tue, 23 Apr 2024 15:25:41 +0200 Subject: [PATCH 1263/1702] NFSD: add write_version to netlink command Introduce write_version netlink command through a "declarative" interface. This patch introduces a change in behavior since for version-set userspace is expected to provide a NFS major/minor version list it wants to enable while all the other ones will be disabled. (procfs write_version command implements imperative interface where the admin writes +3/-3 to enable/disable a single version. Reviewed-by: Jeff Layton Tested-by: Jeff Layton Signed-off-by: Lorenzo Bianconi Signed-off-by: Chuck Lever --- Documentation/netlink/specs/nfsd.yaml | 37 +++++++ fs/nfsd/netlink.c | 24 +++++ fs/nfsd/netlink.h | 5 + fs/nfsd/netns.h | 1 + fs/nfsd/nfsctl.c | 150 ++++++++++++++++++++++++++ fs/nfsd/nfssvc.c | 3 +- include/uapi/linux/nfsd_netlink.h | 18 ++++ 7 files changed, 236 insertions(+), 2 deletions(-) diff --git a/Documentation/netlink/specs/nfsd.yaml b/Documentation/netlink/specs/nfsd.yaml index 703c2d11d0a9f..77adda1425e23 100644 --- a/Documentation/netlink/specs/nfsd.yaml +++ b/Documentation/netlink/specs/nfsd.yaml @@ -78,6 +78,26 @@ attribute-sets: - name: scope type: string + - + name: version + attributes: + - + name: major + type: u32 + - + name: minor + type: u32 + - + name: enabled + type: flag + - + name: server-proto + attributes: + - + name: version + type: nest + nested-attributes: version + multi-attr: true operations: list: @@ -126,3 +146,20 @@ operations: - gracetime - leasetime - scope + - + name: version-set + doc: set nfs enabled versions + attribute-set: server-proto + flags: [ admin-perm ] + do: + request: + attributes: + - version + - + name: version-get + doc: get nfs enabled versions + attribute-set: server-proto + do: + reply: + attributes: + - version diff --git a/fs/nfsd/netlink.c b/fs/nfsd/netlink.c index fe9eef5c7f273..b23b0b84a59aa 100644 --- a/fs/nfsd/netlink.c +++ b/fs/nfsd/netlink.c @@ -10,6 +10,13 @@ #include +/* Common nested types */ +const struct nla_policy nfsd_version_nl_policy[NFSD_A_VERSION_ENABLED + 1] = { + [NFSD_A_VERSION_MAJOR] = { .type = NLA_U32, }, + [NFSD_A_VERSION_MINOR] = { .type = NLA_U32, }, + [NFSD_A_VERSION_ENABLED] = { .type = NLA_FLAG, }, +}; + /* NFSD_CMD_THREADS_SET - do */ static const struct nla_policy nfsd_threads_set_nl_policy[NFSD_A_SERVER_SCOPE + 1] = { [NFSD_A_SERVER_THREADS] = { .type = NLA_U32, }, @@ -18,6 +25,11 @@ static const struct nla_policy nfsd_threads_set_nl_policy[NFSD_A_SERVER_SCOPE + [NFSD_A_SERVER_SCOPE] = { .type = NLA_NUL_STRING, }, }; +/* NFSD_CMD_VERSION_SET - do */ +static const struct nla_policy nfsd_version_set_nl_policy[NFSD_A_SERVER_PROTO_VERSION + 1] = { + [NFSD_A_SERVER_PROTO_VERSION] = NLA_POLICY_NESTED(nfsd_version_nl_policy), +}; + /* Ops table for nfsd */ static const struct genl_split_ops nfsd_nl_ops[] = { { @@ -39,6 +51,18 @@ static const struct genl_split_ops nfsd_nl_ops[] = { .doit = nfsd_nl_threads_get_doit, .flags = GENL_CMD_CAP_DO, }, + { + .cmd = NFSD_CMD_VERSION_SET, + .doit = nfsd_nl_version_set_doit, + .policy = nfsd_version_set_nl_policy, + .maxattr = NFSD_A_SERVER_PROTO_VERSION, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_VERSION_GET, + .doit = nfsd_nl_version_get_doit, + .flags = GENL_CMD_CAP_DO, + }, }; struct genl_family nfsd_nl_family __ro_after_init = { diff --git a/fs/nfsd/netlink.h b/fs/nfsd/netlink.h index 4137fac477e4f..c7c0da2754818 100644 --- a/fs/nfsd/netlink.h +++ b/fs/nfsd/netlink.h @@ -11,6 +11,9 @@ #include +/* Common nested types */ +extern const struct nla_policy nfsd_version_nl_policy[NFSD_A_VERSION_ENABLED + 1]; + int nfsd_nl_rpc_status_get_start(struct netlink_callback *cb); int nfsd_nl_rpc_status_get_done(struct netlink_callback *cb); @@ -18,6 +21,8 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb); int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info); int nfsd_nl_threads_get_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_version_set_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_version_get_doit(struct sk_buff *skb, struct genl_info *info); extern struct genl_family nfsd_nl_family; diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index d4be519b5734e..14ec156563209 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -218,6 +218,7 @@ struct nfsd_net { /* Simple check to find out if a given net was properly initialized */ #define nfsd_netns_ready(nn) ((nn)->sessionid_hashtbl) +extern bool nfsd_support_version(int vers); extern void nfsd_netns_free_versions(struct nfsd_net *nn); extern unsigned int nfsd_net_id; diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 5bfdebb2e7e9d..b6b405437aa07 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -1796,6 +1796,156 @@ int nfsd_nl_threads_get_doit(struct sk_buff *skb, struct genl_info *info) return err; } +/** + * nfsd_nl_version_set_doit - set the nfs enabled versions + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_version_set_doit(struct sk_buff *skb, struct genl_info *info) +{ + const struct nlattr *attr; + struct nfsd_net *nn; + int i, rem; + + if (GENL_REQ_ATTR_CHECK(info, NFSD_A_SERVER_PROTO_VERSION)) + return -EINVAL; + + mutex_lock(&nfsd_mutex); + + nn = net_generic(genl_info_net(info), nfsd_net_id); + if (nn->nfsd_serv) { + mutex_unlock(&nfsd_mutex); + return -EBUSY; + } + + /* clear current supported versions. */ + nfsd_vers(nn, 2, NFSD_CLEAR); + nfsd_vers(nn, 3, NFSD_CLEAR); + for (i = 0; i <= NFSD_SUPPORTED_MINOR_VERSION; i++) + nfsd_minorversion(nn, i, NFSD_CLEAR); + + nlmsg_for_each_attr(attr, info->nlhdr, GENL_HDRLEN, rem) { + struct nlattr *tb[NFSD_A_VERSION_MAX + 1]; + u32 major, minor = 0; + bool enabled; + + if (nla_type(attr) != NFSD_A_SERVER_PROTO_VERSION) + continue; + + if (nla_parse_nested(tb, NFSD_A_VERSION_MAX, attr, + nfsd_version_nl_policy, info->extack) < 0) + continue; + + if (!tb[NFSD_A_VERSION_MAJOR]) + continue; + + major = nla_get_u32(tb[NFSD_A_VERSION_MAJOR]); + if (tb[NFSD_A_VERSION_MINOR]) + minor = nla_get_u32(tb[NFSD_A_VERSION_MINOR]); + + enabled = nla_get_flag(tb[NFSD_A_VERSION_ENABLED]); + + switch (major) { + case 4: + nfsd_minorversion(nn, minor, enabled ? NFSD_SET : NFSD_CLEAR); + break; + case 3: + case 2: + if (!minor) + nfsd_vers(nn, major, enabled ? NFSD_SET : NFSD_CLEAR); + break; + default: + break; + } + } + + mutex_unlock(&nfsd_mutex); + + return 0; +} + +/** + * nfsd_nl_version_get_doit - get the enabled status for all supported nfs versions + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_version_get_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct nfsd_net *nn; + int i, err; + void *hdr; + + skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + hdr = genlmsg_iput(skb, info); + if (!hdr) { + err = -EMSGSIZE; + goto err_free_msg; + } + + mutex_lock(&nfsd_mutex); + nn = net_generic(genl_info_net(info), nfsd_net_id); + + for (i = 2; i <= 4; i++) { + int j; + + for (j = 0; j <= NFSD_SUPPORTED_MINOR_VERSION; j++) { + struct nlattr *attr; + + /* Don't record any versions the kernel doesn't have + * compiled in + */ + if (!nfsd_support_version(i)) + continue; + + /* NFSv{2,3} does not support minor numbers */ + if (i < 4 && j) + continue; + + attr = nla_nest_start(skb, + NFSD_A_SERVER_PROTO_VERSION); + if (!attr) { + err = -EINVAL; + goto err_nfsd_unlock; + } + + if (nla_put_u32(skb, NFSD_A_VERSION_MAJOR, i) || + nla_put_u32(skb, NFSD_A_VERSION_MINOR, j)) { + err = -EINVAL; + goto err_nfsd_unlock; + } + + /* Set the enabled flag if the version is enabled */ + if (nfsd_vers(nn, i, NFSD_TEST) && + (i < 4 || nfsd_minorversion(nn, j, NFSD_TEST)) && + nla_put_flag(skb, NFSD_A_VERSION_ENABLED)) { + err = -EINVAL; + goto err_nfsd_unlock; + } + + nla_nest_end(skb, attr); + } + } + + mutex_unlock(&nfsd_mutex); + genlmsg_end(skb, hdr); + + return genlmsg_reply(skb, info); + +err_nfsd_unlock: + mutex_unlock(&nfsd_mutex); +err_free_msg: + nlmsg_free(skb); + + return err; +} + /** * nfsd_net_init - Prepare the nfsd_net portion of a new net namespace * @net: a freshly-created network namespace diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index e25b9b829749d..cd9a6a1a9fc8f 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -133,8 +133,7 @@ struct svc_program nfsd_program = { .pg_rpcbind_set = nfsd_rpcbind_set, }; -static bool -nfsd_support_version(int vers) +bool nfsd_support_version(int vers) { if (vers >= NFSD_MINVERS && vers < NFSD_NRVERS) return nfsd_version[vers] != NULL; diff --git a/include/uapi/linux/nfsd_netlink.h b/include/uapi/linux/nfsd_netlink.h index 4bbccd5db7cdc..7176cd8e6f095 100644 --- a/include/uapi/linux/nfsd_netlink.h +++ b/include/uapi/linux/nfsd_netlink.h @@ -39,10 +39,28 @@ enum { NFSD_A_SERVER_MAX = (__NFSD_A_SERVER_MAX - 1) }; +enum { + NFSD_A_VERSION_MAJOR = 1, + NFSD_A_VERSION_MINOR, + NFSD_A_VERSION_ENABLED, + + __NFSD_A_VERSION_MAX, + NFSD_A_VERSION_MAX = (__NFSD_A_VERSION_MAX - 1) +}; + +enum { + NFSD_A_SERVER_PROTO_VERSION = 1, + + __NFSD_A_SERVER_PROTO_MAX, + NFSD_A_SERVER_PROTO_MAX = (__NFSD_A_SERVER_PROTO_MAX - 1) +}; + enum { NFSD_CMD_RPC_STATUS_GET = 1, NFSD_CMD_THREADS_SET, NFSD_CMD_THREADS_GET, + NFSD_CMD_VERSION_SET, + NFSD_CMD_VERSION_GET, __NFSD_CMD_MAX, NFSD_CMD_MAX = (__NFSD_CMD_MAX - 1) From a79ec2aecb1b3d9c59241a6155a5554b236de9b7 Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Tue, 23 Apr 2024 15:25:42 +0200 Subject: [PATCH 1264/1702] SUNRPC: introduce svc_xprt_create_from_sa utility routine Add svc_xprt_create_from_sa utility routine and refactor svc_xprt_create() codebase in order to introduce the capability to create a svc port from socket address. Reviewed-by: Jeff Layton Tested-by: Jeff Layton Signed-off-by: Lorenzo Bianconi Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_xprt.h | 3 + net/sunrpc/svc_xprt.c | 133 ++++++++++++++++++-------------- 2 files changed, 78 insertions(+), 58 deletions(-) diff --git a/include/linux/sunrpc/svc_xprt.h b/include/linux/sunrpc/svc_xprt.h index 8e20cd60e2e71..0d9b10dbe07d1 100644 --- a/include/linux/sunrpc/svc_xprt.h +++ b/include/linux/sunrpc/svc_xprt.h @@ -135,6 +135,9 @@ int svc_reg_xprt_class(struct svc_xprt_class *); void svc_unreg_xprt_class(struct svc_xprt_class *); void svc_xprt_init(struct net *, struct svc_xprt_class *, struct svc_xprt *, struct svc_serv *); +int svc_xprt_create_from_sa(struct svc_serv *serv, const char *xprt_name, + struct net *net, struct sockaddr *sap, + int flags, const struct cred *cred); int svc_xprt_create(struct svc_serv *serv, const char *xprt_name, struct net *net, const int family, const unsigned short port, int flags, diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index b4a85a227bd7d..463fe544ae285 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -211,51 +211,6 @@ void svc_xprt_init(struct net *net, struct svc_xprt_class *xcl, } EXPORT_SYMBOL_GPL(svc_xprt_init); -static struct svc_xprt *__svc_xpo_create(struct svc_xprt_class *xcl, - struct svc_serv *serv, - struct net *net, - const int family, - const unsigned short port, - int flags) -{ - struct sockaddr_in sin = { - .sin_family = AF_INET, - .sin_addr.s_addr = htonl(INADDR_ANY), - .sin_port = htons(port), - }; -#if IS_ENABLED(CONFIG_IPV6) - struct sockaddr_in6 sin6 = { - .sin6_family = AF_INET6, - .sin6_addr = IN6ADDR_ANY_INIT, - .sin6_port = htons(port), - }; -#endif - struct svc_xprt *xprt; - struct sockaddr *sap; - size_t len; - - switch (family) { - case PF_INET: - sap = (struct sockaddr *)&sin; - len = sizeof(sin); - break; -#if IS_ENABLED(CONFIG_IPV6) - case PF_INET6: - sap = (struct sockaddr *)&sin6; - len = sizeof(sin6); - break; -#endif - default: - return ERR_PTR(-EAFNOSUPPORT); - } - - xprt = xcl->xcl_ops->xpo_create(serv, net, sap, len, flags); - if (IS_ERR(xprt)) - trace_svc_xprt_create_err(serv->sv_program->pg_name, - xcl->xcl_name, sap, len, xprt); - return xprt; -} - /** * svc_xprt_received - start next receiver thread * @xprt: controlling transport @@ -294,9 +249,8 @@ void svc_add_new_perm_xprt(struct svc_serv *serv, struct svc_xprt *new) } static int _svc_xprt_create(struct svc_serv *serv, const char *xprt_name, - struct net *net, const int family, - const unsigned short port, int flags, - const struct cred *cred) + struct net *net, struct sockaddr *sap, + size_t len, int flags, const struct cred *cred) { struct svc_xprt_class *xcl; @@ -312,8 +266,11 @@ static int _svc_xprt_create(struct svc_serv *serv, const char *xprt_name, goto err; spin_unlock(&svc_xprt_class_lock); - newxprt = __svc_xpo_create(xcl, serv, net, family, port, flags); + newxprt = xcl->xcl_ops->xpo_create(serv, net, sap, len, flags); if (IS_ERR(newxprt)) { + trace_svc_xprt_create_err(serv->sv_program->pg_name, + xcl->xcl_name, sap, len, + newxprt); module_put(xcl->xcl_owner); return PTR_ERR(newxprt); } @@ -329,6 +286,48 @@ static int _svc_xprt_create(struct svc_serv *serv, const char *xprt_name, return -EPROTONOSUPPORT; } +/** + * svc_xprt_create_from_sa - Add a new listener to @serv from socket address + * @serv: target RPC service + * @xprt_name: transport class name + * @net: network namespace + * @sap: socket address pointer + * @flags: SVC_SOCK flags + * @cred: credential to bind to this transport + * + * Return local xprt port on success or %-EPROTONOSUPPORT on failure + */ +int svc_xprt_create_from_sa(struct svc_serv *serv, const char *xprt_name, + struct net *net, struct sockaddr *sap, + int flags, const struct cred *cred) +{ + size_t len; + int err; + + switch (sap->sa_family) { + case AF_INET: + len = sizeof(struct sockaddr_in); + break; +#if IS_ENABLED(CONFIG_IPV6) + case AF_INET6: + len = sizeof(struct sockaddr_in6); + break; +#endif + default: + return -EAFNOSUPPORT; + } + + err = _svc_xprt_create(serv, xprt_name, net, sap, len, flags, cred); + if (err == -EPROTONOSUPPORT) { + request_module("svc%s", xprt_name); + err = _svc_xprt_create(serv, xprt_name, net, sap, len, flags, + cred); + } + + return err; +} +EXPORT_SYMBOL_GPL(svc_xprt_create_from_sa); + /** * svc_xprt_create - Add a new listener to @serv * @serv: target RPC service @@ -339,23 +338,41 @@ static int _svc_xprt_create(struct svc_serv *serv, const char *xprt_name, * @flags: SVC_SOCK flags * @cred: credential to bind to this transport * - * Return values: - * %0: New listener added successfully - * %-EPROTONOSUPPORT: Requested transport type not supported + * Return local xprt port on success or %-EPROTONOSUPPORT on failure */ int svc_xprt_create(struct svc_serv *serv, const char *xprt_name, struct net *net, const int family, const unsigned short port, int flags, const struct cred *cred) { - int err; + struct sockaddr_in sin = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_ANY), + .sin_port = htons(port), + }; +#if IS_ENABLED(CONFIG_IPV6) + struct sockaddr_in6 sin6 = { + .sin6_family = AF_INET6, + .sin6_addr = IN6ADDR_ANY_INIT, + .sin6_port = htons(port), + }; +#endif + struct sockaddr *sap; - err = _svc_xprt_create(serv, xprt_name, net, family, port, flags, cred); - if (err == -EPROTONOSUPPORT) { - request_module("svc%s", xprt_name); - err = _svc_xprt_create(serv, xprt_name, net, family, port, flags, cred); + switch (family) { + case PF_INET: + sap = (struct sockaddr *)&sin; + break; +#if IS_ENABLED(CONFIG_IPV6) + case PF_INET6: + sap = (struct sockaddr *)&sin6; + break; +#endif + default: + return -EAFNOSUPPORT; } - return err; + + return svc_xprt_create_from_sa(serv, xprt_name, net, sap, flags, cred); } EXPORT_SYMBOL_GPL(svc_xprt_create); From cf619507ae8d30b5735e9356d85a8df2bcf2b6ca Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 23 Apr 2024 15:25:43 +0200 Subject: [PATCH 1265/1702] SUNRPC: add a new svc_find_listener helper svc_find_listener will return the transport instance pointer for the endpoint accepting connections/peer traffic from the specified transport class and matching sockaddr. Signed-off-by: Jeff Layton Signed-off-by: Lorenzo Bianconi Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_xprt.h | 2 ++ net/sunrpc/svc_xprt.c | 34 +++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/include/linux/sunrpc/svc_xprt.h b/include/linux/sunrpc/svc_xprt.h index 0d9b10dbe07d1..0981e35a9feda 100644 --- a/include/linux/sunrpc/svc_xprt.h +++ b/include/linux/sunrpc/svc_xprt.h @@ -150,6 +150,8 @@ void svc_xprt_copy_addrs(struct svc_rqst *rqstp, struct svc_xprt *xprt); void svc_xprt_close(struct svc_xprt *xprt); int svc_port_is_privileged(struct sockaddr *sin); int svc_print_xprts(char *buf, int maxlen); +struct svc_xprt *svc_find_listener(struct svc_serv *serv, const char *xcl_name, + struct net *net, const struct sockaddr *sa); struct svc_xprt *svc_find_xprt(struct svc_serv *serv, const char *xcl_name, struct net *net, const sa_family_t af, const unsigned short port); diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index 463fe544ae285..34a3626c56b1b 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -1276,6 +1276,40 @@ static struct svc_deferred_req *svc_deferred_dequeue(struct svc_xprt *xprt) return dr; } +/** + * svc_find_listener - find an RPC transport instance + * @serv: pointer to svc_serv to search + * @xcl_name: C string containing transport's class name + * @net: owner net pointer + * @sa: sockaddr containing address + * + * Return the transport instance pointer for the endpoint accepting + * connections/peer traffic from the specified transport class, + * and matching sockaddr. + */ +struct svc_xprt *svc_find_listener(struct svc_serv *serv, const char *xcl_name, + struct net *net, const struct sockaddr *sa) +{ + struct svc_xprt *xprt; + struct svc_xprt *found = NULL; + + spin_lock_bh(&serv->sv_lock); + list_for_each_entry(xprt, &serv->sv_permsocks, xpt_list) { + if (xprt->xpt_net != net) + continue; + if (strcmp(xprt->xpt_class->xcl_name, xcl_name)) + continue; + if (!rpc_cmp_addr_port(sa, (struct sockaddr *)&xprt->xpt_local)) + continue; + found = xprt; + svc_xprt_get(xprt); + break; + } + spin_unlock_bh(&serv->sv_lock); + return found; +} +EXPORT_SYMBOL_GPL(svc_find_listener); + /** * svc_find_xprt - find an RPC transport instance * @serv: pointer to svc_serv to search From 16a471177496c8e04a9793812c187a2c1a2192fa Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Tue, 23 Apr 2024 15:25:44 +0200 Subject: [PATCH 1266/1702] NFSD: add listener-{set,get} netlink command Introduce write_ports netlink command. For listener-set, userspace is expected to provide a NFS listeners list it wants enabled. All other sockets will be closed. Reviewed-by: Jeff Layton Co-developed-by: Jeff Layton Signed-off-by: Jeff Layton Signed-off-by: Lorenzo Bianconi Signed-off-by: Chuck Lever --- Documentation/netlink/specs/nfsd.yaml | 34 ++++ fs/nfsd/netlink.c | 22 +++ fs/nfsd/netlink.h | 3 + fs/nfsd/nfsctl.c | 220 ++++++++++++++++++++++++++ include/uapi/linux/nfsd_netlink.h | 17 ++ 5 files changed, 296 insertions(+) diff --git a/Documentation/netlink/specs/nfsd.yaml b/Documentation/netlink/specs/nfsd.yaml index 77adda1425e23..d212340971673 100644 --- a/Documentation/netlink/specs/nfsd.yaml +++ b/Documentation/netlink/specs/nfsd.yaml @@ -98,6 +98,23 @@ attribute-sets: type: nest nested-attributes: version multi-attr: true + - + name: sock + attributes: + - + name: addr + type: binary + - + name: transport-name + type: string + - + name: server-sock + attributes: + - + name: addr + type: nest + nested-attributes: sock + multi-attr: true operations: list: @@ -163,3 +180,20 @@ operations: reply: attributes: - version + - + name: listener-set + doc: set nfs running sockets + attribute-set: server-sock + flags: [ admin-perm ] + do: + request: + attributes: + - addr + - + name: listener-get + doc: get nfs running listeners + attribute-set: server-sock + do: + reply: + attributes: + - addr diff --git a/fs/nfsd/netlink.c b/fs/nfsd/netlink.c index b23b0b84a59aa..62d2586d99025 100644 --- a/fs/nfsd/netlink.c +++ b/fs/nfsd/netlink.c @@ -11,6 +11,11 @@ #include /* Common nested types */ +const struct nla_policy nfsd_sock_nl_policy[NFSD_A_SOCK_TRANSPORT_NAME + 1] = { + [NFSD_A_SOCK_ADDR] = { .type = NLA_BINARY, }, + [NFSD_A_SOCK_TRANSPORT_NAME] = { .type = NLA_NUL_STRING, }, +}; + const struct nla_policy nfsd_version_nl_policy[NFSD_A_VERSION_ENABLED + 1] = { [NFSD_A_VERSION_MAJOR] = { .type = NLA_U32, }, [NFSD_A_VERSION_MINOR] = { .type = NLA_U32, }, @@ -30,6 +35,11 @@ static const struct nla_policy nfsd_version_set_nl_policy[NFSD_A_SERVER_PROTO_VE [NFSD_A_SERVER_PROTO_VERSION] = NLA_POLICY_NESTED(nfsd_version_nl_policy), }; +/* NFSD_CMD_LISTENER_SET - do */ +static const struct nla_policy nfsd_listener_set_nl_policy[NFSD_A_SERVER_SOCK_ADDR + 1] = { + [NFSD_A_SERVER_SOCK_ADDR] = NLA_POLICY_NESTED(nfsd_sock_nl_policy), +}; + /* Ops table for nfsd */ static const struct genl_split_ops nfsd_nl_ops[] = { { @@ -63,6 +73,18 @@ static const struct genl_split_ops nfsd_nl_ops[] = { .doit = nfsd_nl_version_get_doit, .flags = GENL_CMD_CAP_DO, }, + { + .cmd = NFSD_CMD_LISTENER_SET, + .doit = nfsd_nl_listener_set_doit, + .policy = nfsd_listener_set_nl_policy, + .maxattr = NFSD_A_SERVER_SOCK_ADDR, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_LISTENER_GET, + .doit = nfsd_nl_listener_get_doit, + .flags = GENL_CMD_CAP_DO, + }, }; struct genl_family nfsd_nl_family __ro_after_init = { diff --git a/fs/nfsd/netlink.h b/fs/nfsd/netlink.h index c7c0da2754818..e3724637d64d5 100644 --- a/fs/nfsd/netlink.h +++ b/fs/nfsd/netlink.h @@ -12,6 +12,7 @@ #include /* Common nested types */ +extern const struct nla_policy nfsd_sock_nl_policy[NFSD_A_SOCK_TRANSPORT_NAME + 1]; extern const struct nla_policy nfsd_version_nl_policy[NFSD_A_VERSION_ENABLED + 1]; int nfsd_nl_rpc_status_get_start(struct netlink_callback *cb); @@ -23,6 +24,8 @@ int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info); int nfsd_nl_threads_get_doit(struct sk_buff *skb, struct genl_info *info); int nfsd_nl_version_set_doit(struct sk_buff *skb, struct genl_info *info); int nfsd_nl_version_get_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_listener_get_doit(struct sk_buff *skb, struct genl_info *info); extern struct genl_family nfsd_nl_family; diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index b6b405437aa07..202140df8f82e 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -1946,6 +1946,226 @@ int nfsd_nl_version_get_doit(struct sk_buff *skb, struct genl_info *info) return err; } +/** + * nfsd_nl_listener_set_doit - set the nfs running sockets + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct net *net = genl_info_net(info); + struct svc_xprt *xprt, *tmp; + const struct nlattr *attr; + struct svc_serv *serv; + LIST_HEAD(permsocks); + struct nfsd_net *nn; + int err, rem; + + mutex_lock(&nfsd_mutex); + + err = nfsd_create_serv(net); + if (err) { + mutex_unlock(&nfsd_mutex); + return err; + } + + nn = net_generic(net, nfsd_net_id); + serv = nn->nfsd_serv; + + spin_lock_bh(&serv->sv_lock); + + /* Move all of the old listener sockets to a temp list */ + list_splice_init(&serv->sv_permsocks, &permsocks); + + /* + * Walk the list of server_socks from userland and move any that match + * back to sv_permsocks + */ + nlmsg_for_each_attr(attr, info->nlhdr, GENL_HDRLEN, rem) { + struct nlattr *tb[NFSD_A_SOCK_MAX + 1]; + const char *xcl_name; + struct sockaddr *sa; + + if (nla_type(attr) != NFSD_A_SERVER_SOCK_ADDR) + continue; + + if (nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr, + nfsd_sock_nl_policy, info->extack) < 0) + continue; + + if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME]) + continue; + + if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(*sa)) + continue; + + xcl_name = nla_data(tb[NFSD_A_SOCK_TRANSPORT_NAME]); + sa = nla_data(tb[NFSD_A_SOCK_ADDR]); + + /* Put back any matching sockets */ + list_for_each_entry_safe(xprt, tmp, &permsocks, xpt_list) { + /* This shouldn't be possible */ + if (WARN_ON_ONCE(xprt->xpt_net != net)) { + list_move(&xprt->xpt_list, &serv->sv_permsocks); + continue; + } + + /* If everything matches, put it back */ + if (!strcmp(xprt->xpt_class->xcl_name, xcl_name) && + rpc_cmp_addr_port(sa, (struct sockaddr *)&xprt->xpt_local)) { + list_move(&xprt->xpt_list, &serv->sv_permsocks); + break; + } + } + } + + /* For now, no removing old sockets while server is running */ + if (serv->sv_nrthreads && !list_empty(&permsocks)) { + list_splice_init(&permsocks, &serv->sv_permsocks); + spin_unlock_bh(&serv->sv_lock); + err = -EBUSY; + goto out_unlock_mtx; + } + + /* Close the remaining sockets on the permsocks list */ + while (!list_empty(&permsocks)) { + xprt = list_first_entry(&permsocks, struct svc_xprt, xpt_list); + list_move(&xprt->xpt_list, &serv->sv_permsocks); + + /* + * Newly-created sockets are born with the BUSY bit set. Clear + * it if there are no threads, since nothing can pick it up + * in that case. + */ + if (!serv->sv_nrthreads) + clear_bit(XPT_BUSY, &xprt->xpt_flags); + + set_bit(XPT_CLOSE, &xprt->xpt_flags); + spin_unlock_bh(&serv->sv_lock); + svc_xprt_close(xprt); + spin_lock_bh(&serv->sv_lock); + } + + spin_unlock_bh(&serv->sv_lock); + + /* walk list of addrs again, open any that still don't exist */ + nlmsg_for_each_attr(attr, info->nlhdr, GENL_HDRLEN, rem) { + struct nlattr *tb[NFSD_A_SOCK_MAX + 1]; + const char *xcl_name; + struct sockaddr *sa; + int ret; + + if (nla_type(attr) != NFSD_A_SERVER_SOCK_ADDR) + continue; + + if (nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr, + nfsd_sock_nl_policy, info->extack) < 0) + continue; + + if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME]) + continue; + + if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(*sa)) + continue; + + xcl_name = nla_data(tb[NFSD_A_SOCK_TRANSPORT_NAME]); + sa = nla_data(tb[NFSD_A_SOCK_ADDR]); + + xprt = svc_find_listener(serv, xcl_name, net, sa); + if (xprt) { + svc_xprt_put(xprt); + continue; + } + + ret = svc_xprt_create_from_sa(serv, xcl_name, net, sa, + SVC_SOCK_ANONYMOUS, + get_current_cred()); + /* always save the latest error */ + if (ret < 0) + err = ret; + } + + if (!serv->sv_nrthreads && list_empty(&nn->nfsd_serv->sv_permsocks)) + nfsd_destroy_serv(net); + +out_unlock_mtx: + mutex_unlock(&nfsd_mutex); + + return err; +} + +/** + * nfsd_nl_listener_get_doit - get the nfs running listeners + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_listener_get_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct svc_xprt *xprt; + struct svc_serv *serv; + struct nfsd_net *nn; + void *hdr; + int err; + + skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + hdr = genlmsg_iput(skb, info); + if (!hdr) { + err = -EMSGSIZE; + goto err_free_msg; + } + + mutex_lock(&nfsd_mutex); + nn = net_generic(genl_info_net(info), nfsd_net_id); + + /* no nfs server? Just send empty socket list */ + if (!nn->nfsd_serv) + goto out_unlock_mtx; + + serv = nn->nfsd_serv; + spin_lock_bh(&serv->sv_lock); + list_for_each_entry(xprt, &serv->sv_permsocks, xpt_list) { + struct nlattr *attr; + + attr = nla_nest_start(skb, NFSD_A_SERVER_SOCK_ADDR); + if (!attr) { + err = -EINVAL; + goto err_serv_unlock; + } + + if (nla_put_string(skb, NFSD_A_SOCK_TRANSPORT_NAME, + xprt->xpt_class->xcl_name) || + nla_put(skb, NFSD_A_SOCK_ADDR, + sizeof(struct sockaddr_storage), + &xprt->xpt_local)) { + err = -EINVAL; + goto err_serv_unlock; + } + + nla_nest_end(skb, attr); + } + spin_unlock_bh(&serv->sv_lock); +out_unlock_mtx: + mutex_unlock(&nfsd_mutex); + genlmsg_end(skb, hdr); + + return genlmsg_reply(skb, info); + +err_serv_unlock: + spin_unlock_bh(&serv->sv_lock); + mutex_unlock(&nfsd_mutex); +err_free_msg: + nlmsg_free(skb); + + return err; +} + /** * nfsd_net_init - Prepare the nfsd_net portion of a new net namespace * @net: a freshly-created network namespace diff --git a/include/uapi/linux/nfsd_netlink.h b/include/uapi/linux/nfsd_netlink.h index 7176cd8e6f095..24c86dbc7ed54 100644 --- a/include/uapi/linux/nfsd_netlink.h +++ b/include/uapi/linux/nfsd_netlink.h @@ -55,12 +55,29 @@ enum { NFSD_A_SERVER_PROTO_MAX = (__NFSD_A_SERVER_PROTO_MAX - 1) }; +enum { + NFSD_A_SOCK_ADDR = 1, + NFSD_A_SOCK_TRANSPORT_NAME, + + __NFSD_A_SOCK_MAX, + NFSD_A_SOCK_MAX = (__NFSD_A_SOCK_MAX - 1) +}; + +enum { + NFSD_A_SERVER_SOCK_ADDR = 1, + + __NFSD_A_SERVER_SOCK_MAX, + NFSD_A_SERVER_SOCK_MAX = (__NFSD_A_SERVER_SOCK_MAX - 1) +}; + enum { NFSD_CMD_RPC_STATUS_GET = 1, NFSD_CMD_THREADS_SET, NFSD_CMD_THREADS_GET, NFSD_CMD_VERSION_SET, NFSD_CMD_VERSION_GET, + NFSD_CMD_LISTENER_SET, + NFSD_CMD_LISTENER_GET, __NFSD_CMD_MAX, NFSD_CMD_MAX = (__NFSD_CMD_MAX - 1) From f2ad13ad08e14ed04395b5bfe2ced5cdbd69b174 Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Fri, 26 Apr 2024 11:47:50 +0800 Subject: [PATCH 1267/1702] SUNRPC: Remove comment for sp_lock It is obsolete since sp_lock was discarded in commit 580a25756a9f ("SUNRPC: discard sp_lock"). Signed-off-by: Guoqing Jiang Reviewed-by: NeilBrown Signed-off-by: Chuck Lever --- net/sunrpc/svc_xprt.c | 1 - 1 file changed, 1 deletion(-) diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index 34a3626c56b1b..dd86d7f1e97e9 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -46,7 +46,6 @@ static LIST_HEAD(svc_xprt_class_list); /* SMP locking strategy: * - * svc_pool->sp_lock protects most of the fields of that pool. * svc_serv->sv_lock protects sv_tempsocks, sv_permsocks, sv_tmpcnt. * when both need to be taken (rare), svc_serv->sv_lock is first. * The "service mutex" protects svc_serv->sv_nrthread. From a8483b9ad92c9d07122efe8697f0f42f6c41d1b1 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 30 Apr 2024 16:05:10 -0400 Subject: [PATCH 1268/1702] NFSD: Record status of async copy operation in struct nfsd4_copy After a client has started an asynchronous COPY operation, a subsequent OFFLOAD_STATUS operation will need to report the status code once that COPY operation has completed. The recorded status record will be used by a subsequent patch. Signed-off-by: Chuck Lever --- fs/nfsd/nfs4proc.c | 21 ++++++++++----------- fs/nfsd/xdr4.h | 1 + 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 3dc173b29803f..7e6580384cdb6 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -1737,7 +1737,7 @@ static void cleanup_async_copy(struct nfsd4_copy *copy) nfs4_put_copy(copy); } -static void nfsd4_send_cb_offload(struct nfsd4_copy *copy, __be32 nfserr) +static void nfsd4_send_cb_offload(struct nfsd4_copy *copy) { struct nfsd4_cb_offload *cbo; @@ -1747,12 +1747,12 @@ static void nfsd4_send_cb_offload(struct nfsd4_copy *copy, __be32 nfserr) memcpy(&cbo->co_res, ©->cp_res, sizeof(copy->cp_res)); memcpy(&cbo->co_fh, ©->fh, sizeof(copy->fh)); - cbo->co_nfserr = nfserr; + cbo->co_nfserr = copy->nfserr; nfsd4_init_cb(&cbo->co_cb, copy->cp_clp, &nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD); trace_nfsd_cb_offload(copy->cp_clp, &cbo->co_res.cb_stateid, - &cbo->co_fh, copy->cp_count, nfserr); + &cbo->co_fh, copy->cp_count, copy->nfserr); nfsd4_run_cb(&cbo->co_cb); } @@ -1766,7 +1766,6 @@ static void nfsd4_send_cb_offload(struct nfsd4_copy *copy, __be32 nfserr) static int nfsd4_do_async_copy(void *data) { struct nfsd4_copy *copy = (struct nfsd4_copy *)data; - __be32 nfserr; trace_nfsd_copy_do_async(copy); if (nfsd4_ssc_is_inter(copy)) { @@ -1777,24 +1776,24 @@ static int nfsd4_do_async_copy(void *data) if (IS_ERR(filp)) { switch (PTR_ERR(filp)) { case -EBADF: - nfserr = nfserr_wrong_type; + copy->nfserr = nfserr_wrong_type; break; default: - nfserr = nfserr_offload_denied; + copy->nfserr = nfserr_offload_denied; } /* ss_mnt will be unmounted by the laundromat */ goto do_callback; } - nfserr = nfsd4_do_copy(copy, filp, copy->nf_dst->nf_file, - false); + copy->nfserr = nfsd4_do_copy(copy, filp, copy->nf_dst->nf_file, + false); nfsd4_cleanup_inter_ssc(copy->ss_nsui, filp, copy->nf_dst); } else { - nfserr = nfsd4_do_copy(copy, copy->nf_src->nf_file, - copy->nf_dst->nf_file, false); + copy->nfserr = nfsd4_do_copy(copy, copy->nf_src->nf_file, + copy->nf_dst->nf_file, false); } do_callback: - nfsd4_send_cb_offload(copy, nfserr); + nfsd4_send_cb_offload(copy); cleanup_async_copy(copy); return 0; } diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h index 446e72b0385e7..62805931e8572 100644 --- a/fs/nfsd/xdr4.h +++ b/fs/nfsd/xdr4.h @@ -694,6 +694,7 @@ struct nfsd4_copy { #define NFSD4_COPY_F_COMMITTED (3) /* response */ + __be32 nfserr; struct nfsd42_write_res cp_res; struct knfsd_fh fh; From cc63c21682a51e24338baf30424b2987e05a556a Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 30 Apr 2024 16:05:11 -0400 Subject: [PATCH 1269/1702] NFSD: Add COPY status code to OFFLOAD_STATUS response Clients that send an OFFLOAD_STATUS might want to distinguish between an async COPY operation that is still running, has completed successfully, or that has failed. The intention of this patch is to make NFSD behave like this: * Copy still running: OFFLOAD_STATUS returns NFS4_OK, the number of bytes copied so far, and an empty osr_status array * Copy completed successfully: OFFLOAD_STATUS returns NFS4_OK, the number of bytes copied, and an osr_status of NFS4_OK * Copy failed: OFFLOAD_STATUS returns NFS4_OK, the number of bytes copied, and an osr_status other than NFS4_OK * Copy operation lost, canceled, or otherwise unrecognized: OFFLOAD_STATUS returns NFS4ERR_BAD_STATEID NB: Though RFC 7862 Section 11.2 lists a small set of NFS status codes that are valid for OFFLOAD_STATUS, there do not seem to be any explicit spec limits on the status codes that may be returned in the osr_status field. At this time we have no unit tests for COPY and its brethren, as pynfs does not yet implement support for NFSv4.2. Signed-off-by: Chuck Lever --- fs/nfsd/nfs4proc.c | 10 ++++++++-- fs/nfsd/nfs4xdr.c | 7 ++++++- fs/nfsd/xdr4.h | 4 +++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 7e6580384cdb6..ea3cc3e870a7f 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -1793,6 +1793,7 @@ static int nfsd4_do_async_copy(void *data) } do_callback: + set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags); nfsd4_send_cb_offload(copy); cleanup_async_copy(copy); return 0; @@ -2002,11 +2003,16 @@ nfsd4_offload_status(struct svc_rqst *rqstp, struct nfsd4_copy *copy; struct nfs4_client *clp = cstate->clp; + os->completed = false; spin_lock(&clp->async_lock); copy = find_async_copy_locked(clp, &os->stateid); - if (copy) + if (copy) { os->count = copy->cp_res.wr_bytes_written; - else + if (test_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags)) { + os->completed = true; + os->status = copy->nfserr; + } + } else status = nfserr_bad_stateid; spin_unlock(&clp->async_lock); diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index ea76100e50caa..c7bfd2180e3f2 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -5271,7 +5271,12 @@ nfsd4_encode_offload_status(struct nfsd4_compoundres *resp, __be32 nfserr, if (nfserr != nfs_ok) return nfserr; /* osr_complete<1> */ - if (xdr_stream_encode_u32(xdr, 0) != XDR_UNIT) + if (os->completed) { + if (xdr_stream_encode_u32(xdr, 1) != XDR_UNIT) + return nfserr_resource; + if (xdr_stream_encode_be32(xdr, os->status) != XDR_UNIT) + return nfserr_resource; + } else if (xdr_stream_encode_u32(xdr, 0) != XDR_UNIT) return nfserr_resource; return nfs_ok; } diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h index 62805931e8572..fbdd42cde1fa5 100644 --- a/fs/nfsd/xdr4.h +++ b/fs/nfsd/xdr4.h @@ -692,6 +692,7 @@ struct nfsd4_copy { #define NFSD4_COPY_F_INTRA (1) #define NFSD4_COPY_F_SYNCHRONOUS (2) #define NFSD4_COPY_F_COMMITTED (3) +#define NFSD4_COPY_F_COMPLETED (4) /* response */ __be32 nfserr; @@ -754,7 +755,8 @@ struct nfsd4_offload_status { /* response */ u64 count; - u32 status; + __be32 status; + bool completed; }; struct nfsd4_copy_notify { From 442d27ff09a218b61020ab56387dbc508ad6bfa6 Mon Sep 17 00:00:00 2001 From: Stephen Smalley Date: Fri, 3 May 2024 09:09:06 -0400 Subject: [PATCH 1270/1702] nfsd: set security label during create operations When security labeling is enabled, the client can pass a file security label as part of a create operation for the new file, similar to mode and other attributes. At present, the security label is received by nfsd and passed down to nfsd_create_setattr(), but nfsd_setattr() is never called and therefore the label is never set on the new file. This bug may have been introduced on or around commit d6a97d3f589a ("NFSD: add security label to struct nfsd_attrs"). Looking at nfsd_setattr() I am uncertain as to whether the same issue presents for file ACLs and therefore requires a similar fix for those. An alternative approach would be to introduce a new LSM hook to set the "create SID" of the current task prior to the actual file creation, which would atomically label the new inode at creation time. This would be better for SELinux and a similar approach has been used previously (see security_dentry_create_files_as) but perhaps not usable by other LSMs. Reproducer: 1. Install a Linux distro with SELinux - Fedora is easiest 2. git clone https://github.com/SELinuxProject/selinux-testsuite 3. Install the requisite dependencies per selinux-testsuite/README.md 4. Run something like the following script: MOUNT=$HOME/selinux-testsuite sudo systemctl start nfs-server sudo exportfs -o rw,no_root_squash,security_label localhost:$MOUNT sudo mkdir -p /mnt/selinux-testsuite sudo mount -t nfs -o vers=4.2 localhost:$MOUNT /mnt/selinux-testsuite pushd /mnt/selinux-testsuite/ sudo make -C policy load pushd tests/filesystem sudo runcon -t test_filesystem_t ./create_file -f trans_test_file \ -e test_filesystem_filetranscon_t -v sudo rm -f trans_test_file popd sudo make -C policy unload popd sudo umount /mnt/selinux-testsuite sudo exportfs -u localhost:$MOUNT sudo rmdir /mnt/selinux-testsuite sudo systemctl stop nfs-server Expected output: Process context: unconfined_u:unconfined_r:test_filesystem_t:s0-s0:c0.c1023 Created file: trans_test_file File context: unconfined_u:object_r:test_filesystem_filetranscon_t:s0 File context is correct Actual output: Process context: unconfined_u:unconfined_r:test_filesystem_t:s0-s0:c0.c1023 Created file: trans_test_file File context: system_u:object_r:test_file_t:s0 File context error, expected: test_filesystem_filetranscon_t got: test_file_t Signed-off-by: Stephen Smalley Reviewed-by: Jeff Layton Reviewed-by: NeilBrown Signed-off-by: Chuck Lever --- fs/nfsd/vfs.c | 2 +- fs/nfsd/vfs.h | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 2e41eb4c3cec7..29b1f3613800a 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -1422,7 +1422,7 @@ nfsd_create_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, * Callers expect new file metadata to be committed even * if the attributes have not changed. */ - if (iap->ia_valid) + if (nfsd_attrs_valid(attrs)) status = nfsd_setattr(rqstp, resfhp, attrs, NULL); else status = nfserrno(commit_metadata(resfhp)); diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h index c60fdb6200fde..57cd70062048f 100644 --- a/fs/nfsd/vfs.h +++ b/fs/nfsd/vfs.h @@ -60,6 +60,14 @@ static inline void nfsd_attrs_free(struct nfsd_attrs *attrs) posix_acl_release(attrs->na_dpacl); } +static inline bool nfsd_attrs_valid(struct nfsd_attrs *attrs) +{ + struct iattr *iap = attrs->na_iattr; + + return (iap->ia_valid || (attrs->na_seclabel && + attrs->na_seclabel->len)); +} + __be32 nfserrno (int errno); int nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp, struct svc_export **expp); From e221c45da3770962418fb30c27d941bbc70d595a Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 6 May 2024 12:30:04 -0400 Subject: [PATCH 1271/1702] knfsd: LOOKUP can return an illegal error value The 'NFS error' NFSERR_OPNOTSUPP is not described by any of the official NFS related RFCs, but appears to have snuck into some older .x files for NFSv2. Either way, it is not in RFC1094, RFC1813 or any of the NFSv4 RFCs, so should not be returned by the knfsd server, and particularly not by the "LOOKUP" operation. Instead, let's return NFSERR_STALE, which is more appropriate if the filesystem encodes the filehandle as FILEID_INVALID. Cc: stable@vger.kernel.org Signed-off-by: Trond Myklebust Signed-off-by: Chuck Lever --- fs/nfsd/nfsfh.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index 40fecf7b224f2..0b75305fb5f53 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -573,7 +573,7 @@ fh_compose(struct svc_fh *fhp, struct svc_export *exp, struct dentry *dentry, _fh_update(fhp, exp, dentry); if (fhp->fh_handle.fh_fileid_type == FILEID_INVALID) { fh_put(fhp); - return nfserr_opnotsupp; + return nfserr_stale; } return 0; @@ -599,7 +599,7 @@ fh_update(struct svc_fh *fhp) _fh_update(fhp, fhp->fh_export, dentry); if (fhp->fh_handle.fh_fileid_type == FILEID_INVALID) - return nfserr_opnotsupp; + return nfserr_stale; return 0; out_bad: printk(KERN_ERR "fh_update: fh not verified!\n"); From 939cb14d51a150e3c12ef7a8ce0ba04ce6131bd2 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 6 May 2024 12:30:05 -0400 Subject: [PATCH 1272/1702] NFS/knfsd: Remove the invalid NFS error 'NFSERR_OPNOTSUPP' NFSERR_OPNOTSUPP is not described by any RFC, and should not be used. Signed-off-by: Trond Myklebust Signed-off-by: Chuck Lever --- fs/nfsd/nfsd.h | 1 - include/trace/misc/nfs.h | 2 -- include/uapi/linux/nfs.h | 1 - 3 files changed, 4 deletions(-) diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 2f6c6f3815b4d..8f4f239d9f8a1 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -230,7 +230,6 @@ void nfsd_lockd_shutdown(void); #define nfserr_nospc cpu_to_be32(NFSERR_NOSPC) #define nfserr_rofs cpu_to_be32(NFSERR_ROFS) #define nfserr_mlink cpu_to_be32(NFSERR_MLINK) -#define nfserr_opnotsupp cpu_to_be32(NFSERR_OPNOTSUPP) #define nfserr_nametoolong cpu_to_be32(NFSERR_NAMETOOLONG) #define nfserr_notempty cpu_to_be32(NFSERR_NOTEMPTY) #define nfserr_dquot cpu_to_be32(NFSERR_DQUOT) diff --git a/include/trace/misc/nfs.h b/include/trace/misc/nfs.h index e43e745915618..7b221d51133a8 100644 --- a/include/trace/misc/nfs.h +++ b/include/trace/misc/nfs.h @@ -28,7 +28,6 @@ TRACE_DEFINE_ENUM(NFSERR_FBIG); TRACE_DEFINE_ENUM(NFSERR_NOSPC); TRACE_DEFINE_ENUM(NFSERR_ROFS); TRACE_DEFINE_ENUM(NFSERR_MLINK); -TRACE_DEFINE_ENUM(NFSERR_OPNOTSUPP); TRACE_DEFINE_ENUM(NFSERR_NAMETOOLONG); TRACE_DEFINE_ENUM(NFSERR_NOTEMPTY); TRACE_DEFINE_ENUM(NFSERR_DQUOT); @@ -64,7 +63,6 @@ TRACE_DEFINE_ENUM(NFSERR_JUKEBOX); { NFSERR_NOSPC, "NOSPC" }, \ { NFSERR_ROFS, "ROFS" }, \ { NFSERR_MLINK, "MLINK" }, \ - { NFSERR_OPNOTSUPP, "OPNOTSUPP" }, \ { NFSERR_NAMETOOLONG, "NAMETOOLONG" }, \ { NFSERR_NOTEMPTY, "NOTEMPTY" }, \ { NFSERR_DQUOT, "DQUOT" }, \ diff --git a/include/uapi/linux/nfs.h b/include/uapi/linux/nfs.h index 946cb62d64b0b..f356f2ba38142 100644 --- a/include/uapi/linux/nfs.h +++ b/include/uapi/linux/nfs.h @@ -61,7 +61,6 @@ NFSERR_NOSPC = 28, /* v2 v3 v4 */ NFSERR_ROFS = 30, /* v2 v3 v4 */ NFSERR_MLINK = 31, /* v3 v4 */ - NFSERR_OPNOTSUPP = 45, /* v2 v3 */ NFSERR_NAMETOOLONG = 63, /* v2 v3 v4 */ NFSERR_NOTEMPTY = 66, /* v2 v3 v4 */ NFSERR_DQUOT = 69, /* v2 v3 v4 */ From 7c18b0a5aa46cc7e5d3a7ef3f9f8e3aa91bb780f Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 7 May 2024 07:59:48 +0200 Subject: [PATCH 1273/1702] clk: samsung: gs101: drop unused HSI2 clock parent data Drop static const arrays with HSI2 clocks parent data which are not referenced by any clock. This might cause -Werror=unused-const-variable warnings. Reported-by: Stephen Boyd Closes: https://lore.kernel.org/all/8bf65df598680f0785c3d6db70acfb9a.sboyd@kernel.org/ Fixes: 093c290084a4 ("clk: samsung: gs101: add support for cmu_hsi2") Reviewed-by: Peter Griffin Link: https://lore.kernel.org/r/20240507055948.34554-1-krzysztof.kozlowski@linaro.org Signed-off-by: Krzysztof Kozlowski --- drivers/clk/samsung/clk-gs101.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/drivers/clk/samsung/clk-gs101.c b/drivers/clk/samsung/clk-gs101.c index e2a6a19925057..ba9570f7a5fab 100644 --- a/drivers/clk/samsung/clk-gs101.c +++ b/drivers/clk/samsung/clk-gs101.c @@ -2601,21 +2601,6 @@ static const unsigned long cmu_hsi2_clk_regs[] __initconst = { QUEUE_CTRL_REG_BLK_HSI2_CMU_HSI2, }; -PNAME(mout_hsi2_ufs_embd_p) = { "oscclk", "dout_cmu_shared0_div4", - "dout_cmu_shared2_div2", "fout_spare_pll" }; - -PNAME(mout_hsi2_pcie_p) = { "oscclk", "dout_cmu_shared2_div2" }; - -PNAME(mout_hsi2_bus_p) = { "dout_cmu_shared0_div4", - "dout_cmu_shared1_div4", - "dout_cmu_shared2_div2", - "dout_cmu_shared3_div2", - "fout_spare_pll", "oscclk", "oscclk", - "oscclk" }; - -PNAME(mout_hsi2_mmc_card_p) = { "fout_shared2_pll", "fout_shared3_pll", - "dout_cmu_shared0_div4", "fout_spare_pll" }; - PNAME(mout_hsi2_bus_user_p) = { "oscclk", "dout_cmu_hsi2_bus" }; PNAME(mout_hsi2_mmc_card_user_p) = { "oscclk", "dout_cmu_hsi2_mmc_card" }; PNAME(mout_hsi2_pcie_user_p) = { "oscclk", "dout_cmu_hsi2_pcie" }; From 327e2c97c46a4d971c5450a9d05b4a673f46c4da Mon Sep 17 00:00:00 2001 From: Michael Kelley Date: Sun, 7 Apr 2024 21:11:41 -0700 Subject: [PATCH 1274/1702] swiotlb: remove alloc_size argument to swiotlb_tbl_map_single() Currently swiotlb_tbl_map_single() takes alloc_align_mask and alloc_size arguments to specify an swiotlb allocation that is larger than mapping_size. This larger allocation is used solely by iommu_dma_map_single() to handle untrusted devices that should not have DMA visibility to memory pages that are partially used for unrelated kernel data. Having two arguments to specify the allocation is redundant. While alloc_align_mask naturally specifies the alignment of the starting address of the allocation, it can also implicitly specify the size by rounding up the mapping_size to that alignment. Additionally, the current approach has an edge case bug. iommu_dma_map_page() already does the rounding up to compute the alloc_size argument. But swiotlb_tbl_map_single() then calculates the alignment offset based on the DMA min_align_mask, and adds that offset to alloc_size. If the offset is non-zero, the addition may result in a value that is larger than the max the swiotlb can allocate. If the rounding up is done _after_ the alignment offset is added to the mapping_size (and the original mapping_size conforms to the value returned by swiotlb_max_mapping_size), then the max that the swiotlb can allocate will not be exceeded. In view of these issues, simplify the swiotlb_tbl_map_single() interface by removing the alloc_size argument. Most call sites pass the same value for mapping_size and alloc_size, and they pass alloc_align_mask as zero. Just remove the redundant argument from these callers, as they will see no functional change. For iommu_dma_map_page() also remove the alloc_size argument, and have swiotlb_tbl_map_single() compute the alloc_size by rounding up mapping_size after adding the offset based on min_align_mask. This has the side effect of fixing the edge case bug but with no other functional change. Also add a sanity test on the alloc_align_mask. While IOMMU code currently ensures the granule is not larger than PAGE_SIZE, if that guarantee were to be removed in the future, the downstream effect on the swiotlb might go unnoticed until strange allocation failures occurred. Tested on an ARM64 system with 16K page size and some kernel test-only hackery to allow modifying the DMA min_align_mask and the granule size that becomes the alloc_align_mask. Tested these combinations with a variety of original memory addresses and sizes, including those that reproduce the edge case bug: * 4K granule and 0 min_align_mask * 4K granule and 0xFFF min_align_mask (4K - 1) * 16K granule and 0xFFF min_align_mask * 64K granule and 0xFFF min_align_mask * 64K granule and 0x3FFF min_align_mask (16K - 1) With the changes, all combinations pass. Signed-off-by: Michael Kelley Reviewed-by: Petr Tesarik Signed-off-by: Christoph Hellwig --- drivers/iommu/dma-iommu.c | 2 +- drivers/xen/swiotlb-xen.c | 2 +- include/linux/swiotlb.h | 2 +- kernel/dma/swiotlb.c | 56 +++++++++++++++++++++++++++++---------- 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index e4cb26f6a9434..7a9e814fc6b33 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1165,7 +1165,7 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, trace_swiotlb_bounced(dev, phys, size); aligned_size = iova_align(iovad, size); - phys = swiotlb_tbl_map_single(dev, phys, size, aligned_size, + phys = swiotlb_tbl_map_single(dev, phys, size, iova_mask(iovad), dir, attrs); if (phys == DMA_MAPPING_ERROR) diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 0e6c6c25d154f..d4f1f8d1ebd88 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -216,7 +216,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, */ trace_swiotlb_bounced(dev, dev_addr, size); - map = swiotlb_tbl_map_single(dev, phys, size, size, 0, dir, attrs); + map = swiotlb_tbl_map_single(dev, phys, size, 0, dir, attrs); if (map == (phys_addr_t)DMA_MAPPING_ERROR) return DMA_MAPPING_ERROR; diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index ea23097e351fd..14bc10c1bb239 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -43,7 +43,7 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask, extern void __init swiotlb_update_mem_attributes(void); phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys, - size_t mapping_size, size_t alloc_size, + size_t mapping_size, unsigned int alloc_aligned_mask, enum dma_data_direction dir, unsigned long attrs); diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index a5e0dfc44d24e..046da973a7e2c 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -1340,15 +1340,40 @@ static unsigned long mem_used(struct io_tlb_mem *mem) #endif /* CONFIG_DEBUG_FS */ +/** + * swiotlb_tbl_map_single() - bounce buffer map a single contiguous physical area + * @dev: Device which maps the buffer. + * @orig_addr: Original (non-bounced) physical IO buffer address + * @mapping_size: Requested size of the actual bounce buffer, excluding + * any pre- or post-padding for alignment + * @alloc_align_mask: Required start and end alignment of the allocated buffer + * @dir: DMA direction + * @attrs: Optional DMA attributes for the map operation + * + * Find and allocate a suitable sequence of IO TLB slots for the request. + * The allocated space starts at an alignment specified by alloc_align_mask, + * and the size of the allocated space is rounded up so that the total amount + * of allocated space is a multiple of (alloc_align_mask + 1). If + * alloc_align_mask is zero, the allocated space may be at any alignment and + * the size is not rounded up. + * + * The returned address is within the allocated space and matches the bits + * of orig_addr that are specified in the DMA min_align_mask for the device. As + * such, this returned address may be offset from the beginning of the allocated + * space. The bounce buffer space starting at the returned address for + * mapping_size bytes is initialized to the contents of the original IO buffer + * area. Any pre-padding (due to an offset) and any post-padding (due to + * rounding-up the size) is not initialized. + */ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr, - size_t mapping_size, size_t alloc_size, - unsigned int alloc_align_mask, enum dma_data_direction dir, - unsigned long attrs) + size_t mapping_size, unsigned int alloc_align_mask, + enum dma_data_direction dir, unsigned long attrs) { struct io_tlb_mem *mem = dev->dma_io_tlb_mem; unsigned int offset; struct io_tlb_pool *pool; unsigned int i; + size_t size; int index; phys_addr_t tlb_addr; unsigned short pad_slots; @@ -1362,20 +1387,24 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr, if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n"); - if (mapping_size > alloc_size) { - dev_warn_once(dev, "Invalid sizes (mapping: %zd bytes, alloc: %zd bytes)", - mapping_size, alloc_size); - return (phys_addr_t)DMA_MAPPING_ERROR; - } + /* + * The default swiotlb memory pool is allocated with PAGE_SIZE + * alignment. If a mapping is requested with larger alignment, + * the mapping may be unable to use the initial slot(s) in all + * sets of IO_TLB_SEGSIZE slots. In such case, a mapping request + * of or near the maximum mapping size would always fail. + */ + dev_WARN_ONCE(dev, alloc_align_mask > ~PAGE_MASK, + "Alloc alignment may prevent fulfilling requests with max mapping_size\n"); offset = swiotlb_align_offset(dev, alloc_align_mask, orig_addr); - index = swiotlb_find_slots(dev, orig_addr, - alloc_size + offset, alloc_align_mask, &pool); + size = ALIGN(mapping_size + offset, alloc_align_mask + 1); + index = swiotlb_find_slots(dev, orig_addr, size, alloc_align_mask, &pool); if (index == -1) { if (!(attrs & DMA_ATTR_NO_WARN)) dev_warn_ratelimited(dev, "swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n", - alloc_size, mem->nslabs, mem_used(mem)); + size, mem->nslabs, mem_used(mem)); return (phys_addr_t)DMA_MAPPING_ERROR; } @@ -1388,7 +1417,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr, offset &= (IO_TLB_SIZE - 1); index += pad_slots; pool->slots[index].pad_slots = pad_slots; - for (i = 0; i < nr_slots(alloc_size + offset); i++) + for (i = 0; i < (nr_slots(size) - pad_slots); i++) pool->slots[index + i].orig_addr = slot_addr(orig_addr, i); tlb_addr = slot_addr(pool->start, index) + offset; /* @@ -1543,8 +1572,7 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size, trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size); - swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, size, 0, dir, - attrs); + swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, attrs); if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR) return DMA_MAPPING_ERROR; From 2650073f1b5858008c32712f3d9e1e808ce7e967 Mon Sep 17 00:00:00 2001 From: Michael Kelley Date: Sun, 7 Apr 2024 21:11:42 -0700 Subject: [PATCH 1275/1702] iommu/dma: fix zeroing of bounce buffer padding used by untrusted devices iommu_dma_map_page() allocates swiotlb memory as a bounce buffer when an untrusted device wants to map only part of the memory in an granule. The goal is to disallow the untrusted device having DMA access to unrelated kernel data that may be sharing the granule. To meet this goal, the bounce buffer itself is zeroed, and any additional swiotlb memory up to alloc_size after the bounce buffer end (i.e., "post-padding") is also zeroed. However, as of commit 901c7280ca0d ("Reinstate some of "swiotlb: rework "fix info leak with DMA_FROM_DEVICE"""), swiotlb_tbl_map_single() always initializes the contents of the bounce buffer to the original memory. Zeroing the bounce buffer is redundant and probably wrong per the discussion in that commit. Only the post-padding needs to be zeroed. Also, when the DMA min_align_mask is non-zero, the allocated bounce buffer space may not start on a granule boundary. The swiotlb memory from the granule boundary to the start of the allocated bounce buffer might belong to some unrelated bounce buffer. So as described in the "second issue" in [1], it can't be zeroed to protect against untrusted devices. But as of commit af133562d5af ("swiotlb: extend buffer pre-padding to alloc_align_mask if necessary"), swiotlb_tbl_map_single() allocates pre-padding slots when necessary to meet min_align_mask requirements, making it possible to zero the pre-padding area as well. Finally, iommu_dma_map_page() uses the swiotlb for untrusted devices and also for certain kmalloc() memory. Current code does the zeroing for both cases, but it is needed only for the untrusted device case. Fix all of this by updating iommu_dma_map_page() to zero both the pre-padding and post-padding areas, but not the actual bounce buffer. Do this only in the case where the bounce buffer is used because of an untrusted device. [1] https://lore.kernel.org/all/20210929023300.335969-1-stevensd@google.com/ Signed-off-by: Michael Kelley Reviewed-by: Petr Tesarik Signed-off-by: Christoph Hellwig --- drivers/iommu/dma-iommu.c | 29 ++++++++++++++++------------- include/linux/iova.h | 5 +++++ 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 7a9e814fc6b33..c745196bc1504 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1154,9 +1154,6 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, */ if (dev_use_swiotlb(dev, size, dir) && iova_offset(iovad, phys | size)) { - void *padding_start; - size_t padding_size, aligned_size; - if (!is_swiotlb_active(dev)) { dev_warn_once(dev, "DMA bounce buffers are inactive, unable to map unaligned transaction.\n"); return DMA_MAPPING_ERROR; @@ -1164,24 +1161,30 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, trace_swiotlb_bounced(dev, phys, size); - aligned_size = iova_align(iovad, size); phys = swiotlb_tbl_map_single(dev, phys, size, iova_mask(iovad), dir, attrs); if (phys == DMA_MAPPING_ERROR) return DMA_MAPPING_ERROR; - /* Cleanup the padding area. */ - padding_start = phys_to_virt(phys); - padding_size = aligned_size; + /* + * Untrusted devices should not see padding areas with random + * leftover kernel data, so zero the pre- and post-padding. + * swiotlb_tbl_map_single() has initialized the bounce buffer + * proper to the contents of the original memory buffer. + */ + if (dev_is_untrusted(dev)) { + size_t start, virt = (size_t)phys_to_virt(phys); - if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && - (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) { - padding_start += size; - padding_size -= size; - } + /* Pre-padding */ + start = iova_align_down(iovad, virt); + memset((void *)start, 0, virt - start); - memset(padding_start, 0, padding_size); + /* Post-padding */ + start = virt + size; + memset((void *)start, 0, + iova_align(iovad, start) - start); + } } if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) diff --git a/include/linux/iova.h b/include/linux/iova.h index 83c00fac2acb1..d2c4fd923efab 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -65,6 +65,11 @@ static inline size_t iova_align(struct iova_domain *iovad, size_t size) return ALIGN(size, iovad->granule); } +static inline size_t iova_align_down(struct iova_domain *iovad, size_t size) +{ + return ALIGN_DOWN(size, iovad->granule); +} + static inline dma_addr_t iova_dma_addr(struct iova_domain *iovad, struct iova *iova) { return (dma_addr_t)iova->pfn_lo << iova_shift(iovad); From fe7514b149e0a8a6f3031d286e52d40163b0b11a Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 7 May 2024 13:20:20 +0200 Subject: [PATCH 1276/1702] dma: compile-out DMA sync op calls when not used Some platforms do have DMA, but DMA there is always direct and coherent. Currently, even on such platforms DMA sync operations are compiled and called. Add a new hidden Kconfig symbol, DMA_NEED_SYNC, and set it only when either sync operations are needed or there is DMA ops or swiotlb or DMA debug is enabled. Compile global dma_sync_*() and dma_need_sync() only when it's set, otherwise provide empty inline stubs. The change allows for future optimizations of DMA sync calls depending on runtime conditions. Signed-off-by: Alexander Lobakin Signed-off-by: Christoph Hellwig --- include/linux/dma-mapping.h | 62 ++++++++++++++++++++----------------- kernel/dma/Kconfig | 5 +++ kernel/dma/mapping.c | 22 +++++++------ 3 files changed, 50 insertions(+), 39 deletions(-) diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 4a658de44ee92..a569b56b25e2c 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -117,14 +117,6 @@ dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, size_t size, enum dma_data_direction dir, unsigned long attrs); void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir, unsigned long attrs); -void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, - enum dma_data_direction dir); -void dma_sync_single_for_device(struct device *dev, dma_addr_t addr, - size_t size, enum dma_data_direction dir); -void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, - int nelems, enum dma_data_direction dir); -void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, - int nelems, enum dma_data_direction dir); void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs); void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, @@ -147,7 +139,6 @@ u64 dma_get_required_mask(struct device *dev); bool dma_addressing_limited(struct device *dev); size_t dma_max_mapping_size(struct device *dev); size_t dma_opt_mapping_size(struct device *dev); -bool dma_need_sync(struct device *dev, dma_addr_t dma_addr); unsigned long dma_get_merge_boundary(struct device *dev); struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size, enum dma_data_direction dir, gfp_t gfp, unsigned long attrs); @@ -195,22 +186,6 @@ static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { } -static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, - size_t size, enum dma_data_direction dir) -{ -} -static inline void dma_sync_single_for_device(struct device *dev, - dma_addr_t addr, size_t size, enum dma_data_direction dir) -{ -} -static inline void dma_sync_sg_for_cpu(struct device *dev, - struct scatterlist *sg, int nelems, enum dma_data_direction dir) -{ -} -static inline void dma_sync_sg_for_device(struct device *dev, - struct scatterlist *sg, int nelems, enum dma_data_direction dir) -{ -} static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) { return -ENOMEM; @@ -277,10 +252,6 @@ static inline size_t dma_opt_mapping_size(struct device *dev) { return 0; } -static inline bool dma_need_sync(struct device *dev, dma_addr_t dma_addr) -{ - return false; -} static inline unsigned long dma_get_merge_boundary(struct device *dev) { return 0; @@ -310,6 +281,39 @@ static inline int dma_mmap_noncontiguous(struct device *dev, } #endif /* CONFIG_HAS_DMA */ +#if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC) +void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, + enum dma_data_direction dir); +void dma_sync_single_for_device(struct device *dev, dma_addr_t addr, + size_t size, enum dma_data_direction dir); +void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, + int nelems, enum dma_data_direction dir); +void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, + int nelems, enum dma_data_direction dir); +bool dma_need_sync(struct device *dev, dma_addr_t dma_addr); +#else /* !CONFIG_HAS_DMA || !CONFIG_DMA_NEED_SYNC */ +static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, + size_t size, enum dma_data_direction dir) +{ +} +static inline void dma_sync_single_for_device(struct device *dev, + dma_addr_t addr, size_t size, enum dma_data_direction dir) +{ +} +static inline void dma_sync_sg_for_cpu(struct device *dev, + struct scatterlist *sg, int nelems, enum dma_data_direction dir) +{ +} +static inline void dma_sync_sg_for_device(struct device *dev, + struct scatterlist *sg, int nelems, enum dma_data_direction dir) +{ +} +static inline bool dma_need_sync(struct device *dev, dma_addr_t dma_addr) +{ + return false; +} +#endif /* !CONFIG_HAS_DMA || !CONFIG_DMA_NEED_SYNC */ + struct page *dma_alloc_pages(struct device *dev, size_t size, dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp); void dma_free_pages(struct device *dev, size_t size, struct page *page, diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig index d62f5957f36be..c06e56be0ca1e 100644 --- a/kernel/dma/Kconfig +++ b/kernel/dma/Kconfig @@ -107,6 +107,11 @@ config DMA_BOUNCE_UNALIGNED_KMALLOC bool depends on SWIOTLB +config DMA_NEED_SYNC + def_bool ARCH_HAS_SYNC_DMA_FOR_DEVICE || ARCH_HAS_SYNC_DMA_FOR_CPU || \ + ARCH_HAS_SYNC_DMA_FOR_CPU_ALL || DMA_API_DEBUG || DMA_OPS || \ + SWIOTLB + config DMA_RESTRICTED_POOL bool "DMA Restricted Pool" depends on OF && OF_RESERVED_MEM && SWIOTLB diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 58db8fd70471a..c78b78e95a263 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -329,6 +329,7 @@ void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size, } EXPORT_SYMBOL(dma_unmap_resource); +#ifdef CONFIG_DMA_NEED_SYNC void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) { @@ -385,6 +386,17 @@ void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, } EXPORT_SYMBOL(dma_sync_sg_for_device); +bool dma_need_sync(struct device *dev, dma_addr_t dma_addr) +{ + const struct dma_map_ops *ops = get_dma_ops(dev); + + if (dma_map_direct(dev, ops)) + return dma_direct_need_sync(dev, dma_addr); + return ops->sync_single_for_cpu || ops->sync_single_for_device; +} +EXPORT_SYMBOL_GPL(dma_need_sync); +#endif /* CONFIG_DMA_NEED_SYNC */ + /* * The whole dma_get_sgtable() idea is fundamentally unsafe - it seems * that the intention is to allow exporting memory allocated via the @@ -841,16 +853,6 @@ size_t dma_opt_mapping_size(struct device *dev) } EXPORT_SYMBOL_GPL(dma_opt_mapping_size); -bool dma_need_sync(struct device *dev, dma_addr_t dma_addr) -{ - const struct dma_map_ops *ops = get_dma_ops(dev); - - if (dma_map_direct(dev, ops)) - return dma_direct_need_sync(dev, dma_addr); - return ops->sync_single_for_cpu || ops->sync_single_for_device; -} -EXPORT_SYMBOL_GPL(dma_need_sync); - unsigned long dma_get_merge_boundary(struct device *dev) { const struct dma_map_ops *ops = get_dma_ops(dev); From f406c8e4b770ca3b0df84a17349e13f2b6b07d10 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 7 May 2024 13:20:21 +0200 Subject: [PATCH 1277/1702] dma: avoid redundant calls for sync operations Quite often, devices do not need dma_sync operations on x86_64 at least. Indeed, when dev_is_dma_coherent(dev) is true and dev_use_swiotlb(dev) is false, iommu_dma_sync_single_for_cpu() and friends do nothing. However, indirectly calling them when CONFIG_RETPOLINE=y consumes about 10% of cycles on a cpu receiving packets from softirq at ~100Gbit rate. Even if/when CONFIG_RETPOLINE is not set, there is a cost of about 3%. Add dev->need_dma_sync boolean and turn it off during the device initialization (dma_set_mask()) depending on the setup: dev_is_dma_coherent() for the direct DMA, !(sync_single_for_device || sync_single_for_cpu) or the new dma_map_ops flag, %DMA_F_CAN_SKIP_SYNC, advertised for non-NULL DMA ops. Then later, if/when swiotlb is used for the first time, the flag is reset back to on, from swiotlb_tbl_map_single(). On iavf, the UDP trafficgen with XDP_DROP in skb mode test shows +3-5% increase for direct DMA. Suggested-by: Christoph Hellwig # direct DMA shortcut Co-developed-by: Eric Dumazet Signed-off-by: Eric Dumazet Signed-off-by: Alexander Lobakin Signed-off-by: Christoph Hellwig --- include/linux/device.h | 4 +++ include/linux/dma-map-ops.h | 12 ++++++++ include/linux/dma-mapping.h | 53 +++++++++++++++++++++++++++++++---- kernel/dma/mapping.c | 55 +++++++++++++++++++++++++++++-------- kernel/dma/swiotlb.c | 6 ++++ 5 files changed, 113 insertions(+), 17 deletions(-) diff --git a/include/linux/device.h b/include/linux/device.h index b9f5464f44ed8..ed95b829f05bb 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -691,6 +691,7 @@ struct device_physical_location { * and optionall (if the coherent mask is large enough) also * for dma allocations. This flag is managed by the dma ops * instance from ->dma_supported. + * @dma_need_sync: The device needs performing DMA sync operations. * * At the lowest level, every device in a Linux system is represented by an * instance of struct device. The device structure contains the information @@ -803,6 +804,9 @@ struct device { #ifdef CONFIG_DMA_OPS_BYPASS bool dma_ops_bypass : 1; #endif +#ifdef CONFIG_DMA_NEED_SYNC + bool dma_need_sync:1; +#endif }; /** diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 4abc60f042092..4893cb89cb524 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -18,8 +18,11 @@ struct iommu_ops; * * DMA_F_PCI_P2PDMA_SUPPORTED: Indicates the dma_map_ops implementation can * handle PCI P2PDMA pages in the map_sg/unmap_sg operation. + * DMA_F_CAN_SKIP_SYNC: DMA sync operations can be skipped if the device is + * coherent and it's not an SWIOTLB buffer. */ #define DMA_F_PCI_P2PDMA_SUPPORTED (1 << 0) +#define DMA_F_CAN_SKIP_SYNC (1 << 1) struct dma_map_ops { unsigned int flags; @@ -273,6 +276,15 @@ static inline bool dev_is_dma_coherent(struct device *dev) } #endif /* CONFIG_ARCH_HAS_DMA_COHERENCE_H */ +static inline void dma_reset_need_sync(struct device *dev) +{ +#ifdef CONFIG_DMA_NEED_SYNC + /* Reset it only once so that the function can be called on hotpath */ + if (unlikely(!dev->dma_need_sync)) + dev->dma_need_sync = true; +#endif +} + /* * Check whether potential kmalloc() buffers are safe for non-coherent DMA. */ diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index a569b56b25e2c..eb4e15893b6c4 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -282,16 +282,59 @@ static inline int dma_mmap_noncontiguous(struct device *dev, #endif /* CONFIG_HAS_DMA */ #if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC) -void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, +void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir); -void dma_sync_single_for_device(struct device *dev, dma_addr_t addr, +void __dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir); -void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, +void __dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction dir); -void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, +void __dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction dir); -bool dma_need_sync(struct device *dev, dma_addr_t dma_addr); +bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr); + +static inline bool dma_dev_need_sync(const struct device *dev) +{ + /* Always call DMA sync operations when debugging is enabled */ + return dev->dma_need_sync || IS_ENABLED(CONFIG_DMA_API_DEBUG); +} + +static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, + size_t size, enum dma_data_direction dir) +{ + if (dma_dev_need_sync(dev)) + __dma_sync_single_for_cpu(dev, addr, size, dir); +} + +static inline void dma_sync_single_for_device(struct device *dev, + dma_addr_t addr, size_t size, enum dma_data_direction dir) +{ + if (dma_dev_need_sync(dev)) + __dma_sync_single_for_device(dev, addr, size, dir); +} + +static inline void dma_sync_sg_for_cpu(struct device *dev, + struct scatterlist *sg, int nelems, enum dma_data_direction dir) +{ + if (dma_dev_need_sync(dev)) + __dma_sync_sg_for_cpu(dev, sg, nelems, dir); +} + +static inline void dma_sync_sg_for_device(struct device *dev, + struct scatterlist *sg, int nelems, enum dma_data_direction dir) +{ + if (dma_dev_need_sync(dev)) + __dma_sync_sg_for_device(dev, sg, nelems, dir); +} + +static inline bool dma_need_sync(struct device *dev, dma_addr_t dma_addr) +{ + return dma_dev_need_sync(dev) ? __dma_need_sync(dev, dma_addr) : false; +} #else /* !CONFIG_HAS_DMA || !CONFIG_DMA_NEED_SYNC */ +static inline bool dma_dev_need_sync(const struct device *dev) +{ + return false; +} static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) { diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index c78b78e95a263..3524bc92c37fd 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -330,7 +330,7 @@ void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size, EXPORT_SYMBOL(dma_unmap_resource); #ifdef CONFIG_DMA_NEED_SYNC -void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, +void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) { const struct dma_map_ops *ops = get_dma_ops(dev); @@ -342,9 +342,9 @@ void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, ops->sync_single_for_cpu(dev, addr, size, dir); debug_dma_sync_single_for_cpu(dev, addr, size, dir); } -EXPORT_SYMBOL(dma_sync_single_for_cpu); +EXPORT_SYMBOL(__dma_sync_single_for_cpu); -void dma_sync_single_for_device(struct device *dev, dma_addr_t addr, +void __dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) { const struct dma_map_ops *ops = get_dma_ops(dev); @@ -356,9 +356,9 @@ void dma_sync_single_for_device(struct device *dev, dma_addr_t addr, ops->sync_single_for_device(dev, addr, size, dir); debug_dma_sync_single_for_device(dev, addr, size, dir); } -EXPORT_SYMBOL(dma_sync_single_for_device); +EXPORT_SYMBOL(__dma_sync_single_for_device); -void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, +void __dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction dir) { const struct dma_map_ops *ops = get_dma_ops(dev); @@ -370,9 +370,9 @@ void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, ops->sync_sg_for_cpu(dev, sg, nelems, dir); debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir); } -EXPORT_SYMBOL(dma_sync_sg_for_cpu); +EXPORT_SYMBOL(__dma_sync_sg_for_cpu); -void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, +void __dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction dir) { const struct dma_map_ops *ops = get_dma_ops(dev); @@ -384,18 +384,47 @@ void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, ops->sync_sg_for_device(dev, sg, nelems, dir); debug_dma_sync_sg_for_device(dev, sg, nelems, dir); } -EXPORT_SYMBOL(dma_sync_sg_for_device); +EXPORT_SYMBOL(__dma_sync_sg_for_device); -bool dma_need_sync(struct device *dev, dma_addr_t dma_addr) +bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr) { const struct dma_map_ops *ops = get_dma_ops(dev); if (dma_map_direct(dev, ops)) + /* + * dma_need_sync could've been reset on first SWIOTLB buffer + * mapping, but @dma_addr is not necessary an SWIOTLB buffer. + * In this case, fall back to more granular check. + */ return dma_direct_need_sync(dev, dma_addr); - return ops->sync_single_for_cpu || ops->sync_single_for_device; + return true; } -EXPORT_SYMBOL_GPL(dma_need_sync); -#endif /* CONFIG_DMA_NEED_SYNC */ +EXPORT_SYMBOL_GPL(__dma_need_sync); + +static void dma_setup_need_sync(struct device *dev) +{ + const struct dma_map_ops *ops = get_dma_ops(dev); + + if (dma_map_direct(dev, ops) || (ops->flags & DMA_F_CAN_SKIP_SYNC)) + /* + * dma_need_sync will be reset to %true on first SWIOTLB buffer + * mapping, if any. During the device initialization, it's + * enough to check only for the DMA coherence. + */ + dev->dma_need_sync = !dev_is_dma_coherent(dev); + else if (!ops->sync_single_for_device && !ops->sync_single_for_cpu && + !ops->sync_sg_for_device && !ops->sync_sg_for_cpu) + /* + * Synchronization is not possible when none of DMA sync ops + * is set. + */ + dev->dma_need_sync = false; + else + dev->dma_need_sync = true; +} +#else /* !CONFIG_DMA_NEED_SYNC */ +static inline void dma_setup_need_sync(struct device *dev) { } +#endif /* !CONFIG_DMA_NEED_SYNC */ /* * The whole dma_get_sgtable() idea is fundamentally unsafe - it seems @@ -785,6 +814,8 @@ int dma_set_mask(struct device *dev, u64 mask) arch_dma_set_mask(dev, mask); *dev->dma_mask = mask; + dma_setup_need_sync(dev); + return 0; } EXPORT_SYMBOL(dma_set_mask); diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 046da973a7e2c..ae3e593eaadb9 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -1408,6 +1408,12 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr, return (phys_addr_t)DMA_MAPPING_ERROR; } + /* + * If dma_need_sync wasn't set, reset it on first SWIOTLB buffer + * mapping to always sync SWIOTLB buffers. + */ + dma_reset_need_sync(dev); + /* * Save away the mapping from the original address to the DMA address. * This is needed when we sync the memory. Then we sync the buffer if From ea01fa703150025806a21c960761c821736f4757 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 7 May 2024 13:20:22 +0200 Subject: [PATCH 1278/1702] iommu/dma: avoid expensive indirect calls for sync operations When IOMMU is on, the actual synchronization happens in the same cases as with the direct DMA. Advertise %DMA_F_CAN_SKIP_SYNC in IOMMU DMA to skip sync ops calls (indirect) for non-SWIOTLB buffers. perf profile before the patch: 18.53% [kernel] [k] gq_rx_skb 14.77% [kernel] [k] napi_reuse_skb 8.95% [kernel] [k] skb_release_data 5.42% [kernel] [k] dev_gro_receive 5.37% [kernel] [k] memcpy <*> 5.26% [kernel] [k] iommu_dma_sync_sg_for_cpu 4.78% [kernel] [k] tcp_gro_receive <*> 4.42% [kernel] [k] iommu_dma_sync_sg_for_device 4.12% [kernel] [k] ipv6_gro_receive 3.65% [kernel] [k] gq_pool_get 3.25% [kernel] [k] skb_gro_receive 2.07% [kernel] [k] napi_gro_frags 1.98% [kernel] [k] tcp6_gro_receive 1.27% [kernel] [k] gq_rx_prep_buffers 1.18% [kernel] [k] gq_rx_napi_handler 0.99% [kernel] [k] csum_partial 0.74% [kernel] [k] csum_ipv6_magic 0.72% [kernel] [k] free_pcp_prepare 0.60% [kernel] [k] __napi_poll 0.58% [kernel] [k] net_rx_action 0.56% [kernel] [k] read_tsc <*> 0.50% [kernel] [k] __x86_indirect_thunk_r11 0.45% [kernel] [k] memset After patch, lines with <*> no longer show up, and overall cpu usage looks much better (~60% instead of ~72%): 25.56% [kernel] [k] gq_rx_skb 9.90% [kernel] [k] napi_reuse_skb 7.39% [kernel] [k] dev_gro_receive 6.78% [kernel] [k] memcpy 6.53% [kernel] [k] skb_release_data 6.39% [kernel] [k] tcp_gro_receive 5.71% [kernel] [k] ipv6_gro_receive 4.35% [kernel] [k] napi_gro_frags 4.34% [kernel] [k] skb_gro_receive 3.50% [kernel] [k] gq_pool_get 3.08% [kernel] [k] gq_rx_napi_handler 2.35% [kernel] [k] tcp6_gro_receive 2.06% [kernel] [k] gq_rx_prep_buffers 1.32% [kernel] [k] csum_partial 0.93% [kernel] [k] csum_ipv6_magic 0.65% [kernel] [k] net_rx_action iavf yields +10% of Mpps on Rx. This also unblocks batched allocations of XSk buffers when IOMMU is active. Co-developed-by: Eric Dumazet Signed-off-by: Eric Dumazet Acked-by: Robin Murphy Signed-off-by: Alexander Lobakin Signed-off-by: Christoph Hellwig --- drivers/iommu/dma-iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index c745196bc1504..2a1dbe5867c45 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1723,7 +1723,8 @@ static size_t iommu_dma_max_mapping_size(struct device *dev) } static const struct dma_map_ops iommu_dma_ops = { - .flags = DMA_F_PCI_P2PDMA_SUPPORTED, + .flags = DMA_F_PCI_P2PDMA_SUPPORTED | + DMA_F_CAN_SKIP_SYNC, .alloc = iommu_dma_alloc, .free = iommu_dma_free, .alloc_pages = dma_common_alloc_pages, From 1f20a5769446a1acae67ac9e63d07a594829a789 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 7 May 2024 13:20:23 +0200 Subject: [PATCH 1279/1702] page_pool: make sure frag API fields don't span between cachelines After commit 5027ec19f104 ("net: page_pool: split the page_pool_params into fast and slow") that made &page_pool contain only "hot" params at the start, cacheline boundary chops frag API fields group in the middle again. To not bother with this each time fast params get expanded or shrunk, let's just align them to `4 * sizeof(long)`, the closest upper pow-2 to their actual size (2 longs + 1 int). This ensures 16-byte alignment for the 32-bit architectures and 32-byte alignment for the 64-bit ones, excluding unnecessary false-sharing. ::page_state_hold_cnt is used quite intensively on hotpath no matter if frag API is used, so move it to the newly created hole in the first cacheline. Signed-off-by: Alexander Lobakin Signed-off-by: Christoph Hellwig --- include/net/page_pool/types.h | 12 +++++++++++- net/core/page_pool.c | 10 ++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h index 5e43a08d3231c..5460cbab5de07 100644 --- a/include/net/page_pool/types.h +++ b/include/net/page_pool/types.h @@ -130,12 +130,22 @@ struct page_pool { struct page_pool_params_fast p; int cpuid; + u32 pages_state_hold_cnt; bool has_init_callback; + /* The following block must stay within one cacheline. On 32-bit + * systems, sizeof(long) == sizeof(int), so that the block size is + * ``3 * sizeof(long)``. On 64-bit systems, the actual size is + * ``2 * sizeof(long) + sizeof(int)``. The closest pow-2 to both of + * them is ``4 * sizeof(long)``, so just use that one for simplicity. + * Having it aligned to a cacheline boundary may be excessive and + * doesn't bring any good. + */ + __cacheline_group_begin(frag) __aligned(4 * sizeof(long)); long frag_users; struct page *frag_page; unsigned int frag_offset; - u32 pages_state_hold_cnt; + __cacheline_group_end(frag); struct delayed_work release_dw; void (*disconnect)(void *pool); diff --git a/net/core/page_pool.c b/net/core/page_pool.c index dd364d738c006..95eac12e8790e 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -172,12 +172,22 @@ static void page_pool_producer_unlock(struct page_pool *pool, spin_unlock_bh(&pool->ring.producer_lock); } +static void page_pool_struct_check(void) +{ + CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_users); + CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_page); + CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_offset); + CACHELINE_ASSERT_GROUP_SIZE(struct page_pool, frag, 4 * sizeof(long)); +} + static int page_pool_init(struct page_pool *pool, const struct page_pool_params *params, int cpuid) { unsigned int ring_qsize = 1024; /* Default */ + page_pool_struct_check(); + memcpy(&pool->p, ¶ms->fast, sizeof(pool->p)); memcpy(&pool->slow, ¶ms->slow, sizeof(pool->slow)); From 403f11ac9ab72fc3bee0b8c80c16e33212ea8cd9 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 7 May 2024 13:20:24 +0200 Subject: [PATCH 1280/1702] page_pool: don't use driver-set flags field directly page_pool::p is driver-defined params, copied directly from the structure passed to page_pool_create(). The structure isn't meant to be modified by the Page Pool core code and this even might look confusing[0][1]. In order to be able to alter some flags, let's define our own, internal fields the same way as the already existing one (::has_init_callback). They are defined as bits in the driver-set params, leave them so here as well, to not waste byte-per-bit or so. Almost 30 bits are still free for future extensions. We could've defined only new flags here or only the ones we may need to alter, but checking some flags in one place while others in another doesn't sound convenient or intuitive. ::flags passed by the driver can now go to the "slow" PP params. Suggested-by: Jakub Kicinski Link[0]: https://lore.kernel.org/netdev/20230703133207.4f0c54ce@kernel.org Suggested-by: Alexander Duyck Link[1]: https://lore.kernel.org/netdev/CAKgT0UfZCGnWgOH96E4GV3ZP6LLbROHM7SHE8NKwq+exX+Gk_Q@mail.gmail.com Signed-off-by: Alexander Lobakin Signed-off-by: Christoph Hellwig --- include/net/page_pool/types.h | 13 ++++++++--- net/core/page_pool.c | 41 +++++++++++++++++++---------------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h index 5460cbab5de07..8c6d9f16bf659 100644 --- a/include/net/page_pool/types.h +++ b/include/net/page_pool/types.h @@ -45,7 +45,6 @@ struct pp_alloc_cache { /** * struct page_pool_params - page pool parameters - * @flags: PP_FLAG_DMA_MAP, PP_FLAG_DMA_SYNC_DEV * @order: 2^order pages on allocation * @pool_size: size of the ptr_ring * @nid: NUMA node id to allocate from pages from @@ -55,10 +54,11 @@ struct pp_alloc_cache { * @dma_dir: DMA mapping direction * @max_len: max DMA sync memory size for PP_FLAG_DMA_SYNC_DEV * @offset: DMA sync address offset for PP_FLAG_DMA_SYNC_DEV + * @netdev: corresponding &net_device for Netlink introspection + * @flags: PP_FLAG_DMA_MAP, PP_FLAG_DMA_SYNC_DEV, PP_FLAG_SYSTEM_POOL */ struct page_pool_params { struct_group_tagged(page_pool_params_fast, fast, - unsigned int flags; unsigned int order; unsigned int pool_size; int nid; @@ -70,6 +70,7 @@ struct page_pool_params { ); struct_group_tagged(page_pool_params_slow, slow, struct net_device *netdev; + unsigned int flags; /* private: used by test code only */ void (*init_callback)(struct page *page, void *arg); void *init_arg; @@ -131,7 +132,13 @@ struct page_pool { int cpuid; u32 pages_state_hold_cnt; - bool has_init_callback; + + bool has_init_callback:1; /* slow::init_callback is set */ + bool dma_map:1; /* Perform DMA mapping */ + bool dma_sync:1; /* Perform DMA sync */ +#ifdef CONFIG_PAGE_POOL_STATS + bool system:1; /* This is a global percpu pool */ +#endif /* The following block must stay within one cacheline. On 32-bit * systems, sizeof(long) == sizeof(int), so that the block size is diff --git a/net/core/page_pool.c b/net/core/page_pool.c index 95eac12e8790e..c2819ff03dd25 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -194,7 +194,7 @@ static int page_pool_init(struct page_pool *pool, pool->cpuid = cpuid; /* Validate only known flags were used */ - if (pool->p.flags & ~(PP_FLAG_ALL)) + if (pool->slow.flags & ~PP_FLAG_ALL) return -EINVAL; if (pool->p.pool_size) @@ -208,22 +208,26 @@ static int page_pool_init(struct page_pool *pool, * DMA_BIDIRECTIONAL is for allowing page used for DMA sending, * which is the XDP_TX use-case. */ - if (pool->p.flags & PP_FLAG_DMA_MAP) { + if (pool->slow.flags & PP_FLAG_DMA_MAP) { if ((pool->p.dma_dir != DMA_FROM_DEVICE) && (pool->p.dma_dir != DMA_BIDIRECTIONAL)) return -EINVAL; + + pool->dma_map = true; } - if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) { + if (pool->slow.flags & PP_FLAG_DMA_SYNC_DEV) { /* In order to request DMA-sync-for-device the page * needs to be mapped */ - if (!(pool->p.flags & PP_FLAG_DMA_MAP)) + if (!(pool->slow.flags & PP_FLAG_DMA_MAP)) return -EINVAL; if (!pool->p.max_len) return -EINVAL; + pool->dma_sync = true; + /* pool->p.offset has to be set according to the address * offset used by the DMA engine to start copying rx data */ @@ -232,7 +236,7 @@ static int page_pool_init(struct page_pool *pool, pool->has_init_callback = !!pool->slow.init_callback; #ifdef CONFIG_PAGE_POOL_STATS - if (!(pool->p.flags & PP_FLAG_SYSTEM_POOL)) { + if (!(pool->slow.flags & PP_FLAG_SYSTEM_POOL)) { pool->recycle_stats = alloc_percpu(struct page_pool_recycle_stats); if (!pool->recycle_stats) return -ENOMEM; @@ -242,12 +246,13 @@ static int page_pool_init(struct page_pool *pool, * (also percpu) page pool instance. */ pool->recycle_stats = &pp_system_recycle_stats; + pool->system = true; } #endif if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0) { #ifdef CONFIG_PAGE_POOL_STATS - if (!(pool->p.flags & PP_FLAG_SYSTEM_POOL)) + if (!pool->system) free_percpu(pool->recycle_stats); #endif return -ENOMEM; @@ -258,7 +263,7 @@ static int page_pool_init(struct page_pool *pool, /* Driver calling page_pool_create() also call page_pool_destroy() */ refcount_set(&pool->user_cnt, 1); - if (pool->p.flags & PP_FLAG_DMA_MAP) + if (pool->dma_map) get_device(pool->p.dev); return 0; @@ -268,11 +273,11 @@ static void page_pool_uninit(struct page_pool *pool) { ptr_ring_cleanup(&pool->ring, NULL); - if (pool->p.flags & PP_FLAG_DMA_MAP) + if (pool->dma_map) put_device(pool->p.dev); #ifdef CONFIG_PAGE_POOL_STATS - if (!(pool->p.flags & PP_FLAG_SYSTEM_POOL)) + if (!pool->system) free_percpu(pool->recycle_stats); #endif } @@ -424,7 +429,7 @@ static bool page_pool_dma_map(struct page_pool *pool, struct page *page) if (page_pool_set_dma_addr(page, dma)) goto unmap_failed; - if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) + if (pool->dma_sync) page_pool_dma_sync_for_device(pool, page, pool->p.max_len); return true; @@ -470,8 +475,7 @@ static struct page *__page_pool_alloc_page_order(struct page_pool *pool, if (unlikely(!page)) return NULL; - if ((pool->p.flags & PP_FLAG_DMA_MAP) && - unlikely(!page_pool_dma_map(pool, page))) { + if (pool->dma_map && unlikely(!page_pool_dma_map(pool, page))) { put_page(page); return NULL; } @@ -491,8 +495,8 @@ static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool, gfp_t gfp) { const int bulk = PP_ALLOC_CACHE_REFILL; - unsigned int pp_flags = pool->p.flags; unsigned int pp_order = pool->p.order; + bool dma_map = pool->dma_map; struct page *page; int i, nr_pages; @@ -517,8 +521,7 @@ static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool, */ for (i = 0; i < nr_pages; i++) { page = pool->alloc.cache[i]; - if ((pp_flags & PP_FLAG_DMA_MAP) && - unlikely(!page_pool_dma_map(pool, page))) { + if (dma_map && unlikely(!page_pool_dma_map(pool, page))) { put_page(page); continue; } @@ -590,7 +593,7 @@ void __page_pool_release_page_dma(struct page_pool *pool, struct page *page) { dma_addr_t dma; - if (!(pool->p.flags & PP_FLAG_DMA_MAP)) + if (!pool->dma_map) /* Always account for inflight pages, even if we didn't * map them */ @@ -673,7 +676,7 @@ static bool __page_pool_page_can_be_recycled(const struct page *page) } /* If the page refcnt == 1, this will try to recycle the page. - * if PP_FLAG_DMA_SYNC_DEV is set, we'll try to sync the DMA area for + * If pool->dma_sync is set, we'll try to sync the DMA area for * the configured size min(dma_sync_size, pool->max_len). * If the page refcnt != 1, then the page will be returned to memory * subsystem. @@ -696,7 +699,7 @@ __page_pool_put_page(struct page_pool *pool, struct page *page, if (likely(__page_pool_page_can_be_recycled(page))) { /* Read barrier done in page_ref_count / READ_ONCE */ - if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) + if (pool->dma_sync) page_pool_dma_sync_for_device(pool, page, dma_sync_size); @@ -809,7 +812,7 @@ static struct page *page_pool_drain_frag(struct page_pool *pool, return NULL; if (__page_pool_page_can_be_recycled(page)) { - if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) + if (pool->dma_sync) page_pool_dma_sync_for_device(pool, page, -1); return page; From 9dcb47a616d552306a01c2032b81c0c920b06847 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 27 Apr 2024 23:55:00 +0900 Subject: [PATCH 1281/1702] kbuild: do not add $(srctree) or $(objtree) to header search paths scripts/Makefile.lib is included not only from scripts/Makefile.build but also from scripts/Makefile.{vmlinux,modfinal} for building generated C files. In scripts/Makefile.{vmlinux,modfinal}, $(obj) and $(src) are empty. Therefore, the header include paths: -I $(srctree)/$(src) -I $(objtree)/$(obj) ... become meaningless code: -I $(srctree)/ -I $(objtree)/ Add these paths only when 'obj' and 'src' are defined. Reported-by: kernel test robot Link: https://lore.kernel.org/oe-kbuild-all/202404170634.BlqTaYA0-lkp@intel.com/ Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Makefile.lib | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index d1d51e38b55d7..e67f066c0cea7 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -213,9 +213,9 @@ endif # $(objtree)/$(obj) for including generated headers from checkin source files ifeq ($(KBUILD_EXTMOD),) ifdef building_out_of_srctree -_c_flags += -I $(srctree)/$(src) -I $(objtree)/$(obj) -_a_flags += -I $(srctree)/$(src) -I $(objtree)/$(obj) -_cpp_flags += -I $(srctree)/$(src) -I $(objtree)/$(obj) +_c_flags += $(addprefix -I $(srctree)/,$(src)) $(addprefix -I $(objtree)/,$(obj)) +_a_flags += $(addprefix -I $(srctree)/,$(src)) $(addprefix -I $(objtree)/,$(obj)) +_cpp_flags += $(addprefix -I $(srctree)/,$(src)) $(addprefix -I $(objtree)/,$(obj)) endif endif From 4bf6a4ebc59201bcd12c932f25edda4c3e36e5df Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 2 May 2024 10:52:58 +0200 Subject: [PATCH 1282/1702] selftests: mm: cow: flag vmsplice() hugetlb tests as XFAIL Patch series "selftests: mm: cow: flag vmsplice() hugetlb tests as XFAIL". The failing hugetlb vmsplice() COW tests keep confusing people, and having tests that have been failing for years and likely will keep failing for years to come because nobody cares enough is rather suboptimal. Let's mark them as XFAIL and document why fixing them is not that easy as it would appear at first sight. More details can be found in [1], especially around how hugetlb pages cannot really be overcommitted, and why we don't particularly care about these vmsplice() leaks for hugetlb -- in contrast to ordinary memory. [1] https://lore.kernel.org/all/8b42a24d-caf0-46ef-9e15-0f88d47d2f21@redhat.com/ This patch (of 2): The vmsplice() hugetlb tests have been failing right from the start, and we documented that in the introducing commit 7dad331be781 ("selftests/vm: anon_cow: hugetlb tests"): Note that some tests cases still fail. This will, for example, be fixed once vmsplice properly uses FOLL_PIN instead of FOLL_GET for pinning. With 2 MiB and 1 GiB hugetlb on x86_64, the expected failures are: Until vmsplice() is changed, these tests will likely keep failing: hugetlb COW reuse logic is harder to change, because using the same COW reuse logic as we use for !hugetlb could harm other (sane) users when running out of free hugetlb pages. More details can be found in [1], especially around how hugetlb pages cannot really be overcommitted, and why we don't particularly care about these vmsplice() leaks for hugetlb -- in contrast to ordinary memory. These (expected) failures keep confusing people, so flag them accordingly. Before: $ ./cow [...] Bail out! 8 out of 778 tests failed # Totals: pass:769 fail:8 xfail:0 xpass:0 skip:1 error:0 $ echo $? 1 After: $ ./cow [...] # Totals: pass:769 fail:0 xfail:8 xpass:0 skip:1 error:0 $ echo $? 0 [1] https://lore.kernel.org/all/8b42a24d-caf0-46ef-9e15-0f88d47d2f21@redhat.com/ Link: https://lkml.kernel.org/r/20240502085259.103784-1-david@redhat.com Link: https://lkml.kernel.org/r/20240502085259.103784-2-david@redhat.com Signed-off-by: David Hildenbrand Cc: Muchun Song Cc: Peter Xu Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/cow.c | 106 +++++++++++++++++++++---------- 1 file changed, 71 insertions(+), 35 deletions(-) diff --git a/tools/testing/selftests/mm/cow.c b/tools/testing/selftests/mm/cow.c index 363bf5f801be5..0549672acbd63 100644 --- a/tools/testing/selftests/mm/cow.c +++ b/tools/testing/selftests/mm/cow.c @@ -199,7 +199,7 @@ static int child_vmsplice_memcmp_fn(char *mem, size_t size, typedef int (*child_fn)(char *mem, size_t size, struct comm_pipes *comm_pipes); static void do_test_cow_in_parent(char *mem, size_t size, bool do_mprotect, - child_fn fn) + child_fn fn, bool xfail) { struct comm_pipes comm_pipes; char buf; @@ -247,33 +247,47 @@ static void do_test_cow_in_parent(char *mem, size_t size, bool do_mprotect, else ret = -EINVAL; - ksft_test_result(!ret, "No leak from parent into child\n"); + if (!ret) { + ksft_test_result_pass("No leak from parent into child\n"); + } else if (xfail) { + /* + * With hugetlb, some vmsplice() tests are currently expected to + * fail because (a) harder to fix and (b) nobody really cares. + * Flag them as expected failure for now. + */ + ksft_test_result_xfail("Leak from parent into child\n"); + } else { + ksft_test_result_fail("Leak from parent into child\n"); + } close_comm_pipes: close_comm_pipes(&comm_pipes); } -static void test_cow_in_parent(char *mem, size_t size) +static void test_cow_in_parent(char *mem, size_t size, bool is_hugetlb) { - do_test_cow_in_parent(mem, size, false, child_memcmp_fn); + do_test_cow_in_parent(mem, size, false, child_memcmp_fn, false); } -static void test_cow_in_parent_mprotect(char *mem, size_t size) +static void test_cow_in_parent_mprotect(char *mem, size_t size, bool is_hugetlb) { - do_test_cow_in_parent(mem, size, true, child_memcmp_fn); + do_test_cow_in_parent(mem, size, true, child_memcmp_fn, false); } -static void test_vmsplice_in_child(char *mem, size_t size) +static void test_vmsplice_in_child(char *mem, size_t size, bool is_hugetlb) { - do_test_cow_in_parent(mem, size, false, child_vmsplice_memcmp_fn); + do_test_cow_in_parent(mem, size, false, child_vmsplice_memcmp_fn, + is_hugetlb); } -static void test_vmsplice_in_child_mprotect(char *mem, size_t size) +static void test_vmsplice_in_child_mprotect(char *mem, size_t size, + bool is_hugetlb) { - do_test_cow_in_parent(mem, size, true, child_vmsplice_memcmp_fn); + do_test_cow_in_parent(mem, size, true, child_vmsplice_memcmp_fn, + is_hugetlb); } static void do_test_vmsplice_in_parent(char *mem, size_t size, - bool before_fork) + bool before_fork, bool xfail) { struct iovec iov = { .iov_base = mem, @@ -355,8 +369,18 @@ static void do_test_vmsplice_in_parent(char *mem, size_t size, } } - ksft_test_result(!memcmp(old, new, transferred), - "No leak from child into parent\n"); + if (!memcmp(old, new, transferred)) { + ksft_test_result_pass("No leak from child into parent\n"); + } else if (xfail) { + /* + * With hugetlb, some vmsplice() tests are currently expected to + * fail because (a) harder to fix and (b) nobody really cares. + * Flag them as expected failure for now. + */ + ksft_test_result_xfail("Leak from child into parent\n"); + } else { + ksft_test_result_fail("Leak from child into parent\n"); + } close_pipe: close(fds[0]); close(fds[1]); @@ -367,14 +391,14 @@ static void do_test_vmsplice_in_parent(char *mem, size_t size, free(new); } -static void test_vmsplice_before_fork(char *mem, size_t size) +static void test_vmsplice_before_fork(char *mem, size_t size, bool is_hugetlb) { - do_test_vmsplice_in_parent(mem, size, true); + do_test_vmsplice_in_parent(mem, size, true, is_hugetlb); } -static void test_vmsplice_after_fork(char *mem, size_t size) +static void test_vmsplice_after_fork(char *mem, size_t size, bool is_hugetlb) { - do_test_vmsplice_in_parent(mem, size, false); + do_test_vmsplice_in_parent(mem, size, false, is_hugetlb); } #ifdef LOCAL_CONFIG_HAVE_LIBURING @@ -529,12 +553,12 @@ static void do_test_iouring(char *mem, size_t size, bool use_fork) close_comm_pipes(&comm_pipes); } -static void test_iouring_ro(char *mem, size_t size) +static void test_iouring_ro(char *mem, size_t size, bool is_hugetlb) { do_test_iouring(mem, size, false); } -static void test_iouring_fork(char *mem, size_t size) +static void test_iouring_fork(char *mem, size_t size, bool is_hugetlb) { do_test_iouring(mem, size, true); } @@ -678,37 +702,41 @@ static void do_test_ro_pin(char *mem, size_t size, enum ro_pin_test test, free(tmp); } -static void test_ro_pin_on_shared(char *mem, size_t size) +static void test_ro_pin_on_shared(char *mem, size_t size, bool is_hugetlb) { do_test_ro_pin(mem, size, RO_PIN_TEST_SHARED, false); } -static void test_ro_fast_pin_on_shared(char *mem, size_t size) +static void test_ro_fast_pin_on_shared(char *mem, size_t size, bool is_hugetlb) { do_test_ro_pin(mem, size, RO_PIN_TEST_SHARED, true); } -static void test_ro_pin_on_ro_previously_shared(char *mem, size_t size) +static void test_ro_pin_on_ro_previously_shared(char *mem, size_t size, + bool is_hugetlb) { do_test_ro_pin(mem, size, RO_PIN_TEST_PREVIOUSLY_SHARED, false); } -static void test_ro_fast_pin_on_ro_previously_shared(char *mem, size_t size) +static void test_ro_fast_pin_on_ro_previously_shared(char *mem, size_t size, + bool is_hugetlb) { do_test_ro_pin(mem, size, RO_PIN_TEST_PREVIOUSLY_SHARED, true); } -static void test_ro_pin_on_ro_exclusive(char *mem, size_t size) +static void test_ro_pin_on_ro_exclusive(char *mem, size_t size, + bool is_hugetlb) { do_test_ro_pin(mem, size, RO_PIN_TEST_RO_EXCLUSIVE, false); } -static void test_ro_fast_pin_on_ro_exclusive(char *mem, size_t size) +static void test_ro_fast_pin_on_ro_exclusive(char *mem, size_t size, + bool is_hugetlb) { do_test_ro_pin(mem, size, RO_PIN_TEST_RO_EXCLUSIVE, true); } -typedef void (*test_fn)(char *mem, size_t size); +typedef void (*test_fn)(char *mem, size_t size, bool hugetlb); static void do_run_with_base_page(test_fn fn, bool swapout) { @@ -740,7 +768,7 @@ static void do_run_with_base_page(test_fn fn, bool swapout) } } - fn(mem, pagesize); + fn(mem, pagesize, false); munmap: munmap(mem, pagesize); } @@ -904,7 +932,7 @@ static void do_run_with_thp(test_fn fn, enum thp_run thp_run, size_t thpsize) break; } - fn(mem, size); + fn(mem, size, false); munmap: munmap(mmap_mem, mmap_size); if (mremap_mem != MAP_FAILED) @@ -997,7 +1025,7 @@ static void run_with_hugetlb(test_fn fn, const char *desc, size_t hugetlbsize) } munmap(dummy, hugetlbsize); - fn(mem, hugetlbsize); + fn(mem, hugetlbsize, true); munmap: munmap(mem, hugetlbsize); } @@ -1036,7 +1064,7 @@ static const struct test_case anon_test_cases[] = { */ { "vmsplice() + unmap in child", - test_vmsplice_in_child + test_vmsplice_in_child, }, /* * vmsplice() test, but do an additional mprotect(PROT_READ)+ @@ -1044,7 +1072,7 @@ static const struct test_case anon_test_cases[] = { */ { "vmsplice() + unmap in child with mprotect() optimization", - test_vmsplice_in_child_mprotect + test_vmsplice_in_child_mprotect, }, /* * vmsplice() [R/O GUP] in parent before fork(), unmap in parent after @@ -1322,23 +1350,31 @@ static void do_test_anon_thp_collapse(char *mem, size_t size, close_comm_pipes(&comm_pipes); } -static void test_anon_thp_collapse_unshared(char *mem, size_t size) +static void test_anon_thp_collapse_unshared(char *mem, size_t size, + bool is_hugetlb) { + assert(!is_hugetlb); do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_UNSHARED); } -static void test_anon_thp_collapse_fully_shared(char *mem, size_t size) +static void test_anon_thp_collapse_fully_shared(char *mem, size_t size, + bool is_hugetlb) { + assert(!is_hugetlb); do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_FULLY_SHARED); } -static void test_anon_thp_collapse_lower_shared(char *mem, size_t size) +static void test_anon_thp_collapse_lower_shared(char *mem, size_t size, + bool is_hugetlb) { + assert(!is_hugetlb); do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_LOWER_SHARED); } -static void test_anon_thp_collapse_upper_shared(char *mem, size_t size) +static void test_anon_thp_collapse_upper_shared(char *mem, size_t size, + bool is_hugetlb) { + assert(!is_hugetlb); do_test_anon_thp_collapse(mem, size, ANON_THP_COLLAPSE_UPPER_SHARED); } From b8a2528835b31718286e7436529917e1f521bf6f Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Thu, 2 May 2024 10:52:59 +0200 Subject: [PATCH 1283/1702] mm/hugetlb: document why hugetlb uses folio_mapcount() for COW reuse decisions Let's document why hugetlb still uses folio_mapcount() and is prone to leaking memory between processes, for example using vmsplice() that still uses FOLL_GET. More details can be found in [1], especially around how hugetlb pages cannot really be overcommitted, and why we don't particularly care about these vmsplice() leaks for hugetlb -- in contrast to ordinary memory. [1] https://lore.kernel.org/all/8b42a24d-caf0-46ef-9e15-0f88d47d2f21@redhat.com/ Link: https://lkml.kernel.org/r/20240502085259.103784-3-david@redhat.com Signed-off-by: David Hildenbrand Suggested-by: Peter Xu Cc: Muchun Song Cc: Shuah Khan Signed-off-by: Andrew Morton --- mm/hugetlb.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 417fc5cdb6eeb..a7efb350f5d07 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5963,6 +5963,13 @@ static vm_fault_t hugetlb_wp(struct folio *pagecache_folio, /* * If no-one else is actually using this page, we're the exclusive * owner and can reuse this page. + * + * Note that we don't rely on the (safer) folio refcount here, because + * copying the hugetlb folio when there are unexpected (temporary) + * folio references could harm simple fork()+exit() users when + * we run out of free hugetlb folios: we would have to kill processes + * in scenarios that used to work. As a side effect, there can still + * be leaks between processes, for example, with FOLL_GET users. */ if (folio_mapcount(old_folio) == 1 && folio_test_anon(old_folio)) { if (!PageAnonExclusive(&old_folio->page)) { From a4c43b8a09805a7b9b39344c1ba304a5641aca77 Mon Sep 17 00:00:00 2001 From: Saurav Shah Date: Thu, 2 May 2024 04:43:17 +0530 Subject: [PATCH 1284/1702] selftests/memfd: fix spelling mistakes Fix spelling mistakes in the comments. Link: https://lkml.kernel.org/r/20240501231317.24648-1-sauravshah.31@gmail.com Signed-off-by: Saurav Shah Cc: Aleksa Sarai Cc: Greg Thelen Cc: Jeff Xu Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/memfd/fuse_test.c | 2 +- tools/testing/selftests/memfd/memfd_test.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/memfd/fuse_test.c b/tools/testing/selftests/memfd/fuse_test.c index 93798c8c5d54b..dbc171a3806db 100644 --- a/tools/testing/selftests/memfd/fuse_test.c +++ b/tools/testing/selftests/memfd/fuse_test.c @@ -306,7 +306,7 @@ int main(int argc, char **argv) * then the kernel did a page-replacement or canceled the read() (or * whatever magic it did..). In that case, the memfd object is still * all zero. - * In case the memfd-object was *not* sealed, the read() was successfull + * In case the memfd-object was *not* sealed, the read() was successful * and the memfd object must *not* be all zero. * Note that in real scenarios, there might be a mixture of both, but * in this test-cases, we have explicit 200ms delays which should be diff --git a/tools/testing/selftests/memfd/memfd_test.c b/tools/testing/selftests/memfd/memfd_test.c index 18f585684e202..95af2d78fd318 100644 --- a/tools/testing/selftests/memfd/memfd_test.c +++ b/tools/testing/selftests/memfd/memfd_test.c @@ -1528,7 +1528,7 @@ static void test_share_open(char *banner, char *b_suffix) /* * Test sharing via fork() - * Test whether seal-modifications work as expected with forked childs. + * Test whether seal-modifications work as expected with forked children. */ static void test_share_fork(char *banner, char *b_suffix) { From 59142d87ab03b8ff969074348f65730d465f42ee Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 1 May 2024 10:26:10 -0700 Subject: [PATCH 1285/1702] memcg: reduce memory size of mem_cgroup_events_index Patch series "memcg: reduce memory consumption by memcg stats", v4. Most of the memory overhead of a memcg object is due to memcg stats maintained by the kernel. Since stats updates happen in performance critical codepaths, the stats are maintained per-cpu and numa specific stats are maintained per-node * per-cpu. This drastically increase the overhead on large machines i.e. large of CPUs and multiple numa nodes. This patch series tries to reduce the overhead by at least not allocating the memory for stats which are not memcg specific. This patch (of 8): mem_cgroup_events_index is a translation table to get the right index of the memcg relevant entry for the general vm_event_item. At the moment, it is defined as integer array. However on a typical system the max entry of vm_event_item (NR_VM_EVENT_ITEMS) is 113, so we don't need to use int as storage type of the array. For now just use int8_t as type and add a BUILD_BUG_ON(). Another benefit of this change is that the translation table fits in 2 cachelines while previously it would require 8 cachelines (assuming 64 bytes cacheline). Link: https://lkml.kernel.org/r/20240501172617.678560-1-shakeel.butt@linux.dev Link: https://lkml.kernel.org/r/20240501172617.678560-2-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Reviewed-by: Roman Gushchin Reviewed-by: Yosry Ahmed Reviewed-by: T.J. Mercier Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Signed-off-by: Andrew Morton --- mm/memcontrol.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index a111e0d981baa..96b72a6e04bac 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -606,11 +606,13 @@ static const unsigned int memcg_vm_event_stat[] = { }; #define NR_MEMCG_EVENTS ARRAY_SIZE(memcg_vm_event_stat) -static int mem_cgroup_events_index[NR_VM_EVENT_ITEMS] __read_mostly; +static int8_t mem_cgroup_events_index[NR_VM_EVENT_ITEMS] __read_mostly; static void init_memcg_events(void) { - int i; + int8_t i; + + BUILD_BUG_ON(NR_VM_EVENT_ITEMS >= S8_MAX); for (i = 0; i < NR_MEMCG_EVENTS; ++i) mem_cgroup_events_index[memcg_vm_event_stat[i]] = i + 1; From 70a64b7919cbd6c12306051ff2825839a9d65605 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 1 May 2024 10:26:11 -0700 Subject: [PATCH 1286/1702] memcg: dynamically allocate lruvec_stats To decouple the dependency of lruvec_stats on NR_VM_NODE_STAT_ITEMS, we need to dynamically allocate lruvec_stats in the mem_cgroup_per_node structure. Also move the definition of lruvec_stats_percpu and lruvec_stats and related functions to the memcontrol.c to facilitate later patches. No functional changes in the patch. Link: https://lkml.kernel.org/r/20240501172617.678560-3-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Reviewed-by: Yosry Ahmed Reviewed-by: T.J. Mercier Reviewed-by: Roman Gushchin Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 62 +++------------------------ mm/memcontrol.c | 87 ++++++++++++++++++++++++++++++++------ 2 files changed, 81 insertions(+), 68 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 9aba0d0462ca6..ab8a6e884375d 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -83,6 +83,8 @@ enum mem_cgroup_events_target { struct memcg_vmstats_percpu; struct memcg_vmstats; +struct lruvec_stats_percpu; +struct lruvec_stats; struct mem_cgroup_reclaim_iter { struct mem_cgroup *position; @@ -90,25 +92,6 @@ struct mem_cgroup_reclaim_iter { unsigned int generation; }; -struct lruvec_stats_percpu { - /* Local (CPU and cgroup) state */ - long state[NR_VM_NODE_STAT_ITEMS]; - - /* Delta calculation for lockless upward propagation */ - long state_prev[NR_VM_NODE_STAT_ITEMS]; -}; - -struct lruvec_stats { - /* Aggregated (CPU and subtree) state */ - long state[NR_VM_NODE_STAT_ITEMS]; - - /* Non-hierarchical (CPU aggregated) state */ - long state_local[NR_VM_NODE_STAT_ITEMS]; - - /* Pending child counts during tree propagation */ - long state_pending[NR_VM_NODE_STAT_ITEMS]; -}; - /* * per-node information in memory controller. */ @@ -116,7 +99,7 @@ struct mem_cgroup_per_node { struct lruvec lruvec; struct lruvec_stats_percpu __percpu *lruvec_stats_percpu; - struct lruvec_stats lruvec_stats; + struct lruvec_stats *lruvec_stats; unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; @@ -1037,42 +1020,9 @@ static inline void mod_memcg_page_state(struct page *page, } unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx); - -static inline unsigned long lruvec_page_state(struct lruvec *lruvec, - enum node_stat_item idx) -{ - struct mem_cgroup_per_node *pn; - long x; - - if (mem_cgroup_disabled()) - return node_page_state(lruvec_pgdat(lruvec), idx); - - pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - x = READ_ONCE(pn->lruvec_stats.state[idx]); -#ifdef CONFIG_SMP - if (x < 0) - x = 0; -#endif - return x; -} - -static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec, - enum node_stat_item idx) -{ - struct mem_cgroup_per_node *pn; - long x = 0; - - if (mem_cgroup_disabled()) - return node_page_state(lruvec_pgdat(lruvec), idx); - - pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - x = READ_ONCE(pn->lruvec_stats.state_local[idx]); -#ifdef CONFIG_SMP - if (x < 0) - x = 0; -#endif - return x; -} +unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx); +unsigned long lruvec_page_state_local(struct lruvec *lruvec, + enum node_stat_item idx); void mem_cgroup_flush_stats(struct mem_cgroup *memcg); void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 96b72a6e04bac..cf6819a3310dd 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -575,6 +575,60 @@ mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) return mz; } +struct lruvec_stats_percpu { + /* Local (CPU and cgroup) state */ + long state[NR_VM_NODE_STAT_ITEMS]; + + /* Delta calculation for lockless upward propagation */ + long state_prev[NR_VM_NODE_STAT_ITEMS]; +}; + +struct lruvec_stats { + /* Aggregated (CPU and subtree) state */ + long state[NR_VM_NODE_STAT_ITEMS]; + + /* Non-hierarchical (CPU aggregated) state */ + long state_local[NR_VM_NODE_STAT_ITEMS]; + + /* Pending child counts during tree propagation */ + long state_pending[NR_VM_NODE_STAT_ITEMS]; +}; + +unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) +{ + struct mem_cgroup_per_node *pn; + long x; + + if (mem_cgroup_disabled()) + return node_page_state(lruvec_pgdat(lruvec), idx); + + pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); + x = READ_ONCE(pn->lruvec_stats->state[idx]); +#ifdef CONFIG_SMP + if (x < 0) + x = 0; +#endif + return x; +} + +unsigned long lruvec_page_state_local(struct lruvec *lruvec, + enum node_stat_item idx) +{ + struct mem_cgroup_per_node *pn; + long x = 0; + + if (mem_cgroup_disabled()) + return node_page_state(lruvec_pgdat(lruvec), idx); + + pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); + x = READ_ONCE(pn->lruvec_stats->state_local[idx]); +#ifdef CONFIG_SMP + if (x < 0) + x = 0; +#endif + return x; +} + /* Subset of vm_event_item to report for memcg event stats */ static const unsigned int memcg_vm_event_stat[] = { PGPGIN, @@ -5486,18 +5540,25 @@ static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) if (!pn) return 1; + pn->lruvec_stats = kzalloc_node(sizeof(struct lruvec_stats), GFP_KERNEL, + node); + if (!pn->lruvec_stats) + goto fail; + pn->lruvec_stats_percpu = alloc_percpu_gfp(struct lruvec_stats_percpu, GFP_KERNEL_ACCOUNT); - if (!pn->lruvec_stats_percpu) { - kfree(pn); - return 1; - } + if (!pn->lruvec_stats_percpu) + goto fail; lruvec_init(&pn->lruvec); pn->memcg = memcg; memcg->nodeinfo[node] = pn; return 0; +fail: + kfree(pn->lruvec_stats); + kfree(pn); + return 1; } static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) @@ -5508,6 +5569,7 @@ static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) return; free_percpu(pn->lruvec_stats_percpu); + kfree(pn->lruvec_stats); kfree(pn); } @@ -5860,18 +5922,19 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) for_each_node_state(nid, N_MEMORY) { struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid]; - struct mem_cgroup_per_node *ppn = NULL; + struct lruvec_stats *lstats = pn->lruvec_stats; + struct lruvec_stats *plstats = NULL; struct lruvec_stats_percpu *lstatc; if (parent) - ppn = parent->nodeinfo[nid]; + plstats = parent->nodeinfo[nid]->lruvec_stats; lstatc = per_cpu_ptr(pn->lruvec_stats_percpu, cpu); for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { - delta = pn->lruvec_stats.state_pending[i]; + delta = lstats->state_pending[i]; if (delta) - pn->lruvec_stats.state_pending[i] = 0; + lstats->state_pending[i] = 0; delta_cpu = 0; v = READ_ONCE(lstatc->state[i]); @@ -5882,12 +5945,12 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) } if (delta_cpu) - pn->lruvec_stats.state_local[i] += delta_cpu; + lstats->state_local[i] += delta_cpu; if (delta) { - pn->lruvec_stats.state[i] += delta; - if (ppn) - ppn->lruvec_stats.state_pending[i] += delta; + lstats->state[i] += delta; + if (plstats) + plstats->state_pending[i] += delta; } } } From aab6103b97f1c7ca6cf33e78d5e6916e53fc265c Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Wed, 1 May 2024 10:26:12 -0700 Subject: [PATCH 1287/1702] mm: memcg: account memory used for memcg vmstats and lruvec stats The percpu memory used by memcg's memory statistics is already accounted. For consistency, let's enable accounting for vmstats and lruvec stats as well. Link: https://lkml.kernel.org/r/20240501172617.678560-4-shakeel.butt@linux.dev Signed-off-by: Roman Gushchin Signed-off-by: Shakeel Butt Reviewed-by: Yosry Ahmed Reviewed-by: T.J. Mercier Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Signed-off-by: Andrew Morton --- mm/memcontrol.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index cf6819a3310dd..42e0062d880b7 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5540,8 +5540,8 @@ static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) if (!pn) return 1; - pn->lruvec_stats = kzalloc_node(sizeof(struct lruvec_stats), GFP_KERNEL, - node); + pn->lruvec_stats = kzalloc_node(sizeof(struct lruvec_stats), + GFP_KERNEL_ACCOUNT, node); if (!pn->lruvec_stats) goto fail; @@ -5612,7 +5612,8 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent) goto fail; } - memcg->vmstats = kzalloc(sizeof(struct memcg_vmstats), GFP_KERNEL); + memcg->vmstats = kzalloc(sizeof(struct memcg_vmstats), + GFP_KERNEL_ACCOUNT); if (!memcg->vmstats) goto fail; From ff48c71c26aaefb090c108d8803abdf0c75f00a9 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 1 May 2024 10:26:13 -0700 Subject: [PATCH 1288/1702] memcg: reduce memory for the lruvec and memcg stats At the moment, the amount of memory allocated for stats related structs in the mem_cgroup corresponds to the size of enum node_stat_item. However not all fields in enum node_stat_item have corresponding memcg stats. So, let's use indirection mechanism similar to the one used for memcg vmstats management. For a given x86_64 config, the size of stats with and without patch is: structs size in bytes w/o with struct lruvec_stats 1128 648 struct lruvec_stats_percpu 752 432 struct memcg_vmstats 1832 1352 struct memcg_vmstats_percpu 1280 960 The memory savings are further compounded by the fact that these structs are allocated for each cpu and for each node. To be precise, for each memcg the memory saved would be: Memory saved = ((21 * 3 * NR_NODES) + (21 * 2 * NR_NODES * NR_CPUS) + (21 * 3) + (21 * 2 * NR_CPUS)) * sizeof(long) Where 21 is the number of fields eliminated. Link: https://lkml.kernel.org/r/20240501172617.678560-5-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Reviewed-by: Roman Gushchin Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: T.J. Mercier Cc: Yosry Ahmed Signed-off-by: Andrew Morton --- mm/memcontrol.c | 134 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 114 insertions(+), 20 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 42e0062d880b7..0dad17d26ce2c 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -575,35 +575,106 @@ mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz) return mz; } +/* Subset of node_stat_item for memcg stats */ +static const unsigned int memcg_node_stat_items[] = { + NR_INACTIVE_ANON, + NR_ACTIVE_ANON, + NR_INACTIVE_FILE, + NR_ACTIVE_FILE, + NR_UNEVICTABLE, + NR_SLAB_RECLAIMABLE_B, + NR_SLAB_UNRECLAIMABLE_B, + WORKINGSET_REFAULT_ANON, + WORKINGSET_REFAULT_FILE, + WORKINGSET_ACTIVATE_ANON, + WORKINGSET_ACTIVATE_FILE, + WORKINGSET_RESTORE_ANON, + WORKINGSET_RESTORE_FILE, + WORKINGSET_NODERECLAIM, + NR_ANON_MAPPED, + NR_FILE_MAPPED, + NR_FILE_PAGES, + NR_FILE_DIRTY, + NR_WRITEBACK, + NR_SHMEM, + NR_SHMEM_THPS, + NR_FILE_THPS, + NR_ANON_THPS, + NR_KERNEL_STACK_KB, + NR_PAGETABLE, + NR_SECONDARY_PAGETABLE, +#ifdef CONFIG_SWAP + NR_SWAPCACHE, +#endif +}; + +static const unsigned int memcg_stat_items[] = { + MEMCG_SWAP, + MEMCG_SOCK, + MEMCG_PERCPU_B, + MEMCG_VMALLOC, + MEMCG_KMEM, + MEMCG_ZSWAP_B, + MEMCG_ZSWAPPED, +}; + +#define NR_MEMCG_NODE_STAT_ITEMS ARRAY_SIZE(memcg_node_stat_items) +#define MEMCG_VMSTAT_SIZE (NR_MEMCG_NODE_STAT_ITEMS + \ + ARRAY_SIZE(memcg_stat_items)) +static int8_t mem_cgroup_stats_index[MEMCG_NR_STAT] __read_mostly; + +static void init_memcg_stats(void) +{ + int8_t i, j = 0; + + BUILD_BUG_ON(MEMCG_NR_STAT >= S8_MAX); + + for (i = 0; i < NR_MEMCG_NODE_STAT_ITEMS; ++i) + mem_cgroup_stats_index[memcg_node_stat_items[i]] = ++j; + + for (i = 0; i < ARRAY_SIZE(memcg_stat_items); ++i) + mem_cgroup_stats_index[memcg_stat_items[i]] = ++j; +} + +static inline int memcg_stats_index(int idx) +{ + return mem_cgroup_stats_index[idx] - 1; +} + struct lruvec_stats_percpu { /* Local (CPU and cgroup) state */ - long state[NR_VM_NODE_STAT_ITEMS]; + long state[NR_MEMCG_NODE_STAT_ITEMS]; /* Delta calculation for lockless upward propagation */ - long state_prev[NR_VM_NODE_STAT_ITEMS]; + long state_prev[NR_MEMCG_NODE_STAT_ITEMS]; }; struct lruvec_stats { /* Aggregated (CPU and subtree) state */ - long state[NR_VM_NODE_STAT_ITEMS]; + long state[NR_MEMCG_NODE_STAT_ITEMS]; /* Non-hierarchical (CPU aggregated) state */ - long state_local[NR_VM_NODE_STAT_ITEMS]; + long state_local[NR_MEMCG_NODE_STAT_ITEMS]; /* Pending child counts during tree propagation */ - long state_pending[NR_VM_NODE_STAT_ITEMS]; + long state_pending[NR_MEMCG_NODE_STAT_ITEMS]; }; unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) { struct mem_cgroup_per_node *pn; long x; + int i; if (mem_cgroup_disabled()) return node_page_state(lruvec_pgdat(lruvec), idx); + i = memcg_stats_index(idx); + if (i < 0) + return 0; + pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - x = READ_ONCE(pn->lruvec_stats->state[idx]); + x = READ_ONCE(pn->lruvec_stats->state[i]); #ifdef CONFIG_SMP if (x < 0) x = 0; @@ -616,12 +687,17 @@ unsigned long lruvec_page_state_local(struct lruvec *lruvec, { struct mem_cgroup_per_node *pn; long x = 0; + int i; if (mem_cgroup_disabled()) return node_page_state(lruvec_pgdat(lruvec), idx); + i = memcg_stats_index(idx); + if (i < 0) + return 0; + pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - x = READ_ONCE(pn->lruvec_stats->state_local[idx]); + x = READ_ONCE(pn->lruvec_stats->state_local[i]); #ifdef CONFIG_SMP if (x < 0) x = 0; @@ -688,11 +764,11 @@ struct memcg_vmstats_percpu { /* The above should fit a single cacheline for memcg_rstat_updated() */ /* Local (CPU and cgroup) page state & events */ - long state[MEMCG_NR_STAT]; + long state[MEMCG_VMSTAT_SIZE]; unsigned long events[NR_MEMCG_EVENTS]; /* Delta calculation for lockless upward propagation */ - long state_prev[MEMCG_NR_STAT]; + long state_prev[MEMCG_VMSTAT_SIZE]; unsigned long events_prev[NR_MEMCG_EVENTS]; /* Cgroup1: threshold notifications & softlimit tree updates */ @@ -702,15 +778,15 @@ struct memcg_vmstats_percpu { struct memcg_vmstats { /* Aggregated (CPU and subtree) page state & events */ - long state[MEMCG_NR_STAT]; + long state[MEMCG_VMSTAT_SIZE]; unsigned long events[NR_MEMCG_EVENTS]; /* Non-hierarchical (CPU aggregated) page state & events */ - long state_local[MEMCG_NR_STAT]; + long state_local[MEMCG_VMSTAT_SIZE]; unsigned long events_local[NR_MEMCG_EVENTS]; /* Pending child counts during tree propagation */ - long state_pending[MEMCG_NR_STAT]; + long state_pending[MEMCG_VMSTAT_SIZE]; unsigned long events_pending[NR_MEMCG_EVENTS]; /* Stats updates since the last flush */ @@ -843,7 +919,13 @@ static void flush_memcg_stats_dwork(struct work_struct *w) unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) { - long x = READ_ONCE(memcg->vmstats->state[idx]); + long x; + int i = memcg_stats_index(idx); + + if (i < 0) + return 0; + + x = READ_ONCE(memcg->vmstats->state[i]); #ifdef CONFIG_SMP if (x < 0) x = 0; @@ -875,18 +957,25 @@ static int memcg_state_val_in_pages(int idx, int val) */ void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val) { - if (mem_cgroup_disabled()) + int i = memcg_stats_index(idx); + + if (mem_cgroup_disabled() || i < 0) return; - __this_cpu_add(memcg->vmstats_percpu->state[idx], val); + __this_cpu_add(memcg->vmstats_percpu->state[i], val); memcg_rstat_updated(memcg, memcg_state_val_in_pages(idx, val)); } /* idx can be of type enum memcg_stat_item or node_stat_item. */ static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx) { - long x = READ_ONCE(memcg->vmstats->state_local[idx]); + long x; + int i = memcg_stats_index(idx); + + if (i < 0) + return 0; + x = READ_ONCE(memcg->vmstats->state_local[i]); #ifdef CONFIG_SMP if (x < 0) x = 0; @@ -900,6 +989,10 @@ static void __mod_memcg_lruvec_state(struct lruvec *lruvec, { struct mem_cgroup_per_node *pn; struct mem_cgroup *memcg; + int i = memcg_stats_index(idx); + + if (i < 0) + return; pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); memcg = pn->memcg; @@ -926,10 +1019,10 @@ static void __mod_memcg_lruvec_state(struct lruvec *lruvec, } /* Update memcg */ - __this_cpu_add(memcg->vmstats_percpu->state[idx], val); + __this_cpu_add(memcg->vmstats_percpu->state[i], val); /* Update lruvec */ - __this_cpu_add(pn->lruvec_stats_percpu->state[idx], val); + __this_cpu_add(pn->lruvec_stats_percpu->state[i], val); memcg_rstat_updated(memcg, memcg_state_val_in_pages(idx, val)); memcg_stats_unlock(); @@ -5697,6 +5790,7 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) page_counter_init(&memcg->kmem, &parent->kmem); page_counter_init(&memcg->tcpmem, &parent->tcpmem); } else { + init_memcg_stats(); init_memcg_events(); page_counter_init(&memcg->memory, NULL); page_counter_init(&memcg->swap, NULL); @@ -5868,7 +5962,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) statc = per_cpu_ptr(memcg->vmstats_percpu, cpu); - for (i = 0; i < MEMCG_NR_STAT; i++) { + for (i = 0; i < MEMCG_VMSTAT_SIZE; i++) { /* * Collect the aggregated propagation counts of groups * below us. We're in a per-cpu loop here and this is @@ -5932,7 +6026,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) lstatc = per_cpu_ptr(pn->lruvec_stats_percpu, cpu); - for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { + for (i = 0; i < NR_MEMCG_NODE_STAT_ITEMS; i++) { delta = lstats->state_pending[i]; if (delta) lstats->state_pending[i] = 0; From 0667c7870a186e15ca7a21e0936070f0977bc80e Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 1 May 2024 10:26:14 -0700 Subject: [PATCH 1289/1702] memcg: cleanup __mod_memcg_lruvec_state There are no memcg specific stats for NR_SHMEM_PMDMAPPED and NR_FILE_PMDMAPPED. Let's remove them. Link: https://lkml.kernel.org/r/20240501172617.678560-6-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Reviewed-by: Yosry Ahmed Reviewed-by: Roman Gushchin Reviewed-by: T.J. Mercier Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Signed-off-by: Andrew Morton --- mm/memcontrol.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 0dad17d26ce2c..b59b670f7b7f9 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1009,8 +1009,6 @@ static void __mod_memcg_lruvec_state(struct lruvec *lruvec, case NR_ANON_MAPPED: case NR_FILE_MAPPED: case NR_ANON_THPS: - case NR_SHMEM_PMDMAPPED: - case NR_FILE_PMDMAPPED: WARN_ON_ONCE(!in_task()); break; default: From 4715c6a753dccd15fd3a8928168f57e349205bd4 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 1 May 2024 10:26:15 -0700 Subject: [PATCH 1290/1702] mm: cleanup WORKINGSET_NODES in workingset WORKINGSET_NODES is not exposed in the memcg stats and thus there is no need to use the memcg specific stat update functions for it. In future if we decide to expose WORKINGSET_NODES in the memcg stats, we can revert this patch. Link: https://lkml.kernel.org/r/20240501172617.678560-7-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Reviewed-by: Roman Gushchin Reviewed-by: T.J. Mercier Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Yosry Ahmed Signed-off-by: Andrew Morton --- mm/workingset.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mm/workingset.c b/mm/workingset.c index f2a0ecaf708d7..c22adb93622a5 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -618,6 +618,7 @@ struct list_lru shadow_nodes; void workingset_update_node(struct xa_node *node) { struct address_space *mapping; + struct page *page = virt_to_page(node); /* * Track non-empty nodes that contain only shadow entries; @@ -633,12 +634,12 @@ void workingset_update_node(struct xa_node *node) if (node->count && node->count == node->nr_values) { if (list_empty(&node->private_list)) { list_lru_add_obj(&shadow_nodes, &node->private_list); - __inc_lruvec_kmem_state(node, WORKINGSET_NODES); + __inc_node_page_state(page, WORKINGSET_NODES); } } else { if (!list_empty(&node->private_list)) { list_lru_del_obj(&shadow_nodes, &node->private_list); - __dec_lruvec_kmem_state(node, WORKINGSET_NODES); + __dec_node_page_state(page, WORKINGSET_NODES); } } } @@ -742,7 +743,7 @@ static enum lru_status shadow_lru_isolate(struct list_head *item, } list_lru_isolate(lru, item); - __dec_lruvec_kmem_state(node, WORKINGSET_NODES); + __dec_node_page_state(virt_to_page(node), WORKINGSET_NODES); spin_unlock(lru_lock); From acb5fe2f1aff090ff3d32328f3028c9fb72453ec Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 1 May 2024 10:26:16 -0700 Subject: [PATCH 1291/1702] memcg: warn for unexpected events and stats To reduce memory usage by the memcg events and stats, the kernel uses indirection table and only allocate stats and events which are being used by the memcg code. To make this more robust, let's add warnings where unexpected stats and events indexes are used. Link: https://lkml.kernel.org/r/20240501172617.678560-8-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Reviewed-by: Roman Gushchin Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: T.J. Mercier Cc: Yosry Ahmed Signed-off-by: Andrew Morton --- mm/memcontrol.c | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b59b670f7b7f9..87411116ec5e6 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -670,7 +670,7 @@ unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) return node_page_state(lruvec_pgdat(lruvec), idx); i = memcg_stats_index(idx); - if (i < 0) + if (WARN_ONCE(i < 0, "%s: missing stat item %d\n", __func__, idx)) return 0; pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); @@ -686,14 +686,14 @@ unsigned long lruvec_page_state_local(struct lruvec *lruvec, enum node_stat_item idx) { struct mem_cgroup_per_node *pn; - long x = 0; + long x; int i; if (mem_cgroup_disabled()) return node_page_state(lruvec_pgdat(lruvec), idx); i = memcg_stats_index(idx); - if (i < 0) + if (WARN_ONCE(i < 0, "%s: missing stat item %d\n", __func__, idx)) return 0; pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); @@ -922,7 +922,7 @@ unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) long x; int i = memcg_stats_index(idx); - if (i < 0) + if (WARN_ONCE(i < 0, "%s: missing stat item %d\n", __func__, idx)) return 0; x = READ_ONCE(memcg->vmstats->state[i]); @@ -959,7 +959,10 @@ void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val) { int i = memcg_stats_index(idx); - if (mem_cgroup_disabled() || i < 0) + if (mem_cgroup_disabled()) + return; + + if (WARN_ONCE(i < 0, "%s: missing stat item %d\n", __func__, idx)) return; __this_cpu_add(memcg->vmstats_percpu->state[i], val); @@ -972,7 +975,7 @@ static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx) long x; int i = memcg_stats_index(idx); - if (i < 0) + if (WARN_ONCE(i < 0, "%s: missing stat item %d\n", __func__, idx)) return 0; x = READ_ONCE(memcg->vmstats->state_local[i]); @@ -991,7 +994,7 @@ static void __mod_memcg_lruvec_state(struct lruvec *lruvec, struct mem_cgroup *memcg; int i = memcg_stats_index(idx); - if (i < 0) + if (WARN_ONCE(i < 0, "%s: missing stat item %d\n", __func__, idx)) return; pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); @@ -1102,34 +1105,38 @@ void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val) void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, unsigned long count) { - int index = memcg_events_index(idx); + int i = memcg_events_index(idx); - if (mem_cgroup_disabled() || index < 0) + if (mem_cgroup_disabled()) + return; + + if (WARN_ONCE(i < 0, "%s: missing stat item %d\n", __func__, idx)) return; memcg_stats_lock(); - __this_cpu_add(memcg->vmstats_percpu->events[index], count); + __this_cpu_add(memcg->vmstats_percpu->events[i], count); memcg_rstat_updated(memcg, count); memcg_stats_unlock(); } static unsigned long memcg_events(struct mem_cgroup *memcg, int event) { - int index = memcg_events_index(event); + int i = memcg_events_index(event); - if (index < 0) + if (WARN_ONCE(i < 0, "%s: missing stat item %d\n", __func__, event)) return 0; - return READ_ONCE(memcg->vmstats->events[index]); + + return READ_ONCE(memcg->vmstats->events[i]); } static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event) { - int index = memcg_events_index(event); + int i = memcg_events_index(event); - if (index < 0) + if (WARN_ONCE(i < 0, "%s: missing stat item %d\n", __func__, event)) return 0; - return READ_ONCE(memcg->vmstats->events_local[index]); + return READ_ONCE(memcg->vmstats->events_local[i]); } static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, From a94032b35e5f97dc1023030d92998418c9feb27b Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 1 May 2024 10:26:17 -0700 Subject: [PATCH 1292/1702] memcg: use proper type for mod_memcg_state The memcg stats update functions can take arbitrary integer but the only input which make sense is enum memcg_stat_item and we don't want these functions to be called with arbitrary integer, so replace the parameter type with enum memcg_stat_item and compiler will be able to warn if memcg stat update functions are called with incorrect index value. Link: https://lkml.kernel.org/r/20240501172617.678560-9-shakeel.butt@linux.dev Signed-off-by: Shakeel Butt Reviewed-by: T.J. Mercier Reviewed-by: Roman Gushchin Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Yosry Ahmed Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 13 +++++++------ mm/memcontrol.c | 3 ++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index ab8a6e884375d..030d34e9d1176 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -974,7 +974,8 @@ void mem_cgroup_print_oom_group(struct mem_cgroup *memcg); void folio_memcg_lock(struct folio *folio); void folio_memcg_unlock(struct folio *folio); -void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val); +void __mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, + int val); /* try to stablize folio_memcg() for all the pages in a memcg */ static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) @@ -995,7 +996,7 @@ static inline void mem_cgroup_unlock_pages(void) /* idx can be of type enum memcg_stat_item or node_stat_item */ static inline void mod_memcg_state(struct mem_cgroup *memcg, - int idx, int val) + enum memcg_stat_item idx, int val) { unsigned long flags; @@ -1005,7 +1006,7 @@ static inline void mod_memcg_state(struct mem_cgroup *memcg, } static inline void mod_memcg_page_state(struct page *page, - int idx, int val) + enum memcg_stat_item idx, int val) { struct mem_cgroup *memcg; @@ -1491,19 +1492,19 @@ static inline void mem_cgroup_print_oom_group(struct mem_cgroup *memcg) } static inline void __mod_memcg_state(struct mem_cgroup *memcg, - int idx, + enum memcg_stat_item idx, int nr) { } static inline void mod_memcg_state(struct mem_cgroup *memcg, - int idx, + enum memcg_stat_item idx, int nr) { } static inline void mod_memcg_page_state(struct page *page, - int idx, int val) + enum memcg_stat_item idx, int val) { } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 87411116ec5e6..e6302f42e40f1 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -955,7 +955,8 @@ static int memcg_state_val_in_pages(int idx, int val) * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item * @val: delta to add to the counter, can be negative */ -void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val) +void __mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, + int val) { int i = memcg_stats_index(idx); From b0d7e15a9f21919382075980a5da9ed85ddfcd11 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Wed, 1 May 2024 15:44:39 +0100 Subject: [PATCH 1293/1702] mm/debug_vm_pgtable: test pmd_leaf() behavior with pmd_mkinvalid() An invalidated pmd should still cause pmd_leaf() to return true. Let's test for that to ensure all arches remain consistent. Link: https://lkml.kernel.org/r/20240501144439.1389048-1-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Reviewed-by: Anshuman Khandual Reviewed-by: David Hildenbrand Cc: Catalin Marinas Signed-off-by: Andrew Morton --- mm/debug_vm_pgtable.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index f1c9a2c5abc04..b104a353b532b 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -982,6 +982,7 @@ static void __init pmd_thp_tests(struct pgtable_debug_args *args) #ifndef __HAVE_ARCH_PMDP_INVALIDATE WARN_ON(!pmd_trans_huge(pmd_mkinvalid(pmd_mkhuge(pmd)))); WARN_ON(!pmd_present(pmd_mkinvalid(pmd_mkhuge(pmd)))); + WARN_ON(!pmd_leaf(pmd_mkinvalid(pmd_mkhuge(pmd)))); #endif /* __HAVE_ARCH_PMDP_INVALIDATE */ } From 3a5a8d343e1cf96eb9971b17cbd4b832ab19b8e7 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Wed, 1 May 2024 15:33:10 +0100 Subject: [PATCH 1294/1702] mm: fix race between __split_huge_pmd_locked() and GUP-fast __split_huge_pmd_locked() can be called for a present THP, devmap or (non-present) migration entry. It calls pmdp_invalidate() unconditionally on the pmdp and only determines if it is present or not based on the returned old pmd. This is a problem for the migration entry case because pmd_mkinvalid(), called by pmdp_invalidate() must only be called for a present pmd. On arm64 at least, pmd_mkinvalid() will mark the pmd such that any future call to pmd_present() will return true. And therefore any lockless pgtable walker could see the migration entry pmd in this state and start interpretting the fields as if it were present, leading to BadThings (TM). GUP-fast appears to be one such lockless pgtable walker. x86 does not suffer the above problem, but instead pmd_mkinvalid() will corrupt the offset field of the swap entry within the swap pte. See link below for discussion of that problem. Fix all of this by only calling pmdp_invalidate() for a present pmd. And for good measure let's add a warning to all implementations of pmdp_invalidate[_ad](). I've manually reviewed all other pmdp_invalidate[_ad]() call sites and believe all others to be conformant. This is a theoretical bug found during code review. I don't have any test case to trigger it in practice. Link: https://lkml.kernel.org/r/20240501143310.1381675-1-ryan.roberts@arm.com Link: https://lore.kernel.org/all/0dd7827a-6334-439a-8fd0-43c98e6af22b@arm.com/ Fixes: 84c3fc4e9c56 ("mm: thp: check pmd migration entry in common path") Signed-off-by: Ryan Roberts Reviewed-by: Zi Yan Reviewed-by: Anshuman Khandual Acked-by: David Hildenbrand Cc: Andreas Larsson Cc: Andy Lutomirski Cc: Aneesh Kumar K.V Cc: Borislav Petkov (AMD) Cc: Catalin Marinas Cc: Christian Borntraeger Cc: Christophe Leroy Cc: Dave Hansen Cc: "David S. Miller" Cc: Ingo Molnar Cc: Jonathan Corbet Cc: Mark Rutland Cc: Naveen N. Rao Cc: Nicholas Piggin Cc: Peter Zijlstra Cc: Sven Schnelle Cc: Thomas Gleixner Cc: Will Deacon Cc: Signed-off-by: Andrew Morton --- Documentation/mm/arch_pgtable_helpers.rst | 6 ++- arch/powerpc/mm/book3s64/pgtable.c | 1 + arch/s390/include/asm/pgtable.h | 4 +- arch/sparc/mm/tlb.c | 1 + arch/x86/mm/pgtable.c | 2 + mm/huge_memory.c | 49 ++++++++++++----------- mm/pgtable-generic.c | 2 + 7 files changed, 39 insertions(+), 26 deletions(-) diff --git a/Documentation/mm/arch_pgtable_helpers.rst b/Documentation/mm/arch_pgtable_helpers.rst index 2466d3363af79..ad50ca6f495eb 100644 --- a/Documentation/mm/arch_pgtable_helpers.rst +++ b/Documentation/mm/arch_pgtable_helpers.rst @@ -140,7 +140,8 @@ PMD Page Table Helpers +---------------------------+--------------------------------------------------+ | pmd_swp_clear_soft_dirty | Clears a soft dirty swapped PMD | +---------------------------+--------------------------------------------------+ -| pmd_mkinvalid | Invalidates a mapped PMD [1] | +| pmd_mkinvalid | Invalidates a present PMD; do not call for | +| | non-present PMD [1] | +---------------------------+--------------------------------------------------+ | pmd_set_huge | Creates a PMD huge mapping | +---------------------------+--------------------------------------------------+ @@ -196,7 +197,8 @@ PUD Page Table Helpers +---------------------------+--------------------------------------------------+ | pud_mkdevmap | Creates a ZONE_DEVICE mapped PUD | +---------------------------+--------------------------------------------------+ -| pud_mkinvalid | Invalidates a mapped PUD [1] | +| pud_mkinvalid | Invalidates a present PUD; do not call for | +| | non-present PUD [1] | +---------------------------+--------------------------------------------------+ | pud_set_huge | Creates a PUD huge mapping | +---------------------------+--------------------------------------------------+ diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c index 83823db3488b9..2975ea0841ba4 100644 --- a/arch/powerpc/mm/book3s64/pgtable.c +++ b/arch/powerpc/mm/book3s64/pgtable.c @@ -170,6 +170,7 @@ pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, { unsigned long old_pmd; + VM_WARN_ON_ONCE(!pmd_present(*pmdp)); old_pmd = pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT, _PAGE_INVALID); flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); return __pmd(old_pmd); diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 2cb2a2e7b34b8..558902edbfece 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -1769,8 +1769,10 @@ static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, static inline pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp) { - pmd_t pmd = __pmd(pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID); + pmd_t pmd; + VM_WARN_ON_ONCE(!pmd_present(*pmdp)); + pmd = __pmd(pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID); return pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd); } diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c index 19642f7ffb52a..8648a50afe889 100644 --- a/arch/sparc/mm/tlb.c +++ b/arch/sparc/mm/tlb.c @@ -249,6 +249,7 @@ pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, { pmd_t old, entry; + VM_WARN_ON_ONCE(!pmd_present(*pmdp)); entry = __pmd(pmd_val(*pmdp) & ~_PAGE_VALID); old = pmdp_establish(vma, address, pmdp, entry); flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index 94767c82fc0d7..93e54ba91fbfd 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -631,6 +631,8 @@ int pmdp_clear_flush_young(struct vm_area_struct *vma, pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) { + VM_WARN_ON_ONCE(!pmd_present(*pmdp)); + /* * No flush is necessary. Once an invalid PTE is established, the PTE's * access and dirty bits cannot be updated. diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 08e4f3343bcd0..ccdcff73284a0 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2430,32 +2430,11 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, return __split_huge_zero_page_pmd(vma, haddr, pmd); } - /* - * Up to this point the pmd is present and huge and userland has the - * whole access to the hugepage during the split (which happens in - * place). If we overwrite the pmd with the not-huge version pointing - * to the pte here (which of course we could if all CPUs were bug - * free), userland could trigger a small page size TLB miss on the - * small sized TLB while the hugepage TLB entry is still established in - * the huge TLB. Some CPU doesn't like that. - * See http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum - * 383 on page 105. Intel should be safe but is also warns that it's - * only safe if the permission and cache attributes of the two entries - * loaded in the two TLB is identical (which should be the case here). - * But it is generally safer to never allow small and huge TLB entries - * for the same virtual address to be loaded simultaneously. So instead - * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the - * current pmd notpresent (atomically because here the pmd_trans_huge - * must remain set at all times on the pmd until the split is complete - * for this pmd), then we flush the SMP TLB and finally we write the - * non-huge version of the pmd entry with pmd_populate. - */ - old_pmd = pmdp_invalidate(vma, haddr, pmd); - - pmd_migration = is_pmd_migration_entry(old_pmd); + pmd_migration = is_pmd_migration_entry(*pmd); if (unlikely(pmd_migration)) { swp_entry_t entry; + old_pmd = *pmd; entry = pmd_to_swp_entry(old_pmd); page = pfn_swap_entry_to_page(entry); write = is_writable_migration_entry(entry); @@ -2466,6 +2445,30 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, soft_dirty = pmd_swp_soft_dirty(old_pmd); uffd_wp = pmd_swp_uffd_wp(old_pmd); } else { + /* + * Up to this point the pmd is present and huge and userland has + * the whole access to the hugepage during the split (which + * happens in place). If we overwrite the pmd with the not-huge + * version pointing to the pte here (which of course we could if + * all CPUs were bug free), userland could trigger a small page + * size TLB miss on the small sized TLB while the hugepage TLB + * entry is still established in the huge TLB. Some CPU doesn't + * like that. See + * http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum + * 383 on page 105. Intel should be safe but is also warns that + * it's only safe if the permission and cache attributes of the + * two entries loaded in the two TLB is identical (which should + * be the case here). But it is generally safer to never allow + * small and huge TLB entries for the same virtual address to be + * loaded simultaneously. So instead of doing "pmd_populate(); + * flush_pmd_tlb_range();" we first mark the current pmd + * notpresent (atomically because here the pmd_trans_huge must + * remain set at all times on the pmd until the split is + * complete for this pmd), then we flush the SMP TLB and finally + * we write the non-huge version of the pmd entry with + * pmd_populate. + */ + old_pmd = pmdp_invalidate(vma, haddr, pmd); page = pmd_page(old_pmd); folio = page_folio(page); if (pmd_dirty(old_pmd)) { diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index 4fcd959dcc4d0..a78a4adf711ac 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -198,6 +198,7 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) { + VM_WARN_ON_ONCE(!pmd_present(*pmdp)); pmd_t old = pmdp_establish(vma, address, pmdp, pmd_mkinvalid(*pmdp)); flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); return old; @@ -208,6 +209,7 @@ pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) { + VM_WARN_ON_ONCE(!pmd_present(*pmdp)); return pmdp_invalidate(vma, address, pmdp); } #endif From 1872b3bcd5874b57ed8c89cae0a2882f484c3885 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 1 May 2024 02:54:20 -0700 Subject: [PATCH 1295/1702] mm: memcg: use READ_ONCE()/WRITE_ONCE() to access stock->nr_pages A memcg pointer in the per-cpu stock can be accessed by drain_all_stock() and consume_stock() in parallel, causing a potential race, which is believed to e harmless. KCSAN shows this data-race clearly in the splat below: BUG: KCSAN: data-race in drain_all_stock.part.0 / try_charge_memcg write to 0xffff88903f8b0788 of 4 bytes by task 35901 on cpu 2: try_charge_memcg (mm/memcontrol.c:2323 mm/memcontrol.c:2746) __mem_cgroup_charge (mm/memcontrol.c:7287 mm/memcontrol.c:7301) do_anonymous_page (mm/memory.c:1054 mm/memory.c:4375 mm/memory.c:4433) __handle_mm_fault (mm/memory.c:3878 mm/memory.c:5300 mm/memory.c:5441) handle_mm_fault (mm/memory.c:5606) do_user_addr_fault (arch/x86/mm/fault.c:1363) exc_page_fault (./arch/x86/include/asm/irqflags.h:37 ./arch/x86/include/asm/irqflags.h:72 arch/x86/mm/fault.c:1513 arch/x86/mm/fault.c:1563) asm_exc_page_fault (./arch/x86/include/asm/idtentry.h:623) read to 0xffff88903f8b0788 of 4 bytes by task 287 on cpu 27: drain_all_stock.part.0 (mm/memcontrol.c:2433) mem_cgroup_css_offline (mm/memcontrol.c:5398 mm/memcontrol.c:5687) css_killed_work_fn (kernel/cgroup/cgroup.c:5521 kernel/cgroup/cgroup.c:5794) process_one_work (kernel/workqueue.c:3254) worker_thread (kernel/workqueue.c:3329 kernel/workqueue.c:3416) kthread (kernel/kthread.c:388) ret_from_fork (arch/x86/kernel/process.c:147) ret_from_fork_asm (arch/x86/entry/entry_64.S:257) value changed: 0x00000014 -> 0x00000013 This happens because drain_all_stock() is reading stock->nr_pages, while consume_stock() might be updating the same address, causing a potential data-race. Make the shared addresses bulletproof regarding to reads and writes, similarly to what stock->cached_objcg and stock->cached. Annotate all accesses to stock->nr_pages with READ_ONCE()/WRITE_ONCE(). Link: https://lkml.kernel.org/r/20240501095420.679208-1-leitao@debian.org Signed-off-by: Breno Leitao Acked-by: Shakeel Butt Reviewed-by: Roman Gushchin Acked-by: Michal Hocko Cc: Johannes Weiner Cc: Muchun Song Signed-off-by: Andrew Morton --- mm/memcontrol.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index e6302f42e40f1..2103ae77cf8d4 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2468,6 +2468,7 @@ static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages) static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) { struct memcg_stock_pcp *stock; + unsigned int stock_pages; unsigned long flags; bool ret = false; @@ -2477,8 +2478,9 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) local_lock_irqsave(&memcg_stock.stock_lock, flags); stock = this_cpu_ptr(&memcg_stock); - if (memcg == READ_ONCE(stock->cached) && stock->nr_pages >= nr_pages) { - stock->nr_pages -= nr_pages; + stock_pages = READ_ONCE(stock->nr_pages); + if (memcg == READ_ONCE(stock->cached) && stock_pages >= nr_pages) { + WRITE_ONCE(stock->nr_pages, stock_pages - nr_pages); ret = true; } @@ -2492,16 +2494,18 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) */ static void drain_stock(struct memcg_stock_pcp *stock) { + unsigned int stock_pages = READ_ONCE(stock->nr_pages); struct mem_cgroup *old = READ_ONCE(stock->cached); if (!old) return; - if (stock->nr_pages) { - page_counter_uncharge(&old->memory, stock->nr_pages); + if (stock_pages) { + page_counter_uncharge(&old->memory, stock_pages); if (do_memsw_account()) - page_counter_uncharge(&old->memsw, stock->nr_pages); - stock->nr_pages = 0; + page_counter_uncharge(&old->memsw, stock_pages); + + WRITE_ONCE(stock->nr_pages, 0); } css_put(&old->css); @@ -2537,6 +2541,7 @@ static void drain_local_stock(struct work_struct *dummy) static void __refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) { struct memcg_stock_pcp *stock; + unsigned int stock_pages; stock = this_cpu_ptr(&memcg_stock); if (READ_ONCE(stock->cached) != memcg) { /* reset if necessary */ @@ -2544,9 +2549,10 @@ static void __refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) css_get(&memcg->css); WRITE_ONCE(stock->cached, memcg); } - stock->nr_pages += nr_pages; + stock_pages = READ_ONCE(stock->nr_pages) + nr_pages; + WRITE_ONCE(stock->nr_pages, stock_pages); - if (stock->nr_pages > MEMCG_CHARGE_BATCH) + if (stock_pages > MEMCG_CHARGE_BATCH) drain_stock(stock); } @@ -2585,7 +2591,7 @@ static void drain_all_stock(struct mem_cgroup *root_memcg) rcu_read_lock(); memcg = READ_ONCE(stock->cached); - if (memcg && stock->nr_pages && + if (memcg && READ_ONCE(stock->nr_pages) && mem_cgroup_is_descendant(memcg, root_memcg)) flush = true; else if (obj_stock_flush_required(stock, root_memcg)) From c14c647bbe23fd96f6bffcc122b9c6c8c46c7928 Mon Sep 17 00:00:00 2001 From: Vishal Verma Date: Tue, 30 Apr 2024 11:44:23 -0600 Subject: [PATCH 1296/1702] dax/bus.c: replace WARN_ON_ONCE() with lockdep asserts Patch series "dax/bus.c: Fixups for dax-bus locking", v3. Commit Fixes: c05ae9d85b47 ("dax/bus.c: replace driver-core lock usage by a local rwsem") introduced a few problems that this series aims to fix. Add back device_lock() where it was correctly used (during device manipulation operations), remove conditional locking in unregister_dax_dev() and unregister_dax_mapping(), use non-interruptible versions of rwsem locks when not called from a user process, and fix up a write vs. read usage of an rwsem. This patch (of 4): In [1], Dan points out that all of the WARN_ON_ONCE() usage in the referenced patch should be replaced with lockdep_assert_held, or lockdep_held_assert_write(). Replace these as appropriate. Link: https://lkml.kernel.org/r/20240430-vv-dax_abi_fixes-v3-0-e3dcd755774c@intel.com Link: https://lore.kernel.org/r/65f0b5ef41817_aa222941a@dwillia2-mobl3.amr.corp.intel.com.notmuch [1] Link: https://lkml.kernel.org/r/20240430-vv-dax_abi_fixes-v3-1-e3dcd755774c@intel.com Fixes: c05ae9d85b47 ("dax/bus.c: replace driver-core lock usage by a local rwsem") Signed-off-by: Vishal Verma Reported-by: Dan Williams Reviewed-by: Dan Williams Cc: Alison Schofield Cc: Dave Jiang Cc: Vishal Verma Signed-off-by: Andrew Morton --- drivers/dax/bus.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c index 797e1ebff2997..7924dd542a139 100644 --- a/drivers/dax/bus.c +++ b/drivers/dax/bus.c @@ -192,7 +192,7 @@ static u64 dev_dax_size(struct dev_dax *dev_dax) u64 size = 0; int i; - WARN_ON_ONCE(!rwsem_is_locked(&dax_dev_rwsem)); + lockdep_assert_held(&dax_dev_rwsem); for (i = 0; i < dev_dax->nr_range; i++) size += range_len(&dev_dax->ranges[i].range); @@ -302,7 +302,7 @@ static unsigned long long dax_region_avail_size(struct dax_region *dax_region) resource_size_t size = resource_size(&dax_region->res); struct resource *res; - WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem)); + lockdep_assert_held(&dax_region_rwsem); for_each_dax_region_resource(dax_region, res) size -= resource_size(res); @@ -447,7 +447,7 @@ static void trim_dev_dax_range(struct dev_dax *dev_dax) struct range *range = &dev_dax->ranges[i].range; struct dax_region *dax_region = dev_dax->region; - WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem)); + lockdep_assert_held_write(&dax_region_rwsem); dev_dbg(&dev_dax->dev, "delete range[%d]: %#llx:%#llx\n", i, (unsigned long long)range->start, (unsigned long long)range->end); @@ -507,7 +507,7 @@ static int __free_dev_dax_id(struct dev_dax *dev_dax) struct dax_region *dax_region; int rc = dev_dax->id; - WARN_ON_ONCE(!rwsem_is_locked(&dax_dev_rwsem)); + lockdep_assert_held_write(&dax_dev_rwsem); if (!dev_dax->dyn_id || dev_dax->id < 0) return -1; @@ -713,7 +713,7 @@ static void __unregister_dax_mapping(void *data) dev_dbg(dev, "%s\n", __func__); - WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem)); + lockdep_assert_held_write(&dax_region_rwsem); dev_dax->ranges[mapping->range_id].mapping = NULL; mapping->range_id = -1; @@ -830,7 +830,7 @@ static int devm_register_dax_mapping(struct dev_dax *dev_dax, int range_id) struct device *dev; int rc; - WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem)); + lockdep_assert_held_write(&dax_region_rwsem); if (dev_WARN_ONCE(&dev_dax->dev, !dax_region->dev->driver, "region disabled\n")) @@ -876,7 +876,7 @@ static int alloc_dev_dax_range(struct dev_dax *dev_dax, u64 start, struct resource *alloc; int i, rc; - WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem)); + lockdep_assert_held_write(&dax_region_rwsem); /* handle the seed alloc special case */ if (!size) { @@ -935,7 +935,7 @@ static int adjust_dev_dax_range(struct dev_dax *dev_dax, struct resource *res, r struct device *dev = &dev_dax->dev; int rc; - WARN_ON_ONCE(!rwsem_is_locked(&dax_region_rwsem)); + lockdep_assert_held_write(&dax_region_rwsem); if (dev_WARN_ONCE(dev, !size, "deletion is handled by dev_dax_shrink\n")) return -EINVAL; From 6f6544f27e41f9d7dca55c288f12175a9c48dfe2 Mon Sep 17 00:00:00 2001 From: Vishal Verma Date: Tue, 30 Apr 2024 11:44:24 -0600 Subject: [PATCH 1297/1702] dax/bus.c: fix locking for unregister_dax_dev / unregister_dax_mapping paths Commit c05ae9d85b47 ("dax/bus.c: replace driver-core lock usage by a local rwsem") aimed to undo device_lock() abuses for protecting changes to dax-driver internal data-structures like the dax_region resource tree to device-dax-instance range structures. However, the device_lock() was legitimately enforcing that devices to be deleted were not current actively attached to any driver nor assigned any capacity from the region. As a result of the device_lock restoration in delete_store(), the conditional locking in unregister_dev_dax() and unregister_dax_mapping() can be removed. Link: https://lkml.kernel.org/r/20240430-vv-dax_abi_fixes-v3-2-e3dcd755774c@intel.com Fixes: c05ae9d85b47 ("dax/bus.c: replace driver-core lock usage by a local rwsem") Signed-off-by: Vishal Verma Reported-by: Dan Williams Reviewed-by: Dan Williams Cc: Alison Schofield Cc: Dave Jiang Signed-off-by: Andrew Morton --- drivers/dax/bus.c | 42 ++++++++---------------------------------- 1 file changed, 8 insertions(+), 34 deletions(-) diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c index 7924dd542a139..e2c7354ce3281 100644 --- a/drivers/dax/bus.c +++ b/drivers/dax/bus.c @@ -465,26 +465,17 @@ static void free_dev_dax_ranges(struct dev_dax *dev_dax) trim_dev_dax_range(dev_dax); } -static void __unregister_dev_dax(void *dev) +static void unregister_dev_dax(void *dev) { struct dev_dax *dev_dax = to_dev_dax(dev); dev_dbg(dev, "%s\n", __func__); + down_write(&dax_region_rwsem); kill_dev_dax(dev_dax); device_del(dev); free_dev_dax_ranges(dev_dax); put_device(dev); -} - -static void unregister_dev_dax(void *dev) -{ - if (rwsem_is_locked(&dax_region_rwsem)) - return __unregister_dev_dax(dev); - - if (WARN_ON_ONCE(down_write_killable(&dax_region_rwsem) != 0)) - return; - __unregister_dev_dax(dev); up_write(&dax_region_rwsem); } @@ -560,15 +551,10 @@ static ssize_t delete_store(struct device *dev, struct device_attribute *attr, if (!victim) return -ENXIO; - rc = down_write_killable(&dax_region_rwsem); - if (rc) - return rc; - rc = down_write_killable(&dax_dev_rwsem); - if (rc) { - up_write(&dax_region_rwsem); - return rc; - } + device_lock(dev); + device_lock(victim); dev_dax = to_dev_dax(victim); + down_write(&dax_dev_rwsem); if (victim->driver || dev_dax_size(dev_dax)) rc = -EBUSY; else { @@ -589,11 +575,12 @@ static ssize_t delete_store(struct device *dev, struct device_attribute *attr, rc = -EBUSY; } up_write(&dax_dev_rwsem); + device_unlock(victim); /* won the race to invalidate the device, clean it up */ if (do_del) devm_release_action(dev, unregister_dev_dax, victim); - up_write(&dax_region_rwsem); + device_unlock(dev); put_device(victim); return rc; @@ -705,7 +692,7 @@ static void dax_mapping_release(struct device *dev) put_device(parent); } -static void __unregister_dax_mapping(void *data) +static void unregister_dax_mapping(void *data) { struct device *dev = data; struct dax_mapping *mapping = to_dax_mapping(dev); @@ -713,25 +700,12 @@ static void __unregister_dax_mapping(void *data) dev_dbg(dev, "%s\n", __func__); - lockdep_assert_held_write(&dax_region_rwsem); - dev_dax->ranges[mapping->range_id].mapping = NULL; mapping->range_id = -1; device_unregister(dev); } -static void unregister_dax_mapping(void *data) -{ - if (rwsem_is_locked(&dax_region_rwsem)) - return __unregister_dax_mapping(data); - - if (WARN_ON_ONCE(down_write_killable(&dax_region_rwsem) != 0)) - return; - __unregister_dax_mapping(data); - up_write(&dax_region_rwsem); -} - static struct dev_dax_range *get_dax_range(struct device *dev) { struct dax_mapping *mapping = to_dax_mapping(dev); From e39dbcfba714c4c2e924e96fc8fdde1080a5a737 Mon Sep 17 00:00:00 2001 From: Vishal Verma Date: Tue, 30 Apr 2024 11:44:25 -0600 Subject: [PATCH 1298/1702] dax/bus.c: don't use down_write_killable for non-user processes Change an instance of down_write_killable() to a simple down_write() where there is no user process that might want to interrupt the operation. Link: https://lkml.kernel.org/r/20240430-vv-dax_abi_fixes-v3-3-e3dcd755774c@intel.com Fixes: c05ae9d85b47 ("dax/bus.c: replace driver-core lock usage by a local rwsem") Signed-off-by: Vishal Verma Reported-by: Dan Williams Reviewed-by: Dan Williams Cc: Alison Schofield Cc: Dave Jiang Signed-off-by: Andrew Morton --- drivers/dax/bus.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c index e2c7354ce3281..0011a6e6a8f2a 100644 --- a/drivers/dax/bus.c +++ b/drivers/dax/bus.c @@ -1540,12 +1540,8 @@ static struct dev_dax *__devm_create_dev_dax(struct dev_dax_data *data) struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data) { struct dev_dax *dev_dax; - int rc; - - rc = down_write_killable(&dax_region_rwsem); - if (rc) - return ERR_PTR(rc); + down_write(&dax_region_rwsem); dev_dax = __devm_create_dev_dax(data); up_write(&dax_region_rwsem); From 2acf04532d6d655d8c3b2ee4ddeb320107043086 Mon Sep 17 00:00:00 2001 From: Vishal Verma Date: Tue, 30 Apr 2024 11:44:26 -0600 Subject: [PATCH 1299/1702] dax/bus.c: use the right locking mode (read vs write) in size_show In size_show(), the dax_dev_rwsem only needs a read lock, but was acquiring a write lock. Change it to down_read_interruptible() so it doesn't unnecessarily hold a write lock. Link: https://lkml.kernel.org/r/20240430-vv-dax_abi_fixes-v3-4-e3dcd755774c@intel.com Fixes: c05ae9d85b47 ("dax/bus.c: replace driver-core lock usage by a local rwsem") Signed-off-by: Vishal Verma Reviewed-by: Dan Williams Cc: Alison Schofield Cc: Dave Jiang Signed-off-by: Andrew Morton --- drivers/dax/bus.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c index 0011a6e6a8f2a..f24b67c64d5ec 100644 --- a/drivers/dax/bus.c +++ b/drivers/dax/bus.c @@ -937,11 +937,11 @@ static ssize_t size_show(struct device *dev, unsigned long long size; int rc; - rc = down_write_killable(&dax_dev_rwsem); + rc = down_read_interruptible(&dax_dev_rwsem); if (rc) return rc; size = dev_dax_size(dev_dax); - up_write(&dax_dev_rwsem); + up_read(&dax_dev_rwsem); return sysfs_emit(buf, "%llu\n", size); } From cc48be374b654e1533c40cd64ebc4e4b0a637317 Mon Sep 17 00:00:00 2001 From: Frank van der Linden Date: Tue, 30 Apr 2024 16:14:37 +0000 Subject: [PATCH 1300/1702] mm/hugetlb: align cma on allocation order, not demotion order Align the CMA area for hugetlb gigantic pages to their size, not the size that they can be demoted to. Otherwise there might be misaligned sections at the start and end of the CMA area that will never be used for hugetlb page allocations. Link: https://lkml.kernel.org/r/20240430161437.2100295-1-fvdl@google.com Fixes: a01f43901cfb ("hugetlb: be sure to free demoted CMA pages to CMA") Signed-off-by: Frank van der Linden Reviewed-by: David Hildenbrand Reviewed-by: Roman Gushchin Cc: Muchun Song Signed-off-by: Andrew Morton --- mm/hugetlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index a7efb350f5d07..33d175add0447 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -7800,7 +7800,7 @@ void __init hugetlb_cma_reserve(int order) * huge page demotion. */ res = cma_declare_contiguous_nid(0, size, 0, - PAGE_SIZE << HUGETLB_PAGE_ORDER, + PAGE_SIZE << order, HUGETLB_PAGE_ORDER, false, name, &hugetlb_cma[nid], nid); if (res) { From 67f4c91a449a6cc936679e43a916d79ee134464a Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Tue, 30 Apr 2024 15:15:08 +0200 Subject: [PATCH 1301/1702] selftests: mm: gup_longterm: test unsharing logic when R/O pinning In our FOLL_LONGTERM tests, we prefault the page tables for the GUP-fast test cases to be able to find a PTE and exercise the "longterm pinning allowed" logic on the GUP-fast path where possible. For now, we always prefault the page tables writable, resulting in PTEs that are writable. Let's cover more cases to also test if our unsharing logic works as expected (and is able to make progress when there is nothing to unshare) by mprotect'ing the range R/O when R/O-pinning, so we don't get PTEs that are writable. This change would have found an issue introduced by commit a12083d721d7 ("mm/gup: handle hugepd for follow_page()"), whereby R/O pinning was not able to make progress in all cases, because unsharing logic was not provided with the VMA to decide at some point that long-term R/O pinning a !anon page is fine. Link: https://lkml.kernel.org/r/20240430131508.86924-1-david@redhat.com Signed-off-by: David Hildenbrand Acked-by: Peter Xu Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/gup_longterm.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/mm/gup_longterm.c b/tools/testing/selftests/mm/gup_longterm.c index ad168d35b23b7..6c6d99c1dc765 100644 --- a/tools/testing/selftests/mm/gup_longterm.c +++ b/tools/testing/selftests/mm/gup_longterm.c @@ -118,15 +118,22 @@ static void do_test(int fd, size_t size, enum test_type type, bool shared) return; } - /* - * Fault in the page writable such that GUP-fast can eventually pin - * it immediately. - */ + /* Fault in the page such that GUP-fast can pin it directly. */ memset(mem, 0, size); switch (type) { case TEST_TYPE_RO: case TEST_TYPE_RO_FAST: + /* + * Cover more cases regarding unsharing decisions when + * long-term R/O pinning by mapping the page R/O. + */ + ret = mprotect(mem, size, PROT_READ); + if (ret) { + ksft_test_result_fail("mprotect() failed\n"); + goto munmap; + } + /* FALLTHROUGH */ case TEST_TYPE_RW: case TEST_TYPE_RW_FAST: { struct pin_longterm_test args; @@ -228,6 +235,7 @@ static void do_test(int fd, size_t size, enum test_type type, bool shared) assert(false); } +munmap: munmap(mem, size); } From 01d89b93e1769928529d339d6e4f780129454161 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Tue, 30 Apr 2024 09:13:03 -0400 Subject: [PATCH 1302/1702] mm/gup: fix hugepd handling in hugetlb rework Commit a12083d721d7 added hugepd handling for gup-slow, reusing gup-fast functions. follow_hugepd() correctly took the vma pointer in, however didn't pass it over into the lower functions, which was overlooked. The issue is gup_fast_hugepte() uses the vma pointer to make the correct decision on whether an unshare is needed for a FOLL_PIN|FOLL_LONGTERM. Now without vma ponter it will constantly return "true" (needs an unshare) for a page cache, even though in the SHARED case it will be wrong to unshare. The other problem is, even if an unshare is needed, it now returns 0 rather than -EMLINK, which will not trigger a follow up FAULT_FLAG_UNSHARE fault. That will need to be fixed too when the unshare is wanted. gup_longterm test didn't expose this issue in the past because it didn't yet test R/O unshare in this case, another separate patch will enable that in future tests. Fix it by passing vma correctly to the bottom, rename gup_fast_hugepte() back to gup_hugepte() as it is shared between the fast/slow paths, and also allow -EMLINK to be returned properly by gup_hugepte() even though gup-fast will take it the same as zero. Link: https://lkml.kernel.org/r/20240430131303.264331-1-peterx@redhat.com Fixes: a12083d721d7 ("mm/gup: handle hugepd for follow_page()") Signed-off-by: Peter Xu Reported-by: David Hildenbrand Reviewed-by: David Hildenbrand Cc: Aneesh Kumar K.V Cc: Christophe Leroy Cc: Jason Gunthorpe Cc: John Hubbard Cc: Lorenzo Stoakes Cc: Muchun Song Signed-off-by: Andrew Morton --- mm/gup.c | 64 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index 2f7baf96f6559..ca0f5cedce9b2 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -525,9 +525,17 @@ static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end, return (__boundary - 1 < end - 1) ? __boundary : end; } -static int gup_fast_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, - unsigned long end, unsigned int flags, struct page **pages, - int *nr) +/* + * Returns 1 if succeeded, 0 if failed, -EMLINK if unshare needed. + * + * NOTE: for the same entry, gup-fast and gup-slow can return different + * results (0 v.s. -EMLINK) depending on whether vma is available. This is + * the expected behavior, where we simply want gup-fast to fallback to + * gup-slow to take the vma reference first. + */ +static int gup_hugepte(struct vm_area_struct *vma, pte_t *ptep, unsigned long sz, + unsigned long addr, unsigned long end, unsigned int flags, + struct page **pages, int *nr) { unsigned long pte_end; struct page *page; @@ -559,9 +567,9 @@ static int gup_fast_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, return 0; } - if (!pte_write(pte) && gup_must_unshare(NULL, flags, &folio->page)) { + if (!pte_write(pte) && gup_must_unshare(vma, flags, &folio->page)) { gup_put_folio(folio, refs, flags); - return 0; + return -EMLINK; } *nr += refs; @@ -577,19 +585,22 @@ static int gup_fast_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, * of the other folios. See writable_file_mapping_allowed() and * gup_fast_folio_allowed() for more information. */ -static int gup_fast_hugepd(hugepd_t hugepd, unsigned long addr, - unsigned int pdshift, unsigned long end, unsigned int flags, - struct page **pages, int *nr) +static int gup_hugepd(struct vm_area_struct *vma, hugepd_t hugepd, + unsigned long addr, unsigned int pdshift, + unsigned long end, unsigned int flags, + struct page **pages, int *nr) { pte_t *ptep; unsigned long sz = 1UL << hugepd_shift(hugepd); unsigned long next; + int ret; ptep = hugepte_offset(hugepd, addr, pdshift); do { next = hugepte_addr_end(addr, end, sz); - if (!gup_fast_hugepte(ptep, sz, addr, end, flags, pages, nr)) - return 0; + ret = gup_hugepte(vma, ptep, sz, addr, end, flags, pages, nr); + if (ret != 1) + return ret; } while (ptep++, addr = next, addr != end); return 1; @@ -613,22 +624,25 @@ static struct page *follow_hugepd(struct vm_area_struct *vma, hugepd_t hugepd, h = hstate_vma(vma); ptep = hugepte_offset(hugepd, addr, pdshift); ptl = huge_pte_lock(h, vma->vm_mm, ptep); - ret = gup_fast_hugepd(hugepd, addr, pdshift, addr + PAGE_SIZE, - flags, &page, &nr); + ret = gup_hugepd(vma, hugepd, addr, pdshift, addr + PAGE_SIZE, + flags, &page, &nr); spin_unlock(ptl); - if (ret) { + if (ret == 1) { + /* GUP succeeded */ WARN_ON_ONCE(nr != 1); ctx->page_mask = (1U << huge_page_order(h)) - 1; return page; } - return NULL; + /* ret can be either 0 (translates to NULL) or negative */ + return ERR_PTR(ret); } #else /* CONFIG_ARCH_HAS_HUGEPD */ -static inline int gup_fast_hugepd(hugepd_t hugepd, unsigned long addr, - unsigned int pdshift, unsigned long end, unsigned int flags, - struct page **pages, int *nr) +static inline int gup_hugepd(struct vm_area_struct *vma, hugepd_t hugepd, + unsigned long addr, unsigned int pdshift, + unsigned long end, unsigned int flags, + struct page **pages, int *nr) { return 0; } @@ -3261,8 +3275,8 @@ static int gup_fast_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, * architecture have different format for hugetlbfs * pmd format and THP pmd format */ - if (!gup_fast_hugepd(__hugepd(pmd_val(pmd)), addr, - PMD_SHIFT, next, flags, pages, nr)) + if (gup_hugepd(NULL, __hugepd(pmd_val(pmd)), addr, + PMD_SHIFT, next, flags, pages, nr) != 1) return 0; } else if (!gup_fast_pte_range(pmd, pmdp, addr, next, flags, pages, nr)) @@ -3291,8 +3305,8 @@ static int gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, pages, nr)) return 0; } else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) { - if (!gup_fast_hugepd(__hugepd(pud_val(pud)), addr, - PUD_SHIFT, next, flags, pages, nr)) + if (gup_hugepd(NULL, __hugepd(pud_val(pud)), addr, + PUD_SHIFT, next, flags, pages, nr) != 1) return 0; } else if (!gup_fast_pmd_range(pudp, pud, addr, next, flags, pages, nr)) @@ -3318,8 +3332,8 @@ static int gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, return 0; BUILD_BUG_ON(p4d_leaf(p4d)); if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) { - if (!gup_fast_hugepd(__hugepd(p4d_val(p4d)), addr, - P4D_SHIFT, next, flags, pages, nr)) + if (gup_hugepd(NULL, __hugepd(p4d_val(p4d)), addr, + P4D_SHIFT, next, flags, pages, nr) != 1) return 0; } else if (!gup_fast_pud_range(p4dp, p4d, addr, next, flags, pages, nr)) @@ -3347,8 +3361,8 @@ static void gup_fast_pgd_range(unsigned long addr, unsigned long end, pages, nr)) return; } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) { - if (!gup_fast_hugepd(__hugepd(pgd_val(pgd)), addr, - PGDIR_SHIFT, next, flags, pages, nr)) + if (gup_hugepd(NULL, __hugepd(pgd_val(pgd)), addr, + PGDIR_SHIFT, next, flags, pages, nr) != 1) return; } else if (!gup_fast_p4d_range(pgdp, pgd, addr, next, flags, pages, nr)) From 69a5f999176d10b2618db00b12e661137b42f5f2 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 29 Apr 2024 15:44:48 -0700 Subject: [PATCH 1303/1702] mm/damon/paddr: avoid unnecessary page level access check for pageout DAMOS action Patch series "mm/damon/paddr: simplify page level access re-check for pageout. The 'pageout' DAMOS action implementation of 'paddr' asks reclaim_pages() to do page level access check again. But the user can ask 'paddr' to do the page level access check on its own, using DAMOS filter of 'young page' type. Meanwhile, 'paddr' is the only user of reclaim_pages() that asks the page level access check. Make 'paddr' does the page level access check on its own always, and simplify reclaim_pages() by removing the page level access check request handling logic. As a result of the change for reclaim_pages(), reclaim_folio_list(), which is called by reclaim_pages(), also no more need to do the page level access check. Simplify the function, too. This patch (of 4): 'pageout' DAMOS action implementation of 'paddr' asks reclaim_pages() to do the page level access check. User could ask DAMOS to do the page level access check on its own using 'young page' type DAMOS filter. In the case, pageout DAMOS action unnecessarily asks reclaim_pages() to do the check again. Ask the page level access check only if the scheme is not having the filter. Link: https://lkml.kernel.org/r/20240429224451.67081-1-sj@kernel.org Link: https://lkml.kernel.org/r/20240429224451.67081-2-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton --- mm/damon/paddr.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 5685ba485097d..d5f2f7ddf8638 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -244,6 +244,16 @@ static unsigned long damon_pa_pageout(struct damon_region *r, struct damos *s) { unsigned long addr, applied; LIST_HEAD(folio_list); + bool ignore_references = false; + struct damos_filter *filter; + + /* respect user's page level reference check handling request */ + damos_for_each_filter(filter, s) { + if (filter->type == DAMOS_FILTER_TYPE_YOUNG) { + ignore_references = true; + break; + } + } for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) { struct folio *folio = damon_get_folio(PHYS_PFN(addr)); @@ -265,7 +275,7 @@ static unsigned long damon_pa_pageout(struct damon_region *r, struct damos *s) put_folio: folio_put(folio); } - applied = reclaim_pages(&folio_list, false); + applied = reclaim_pages(&folio_list, ignore_references); cond_resched(); return applied * PAGE_SIZE; } From ebd3f70c630a9a1aa1e27a63a7e0ffe97a0c1189 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 29 Apr 2024 15:44:49 -0700 Subject: [PATCH 1304/1702] mm/damon/paddr: do page level access check for pageout DAMOS action on its own 'pageout' DAMOS action implementation of 'paddr' DAMON operations set asks reclaim_pages() to do page level access check if the user is not asking DAMOS to do that on its own. Simplify the logic by making the check always be done by 'paddr'. Link: https://lkml.kernel.org/r/20240429224451.67081-3-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton --- mm/damon/paddr.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index d5f2f7ddf8638..974edef1740d0 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -244,16 +244,22 @@ static unsigned long damon_pa_pageout(struct damon_region *r, struct damos *s) { unsigned long addr, applied; LIST_HEAD(folio_list); - bool ignore_references = false; + bool install_young_filter = true; struct damos_filter *filter; - /* respect user's page level reference check handling request */ + /* check access in page level again by default */ damos_for_each_filter(filter, s) { if (filter->type == DAMOS_FILTER_TYPE_YOUNG) { - ignore_references = true; + install_young_filter = false; break; } } + if (install_young_filter) { + filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, true); + if (!filter) + return 0; + damos_add_filter(s, filter); + } for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) { struct folio *folio = damon_get_folio(PHYS_PFN(addr)); @@ -275,7 +281,9 @@ static unsigned long damon_pa_pageout(struct damon_region *r, struct damos *s) put_folio: folio_put(folio); } - applied = reclaim_pages(&folio_list, ignore_references); + if (install_young_filter) + damos_destroy_filter(filter); + applied = reclaim_pages(&folio_list, true); cond_resched(); return applied * PAGE_SIZE; } From 14f5be2a2d9bb7eb21807b6e62de73dd24082b73 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 29 Apr 2024 15:44:50 -0700 Subject: [PATCH 1305/1702] mm/vmscan: remove ignore_references argument of reclaim_pages() All reclaim_pages() callers are setting 'ignore_references' parameter 'true'. In other words, the parameter is not really being used. Remove the argument to make it simple. Link: https://lkml.kernel.org/r/20240429224451.67081-4-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton --- mm/damon/paddr.c | 2 +- mm/internal.h | 2 +- mm/madvise.c | 4 ++-- mm/vmscan.c | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 974edef1740d0..18797c1b419bf 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -283,7 +283,7 @@ static unsigned long damon_pa_pageout(struct damon_region *r, struct damos *s) } if (install_young_filter) damos_destroy_filter(filter); - applied = reclaim_pages(&folio_list, true); + applied = reclaim_pages(&folio_list); cond_resched(); return applied * PAGE_SIZE; } diff --git a/mm/internal.h b/mm/internal.h index 6803c7b17c1f3..2adabe3694036 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1052,7 +1052,7 @@ extern unsigned long __must_check vm_mmap_pgoff(struct file *, unsigned long, unsigned long, unsigned long); extern void set_pageblock_order(void); -unsigned long reclaim_pages(struct list_head *folio_list, bool ignore_references); +unsigned long reclaim_pages(struct list_head *folio_list); unsigned int reclaim_clean_pages_from_list(struct zone *zone, struct list_head *folio_list); /* The ALLOC_WMARK bits are used as an index to zone->watermark */ diff --git a/mm/madvise.c b/mm/madvise.c index 56efea02e26c3..c8ba3f3eb54d7 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -423,7 +423,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, huge_unlock: spin_unlock(ptl); if (pageout) - reclaim_pages(&folio_list, true); + reclaim_pages(&folio_list); return 0; } @@ -547,7 +547,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, pte_unmap_unlock(start_pte, ptl); } if (pageout) - reclaim_pages(&folio_list, true); + reclaim_pages(&folio_list); cond_resched(); return 0; diff --git a/mm/vmscan.c b/mm/vmscan.c index d194e7240df44..fe923a4a56bbb 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2133,7 +2133,7 @@ static unsigned int reclaim_folio_list(struct list_head *folio_list, return nr_reclaimed; } -unsigned long reclaim_pages(struct list_head *folio_list, bool ignore_references) +unsigned long reclaim_pages(struct list_head *folio_list) { int nid; unsigned int nr_reclaimed = 0; @@ -2156,11 +2156,11 @@ unsigned long reclaim_pages(struct list_head *folio_list, bool ignore_references } nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid), - ignore_references); + true); nid = folio_nid(lru_to_folio(folio_list)); } while (!list_empty(folio_list)); - nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid), ignore_references); + nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid), true); memalloc_noreclaim_restore(noreclaim_flag); From c961bddb7df086a557dfcd8c1846f49e75d62888 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 29 Apr 2024 15:44:51 -0700 Subject: [PATCH 1306/1702] mm/vmscan: remove ignore_references argument of reclaim_folio_list() All reclaim_folio_list() callers are passing 'true' for 'ignore_references' parameter. In other words, the parameter is not really being used. Simplify the code by removing the parameter. Link: https://lkml.kernel.org/r/20240429224451.67081-5-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton --- mm/vmscan.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index fe923a4a56bbb..d55e8d07ffc4a 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2109,8 +2109,7 @@ static void shrink_active_list(unsigned long nr_to_scan, } static unsigned int reclaim_folio_list(struct list_head *folio_list, - struct pglist_data *pgdat, - bool ignore_references) + struct pglist_data *pgdat) { struct reclaim_stat dummy_stat; unsigned int nr_reclaimed; @@ -2123,7 +2122,7 @@ static unsigned int reclaim_folio_list(struct list_head *folio_list, .no_demotion = 1, }; - nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, ignore_references); + nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, true); while (!list_empty(folio_list)) { folio = lru_to_folio(folio_list); list_del(&folio->lru); @@ -2155,12 +2154,11 @@ unsigned long reclaim_pages(struct list_head *folio_list) continue; } - nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid), - true); + nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); nid = folio_nid(lru_to_folio(folio_list)); } while (!list_empty(folio_list)); - nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid), true); + nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); memalloc_noreclaim_restore(noreclaim_flag); From 447bac3d292fe934010bcf25e1edd539aa204988 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 29 Apr 2024 20:00:55 +0100 Subject: [PATCH 1307/1702] thp: remove HPAGE_PMD_ORDER minimum assertion We now handle order-1 folios correctly, so we don't need this assertion any more. Link: https://lkml.kernel.org/r/20240429190114.3126789-1-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- mm/huge_memory.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index ccdcff73284a0..317de2afd3718 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -740,11 +740,6 @@ static int __init hugepage_init(void) * hugepages can't be allocated by the buddy allocator */ MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER > MAX_PAGE_ORDER); - /* - * we use page->mapping and page->index in second tail page - * as list_head: assuming THP order >= 2 - */ - MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER < 2); err = hugepage_init_sysfs(&hugepage_kobj); if (err) From 99b150d84e4939735cfce245e32e3d29312c68ec Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 16 Apr 2024 18:28:54 +0100 Subject: [PATCH 1308/1702] ext4: convert bd_bitmap_page to bd_bitmap_folio There is no need to make this a multi-page folio, so leave all the infrastructure around it in pages. But since we're locking it, playing with its refcount and checking whether it's uptodate, it needs to move to the folio API. Signed-off-by: Matthew Wilcox (Oracle) Link: https://lore.kernel.org/r/20240416172900.244637-2-willy@infradead.org Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc.c | 98 ++++++++++++++++++++++++----------------------- fs/ext4/mballoc.h | 2 +- 2 files changed, 52 insertions(+), 48 deletions(-) diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index e89c1bfb7ce31..174c029a79f64 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -1452,9 +1452,10 @@ static int ext4_mb_get_buddy_page_lock(struct super_block *sb, int block, pnum, poff; int blocks_per_page; struct page *page; + struct folio *folio; e4b->bd_buddy_page = NULL; - e4b->bd_bitmap_page = NULL; + e4b->bd_bitmap_folio = NULL; blocks_per_page = PAGE_SIZE / sb->s_blocksize; /* @@ -1465,12 +1466,13 @@ static int ext4_mb_get_buddy_page_lock(struct super_block *sb, block = group * 2; pnum = block / blocks_per_page; poff = block % blocks_per_page; - page = find_or_create_page(inode->i_mapping, pnum, gfp); - if (!page) - return -ENOMEM; - BUG_ON(page->mapping != inode->i_mapping); - e4b->bd_bitmap_page = page; - e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); + folio = __filemap_get_folio(inode->i_mapping, pnum, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); + if (IS_ERR(folio)) + return PTR_ERR(folio); + BUG_ON(folio->mapping != inode->i_mapping); + e4b->bd_bitmap_folio = folio; + e4b->bd_bitmap = folio_address(folio) + (poff * sb->s_blocksize); if (blocks_per_page >= 2) { /* buddy and bitmap are on the same page */ @@ -1488,9 +1490,9 @@ static int ext4_mb_get_buddy_page_lock(struct super_block *sb, static void ext4_mb_put_buddy_page_lock(struct ext4_buddy *e4b) { - if (e4b->bd_bitmap_page) { - unlock_page(e4b->bd_bitmap_page); - put_page(e4b->bd_bitmap_page); + if (e4b->bd_bitmap_folio) { + folio_unlock(e4b->bd_bitmap_folio); + folio_put(e4b->bd_bitmap_folio); } if (e4b->bd_buddy_page) { unlock_page(e4b->bd_buddy_page); @@ -1510,6 +1512,7 @@ int ext4_mb_init_group(struct super_block *sb, ext4_group_t group, gfp_t gfp) struct ext4_group_info *this_grp; struct ext4_buddy e4b; struct page *page; + struct folio *folio; int ret = 0; might_sleep(); @@ -1536,11 +1539,11 @@ int ext4_mb_init_group(struct super_block *sb, ext4_group_t group, gfp_t gfp) goto err; } - page = e4b.bd_bitmap_page; - ret = ext4_mb_init_cache(page, NULL, gfp); + folio = e4b.bd_bitmap_folio; + ret = ext4_mb_init_cache(&folio->page, NULL, gfp); if (ret) goto err; - if (!PageUptodate(page)) { + if (!folio_test_uptodate(folio)) { ret = -EIO; goto err; } @@ -1582,6 +1585,7 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, int pnum; int poff; struct page *page; + struct folio *folio; int ret; struct ext4_group_info *grp; struct ext4_sb_info *sbi = EXT4_SB(sb); @@ -1600,7 +1604,7 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, e4b->bd_sb = sb; e4b->bd_group = group; e4b->bd_buddy_page = NULL; - e4b->bd_bitmap_page = NULL; + e4b->bd_bitmap_folio = NULL; if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { /* @@ -1621,53 +1625,53 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, pnum = block / blocks_per_page; poff = block % blocks_per_page; - /* we could use find_or_create_page(), but it locks page - * what we'd like to avoid in fast path ... */ - page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED); - if (page == NULL || !PageUptodate(page)) { - if (page) + /* Avoid locking the folio in the fast path ... */ + folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_ACCESSED, 0); + if (IS_ERR(folio) || !folio_test_uptodate(folio)) { + if (!IS_ERR(folio)) /* - * drop the page reference and try - * to get the page with lock. If we + * drop the folio reference and try + * to get the folio with lock. If we * are not uptodate that implies - * somebody just created the page but - * is yet to initialize the same. So + * somebody just created the folio but + * is yet to initialize it. So * wait for it to initialize. */ - put_page(page); - page = find_or_create_page(inode->i_mapping, pnum, gfp); - if (page) { - if (WARN_RATELIMIT(page->mapping != inode->i_mapping, - "ext4: bitmap's paging->mapping != inode->i_mapping\n")) { + folio_put(folio); + folio = __filemap_get_folio(inode->i_mapping, pnum, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); + if (!IS_ERR(folio)) { + if (WARN_RATELIMIT(folio->mapping != inode->i_mapping, + "ext4: bitmap's mapping != inode->i_mapping\n")) { /* should never happen */ - unlock_page(page); + folio_unlock(folio); ret = -EINVAL; goto err; } - if (!PageUptodate(page)) { - ret = ext4_mb_init_cache(page, NULL, gfp); + if (!folio_test_uptodate(folio)) { + ret = ext4_mb_init_cache(&folio->page, NULL, gfp); if (ret) { - unlock_page(page); + folio_unlock(folio); goto err; } - mb_cmp_bitmaps(e4b, page_address(page) + + mb_cmp_bitmaps(e4b, folio_address(folio) + (poff * sb->s_blocksize)); } - unlock_page(page); + folio_unlock(folio); } } - if (page == NULL) { - ret = -ENOMEM; + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); goto err; } - if (!PageUptodate(page)) { + if (!folio_test_uptodate(folio)) { ret = -EIO; goto err; } /* Pages marked accessed already */ - e4b->bd_bitmap_page = page; - e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); + e4b->bd_bitmap_folio = folio; + e4b->bd_bitmap = folio_address(folio) + (poff * sb->s_blocksize); block++; pnum = block / blocks_per_page; @@ -1715,8 +1719,8 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, err: if (page) put_page(page); - if (e4b->bd_bitmap_page) - put_page(e4b->bd_bitmap_page); + if (e4b->bd_bitmap_folio) + folio_put(e4b->bd_bitmap_folio); e4b->bd_buddy = NULL; e4b->bd_bitmap = NULL; @@ -1731,8 +1735,8 @@ static int ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group, static void ext4_mb_unload_buddy(struct ext4_buddy *e4b) { - if (e4b->bd_bitmap_page) - put_page(e4b->bd_bitmap_page); + if (e4b->bd_bitmap_folio) + folio_put(e4b->bd_bitmap_folio); if (e4b->bd_buddy_page) put_page(e4b->bd_buddy_page); } @@ -2157,7 +2161,7 @@ static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, * double allocate blocks. The reference is dropped * in ext4_mb_release_context */ - ac->ac_bitmap_page = e4b->bd_bitmap_page; + ac->ac_bitmap_page = &e4b->bd_bitmap_folio->page; get_page(ac->ac_bitmap_page); ac->ac_buddy_page = e4b->bd_buddy_page; get_page(ac->ac_buddy_page); @@ -3894,7 +3898,7 @@ static void ext4_free_data_in_buddy(struct super_block *sb, * balance refcounts from ext4_mb_free_metadata() */ put_page(e4b.bd_buddy_page); - put_page(e4b.bd_bitmap_page); + folio_put(e4b.bd_bitmap_folio); } ext4_unlock_group(sb, entry->efd_group); ext4_mb_unload_buddy(&e4b); @@ -6316,7 +6320,7 @@ ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b, struct rb_node *parent = NULL, *new_node; BUG_ON(!ext4_handle_valid(handle)); - BUG_ON(e4b->bd_bitmap_page == NULL); + BUG_ON(e4b->bd_bitmap_folio == NULL); BUG_ON(e4b->bd_buddy_page == NULL); new_node = &new_entry->efd_node; @@ -6329,7 +6333,7 @@ ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b, * on-disk bitmap and lose not-yet-available * blocks */ get_page(e4b->bd_buddy_page); - get_page(e4b->bd_bitmap_page); + folio_get(e4b->bd_bitmap_folio); } while (*n) { parent = *n; diff --git a/fs/ext4/mballoc.h b/fs/ext4/mballoc.h index 7ee13d8eecd79..f8b7e9d2e463e 100644 --- a/fs/ext4/mballoc.h +++ b/fs/ext4/mballoc.h @@ -217,7 +217,7 @@ struct ext4_allocation_context { struct ext4_buddy { struct page *bd_buddy_page; void *bd_buddy; - struct page *bd_bitmap_page; + struct folio *bd_bitmap_folio; void *bd_bitmap; struct ext4_group_info *bd_info; struct super_block *bd_sb; From 5eea586b47f05b5f5518cf8f9dd9283a01a8066d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 16 Apr 2024 18:28:55 +0100 Subject: [PATCH 1309/1702] ext4: convert bd_buddy_page to bd_buddy_folio There is no need to make this a multi-page folio, so leave all the infrastructure around it in pages. But since we're locking it, playing with its refcount and checking whether it's uptodate, it needs to move to the folio API. Signed-off-by: Matthew Wilcox (Oracle) Link: https://lore.kernel.org/r/20240416172900.244637-3-willy@infradead.org Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc.c | 91 +++++++++++++++++++++++------------------------ fs/ext4/mballoc.h | 2 +- 2 files changed, 46 insertions(+), 47 deletions(-) diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 174c029a79f64..1a76e5741d36c 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -1443,7 +1443,7 @@ static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) * Lock the buddy and bitmap pages. This make sure other parallel init_group * on the same buddy page doesn't happen whild holding the buddy page lock. * Return locked buddy and bitmap pages on e4b struct. If buddy and bitmap - * are on the same page e4b->bd_buddy_page is NULL and return value is 0. + * are on the same page e4b->bd_buddy_folio is NULL and return value is 0. */ static int ext4_mb_get_buddy_page_lock(struct super_block *sb, ext4_group_t group, struct ext4_buddy *e4b, gfp_t gfp) @@ -1451,10 +1451,9 @@ static int ext4_mb_get_buddy_page_lock(struct super_block *sb, struct inode *inode = EXT4_SB(sb)->s_buddy_cache; int block, pnum, poff; int blocks_per_page; - struct page *page; struct folio *folio; - e4b->bd_buddy_page = NULL; + e4b->bd_buddy_folio = NULL; e4b->bd_bitmap_folio = NULL; blocks_per_page = PAGE_SIZE / sb->s_blocksize; @@ -1480,11 +1479,12 @@ static int ext4_mb_get_buddy_page_lock(struct super_block *sb, } /* blocks_per_page == 1, hence we need another page for the buddy */ - page = find_or_create_page(inode->i_mapping, block + 1, gfp); - if (!page) - return -ENOMEM; - BUG_ON(page->mapping != inode->i_mapping); - e4b->bd_buddy_page = page; + folio = __filemap_get_folio(inode->i_mapping, block + 1, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); + if (IS_ERR(folio)) + return PTR_ERR(folio); + BUG_ON(folio->mapping != inode->i_mapping); + e4b->bd_buddy_folio = folio; return 0; } @@ -1494,9 +1494,9 @@ static void ext4_mb_put_buddy_page_lock(struct ext4_buddy *e4b) folio_unlock(e4b->bd_bitmap_folio); folio_put(e4b->bd_bitmap_folio); } - if (e4b->bd_buddy_page) { - unlock_page(e4b->bd_buddy_page); - put_page(e4b->bd_buddy_page); + if (e4b->bd_buddy_folio) { + folio_unlock(e4b->bd_buddy_folio); + folio_put(e4b->bd_buddy_folio); } } @@ -1511,7 +1511,6 @@ int ext4_mb_init_group(struct super_block *sb, ext4_group_t group, gfp_t gfp) struct ext4_group_info *this_grp; struct ext4_buddy e4b; - struct page *page; struct folio *folio; int ret = 0; @@ -1548,7 +1547,7 @@ int ext4_mb_init_group(struct super_block *sb, ext4_group_t group, gfp_t gfp) goto err; } - if (e4b.bd_buddy_page == NULL) { + if (e4b.bd_buddy_folio == NULL) { /* * If both the bitmap and buddy are in * the same page we don't need to force @@ -1558,11 +1557,11 @@ int ext4_mb_init_group(struct super_block *sb, ext4_group_t group, gfp_t gfp) goto err; } /* init buddy cache */ - page = e4b.bd_buddy_page; - ret = ext4_mb_init_cache(page, e4b.bd_bitmap, gfp); + folio = e4b.bd_buddy_folio; + ret = ext4_mb_init_cache(&folio->page, e4b.bd_bitmap, gfp); if (ret) goto err; - if (!PageUptodate(page)) { + if (!folio_test_uptodate(folio)) { ret = -EIO; goto err; } @@ -1584,7 +1583,6 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, int block; int pnum; int poff; - struct page *page; struct folio *folio; int ret; struct ext4_group_info *grp; @@ -1603,7 +1601,7 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, e4b->bd_info = grp; e4b->bd_sb = sb; e4b->bd_group = group; - e4b->bd_buddy_page = NULL; + e4b->bd_buddy_folio = NULL; e4b->bd_bitmap_folio = NULL; if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { @@ -1669,7 +1667,7 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, goto err; } - /* Pages marked accessed already */ + /* Folios marked accessed already */ e4b->bd_bitmap_folio = folio; e4b->bd_bitmap = folio_address(folio) + (poff * sb->s_blocksize); @@ -1677,48 +1675,49 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, pnum = block / blocks_per_page; poff = block % blocks_per_page; - page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED); - if (page == NULL || !PageUptodate(page)) { - if (page) - put_page(page); - page = find_or_create_page(inode->i_mapping, pnum, gfp); - if (page) { - if (WARN_RATELIMIT(page->mapping != inode->i_mapping, - "ext4: buddy bitmap's page->mapping != inode->i_mapping\n")) { + folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_ACCESSED, 0); + if (IS_ERR(folio) || !folio_test_uptodate(folio)) { + if (!IS_ERR(folio)) + folio_put(folio); + folio = __filemap_get_folio(inode->i_mapping, pnum, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); + if (!IS_ERR(folio)) { + if (WARN_RATELIMIT(folio->mapping != inode->i_mapping, + "ext4: buddy bitmap's mapping != inode->i_mapping\n")) { /* should never happen */ - unlock_page(page); + folio_unlock(folio); ret = -EINVAL; goto err; } - if (!PageUptodate(page)) { - ret = ext4_mb_init_cache(page, e4b->bd_bitmap, + if (!folio_test_uptodate(folio)) { + ret = ext4_mb_init_cache(&folio->page, e4b->bd_bitmap, gfp); if (ret) { - unlock_page(page); + folio_unlock(folio); goto err; } } - unlock_page(page); + folio_unlock(folio); } } - if (page == NULL) { - ret = -ENOMEM; + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); goto err; } - if (!PageUptodate(page)) { + if (!folio_test_uptodate(folio)) { ret = -EIO; goto err; } - /* Pages marked accessed already */ - e4b->bd_buddy_page = page; - e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize); + /* Folios marked accessed already */ + e4b->bd_buddy_folio = folio; + e4b->bd_buddy = folio_address(folio) + (poff * sb->s_blocksize); return 0; err: - if (page) - put_page(page); + if (folio) + folio_put(folio); if (e4b->bd_bitmap_folio) folio_put(e4b->bd_bitmap_folio); @@ -1737,8 +1736,8 @@ static void ext4_mb_unload_buddy(struct ext4_buddy *e4b) { if (e4b->bd_bitmap_folio) folio_put(e4b->bd_bitmap_folio); - if (e4b->bd_buddy_page) - put_page(e4b->bd_buddy_page); + if (e4b->bd_buddy_folio) + folio_put(e4b->bd_buddy_folio); } @@ -2163,7 +2162,7 @@ static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, */ ac->ac_bitmap_page = &e4b->bd_bitmap_folio->page; get_page(ac->ac_bitmap_page); - ac->ac_buddy_page = e4b->bd_buddy_page; + ac->ac_buddy_page = &e4b->bd_buddy_folio->page; get_page(ac->ac_buddy_page); /* store last allocated for subsequent stream allocation */ if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { @@ -3897,7 +3896,7 @@ static void ext4_free_data_in_buddy(struct super_block *sb, /* No more items in the per group rb tree * balance refcounts from ext4_mb_free_metadata() */ - put_page(e4b.bd_buddy_page); + folio_put(e4b.bd_buddy_folio); folio_put(e4b.bd_bitmap_folio); } ext4_unlock_group(sb, entry->efd_group); @@ -6321,7 +6320,7 @@ ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b, BUG_ON(!ext4_handle_valid(handle)); BUG_ON(e4b->bd_bitmap_folio == NULL); - BUG_ON(e4b->bd_buddy_page == NULL); + BUG_ON(e4b->bd_buddy_folio == NULL); new_node = &new_entry->efd_node; cluster = new_entry->efd_start_cluster; @@ -6332,7 +6331,7 @@ ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b, * otherwise we'll refresh it from * on-disk bitmap and lose not-yet-available * blocks */ - get_page(e4b->bd_buddy_page); + folio_get(e4b->bd_buddy_folio); folio_get(e4b->bd_bitmap_folio); } while (*n) { diff --git a/fs/ext4/mballoc.h b/fs/ext4/mballoc.h index f8b7e9d2e463e..268c838ddc2d8 100644 --- a/fs/ext4/mballoc.h +++ b/fs/ext4/mballoc.h @@ -215,7 +215,7 @@ struct ext4_allocation_context { #define AC_STATUS_BREAK 3 struct ext4_buddy { - struct page *bd_buddy_page; + struct folio *bd_buddy_folio; void *bd_buddy; struct folio *bd_bitmap_folio; void *bd_bitmap; From e1622a0d558200b0ccdfbf69ee29b9ac1f5c2442 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 16 Apr 2024 18:28:56 +0100 Subject: [PATCH 1310/1702] ext4: convert ext4_mb_init_cache() to take a folio All callers now have a folio, so convert this function from operating on a page to operating on a folio. The folio is assumed to be a single page. Signe-off-by: Matthew Wilcox (Oracle) Link: https://lore.kernel.org/r/20240416172900.244637-4-willy@infradead.org Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc.c | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 1a76e5741d36c..71950abb457e8 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -1274,7 +1274,7 @@ static void mb_regenerate_buddy(struct ext4_buddy *e4b) * for this page; do not hold this lock when calling this routine! */ -static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) +static int ext4_mb_init_cache(struct folio *folio, char *incore, gfp_t gfp) { ext4_group_t ngroups; unsigned int blocksize; @@ -1292,13 +1292,13 @@ static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) char *bitmap; struct ext4_group_info *grinfo; - inode = page->mapping->host; + inode = folio->mapping->host; sb = inode->i_sb; ngroups = ext4_get_groups_count(sb); blocksize = i_blocksize(inode); blocks_per_page = PAGE_SIZE / blocksize; - mb_debug(sb, "init page %lu\n", page->index); + mb_debug(sb, "init folio %lu\n", folio->index); groups_per_page = blocks_per_page >> 1; if (groups_per_page == 0) @@ -1313,9 +1313,9 @@ static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) } else bh = &bhs; - first_group = page->index * blocks_per_page / 2; + first_group = folio->index * blocks_per_page / 2; - /* read all groups the page covers into the cache */ + /* read all groups the folio covers into the cache */ for (i = 0, group = first_group; i < groups_per_page; i++, group++) { if (group >= ngroups) break; @@ -1326,10 +1326,11 @@ static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) /* * If page is uptodate then we came here after online resize * which added some new uninitialized group info structs, so - * we must skip all initialized uptodate buddies on the page, + * we must skip all initialized uptodate buddies on the folio, * which may be currently in use by an allocating task. */ - if (PageUptodate(page) && !EXT4_MB_GRP_NEED_INIT(grinfo)) { + if (folio_test_uptodate(folio) && + !EXT4_MB_GRP_NEED_INIT(grinfo)) { bh[i] = NULL; continue; } @@ -1353,7 +1354,7 @@ static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) err = err2; } - first_block = page->index * blocks_per_page; + first_block = folio->index * blocks_per_page; for (i = 0; i < blocks_per_page; i++) { group = (first_block + i) >> 1; if (group >= ngroups) @@ -1374,7 +1375,7 @@ static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) * above * */ - data = page_address(page) + (i * blocksize); + data = folio_address(folio) + (i * blocksize); bitmap = bh[group - first_group]->b_data; /* @@ -1389,8 +1390,8 @@ static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) if ((first_block + i) & 1) { /* this is block of buddy */ BUG_ON(incore == NULL); - mb_debug(sb, "put buddy for group %u in page %lu/%x\n", - group, page->index, i * blocksize); + mb_debug(sb, "put buddy for group %u in folio %lu/%x\n", + group, folio->index, i * blocksize); trace_ext4_mb_buddy_bitmap_load(sb, group); grinfo->bb_fragments = 0; memset(grinfo->bb_counters, 0, @@ -1408,8 +1409,8 @@ static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) } else { /* this is block of bitmap */ BUG_ON(incore != NULL); - mb_debug(sb, "put bitmap for group %u in page %lu/%x\n", - group, page->index, i * blocksize); + mb_debug(sb, "put bitmap for group %u in folio %lu/%x\n", + group, folio->index, i * blocksize); trace_ext4_mb_bitmap_load(sb, group); /* see comments in ext4_mb_put_pa() */ @@ -1427,7 +1428,7 @@ static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) incore = data; } } - SetPageUptodate(page); + folio_mark_uptodate(folio); out: if (bh) { @@ -1539,7 +1540,7 @@ int ext4_mb_init_group(struct super_block *sb, ext4_group_t group, gfp_t gfp) } folio = e4b.bd_bitmap_folio; - ret = ext4_mb_init_cache(&folio->page, NULL, gfp); + ret = ext4_mb_init_cache(folio, NULL, gfp); if (ret) goto err; if (!folio_test_uptodate(folio)) { @@ -1558,7 +1559,7 @@ int ext4_mb_init_group(struct super_block *sb, ext4_group_t group, gfp_t gfp) } /* init buddy cache */ folio = e4b.bd_buddy_folio; - ret = ext4_mb_init_cache(&folio->page, e4b.bd_bitmap, gfp); + ret = ext4_mb_init_cache(folio, e4b.bd_bitmap, gfp); if (ret) goto err; if (!folio_test_uptodate(folio)) { @@ -1647,7 +1648,7 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, goto err; } if (!folio_test_uptodate(folio)) { - ret = ext4_mb_init_cache(&folio->page, NULL, gfp); + ret = ext4_mb_init_cache(folio, NULL, gfp); if (ret) { folio_unlock(folio); goto err; @@ -1690,7 +1691,7 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, goto err; } if (!folio_test_uptodate(folio)) { - ret = ext4_mb_init_cache(&folio->page, e4b->bd_bitmap, + ret = ext4_mb_init_cache(folio, e4b->bd_bitmap, gfp); if (ret) { folio_unlock(folio); From ccedf35b5daa429c0e731eac6fb32b0208302c6b Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 16 Apr 2024 18:28:57 +0100 Subject: [PATCH 1311/1702] ext4: convert ac_bitmap_page to ac_bitmap_folio This just carries around the bd_bitmap_folio so should also be a folio. Signed-off-by: Matthew Wilcox (Oracle) Link: https://lore.kernel.org/r/20240416172900.244637-5-willy@infradead.org Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc.c | 8 ++++---- fs/ext4/mballoc.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 71950abb457e8..218480b20058e 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -2161,8 +2161,8 @@ static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, * double allocate blocks. The reference is dropped * in ext4_mb_release_context */ - ac->ac_bitmap_page = &e4b->bd_bitmap_folio->page; - get_page(ac->ac_bitmap_page); + ac->ac_bitmap_folio = e4b->bd_bitmap_folio; + folio_get(ac->ac_bitmap_folio); ac->ac_buddy_page = &e4b->bd_buddy_folio->page; get_page(ac->ac_buddy_page); /* store last allocated for subsequent stream allocation */ @@ -6002,8 +6002,8 @@ static void ext4_mb_release_context(struct ext4_allocation_context *ac) ext4_mb_put_pa(ac, ac->ac_sb, pa); } - if (ac->ac_bitmap_page) - put_page(ac->ac_bitmap_page); + if (ac->ac_bitmap_folio) + folio_put(ac->ac_bitmap_folio); if (ac->ac_buddy_page) put_page(ac->ac_buddy_page); if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) diff --git a/fs/ext4/mballoc.h b/fs/ext4/mballoc.h index 268c838ddc2d8..469e5516fb77e 100644 --- a/fs/ext4/mballoc.h +++ b/fs/ext4/mballoc.h @@ -204,7 +204,7 @@ struct ext4_allocation_context { __u8 ac_2order; /* if request is to allocate 2^N blocks and * N > 0, the field stores N, otherwise 0 */ __u8 ac_op; /* operation, for history only */ - struct page *ac_bitmap_page; + struct folio *ac_bitmap_folio; struct page *ac_buddy_page; struct ext4_prealloc_space *ac_pa; struct ext4_locality_group *ac_lg; From c84f1510fba9fb181e6a1aa7b5fcfc67381b39c9 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 16 Apr 2024 18:28:58 +0100 Subject: [PATCH 1312/1702] ext4: convert ac_buddy_page to ac_buddy_folio This just carries around the bd_buddy_folio so should also be a folio. Signed-off-by: Matthew Wilcox (Oracle) Link: https://lore.kernel.org/r/20240416172900.244637-6-willy@infradead.org Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc.c | 8 ++++---- fs/ext4/mballoc.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 218480b20058e..ac0ee747f4b94 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -2163,8 +2163,8 @@ static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, */ ac->ac_bitmap_folio = e4b->bd_bitmap_folio; folio_get(ac->ac_bitmap_folio); - ac->ac_buddy_page = &e4b->bd_buddy_folio->page; - get_page(ac->ac_buddy_page); + ac->ac_buddy_folio = e4b->bd_buddy_folio; + folio_get(ac->ac_buddy_folio); /* store last allocated for subsequent stream allocation */ if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { spin_lock(&sbi->s_md_lock); @@ -6004,8 +6004,8 @@ static void ext4_mb_release_context(struct ext4_allocation_context *ac) } if (ac->ac_bitmap_folio) folio_put(ac->ac_bitmap_folio); - if (ac->ac_buddy_page) - put_page(ac->ac_buddy_page); + if (ac->ac_buddy_folio) + folio_put(ac->ac_buddy_folio); if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) mutex_unlock(&ac->ac_lg->lg_mutex); ext4_mb_collect_stats(ac); diff --git a/fs/ext4/mballoc.h b/fs/ext4/mballoc.h index 469e5516fb77e..d8553f1498d3c 100644 --- a/fs/ext4/mballoc.h +++ b/fs/ext4/mballoc.h @@ -205,7 +205,7 @@ struct ext4_allocation_context { * N > 0, the field stores N, otherwise 0 */ __u8 ac_op; /* operation, for history only */ struct folio *ac_bitmap_folio; - struct page *ac_buddy_page; + struct folio *ac_buddy_folio; struct ext4_prealloc_space *ac_pa; struct ext4_locality_group *ac_lg; }; From 3f4830abd236d0428e50451e1ecb62e14c365e9b Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 17 Apr 2024 21:10:40 +0300 Subject: [PATCH 1313/1702] ext4: fix potential unnitialized variable Smatch complains "err" can be uninitialized in the caller. fs/ext4/indirect.c:349 ext4_alloc_branch() error: uninitialized symbol 'err'. Set the error to zero on the success path. Fixes: 8016e29f4362 ("ext4: fast commit recovery path") Signed-off-by: Dan Carpenter Link: https://lore.kernel.org/r/363a4673-0fb8-4adf-b4fb-90a499077276@moroto.mountain Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index ac0ee747f4b94..648989c125f2f 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -6126,6 +6126,7 @@ ext4_mb_new_blocks_simple(struct ext4_allocation_request *ar, int *errp) ext4_mb_mark_bb(sb, block, 1, true); ar->len = 1; + *errp = 0; return block; } From df0b5afc62f3368d657a8fe4a8d393ac481474c2 Mon Sep 17 00:00:00 2001 From: Zhang Yi Date: Fri, 19 Apr 2024 10:30:05 +0800 Subject: [PATCH 1314/1702] ext4: remove the redundant folio_wait_stable() __filemap_get_folio() with FGP_WRITEBEGIN parameter has already wait for stable folio, so remove the redundant folio_wait_stable() in ext4_da_write_begin(), it was left over from the commit cc883236b792 ("ext4: drop unnecessary journal handle in delalloc write") that removed the retry getting page logic. Fixes: cc883236b792 ("ext4: drop unnecessary journal handle in delalloc write") Signed-off-by: Zhang Yi Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240419023005.2719050-1-yi.zhang@huaweicloud.com Signed-off-by: Theodore Ts'o --- fs/ext4/inode.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 989e28c24a8cb..4bae9ccf5fe01 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -2887,9 +2887,6 @@ static int ext4_da_write_begin(struct file *file, struct address_space *mapping, if (IS_ERR(folio)) return PTR_ERR(folio); - /* In case writeback began while the folio was unlocked */ - folio_wait_stable(folio); - #ifdef CONFIG_FS_ENCRYPTION ret = ext4_block_write_begin(folio, pos, len, ext4_da_get_block_prep); #else From 8b57de1c5edde3faf8a4f6a440b7ec16bb3c81d4 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 10 Apr 2024 12:28:03 +0100 Subject: [PATCH 1315/1702] jbd2: remove redundant assignement to variable err The variable err is being assigned a value that is never read, it is being re-assigned inside the following while loop and also after the while loop. The assignment is redundant and can be removed. Cleans up clang scan build warning: fs/jbd2/commit.c:574:2: warning: Value stored to 'err' is never read [deadcode.DeadStores] Signed-off-by: Colin Ian King Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240410112803.232993-1-colin.i.king@gmail.com Signed-off-by: Theodore Ts'o --- fs/jbd2/commit.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c index 5e122586e06ed..78a9d08ae9f85 100644 --- a/fs/jbd2/commit.c +++ b/fs/jbd2/commit.c @@ -571,7 +571,6 @@ void jbd2_journal_commit_transaction(journal_t *journal) J_ASSERT(commit_transaction->t_nr_buffers <= atomic_read(&commit_transaction->t_outstanding_credits)); - err = 0; bufs = 0; descriptor = NULL; while (commit_transaction->t_buffers) { From 0c0b4a49d3e7f49690a6827a41faeffad5df7e21 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Sat, 4 May 2024 15:55:25 +0800 Subject: [PATCH 1316/1702] ext4: fix mb_cache_entry's e_refcnt leak in ext4_xattr_block_cache_find() Syzbot reports a warning as follows: ============================================ WARNING: CPU: 0 PID: 5075 at fs/mbcache.c:419 mb_cache_destroy+0x224/0x290 Modules linked in: CPU: 0 PID: 5075 Comm: syz-executor199 Not tainted 6.9.0-rc6-gb947cc5bf6d7 RIP: 0010:mb_cache_destroy+0x224/0x290 fs/mbcache.c:419 Call Trace: ext4_put_super+0x6d4/0xcd0 fs/ext4/super.c:1375 generic_shutdown_super+0x136/0x2d0 fs/super.c:641 kill_block_super+0x44/0x90 fs/super.c:1675 ext4_kill_sb+0x68/0xa0 fs/ext4/super.c:7327 [...] ============================================ This is because when finding an entry in ext4_xattr_block_cache_find(), if ext4_sb_bread() returns -ENOMEM, the ce's e_refcnt, which has already grown in the __entry_find(), won't be put away, and eventually trigger the above issue in mb_cache_destroy() due to reference count leakage. So call mb_cache_entry_put() on the -ENOMEM error branch as a quick fix. Reported-by: syzbot+dd43bd0f7474512edc47@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=dd43bd0f7474512edc47 Fixes: fb265c9cb49e ("ext4: add ext4_sb_bread() to disambiguate ENOMEM cases") Cc: stable@kernel.org Signed-off-by: Baokun Li Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240504075526.2254349-2-libaokun@huaweicloud.com Signed-off-by: Theodore Ts'o --- fs/ext4/xattr.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index 04f90df8dbae9..3e9b088800b1b 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -3117,8 +3117,10 @@ ext4_xattr_block_cache_find(struct inode *inode, bh = ext4_sb_bread(inode->i_sb, ce->e_value, REQ_PRIO); if (IS_ERR(bh)) { - if (PTR_ERR(bh) == -ENOMEM) + if (PTR_ERR(bh) == -ENOMEM) { + mb_cache_entry_put(ea_block_cache, ce); return NULL; + } bh = NULL; EXT4_ERROR_INODE(inode, "block %lu read error", (unsigned long)ce->e_value); From dc1c4663bc493f323d6b2f9dd55c044ea920dacf Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Sat, 4 May 2024 15:55:26 +0800 Subject: [PATCH 1317/1702] ext4: propagate errors from ext4_sb_bread() in ext4_xattr_block_cache_find() In ext4_xattr_block_cache_find(), when ext4_sb_bread() returns an error, we will either continue to find the next ea block or return NULL to try to insert a new ea block. But whether ext4_sb_bread() returns -EIO or -ENOMEM, the next operation is most likely to fail with the same error. So propagate the error returned by ext4_sb_bread() to make ext4_xattr_block_set() fail to reduce pointless operations. Signed-off-by: Baokun Li Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240504075526.2254349-3-libaokun@huaweicloud.com Signed-off-by: Theodore Ts'o --- fs/ext4/xattr.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index 3e9b088800b1b..6460879b9fcbb 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -2028,8 +2028,13 @@ ext4_xattr_block_set(handle_t *handle, struct inode *inode, inserted: if (!IS_LAST_ENTRY(s->first)) { - new_bh = ext4_xattr_block_cache_find(inode, header(s->base), - &ce); + new_bh = ext4_xattr_block_cache_find(inode, header(s->base), &ce); + if (IS_ERR(new_bh)) { + error = PTR_ERR(new_bh); + new_bh = NULL; + goto cleanup; + } + if (new_bh) { /* We found an identical block in the cache. */ if (new_bh == bs->bh) @@ -3094,8 +3099,8 @@ ext4_xattr_cmp(struct ext4_xattr_header *header1, * * Find an identical extended attribute block. * - * Returns a pointer to the block found, or NULL if such a block was - * not found or an error occurred. + * Returns a pointer to the block found, or NULL if such a block was not + * found, or an error pointer if an error occurred while reading ea block. */ static struct buffer_head * ext4_xattr_block_cache_find(struct inode *inode, @@ -3117,13 +3122,11 @@ ext4_xattr_block_cache_find(struct inode *inode, bh = ext4_sb_bread(inode->i_sb, ce->e_value, REQ_PRIO); if (IS_ERR(bh)) { - if (PTR_ERR(bh) == -ENOMEM) { - mb_cache_entry_put(ea_block_cache, ce); - return NULL; - } - bh = NULL; - EXT4_ERROR_INODE(inode, "block %lu read error", - (unsigned long)ce->e_value); + if (PTR_ERR(bh) != -ENOMEM) + EXT4_ERROR_INODE(inode, "block %lu read error", + (unsigned long)ce->e_value); + mb_cache_entry_put(ea_block_cache, ce); + return bh; } else if (ext4_xattr_cmp(header, BHDR(bh)) == 0) { *pce = ce; return bh; From 098c290a490d0121e209617097ea563e5e607066 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Wed, 24 Apr 2024 09:42:08 +0100 Subject: [PATCH 1318/1702] clock, reset: microchip: move all mpfs reset code to the reset subsystem Stephen and Philipp, while reviewing patches, said that all of the aux device creation and the register read/write code could be moved to the reset subsystem, leaving the clock driver with no implementations of reset_* functions at all. Move them. Suggested-by: Philipp Zabel Suggested-by: Stephen Boyd Signed-off-by: Conor Dooley Link: https://lore.kernel.org/r/20240424-strangle-sharpener-34755c5e6e3e@spud Signed-off-by: Stephen Boyd --- drivers/clk/microchip/clk-mpfs.c | 92 +------------------------------ drivers/reset/reset-mpfs.c | 95 +++++++++++++++++++++++++++++--- include/soc/microchip/mpfs.h | 10 ++-- 3 files changed, 93 insertions(+), 104 deletions(-) diff --git a/drivers/clk/microchip/clk-mpfs.c b/drivers/clk/microchip/clk-mpfs.c index 22eab91a67129..28ec0da88cb38 100644 --- a/drivers/clk/microchip/clk-mpfs.c +++ b/drivers/clk/microchip/clk-mpfs.c @@ -4,12 +4,10 @@ * * Copyright (C) 2020-2022 Microchip Technology Inc. All rights reserved. */ -#include #include #include #include #include -#include #include #include @@ -361,93 +359,6 @@ static int mpfs_clk_register_periphs(struct device *dev, struct mpfs_periph_hw_c return 0; } -/* - * Peripheral clock resets - */ - -#if IS_ENABLED(CONFIG_RESET_CONTROLLER) - -u32 mpfs_reset_read(struct device *dev) -{ - struct mpfs_clock_data *clock_data = dev_get_drvdata(dev->parent); - - return readl_relaxed(clock_data->base + REG_SUBBLK_RESET_CR); -} -EXPORT_SYMBOL_NS_GPL(mpfs_reset_read, MCHP_CLK_MPFS); - -void mpfs_reset_write(struct device *dev, u32 val) -{ - struct mpfs_clock_data *clock_data = dev_get_drvdata(dev->parent); - - writel_relaxed(val, clock_data->base + REG_SUBBLK_RESET_CR); -} -EXPORT_SYMBOL_NS_GPL(mpfs_reset_write, MCHP_CLK_MPFS); - -static void mpfs_reset_unregister_adev(void *_adev) -{ - struct auxiliary_device *adev = _adev; - - auxiliary_device_delete(adev); - auxiliary_device_uninit(adev); -} - -static void mpfs_reset_adev_release(struct device *dev) -{ - struct auxiliary_device *adev = to_auxiliary_dev(dev); - - kfree(adev); -} - -static struct auxiliary_device *mpfs_reset_adev_alloc(struct mpfs_clock_data *clk_data) -{ - struct auxiliary_device *adev; - int ret; - - adev = kzalloc(sizeof(*adev), GFP_KERNEL); - if (!adev) - return ERR_PTR(-ENOMEM); - - adev->name = "reset-mpfs"; - adev->dev.parent = clk_data->dev; - adev->dev.release = mpfs_reset_adev_release; - adev->id = 666u; - - ret = auxiliary_device_init(adev); - if (ret) { - kfree(adev); - return ERR_PTR(ret); - } - - return adev; -} - -static int mpfs_reset_controller_register(struct mpfs_clock_data *clk_data) -{ - struct auxiliary_device *adev; - int ret; - - adev = mpfs_reset_adev_alloc(clk_data); - if (IS_ERR(adev)) - return PTR_ERR(adev); - - ret = auxiliary_device_add(adev); - if (ret) { - auxiliary_device_uninit(adev); - return ret; - } - - return devm_add_action_or_reset(clk_data->dev, mpfs_reset_unregister_adev, adev); -} - -#else /* !CONFIG_RESET_CONTROLLER */ - -static int mpfs_reset_controller_register(struct mpfs_clock_data *clk_data) -{ - return 0; -} - -#endif /* !CONFIG_RESET_CONTROLLER */ - static int mpfs_clk_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -499,7 +410,7 @@ static int mpfs_clk_probe(struct platform_device *pdev) if (ret) return ret; - return mpfs_reset_controller_register(clk_data); + return mpfs_reset_controller_register(dev, clk_data->base + REG_SUBBLK_RESET_CR); } static const struct of_device_id mpfs_clk_of_match_table[] = { @@ -532,3 +443,4 @@ MODULE_DESCRIPTION("Microchip PolarFire SoC Clock Driver"); MODULE_AUTHOR("Padmarao Begari "); MODULE_AUTHOR("Daire McNamara "); MODULE_AUTHOR("Conor Dooley "); +MODULE_IMPORT_NS(MCHP_CLK_MPFS); diff --git a/drivers/reset/reset-mpfs.c b/drivers/reset/reset-mpfs.c index 7f3fb2d472f41..710f9c1676f93 100644 --- a/drivers/reset/reset-mpfs.c +++ b/drivers/reset/reset-mpfs.c @@ -8,9 +8,11 @@ */ #include #include +#include #include #include #include +#include #include #include #include @@ -28,20 +30,30 @@ /* block concurrent access to the soft reset register */ static DEFINE_SPINLOCK(mpfs_reset_lock); +struct mpfs_reset { + void __iomem *base; + struct reset_controller_dev rcdev; +}; + +static inline struct mpfs_reset *to_mpfs_reset(struct reset_controller_dev *rcdev) +{ + return container_of(rcdev, struct mpfs_reset, rcdev); +} + /* * Peripheral clock resets */ - static int mpfs_assert(struct reset_controller_dev *rcdev, unsigned long id) { + struct mpfs_reset *rst = to_mpfs_reset(rcdev); unsigned long flags; u32 reg; spin_lock_irqsave(&mpfs_reset_lock, flags); - reg = mpfs_reset_read(rcdev->dev); + reg = readl(rst->base); reg |= BIT(id); - mpfs_reset_write(rcdev->dev, reg); + writel(reg, rst->base); spin_unlock_irqrestore(&mpfs_reset_lock, flags); @@ -50,14 +62,15 @@ static int mpfs_assert(struct reset_controller_dev *rcdev, unsigned long id) static int mpfs_deassert(struct reset_controller_dev *rcdev, unsigned long id) { + struct mpfs_reset *rst = to_mpfs_reset(rcdev); unsigned long flags; u32 reg; spin_lock_irqsave(&mpfs_reset_lock, flags); - reg = mpfs_reset_read(rcdev->dev); + reg = readl(rst->base); reg &= ~BIT(id); - mpfs_reset_write(rcdev->dev, reg); + writel(reg, rst->base); spin_unlock_irqrestore(&mpfs_reset_lock, flags); @@ -66,7 +79,8 @@ static int mpfs_deassert(struct reset_controller_dev *rcdev, unsigned long id) static int mpfs_status(struct reset_controller_dev *rcdev, unsigned long id) { - u32 reg = mpfs_reset_read(rcdev->dev); + struct mpfs_reset *rst = to_mpfs_reset(rcdev); + u32 reg = readl(rst->base); /* * It is safe to return here as MPFS_NUM_RESETS makes sure the sign bit @@ -121,11 +135,15 @@ static int mpfs_reset_probe(struct auxiliary_device *adev, { struct device *dev = &adev->dev; struct reset_controller_dev *rcdev; + struct mpfs_reset *rst; - rcdev = devm_kzalloc(dev, sizeof(*rcdev), GFP_KERNEL); - if (!rcdev) + rst = devm_kzalloc(dev, sizeof(*rst), GFP_KERNEL); + if (!rst) return -ENOMEM; + rst->base = (void __iomem *)adev->dev.platform_data; + + rcdev = &rst->rcdev; rcdev->dev = dev; rcdev->dev->parent = dev->parent; rcdev->ops = &mpfs_reset_ops; @@ -137,9 +155,68 @@ static int mpfs_reset_probe(struct auxiliary_device *adev, return devm_reset_controller_register(dev, rcdev); } +static void mpfs_reset_unregister_adev(void *_adev) +{ + struct auxiliary_device *adev = _adev; + + auxiliary_device_delete(adev); + auxiliary_device_uninit(adev); +} + +static void mpfs_reset_adev_release(struct device *dev) +{ + struct auxiliary_device *adev = to_auxiliary_dev(dev); + + kfree(adev); +} + +static struct auxiliary_device *mpfs_reset_adev_alloc(struct device *clk_dev) +{ + struct auxiliary_device *adev; + int ret; + + adev = kzalloc(sizeof(*adev), GFP_KERNEL); + if (!adev) + return ERR_PTR(-ENOMEM); + + adev->name = "reset-mpfs"; + adev->dev.parent = clk_dev; + adev->dev.release = mpfs_reset_adev_release; + adev->id = 666u; + + ret = auxiliary_device_init(adev); + if (ret) { + kfree(adev); + return ERR_PTR(ret); + } + + return adev; +} + +int mpfs_reset_controller_register(struct device *clk_dev, void __iomem *base) +{ + struct auxiliary_device *adev; + int ret; + + adev = mpfs_reset_adev_alloc(clk_dev); + if (IS_ERR(adev)) + return PTR_ERR(adev); + + ret = auxiliary_device_add(adev); + if (ret) { + auxiliary_device_uninit(adev); + return ret; + } + + adev->dev.platform_data = (__force void *)base; + + return devm_add_action_or_reset(clk_dev, mpfs_reset_unregister_adev, adev); +} +EXPORT_SYMBOL_NS_GPL(mpfs_reset_controller_register, MCHP_CLK_MPFS); + static const struct auxiliary_device_id mpfs_reset_ids[] = { { - .name = "clk_mpfs.reset-mpfs", + .name = "reset_mpfs.reset-mpfs", }, { } }; diff --git a/include/soc/microchip/mpfs.h b/include/soc/microchip/mpfs.h index 09722f83b0ca8..d7e612b5e22ef 100644 --- a/include/soc/microchip/mpfs.h +++ b/include/soc/microchip/mpfs.h @@ -43,11 +43,11 @@ struct mtd_info *mpfs_sys_controller_get_flash(struct mpfs_sys_controller *mpfs_ #endif /* if IS_ENABLED(CONFIG_POLARFIRE_SOC_SYS_CTRL) */ #if IS_ENABLED(CONFIG_MCHP_CLK_MPFS) - -u32 mpfs_reset_read(struct device *dev); - -void mpfs_reset_write(struct device *dev, u32 val); - +#if IS_ENABLED(CONFIG_RESET_CONTROLLER) +int mpfs_reset_controller_register(struct device *clk_dev, void __iomem *base); +#else +static inline int mpfs_reset_controller_register(struct device *clk_dev, void __iomem *base) { return 0; } +#endif /* if IS_ENABLED(CONFIG_RESET_CONTROLLER) */ #endif /* if IS_ENABLED(CONFIG_MCHP_CLK_MPFS) */ #endif /* __SOC_MPFS_H__ */ From 12b52b83c9276e7cad13fedbbe38bde570164775 Mon Sep 17 00:00:00 2001 From: Gabriel Fernandez Date: Fri, 19 Apr 2024 17:27:20 +0200 Subject: [PATCH 1319/1702] dt-bindings: clocks: stm32mp25: add access-controllers description access-controllers is an optional property that allows to refer to domain access controller. The RCC driver will be able to check if we are allowed to register clocks for a peripheral. Signed-off-by: Gabriel Fernandez Link: https://lore.kernel.org/r/20240419152723.570159-2-gabriel.fernandez@foss.st.com Acked-by: Rob Herring (Arm) Signed-off-by: Stephen Boyd --- Documentation/devicetree/bindings/clock/st,stm32mp25-rcc.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/devicetree/bindings/clock/st,stm32mp25-rcc.yaml b/Documentation/devicetree/bindings/clock/st,stm32mp25-rcc.yaml index 0929fa7e271b3..88e52f10d1ecc 100644 --- a/Documentation/devicetree/bindings/clock/st,stm32mp25-rcc.yaml +++ b/Documentation/devicetree/bindings/clock/st,stm32mp25-rcc.yaml @@ -114,6 +114,10 @@ properties: - description: CK_SCMI_PLL3 PLL3 clock - description: clk_dsi_txbyte DSI byte clock + access-controllers: + minItems: 1 + maxItems: 2 + required: - compatible - reg From e00f2540a581f8b8c165e5ae8afe52e4ad038550 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 18 Mar 2024 08:18:10 -0700 Subject: [PATCH 1320/1702] clk: qcom: Fix SC_CAMCC_8280XP dependencies CONFIG_SC_GCC_8280XP depends on ARM64 but it is selected by CONFIG_SC_CAMCC_8280XP, which can be selected on ARM, resulting in a Kconfig warning. WARNING: unmet direct dependencies detected for SC_GCC_8280XP Depends on [n]: COMMON_CLK [=y] && COMMON_CLK_QCOM [=y] && (ARM64 || COMPILE_TEST [=n]) Selected by [y]: - SC_CAMCC_8280XP [=y] && COMMON_CLK [=y] && COMMON_CLK_QCOM [=y] Add the same dependencies to CONFIG_SC_CAMCC_8280XP to resolve the warning. Fixes: ff93872a9c61 ("clk: qcom: camcc-sc8280xp: Add sc8280xp CAMCC") Signed-off-by: Nathan Chancellor Reviewed-by: Dmitry Baryshkov Link: https://lore.kernel.org/r/20240318-fix-some-qcom-kconfig-deps-v1-1-ea0773e3df5a@kernel.org Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig index 8ab08e7b5b6c6..b9ff32cab329b 100644 --- a/drivers/clk/qcom/Kconfig +++ b/drivers/clk/qcom/Kconfig @@ -474,6 +474,7 @@ config SC_CAMCC_7280 config SC_CAMCC_8280XP tristate "SC8280XP Camera Clock Controller" + depends on ARM64 || COMPILE_TEST select SC_GCC_8280XP help Support for the camera clock controller on Qualcomm Technologies, Inc From 07fb0a76bb757990b99fc2ab78ad7d1709cc441d Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 18 Mar 2024 08:18:11 -0700 Subject: [PATCH 1321/1702] clk: qcom: Fix SM_GPUCC_8650 dependencies CONFIG_SM_GCC_8650 depends on ARM64 but it is selected by CONFIG_SM_GPUCC_8650, which can be selected on ARM, resulting in a Kconfig warning. WARNING: unmet direct dependencies detected for SM_GCC_8650 Depends on [n]: COMMON_CLK [=y] && COMMON_CLK_QCOM [=y] && (ARM64 || COMPILE_TEST [=n]) Selected by [y]: - SM_GPUCC_8650 [=y] && COMMON_CLK [=y] && COMMON_CLK_QCOM [=y] Add the same dependencies to CONFIG_SM_GPUCC_8650 to resolve the warning. Fixes: 8676fd4f3874 ("clk: qcom: add the SM8650 GPU Clock Controller driver") Signed-off-by: Nathan Chancellor Reviewed-by: Dmitry Baryshkov Link: https://lore.kernel.org/r/20240318-fix-some-qcom-kconfig-deps-v1-2-ea0773e3df5a@kernel.org Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig index b9ff32cab329b..1bb51a0588726 100644 --- a/drivers/clk/qcom/Kconfig +++ b/drivers/clk/qcom/Kconfig @@ -1095,6 +1095,7 @@ config SM_GPUCC_8550 config SM_GPUCC_8650 tristate "SM8650 Graphics Clock Controller" + depends on ARM64 || COMPILE_TEST select SM_GCC_8650 help Support for the graphics clock controller on SM8650 devices. From c55f7ee2ec239b6afd8639c7ac06493876deb0ea Mon Sep 17 00:00:00 2001 From: Gabor Juhos Date: Tue, 26 Mar 2024 14:34:11 +0100 Subject: [PATCH 1322/1702] clk: qcom: apss-ipq-pll: fix PLL rate for IPQ5018 According to ipq5018.dtsi, the maximum supported rate by the CPU is 1.008 GHz on the IPQ5018 platform, however the current configuration of the PLL results in 1.2 GHz rate. Change the 'L' value in the PLL configuration to limit the rate to 1.008 GHz. The downstream kernel also uses the same value [1]. Also add a comment to indicate the desired frequency. [1] https://git.codelinaro.org/clo/qsdk/oss/kernel/linux-ipq-5.4/-/blob/NHSS.QSDK.12.4/drivers/clk/qcom/apss-ipq5018.c?ref_type=heads#L151 Fixes: 50492f929486 ("clk: qcom: apss-ipq-pll: add support for IPQ5018") Signed-off-by: Gabor Juhos Reviewed-by: Konrad Dybcio Link: https://lore.kernel.org/r/20240326-fix-ipq5018-apss-pll-rate-v1-1-82ab31c9da7e@gmail.com Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/apss-ipq-pll.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/clk/qcom/apss-ipq-pll.c b/drivers/clk/qcom/apss-ipq-pll.c index 1a6f4db253796..5f7f537e4ecbe 100644 --- a/drivers/clk/qcom/apss-ipq-pll.c +++ b/drivers/clk/qcom/apss-ipq-pll.c @@ -66,8 +66,9 @@ static struct clk_alpha_pll ipq_pll_stromer_plus = { }, }; +/* 1.008 GHz configuration */ static const struct alpha_pll_config ipq5018_pll_config = { - .l = 0x32, + .l = 0x2a, .config_ctl_val = 0x4001075b, .config_ctl_hi_val = 0x304, .main_output_mask = BIT(0), From 3c5b3e17b8fd1f1add5a9477306c355fab126977 Mon Sep 17 00:00:00 2001 From: Gabor Juhos Date: Thu, 28 Mar 2024 08:54:31 +0100 Subject: [PATCH 1323/1702] clk: qcom: clk-alpha-pll: fix rate setting for Stromer PLLs The clk_alpha_pll_stromer_set_rate() function writes inproper values into the ALPHA_VAL{,_U} registers which results in wrong clock rates when the alpha value is used. The broken behaviour can be seen on IPQ5018 for example, when dynamic scaling sets the CPU frequency to 800000 KHz. In this case the CPU cores are running only at 792031 KHz: # cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq 800000 # cat /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_cur_freq 792031 This happens because the function ignores the fact that the alpha value calculated by the alpha_pll_round_rate() function is only 32 bits wide which must be extended to 40 bits if it is used on a hardware which supports 40 bits wide values. Extend the clk_alpha_pll_stromer_set_rate() function to convert the alpha value to 40 bits before wrinting that into the registers in order to ensure that the hardware really uses the requested rate. After the change the CPU frequency is correct: # cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq 800000 # cat /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_cur_freq 800000 Cc: stable@vger.kernel.org Fixes: e47a4f55f240 ("clk: qcom: clk-alpha-pll: Add support for Stromer PLLs") Reviewed-by: Dmitry Baryshkov Signed-off-by: Gabor Juhos Link: https://lore.kernel.org/r/20240328-alpha-pll-fix-stromer-set-rate-v3-1-1b79714c78bc@gmail.com Signed-off-by: Bjorn Andersson --- drivers/clk/qcom/clk-alpha-pll.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index 236794cb3e501..d4227909d1fe1 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -2508,6 +2508,8 @@ static int clk_alpha_pll_stromer_set_rate(struct clk_hw *hw, unsigned long rate, rate = alpha_pll_round_rate(rate, prate, &l, &a, ALPHA_REG_BITWIDTH); regmap_write(pll->clkr.regmap, PLL_L_VAL(pll), l); + + a <<= ALPHA_REG_BITWIDTH - ALPHA_BITWIDTH; regmap_write(pll->clkr.regmap, PLL_ALPHA_VAL(pll), a); regmap_write(pll->clkr.regmap, PLL_ALPHA_VAL_U(pll), a >> ALPHA_BITWIDTH); From 4321de4497b24fbf22389331f4ecd4039a451aa9 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 7 May 2024 13:20:25 +0200 Subject: [PATCH 1324/1702] page_pool: check for DMA sync shortcut earlier We can save a couple more function calls in the Page Pool code if we check for dma_need_sync() earlier, just when we test pp->p.dma_sync. Move both these checks into an inline wrapper and call the PP wrapper over the generic DMA sync function only when both are true. You can't cache the result of dma_need_sync() in &page_pool, as it may change anytime if an SWIOTLB buffer is allocated or mapped. Signed-off-by: Alexander Lobakin Signed-off-by: Christoph Hellwig --- net/core/page_pool.c | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/net/core/page_pool.c b/net/core/page_pool.c index c2819ff03dd25..4f9d1bd7f4d18 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -398,16 +398,26 @@ static struct page *__page_pool_get_cached(struct page_pool *pool) return page; } -static void page_pool_dma_sync_for_device(struct page_pool *pool, - struct page *page, - unsigned int dma_sync_size) +static void __page_pool_dma_sync_for_device(const struct page_pool *pool, + struct page *page, + u32 dma_sync_size) { +#if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC) dma_addr_t dma_addr = page_pool_get_dma_addr(page); dma_sync_size = min(dma_sync_size, pool->p.max_len); - dma_sync_single_range_for_device(pool->p.dev, dma_addr, - pool->p.offset, dma_sync_size, - pool->p.dma_dir); + __dma_sync_single_for_device(pool->p.dev, dma_addr + pool->p.offset, + dma_sync_size, pool->p.dma_dir); +#endif +} + +static __always_inline void +page_pool_dma_sync_for_device(const struct page_pool *pool, + struct page *page, + u32 dma_sync_size) +{ + if (pool->dma_sync && dma_dev_need_sync(pool->p.dev)) + __page_pool_dma_sync_for_device(pool, page, dma_sync_size); } static bool page_pool_dma_map(struct page_pool *pool, struct page *page) @@ -429,8 +439,7 @@ static bool page_pool_dma_map(struct page_pool *pool, struct page *page) if (page_pool_set_dma_addr(page, dma)) goto unmap_failed; - if (pool->dma_sync) - page_pool_dma_sync_for_device(pool, page, pool->p.max_len); + page_pool_dma_sync_for_device(pool, page, pool->p.max_len); return true; @@ -699,9 +708,7 @@ __page_pool_put_page(struct page_pool *pool, struct page *page, if (likely(__page_pool_page_can_be_recycled(page))) { /* Read barrier done in page_ref_count / READ_ONCE */ - if (pool->dma_sync) - page_pool_dma_sync_for_device(pool, page, - dma_sync_size); + page_pool_dma_sync_for_device(pool, page, dma_sync_size); if (allow_direct && in_softirq() && page_pool_recycle_in_cache(page, pool)) @@ -812,9 +819,7 @@ static struct page *page_pool_drain_frag(struct page_pool *pool, return NULL; if (__page_pool_page_can_be_recycled(page)) { - if (pool->dma_sync) - page_pool_dma_sync_for_device(pool, page, -1); - + page_pool_dma_sync_for_device(pool, page, -1); return page; } From 163943ac00cb31ac1a88ce5f78a7e2ead37329ec Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 7 May 2024 13:20:26 +0200 Subject: [PATCH 1325/1702] xsk: use generic DMA sync shortcut instead of a custom one XSk infra's been using its own DMA sync shortcut to try avoiding redundant function calls. Now that there is a generic one, remove the custom implementation and rely on the generic helpers. xsk_buff_dma_sync_for_cpu() doesn't need the second argument anymore, remove it. Signed-off-by: Alexander Lobakin Signed-off-by: Christoph Hellwig --- drivers/net/ethernet/engleder/tsnep_main.c | 2 +- .../net/ethernet/freescale/dpaa2/dpaa2-xsk.c | 2 +- drivers/net/ethernet/intel/i40e/i40e_xsk.c | 2 +- drivers/net/ethernet/intel/ice/ice_xsk.c | 2 +- drivers/net/ethernet/intel/igc/igc_main.c | 2 +- drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 2 +- .../ethernet/mellanox/mlx5/core/en/xsk/rx.c | 4 +-- .../net/ethernet/mellanox/mlx5/core/en_rx.c | 2 +- drivers/net/ethernet/netronome/nfp/nfd3/xsk.c | 2 +- .../net/ethernet/stmicro/stmmac/stmmac_main.c | 2 +- include/net/xdp_sock_drv.h | 7 ++--- include/net/xsk_buff_pool.h | 14 +++------ net/xdp/xsk_buff_pool.c | 29 +++---------------- 13 files changed, 21 insertions(+), 51 deletions(-) diff --git a/drivers/net/ethernet/engleder/tsnep_main.c b/drivers/net/ethernet/engleder/tsnep_main.c index 4b15af6b7122e..44da335d66bda 100644 --- a/drivers/net/ethernet/engleder/tsnep_main.c +++ b/drivers/net/ethernet/engleder/tsnep_main.c @@ -1587,7 +1587,7 @@ static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi, length = __le32_to_cpu(entry->desc_wb->properties) & TSNEP_DESC_LENGTH_MASK; xsk_buff_set_size(entry->xdp, length - ETH_FCS_LEN); - xsk_buff_dma_sync_for_cpu(entry->xdp, rx->xsk_pool); + xsk_buff_dma_sync_for_cpu(entry->xdp); /* RX metadata with timestamps is in front of actual data, * subtract metadata size to get length of actual data and diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk.c index 051748b997f3f..a466c23791461 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk.c @@ -55,7 +55,7 @@ static u32 dpaa2_xsk_run_xdp(struct dpaa2_eth_priv *priv, xdp_set_data_meta_invalid(xdp_buff); xdp_buff->rxq = &ch->xdp_rxq; - xsk_buff_dma_sync_for_cpu(xdp_buff, ch->xsk_pool); + xsk_buff_dma_sync_for_cpu(xdp_buff); xdp_act = bpf_prog_run_xdp(xdp_prog, xdp_buff); /* xdp.data pointer may have changed */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c index 11500003af0d4..d20ce517426e2 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c +++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c @@ -483,7 +483,7 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) bi = *i40e_rx_bi(rx_ring, next_to_process); xsk_buff_set_size(bi, size); - xsk_buff_dma_sync_for_cpu(bi, rx_ring->xsk_pool); + xsk_buff_dma_sync_for_cpu(bi); if (!first) first = bi; diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c index 1857220d27fee..cecd5b1e07570 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.c +++ b/drivers/net/ethernet/intel/ice/ice_xsk.c @@ -879,7 +879,7 @@ int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, int budget) ICE_RX_FLX_DESC_PKT_LEN_M; xsk_buff_set_size(xdp, size); - xsk_buff_dma_sync_for_cpu(xdp, xsk_pool); + xsk_buff_dma_sync_for_cpu(xdp); if (!first) { first = xdp; diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 4d975d620a8e4..07692e2a7c640 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -2813,7 +2813,7 @@ static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget) } bi->xdp->data_end = bi->xdp->data + size; - xsk_buff_dma_sync_for_cpu(bi->xdp, ring->xsk_pool); + xsk_buff_dma_sync_for_cpu(bi->xdp); res = __igc_xdp_run_prog(adapter, prog, bi->xdp); switch (res) { diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index d34d715c59ebc..ee2d0ec12b2dc 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -304,7 +304,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, } bi->xdp->data_end = bi->xdp->data + size; - xsk_buff_dma_sync_for_cpu(bi->xdp, rx_ring->xsk_pool); + xsk_buff_dma_sync_for_cpu(bi->xdp); xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, bi->xdp); if (likely(xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR))) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c index b8dd744536553..1b7132fa70de2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c @@ -270,7 +270,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, /* mxbuf->rq is set on allocation, but cqe is per-packet so set it here */ mxbuf->cqe = cqe; xsk_buff_set_size(&mxbuf->xdp, cqe_bcnt); - xsk_buff_dma_sync_for_cpu(&mxbuf->xdp, rq->xsk_pool); + xsk_buff_dma_sync_for_cpu(&mxbuf->xdp); net_prefetch(mxbuf->xdp.data); /* Possible flows: @@ -319,7 +319,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, /* mxbuf->rq is set on allocation, but cqe is per-packet so set it here */ mxbuf->cqe = cqe; xsk_buff_set_size(&mxbuf->xdp, cqe_bcnt); - xsk_buff_dma_sync_for_cpu(&mxbuf->xdp, rq->xsk_pool); + xsk_buff_dma_sync_for_cpu(&mxbuf->xdp); net_prefetch(mxbuf->xdp.data); prog = rcu_dereference(rq->xdp_prog); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index d601b5faaed5b..b5333da20e8a7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -917,7 +917,7 @@ INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq) if (!rq->xsk_pool) { count = mlx5e_refill_rx_wqes(rq, head, wqe_bulk); - } else if (likely(!rq->xsk_pool->dma_need_sync)) { + } else if (likely(!dma_dev_need_sync(rq->pdev))) { mlx5e_xsk_free_rx_wqes(rq, head, wqe_bulk); count = mlx5e_xsk_alloc_rx_wqes_batched(rq, head, wqe_bulk); } else { diff --git a/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c b/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c index 45be6954d5aae..01cfa9cc1b5e5 100644 --- a/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c +++ b/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c @@ -184,7 +184,7 @@ nfp_nfd3_xsk_rx(struct nfp_net_rx_ring *rx_ring, int budget, xrxbuf->xdp->data += meta_len; xrxbuf->xdp->data_end = xrxbuf->xdp->data + pkt_len; xdp_set_data_meta_invalid(xrxbuf->xdp); - xsk_buff_dma_sync_for_cpu(xrxbuf->xdp, r_vec->xsk_pool); + xsk_buff_dma_sync_for_cpu(xrxbuf->xdp); net_prefetch(xrxbuf->xdp->data); if (meta_len) { diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 7c6fb14b55550..206cba44dd302 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -5352,7 +5352,7 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) /* RX buffer is good and fit into a XSK pool buffer */ buf->xdp->data_end = buf->xdp->data + buf1_len; - xsk_buff_dma_sync_for_cpu(buf->xdp, rx_q->xsk_pool); + xsk_buff_dma_sync_for_cpu(buf->xdp); prog = READ_ONCE(priv->xdp_prog); res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index c9aec9ab61912..0a5dca2b2b3f6 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -219,13 +219,10 @@ static inline struct xsk_tx_metadata *xsk_buff_get_metadata(struct xsk_buff_pool return meta; } -static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool) +static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp) { struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); - if (!pool->dma_need_sync) - return; - xp_dma_sync_for_cpu(xskb); } @@ -402,7 +399,7 @@ static inline struct xsk_tx_metadata *xsk_buff_get_metadata(struct xsk_buff_pool return NULL; } -static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool) +static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp) { } diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h index 99dd7376df6a7..bacb33f1e3e58 100644 --- a/include/net/xsk_buff_pool.h +++ b/include/net/xsk_buff_pool.h @@ -43,7 +43,6 @@ struct xsk_dma_map { refcount_t users; struct list_head list; /* Protected by the RTNL_LOCK */ u32 dma_pages_cnt; - bool dma_need_sync; }; struct xsk_buff_pool { @@ -82,7 +81,6 @@ struct xsk_buff_pool { u8 tx_metadata_len; /* inherited from umem */ u8 cached_need_wakeup; bool uses_need_wakeup; - bool dma_need_sync; bool unaligned; bool tx_sw_csum; void *addrs; @@ -155,21 +153,17 @@ static inline dma_addr_t xp_get_frame_dma(struct xdp_buff_xsk *xskb) return xskb->frame_dma; } -void xp_dma_sync_for_cpu_slow(struct xdp_buff_xsk *xskb); static inline void xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb) { - xp_dma_sync_for_cpu_slow(xskb); + dma_sync_single_for_cpu(xskb->pool->dev, xskb->dma, + xskb->pool->frame_len, + DMA_BIDIRECTIONAL); } -void xp_dma_sync_for_device_slow(struct xsk_buff_pool *pool, dma_addr_t dma, - size_t size); static inline void xp_dma_sync_for_device(struct xsk_buff_pool *pool, dma_addr_t dma, size_t size) { - if (!pool->dma_need_sync) - return; - - xp_dma_sync_for_device_slow(pool, dma, size); + dma_sync_single_for_device(pool->dev, dma, size, DMA_BIDIRECTIONAL); } /* Masks for xdp_umem_page flags. diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c index ce60ecd48a4dc..c0e0204b96304 100644 --- a/net/xdp/xsk_buff_pool.c +++ b/net/xdp/xsk_buff_pool.c @@ -338,7 +338,6 @@ static struct xsk_dma_map *xp_create_dma_map(struct device *dev, struct net_devi dma_map->netdev = netdev; dma_map->dev = dev; - dma_map->dma_need_sync = false; dma_map->dma_pages_cnt = nr_pages; refcount_set(&dma_map->users, 1); list_add(&dma_map->list, &umem->xsk_dma_list); @@ -424,7 +423,6 @@ static int xp_init_dma_info(struct xsk_buff_pool *pool, struct xsk_dma_map *dma_ pool->dev = dma_map->dev; pool->dma_pages_cnt = dma_map->dma_pages_cnt; - pool->dma_need_sync = dma_map->dma_need_sync; memcpy(pool->dma_pages, dma_map->dma_pages, pool->dma_pages_cnt * sizeof(*pool->dma_pages)); @@ -460,8 +458,6 @@ int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev, __xp_dma_unmap(dma_map, attrs); return -ENOMEM; } - if (dma_need_sync(dev, dma)) - dma_map->dma_need_sync = true; dma_map->dma_pages[i] = dma; } @@ -557,11 +553,9 @@ struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool) xskb->xdp.data_meta = xskb->xdp.data; xskb->xdp.flags = 0; - if (pool->dma_need_sync) { - dma_sync_single_range_for_device(pool->dev, xskb->dma, 0, - pool->frame_len, - DMA_BIDIRECTIONAL); - } + if (pool->dev) + xp_dma_sync_for_device(pool, xskb->dma, pool->frame_len); + return &xskb->xdp; } EXPORT_SYMBOL(xp_alloc); @@ -633,7 +627,7 @@ u32 xp_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max) { u32 nb_entries1 = 0, nb_entries2; - if (unlikely(pool->dma_need_sync)) { + if (unlikely(pool->dev && dma_dev_need_sync(pool->dev))) { struct xdp_buff *buff; /* Slow path */ @@ -693,18 +687,3 @@ dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr) (addr & ~PAGE_MASK); } EXPORT_SYMBOL(xp_raw_get_dma); - -void xp_dma_sync_for_cpu_slow(struct xdp_buff_xsk *xskb) -{ - dma_sync_single_range_for_cpu(xskb->pool->dev, xskb->dma, 0, - xskb->pool->frame_len, DMA_BIDIRECTIONAL); -} -EXPORT_SYMBOL(xp_dma_sync_for_cpu_slow); - -void xp_dma_sync_for_device_slow(struct xsk_buff_pool *pool, dma_addr_t dma, - size_t size) -{ - dma_sync_single_range_for_device(pool->dev, dma, 0, - size, DMA_BIDIRECTIONAL); -} -EXPORT_SYMBOL(xp_dma_sync_for_device_slow); From 9d351132ed706ae24325809afa821cabf6d72568 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 8 May 2024 09:47:46 +0200 Subject: [PATCH 1326/1702] perf/x86/cstate: Remove unused 'struct perf_cstate_msr' Use of this structure was removed in: 8f2a28c5859b ("perf/x86/cstate: Use new probe function") Remove the now stale type as well. Signed-off-by: Ingo Molnar Cc: linux-kernel@vger.kernel.org Cc: Peter Zijlstra Cc: Arnaldo Carvalho de Melo Cc: Namhyung Kim Signed-off-by: Ingo Molnar --- arch/x86/events/intel/cstate.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index 54eb142810fbb..e64eaa8dda5a4 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -143,12 +143,6 @@ struct cstate_model { #define SLM_PKG_C6_USE_C7_MSR (1UL << 0) #define KNL_CORE_C6_MSR (1UL << 1) -struct perf_cstate_msr { - u64 msr; - struct perf_pmu_events_attr *attr; -}; - - /* cstate_core PMU */ static struct pmu cstate_core_pmu; static bool has_cstate_core; From 2b8af5001abdf583da3a63201cc6137553019515 Mon Sep 17 00:00:00 2001 From: Michael Margolin Date: Mon, 6 May 2024 15:18:29 +0000 Subject: [PATCH 1327/1702] RDMA/efa: Support QP with unsolicited write w/ imm. receive Add a new EFA flags attribute for QP creation, and support unsolicited write with immediate flag. QPs created with this flag set will not consume receive work requests for incoming RDMA write with immediate. Expose device capability bit for this feature support. Reviewed-by: Daniel Kranzdorf Reviewed-by: Firas Jahjah Signed-off-by: Michael Margolin Link: https://lore.kernel.org/r/20240506151829.6475-1-mrgolin@amazon.com Signed-off-by: Leon Romanovsky --- .../infiniband/hw/efa/efa_admin_cmds_defs.h | 11 +++++++++-- drivers/infiniband/hw/efa/efa_com_cmd.c | 3 +++ drivers/infiniband/hw/efa/efa_com_cmd.h | 1 + drivers/infiniband/hw/efa/efa_verbs.c | 19 ++++++++++++++++++- include/uapi/rdma/efa-abi.h | 7 +++++++ 5 files changed, 38 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h index 7377c8a9f4d5d..4296662e59c39 100644 --- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h @@ -110,7 +110,10 @@ struct efa_admin_create_qp_cmd { * virtual (IOVA returned by MR registration) * 1 : rq_virt - If set, RQ ring base address is * virtual (IOVA returned by MR registration) - * 7:2 : reserved - MBZ + * 2 : unsolicited_write_recv - If set, work requests + * will not be consumed for incoming RDMA write with + * immediate + * 7:3 : reserved - MBZ */ u8 flags; @@ -663,7 +666,9 @@ struct efa_admin_feature_device_attr_desc { * polling is supported * 3 : rdma_write - If set, RDMA Write is supported * on TX queues - * 31:4 : reserved - MBZ + * 4 : unsolicited_write_recv - If set, unsolicited + * write with imm. receive is supported + * 31:5 : reserved - MBZ */ u32 device_caps; @@ -1009,6 +1014,7 @@ struct efa_admin_host_info { /* create_qp_cmd */ #define EFA_ADMIN_CREATE_QP_CMD_SQ_VIRT_MASK BIT(0) #define EFA_ADMIN_CREATE_QP_CMD_RQ_VIRT_MASK BIT(1) +#define EFA_ADMIN_CREATE_QP_CMD_UNSOLICITED_WRITE_RECV_MASK BIT(2) /* modify_qp_cmd */ #define EFA_ADMIN_MODIFY_QP_CMD_QP_STATE_MASK BIT(0) @@ -1044,6 +1050,7 @@ struct efa_admin_host_info { #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RNR_RETRY_MASK BIT(1) #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_DATA_POLLING_128_MASK BIT(2) #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_WRITE_MASK BIT(3) +#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_UNSOLICITED_WRITE_RECV_MASK BIT(4) /* create_eq_cmd */ #define EFA_ADMIN_CREATE_EQ_CMD_ENTRY_SIZE_WORDS_MASK GENMASK(4, 0) diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c index d3398c7b0bd0f..5b9c2b16df0e5 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.c +++ b/drivers/infiniband/hw/efa/efa_com_cmd.c @@ -32,6 +32,9 @@ int efa_com_create_qp(struct efa_com_dev *edev, params->rq_depth; create_qp_cmd.uar = params->uarn; + if (params->unsolicited_write_recv) + EFA_SET(&create_qp_cmd.flags, EFA_ADMIN_CREATE_QP_CMD_UNSOLICITED_WRITE_RECV, 1); + err = efa_com_cmd_exec(aq, (struct efa_admin_aq_entry *)&create_qp_cmd, sizeof(create_qp_cmd), diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h index 720a99ba0f7d1..9714105fcf7ec 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.h +++ b/drivers/infiniband/hw/efa/efa_com_cmd.h @@ -27,6 +27,7 @@ struct efa_com_create_qp_params { u16 pd; u16 uarn; u8 qp_type; + u8 unsolicited_write_recv : 1; }; struct efa_com_create_qp_result { diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 2f412db2edcd3..8f7a13b79cdc5 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -263,6 +263,9 @@ int efa_query_device(struct ib_device *ibdev, if (EFA_DEV_CAP(dev, RDMA_WRITE)) resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RDMA_WRITE; + if (EFA_DEV_CAP(dev, UNSOLICITED_WRITE_RECV)) + resp.device_caps |= EFA_QUERY_DEVICE_CAPS_UNSOLICITED_WRITE_RECV; + if (dev->neqs) resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_NOTIFICATIONS; @@ -639,6 +642,7 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, struct efa_ibv_create_qp cmd = {}; struct efa_qp *qp = to_eqp(ibqp); struct efa_ucontext *ucontext; + u16 supported_efa_flags = 0; int err; ucontext = rdma_udata_to_drv_context(udata, struct efa_ucontext, @@ -676,13 +680,23 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, goto err_out; } - if (cmd.comp_mask) { + if (cmd.comp_mask || !is_reserved_cleared(cmd.reserved_90)) { ibdev_dbg(&dev->ibdev, "Incompatible ABI params, unknown fields in udata\n"); err = -EINVAL; goto err_out; } + if (EFA_DEV_CAP(dev, UNSOLICITED_WRITE_RECV)) + supported_efa_flags |= EFA_CREATE_QP_WITH_UNSOLICITED_WRITE_RECV; + + if (cmd.flags & ~supported_efa_flags) { + ibdev_dbg(&dev->ibdev, "Unsupported EFA QP create flags[%#x], supported[%#x]\n", + cmd.flags, supported_efa_flags); + err = -EOPNOTSUPP; + goto err_out; + } + create_qp_params.uarn = ucontext->uarn; create_qp_params.pd = to_epd(ibqp->pd)->pdn; @@ -722,6 +736,9 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, create_qp_params.rq_base_addr = qp->rq_dma_addr; } + if (cmd.flags & EFA_CREATE_QP_WITH_UNSOLICITED_WRITE_RECV) + create_qp_params.unsolicited_write_recv = true; + err = efa_com_create_qp(&dev->edev, &create_qp_params, &create_qp_resp); if (err) diff --git a/include/uapi/rdma/efa-abi.h b/include/uapi/rdma/efa-abi.h index 701e2d567e411..d689b8b34189d 100644 --- a/include/uapi/rdma/efa-abi.h +++ b/include/uapi/rdma/efa-abi.h @@ -85,11 +85,17 @@ enum { EFA_QP_DRIVER_TYPE_SRD = 0, }; +enum { + EFA_CREATE_QP_WITH_UNSOLICITED_WRITE_RECV = 1 << 0, +}; + struct efa_ibv_create_qp { __u32 comp_mask; __u32 rq_ring_size; /* bytes */ __u32 sq_ring_size; /* bytes */ __u32 driver_qp_type; + __u16 flags; + __u8 reserved_90[6]; }; struct efa_ibv_create_qp_resp { @@ -123,6 +129,7 @@ enum { EFA_QUERY_DEVICE_CAPS_CQ_WITH_SGID = 1 << 3, EFA_QUERY_DEVICE_CAPS_DATA_POLLING_128 = 1 << 4, EFA_QUERY_DEVICE_CAPS_RDMA_WRITE = 1 << 5, + EFA_QUERY_DEVICE_CAPS_UNSOLICITED_WRITE_RECV = 1 << 6, }; struct efa_ibv_ex_query_device_resp { From 055e09ac54ae9c8396c1086fe06a73e0ce9bdd10 Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Wed, 17 Apr 2024 23:11:23 +0300 Subject: [PATCH 1328/1702] cpumask: delete unused reset_cpu_possible_mask() Link: https://lkml.kernel.org/r/20240417201123.2961-1-adobriyan@gmail.com Signed-off-by: Alexey Dobriyan Cc: Rasmus Villemoes Cc: Yury Norov Signed-off-by: Andrew Morton --- include/linux/cpumask.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 1c29947db8489..04536a29f10fc 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -1017,11 +1017,6 @@ void init_cpu_present(const struct cpumask *src); void init_cpu_possible(const struct cpumask *src); void init_cpu_online(const struct cpumask *src); -static inline void reset_cpu_possible_mask(void) -{ - bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS); -} - static inline void set_cpu_possible(unsigned int cpu, bool possible) { From 4707c13de3e42a47f0d99fe5fb58fa9dd23b455e Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Thu, 18 Apr 2024 11:58:43 +0800 Subject: [PATCH 1329/1702] crash: add prefix for crash dumping messages Add pr_fmt() to kernel/crash_core.c to add the module name to debugging message printed as prefix. And also add prefix 'crashkernel:' to two lines of message printing code in kernel/crash_reserve.c. In kernel/crash_reserve.c, almost all debugging messages have 'crashkernel:' prefix or there's keyword crashkernel at the beginning or in the middle, adding pr_fmt() makes it redundant. Link: https://lkml.kernel.org/r/20240418035843.1562887-1-bhe@redhat.com Signed-off-by: Baoquan He Cc: Dave Young Cc: Jiri Slaby Signed-off-by: Andrew Morton --- kernel/crash_core.c | 2 ++ kernel/crash_reserve.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 78b5dc7cee3ab..1e7ac977f7c0b 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -4,6 +4,8 @@ * Copyright (C) 2002-2004 Eric Biederman */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include #include #include diff --git a/kernel/crash_reserve.c b/kernel/crash_reserve.c index 066668799f757..5b2722a93a48c 100644 --- a/kernel/crash_reserve.c +++ b/kernel/crash_reserve.c @@ -109,7 +109,7 @@ static int __init parse_crashkernel_mem(char *cmdline, size = memparse(cur, &tmp); if (cur == tmp) { - pr_warn("Memory value expected\n"); + pr_warn("crashkernel: Memory value expected\n"); return -EINVAL; } cur = tmp; @@ -132,7 +132,7 @@ static int __init parse_crashkernel_mem(char *cmdline, cur++; *crash_base = memparse(cur, &tmp); if (cur == tmp) { - pr_warn("Memory value expected after '@'\n"); + pr_warn("crahskernel: Memory value expected after '@'\n"); return -EINVAL; } } From f4af41bf177add167e39e4b0203460b1d0b531f6 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Tue, 9 Apr 2024 12:22:38 +0800 Subject: [PATCH 1330/1702] kexec: fix the unexpected kexec_dprintk() macro Jiri reported that the current kexec_dprintk() always prints out debugging message whenever kexec/kdmmp loading is triggered. That is not wanted. The debugging message is supposed to be printed out when 'kexec -s -d' is specified for kexec/kdump loading. After investigating, the reason is the current kexec_dprintk() takes printk(KERN_INFO) or printk(KERN_DEBUG) depending on whether '-d' is specified. However, distros usually have defaulg log level like below: [~]# cat /proc/sys/kernel/printk 7 4 1 7 So, even though '-d' is not specified, printk(KERN_DEBUG) also always prints out. I thought printk(KERN_DEBUG) is equal to pr_debug(), it's not. Fix it by changing to use pr_info() instead which are expected to work. Link: https://lkml.kernel.org/r/20240409042238.1240462-1-bhe@redhat.com Fixes: cbc2fe9d9cb2 ("kexec_file: add kexec_file flag to control debug printing") Signed-off-by: Baoquan He Reported-by: Jiri Slaby Closes: https://lore.kernel.org/all/4c775fca-5def-4a2d-8437-7130b02722a2@kernel.org Reviewed-by: Dave Young Cc: Signed-off-by: Andrew Morton --- include/linux/kexec.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 060835bb82d52..f31bd304df451 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -461,10 +461,8 @@ static inline void arch_kexec_pre_free_pages(void *vaddr, unsigned int pages) { extern bool kexec_file_dbg_print; -#define kexec_dprintk(fmt, ...) \ - printk("%s" fmt, \ - kexec_file_dbg_print ? KERN_INFO : KERN_DEBUG, \ - ##__VA_ARGS__) +#define kexec_dprintk(fmt, arg...) \ + do { if (kexec_file_dbg_print) pr_info(fmt, ##arg); } while (0) #else /* !CONFIG_KEXEC_CORE */ struct pt_regs; From 36defdd9d7c605ba042962ef1eeb66a9d3ff5884 Mon Sep 17 00:00:00 2001 From: Eric Sandeen Date: Thu, 25 Apr 2024 03:27:16 +0900 Subject: [PATCH 1331/1702] nilfs2: convert to use the new mount API Convert nilfs2 to use the new mount API. [sandeen@redhat.com: v2] Link: https://lkml.kernel.org/r/33d078a7-9072-4d8e-a3a9-dec23d4191da@redhat.com Link: https://lkml.kernel.org/r/20240425190526.10905-1-konishi.ryusuke@gmail.com [konishi.ryusuke: fixed missing SB_RDONLY flag repair in nilfs_reconfigure] Link: https://lkml.kernel.org/r/33d078a7-9072-4d8e-a3a9-dec23d4191da@redhat.com Link: https://lkml.kernel.org/r/20240424182716.6024-1-konishi.ryusuke@gmail.com Signed-off-by: Eric Sandeen Signed-off-by: Ryusuke Konishi Signed-off-by: Andrew Morton --- fs/nilfs2/nilfs.h | 4 +- fs/nilfs2/super.c | 388 ++++++++++++++++++------------------------ fs/nilfs2/the_nilfs.c | 5 +- fs/nilfs2/the_nilfs.h | 6 +- 4 files changed, 174 insertions(+), 229 deletions(-) diff --git a/fs/nilfs2/nilfs.h b/fs/nilfs2/nilfs.h index 2e29b98ba8bab..728e90be3570b 100644 --- a/fs/nilfs2/nilfs.h +++ b/fs/nilfs2/nilfs.h @@ -335,8 +335,8 @@ void __nilfs_error(struct super_block *sb, const char *function, extern struct nilfs_super_block * nilfs_read_super_block(struct super_block *, u64, int, struct buffer_head **); -extern int nilfs_store_magic_and_option(struct super_block *, - struct nilfs_super_block *, char *); +extern int nilfs_store_magic(struct super_block *sb, + struct nilfs_super_block *sbp); extern int nilfs_check_feature_compatibility(struct super_block *, struct nilfs_super_block *); extern void nilfs_set_log_cursor(struct nilfs_super_block *, diff --git a/fs/nilfs2/super.c b/fs/nilfs2/super.c index ac24ed109ce93..e835e1f5a7120 100644 --- a/fs/nilfs2/super.c +++ b/fs/nilfs2/super.c @@ -29,13 +29,13 @@ #include #include #include -#include #include #include #include #include #include #include +#include #include "nilfs.h" #include "export.h" #include "mdt.h" @@ -61,7 +61,6 @@ struct kmem_cache *nilfs_segbuf_cachep; struct kmem_cache *nilfs_btree_path_cache; static int nilfs_setup_super(struct super_block *sb, int is_mount); -static int nilfs_remount(struct super_block *sb, int *flags, char *data); void __nilfs_msg(struct super_block *sb, const char *fmt, ...) { @@ -702,105 +701,98 @@ static const struct super_operations nilfs_sops = { .freeze_fs = nilfs_freeze, .unfreeze_fs = nilfs_unfreeze, .statfs = nilfs_statfs, - .remount_fs = nilfs_remount, .show_options = nilfs_show_options }; enum { - Opt_err_cont, Opt_err_panic, Opt_err_ro, - Opt_barrier, Opt_nobarrier, Opt_snapshot, Opt_order, Opt_norecovery, - Opt_discard, Opt_nodiscard, Opt_err, + Opt_err, Opt_barrier, Opt_snapshot, Opt_order, Opt_norecovery, + Opt_discard, }; -static match_table_t tokens = { - {Opt_err_cont, "errors=continue"}, - {Opt_err_panic, "errors=panic"}, - {Opt_err_ro, "errors=remount-ro"}, - {Opt_barrier, "barrier"}, - {Opt_nobarrier, "nobarrier"}, - {Opt_snapshot, "cp=%u"}, - {Opt_order, "order=%s"}, - {Opt_norecovery, "norecovery"}, - {Opt_discard, "discard"}, - {Opt_nodiscard, "nodiscard"}, - {Opt_err, NULL} +static const struct constant_table nilfs_param_err[] = { + {"continue", NILFS_MOUNT_ERRORS_CONT}, + {"panic", NILFS_MOUNT_ERRORS_PANIC}, + {"remount-ro", NILFS_MOUNT_ERRORS_RO}, + {} }; -static int parse_options(char *options, struct super_block *sb, int is_remount) -{ - struct the_nilfs *nilfs = sb->s_fs_info; - char *p; - substring_t args[MAX_OPT_ARGS]; - - if (!options) - return 1; - - while ((p = strsep(&options, ",")) != NULL) { - int token; +static const struct fs_parameter_spec nilfs_param_spec[] = { + fsparam_enum ("errors", Opt_err, nilfs_param_err), + fsparam_flag_no ("barrier", Opt_barrier), + fsparam_u64 ("cp", Opt_snapshot), + fsparam_string ("order", Opt_order), + fsparam_flag ("norecovery", Opt_norecovery), + fsparam_flag_no ("discard", Opt_discard), + {} +}; - if (!*p) - continue; +struct nilfs_fs_context { + unsigned long ns_mount_opt; + __u64 cno; +}; - token = match_token(p, tokens, args); - switch (token) { - case Opt_barrier: - nilfs_set_opt(nilfs, BARRIER); - break; - case Opt_nobarrier: +static int nilfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct nilfs_fs_context *nilfs = fc->fs_private; + int is_remount = fc->purpose == FS_CONTEXT_FOR_RECONFIGURE; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, nilfs_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_barrier: + if (result.negated) nilfs_clear_opt(nilfs, BARRIER); - break; - case Opt_order: - if (strcmp(args[0].from, "relaxed") == 0) - /* Ordered data semantics */ - nilfs_clear_opt(nilfs, STRICT_ORDER); - else if (strcmp(args[0].from, "strict") == 0) - /* Strict in-order semantics */ - nilfs_set_opt(nilfs, STRICT_ORDER); - else - return 0; - break; - case Opt_err_panic: - nilfs_write_opt(nilfs, ERROR_MODE, ERRORS_PANIC); - break; - case Opt_err_ro: - nilfs_write_opt(nilfs, ERROR_MODE, ERRORS_RO); - break; - case Opt_err_cont: - nilfs_write_opt(nilfs, ERROR_MODE, ERRORS_CONT); - break; - case Opt_snapshot: - if (is_remount) { - nilfs_err(sb, - "\"%s\" option is invalid for remount", - p); - return 0; - } - break; - case Opt_norecovery: - nilfs_set_opt(nilfs, NORECOVERY); - break; - case Opt_discard: - nilfs_set_opt(nilfs, DISCARD); - break; - case Opt_nodiscard: - nilfs_clear_opt(nilfs, DISCARD); - break; - default: - nilfs_err(sb, "unrecognized mount option \"%s\"", p); - return 0; + else + nilfs_set_opt(nilfs, BARRIER); + break; + case Opt_order: + if (strcmp(param->string, "relaxed") == 0) + /* Ordered data semantics */ + nilfs_clear_opt(nilfs, STRICT_ORDER); + else if (strcmp(param->string, "strict") == 0) + /* Strict in-order semantics */ + nilfs_set_opt(nilfs, STRICT_ORDER); + else + return -EINVAL; + break; + case Opt_err: + nilfs->ns_mount_opt &= ~NILFS_MOUNT_ERROR_MODE; + nilfs->ns_mount_opt |= result.uint_32; + break; + case Opt_snapshot: + if (is_remount) { + struct super_block *sb = fc->root->d_sb; + + nilfs_err(sb, + "\"%s\" option is invalid for remount", + param->key); + return -EINVAL; + } + if (result.uint_64 == 0) { + nilfs_err(NULL, + "invalid option \"cp=0\": invalid checkpoint number 0"); + return -EINVAL; } + nilfs->cno = result.uint_64; + break; + case Opt_norecovery: + nilfs_set_opt(nilfs, NORECOVERY); + break; + case Opt_discard: + if (result.negated) + nilfs_clear_opt(nilfs, DISCARD); + else + nilfs_set_opt(nilfs, DISCARD); + break; + default: + return -EINVAL; } - return 1; -} - -static inline void -nilfs_set_default_options(struct super_block *sb, - struct nilfs_super_block *sbp) -{ - struct the_nilfs *nilfs = sb->s_fs_info; - nilfs->ns_mount_opt = - NILFS_MOUNT_ERRORS_RO | NILFS_MOUNT_BARRIER; + return 0; } static int nilfs_setup_super(struct super_block *sb, int is_mount) @@ -857,9 +849,8 @@ struct nilfs_super_block *nilfs_read_super_block(struct super_block *sb, return (struct nilfs_super_block *)((char *)(*pbh)->b_data + offset); } -int nilfs_store_magic_and_option(struct super_block *sb, - struct nilfs_super_block *sbp, - char *data) +int nilfs_store_magic(struct super_block *sb, + struct nilfs_super_block *sbp) { struct the_nilfs *nilfs = sb->s_fs_info; @@ -870,14 +861,12 @@ int nilfs_store_magic_and_option(struct super_block *sb, sb->s_flags |= SB_NOATIME; #endif - nilfs_set_default_options(sb, sbp); - nilfs->ns_resuid = le16_to_cpu(sbp->s_def_resuid); nilfs->ns_resgid = le16_to_cpu(sbp->s_def_resgid); nilfs->ns_interval = le32_to_cpu(sbp->s_c_interval); nilfs->ns_watermark = le32_to_cpu(sbp->s_c_block_max); - return !parse_options(data, sb, 0) ? -EINVAL : 0; + return 0; } int nilfs_check_feature_compatibility(struct super_block *sb, @@ -1035,17 +1024,17 @@ int nilfs_checkpoint_is_mounted(struct super_block *sb, __u64 cno) /** * nilfs_fill_super() - initialize a super block instance * @sb: super_block - * @data: mount options - * @silent: silent mode flag + * @fc: filesystem context * * This function is called exclusively by nilfs->ns_mount_mutex. * So, the recovery process is protected from other simultaneous mounts. */ static int -nilfs_fill_super(struct super_block *sb, void *data, int silent) +nilfs_fill_super(struct super_block *sb, struct fs_context *fc) { struct the_nilfs *nilfs; struct nilfs_root *fsroot; + struct nilfs_fs_context *ctx = fc->fs_private; __u64 cno; int err; @@ -1055,10 +1044,13 @@ nilfs_fill_super(struct super_block *sb, void *data, int silent) sb->s_fs_info = nilfs; - err = init_nilfs(nilfs, sb, (char *)data); + err = init_nilfs(nilfs, sb); if (err) goto failed_nilfs; + /* Copy in parsed mount options */ + nilfs->ns_mount_opt = ctx->ns_mount_opt; + sb->s_op = &nilfs_sops; sb->s_export_op = &nilfs_export_ops; sb->s_root = NULL; @@ -1117,34 +1109,25 @@ nilfs_fill_super(struct super_block *sb, void *data, int silent) return err; } -static int nilfs_remount(struct super_block *sb, int *flags, char *data) +static int nilfs_reconfigure(struct fs_context *fc) { + struct nilfs_fs_context *ctx = fc->fs_private; + struct super_block *sb = fc->root->d_sb; struct the_nilfs *nilfs = sb->s_fs_info; - unsigned long old_sb_flags; - unsigned long old_mount_opt; int err; sync_filesystem(sb); - old_sb_flags = sb->s_flags; - old_mount_opt = nilfs->ns_mount_opt; - - if (!parse_options(data, sb, 1)) { - err = -EINVAL; - goto restore_opts; - } - sb->s_flags = (sb->s_flags & ~SB_POSIXACL); err = -EINVAL; if (!nilfs_valid_fs(nilfs)) { nilfs_warn(sb, "couldn't remount because the filesystem is in an incomplete recovery state"); - goto restore_opts; + goto ignore_opts; } - - if ((bool)(*flags & SB_RDONLY) == sb_rdonly(sb)) + if ((bool)(fc->sb_flags & SB_RDONLY) == sb_rdonly(sb)) goto out; - if (*flags & SB_RDONLY) { + if (fc->sb_flags & SB_RDONLY) { sb->s_flags |= SB_RDONLY; /* @@ -1172,138 +1155,67 @@ static int nilfs_remount(struct super_block *sb, int *flags, char *data) "couldn't remount RDWR because of unsupported optional features (%llx)", (unsigned long long)features); err = -EROFS; - goto restore_opts; + goto ignore_opts; } sb->s_flags &= ~SB_RDONLY; root = NILFS_I(d_inode(sb->s_root))->i_root; err = nilfs_attach_log_writer(sb, root); - if (err) - goto restore_opts; + if (err) { + sb->s_flags |= SB_RDONLY; + goto ignore_opts; + } down_write(&nilfs->ns_sem); nilfs_setup_super(sb, true); up_write(&nilfs->ns_sem); } out: - return 0; - - restore_opts: - sb->s_flags = old_sb_flags; - nilfs->ns_mount_opt = old_mount_opt; - return err; -} - -struct nilfs_super_data { - __u64 cno; - int flags; -}; - -static int nilfs_parse_snapshot_option(const char *option, - const substring_t *arg, - struct nilfs_super_data *sd) -{ - unsigned long long val; - const char *msg = NULL; - int err; - - if (!(sd->flags & SB_RDONLY)) { - msg = "read-only option is not specified"; - goto parse_error; - } - - err = kstrtoull(arg->from, 0, &val); - if (err) { - if (err == -ERANGE) - msg = "too large checkpoint number"; - else - msg = "malformed argument"; - goto parse_error; - } else if (val == 0) { - msg = "invalid checkpoint number 0"; - goto parse_error; - } - sd->cno = val; - return 0; - -parse_error: - nilfs_err(NULL, "invalid option \"%s\": %s", option, msg); - return 1; -} - -/** - * nilfs_identify - pre-read mount options needed to identify mount instance - * @data: mount options - * @sd: nilfs_super_data - */ -static int nilfs_identify(char *data, struct nilfs_super_data *sd) -{ - char *p, *options = data; - substring_t args[MAX_OPT_ARGS]; - int token; - int ret = 0; - - do { - p = strsep(&options, ","); - if (p != NULL && *p) { - token = match_token(p, tokens, args); - if (token == Opt_snapshot) - ret = nilfs_parse_snapshot_option(p, &args[0], - sd); - } - if (!options) - break; - BUG_ON(options == data); - *(options - 1) = ','; - } while (!ret); - return ret; -} + sb->s_flags = (sb->s_flags & ~SB_POSIXACL); + /* Copy over parsed remount options */ + nilfs->ns_mount_opt = ctx->ns_mount_opt; -static int nilfs_set_bdev_super(struct super_block *s, void *data) -{ - s->s_dev = *(dev_t *)data; return 0; -} -static int nilfs_test_bdev_super(struct super_block *s, void *data) -{ - return !(s->s_iflags & SB_I_RETIRED) && s->s_dev == *(dev_t *)data; + ignore_opts: + return err; } -static struct dentry * -nilfs_mount(struct file_system_type *fs_type, int flags, - const char *dev_name, void *data) +static int +nilfs_get_tree(struct fs_context *fc) { - struct nilfs_super_data sd = { .flags = flags }; + struct nilfs_fs_context *ctx = fc->fs_private; struct super_block *s; dev_t dev; int err; - if (nilfs_identify(data, &sd)) - return ERR_PTR(-EINVAL); + if (ctx->cno && !(fc->sb_flags & SB_RDONLY)) { + nilfs_err(NULL, + "invalid option \"cp=%llu\": read-only option is not specified", + ctx->cno); + return -EINVAL; + } - err = lookup_bdev(dev_name, &dev); + err = lookup_bdev(fc->source, &dev); if (err) - return ERR_PTR(err); + return err; - s = sget(fs_type, nilfs_test_bdev_super, nilfs_set_bdev_super, flags, - &dev); + s = sget_dev(fc, dev); if (IS_ERR(s)) - return ERR_CAST(s); + return PTR_ERR(s); if (!s->s_root) { - err = setup_bdev_super(s, flags, NULL); + err = setup_bdev_super(s, fc->sb_flags, fc); if (!err) - err = nilfs_fill_super(s, data, - flags & SB_SILENT ? 1 : 0); + err = nilfs_fill_super(s, fc); if (err) goto failed_super; s->s_flags |= SB_ACTIVE; - } else if (!sd.cno) { + } else if (!ctx->cno) { if (nilfs_tree_is_busy(s->s_root)) { - if ((flags ^ s->s_flags) & SB_RDONLY) { + if ((fc->sb_flags ^ s->s_flags) & SB_RDONLY) { nilfs_err(s, "the device already has a %s mount.", sb_rdonly(s) ? "read-only" : "read/write"); @@ -1312,37 +1224,75 @@ nilfs_mount(struct file_system_type *fs_type, int flags, } } else { /* - * Try remount to setup mount states if the current + * Try reconfigure to setup mount states if the current * tree is not mounted and only snapshots use this sb. + * + * Since nilfs_reconfigure() requires fc->root to be + * set, set it first and release it on failure. */ - err = nilfs_remount(s, &flags, data); - if (err) + fc->root = dget(s->s_root); + err = nilfs_reconfigure(fc); + if (err) { + dput(fc->root); + fc->root = NULL; /* prevent double release */ goto failed_super; + } + return 0; } } - if (sd.cno) { + if (ctx->cno) { struct dentry *root_dentry; - err = nilfs_attach_snapshot(s, sd.cno, &root_dentry); + err = nilfs_attach_snapshot(s, ctx->cno, &root_dentry); if (err) goto failed_super; - return root_dentry; + fc->root = root_dentry; + return 0; } - return dget(s->s_root); + fc->root = dget(s->s_root); + return 0; failed_super: deactivate_locked_super(s); - return ERR_PTR(err); + return err; +} + +static void nilfs_free_fc(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static const struct fs_context_operations nilfs_context_ops = { + .parse_param = nilfs_parse_param, + .get_tree = nilfs_get_tree, + .reconfigure = nilfs_reconfigure, + .free = nilfs_free_fc, +}; + +static int nilfs_init_fs_context(struct fs_context *fc) +{ + struct nilfs_fs_context *ctx; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->ns_mount_opt = NILFS_MOUNT_ERRORS_RO | NILFS_MOUNT_BARRIER; + fc->fs_private = ctx; + fc->ops = &nilfs_context_ops; + + return 0; } struct file_system_type nilfs_fs_type = { .owner = THIS_MODULE, .name = "nilfs2", - .mount = nilfs_mount, .kill_sb = kill_block_super, .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = nilfs_init_fs_context, + .parameters = nilfs_param_spec, }; MODULE_ALIAS_FS("nilfs2"); diff --git a/fs/nilfs2/the_nilfs.c b/fs/nilfs2/the_nilfs.c index 2ae2c1bbf6d17..db322068678f4 100644 --- a/fs/nilfs2/the_nilfs.c +++ b/fs/nilfs2/the_nilfs.c @@ -659,7 +659,6 @@ static int nilfs_load_super_block(struct the_nilfs *nilfs, * init_nilfs - initialize a NILFS instance. * @nilfs: the_nilfs structure * @sb: super block - * @data: mount options * * init_nilfs() performs common initialization per block device (e.g. * reading the super block, getting disk layout information, initializing @@ -668,7 +667,7 @@ static int nilfs_load_super_block(struct the_nilfs *nilfs, * Return Value: On success, 0 is returned. On error, a negative error * code is returned. */ -int init_nilfs(struct the_nilfs *nilfs, struct super_block *sb, char *data) +int init_nilfs(struct the_nilfs *nilfs, struct super_block *sb) { struct nilfs_super_block *sbp; int blocksize; @@ -686,7 +685,7 @@ int init_nilfs(struct the_nilfs *nilfs, struct super_block *sb, char *data) if (err) goto out; - err = nilfs_store_magic_and_option(sb, sbp, data); + err = nilfs_store_magic(sb, sbp); if (err) goto failed_sbh; diff --git a/fs/nilfs2/the_nilfs.h b/fs/nilfs2/the_nilfs.h index cd4ae1b8ae165..85da0629415df 100644 --- a/fs/nilfs2/the_nilfs.h +++ b/fs/nilfs2/the_nilfs.h @@ -219,10 +219,6 @@ THE_NILFS_FNS(PURGING, purging) #define nilfs_set_opt(nilfs, opt) \ ((nilfs)->ns_mount_opt |= NILFS_MOUNT_##opt) #define nilfs_test_opt(nilfs, opt) ((nilfs)->ns_mount_opt & NILFS_MOUNT_##opt) -#define nilfs_write_opt(nilfs, mask, opt) \ - ((nilfs)->ns_mount_opt = \ - (((nilfs)->ns_mount_opt & ~NILFS_MOUNT_##mask) | \ - NILFS_MOUNT_##opt)) \ /** * struct nilfs_root - nilfs root object @@ -276,7 +272,7 @@ static inline int nilfs_sb_will_flip(struct the_nilfs *nilfs) void nilfs_set_last_segment(struct the_nilfs *, sector_t, u64, __u64); struct the_nilfs *alloc_nilfs(struct super_block *sb); void destroy_nilfs(struct the_nilfs *nilfs); -int init_nilfs(struct the_nilfs *nilfs, struct super_block *sb, char *data); +int init_nilfs(struct the_nilfs *nilfs, struct super_block *sb); int load_nilfs(struct the_nilfs *nilfs, struct super_block *sb); unsigned long nilfs_nrsvsegs(struct the_nilfs *nilfs, unsigned long nsegs); void nilfs_set_nsegments(struct the_nilfs *nilfs, unsigned long nsegs); From f492fb365699448949a2eb4aa295e0e8fbf79b10 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 23 Apr 2024 23:30:18 +0100 Subject: [PATCH 1332/1702] ocfs2: remove redundant assignment to variable status Variable status is being assigned and error code that is never read, it is being assigned inside of a do-while loop. The assignment is redundant and can be removed. Cleans up clang scan build warning: fs/ocfs2/dlm/dlmdomain.c:1530:2: warning: Value stored to 'status' is never read [deadcode.DeadStores] Link: https://lkml.kernel.org/r/20240423223018.1573213-1-colin.i.king@gmail.com Signed-off-by: Colin Ian King Acked-by: Joseph Qi Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Cc: Heming Zhao Cc: Dan Carpenter Signed-off-by: Andrew Morton --- fs/ocfs2/dlm/dlmdomain.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/ocfs2/dlm/dlmdomain.c b/fs/ocfs2/dlm/dlmdomain.c index 2e0a2f3382829..2018501b22493 100644 --- a/fs/ocfs2/dlm/dlmdomain.c +++ b/fs/ocfs2/dlm/dlmdomain.c @@ -1527,7 +1527,6 @@ static void dlm_send_join_asserts(struct dlm_ctxt *dlm, { int status, node, live; - status = 0; node = -1; while ((node = find_next_bit(node_map, O2NM_MAX_NODES, node + 1)) < O2NM_MAX_NODES) { From bbed8b9ffed16572357c5a348c7172846d106cfe Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Tue, 23 Apr 2024 17:27:57 -0300 Subject: [PATCH 1333/1702] tools lib rbtree: pick some improvements from the kernel rbtree code The tools/lib/rbtree.c code came from the kernel. Remove the EXPORT_SYMBOL() that make sense only there. Unfortunately it is not being checked with tools/perf/check_headers.sh. Will try to remedy this. Until then pick the improvements from: b0687c1119b4e8c8 ("lib/rbtree: use '+' instead of '|' for setting color.") That I noticed by doing: diff -u tools/lib/rbtree.c lib/rbtree.c diff -u tools/include/linux/rbtree_augmented.h include/linux/rbtree_augmented.h There is one other cases, but lets pick it in separate patches. Link: https://lkml.kernel.org/r/ZigZzeFoukzRKG1Q@x1 Signed-off-by: Arnaldo Carvalho de Melo Cc: Adrian Hunter Cc: Ian Rogers Cc: Jiri Olsa Cc: Kan Liang Cc: Namhyung Kim Cc: Noah Goldstein Signed-off-by: Andrew Morton --- tools/include/linux/rbtree_augmented.h | 4 ++-- tools/lib/rbtree.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/include/linux/rbtree_augmented.h b/tools/include/linux/rbtree_augmented.h index 570bb9794421b..95483c7d81df7 100644 --- a/tools/include/linux/rbtree_augmented.h +++ b/tools/include/linux/rbtree_augmented.h @@ -158,13 +158,13 @@ RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ static inline void rb_set_parent(struct rb_node *rb, struct rb_node *p) { - rb->__rb_parent_color = rb_color(rb) | (unsigned long)p; + rb->__rb_parent_color = rb_color(rb) + (unsigned long)p; } static inline void rb_set_parent_color(struct rb_node *rb, struct rb_node *p, int color) { - rb->__rb_parent_color = (unsigned long)p | color; + rb->__rb_parent_color = (unsigned long)p + color; } static inline void diff --git a/tools/lib/rbtree.c b/tools/lib/rbtree.c index 727396de6be54..9e7307186b7f4 100644 --- a/tools/lib/rbtree.c +++ b/tools/lib/rbtree.c @@ -58,7 +58,7 @@ static inline void rb_set_black(struct rb_node *rb) { - rb->__rb_parent_color |= RB_BLACK; + rb->__rb_parent_color += RB_BLACK; } static inline struct rb_node *rb_red_parent(struct rb_node *red) From 1f65ce65a3749f858b2b3cd0898483ea61313549 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 23 Apr 2024 22:23:08 +0300 Subject: [PATCH 1334/1702] media: rc: add missing io.h Patch series "kfifo: Clean up kfifo.h", v2. To reduce dependency hell a degree, clean up kfifo.h (mainly getting rid of kernel.h in the global header). This patch (of 3): In many remote control drivers the io.h is implied by others. This is not good as it prevents from cleanups done in other headers. Add missing include. Link: https://lkml.kernel.org/r/20240423192529.3249134-1-andriy.shevchenko@linux.intel.com Link: https://lkml.kernel.org/r/20240423192529.3249134-2-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Alain Volmat Cc: AngeloGioacchino Del Regno Cc: Chen-Yu Tsai Cc: Hans Verkuil Cc: Jernej Skrabec Cc: Matthias Brugger Cc: Mauro Carvalho Chehab Cc: Patrice Chotard Cc: Rob Herring Cc: Samuel Holland Cc: Sean Wang Cc: Sean Young Cc: Stefani Seibold Signed-off-by: Andrew Morton --- drivers/media/rc/mtk-cir.c | 1 + drivers/media/rc/serial_ir.c | 1 + drivers/media/rc/st_rc.c | 1 + drivers/media/rc/sunxi-cir.c | 1 + 4 files changed, 4 insertions(+) diff --git a/drivers/media/rc/mtk-cir.c b/drivers/media/rc/mtk-cir.c index 4e294e59d3cba..b2f82b2d1c8d4 100644 --- a/drivers/media/rc/mtk-cir.c +++ b/drivers/media/rc/mtk-cir.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/media/rc/serial_ir.c b/drivers/media/rc/serial_ir.c index 96ae0294ac102..fc5fd39271772 100644 --- a/drivers/media/rc/serial_ir.c +++ b/drivers/media/rc/serial_ir.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/media/rc/st_rc.c b/drivers/media/rc/st_rc.c index 28477aa955635..988b09191c4c7 100644 --- a/drivers/media/rc/st_rc.c +++ b/drivers/media/rc/st_rc.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/media/rc/sunxi-cir.c b/drivers/media/rc/sunxi-cir.c index bf58c965ead8c..b49df8355e6b3 100644 --- a/drivers/media/rc/sunxi-cir.c +++ b/drivers/media/rc/sunxi-cir.c @@ -12,6 +12,7 @@ #include #include +#include #include #include #include From 495ae16a2895d6475384bca59f5128c9964dee0c Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 23 Apr 2024 22:23:09 +0300 Subject: [PATCH 1335/1702] media: stih-cec: add missing io.h In the driver the io.h is implied by others. This is not good as it prevents from cleanups done in other headers. Add missing include. Link: https://lkml.kernel.org/r/20240423192529.3249134-3-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Alain Volmat Cc: AngeloGioacchino Del Regno Cc: Chen-Yu Tsai Cc: Hans Verkuil Cc: Jernej Skrabec Cc: Matthias Brugger Cc: Mauro Carvalho Chehab Cc: Patrice Chotard Cc: Rob Herring Cc: Samuel Holland Cc: Sean Wang Cc: Sean Young Cc: Stefani Seibold Signed-off-by: Andrew Morton --- drivers/media/cec/platform/sti/stih-cec.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/media/cec/platform/sti/stih-cec.c b/drivers/media/cec/platform/sti/stih-cec.c index a20fc5c0c88d0..99978a7c8d9b7 100644 --- a/drivers/media/cec/platform/sti/stih-cec.c +++ b/drivers/media/cec/platform/sti/stih-cec.c @@ -6,6 +6,7 @@ */ #include #include +#include #include #include #include From 22bcc915ae910bc823d8351542f6d9e7623fff24 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 23 Apr 2024 22:23:10 +0300 Subject: [PATCH 1336/1702] kfifo: don't use "proxy" headers Update header inclusions to follow IWYU (Include What You Use) principle. Link: https://lkml.kernel.org/r/20240423192529.3249134-4-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Alain Volmat Cc: AngeloGioacchino Del Regno Cc: Chen-Yu Tsai Cc: Hans Verkuil Cc: Jernej Skrabec Cc: Matthias Brugger Cc: Mauro Carvalho Chehab Cc: Patrice Chotard Cc: Rob Herring Cc: Samuel Holland Cc: Sean Wang Cc: Sean Young Cc: Stefani Seibold Signed-off-by: Andrew Morton --- include/linux/kfifo.h | 9 +++++++-- lib/kfifo.c | 8 ++++---- samples/kfifo/dma-example.c | 3 ++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/include/linux/kfifo.h b/include/linux/kfifo.h index 0b35a41440ff1..6b28d642f3325 100644 --- a/include/linux/kfifo.h +++ b/include/linux/kfifo.h @@ -36,10 +36,15 @@ * to lock the reader. */ -#include +#include #include #include -#include +#include + +#include +#include + +struct scatterlist; struct __kfifo { unsigned int in; diff --git a/lib/kfifo.c b/lib/kfifo.c index 12f5a347aa137..15acdee4a8f32 100644 --- a/lib/kfifo.c +++ b/lib/kfifo.c @@ -5,13 +5,13 @@ * Copyright (C) 2009/2010 Stefani Seibold */ -#include -#include -#include #include +#include +#include #include +#include +#include #include -#include /* * internal helper to calculate the unused elements in a fifo diff --git a/samples/kfifo/dma-example.c b/samples/kfifo/dma-example.c index 0cf27483cb361..74fe915b7ffe8 100644 --- a/samples/kfifo/dma-example.c +++ b/samples/kfifo/dma-example.c @@ -6,8 +6,9 @@ */ #include -#include #include +#include +#include /* * This module shows how to handle fifo dma operations. From ec0b6d17a5f89da2182ec8e2f978c20bbedf6ae2 Mon Sep 17 00:00:00 2001 From: Florian Rommel Date: Thu, 25 Apr 2024 17:34:58 +0200 Subject: [PATCH 1337/1702] scripts/gdb: fix failing KGDB detection during probe Patch series "scripts/gdb: Fixes for $lx_current and $lx_per_cpu". This series fixes several bugs in the GDB scripts related to the $lx_current and $lx_per_cpu functions. The changes were tested with GDB 10, 11, 12, 13, and 14. Patch 1 fixes false-negative results when probing for KGDB Patch 2 fixes the $lx_per_cpu function, which is currently non-functional in QEMU-GDB and KGDB. Patch 3 fixes an additional bug in $lx_per_cpu that occurs with KGDB. Patch 4 fixes the incorrect detection of the current CPU number in KGDB, which silently breaks $lx_per_cpu and $lx_current. This patch (of 4): The KGDB probe function sometimes failed to detect KGDB for SMP machines as it assumed that task 2 (kthreadd) is running on CPU 0, which is not necessarily the case. Now, the detection is agnostic to kthreadd's CPU. Link: https://lkml.kernel.org/r/20240425153501.749966-1-mail@florommel.de Link: https://lkml.kernel.org/r/20240425153501.749966-2-mail@florommel.de Signed-off-by: Florian Rommel Cc: Andrew Jones Cc: Deepak Gupta Cc: Jan Kiszka Cc: Kieran Bingham Cc: Kuan-Ying Lee Cc: Palmer Dabbelt Signed-off-by: Andrew Morton --- scripts/gdb/linux/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gdb/linux/utils.py b/scripts/gdb/linux/utils.py index 7d5278d815fa1..245ab297ea84a 100644 --- a/scripts/gdb/linux/utils.py +++ b/scripts/gdb/linux/utils.py @@ -196,7 +196,7 @@ def probe_qemu(): def probe_kgdb(): try: thread_info = gdb.execute("info thread 2", to_string=True) - return "shadowCPU0" in thread_info + return "shadowCPU" in thread_info except gdb.error: return False From db08c53fdd542bb7f83bd57c3cfa3e1b95c9b54d Mon Sep 17 00:00:00 2001 From: Florian Rommel Date: Thu, 25 Apr 2024 17:34:59 +0200 Subject: [PATCH 1338/1702] scripts/gdb: fix parameter handling in $lx_per_cpu Before, the script tried to get the address by constructing a pointer to the parameter (by name). However, since GDB now passes the parameter as a GdbValue, we cannot get its name. Instead, we retrieve the address through GdbValue's address attribute. Before: >>> p $lx_per_cpu(cpu_info) Traceback (most recent call last): File "./scripts/gdb/linux/cpus.py", line 152, in invoke var_ptr = gdb.parse_and_eval("&" + var_name.string()) ^^^^^^^^^^^^^^^^^ gdb.error: Trying to read string with inappropriate type `struct cpuinfo_x86'. Link: https://lkml.kernel.org/r/20240425153501.749966-3-mail@florommel.de Signed-off-by: Florian Rommel Cc: Andrew Jones Cc: Deepak Gupta Cc: Jan Kiszka Cc: Kieran Bingham Cc: Kuan-Ying Lee Cc: Palmer Dabbelt Signed-off-by: Andrew Morton --- scripts/gdb/linux/cpus.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/gdb/linux/cpus.py b/scripts/gdb/linux/cpus.py index cba589e5b57d6..2b51a3abd3639 100644 --- a/scripts/gdb/linux/cpus.py +++ b/scripts/gdb/linux/cpus.py @@ -152,9 +152,8 @@ class PerCpu(gdb.Function): def __init__(self): super(PerCpu, self).__init__("lx_per_cpu") - def invoke(self, var_name, cpu=-1): - var_ptr = gdb.parse_and_eval("&" + var_name.string()) - return per_cpu(var_ptr, cpu) + def invoke(self, var, cpu=-1): + return per_cpu(var.address, cpu) PerCpu() From 7566b063e9e4af908123ebe8b80cc0d0c7429507 Mon Sep 17 00:00:00 2001 From: Florian Rommel Date: Thu, 25 Apr 2024 17:35:00 +0200 Subject: [PATCH 1339/1702] scripts/gdb: make get_thread_info accept pointers get_thread_info ($lx_thread_info) only accepted a dereferenced task parameter. Passing a pointer to a task_struct (like $lx_per_cpu does with KGDB) threw an exception. With this patch, both (dereferenced values and pointers) are accepted. Before (on x86, KGDB): >>> p $lx_per_cpu(cpu_info) Traceback (most recent call last): File "./scripts/gdb/linux/cpus.py", line 158, in invoke return per_cpu(var_ptr, cpu) ^^^^^^^^^^^^^^^^^^^^^ File "./scripts/gdb/linux/cpus.py", line 42, in per_cpu cpu = get_current_cpu() ^^^^^^^^^^^^^^^^^ File "./scripts/gdb/linux/cpus.py", line 33, in get_current_cpu return tasks.get_thread_info(tasks.get_task_by_pid(tid))['cpu'] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "./scripts/gdb/linux/tasks.py", line 88, in get_thread_info if task.type.fields()[0].type == thread_info_type.get_type(): ~~~~~~~~~~~~~~~~~~^^^ IndexError: list index out of range Link: https://lkml.kernel.org/r/20240425153501.749966-4-mail@florommel.de Signed-off-by: Florian Rommel Cc: Andrew Jones Cc: Deepak Gupta Cc: Jan Kiszka Cc: Kieran Bingham Cc: Kuan-Ying Lee Cc: Palmer Dabbelt Signed-off-by: Andrew Morton --- scripts/gdb/linux/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gdb/linux/tasks.py b/scripts/gdb/linux/tasks.py index 6793d6e86e777..62348397c1f5c 100644 --- a/scripts/gdb/linux/tasks.py +++ b/scripts/gdb/linux/tasks.py @@ -85,7 +85,7 @@ def invoke(self, arg, from_tty): def get_thread_info(task): thread_info_ptr_type = thread_info_type.get_type().pointer() - if task.type.fields()[0].type == thread_info_type.get_type(): + if task_type.get_type().fields()[0].type == thread_info_type.get_type(): return task['thread_info'] thread_info = task['stack'].cast(thread_info_ptr_type) return thread_info.dereference() From 40eea5abbb9ccae6df55dfd94c3c85c023e2521b Mon Sep 17 00:00:00 2001 From: Florian Rommel Date: Thu, 25 Apr 2024 17:35:01 +0200 Subject: [PATCH 1340/1702] scripts/gdb: fix detection of current CPU in KGDB Directly read the current CPU number from the kgdb_active variable. Before, the active CPU was obtained through the current task, which required searching the task list for the pid of GDB's selected thread. Obtaining the pid was buggy: GDB may use selected_thread().ptid[1] (LWPID) instead of .ptid[2] (TID) to store the threads pid; see https://sourceware.org/gdb/current/onlinedocs/gdb.html/Threads-In-Python.html As a result, the detection could return the wrong CPU number, leading to incorrect results for $lx_per_cpu and $lx_current. As a side effect, the patch significantly speeds up $lx_per_cpu and $lx_current in KGDB by avoiding the task-list iteration. Link: https://lkml.kernel.org/r/20240425153501.749966-5-mail@florommel.de Signed-off-by: Florian Rommel Cc: Andrew Jones Cc: Deepak Gupta Cc: Jan Kiszka Cc: Kieran Bingham Cc: Kuan-Ying Lee Cc: Palmer Dabbelt Signed-off-by: Andrew Morton --- scripts/gdb/linux/cpus.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/gdb/linux/cpus.py b/scripts/gdb/linux/cpus.py index 2b51a3abd3639..2f11c4f9c345a 100644 --- a/scripts/gdb/linux/cpus.py +++ b/scripts/gdb/linux/cpus.py @@ -26,11 +26,7 @@ def get_current_cpu(): if utils.get_gdbserver_type() == utils.GDBSERVER_QEMU: return gdb.selected_thread().num - 1 elif utils.get_gdbserver_type() == utils.GDBSERVER_KGDB: - tid = gdb.selected_thread().ptid[2] - if tid > (0x100000000 - MAX_CPUS - 2): - return 0x100000000 - tid - 2 - else: - return tasks.get_thread_info(tasks.get_task_by_pid(tid))['cpu'] + return gdb.parse_and_eval("kgdb_active.counter") else: raise gdb.GdbError("Sorry, obtaining the current CPU is not yet " "supported with this gdb server.") From 675f02e5e65651553c945a464e4f7630859f2032 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sat, 20 Apr 2024 03:50:17 +0100 Subject: [PATCH 1341/1702] squashfs: convert squashfs_symlink_read_folio to use folio APIs Remove use of page APIs, return the errno instead of 0, switch from kmap_atomic to kmap_local and use folio_end_read() to unify the two exit paths. Link: https://lkml.kernel.org/r/20240420025029.2166544-23-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Tested-by: Phillip Lougher Reviewed-by: Phillip Lougher Signed-off-by: Andrew Morton --- fs/squashfs/symlink.c | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/fs/squashfs/symlink.c b/fs/squashfs/symlink.c index 2bf977a52c2c3..6ef735bd841a1 100644 --- a/fs/squashfs/symlink.c +++ b/fs/squashfs/symlink.c @@ -32,20 +32,19 @@ static int squashfs_symlink_read_folio(struct file *file, struct folio *folio) { - struct page *page = &folio->page; - struct inode *inode = page->mapping->host; + struct inode *inode = folio->mapping->host; struct super_block *sb = inode->i_sb; struct squashfs_sb_info *msblk = sb->s_fs_info; - int index = page->index << PAGE_SHIFT; + int index = folio_pos(folio); u64 block = squashfs_i(inode)->start; int offset = squashfs_i(inode)->offset; int length = min_t(int, i_size_read(inode) - index, PAGE_SIZE); - int bytes, copied; + int bytes, copied, error; void *pageaddr; struct squashfs_cache_entry *entry; TRACE("Entered squashfs_symlink_readpage, page index %ld, start block " - "%llx, offset %x\n", page->index, block, offset); + "%llx, offset %x\n", folio->index, block, offset); /* * Skip index bytes into symlink metadata. @@ -57,14 +56,15 @@ static int squashfs_symlink_read_folio(struct file *file, struct folio *folio) ERROR("Unable to read symlink [%llx:%x]\n", squashfs_i(inode)->start, squashfs_i(inode)->offset); - goto error_out; + error = bytes; + goto out; } } /* * Read length bytes from symlink metadata. Squashfs_read_metadata * is not used here because it can sleep and we want to use - * kmap_atomic to map the page. Instead call the underlying + * kmap_local to map the folio. Instead call the underlying * squashfs_cache_get routine. As length bytes may overlap metadata * blocks, we may need to call squashfs_cache_get multiple times. */ @@ -75,29 +75,26 @@ static int squashfs_symlink_read_folio(struct file *file, struct folio *folio) squashfs_i(inode)->start, squashfs_i(inode)->offset); squashfs_cache_put(entry); - goto error_out; + error = entry->error; + goto out; } - pageaddr = kmap_atomic(page); + pageaddr = kmap_local_folio(folio, 0); copied = squashfs_copy_data(pageaddr + bytes, entry, offset, length - bytes); if (copied == length - bytes) memset(pageaddr + length, 0, PAGE_SIZE - length); else block = entry->next_index; - kunmap_atomic(pageaddr); + kunmap_local(pageaddr); squashfs_cache_put(entry); } - flush_dcache_page(page); - SetPageUptodate(page); - unlock_page(page); - return 0; - -error_out: - SetPageError(page); - unlock_page(page); - return 0; + flush_dcache_folio(folio); + error = 0; +out: + folio_end_read(folio, error == 0); + return error; } From bbf45b7e68555569489ab2428dd9c23960cdc9bf Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sat, 20 Apr 2024 03:50:18 +0100 Subject: [PATCH 1342/1702] squashfs: remove calls to set the folio error flag Nobody checks the error flag on squashfs folios, so stop setting it. Link: https://lkml.kernel.org/r/20240420025029.2166544-24-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Tested-by: Phillip Lougher Reviewed-by: Phillip Lougher Signed-off-by: Andrew Morton --- fs/squashfs/file.c | 6 +----- fs/squashfs/file_direct.c | 3 +-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/fs/squashfs/file.c b/fs/squashfs/file.c index e8df6430444b0..a8c1e7f9a6092 100644 --- a/fs/squashfs/file.c +++ b/fs/squashfs/file.c @@ -375,8 +375,6 @@ void squashfs_fill_page(struct page *page, struct squashfs_cache_entry *buffer, flush_dcache_page(page); if (copied == avail) SetPageUptodate(page); - else - SetPageError(page); } /* Copy data into page cache */ @@ -471,7 +469,7 @@ static int squashfs_read_folio(struct file *file, struct folio *folio) res = read_blocklist(inode, index, &block); if (res < 0) - goto error_out; + goto out; if (res == 0) res = squashfs_readpage_sparse(page, expected); @@ -483,8 +481,6 @@ static int squashfs_read_folio(struct file *file, struct folio *folio) if (!res) return 0; -error_out: - SetPageError(page); out: pageaddr = kmap_atomic(page); memset(pageaddr, 0, PAGE_SIZE); diff --git a/fs/squashfs/file_direct.c b/fs/squashfs/file_direct.c index 763a3f7a75f6d..2a689ce71de98 100644 --- a/fs/squashfs/file_direct.c +++ b/fs/squashfs/file_direct.c @@ -106,14 +106,13 @@ int squashfs_readpage_block(struct page *target_page, u64 block, int bsize, return 0; mark_errored: - /* Decompression failed, mark pages as errored. Target_page is + /* Decompression failed. Target_page is * dealt with by the caller */ for (i = 0; i < pages; i++) { if (page[i] == NULL || page[i] == target_page) continue; flush_dcache_page(page[i]); - SetPageError(page[i]); unlock_page(page[i]); put_page(page[i]); } From 91d743a9c8299de1fc1b47428d8bb4c85face00f Mon Sep 17 00:00:00 2001 From: Ryusuke Konishi Date: Tue, 30 Apr 2024 17:00:19 +0900 Subject: [PATCH 1343/1702] nilfs2: make superblock data array index computation sparse friendly Upon running sparse, "warning: dubious: x & !y" is output at an array index calculation within nilfs_load_super_block(). The calculation is not wrong, but to eliminate the sparse warning, replace it with an equivalent calculation. Also, add a comment to make it easier to understand what the unintuitive array index calculation is doing and whether it's correct. Link: https://lkml.kernel.org/r/20240430080019.4242-3-konishi.ryusuke@gmail.com Fixes: e339ad31f599 ("nilfs2: introduce secondary super block") Signed-off-by: Ryusuke Konishi Cc: Bart Van Assche Cc: Jens Axboe Cc: kernel test robot Signed-off-by: Andrew Morton --- fs/nilfs2/the_nilfs.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/fs/nilfs2/the_nilfs.c b/fs/nilfs2/the_nilfs.c index db322068678f4..f41d7b6d432c6 100644 --- a/fs/nilfs2/the_nilfs.c +++ b/fs/nilfs2/the_nilfs.c @@ -592,7 +592,7 @@ static int nilfs_load_super_block(struct the_nilfs *nilfs, struct nilfs_super_block **sbp = nilfs->ns_sbp; struct buffer_head **sbh = nilfs->ns_sbh; u64 sb2off, devsize = bdev_nr_bytes(nilfs->ns_bdev); - int valid[2], swp = 0; + int valid[2], swp = 0, older; if (devsize < NILFS_SEG_MIN_BLOCKS * NILFS_MIN_BLOCK_SIZE + 4096) { nilfs_err(sb, "device size too small"); @@ -648,9 +648,25 @@ static int nilfs_load_super_block(struct the_nilfs *nilfs, if (swp) nilfs_swap_super_block(nilfs); + /* + * Calculate the array index of the older superblock data. + * If one has been dropped, set index 0 pointing to the remaining one, + * otherwise set index 1 pointing to the old one (including if both + * are the same). + * + * Divided case valid[0] valid[1] swp -> older + * ------------------------------------------------------------- + * Both SBs are invalid 0 0 N/A (Error) + * SB1 is invalid 0 1 1 0 + * SB2 is invalid 1 0 0 0 + * SB2 is newer 1 1 1 0 + * SB2 is older or the same 1 1 0 1 + */ + older = valid[1] ^ swp; + nilfs->ns_sbwcount = 0; nilfs->ns_sbwtime = le64_to_cpu(sbp[0]->s_wtime); - nilfs->ns_prot_seq = le64_to_cpu(sbp[valid[1] & !swp]->s_last_seq); + nilfs->ns_prot_seq = le64_to_cpu(sbp[older]->s_last_seq); *sbpp = sbp[0]; return 0; } From 602ba77361160d99c77e3dadf7d891364f8c71b3 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Mon, 29 Apr 2024 23:02:35 -0700 Subject: [PATCH 1344/1702] watchdog: handle comma separated nmi_watchdog command line Per the document, the kernel can accept comma separated command line like nmi_watchdog=nopanic,0. However, the code doesn't really handle it. Fix the kernel to handle it properly. Link: https://lkml.kernel.org/r/20240430060236.1878002-1-song@kernel.org Signed-off-by: Song Liu Cc: Peter Zijlstra Signed-off-by: Andrew Morton --- kernel/watchdog.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kernel/watchdog.c b/kernel/watchdog.c index d7b2125503af3..7f54484de16f7 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -71,6 +71,7 @@ void __init hardlockup_detector_disable(void) static int __init hardlockup_panic_setup(char *str) { +next: if (!strncmp(str, "panic", 5)) hardlockup_panic = 1; else if (!strncmp(str, "nopanic", 7)) @@ -79,6 +80,12 @@ static int __init hardlockup_panic_setup(char *str) watchdog_hardlockup_user_enabled = 0; else if (!strncmp(str, "1", 1)) watchdog_hardlockup_user_enabled = 1; + while (*(str++)) { + if (*str == ',') { + str++; + goto next; + } + } return 1; } __setup("nmi_watchdog=", hardlockup_panic_setup); From 393fb313a2e150b768e4850658679e2afff431e9 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Mon, 29 Apr 2024 23:02:36 -0700 Subject: [PATCH 1345/1702] watchdog: allow nmi watchdog to use raw perf event NMI watchdog permanently consumes one hardware counters per CPU on the system. For systems that use many hardware counters, this causes more aggressive time multiplexing of perf events. OTOH, some CPUs (mostly Intel) support "ref-cycles" event, which is rarely used. Add kernel cmdline arg nmi_watchdog=rNNN to configure the watchdog to use raw event. For example, on Intel CPUs, we can use "r300" to configure the watchdog to use ref-cycles event. If the raw event does not work, fall back to use "cycles". [akpm@linux-foundation.org: fix kerneldoc] Link: https://lkml.kernel.org/r/20240430060236.1878002-2-song@kernel.org Signed-off-by: Song Liu Cc: Peter Zijlstra Cc: "Matthew Wilcox (Oracle)" Signed-off-by: Andrew Morton --- .../admin-guide/kernel-parameters.txt | 5 +- include/linux/nmi.h | 2 + kernel/watchdog.c | 2 + kernel/watchdog_perf.c | 46 +++++++++++++++++++ 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 902ecd92a29fb..1fa79a3d0d1a2 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3773,10 +3773,12 @@ Format: [state][,regs][,debounce][,die] nmi_watchdog= [KNL,BUGS=X86] Debugging features for SMP kernels - Format: [panic,][nopanic,][num] + Format: [panic,][nopanic,][rNNN,][num] Valid num: 0 or 1 0 - turn hardlockup detector in nmi_watchdog off 1 - turn hardlockup detector in nmi_watchdog on + rNNN - configure the watchdog with raw perf event 0xNNN + When panic is specified, panic when an NMI watchdog timeout occurs (or 'nopanic' to not panic on an NMI watchdog, if CONFIG_BOOTPARAM_HARDLOCKUP_PANIC is set) @@ -7464,4 +7466,3 @@ memory, and other data can't be written using xmon commands. off xmon is disabled. - diff --git a/include/linux/nmi.h b/include/linux/nmi.h index f53438eae815d..a8dfb38c9bb6f 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -105,10 +105,12 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs); extern void hardlockup_detector_perf_stop(void); extern void hardlockup_detector_perf_restart(void); extern void hardlockup_detector_perf_cleanup(void); +extern void hardlockup_config_perf_event(const char *str); #else static inline void hardlockup_detector_perf_stop(void) { } static inline void hardlockup_detector_perf_restart(void) { } static inline void hardlockup_detector_perf_cleanup(void) { } +static inline void hardlockup_config_perf_event(const char *str) { } #endif void watchdog_hardlockup_stop(void); diff --git a/kernel/watchdog.c b/kernel/watchdog.c index 7f54484de16f7..ab0129b15f251 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -80,6 +80,8 @@ static int __init hardlockup_panic_setup(char *str) watchdog_hardlockup_user_enabled = 0; else if (!strncmp(str, "1", 1)) watchdog_hardlockup_user_enabled = 1; + else if (!strncmp(str, "r", 1)) + hardlockup_config_perf_event(str + 1); while (*(str++)) { if (*str == ',') { str++; diff --git a/kernel/watchdog_perf.c b/kernel/watchdog_perf.c index 8ea00c4a24b2d..5f7d1f0d42686 100644 --- a/kernel/watchdog_perf.c +++ b/kernel/watchdog_perf.c @@ -90,6 +90,14 @@ static struct perf_event_attr wd_hw_attr = { .disabled = 1, }; +static struct perf_event_attr fallback_wd_hw_attr = { + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, + .size = sizeof(struct perf_event_attr), + .pinned = 1, + .disabled = 1, +}; + /* Callback function for perf event subsystem */ static void watchdog_overflow_callback(struct perf_event *event, struct perf_sample_data *data, @@ -122,6 +130,13 @@ static int hardlockup_detector_event_create(void) /* Try to register using hardware perf events */ evt = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback, NULL); + if (IS_ERR(evt)) { + wd_attr = &fallback_wd_hw_attr; + wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh); + evt = perf_event_create_kernel_counter(wd_attr, cpu, NULL, + watchdog_overflow_callback, NULL); + } + if (IS_ERR(evt)) { pr_debug("Perf event create on CPU %d failed with %ld\n", cpu, PTR_ERR(evt)); @@ -259,3 +274,34 @@ int __init watchdog_hardlockup_probe(void) } return ret; } + +/** + * hardlockup_config_perf_event - Overwrite config of wd_hw_attr. + * + * @str: number which identifies the raw perf event to use + */ +void __init hardlockup_config_perf_event(const char *str) +{ + u64 config; + char buf[24]; + char *comma = strchr(str, ','); + + if (!comma) { + if (kstrtoull(str, 16, &config)) + return; + } else { + unsigned int len = comma - str; + + if (len >= sizeof(buf)) + return; + + if (strscpy(buf, str, sizeof(buf)) < 0) + return; + buf[len] = 0; + if (kstrtoull(buf, 16, &config)) + return; + } + + wd_hw_attr.type = PERF_TYPE_RAW; + wd_hw_attr.config = config; +} From 8fcb916cac8985188a3f825966ffa99507a90cb1 Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Sat, 4 May 2024 16:41:07 -0700 Subject: [PATCH 1346/1702] kernel/watchdog_perf.c: tidy up kerneldoc It is unconventional to have a blank line between name-of-function and description-of-args. Cc: Peter Zijlstra Cc: Song Liu Cc: "Matthew Wilcox (Oracle)" Cc: Ryusuke Konishi Signed-off-by: Andrew Morton --- kernel/watchdog_perf.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/kernel/watchdog_perf.c b/kernel/watchdog_perf.c index 5f7d1f0d42686..d577c4a8321ee 100644 --- a/kernel/watchdog_perf.c +++ b/kernel/watchdog_perf.c @@ -148,7 +148,6 @@ static int hardlockup_detector_event_create(void) /** * watchdog_hardlockup_enable - Enable the local event - * * @cpu: The CPU to enable hard lockup on. */ void watchdog_hardlockup_enable(unsigned int cpu) @@ -167,7 +166,6 @@ void watchdog_hardlockup_enable(unsigned int cpu) /** * watchdog_hardlockup_disable - Disable the local event - * * @cpu: The CPU to enable hard lockup on. */ void watchdog_hardlockup_disable(unsigned int cpu) @@ -277,7 +275,6 @@ int __init watchdog_hardlockup_probe(void) /** * hardlockup_config_perf_event - Overwrite config of wd_hw_attr. - * * @str: number which identifies the raw perf event to use */ void __init hardlockup_config_perf_event(const char *str) From cf7385cb26ac4f0ee6c7385960525ad534323252 Mon Sep 17 00:00:00 2001 From: Sergey Shtylyov Date: Sun, 14 Apr 2024 11:51:39 +0300 Subject: [PATCH 1347/1702] of: module: add buffer overflow check in of_modalias() In of_modalias(), if the buffer happens to be too small even for the 1st snprintf() call, the len parameter will become negative and str parameter (if not NULL initially) will point beyond the buffer's end. Add the buffer overflow check after the 1st snprintf() call and fix such check after the strlen() call (accounting for the terminating NUL char). Fixes: bc575064d688 ("of/device: use of_property_for_each_string to parse compatible strings") Signed-off-by: Sergey Shtylyov Link: https://lore.kernel.org/r/bbfc6be0-c687-62b6-d015-5141b93f313e@omp.ru Signed-off-by: Rob Herring --- drivers/of/module.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/of/module.c b/drivers/of/module.c index 0e8aa974f0f2b..6c2c3e91f7ed8 100644 --- a/drivers/of/module.c +++ b/drivers/of/module.c @@ -21,14 +21,15 @@ ssize_t of_modalias(const struct device_node *np, char *str, ssize_t len) csize = snprintf(str, len, "of:N%pOFn%c%s", np, 'T', of_node_get_device_type(np)); tsize = csize; + if (csize >= len) + csize = len > 0 ? len - 1 : 0; len -= csize; - if (str) - str += csize; + str += csize; of_property_for_each_string(np, "compatible", p, compat) { csize = strlen(compat) + 1; tsize += csize; - if (csize > len) + if (csize >= len) continue; csize = snprintf(str, len, "C%s", compat); From 28081ebd17fb822d35633afbd8111271207d95f2 Mon Sep 17 00:00:00 2001 From: "Rob Herring (Arm)" Date: Wed, 17 Apr 2024 15:04:30 -0500 Subject: [PATCH 1348/1702] dt-bindings: PCI: qcom,pcie-sm8350: Drop redundant 'oneOf' sub-schema The first entry in the 'oneOf' schema doesn't work because the top level schema requires exactly 8 interrupt entries. The 2nd entry is just redundant with the top level. Since 1 entry appears to have been a mistake, let's just drop the entire 'oneOf' rather than reworking the top-level to allow 1 entry. Reviewed-by: Manivannan Sadhasivam Reviewed-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240417200431.3173953-1-robh@kernel.org Signed-off-by: Rob Herring (Arm) --- .../bindings/pci/qcom,pcie-sm8350.yaml | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8350.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8350.yaml index 9eb6e457b07f1..2a4cc41fc710a 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8350.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8350.yaml @@ -71,28 +71,6 @@ properties: items: - const: pci -oneOf: - - properties: - interrupts: - maxItems: 1 - interrupt-names: - items: - - const: msi - - - properties: - interrupts: - minItems: 8 - interrupt-names: - items: - - const: msi0 - - const: msi1 - - const: msi2 - - const: msi3 - - const: msi4 - - const: msi5 - - const: msi6 - - const: msi7 - allOf: - $ref: qcom,pcie-common.yaml# From 5ad1f33c29c311787943a92c56903da74f19fd5d Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 8 May 2024 10:17:12 -0400 Subject: [PATCH 1349/1702] bcachefs: Fix sb_clean_validate endianness conversion Signed-off-by: Kent Overstreet --- fs/bcachefs/sb-clean.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/bcachefs/sb-clean.c b/fs/bcachefs/sb-clean.c index 194e55b11137b..d27804d4987a0 100644 --- a/fs/bcachefs/sb-clean.c +++ b/fs/bcachefs/sb-clean.c @@ -283,7 +283,7 @@ static int bch2_sb_clean_validate(struct bch_sb *sb, entry = vstruct_next(entry)) { if ((void *) vstruct_next(entry) > vstruct_end(&clean->field)) { prt_str(err, "entry type "); - bch2_prt_jset_entry_type(err, le16_to_cpu(entry->type)); + bch2_prt_jset_entry_type(err, entry->type); prt_str(err, " overruns end of section"); return -BCH_ERR_invalid_sb_clean; } From 5dfd3746b6c486db18bc75de89c7abce41c7826c Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 8 May 2024 00:29:24 -0400 Subject: [PATCH 1350/1702] bcachefs: Fix needs_whiteout BUG_ON() in bkey_sort() Btree nodes are log structured; thus, we need to emit whiteouts when we're deleting a key that's been written out to disk. k->needs_whiteout tracks whether a key will need a whiteout when it's deleted, and this requires some careful handling; e.g. the key we're deleting may not have been written out to disk, but it may have overwritten a key that was - thus we need to carry this flag around on overwrites. Invariants: There may be multiple key for the same position in a given node (because of overwrites), but only one of them will be a live (non deleted) key, and only one key for a given position will have the needs_whiteout flag set. Additionally, we don't want to carry around whiteouts that need to be written in the main searchable part of a btree node - btree_iter_peek() will have to skip past them, and this can lead to an O(n^2) issues when doing sequential deletions (e.g. inode rm/truncate). So there's a separate region in the btree node buffer for unwritten whiteouts; these are merge sorted with the rest of the keys we're writing in the btree node write path. The unwritten whiteouts was a later optimization that bch2_sort_keys() didn't take into account; the unwritten whiteouts area means that we never have deleted keys with needs_whiteout set in the main searchable part of a btree node. That means we can simplify and optimize some sort paths, and eliminate an assertion that syzbot found: - Unless we're in the btree node write path, it's always ok to drop whiteouts when sorting - When sorting for a btree node write, we drop the whiteout if it's not from the unwritten whiteouts area, or if it's overwritten by a real key at the same position. This completely eliminates some tricky logic for propagating the needs_whiteout flag: syzbot was able to hit the assertion that checked that there shouldn't be more than one key at the same pos with needs_whiteout set, likely due to a combination of flipping on needs_whiteout on all written keys (they need whiteouts if overwritten), combined with not always dropping unneeded whiteouts, and the tricky logic in the sort path for preserving needs_whiteout that wasn't really needed. Signed-off-by: Kent Overstreet --- fs/bcachefs/bkey_sort.c | 79 ++++++++++++++++++++++++----------------- fs/bcachefs/bkey_sort.h | 4 +-- fs/bcachefs/btree_io.c | 18 +++++----- 3 files changed, 56 insertions(+), 45 deletions(-) diff --git a/fs/bcachefs/bkey_sort.c b/fs/bcachefs/bkey_sort.c index bcca9e76a0b4b..4536eb50fc406 100644 --- a/fs/bcachefs/bkey_sort.c +++ b/fs/bcachefs/bkey_sort.c @@ -6,9 +6,9 @@ #include "bset.h" #include "extents.h" -typedef int (*sort_cmp_fn)(struct btree *, - struct bkey_packed *, - struct bkey_packed *); +typedef int (*sort_cmp_fn)(const struct btree *, + const struct bkey_packed *, + const struct bkey_packed *); static inline bool sort_iter_end(struct sort_iter *iter) { @@ -70,9 +70,9 @@ static inline struct bkey_packed *sort_iter_next(struct sort_iter *iter, /* * If keys compare equal, compare by pointer order: */ -static inline int key_sort_fix_overlapping_cmp(struct btree *b, - struct bkey_packed *l, - struct bkey_packed *r) +static inline int key_sort_fix_overlapping_cmp(const struct btree *b, + const struct bkey_packed *l, + const struct bkey_packed *r) { return bch2_bkey_cmp_packed(b, l, r) ?: cmp_int((unsigned long) l, (unsigned long) r); @@ -154,46 +154,59 @@ bch2_sort_repack(struct bset *dst, struct btree *src, return nr; } -static inline int sort_keys_cmp(struct btree *b, - struct bkey_packed *l, - struct bkey_packed *r) +static inline int keep_unwritten_whiteouts_cmp(const struct btree *b, + const struct bkey_packed *l, + const struct bkey_packed *r) { return bch2_bkey_cmp_packed_inlined(b, l, r) ?: (int) bkey_deleted(r) - (int) bkey_deleted(l) ?: - (int) l->needs_whiteout - (int) r->needs_whiteout; + (long) l - (long) r; } -unsigned bch2_sort_keys(struct bkey_packed *dst, - struct sort_iter *iter, - bool filter_whiteouts) +#include "btree_update_interior.h" + +/* + * For sorting in the btree node write path: whiteouts not in the unwritten + * whiteouts area are dropped, whiteouts in the unwritten whiteouts area are + * dropped if overwritten by real keys: + */ +unsigned bch2_sort_keys_keep_unwritten_whiteouts(struct bkey_packed *dst, struct sort_iter *iter) { - const struct bkey_format *f = &iter->b->format; struct bkey_packed *in, *next, *out = dst; - sort_iter_sort(iter, sort_keys_cmp); + sort_iter_sort(iter, keep_unwritten_whiteouts_cmp); - while ((in = sort_iter_next(iter, sort_keys_cmp))) { - bool needs_whiteout = false; + while ((in = sort_iter_next(iter, keep_unwritten_whiteouts_cmp))) { + if (bkey_deleted(in) && in < unwritten_whiteouts_start(iter->b)) + continue; - if (bkey_deleted(in) && - (filter_whiteouts || !in->needs_whiteout)) + if ((next = sort_iter_peek(iter)) && + !bch2_bkey_cmp_packed_inlined(iter->b, in, next)) continue; - while ((next = sort_iter_peek(iter)) && - !bch2_bkey_cmp_packed_inlined(iter->b, in, next)) { - BUG_ON(in->needs_whiteout && - next->needs_whiteout); - needs_whiteout |= in->needs_whiteout; - in = sort_iter_next(iter, sort_keys_cmp); - } + bkey_p_copy(out, in); + out = bkey_p_next(out); + } - if (bkey_deleted(in)) { - memcpy_u64s_small(out, in, bkeyp_key_u64s(f, in)); - set_bkeyp_val_u64s(f, out, 0); - } else { - bkey_p_copy(out, in); - } - out->needs_whiteout |= needs_whiteout; + return (u64 *) out - (u64 *) dst; +} + +/* + * Main sort routine for compacting a btree node in memory: we always drop + * whiteouts because any whiteouts that need to be written are in the unwritten + * whiteouts area: + */ +unsigned bch2_sort_keys(struct bkey_packed *dst, struct sort_iter *iter) +{ + struct bkey_packed *in, *out = dst; + + sort_iter_sort(iter, bch2_bkey_cmp_packed_inlined); + + while ((in = sort_iter_next(iter, bch2_bkey_cmp_packed_inlined))) { + if (bkey_deleted(in)) + continue; + + bkey_p_copy(out, in); out = bkey_p_next(out); } diff --git a/fs/bcachefs/bkey_sort.h b/fs/bcachefs/bkey_sort.h index 7c0f0b160f185..9be969d468906 100644 --- a/fs/bcachefs/bkey_sort.h +++ b/fs/bcachefs/bkey_sort.h @@ -48,7 +48,7 @@ bch2_sort_repack(struct bset *, struct btree *, struct btree_node_iter *, struct bkey_format *, bool); -unsigned bch2_sort_keys(struct bkey_packed *, - struct sort_iter *, bool); +unsigned bch2_sort_keys_keep_unwritten_whiteouts(struct bkey_packed *, struct sort_iter *); +unsigned bch2_sort_keys(struct bkey_packed *, struct sort_iter *); #endif /* _BCACHEFS_BKEY_SORT_H */ diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c index debb0edc3455a..94753c5b0e897 100644 --- a/fs/bcachefs/btree_io.c +++ b/fs/bcachefs/btree_io.c @@ -288,8 +288,7 @@ bool bch2_compact_whiteouts(struct bch_fs *c, struct btree *b, static void btree_node_sort(struct bch_fs *c, struct btree *b, unsigned start_idx, - unsigned end_idx, - bool filter_whiteouts) + unsigned end_idx) { struct btree_node *out; struct sort_iter_stack sort_iter; @@ -320,7 +319,7 @@ static void btree_node_sort(struct bch_fs *c, struct btree *b, start_time = local_clock(); - u64s = bch2_sort_keys(out->keys.start, &sort_iter.iter, filter_whiteouts); + u64s = bch2_sort_keys(out->keys.start, &sort_iter.iter); out->keys.u64s = cpu_to_le16(u64s); @@ -426,13 +425,12 @@ static bool btree_node_compact(struct bch_fs *c, struct btree *b) break; if (b->nsets - unwritten_idx > 1) { - btree_node_sort(c, b, unwritten_idx, - b->nsets, false); + btree_node_sort(c, b, unwritten_idx, b->nsets); ret = true; } if (unwritten_idx > 1) { - btree_node_sort(c, b, 0, unwritten_idx, false); + btree_node_sort(c, b, 0, unwritten_idx); ret = true; } @@ -2095,11 +2093,11 @@ void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, unsigned flags) unwritten_whiteouts_end(b)); SET_BSET_SEPARATE_WHITEOUTS(i, false); - b->whiteout_u64s = 0; - - u64s = bch2_sort_keys(i->start, &sort_iter.iter, false); + u64s = bch2_sort_keys_keep_unwritten_whiteouts(i->start, &sort_iter.iter); le16_add_cpu(&i->u64s, u64s); + b->whiteout_u64s = 0; + BUG_ON(!b->written && i->u64s != b->data->keys.u64s); set_needs_whiteout(i, false); @@ -2249,7 +2247,7 @@ bool bch2_btree_post_write_cleanup(struct bch_fs *c, struct btree *b) * single bset: */ if (b->nsets > 1) { - btree_node_sort(c, b, 0, b->nsets, true); + btree_node_sort(c, b, 0, b->nsets); invalidated_iter = true; } else { invalidated_iter = bch2_drop_whiteouts(b, COMPACT_ALL); From 61692c7812ab2aca17a3751f6e7798acbdae4b6b Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 8 May 2024 10:58:26 -0400 Subject: [PATCH 1351/1702] bcachefs: bch2_bkey_format_field_overflows() Fix another shift-by-64 by factoring out a common helper for bch2_bkey_format_invalid() and bformat_needs_redo() (where it was already fixed). Reported-by: syzbot+9833a1d29d4a44361e2c@syzkaller.appspotmail.com Signed-off-by: Kent Overstreet --- fs/bcachefs/bkey.c | 13 +++++-------- fs/bcachefs/bkey.h | 23 +++++++++++++++++++++++ fs/bcachefs/move.c | 20 ++------------------ 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/fs/bcachefs/bkey.c b/fs/bcachefs/bkey.c index 76e79a15ba08f..952299a2e4163 100644 --- a/fs/bcachefs/bkey.c +++ b/fs/bcachefs/bkey.c @@ -656,20 +656,17 @@ int bch2_bkey_format_invalid(struct bch_fs *c, * unpacked format: */ for (i = 0; i < f->nr_fields; i++) { - if (!c || c->sb.version_min >= bcachefs_metadata_version_snapshot) { + if ((!c || c->sb.version_min >= bcachefs_metadata_version_snapshot) && + bch2_bkey_format_field_overflows(f, i)) { unsigned unpacked_bits = bch2_bkey_format_current.bits_per_field[i]; u64 unpacked_max = ~((~0ULL << 1) << (unpacked_bits - 1)); u64 packed_max = f->bits_per_field[i] ? ~((~0ULL << 1) << (f->bits_per_field[i] - 1)) : 0; - u64 field_offset = le64_to_cpu(f->field_offset[i]); - if (packed_max + field_offset < packed_max || - packed_max + field_offset > unpacked_max) { - prt_printf(err, "field %u too large: %llu + %llu > %llu", - i, packed_max, field_offset, unpacked_max); - return -BCH_ERR_invalid; - } + prt_printf(err, "field %u too large: %llu + %llu > %llu", + i, packed_max, le64_to_cpu(f->field_offset[i]), unpacked_max); + return -BCH_ERR_invalid; } bits += f->bits_per_field[i]; diff --git a/fs/bcachefs/bkey.h b/fs/bcachefs/bkey.h index 3a45d128f608d..1bb0bd4371b04 100644 --- a/fs/bcachefs/bkey.h +++ b/fs/bcachefs/bkey.h @@ -574,6 +574,29 @@ static inline void bch2_bkey_format_add_key(struct bkey_format_state *s, const s void bch2_bkey_format_add_pos(struct bkey_format_state *, struct bpos); struct bkey_format bch2_bkey_format_done(struct bkey_format_state *); + +static inline bool bch2_bkey_format_field_overflows(struct bkey_format *f, unsigned i) +{ + unsigned f_bits = f->bits_per_field[i]; + unsigned unpacked_bits = bch2_bkey_format_current.bits_per_field[i]; + u64 unpacked_mask = ~((~0ULL << 1) << (unpacked_bits - 1)); + u64 field_offset = le64_to_cpu(f->field_offset[i]); + + if (f_bits > unpacked_bits) + return true; + + if ((f_bits == unpacked_bits) && field_offset) + return true; + + u64 f_mask = f_bits + ? ~((~0ULL << (f_bits - 1)) << 1) + : 0; + + if (((field_offset + f_mask) & unpacked_mask) < field_offset) + return true; + return false; +} + int bch2_bkey_format_invalid(struct bch_fs *, struct bkey_format *, enum bkey_invalid_flags, struct printbuf *); void bch2_bkey_format_to_text(struct printbuf *, const struct bkey_format *); diff --git a/fs/bcachefs/move.c b/fs/bcachefs/move.c index 4d94b7742dbba..45984a688e5b6 100644 --- a/fs/bcachefs/move.c +++ b/fs/bcachefs/move.c @@ -975,26 +975,10 @@ static bool migrate_btree_pred(struct bch_fs *c, void *arg, */ static bool bformat_needs_redo(struct bkey_format *f) { - for (unsigned i = 0; i < f->nr_fields; i++) { - unsigned f_bits = f->bits_per_field[i]; - unsigned unpacked_bits = bch2_bkey_format_current.bits_per_field[i]; - u64 unpacked_mask = ~((~0ULL << 1) << (unpacked_bits - 1)); - u64 field_offset = le64_to_cpu(f->field_offset[i]); - - if (f_bits > unpacked_bits) - return true; - - if ((f_bits == unpacked_bits) && field_offset) + for (unsigned i = 0; i < f->nr_fields; i++) + if (bch2_bkey_format_field_overflows(f, i)) return true; - u64 f_mask = f_bits - ? ~((~0ULL << (f_bits - 1)) << 1) - : 0; - - if (((field_offset + f_mask) & unpacked_mask) < field_offset) - return true; - } - return false; } From 74768337de7f95666fb68a70c73eb1728126cff7 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 8 May 2024 02:50:42 -0400 Subject: [PATCH 1352/1702] bcachefs: Fix xattr_to_text() unsafety Signed-off-by: Kent Overstreet --- fs/bcachefs/xattr.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fs/bcachefs/xattr.c b/fs/bcachefs/xattr.c index 754f17bba68ed..4b3a2df2da323 100644 --- a/fs/bcachefs/xattr.c +++ b/fs/bcachefs/xattr.c @@ -118,11 +118,17 @@ void bch2_xattr_to_text(struct printbuf *out, struct bch_fs *c, else prt_printf(out, "(unknown type %u)", xattr.v->x_type); + unsigned name_len = xattr.v->x_name_len; + unsigned val_len = le16_to_cpu(xattr.v->x_val_len); + unsigned max_name_val_bytes = bkey_val_bytes(xattr.k) - + offsetof(struct bch_xattr, x_name); + + val_len = min_t(int, val_len, max_name_val_bytes - name_len); + name_len = min(name_len, max_name_val_bytes); + prt_printf(out, "%.*s:%.*s", - xattr.v->x_name_len, - xattr.v->x_name, - le16_to_cpu(xattr.v->x_val_len), - (char *) xattr_val(xattr.v)); + name_len, xattr.v->x_name, + val_len, (char *) xattr_val(xattr.v)); if (xattr.v->x_type == KEY_TYPE_XATTR_INDEX_POSIX_ACL_ACCESS || xattr.v->x_type == KEY_TYPE_XATTR_INDEX_POSIX_ACL_DEFAULT) { From 49e9d01f669e5cc5adae391589507541f8ebce07 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sun, 5 May 2024 10:46:18 +0200 Subject: [PATCH 1353/1702] dt-bindings: Use full path to other schemas When referencing other schema, it is preferred to use an absolute path (/schemas/....), which allows also an seamless move of particular schema out of Linux kernel to dtschema. Signed-off-by: Krzysztof Kozlowski Acked-by: Miquel Raynal Link: https://lore.kernel.org/r/20240505084618.135705-1-krzysztof.kozlowski@linaro.org Signed-off-by: Rob Herring (Arm) --- Documentation/devicetree/bindings/input/azoteq,iqs7222.yaml | 2 +- Documentation/devicetree/bindings/media/amphion,vpu.yaml | 2 +- Documentation/devicetree/bindings/mtd/mtd.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/input/azoteq,iqs7222.yaml b/Documentation/devicetree/bindings/input/azoteq,iqs7222.yaml index 5b1769c19b173..418c168b223b7 100644 --- a/Documentation/devicetree/bindings/input/azoteq,iqs7222.yaml +++ b/Documentation/devicetree/bindings/input/azoteq,iqs7222.yaml @@ -784,7 +784,7 @@ patternProperties: gpio-2: GPIO4 allOf: - - $ref: ../pinctrl/pincfg-node.yaml# + - $ref: /schemas/pinctrl/pincfg-node.yaml# properties: drive-open-drain: true diff --git a/Documentation/devicetree/bindings/media/amphion,vpu.yaml b/Documentation/devicetree/bindings/media/amphion,vpu.yaml index c0d83d7552399..9801de3ed84e8 100644 --- a/Documentation/devicetree/bindings/media/amphion,vpu.yaml +++ b/Documentation/devicetree/bindings/media/amphion,vpu.yaml @@ -44,7 +44,7 @@ patternProperties: description: Each vpu encoder or decoder correspond a MU, which used for communication between driver and firmware. Implement via mailbox on driver. - $ref: ../mailbox/fsl,mu.yaml# + $ref: /schemas/mailbox/fsl,mu.yaml# "^vpu-core@[0-9a-f]+$": diff --git a/Documentation/devicetree/bindings/mtd/mtd.yaml b/Documentation/devicetree/bindings/mtd/mtd.yaml index ee442ecb11cd9..bbb56216a4e25 100644 --- a/Documentation/devicetree/bindings/mtd/mtd.yaml +++ b/Documentation/devicetree/bindings/mtd/mtd.yaml @@ -48,8 +48,8 @@ patternProperties: type: object allOf: - - $ref: ../nvmem/nvmem.yaml# - - $ref: ../nvmem/nvmem-deprecated-cells.yaml# + - $ref: /schemas/nvmem/nvmem.yaml# + - $ref: /schemas/nvmem/nvmem-deprecated-cells.yaml# unevaluatedProperties: false From 6d828691858396e95e9856fad6a08c57f06b6e2d Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 3 May 2024 14:17:54 -0400 Subject: [PATCH 1354/1702] bcachefs: Better write_super() error messages When a superblock write is silently dropped or it's been modified by another process we need to know which device it was. Signed-off-by: Kent Overstreet --- fs/bcachefs/super-io.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/fs/bcachefs/super-io.c b/fs/bcachefs/super-io.c index bfdb15e7d778e..1cadc99898f56 100644 --- a/fs/bcachefs/super-io.c +++ b/fs/bcachefs/super-io.c @@ -1020,26 +1020,35 @@ int bch2_write_super(struct bch_fs *c) continue; if (le64_to_cpu(ca->sb_read_scratch->seq) < ca->disk_sb.seq) { - bch2_fs_fatal_error(c, + struct printbuf buf = PRINTBUF; + prt_char(&buf, ' '); + prt_bdevname(&buf, ca->disk_sb.bdev); + prt_printf(&buf, ": Superblock write was silently dropped! (seq %llu expected %llu)", le64_to_cpu(ca->sb_read_scratch->seq), ca->disk_sb.seq); - percpu_ref_put(&ca->io_ref); + bch2_fs_fatal_error(c, "%s", buf.buf); + printbuf_exit(&buf); ret = -BCH_ERR_erofs_sb_err; - goto out; } if (le64_to_cpu(ca->sb_read_scratch->seq) > ca->disk_sb.seq) { - bch2_fs_fatal_error(c, + struct printbuf buf = PRINTBUF; + prt_char(&buf, ' '); + prt_bdevname(&buf, ca->disk_sb.bdev); + prt_printf(&buf, ": Superblock modified by another process (seq %llu expected %llu)", le64_to_cpu(ca->sb_read_scratch->seq), ca->disk_sb.seq); - percpu_ref_put(&ca->io_ref); + bch2_fs_fatal_error(c, "%s", buf.buf); + printbuf_exit(&buf); ret = -BCH_ERR_erofs_sb_err; - goto out; } } + if (ret) + goto out; + do { wrote = false; darray_for_each(online_devices, cap) { From 62606398d5d718186d914d8f30be34177dc3f26d Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 28 Apr 2024 21:24:45 -0400 Subject: [PATCH 1355/1702] bcachefs: Run upgrade/downgrade even in -o nochanges mode We need to be able to test these paths in dry run mode. Signed-off-by: Kent Overstreet --- fs/bcachefs/recovery.c | 84 +++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/fs/bcachefs/recovery.c b/fs/bcachefs/recovery.c index 8091d0686029f..08c0422a722dc 100644 --- a/fs/bcachefs/recovery.c +++ b/fs/bcachefs/recovery.c @@ -597,56 +597,54 @@ int bch2_fs_recovery(struct bch_fs *c) if (c->opts.norecovery) c->opts.recovery_pass_last = BCH_RECOVERY_PASS_journal_replay - 1; - if (!c->opts.nochanges) { - mutex_lock(&c->sb_lock); - struct bch_sb_field_ext *ext = bch2_sb_field_get(c->disk_sb.sb, ext); - bool write_sb = false; - - if (BCH_SB_HAS_TOPOLOGY_ERRORS(c->disk_sb.sb)) { - ext->recovery_passes_required[0] |= - cpu_to_le64(bch2_recovery_passes_to_stable(BIT_ULL(BCH_RECOVERY_PASS_check_topology))); - write_sb = true; - } + mutex_lock(&c->sb_lock); + struct bch_sb_field_ext *ext = bch2_sb_field_get(c->disk_sb.sb, ext); + bool write_sb = false; - u64 sb_passes = bch2_recovery_passes_from_stable(le64_to_cpu(ext->recovery_passes_required[0])); - if (sb_passes) { - struct printbuf buf = PRINTBUF; - prt_str(&buf, "superblock requires following recovery passes to be run:\n "); - prt_bitflags(&buf, bch2_recovery_passes, sb_passes); - bch_info(c, "%s", buf.buf); - printbuf_exit(&buf); - } + if (BCH_SB_HAS_TOPOLOGY_ERRORS(c->disk_sb.sb)) { + ext->recovery_passes_required[0] |= + cpu_to_le64(bch2_recovery_passes_to_stable(BIT_ULL(BCH_RECOVERY_PASS_check_topology))); + write_sb = true; + } - if (bch2_check_version_downgrade(c)) { - struct printbuf buf = PRINTBUF; + u64 sb_passes = bch2_recovery_passes_from_stable(le64_to_cpu(ext->recovery_passes_required[0])); + if (sb_passes) { + struct printbuf buf = PRINTBUF; + prt_str(&buf, "superblock requires following recovery passes to be run:\n "); + prt_bitflags(&buf, bch2_recovery_passes, sb_passes); + bch_info(c, "%s", buf.buf); + printbuf_exit(&buf); + } - prt_str(&buf, "Version downgrade required:"); + if (bch2_check_version_downgrade(c)) { + struct printbuf buf = PRINTBUF; - __le64 passes = ext->recovery_passes_required[0]; - bch2_sb_set_downgrade(c, - BCH_VERSION_MINOR(bcachefs_metadata_version_current), - BCH_VERSION_MINOR(c->sb.version)); - passes = ext->recovery_passes_required[0] & ~passes; - if (passes) { - prt_str(&buf, "\n running recovery passes: "); - prt_bitflags(&buf, bch2_recovery_passes, - bch2_recovery_passes_from_stable(le64_to_cpu(passes))); - } + prt_str(&buf, "Version downgrade required:"); - bch_info(c, "%s", buf.buf); - printbuf_exit(&buf); - write_sb = true; + __le64 passes = ext->recovery_passes_required[0]; + bch2_sb_set_downgrade(c, + BCH_VERSION_MINOR(bcachefs_metadata_version_current), + BCH_VERSION_MINOR(c->sb.version)); + passes = ext->recovery_passes_required[0] & ~passes; + if (passes) { + prt_str(&buf, "\n running recovery passes: "); + prt_bitflags(&buf, bch2_recovery_passes, + bch2_recovery_passes_from_stable(le64_to_cpu(passes))); } - if (check_version_upgrade(c)) - write_sb = true; + bch_info(c, "%s", buf.buf); + printbuf_exit(&buf); + write_sb = true; + } - if (write_sb) - bch2_write_super(c); + if (check_version_upgrade(c)) + write_sb = true; - c->recovery_passes_explicit |= bch2_recovery_passes_from_stable(le64_to_cpu(ext->recovery_passes_required[0])); - mutex_unlock(&c->sb_lock); - } + if (write_sb) + bch2_write_super(c); + + c->recovery_passes_explicit |= bch2_recovery_passes_from_stable(le64_to_cpu(ext->recovery_passes_required[0])); + mutex_unlock(&c->sb_lock); if (c->opts.fsck && IS_ENABLED(CONFIG_BCACHEFS_DEBUG)) c->recovery_passes_explicit |= BIT_ULL(BCH_RECOVERY_PASS_check_topology); @@ -832,8 +830,8 @@ int bch2_fs_recovery(struct bch_fs *c) } mutex_lock(&c->sb_lock); - struct bch_sb_field_ext *ext = bch2_sb_field_get(c->disk_sb.sb, ext); - bool write_sb = false; + ext = bch2_sb_field_get(c->disk_sb.sb, ext); + write_sb = false; if (BCH_SB_VERSION_UPGRADE_COMPLETE(c->disk_sb.sb) != le16_to_cpu(c->disk_sb.sb->version)) { SET_BCH_SB_VERSION_UPGRADE_COMPLETE(c->disk_sb.sb, le16_to_cpu(c->disk_sb.sb->version)); From acce32a51e6c32e0ad8f921f9af2226e6750bb51 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 10 Apr 2024 14:32:45 -0400 Subject: [PATCH 1356/1702] bcachefs: printbuf improvements - fix assorted (harmless) off-by-one errors - we were inconsistent on whether out->pos stays <= out->size on overflow; now it does, and printbuf.overflow exists to indicate if a printbuf has overflowed - factor out printbuf_advance_pos() - printbuf_nul_terminate_reserved(); use this to reduce the number of printbuf_make_room() calls Signed-off-by: Kent Overstreet --- fs/bcachefs/printbuf.c | 65 ++++++++++++++++++++---------------------- fs/bcachefs/printbuf.h | 53 ++++++++++++++++------------------ 2 files changed, 55 insertions(+), 63 deletions(-) diff --git a/fs/bcachefs/printbuf.c b/fs/bcachefs/printbuf.c index b27d22925929a..46ce63312a3fb 100644 --- a/fs/bcachefs/printbuf.c +++ b/fs/bcachefs/printbuf.c @@ -17,28 +17,28 @@ static inline unsigned printbuf_linelen(struct printbuf *buf) int bch2_printbuf_make_room(struct printbuf *out, unsigned extra) { - unsigned new_size; - char *buf; - - if (!out->heap_allocated) - return 0; - /* Reserved space for terminating nul: */ extra += 1; - if (out->pos + extra < out->size) + if (out->pos + extra <= out->size) return 0; - new_size = roundup_pow_of_two(out->size + extra); + if (!out->heap_allocated) { + out->overflow = true; + return 0; + } + + unsigned new_size = roundup_pow_of_two(out->size + extra); /* * Note: output buffer must be freeable with kfree(), it's not required * that the user use printbuf_exit(). */ - buf = krealloc(out->buf, new_size, !out->atomic ? GFP_KERNEL : GFP_NOWAIT); + char *buf = krealloc(out->buf, new_size, !out->atomic ? GFP_KERNEL : GFP_NOWAIT); if (!buf) { out->allocation_failure = true; + out->overflow = true; return -ENOMEM; } @@ -47,6 +47,11 @@ int bch2_printbuf_make_room(struct printbuf *out, unsigned extra) return 0; } +static void printbuf_advance_pos(struct printbuf *out, unsigned len) +{ + out->pos += min(len, printbuf_remaining(out)); +} + void bch2_prt_vprintf(struct printbuf *out, const char *fmt, va_list args) { int len; @@ -55,14 +60,12 @@ void bch2_prt_vprintf(struct printbuf *out, const char *fmt, va_list args) va_list args2; va_copy(args2, args); - len = vsnprintf(out->buf + out->pos, printbuf_remaining(out), fmt, args2); + len = vsnprintf(out->buf + out->pos, printbuf_remaining_size(out), fmt, args2); va_end(args2); - } while (len + 1 >= printbuf_remaining(out) && - !bch2_printbuf_make_room(out, len + 1)); + } while (len > printbuf_remaining(out) && + !bch2_printbuf_make_room(out, len)); - len = min_t(size_t, len, - printbuf_remaining(out) ? printbuf_remaining(out) - 1 : 0); - out->pos += len; + printbuf_advance_pos(out, len); } void bch2_prt_printf(struct printbuf *out, const char *fmt, ...) @@ -72,14 +75,12 @@ void bch2_prt_printf(struct printbuf *out, const char *fmt, ...) do { va_start(args, fmt); - len = vsnprintf(out->buf + out->pos, printbuf_remaining(out), fmt, args); + len = vsnprintf(out->buf + out->pos, printbuf_remaining_size(out), fmt, args); va_end(args); - } while (len + 1 >= printbuf_remaining(out) && - !bch2_printbuf_make_room(out, len + 1)); + } while (len > printbuf_remaining(out) && + !bch2_printbuf_make_room(out, len)); - len = min_t(size_t, len, - printbuf_remaining(out) ? printbuf_remaining(out) - 1 : 0); - out->pos += len; + printbuf_advance_pos(out, len); } /** @@ -194,18 +195,15 @@ void bch2_printbuf_indent_sub(struct printbuf *buf, unsigned spaces) void bch2_prt_newline(struct printbuf *buf) { - unsigned i; - bch2_printbuf_make_room(buf, 1 + buf->indent); - __prt_char(buf, '\n'); + __prt_char_reserved(buf, '\n'); buf->last_newline = buf->pos; - for (i = 0; i < buf->indent; i++) - __prt_char(buf, ' '); + __prt_chars_reserved(buf, ' ', buf->indent); - printbuf_nul_terminate(buf); + printbuf_nul_terminate_reserved(buf); buf->last_field = buf->pos; buf->cur_tabstop = 0; @@ -262,7 +260,7 @@ static void __prt_tab_rjust(struct printbuf *buf) memset(buf->buf + buf->last_field, ' ', min((unsigned) pad, buf->size - buf->last_field)); - buf->pos += pad; + printbuf_advance_pos(buf, pad); printbuf_nul_terminate(buf); } @@ -348,9 +346,10 @@ void bch2_prt_bytes_indented(struct printbuf *out, const char *str, unsigned cou void bch2_prt_human_readable_u64(struct printbuf *out, u64 v) { bch2_printbuf_make_room(out, 10); - out->pos += string_get_size(v, 1, !out->si_units, - out->buf + out->pos, - printbuf_remaining_size(out)); + unsigned len = string_get_size(v, 1, !out->si_units, + out->buf + out->pos, + printbuf_remaining_size(out)); + printbuf_advance_pos(out, len); } /** @@ -402,9 +401,7 @@ void bch2_prt_string_option(struct printbuf *out, const char * const list[], size_t selected) { - size_t i; - - for (i = 0; list[i]; i++) + for (size_t i = 0; list[i]; i++) bch2_prt_printf(out, i == selected ? "[%s] " : "%s ", list[i]); } diff --git a/fs/bcachefs/printbuf.h b/fs/bcachefs/printbuf.h index 9a4a56c409371..9ecc56bc96359 100644 --- a/fs/bcachefs/printbuf.h +++ b/fs/bcachefs/printbuf.h @@ -86,6 +86,7 @@ struct printbuf { u8 atomic; bool allocation_failure:1; bool heap_allocated:1; + bool overflow:1; enum printbuf_si si_units:1; bool human_readable_units:1; bool has_indent_or_tabstops:1; @@ -142,7 +143,9 @@ void bch2_prt_bitflags_vector(struct printbuf *, const char * const[], */ static inline unsigned printbuf_remaining_size(struct printbuf *out) { - return out->pos < out->size ? out->size - out->pos : 0; + if (WARN_ON(out->size && out->pos >= out->size)) + out->pos = out->size - 1; + return out->size - out->pos; } /* @@ -151,7 +154,7 @@ static inline unsigned printbuf_remaining_size(struct printbuf *out) */ static inline unsigned printbuf_remaining(struct printbuf *out) { - return out->pos < out->size ? out->size - out->pos - 1 : 0; + return out->size ? printbuf_remaining_size(out) - 1 : 0; } static inline unsigned printbuf_written(struct printbuf *out) @@ -159,30 +162,25 @@ static inline unsigned printbuf_written(struct printbuf *out) return out->size ? min(out->pos, out->size - 1) : 0; } -/* - * Returns true if output was truncated: - */ -static inline bool printbuf_overflowed(struct printbuf *out) +static inline void printbuf_nul_terminate_reserved(struct printbuf *out) { - return out->pos >= out->size; + if (WARN_ON(out->size && out->pos >= out->size)) + out->pos = out->size - 1; + if (out->size) + out->buf[out->pos] = 0; } static inline void printbuf_nul_terminate(struct printbuf *out) { bch2_printbuf_make_room(out, 1); - - if (out->pos < out->size) - out->buf[out->pos] = 0; - else if (out->size) - out->buf[out->size - 1] = 0; + printbuf_nul_terminate_reserved(out); } /* Doesn't call bch2_printbuf_make_room(), doesn't nul terminate: */ static inline void __prt_char_reserved(struct printbuf *out, char c) { if (printbuf_remaining(out)) - out->buf[out->pos] = c; - out->pos++; + out->buf[out->pos++] = c; } /* Doesn't nul terminate: */ @@ -194,37 +192,34 @@ static inline void __prt_char(struct printbuf *out, char c) static inline void prt_char(struct printbuf *out, char c) { - __prt_char(out, c); - printbuf_nul_terminate(out); + bch2_printbuf_make_room(out, 2); + __prt_char_reserved(out, c); + printbuf_nul_terminate_reserved(out); } static inline void __prt_chars_reserved(struct printbuf *out, char c, unsigned n) { - unsigned i, can_print = min(n, printbuf_remaining(out)); + unsigned can_print = min(n, printbuf_remaining(out)); - for (i = 0; i < can_print; i++) + for (unsigned i = 0; i < can_print; i++) out->buf[out->pos++] = c; - out->pos += n - can_print; } static inline void prt_chars(struct printbuf *out, char c, unsigned n) { bch2_printbuf_make_room(out, n); __prt_chars_reserved(out, c, n); - printbuf_nul_terminate(out); + printbuf_nul_terminate_reserved(out); } static inline void prt_bytes(struct printbuf *out, const void *b, unsigned n) { - unsigned i, can_print; - bch2_printbuf_make_room(out, n); - can_print = min(n, printbuf_remaining(out)); + unsigned can_print = min(n, printbuf_remaining(out)); - for (i = 0; i < can_print; i++) + for (unsigned i = 0; i < can_print; i++) out->buf[out->pos++] = ((char *) b)[i]; - out->pos += n - can_print; printbuf_nul_terminate(out); } @@ -241,18 +236,18 @@ static inline void prt_str_indented(struct printbuf *out, const char *str) static inline void prt_hex_byte(struct printbuf *out, u8 byte) { - bch2_printbuf_make_room(out, 2); + bch2_printbuf_make_room(out, 3); __prt_char_reserved(out, hex_asc_hi(byte)); __prt_char_reserved(out, hex_asc_lo(byte)); - printbuf_nul_terminate(out); + printbuf_nul_terminate_reserved(out); } static inline void prt_hex_byte_upper(struct printbuf *out, u8 byte) { - bch2_printbuf_make_room(out, 2); + bch2_printbuf_make_room(out, 3); __prt_char_reserved(out, hex_asc_upper_hi(byte)); __prt_char_reserved(out, hex_asc_upper_lo(byte)); - printbuf_nul_terminate(out); + printbuf_nul_terminate_reserved(out); } /** From 2dcb605e86891f17bea3d77ba05f5d12cdf38573 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 10 Apr 2024 14:32:45 -0400 Subject: [PATCH 1357/1702] bcachefs: printbufs: prt_printf() now handles \t\r\n Signed-off-by: Kent Overstreet --- fs/bcachefs/printbuf.c | 169 ++++++++++++++++++++++++++--------------- 1 file changed, 106 insertions(+), 63 deletions(-) diff --git a/fs/bcachefs/printbuf.c b/fs/bcachefs/printbuf.c index 46ce63312a3fb..8b0369185f5cf 100644 --- a/fs/bcachefs/printbuf.c +++ b/fs/bcachefs/printbuf.c @@ -10,9 +10,24 @@ #include "printbuf.h" +static inline unsigned __printbuf_linelen(struct printbuf *buf, unsigned pos) +{ + return pos - buf->last_newline; +} + static inline unsigned printbuf_linelen(struct printbuf *buf) { - return buf->pos - buf->last_newline; + return __printbuf_linelen(buf, buf->pos); +} + +/* + * Returns spaces from start of line, if set, or 0 if unset: + */ +static inline unsigned cur_tabstop(struct printbuf *buf) +{ + return buf->cur_tabstop < buf->nr_tabstops + ? buf->_tabstops[buf->cur_tabstop] + : 0; } int bch2_printbuf_make_room(struct printbuf *out, unsigned extra) @@ -52,6 +67,87 @@ static void printbuf_advance_pos(struct printbuf *out, unsigned len) out->pos += min(len, printbuf_remaining(out)); } +static void printbuf_insert_spaces(struct printbuf *out, unsigned pos, unsigned nr) +{ + unsigned move = out->pos - pos; + + bch2_printbuf_make_room(out, nr); + + if (pos + nr < out->size) + memmove(out->buf + pos + nr, + out->buf + pos, + min(move, out->size - 1 - pos - nr)); + + if (pos < out->size) + memset(out->buf + pos, ' ', min(nr, out->size - pos)); + + printbuf_advance_pos(out, nr); + printbuf_nul_terminate_reserved(out); +} + +static void __printbuf_do_indent(struct printbuf *out, unsigned pos) +{ + while (true) { + int pad; + unsigned len = out->pos - pos; + char *p = out->buf + pos; + char *n = memscan(p, '\n', len); + if (cur_tabstop(out)) { + n = min(n, (char *) memscan(p, '\r', len)); + n = min(n, (char *) memscan(p, '\t', len)); + } + + pos = n - out->buf; + if (pos == out->pos) + break; + + switch (*n) { + case '\n': + pos++; + out->last_newline = pos; + + printbuf_insert_spaces(out, pos, out->indent); + + pos = min(pos + out->indent, out->pos); + out->last_field = pos; + out->cur_tabstop = 0; + break; + case '\r': + memmove(n, n + 1, out->pos - pos); + --out->pos; + pad = (int) cur_tabstop(out) - (int) __printbuf_linelen(out, pos); + if (pad > 0) { + printbuf_insert_spaces(out, out->last_field, pad); + pos += pad; + } + + out->last_field = pos; + out->cur_tabstop++; + break; + case '\t': + pad = (int) cur_tabstop(out) - (int) __printbuf_linelen(out, pos) - 1; + if (pad > 0) { + *n = ' '; + printbuf_insert_spaces(out, pos, pad - 1); + pos += pad; + } else { + memmove(n, n + 1, out->pos - pos); + --out->pos; + } + + out->last_field = pos; + out->cur_tabstop++; + break; + } + } +} + +static inline void printbuf_do_indent(struct printbuf *out, unsigned pos) +{ + if (out->has_indent_or_tabstops && !out->suppress_indent_tabstop_handling) + __printbuf_do_indent(out, pos); +} + void bch2_prt_vprintf(struct printbuf *out, const char *fmt, va_list args) { int len; @@ -65,7 +161,9 @@ void bch2_prt_vprintf(struct printbuf *out, const char *fmt, va_list args) } while (len > printbuf_remaining(out) && !bch2_printbuf_make_room(out, len)); + unsigned indent_pos = out->pos; printbuf_advance_pos(out, len); + printbuf_do_indent(out, indent_pos); } void bch2_prt_printf(struct printbuf *out, const char *fmt, ...) @@ -80,7 +178,9 @@ void bch2_prt_printf(struct printbuf *out, const char *fmt, ...) } while (len > printbuf_remaining(out) && !bch2_printbuf_make_room(out, len)); + unsigned indent_pos = out->pos; printbuf_advance_pos(out, len); + printbuf_do_indent(out, indent_pos); } /** @@ -209,16 +309,6 @@ void bch2_prt_newline(struct printbuf *buf) buf->cur_tabstop = 0; } -/* - * Returns spaces from start of line, if set, or 0 if unset: - */ -static inline unsigned cur_tabstop(struct printbuf *buf) -{ - return buf->cur_tabstop < buf->nr_tabstops - ? buf->_tabstops[buf->cur_tabstop] - : 0; -} - static void __prt_tab(struct printbuf *out) { int spaces = max_t(int, 0, cur_tabstop(out) - printbuf_linelen(out)); @@ -245,24 +335,9 @@ void bch2_prt_tab(struct printbuf *out) static void __prt_tab_rjust(struct printbuf *buf) { - unsigned move = buf->pos - buf->last_field; int pad = (int) cur_tabstop(buf) - (int) printbuf_linelen(buf); - - if (pad > 0) { - bch2_printbuf_make_room(buf, pad); - - if (buf->last_field + pad < buf->size) - memmove(buf->buf + buf->last_field + pad, - buf->buf + buf->last_field, - min(move, buf->size - 1 - buf->last_field - pad)); - - if (buf->last_field < buf->size) - memset(buf->buf + buf->last_field, ' ', - min((unsigned) pad, buf->size - buf->last_field)); - - printbuf_advance_pos(buf, pad); - printbuf_nul_terminate(buf); - } + if (pad > 0) + printbuf_insert_spaces(buf, buf->last_field, pad); buf->last_field = buf->pos; buf->cur_tabstop++; @@ -299,41 +374,9 @@ void bch2_prt_tab_rjust(struct printbuf *buf) */ void bch2_prt_bytes_indented(struct printbuf *out, const char *str, unsigned count) { - const char *unprinted_start = str; - const char *end = str + count; - - if (!out->has_indent_or_tabstops || out->suppress_indent_tabstop_handling) { - prt_bytes(out, str, count); - return; - } - - while (str != end) { - switch (*str) { - case '\n': - prt_bytes(out, unprinted_start, str - unprinted_start); - unprinted_start = str + 1; - bch2_prt_newline(out); - break; - case '\t': - if (likely(cur_tabstop(out))) { - prt_bytes(out, unprinted_start, str - unprinted_start); - unprinted_start = str + 1; - __prt_tab(out); - } - break; - case '\r': - if (likely(cur_tabstop(out))) { - prt_bytes(out, unprinted_start, str - unprinted_start); - unprinted_start = str + 1; - __prt_tab_rjust(out); - } - break; - } - - str++; - } - - prt_bytes(out, unprinted_start, str - unprinted_start); + unsigned indent_pos = out->pos; + prt_bytes(out, str, count); + printbuf_do_indent(out, indent_pos); } /** From 7423330e30ab916d56ce9a9d49e37e40ead5b153 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 10 Apr 2024 16:08:24 -0400 Subject: [PATCH 1358/1702] bcachefs: prt_printf() now respects \r\n\t Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 32 +++------ fs/bcachefs/backpointers.c | 3 +- fs/bcachefs/btree_io.c | 15 ++--- fs/bcachefs/btree_iter.c | 18 ++--- fs/bcachefs/btree_key_cache.c | 9 +-- fs/bcachefs/btree_locking.c | 6 +- fs/bcachefs/btree_trans_commit.c | 6 +- fs/bcachefs/buckets.c | 21 ++---- fs/bcachefs/checksum.c | 12 ++-- fs/bcachefs/debug.c | 64 +++++------------- fs/bcachefs/inode.c | 24 ++----- fs/bcachefs/io_read.c | 3 +- fs/bcachefs/io_write.c | 3 +- fs/bcachefs/journal.c | 97 +++++++++++--------------- fs/bcachefs/move.c | 37 ++++------ fs/bcachefs/quota.c | 112 ++++++------------------------- fs/bcachefs/sb-clean.c | 6 +- fs/bcachefs/sb-counters.c | 15 ++--- fs/bcachefs/sb-downgrade.c | 9 +-- fs/bcachefs/sb-members.c | 96 +++++++------------------- fs/bcachefs/super-io.c | 70 ++++++------------- fs/bcachefs/sysfs.c | 96 ++++++-------------------- fs/bcachefs/util.c | 61 +++++------------ 23 files changed, 226 insertions(+), 589 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 534ba2b02bd65..6a7d249f108c9 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -330,27 +330,17 @@ void bch2_alloc_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c prt_printf(out, "gen %u oldest_gen %u data_type ", a->gen, a->oldest_gen); bch2_prt_data_type(out, a->data_type); prt_newline(out); - prt_printf(out, "journal_seq %llu", a->journal_seq); - prt_newline(out); - prt_printf(out, "need_discard %llu", BCH_ALLOC_V4_NEED_DISCARD(a)); - prt_newline(out); - prt_printf(out, "need_inc_gen %llu", BCH_ALLOC_V4_NEED_INC_GEN(a)); - prt_newline(out); - prt_printf(out, "dirty_sectors %u", a->dirty_sectors); - prt_newline(out); - prt_printf(out, "cached_sectors %u", a->cached_sectors); - prt_newline(out); - prt_printf(out, "stripe %u", a->stripe); - prt_newline(out); - prt_printf(out, "stripe_redundancy %u", a->stripe_redundancy); - prt_newline(out); - prt_printf(out, "io_time[READ] %llu", a->io_time[READ]); - prt_newline(out); - prt_printf(out, "io_time[WRITE] %llu", a->io_time[WRITE]); - prt_newline(out); - prt_printf(out, "fragmentation %llu", a->fragmentation_lru); - prt_newline(out); - prt_printf(out, "bp_start %llu", BCH_ALLOC_V4_BACKPOINTERS_START(a)); + prt_printf(out, "journal_seq %llu\n", a->journal_seq); + prt_printf(out, "need_discard %llu\n", BCH_ALLOC_V4_NEED_DISCARD(a)); + prt_printf(out, "need_inc_gen %llu\n", BCH_ALLOC_V4_NEED_INC_GEN(a)); + prt_printf(out, "dirty_sectors %u\n", a->dirty_sectors); + prt_printf(out, "cached_sectors %u\n", a->cached_sectors); + prt_printf(out, "stripe %u\n", a->stripe); + prt_printf(out, "stripe_redundancy %u\n", a->stripe_redundancy); + prt_printf(out, "io_time[READ] %llu\n", a->io_time[READ]); + prt_printf(out, "io_time[WRITE] %llu\n", a->io_time[WRITE]); + prt_printf(out, "fragmentation %llu\n", a->fragmentation_lru); + prt_printf(out, "bp_start %llu\n", BCH_ALLOC_V4_BACKPOINTERS_START(a)); printbuf_indent_sub(out, 2); } diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c index af7a71de1bdfe..fc9506c71eae1 100644 --- a/fs/bcachefs/backpointers.c +++ b/fs/bcachefs/backpointers.c @@ -117,8 +117,7 @@ static noinline int backpointer_mod_err(struct btree_trans *trans, bch_err(c, "%s", buf.buf); } else if (c->curr_recovery_pass > BCH_RECOVERY_PASS_check_extents_to_backpointers) { - prt_printf(&buf, "backpointer not found when deleting"); - prt_newline(&buf); + prt_printf(&buf, "backpointer not found when deleting\n"); printbuf_indent_add(&buf, 2); prt_printf(&buf, "searching for "); diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c index 94753c5b0e897..fc046c9ccb450 100644 --- a/fs/bcachefs/btree_io.c +++ b/fs/bcachefs/btree_io.c @@ -522,7 +522,9 @@ static void btree_err_msg(struct printbuf *out, struct bch_fs *c, prt_printf(out, "at btree "); bch2_btree_pos_to_text(out, c, b); - prt_printf(out, "\n node offset %u/%u", + printbuf_indent_add(out, 2); + + prt_printf(out, "\nnode offset %u/%u", b->written, btree_ptr_sectors_written(&b->key)); if (i) prt_printf(out, " bset u64s %u", le16_to_cpu(i->u64s)); @@ -2344,20 +2346,13 @@ void bch2_btree_write_stats_to_text(struct printbuf *out, struct bch_fs *c) printbuf_tabstop_push(out, 20); printbuf_tabstop_push(out, 10); - prt_tab(out); - prt_str(out, "nr"); - prt_tab(out); - prt_str(out, "size"); - prt_newline(out); + prt_printf(out, "\tnr\tsize\n"); for (unsigned i = 0; i < BTREE_WRITE_TYPE_NR; i++) { u64 nr = atomic64_read(&c->btree_write_stats[i].nr); u64 bytes = atomic64_read(&c->btree_write_stats[i].bytes); - prt_printf(out, "%s:", bch2_btree_write_types[i]); - prt_tab(out); - prt_u64(out, nr); - prt_tab(out); + prt_printf(out, "%s:\t%llu\t", bch2_btree_write_types[i], nr); prt_human_readable_u64(out, nr ? div64_u64(bytes, nr) : 0); prt_newline(out); } diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index 2a211a4bebd15..080cb50168b05 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -1387,19 +1387,17 @@ void __noreturn bch2_trans_in_restart_error(struct btree_trans *trans) noinline __cold void bch2_trans_updates_to_text(struct printbuf *buf, struct btree_trans *trans) { - prt_printf(buf, "transaction updates for %s journal seq %llu", + prt_printf(buf, "transaction updates for %s journal seq %llu\n", trans->fn, trans->journal_res.seq); - prt_newline(buf); printbuf_indent_add(buf, 2); trans_for_each_update(trans, i) { struct bkey_s_c old = { &i->old_k, i->old_v }; - prt_printf(buf, "update: btree=%s cached=%u %pS", + prt_printf(buf, "update: btree=%s cached=%u %pS\n", bch2_btree_id_str(i->btree_id), i->cached, (void *) i->ip_allocated); - prt_newline(buf); prt_printf(buf, " old "); bch2_bkey_val_to_text(buf, trans->c, old); @@ -3166,13 +3164,11 @@ bch2_btree_bkey_cached_common_to_text(struct printbuf *out, pid = owner ? owner->pid : 0; rcu_read_unlock(); - prt_tab(out); - prt_printf(out, "%px %c l=%u %s:", b, b->cached ? 'c' : 'b', + prt_printf(out, "\t%px %c l=%u %s:", b, b->cached ? 'c' : 'b', b->level, bch2_btree_id_str(b->btree_id)); bch2_bpos_to_text(out, btree_node_pos(b)); - prt_tab(out); - prt_printf(out, " locks %u:%u:%u held by pid %u", + prt_printf(out, "\t locks %u:%u:%u held by pid %u", c.n[0], c.n[1], c.n[2], pid); } @@ -3229,10 +3225,8 @@ void bch2_btree_trans_to_text(struct printbuf *out, struct btree_trans *trans) b = READ_ONCE(trans->locking); if (b) { - prt_printf(out, " blocked for %lluus on", - div_u64(local_clock() - trans->locking_wait.start_time, - 1000)); - prt_newline(out); + prt_printf(out, " blocked for %lluus on\n", + div_u64(local_clock() - trans->locking_wait.start_time, 1000)); prt_printf(out, " %c", lock_types[trans->locking_wait.lock_want]); bch2_btree_bkey_cached_common_to_text(out, b); prt_newline(out); diff --git a/fs/bcachefs/btree_key_cache.c b/fs/bcachefs/btree_key_cache.c index 7dafa1accec22..3dea89c6cf7e5 100644 --- a/fs/bcachefs/btree_key_cache.c +++ b/fs/bcachefs/btree_key_cache.c @@ -1039,12 +1039,9 @@ int bch2_fs_btree_key_cache_init(struct btree_key_cache *bc) void bch2_btree_key_cache_to_text(struct printbuf *out, struct btree_key_cache *c) { - prt_printf(out, "nr_freed:\t%lu", atomic_long_read(&c->nr_freed)); - prt_newline(out); - prt_printf(out, "nr_keys:\t%lu", atomic_long_read(&c->nr_keys)); - prt_newline(out); - prt_printf(out, "nr_dirty:\t%lu", atomic_long_read(&c->nr_dirty)); - prt_newline(out); + prt_printf(out, "nr_freed:\t%lu\n", atomic_long_read(&c->nr_freed)); + prt_printf(out, "nr_keys:\t%lu\n", atomic_long_read(&c->nr_keys)); + prt_printf(out, "nr_dirty:\t%lu\n", atomic_long_read(&c->nr_dirty)); } void bch2_btree_key_cache_exit(void) diff --git a/fs/bcachefs/btree_locking.c b/fs/bcachefs/btree_locking.c index f2caf491957ef..4ee4855395b9b 100644 --- a/fs/bcachefs/btree_locking.c +++ b/fs/bcachefs/btree_locking.c @@ -83,8 +83,7 @@ static noinline void print_cycle(struct printbuf *out, struct lock_graph *g) { struct trans_waiting_for_lock *i; - prt_printf(out, "Found lock cycle (%u entries):", g->nr); - prt_newline(out); + prt_printf(out, "Found lock cycle (%u entries):\n", g->nr); for (i = g->g; i < g->g + g->nr; i++) { struct task_struct *task = READ_ONCE(i->trans->locking_wait.task); @@ -224,8 +223,7 @@ static noinline int break_cycle(struct lock_graph *g, struct printbuf *cycle) bch2_btree_trans_to_text(&buf, trans); - prt_printf(&buf, "backtrace:"); - prt_newline(&buf); + prt_printf(&buf, "backtrace:\n"); printbuf_indent_add(&buf, 2); bch2_prt_task_backtrace(&buf, trans->locking_wait.task, 2, GFP_NOWAIT); printbuf_indent_sub(&buf, 2); diff --git a/fs/bcachefs/btree_trans_commit.c b/fs/bcachefs/btree_trans_commit.c index bbec91e8e6506..79cbf54c89128 100644 --- a/fs/bcachefs/btree_trans_commit.c +++ b/fs/bcachefs/btree_trans_commit.c @@ -773,9 +773,8 @@ static noinline int bch2_trans_commit_bkey_invalid(struct btree_trans *trans, struct bch_fs *c = trans->c; printbuf_reset(err); - prt_printf(err, "invalid bkey on insert from %s -> %ps", + prt_printf(err, "invalid bkey on insert from %s -> %ps\n", trans->fn, (void *) i->ip_allocated); - prt_newline(err); printbuf_indent_add(err, 2); bch2_bkey_val_to_text(err, c, bkey_i_to_s_c(i->k)); @@ -796,8 +795,7 @@ static noinline int bch2_trans_commit_journal_entry_invalid(struct btree_trans * struct bch_fs *c = trans->c; struct printbuf buf = PRINTBUF; - prt_printf(&buf, "invalid bkey on insert from %s", trans->fn); - prt_newline(&buf); + prt_printf(&buf, "invalid bkey on insert from %s\n", trans->fn); printbuf_indent_add(&buf, 2); bch2_journal_entry_to_text(&buf, c, i); diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 82f179258867b..6ac6f61cfbc4c 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -274,25 +274,14 @@ void bch2_dev_usage_init(struct bch_dev *ca) void bch2_dev_usage_to_text(struct printbuf *out, struct bch_dev_usage *usage) { - prt_tab(out); - prt_str(out, "buckets"); - prt_tab_rjust(out); - prt_str(out, "sectors"); - prt_tab_rjust(out); - prt_str(out, "fragmented"); - prt_tab_rjust(out); - prt_newline(out); + prt_printf(out, "\tbuckets\rsectors\rfragmented\r\n"); for (unsigned i = 0; i < BCH_DATA_NR; i++) { bch2_prt_data_type(out, i); - prt_tab(out); - prt_u64(out, usage->d[i].buckets); - prt_tab_rjust(out); - prt_u64(out, usage->d[i].sectors); - prt_tab_rjust(out); - prt_u64(out, usage->d[i].fragmented); - prt_tab_rjust(out); - prt_newline(out); + prt_printf(out, "\t%llu\r%llu\r%llu\r\n", + usage->d[i].buckets, + usage->d[i].sectors, + usage->d[i].fragmented); } } diff --git a/fs/bcachefs/checksum.c b/fs/bcachefs/checksum.c index 088fd2e7bdf12..2b3677ae09425 100644 --- a/fs/bcachefs/checksum.c +++ b/fs/bcachefs/checksum.c @@ -494,14 +494,10 @@ static void bch2_sb_crypt_to_text(struct printbuf *out, struct bch_sb *sb, { struct bch_sb_field_crypt *crypt = field_to_type(f, crypt); - prt_printf(out, "KFD: %llu", BCH_CRYPT_KDF_TYPE(crypt)); - prt_newline(out); - prt_printf(out, "scrypt n: %llu", BCH_KDF_SCRYPT_N(crypt)); - prt_newline(out); - prt_printf(out, "scrypt r: %llu", BCH_KDF_SCRYPT_R(crypt)); - prt_newline(out); - prt_printf(out, "scrypt p: %llu", BCH_KDF_SCRYPT_P(crypt)); - prt_newline(out); + prt_printf(out, "KFD: %llu\n", BCH_CRYPT_KDF_TYPE(crypt)); + prt_printf(out, "scrypt n: %llu\n", BCH_KDF_SCRYPT_N(crypt)); + prt_printf(out, "scrypt r: %llu\n", BCH_KDF_SCRYPT_R(crypt)); + prt_printf(out, "scrypt p: %llu\n", BCH_KDF_SCRYPT_P(crypt)); } const struct bch_sb_field_ops bch_sb_field_ops_crypt = { diff --git a/fs/bcachefs/debug.c b/fs/bcachefs/debug.c index cd99b73994144..d0ec74e6dc959 100644 --- a/fs/bcachefs/debug.c +++ b/fs/bcachefs/debug.c @@ -492,51 +492,26 @@ static void bch2_cached_btree_node_to_text(struct printbuf *out, struct bch_fs * if (!out->nr_tabstops) printbuf_tabstop_push(out, 32); - prt_printf(out, "%px btree=%s l=%u ", - b, - bch2_btree_id_str(b->c.btree_id), - b->c.level); - prt_newline(out); + prt_printf(out, "%px btree=%s l=%u\n", b, bch2_btree_id_str(b->c.btree_id), b->c.level); printbuf_indent_add(out, 2); bch2_bkey_val_to_text(out, c, bkey_i_to_s_c(&b->key)); prt_newline(out); - prt_printf(out, "flags: "); - prt_tab(out); + prt_printf(out, "flags:\t"); prt_bitflags(out, bch2_btree_node_flags, b->flags); prt_newline(out); - prt_printf(out, "pcpu read locks: "); - prt_tab(out); - prt_printf(out, "%u", b->c.lock.readers != NULL); - prt_newline(out); - - prt_printf(out, "written:"); - prt_tab(out); - prt_printf(out, "%u", b->written); - prt_newline(out); - - prt_printf(out, "writes blocked:"); - prt_tab(out); - prt_printf(out, "%u", !list_empty_careful(&b->write_blocked)); - prt_newline(out); - - prt_printf(out, "will make reachable:"); - prt_tab(out); - prt_printf(out, "%lx", b->will_make_reachable); - prt_newline(out); - - prt_printf(out, "journal pin %px:", &b->writes[0].journal); - prt_tab(out); - prt_printf(out, "%llu", b->writes[0].journal.seq); - prt_newline(out); + prt_printf(out, "pcpu read locks:\t%u\n", b->c.lock.readers != NULL); + prt_printf(out, "written:\t%u\n", b->written); + prt_printf(out, "writes blocked:\t%u\n", !list_empty_careful(&b->write_blocked)); + prt_printf(out, "will make reachable:\t%lx\n", b->will_make_reachable); - prt_printf(out, "journal pin %px:", &b->writes[1].journal); - prt_tab(out); - prt_printf(out, "%llu", b->writes[1].journal.seq); - prt_newline(out); + prt_printf(out, "journal pin %px:\t%llu\n", + &b->writes[0].journal, b->writes[0].journal.seq); + prt_printf(out, "journal pin %px:\t%llu\n", + &b->writes[1].journal, b->writes[1].journal.seq); printbuf_indent_sub(out, 2); } @@ -625,8 +600,7 @@ static ssize_t bch2_btree_transactions_read(struct file *file, char __user *buf, bch2_btree_trans_to_text(&i->buf, trans); - prt_printf(&i->buf, "backtrace:"); - prt_newline(&i->buf); + prt_printf(&i->buf, "backtrace:\n"); printbuf_indent_add(&i->buf, 2); bch2_prt_task_backtrace(&i->buf, task, 0, GFP_KERNEL); printbuf_indent_sub(&i->buf, 2); @@ -782,25 +756,20 @@ static ssize_t btree_transaction_stats_read(struct file *file, char __user *buf, !bch2_btree_transaction_fns[i->iter]) break; - prt_printf(&i->buf, "%s: ", bch2_btree_transaction_fns[i->iter]); - prt_newline(&i->buf); + prt_printf(&i->buf, "%s:\n", bch2_btree_transaction_fns[i->iter]); printbuf_indent_add(&i->buf, 2); mutex_lock(&s->lock); - prt_printf(&i->buf, "Max mem used: %u", s->max_mem); - prt_newline(&i->buf); - - prt_printf(&i->buf, "Transaction duration:"); - prt_newline(&i->buf); + prt_printf(&i->buf, "Max mem used: %u\n", s->max_mem); + prt_printf(&i->buf, "Transaction duration:\n"); printbuf_indent_add(&i->buf, 2); bch2_time_stats_to_text(&i->buf, &s->duration); printbuf_indent_sub(&i->buf, 2); if (IS_ENABLED(CONFIG_BCACHEFS_LOCK_TIME_STATS)) { - prt_printf(&i->buf, "Lock hold times:"); - prt_newline(&i->buf); + prt_printf(&i->buf, "Lock hold times:\n"); printbuf_indent_add(&i->buf, 2); bch2_time_stats_to_text(&i->buf, &s->lock_hold_times); @@ -808,8 +777,7 @@ static ssize_t btree_transaction_stats_read(struct file *file, char __user *buf, } if (s->max_paths_text) { - prt_printf(&i->buf, "Maximum allocated btree paths (%u):", s->nr_max_paths); - prt_newline(&i->buf); + prt_printf(&i->buf, "Maximum allocated btree paths (%u):\n", s->nr_max_paths); printbuf_indent_add(&i->buf, 2); prt_str_indented(&i->buf, s->max_paths_text); diff --git a/fs/bcachefs/inode.c b/fs/bcachefs/inode.c index 0f95d7fb5ec0b..dc72c59773fec 100644 --- a/fs/bcachefs/inode.c +++ b/fs/bcachefs/inode.c @@ -535,29 +535,19 @@ static void __bch2_inode_unpacked_to_text(struct printbuf *out, struct bch_inode_unpacked *inode) { printbuf_indent_add(out, 2); - prt_printf(out, "mode=%o", inode->bi_mode); - prt_newline(out); + prt_printf(out, "mode=%o\n", inode->bi_mode); prt_str(out, "flags="); prt_bitflags(out, bch2_inode_flag_strs, inode->bi_flags & ((1U << 20) - 1)); - prt_printf(out, " (%x)", inode->bi_flags); - prt_newline(out); + prt_printf(out, " (%x)\n", inode->bi_flags); - prt_printf(out, "journal_seq=%llu", inode->bi_journal_seq); - prt_newline(out); - - prt_printf(out, "bi_size=%llu", inode->bi_size); - prt_newline(out); - - prt_printf(out, "bi_sectors=%llu", inode->bi_sectors); - prt_newline(out); - - prt_printf(out, "bi_version=%llu", inode->bi_version); - prt_newline(out); + prt_printf(out, "journal_seq=%llu\n", inode->bi_journal_seq); + prt_printf(out, "bi_size=%llu\n", inode->bi_size); + prt_printf(out, "bi_sectors=%llu\n", inode->bi_sectors); + prt_printf(out, "bi_version=%llu\n", inode->bi_version); #define x(_name, _bits) \ - prt_printf(out, #_name "=%llu", (u64) inode->_name); \ - prt_newline(out); + prt_printf(out, #_name "=%llu\n", (u64) inode->_name); BCH_INODE_FIELDS_v3() #undef x printbuf_indent_sub(out, 2); diff --git a/fs/bcachefs/io_read.c b/fs/bcachefs/io_read.c index 8a556e6d1ab6f..c07902b6c759f 100644 --- a/fs/bcachefs/io_read.c +++ b/fs/bcachefs/io_read.c @@ -771,9 +771,8 @@ static noinline void read_from_stale_dirty_pointer(struct btree_trans *trans, PTR_BUCKET_POS(c, &ptr), BTREE_ITER_CACHED); - prt_printf(&buf, "Attempting to read from stale dirty pointer:"); + prt_printf(&buf, "Attempting to read from stale dirty pointer:\n"); printbuf_indent_add(&buf, 2); - prt_newline(&buf); bch2_bkey_val_to_text(&buf, c, k); prt_newline(&buf); diff --git a/fs/bcachefs/io_write.c b/fs/bcachefs/io_write.c index 40d7df7607dfd..c11e0944b0698 100644 --- a/fs/bcachefs/io_write.c +++ b/fs/bcachefs/io_write.c @@ -1649,8 +1649,7 @@ void bch2_write_op_to_text(struct printbuf *out, struct bch_write_op *op) prt_bitflags(out, bch2_write_flags, op->flags); prt_newline(out); - prt_printf(out, "ref: %u", closure_nr_remaining(&op->cl)); - prt_newline(out); + prt_printf(out, "ref: %u\n", closure_nr_remaining(&op->cl)); printbuf_indent_sub(out, 2); } diff --git a/fs/bcachefs/journal.c b/fs/bcachefs/journal.c index a8b08e76d0d01..c40b5c40d63ff 100644 --- a/fs/bcachefs/journal.c +++ b/fs/bcachefs/journal.c @@ -53,29 +53,19 @@ static void bch2_journal_buf_to_text(struct printbuf *out, struct journal *j, u6 unsigned i = seq & JOURNAL_BUF_MASK; struct journal_buf *buf = j->buf + i; - prt_str(out, "seq:"); - prt_tab(out); - prt_printf(out, "%llu", seq); - prt_newline(out); + prt_printf(out, "seq:\t%llu\n", seq); printbuf_indent_add(out, 2); - prt_str(out, "refcount:"); - prt_tab(out); - prt_printf(out, "%u", journal_state_count(s, i)); - prt_newline(out); + prt_printf(out, "refcount:\t%u\n", journal_state_count(s, i)); - prt_str(out, "size:"); - prt_tab(out); + prt_printf(out, "size:\t"); prt_human_readable_u64(out, vstruct_bytes(buf->data)); prt_newline(out); - prt_str(out, "expires:"); - prt_tab(out); - prt_printf(out, "%li jiffies", buf->expires - jiffies); - prt_newline(out); + prt_printf(out, "expires:\t"); + prt_printf(out, "%li jiffies\n", buf->expires - jiffies); - prt_str(out, "flags:"); - prt_tab(out); + prt_printf(out, "flags:\t"); if (buf->noflush) prt_str(out, "noflush "); if (buf->must_flush) @@ -1422,12 +1412,12 @@ void __bch2_journal_debug_to_text(struct printbuf *out, struct journal *j) s = READ_ONCE(j->reservations); prt_printf(out, "dirty journal entries:\t%llu/%llu\n", fifo_used(&j->pin), j->pin.size); - prt_printf(out, "seq:\t\t\t%llu\n", journal_cur_seq(j)); - prt_printf(out, "seq_ondisk:\t\t%llu\n", j->seq_ondisk); - prt_printf(out, "last_seq:\t\t%llu\n", journal_last_seq(j)); + prt_printf(out, "seq:\t%llu\n", journal_cur_seq(j)); + prt_printf(out, "seq_ondisk:\t%llu\n", j->seq_ondisk); + prt_printf(out, "last_seq:\t%llu\n", journal_last_seq(j)); prt_printf(out, "last_seq_ondisk:\t%llu\n", j->last_seq_ondisk); prt_printf(out, "flushed_seq_ondisk:\t%llu\n", j->flushed_seq_ondisk); - prt_printf(out, "watermark:\t\t%s\n", bch2_watermarks[j->watermark]); + prt_printf(out, "watermark:\t%s\n", bch2_watermarks[j->watermark]); prt_printf(out, "each entry reserved:\t%u\n", j->entry_u64s_reserved); prt_printf(out, "nr flush writes:\t%llu\n", j->nr_flush_writes); prt_printf(out, "nr noflush writes:\t%llu\n", j->nr_noflush_writes); @@ -1436,48 +1426,48 @@ void __bch2_journal_debug_to_text(struct printbuf *out, struct journal *j) prt_newline(out); prt_printf(out, "nr direct reclaim:\t%llu\n", j->nr_direct_reclaim); prt_printf(out, "nr background reclaim:\t%llu\n", j->nr_background_reclaim); - prt_printf(out, "reclaim kicked:\t\t%u\n", j->reclaim_kicked); + prt_printf(out, "reclaim kicked:\t%u\n", j->reclaim_kicked); prt_printf(out, "reclaim runs in:\t%u ms\n", time_after(j->next_reclaim, now) ? jiffies_to_msecs(j->next_reclaim - jiffies) : 0); - prt_printf(out, "blocked:\t\t%u\n", j->blocked); + prt_printf(out, "blocked:\t%u\n", j->blocked); prt_printf(out, "current entry sectors:\t%u\n", j->cur_entry_sectors); prt_printf(out, "current entry error:\t%s\n", bch2_journal_errors[j->cur_entry_error]); - prt_printf(out, "current entry:\t\t"); + prt_printf(out, "current entry:\t"); switch (s.cur_entry_offset) { case JOURNAL_ENTRY_ERROR_VAL: - prt_printf(out, "error"); + prt_printf(out, "error\n"); break; case JOURNAL_ENTRY_CLOSED_VAL: - prt_printf(out, "closed"); + prt_printf(out, "closed\n"); break; default: - prt_printf(out, "%u/%u", s.cur_entry_offset, j->cur_entry_u64s); + prt_printf(out, "%u/%u\n", s.cur_entry_offset, j->cur_entry_u64s); break; } - prt_newline(out); - prt_printf(out, "unwritten entries:"); - prt_newline(out); + prt_printf(out, "unwritten entries:\n"); bch2_journal_bufs_to_text(out, j); prt_printf(out, - "replay done:\t\t%i\n", + "replay done:\t%i\n", test_bit(JOURNAL_REPLAY_DONE, &j->flags)); prt_printf(out, "space:\n"); - prt_printf(out, "\tdiscarded\t%u:%u\n", + printbuf_indent_add(out, 2); + prt_printf(out, "discarded\t%u:%u\n", j->space[journal_space_discarded].next_entry, j->space[journal_space_discarded].total); - prt_printf(out, "\tclean ondisk\t%u:%u\n", + prt_printf(out, "clean ondisk\t%u:%u\n", j->space[journal_space_clean_ondisk].next_entry, j->space[journal_space_clean_ondisk].total); - prt_printf(out, "\tclean\t\t%u:%u\n", + prt_printf(out, "clean\t%u:%u\n", j->space[journal_space_clean].next_entry, j->space[journal_space_clean].total); - prt_printf(out, "\ttotal\t\t%u:%u\n", + prt_printf(out, "total\t%u:%u\n", j->space[journal_space_total].next_entry, j->space[journal_space_total].total); + printbuf_indent_sub(out, 2); for_each_member_device_rcu(c, ca, &c->rw_devs[BCH_DATA_journal]) { struct journal_device *ja = &ca->journal; @@ -1488,14 +1478,16 @@ void __bch2_journal_debug_to_text(struct printbuf *out, struct journal *j) if (!ja->nr) continue; - prt_printf(out, "dev %u:\n", ca->dev_idx); - prt_printf(out, "\tnr\t\t%u\n", ja->nr); - prt_printf(out, "\tbucket size\t%u\n", ca->mi.bucket_size); - prt_printf(out, "\tavailable\t%u:%u\n", bch2_journal_dev_buckets_available(j, ja, journal_space_discarded), ja->sectors_free); - prt_printf(out, "\tdiscard_idx\t%u\n", ja->discard_idx); - prt_printf(out, "\tdirty_ondisk\t%u (seq %llu)\n", ja->dirty_idx_ondisk, ja->bucket_seq[ja->dirty_idx_ondisk]); - prt_printf(out, "\tdirty_idx\t%u (seq %llu)\n", ja->dirty_idx, ja->bucket_seq[ja->dirty_idx]); - prt_printf(out, "\tcur_idx\t\t%u (seq %llu)\n", ja->cur_idx, ja->bucket_seq[ja->cur_idx]); + prt_printf(out, "dev %u:\n", ca->dev_idx); + printbuf_indent_add(out, 2); + prt_printf(out, "nr\t%u\n", ja->nr); + prt_printf(out, "bucket size\t%u\n", ca->mi.bucket_size); + prt_printf(out, "available\t%u:%u\n", bch2_journal_dev_buckets_available(j, ja, journal_space_discarded), ja->sectors_free); + prt_printf(out, "discard_idx\t%u\n", ja->discard_idx); + prt_printf(out, "dirty_ondisk\t%u (seq %llu)\n",ja->dirty_idx_ondisk, ja->bucket_seq[ja->dirty_idx_ondisk]); + prt_printf(out, "dirty_idx\t%u (seq %llu)\n", ja->dirty_idx, ja->bucket_seq[ja->dirty_idx]); + prt_printf(out, "cur_idx\t%u (seq %llu)\n", ja->cur_idx, ja->bucket_seq[ja->cur_idx]); + printbuf_indent_sub(out, 2); } rcu_read_unlock(); @@ -1527,25 +1519,18 @@ bool bch2_journal_seq_pins_to_text(struct printbuf *out, struct journal *j, u64 pin_list = journal_seq_pin(j, *seq); - prt_printf(out, "%llu: count %u", *seq, atomic_read(&pin_list->count)); - prt_newline(out); + prt_printf(out, "%llu: count %u\n", *seq, atomic_read(&pin_list->count)); printbuf_indent_add(out, 2); for (unsigned i = 0; i < ARRAY_SIZE(pin_list->list); i++) - list_for_each_entry(pin, &pin_list->list[i], list) { - prt_printf(out, "\t%px %ps", pin, pin->flush); - prt_newline(out); - } + list_for_each_entry(pin, &pin_list->list[i], list) + prt_printf(out, "\t%px %ps\n", pin, pin->flush); - if (!list_empty(&pin_list->flushed)) { - prt_printf(out, "flushed:"); - prt_newline(out); - } + if (!list_empty(&pin_list->flushed)) + prt_printf(out, "flushed:\n"); - list_for_each_entry(pin, &pin_list->flushed, list) { - prt_printf(out, "\t%px %ps", pin, pin->flush); - prt_newline(out); - } + list_for_each_entry(pin, &pin_list->flushed, list) + prt_printf(out, "\t%px %ps\n", pin, pin->flush); printbuf_indent_sub(out, 2); diff --git a/fs/bcachefs/move.c b/fs/bcachefs/move.c index 45984a688e5b6..e491203f21c1c 100644 --- a/fs/bcachefs/move.c +++ b/fs/bcachefs/move.c @@ -41,28 +41,23 @@ static void bch2_data_update_opts_to_text(struct printbuf *out, struct bch_fs *c struct data_update_opts *data_opts) { printbuf_tabstop_push(out, 20); - prt_str(out, "rewrite ptrs:"); - prt_tab(out); + prt_str(out, "rewrite ptrs:\t"); bch2_prt_u64_base2(out, data_opts->rewrite_ptrs); prt_newline(out); - prt_str(out, "kill ptrs: "); - prt_tab(out); + prt_str(out, "kill ptrs:\t"); bch2_prt_u64_base2(out, data_opts->kill_ptrs); prt_newline(out); - prt_str(out, "target: "); - prt_tab(out); + prt_str(out, "target:\t"); bch2_target_to_text(out, c, data_opts->target); prt_newline(out); - prt_str(out, "compression: "); - prt_tab(out); + prt_str(out, "compression:\t"); bch2_compression_opt_to_text(out, background_compression(*io_opts)); prt_newline(out); - prt_str(out, "extra replicas: "); - prt_tab(out); + prt_str(out, "extra replicas:\t"); prt_u64(out, data_opts->extra_replicas); } @@ -1127,23 +1122,17 @@ void bch2_move_stats_to_text(struct printbuf *out, struct bch_move_stats *stats) prt_newline(out); printbuf_indent_add(out, 2); - prt_str(out, "keys moved: "); - prt_u64(out, atomic64_read(&stats->keys_moved)); - prt_newline(out); - - prt_str(out, "keys raced: "); - prt_u64(out, atomic64_read(&stats->keys_raced)); - prt_newline(out); - - prt_str(out, "bytes seen: "); + prt_printf(out, "keys moved: %llu\n", atomic64_read(&stats->keys_moved)); + prt_printf(out, "keys raced: %llu\n", atomic64_read(&stats->keys_raced)); + prt_printf(out, "bytes seen: "); prt_human_readable_u64(out, atomic64_read(&stats->sectors_seen) << 9); prt_newline(out); - prt_str(out, "bytes moved: "); + prt_printf(out, "bytes moved: "); prt_human_readable_u64(out, atomic64_read(&stats->sectors_moved) << 9); prt_newline(out); - prt_str(out, "bytes raced: "); + prt_printf(out, "bytes raced: "); prt_human_readable_u64(out, atomic64_read(&stats->sectors_raced) << 9); prt_newline(out); @@ -1157,19 +1146,17 @@ static void bch2_moving_ctxt_to_text(struct printbuf *out, struct bch_fs *c, str bch2_move_stats_to_text(out, ctxt->stats); printbuf_indent_add(out, 2); - prt_printf(out, "reads: ios %u/%u sectors %u/%u", + prt_printf(out, "reads: ios %u/%u sectors %u/%u\n", atomic_read(&ctxt->read_ios), c->opts.move_ios_in_flight, atomic_read(&ctxt->read_sectors), c->opts.move_bytes_in_flight >> 9); - prt_newline(out); - prt_printf(out, "writes: ios %u/%u sectors %u/%u", + prt_printf(out, "writes: ios %u/%u sectors %u/%u\n", atomic_read(&ctxt->write_ios), c->opts.move_ios_in_flight, atomic_read(&ctxt->write_sectors), c->opts.move_bytes_in_flight >> 9); - prt_newline(out); printbuf_indent_add(out, 2); diff --git a/fs/bcachefs/quota.c b/fs/bcachefs/quota.c index 556da07381069..5660798c64533 100644 --- a/fs/bcachefs/quota.c +++ b/fs/bcachefs/quota.c @@ -97,45 +97,14 @@ static void qc_info_to_text(struct printbuf *out, struct qc_info *i) printbuf_tabstops_reset(out); printbuf_tabstop_push(out, 20); - prt_str(out, "i_fieldmask"); - prt_tab(out); - prt_printf(out, "%x", i->i_fieldmask); - prt_newline(out); - - prt_str(out, "i_flags"); - prt_tab(out); - prt_printf(out, "%u", i->i_flags); - prt_newline(out); - - prt_str(out, "i_spc_timelimit"); - prt_tab(out); - prt_printf(out, "%u", i->i_spc_timelimit); - prt_newline(out); - - prt_str(out, "i_ino_timelimit"); - prt_tab(out); - prt_printf(out, "%u", i->i_ino_timelimit); - prt_newline(out); - - prt_str(out, "i_rt_spc_timelimit"); - prt_tab(out); - prt_printf(out, "%u", i->i_rt_spc_timelimit); - prt_newline(out); - - prt_str(out, "i_spc_warnlimit"); - prt_tab(out); - prt_printf(out, "%u", i->i_spc_warnlimit); - prt_newline(out); - - prt_str(out, "i_ino_warnlimit"); - prt_tab(out); - prt_printf(out, "%u", i->i_ino_warnlimit); - prt_newline(out); - - prt_str(out, "i_rt_spc_warnlimit"); - prt_tab(out); - prt_printf(out, "%u", i->i_rt_spc_warnlimit); - prt_newline(out); + prt_printf(out, "i_fieldmask\t%x\n", i->i_fieldmask); + prt_printf(out, "i_flags\t%u\n", i->i_flags); + prt_printf(out, "i_spc_timelimit\t%u\n", i->i_spc_timelimit); + prt_printf(out, "i_ino_timelimit\t%u\n", i->i_ino_timelimit); + prt_printf(out, "i_rt_spc_timelimit\t%u\n", i->i_rt_spc_timelimit); + prt_printf(out, "i_spc_warnlimit\t%u\n", i->i_spc_warnlimit); + prt_printf(out, "i_ino_warnlimit\t%u\n", i->i_ino_warnlimit); + prt_printf(out, "i_rt_spc_warnlimit\t%u\n", i->i_rt_spc_warnlimit); } static void qc_dqblk_to_text(struct printbuf *out, struct qc_dqblk *q) @@ -143,60 +112,17 @@ static void qc_dqblk_to_text(struct printbuf *out, struct qc_dqblk *q) printbuf_tabstops_reset(out); printbuf_tabstop_push(out, 20); - prt_str(out, "d_fieldmask"); - prt_tab(out); - prt_printf(out, "%x", q->d_fieldmask); - prt_newline(out); - - prt_str(out, "d_spc_hardlimit"); - prt_tab(out); - prt_printf(out, "%llu", q->d_spc_hardlimit); - prt_newline(out); - - prt_str(out, "d_spc_softlimit"); - prt_tab(out); - prt_printf(out, "%llu", q->d_spc_softlimit); - prt_newline(out); - - prt_str(out, "d_ino_hardlimit"); - prt_tab(out); - prt_printf(out, "%llu", q->d_ino_hardlimit); - prt_newline(out); - - prt_str(out, "d_ino_softlimit"); - prt_tab(out); - prt_printf(out, "%llu", q->d_ino_softlimit); - prt_newline(out); - - prt_str(out, "d_space"); - prt_tab(out); - prt_printf(out, "%llu", q->d_space); - prt_newline(out); - - prt_str(out, "d_ino_count"); - prt_tab(out); - prt_printf(out, "%llu", q->d_ino_count); - prt_newline(out); - - prt_str(out, "d_ino_timer"); - prt_tab(out); - prt_printf(out, "%llu", q->d_ino_timer); - prt_newline(out); - - prt_str(out, "d_spc_timer"); - prt_tab(out); - prt_printf(out, "%llu", q->d_spc_timer); - prt_newline(out); - - prt_str(out, "d_ino_warns"); - prt_tab(out); - prt_printf(out, "%i", q->d_ino_warns); - prt_newline(out); - - prt_str(out, "d_spc_warns"); - prt_tab(out); - prt_printf(out, "%i", q->d_spc_warns); - prt_newline(out); + prt_printf(out, "d_fieldmask\t%x\n", q->d_fieldmask); + prt_printf(out, "d_spc_hardlimit\t%llu\n", q->d_spc_hardlimit); + prt_printf(out, "d_spc_softlimit\t%llu\n", q->d_spc_softlimit); + prt_printf(out, "d_ino_hardlimit\%llu\n", q->d_ino_hardlimit); + prt_printf(out, "d_ino_softlimit\t%llu\n", q->d_ino_softlimit); + prt_printf(out, "d_space\t%llu\n", q->d_space); + prt_printf(out, "d_ino_count\t%llu\n", q->d_ino_count); + prt_printf(out, "d_ino_timer\t%llu\n", q->d_ino_timer); + prt_printf(out, "d_spc_timer\t%llu\n", q->d_spc_timer); + prt_printf(out, "d_ino_warns\t%i\n", q->d_ino_warns); + prt_printf(out, "d_spc_warns\t%i\n", q->d_spc_warns); } static inline unsigned __next_qtype(unsigned i, unsigned qtypes) diff --git a/fs/bcachefs/sb-clean.c b/fs/bcachefs/sb-clean.c index d27804d4987a0..c2d4be52c8c52 100644 --- a/fs/bcachefs/sb-clean.c +++ b/fs/bcachefs/sb-clean.c @@ -298,10 +298,8 @@ static void bch2_sb_clean_to_text(struct printbuf *out, struct bch_sb *sb, struct bch_sb_field_clean *clean = field_to_type(f, clean); struct jset_entry *entry; - prt_printf(out, "flags: %x", le32_to_cpu(clean->flags)); - prt_newline(out); - prt_printf(out, "journal_seq: %llu", le64_to_cpu(clean->journal_seq)); - prt_newline(out); + prt_printf(out, "flags: %x\n", le32_to_cpu(clean->flags)); + prt_printf(out, "journal_seq: %llu\n", le64_to_cpu(clean->journal_seq)); for (entry = clean->start; entry != vstruct_end(&clean->field); diff --git a/fs/bcachefs/sb-counters.c b/fs/bcachefs/sb-counters.c index 7dc898761bb31..7282f78861926 100644 --- a/fs/bcachefs/sb-counters.c +++ b/fs/bcachefs/sb-counters.c @@ -31,19 +31,12 @@ static void bch2_sb_counters_to_text(struct printbuf *out, struct bch_sb *sb, struct bch_sb_field *f) { struct bch_sb_field_counters *ctrs = field_to_type(f, counters); - unsigned int i; unsigned int nr = bch2_sb_counter_nr_entries(ctrs); - for (i = 0; i < nr; i++) { - if (i < BCH_COUNTER_NR) - prt_printf(out, "%s ", bch2_counter_names[i]); - else - prt_printf(out, "(unknown)"); - - prt_tab(out); - prt_printf(out, "%llu", le64_to_cpu(ctrs->d[i])); - prt_newline(out); - } + for (unsigned i = 0; i < nr; i++) + prt_printf(out, "%s \t%llu\n", + i < BCH_COUNTER_NR ? bch2_counter_names[i] : "(unknown)", + le64_to_cpu(ctrs->d[i])); }; int bch2_sb_counters_to_cpu(struct bch_fs *c) diff --git a/fs/bcachefs/sb-downgrade.c b/fs/bcachefs/sb-downgrade.c index a98ef940b7a32..90b06f8aa1df0 100644 --- a/fs/bcachefs/sb-downgrade.c +++ b/fs/bcachefs/sb-downgrade.c @@ -164,19 +164,16 @@ static void bch2_sb_downgrade_to_text(struct printbuf *out, struct bch_sb *sb, printbuf_tabstop_push(out, 16); for_each_downgrade_entry(e, i) { - prt_str(out, "version:"); - prt_tab(out); + prt_str(out, "version:\t"); bch2_version_to_text(out, le16_to_cpu(i->version)); prt_newline(out); - prt_str(out, "recovery passes:"); - prt_tab(out); + prt_str(out, "recovery passes:\t"); prt_bitflags(out, bch2_recovery_passes, bch2_recovery_passes_from_stable(le64_to_cpu(i->recovery_passes[0]))); prt_newline(out); - prt_str(out, "errors:"); - prt_tab(out); + prt_str(out, "errors:\t"); bool first = true; for (unsigned j = 0; j < le16_to_cpu(i->nr_errors); j++) { if (!first) diff --git a/fs/bcachefs/sb-members.c b/fs/bcachefs/sb-members.c index 44b3f0cb7b497..0007fcad5baa8 100644 --- a/fs/bcachefs/sb-members.c +++ b/fs/bcachefs/sb-members.c @@ -167,15 +167,11 @@ static void member_to_text(struct printbuf *out, if (!bch2_member_exists(&m)) return; - prt_printf(out, "Device:"); - prt_tab(out); - prt_printf(out, "%u", i); - prt_newline(out); + prt_printf(out, "Device:\t%u\n", i); printbuf_indent_add(out, 2); - prt_printf(out, "Label:"); - prt_tab(out); + prt_printf(out, "Label:\t"); if (BCH_MEMBER_GROUP(&m)) { unsigned idx = BCH_MEMBER_GROUP(&m) - 1; @@ -189,96 +185,59 @@ static void member_to_text(struct printbuf *out, } prt_newline(out); - prt_printf(out, "UUID:"); - prt_tab(out); + prt_printf(out, "UUID:\t"); pr_uuid(out, m.uuid.b); prt_newline(out); - prt_printf(out, "Size:"); - prt_tab(out); + prt_printf(out, "Size:\t"); prt_units_u64(out, device_size << 9); prt_newline(out); - for (unsigned i = 0; i < BCH_MEMBER_ERROR_NR; i++) { - prt_printf(out, "%s errors:", bch2_member_error_strs[i]); - prt_tab(out); - prt_u64(out, le64_to_cpu(m.errors[i])); - prt_newline(out); - } + for (unsigned i = 0; i < BCH_MEMBER_ERROR_NR; i++) + prt_printf(out, "%s errors:\t%llu\n", bch2_member_error_strs[i], le64_to_cpu(m.errors[i])); - for (unsigned i = 0; i < BCH_IOPS_NR; i++) { - prt_printf(out, "%s iops:", bch2_iops_measurements[i]); - prt_tab(out); - prt_printf(out, "%u", le32_to_cpu(m.iops[i])); - prt_newline(out); - } + for (unsigned i = 0; i < BCH_IOPS_NR; i++) + prt_printf(out, "%s iops:\t%u\n", bch2_iops_measurements[i], le32_to_cpu(m.iops[i])); - prt_printf(out, "Bucket size:"); - prt_tab(out); + prt_printf(out, "Bucket size:\t"); prt_units_u64(out, bucket_size << 9); prt_newline(out); - prt_printf(out, "First bucket:"); - prt_tab(out); - prt_printf(out, "%u", le16_to_cpu(m.first_bucket)); - prt_newline(out); + prt_printf(out, "First bucket:\t%u\n", le16_to_cpu(m.first_bucket)); + prt_printf(out, "Buckets:\t%llu\n", le64_to_cpu(m.nbuckets)); - prt_printf(out, "Buckets:"); - prt_tab(out); - prt_printf(out, "%llu", le64_to_cpu(m.nbuckets)); - prt_newline(out); - - prt_printf(out, "Last mount:"); - prt_tab(out); + prt_printf(out, "Last mount:\t"); if (m.last_mount) bch2_prt_datetime(out, le64_to_cpu(m.last_mount)); else prt_printf(out, "(never)"); prt_newline(out); - prt_printf(out, "Last superblock write:"); - prt_tab(out); - prt_u64(out, le64_to_cpu(m.seq)); - prt_newline(out); + prt_printf(out, "Last superblock write:\t%llu\n", le64_to_cpu(m.seq)); - prt_printf(out, "State:"); - prt_tab(out); - prt_printf(out, "%s", + prt_printf(out, "State:\t%s\n", BCH_MEMBER_STATE(&m) < BCH_MEMBER_STATE_NR ? bch2_member_states[BCH_MEMBER_STATE(&m)] : "unknown"); - prt_newline(out); - prt_printf(out, "Data allowed:"); - prt_tab(out); + prt_printf(out, "Data allowed:\t"); if (BCH_MEMBER_DATA_ALLOWED(&m)) prt_bitflags(out, __bch2_data_types, BCH_MEMBER_DATA_ALLOWED(&m)); else prt_printf(out, "(none)"); prt_newline(out); - prt_printf(out, "Has data:"); - prt_tab(out); + prt_printf(out, "Has data:\t"); if (data_have) prt_bitflags(out, __bch2_data_types, data_have); else prt_printf(out, "(none)"); prt_newline(out); - prt_str(out, "Durability:"); - prt_tab(out); - prt_printf(out, "%llu", BCH_MEMBER_DURABILITY(&m) ? BCH_MEMBER_DURABILITY(&m) - 1 : 1); - prt_newline(out); + prt_printf(out, "Durability:\t%llu\n", BCH_MEMBER_DURABILITY(&m) ? BCH_MEMBER_DURABILITY(&m) - 1 : 1); - prt_printf(out, "Discard:"); - prt_tab(out); - prt_printf(out, "%llu", BCH_MEMBER_DISCARD(&m)); - prt_newline(out); - - prt_printf(out, "Freespace initialized:"); - prt_tab(out); - prt_printf(out, "%llu", BCH_MEMBER_FREESPACE_INITIALIZED(&m)); - prt_newline(out); + prt_printf(out, "Discard:\t%llu\n", BCH_MEMBER_DISCARD(&m)); + prt_printf(out, "Freespace initialized:\t%llu\n", BCH_MEMBER_FREESPACE_INITIALIZED(&m)); printbuf_indent_sub(out, 2); } @@ -390,12 +349,8 @@ void bch2_dev_io_errors_to_text(struct printbuf *out, struct bch_dev *ca) prt_newline(out); printbuf_indent_add(out, 2); - for (unsigned i = 0; i < BCH_MEMBER_ERROR_NR; i++) { - prt_printf(out, "%s:", bch2_member_error_strs[i]); - prt_tab(out); - prt_u64(out, atomic64_read(&ca->errors[i])); - prt_newline(out); - } + for (unsigned i = 0; i < BCH_MEMBER_ERROR_NR; i++) + prt_printf(out, "%s:\t%llu\n", bch2_member_error_strs[i], atomic64_read(&ca->errors[i])); printbuf_indent_sub(out, 2); prt_str(out, "IO errors since "); @@ -404,12 +359,9 @@ void bch2_dev_io_errors_to_text(struct printbuf *out, struct bch_dev *ca) prt_newline(out); printbuf_indent_add(out, 2); - for (unsigned i = 0; i < BCH_MEMBER_ERROR_NR; i++) { - prt_printf(out, "%s:", bch2_member_error_strs[i]); - prt_tab(out); - prt_u64(out, atomic64_read(&ca->errors[i]) - le64_to_cpu(m.errors_at_reset[i])); - prt_newline(out); - } + for (unsigned i = 0; i < BCH_MEMBER_ERROR_NR; i++) + prt_printf(out, "%s:\t%llu\n", bch2_member_error_strs[i], + atomic64_read(&ca->errors[i]) - le64_to_cpu(m.errors_at_reset[i])); printbuf_indent_sub(out, 2); } diff --git a/fs/bcachefs/super-io.c b/fs/bcachefs/super-io.c index 1cadc99898f56..52f0ffd4a0d27 100644 --- a/fs/bcachefs/super-io.c +++ b/fs/bcachefs/super-io.c @@ -1176,8 +1176,7 @@ static void bch2_sb_ext_to_text(struct printbuf *out, struct bch_sb *sb, { struct bch_sb_field_ext *e = field_to_type(f, ext); - prt_printf(out, "Recovery passes required:"); - prt_tab(out); + prt_printf(out, "Recovery passes required:\t"); prt_bitflags(out, bch2_recovery_passes, bch2_recovery_passes_from_stable(le64_to_cpu(e->recovery_passes_required[0]))); prt_newline(out); @@ -1186,16 +1185,14 @@ static void bch2_sb_ext_to_text(struct printbuf *out, struct bch_sb *sb, if (errors_silent) { le_bitvector_to_cpu(errors_silent, (void *) e->errors_silent, sizeof(e->errors_silent) * 8); - prt_printf(out, "Errors to silently fix:"); - prt_tab(out); + prt_printf(out, "Errors to silently fix:\t"); prt_bitflags_vector(out, bch2_sb_error_strs, errors_silent, sizeof(e->errors_silent) * 8); prt_newline(out); kfree(errors_silent); } - prt_printf(out, "Btrees with missing data:"); - prt_tab(out); + prt_printf(out, "Btrees with missing data:\t"); prt_bitflags(out, __bch2_btree_ids, le64_to_cpu(e->btrees_lost_data)); prt_newline(out); } @@ -1305,95 +1302,71 @@ void bch2_sb_to_text(struct printbuf *out, struct bch_sb *sb, for (int i = 0; i < sb->nr_devices; i++) nr_devices += bch2_dev_exists(sb, i); - prt_printf(out, "External UUID:"); - prt_tab(out); + prt_printf(out, "External UUID:\t"); pr_uuid(out, sb->user_uuid.b); prt_newline(out); - prt_printf(out, "Internal UUID:"); - prt_tab(out); + prt_printf(out, "Internal UUID:\t"); pr_uuid(out, sb->uuid.b); prt_newline(out); - prt_printf(out, "Magic number:"); - prt_tab(out); + prt_printf(out, "Magic number:\t"); pr_uuid(out, sb->magic.b); prt_newline(out); - prt_str(out, "Device index:"); - prt_tab(out); - prt_printf(out, "%u", sb->dev_idx); - prt_newline(out); + prt_printf(out, "Device index:\t%u\n", sb->dev_idx); - prt_str(out, "Label:"); - prt_tab(out); + prt_str(out, "Label:\t"); prt_printf(out, "%.*s", (int) sizeof(sb->label), sb->label); prt_newline(out); - prt_str(out, "Version:"); - prt_tab(out); + prt_str(out, "Version:\t"); bch2_version_to_text(out, le16_to_cpu(sb->version)); prt_newline(out); - prt_str(out, "Version upgrade complete:"); - prt_tab(out); + prt_str(out, "Version upgrade complete:\t"); bch2_version_to_text(out, BCH_SB_VERSION_UPGRADE_COMPLETE(sb)); prt_newline(out); - prt_printf(out, "Oldest version on disk:"); - prt_tab(out); + prt_printf(out, "Oldest version on disk:\t"); bch2_version_to_text(out, le16_to_cpu(sb->version_min)); prt_newline(out); - prt_printf(out, "Created:"); - prt_tab(out); + prt_printf(out, "Created:\t"); if (sb->time_base_lo) bch2_prt_datetime(out, div_u64(le64_to_cpu(sb->time_base_lo), NSEC_PER_SEC)); else prt_printf(out, "(not set)"); prt_newline(out); - prt_printf(out, "Sequence number:"); - prt_tab(out); + prt_printf(out, "Sequence number:\t"); prt_printf(out, "%llu", le64_to_cpu(sb->seq)); prt_newline(out); - prt_printf(out, "Time of last write:"); - prt_tab(out); + prt_printf(out, "Time of last write:\t"); bch2_prt_datetime(out, le64_to_cpu(sb->write_time)); prt_newline(out); - prt_printf(out, "Superblock size:"); - prt_tab(out); + prt_printf(out, "Superblock size:\t"); prt_units_u64(out, vstruct_bytes(sb)); prt_str(out, "/"); prt_units_u64(out, 512ULL << sb->layout.sb_max_size_bits); prt_newline(out); - prt_printf(out, "Clean:"); - prt_tab(out); - prt_printf(out, "%llu", BCH_SB_CLEAN(sb)); - prt_newline(out); - - prt_printf(out, "Devices:"); - prt_tab(out); - prt_printf(out, "%u", nr_devices); - prt_newline(out); + prt_printf(out, "Clean:\t%llu\n", BCH_SB_CLEAN(sb)); + prt_printf(out, "Devices:\t%u\n", nr_devices); - prt_printf(out, "Sections:"); + prt_printf(out, "Sections:\t"); vstruct_for_each(sb, f) fields_have |= 1 << le32_to_cpu(f->type); - prt_tab(out); prt_bitflags(out, bch2_sb_fields, fields_have); prt_newline(out); - prt_printf(out, "Features:"); - prt_tab(out); + prt_printf(out, "Features:\t"); prt_bitflags(out, bch2_sb_features, le64_to_cpu(sb->features[0])); prt_newline(out); - prt_printf(out, "Compat features:"); - prt_tab(out); + prt_printf(out, "Compat features:\t"); prt_bitflags(out, bch2_sb_compat, le64_to_cpu(sb->compat[0])); prt_newline(out); @@ -1410,8 +1383,7 @@ void bch2_sb_to_text(struct printbuf *out, struct bch_sb *sb, if (opt->get_sb != BCH2_NO_SB_OPT) { u64 v = bch2_opt_from_sb(sb, id); - prt_printf(out, "%s:", opt->attr.name); - prt_tab(out); + prt_printf(out, "%s:\t", opt->attr.name); bch2_opt_to_text(out, NULL, sb, opt, v, OPT_HUMAN_READABLE|OPT_SHOW_FULL_LIST); prt_newline(out); diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index 5be92fe3f4ea4..b6bfd7cbce184 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -189,12 +189,8 @@ static void bch2_write_refs_to_text(struct printbuf *out, struct bch_fs *c) { bch2_printbuf_tabstop_push(out, 24); - for (unsigned i = 0; i < ARRAY_SIZE(c->writes); i++) { - prt_str(out, bch2_write_refs[i]); - prt_tab(out); - prt_printf(out, "%li", atomic_long_read(&c->writes[i])); - prt_newline(out); - } + for (unsigned i = 0; i < ARRAY_SIZE(c->writes); i++) + prt_printf(out, "%s\t%li\n", bch2_write_refs[i], atomic_long_read(&c->writes[i])); } #endif @@ -313,22 +309,11 @@ static int bch2_compression_stats_to_text(struct printbuf *out, struct bch_fs *c if (ret) return ret; - prt_str(out, "type"); printbuf_tabstop_push(out, 12); - prt_tab(out); - - prt_str(out, "compressed"); printbuf_tabstop_push(out, 16); - prt_tab_rjust(out); - - prt_str(out, "uncompressed"); printbuf_tabstop_push(out, 16); - prt_tab_rjust(out); - - prt_str(out, "average extent size"); printbuf_tabstop_push(out, 24); - prt_tab_rjust(out); - prt_newline(out); + prt_printf(out, "type\tcompressed\runcompressed\raverage extent size\r\n"); for (unsigned i = 0; i < ARRAY_SIZE(s); i++) { bch2_prt_compression_type(out, i); @@ -594,13 +579,11 @@ SHOW(bch2_fs_counters) if (attr == &sysfs_##t) { \ counter = percpu_u64_get(&c->counters[BCH_COUNTER_##t]);\ counter_since_mount = counter - c->counters_on_mount[BCH_COUNTER_##t];\ - prt_printf(out, "since mount:"); \ - prt_tab(out); \ + prt_printf(out, "since mount:\t"); \ prt_human_readable_u64(out, counter_since_mount); \ prt_newline(out); \ \ - prt_printf(out, "since filesystem creation:"); \ - prt_tab(out); \ + prt_printf(out, "since filesystem creation:\t"); \ prt_human_readable_u64(out, counter); \ prt_newline(out); \ } @@ -796,11 +779,11 @@ static void dev_alloc_debug_to_text(struct printbuf *out, struct bch_dev *ca) { struct bch_fs *c = ca->fs; struct bch_dev_usage stats = bch2_dev_usage_read(ca); - unsigned i, nr[BCH_DATA_NR]; + unsigned nr[BCH_DATA_NR]; memset(nr, 0, sizeof(nr)); - for (i = 0; i < ARRAY_SIZE(c->open_buckets); i++) + for (unsigned i = 0; i < ARRAY_SIZE(c->open_buckets); i++) nr[c->open_buckets[i].data_type]++; printbuf_tabstop_push(out, 8); @@ -813,65 +796,24 @@ static void dev_alloc_debug_to_text(struct printbuf *out, struct bch_dev *ca) prt_newline(out); - prt_printf(out, "reserves:"); - prt_newline(out); - for (i = 0; i < BCH_WATERMARK_NR; i++) { - prt_str(out, bch2_watermarks[i]); - prt_tab(out); - prt_u64(out, bch2_dev_buckets_reserved(ca, i)); - prt_tab_rjust(out); - prt_newline(out); - } + prt_printf(out, "reserves:\n"); + for (unsigned i = 0; i < BCH_WATERMARK_NR; i++) + prt_printf(out, "%s\t%llu\r\n", bch2_watermarks[i], bch2_dev_buckets_reserved(ca, i)); prt_newline(out); printbuf_tabstops_reset(out); printbuf_tabstop_push(out, 24); - prt_str(out, "freelist_wait"); - prt_tab(out); - prt_str(out, c->freelist_wait.list.first ? "waiting" : "empty"); - prt_newline(out); - - prt_str(out, "open buckets allocated"); - prt_tab(out); - prt_u64(out, OPEN_BUCKETS_COUNT - c->open_buckets_nr_free); - prt_newline(out); - - prt_str(out, "open buckets this dev"); - prt_tab(out); - prt_u64(out, ca->nr_open_buckets); - prt_newline(out); - - prt_str(out, "open buckets total"); - prt_tab(out); - prt_u64(out, OPEN_BUCKETS_COUNT); - prt_newline(out); - - prt_str(out, "open_buckets_wait"); - prt_tab(out); - prt_str(out, c->open_buckets_wait.list.first ? "waiting" : "empty"); - prt_newline(out); - - prt_str(out, "open_buckets_btree"); - prt_tab(out); - prt_u64(out, nr[BCH_DATA_btree]); - prt_newline(out); - - prt_str(out, "open_buckets_user"); - prt_tab(out); - prt_u64(out, nr[BCH_DATA_user]); - prt_newline(out); - - prt_str(out, "buckets_to_invalidate"); - prt_tab(out); - prt_u64(out, should_invalidate_buckets(ca, stats)); - prt_newline(out); - - prt_str(out, "btree reserve cache"); - prt_tab(out); - prt_u64(out, c->btree_reserve_cache_nr); - prt_newline(out); + prt_printf(out, "freelist_wait\t%s\n", c->freelist_wait.list.first ? "waiting" : "empty"); + prt_printf(out, "open buckets allocated\t%i\n", OPEN_BUCKETS_COUNT - c->open_buckets_nr_free); + prt_printf(out, "open buckets this dev\t%i\n", ca->nr_open_buckets); + prt_printf(out, "open buckets total\t%u\n", OPEN_BUCKETS_COUNT); + prt_printf(out, "open_buckets_wait\t%s\n", c->open_buckets_wait.list.first ? "waiting" : "empty"); + prt_printf(out, "open_buckets_btree\t%u\n", nr[BCH_DATA_btree]); + prt_printf(out, "open_buckets_user\t%u\n", nr[BCH_DATA_user]); + prt_printf(out, "buckets_to_invalidate\t%llu\n", should_invalidate_buckets(ca, stats)); + prt_printf(out, "btree reserve cache\t%u\n", c->btree_reserve_cache_nr); } static const char * const bch2_rw[] = { diff --git a/fs/bcachefs/util.c b/fs/bcachefs/util.c index 92c6ad75e702a..de331dec2a99c 100644 --- a/fs/bcachefs/util.c +++ b/fs/bcachefs/util.c @@ -348,15 +348,12 @@ static void bch2_pr_time_units_aligned(struct printbuf *out, u64 ns) { const struct time_unit *u = bch2_pick_time_units(ns); - prt_printf(out, "%llu ", div64_u64(ns, u->nsecs)); - prt_tab_rjust(out); - prt_printf(out, "%s", u->name); + prt_printf(out, "%llu \r%s", div64_u64(ns, u->nsecs), u->name); } static inline void pr_name_and_units(struct printbuf *out, const char *name, u64 ns) { - prt_str(out, name); - prt_tab(out); + prt_printf(out, "%s\t", name); bch2_pr_time_units_aligned(out, ns); prt_newline(out); } @@ -389,12 +386,8 @@ void bch2_time_stats_to_text(struct printbuf *out, struct bch2_time_stats *stats } printbuf_tabstop_push(out, out->indent + TABSTOP_SIZE); - prt_printf(out, "count:"); - prt_tab(out); - prt_printf(out, "%llu ", - stats->duration_stats.n); + prt_printf(out, "count:\t%llu\n", stats->duration_stats.n); printbuf_tabstop_pop(out); - prt_newline(out); printbuf_tabstops_reset(out); @@ -403,13 +396,8 @@ void bch2_time_stats_to_text(struct printbuf *out, struct bch2_time_stats *stats printbuf_tabstop_push(out, 0); printbuf_tabstop_push(out, TABSTOP_SIZE + 2); - prt_tab(out); - prt_printf(out, "since mount"); - prt_tab_rjust(out); - prt_tab(out); + prt_printf(out, "\tsince mount\r\trecent\r\n"); prt_printf(out, "recent"); - prt_tab_rjust(out); - prt_newline(out); printbuf_tabstops_reset(out); printbuf_tabstop_push(out, out->indent + 20); @@ -417,23 +405,20 @@ void bch2_time_stats_to_text(struct printbuf *out, struct bch2_time_stats *stats printbuf_tabstop_push(out, 2); printbuf_tabstop_push(out, TABSTOP_SIZE); - prt_printf(out, "duration of events"); - prt_newline(out); + prt_printf(out, "duration of events\n"); printbuf_indent_add(out, 2); pr_name_and_units(out, "min:", stats->min_duration); pr_name_and_units(out, "max:", stats->max_duration); pr_name_and_units(out, "total:", stats->total_duration); - prt_printf(out, "mean:"); - prt_tab(out); + prt_printf(out, "mean:\t"); bch2_pr_time_units_aligned(out, d_mean); prt_tab(out); bch2_pr_time_units_aligned(out, mean_and_variance_weighted_get_mean(stats->duration_stats_weighted, TIME_STATS_MV_WEIGHT)); prt_newline(out); - prt_printf(out, "stddev:"); - prt_tab(out); + prt_printf(out, "stddev:\t"); bch2_pr_time_units_aligned(out, d_stddev); prt_tab(out); bch2_pr_time_units_aligned(out, mean_and_variance_weighted_get_stddev(stats->duration_stats_weighted, TIME_STATS_MV_WEIGHT)); @@ -441,22 +426,19 @@ void bch2_time_stats_to_text(struct printbuf *out, struct bch2_time_stats *stats printbuf_indent_sub(out, 2); prt_newline(out); - prt_printf(out, "time between events"); - prt_newline(out); + prt_printf(out, "time between events\n"); printbuf_indent_add(out, 2); pr_name_and_units(out, "min:", stats->min_freq); pr_name_and_units(out, "max:", stats->max_freq); - prt_printf(out, "mean:"); - prt_tab(out); + prt_printf(out, "mean:\t"); bch2_pr_time_units_aligned(out, f_mean); prt_tab(out); bch2_pr_time_units_aligned(out, mean_and_variance_weighted_get_mean(stats->freq_stats_weighted, TIME_STATS_MV_WEIGHT)); prt_newline(out); - prt_printf(out, "stddev:"); - prt_tab(out); + prt_printf(out, "stddev:\t"); bch2_pr_time_units_aligned(out, f_stddev); prt_tab(out); bch2_pr_time_units_aligned(out, mean_and_variance_weighted_get_stddev(stats->freq_stats_weighted, TIME_STATS_MV_WEIGHT)); @@ -589,40 +571,31 @@ void bch2_pd_controller_debug_to_text(struct printbuf *out, struct bch_pd_contro if (!out->nr_tabstops) printbuf_tabstop_push(out, 20); - prt_printf(out, "rate:"); - prt_tab(out); + prt_printf(out, "rate:\t"); prt_human_readable_s64(out, pd->rate.rate); prt_newline(out); - prt_printf(out, "target:"); - prt_tab(out); + prt_printf(out, "target:\t"); prt_human_readable_u64(out, pd->last_target); prt_newline(out); - prt_printf(out, "actual:"); - prt_tab(out); + prt_printf(out, "actual:\t"); prt_human_readable_u64(out, pd->last_actual); prt_newline(out); - prt_printf(out, "proportional:"); - prt_tab(out); + prt_printf(out, "proportional:\t"); prt_human_readable_s64(out, pd->last_proportional); prt_newline(out); - prt_printf(out, "derivative:"); - prt_tab(out); + prt_printf(out, "derivative:\t"); prt_human_readable_s64(out, pd->last_derivative); prt_newline(out); - prt_printf(out, "change:"); - prt_tab(out); + prt_printf(out, "change:\t"); prt_human_readable_s64(out, pd->last_change); prt_newline(out); - prt_printf(out, "next io:"); - prt_tab(out); - prt_printf(out, "%llims", div64_s64(pd->rate.next - local_clock(), NSEC_PER_MSEC)); - prt_newline(out); + prt_printf(out, "next io:\t%llims\n", div64_s64(pd->rate.next - local_clock(), NSEC_PER_MSEC)); } /* misc: */ From 9089376f709e726e9bd2914284b9b98af65de749 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 10 Apr 2024 14:41:17 -0400 Subject: [PATCH 1359/1702] bcachefs: bch2_btree_node_header_to_text() better btree node read path error messages Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_io.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c index fc046c9ccb450..ed80556b80fe2 100644 --- a/fs/bcachefs/btree_io.c +++ b/fs/bcachefs/btree_io.c @@ -23,6 +23,18 @@ #include +static void bch2_btree_node_header_to_text(struct printbuf *out, struct btree_node *bn) +{ + prt_printf(out, "btree=%s l=%u seq %llux\n", + bch2_btree_id_str(BTREE_NODE_ID(bn)), + (unsigned) BTREE_NODE_LEVEL(bn), bn->keys.seq); + prt_str(out, "min: "); + bch2_bpos_to_text(out, bn->min_key); + prt_newline(out); + prt_str(out, "max: "); + bch2_bpos_to_text(out, bn->max_key); +} + void bch2_btree_node_io_unlock(struct btree *b) { EBUG_ON(!btree_node_write_in_flight(b)); @@ -1021,18 +1033,19 @@ int bch2_btree_node_read_done(struct bch_fs *c, struct bch_dev *ca, -BCH_ERR_btree_node_read_err_must_retry, c, ca, b, NULL, btree_node_bad_seq, - "got wrong btree node (want %llx got %llx)\n" - "got btree %s level %llu pos %s", - bp->seq, b->data->keys.seq, - bch2_btree_id_str(BTREE_NODE_ID(b->data)), - BTREE_NODE_LEVEL(b->data), - buf.buf); + "got wrong btree node: got\n%s", + (printbuf_reset(&buf), + bch2_btree_node_header_to_text(&buf, b->data), + buf.buf)); } else { btree_err_on(!b->data->keys.seq, -BCH_ERR_btree_node_read_err_must_retry, c, ca, b, NULL, btree_node_bad_seq, - "bad btree header: seq 0"); + "bad btree header: seq 0\n%s", + (printbuf_reset(&buf), + bch2_btree_node_header_to_text(&buf, b->data), + buf.buf)); } while (b->written < (ptr_written ?: btree_sectors(c))) { From 6ab71b4a8e281f30a3dfd6f7831d9a2c67e162e4 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 9 Apr 2024 00:04:31 -0400 Subject: [PATCH 1360/1702] bcachefs: bch2_journal_keys_dump() debug helper Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_journal_iter.c | 17 +++++++++++++++++ fs/bcachefs/btree_journal_iter.h | 2 ++ 2 files changed, 19 insertions(+) diff --git a/fs/bcachefs/btree_journal_iter.c b/fs/bcachefs/btree_journal_iter.c index 1e8cf49a69353..332dbf164929e 100644 --- a/fs/bcachefs/btree_journal_iter.c +++ b/fs/bcachefs/btree_journal_iter.c @@ -623,3 +623,20 @@ void bch2_shoot_down_journal_keys(struct bch_fs *c, enum btree_id btree, keys->data[dst++] = *i; keys->nr = keys->gap = dst; } + +void bch2_journal_keys_dump(struct bch_fs *c) +{ + struct journal_keys *keys = &c->journal_keys; + struct printbuf buf = PRINTBUF; + + pr_info("%zu keys:", keys->nr); + + move_gap(keys, keys->nr); + + darray_for_each(*keys, i) { + printbuf_reset(&buf); + bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(i->k)); + pr_err("%s l=%u %s", bch2_btree_id_str(i->btree_id), i->level, buf.buf); + } + printbuf_exit(&buf); +} diff --git a/fs/bcachefs/btree_journal_iter.h b/fs/bcachefs/btree_journal_iter.h index af25046ebcaa7..1ba4a79b0ef90 100644 --- a/fs/bcachefs/btree_journal_iter.h +++ b/fs/bcachefs/btree_journal_iter.h @@ -70,4 +70,6 @@ void bch2_shoot_down_journal_keys(struct bch_fs *, enum btree_id, unsigned, unsigned, struct bpos, struct bpos); +void bch2_journal_keys_dump(struct bch_fs *); + #endif /* _BCACHEFS_BTREE_JOURNAL_ITER_H */ From ac01928b8e9688aca3c23134d4c889c4839c360a Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 7 Apr 2024 16:16:49 -0400 Subject: [PATCH 1361/1702] bcachefs: bch2_hash_lookup() now returns bkey_s_c small cleanup Signed-off-by: Kent Overstreet --- fs/bcachefs/acl.c | 37 +++++++++++-------------------------- fs/bcachefs/dirent.c | 30 +++++++++--------------------- fs/bcachefs/fs.c | 10 +++------- fs/bcachefs/fsck.c | 8 ++++---- fs/bcachefs/str_hash.h | 27 +++++++++++++-------------- fs/bcachefs/xattr.c | 25 +++++++++---------------- 6 files changed, 49 insertions(+), 88 deletions(-) diff --git a/fs/bcachefs/acl.c b/fs/bcachefs/acl.c index 5c180fdc3efbd..b4b112e33be4c 100644 --- a/fs/bcachefs/acl.c +++ b/fs/bcachefs/acl.c @@ -282,18 +282,12 @@ struct posix_acl *bch2_get_acl(struct mnt_idmap *idmap, struct btree_trans *trans = bch2_trans_get(c); struct btree_iter iter = { NULL }; struct posix_acl *acl = NULL; - struct bkey_s_c k; - int ret; retry: bch2_trans_begin(trans); - ret = bch2_hash_lookup(trans, &iter, bch2_xattr_hash_desc, - &hash, inode_inum(inode), &search, 0); - if (ret) - goto err; - - k = bch2_btree_iter_peek_slot(&iter); - ret = bkey_err(k); + struct bkey_s_c k = bch2_hash_lookup(trans, &iter, bch2_xattr_hash_desc, + &hash, inode_inum(inode), &search, 0); + int ret = bkey_err(k); if (ret) goto err; @@ -414,39 +408,30 @@ int bch2_acl_chmod(struct btree_trans *trans, subvol_inum inum, struct bch_hash_info hash_info = bch2_hash_info_init(trans->c, inode); struct xattr_search_key search = X_SEARCH(KEY_TYPE_XATTR_INDEX_POSIX_ACL_ACCESS, "", 0); struct btree_iter iter; - struct bkey_s_c_xattr xattr; - struct bkey_i_xattr *new; struct posix_acl *acl = NULL; - struct bkey_s_c k; - int ret; - ret = bch2_hash_lookup(trans, &iter, bch2_xattr_hash_desc, + struct bkey_s_c k = bch2_hash_lookup(trans, &iter, bch2_xattr_hash_desc, &hash_info, inum, &search, BTREE_ITER_INTENT); + int ret = bkey_err(k); if (ret) return bch2_err_matches(ret, ENOENT) ? 0 : ret; - k = bch2_btree_iter_peek_slot(&iter); - ret = bkey_err(k); - if (ret) - goto err; - xattr = bkey_s_c_to_xattr(k); + struct bkey_s_c_xattr xattr = bkey_s_c_to_xattr(k); acl = bch2_acl_from_disk(trans, xattr_val(xattr.v), le16_to_cpu(xattr.v->x_val_len)); ret = PTR_ERR_OR_ZERO(acl); - if (IS_ERR_OR_NULL(acl)) + if (ret) goto err; - ret = allocate_dropping_locks_errcode(trans, - __posix_acl_chmod(&acl, _gfp, mode)); + ret = allocate_dropping_locks_errcode(trans, __posix_acl_chmod(&acl, _gfp, mode)); if (ret) goto err; - new = bch2_acl_to_xattr(trans, acl, ACL_TYPE_ACCESS); - if (IS_ERR(new)) { - ret = PTR_ERR(new); + struct bkey_i_xattr *new = bch2_acl_to_xattr(trans, acl, ACL_TYPE_ACCESS); + ret = PTR_ERR_OR_ZERO(new); + if (ret) goto err; - } new->k.p = iter.pos; ret = bch2_trans_update(trans, &iter, &new->k_i, 0); diff --git a/fs/bcachefs/dirent.c b/fs/bcachefs/dirent.c index d37bd07afbfe4..d4e3ca5045d61 100644 --- a/fs/bcachefs/dirent.c +++ b/fs/bcachefs/dirent.c @@ -301,13 +301,9 @@ int bch2_dirent_rename(struct btree_trans *trans, memset(dst_inum, 0, sizeof(*dst_inum)); /* Lookup src: */ - ret = bch2_hash_lookup(trans, &src_iter, bch2_dirent_hash_desc, - src_hash, src_dir, src_name, - BTREE_ITER_INTENT); - if (ret) - goto out; - - old_src = bch2_btree_iter_peek_slot(&src_iter); + old_src = bch2_hash_lookup(trans, &src_iter, bch2_dirent_hash_desc, + src_hash, src_dir, src_name, + BTREE_ITER_INTENT); ret = bkey_err(old_src); if (ret) goto out; @@ -329,13 +325,9 @@ int bch2_dirent_rename(struct btree_trans *trans, if (ret) goto out; } else { - ret = bch2_hash_lookup(trans, &dst_iter, bch2_dirent_hash_desc, - dst_hash, dst_dir, dst_name, - BTREE_ITER_INTENT); - if (ret) - goto out; - - old_dst = bch2_btree_iter_peek_slot(&dst_iter); + old_dst = bch2_hash_lookup(trans, &dst_iter, bch2_dirent_hash_desc, + dst_hash, dst_dir, dst_name, + BTREE_ITER_INTENT); ret = bkey_err(old_dst); if (ret) goto out; @@ -479,13 +471,9 @@ int bch2_dirent_lookup_trans(struct btree_trans *trans, const struct qstr *name, subvol_inum *inum, unsigned flags) { - int ret = bch2_hash_lookup(trans, iter, bch2_dirent_hash_desc, - hash_info, dir, name, flags); - if (ret) - return ret; - - struct bkey_s_c k = bch2_btree_iter_peek_slot(iter); - ret = bkey_err(k); + struct bkey_s_c k = bch2_hash_lookup(trans, iter, bch2_dirent_hash_desc, + hash_info, dir, name, flags); + int ret = bkey_err(k); if (ret) goto err; diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c index 6f114803c6f23..3d15275909bd2 100644 --- a/fs/bcachefs/fs.c +++ b/fs/bcachefs/fs.c @@ -377,16 +377,12 @@ static struct bch_inode_info *bch2_lookup_trans(struct btree_trans *trans, struct btree_iter dirent_iter = {}; subvol_inum inum = {}; - int ret = bch2_hash_lookup(trans, &dirent_iter, bch2_dirent_hash_desc, - dir_hash_info, dir, name, 0); + struct bkey_s_c k = bch2_hash_lookup(trans, &dirent_iter, bch2_dirent_hash_desc, + dir_hash_info, dir, name, 0); + int ret = bkey_err(k); if (ret) return ERR_PTR(ret); - struct bkey_s_c k = bch2_btree_iter_peek_slot(&dirent_iter); - ret = bkey_err(k); - if (ret) - goto err; - ret = bch2_dirent_read_target(trans, dir, bkey_s_c_to_dirent(k), &inum); if (ret > 0) ret = -ENOENT; diff --git a/fs/bcachefs/fsck.c b/fs/bcachefs/fsck.c index 8e2010212cc37..caef81aa618c9 100644 --- a/fs/bcachefs/fsck.c +++ b/fs/bcachefs/fsck.c @@ -127,13 +127,13 @@ static int lookup_dirent_in_snapshot(struct btree_trans *trans, u64 *target, unsigned *type, u32 snapshot) { struct btree_iter iter; - struct bkey_s_c_dirent d; - int ret = bch2_hash_lookup_in_snapshot(trans, &iter, bch2_dirent_hash_desc, - &hash_info, dir, name, 0, snapshot); + struct bkey_s_c k = bch2_hash_lookup_in_snapshot(trans, &iter, bch2_dirent_hash_desc, + &hash_info, dir, name, 0, snapshot); + int ret = bkey_err(k); if (ret) return ret; - d = bkey_s_c_to_dirent(bch2_btree_iter_peek_slot(&iter)); + struct bkey_s_c_dirent d = bkey_s_c_to_dirent(bch2_btree_iter_peek_slot(&iter)); *target = le64_to_cpu(d.v->d_inum); *type = d.v->d_type; bch2_trans_iter_exit(trans, &iter); diff --git a/fs/bcachefs/str_hash.h b/fs/bcachefs/str_hash.h index 3976f80721bf1..11f5f5d27d23d 100644 --- a/fs/bcachefs/str_hash.h +++ b/fs/bcachefs/str_hash.h @@ -159,7 +159,7 @@ static inline bool is_visible_key(struct bch_hash_desc desc, subvol_inum inum, s desc.is_visible(inum, k)); } -static __always_inline int +static __always_inline struct bkey_s_c bch2_hash_lookup_in_snapshot(struct btree_trans *trans, struct btree_iter *iter, const struct bch_hash_desc desc, @@ -176,7 +176,7 @@ bch2_hash_lookup_in_snapshot(struct btree_trans *trans, BTREE_ITER_SLOTS|flags, k, ret) { if (is_visible_key(desc, inum, k)) { if (!desc.cmp_key(k, key)) - return 0; + return k; } else if (k.k->type == KEY_TYPE_hash_whiteout) { ; } else { @@ -186,10 +186,10 @@ bch2_hash_lookup_in_snapshot(struct btree_trans *trans, } bch2_trans_iter_exit(trans, iter); - return ret ?: -BCH_ERR_ENOENT_str_hash_lookup; + return bkey_s_c_err(ret ?: -BCH_ERR_ENOENT_str_hash_lookup); } -static __always_inline int +static __always_inline struct bkey_s_c bch2_hash_lookup(struct btree_trans *trans, struct btree_iter *iter, const struct bch_hash_desc desc, @@ -198,8 +198,11 @@ bch2_hash_lookup(struct btree_trans *trans, unsigned flags) { u32 snapshot; - return bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot) ?: - bch2_hash_lookup_in_snapshot(trans, iter, desc, info, inum, key, flags, snapshot); + int ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot); + if (ret) + return bkey_s_c_err(ret); + + return bch2_hash_lookup_in_snapshot(trans, iter, desc, info, inum, key, flags, snapshot); } static __always_inline int @@ -369,14 +372,10 @@ int bch2_hash_delete(struct btree_trans *trans, subvol_inum inum, const void *key) { struct btree_iter iter; - int ret; - - ret = bch2_hash_lookup(trans, &iter, desc, info, inum, key, - BTREE_ITER_INTENT); - if (ret) - return ret; - - ret = bch2_hash_delete_at(trans, desc, info, &iter, 0); + struct bkey_s_c k = bch2_hash_lookup(trans, &iter, desc, info, inum, key, + BTREE_ITER_INTENT); + int ret = bkey_err(k) ?: + bch2_hash_delete_at(trans, desc, info, &iter, 0); bch2_trans_iter_exit(trans, &iter); return ret; } diff --git a/fs/bcachefs/xattr.c b/fs/bcachefs/xattr.c index 4b3a2df2da323..d787f1d4cc780 100644 --- a/fs/bcachefs/xattr.c +++ b/fs/bcachefs/xattr.c @@ -144,21 +144,13 @@ static int bch2_xattr_get_trans(struct btree_trans *trans, struct bch_inode_info struct bch_hash_info hash = bch2_hash_info_init(trans->c, &inode->ei_inode); struct xattr_search_key search = X_SEARCH(type, name, strlen(name)); struct btree_iter iter; - struct bkey_s_c_xattr xattr; - struct bkey_s_c k; - int ret; - - ret = bch2_hash_lookup(trans, &iter, bch2_xattr_hash_desc, &hash, - inode_inum(inode), &search, 0); - if (ret) - goto err1; - - k = bch2_btree_iter_peek_slot(&iter); - ret = bkey_err(k); + struct bkey_s_c k = bch2_hash_lookup(trans, &iter, bch2_xattr_hash_desc, &hash, + inode_inum(inode), &search, 0); + int ret = bkey_err(k); if (ret) - goto err2; + return ret; - xattr = bkey_s_c_to_xattr(k); + struct bkey_s_c_xattr xattr = bkey_s_c_to_xattr(k); ret = le16_to_cpu(xattr.v->x_val_len); if (buffer) { if (ret > size) @@ -166,10 +158,8 @@ static int bch2_xattr_get_trans(struct btree_trans *trans, struct bch_inode_info else memcpy(buffer, xattr_val(xattr.v), ret); } -err2: bch2_trans_iter_exit(trans, &iter); -err1: - return ret < 0 && bch2_err_matches(ret, ENOENT) ? -ENODATA : ret; + return ret; } int bch2_xattr_set(struct btree_trans *trans, subvol_inum inum, @@ -365,6 +355,9 @@ static int bch2_xattr_get_handler(const struct xattr_handler *handler, int ret = bch2_trans_do(c, NULL, NULL, 0, bch2_xattr_get_trans(trans, inode, name, buffer, size, handler->flags)); + if (ret < 0 && bch2_err_matches(ret, ENOENT)) + ret = -ENODATA; + return bch2_err_class(ret); } From 5577881455cdfe0c39781e43215138faf5ed2118 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 5 Apr 2024 16:21:39 -0400 Subject: [PATCH 1362/1702] bcachefs: add btree_node_merging_disabled debug param Signed-off-by: Kent Overstreet --- fs/bcachefs/bcachefs.h | 2 ++ fs/bcachefs/btree_update_interior.h | 3 +++ 2 files changed, 5 insertions(+) diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index 91c3c1fef233d..6366a6a90944d 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -359,6 +359,8 @@ do { \ #define BCH_DEBUG_PARAMS_ALWAYS() \ BCH_DEBUG_PARAM(key_merging_disabled, \ "Disables merging of extents") \ + BCH_DEBUG_PARAM(btree_node_merging_disabled, \ + "Disables merging of btree nodes") \ BCH_DEBUG_PARAM(btree_gc_always_rewrite, \ "Causes mark and sweep to compact and rewrite every " \ "btree node it traverses") \ diff --git a/fs/bcachefs/btree_update_interior.h b/fs/bcachefs/btree_update_interior.h index c1a479ebaad12..ca1a3be43af8b 100644 --- a/fs/bcachefs/btree_update_interior.h +++ b/fs/bcachefs/btree_update_interior.h @@ -144,6 +144,9 @@ static inline int bch2_foreground_maybe_merge_sibling(struct btree_trans *trans, EBUG_ON(!btree_node_locked(path, level)); + if (bch2_btree_node_merging_disabled) + return 0; + b = path->l[level].b; if (b->sib_u64s[sib] > trans->c->btree_foreground_merge_threshold) return 0; From 00589cadb1fb12548e096a29b1b9c0ccb313f2fa Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 5 Apr 2024 21:32:06 -0400 Subject: [PATCH 1363/1702] bcachefs: bch2_btree_path_to_text() Long form version of bch2_btree_path_to_text() - useful in error messages and tracepoints. Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.c | 52 +++++++++++++++++++++++++++++++++---- fs/bcachefs/btree_iter.h | 1 + fs/bcachefs/btree_locking.c | 5 ++++ 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index 080cb50168b05..ca3b6a0ea1f8e 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -1426,23 +1426,63 @@ void bch2_dump_trans_updates(struct btree_trans *trans) printbuf_exit(&buf); } -static void bch2_btree_path_to_text(struct printbuf *out, struct btree_trans *trans, btree_path_idx_t path_idx) +static void bch2_btree_path_to_text_short(struct printbuf *out, struct btree_trans *trans, btree_path_idx_t path_idx) { struct btree_path *path = trans->paths + path_idx; - prt_printf(out, "path: idx %2u ref %u:%u %c %c btree=%s l=%u pos ", + prt_printf(out, "path: idx %2u ref %u:%u %c %c %c btree=%s l=%u pos ", path_idx, path->ref, path->intent_ref, path->preserve ? 'P' : ' ', path->should_be_locked ? 'S' : ' ', + path->cached ? 'C' : 'B', bch2_btree_id_str(path->btree_id), path->level); bch2_bpos_to_text(out, path->pos); - prt_printf(out, " locks %u", path->nodes_locked); #ifdef TRACK_PATH_ALLOCATED prt_printf(out, " %pS", (void *) path->ip_allocated); #endif +} + +static const char *btree_node_locked_str(enum btree_node_locked_type t) +{ + switch (t) { + case BTREE_NODE_UNLOCKED: + return "unlocked"; + case BTREE_NODE_READ_LOCKED: + return "read"; + case BTREE_NODE_INTENT_LOCKED: + return "intent"; + case BTREE_NODE_WRITE_LOCKED: + return "write"; + default: + return NULL; + } +} + +void bch2_btree_path_to_text(struct printbuf *out, struct btree_trans *trans, btree_path_idx_t path_idx) +{ + bch2_btree_path_to_text_short(out, trans, path_idx); + + struct btree_path *path = trans->paths + path_idx; + + prt_printf(out, " uptodate %u locks_want %u", path->uptodate, path->locks_want); prt_newline(out); + + printbuf_indent_add(out, 2); + for (unsigned l = 0; l < BTREE_MAX_DEPTH; l++) { + prt_printf(out, "l=%u locks %s seq %u node ", l, + btree_node_locked_str(btree_node_locked_type(path, l)), + path->l[l].lock_seq); + + int ret = PTR_ERR_OR_ZERO(path->l[l].b); + if (ret) + prt_str(out, bch2_err_str(ret)); + else + prt_printf(out, "%px", path->l[l].b); + prt_newline(out); + } + printbuf_indent_sub(out, 2); } static noinline __cold @@ -1454,8 +1494,10 @@ void __bch2_trans_paths_to_text(struct printbuf *out, struct btree_trans *trans, if (!nosort) btree_trans_sort_paths(trans); - trans_for_each_path_idx_inorder(trans, iter) - bch2_btree_path_to_text(out, trans, iter.path_idx); + trans_for_each_path_idx_inorder(trans, iter) { + bch2_btree_path_to_text_short(out, trans, iter.path_idx); + prt_newline(out); + } } noinline __cold diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index 1c70836dd7cce..fcf2742b6ad9a 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -861,6 +861,7 @@ __bch2_btree_iter_peek_and_restart(struct btree_trans *trans, }) void bch2_trans_updates_to_text(struct printbuf *, struct btree_trans *); +void bch2_btree_path_to_text(struct printbuf *, struct btree_trans *, btree_path_idx_t); void bch2_trans_paths_to_text(struct printbuf *, struct btree_trans *); void bch2_dump_trans_updates(struct btree_trans *); void bch2_dump_trans_paths_updates(struct btree_trans *); diff --git a/fs/bcachefs/btree_locking.c b/fs/bcachefs/btree_locking.c index 4ee4855395b9b..8d1c4f78db5eb 100644 --- a/fs/bcachefs/btree_locking.c +++ b/fs/bcachefs/btree_locking.c @@ -836,6 +836,11 @@ void bch2_btree_path_verify_locks(struct btree_path *path) { unsigned l; + /* + * A path may be uptodate and yet have nothing locked if and only if + * there is no node at path->level, which generally means we were + * iterating over all nodes and got to the end of the btree + */ if (!path->nodes_locked) { BUG_ON(path->uptodate == BTREE_ITER_UPTODATE && btree_path_node(path, path->level)); From 497c982f057d3a20af4df313d997e81ca903ce4e Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 20 Feb 2024 21:17:15 -0500 Subject: [PATCH 1364/1702] bcachefs: New assertion for writing to the journal after shutdown Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_update.c | 2 +- fs/bcachefs/journal.c | 6 ++++-- fs/bcachefs/journal.h | 4 ++-- fs/bcachefs/journal_reclaim.c | 2 +- fs/bcachefs/journal_types.h | 2 +- fs/bcachefs/super.c | 1 + 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/fs/bcachefs/btree_update.c b/fs/bcachefs/btree_update.c index 8e47e260eba59..c46ee4c813ae2 100644 --- a/fs/bcachefs/btree_update.c +++ b/fs/bcachefs/btree_update.c @@ -852,7 +852,7 @@ __bch2_fs_log_msg(struct bch_fs *c, unsigned commit_flags, const char *fmt, if (ret) goto err; - if (!test_bit(JOURNAL_STARTED, &c->journal.flags)) { + if (!test_bit(JOURNAL_RUNNING, &c->journal.flags)) { ret = darray_make_room(&c->journal.early_journal_entries, jset_u64s(u64s)); if (ret) goto err; diff --git a/fs/bcachefs/journal.c b/fs/bcachefs/journal.c index c40b5c40d63ff..d302d8e836780 100644 --- a/fs/bcachefs/journal.c +++ b/fs/bcachefs/journal.c @@ -1177,12 +1177,14 @@ void bch2_fs_journal_stop(struct journal *j) bch2_journal_meta(j); journal_quiesce(j); + cancel_delayed_work_sync(&j->write_work); BUG_ON(!bch2_journal_error(j) && test_bit(JOURNAL_REPLAY_DONE, &j->flags) && j->last_empty_seq != journal_cur_seq(j)); - cancel_delayed_work_sync(&j->write_work); + if (!bch2_journal_error(j)) + clear_bit(JOURNAL_RUNNING, &j->flags); } int bch2_fs_journal_start(struct journal *j, u64 cur_seq) @@ -1256,7 +1258,7 @@ int bch2_fs_journal_start(struct journal *j, u64 cur_seq) spin_lock(&j->lock); - set_bit(JOURNAL_STARTED, &j->flags); + set_bit(JOURNAL_RUNNING, &j->flags); j->last_flush_write = jiffies; j->reservations.idx = j->reservations.unwritten_idx = journal_cur_seq(j); diff --git a/fs/bcachefs/journal.h b/fs/bcachefs/journal.h index 7c7528f839c56..4b8c709bc317b 100644 --- a/fs/bcachefs/journal.h +++ b/fs/bcachefs/journal.h @@ -372,7 +372,7 @@ static inline int bch2_journal_res_get(struct journal *j, struct journal_res *re int ret; EBUG_ON(res->ref); - EBUG_ON(!test_bit(JOURNAL_STARTED, &j->flags)); + EBUG_ON(!test_bit(JOURNAL_RUNNING, &j->flags)); res->u64s = u64s; @@ -418,7 +418,7 @@ struct bch_dev; static inline void bch2_journal_set_replay_done(struct journal *j) { - BUG_ON(!test_bit(JOURNAL_STARTED, &j->flags)); + BUG_ON(!test_bit(JOURNAL_RUNNING, &j->flags)); set_bit(JOURNAL_REPLAY_DONE, &j->flags); } diff --git a/fs/bcachefs/journal_reclaim.c b/fs/bcachefs/journal_reclaim.c index 04a577848b015..a0c9f7ac611df 100644 --- a/fs/bcachefs/journal_reclaim.c +++ b/fs/bcachefs/journal_reclaim.c @@ -833,7 +833,7 @@ bool bch2_journal_flush_pins(struct journal *j, u64 seq_to_flush) /* time_stats this */ bool did_work = false; - if (!test_bit(JOURNAL_STARTED, &j->flags)) + if (!test_bit(JOURNAL_RUNNING, &j->flags)) return false; closure_wait_event(&j->async_wait, diff --git a/fs/bcachefs/journal_types.h b/fs/bcachefs/journal_types.h index b5161b5d76a00..08debe6bfeeff 100644 --- a/fs/bcachefs/journal_types.h +++ b/fs/bcachefs/journal_types.h @@ -131,7 +131,7 @@ enum journal_space_from { enum journal_flags { JOURNAL_REPLAY_DONE, - JOURNAL_STARTED, + JOURNAL_RUNNING, JOURNAL_MAY_SKIP_FLUSH, JOURNAL_NEED_FLUSH_WRITE, JOURNAL_SPACE_LOW, diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index dddf57ec4511f..ca2f3debd7fd6 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -468,6 +468,7 @@ static int __bch2_fs_read_write(struct bch_fs *c, bool early) * at least one non-flush write in the journal or recovery will fail: */ set_bit(JOURNAL_NEED_FLUSH_WRITE, &c->journal.flags); + set_bit(JOURNAL_RUNNING, &c->journal.flags); for_each_rw_member(c, ca) bch2_dev_allocator_add(c, ca); From 19391b92947c75267cdacb1acefe8ad0d278a9dd Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 28 Mar 2024 00:39:11 -0400 Subject: [PATCH 1365/1702] bcachefs: allow for custom action in fsck error messages Be more explicit to the user about what we're doing. Signed-off-by: Kent Overstreet --- fs/bcachefs/error.c | 59 ++++++++++++++++++++++++++++++++-------- fs/bcachefs/journal_io.c | 2 +- fs/bcachefs/snapshot.c | 2 +- 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/fs/bcachefs/error.c b/fs/bcachefs/error.c index 82a6656c941c5..c66eeffcd7f2a 100644 --- a/fs/bcachefs/error.c +++ b/fs/bcachefs/error.c @@ -176,6 +176,21 @@ static struct fsck_err_state *fsck_err_get(struct bch_fs *c, const char *fmt) return s; } +/* s/fix?/fixing/ s/recreate?/recreating/ */ +static void prt_actioning(struct printbuf *out, const char *action) +{ + unsigned len = strlen(action); + + BUG_ON(action[len - 1] != '?'); + --len; + + if (action[len - 1] == 'e') + --len; + + prt_bytes(out, action, len); + prt_str(out, "ing"); +} + int bch2_fsck_err(struct bch_fs *c, enum bch_fsck_flags flags, enum bch_sb_error_id err, @@ -186,6 +201,7 @@ int bch2_fsck_err(struct bch_fs *c, bool print = true, suppressing = false, inconsistent = false; struct printbuf buf = PRINTBUF, *out = &buf; int ret = -BCH_ERR_fsck_ignore; + const char *action_orig = "fix?", *action = action_orig; if ((flags & FSCK_CAN_FIX) && test_bit(err, c->sb.errors_silent)) @@ -197,6 +213,19 @@ int bch2_fsck_err(struct bch_fs *c, prt_vprintf(out, fmt, args); va_end(args); + /* Custom fix/continue/recreate/etc.? */ + if (out->buf[out->pos - 1] == '?') { + const char *p = strrchr(out->buf, ','); + if (p) { + out->pos = p - out->buf; + action = kstrdup(p + 2, GFP_KERNEL); + if (!action) { + ret = -ENOMEM; + goto err; + } + } + } + mutex_lock(&c->fsck_error_msgs_lock); s = fsck_err_get(c, fmt); if (s) { @@ -208,12 +237,16 @@ int bch2_fsck_err(struct bch_fs *c, if (s->last_msg && !strcmp(buf.buf, s->last_msg)) { ret = s->ret; mutex_unlock(&c->fsck_error_msgs_lock); - printbuf_exit(&buf); - return ret; + goto err; } kfree(s->last_msg); s->last_msg = kstrdup(buf.buf, GFP_KERNEL); + if (!s->last_msg) { + mutex_unlock(&c->fsck_error_msgs_lock); + ret = -ENOMEM; + goto err; + } if (c->opts.ratelimit_errors && !(flags & FSCK_NO_RATELIMIT) && @@ -239,7 +272,8 @@ int bch2_fsck_err(struct bch_fs *c, inconsistent = true; ret = -BCH_ERR_fsck_errors_not_fixed; } else if (flags & FSCK_CAN_FIX) { - prt_str(out, ", fixing"); + prt_str(out, ", "); + prt_actioning(out, action); ret = -BCH_ERR_fsck_fix; } else { prt_str(out, ", continuing"); @@ -254,16 +288,16 @@ int bch2_fsck_err(struct bch_fs *c, : c->opts.fix_errors; if (fix == FSCK_FIX_ask) { - int ask; + prt_str(out, ", "); + prt_str(out, action); - prt_str(out, ": fix?"); if (bch2_fs_stdio_redirect(c)) bch2_print(c, "%s", out->buf); else bch2_print_string_as_lines(KERN_ERR, out->buf); print = false; - ask = bch2_fsck_ask_yn(c); + int ask = bch2_fsck_ask_yn(c); if (ask >= YN_ALLNO && s) s->fix = ask == YN_ALLNO @@ -276,10 +310,12 @@ int bch2_fsck_err(struct bch_fs *c, } else if (fix == FSCK_FIX_yes || (c->opts.nochanges && !(flags & FSCK_CAN_IGNORE))) { - prt_str(out, ", fixing"); + prt_str(out, ", "); + prt_actioning(out, action); ret = -BCH_ERR_fsck_fix; } else { - prt_str(out, ", not fixing"); + prt_str(out, ", not "); + prt_actioning(out, action); } } else if (flags & FSCK_NEED_FSCK) { prt_str(out, " (run fsck to correct)"); @@ -311,8 +347,6 @@ int bch2_fsck_err(struct bch_fs *c, mutex_unlock(&c->fsck_error_msgs_lock); - printbuf_exit(&buf); - if (inconsistent) bch2_inconsistent_error(c); @@ -322,7 +356,10 @@ int bch2_fsck_err(struct bch_fs *c, set_bit(BCH_FS_errors_not_fixed, &c->flags); set_bit(BCH_FS_error, &c->flags); } - +err: + if (action != action_orig) + kfree(action); + printbuf_exit(&buf); return ret; } diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c index eb1f9d6f5a196..cd14636bc2ab4 100644 --- a/fs/bcachefs/journal_io.c +++ b/fs/bcachefs/journal_io.c @@ -1366,7 +1366,7 @@ int bch2_journal_read(struct bch_fs *c, fsck_err(c, journal_entries_missing, "journal entries %llu-%llu missing! (replaying %llu-%llu)\n" " prev at %s\n" - " next at %s", + " next at %s, continue?", missing_start, missing_end, *last_seq, *blacklist_seq - 1, buf1.buf, buf2.buf); diff --git a/fs/bcachefs/snapshot.c b/fs/bcachefs/snapshot.c index 544322d5c2517..c415ea0a649b0 100644 --- a/fs/bcachefs/snapshot.c +++ b/fs/bcachefs/snapshot.c @@ -1018,7 +1018,7 @@ int bch2_reconstruct_snapshots(struct bch_fs *c) darray_for_each(*t, id) { if (fsck_err_on(!bch2_snapshot_equiv(c, *id), c, snapshot_node_missing, - "snapshot node %u from tree %s missing", *id, buf.buf)) { + "snapshot node %u from tree %s missing, recreate?", *id, buf.buf)) { if (t->nr > 1) { bch_err(c, "cannot reconstruct snapshot trees with multiple nodes"); ret = -BCH_ERR_fsck_repair_unimplemented; From 4dcd90b6d1b76b87722059db0f368aa48e7703da Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 30 Mar 2024 22:42:29 -0400 Subject: [PATCH 1366/1702] bcachefs: Don't read journal just for fsck reading the journal can take a decent amount of time compared to the rest of fsck, let's only read it when required. Signed-off-by: Kent Overstreet --- fs/bcachefs/recovery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/bcachefs/recovery.c b/fs/bcachefs/recovery.c index 08c0422a722dc..9cf262f1a3cd1 100644 --- a/fs/bcachefs/recovery.c +++ b/fs/bcachefs/recovery.c @@ -658,7 +658,7 @@ int bch2_fs_recovery(struct bch_fs *c) goto err; } - if (!c->sb.clean || c->opts.fsck || c->opts.retain_recovery_info) { + if (!c->sb.clean || c->opts.retain_recovery_info) { struct genradix_iter iter; struct journal_replay **i; From be31bf439c2154e4c1174186cf2bd6fd1e6b7a88 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 21 Mar 2024 23:21:56 -0400 Subject: [PATCH 1367/1702] bcachefs: When traversing to interior nodes, propagate result to paths to same leaf node Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index ca3b6a0ea1f8e..39c89914cb536 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -1162,6 +1162,7 @@ int bch2_btree_path_traverse_one(struct btree_trans *trans, goto out_uptodate; path->level = btree_path_up_until_good_node(trans, path, 0); + unsigned max_level = path->level; EBUG_ON(btree_path_node(path, path->level) && !btree_node_locked(path, path->level)); @@ -1192,6 +1193,16 @@ int bch2_btree_path_traverse_one(struct btree_trans *trans, goto out; } } + + if (unlikely(max_level > path->level)) { + struct btree_path *linked; + unsigned iter; + + trans_for_each_path_with_node(trans, path_l(path)->b, linked, iter) + for (unsigned j = path->level + 1; j < max_level; j++) + linked->l[j] = path->l[j]; + } + out_uptodate: path->uptodate = BTREE_ITER_UPTODATE; out: From 0ddb5f0854a8f42d949f5d8a49980d8174f266a3 Mon Sep 17 00:00:00 2001 From: Kuan-Wei Chiu Date: Sun, 7 Apr 2024 11:39:04 +0800 Subject: [PATCH 1368/1702] bcachefs: Optimize eytzinger0_sort() with bottom-up heapsort This optimization reduces the average number of comparisons required from 2*n*log2(n) - 3*n + o(n) to n*log2(n) + 0.37*n + o(n). When n is sufficiently large, it results in approximately 50% fewer comparisons. Currently, eytzinger0_sort employs the textbook version of heapsort, where during the heapify process, each level requires two comparisons to determine the maximum among three elements. In contrast, the bottom-up heapsort, during heapify, only compares two children at each level until reaching a leaf node. Then, it backtracks from the leaf node to find the correct position. Since heapify typically continues until very close to the leaf node, the standard heapify requires about 2*log2(n) comparisons, while the bottom-up variant only needs log2(n) comparisons. The experimental data presented below is based on an array generated by get_random_u32(). | N | comparisons(old) | comparisons(new) | time(old) | time(new) | |-------|------------------|------------------|-----------|-----------| | 10000 | 235381 | 136615 | 25545 us | 20366 us | | 20000 | 510694 | 293425 | 31336 us | 18312 us | | 30000 | 800384 | 457412 | 35042 us | 27386 us | | 40000 | 1101617 | 626831 | 48779 us | 38253 us | | 50000 | 1409762 | 799637 | 62238 us | 46950 us | | 60000 | 1721191 | 974521 | 75588 us | 58367 us | | 70000 | 2038536 | 1152171 | 90823 us | 68778 us | | 80000 | 2362958 | 1333472 | 104165 us | 78625 us | | 90000 | 2690900 | 1516065 | 116111 us | 89573 us | | 100000| 3019413 | 1699879 | 133638 us | 100998 us | Refs: BOTTOM-UP-HEAPSORT, a new variant of HEAPSORT beating, on an average, QUICKSORT (if n is not very small) Ingo Wegener Theoretical Computer Science, 118(1); Pages 81-98, 13 September 1993 https://doi.org/10.1016/0304-3975(93)90364-Y Signed-off-by: Kuan-Wei Chiu Signed-off-by: Kent Overstreet --- fs/bcachefs/eytzinger.c | 105 +++++++++++++++++++++++++++++++++------- 1 file changed, 88 insertions(+), 17 deletions(-) diff --git a/fs/bcachefs/eytzinger.c b/fs/bcachefs/eytzinger.c index 0f955c3c76a7b..2eaffe37b5e76 100644 --- a/fs/bcachefs/eytzinger.c +++ b/fs/bcachefs/eytzinger.c @@ -171,7 +171,7 @@ void eytzinger0_sort_r(void *base, size_t n, size_t size, swap_r_func_t swap_func, const void *priv) { - int i, c, r; + int i, j, k; /* called from 'sort' without swap function, let's pick the default */ if (swap_func == SWAP_WRAPPER && !((struct wrapper *)priv)->swap_func) @@ -188,17 +188,22 @@ void eytzinger0_sort_r(void *base, size_t n, size_t size, /* heapify */ for (i = n / 2 - 1; i >= 0; --i) { - for (r = i; r * 2 + 1 < n; r = c) { - c = r * 2 + 1; + /* Find the sift-down path all the way to the leaves. */ + for (j = i; k = j * 2 + 1, k + 1 < n;) + j = eytzinger0_do_cmp(base, n, size, cmp_func, priv, k, k + 1) > 0 ? k : k + 1; - if (c + 1 < n && - eytzinger0_do_cmp(base, n, size, cmp_func, priv, c, c + 1) < 0) - c++; + /* Special case for the last leaf with no sibling. */ + if (j * 2 + 2 == n) + j = j * 2 + 1; - if (eytzinger0_do_cmp(base, n, size, cmp_func, priv, r, c) >= 0) - break; + /* Backtrack to the correct location. */ + while (j != i && eytzinger0_do_cmp(base, n, size, cmp_func, priv, i, j) >= 0) + j = (j - 1) / 2; - eytzinger0_do_swap(base, n, size, swap_func, priv, r, c); + /* Shift the element into its correct place. */ + for (k = j; j != i;) { + j = (j - 1) / 2; + eytzinger0_do_swap(base, n, size, swap_func, priv, j, k); } } @@ -206,17 +211,22 @@ void eytzinger0_sort_r(void *base, size_t n, size_t size, for (i = n - 1; i > 0; --i) { eytzinger0_do_swap(base, n, size, swap_func, priv, 0, i); - for (r = 0; r * 2 + 1 < i; r = c) { - c = r * 2 + 1; + /* Find the sift-down path all the way to the leaves. */ + for (j = 0; k = j * 2 + 1, k + 1 < i;) + j = eytzinger0_do_cmp(base, n, size, cmp_func, priv, k, k + 1) > 0 ? k : k + 1; - if (c + 1 < i && - eytzinger0_do_cmp(base, n, size, cmp_func, priv, c, c + 1) < 0) - c++; + /* Special case for the last leaf with no sibling. */ + if (j * 2 + 2 == i) + j = j * 2 + 1; - if (eytzinger0_do_cmp(base, n, size, cmp_func, priv, r, c) >= 0) - break; + /* Backtrack to the correct location. */ + while (j && eytzinger0_do_cmp(base, n, size, cmp_func, priv, 0, j) >= 0) + j = (j - 1) / 2; - eytzinger0_do_swap(base, n, size, swap_func, priv, r, c); + /* Shift the element into its correct place. */ + for (k = j; j;) { + j = (j - 1) / 2; + eytzinger0_do_swap(base, n, size, swap_func, priv, j, k); } } } @@ -232,3 +242,64 @@ void eytzinger0_sort(void *base, size_t n, size_t size, return eytzinger0_sort_r(base, n, size, _CMP_WRAPPER, SWAP_WRAPPER, &w); } + +#if 0 +#include +#include +#include + +static u64 cmp_count; + +static int mycmp(const void *a, const void *b) +{ + u32 _a = *(u32 *)a; + u32 _b = *(u32 *)b; + + cmp_count++; + if (_a < _b) + return -1; + else if (_a > _b) + return 1; + else + return 0; +} + +static int test(void) +{ + size_t N, i; + ktime_t start, end; + s64 delta; + u32 *arr; + + for (N = 10000; N <= 100000; N += 10000) { + arr = kmalloc_array(N, sizeof(u32), GFP_KERNEL); + cmp_count = 0; + + for (i = 0; i < N; i++) + arr[i] = get_random_u32(); + + start = ktime_get(); + eytzinger0_sort(arr, N, sizeof(u32), mycmp, NULL); + end = ktime_get(); + + delta = ktime_us_delta(end, start); + printk(KERN_INFO "time: %lld\n", delta); + printk(KERN_INFO "comparisons: %lld\n", cmp_count); + + u32 prev = 0; + + eytzinger0_for_each(i, N) { + if (prev > arr[i]) + goto err; + prev = arr[i]; + } + + kfree(arr); + } + return 0; + +err: + kfree(arr); + return -1; +} +#endif From a21107eeb17a348a20248b343c80c2aea3c65b9f Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 6 Apr 2024 01:44:05 -0400 Subject: [PATCH 1369/1702] bcachefs: kill for_each_btree_key_old() Dead code Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index fcf2742b6ad9a..0427097bb9666 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -794,14 +794,6 @@ __bch2_btree_iter_peek_and_restart(struct btree_trans *trans, return k; } -#define for_each_btree_key_old(_trans, _iter, _btree_id, \ - _start, _flags, _k, _ret) \ - for (bch2_trans_iter_init((_trans), &(_iter), (_btree_id), \ - (_start), (_flags)); \ - (_k) = __bch2_btree_iter_peek_and_restart((_trans), &(_iter), _flags),\ - !((_ret) = bkey_err(_k)) && (_k).k; \ - bch2_btree_iter_advance(&(_iter))) - #define for_each_btree_key_upto_norestart(_trans, _iter, _btree_id, \ _start, _end, _flags, _k, _ret) \ for (bch2_trans_iter_init((_trans), &(_iter), (_btree_id), \ From 665e8b32393523f0fb4752ccd4831ae6a10b1145 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 7 Apr 2024 00:52:47 -0400 Subject: [PATCH 1370/1702] bcachefs: for_each_btree_key_continue() Signed-off-by: Kent Overstreet --- fs/bcachefs/backpointers.c | 28 +++++----------------------- fs/bcachefs/btree_iter.h | 21 +++++++++++++++------ 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c index fc9506c71eae1..0ca53392f7876 100644 --- a/fs/bcachefs/backpointers.c +++ b/fs/bcachefs/backpointers.c @@ -793,31 +793,13 @@ static int bch2_check_extents_to_backpointers_pass(struct btree_trans *trans, while (level >= depth) { struct btree_iter iter; - bch2_trans_node_iter_init(trans, &iter, btree_id, POS_MIN, 0, - level, + bch2_trans_node_iter_init(trans, &iter, btree_id, POS_MIN, 0, level, BTREE_ITER_PREFETCH); - while (1) { - bch2_trans_begin(trans); - - struct bkey_s_c k = bch2_btree_iter_peek(&iter); - if (!k.k) - break; - ret = bkey_err(k) ?: - check_extent_to_backpointers(trans, s, btree_id, level, k) ?: - bch2_trans_commit(trans, NULL, NULL, - BCH_TRANS_COMMIT_no_enospc); - if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) { - ret = 0; - continue; - } - if (ret) - break; - if (bpos_eq(iter.pos, SPOS_MAX)) - break; - bch2_btree_iter_advance(&iter); - } - bch2_trans_iter_exit(trans, &iter); + ret = for_each_btree_key_continue(trans, iter, 0, k, ({ + check_extent_to_backpointers(trans, s, btree_id, level, k) ?: + bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc); + })); if (ret) return ret; diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index 0427097bb9666..52030929da0ad 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -699,16 +699,12 @@ transaction_restart: \ _ret2 ?: trans_was_restarted(_trans, _restart_count); \ }) -#define for_each_btree_key_upto(_trans, _iter, _btree_id, \ - _start, _end, _flags, _k, _do) \ +#define for_each_btree_key_upto_continue(_trans, _iter, \ + _end, _flags, _k, _do) \ ({ \ - struct btree_iter _iter; \ struct bkey_s_c _k; \ int _ret3 = 0; \ \ - bch2_trans_iter_init((_trans), &(_iter), (_btree_id), \ - (_start), (_flags)); \ - \ do { \ _ret3 = lockrestart_do(_trans, ({ \ (_k) = bch2_btree_iter_peek_upto_type(&(_iter), \ @@ -724,6 +720,19 @@ transaction_restart: \ _ret3; \ }) +#define for_each_btree_key_continue(_trans, _iter, _flags, _k, _do) \ + for_each_btree_key_upto_continue(_trans, _iter, SPOS_MAX, _flags, _k, _do) + +#define for_each_btree_key_upto(_trans, _iter, _btree_id, \ + _start, _end, _flags, _k, _do) \ +({ \ + struct btree_iter _iter; \ + bch2_trans_iter_init((_trans), &(_iter), (_btree_id), \ + (_start), (_flags)); \ + \ + for_each_btree_key_upto_continue(_trans, _iter, _end, _flags, _k, _do);\ +}) + #define for_each_btree_key(_trans, _iter, _btree_id, \ _start, _flags, _k, _do) \ for_each_btree_key_upto(_trans, _iter, _btree_id, _start, \ From 68e142405cf4cf01461012ec72d675038c514b92 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 6 Apr 2024 00:11:01 -0400 Subject: [PATCH 1371/1702] bcachefs: bch2_gc() is now private to btree_gc.c Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_gc.c | 14 ++++++-------- fs/bcachefs/btree_gc.h | 2 +- fs/bcachefs/recovery_passes.c | 5 ----- fs/bcachefs/sysfs.c | 12 +----------- 4 files changed, 8 insertions(+), 25 deletions(-) diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 791470b0c6545..606e1120ab7cd 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -1756,7 +1756,7 @@ static void bch2_gc_stripes_reset(struct bch_fs *c, bool metadata_only) * move around - if references move backwards in the ordering GC * uses, GC could skip past them */ -int bch2_gc(struct bch_fs *c, bool initial, bool metadata_only) +static int bch2_gc(struct bch_fs *c, bool initial, bool metadata_only) { unsigned iter = 0; int ret; @@ -1843,6 +1843,11 @@ int bch2_gc(struct bch_fs *c, bool initial, bool metadata_only) return ret; } +int bch2_check_allocations(struct bch_fs *c) +{ + return bch2_gc(c, true, false); +} + static int gc_btree_gens_key(struct btree_trans *trans, struct btree_iter *iter, struct bkey_s_c k) @@ -2024,14 +2029,7 @@ static int bch2_gc_thread(void *arg) last = atomic64_read(&clock->now); last_kick = atomic_read(&c->kick_gc); - /* - * Full gc is currently incompatible with btree key cache: - */ -#if 0 - ret = bch2_gc(c, false, false); -#else bch2_gc_gens(c); -#endif debug_check_no_locks_held(); } diff --git a/fs/bcachefs/btree_gc.h b/fs/bcachefs/btree_gc.h index 607575f83a002..0d6c0a2df613e 100644 --- a/fs/bcachefs/btree_gc.h +++ b/fs/bcachefs/btree_gc.h @@ -6,7 +6,7 @@ #include "btree_types.h" int bch2_check_topology(struct bch_fs *); -int bch2_gc(struct bch_fs *, bool, bool); +int bch2_check_allocations(struct bch_fs *); int bch2_gc_gens(struct bch_fs *); void bch2_gc_thread_stop(struct bch_fs *); int bch2_gc_thread_start(struct bch_fs *); diff --git a/fs/bcachefs/recovery_passes.c b/fs/bcachefs/recovery_passes.c index 0cec0f7d97035..c065c0d012125 100644 --- a/fs/bcachefs/recovery_passes.c +++ b/fs/bcachefs/recovery_passes.c @@ -26,11 +26,6 @@ const char * const bch2_recovery_passes[] = { NULL }; -static int bch2_check_allocations(struct bch_fs *c) -{ - return bch2_gc(c, true, false); -} - static int bch2_set_may_go_rw(struct bch_fs *c) { struct journal_keys *keys = &c->journal_keys; diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index b6bfd7cbce184..4011ab4e710b7 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -501,18 +501,8 @@ STORE(bch2_fs) if (attr == &sysfs_btree_wakeup) bch2_btree_wakeup_all(c); - if (attr == &sysfs_trigger_gc) { - /* - * Full gc is currently incompatible with btree key cache: - */ -#if 0 - down_read(&c->state_lock); - bch2_gc(c, false, false); - up_read(&c->state_lock); -#else + if (attr == &sysfs_trigger_gc) bch2_gc_gens(c); -#endif - } if (attr == &sysfs_trigger_discards) bch2_do_discards(c); From d1b213a00ddc1166bc210868371d4f619286d69b Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 7 Apr 2024 17:05:01 -0400 Subject: [PATCH 1372/1702] bcachefs: Finish converting reconstruct_alloc to errors_silent with errors_silent, reconstruct_alloc no longer requires fsck and fix_errors to work Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_gc.c | 4 +--- fs/bcachefs/recovery.c | 11 +++++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 606e1120ab7cd..130f4a3885b25 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -1209,9 +1209,7 @@ static int bch2_gc_done(struct bch_fs *c, { struct bch_dev *ca = NULL; struct printbuf buf = PRINTBUF; - bool verify = !metadata_only && - !c->opts.reconstruct_alloc && - (!initial || (c->sb.compat & (1ULL << BCH_COMPAT_alloc_info))); + bool verify = !metadata_only; unsigned i; int ret = 0; diff --git a/fs/bcachefs/recovery.c b/fs/bcachefs/recovery.c index 9cf262f1a3cd1..d22579a036c74 100644 --- a/fs/bcachefs/recovery.c +++ b/fs/bcachefs/recovery.c @@ -65,9 +65,20 @@ static void bch2_reconstruct_alloc(struct bch_fs *c) __set_bit_le64(BCH_FSCK_ERR_ptr_to_missing_alloc_key, ext->errors_silent); __set_bit_le64(BCH_FSCK_ERR_ptr_gen_newer_than_bucket_gen, ext->errors_silent); __set_bit_le64(BCH_FSCK_ERR_stale_dirty_ptr, ext->errors_silent); + + __set_bit_le64(BCH_FSCK_ERR_dev_usage_buckets_wrong, ext->errors_silent); + __set_bit_le64(BCH_FSCK_ERR_dev_usage_sectors_wrong, ext->errors_silent); + __set_bit_le64(BCH_FSCK_ERR_dev_usage_fragmented_wrong, ext->errors_silent); + + __set_bit_le64(BCH_FSCK_ERR_fs_usage_btree_wrong, ext->errors_silent); + __set_bit_le64(BCH_FSCK_ERR_fs_usage_cached_wrong, ext->errors_silent); + __set_bit_le64(BCH_FSCK_ERR_fs_usage_persistent_reserved_wrong, ext->errors_silent); + __set_bit_le64(BCH_FSCK_ERR_fs_usage_replicas_wrong, ext->errors_silent); + __set_bit_le64(BCH_FSCK_ERR_alloc_key_data_type_wrong, ext->errors_silent); __set_bit_le64(BCH_FSCK_ERR_alloc_key_gen_wrong, ext->errors_silent); __set_bit_le64(BCH_FSCK_ERR_alloc_key_dirty_sectors_wrong, ext->errors_silent); + __set_bit_le64(BCH_FSCK_ERR_alloc_key_cached_sectors_wrong, ext->errors_silent); __set_bit_le64(BCH_FSCK_ERR_alloc_key_stripe_wrong, ext->errors_silent); __set_bit_le64(BCH_FSCK_ERR_alloc_key_stripe_redundancy_wrong, ext->errors_silent); __set_bit_le64(BCH_FSCK_ERR_need_discard_key_wrong, ext->errors_silent); From 58dda9c10e3fdea488180a74de622432de6b4568 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 6 Apr 2024 22:40:12 -0400 Subject: [PATCH 1373/1702] bcachefs: kill metadata only gc Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_gc.c | 188 +++++++++++++---------------------------- 1 file changed, 59 insertions(+), 129 deletions(-) diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 130f4a3885b25..614959f0f669b 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -899,18 +899,18 @@ static int btree_gc_mark_node(struct btree_trans *trans, struct btree *b, bool i } static int bch2_gc_btree(struct btree_trans *trans, enum btree_id btree_id, - bool initial, bool metadata_only) + bool initial) { struct bch_fs *c = trans->c; struct btree_iter iter; struct btree *b; - unsigned depth = metadata_only ? 1 : 0; + unsigned target_depth = btree_type_has_ptrs(btree_id) ? 0 : 1; int ret = 0; gc_pos_set(c, gc_pos_btree(btree_id, POS_MIN, 0)); __for_each_btree_node(trans, iter, btree_id, POS_MIN, - 0, depth, BTREE_ITER_PREFETCH, b, ret) { + 0, target_depth, BTREE_ITER_PREFETCH, b, ret) { bch2_verify_btree_nr_keys(b); gc_pos_set(c, gc_pos_btree_node(b)); @@ -1029,12 +1029,15 @@ static int bch2_gc_btree_init_recurse(struct btree_trans *trans, struct btree *b } static int bch2_gc_btree_init(struct btree_trans *trans, - enum btree_id btree_id, - bool metadata_only) + enum btree_id btree_id) { struct bch_fs *c = trans->c; struct btree *b; - unsigned target_depth = metadata_only ? 1 : 0; + /* + * We need to make sure every leaf node is readable before going RW + unsigned target_depth = btree_node_type_needs_gc(__btree_node_type(0, btree_id)) ? 0 : 1; + */ + unsigned target_depth = 0; struct printbuf buf = PRINTBUF; int ret = 0; @@ -1046,8 +1049,7 @@ static int bch2_gc_btree_init(struct btree_trans *trans, if (mustfix_fsck_err_on(!bpos_eq(b->data->min_key, POS_MIN), c, btree_root_bad_min_key, "btree root with incorrect min_key: %s", buf.buf)) { - bch_err(c, "repair unimplemented"); - ret = -BCH_ERR_fsck_repair_unimplemented; + ret = bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_topology); goto fsck_err; } @@ -1056,8 +1058,7 @@ static int bch2_gc_btree_init(struct btree_trans *trans, if (mustfix_fsck_err_on(!bpos_eq(b->data->max_key, SPOS_MAX), c, btree_root_bad_max_key, "btree root with incorrect max_key: %s", buf.buf)) { - bch_err(c, "repair unimplemented"); - ret = -BCH_ERR_fsck_repair_unimplemented; + ret = bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_topology); goto fsck_err; } @@ -1084,7 +1085,7 @@ static inline int btree_id_gc_phase_cmp(enum btree_id l, enum btree_id r) (int) btree_id_to_gc_phase(r); } -static int bch2_gc_btrees(struct bch_fs *c, bool initial, bool metadata_only) +static int bch2_gc_btrees(struct bch_fs *c, bool initial) { struct btree_trans *trans = bch2_trans_get(c); enum btree_id ids[BTREE_ID_NR]; @@ -1095,18 +1096,15 @@ static int bch2_gc_btrees(struct bch_fs *c, bool initial, bool metadata_only) ids[i] = i; bubble_sort(ids, BTREE_ID_NR, btree_id_gc_phase_cmp); - for (i = 0; i < BTREE_ID_NR && !ret; i++) - ret = initial - ? bch2_gc_btree_init(trans, ids[i], metadata_only) - : bch2_gc_btree(trans, ids[i], initial, metadata_only); + for (i = 0; i < btree_id_nr_alive(c) && !ret; i++) { + unsigned btree = i < BTREE_ID_NR ? ids[i] : i; - for (i = BTREE_ID_NR; i < btree_id_nr_alive(c) && !ret; i++) { - if (!bch2_btree_id_root(c, i)->alive) + if (IS_ERR_OR_NULL(bch2_btree_id_root(c, btree)->b)) continue; ret = initial - ? bch2_gc_btree_init(trans, i, metadata_only) - : bch2_gc_btree(trans, i, initial, metadata_only); + ? bch2_gc_btree_init(trans, btree) + : bch2_gc_btree(trans, btree, initial); } bch2_trans_put(trans); @@ -1169,24 +1167,6 @@ static void bch2_mark_superblocks(struct bch_fs *c) mutex_unlock(&c->sb_lock); } -#if 0 -/* Also see bch2_pending_btree_node_free_insert_done() */ -static void bch2_mark_pending_btree_node_frees(struct bch_fs *c) -{ - struct btree_update *as; - struct pending_btree_node_free *d; - - mutex_lock(&c->btree_interior_update_lock); - gc_pos_set(c, gc_phase(GC_PHASE_PENDING_DELETE)); - - for_each_pending_btree_node_free(c, as, d) - if (d->index_update_done) - bch2_mark_key(c, bkey_i_to_s_c(&d->key), BTREE_TRIGGER_GC); - - mutex_unlock(&c->btree_interior_update_lock); -} -#endif - static void bch2_gc_free(struct bch_fs *c) { genradix_free(&c->reflink_gc_table); @@ -1204,26 +1184,23 @@ static void bch2_gc_free(struct bch_fs *c) c->usage_gc = NULL; } -static int bch2_gc_done(struct bch_fs *c, - bool initial, bool metadata_only) +static int bch2_gc_done(struct bch_fs *c) { struct bch_dev *ca = NULL; struct printbuf buf = PRINTBUF; - bool verify = !metadata_only; unsigned i; int ret = 0; percpu_down_write(&c->mark_lock); -#define copy_field(_err, _f, _msg, ...) \ - if (dst->_f != src->_f && \ - (!verify || \ - fsck_err(c, _err, _msg ": got %llu, should be %llu" \ - , ##__VA_ARGS__, dst->_f, src->_f))) \ +#define copy_field(_err, _f, _msg, ...) \ + if (fsck_err_on(dst->_f != src->_f, c, _err, \ + _msg ": got %llu, should be %llu" , ##__VA_ARGS__, \ + dst->_f, src->_f)) \ dst->_f = src->_f -#define copy_dev_field(_err, _f, _msg, ...) \ +#define copy_dev_field(_err, _f, _msg, ...) \ copy_field(_err, _f, "dev %u has wrong " _msg, ca->dev_idx, ##__VA_ARGS__) -#define copy_fs_field(_err, _f, _msg, ...) \ +#define copy_fs_field(_err, _f, _msg, ...) \ copy_field(_err, _f, "fs has wrong " _msg, ##__VA_ARGS__) for (i = 0; i < ARRAY_SIZE(c->usage); i++) @@ -1256,31 +1233,24 @@ static int bch2_gc_done(struct bch_fs *c, copy_fs_field(fs_usage_btree_wrong, b.btree, "btree"); - if (!metadata_only) { - copy_fs_field(fs_usage_data_wrong, - b.data, "data"); - copy_fs_field(fs_usage_cached_wrong, - b.cached, "cached"); - copy_fs_field(fs_usage_reserved_wrong, - b.reserved, "reserved"); - copy_fs_field(fs_usage_nr_inodes_wrong, - b.nr_inodes,"nr_inodes"); - - for (i = 0; i < BCH_REPLICAS_MAX; i++) - copy_fs_field(fs_usage_persistent_reserved_wrong, - persistent_reserved[i], - "persistent_reserved[%i]", i); - } + copy_fs_field(fs_usage_data_wrong, + b.data, "data"); + copy_fs_field(fs_usage_cached_wrong, + b.cached, "cached"); + copy_fs_field(fs_usage_reserved_wrong, + b.reserved, "reserved"); + copy_fs_field(fs_usage_nr_inodes_wrong, + b.nr_inodes,"nr_inodes"); + + for (i = 0; i < BCH_REPLICAS_MAX; i++) + copy_fs_field(fs_usage_persistent_reserved_wrong, + persistent_reserved[i], + "persistent_reserved[%i]", i); for (i = 0; i < c->replicas.nr; i++) { struct bch_replicas_entry_v1 *e = cpu_replicas_entry(&c->replicas, i); - if (metadata_only && - (e->data_type == BCH_DATA_user || - e->data_type == BCH_DATA_cached)) - continue; - printbuf_reset(&buf); bch2_replicas_entry_to_text(&buf, e); @@ -1359,8 +1329,7 @@ static inline bool bch2_alloc_v4_cmp(struct bch_alloc_v4 l, static int bch2_alloc_write_key(struct btree_trans *trans, struct btree_iter *iter, - struct bkey_s_c k, - bool metadata_only) + struct bkey_s_c k) { struct bch_fs *c = trans->c; struct bch_dev *ca = bch_dev_bkey_exists(c, iter->pos.inode); @@ -1400,12 +1369,6 @@ static int bch2_alloc_write_key(struct btree_trans *trans, bch2_dev_usage_update_m(c, ca, &old_gc, &gc); percpu_up_read(&c->mark_lock); - if (metadata_only && - gc.data_type != BCH_DATA_sb && - gc.data_type != BCH_DATA_journal && - gc.data_type != BCH_DATA_btree) - return 0; - if (gen_after(old->gen, gc.gen)) return 0; @@ -1463,7 +1426,7 @@ static int bch2_alloc_write_key(struct btree_trans *trans, return ret; } -static int bch2_gc_alloc_done(struct bch_fs *c, bool metadata_only) +static int bch2_gc_alloc_done(struct bch_fs *c) { int ret = 0; @@ -1474,7 +1437,7 @@ static int bch2_gc_alloc_done(struct bch_fs *c, bool metadata_only) POS(ca->dev_idx, ca->mi.nbuckets - 1), BTREE_ITER_SLOTS|BTREE_ITER_PREFETCH, k, NULL, NULL, BCH_TRANS_COMMIT_lazy_rw, - bch2_alloc_write_key(trans, &iter, k, metadata_only))); + bch2_alloc_write_key(trans, &iter, k))); if (ret) { percpu_ref_put(&ca->ref); break; @@ -1485,7 +1448,7 @@ static int bch2_gc_alloc_done(struct bch_fs *c, bool metadata_only) return ret; } -static int bch2_gc_alloc_start(struct bch_fs *c, bool metadata_only) +static int bch2_gc_alloc_start(struct bch_fs *c) { for_each_member_device(c, ca) { struct bucket_array *buckets = kvmalloc(sizeof(struct bucket_array) + @@ -1513,36 +1476,19 @@ static int bch2_gc_alloc_start(struct bch_fs *c, bool metadata_only) g->gen_valid = 1; g->gen = a->gen; - - if (metadata_only && - (a->data_type == BCH_DATA_user || - a->data_type == BCH_DATA_cached || - a->data_type == BCH_DATA_parity)) { - g->data_type = a->data_type; - g->dirty_sectors = a->dirty_sectors; - g->cached_sectors = a->cached_sectors; - g->stripe = a->stripe; - g->stripe_redundancy = a->stripe_redundancy; - } - 0; }))); bch_err_fn(c, ret); return ret; } -static void bch2_gc_alloc_reset(struct bch_fs *c, bool metadata_only) +static void bch2_gc_alloc_reset(struct bch_fs *c) { for_each_member_device(c, ca) { struct bucket_array *buckets = gc_bucket_array(ca); struct bucket *g; for_each_bucket(g, buckets) { - if (metadata_only && - (g->data_type == BCH_DATA_user || - g->data_type == BCH_DATA_cached || - g->data_type == BCH_DATA_parity)) - continue; g->data_type = 0; g->dirty_sectors = 0; g->cached_sectors = 0; @@ -1599,13 +1545,10 @@ static int bch2_gc_write_reflink_key(struct btree_trans *trans, return ret; } -static int bch2_gc_reflink_done(struct bch_fs *c, bool metadata_only) +static int bch2_gc_reflink_done(struct bch_fs *c) { size_t idx = 0; - if (metadata_only) - return 0; - int ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_reflink, POS_MIN, @@ -1616,13 +1559,8 @@ static int bch2_gc_reflink_done(struct bch_fs *c, bool metadata_only) return ret; } -static int bch2_gc_reflink_start(struct bch_fs *c, - bool metadata_only) +static int bch2_gc_reflink_start(struct bch_fs *c) { - - if (metadata_only) - return 0; - c->reflink_gc_nr = 0; int ret = bch2_trans_run(c, @@ -1650,7 +1588,7 @@ static int bch2_gc_reflink_start(struct bch_fs *c, return ret; } -static void bch2_gc_reflink_reset(struct bch_fs *c, bool metadata_only) +static void bch2_gc_reflink_reset(struct bch_fs *c) { struct genradix_iter iter; struct reflink_gc *r; @@ -1712,11 +1650,8 @@ static int bch2_gc_write_stripes_key(struct btree_trans *trans, return ret; } -static int bch2_gc_stripes_done(struct bch_fs *c, bool metadata_only) +static int bch2_gc_stripes_done(struct bch_fs *c) { - if (metadata_only) - return 0; - return bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_stripes, POS_MIN, @@ -1725,7 +1660,7 @@ static int bch2_gc_stripes_done(struct bch_fs *c, bool metadata_only) bch2_gc_write_stripes_key(trans, &iter, k))); } -static void bch2_gc_stripes_reset(struct bch_fs *c, bool metadata_only) +static void bch2_gc_stripes_reset(struct bch_fs *c) { genradix_free(&c->gc_stripes); } @@ -1735,7 +1670,6 @@ static void bch2_gc_stripes_reset(struct bch_fs *c, bool metadata_only) * * @c: filesystem object * @initial: are we in recovery? - * @metadata_only: are we just checking metadata references, or everything? * * Returns: 0 on success, or standard errcode on failure * @@ -1754,7 +1688,7 @@ static void bch2_gc_stripes_reset(struct bch_fs *c, bool metadata_only) * move around - if references move backwards in the ordering GC * uses, GC could skip past them */ -static int bch2_gc(struct bch_fs *c, bool initial, bool metadata_only) +static int bch2_gc(struct bch_fs *c, bool initial) { unsigned iter = 0; int ret; @@ -1766,8 +1700,8 @@ static int bch2_gc(struct bch_fs *c, bool initial, bool metadata_only) bch2_btree_interior_updates_flush(c); ret = bch2_gc_start(c) ?: - bch2_gc_alloc_start(c, metadata_only) ?: - bch2_gc_reflink_start(c, metadata_only); + bch2_gc_alloc_start(c) ?: + bch2_gc_reflink_start(c); if (ret) goto out; again: @@ -1775,14 +1709,10 @@ static int bch2_gc(struct bch_fs *c, bool initial, bool metadata_only) bch2_mark_superblocks(c); - ret = bch2_gc_btrees(c, initial, metadata_only); - + ret = bch2_gc_btrees(c, initial); if (ret) goto out; -#if 0 - bch2_mark_pending_btree_node_frees(c); -#endif c->gc_count++; if (test_bit(BCH_FS_need_another_gc, &c->flags) || @@ -1800,9 +1730,9 @@ static int bch2_gc(struct bch_fs *c, bool initial, bool metadata_only) clear_bit(BCH_FS_need_another_gc, &c->flags); __gc_pos_set(c, gc_phase(GC_PHASE_NOT_RUNNING)); - bch2_gc_stripes_reset(c, metadata_only); - bch2_gc_alloc_reset(c, metadata_only); - bch2_gc_reflink_reset(c, metadata_only); + bch2_gc_stripes_reset(c); + bch2_gc_alloc_reset(c); + bch2_gc_reflink_reset(c); ret = bch2_gc_reset(c); if (ret) goto out; @@ -1815,10 +1745,10 @@ static int bch2_gc(struct bch_fs *c, bool initial, bool metadata_only) if (!ret) { bch2_journal_block(&c->journal); - ret = bch2_gc_alloc_done(c, metadata_only) ?: - bch2_gc_done(c, initial, metadata_only) ?: - bch2_gc_stripes_done(c, metadata_only) ?: - bch2_gc_reflink_done(c, metadata_only); + ret = bch2_gc_alloc_done(c) ?: + bch2_gc_done(c) ?: + bch2_gc_stripes_done(c) ?: + bch2_gc_reflink_done(c); bch2_journal_unblock(&c->journal); } @@ -1843,7 +1773,7 @@ static int bch2_gc(struct bch_fs *c, bool initial, bool metadata_only) int bch2_check_allocations(struct bch_fs *c) { - return bch2_gc(c, true, false); + return bch2_gc(c, true); } static int gc_btree_gens_key(struct btree_trans *trans, From b982d645a40e29296a5dbc3c5da067e93e7abd62 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 6 Apr 2024 23:20:49 -0400 Subject: [PATCH 1374/1702] bcachefs: move topology repair kick to gc_btrees() Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_gc.c | 48 ++++++++---------------------------------- 1 file changed, 9 insertions(+), 39 deletions(-) diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 614959f0f669b..b86a821f388a0 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -52,12 +52,6 @@ static struct bkey_s unsafe_bkey_s_c_to_s(struct bkey_s_c k) }}}; } -static bool should_restart_for_topology_repair(struct bch_fs *c) -{ - return c->opts.fix_errors != FSCK_FIX_no && - !(c->recovery_passes_complete & BIT_ULL(BCH_RECOVERY_PASS_check_topology)); -} - static inline void __gc_pos_set(struct bch_fs *c, struct gc_pos new_pos) { preempt_disable(); @@ -881,9 +875,6 @@ static int btree_gc_mark_node(struct btree_trans *trans, struct btree *b, bool i if (ret) return ret; - if (!btree_node_type_needs_gc(btree_node_type(b))) - return 0; - bch2_btree_node_iter_init_from_start(&iter, b); while ((k = bch2_btree_node_iter_peek_unpack(&iter, b, &unpacked)).k) { @@ -982,36 +973,9 @@ static int bch2_gc_btree_init_recurse(struct btree_trans *trans, struct btree *b b->c.btree_id, b->c.level - 1, false); ret = PTR_ERR_OR_ZERO(child); - - if (bch2_err_matches(ret, EIO)) { - bch2_topology_error(c); - - if (__fsck_err(c, - FSCK_CAN_FIX| - FSCK_CAN_IGNORE| - FSCK_NO_RATELIMIT, - btree_node_read_error, - "Unreadable btree node at btree %s level %u:\n" - " %s", - bch2_btree_id_str(b->c.btree_id), - b->c.level - 1, - (printbuf_reset(&buf), - bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(cur.k)), buf.buf)) && - should_restart_for_topology_repair(c)) { - bch_info(c, "Halting mark and sweep to start topology repair pass"); - ret = bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_topology); - goto fsck_err; - } else { - /* Continue marking when opted to not - * fix the error: */ - ret = 0; - set_bit(BCH_FS_initial_gc_unfixed, &c->flags); - continue; - } - } else if (ret) { - bch_err_msg(c, ret, "getting btree node"); + bch_err_msg(c, ret, "getting btree node"); + if (ret) break; - } ret = bch2_gc_btree_init_recurse(trans, child, target_depth); @@ -1105,8 +1069,14 @@ static int bch2_gc_btrees(struct bch_fs *c, bool initial) ret = initial ? bch2_gc_btree_init(trans, btree) : bch2_gc_btree(trans, btree, initial); - } + if (mustfix_fsck_err_on(bch2_err_matches(ret, EIO), + c, btree_node_read_error, + "btree node read error for %s", + bch2_btree_id_str(btree))) + ret = bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_topology); + } +fsck_err: bch2_trans_put(trans); bch_err_fn(c, ret); return ret; From d1adfe4e7e4e7ea225547a07c4b79c314c50c6fb Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 6 Apr 2024 23:26:36 -0400 Subject: [PATCH 1375/1702] bcachefs: move root node topo checks to node_check_topology() Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_gc.c | 24 +----------------------- fs/bcachefs/btree_update_interior.c | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index b86a821f388a0..6b466b2ebebec 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -996,36 +996,16 @@ static int bch2_gc_btree_init(struct btree_trans *trans, enum btree_id btree_id) { struct bch_fs *c = trans->c; - struct btree *b; /* * We need to make sure every leaf node is readable before going RW unsigned target_depth = btree_node_type_needs_gc(__btree_node_type(0, btree_id)) ? 0 : 1; */ unsigned target_depth = 0; - struct printbuf buf = PRINTBUF; int ret = 0; - b = bch2_btree_id_root(c, btree_id)->b; + struct btree *b = bch2_btree_id_root(c, btree_id)->b; six_lock_read(&b->c.lock, NULL, NULL); - printbuf_reset(&buf); - bch2_bpos_to_text(&buf, b->data->min_key); - if (mustfix_fsck_err_on(!bpos_eq(b->data->min_key, POS_MIN), c, - btree_root_bad_min_key, - "btree root with incorrect min_key: %s", buf.buf)) { - ret = bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_topology); - goto fsck_err; - } - - printbuf_reset(&buf); - bch2_bpos_to_text(&buf, b->data->max_key); - if (mustfix_fsck_err_on(!bpos_eq(b->data->max_key, SPOS_MAX), c, - btree_root_bad_max_key, - "btree root with incorrect max_key: %s", buf.buf)) { - ret = bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_check_topology); - goto fsck_err; - } - if (b->c.level >= target_depth) ret = bch2_gc_btree_init_recurse(trans, b, target_depth); @@ -1035,11 +1015,9 @@ static int bch2_gc_btree_init(struct btree_trans *trans, ret = bch2_gc_mark_key(trans, b->c.btree_id, b->c.level + 1, true, &k, true); } -fsck_err: six_unlock_read(&b->c.lock); bch_err_fn(c, ret); - printbuf_exit(&buf); return ret; } diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index b4efd8cc4d1a2..875b1795e4086 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -73,6 +73,24 @@ int bch2_btree_node_check_topology(struct btree_trans *trans, struct btree *b) !bpos_eq(bkey_i_to_btree_ptr_v2(&b->key)->v.min_key, b->data->min_key)); + if (b == btree_node_root(c, b)) { + if (!bpos_eq(b->data->min_key, POS_MIN)) { + printbuf_reset(&buf); + bch2_bpos_to_text(&buf, b->data->min_key); + need_fsck_err(c, btree_root_bad_min_key, + "btree root with incorrect min_key: %s", buf.buf); + goto topology_repair; + } + + if (!bpos_eq(b->data->max_key, SPOS_MAX)) { + printbuf_reset(&buf); + bch2_bpos_to_text(&buf, b->data->max_key); + need_fsck_err(c, btree_root_bad_max_key, + "btree root with incorrect max_key: %s", buf.buf); + goto topology_repair; + } + } + if (!b->c.level) return 0; From ba665494fbf8ea63656b7d45755595cfc6214b50 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 6 Apr 2024 23:39:12 -0400 Subject: [PATCH 1376/1702] bcachefs: gc_btree_init_recurse() uses gc_mark_node() Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_gc.c | 64 +++++++++++++++--------------------------- 1 file changed, 22 insertions(+), 42 deletions(-) diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 6b466b2ebebec..18b33811ae401 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -866,8 +866,7 @@ static int bch2_gc_mark_key(struct btree_trans *trans, enum btree_id btree_id, static int btree_gc_mark_node(struct btree_trans *trans, struct btree *b, bool initial) { - struct btree_node_iter iter; - struct bkey unpacked; + struct btree_and_journal_iter iter; struct bkey_s_c k; int ret = 0; @@ -875,18 +874,18 @@ static int btree_gc_mark_node(struct btree_trans *trans, struct btree *b, bool i if (ret) return ret; - bch2_btree_node_iter_init_from_start(&iter, b); + bch2_btree_and_journal_iter_init_node_iter(trans, &iter, b); - while ((k = bch2_btree_node_iter_peek_unpack(&iter, b, &unpacked)).k) { - ret = bch2_gc_mark_key(trans, b->c.btree_id, b->c.level, false, - &k, initial); + while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) { + ret = bch2_gc_mark_key(trans, b->c.btree_id, b->c.level, false, &k, initial); if (ret) - return ret; + break; - bch2_btree_node_iter_advance(&iter, b); + bch2_btree_and_journal_iter_advance(&iter); } - return 0; + bch2_btree_and_journal_iter_exit(&iter); + return ret; } static int bch2_gc_btree(struct btree_trans *trans, enum btree_id btree_id, @@ -932,44 +931,26 @@ static int bch2_gc_btree(struct btree_trans *trans, enum btree_id btree_id, static int bch2_gc_btree_init_recurse(struct btree_trans *trans, struct btree *b, unsigned target_depth) { - struct bch_fs *c = trans->c; - struct btree_and_journal_iter iter; - struct bkey_s_c k; - struct bkey_buf cur; - struct printbuf buf = PRINTBUF; - int ret = 0; - - ret = bch2_btree_node_check_topology(trans, b); + int ret = btree_gc_mark_node(trans, b, true); if (ret) return ret; - bch2_btree_and_journal_iter_init_node_iter(trans, &iter, b); - bch2_bkey_buf_init(&cur); - - while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) { - BUG_ON(bpos_lt(k.k->p, b->data->min_key)); - BUG_ON(bpos_gt(k.k->p, b->data->max_key)); - - ret = bch2_gc_mark_key(trans, b->c.btree_id, b->c.level, - false, &k, true); - if (ret) - goto fsck_err; - - bch2_btree_and_journal_iter_advance(&iter); - } - if (b->c.level > target_depth) { - bch2_btree_and_journal_iter_exit(&iter); + struct bch_fs *c = trans->c; + struct btree_and_journal_iter iter; + struct bkey_s_c k; + struct bkey_buf cur; + + bch2_bkey_buf_init(&cur); bch2_btree_and_journal_iter_init_node_iter(trans, &iter, b); iter.prefetch = true; while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) { - struct btree *child; - bch2_bkey_buf_reassemble(&cur, c, k); bch2_btree_and_journal_iter_advance(&iter); - child = bch2_btree_node_get_noiter(trans, cur.k, + struct btree *child = + bch2_btree_node_get_noiter(trans, cur.k, b->c.btree_id, b->c.level - 1, false); ret = PTR_ERR_OR_ZERO(child); @@ -977,18 +958,17 @@ static int bch2_gc_btree_init_recurse(struct btree_trans *trans, struct btree *b if (ret) break; - ret = bch2_gc_btree_init_recurse(trans, child, - target_depth); + ret = bch2_gc_btree_init_recurse(trans, child, target_depth); six_unlock_read(&child->c.lock); if (ret) break; } + + bch2_bkey_buf_exit(&cur, c); + bch2_btree_and_journal_iter_exit(&iter); } -fsck_err: - bch2_bkey_buf_exit(&cur, c); - bch2_btree_and_journal_iter_exit(&iter); - printbuf_exit(&buf); + return ret; } From c281db0fa59180bd8901ed88c526616143ef284b Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 7 Apr 2024 02:11:03 -0400 Subject: [PATCH 1377/1702] bcachefs: mark_superblock cleanup Consolidate mark_superblock() and trans_mark_superblock(), like we did with the other trigger paths. Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_gc.c | 54 ++------------ fs/bcachefs/buckets.c | 160 +++++++++++++++++++++-------------------- fs/bcachefs/buckets.h | 11 ++- fs/bcachefs/journal.c | 5 +- fs/bcachefs/super.c | 6 +- 5 files changed, 97 insertions(+), 139 deletions(-) diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 18b33811ae401..512be84aec239 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -1040,59 +1040,14 @@ static int bch2_gc_btrees(struct bch_fs *c, bool initial) return ret; } -static void mark_metadata_sectors(struct bch_fs *c, struct bch_dev *ca, - u64 start, u64 end, - enum bch_data_type type, - unsigned flags) -{ - u64 b = sector_to_bucket(ca, start); - - do { - unsigned sectors = - min_t(u64, bucket_to_sector(ca, b + 1), end) - start; - - bch2_mark_metadata_bucket(c, ca, b, type, sectors, - gc_phase(GC_PHASE_SB), flags); - b++; - start += sectors; - } while (start < end); -} - -static void bch2_mark_dev_superblock(struct bch_fs *c, struct bch_dev *ca, - unsigned flags) -{ - struct bch_sb_layout *layout = &ca->disk_sb.sb->layout; - unsigned i; - u64 b; - - for (i = 0; i < layout->nr_superblocks; i++) { - u64 offset = le64_to_cpu(layout->sb_offset[i]); - - if (offset == BCH_SB_SECTOR) - mark_metadata_sectors(c, ca, 0, BCH_SB_SECTOR, - BCH_DATA_sb, flags); - - mark_metadata_sectors(c, ca, offset, - offset + (1 << layout->sb_max_size_bits), - BCH_DATA_sb, flags); - } - - for (i = 0; i < ca->journal.nr; i++) { - b = ca->journal.buckets[i]; - bch2_mark_metadata_bucket(c, ca, b, BCH_DATA_journal, - ca->mi.bucket_size, - gc_phase(GC_PHASE_SB), flags); - } -} - -static void bch2_mark_superblocks(struct bch_fs *c) +static int bch2_mark_superblocks(struct bch_fs *c) { mutex_lock(&c->sb_lock); gc_pos_set(c, gc_phase(GC_PHASE_SB)); - for_each_online_member(c, ca) - bch2_mark_dev_superblock(c, ca, BTREE_TRIGGER_GC); + int ret = bch2_trans_mark_dev_sbs_flags(c, BTREE_TRIGGER_GC); mutex_unlock(&c->sb_lock); + return ret; } static void bch2_gc_free(struct bch_fs *c) @@ -1635,7 +1590,8 @@ static int bch2_gc(struct bch_fs *c, bool initial) again: gc_pos_set(c, gc_phase(GC_PHASE_START)); - bch2_mark_superblocks(c); + ret = bch2_mark_superblocks(c); + BUG_ON(ret); ret = bch2_gc_btrees(c, initial); if (ret) diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 6ac6f61cfbc4c..015321a87170a 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -485,59 +485,6 @@ int bch2_update_cached_sectors_list(struct btree_trans *trans, unsigned dev, s64 return bch2_update_replicas_list(trans, &r.e, sectors); } -int bch2_mark_metadata_bucket(struct bch_fs *c, struct bch_dev *ca, - size_t b, enum bch_data_type data_type, - unsigned sectors, struct gc_pos pos, - unsigned flags) -{ - struct bucket old, new, *g; - int ret = 0; - - BUG_ON(!(flags & BTREE_TRIGGER_GC)); - BUG_ON(data_type != BCH_DATA_sb && - data_type != BCH_DATA_journal); - - /* - * Backup superblock might be past the end of our normal usable space: - */ - if (b >= ca->mi.nbuckets) - return 0; - - percpu_down_read(&c->mark_lock); - g = gc_bucket(ca, b); - - bucket_lock(g); - old = *g; - - if (bch2_fs_inconsistent_on(g->data_type && - g->data_type != data_type, c, - "different types of data in same bucket: %s, %s", - bch2_data_type_str(g->data_type), - bch2_data_type_str(data_type))) { - ret = -EIO; - goto err; - } - - if (bch2_fs_inconsistent_on((u64) g->dirty_sectors + sectors > ca->mi.bucket_size, c, - "bucket %u:%zu gen %u data type %s sector count overflow: %u + %u > bucket size", - ca->dev_idx, b, g->gen, - bch2_data_type_str(g->data_type ?: data_type), - g->dirty_sectors, sectors)) { - ret = -EIO; - goto err; - } - - g->data_type = data_type; - g->dirty_sectors += sectors; - new = *g; -err: - bucket_unlock(g); - if (!ret) - bch2_dev_usage_update_m(c, ca, &old, &new); - percpu_up_read(&c->mark_lock); - return ret; -} - int bch2_check_bucket_ref(struct btree_trans *trans, struct bkey_s_c k, const struct bch_extent_ptr *ptr, @@ -1107,22 +1054,16 @@ int bch2_trigger_reservation(struct btree_trans *trans, /* Mark superblocks: */ static int __bch2_trans_mark_metadata_bucket(struct btree_trans *trans, - struct bch_dev *ca, size_t b, + struct bch_dev *ca, u64 b, enum bch_data_type type, unsigned sectors) { struct bch_fs *c = trans->c; struct btree_iter iter; - struct bkey_i_alloc_v4 *a; int ret = 0; - /* - * Backup superblock might be past the end of our normal usable space: - */ - if (b >= ca->mi.nbuckets) - return 0; - - a = bch2_trans_start_alloc_update(trans, &iter, POS(ca->dev_idx, b)); + struct bkey_i_alloc_v4 *a = + bch2_trans_start_alloc_update(trans, &iter, POS(ca->dev_idx, b)); if (IS_ERR(a)) return PTR_ERR(a); @@ -1150,20 +1091,78 @@ static int __bch2_trans_mark_metadata_bucket(struct btree_trans *trans, return ret; } +static int bch2_mark_metadata_bucket(struct bch_fs *c, struct bch_dev *ca, + u64 b, enum bch_data_type data_type, + unsigned sectors, unsigned flags) +{ + struct bucket old, new, *g; + int ret = 0; + + percpu_down_read(&c->mark_lock); + g = gc_bucket(ca, b); + + bucket_lock(g); + old = *g; + + if (bch2_fs_inconsistent_on(g->data_type && + g->data_type != data_type, c, + "different types of data in same bucket: %s, %s", + bch2_data_type_str(g->data_type), + bch2_data_type_str(data_type))) { + ret = -EIO; + goto err; + } + + if (bch2_fs_inconsistent_on((u64) g->dirty_sectors + sectors > ca->mi.bucket_size, c, + "bucket %u:%llu gen %u data type %s sector count overflow: %u + %u > bucket size", + ca->dev_idx, b, g->gen, + bch2_data_type_str(g->data_type ?: data_type), + g->dirty_sectors, sectors)) { + ret = -EIO; + goto err; + } + + g->data_type = data_type; + g->dirty_sectors += sectors; + new = *g; +err: + bucket_unlock(g); + if (!ret) + bch2_dev_usage_update_m(c, ca, &old, &new); + percpu_up_read(&c->mark_lock); + return ret; +} + int bch2_trans_mark_metadata_bucket(struct btree_trans *trans, - struct bch_dev *ca, size_t b, + struct bch_dev *ca, u64 b, enum bch_data_type type, - unsigned sectors) + unsigned sectors, unsigned flags) { - return commit_do(trans, NULL, NULL, 0, - __bch2_trans_mark_metadata_bucket(trans, ca, b, type, sectors)); + BUG_ON(type != BCH_DATA_free && + type != BCH_DATA_sb && + type != BCH_DATA_journal); + + /* + * Backup superblock might be past the end of our normal usable space: + */ + if (b >= ca->mi.nbuckets) + return 0; + + if (flags & BTREE_TRIGGER_GC) + return bch2_mark_metadata_bucket(trans->c, ca, b, type, sectors, flags); + else if (flags & BTREE_TRIGGER_TRANSACTIONAL) + return commit_do(trans, NULL, NULL, 0, + __bch2_trans_mark_metadata_bucket(trans, ca, b, type, sectors)); + else + BUG(); } static int bch2_trans_mark_metadata_sectors(struct btree_trans *trans, struct bch_dev *ca, u64 start, u64 end, enum bch_data_type type, - u64 *bucket, unsigned *bucket_sectors) + u64 *bucket, unsigned *bucket_sectors, + unsigned flags) { do { u64 b = sector_to_bucket(ca, start); @@ -1172,7 +1171,7 @@ static int bch2_trans_mark_metadata_sectors(struct btree_trans *trans, if (b != *bucket && *bucket_sectors) { int ret = bch2_trans_mark_metadata_bucket(trans, ca, *bucket, - type, *bucket_sectors); + type, *bucket_sectors, flags); if (ret) return ret; @@ -1188,7 +1187,7 @@ static int bch2_trans_mark_metadata_sectors(struct btree_trans *trans, } static int __bch2_trans_mark_dev_sb(struct btree_trans *trans, - struct bch_dev *ca) + struct bch_dev *ca, unsigned flags) { struct bch_sb_layout *layout = &ca->disk_sb.sb->layout; u64 bucket = 0; @@ -1201,21 +1200,21 @@ static int __bch2_trans_mark_dev_sb(struct btree_trans *trans, if (offset == BCH_SB_SECTOR) { ret = bch2_trans_mark_metadata_sectors(trans, ca, 0, BCH_SB_SECTOR, - BCH_DATA_sb, &bucket, &bucket_sectors); + BCH_DATA_sb, &bucket, &bucket_sectors, flags); if (ret) return ret; } ret = bch2_trans_mark_metadata_sectors(trans, ca, offset, offset + (1 << layout->sb_max_size_bits), - BCH_DATA_sb, &bucket, &bucket_sectors); + BCH_DATA_sb, &bucket, &bucket_sectors, flags); if (ret) return ret; } if (bucket_sectors) { ret = bch2_trans_mark_metadata_bucket(trans, ca, - bucket, BCH_DATA_sb, bucket_sectors); + bucket, BCH_DATA_sb, bucket_sectors, flags); if (ret) return ret; } @@ -1223,7 +1222,7 @@ static int __bch2_trans_mark_dev_sb(struct btree_trans *trans, for (i = 0; i < ca->journal.nr; i++) { ret = bch2_trans_mark_metadata_bucket(trans, ca, ca->journal.buckets[i], - BCH_DATA_journal, ca->mi.bucket_size); + BCH_DATA_journal, ca->mi.bucket_size, flags); if (ret) return ret; } @@ -1231,18 +1230,18 @@ static int __bch2_trans_mark_dev_sb(struct btree_trans *trans, return 0; } -int bch2_trans_mark_dev_sb(struct bch_fs *c, struct bch_dev *ca) +int bch2_trans_mark_dev_sb(struct bch_fs *c, struct bch_dev *ca, unsigned flags) { - int ret = bch2_trans_run(c, __bch2_trans_mark_dev_sb(trans, ca)); - + int ret = bch2_trans_run(c, + __bch2_trans_mark_dev_sb(trans, ca, flags)); bch_err_fn(c, ret); return ret; } -int bch2_trans_mark_dev_sbs(struct bch_fs *c) +int bch2_trans_mark_dev_sbs_flags(struct bch_fs *c, unsigned flags) { for_each_online_member(c, ca) { - int ret = bch2_trans_mark_dev_sb(c, ca); + int ret = bch2_trans_mark_dev_sb(c, ca, flags); if (ret) { percpu_ref_put(&ca->ref); return ret; @@ -1252,6 +1251,11 @@ int bch2_trans_mark_dev_sbs(struct bch_fs *c) return 0; } +int bch2_trans_mark_dev_sbs(struct bch_fs *c) +{ + return bch2_trans_mark_dev_sbs_flags(c, BTREE_TRIGGER_TRANSACTIONAL); +} + /* Disk reservations: */ #define SECTORS_CACHE 1024 diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h index f9af5adabe836..bae76e58311d9 100644 --- a/fs/bcachefs/buckets.h +++ b/fs/bcachefs/buckets.h @@ -337,10 +337,6 @@ int bch2_check_bucket_ref(struct btree_trans *, struct bkey_s_c, const struct bch_extent_ptr *, s64, enum bch_data_type, u8, u8, u32); -int bch2_mark_metadata_bucket(struct bch_fs *, struct bch_dev *, - size_t, enum bch_data_type, unsigned, - struct gc_pos, unsigned); - int bch2_trigger_extent(struct btree_trans *, enum btree_id, unsigned, struct bkey_s_c, struct bkey_s, unsigned); int bch2_trigger_reservation(struct btree_trans *, enum btree_id, unsigned, @@ -362,9 +358,10 @@ void bch2_trans_account_disk_usage_change(struct btree_trans *); void bch2_trans_fs_usage_revert(struct btree_trans *, struct replicas_delta_list *); int bch2_trans_fs_usage_apply(struct btree_trans *, struct replicas_delta_list *); -int bch2_trans_mark_metadata_bucket(struct btree_trans *, struct bch_dev *, - size_t, enum bch_data_type, unsigned); -int bch2_trans_mark_dev_sb(struct bch_fs *, struct bch_dev *); +int bch2_trans_mark_metadata_bucket(struct btree_trans *, struct bch_dev *, u64, + enum bch_data_type, unsigned, unsigned); +int bch2_trans_mark_dev_sb(struct bch_fs *, struct bch_dev *, unsigned); +int bch2_trans_mark_dev_sbs_flags(struct bch_fs *, unsigned); int bch2_trans_mark_dev_sbs(struct bch_fs *); static inline bool is_superblock_bucket(struct bch_dev *ca, u64 b) diff --git a/fs/bcachefs/journal.c b/fs/bcachefs/journal.c index d302d8e836780..fc02cb4ee74ff 100644 --- a/fs/bcachefs/journal.c +++ b/fs/bcachefs/journal.c @@ -946,7 +946,7 @@ static int __bch2_set_nr_journal_buckets(struct bch_dev *ca, unsigned nr, ret = bch2_trans_run(c, bch2_trans_mark_metadata_bucket(trans, ca, ob[nr_got]->bucket, BCH_DATA_journal, - ca->mi.bucket_size)); + ca->mi.bucket_size, BTREE_TRIGGER_TRANSACTIONAL)); if (ret) { bch2_open_bucket_put(c, ob[nr_got]); bch_err_msg(c, ret, "marking new journal buckets"); @@ -1026,7 +1026,8 @@ static int __bch2_set_nr_journal_buckets(struct bch_dev *ca, unsigned nr, for (i = 0; i < nr_got; i++) bch2_trans_run(c, bch2_trans_mark_metadata_bucket(trans, ca, - bu[i], BCH_DATA_free, 0)); + bu[i], BCH_DATA_free, 0, + BTREE_TRIGGER_TRANSACTIONAL)); err_free: if (!new_fs) for (i = 0; i < nr_got; i++) diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index ca2f3debd7fd6..9073915f22a13 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -1822,7 +1822,7 @@ int bch2_dev_add(struct bch_fs *c, const char *path) bch2_dev_usage_journal_reserve(c); - ret = bch2_trans_mark_dev_sb(c, ca); + ret = bch2_trans_mark_dev_sb(c, ca, BTREE_TRIGGER_TRANSACTIONAL); bch_err_msg(ca, ret, "marking new superblock"); if (ret) goto err_late; @@ -1887,7 +1887,7 @@ int bch2_dev_online(struct bch_fs *c, const char *path) ca = bch_dev_locked(c, dev_idx); - ret = bch2_trans_mark_dev_sb(c, ca); + ret = bch2_trans_mark_dev_sb(c, ca, BTREE_TRIGGER_TRANSACTIONAL); bch_err_msg(c, ret, "bringing %s online: error from bch2_trans_mark_dev_sb", path); if (ret) goto err; @@ -1980,7 +1980,7 @@ int bch2_dev_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets) if (ret) goto err; - ret = bch2_trans_mark_dev_sb(c, ca); + ret = bch2_trans_mark_dev_sb(c, ca, BTREE_TRIGGER_TRANSACTIONAL); if (ret) goto err; From bf5f6a689b6037cb164e110dca37042aab9c0e09 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 7 Apr 2024 17:54:00 -0400 Subject: [PATCH 1378/1702] bcachefs: __BTREE_ITER_ALL_SNAPSHOTS -> BTREE_ITER_SNAPSHOT_FIELD Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.c | 4 ++-- fs/bcachefs/btree_iter.h | 2 +- fs/bcachefs/btree_types.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index 39c89914cb536..ccf2e41b8aa79 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -258,7 +258,7 @@ static void bch2_btree_iter_verify(struct btree_iter *iter) BUG_ON((iter->flags & BTREE_ITER_IS_EXTENTS) && (iter->flags & BTREE_ITER_ALL_SNAPSHOTS)); - BUG_ON(!(iter->flags & __BTREE_ITER_ALL_SNAPSHOTS) && + BUG_ON(!(iter->flags & BTREE_ITER_SNAPSHOT_FIELD) && (iter->flags & BTREE_ITER_ALL_SNAPSHOTS) && !btree_type_has_snapshot_field(iter->btree_id)); @@ -2809,7 +2809,7 @@ void bch2_trans_node_iter_init(struct btree_trans *trans, unsigned flags) { flags |= BTREE_ITER_NOT_EXTENTS; - flags |= __BTREE_ITER_ALL_SNAPSHOTS; + flags |= BTREE_ITER_SNAPSHOT_FIELD; flags |= BTREE_ITER_ALL_SNAPSHOTS; bch2_trans_iter_init_common(trans, iter, btree_id, pos, locks_want, depth, diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index 52030929da0ad..57323d5e94a99 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -420,7 +420,7 @@ static inline unsigned __bch2_btree_iter_flags(struct btree_trans *trans, btree_id_is_extents(btree_id)) flags |= BTREE_ITER_IS_EXTENTS; - if (!(flags & __BTREE_ITER_ALL_SNAPSHOTS) && + if (!(flags & BTREE_ITER_SNAPSHOT_FIELD) && !btree_type_has_snapshot_field(btree_id)) flags &= ~BTREE_ITER_ALL_SNAPSHOTS; diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index c69b233c41bb3..d4649129e5d36 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -210,7 +210,7 @@ static const __maybe_unused u16 BTREE_ITER_CACHED = 1 << 5; static const __maybe_unused u16 BTREE_ITER_WITH_KEY_CACHE = 1 << 6; static const __maybe_unused u16 BTREE_ITER_WITH_UPDATES = 1 << 7; static const __maybe_unused u16 BTREE_ITER_WITH_JOURNAL = 1 << 8; -static const __maybe_unused u16 __BTREE_ITER_ALL_SNAPSHOTS = 1 << 9; +static const __maybe_unused u16 BTREE_ITER_SNAPSHOT_FIELD = 1 << 9; static const __maybe_unused u16 BTREE_ITER_ALL_SNAPSHOTS = 1 << 10; static const __maybe_unused u16 BTREE_ITER_FILTER_SNAPSHOTS = 1 << 11; static const __maybe_unused u16 BTREE_ITER_NOPRESERVE = 1 << 12; From 5dd8c60e1e044816d789098ce2454a130e06b03d Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 7 Apr 2024 18:05:34 -0400 Subject: [PATCH 1379/1702] bcachefs: iter/update/trigger/str_hash flag cleanup Combine iter/update/trigger/str_hash flags into a single enum, and x-macroize them for a to_text() function later. These flags are all for a specific iter/key/update context, so it makes sense to group them together - iter/update/trigger flags were already given distinct bits, this cleans up and unifies that handling. Signed-off-by: Kent Overstreet --- fs/bcachefs/acl.c | 4 +- fs/bcachefs/alloc_background.c | 60 +++++----- fs/bcachefs/alloc_foreground.c | 8 +- fs/bcachefs/backpointers.c | 14 +-- fs/bcachefs/bkey_methods.h | 65 ++--------- fs/bcachefs/btree_gc.c | 22 ++-- fs/bcachefs/btree_iter.c | 166 ++++++++++++++-------------- fs/bcachefs/btree_iter.h | 34 +++--- fs/bcachefs/btree_key_cache.c | 28 ++--- fs/bcachefs/btree_trans_commit.c | 24 ++-- fs/bcachefs/btree_types.h | 109 +++++++++++++----- fs/bcachefs/btree_update.c | 88 +++++++-------- fs/bcachefs/btree_update.h | 12 +- fs/bcachefs/btree_update_interior.c | 20 ++-- fs/bcachefs/btree_write_buffer.c | 8 +- fs/bcachefs/buckets.c | 77 ++++++------- fs/bcachefs/buckets.h | 19 ++-- fs/bcachefs/data_update.c | 12 +- fs/bcachefs/debug.c | 8 +- fs/bcachefs/dirent.c | 21 ++-- fs/bcachefs/dirent.h | 4 +- fs/bcachefs/ec.c | 26 ++--- fs/bcachefs/ec.h | 3 +- fs/bcachefs/extent_update.c | 2 +- fs/bcachefs/fs-common.c | 34 +++--- fs/bcachefs/fs-io-buffered.c | 2 +- fs/bcachefs/fs-io-direct.c | 2 +- fs/bcachefs/fs-io-pagecache.c | 2 +- fs/bcachefs/fs-io.c | 4 +- fs/bcachefs/fs.c | 6 +- fs/bcachefs/fsck.c | 74 ++++++------- fs/bcachefs/inode.c | 30 ++--- fs/bcachefs/inode.h | 2 +- fs/bcachefs/io_misc.c | 10 +- fs/bcachefs/io_read.c | 10 +- fs/bcachefs/io_write.c | 16 +-- fs/bcachefs/journal.c | 4 +- fs/bcachefs/journal_seq_blacklist.c | 2 +- fs/bcachefs/logged_ops.c | 2 +- fs/bcachefs/lru.c | 2 +- fs/bcachefs/migrate.c | 8 +- fs/bcachefs/move.c | 14 +-- fs/bcachefs/movinggc.c | 2 +- fs/bcachefs/opts.h | 2 +- fs/bcachefs/quota.c | 6 +- fs/bcachefs/rebalance.c | 8 +- fs/bcachefs/recovery.c | 14 +-- fs/bcachefs/reflink.c | 55 ++++----- fs/bcachefs/reflink.h | 6 +- fs/bcachefs/snapshot.c | 40 +++---- fs/bcachefs/str_hash.h | 45 +++----- fs/bcachefs/subvolume.c | 18 +-- fs/bcachefs/super.c | 18 +-- fs/bcachefs/sysfs.c | 2 +- fs/bcachefs/tests.c | 16 +-- fs/bcachefs/xattr.c | 6 +- 56 files changed, 652 insertions(+), 644 deletions(-) diff --git a/fs/bcachefs/acl.c b/fs/bcachefs/acl.c index b4b112e33be4c..250d6c6d3a3a1 100644 --- a/fs/bcachefs/acl.c +++ b/fs/bcachefs/acl.c @@ -360,7 +360,7 @@ int bch2_set_acl(struct mnt_idmap *idmap, ret = bch2_subvol_is_ro_trans(trans, inode->ei_subvol) ?: bch2_inode_peek(trans, &inode_iter, &inode_u, inode_inum(inode), - BTREE_ITER_INTENT); + BTREE_ITER_intent); if (ret) goto btree_err; @@ -411,7 +411,7 @@ int bch2_acl_chmod(struct btree_trans *trans, subvol_inum inum, struct posix_acl *acl = NULL; struct bkey_s_c k = bch2_hash_lookup(trans, &iter, bch2_xattr_hash_desc, - &hash_info, inum, &search, BTREE_ITER_INTENT); + &hash_info, inum, &search, BTREE_ITER_intent); int ret = bkey_err(k); if (ret) return bch2_err_matches(ret, ENOENT) ? 0 : ret; diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 6a7d249f108c9..d3b86ce0fae61 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -437,9 +437,9 @@ bch2_trans_start_alloc_update(struct btree_trans *trans, struct btree_iter *iter int ret; k = bch2_bkey_get_iter(trans, iter, BTREE_ID_alloc, pos, - BTREE_ITER_WITH_UPDATES| - BTREE_ITER_CACHED| - BTREE_ITER_INTENT); + BTREE_ITER_with_updates| + BTREE_ITER_cached| + BTREE_ITER_intent); ret = bkey_err(k); if (unlikely(ret)) return ERR_PTR(ret); @@ -510,7 +510,7 @@ int bch2_bucket_gens_init(struct bch_fs *c) int ret; ret = for_each_btree_key(trans, iter, BTREE_ID_alloc, POS_MIN, - BTREE_ITER_PREFETCH, k, ({ + BTREE_ITER_prefetch, k, ({ /* * Not a fsck error because this is checked/repaired by * bch2_check_alloc_key() which runs later: @@ -563,7 +563,7 @@ int bch2_alloc_read(struct bch_fs *c) if (c->sb.version_upgrade_complete >= bcachefs_metadata_version_bucket_gens) { ret = for_each_btree_key(trans, iter, BTREE_ID_bucket_gens, POS_MIN, - BTREE_ITER_PREFETCH, k, ({ + BTREE_ITER_prefetch, k, ({ u64 start = bucket_gens_pos_to_alloc(k.k->p, 0).offset; u64 end = bucket_gens_pos_to_alloc(bpos_nosnap_successor(k.k->p), 0).offset; @@ -589,7 +589,7 @@ int bch2_alloc_read(struct bch_fs *c) })); } else { ret = for_each_btree_key(trans, iter, BTREE_ID_alloc, POS_MIN, - BTREE_ITER_PREFETCH, k, ({ + BTREE_ITER_prefetch, k, ({ /* * Not a fsck error because this is checked/repaired by * bch2_check_alloc_key() which runs later: @@ -657,7 +657,7 @@ static int bch2_bucket_do_index(struct btree_trans *trans, old = bch2_bkey_get_iter(trans, &iter, btree, bkey_start_pos(&k->k), - BTREE_ITER_INTENT); + BTREE_ITER_intent); ret = bkey_err(old); if (ret) return ret; @@ -701,8 +701,8 @@ static noinline int bch2_bucket_gen_update(struct btree_trans *trans, return ret; k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_bucket_gens, pos, - BTREE_ITER_INTENT| - BTREE_ITER_WITH_UPDATES); + BTREE_ITER_intent| + BTREE_ITER_with_updates); ret = bkey_err(k); if (ret) return ret; @@ -738,7 +738,7 @@ int bch2_trigger_alloc(struct btree_trans *trans, struct bch_alloc_v4 old_a_convert; const struct bch_alloc_v4 *old_a = bch2_alloc_to_v4(old, &old_a_convert); - if (flags & BTREE_TRIGGER_TRANSACTIONAL) { + if (flags & BTREE_TRIGGER_transactional) { struct bch_alloc_v4 *new_a = bkey_s_to_alloc_v4(new).v; new_a->data_type = alloc_data_type(*new_a, new_a->data_type); @@ -802,7 +802,7 @@ int bch2_trigger_alloc(struct btree_trans *trans, * not: */ - if ((flags & BTREE_TRIGGER_BUCKET_INVALIDATE) && + if ((flags & BTREE_TRIGGER_bucket_invalidate) && old_a->cached_sectors) { ret = bch2_update_cached_sectors_list(trans, new.k->p.inode, -((s64) old_a->cached_sectors)); @@ -811,12 +811,12 @@ int bch2_trigger_alloc(struct btree_trans *trans, } } - if ((flags & BTREE_TRIGGER_ATOMIC) && (flags & BTREE_TRIGGER_INSERT)) { + if ((flags & BTREE_TRIGGER_atomic) && (flags & BTREE_TRIGGER_insert)) { struct bch_alloc_v4 *new_a = bkey_s_to_alloc_v4(new).v; u64 journal_seq = trans->journal_res.seq; u64 bucket_journal_seq = new_a->journal_seq; - if ((flags & BTREE_TRIGGER_INSERT) && + if ((flags & BTREE_TRIGGER_insert) && data_type_is_empty(old_a->data_type) != data_type_is_empty(new_a->data_type) && new.k->type == KEY_TYPE_alloc_v4) { @@ -877,8 +877,8 @@ int bch2_trigger_alloc(struct btree_trans *trans, bch2_do_gc_gens(c); } - if ((flags & BTREE_TRIGGER_GC) && - (flags & BTREE_TRIGGER_BUCKET_INVALIDATE)) { + if ((flags & BTREE_TRIGGER_gc) && + (flags & BTREE_TRIGGER_bucket_invalidate)) { struct bch_alloc_v4 new_a_convert; const struct bch_alloc_v4 *new_a = bch2_alloc_to_v4(new.s_c, &new_a_convert); @@ -903,7 +903,7 @@ int bch2_trigger_alloc(struct btree_trans *trans, } /* - * This synthesizes deleted extents for holes, similar to BTREE_ITER_SLOTS for + * This synthesizes deleted extents for holes, similar to BTREE_ITER_slots for * extents style btrees, but works on non-extents btrees: */ static struct bkey_s_c bch2_get_key_or_hole(struct btree_iter *iter, struct bpos end, struct bkey *hole) @@ -1401,13 +1401,13 @@ int bch2_check_alloc_info(struct bch_fs *c) int ret = 0; bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc, POS_MIN, - BTREE_ITER_PREFETCH); + BTREE_ITER_prefetch); bch2_trans_iter_init(trans, &discard_iter, BTREE_ID_need_discard, POS_MIN, - BTREE_ITER_PREFETCH); + BTREE_ITER_prefetch); bch2_trans_iter_init(trans, &freespace_iter, BTREE_ID_freespace, POS_MIN, - BTREE_ITER_PREFETCH); + BTREE_ITER_prefetch); bch2_trans_iter_init(trans, &bucket_gens_iter, BTREE_ID_bucket_gens, POS_MIN, - BTREE_ITER_PREFETCH); + BTREE_ITER_prefetch); while (1) { struct bpos next; @@ -1469,13 +1469,13 @@ int bch2_check_alloc_info(struct bch_fs *c) ret = for_each_btree_key(trans, iter, BTREE_ID_need_discard, POS_MIN, - BTREE_ITER_PREFETCH, k, + BTREE_ITER_prefetch, k, bch2_check_discard_freespace_key(trans, &iter)); if (ret) goto err; bch2_trans_iter_init(trans, &iter, BTREE_ID_freespace, POS_MIN, - BTREE_ITER_PREFETCH); + BTREE_ITER_prefetch); while (1) { bch2_trans_begin(trans); k = bch2_btree_iter_peek(&iter); @@ -1505,7 +1505,7 @@ int bch2_check_alloc_info(struct bch_fs *c) ret = for_each_btree_key_commit(trans, iter, BTREE_ID_bucket_gens, POS_MIN, - BTREE_ITER_PREFETCH, k, + BTREE_ITER_prefetch, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, bch2_check_bucket_gens_key(trans, &iter, k)); err: @@ -1552,7 +1552,7 @@ static int bch2_check_alloc_to_lru_ref(struct btree_trans *trans, a_mut->v.io_time[READ] = atomic64_read(&c->io_clock[READ].now); ret = bch2_trans_update(trans, alloc_iter, - &a_mut->k_i, BTREE_TRIGGER_NORUN); + &a_mut->k_i, BTREE_TRIGGER_norun); if (ret) goto err; @@ -1591,7 +1591,7 @@ int bch2_check_alloc_to_lru_refs(struct bch_fs *c) { int ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_alloc, - POS_MIN, BTREE_ITER_PREFETCH, k, + POS_MIN, BTREE_ITER_prefetch, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, bch2_check_alloc_to_lru_ref(trans, &iter))); bch_err_fn(c, ret); @@ -1693,7 +1693,7 @@ static int bch2_discard_one_bucket(struct btree_trans *trans, k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_alloc, need_discard_iter->pos, - BTREE_ITER_CACHED); + BTREE_ITER_cached); ret = bkey_err(k); if (ret) goto out; @@ -1817,7 +1817,7 @@ void bch2_do_discards(struct bch_fs *c) static int bch2_clear_bucket_needs_discard(struct btree_trans *trans, struct bpos bucket) { struct btree_iter iter; - bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc, bucket, BTREE_ITER_INTENT); + bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc, bucket, BTREE_ITER_intent); struct bkey_s_c k = bch2_btree_iter_peek_slot(&iter); int ret = bkey_err(k); if (ret) @@ -1952,7 +1952,7 @@ static int invalidate_one_bucket(struct btree_trans *trans, a->v.io_time[WRITE] = atomic64_read(&c->io_clock[WRITE].now); ret = bch2_trans_update(trans, &alloc_iter, &a->k_i, - BTREE_TRIGGER_BUCKET_INVALIDATE) ?: + BTREE_TRIGGER_bucket_invalidate) ?: bch2_trans_commit(trans, NULL, NULL, BCH_WATERMARK_btree| BCH_TRANS_COMMIT_no_enospc); @@ -2004,7 +2004,7 @@ static void bch2_do_invalidates_work(struct work_struct *work) ret = for_each_btree_key_upto(trans, iter, BTREE_ID_lru, lru_pos(ca->dev_idx, 0, 0), lru_pos(ca->dev_idx, U64_MAX, LRU_TIME_MAX), - BTREE_ITER_INTENT, k, + BTREE_ITER_intent, k, invalidate_one_bucket(trans, &iter, k, &nr_to_invalidate)); if (ret < 0) { @@ -2041,7 +2041,7 @@ int bch2_dev_freespace_init(struct bch_fs *c, struct bch_dev *ca, bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc, POS(ca->dev_idx, max_t(u64, ca->mi.first_bucket, bucket_start)), - BTREE_ITER_PREFETCH); + BTREE_ITER_prefetch); /* * Scan the alloc btree for every bucket on @ca, and add buckets to the * freespace/need_discard/need_gc_gens btrees as needed: diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index a1fc30adf9129..b68e1fd782f33 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -300,7 +300,7 @@ static struct open_bucket *try_alloc_bucket(struct btree_trans *trans, struct bc k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_alloc, POS(ca->dev_idx, b), - BTREE_ITER_CACHED); + BTREE_ITER_cached); ret = bkey_err(k); if (ret) { ob = ERR_PTR(ret); @@ -344,7 +344,7 @@ static struct open_bucket *try_alloc_bucket(struct btree_trans *trans, struct bc ret = bch2_get_next_backpointer(trans, POS(ca->dev_idx, b), -1, &bp_pos, &bp, - BTREE_ITER_NOPRESERVE); + BTREE_ITER_nopreserve); if (ret) { ob = ERR_PTR(ret); goto err; @@ -404,7 +404,7 @@ bch2_bucket_alloc_early(struct btree_trans *trans, */ again: for_each_btree_key_norestart(trans, iter, BTREE_ID_alloc, POS(ca->dev_idx, alloc_cursor), - BTREE_ITER_SLOTS, k, ret) { + BTREE_ITER_slots, k, ret) { struct bch_alloc_v4 a_convert; const struct bch_alloc_v4 *a; @@ -420,7 +420,7 @@ bch2_bucket_alloc_early(struct btree_trans *trans, continue; /* now check the cached key to serialize concurrent allocs of the bucket */ - ck = bch2_bkey_get_iter(trans, &citer, BTREE_ID_alloc, k.k->p, BTREE_ITER_CACHED); + ck = bch2_bkey_get_iter(trans, &citer, BTREE_ID_alloc, k.k->p, BTREE_ITER_cached); ret = bkey_err(ck); if (ret) break; diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c index 0ca53392f7876..5586820fcf2c4 100644 --- a/fs/bcachefs/backpointers.c +++ b/fs/bcachefs/backpointers.c @@ -170,9 +170,9 @@ int bch2_bucket_backpointer_mod_nowritebuffer(struct btree_trans *trans, k = bch2_bkey_get_iter(trans, &bp_iter, BTREE_ID_backpointers, bp_k->k.p, - BTREE_ITER_INTENT| - BTREE_ITER_SLOTS| - BTREE_ITER_WITH_UPDATES); + BTREE_ITER_intent| + BTREE_ITER_slots| + BTREE_ITER_with_updates); ret = bkey_err(k); if (ret) goto err; @@ -212,7 +212,7 @@ int bch2_get_next_backpointer(struct btree_trans *trans, if (gen >= 0) { k = bch2_bkey_get_iter(trans, &alloc_iter, BTREE_ID_alloc, - bucket, BTREE_ITER_CACHED|iter_flags); + bucket, BTREE_ITER_cached|iter_flags); ret = bkey_err(k); if (ret) goto out; @@ -759,7 +759,7 @@ static int bch2_get_btree_in_memory_pos(struct btree_trans *trans, __for_each_btree_node(trans, iter, btree, btree == start.btree ? start.pos : POS_MIN, - 0, depth, BTREE_ITER_PREFETCH, b, ret) { + 0, depth, BTREE_ITER_prefetch, b, ret) { mem_may_pin -= btree_buf_bytes(b); if (mem_may_pin <= 0) { c->btree_cache.pinned_nodes_end = *end = @@ -794,7 +794,7 @@ static int bch2_check_extents_to_backpointers_pass(struct btree_trans *trans, while (level >= depth) { struct btree_iter iter; bch2_trans_node_iter_init(trans, &iter, btree_id, POS_MIN, 0, level, - BTREE_ITER_PREFETCH); + BTREE_ITER_prefetch); ret = for_each_btree_key_continue(trans, iter, 0, k, ({ check_extent_to_backpointers(trans, s, btree_id, level, k) ?: @@ -917,7 +917,7 @@ static int bch2_check_backpointers_to_extents_pass(struct btree_trans *trans, struct bpos last_flushed_pos = SPOS_MAX; return for_each_btree_key_commit(trans, iter, BTREE_ID_backpointers, - POS_MIN, BTREE_ITER_PREFETCH, k, + POS_MIN, BTREE_ITER_prefetch, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, check_one_backpointer(trans, start, end, bkey_s_c_to_backpointer(k), diff --git a/fs/bcachefs/bkey_methods.h b/fs/bcachefs/bkey_methods.h index 03efe8ee565a9..7351c0d38534f 100644 --- a/fs/bcachefs/bkey_methods.h +++ b/fs/bcachefs/bkey_methods.h @@ -29,7 +29,8 @@ struct bkey_ops { bool (*key_normalize)(struct bch_fs *, struct bkey_s); bool (*key_merge)(struct bch_fs *, struct bkey_s, struct bkey_s_c); int (*trigger)(struct btree_trans *, enum btree_id, unsigned, - struct bkey_s_c, struct bkey_s, unsigned); + struct bkey_s_c, struct bkey_s, + enum btree_iter_update_trigger_flags); void (*compat)(enum btree_id id, unsigned version, unsigned big_endian, int write, struct bkey_s); @@ -76,56 +77,10 @@ static inline bool bch2_bkey_maybe_mergable(const struct bkey *l, const struct b bool bch2_bkey_merge(struct bch_fs *, struct bkey_s, struct bkey_s_c); -enum btree_update_flags { - __BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE = __BTREE_ITER_FLAGS_END, - __BTREE_UPDATE_NOJOURNAL, - __BTREE_UPDATE_KEY_CACHE_RECLAIM, - - __BTREE_TRIGGER_NORUN, - __BTREE_TRIGGER_TRANSACTIONAL, - __BTREE_TRIGGER_ATOMIC, - __BTREE_TRIGGER_GC, - __BTREE_TRIGGER_INSERT, - __BTREE_TRIGGER_OVERWRITE, - __BTREE_TRIGGER_BUCKET_INVALIDATE, -}; - -#define BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE (1U << __BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) -#define BTREE_UPDATE_NOJOURNAL (1U << __BTREE_UPDATE_NOJOURNAL) -#define BTREE_UPDATE_KEY_CACHE_RECLAIM (1U << __BTREE_UPDATE_KEY_CACHE_RECLAIM) - -/* Don't run triggers at all */ -#define BTREE_TRIGGER_NORUN (1U << __BTREE_TRIGGER_NORUN) - -/* - * If set, we're running transactional triggers as part of a transaction commit: - * triggers may generate new updates - * - * If cleared, and either BTREE_TRIGGER_INSERT|BTREE_TRIGGER_OVERWRITE are set, - * we're running atomic triggers during a transaction commit: we have our - * journal reservation, we're holding btree node write locks, and we know the - * transaction is going to commit (returning an error here is a fatal error, - * causing us to go emergency read-only) - */ -#define BTREE_TRIGGER_TRANSACTIONAL (1U << __BTREE_TRIGGER_TRANSACTIONAL) -#define BTREE_TRIGGER_ATOMIC (1U << __BTREE_TRIGGER_ATOMIC) - -/* We're in gc/fsck: running triggers to recalculate e.g. disk usage */ -#define BTREE_TRIGGER_GC (1U << __BTREE_TRIGGER_GC) - -/* @new is entering the btree */ -#define BTREE_TRIGGER_INSERT (1U << __BTREE_TRIGGER_INSERT) - -/* @old is leaving the btree */ -#define BTREE_TRIGGER_OVERWRITE (1U << __BTREE_TRIGGER_OVERWRITE) - -/* signal from bucket invalidate path to alloc trigger */ -#define BTREE_TRIGGER_BUCKET_INVALIDATE (1U << __BTREE_TRIGGER_BUCKET_INVALIDATE) - static inline int bch2_key_trigger(struct btree_trans *trans, enum btree_id btree, unsigned level, struct bkey_s_c old, struct bkey_s new, - unsigned flags) + enum btree_iter_update_trigger_flags flags) { const struct bkey_ops *ops = bch2_bkey_type_ops(old.k->type ?: new.k->type); @@ -135,8 +90,9 @@ static inline int bch2_key_trigger(struct btree_trans *trans, } static inline int bch2_key_trigger_old(struct btree_trans *trans, - enum btree_id btree_id, unsigned level, - struct bkey_s_c old, unsigned flags) + enum btree_id btree_id, unsigned level, + struct bkey_s_c old, + enum btree_iter_update_trigger_flags flags) { struct bkey_i deleted; @@ -144,12 +100,13 @@ static inline int bch2_key_trigger_old(struct btree_trans *trans, deleted.k.p = old.k->p; return bch2_key_trigger(trans, btree_id, level, old, bkey_i_to_s(&deleted), - BTREE_TRIGGER_OVERWRITE|flags); + BTREE_TRIGGER_overwrite|flags); } static inline int bch2_key_trigger_new(struct btree_trans *trans, - enum btree_id btree_id, unsigned level, - struct bkey_s new, unsigned flags) + enum btree_id btree_id, unsigned level, + struct bkey_s new, + enum btree_iter_update_trigger_flags flags) { struct bkey_i deleted; @@ -157,7 +114,7 @@ static inline int bch2_key_trigger_new(struct btree_trans *trans, deleted.k.p = new.k->p; return bch2_key_trigger(trans, btree_id, level, bkey_i_to_s_c(&deleted), new, - BTREE_TRIGGER_INSERT|flags); + BTREE_TRIGGER_insert|flags); } void bch2_bkey_renumber(enum btree_node_type, struct bkey_packed *, int); diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 512be84aec239..c98b0ce455243 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -856,7 +856,7 @@ static int bch2_gc_mark_key(struct btree_trans *trans, enum btree_id btree_id, ret = commit_do(trans, NULL, NULL, 0, bch2_key_trigger(trans, btree_id, level, old, - unsafe_bkey_s_c_to_s(*k), BTREE_TRIGGER_GC)); + unsafe_bkey_s_c_to_s(*k), BTREE_TRIGGER_gc)); fsck_err: err: printbuf_exit(&buf); @@ -900,7 +900,7 @@ static int bch2_gc_btree(struct btree_trans *trans, enum btree_id btree_id, gc_pos_set(c, gc_pos_btree(btree_id, POS_MIN, 0)); __for_each_btree_node(trans, iter, btree_id, POS_MIN, - 0, target_depth, BTREE_ITER_PREFETCH, b, ret) { + 0, target_depth, BTREE_ITER_prefetch, b, ret) { bch2_verify_btree_nr_keys(b); gc_pos_set(c, gc_pos_btree_node(b)); @@ -1045,7 +1045,7 @@ static int bch2_mark_superblocks(struct bch_fs *c) mutex_lock(&c->sb_lock); gc_pos_set(c, gc_phase(GC_PHASE_SB)); - int ret = bch2_trans_mark_dev_sbs_flags(c, BTREE_TRIGGER_GC); + int ret = bch2_trans_mark_dev_sbs_flags(c, BTREE_TRIGGER_gc); mutex_unlock(&c->sb_lock); return ret; } @@ -1304,7 +1304,7 @@ static int bch2_alloc_write_key(struct btree_trans *trans, if (a->v.data_type == BCH_DATA_cached && !a->v.io_time[READ]) a->v.io_time[READ] = max_t(u64, 1, atomic64_read(&c->io_clock[READ].now)); - ret = bch2_trans_update(trans, iter, &a->k_i, BTREE_TRIGGER_NORUN); + ret = bch2_trans_update(trans, iter, &a->k_i, BTREE_TRIGGER_norun); fsck_err: return ret; } @@ -1318,7 +1318,7 @@ static int bch2_gc_alloc_done(struct bch_fs *c) for_each_btree_key_upto_commit(trans, iter, BTREE_ID_alloc, POS(ca->dev_idx, ca->mi.first_bucket), POS(ca->dev_idx, ca->mi.nbuckets - 1), - BTREE_ITER_SLOTS|BTREE_ITER_PREFETCH, k, + BTREE_ITER_slots|BTREE_ITER_prefetch, k, NULL, NULL, BCH_TRANS_COMMIT_lazy_rw, bch2_alloc_write_key(trans, &iter, k))); if (ret) { @@ -1350,7 +1350,7 @@ static int bch2_gc_alloc_start(struct bch_fs *c) int ret = bch2_trans_run(c, for_each_btree_key(trans, iter, BTREE_ID_alloc, POS_MIN, - BTREE_ITER_PREFETCH, k, ({ + BTREE_ITER_prefetch, k, ({ struct bch_dev *ca = bch_dev_bkey_exists(c, k.k->p.inode); struct bucket *g = gc_bucket(ca, k.k->p.offset); @@ -1435,7 +1435,7 @@ static int bch2_gc_reflink_done(struct bch_fs *c) int ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_reflink, POS_MIN, - BTREE_ITER_PREFETCH, k, + BTREE_ITER_prefetch, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, bch2_gc_write_reflink_key(trans, &iter, k, &idx))); c->reflink_gc_nr = 0; @@ -1448,7 +1448,7 @@ static int bch2_gc_reflink_start(struct bch_fs *c) int ret = bch2_trans_run(c, for_each_btree_key(trans, iter, BTREE_ID_reflink, POS_MIN, - BTREE_ITER_PREFETCH, k, ({ + BTREE_ITER_prefetch, k, ({ const __le64 *refcount = bkey_refcount_c(k); if (!refcount) @@ -1538,7 +1538,7 @@ static int bch2_gc_stripes_done(struct bch_fs *c) return bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_stripes, POS_MIN, - BTREE_ITER_PREFETCH, k, + BTREE_ITER_prefetch, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, bch2_gc_write_stripes_key(trans, &iter, k))); } @@ -1762,7 +1762,7 @@ int bch2_gc_gens(struct bch_fs *c) ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, i, POS_MIN, - BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, + BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, @@ -1774,7 +1774,7 @@ int bch2_gc_gens(struct bch_fs *c) ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_alloc, POS_MIN, - BTREE_ITER_PREFETCH, + BTREE_ITER_prefetch, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index ccf2e41b8aa79..db47063e5844e 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -61,7 +61,7 @@ static inline int btree_path_cmp(const struct btree_path *l, static inline struct bpos bkey_successor(struct btree_iter *iter, struct bpos p) { /* Are we iterating over keys in all snapshots? */ - if (iter->flags & BTREE_ITER_ALL_SNAPSHOTS) { + if (iter->flags & BTREE_ITER_all_snapshots) { p = bpos_successor(p); } else { p = bpos_nosnap_successor(p); @@ -74,7 +74,7 @@ static inline struct bpos bkey_successor(struct btree_iter *iter, struct bpos p) static inline struct bpos bkey_predecessor(struct btree_iter *iter, struct bpos p) { /* Are we iterating over keys in all snapshots? */ - if (iter->flags & BTREE_ITER_ALL_SNAPSHOTS) { + if (iter->flags & BTREE_ITER_all_snapshots) { p = bpos_predecessor(p); } else { p = bpos_nosnap_predecessor(p); @@ -88,7 +88,7 @@ static inline struct bpos btree_iter_search_key(struct btree_iter *iter) { struct bpos pos = iter->pos; - if ((iter->flags & BTREE_ITER_IS_EXTENTS) && + if ((iter->flags & BTREE_ITER_is_extents) && !bkey_eq(pos, POS_MAX)) pos = bkey_successor(iter, pos); return pos; @@ -253,13 +253,13 @@ static void bch2_btree_iter_verify(struct btree_iter *iter) BUG_ON(iter->btree_id >= BTREE_ID_NR); - BUG_ON(!!(iter->flags & BTREE_ITER_CACHED) != btree_iter_path(trans, iter)->cached); + BUG_ON(!!(iter->flags & BTREE_ITER_cached) != btree_iter_path(trans, iter)->cached); - BUG_ON((iter->flags & BTREE_ITER_IS_EXTENTS) && - (iter->flags & BTREE_ITER_ALL_SNAPSHOTS)); + BUG_ON((iter->flags & BTREE_ITER_is_extents) && + (iter->flags & BTREE_ITER_all_snapshots)); - BUG_ON(!(iter->flags & BTREE_ITER_SNAPSHOT_FIELD) && - (iter->flags & BTREE_ITER_ALL_SNAPSHOTS) && + BUG_ON(!(iter->flags & BTREE_ITER_snapshot_field) && + (iter->flags & BTREE_ITER_all_snapshots) && !btree_type_has_snapshot_field(iter->btree_id)); if (iter->update_path) @@ -269,10 +269,10 @@ static void bch2_btree_iter_verify(struct btree_iter *iter) static void bch2_btree_iter_verify_entry_exit(struct btree_iter *iter) { - BUG_ON((iter->flags & BTREE_ITER_FILTER_SNAPSHOTS) && + BUG_ON((iter->flags & BTREE_ITER_filter_snapshots) && !iter->pos.snapshot); - BUG_ON(!(iter->flags & BTREE_ITER_ALL_SNAPSHOTS) && + BUG_ON(!(iter->flags & BTREE_ITER_all_snapshots) && iter->pos.snapshot != iter->snapshot); BUG_ON(bkey_lt(iter->pos, bkey_start_pos(&iter->k)) || @@ -289,7 +289,7 @@ static int bch2_btree_iter_verify_ret(struct btree_iter *iter, struct bkey_s_c k if (!bch2_debug_check_iterators) return 0; - if (!(iter->flags & BTREE_ITER_FILTER_SNAPSHOTS)) + if (!(iter->flags & BTREE_ITER_filter_snapshots)) return 0; if (bkey_err(k) || !k.k) @@ -300,8 +300,8 @@ static int bch2_btree_iter_verify_ret(struct btree_iter *iter, struct bkey_s_c k k.k->p.snapshot)); bch2_trans_iter_init(trans, ©, iter->btree_id, iter->pos, - BTREE_ITER_NOPRESERVE| - BTREE_ITER_ALL_SNAPSHOTS); + BTREE_ITER_nopreserve| + BTREE_ITER_all_snapshots); prev = bch2_btree_iter_prev(©); if (!prev.k) goto out; @@ -897,7 +897,7 @@ static noinline int btree_node_iter_and_journal_peek(struct btree_trans *trans, bch2_bkey_buf_reassemble(out, c, k); - if ((flags & BTREE_ITER_PREFETCH) && + if ((flags & BTREE_ITER_prefetch) && c->opts.btree_node_prefetch) ret = btree_path_prefetch_j(trans, path, &jiter); @@ -944,7 +944,7 @@ static __always_inline int btree_path_down(struct btree_trans *trans, bch2_bkey_buf_unpack(&tmp, c, l->b, k); - if ((flags & BTREE_ITER_PREFETCH) && + if ((flags & BTREE_ITER_prefetch) && c->opts.btree_node_prefetch) { ret = btree_path_prefetch(trans, path); if (ret) @@ -1659,8 +1659,8 @@ btree_path_idx_t bch2_path_get(struct btree_trans *trans, unsigned flags, unsigned long ip) { struct btree_path *path; - bool cached = flags & BTREE_ITER_CACHED; - bool intent = flags & BTREE_ITER_INTENT; + bool cached = flags & BTREE_ITER_cached; + bool intent = flags & BTREE_ITER_intent; struct trans_for_each_path_inorder_iter iter; btree_path_idx_t path_pos = 0, path_idx; @@ -1708,7 +1708,7 @@ btree_path_idx_t bch2_path_get(struct btree_trans *trans, trans->paths_sorted = false; } - if (!(flags & BTREE_ITER_NOPRESERVE)) + if (!(flags & BTREE_ITER_nopreserve)) path->preserve = true; if (path->intent_ref) @@ -1786,7 +1786,7 @@ bch2_btree_iter_traverse(struct btree_iter *iter) iter->path = bch2_btree_path_set_pos(trans, iter->path, btree_iter_search_key(iter), - iter->flags & BTREE_ITER_INTENT, + iter->flags & BTREE_ITER_intent, btree_iter_ip_allocated(iter)); ret = bch2_btree_path_traverse(iter->trans, iter->path, iter->flags); @@ -1825,7 +1825,7 @@ struct btree *bch2_btree_iter_peek_node(struct btree_iter *iter) iter->k.p = iter->pos = b->key.k.p; iter->path = bch2_btree_path_set_pos(trans, iter->path, b->key.k.p, - iter->flags & BTREE_ITER_INTENT, + iter->flags & BTREE_ITER_intent, btree_iter_ip_allocated(iter)); btree_path_set_should_be_locked(btree_iter_path(trans, iter)); out: @@ -1892,7 +1892,7 @@ struct btree *bch2_btree_iter_next_node(struct btree_iter *iter) */ iter->path = bch2_btree_path_set_pos(trans, iter->path, bpos_successor(iter->pos), - iter->flags & BTREE_ITER_INTENT, + iter->flags & BTREE_ITER_intent, btree_iter_ip_allocated(iter)); path = btree_iter_path(trans, iter); @@ -1910,7 +1910,7 @@ struct btree *bch2_btree_iter_next_node(struct btree_iter *iter) iter->k.p = iter->pos = b->key.k.p; iter->path = bch2_btree_path_set_pos(trans, iter->path, b->key.k.p, - iter->flags & BTREE_ITER_INTENT, + iter->flags & BTREE_ITER_intent, btree_iter_ip_allocated(iter)); btree_path_set_should_be_locked(btree_iter_path(trans, iter)); EBUG_ON(btree_iter_path(trans, iter)->uptodate); @@ -1929,11 +1929,11 @@ struct btree *bch2_btree_iter_next_node(struct btree_iter *iter) inline bool bch2_btree_iter_advance(struct btree_iter *iter) { struct bpos pos = iter->k.p; - bool ret = !(iter->flags & BTREE_ITER_ALL_SNAPSHOTS + bool ret = !(iter->flags & BTREE_ITER_all_snapshots ? bpos_eq(pos, SPOS_MAX) : bkey_eq(pos, SPOS_MAX)); - if (ret && !(iter->flags & BTREE_ITER_IS_EXTENTS)) + if (ret && !(iter->flags & BTREE_ITER_is_extents)) pos = bkey_successor(iter, pos); bch2_btree_iter_set_pos(iter, pos); return ret; @@ -1942,11 +1942,11 @@ inline bool bch2_btree_iter_advance(struct btree_iter *iter) inline bool bch2_btree_iter_rewind(struct btree_iter *iter) { struct bpos pos = bkey_start_pos(&iter->k); - bool ret = !(iter->flags & BTREE_ITER_ALL_SNAPSHOTS + bool ret = !(iter->flags & BTREE_ITER_all_snapshots ? bpos_eq(pos, POS_MIN) : bkey_eq(pos, POS_MIN)); - if (ret && !(iter->flags & BTREE_ITER_IS_EXTENTS)) + if (ret && !(iter->flags & BTREE_ITER_is_extents)) pos = bkey_predecessor(iter, pos); bch2_btree_iter_set_pos(iter, pos); return ret; @@ -2057,7 +2057,7 @@ struct bkey_s_c btree_trans_peek_key_cache(struct btree_iter *iter, struct bpos struct bkey_s_c k; int ret; - if ((iter->flags & BTREE_ITER_KEY_CACHE_FILL) && + if ((iter->flags & BTREE_ITER_key_cache_fill) && bpos_eq(iter->pos, pos)) return bkey_s_c_null; @@ -2066,17 +2066,17 @@ struct bkey_s_c btree_trans_peek_key_cache(struct btree_iter *iter, struct bpos if (!iter->key_cache_path) iter->key_cache_path = bch2_path_get(trans, iter->btree_id, pos, - iter->flags & BTREE_ITER_INTENT, 0, - iter->flags|BTREE_ITER_CACHED| - BTREE_ITER_CACHED_NOFILL, + iter->flags & BTREE_ITER_intent, 0, + iter->flags|BTREE_ITER_cached| + BTREE_ITER_cached_nofill, _THIS_IP_); iter->key_cache_path = bch2_btree_path_set_pos(trans, iter->key_cache_path, pos, - iter->flags & BTREE_ITER_INTENT, + iter->flags & BTREE_ITER_intent, btree_iter_ip_allocated(iter)); ret = bch2_btree_path_traverse(trans, iter->key_cache_path, - iter->flags|BTREE_ITER_CACHED) ?: + iter->flags|BTREE_ITER_cached) ?: bch2_btree_path_relock(trans, btree_iter_path(trans, iter), _THIS_IP_); if (unlikely(ret)) return bkey_s_c_err(ret); @@ -2104,7 +2104,7 @@ static struct bkey_s_c __bch2_btree_iter_peek(struct btree_iter *iter, struct bp struct btree_path_level *l; iter->path = bch2_btree_path_set_pos(trans, iter->path, search_key, - iter->flags & BTREE_ITER_INTENT, + iter->flags & BTREE_ITER_intent, btree_iter_ip_allocated(iter)); ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); @@ -2129,7 +2129,7 @@ static struct bkey_s_c __bch2_btree_iter_peek(struct btree_iter *iter, struct bp k = btree_path_level_peek_all(trans->c, l, &iter->k); - if (unlikely(iter->flags & BTREE_ITER_WITH_KEY_CACHE) && + if (unlikely(iter->flags & BTREE_ITER_with_key_cache) && k.k && (k2 = btree_trans_peek_key_cache(iter, k.k->p)).k) { k = k2; @@ -2140,10 +2140,10 @@ static struct bkey_s_c __bch2_btree_iter_peek(struct btree_iter *iter, struct bp } } - if (unlikely(iter->flags & BTREE_ITER_WITH_JOURNAL)) + if (unlikely(iter->flags & BTREE_ITER_with_journal)) k = btree_trans_peek_journal(trans, iter, k); - if (unlikely((iter->flags & BTREE_ITER_WITH_UPDATES) && + if (unlikely((iter->flags & BTREE_ITER_with_updates) && trans->nr_updates)) bch2_btree_trans_peek_updates(trans, iter, &k); @@ -2195,11 +2195,11 @@ struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *iter, struct bpos e struct bpos iter_pos; int ret; - EBUG_ON((iter->flags & BTREE_ITER_FILTER_SNAPSHOTS) && bkey_eq(end, POS_MAX)); + EBUG_ON((iter->flags & BTREE_ITER_filter_snapshots) && bkey_eq(end, POS_MAX)); if (iter->update_path) { bch2_path_put_nokeep(trans, iter->update_path, - iter->flags & BTREE_ITER_INTENT); + iter->flags & BTREE_ITER_intent); iter->update_path = 0; } @@ -2222,7 +2222,7 @@ struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *iter, struct bpos e * isn't monotonically increasing before FILTER_SNAPSHOTS, and * that's what we check against in extents mode: */ - if (unlikely(!(iter->flags & BTREE_ITER_IS_EXTENTS) + if (unlikely(!(iter->flags & BTREE_ITER_is_extents) ? bkey_gt(k.k->p, end) : k.k->p.inode > end.inode)) goto end; @@ -2230,13 +2230,13 @@ struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *iter, struct bpos e if (iter->update_path && !bkey_eq(trans->paths[iter->update_path].pos, k.k->p)) { bch2_path_put_nokeep(trans, iter->update_path, - iter->flags & BTREE_ITER_INTENT); + iter->flags & BTREE_ITER_intent); iter->update_path = 0; } - if ((iter->flags & BTREE_ITER_FILTER_SNAPSHOTS) && - (iter->flags & BTREE_ITER_INTENT) && - !(iter->flags & BTREE_ITER_IS_EXTENTS) && + if ((iter->flags & BTREE_ITER_filter_snapshots) && + (iter->flags & BTREE_ITER_intent) && + !(iter->flags & BTREE_ITER_is_extents) && !iter->update_path) { struct bpos pos = k.k->p; @@ -2251,12 +2251,12 @@ struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *iter, struct bpos e * advance, same as on exit for iter->path, but only up * to snapshot */ - __btree_path_get(trans->paths + iter->path, iter->flags & BTREE_ITER_INTENT); + __btree_path_get(trans->paths + iter->path, iter->flags & BTREE_ITER_intent); iter->update_path = iter->path; iter->update_path = bch2_btree_path_set_pos(trans, iter->update_path, pos, - iter->flags & BTREE_ITER_INTENT, + iter->flags & BTREE_ITER_intent, _THIS_IP_); ret = bch2_btree_path_traverse(trans, iter->update_path, iter->flags); if (unlikely(ret)) { @@ -2269,7 +2269,7 @@ struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *iter, struct bpos e * We can never have a key in a leaf node at POS_MAX, so * we don't have to check these successor() calls: */ - if ((iter->flags & BTREE_ITER_FILTER_SNAPSHOTS) && + if ((iter->flags & BTREE_ITER_filter_snapshots) && !bch2_snapshot_is_ancestor(trans->c, iter->snapshot, k.k->p.snapshot)) { @@ -2278,7 +2278,7 @@ struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *iter, struct bpos e } if (bkey_whiteout(k.k) && - !(iter->flags & BTREE_ITER_ALL_SNAPSHOTS)) { + !(iter->flags & BTREE_ITER_all_snapshots)) { search_key = bkey_successor(iter, k.k->p); continue; } @@ -2288,12 +2288,12 @@ struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *iter, struct bpos e * equal to the key we just returned - except extents can * straddle iter->pos: */ - if (!(iter->flags & BTREE_ITER_IS_EXTENTS)) + if (!(iter->flags & BTREE_ITER_is_extents)) iter_pos = k.k->p; else iter_pos = bkey_max(iter->pos, bkey_start_pos(k.k)); - if (unlikely(!(iter->flags & BTREE_ITER_IS_EXTENTS) + if (unlikely(!(iter->flags & BTREE_ITER_is_extents) ? bkey_gt(iter_pos, end) : bkey_ge(iter_pos, end))) goto end; @@ -2304,7 +2304,7 @@ struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *iter, struct bpos e iter->pos = iter_pos; iter->path = bch2_btree_path_set_pos(trans, iter->path, k.k->p, - iter->flags & BTREE_ITER_INTENT, + iter->flags & BTREE_ITER_intent, btree_iter_ip_allocated(iter)); btree_path_set_should_be_locked(btree_iter_path(trans, iter)); @@ -2317,7 +2317,7 @@ struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *iter, struct bpos e btree_path_set_should_be_locked(trans->paths + iter->update_path); } - if (!(iter->flags & BTREE_ITER_ALL_SNAPSHOTS)) + if (!(iter->flags & BTREE_ITER_all_snapshots)) iter->pos.snapshot = iter->snapshot; ret = bch2_btree_iter_verify_ret(iter, k); @@ -2370,18 +2370,18 @@ struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_iter *iter) EBUG_ON(btree_iter_path(trans, iter)->cached || btree_iter_path(trans, iter)->level); - if (iter->flags & BTREE_ITER_WITH_JOURNAL) + if (iter->flags & BTREE_ITER_with_journal) return bkey_s_c_err(-BCH_ERR_btree_iter_with_journal_not_supported); bch2_btree_iter_verify(iter); bch2_btree_iter_verify_entry_exit(iter); - if (iter->flags & BTREE_ITER_FILTER_SNAPSHOTS) + if (iter->flags & BTREE_ITER_filter_snapshots) search_key.snapshot = U32_MAX; while (1) { iter->path = bch2_btree_path_set_pos(trans, iter->path, search_key, - iter->flags & BTREE_ITER_INTENT, + iter->flags & BTREE_ITER_intent, btree_iter_ip_allocated(iter)); ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); @@ -2396,17 +2396,17 @@ struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_iter *iter) k = btree_path_level_peek(trans, path, &path->l[0], &iter->k); if (!k.k || - ((iter->flags & BTREE_ITER_IS_EXTENTS) + ((iter->flags & BTREE_ITER_is_extents) ? bpos_ge(bkey_start_pos(k.k), search_key) : bpos_gt(k.k->p, search_key))) k = btree_path_level_prev(trans, path, &path->l[0], &iter->k); - if (unlikely((iter->flags & BTREE_ITER_WITH_UPDATES) && + if (unlikely((iter->flags & BTREE_ITER_with_updates) && trans->nr_updates)) bch2_btree_trans_peek_prev_updates(trans, iter, &k); if (likely(k.k)) { - if (iter->flags & BTREE_ITER_FILTER_SNAPSHOTS) { + if (iter->flags & BTREE_ITER_filter_snapshots) { if (k.k->p.snapshot == iter->snapshot) goto got_key; @@ -2417,7 +2417,7 @@ struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_iter *iter) */ if (saved_path && !bkey_eq(k.k->p, saved_k.p)) { bch2_path_put_nokeep(trans, iter->path, - iter->flags & BTREE_ITER_INTENT); + iter->flags & BTREE_ITER_intent); iter->path = saved_path; saved_path = 0; iter->k = saved_k; @@ -2430,9 +2430,9 @@ struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_iter *iter) k.k->p.snapshot)) { if (saved_path) bch2_path_put_nokeep(trans, saved_path, - iter->flags & BTREE_ITER_INTENT); + iter->flags & BTREE_ITER_intent); saved_path = btree_path_clone(trans, iter->path, - iter->flags & BTREE_ITER_INTENT); + iter->flags & BTREE_ITER_intent); path = btree_iter_path(trans, iter); saved_k = *k.k; saved_v = k.v; @@ -2443,9 +2443,9 @@ struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_iter *iter) } got_key: if (bkey_whiteout(k.k) && - !(iter->flags & BTREE_ITER_ALL_SNAPSHOTS)) { + !(iter->flags & BTREE_ITER_all_snapshots)) { search_key = bkey_predecessor(iter, k.k->p); - if (iter->flags & BTREE_ITER_FILTER_SNAPSHOTS) + if (iter->flags & BTREE_ITER_filter_snapshots) search_key.snapshot = U32_MAX; continue; } @@ -2469,11 +2469,11 @@ struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_iter *iter) if (bkey_lt(k.k->p, iter->pos)) iter->pos = k.k->p; - if (iter->flags & BTREE_ITER_FILTER_SNAPSHOTS) + if (iter->flags & BTREE_ITER_filter_snapshots) iter->pos.snapshot = iter->snapshot; out_no_locked: if (saved_path) - bch2_path_put_nokeep(trans, saved_path, iter->flags & BTREE_ITER_INTENT); + bch2_path_put_nokeep(trans, saved_path, iter->flags & BTREE_ITER_intent); bch2_btree_iter_verify_entry_exit(iter); bch2_btree_iter_verify(iter); @@ -2505,10 +2505,10 @@ struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_iter *iter) bch2_btree_iter_verify(iter); bch2_btree_iter_verify_entry_exit(iter); - EBUG_ON(btree_iter_path(trans, iter)->level && (iter->flags & BTREE_ITER_WITH_KEY_CACHE)); + EBUG_ON(btree_iter_path(trans, iter)->level && (iter->flags & BTREE_ITER_with_key_cache)); /* extents can't span inode numbers: */ - if ((iter->flags & BTREE_ITER_IS_EXTENTS) && + if ((iter->flags & BTREE_ITER_is_extents) && unlikely(iter->pos.offset == KEY_OFFSET_MAX)) { if (iter->pos.inode == KEY_INODE_MAX) return bkey_s_c_null; @@ -2518,7 +2518,7 @@ struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_iter *iter) search_key = btree_iter_search_key(iter); iter->path = bch2_btree_path_set_pos(trans, iter->path, search_key, - iter->flags & BTREE_ITER_INTENT, + iter->flags & BTREE_ITER_intent, btree_iter_ip_allocated(iter)); ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); @@ -2527,22 +2527,22 @@ struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_iter *iter) goto out_no_locked; } - if ((iter->flags & BTREE_ITER_CACHED) || - !(iter->flags & (BTREE_ITER_IS_EXTENTS|BTREE_ITER_FILTER_SNAPSHOTS))) { + if ((iter->flags & BTREE_ITER_cached) || + !(iter->flags & (BTREE_ITER_is_extents|BTREE_ITER_filter_snapshots))) { k = bkey_s_c_null; - if (unlikely((iter->flags & BTREE_ITER_WITH_UPDATES) && + if (unlikely((iter->flags & BTREE_ITER_with_updates) && trans->nr_updates)) { bch2_btree_trans_peek_slot_updates(trans, iter, &k); if (k.k) goto out; } - if (unlikely(iter->flags & BTREE_ITER_WITH_JOURNAL) && + if (unlikely(iter->flags & BTREE_ITER_with_journal) && (k = btree_trans_peek_slot_journal(trans, iter)).k) goto out; - if (unlikely(iter->flags & BTREE_ITER_WITH_KEY_CACHE) && + if (unlikely(iter->flags & BTREE_ITER_with_key_cache) && (k = btree_trans_peek_key_cache(iter, iter->pos)).k) { if (!bkey_err(k)) iter->k = *k.k; @@ -2557,12 +2557,12 @@ struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_iter *iter) struct bpos next; struct bpos end = iter->pos; - if (iter->flags & BTREE_ITER_IS_EXTENTS) + if (iter->flags & BTREE_ITER_is_extents) end.offset = U64_MAX; EBUG_ON(btree_iter_path(trans, iter)->level); - if (iter->flags & BTREE_ITER_INTENT) { + if (iter->flags & BTREE_ITER_intent) { struct btree_iter iter2; bch2_trans_copy_iter(&iter2, iter); @@ -2593,7 +2593,7 @@ struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_iter *iter) bkey_init(&iter->k); iter->k.p = iter->pos; - if (iter->flags & BTREE_ITER_IS_EXTENTS) { + if (iter->flags & BTREE_ITER_is_extents) { bch2_key_resize(&iter->k, min_t(u64, KEY_SIZE_MAX, (next.inode == iter->pos.inode @@ -2777,13 +2777,13 @@ void bch2_trans_iter_exit(struct btree_trans *trans, struct btree_iter *iter) { if (iter->update_path) bch2_path_put_nokeep(trans, iter->update_path, - iter->flags & BTREE_ITER_INTENT); + iter->flags & BTREE_ITER_intent); if (iter->path) bch2_path_put(trans, iter->path, - iter->flags & BTREE_ITER_INTENT); + iter->flags & BTREE_ITER_intent); if (iter->key_cache_path) bch2_path_put(trans, iter->key_cache_path, - iter->flags & BTREE_ITER_INTENT); + iter->flags & BTREE_ITER_intent); iter->path = 0; iter->update_path = 0; iter->key_cache_path = 0; @@ -2808,9 +2808,9 @@ void bch2_trans_node_iter_init(struct btree_trans *trans, unsigned depth, unsigned flags) { - flags |= BTREE_ITER_NOT_EXTENTS; - flags |= BTREE_ITER_SNAPSHOT_FIELD; - flags |= BTREE_ITER_ALL_SNAPSHOTS; + flags |= BTREE_ITER_not_extents; + flags |= BTREE_ITER_snapshot_field; + flags |= BTREE_ITER_all_snapshots; bch2_trans_iter_init_common(trans, iter, btree_id, pos, locks_want, depth, __bch2_btree_iter_flags(trans, btree_id, flags), @@ -2833,9 +2833,9 @@ void bch2_trans_copy_iter(struct btree_iter *dst, struct btree_iter *src) dst->ip_allocated = _RET_IP_; #endif if (src->path) - __btree_path_get(trans->paths + src->path, src->flags & BTREE_ITER_INTENT); + __btree_path_get(trans->paths + src->path, src->flags & BTREE_ITER_intent); if (src->update_path) - __btree_path_get(trans->paths + src->update_path, src->flags & BTREE_ITER_INTENT); + __btree_path_get(trans->paths + src->update_path, src->flags & BTREE_ITER_intent); dst->key_cache_path = 0; } diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index 57323d5e94a99..abbf7bc8d9c53 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -386,10 +386,10 @@ static inline void bch2_btree_iter_set_pos(struct btree_iter *iter, struct bpos if (unlikely(iter->update_path)) bch2_path_put(trans, iter->update_path, - iter->flags & BTREE_ITER_INTENT); + iter->flags & BTREE_ITER_intent); iter->update_path = 0; - if (!(iter->flags & BTREE_ITER_ALL_SNAPSHOTS)) + if (!(iter->flags & BTREE_ITER_all_snapshots)) new_pos.snapshot = iter->snapshot; __bch2_btree_iter_set_pos(iter, new_pos); @@ -397,7 +397,7 @@ static inline void bch2_btree_iter_set_pos(struct btree_iter *iter, struct bpos static inline void bch2_btree_iter_set_pos_to_extent_start(struct btree_iter *iter) { - BUG_ON(!(iter->flags & BTREE_ITER_IS_EXTENTS)); + BUG_ON(!(iter->flags & BTREE_ITER_is_extents)); iter->pos = bkey_start_pos(&iter->k); } @@ -416,20 +416,20 @@ static inline unsigned __bch2_btree_iter_flags(struct btree_trans *trans, unsigned btree_id, unsigned flags) { - if (!(flags & (BTREE_ITER_ALL_SNAPSHOTS|BTREE_ITER_NOT_EXTENTS)) && + if (!(flags & (BTREE_ITER_all_snapshots|BTREE_ITER_not_extents)) && btree_id_is_extents(btree_id)) - flags |= BTREE_ITER_IS_EXTENTS; + flags |= BTREE_ITER_is_extents; - if (!(flags & BTREE_ITER_SNAPSHOT_FIELD) && + if (!(flags & BTREE_ITER_snapshot_field) && !btree_type_has_snapshot_field(btree_id)) - flags &= ~BTREE_ITER_ALL_SNAPSHOTS; + flags &= ~BTREE_ITER_all_snapshots; - if (!(flags & BTREE_ITER_ALL_SNAPSHOTS) && + if (!(flags & BTREE_ITER_all_snapshots) && btree_type_has_snapshots(btree_id)) - flags |= BTREE_ITER_FILTER_SNAPSHOTS; + flags |= BTREE_ITER_filter_snapshots; if (trans->journal_replay_not_finished) - flags |= BTREE_ITER_WITH_JOURNAL; + flags |= BTREE_ITER_with_journal; return flags; } @@ -439,10 +439,10 @@ static inline unsigned bch2_btree_iter_flags(struct btree_trans *trans, unsigned flags) { if (!btree_id_cached(trans->c, btree_id)) { - flags &= ~BTREE_ITER_CACHED; - flags &= ~BTREE_ITER_WITH_KEY_CACHE; - } else if (!(flags & BTREE_ITER_CACHED)) - flags |= BTREE_ITER_WITH_KEY_CACHE; + flags &= ~BTREE_ITER_cached; + flags &= ~BTREE_ITER_with_key_cache; + } else if (!(flags & BTREE_ITER_cached)) + flags |= BTREE_ITER_with_key_cache; return __bch2_btree_iter_flags(trans, btree_id, flags); } @@ -619,14 +619,14 @@ u32 bch2_trans_begin(struct btree_trans *); static inline struct bkey_s_c bch2_btree_iter_peek_prev_type(struct btree_iter *iter, unsigned flags) { - return flags & BTREE_ITER_SLOTS ? bch2_btree_iter_peek_slot(iter) : + return flags & BTREE_ITER_slots ? bch2_btree_iter_peek_slot(iter) : bch2_btree_iter_peek_prev(iter); } static inline struct bkey_s_c bch2_btree_iter_peek_type(struct btree_iter *iter, unsigned flags) { - return flags & BTREE_ITER_SLOTS ? bch2_btree_iter_peek_slot(iter) : + return flags & BTREE_ITER_slots ? bch2_btree_iter_peek_slot(iter) : bch2_btree_iter_peek(iter); } @@ -634,7 +634,7 @@ static inline struct bkey_s_c bch2_btree_iter_peek_upto_type(struct btree_iter * struct bpos end, unsigned flags) { - if (!(flags & BTREE_ITER_SLOTS)) + if (!(flags & BTREE_ITER_slots)) return bch2_btree_iter_peek_upto(iter, end); if (bkey_gt(iter->pos, end)) diff --git a/fs/bcachefs/btree_key_cache.c b/fs/bcachefs/btree_key_cache.c index 3dea89c6cf7e5..547ba9329678e 100644 --- a/fs/bcachefs/btree_key_cache.c +++ b/fs/bcachefs/btree_key_cache.c @@ -383,9 +383,9 @@ static int btree_key_cache_fill(struct btree_trans *trans, int ret; bch2_trans_iter_init(trans, &iter, ck->key.btree_id, ck->key.pos, - BTREE_ITER_KEY_CACHE_FILL| - BTREE_ITER_CACHED_NOFILL); - iter.flags &= ~BTREE_ITER_WITH_JOURNAL; + BTREE_ITER_key_cache_fill| + BTREE_ITER_cached_nofill); + iter.flags &= ~BTREE_ITER_with_journal; k = bch2_btree_iter_peek_slot(&iter); ret = bkey_err(k); if (ret) @@ -515,7 +515,7 @@ bch2_btree_path_traverse_cached_slowpath(struct btree_trans *trans, struct btree fill: path->uptodate = BTREE_ITER_UPTODATE; - if (!ck->valid && !(flags & BTREE_ITER_CACHED_NOFILL)) { + if (!ck->valid && !(flags & BTREE_ITER_cached_nofill)) { /* * Using the underscore version because we haven't set * path->uptodate yet: @@ -622,13 +622,13 @@ static int btree_key_cache_flush_pos(struct btree_trans *trans, int ret; bch2_trans_iter_init(trans, &b_iter, key.btree_id, key.pos, - BTREE_ITER_SLOTS| - BTREE_ITER_INTENT| - BTREE_ITER_ALL_SNAPSHOTS); + BTREE_ITER_slots| + BTREE_ITER_intent| + BTREE_ITER_all_snapshots); bch2_trans_iter_init(trans, &c_iter, key.btree_id, key.pos, - BTREE_ITER_CACHED| - BTREE_ITER_INTENT); - b_iter.flags &= ~BTREE_ITER_WITH_KEY_CACHE; + BTREE_ITER_cached| + BTREE_ITER_intent); + b_iter.flags &= ~BTREE_ITER_with_key_cache; ret = bch2_btree_iter_traverse(&c_iter); if (ret) @@ -666,9 +666,9 @@ static int btree_key_cache_flush_pos(struct btree_trans *trans, ret = bch2_btree_iter_traverse(&b_iter) ?: bch2_trans_update(trans, &b_iter, ck->k, - BTREE_UPDATE_KEY_CACHE_RECLAIM| - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE| - BTREE_TRIGGER_NORUN) ?: + BTREE_UPDATE_key_cache_reclaim| + BTREE_UPDATE_internal_snapshot_node| + BTREE_TRIGGER_norun) ?: bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_check_rw| BCH_TRANS_COMMIT_no_enospc| @@ -790,7 +790,7 @@ bool bch2_btree_insert_key_cached(struct btree_trans *trans, * flushing. The flush callback will not proceed unless ->seq matches * the latest pin, so make sure it starts with a consistent value. */ - if (!(insert_entry->flags & BTREE_UPDATE_NOJOURNAL) || + if (!(insert_entry->flags & BTREE_UPDATE_nojournal) || !journal_pin_active(&ck->journal)) { ck->seq = trans->journal_res.seq; } diff --git a/fs/bcachefs/btree_trans_commit.c b/fs/bcachefs/btree_trans_commit.c index 79cbf54c89128..c5317e74ee069 100644 --- a/fs/bcachefs/btree_trans_commit.c +++ b/fs/bcachefs/btree_trans_commit.c @@ -315,7 +315,7 @@ static inline void btree_insert_entry_checks(struct btree_trans *trans, BUG_ON(i->btree_id != path->btree_id); EBUG_ON(!i->level && btree_type_has_snapshots(i->btree_id) && - !(i->flags & BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) && + !(i->flags & BTREE_UPDATE_internal_snapshot_node) && test_bit(JOURNAL_REPLAY_DONE, &trans->c->journal.flags) && i->k->k.p.snapshot && bch2_snapshot_is_internal_node(trans->c, i->k->k.p.snapshot) > 0); @@ -443,13 +443,13 @@ static int run_one_mem_trigger(struct btree_trans *trans, verify_update_old_key(trans, i); - if (unlikely(flags & BTREE_TRIGGER_NORUN)) + if (unlikely(flags & BTREE_TRIGGER_norun)) return 0; if (old_ops->trigger == new_ops->trigger) { ret = bch2_key_trigger(trans, i->btree_id, i->level, old, bkey_i_to_s(new), - BTREE_TRIGGER_INSERT|BTREE_TRIGGER_OVERWRITE|flags); + BTREE_TRIGGER_insert|BTREE_TRIGGER_overwrite|flags); } else { ret = bch2_key_trigger_new(trans, i->btree_id, i->level, bkey_i_to_s(new), flags) ?: @@ -472,11 +472,11 @@ static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_ struct bkey_s_c old = { &old_k, i->old_v }; const struct bkey_ops *old_ops = bch2_bkey_type_ops(old.k->type); const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type); - unsigned flags = i->flags|BTREE_TRIGGER_TRANSACTIONAL; + unsigned flags = i->flags|BTREE_TRIGGER_transactional; verify_update_old_key(trans, i); - if ((i->flags & BTREE_TRIGGER_NORUN) || + if ((i->flags & BTREE_TRIGGER_norun) || !(BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS & (1U << i->bkey_type))) return 0; @@ -486,8 +486,8 @@ static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_ i->overwrite_trigger_run = true; i->insert_trigger_run = true; return bch2_key_trigger(trans, i->btree_id, i->level, old, bkey_i_to_s(i->k), - BTREE_TRIGGER_INSERT| - BTREE_TRIGGER_OVERWRITE|flags) ?: 1; + BTREE_TRIGGER_insert| + BTREE_TRIGGER_overwrite|flags) ?: 1; } else if (overwrite && !i->overwrite_trigger_run) { i->overwrite_trigger_run = true; return bch2_key_trigger_old(trans, i->btree_id, i->level, old, flags) ?: 1; @@ -572,7 +572,7 @@ static int bch2_trans_commit_run_triggers(struct btree_trans *trans) #ifdef CONFIG_BCACHEFS_DEBUG trans_for_each_update(trans, i) - BUG_ON(!(i->flags & BTREE_TRIGGER_NORUN) && + BUG_ON(!(i->flags & BTREE_TRIGGER_norun) && (BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS & (1U << i->bkey_type)) && (!i->insert_trigger_run || !i->overwrite_trigger_run)); #endif @@ -590,7 +590,7 @@ static noinline int bch2_trans_commit_run_gc_triggers(struct btree_trans *trans) if (btree_node_type_needs_gc(__btree_node_type(i->level, i->btree_id)) && gc_visited(trans->c, gc_pos_btree_node(insert_l(trans, i)->b))) { - int ret = run_one_mem_trigger(trans, i, i->flags|BTREE_TRIGGER_GC); + int ret = run_one_mem_trigger(trans, i, i->flags|BTREE_TRIGGER_gc); if (ret) return ret; } @@ -686,7 +686,7 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags, trans_for_each_update(trans, i) if (BTREE_NODE_TYPE_HAS_ATOMIC_TRIGGERS & (1U << i->bkey_type)) { - ret = run_one_mem_trigger(trans, i, BTREE_TRIGGER_ATOMIC|i->flags); + ret = run_one_mem_trigger(trans, i, BTREE_TRIGGER_atomic|i->flags); if (ret) goto fatal_err; } @@ -705,7 +705,7 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags, if (i->key_cache_already_flushed) continue; - if (i->flags & BTREE_UPDATE_NOJOURNAL) + if (i->flags & BTREE_UPDATE_nojournal) continue; verify_update_old_key(trans, i); @@ -1063,7 +1063,7 @@ int __bch2_trans_commit(struct btree_trans *trans, unsigned flags) if (i->key_cache_already_flushed) continue; - if (i->flags & BTREE_UPDATE_NOJOURNAL) + if (i->flags & BTREE_UPDATE_nojournal) continue; /* we're going to journal the key being updated: */ diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index d4649129e5d36..123abeec4ce9c 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -187,36 +187,87 @@ struct btree_node_iter { } data[MAX_BSETS]; }; +#define BTREE_ITER_FLAGS() \ + x(slots) \ + x(intent) \ + x(prefetch) \ + x(is_extents) \ + x(not_extents) \ + x(cached) \ + x(with_key_cache) \ + x(with_updates) \ + x(with_journal) \ + x(snapshot_field) \ + x(all_snapshots) \ + x(filter_snapshots) \ + x(nopreserve) \ + x(cached_nofill) \ + x(key_cache_fill) \ + +#define STR_HASH_FLAGS() \ + x(must_create) \ + x(must_replace) + +#define BTREE_UPDATE_FLAGS() \ + x(internal_snapshot_node) \ + x(nojournal) \ + x(key_cache_reclaim) + + /* - * Iterate over all possible positions, synthesizing deleted keys for holes: - */ -static const __maybe_unused u16 BTREE_ITER_SLOTS = 1 << 0; -/* - * Indicates that intent locks should be taken on leaf nodes, because we expect - * to be doing updates: - */ -static const __maybe_unused u16 BTREE_ITER_INTENT = 1 << 1; -/* - * Causes the btree iterator code to prefetch additional btree nodes from disk: - */ -static const __maybe_unused u16 BTREE_ITER_PREFETCH = 1 << 2; -/* - * Used in bch2_btree_iter_traverse(), to indicate whether we're searching for - * @pos or the first key strictly greater than @pos + * BTREE_TRIGGER_norun - don't run triggers at all + * + * BTREE_TRIGGER_transactional - we're running transactional triggers as part of + * a transaction commit: triggers may generate new updates + * + * BTREE_TRIGGER_atomic - we're running atomic triggers during a transaction + * commit: we have our journal reservation, we're holding btree node write + * locks, and we know the transaction is going to commit (returning an error + * here is a fatal error, causing us to go emergency read-only) + * + * BTREE_TRIGGER_gc - we're in gc/fsck: running triggers to recalculate e.g. disk usage + * + * BTREE_TRIGGER_insert - @new is entering the btree + * BTREE_TRIGGER_overwrite - @old is leaving the btree + * + * BTREE_TRIGGER_bucket_invalidate - signal from bucket invalidate path to alloc + * trigger */ -static const __maybe_unused u16 BTREE_ITER_IS_EXTENTS = 1 << 3; -static const __maybe_unused u16 BTREE_ITER_NOT_EXTENTS = 1 << 4; -static const __maybe_unused u16 BTREE_ITER_CACHED = 1 << 5; -static const __maybe_unused u16 BTREE_ITER_WITH_KEY_CACHE = 1 << 6; -static const __maybe_unused u16 BTREE_ITER_WITH_UPDATES = 1 << 7; -static const __maybe_unused u16 BTREE_ITER_WITH_JOURNAL = 1 << 8; -static const __maybe_unused u16 BTREE_ITER_SNAPSHOT_FIELD = 1 << 9; -static const __maybe_unused u16 BTREE_ITER_ALL_SNAPSHOTS = 1 << 10; -static const __maybe_unused u16 BTREE_ITER_FILTER_SNAPSHOTS = 1 << 11; -static const __maybe_unused u16 BTREE_ITER_NOPRESERVE = 1 << 12; -static const __maybe_unused u16 BTREE_ITER_CACHED_NOFILL = 1 << 13; -static const __maybe_unused u16 BTREE_ITER_KEY_CACHE_FILL = 1 << 14; -#define __BTREE_ITER_FLAGS_END 15 +#define BTREE_TRIGGER_FLAGS() \ + x(norun) \ + x(transactional) \ + x(atomic) \ + x(gc) \ + x(insert) \ + x(overwrite) \ + x(bucket_invalidate) + +enum { +#define x(n) BTREE_ITER_FLAG_BIT_##n, + BTREE_ITER_FLAGS() + STR_HASH_FLAGS() + BTREE_UPDATE_FLAGS() + BTREE_TRIGGER_FLAGS() +#undef x +}; + +/* iter flags must fit in a u16: */ +//BUILD_BUG_ON(BTREE_ITER_FLAG_BIT_key_cache_fill > 15); + +enum btree_iter_update_trigger_flags { +#define x(n) BTREE_ITER_##n = 1U << BTREE_ITER_FLAG_BIT_##n, + BTREE_ITER_FLAGS() +#undef x +#define x(n) STR_HASH_##n = 1U << BTREE_ITER_FLAG_BIT_##n, + STR_HASH_FLAGS() +#undef x +#define x(n) BTREE_UPDATE_##n = 1U << BTREE_ITER_FLAG_BIT_##n, + BTREE_UPDATE_FLAGS() +#undef x +#define x(n) BTREE_TRIGGER_##n = 1U << BTREE_ITER_FLAG_BIT_##n, + BTREE_TRIGGER_FLAGS() +#undef x +}; enum btree_path_uptodate { BTREE_ITER_UPTODATE = 0, @@ -307,7 +358,7 @@ struct btree_iter { */ struct bkey k; - /* BTREE_ITER_WITH_JOURNAL: */ + /* BTREE_ITER_with_journal: */ size_t journal_idx; #ifdef TRACK_PATH_ALLOCATED unsigned long ip_allocated; diff --git a/fs/bcachefs/btree_update.c b/fs/bcachefs/btree_update.c index c46ee4c813ae2..da4b17704296d 100644 --- a/fs/bcachefs/btree_update.c +++ b/fs/bcachefs/btree_update.c @@ -25,14 +25,14 @@ static inline int btree_insert_entry_cmp(const struct btree_insert_entry *l, static int __must_check bch2_trans_update_by_path(struct btree_trans *, btree_path_idx_t, - struct bkey_i *, enum btree_update_flags, + struct bkey_i *, enum btree_iter_update_trigger_flags, unsigned long ip); static noinline int extent_front_merge(struct btree_trans *trans, struct btree_iter *iter, struct bkey_s_c k, struct bkey_i **insert, - enum btree_update_flags flags) + enum btree_iter_update_trigger_flags flags) { struct bch_fs *c = trans->c; struct bkey_i *update; @@ -104,8 +104,8 @@ static int need_whiteout_for_snapshot(struct btree_trans *trans, pos.snapshot++; for_each_btree_key_norestart(trans, iter, btree_id, pos, - BTREE_ITER_ALL_SNAPSHOTS| - BTREE_ITER_NOPRESERVE, k, ret) { + BTREE_ITER_all_snapshots| + BTREE_ITER_nopreserve, k, ret) { if (!bkey_eq(k.k->p, pos)) break; @@ -138,8 +138,8 @@ int __bch2_insert_snapshot_whiteouts(struct btree_trans *trans, darray_init(&s); bch2_trans_iter_init(trans, &old_iter, id, old_pos, - BTREE_ITER_NOT_EXTENTS| - BTREE_ITER_ALL_SNAPSHOTS); + BTREE_ITER_not_extents| + BTREE_ITER_all_snapshots); while ((old_k = bch2_btree_iter_prev(&old_iter)).k && !(ret = bkey_err(old_k)) && bkey_eq(old_pos, old_k.k->p)) { @@ -151,8 +151,8 @@ int __bch2_insert_snapshot_whiteouts(struct btree_trans *trans, continue; new_k = bch2_bkey_get_iter(trans, &new_iter, id, whiteout_pos, - BTREE_ITER_NOT_EXTENTS| - BTREE_ITER_INTENT); + BTREE_ITER_not_extents| + BTREE_ITER_intent); ret = bkey_err(new_k); if (ret) break; @@ -168,7 +168,7 @@ int __bch2_insert_snapshot_whiteouts(struct btree_trans *trans, update->k.type = KEY_TYPE_whiteout; ret = bch2_trans_update(trans, &new_iter, update, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + BTREE_UPDATE_internal_snapshot_node); } bch2_trans_iter_exit(trans, &new_iter); @@ -185,7 +185,7 @@ int __bch2_insert_snapshot_whiteouts(struct btree_trans *trans, int bch2_trans_update_extent_overwrite(struct btree_trans *trans, struct btree_iter *iter, - enum btree_update_flags flags, + enum btree_iter_update_trigger_flags flags, struct bkey_s_c old, struct bkey_s_c new) { @@ -218,7 +218,7 @@ int bch2_trans_update_extent_overwrite(struct btree_trans *trans, ret = bch2_insert_snapshot_whiteouts(trans, btree_id, old.k->p, update->k.p) ?: bch2_btree_insert_nonextent(trans, btree_id, update, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE|flags); + BTREE_UPDATE_internal_snapshot_node|flags); if (ret) return ret; } @@ -235,7 +235,7 @@ int bch2_trans_update_extent_overwrite(struct btree_trans *trans, ret = bch2_insert_snapshot_whiteouts(trans, btree_id, old.k->p, update->k.p) ?: bch2_btree_insert_nonextent(trans, btree_id, update, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE|flags); + BTREE_UPDATE_internal_snapshot_node|flags); if (ret) return ret; } @@ -260,7 +260,7 @@ int bch2_trans_update_extent_overwrite(struct btree_trans *trans, } ret = bch2_btree_insert_nonextent(trans, btree_id, update, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE|flags); + BTREE_UPDATE_internal_snapshot_node|flags); if (ret) return ret; } @@ -273,7 +273,7 @@ int bch2_trans_update_extent_overwrite(struct btree_trans *trans, bch2_cut_front(new.k->p, update); ret = bch2_trans_update_by_path(trans, iter->path, update, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE| + BTREE_UPDATE_internal_snapshot_node| flags, _RET_IP_); if (ret) return ret; @@ -285,7 +285,7 @@ int bch2_trans_update_extent_overwrite(struct btree_trans *trans, static int bch2_trans_update_extent(struct btree_trans *trans, struct btree_iter *orig_iter, struct bkey_i *insert, - enum btree_update_flags flags) + enum btree_iter_update_trigger_flags flags) { struct btree_iter iter; struct bkey_s_c k; @@ -293,9 +293,9 @@ static int bch2_trans_update_extent(struct btree_trans *trans, int ret = 0; bch2_trans_iter_init(trans, &iter, btree_id, bkey_start_pos(&insert->k), - BTREE_ITER_INTENT| - BTREE_ITER_WITH_UPDATES| - BTREE_ITER_NOT_EXTENTS); + BTREE_ITER_intent| + BTREE_ITER_with_updates| + BTREE_ITER_not_extents); k = bch2_btree_iter_peek_upto(&iter, POS(insert->k.p.inode, U64_MAX)); if ((ret = bkey_err(k))) goto err; @@ -346,7 +346,7 @@ static int bch2_trans_update_extent(struct btree_trans *trans, static noinline int flush_new_cached_update(struct btree_trans *trans, struct btree_insert_entry *i, - enum btree_update_flags flags, + enum btree_iter_update_trigger_flags flags, unsigned long ip) { struct bkey k; @@ -354,7 +354,7 @@ static noinline int flush_new_cached_update(struct btree_trans *trans, btree_path_idx_t path_idx = bch2_path_get(trans, i->btree_id, i->old_k.p, 1, 0, - BTREE_ITER_INTENT, _THIS_IP_); + BTREE_ITER_intent, _THIS_IP_); ret = bch2_btree_path_traverse(trans, path_idx, 0); if (ret) goto out; @@ -372,7 +372,7 @@ static noinline int flush_new_cached_update(struct btree_trans *trans, goto out; i->key_cache_already_flushed = true; - i->flags |= BTREE_TRIGGER_NORUN; + i->flags |= BTREE_TRIGGER_norun; btree_path_set_should_be_locked(btree_path); ret = bch2_trans_update_by_path(trans, path_idx, i->k, flags, ip); @@ -383,7 +383,7 @@ static noinline int flush_new_cached_update(struct btree_trans *trans, static int __must_check bch2_trans_update_by_path(struct btree_trans *trans, btree_path_idx_t path_idx, - struct bkey_i *k, enum btree_update_flags flags, + struct bkey_i *k, enum btree_iter_update_trigger_flags flags, unsigned long ip) { struct bch_fs *c = trans->c; @@ -479,15 +479,15 @@ static noinline int bch2_trans_update_get_key_cache(struct btree_trans *trans, if (!iter->key_cache_path) iter->key_cache_path = bch2_path_get(trans, path->btree_id, path->pos, 1, 0, - BTREE_ITER_INTENT| - BTREE_ITER_CACHED, _THIS_IP_); + BTREE_ITER_intent| + BTREE_ITER_cached, _THIS_IP_); iter->key_cache_path = bch2_btree_path_set_pos(trans, iter->key_cache_path, path->pos, - iter->flags & BTREE_ITER_INTENT, + iter->flags & BTREE_ITER_intent, _THIS_IP_); - ret = bch2_btree_path_traverse(trans, iter->key_cache_path, BTREE_ITER_CACHED); + ret = bch2_btree_path_traverse(trans, iter->key_cache_path, BTREE_ITER_cached); if (unlikely(ret)) return ret; @@ -505,17 +505,17 @@ static noinline int bch2_trans_update_get_key_cache(struct btree_trans *trans, } int __must_check bch2_trans_update(struct btree_trans *trans, struct btree_iter *iter, - struct bkey_i *k, enum btree_update_flags flags) + struct bkey_i *k, enum btree_iter_update_trigger_flags flags) { btree_path_idx_t path_idx = iter->update_path ?: iter->path; int ret; - if (iter->flags & BTREE_ITER_IS_EXTENTS) + if (iter->flags & BTREE_ITER_is_extents) return bch2_trans_update_extent(trans, iter, k, flags); if (bkey_deleted(&k->k) && - !(flags & BTREE_UPDATE_KEY_CACHE_RECLAIM) && - (iter->flags & BTREE_ITER_FILTER_SNAPSHOTS)) { + !(flags & BTREE_UPDATE_key_cache_reclaim) && + (iter->flags & BTREE_ITER_filter_snapshots)) { ret = need_whiteout_for_snapshot(trans, iter->btree_id, k->k.p); if (unlikely(ret < 0)) return ret; @@ -528,7 +528,7 @@ int __must_check bch2_trans_update(struct btree_trans *trans, struct btree_iter * Ensure that updates to cached btrees go to the key cache: */ struct btree_path *path = trans->paths + path_idx; - if (!(flags & BTREE_UPDATE_KEY_CACHE_RECLAIM) && + if (!(flags & BTREE_UPDATE_key_cache_reclaim) && !path->cached && !path->level && btree_id_cached(trans->c, path->btree_id)) { @@ -587,7 +587,7 @@ int bch2_bkey_get_empty_slot(struct btree_trans *trans, struct btree_iter *iter, struct bkey_s_c k; int ret = 0; - bch2_trans_iter_init(trans, iter, btree, POS_MAX, BTREE_ITER_INTENT); + bch2_trans_iter_init(trans, iter, btree, POS_MAX, BTREE_ITER_intent); k = bch2_btree_iter_prev(iter); ret = bkey_err(k); if (ret) @@ -621,15 +621,15 @@ void bch2_trans_commit_hook(struct btree_trans *trans, int bch2_btree_insert_nonextent(struct btree_trans *trans, enum btree_id btree, struct bkey_i *k, - enum btree_update_flags flags) + enum btree_iter_update_trigger_flags flags) { struct btree_iter iter; int ret; bch2_trans_iter_init(trans, &iter, btree, k->k.p, - BTREE_ITER_CACHED| - BTREE_ITER_NOT_EXTENTS| - BTREE_ITER_INTENT); + BTREE_ITER_cached| + BTREE_ITER_not_extents| + BTREE_ITER_intent); ret = bch2_btree_iter_traverse(&iter) ?: bch2_trans_update(trans, &iter, k, flags); bch2_trans_iter_exit(trans, &iter); @@ -637,14 +637,14 @@ int bch2_btree_insert_nonextent(struct btree_trans *trans, } int bch2_btree_insert_trans(struct btree_trans *trans, enum btree_id id, - struct bkey_i *k, enum btree_update_flags flags) + struct bkey_i *k, enum btree_iter_update_trigger_flags flags) { struct btree_iter iter; int ret; bch2_trans_iter_init(trans, &iter, id, bkey_start_pos(&k->k), - BTREE_ITER_CACHED| - BTREE_ITER_INTENT); + BTREE_ITER_cached| + BTREE_ITER_intent); ret = bch2_btree_iter_traverse(&iter) ?: bch2_trans_update(trans, &iter, k, flags); bch2_trans_iter_exit(trans, &iter); @@ -698,8 +698,8 @@ int bch2_btree_delete(struct btree_trans *trans, int ret; bch2_trans_iter_init(trans, &iter, btree, pos, - BTREE_ITER_CACHED| - BTREE_ITER_INTENT); + BTREE_ITER_cached| + BTREE_ITER_intent); ret = bch2_btree_iter_traverse(&iter) ?: bch2_btree_delete_at(trans, &iter, update_flags); bch2_trans_iter_exit(trans, &iter); @@ -717,7 +717,7 @@ int bch2_btree_delete_range_trans(struct btree_trans *trans, enum btree_id id, struct bkey_s_c k; int ret = 0; - bch2_trans_iter_init(trans, &iter, id, start, BTREE_ITER_INTENT); + bch2_trans_iter_init(trans, &iter, id, start, BTREE_ITER_intent); while ((k = bch2_btree_iter_peek_upto(&iter, end)).k) { struct disk_reservation disk_res = bch2_disk_reservation_init(trans->c, 0); @@ -745,7 +745,7 @@ int bch2_btree_delete_range_trans(struct btree_trans *trans, enum btree_id id, */ delete.k.p = iter.pos; - if (iter.flags & BTREE_ITER_IS_EXTENTS) + if (iter.flags & BTREE_ITER_is_extents) bch2_key_resize(&delete.k, bpos_min(end, k.k->p).offset - iter.pos.offset); @@ -804,7 +804,7 @@ int bch2_btree_bit_mod(struct btree_trans *trans, enum btree_id btree, k->k.p = pos; struct btree_iter iter; - bch2_trans_iter_init(trans, &iter, btree, pos, BTREE_ITER_INTENT); + bch2_trans_iter_init(trans, &iter, btree, pos, BTREE_ITER_intent); ret = bch2_btree_iter_traverse(&iter) ?: bch2_trans_update(trans, &iter, k, 0); diff --git a/fs/bcachefs/btree_update.h b/fs/bcachefs/btree_update.h index cc7c53e83f89d..3a04ede35475f 100644 --- a/fs/bcachefs/btree_update.h +++ b/fs/bcachefs/btree_update.h @@ -50,10 +50,10 @@ int bch2_btree_delete_at(struct btree_trans *, struct btree_iter *, unsigned); int bch2_btree_delete(struct btree_trans *, enum btree_id, struct bpos, unsigned); int bch2_btree_insert_nonextent(struct btree_trans *, enum btree_id, - struct bkey_i *, enum btree_update_flags); + struct bkey_i *, enum btree_iter_update_trigger_flags); int bch2_btree_insert_trans(struct btree_trans *, enum btree_id, struct bkey_i *, - enum btree_update_flags); + enum btree_iter_update_trigger_flags); int bch2_btree_insert(struct bch_fs *, enum btree_id, struct bkey_i *, struct disk_reservation *, int flags); @@ -94,14 +94,14 @@ static inline int bch2_insert_snapshot_whiteouts(struct btree_trans *trans, } int bch2_trans_update_extent_overwrite(struct btree_trans *, struct btree_iter *, - enum btree_update_flags, + enum btree_iter_update_trigger_flags, struct bkey_s_c, struct bkey_s_c); int bch2_bkey_get_empty_slot(struct btree_trans *, struct btree_iter *, enum btree_id, struct bpos); int __must_check bch2_trans_update(struct btree_trans *, struct btree_iter *, - struct bkey_i *, enum btree_update_flags); + struct bkey_i *, enum btree_iter_update_trigger_flags); struct jset_entry *__bch2_trans_jset_entry_alloc(struct btree_trans *, unsigned); @@ -276,7 +276,7 @@ static inline struct bkey_i *__bch2_bkey_get_mut_noupdate(struct btree_trans *tr unsigned flags, unsigned type, unsigned min_bytes) { struct bkey_s_c k = __bch2_bkey_get_iter(trans, iter, - btree_id, pos, flags|BTREE_ITER_INTENT, type); + btree_id, pos, flags|BTREE_ITER_intent, type); struct bkey_i *ret = IS_ERR(k.k) ? ERR_CAST(k.k) : __bch2_bkey_make_mut_noupdate(trans, k, 0, min_bytes); @@ -299,7 +299,7 @@ static inline struct bkey_i *__bch2_bkey_get_mut(struct btree_trans *trans, unsigned flags, unsigned type, unsigned min_bytes) { struct bkey_i *mut = __bch2_bkey_get_mut_noupdate(trans, iter, - btree_id, pos, flags|BTREE_ITER_INTENT, type, min_bytes); + btree_id, pos, flags|BTREE_ITER_intent, type, min_bytes); int ret; if (IS_ERR(mut)) diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index 875b1795e4086..0b6645c2266fb 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -44,8 +44,8 @@ static btree_path_idx_t get_unlocked_mut_path(struct btree_trans *trans, struct bpos pos) { btree_path_idx_t path_idx = bch2_path_get(trans, btree_id, pos, level + 1, level, - BTREE_ITER_NOPRESERVE| - BTREE_ITER_INTENT, _RET_IP_); + BTREE_ITER_nopreserve| + BTREE_ITER_intent, _RET_IP_); path_idx = bch2_btree_path_make_mut(trans, path_idx, true, _RET_IP_); struct btree_path *path = trans->paths + path_idx; @@ -664,7 +664,7 @@ static int btree_update_nodes_written_trans(struct btree_trans *trans, unsigned level = bkey_i_to_btree_ptr_v2(k)->v.mem_ptr; ret = bch2_key_trigger_old(trans, as->btree_id, level, bkey_i_to_s_c(k), - BTREE_TRIGGER_TRANSACTIONAL); + BTREE_TRIGGER_transactional); if (ret) return ret; } @@ -673,7 +673,7 @@ static int btree_update_nodes_written_trans(struct btree_trans *trans, unsigned level = bkey_i_to_btree_ptr_v2(k)->v.mem_ptr; ret = bch2_key_trigger_new(trans, as->btree_id, level, bkey_i_to_s(k), - BTREE_TRIGGER_TRANSACTIONAL); + BTREE_TRIGGER_transactional); if (ret) return ret; } @@ -1997,7 +1997,7 @@ int __bch2_foreground_maybe_merge(struct btree_trans *trans, : bpos_successor(b->data->max_key); sib_path = bch2_path_get(trans, btree, sib_pos, - U8_MAX, level, BTREE_ITER_INTENT, _THIS_IP_); + U8_MAX, level, BTREE_ITER_intent, _THIS_IP_); ret = bch2_btree_path_traverse(trans, sib_path, false); if (ret) goto err; @@ -2351,10 +2351,10 @@ static int __bch2_btree_node_update_key(struct btree_trans *trans, if (!skip_triggers) { ret = bch2_key_trigger_old(trans, b->c.btree_id, b->c.level + 1, bkey_i_to_s_c(&b->key), - BTREE_TRIGGER_TRANSACTIONAL) ?: + BTREE_TRIGGER_transactional) ?: bch2_key_trigger_new(trans, b->c.btree_id, b->c.level + 1, bkey_i_to_s(new_key), - BTREE_TRIGGER_TRANSACTIONAL); + BTREE_TRIGGER_transactional); if (ret) return ret; } @@ -2371,7 +2371,7 @@ static int __bch2_btree_node_update_key(struct btree_trans *trans, bch2_trans_copy_iter(&iter2, iter); iter2.path = bch2_btree_path_make_mut(trans, iter2.path, - iter2.flags & BTREE_ITER_INTENT, + iter2.flags & BTREE_ITER_intent, _THIS_IP_); struct btree_path *path2 = btree_iter_path(trans, &iter2); @@ -2383,7 +2383,7 @@ static int __bch2_btree_node_update_key(struct btree_trans *trans, trans->paths_sorted = false; ret = bch2_btree_iter_traverse(&iter2) ?: - bch2_trans_update(trans, &iter2, new_key, BTREE_TRIGGER_NORUN); + bch2_trans_update(trans, &iter2, new_key, BTREE_TRIGGER_norun); if (ret) goto err; } else { @@ -2491,7 +2491,7 @@ int bch2_btree_node_update_key_get_iter(struct btree_trans *trans, bch2_trans_node_iter_init(trans, &iter, b->c.btree_id, b->key.k.p, BTREE_MAX_DEPTH, b->c.level, - BTREE_ITER_INTENT); + BTREE_ITER_intent); ret = bch2_btree_iter_traverse(&iter); if (ret) goto out; diff --git a/fs/bcachefs/btree_write_buffer.c b/fs/bcachefs/btree_write_buffer.c index 36a6f42aba5e6..75c8a196b3f63 100644 --- a/fs/bcachefs/btree_write_buffer.c +++ b/fs/bcachefs/btree_write_buffer.c @@ -122,7 +122,7 @@ static noinline int wb_flush_one_slowpath(struct btree_trans *trans, trans->journal_res.seq = wb->journal_seq; return bch2_trans_update(trans, iter, &wb->k, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) ?: + BTREE_UPDATE_internal_snapshot_node) ?: bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc| BCH_TRANS_COMMIT_no_check_rw| @@ -191,13 +191,13 @@ btree_write_buffered_insert(struct btree_trans *trans, int ret; bch2_trans_iter_init(trans, &iter, wb->btree, bkey_start_pos(&wb->k.k), - BTREE_ITER_CACHED|BTREE_ITER_INTENT); + BTREE_ITER_cached|BTREE_ITER_intent); trans->journal_res.seq = wb->journal_seq; ret = bch2_btree_iter_traverse(&iter) ?: bch2_trans_update(trans, &iter, &wb->k, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + BTREE_UPDATE_internal_snapshot_node); bch2_trans_iter_exit(trans, &iter); return ret; } @@ -332,7 +332,7 @@ static int bch2_btree_write_buffer_flush_locked(struct btree_trans *trans) if (!iter.path || iter.btree_id != k->btree) { bch2_trans_iter_exit(trans, &iter); bch2_trans_iter_init(trans, &iter, k->btree, k->k.k.p, - BTREE_ITER_INTENT|BTREE_ITER_ALL_SNAPSHOTS); + BTREE_ITER_intent|BTREE_ITER_all_snapshots); } bch2_btree_iter_set_pos(&iter, k->k.k.p); diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 015321a87170a..20f471c08b2e0 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -752,16 +752,17 @@ static int bch2_trigger_pointer(struct btree_trans *trans, enum btree_id btree_id, unsigned level, struct bkey_s_c k, struct extent_ptr_decoded p, const union bch_extent_entry *entry, - s64 *sectors, unsigned flags) + s64 *sectors, + enum btree_iter_update_trigger_flags flags) { - bool insert = !(flags & BTREE_TRIGGER_OVERWRITE); + bool insert = !(flags & BTREE_TRIGGER_overwrite); struct bpos bucket; struct bch_backpointer bp; bch2_extent_ptr_to_bp(trans->c, btree_id, level, k, p, entry, &bucket, &bp); *sectors = insert ? bp.bucket_len : -((s64) bp.bucket_len); - if (flags & BTREE_TRIGGER_TRANSACTIONAL) { + if (flags & BTREE_TRIGGER_transactional) { struct btree_iter iter; struct bkey_i_alloc_v4 *a = bch2_trans_start_alloc_update(trans, &iter, bucket); int ret = PTR_ERR_OR_ZERO(a); @@ -784,7 +785,7 @@ static int bch2_trigger_pointer(struct btree_trans *trans, } } - if (flags & BTREE_TRIGGER_GC) { + if (flags & BTREE_TRIGGER_gc) { struct bch_fs *c = trans->c; struct bch_dev *ca = bch_dev_bkey_exists(c, p.ptr.dev); enum bch_data_type data_type = bch2_bkey_ptr_data_type(k, p, entry); @@ -820,13 +821,14 @@ static int bch2_trigger_stripe_ptr(struct btree_trans *trans, struct bkey_s_c k, struct extent_ptr_decoded p, enum bch_data_type data_type, - s64 sectors, unsigned flags) + s64 sectors, + enum btree_iter_update_trigger_flags flags) { - if (flags & BTREE_TRIGGER_TRANSACTIONAL) { + if (flags & BTREE_TRIGGER_transactional) { struct btree_iter iter; struct bkey_i_stripe *s = bch2_bkey_get_mut_typed(trans, &iter, BTREE_ID_stripes, POS(0, p.ec.idx), - BTREE_ITER_WITH_UPDATES, stripe); + BTREE_ITER_with_updates, stripe); int ret = PTR_ERR_OR_ZERO(s); if (unlikely(ret)) { bch2_trans_inconsistent_on(bch2_err_matches(ret, ENOENT), trans, @@ -856,10 +858,10 @@ static int bch2_trigger_stripe_ptr(struct btree_trans *trans, return ret; } - if (flags & BTREE_TRIGGER_GC) { + if (flags & BTREE_TRIGGER_gc) { struct bch_fs *c = trans->c; - BUG_ON(!(flags & BTREE_TRIGGER_GC)); + BUG_ON(!(flags & BTREE_TRIGGER_gc)); struct gc_stripe *m = genradix_ptr_alloc(&c->gc_stripes, p.ec.idx, GFP_KERNEL); if (!m) { @@ -895,9 +897,10 @@ static int bch2_trigger_stripe_ptr(struct btree_trans *trans, static int __trigger_extent(struct btree_trans *trans, enum btree_id btree_id, unsigned level, - struct bkey_s_c k, unsigned flags) + struct bkey_s_c k, + enum btree_iter_update_trigger_flags flags) { - bool gc = flags & BTREE_TRIGGER_GC; + bool gc = flags & BTREE_TRIGGER_gc; struct bch_fs *c = trans->c; struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); const union bch_extent_entry *entry; @@ -969,7 +972,7 @@ static int __trigger_extent(struct btree_trans *trans, int bch2_trigger_extent(struct btree_trans *trans, enum btree_id btree_id, unsigned level, struct bkey_s_c old, struct bkey_s new, - unsigned flags) + enum btree_iter_update_trigger_flags flags) { struct bkey_ptrs_c new_ptrs = bch2_bkey_ptrs_c(new.s_c); struct bkey_ptrs_c old_ptrs = bch2_bkey_ptrs_c(old); @@ -983,7 +986,7 @@ int bch2_trigger_extent(struct btree_trans *trans, new_ptrs_bytes)) return 0; - if (flags & BTREE_TRIGGER_TRANSACTIONAL) { + if (flags & BTREE_TRIGGER_transactional) { struct bch_fs *c = trans->c; int mod = (int) bch2_bkey_needs_rebalance(c, new.s_c) - (int) bch2_bkey_needs_rebalance(c, old); @@ -996,7 +999,7 @@ int bch2_trigger_extent(struct btree_trans *trans, } } - if (flags & (BTREE_TRIGGER_TRANSACTIONAL|BTREE_TRIGGER_GC)) + if (flags & (BTREE_TRIGGER_transactional|BTREE_TRIGGER_gc)) return trigger_run_overwrite_then_insert(__trigger_extent, trans, btree_id, level, old, new, flags); return 0; @@ -1005,17 +1008,17 @@ int bch2_trigger_extent(struct btree_trans *trans, /* KEY_TYPE_reservation */ static int __trigger_reservation(struct btree_trans *trans, - enum btree_id btree_id, unsigned level, - struct bkey_s_c k, unsigned flags) + enum btree_id btree_id, unsigned level, struct bkey_s_c k, + enum btree_iter_update_trigger_flags flags) { struct bch_fs *c = trans->c; unsigned replicas = bkey_s_c_to_reservation(k).v->nr_replicas; s64 sectors = (s64) k.k->size * replicas; - if (flags & BTREE_TRIGGER_OVERWRITE) + if (flags & BTREE_TRIGGER_overwrite) sectors = -sectors; - if (flags & BTREE_TRIGGER_TRANSACTIONAL) { + if (flags & BTREE_TRIGGER_transactional) { int ret = bch2_replicas_deltas_realloc(trans, 0); if (ret) return ret; @@ -1026,7 +1029,7 @@ static int __trigger_reservation(struct btree_trans *trans, d->persistent_reserved[replicas - 1] += sectors; } - if (flags & BTREE_TRIGGER_GC) { + if (flags & BTREE_TRIGGER_gc) { percpu_down_read(&c->mark_lock); preempt_disable(); @@ -1046,7 +1049,7 @@ static int __trigger_reservation(struct btree_trans *trans, int bch2_trigger_reservation(struct btree_trans *trans, enum btree_id btree_id, unsigned level, struct bkey_s_c old, struct bkey_s new, - unsigned flags) + enum btree_iter_update_trigger_flags flags) { return trigger_run_overwrite_then_insert(__trigger_reservation, trans, btree_id, level, old, new, flags); } @@ -1092,8 +1095,8 @@ static int __bch2_trans_mark_metadata_bucket(struct btree_trans *trans, } static int bch2_mark_metadata_bucket(struct bch_fs *c, struct bch_dev *ca, - u64 b, enum bch_data_type data_type, - unsigned sectors, unsigned flags) + u64 b, enum bch_data_type data_type, unsigned sectors, + enum btree_iter_update_trigger_flags flags) { struct bucket old, new, *g; int ret = 0; @@ -1134,9 +1137,9 @@ static int bch2_mark_metadata_bucket(struct bch_fs *c, struct bch_dev *ca, } int bch2_trans_mark_metadata_bucket(struct btree_trans *trans, - struct bch_dev *ca, u64 b, - enum bch_data_type type, - unsigned sectors, unsigned flags) + struct bch_dev *ca, u64 b, + enum bch_data_type type, unsigned sectors, + enum btree_iter_update_trigger_flags flags) { BUG_ON(type != BCH_DATA_free && type != BCH_DATA_sb && @@ -1148,9 +1151,9 @@ int bch2_trans_mark_metadata_bucket(struct btree_trans *trans, if (b >= ca->mi.nbuckets) return 0; - if (flags & BTREE_TRIGGER_GC) + if (flags & BTREE_TRIGGER_gc) return bch2_mark_metadata_bucket(trans->c, ca, b, type, sectors, flags); - else if (flags & BTREE_TRIGGER_TRANSACTIONAL) + else if (flags & BTREE_TRIGGER_transactional) return commit_do(trans, NULL, NULL, 0, __bch2_trans_mark_metadata_bucket(trans, ca, b, type, sectors)); else @@ -1158,11 +1161,9 @@ int bch2_trans_mark_metadata_bucket(struct btree_trans *trans, } static int bch2_trans_mark_metadata_sectors(struct btree_trans *trans, - struct bch_dev *ca, - u64 start, u64 end, - enum bch_data_type type, - u64 *bucket, unsigned *bucket_sectors, - unsigned flags) + struct bch_dev *ca, u64 start, u64 end, + enum bch_data_type type, u64 *bucket, unsigned *bucket_sectors, + enum btree_iter_update_trigger_flags flags) { do { u64 b = sector_to_bucket(ca, start); @@ -1186,8 +1187,8 @@ static int bch2_trans_mark_metadata_sectors(struct btree_trans *trans, return 0; } -static int __bch2_trans_mark_dev_sb(struct btree_trans *trans, - struct bch_dev *ca, unsigned flags) +static int __bch2_trans_mark_dev_sb(struct btree_trans *trans, struct bch_dev *ca, + enum btree_iter_update_trigger_flags flags) { struct bch_sb_layout *layout = &ca->disk_sb.sb->layout; u64 bucket = 0; @@ -1230,7 +1231,8 @@ static int __bch2_trans_mark_dev_sb(struct btree_trans *trans, return 0; } -int bch2_trans_mark_dev_sb(struct bch_fs *c, struct bch_dev *ca, unsigned flags) +int bch2_trans_mark_dev_sb(struct bch_fs *c, struct bch_dev *ca, + enum btree_iter_update_trigger_flags flags) { int ret = bch2_trans_run(c, __bch2_trans_mark_dev_sb(trans, ca, flags)); @@ -1238,7 +1240,8 @@ int bch2_trans_mark_dev_sb(struct bch_fs *c, struct bch_dev *ca, unsigned flags) return ret; } -int bch2_trans_mark_dev_sbs_flags(struct bch_fs *c, unsigned flags) +int bch2_trans_mark_dev_sbs_flags(struct bch_fs *c, + enum btree_iter_update_trigger_flags flags) { for_each_online_member(c, ca) { int ret = bch2_trans_mark_dev_sb(c, ca, flags); @@ -1253,7 +1256,7 @@ int bch2_trans_mark_dev_sbs_flags(struct bch_fs *c, unsigned flags) int bch2_trans_mark_dev_sbs(struct bch_fs *c) { - return bch2_trans_mark_dev_sbs_flags(c, BTREE_TRIGGER_TRANSACTIONAL); + return bch2_trans_mark_dev_sbs_flags(c, BTREE_TRIGGER_transactional); } /* Disk reservations: */ diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h index bae76e58311d9..a84e14468276e 100644 --- a/fs/bcachefs/buckets.h +++ b/fs/bcachefs/buckets.h @@ -338,18 +338,20 @@ int bch2_check_bucket_ref(struct btree_trans *, struct bkey_s_c, s64, enum bch_data_type, u8, u8, u32); int bch2_trigger_extent(struct btree_trans *, enum btree_id, unsigned, - struct bkey_s_c, struct bkey_s, unsigned); + struct bkey_s_c, struct bkey_s, + enum btree_iter_update_trigger_flags); int bch2_trigger_reservation(struct btree_trans *, enum btree_id, unsigned, - struct bkey_s_c, struct bkey_s, unsigned); + struct bkey_s_c, struct bkey_s, + enum btree_iter_update_trigger_flags); #define trigger_run_overwrite_then_insert(_fn, _trans, _btree_id, _level, _old, _new, _flags)\ ({ \ int ret = 0; \ \ if (_old.k->type) \ - ret = _fn(_trans, _btree_id, _level, _old, _flags & ~BTREE_TRIGGER_INSERT); \ + ret = _fn(_trans, _btree_id, _level, _old, _flags & ~BTREE_TRIGGER_insert); \ if (!ret && _new.k->type) \ - ret = _fn(_trans, _btree_id, _level, _new.s_c, _flags & ~BTREE_TRIGGER_OVERWRITE);\ + ret = _fn(_trans, _btree_id, _level, _new.s_c, _flags & ~BTREE_TRIGGER_overwrite);\ ret; \ }) @@ -359,9 +361,12 @@ void bch2_trans_fs_usage_revert(struct btree_trans *, struct replicas_delta_list int bch2_trans_fs_usage_apply(struct btree_trans *, struct replicas_delta_list *); int bch2_trans_mark_metadata_bucket(struct btree_trans *, struct bch_dev *, u64, - enum bch_data_type, unsigned, unsigned); -int bch2_trans_mark_dev_sb(struct bch_fs *, struct bch_dev *, unsigned); -int bch2_trans_mark_dev_sbs_flags(struct bch_fs *, unsigned); + enum bch_data_type, unsigned, + enum btree_iter_update_trigger_flags); +int bch2_trans_mark_dev_sb(struct bch_fs *, struct bch_dev *, + enum btree_iter_update_trigger_flags); +int bch2_trans_mark_dev_sbs_flags(struct bch_fs *, + enum btree_iter_update_trigger_flags); int bch2_trans_mark_dev_sbs(struct bch_fs *); static inline bool is_superblock_bucket(struct bch_dev *ca, u64 b) diff --git a/fs/bcachefs/data_update.c b/fs/bcachefs/data_update.c index 0022b51ce3c09..3c0f0801468eb 100644 --- a/fs/bcachefs/data_update.c +++ b/fs/bcachefs/data_update.c @@ -106,7 +106,7 @@ static int __bch2_data_update_index_update(struct btree_trans *trans, bch2_trans_iter_init(trans, &iter, m->btree_id, bkey_start_pos(&bch2_keylist_front(keys)->k), - BTREE_ITER_SLOTS|BTREE_ITER_INTENT); + BTREE_ITER_slots|BTREE_ITER_intent); while (1) { struct bkey_s_c k; @@ -288,7 +288,7 @@ static int __bch2_data_update_index_update(struct btree_trans *trans, k.k->p, insert->k.p) ?: bch2_bkey_set_needs_rebalance(c, insert, &op->opts) ?: bch2_trans_update(trans, &iter, insert, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) ?: + BTREE_UPDATE_internal_snapshot_node) ?: bch2_trans_commit(trans, &op->res, NULL, BCH_TRANS_COMMIT_no_check_rw| @@ -387,7 +387,7 @@ static void bch2_update_unwritten_extent(struct btree_trans *trans, unsigned sectors = bio_sectors(bio); bch2_trans_iter_init(trans, &iter, update->btree_id, update->op.pos, - BTREE_ITER_SLOTS); + BTREE_ITER_slots); ret = lockrestart_do(trans, ({ k = bch2_btree_iter_peek_slot(&iter); bkey_err(k); @@ -480,15 +480,15 @@ int bch2_extent_drop_ptrs(struct btree_trans *trans, /* * Since we're not inserting through an extent iterator - * (BTREE_ITER_ALL_SNAPSHOTS iterators aren't extent iterators), + * (BTREE_ITER_all_snapshots iterators aren't extent iterators), * we aren't using the extent overwrite path to delete, we're * just using the normal key deletion path: */ - if (bkey_deleted(&n->k) && !(iter->flags & BTREE_ITER_IS_EXTENTS)) + if (bkey_deleted(&n->k) && !(iter->flags & BTREE_ITER_is_extents)) n->k.size = 0; return bch2_trans_relock(trans) ?: - bch2_trans_update(trans, iter, n, BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) ?: + bch2_trans_update(trans, iter, n, BTREE_UPDATE_internal_snapshot_node) ?: bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc); } diff --git a/fs/bcachefs/debug.c b/fs/bcachefs/debug.c index d0ec74e6dc959..8310eff12a172 100644 --- a/fs/bcachefs/debug.c +++ b/fs/bcachefs/debug.c @@ -375,8 +375,8 @@ static ssize_t bch2_read_btree(struct file *file, char __user *buf, return flush_buf(i) ?: bch2_trans_run(i->c, for_each_btree_key(trans, iter, i->id, i->from, - BTREE_ITER_PREFETCH| - BTREE_ITER_ALL_SNAPSHOTS, k, ({ + BTREE_ITER_prefetch| + BTREE_ITER_all_snapshots, k, ({ bch2_bkey_val_to_text(&i->buf, i->c, k); prt_newline(&i->buf); bch2_trans_unlock(trans); @@ -459,8 +459,8 @@ static ssize_t bch2_read_bfloat_failed(struct file *file, char __user *buf, return flush_buf(i) ?: bch2_trans_run(i->c, for_each_btree_key(trans, iter, i->id, i->from, - BTREE_ITER_PREFETCH| - BTREE_ITER_ALL_SNAPSHOTS, k, ({ + BTREE_ITER_prefetch| + BTREE_ITER_all_snapshots, k, ({ struct btree_path_level *l = &btree_iter_path(trans, &iter)->l[0]; struct bkey_packed *_k = diff --git a/fs/bcachefs/dirent.c b/fs/bcachefs/dirent.c index d4e3ca5045d61..ed256c3aa8a5f 100644 --- a/fs/bcachefs/dirent.c +++ b/fs/bcachefs/dirent.c @@ -205,7 +205,7 @@ int bch2_dirent_create_snapshot(struct btree_trans *trans, const struct bch_hash_info *hash_info, u8 type, const struct qstr *name, u64 dst_inum, u64 *dir_offset, - bch_str_hash_flags_t str_hash_flags) + enum btree_iter_update_trigger_flags flags) { subvol_inum dir_inum = { .subvol = dir_subvol, .inum = dir }; struct bkey_i_dirent *dirent; @@ -220,9 +220,8 @@ int bch2_dirent_create_snapshot(struct btree_trans *trans, dirent->k.p.snapshot = snapshot; ret = bch2_hash_set_in_snapshot(trans, bch2_dirent_hash_desc, hash_info, - dir_inum, snapshot, - &dirent->k_i, str_hash_flags, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + dir_inum, snapshot, &dirent->k_i, + flags|BTREE_UPDATE_internal_snapshot_node); *dir_offset = dirent->k.p.offset; return ret; @@ -232,7 +231,7 @@ int bch2_dirent_create(struct btree_trans *trans, subvol_inum dir, const struct bch_hash_info *hash_info, u8 type, const struct qstr *name, u64 dst_inum, u64 *dir_offset, - bch_str_hash_flags_t str_hash_flags) + enum btree_iter_update_trigger_flags flags) { struct bkey_i_dirent *dirent; int ret; @@ -243,7 +242,7 @@ int bch2_dirent_create(struct btree_trans *trans, subvol_inum dir, return ret; ret = bch2_hash_set(trans, bch2_dirent_hash_desc, hash_info, - dir, &dirent->k_i, str_hash_flags); + dir, &dirent->k_i, flags); *dir_offset = dirent->k.p.offset; return ret; @@ -272,7 +271,7 @@ int bch2_dirent_read_target(struct btree_trans *trans, subvol_inum dir, } else { target->subvol = le32_to_cpu(d.v->d_child_subvol); - ret = bch2_subvolume_get(trans, target->subvol, true, BTREE_ITER_CACHED, &s); + ret = bch2_subvolume_get(trans, target->subvol, true, BTREE_ITER_cached, &s); target->inum = le64_to_cpu(s.inode); } @@ -303,7 +302,7 @@ int bch2_dirent_rename(struct btree_trans *trans, /* Lookup src: */ old_src = bch2_hash_lookup(trans, &src_iter, bch2_dirent_hash_desc, src_hash, src_dir, src_name, - BTREE_ITER_INTENT); + BTREE_ITER_intent); ret = bkey_err(old_src); if (ret) goto out; @@ -327,7 +326,7 @@ int bch2_dirent_rename(struct btree_trans *trans, } else { old_dst = bch2_hash_lookup(trans, &dst_iter, bch2_dirent_hash_desc, dst_hash, dst_dir, dst_name, - BTREE_ITER_INTENT); + BTREE_ITER_intent); ret = bkey_err(old_dst); if (ret) goto out; @@ -442,7 +441,7 @@ int bch2_dirent_rename(struct btree_trans *trans, if (delete_src) { bch2_btree_iter_set_snapshot(&src_iter, old_src.k->p.snapshot); ret = bch2_btree_iter_traverse(&src_iter) ?: - bch2_btree_delete_at(trans, &src_iter, BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + bch2_btree_delete_at(trans, &src_iter, BTREE_UPDATE_internal_snapshot_node); if (ret) goto out; } @@ -450,7 +449,7 @@ int bch2_dirent_rename(struct btree_trans *trans, if (delete_dst) { bch2_btree_iter_set_snapshot(&dst_iter, old_dst.k->p.snapshot); ret = bch2_btree_iter_traverse(&dst_iter) ?: - bch2_btree_delete_at(trans, &dst_iter, BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + bch2_btree_delete_at(trans, &dst_iter, BTREE_UPDATE_internal_snapshot_node); if (ret) goto out; } diff --git a/fs/bcachefs/dirent.h b/fs/bcachefs/dirent.h index bee55cca2aa0d..635ceb3c07fbc 100644 --- a/fs/bcachefs/dirent.h +++ b/fs/bcachefs/dirent.h @@ -38,11 +38,11 @@ int bch2_dirent_read_target(struct btree_trans *, subvol_inum, int bch2_dirent_create_snapshot(struct btree_trans *, u32, u64, u32, const struct bch_hash_info *, u8, const struct qstr *, u64, u64 *, - bch_str_hash_flags_t); + enum btree_iter_update_trigger_flags); int bch2_dirent_create(struct btree_trans *, subvol_inum, const struct bch_hash_info *, u8, const struct qstr *, u64, u64 *, - bch_str_hash_flags_t); + enum btree_iter_update_trigger_flags); static inline unsigned vfs_d_type(unsigned type) { diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 556a217108d32..86999ed80439b 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -244,7 +244,7 @@ static int bch2_trans_mark_stripe_bucket(struct btree_trans *trans, static int mark_stripe_bucket(struct btree_trans *trans, struct bkey_s_c k, unsigned ptr_idx, - unsigned flags) + enum btree_iter_update_trigger_flags flags) { struct bch_fs *c = trans->c; const struct bch_stripe *s = bkey_s_c_to_stripe(k).v; @@ -258,7 +258,7 @@ static int mark_stripe_bucket(struct btree_trans *trans, struct printbuf buf = PRINTBUF; int ret = 0; - BUG_ON(!(flags & BTREE_TRIGGER_GC)); + BUG_ON(!(flags & BTREE_TRIGGER_gc)); /* * XXX doesn't handle deletion */ @@ -302,7 +302,7 @@ static int mark_stripe_bucket(struct btree_trans *trans, int bch2_trigger_stripe(struct btree_trans *trans, enum btree_id btree_id, unsigned level, struct bkey_s_c old, struct bkey_s _new, - unsigned flags) + enum btree_iter_update_trigger_flags flags) { struct bkey_s_c new = _new.s_c; struct bch_fs *c = trans->c; @@ -312,7 +312,7 @@ int bch2_trigger_stripe(struct btree_trans *trans, const struct bch_stripe *new_s = new.k->type == KEY_TYPE_stripe ? bkey_s_c_to_stripe(new).v : NULL; - if (flags & BTREE_TRIGGER_TRANSACTIONAL) { + if (flags & BTREE_TRIGGER_transactional) { /* * If the pointers aren't changing, we don't need to do anything: */ @@ -371,7 +371,7 @@ int bch2_trigger_stripe(struct btree_trans *trans, } } - if (flags & BTREE_TRIGGER_ATOMIC) { + if (flags & BTREE_TRIGGER_atomic) { struct stripe *m = genradix_ptr(&c->stripes, idx); if (!m) { @@ -410,7 +410,7 @@ int bch2_trigger_stripe(struct btree_trans *trans, } } - if (flags & BTREE_TRIGGER_GC) { + if (flags & BTREE_TRIGGER_gc) { struct gc_stripe *m = genradix_ptr_alloc(&c->gc_stripes, idx, GFP_KERNEL); @@ -769,7 +769,7 @@ static int get_stripe_key_trans(struct btree_trans *trans, u64 idx, int ret; k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_stripes, - POS(0, idx), BTREE_ITER_SLOTS); + POS(0, idx), BTREE_ITER_slots); ret = bkey_err(k); if (ret) goto err; @@ -1060,7 +1060,7 @@ static int ec_stripe_delete(struct btree_trans *trans, u64 idx) int ret; k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_stripes, POS(0, idx), - BTREE_ITER_INTENT); + BTREE_ITER_intent); ret = bkey_err(k); if (ret) goto err; @@ -1131,7 +1131,7 @@ static int ec_stripe_key_update(struct btree_trans *trans, int ret; k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_stripes, - new->k.p, BTREE_ITER_INTENT); + new->k.p, BTREE_ITER_intent); ret = bkey_err(k); if (ret) goto err; @@ -1189,7 +1189,7 @@ static int ec_stripe_update_extent(struct btree_trans *trans, int ret, dev, block; ret = bch2_get_next_backpointer(trans, bucket, gen, - bp_pos, &bp, BTREE_ITER_CACHED); + bp_pos, &bp, BTREE_ITER_cached); if (ret) return ret; if (bpos_eq(*bp_pos, SPOS_MAX)) @@ -1214,7 +1214,7 @@ static int ec_stripe_update_extent(struct btree_trans *trans, return -EIO; } - k = bch2_backpointer_get_key(trans, &iter, *bp_pos, bp, BTREE_ITER_INTENT); + k = bch2_backpointer_get_key(trans, &iter, *bp_pos, bp, BTREE_ITER_intent); ret = bkey_err(k); if (ret) return ret; @@ -1937,7 +1937,7 @@ static int __bch2_ec_stripe_head_reserve(struct btree_trans *trans, struct ec_st } for_each_btree_key_norestart(trans, iter, BTREE_ID_stripes, start_pos, - BTREE_ITER_SLOTS|BTREE_ITER_INTENT, k, ret) { + BTREE_ITER_slots|BTREE_ITER_intent, k, ret) { if (bkey_gt(k.k->p, POS(0, U32_MAX))) { if (start_pos.offset) { start_pos = min_pos; @@ -2127,7 +2127,7 @@ int bch2_stripes_read(struct bch_fs *c) { int ret = bch2_trans_run(c, for_each_btree_key(trans, iter, BTREE_ID_stripes, POS_MIN, - BTREE_ITER_PREFETCH, k, ({ + BTREE_ITER_prefetch, k, ({ if (k.k->type != KEY_TYPE_stripe) continue; diff --git a/fs/bcachefs/ec.h b/fs/bcachefs/ec.h index f042616888b0a..a1c2bc74ef458 100644 --- a/fs/bcachefs/ec.h +++ b/fs/bcachefs/ec.h @@ -13,7 +13,8 @@ int bch2_stripe_invalid(struct bch_fs *, struct bkey_s_c, void bch2_stripe_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); int bch2_trigger_stripe(struct btree_trans *, enum btree_id, unsigned, - struct bkey_s_c, struct bkey_s, unsigned); + struct bkey_s_c, struct bkey_s, + enum btree_iter_update_trigger_flags); #define bch2_bkey_ops_stripe ((struct bkey_ops) { \ .key_invalid = bch2_stripe_invalid, \ diff --git a/fs/bcachefs/extent_update.c b/fs/bcachefs/extent_update.c index b9033bb4f11cf..5f4fecb358da0 100644 --- a/fs/bcachefs/extent_update.c +++ b/fs/bcachefs/extent_update.c @@ -72,7 +72,7 @@ static int count_iters_for_insert(struct btree_trans *trans, for_each_btree_key_norestart(trans, iter, BTREE_ID_reflink, POS(0, idx + offset), - BTREE_ITER_SLOTS, r_k, ret2) { + BTREE_ITER_slots, r_k, ret2) { if (bkey_ge(bkey_start_pos(r_k.k), POS(0, idx + sectors))) break; diff --git a/fs/bcachefs/fs-common.c b/fs/bcachefs/fs-common.c index 624e6f963240f..6c737def357e3 100644 --- a/fs/bcachefs/fs-common.c +++ b/fs/bcachefs/fs-common.c @@ -42,7 +42,7 @@ int bch2_create_trans(struct btree_trans *trans, if (ret) goto err; - ret = bch2_inode_peek(trans, &dir_iter, dir_u, dir, BTREE_ITER_INTENT); + ret = bch2_inode_peek(trans, &dir_iter, dir_u, dir, BTREE_ITER_intent); if (ret) goto err; @@ -70,7 +70,7 @@ int bch2_create_trans(struct btree_trans *trans, struct bch_subvolume s; ret = bch2_subvolume_get(trans, snapshot_src.subvol, true, - BTREE_ITER_CACHED, &s); + BTREE_ITER_cached, &s); if (ret) goto err; @@ -78,7 +78,7 @@ int bch2_create_trans(struct btree_trans *trans, } ret = bch2_inode_peek(trans, &inode_iter, new_inode, snapshot_src, - BTREE_ITER_INTENT); + BTREE_ITER_intent); if (ret) goto err; @@ -163,7 +163,7 @@ int bch2_create_trans(struct btree_trans *trans, name, dir_target, &dir_offset, - BCH_HASH_SET_MUST_CREATE); + STR_HASH_must_create); if (ret) goto err; @@ -171,7 +171,7 @@ int bch2_create_trans(struct btree_trans *trans, new_inode->bi_dir_offset = dir_offset; } - inode_iter.flags &= ~BTREE_ITER_ALL_SNAPSHOTS; + inode_iter.flags &= ~BTREE_ITER_all_snapshots; bch2_btree_iter_set_snapshot(&inode_iter, snapshot); ret = bch2_btree_iter_traverse(&inode_iter) ?: @@ -198,7 +198,7 @@ int bch2_link_trans(struct btree_trans *trans, if (dir.subvol != inum.subvol) return -EXDEV; - ret = bch2_inode_peek(trans, &inode_iter, inode_u, inum, BTREE_ITER_INTENT); + ret = bch2_inode_peek(trans, &inode_iter, inode_u, inum, BTREE_ITER_intent); if (ret) goto err; @@ -207,7 +207,7 @@ int bch2_link_trans(struct btree_trans *trans, if (ret) return ret; - ret = bch2_inode_peek(trans, &dir_iter, dir_u, dir, BTREE_ITER_INTENT); + ret = bch2_inode_peek(trans, &dir_iter, dir_u, dir, BTREE_ITER_intent); if (ret) goto err; @@ -223,7 +223,7 @@ int bch2_link_trans(struct btree_trans *trans, ret = bch2_dirent_create(trans, dir, &dir_hash, mode_to_type(inode_u->bi_mode), name, inum.inum, &dir_offset, - BCH_HASH_SET_MUST_CREATE); + STR_HASH_must_create); if (ret) goto err; @@ -255,19 +255,19 @@ int bch2_unlink_trans(struct btree_trans *trans, struct bkey_s_c k; int ret; - ret = bch2_inode_peek(trans, &dir_iter, dir_u, dir, BTREE_ITER_INTENT); + ret = bch2_inode_peek(trans, &dir_iter, dir_u, dir, BTREE_ITER_intent); if (ret) goto err; dir_hash = bch2_hash_info_init(c, dir_u); ret = bch2_dirent_lookup_trans(trans, &dirent_iter, dir, &dir_hash, - name, &inum, BTREE_ITER_INTENT); + name, &inum, BTREE_ITER_intent); if (ret) goto err; ret = bch2_inode_peek(trans, &inode_iter, inode_u, inum, - BTREE_ITER_INTENT); + BTREE_ITER_intent); if (ret) goto err; @@ -322,7 +322,7 @@ int bch2_unlink_trans(struct btree_trans *trans, ret = bch2_hash_delete_at(trans, bch2_dirent_hash_desc, &dir_hash, &dirent_iter, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) ?: + BTREE_UPDATE_internal_snapshot_node) ?: bch2_inode_write(trans, &dir_iter, dir_u) ?: bch2_inode_write(trans, &inode_iter, inode_u); err: @@ -363,7 +363,7 @@ static int subvol_update_parent(struct btree_trans *trans, u32 subvol, u32 new_p struct bkey_i_subvolume *s = bch2_bkey_get_mut_typed(trans, &iter, BTREE_ID_subvolumes, POS(0, subvol), - BTREE_ITER_CACHED, subvolume); + BTREE_ITER_cached, subvolume); int ret = PTR_ERR_OR_ZERO(s); if (ret) return ret; @@ -394,7 +394,7 @@ int bch2_rename_trans(struct btree_trans *trans, int ret; ret = bch2_inode_peek(trans, &src_dir_iter, src_dir_u, src_dir, - BTREE_ITER_INTENT); + BTREE_ITER_intent); if (ret) goto err; @@ -403,7 +403,7 @@ int bch2_rename_trans(struct btree_trans *trans, if (dst_dir.inum != src_dir.inum || dst_dir.subvol != src_dir.subvol) { ret = bch2_inode_peek(trans, &dst_dir_iter, dst_dir_u, dst_dir, - BTREE_ITER_INTENT); + BTREE_ITER_intent); if (ret) goto err; @@ -423,13 +423,13 @@ int bch2_rename_trans(struct btree_trans *trans, goto err; ret = bch2_inode_peek(trans, &src_inode_iter, src_inode_u, src_inum, - BTREE_ITER_INTENT); + BTREE_ITER_intent); if (ret) goto err; if (dst_inum.inum) { ret = bch2_inode_peek(trans, &dst_inode_iter, dst_inode_u, dst_inum, - BTREE_ITER_INTENT); + BTREE_ITER_intent); if (ret) goto err; } diff --git a/fs/bcachefs/fs-io-buffered.c b/fs/bcachefs/fs-io-buffered.c index 39292e7ef342c..120430abdae74 100644 --- a/fs/bcachefs/fs-io-buffered.c +++ b/fs/bcachefs/fs-io-buffered.c @@ -176,7 +176,7 @@ static void bchfs_read(struct btree_trans *trans, bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, SPOS(inum.inum, rbio->bio.bi_iter.bi_sector, snapshot), - BTREE_ITER_SLOTS); + BTREE_ITER_slots); while (1) { struct bkey_s_c k; unsigned bytes, sectors, offset_into_extent; diff --git a/fs/bcachefs/fs-io-direct.c b/fs/bcachefs/fs-io-direct.c index b889370a50881..09d21aef879af 100644 --- a/fs/bcachefs/fs-io-direct.c +++ b/fs/bcachefs/fs-io-direct.c @@ -254,7 +254,7 @@ static bool bch2_check_range_allocated(struct bch_fs *c, subvol_inum inum, for_each_btree_key_norestart(trans, iter, BTREE_ID_extents, SPOS(inum.inum, offset, snapshot), - BTREE_ITER_SLOTS, k, err) { + BTREE_ITER_slots, k, err) { if (bkey_ge(bkey_start_pos(k.k), POS(inum.inum, end))) break; diff --git a/fs/bcachefs/fs-io-pagecache.c b/fs/bcachefs/fs-io-pagecache.c index d359aa9b33b82..872283e5bd1e2 100644 --- a/fs/bcachefs/fs-io-pagecache.c +++ b/fs/bcachefs/fs-io-pagecache.c @@ -214,7 +214,7 @@ int bch2_folio_set(struct bch_fs *c, subvol_inum inum, for_each_btree_key_norestart(trans, iter, BTREE_ID_extents, SPOS(inum.inum, offset, snapshot), - BTREE_ITER_SLOTS, k, ret) { + BTREE_ITER_slots, k, ret) { unsigned nr_ptrs = bch2_bkey_nr_ptrs_fully_allocated(k); unsigned state = bkey_to_sector_state(k); diff --git a/fs/bcachefs/fs-io.c b/fs/bcachefs/fs-io.c index 20b40477425f4..442bcb0793c41 100644 --- a/fs/bcachefs/fs-io.c +++ b/fs/bcachefs/fs-io.c @@ -594,7 +594,7 @@ static int __bchfs_fallocate(struct bch_inode_info *inode, int mode, bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, POS(inode->v.i_ino, start_sector), - BTREE_ITER_SLOTS|BTREE_ITER_INTENT); + BTREE_ITER_slots|BTREE_ITER_intent); while (!ret && bkey_lt(iter.pos, end_pos)) { s64 i_sectors_delta = 0; @@ -1009,7 +1009,7 @@ static loff_t bch2_seek_hole(struct file *file, u64 offset) for_each_btree_key_norestart(trans, iter, BTREE_ID_extents, SPOS(inode->v.i_ino, offset >> 9, snapshot), - BTREE_ITER_SLOTS, k, ret) { + BTREE_ITER_slots, k, ret) { if (k.k->p.inode != inode->v.i_ino) { next_hole = bch2_seek_pagecache_hole(&inode->v, offset, MAX_LFS_FILESIZE, 0, false); diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c index 3d15275909bd2..81cfc74828fd7 100644 --- a/fs/bcachefs/fs.c +++ b/fs/bcachefs/fs.c @@ -90,7 +90,7 @@ int __must_check bch2_write_inode(struct bch_fs *c, bch2_trans_begin(trans); ret = bch2_inode_peek(trans, &iter, &inode_u, inode_inum(inode), - BTREE_ITER_INTENT) ?: + BTREE_ITER_intent) ?: (set ? set(trans, inode, &inode_u, p) : 0) ?: bch2_inode_write(trans, &iter, &inode_u) ?: bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc); @@ -323,7 +323,7 @@ __bch2_create(struct mnt_idmap *idmap, inum.inum = inode_u.bi_inum; ret = bch2_subvolume_get(trans, inum.subvol, true, - BTREE_ITER_WITH_UPDATES, &subvol) ?: + BTREE_ITER_with_updates, &subvol) ?: bch2_trans_commit(trans, NULL, &journal_seq, 0); if (unlikely(ret)) { bch2_quota_acct(c, bch_qid(&inode_u), Q_INO, -1, @@ -783,7 +783,7 @@ int bch2_setattr_nonsize(struct mnt_idmap *idmap, acl = NULL; ret = bch2_inode_peek(trans, &inode_iter, &inode_u, inode_inum(inode), - BTREE_ITER_INTENT); + BTREE_ITER_intent); if (ret) goto btree_err; diff --git a/fs/bcachefs/fsck.c b/fs/bcachefs/fsck.c index caef81aa618c9..d166c17109dd4 100644 --- a/fs/bcachefs/fsck.c +++ b/fs/bcachefs/fsck.c @@ -79,7 +79,7 @@ static int lookup_first_inode(struct btree_trans *trans, u64 inode_nr, bch2_trans_iter_init(trans, &iter, BTREE_ID_inodes, POS(0, inode_nr), - BTREE_ITER_ALL_SNAPSHOTS); + BTREE_ITER_all_snapshots); k = bch2_btree_iter_peek(&iter); ret = bkey_err(k); if (ret) @@ -154,12 +154,12 @@ static int __remove_dirent(struct btree_trans *trans, struct bpos pos) dir_hash_info = bch2_hash_info_init(c, &dir_inode); - bch2_trans_iter_init(trans, &iter, BTREE_ID_dirents, pos, BTREE_ITER_INTENT); + bch2_trans_iter_init(trans, &iter, BTREE_ID_dirents, pos, BTREE_ITER_intent); ret = bch2_btree_iter_traverse(&iter) ?: bch2_hash_delete_at(trans, bch2_dirent_hash_desc, &dir_hash_info, &iter, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + BTREE_UPDATE_internal_snapshot_node); bch2_trans_iter_exit(trans, &iter); err: bch_err_fn(c, ret); @@ -274,9 +274,9 @@ static int lookup_lostfound(struct btree_trans *trans, u32 snapshot, &lostfound_str, lostfound->bi_inum, &lostfound->bi_dir_offset, - BCH_HASH_SET_MUST_CREATE) ?: + STR_HASH_must_create) ?: bch2_inode_write_flags(trans, &lostfound_iter, lostfound, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + BTREE_UPDATE_internal_snapshot_node); err: bch_err_msg(c, ret, "creating lost+found"); bch2_trans_iter_exit(trans, &lostfound_iter); @@ -333,7 +333,7 @@ static int reattach_inode(struct btree_trans *trans, &name, inode->bi_subvol ?: inode->bi_inum, &dir_offset, - BCH_HASH_SET_MUST_CREATE); + STR_HASH_must_create); if (ret) return ret; @@ -708,7 +708,7 @@ static int get_inodes_all_snapshots(struct btree_trans *trans, w->inodes.nr = 0; for_each_btree_key_norestart(trans, iter, BTREE_ID_inodes, POS(0, inum), - BTREE_ITER_ALL_SNAPSHOTS, k, ret) { + BTREE_ITER_all_snapshots, k, ret) { if (k.k->p.offset != inum) break; @@ -799,7 +799,7 @@ static int __get_visible_inodes(struct btree_trans *trans, w->inodes.nr = 0; for_each_btree_key_norestart(trans, iter, BTREE_ID_inodes, POS(0, inum), - BTREE_ITER_ALL_SNAPSHOTS, k, ret) { + BTREE_ITER_all_snapshots, k, ret) { u32 equiv = bch2_snapshot_equiv(c, k.k->p.snapshot); if (k.k->p.offset != inum) @@ -832,7 +832,7 @@ static int check_key_has_snapshot(struct btree_trans *trans, "key in missing snapshot: %s", (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) ret = bch2_btree_delete_at(trans, iter, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) ?: 1; + BTREE_UPDATE_internal_snapshot_node) ?: 1; fsck_err: printbuf_exit(&buf); return ret; @@ -861,8 +861,8 @@ static int hash_redo_key(struct btree_trans *trans, bch2_hash_set_in_snapshot(trans, desc, hash_info, (subvol_inum) { 0, k.k->p.inode }, k.k->p.snapshot, tmp, - BCH_HASH_SET_MUST_CREATE, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) ?: + STR_HASH_must_create| + BTREE_UPDATE_internal_snapshot_node) ?: bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc); } @@ -891,7 +891,7 @@ static int hash_check_key(struct btree_trans *trans, for_each_btree_key_norestart(trans, iter, desc.btree_id, SPOS(hash_k.k->p.inode, hash, hash_k.k->p.snapshot), - BTREE_ITER_SLOTS, k, ret) { + BTREE_ITER_slots, k, ret) { if (bkey_eq(k.k->p, hash_k.k->p)) break; @@ -1233,7 +1233,7 @@ int bch2_check_inodes(struct bch_fs *c) int ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_inodes, POS_MIN, - BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, k, + BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, check_inode(trans, &iter, k, &prev, &s, full))); @@ -1362,8 +1362,8 @@ static int overlapping_extents_found(struct btree_trans *trans, BUG_ON(bkey_le(pos1, bkey_start_pos(&pos2))); bch2_trans_iter_init(trans, &iter1, btree, pos1, - BTREE_ITER_ALL_SNAPSHOTS| - BTREE_ITER_NOT_EXTENTS); + BTREE_ITER_all_snapshots| + BTREE_ITER_not_extents); k1 = bch2_btree_iter_peek_upto(&iter1, POS(pos1.inode, U64_MAX)); ret = bkey_err(k1); if (ret) @@ -1425,7 +1425,7 @@ static int overlapping_extents_found(struct btree_trans *trans, trans->extra_disk_res += bch2_bkey_sectors_compressed(k2); ret = bch2_trans_update_extent_overwrite(trans, old_iter, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE, + BTREE_UPDATE_internal_snapshot_node, k1, k2) ?: bch2_trans_commit(trans, &res, NULL, BCH_TRANS_COMMIT_no_enospc); bch2_disk_reservation_put(c, &res); @@ -1625,7 +1625,7 @@ static int check_extent(struct btree_trans *trans, struct btree_iter *iter, bch2_btree_iter_set_snapshot(&iter2, i->snapshot); ret = bch2_btree_iter_traverse(&iter2) ?: bch2_btree_delete_at(trans, &iter2, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + BTREE_UPDATE_internal_snapshot_node); bch2_trans_iter_exit(trans, &iter2); if (ret) goto err; @@ -1652,7 +1652,7 @@ static int check_extent(struct btree_trans *trans, struct btree_iter *iter, bch_err_fn(c, ret); return ret; delete: - ret = bch2_btree_delete_at(trans, iter, BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + ret = bch2_btree_delete_at(trans, iter, BTREE_UPDATE_internal_snapshot_node); goto out; } @@ -1673,7 +1673,7 @@ int bch2_check_extents(struct bch_fs *c) int ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_extents, POS(BCACHEFS_ROOT_INO, 0), - BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, k, + BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, &res, NULL, BCH_TRANS_COMMIT_no_enospc, ({ bch2_disk_reservation_put(c, &res); @@ -1698,7 +1698,7 @@ int bch2_check_indirect_extents(struct bch_fs *c) int ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_reflink, POS_MIN, - BTREE_ITER_PREFETCH, k, + BTREE_ITER_prefetch, k, &res, NULL, BCH_TRANS_COMMIT_no_enospc, ({ bch2_disk_reservation_put(c, &res); @@ -2104,7 +2104,7 @@ static int check_dirent(struct btree_trans *trans, struct btree_iter *iter, (printbuf_reset(&buf), bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { ret = bch2_btree_delete_at(trans, iter, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + BTREE_UPDATE_internal_snapshot_node); goto out; } @@ -2191,7 +2191,7 @@ int bch2_check_dirents(struct bch_fs *c) int ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_dirents, POS(BCACHEFS_ROOT_INO, 0), - BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, + BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, @@ -2255,7 +2255,7 @@ int bch2_check_xattrs(struct bch_fs *c) ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_xattrs, POS(BCACHEFS_ROOT_INO, 0), - BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, + BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, @@ -2422,7 +2422,7 @@ int bch2_check_subvolume_structure(struct bch_fs *c) { int ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, - BTREE_ID_subvolumes, POS_MIN, BTREE_ITER_PREFETCH, k, + BTREE_ID_subvolumes, POS_MIN, BTREE_ITER_prefetch, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, check_subvol_path(trans, &iter, k))); bch_err_fn(c, ret); @@ -2559,9 +2559,9 @@ int bch2_check_directory_structure(struct bch_fs *c) ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_inodes, POS_MIN, - BTREE_ITER_INTENT| - BTREE_ITER_PREFETCH| - BTREE_ITER_ALL_SNAPSHOTS, k, + BTREE_ITER_intent| + BTREE_ITER_prefetch| + BTREE_ITER_all_snapshots, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, ({ if (!bkey_is_inode(k.k)) continue; @@ -2661,9 +2661,9 @@ static int check_nlinks_find_hardlinks(struct bch_fs *c, int ret = bch2_trans_run(c, for_each_btree_key(trans, iter, BTREE_ID_inodes, POS(0, start), - BTREE_ITER_INTENT| - BTREE_ITER_PREFETCH| - BTREE_ITER_ALL_SNAPSHOTS, k, ({ + BTREE_ITER_intent| + BTREE_ITER_prefetch| + BTREE_ITER_all_snapshots, k, ({ if (!bkey_is_inode(k.k)) continue; @@ -2704,9 +2704,9 @@ static int check_nlinks_walk_dirents(struct bch_fs *c, struct nlink_table *links int ret = bch2_trans_run(c, for_each_btree_key(trans, iter, BTREE_ID_dirents, POS_MIN, - BTREE_ITER_INTENT| - BTREE_ITER_PREFETCH| - BTREE_ITER_ALL_SNAPSHOTS, k, ({ + BTREE_ITER_intent| + BTREE_ITER_prefetch| + BTREE_ITER_all_snapshots, k, ({ ret = snapshots_seen_update(c, &s, iter.btree_id, k.k->p); if (ret) break; @@ -2781,7 +2781,7 @@ static int check_nlinks_update_hardlinks(struct bch_fs *c, int ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_inodes, POS(0, range_start), - BTREE_ITER_INTENT|BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, k, + BTREE_ITER_intent|BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, check_nlinks_update_inode(trans, &iter, k, links, &idx, range_end))); if (ret < 0) { @@ -2849,7 +2849,7 @@ static int fix_reflink_p_key(struct btree_trans *trans, struct btree_iter *iter, u->v.front_pad = 0; u->v.back_pad = 0; - return bch2_trans_update(trans, iter, &u->k_i, BTREE_TRIGGER_NORUN); + return bch2_trans_update(trans, iter, &u->k_i, BTREE_TRIGGER_norun); } int bch2_fix_reflink_p(struct bch_fs *c) @@ -2860,8 +2860,8 @@ int bch2_fix_reflink_p(struct bch_fs *c) int ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_extents, POS_MIN, - BTREE_ITER_INTENT|BTREE_ITER_PREFETCH| - BTREE_ITER_ALL_SNAPSHOTS, k, + BTREE_ITER_intent|BTREE_ITER_prefetch| + BTREE_ITER_all_snapshots, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, fix_reflink_p_key(trans, &iter, k))); bch_err_fn(c, ret); diff --git a/fs/bcachefs/inode.c b/fs/bcachefs/inode.c index dc72c59773fec..fc8b30441bbc0 100644 --- a/fs/bcachefs/inode.c +++ b/fs/bcachefs/inode.c @@ -339,7 +339,7 @@ int bch2_inode_peek_nowarn(struct btree_trans *trans, k = bch2_bkey_get_iter(trans, iter, BTREE_ID_inodes, SPOS(0, inum.inum, snapshot), - flags|BTREE_ITER_CACHED); + flags|BTREE_ITER_cached); ret = bkey_err(k); if (ret) return ret; @@ -371,7 +371,7 @@ int bch2_inode_peek(struct btree_trans *trans, int bch2_inode_write_flags(struct btree_trans *trans, struct btree_iter *iter, struct bch_inode_unpacked *inode, - enum btree_update_flags flags) + enum btree_iter_update_trigger_flags flags) { struct bkey_inode_buf *inode_p; @@ -399,7 +399,7 @@ int __bch2_fsck_write_inode(struct btree_trans *trans, return bch2_btree_insert_nonextent(trans, BTREE_ID_inodes, &inode_p->inode.k_i, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + BTREE_UPDATE_internal_snapshot_node); } int bch2_fsck_write_inode(struct btree_trans *trans, @@ -598,7 +598,7 @@ int bch2_trigger_inode(struct btree_trans *trans, { s64 nr = (s64) bkey_is_inode(new.k) - (s64) bkey_is_inode(old.k); - if (flags & BTREE_TRIGGER_TRANSACTIONAL) { + if (flags & BTREE_TRIGGER_transactional) { if (nr) { int ret = bch2_replicas_deltas_realloc(trans, 0); if (ret) @@ -617,13 +617,13 @@ int bch2_trigger_inode(struct btree_trans *trans, } } - if ((flags & BTREE_TRIGGER_ATOMIC) && (flags & BTREE_TRIGGER_INSERT)) { + if ((flags & BTREE_TRIGGER_atomic) && (flags & BTREE_TRIGGER_insert)) { BUG_ON(!trans->journal_res.seq); bkey_s_to_inode_v3(new).v->bi_journal_seq = cpu_to_le64(trans->journal_res.seq); } - if (flags & BTREE_TRIGGER_GC) { + if (flags & BTREE_TRIGGER_gc) { struct bch_fs *c = trans->c; percpu_down_read(&c->mark_lock); @@ -752,8 +752,8 @@ int bch2_inode_create(struct btree_trans *trans, pos = start; bch2_trans_iter_init(trans, iter, BTREE_ID_inodes, POS(0, pos), - BTREE_ITER_ALL_SNAPSHOTS| - BTREE_ITER_INTENT); + BTREE_ITER_all_snapshots| + BTREE_ITER_intent); again: while ((k = bch2_btree_iter_peek(iter)).k && !(ret = bkey_err(k)) && @@ -814,7 +814,7 @@ static int bch2_inode_delete_keys(struct btree_trans *trans, * extent iterator: */ bch2_trans_iter_init(trans, &iter, id, POS(inum.inum, 0), - BTREE_ITER_INTENT); + BTREE_ITER_intent); while (1) { bch2_trans_begin(trans); @@ -836,7 +836,7 @@ static int bch2_inode_delete_keys(struct btree_trans *trans, bkey_init(&delete.k); delete.k.p = iter.pos; - if (iter.flags & BTREE_ITER_IS_EXTENTS) + if (iter.flags & BTREE_ITER_is_extents) bch2_key_resize(&delete.k, bpos_min(end, k.k->p).offset - iter.pos.offset); @@ -885,7 +885,7 @@ int bch2_inode_rm(struct bch_fs *c, subvol_inum inum) k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_inodes, SPOS(0, inum.inum, snapshot), - BTREE_ITER_INTENT|BTREE_ITER_CACHED); + BTREE_ITER_intent|BTREE_ITER_cached); ret = bkey_err(k); if (ret) goto err; @@ -1045,7 +1045,7 @@ int bch2_inode_rm_snapshot(struct btree_trans *trans, u64 inum, u32 snapshot) bch2_trans_begin(trans); k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_inodes, - SPOS(0, inum, snapshot), BTREE_ITER_INTENT); + SPOS(0, inum, snapshot), BTREE_ITER_intent); ret = bkey_err(k); if (ret) goto err; @@ -1090,7 +1090,7 @@ static int may_delete_deleted_inode(struct btree_trans *trans, struct bch_inode_unpacked inode; int ret; - k = bch2_bkey_get_iter(trans, &inode_iter, BTREE_ID_inodes, pos, BTREE_ITER_CACHED); + k = bch2_bkey_get_iter(trans, &inode_iter, BTREE_ID_inodes, pos, BTREE_ITER_cached); ret = bkey_err(k); if (ret) return ret; @@ -1142,7 +1142,7 @@ static int may_delete_deleted_inode(struct btree_trans *trans, inode.bi_flags &= ~BCH_INODE_unlinked; ret = bch2_inode_write_flags(trans, &inode_iter, &inode, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + BTREE_UPDATE_internal_snapshot_node); bch_err_msg(c, ret, "clearing inode unlinked flag"); if (ret) goto out; @@ -1189,7 +1189,7 @@ int bch2_delete_dead_inodes(struct bch_fs *c) * flushed and we'd spin: */ ret = for_each_btree_key_commit(trans, iter, BTREE_ID_deleted_inodes, POS_MIN, - BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, k, + BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, ({ ret = may_delete_deleted_inode(trans, &iter, k.k->p, &need_another_pass); if (ret > 0) { diff --git a/fs/bcachefs/inode.h b/fs/bcachefs/inode.h index 056298050550f..08e8505dfa33c 100644 --- a/fs/bcachefs/inode.h +++ b/fs/bcachefs/inode.h @@ -101,7 +101,7 @@ int bch2_inode_peek(struct btree_trans *, struct btree_iter *, struct bch_inode_unpacked *, subvol_inum, unsigned); int bch2_inode_write_flags(struct btree_trans *, struct btree_iter *, - struct bch_inode_unpacked *, enum btree_update_flags); + struct bch_inode_unpacked *, enum btree_iter_update_trigger_flags); static inline int bch2_inode_write(struct btree_trans *trans, struct btree_iter *iter, diff --git a/fs/bcachefs/io_misc.c b/fs/bcachefs/io_misc.c index 82f9170dab3fd..4ec979b4b23e4 100644 --- a/fs/bcachefs/io_misc.c +++ b/fs/bcachefs/io_misc.c @@ -198,7 +198,7 @@ int bch2_fpunch(struct bch_fs *c, subvol_inum inum, u64 start, u64 end, bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, POS(inum.inum, start), - BTREE_ITER_INTENT); + BTREE_ITER_intent); ret = bch2_fpunch_at(trans, &iter, inum, end, i_sectors_delta); @@ -230,7 +230,7 @@ static int truncate_set_isize(struct btree_trans *trans, struct bch_inode_unpacked inode_u; int ret; - ret = bch2_inode_peek(trans, &iter, &inode_u, inum, BTREE_ITER_INTENT) ?: + ret = bch2_inode_peek(trans, &iter, &inode_u, inum, BTREE_ITER_intent) ?: (inode_u.bi_size = new_i_size, 0) ?: bch2_inode_write(trans, &iter, &inode_u); @@ -256,7 +256,7 @@ static int __bch2_resume_logged_op_truncate(struct btree_trans *trans, bch2_trans_iter_init(trans, &fpunch_iter, BTREE_ID_extents, POS(inum.inum, round_up(new_i_size, block_bytes(c)) >> 9), - BTREE_ITER_INTENT); + BTREE_ITER_intent); ret = bch2_fpunch_at(trans, &fpunch_iter, inum, U64_MAX, i_sectors_delta); bch2_trans_iter_exit(trans, &fpunch_iter); @@ -317,7 +317,7 @@ static int adjust_i_size(struct btree_trans *trans, subvol_inum inum, u64 offset offset <<= 9; len <<= 9; - ret = bch2_inode_peek(trans, &iter, &inode_u, inum, BTREE_ITER_INTENT); + ret = bch2_inode_peek(trans, &iter, &inode_u, inum, BTREE_ITER_intent); if (ret) return ret; @@ -365,7 +365,7 @@ static int __bch2_resume_logged_op_finsert(struct btree_trans *trans, bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, POS(inum.inum, 0), - BTREE_ITER_INTENT); + BTREE_ITER_intent); switch (op->v.state) { case LOGGED_OP_FINSERT_start: diff --git a/fs/bcachefs/io_read.c b/fs/bcachefs/io_read.c index c07902b6c759f..475ab6c02dd16 100644 --- a/fs/bcachefs/io_read.c +++ b/fs/bcachefs/io_read.c @@ -378,7 +378,7 @@ static void bch2_read_retry_nodecode(struct bch_fs *c, struct bch_read_bio *rbio bch2_bkey_buf_init(&sk); bch2_trans_iter_init(trans, &iter, rbio->data_btree, - rbio->read_pos, BTREE_ITER_SLOTS); + rbio->read_pos, BTREE_ITER_slots); retry: rbio->bio.bi_status = 0; @@ -487,7 +487,7 @@ static int __bch2_rbio_narrow_crcs(struct btree_trans *trans, return 0; k = bch2_bkey_get_iter(trans, &iter, rbio->data_btree, rbio->data_pos, - BTREE_ITER_SLOTS|BTREE_ITER_INTENT); + BTREE_ITER_slots|BTREE_ITER_intent); if ((ret = bkey_err(k))) goto out; @@ -523,7 +523,7 @@ static int __bch2_rbio_narrow_crcs(struct btree_trans *trans, goto out; ret = bch2_trans_update(trans, &iter, new, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + BTREE_UPDATE_internal_snapshot_node); out: bch2_trans_iter_exit(trans, &iter); return ret; @@ -769,7 +769,7 @@ static noinline void read_from_stale_dirty_pointer(struct btree_trans *trans, bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc, PTR_BUCKET_POS(c, &ptr), - BTREE_ITER_CACHED); + BTREE_ITER_cached); prt_printf(&buf, "Attempting to read from stale dirty pointer:\n"); printbuf_indent_add(&buf, 2); @@ -1112,7 +1112,7 @@ void __bch2_read(struct bch_fs *c, struct bch_read_bio *rbio, bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, SPOS(inum.inum, bvec_iter.bi_sector, snapshot), - BTREE_ITER_SLOTS); + BTREE_ITER_slots); while (1) { unsigned bytes, sectors, offset_into_extent; enum btree_id data_btree = BTREE_ID_extents; diff --git a/fs/bcachefs/io_write.c b/fs/bcachefs/io_write.c index c11e0944b0698..9826772c0eb4c 100644 --- a/fs/bcachefs/io_write.c +++ b/fs/bcachefs/io_write.c @@ -166,7 +166,7 @@ int bch2_sum_sector_overwrites(struct btree_trans *trans, bch2_trans_copy_iter(&iter, extent_iter); for_each_btree_key_upto_continue_norestart(iter, - new->k.p, BTREE_ITER_SLOTS, old, ret) { + new->k.p, BTREE_ITER_slots, old, ret) { s64 sectors = min(new->k.p.offset, old.k->p.offset) - max(bkey_start_offset(&new->k), bkey_start_offset(old.k)); @@ -210,14 +210,14 @@ static inline int bch2_extent_update_i_size_sectors(struct btree_trans *trans, * to be journalled - if we crash, the bi_journal_seq update will be * lost, but that's fine. */ - unsigned inode_update_flags = BTREE_UPDATE_NOJOURNAL; + unsigned inode_update_flags = BTREE_UPDATE_nojournal; struct btree_iter iter; struct bkey_s_c k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_inodes, SPOS(0, extent_iter->pos.inode, extent_iter->snapshot), - BTREE_ITER_CACHED); + BTREE_ITER_cached); int ret = bkey_err(k); if (unlikely(ret)) return ret; @@ -259,7 +259,7 @@ static inline int bch2_extent_update_i_size_sectors(struct btree_trans *trans, } ret = bch2_trans_update(trans, &iter, &inode->k_i, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE| + BTREE_UPDATE_internal_snapshot_node| inode_update_flags); err: bch2_trans_iter_exit(trans, &iter); @@ -368,7 +368,7 @@ static int bch2_write_index_default(struct bch_write_op *op) bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, bkey_start_pos(&sk.k->k), - BTREE_ITER_SLOTS|BTREE_ITER_INTENT); + BTREE_ITER_slots|BTREE_ITER_intent); ret = bch2_bkey_set_needs_rebalance(c, sk.k, &op->opts) ?: bch2_extent_update(trans, inum, &iter, sk.k, @@ -1158,7 +1158,7 @@ static int bch2_nocow_write_convert_one_unwritten(struct btree_trans *trans, return bch2_extent_update_i_size_sectors(trans, iter, min(new->k.p.offset << 9, new_i_size), 0) ?: bch2_trans_update(trans, iter, new, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + BTREE_UPDATE_internal_snapshot_node); } static void bch2_nocow_write_convert_unwritten(struct bch_write_op *op) @@ -1169,7 +1169,7 @@ static void bch2_nocow_write_convert_unwritten(struct bch_write_op *op) for_each_keylist_key(&op->insert_keys, orig) { int ret = for_each_btree_key_upto_commit(trans, iter, BTREE_ID_extents, bkey_start_pos(&orig->k), orig->k.p, - BTREE_ITER_INTENT, k, + BTREE_ITER_intent, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, ({ bch2_nocow_write_convert_one_unwritten(trans, &iter, orig, k, op->new_i_size); })); @@ -1242,7 +1242,7 @@ static void bch2_nocow_write(struct bch_write_op *op) bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, SPOS(op->pos.inode, op->pos.offset, snapshot), - BTREE_ITER_SLOTS); + BTREE_ITER_slots); while (1) { struct bio *bio = &op->wbio.bio; diff --git a/fs/bcachefs/journal.c b/fs/bcachefs/journal.c index fc02cb4ee74ff..1b62034e197d6 100644 --- a/fs/bcachefs/journal.c +++ b/fs/bcachefs/journal.c @@ -946,7 +946,7 @@ static int __bch2_set_nr_journal_buckets(struct bch_dev *ca, unsigned nr, ret = bch2_trans_run(c, bch2_trans_mark_metadata_bucket(trans, ca, ob[nr_got]->bucket, BCH_DATA_journal, - ca->mi.bucket_size, BTREE_TRIGGER_TRANSACTIONAL)); + ca->mi.bucket_size, BTREE_TRIGGER_transactional)); if (ret) { bch2_open_bucket_put(c, ob[nr_got]); bch_err_msg(c, ret, "marking new journal buckets"); @@ -1027,7 +1027,7 @@ static int __bch2_set_nr_journal_buckets(struct bch_dev *ca, unsigned nr, bch2_trans_run(c, bch2_trans_mark_metadata_bucket(trans, ca, bu[i], BCH_DATA_free, 0, - BTREE_TRIGGER_TRANSACTIONAL)); + BTREE_TRIGGER_transactional)); err_free: if (!new_fs) for (i = 0; i < nr_got; i++) diff --git a/fs/bcachefs/journal_seq_blacklist.c b/fs/bcachefs/journal_seq_blacklist.c index 37a024e034d49..c87c68ae27f43 100644 --- a/fs/bcachefs/journal_seq_blacklist.c +++ b/fs/bcachefs/journal_seq_blacklist.c @@ -233,7 +233,7 @@ void bch2_blacklist_entries_gc(struct work_struct *work) struct btree *b; bch2_trans_node_iter_init(trans, &iter, i, POS_MIN, - 0, 0, BTREE_ITER_PREFETCH); + 0, 0, BTREE_ITER_prefetch); retry: bch2_trans_begin(trans); diff --git a/fs/bcachefs/logged_ops.c b/fs/bcachefs/logged_ops.c index b82f8209041ff..f49fdca1d07dc 100644 --- a/fs/bcachefs/logged_ops.c +++ b/fs/bcachefs/logged_ops.c @@ -56,7 +56,7 @@ int bch2_resume_logged_ops(struct bch_fs *c) int ret = bch2_trans_run(c, for_each_btree_key(trans, iter, BTREE_ID_logged_ops, POS_MIN, - BTREE_ITER_PREFETCH, k, + BTREE_ITER_prefetch, k, resume_logged_op(trans, &iter, k))); bch_err_fn(c, ret); return ret; diff --git a/fs/bcachefs/lru.c b/fs/bcachefs/lru.c index 26569043e3680..19285fcd252c5 100644 --- a/fs/bcachefs/lru.c +++ b/fs/bcachefs/lru.c @@ -149,7 +149,7 @@ int bch2_check_lrus(struct bch_fs *c) struct bpos last_flushed_pos = POS_MIN; int ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, - BTREE_ID_lru, POS_MIN, BTREE_ITER_PREFETCH, k, + BTREE_ID_lru, POS_MIN, BTREE_ITER_prefetch, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc|BCH_TRANS_COMMIT_lazy_rw, bch2_check_lru_key(trans, &iter, k, &last_flushed_pos))); bch_err_fn(c, ret); diff --git a/fs/bcachefs/migrate.c b/fs/bcachefs/migrate.c index 69098eeb5d48e..ddc187fb693d1 100644 --- a/fs/bcachefs/migrate.c +++ b/fs/bcachefs/migrate.c @@ -49,7 +49,7 @@ static int bch2_dev_usrdata_drop_key(struct btree_trans *trans, if (!bch2_bkey_has_device_c(k, dev_idx)) return 0; - n = bch2_bkey_make_mut(trans, iter, &k, BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + n = bch2_bkey_make_mut(trans, iter, &k, BTREE_UPDATE_internal_snapshot_node); ret = PTR_ERR_OR_ZERO(n); if (ret) return ret; @@ -67,7 +67,7 @@ static int bch2_dev_usrdata_drop_key(struct btree_trans *trans, /* * Since we're not inserting through an extent iterator - * (BTREE_ITER_ALL_SNAPSHOTS iterators aren't extent iterators), + * (BTREE_ITER_all_snapshots iterators aren't extent iterators), * we aren't using the extent overwrite path to delete, we're * just using the normal key deletion path: */ @@ -87,7 +87,7 @@ static int bch2_dev_usrdata_drop(struct bch_fs *c, unsigned dev_idx, int flags) continue; ret = for_each_btree_key_commit(trans, iter, id, POS_MIN, - BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, k, + BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, bch2_dev_usrdata_drop_key(trans, &iter, k, dev_idx, flags)); if (ret) @@ -119,7 +119,7 @@ static int bch2_dev_metadata_drop(struct bch_fs *c, unsigned dev_idx, int flags) for (id = 0; id < BTREE_ID_NR; id++) { bch2_trans_node_iter_init(trans, &iter, id, POS_MIN, 0, 0, - BTREE_ITER_PREFETCH); + BTREE_ITER_prefetch); retry: ret = 0; while (bch2_trans_begin(trans), diff --git a/fs/bcachefs/move.c b/fs/bcachefs/move.c index e491203f21c1c..db383dc239bb0 100644 --- a/fs/bcachefs/move.c +++ b/fs/bcachefs/move.c @@ -416,7 +416,7 @@ struct bch_io_opts *bch2_move_get_io_opts(struct btree_trans *trans, io_opts->d.nr = 0; ret = for_each_btree_key(trans, iter, BTREE_ID_inodes, POS(0, extent_k.k->p.inode), - BTREE_ITER_ALL_SNAPSHOTS, k, ({ + BTREE_ITER_all_snapshots, k, ({ if (k.k->p.offset != extent_k.k->p.inode) break; @@ -462,7 +462,7 @@ int bch2_move_get_io_opts_one(struct btree_trans *trans, k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_inodes, SPOS(0, extent_k.k->p.inode, extent_k.k->p.snapshot), - BTREE_ITER_CACHED); + BTREE_ITER_cached); ret = bkey_err(k); if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) return ret; @@ -548,8 +548,8 @@ static int bch2_move_data_btree(struct moving_context *ctxt, } bch2_trans_iter_init(trans, &iter, btree_id, start, - BTREE_ITER_PREFETCH| - BTREE_ITER_ALL_SNAPSHOTS); + BTREE_ITER_prefetch| + BTREE_ITER_all_snapshots); if (ctxt->rate) bch2_ratelimit_reset(ctxt->rate); @@ -700,7 +700,7 @@ int bch2_evacuate_bucket(struct moving_context *ctxt, bch2_trans_begin(trans); bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc, - bucket, BTREE_ITER_CACHED); + bucket, BTREE_ITER_cached); ret = lockrestart_do(trans, bkey_err(k = bch2_btree_iter_peek_slot(&iter))); bch2_trans_iter_exit(trans, &iter); @@ -727,7 +727,7 @@ int bch2_evacuate_bucket(struct moving_context *ctxt, ret = bch2_get_next_backpointer(trans, bucket, gen, &bp_pos, &bp, - BTREE_ITER_CACHED); + BTREE_ITER_cached); if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) continue; if (ret) @@ -863,7 +863,7 @@ static int bch2_move_btree(struct bch_fs *c, continue; bch2_trans_node_iter_init(trans, &iter, btree, POS_MIN, 0, 0, - BTREE_ITER_PREFETCH); + BTREE_ITER_prefetch); retry: ret = 0; while (bch2_trans_begin(trans), diff --git a/fs/bcachefs/movinggc.c b/fs/bcachefs/movinggc.c index 0d2b82d8d11f3..d0afd8bc0193b 100644 --- a/fs/bcachefs/movinggc.c +++ b/fs/bcachefs/movinggc.c @@ -84,7 +84,7 @@ static int bch2_bucket_is_movable(struct btree_trans *trans, return 0; k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_alloc, - b->k.bucket, BTREE_ITER_CACHED); + b->k.bucket, BTREE_ITER_cached); ret = bkey_err(k); if (ret) return ret; diff --git a/fs/bcachefs/opts.h b/fs/bcachefs/opts.h index 84e452835a17d..74f4d92849bb6 100644 --- a/fs/bcachefs/opts.h +++ b/fs/bcachefs/opts.h @@ -480,7 +480,7 @@ enum fsck_err_opts { OPT_FS|OPT_MOUNT|OPT_RUNTIME, \ OPT_BOOL(), \ BCH2_NO_SB_OPT, true, \ - NULL, "BTREE_ITER_PREFETCH casuse btree nodes to be\n"\ + NULL, "BTREE_ITER_prefetch casuse btree nodes to be\n"\ " prefetched sequentially") struct bch_opts { diff --git a/fs/bcachefs/quota.c b/fs/bcachefs/quota.c index 5660798c64533..9d7498e774e20 100644 --- a/fs/bcachefs/quota.c +++ b/fs/bcachefs/quota.c @@ -536,10 +536,10 @@ int bch2_fs_quota_read(struct bch_fs *c) int ret = bch2_trans_run(c, for_each_btree_key(trans, iter, BTREE_ID_quotas, POS_MIN, - BTREE_ITER_PREFETCH, k, + BTREE_ITER_prefetch, k, __bch2_quota_set(c, k, NULL)) ?: for_each_btree_key(trans, iter, BTREE_ID_inodes, POS_MIN, - BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, k, + BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, bch2_fs_quota_read_inode(trans, &iter, k))); bch_err_fn(c, ret); return ret; @@ -826,7 +826,7 @@ static int bch2_set_quota_trans(struct btree_trans *trans, int ret; k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_quotas, new_quota->k.p, - BTREE_ITER_SLOTS|BTREE_ITER_INTENT); + BTREE_ITER_slots|BTREE_ITER_intent); ret = bkey_err(k); if (unlikely(ret)) return ret; diff --git a/fs/bcachefs/rebalance.c b/fs/bcachefs/rebalance.c index 56336f3dd1d07..c640c0f13976c 100644 --- a/fs/bcachefs/rebalance.c +++ b/fs/bcachefs/rebalance.c @@ -42,7 +42,7 @@ static int __bch2_set_rebalance_needs_scan(struct btree_trans *trans, u64 inum) bch2_trans_iter_init(trans, &iter, BTREE_ID_rebalance_work, SPOS(inum, REBALANCE_WORK_SCAN_OFFSET, U32_MAX), - BTREE_ITER_INTENT); + BTREE_ITER_intent); k = bch2_btree_iter_peek_slot(&iter); ret = bkey_err(k); if (ret) @@ -89,7 +89,7 @@ static int bch2_clear_rebalance_needs_scan(struct btree_trans *trans, u64 inum, bch2_trans_iter_init(trans, &iter, BTREE_ID_rebalance_work, SPOS(inum, REBALANCE_WORK_SCAN_OFFSET, U32_MAX), - BTREE_ITER_INTENT); + BTREE_ITER_intent); k = bch2_btree_iter_peek_slot(&iter); ret = bkey_err(k); if (ret) @@ -140,7 +140,7 @@ static struct bkey_s_c next_rebalance_extent(struct btree_trans *trans, bch2_trans_iter_init(trans, extent_iter, work_pos.inode ? BTREE_ID_extents : BTREE_ID_reflink, work_pos, - BTREE_ITER_ALL_SNAPSHOTS); + BTREE_ITER_all_snapshots); k = bch2_btree_iter_peek_slot(extent_iter); if (bkey_err(k)) return k; @@ -328,7 +328,7 @@ static int do_rebalance(struct moving_context *ctxt) bch2_trans_iter_init(trans, &rebalance_work_iter, BTREE_ID_rebalance_work, POS_MIN, - BTREE_ITER_ALL_SNAPSHOTS); + BTREE_ITER_all_snapshots); while (!bch2_move_ratelimit(ctxt)) { if (!r->enabled) { diff --git a/fs/bcachefs/recovery.c b/fs/bcachefs/recovery.c index d22579a036c74..e997217374c64 100644 --- a/fs/bcachefs/recovery.c +++ b/fs/bcachefs/recovery.c @@ -136,9 +136,9 @@ static int bch2_journal_replay_key(struct btree_trans *trans, { struct btree_iter iter; unsigned iter_flags = - BTREE_ITER_INTENT| - BTREE_ITER_NOT_EXTENTS; - unsigned update_flags = BTREE_TRIGGER_NORUN; + BTREE_ITER_intent| + BTREE_ITER_not_extents; + unsigned update_flags = BTREE_TRIGGER_norun; int ret; if (k->overwritten) @@ -147,17 +147,17 @@ static int bch2_journal_replay_key(struct btree_trans *trans, trans->journal_res.seq = k->journal_seq; /* - * BTREE_UPDATE_KEY_CACHE_RECLAIM disables key cache lookup/update to + * BTREE_UPDATE_key_cache_reclaim disables key cache lookup/update to * keep the key cache coherent with the underlying btree. Nothing * besides the allocator is doing updates yet so we don't need key cache * coherency for non-alloc btrees, and key cache fills for snapshots - * btrees use BTREE_ITER_FILTER_SNAPSHOTS, which isn't available until + * btrees use BTREE_ITER_filter_snapshots, which isn't available until * the snapshots recovery pass runs. */ if (!k->level && k->btree_id == BTREE_ID_alloc) - iter_flags |= BTREE_ITER_CACHED; + iter_flags |= BTREE_ITER_cached; else - update_flags |= BTREE_UPDATE_KEY_CACHE_RECLAIM; + update_flags |= BTREE_UPDATE_key_cache_reclaim; bch2_trans_node_iter_init(trans, &iter, k->btree_id, k->k->k.p, BTREE_MAX_DEPTH, k->level, diff --git a/fs/bcachefs/reflink.c b/fs/bcachefs/reflink.c index ff7864731a073..a0cf5d97513f1 100644 --- a/fs/bcachefs/reflink.c +++ b/fs/bcachefs/reflink.c @@ -74,20 +74,20 @@ bool bch2_reflink_p_merge(struct bch_fs *c, struct bkey_s _l, struct bkey_s_c _r } static int trans_trigger_reflink_p_segment(struct btree_trans *trans, - struct bkey_s_c_reflink_p p, - u64 *idx, unsigned flags) + struct bkey_s_c_reflink_p p, u64 *idx, + enum btree_iter_update_trigger_flags flags) { struct bch_fs *c = trans->c; struct btree_iter iter; struct bkey_i *k; __le64 *refcount; - int add = !(flags & BTREE_TRIGGER_OVERWRITE) ? 1 : -1; + int add = !(flags & BTREE_TRIGGER_overwrite) ? 1 : -1; struct printbuf buf = PRINTBUF; int ret; k = bch2_bkey_get_mut_noupdate(trans, &iter, BTREE_ID_reflink, POS(0, *idx), - BTREE_ITER_WITH_UPDATES); + BTREE_ITER_with_updates); ret = PTR_ERR_OR_ZERO(k); if (ret) goto err; @@ -102,7 +102,7 @@ static int trans_trigger_reflink_p_segment(struct btree_trans *trans, goto err; } - if (!*refcount && (flags & BTREE_TRIGGER_OVERWRITE)) { + if (!*refcount && (flags & BTREE_TRIGGER_overwrite)) { bch2_bkey_val_to_text(&buf, c, p.s_c); bch2_trans_inconsistent(trans, "indirect extent refcount underflow at %llu while marking\n %s", @@ -111,7 +111,7 @@ static int trans_trigger_reflink_p_segment(struct btree_trans *trans, goto err; } - if (flags & BTREE_TRIGGER_INSERT) { + if (flags & BTREE_TRIGGER_insert) { struct bch_reflink_p *v = (struct bch_reflink_p *) p.v; u64 pad; @@ -141,12 +141,13 @@ static int trans_trigger_reflink_p_segment(struct btree_trans *trans, } static s64 gc_trigger_reflink_p_segment(struct btree_trans *trans, - struct bkey_s_c_reflink_p p, - u64 *idx, unsigned flags, size_t r_idx) + struct bkey_s_c_reflink_p p, u64 *idx, + enum btree_iter_update_trigger_flags flags, + size_t r_idx) { struct bch_fs *c = trans->c; struct reflink_gc *r; - int add = !(flags & BTREE_TRIGGER_OVERWRITE) ? 1 : -1; + int add = !(flags & BTREE_TRIGGER_overwrite) ? 1 : -1; u64 start = le64_to_cpu(p.v->idx); u64 end = le64_to_cpu(p.v->idx) + p.k->size; u64 next_idx = end + le32_to_cpu(p.v->back_pad); @@ -189,7 +190,7 @@ static s64 gc_trigger_reflink_p_segment(struct btree_trans *trans, set_bkey_val_u64s(&update->k, 0); } - ret = bch2_btree_insert_trans(trans, BTREE_ID_extents, update, BTREE_TRIGGER_NORUN); + ret = bch2_btree_insert_trans(trans, BTREE_ID_extents, update, BTREE_TRIGGER_norun); } *idx = next_idx; @@ -200,8 +201,8 @@ static s64 gc_trigger_reflink_p_segment(struct btree_trans *trans, } static int __trigger_reflink_p(struct btree_trans *trans, - enum btree_id btree_id, unsigned level, - struct bkey_s_c k, unsigned flags) + enum btree_id btree_id, unsigned level, struct bkey_s_c k, + enum btree_iter_update_trigger_flags flags) { struct bch_fs *c = trans->c; struct bkey_s_c_reflink_p p = bkey_s_c_to_reflink_p(k); @@ -210,12 +211,12 @@ static int __trigger_reflink_p(struct btree_trans *trans, u64 idx = le64_to_cpu(p.v->idx) - le32_to_cpu(p.v->front_pad); u64 end = le64_to_cpu(p.v->idx) + p.k->size + le32_to_cpu(p.v->back_pad); - if (flags & BTREE_TRIGGER_TRANSACTIONAL) { + if (flags & BTREE_TRIGGER_transactional) { while (idx < end && !ret) ret = trans_trigger_reflink_p_segment(trans, p, &idx, flags); } - if (flags & BTREE_TRIGGER_GC) { + if (flags & BTREE_TRIGGER_gc) { size_t l = 0, r = c->reflink_gc_nr; while (l < r) { @@ -238,10 +239,10 @@ int bch2_trigger_reflink_p(struct btree_trans *trans, enum btree_id btree_id, unsigned level, struct bkey_s_c old, struct bkey_s new, - unsigned flags) + enum btree_iter_update_trigger_flags flags) { - if ((flags & BTREE_TRIGGER_TRANSACTIONAL) && - (flags & BTREE_TRIGGER_INSERT)) { + if ((flags & BTREE_TRIGGER_transactional) && + (flags & BTREE_TRIGGER_insert)) { struct bch_reflink_p *v = bkey_s_to_reflink_p(new).v; v->front_pad = v->back_pad = 0; @@ -283,21 +284,21 @@ bool bch2_reflink_v_merge(struct bch_fs *c, struct bkey_s _l, struct bkey_s_c _r static inline void check_indirect_extent_deleting(struct bkey_s new, unsigned *flags) { - if ((*flags & BTREE_TRIGGER_INSERT) && !*bkey_refcount(new)) { + if ((*flags & BTREE_TRIGGER_insert) && !*bkey_refcount(new)) { new.k->type = KEY_TYPE_deleted; new.k->size = 0; set_bkey_val_u64s(new.k, 0); - *flags &= ~BTREE_TRIGGER_INSERT; + *flags &= ~BTREE_TRIGGER_insert; } } int bch2_trigger_reflink_v(struct btree_trans *trans, enum btree_id btree_id, unsigned level, struct bkey_s_c old, struct bkey_s new, - unsigned flags) + enum btree_iter_update_trigger_flags flags) { - if ((flags & BTREE_TRIGGER_TRANSACTIONAL) && - (flags & BTREE_TRIGGER_INSERT)) + if ((flags & BTREE_TRIGGER_transactional) && + (flags & BTREE_TRIGGER_insert)) check_indirect_extent_deleting(new, &flags); return bch2_trigger_extent(trans, btree_id, level, old, new, flags); @@ -349,7 +350,7 @@ static int bch2_make_extent_indirect(struct btree_trans *trans, bch2_check_set_feature(c, BCH_FEATURE_reflink_inline_data); bch2_trans_iter_init(trans, &reflink_iter, BTREE_ID_reflink, POS_MAX, - BTREE_ITER_INTENT); + BTREE_ITER_intent); k = bch2_btree_iter_peek_prev(&reflink_iter); ret = bkey_err(k); if (ret) @@ -394,7 +395,7 @@ static int bch2_make_extent_indirect(struct btree_trans *trans, r_p->v.idx = cpu_to_le64(bkey_start_offset(&r_v->k)); ret = bch2_trans_update(trans, extent_iter, &r_p->k_i, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + BTREE_UPDATE_internal_snapshot_node); err: bch2_trans_iter_exit(trans, &reflink_iter); @@ -455,9 +456,9 @@ s64 bch2_remap_range(struct bch_fs *c, goto err; bch2_trans_iter_init(trans, &src_iter, BTREE_ID_extents, src_start, - BTREE_ITER_INTENT); + BTREE_ITER_intent); bch2_trans_iter_init(trans, &dst_iter, BTREE_ID_extents, dst_start, - BTREE_ITER_INTENT); + BTREE_ITER_intent); while ((ret == 0 || bch2_err_matches(ret, BCH_ERR_transaction_restart)) && @@ -567,7 +568,7 @@ s64 bch2_remap_range(struct bch_fs *c, bch2_trans_begin(trans); ret2 = bch2_inode_peek(trans, &inode_iter, &inode_u, - dst_inum, BTREE_ITER_INTENT); + dst_inum, BTREE_ITER_intent); if (!ret2 && inode_u.bi_size < new_i_size) { diff --git a/fs/bcachefs/reflink.h b/fs/bcachefs/reflink.h index 4d8867289717b..e34c4bdf7a76b 100644 --- a/fs/bcachefs/reflink.h +++ b/fs/bcachefs/reflink.h @@ -10,7 +10,8 @@ void bch2_reflink_p_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); bool bch2_reflink_p_merge(struct bch_fs *, struct bkey_s, struct bkey_s_c); int bch2_trigger_reflink_p(struct btree_trans *, enum btree_id, unsigned, - struct bkey_s_c, struct bkey_s, unsigned); + struct bkey_s_c, struct bkey_s, + enum btree_iter_update_trigger_flags); #define bch2_bkey_ops_reflink_p ((struct bkey_ops) { \ .key_invalid = bch2_reflink_p_invalid, \ @@ -25,7 +26,8 @@ int bch2_reflink_v_invalid(struct bch_fs *, struct bkey_s_c, void bch2_reflink_v_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); int bch2_trigger_reflink_v(struct btree_trans *, enum btree_id, unsigned, - struct bkey_s_c, struct bkey_s, unsigned); + struct bkey_s_c, struct bkey_s, + enum btree_iter_update_trigger_flags); #define bch2_bkey_ops_reflink_v ((struct bkey_ops) { \ .key_invalid = bch2_reflink_v_invalid, \ diff --git a/fs/bcachefs/snapshot.c b/fs/bcachefs/snapshot.c index c415ea0a649b0..0b26dee17a5ab 100644 --- a/fs/bcachefs/snapshot.c +++ b/fs/bcachefs/snapshot.c @@ -49,7 +49,7 @@ int bch2_snapshot_tree_lookup(struct btree_trans *trans, u32 id, struct bch_snapshot_tree *s) { int ret = bch2_bkey_get_val_typed(trans, BTREE_ID_snapshot_trees, POS(0, id), - BTREE_ITER_WITH_UPDATES, snapshot_tree, s); + BTREE_ITER_with_updates, snapshot_tree, s); if (bch2_err_matches(ret, ENOENT)) ret = -BCH_ERR_ENOENT_snapshot_tree; @@ -361,7 +361,7 @@ int bch2_snapshot_lookup(struct btree_trans *trans, u32 id, struct bch_snapshot *s) { return bch2_bkey_get_val_typed(trans, BTREE_ID_snapshots, POS(0, id), - BTREE_ITER_WITH_UPDATES, snapshot, s); + BTREE_ITER_with_updates, snapshot, s); } static int bch2_snapshot_live(struct btree_trans *trans, u32 id) @@ -618,7 +618,7 @@ int bch2_check_snapshot_trees(struct bch_fs *c) int ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_snapshot_trees, POS_MIN, - BTREE_ITER_PREFETCH, k, + BTREE_ITER_prefetch, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, check_snapshot_tree(trans, &iter, k))); bch_err_fn(c, ret); @@ -695,7 +695,7 @@ static int snapshot_tree_ptr_repair(struct btree_trans *trans, root = bch2_bkey_get_iter_typed(trans, &root_iter, BTREE_ID_snapshots, POS(0, root_id), - BTREE_ITER_WITH_UPDATES, snapshot); + BTREE_ITER_with_updates, snapshot); ret = bkey_err(root); if (ret) goto err; @@ -886,7 +886,7 @@ int bch2_check_snapshots(struct bch_fs *c) int ret = bch2_trans_run(c, for_each_btree_key_reverse_commit(trans, iter, BTREE_ID_snapshots, POS_MAX, - BTREE_ITER_PREFETCH, k, + BTREE_ITER_prefetch, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, check_snapshot(trans, &iter, k))); bch_err_fn(c, ret); @@ -1001,7 +1001,7 @@ int bch2_reconstruct_snapshots(struct bch_fs *c) r.btree = btree; ret = for_each_btree_key(trans, iter, btree, POS_MIN, - BTREE_ITER_ALL_SNAPSHOTS|BTREE_ITER_PREFETCH, k, ({ + BTREE_ITER_all_snapshots|BTREE_ITER_prefetch, k, ({ get_snapshot_trees(c, &r, k.k->p); })); if (ret) @@ -1090,7 +1090,7 @@ static int bch2_snapshot_node_delete(struct btree_trans *trans, u32 id) int ret = 0; s = bch2_bkey_get_iter_typed(trans, &iter, BTREE_ID_snapshots, POS(0, id), - BTREE_ITER_INTENT, snapshot); + BTREE_ITER_intent, snapshot); ret = bkey_err(s); bch2_fs_inconsistent_on(bch2_err_matches(ret, ENOENT), c, "missing snapshot %u", id); @@ -1199,7 +1199,7 @@ static int create_snapids(struct btree_trans *trans, u32 parent, u32 tree, int ret; bch2_trans_iter_init(trans, &iter, BTREE_ID_snapshots, - POS_MIN, BTREE_ITER_INTENT); + POS_MIN, BTREE_ITER_intent); k = bch2_btree_iter_peek(&iter); ret = bkey_err(k); if (ret) @@ -1367,7 +1367,7 @@ static int snapshot_delete_key(struct btree_trans *trans, if (snapshot_list_has_id(deleted, k.k->p.snapshot) || snapshot_list_has_id(equiv_seen, equiv)) { return bch2_btree_delete_at(trans, iter, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + BTREE_UPDATE_internal_snapshot_node); } else { return snapshot_list_add(c, equiv_seen, equiv); } @@ -1404,15 +1404,15 @@ static int move_key_to_correct_snapshot(struct btree_trans *trans, new->k.p.snapshot = equiv; bch2_trans_iter_init(trans, &new_iter, iter->btree_id, new->k.p, - BTREE_ITER_ALL_SNAPSHOTS| - BTREE_ITER_CACHED| - BTREE_ITER_INTENT); + BTREE_ITER_all_snapshots| + BTREE_ITER_cached| + BTREE_ITER_intent); ret = bch2_btree_iter_traverse(&new_iter) ?: bch2_trans_update(trans, &new_iter, new, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) ?: + BTREE_UPDATE_internal_snapshot_node) ?: bch2_btree_delete_at(trans, iter, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); + BTREE_UPDATE_internal_snapshot_node); bch2_trans_iter_exit(trans, &new_iter); if (ret) return ret; @@ -1603,12 +1603,12 @@ int bch2_delete_dead_snapshots(struct bch_fs *c) ret = for_each_btree_key_commit(trans, iter, id, POS_MIN, - BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, k, + BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, &res, NULL, BCH_TRANS_COMMIT_no_enospc, snapshot_delete_key(trans, &iter, k, &deleted, &equiv_seen, &last_pos)) ?: for_each_btree_key_commit(trans, iter, id, POS_MIN, - BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, k, + BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, &res, NULL, BCH_TRANS_COMMIT_no_enospc, move_key_to_correct_snapshot(trans, &iter, k)); @@ -1643,7 +1643,7 @@ int bch2_delete_dead_snapshots(struct bch_fs *c) * nodes some depth fields will be off: */ ret = for_each_btree_key_commit(trans, iter, BTREE_ID_snapshots, POS_MIN, - BTREE_ITER_INTENT, k, + BTREE_ITER_intent, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, bch2_fix_child_of_deleted_snapshot(trans, &iter, k, &deleted_interior)); if (ret) @@ -1699,8 +1699,8 @@ int __bch2_key_has_snapshot_overwrites(struct btree_trans *trans, int ret; bch2_trans_iter_init(trans, &iter, id, pos, - BTREE_ITER_NOT_EXTENTS| - BTREE_ITER_ALL_SNAPSHOTS); + BTREE_ITER_not_extents| + BTREE_ITER_all_snapshots); while (1) { k = bch2_btree_iter_prev(&iter); ret = bkey_err(k); @@ -1752,7 +1752,7 @@ static int bch2_propagate_key_to_snapshot_leaf(struct btree_trans *trans, pos.snapshot = leaf_id; - bch2_trans_iter_init(trans, &iter, btree, pos, BTREE_ITER_INTENT); + bch2_trans_iter_init(trans, &iter, btree, pos, BTREE_ITER_intent); k = bch2_btree_iter_peek_slot(&iter); ret = bkey_err(k); if (ret) diff --git a/fs/bcachefs/str_hash.h b/fs/bcachefs/str_hash.h index 11f5f5d27d23d..cbad9b27874fe 100644 --- a/fs/bcachefs/str_hash.h +++ b/fs/bcachefs/str_hash.h @@ -15,16 +15,6 @@ #include #include -typedef unsigned __bitwise bch_str_hash_flags_t; - -enum bch_str_hash_flags { - __BCH_HASH_SET_MUST_CREATE, - __BCH_HASH_SET_MUST_REPLACE, -}; - -#define BCH_HASH_SET_MUST_CREATE (__force bch_str_hash_flags_t) BIT(__BCH_HASH_SET_MUST_CREATE) -#define BCH_HASH_SET_MUST_REPLACE (__force bch_str_hash_flags_t) BIT(__BCH_HASH_SET_MUST_REPLACE) - static inline enum bch_str_hash_type bch2_str_hash_opt_to_type(struct bch_fs *c, enum bch_str_hash_opts opt) { @@ -165,7 +155,8 @@ bch2_hash_lookup_in_snapshot(struct btree_trans *trans, const struct bch_hash_desc desc, const struct bch_hash_info *info, subvol_inum inum, const void *key, - unsigned flags, u32 snapshot) + enum btree_iter_update_trigger_flags flags, + u32 snapshot) { struct bkey_s_c k; int ret; @@ -173,7 +164,7 @@ bch2_hash_lookup_in_snapshot(struct btree_trans *trans, for_each_btree_key_upto_norestart(trans, *iter, desc.btree_id, SPOS(inum.inum, desc.hash_key(info, key), snapshot), POS(inum.inum, U64_MAX), - BTREE_ITER_SLOTS|flags, k, ret) { + BTREE_ITER_slots|flags, k, ret) { if (is_visible_key(desc, inum, k)) { if (!desc.cmp_key(k, key)) return k; @@ -195,7 +186,7 @@ bch2_hash_lookup(struct btree_trans *trans, const struct bch_hash_desc desc, const struct bch_hash_info *info, subvol_inum inum, const void *key, - unsigned flags) + enum btree_iter_update_trigger_flags flags) { u32 snapshot; int ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot); @@ -223,7 +214,7 @@ bch2_hash_hole(struct btree_trans *trans, for_each_btree_key_upto_norestart(trans, *iter, desc.btree_id, SPOS(inum.inum, desc.hash_key(info, key), snapshot), POS(inum.inum, U64_MAX), - BTREE_ITER_SLOTS|BTREE_ITER_INTENT, k, ret) + BTREE_ITER_slots|BTREE_ITER_intent, k, ret) if (!is_visible_key(desc, inum, k)) return 0; bch2_trans_iter_exit(trans, iter); @@ -245,7 +236,7 @@ int bch2_hash_needs_whiteout(struct btree_trans *trans, bch2_btree_iter_advance(&iter); - for_each_btree_key_continue_norestart(iter, BTREE_ITER_SLOTS, k, ret) { + for_each_btree_key_continue_norestart(iter, BTREE_ITER_slots, k, ret) { if (k.k->type != desc.key_type && k.k->type != KEY_TYPE_hash_whiteout) break; @@ -267,8 +258,7 @@ int bch2_hash_set_in_snapshot(struct btree_trans *trans, const struct bch_hash_info *info, subvol_inum inum, u32 snapshot, struct bkey_i *insert, - bch_str_hash_flags_t str_hash_flags, - int update_flags) + enum btree_iter_update_trigger_flags flags) { struct btree_iter iter, slot = { NULL }; struct bkey_s_c k; @@ -280,7 +270,7 @@ int bch2_hash_set_in_snapshot(struct btree_trans *trans, desc.hash_bkey(info, bkey_i_to_s_c(insert)), snapshot), POS(insert->k.p.inode, U64_MAX), - BTREE_ITER_SLOTS|BTREE_ITER_INTENT, k, ret) { + BTREE_ITER_slots|BTREE_ITER_intent, k, ret) { if (is_visible_key(desc, inum, k)) { if (!desc.cmp_bkey(k, bkey_i_to_s_c(insert))) goto found; @@ -289,8 +279,7 @@ int bch2_hash_set_in_snapshot(struct btree_trans *trans, continue; } - if (!slot.path && - !(str_hash_flags & BCH_HASH_SET_MUST_REPLACE)) + if (!slot.path && !(flags & STR_HASH_must_replace)) bch2_trans_copy_iter(&slot, &iter); if (k.k->type != KEY_TYPE_hash_whiteout) @@ -308,16 +297,16 @@ int bch2_hash_set_in_snapshot(struct btree_trans *trans, found = true; not_found: - if (!found && (str_hash_flags & BCH_HASH_SET_MUST_REPLACE)) { + if (!found && (flags & STR_HASH_must_replace)) { ret = -BCH_ERR_ENOENT_str_hash_set_must_replace; - } else if (found && (str_hash_flags & BCH_HASH_SET_MUST_CREATE)) { + } else if (found && (flags & STR_HASH_must_create)) { ret = -EEXIST; } else { if (!found && slot.path) swap(iter, slot); insert->k.p = iter.pos; - ret = bch2_trans_update(trans, &iter, insert, update_flags); + ret = bch2_trans_update(trans, &iter, insert, flags); } goto out; @@ -329,14 +318,14 @@ int bch2_hash_set(struct btree_trans *trans, const struct bch_hash_info *info, subvol_inum inum, struct bkey_i *insert, - bch_str_hash_flags_t str_hash_flags) + enum btree_iter_update_trigger_flags flags) { insert->k.p.inode = inum.inum; u32 snapshot; return bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot) ?: bch2_hash_set_in_snapshot(trans, desc, info, inum, - snapshot, insert, str_hash_flags, 0); + snapshot, insert, flags); } static __always_inline @@ -344,7 +333,7 @@ int bch2_hash_delete_at(struct btree_trans *trans, const struct bch_hash_desc desc, const struct bch_hash_info *info, struct btree_iter *iter, - unsigned update_flags) + enum btree_iter_update_trigger_flags flags) { struct bkey_i *delete; int ret; @@ -362,7 +351,7 @@ int bch2_hash_delete_at(struct btree_trans *trans, delete->k.p = iter->pos; delete->k.type = ret ? KEY_TYPE_hash_whiteout : KEY_TYPE_deleted; - return bch2_trans_update(trans, iter, delete, update_flags); + return bch2_trans_update(trans, iter, delete, flags); } static __always_inline @@ -373,7 +362,7 @@ int bch2_hash_delete(struct btree_trans *trans, { struct btree_iter iter; struct bkey_s_c k = bch2_hash_lookup(trans, &iter, desc, info, inum, key, - BTREE_ITER_INTENT); + BTREE_ITER_intent); int ret = bkey_err(k) ?: bch2_hash_delete_at(trans, desc, info, &iter, 0); bch2_trans_iter_exit(trans, &iter); diff --git a/fs/bcachefs/subvolume.c b/fs/bcachefs/subvolume.c index 88a79c8232768..33292dae4d509 100644 --- a/fs/bcachefs/subvolume.c +++ b/fs/bcachefs/subvolume.c @@ -162,7 +162,7 @@ int bch2_check_subvols(struct bch_fs *c) { int ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, - BTREE_ID_subvolumes, POS_MIN, BTREE_ITER_PREFETCH, k, + BTREE_ID_subvolumes, POS_MIN, BTREE_ITER_prefetch, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, check_subvol(trans, &iter, k))); bch_err_fn(c, ret); @@ -198,7 +198,7 @@ int bch2_check_subvol_children(struct bch_fs *c) { int ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, - BTREE_ID_subvolume_children, POS_MIN, BTREE_ITER_PREFETCH, k, + BTREE_ID_subvolume_children, POS_MIN, BTREE_ITER_prefetch, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, check_subvol_child(trans, &iter, k))); bch_err_fn(c, ret); @@ -247,7 +247,7 @@ int bch2_subvolume_trigger(struct btree_trans *trans, struct bkey_s_c old, struct bkey_s new, unsigned flags) { - if (flags & BTREE_TRIGGER_TRANSACTIONAL) { + if (flags & BTREE_TRIGGER_transactional) { struct bpos children_pos_old = subvolume_children_pos(old); struct bpos children_pos_new = subvolume_children_pos(new.s_c); @@ -333,7 +333,7 @@ int bch2_subvolume_get_snapshot(struct btree_trans *trans, u32 subvolid, subvol = bch2_bkey_get_iter_typed(trans, &iter, BTREE_ID_subvolumes, POS(0, subvolid), - BTREE_ITER_CACHED|BTREE_ITER_WITH_UPDATES, + BTREE_ITER_cached|BTREE_ITER_with_updates, subvolume); ret = bkey_err(subvol); bch2_fs_inconsistent_on(bch2_err_matches(ret, ENOENT), trans->c, @@ -383,9 +383,9 @@ static int bch2_subvolumes_reparent(struct btree_trans *trans, u32 subvolid_to_d return lockrestart_do(trans, bch2_subvolume_get(trans, subvolid_to_delete, true, - BTREE_ITER_CACHED, &s)) ?: + BTREE_ITER_cached, &s)) ?: for_each_btree_key_commit(trans, iter, - BTREE_ID_subvolumes, POS_MIN, BTREE_ITER_PREFETCH, k, + BTREE_ID_subvolumes, POS_MIN, BTREE_ITER_prefetch, k, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, bch2_subvolume_reparent(trans, &iter, k, subvolid_to_delete, le32_to_cpu(s.creation_parent))); @@ -404,7 +404,7 @@ static int __bch2_subvolume_delete(struct btree_trans *trans, u32 subvolid) subvol = bch2_bkey_get_iter_typed(trans, &iter, BTREE_ID_subvolumes, POS(0, subvolid), - BTREE_ITER_CACHED|BTREE_ITER_INTENT, + BTREE_ITER_cached|BTREE_ITER_intent, subvolume); ret = bkey_err(subvol); bch2_fs_inconsistent_on(bch2_err_matches(ret, ENOENT), trans->c, @@ -505,7 +505,7 @@ int bch2_subvolume_unlink(struct btree_trans *trans, u32 subvolid) n = bch2_bkey_get_mut_typed(trans, &iter, BTREE_ID_subvolumes, POS(0, subvolid), - BTREE_ITER_CACHED, subvolume); + BTREE_ITER_cached, subvolume); ret = PTR_ERR_OR_ZERO(n); if (unlikely(ret)) { bch2_fs_inconsistent_on(bch2_err_matches(ret, ENOENT), trans->c, @@ -547,7 +547,7 @@ int bch2_subvolume_create(struct btree_trans *trans, u64 inode, src_subvol = bch2_bkey_get_mut_typed(trans, &src_iter, BTREE_ID_subvolumes, POS(0, src_subvolid), - BTREE_ITER_CACHED, subvolume); + BTREE_ITER_cached, subvolume); ret = PTR_ERR_OR_ZERO(src_subvol); if (unlikely(ret)) { bch2_fs_inconsistent_on(bch2_err_matches(ret, ENOENT), c, diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index 9073915f22a13..c161313913001 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -1600,17 +1600,17 @@ static int bch2_dev_remove_alloc(struct bch_fs *c, struct bch_dev *ca) * with bch2_do_invalidates() and bch2_do_discards() */ ret = bch2_btree_delete_range(c, BTREE_ID_lru, start, end, - BTREE_TRIGGER_NORUN, NULL) ?: + BTREE_TRIGGER_norun, NULL) ?: bch2_btree_delete_range(c, BTREE_ID_need_discard, start, end, - BTREE_TRIGGER_NORUN, NULL) ?: + BTREE_TRIGGER_norun, NULL) ?: bch2_btree_delete_range(c, BTREE_ID_freespace, start, end, - BTREE_TRIGGER_NORUN, NULL) ?: + BTREE_TRIGGER_norun, NULL) ?: bch2_btree_delete_range(c, BTREE_ID_backpointers, start, end, - BTREE_TRIGGER_NORUN, NULL) ?: + BTREE_TRIGGER_norun, NULL) ?: bch2_btree_delete_range(c, BTREE_ID_alloc, start, end, - BTREE_TRIGGER_NORUN, NULL) ?: + BTREE_TRIGGER_norun, NULL) ?: bch2_btree_delete_range(c, BTREE_ID_bucket_gens, start, end, - BTREE_TRIGGER_NORUN, NULL); + BTREE_TRIGGER_norun, NULL); bch_err_msg(c, ret, "removing dev alloc info"); return ret; } @@ -1822,7 +1822,7 @@ int bch2_dev_add(struct bch_fs *c, const char *path) bch2_dev_usage_journal_reserve(c); - ret = bch2_trans_mark_dev_sb(c, ca, BTREE_TRIGGER_TRANSACTIONAL); + ret = bch2_trans_mark_dev_sb(c, ca, BTREE_TRIGGER_transactional); bch_err_msg(ca, ret, "marking new superblock"); if (ret) goto err_late; @@ -1887,7 +1887,7 @@ int bch2_dev_online(struct bch_fs *c, const char *path) ca = bch_dev_locked(c, dev_idx); - ret = bch2_trans_mark_dev_sb(c, ca, BTREE_TRIGGER_TRANSACTIONAL); + ret = bch2_trans_mark_dev_sb(c, ca, BTREE_TRIGGER_transactional); bch_err_msg(c, ret, "bringing %s online: error from bch2_trans_mark_dev_sb", path); if (ret) goto err; @@ -1980,7 +1980,7 @@ int bch2_dev_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets) if (ret) goto err; - ret = bch2_trans_mark_dev_sb(c, ca, BTREE_TRIGGER_TRANSACTIONAL); + ret = bch2_trans_mark_dev_sb(c, ca, BTREE_TRIGGER_transactional); if (ret) goto err; diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index 4011ab4e710b7..607c930831bfb 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -274,7 +274,7 @@ static int bch2_compression_stats_to_text(struct printbuf *out, struct bch_fs *c continue; ret = for_each_btree_key(trans, iter, id, POS_MIN, - BTREE_ITER_ALL_SNAPSHOTS, k, ({ + BTREE_ITER_all_snapshots, k, ({ struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); struct bch_extent_crc_unpacked crc; const union bch_extent_entry *entry; diff --git a/fs/bcachefs/tests.c b/fs/bcachefs/tests.c index bfec656f94c07..68104b2056d94 100644 --- a/fs/bcachefs/tests.c +++ b/fs/bcachefs/tests.c @@ -40,7 +40,7 @@ static int test_delete(struct bch_fs *c, u64 nr) k.k.p.snapshot = U32_MAX; bch2_trans_iter_init(trans, &iter, BTREE_ID_xattrs, k.k.p, - BTREE_ITER_INTENT); + BTREE_ITER_intent); ret = commit_do(trans, NULL, NULL, 0, bch2_btree_iter_traverse(&iter) ?: @@ -81,7 +81,7 @@ static int test_delete_written(struct bch_fs *c, u64 nr) k.k.p.snapshot = U32_MAX; bch2_trans_iter_init(trans, &iter, BTREE_ID_xattrs, k.k.p, - BTREE_ITER_INTENT); + BTREE_ITER_intent); ret = commit_do(trans, NULL, NULL, 0, bch2_btree_iter_traverse(&iter) ?: @@ -261,7 +261,7 @@ static int test_iterate_slots(struct bch_fs *c, u64 nr) ret = bch2_trans_run(c, for_each_btree_key_upto(trans, iter, BTREE_ID_xattrs, SPOS(0, 0, U32_MAX), POS(0, U64_MAX), - BTREE_ITER_SLOTS, k, ({ + BTREE_ITER_slots, k, ({ if (i >= nr * 2) break; @@ -322,7 +322,7 @@ static int test_iterate_slots_extents(struct bch_fs *c, u64 nr) ret = bch2_trans_run(c, for_each_btree_key_upto(trans, iter, BTREE_ID_extents, SPOS(0, 0, U32_MAX), POS(0, U64_MAX), - BTREE_ITER_SLOTS, k, ({ + BTREE_ITER_slots, k, ({ if (i == nr) break; BUG_ON(bkey_deleted(k.k) != !(i % 16)); @@ -452,7 +452,7 @@ static int insert_test_overlapping_extent(struct bch_fs *c, u64 inum, u64 start, ret = bch2_trans_do(c, NULL, NULL, 0, bch2_btree_insert_nonextent(trans, BTREE_ID_extents, &k.k_i, - BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE)); + BTREE_UPDATE_internal_snapshot_node)); bch_err_fn(c, ret); return ret; } @@ -671,7 +671,7 @@ static int __do_delete(struct btree_trans *trans, struct bpos pos) int ret = 0; bch2_trans_iter_init(trans, &iter, BTREE_ID_xattrs, pos, - BTREE_ITER_INTENT); + BTREE_ITER_intent); k = bch2_btree_iter_peek_upto(&iter, POS(0, U64_MAX)); ret = bkey_err(k); if (ret) @@ -714,7 +714,7 @@ static int seq_insert(struct bch_fs *c, u64 nr) return bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_xattrs, SPOS(0, 0, U32_MAX), - BTREE_ITER_SLOTS|BTREE_ITER_INTENT, k, + BTREE_ITER_slots|BTREE_ITER_intent, k, NULL, NULL, 0, ({ if (iter.pos.offset >= nr) break; @@ -737,7 +737,7 @@ static int seq_overwrite(struct bch_fs *c, u64 nr) return bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_xattrs, SPOS(0, 0, U32_MAX), - BTREE_ITER_INTENT, k, + BTREE_ITER_intent, k, NULL, NULL, 0, ({ struct bkey_i_cookie u; diff --git a/fs/bcachefs/xattr.c b/fs/bcachefs/xattr.c index d787f1d4cc780..786ccd7a7662d 100644 --- a/fs/bcachefs/xattr.c +++ b/fs/bcachefs/xattr.c @@ -173,7 +173,7 @@ int bch2_xattr_set(struct btree_trans *trans, subvol_inum inum, int ret; ret = bch2_subvol_is_ro_trans(trans, inum.subvol) ?: - bch2_inode_peek(trans, &inode_iter, inode_u, inum, BTREE_ITER_INTENT); + bch2_inode_peek(trans, &inode_iter, inode_u, inum, BTREE_ITER_intent); if (ret) return ret; @@ -208,8 +208,8 @@ int bch2_xattr_set(struct btree_trans *trans, subvol_inum inum, ret = bch2_hash_set(trans, bch2_xattr_hash_desc, hash_info, inum, &xattr->k_i, - (flags & XATTR_CREATE ? BCH_HASH_SET_MUST_CREATE : 0)| - (flags & XATTR_REPLACE ? BCH_HASH_SET_MUST_REPLACE : 0)); + (flags & XATTR_CREATE ? STR_HASH_must_create : 0)| + (flags & XATTR_REPLACE ? STR_HASH_must_replace : 0)); } else { struct xattr_search_key search = X_SEARCH(type, name, strlen(name)); From 65bd442397274347e721a89c2c4882a392bae982 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 7 Apr 2024 19:13:45 -0400 Subject: [PATCH 1380/1702] bcachefs: bch2_btree_insert_trans() no longer specifies BTREE_ITER_cached Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_update.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/fs/bcachefs/btree_update.c b/fs/bcachefs/btree_update.c index da4b17704296d..ee10c1f845b80 100644 --- a/fs/bcachefs/btree_update.c +++ b/fs/bcachefs/btree_update.c @@ -640,13 +640,10 @@ int bch2_btree_insert_trans(struct btree_trans *trans, enum btree_id id, struct bkey_i *k, enum btree_iter_update_trigger_flags flags) { struct btree_iter iter; - int ret; - bch2_trans_iter_init(trans, &iter, id, bkey_start_pos(&k->k), - BTREE_ITER_cached| - BTREE_ITER_intent); - ret = bch2_btree_iter_traverse(&iter) ?: - bch2_trans_update(trans, &iter, k, flags); + BTREE_ITER_intent|flags); + int ret = bch2_btree_iter_traverse(&iter) ?: + bch2_trans_update(trans, &iter, k, flags); bch2_trans_iter_exit(trans, &iter); return ret; } From 0c0cbfdb84725e9933a24ecf47c61bdeeda06ba2 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 10 Apr 2024 13:18:22 -0400 Subject: [PATCH 1381/1702] bcachefs: bch2_dir_emit() - drop_locks_do() conversion Add a new helper that calls dir_emit() and updates ctx->pos on success; this lets us convert bch2_readdir() to drop_locks_do(). Signed-off-by: Kent Overstreet --- fs/bcachefs/dirent.c | 46 +++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/fs/bcachefs/dirent.c b/fs/bcachefs/dirent.c index ed256c3aa8a5f..dd9de8e5a28a9 100644 --- a/fs/bcachefs/dirent.c +++ b/fs/bcachefs/dirent.c @@ -528,16 +528,26 @@ int bch2_empty_dir_trans(struct btree_trans *trans, subvol_inum dir) bch2_empty_dir_snapshot(trans, dir.inum, dir.subvol, snapshot); } +static int bch2_dir_emit(struct dir_context *ctx, struct bkey_s_c_dirent d, subvol_inum target) +{ + struct qstr name = bch2_dirent_get_name(d); + bool ret = dir_emit(ctx, name.name, + name.len, + target.inum, + vfs_d_type(d.v->d_type)); + if (ret) + ctx->pos = d.k->p.offset + 1; + return ret; +} + int bch2_readdir(struct bch_fs *c, subvol_inum inum, struct dir_context *ctx) { struct btree_trans *trans = bch2_trans_get(c); struct btree_iter iter; struct bkey_s_c k; - struct bkey_s_c_dirent dirent; subvol_inum target; u32 snapshot; struct bkey_buf sk; - struct qstr name; int ret; bch2_bkey_buf_init(&sk); @@ -554,7 +564,9 @@ int bch2_readdir(struct bch_fs *c, subvol_inum inum, struct dir_context *ctx) if (k.k->type != KEY_TYPE_dirent) continue; - dirent = bkey_s_c_to_dirent(k); + /* dir_emit() can fault and block: */ + bch2_bkey_buf_reassemble(&sk, c, k); + struct bkey_s_c_dirent dirent = bkey_i_to_s_c_dirent(sk.k); ret = bch2_dirent_read_target(trans, inum, dirent, &target); if (ret < 0) @@ -562,28 +574,22 @@ int bch2_readdir(struct bch_fs *c, subvol_inum inum, struct dir_context *ctx) if (ret) continue; - /* dir_emit() can fault and block: */ - bch2_bkey_buf_reassemble(&sk, c, k); - dirent = bkey_i_to_s_c_dirent(sk.k); - bch2_trans_unlock(trans); - - name = bch2_dirent_get_name(dirent); - - ctx->pos = dirent.k->p.offset; - if (!dir_emit(ctx, name.name, - name.len, - target.inum, - vfs_d_type(dirent.v->d_type))) - break; - ctx->pos = dirent.k->p.offset + 1; - /* * read_target looks up subvolumes, we can overflow paths if the * directory has many subvolumes in it + * + * XXX: btree_trans_too_many_iters() is something we'd like to + * get rid of, and there's no good reason to be using it here + * except that we don't yet have a for_each_btree_key() helper + * that does subvolume_get_snapshot(). */ - ret = btree_trans_too_many_iters(trans); - if (ret) + ret = drop_locks_do(trans, + bch2_dir_emit(ctx, dirent, target)) ?: + btree_trans_too_many_iters(trans); + if (ret) { + ret = ret < 0 ? ret : 0; break; + } } bch2_trans_iter_exit(trans, &iter); err: From 923ed0ae5ebd0bb0dbc64a88348fa1835a3644ee Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 9 Apr 2024 19:45:41 -0400 Subject: [PATCH 1382/1702] bcachefs: bch2_trans_relock_fail() - factor out slowpath Factor out slowpath into a separate helper Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_locking.c | 98 ++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 46 deletions(-) diff --git a/fs/bcachefs/btree_locking.c b/fs/bcachefs/btree_locking.c index 8d1c4f78db5eb..e6adb2df3a571 100644 --- a/fs/bcachefs/btree_locking.c +++ b/fs/bcachefs/btree_locking.c @@ -723,51 +723,51 @@ void bch2_trans_downgrade(struct btree_trans *trans) bch2_btree_path_downgrade(trans, path); } -int bch2_trans_relock(struct btree_trans *trans) +static inline void __bch2_trans_unlock(struct btree_trans *trans) { struct btree_path *path; unsigned i; - if (unlikely(trans->restarted)) - return -((int) trans->restarted); + trans_for_each_path(trans, path, i) + __bch2_btree_path_unlock(trans, path); +} - trans_for_each_path(trans, path, i) { - struct get_locks_fail f; +static noinline __cold int bch2_trans_relock_fail(struct btree_trans *trans, struct btree_path *path, + struct get_locks_fail *f, bool trace) +{ + if (!trace) + goto out; - if (path->should_be_locked && - !btree_path_get_locks(trans, path, false, &f)) { - if (trace_trans_restart_relock_enabled()) { - struct printbuf buf = PRINTBUF; - - bch2_bpos_to_text(&buf, path->pos); - prt_printf(&buf, " l=%u seq=%u node seq=", - f.l, path->l[f.l].lock_seq); - if (IS_ERR_OR_NULL(f.b)) { - prt_str(&buf, bch2_err_str(PTR_ERR(f.b))); - } else { - prt_printf(&buf, "%u", f.b->c.lock.seq); - - struct six_lock_count c = - bch2_btree_node_lock_counts(trans, NULL, &f.b->c, f.l); - prt_printf(&buf, " self locked %u.%u.%u", c.n[0], c.n[1], c.n[2]); - - c = six_lock_counts(&f.b->c.lock); - prt_printf(&buf, " total locked %u.%u.%u", c.n[0], c.n[1], c.n[2]); - } + if (trace_trans_restart_relock_enabled()) { + struct printbuf buf = PRINTBUF; - trace_trans_restart_relock(trans, _RET_IP_, buf.buf); - printbuf_exit(&buf); - } + bch2_bpos_to_text(&buf, path->pos); + prt_printf(&buf, " l=%u seq=%u node seq=", f->l, path->l[f->l].lock_seq); + if (IS_ERR_OR_NULL(f->b)) { + prt_str(&buf, bch2_err_str(PTR_ERR(f->b))); + } else { + prt_printf(&buf, "%u", f->b->c.lock.seq); - count_event(trans->c, trans_restart_relock); - return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock); + struct six_lock_count c = + bch2_btree_node_lock_counts(trans, NULL, &f->b->c, f->l); + prt_printf(&buf, " self locked %u.%u.%u", c.n[0], c.n[1], c.n[2]); + + c = six_lock_counts(&f->b->c.lock); + prt_printf(&buf, " total locked %u.%u.%u", c.n[0], c.n[1], c.n[2]); } + + trace_trans_restart_relock(trans, _RET_IP_, buf.buf); + printbuf_exit(&buf); } - return 0; + count_event(trans->c, trans_restart_relock); +out: + __bch2_trans_unlock(trans); + bch2_trans_verify_locks(trans); + return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock); } -int bch2_trans_relock_notrace(struct btree_trans *trans) +static inline int __bch2_trans_relock(struct btree_trans *trans, bool trace) { struct btree_path *path; unsigned i; @@ -775,30 +775,36 @@ int bch2_trans_relock_notrace(struct btree_trans *trans) if (unlikely(trans->restarted)) return -((int) trans->restarted); - trans_for_each_path(trans, path, i) + trans_for_each_path(trans, path, i) { + struct get_locks_fail f; + if (path->should_be_locked && - !bch2_btree_path_relock_norestart(trans, path)) { - return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock); - } + !btree_path_get_locks(trans, path, false, &f)) + return bch2_trans_relock_fail(trans, path, &f, trace); + } + + bch2_trans_verify_locks(trans); return 0; } -void bch2_trans_unlock_noassert(struct btree_trans *trans) +int bch2_trans_relock(struct btree_trans *trans) { - struct btree_path *path; - unsigned i; + return __bch2_trans_relock(trans, true); +} - trans_for_each_path(trans, path, i) - __bch2_btree_path_unlock(trans, path); +int bch2_trans_relock_notrace(struct btree_trans *trans) +{ + return __bch2_trans_relock(trans, false); } -void bch2_trans_unlock(struct btree_trans *trans) +void bch2_trans_unlock_noassert(struct btree_trans *trans) { - struct btree_path *path; - unsigned i; + __bch2_trans_unlock(trans); +} - trans_for_each_path(trans, path, i) - __bch2_btree_path_unlock(trans, path); +void bch2_trans_unlock(struct btree_trans *trans) +{ + __bch2_trans_unlock(trans); } void bch2_trans_unlock_long(struct btree_trans *trans) From d155272b6e58a4b372982de777fde7dc503a8f96 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 11 Apr 2024 21:18:35 -0400 Subject: [PATCH 1383/1702] bcachefs: bucket_valid() cut out a branch from doing it the obvious way Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.h | 3 +-- fs/bcachefs/bcachefs.h | 1 + fs/bcachefs/buckets.h | 14 +++++++++----- fs/bcachefs/extents.c | 4 +--- fs/bcachefs/sb-members.h | 2 ++ fs/bcachefs/sb-members_types.h | 21 +++++++++++++++++++++ fs/bcachefs/super_types.h | 15 --------------- 7 files changed, 35 insertions(+), 25 deletions(-) create mode 100644 fs/bcachefs/sb-members_types.h diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h index 2790e516383d5..2bcc648081c38 100644 --- a/fs/bcachefs/alloc_background.h +++ b/fs/bcachefs/alloc_background.h @@ -21,8 +21,7 @@ static inline bool bch2_dev_bucket_exists(struct bch_fs *c, struct bpos pos) return false; ca = bch_dev_bkey_exists(c, pos.inode); - return pos.offset >= ca->mi.first_bucket && - pos.offset < ca->mi.nbuckets; + return bucket_valid(ca, pos.offset); } static inline u64 bucket_to_u64(struct bpos bucket) diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index 6366a6a90944d..3dad475334918 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -470,6 +470,7 @@ enum bch_time_stats { #include "quota_types.h" #include "rebalance_types.h" #include "replicas_types.h" +#include "sb-members_types.h" #include "subvolume_types.h" #include "super_types.h" #include "thread_with_file_types.h" diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h index a84e14468276e..97b4bf8961e2b 100644 --- a/fs/bcachefs/buckets.h +++ b/fs/bcachefs/buckets.h @@ -12,7 +12,7 @@ #include "extents.h" #include "sb-members.h" -static inline size_t sector_to_bucket(const struct bch_dev *ca, sector_t s) +static inline u64 sector_to_bucket(const struct bch_dev *ca, sector_t s) { return div_u64(s, ca->mi.bucket_size); } @@ -30,12 +30,16 @@ static inline sector_t bucket_remainder(const struct bch_dev *ca, sector_t s) return remainder; } -static inline size_t sector_to_bucket_and_offset(const struct bch_dev *ca, sector_t s, - u32 *offset) +static inline u64 sector_to_bucket_and_offset(const struct bch_dev *ca, sector_t s, u32 *offset) { return div_u64_rem(s, ca->mi.bucket_size, offset); } +static inline bool bucket_valid(const struct bch_dev *ca, u64 b) +{ + return b - ca->mi.first_bucket < ca->mi.nbuckets_minus_first; +} + #define for_each_bucket(_b, _buckets) \ for (_b = (_buckets)->b + (_buckets)->first_bucket; \ _b < (_buckets)->b + (_buckets)->nbuckets; _b++) @@ -94,7 +98,7 @@ static inline struct bucket *gc_bucket(struct bch_dev *ca, size_t b) { struct bucket_array *buckets = gc_bucket_array(ca); - BUG_ON(b < buckets->first_bucket || b >= buckets->nbuckets); + BUG_ON(!bucket_valid(ca, b)); return buckets->b + b; } @@ -111,7 +115,7 @@ static inline u8 *bucket_gen(struct bch_dev *ca, size_t b) { struct bucket_gens *gens = bucket_gens(ca); - BUG_ON(b < gens->first_bucket || b >= gens->nbuckets); + BUG_ON(!bucket_valid(ca, b)); return gens->b + b; } diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index 1a331e5392048..e9db8f6d6bc11 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -998,9 +998,7 @@ void bch2_extent_ptr_to_text(struct printbuf *out, struct bch_fs *c, const struc prt_str(out, " cached"); if (ptr->unwritten) prt_str(out, " unwritten"); - if (b >= ca->mi.first_bucket && - b < ca->mi.nbuckets && - ptr_stale(ca, ptr)) + if (bucket_valid(ca, b) && ptr_stale(ca, ptr)) prt_printf(out, " stale"); } } diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index 5bf27d30ca296..fa55b96662413 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -210,6 +210,8 @@ static inline struct bch_member_cpu bch2_mi_to_cpu(struct bch_member *mi) { return (struct bch_member_cpu) { .nbuckets = le64_to_cpu(mi->nbuckets), + .nbuckets_minus_first = le64_to_cpu(mi->nbuckets) - + le16_to_cpu(mi->first_bucket), .first_bucket = le16_to_cpu(mi->first_bucket), .bucket_size = le16_to_cpu(mi->bucket_size), .group = BCH_MEMBER_GROUP(mi), diff --git a/fs/bcachefs/sb-members_types.h b/fs/bcachefs/sb-members_types.h new file mode 100644 index 0000000000000..c0eda888fe39a --- /dev/null +++ b/fs/bcachefs/sb-members_types.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _BCACHEFS_SB_MEMBERS_TYPES_H +#define _BCACHEFS_SB_MEMBERS_TYPES_H + +struct bch_member_cpu { + u64 nbuckets; /* device size */ + u64 nbuckets_minus_first; + u16 first_bucket; /* index of first bucket used */ + u16 bucket_size; /* sectors */ + u16 group; + u8 state; + u8 discard; + u8 data_allowed; + u8 durability; + u8 freespace_initialized; + u8 valid; + u8 btree_bitmap_shift; + u64 btree_allocated_bitmap; +}; + +#endif /* _BCACHEFS_SB_MEMBERS_H */ diff --git a/fs/bcachefs/super_types.h b/fs/bcachefs/super_types.h index 11bcef170c2c2..368a63d938cfc 100644 --- a/fs/bcachefs/super_types.h +++ b/fs/bcachefs/super_types.h @@ -26,19 +26,4 @@ struct bch_devs_list { u8 data[BCH_BKEY_PTRS_MAX]; }; -struct bch_member_cpu { - u64 nbuckets; /* device size */ - u16 first_bucket; /* index of first bucket used */ - u16 bucket_size; /* sectors */ - u16 group; - u8 state; - u8 discard; - u8 data_allowed; - u8 durability; - u8 freespace_initialized; - u8 valid; - u8 btree_bitmap_shift; - u64 btree_allocated_bitmap; -}; - #endif /* _BCACHEFS_SUPER_TYPES_H */ From 2f724563fcd76166b9922c506078a4afa4e3a90a Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 11 Apr 2024 23:31:55 -0400 Subject: [PATCH 1384/1702] bcachefs: member helper cleanups Some renaming for better consistency bch2_member_exists -> bch2_member_alive bch2_dev_exists -> bch2_member_exists bch2_dev_exsits2 -> bch2_dev_exists bch_dev_locked -> bch2_dev_locked bch_dev_bkey_exists -> bch2_dev_bkey_exists new helper - bch2_dev_safe Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 32 ++++++++++++++++---------------- fs/bcachefs/alloc_background.h | 4 ++-- fs/bcachefs/alloc_foreground.c | 14 +++++++------- fs/bcachefs/alloc_foreground.h | 2 +- fs/bcachefs/backpointers.c | 10 +++++----- fs/bcachefs/backpointers.h | 4 ++-- fs/bcachefs/bcachefs.h | 5 ----- fs/bcachefs/btree_gc.c | 16 ++++++++-------- fs/bcachefs/btree_io.c | 16 ++++++++-------- fs/bcachefs/buckets.c | 4 ++-- fs/bcachefs/buckets.h | 4 ++-- fs/bcachefs/data_update.c | 6 +++--- fs/bcachefs/debug.c | 4 ++-- fs/bcachefs/disk_groups.c | 4 ++-- fs/bcachefs/ec.c | 10 +++++----- fs/bcachefs/extents.c | 29 +++++++++++++++-------------- fs/bcachefs/io_read.c | 8 ++++---- fs/bcachefs/io_write.c | 12 ++++++------ fs/bcachefs/journal_io.c | 10 +++++----- fs/bcachefs/move.c | 2 +- fs/bcachefs/recovery.c | 2 +- fs/bcachefs/replicas.c | 6 +++--- fs/bcachefs/sb-members.c | 4 ++-- fs/bcachefs/sb-members.h | 32 ++++++++++++++++++++++---------- fs/bcachefs/super-io.c | 2 +- fs/bcachefs/super.c | 17 ++++++++--------- 26 files changed, 133 insertions(+), 126 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index d3b86ce0fae61..2f91ff67453f2 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -576,10 +576,10 @@ int bch2_alloc_read(struct bch_fs *c) * Not a fsck error because this is checked/repaired by * bch2_check_alloc_key() which runs later: */ - if (!bch2_dev_exists2(c, k.k->p.inode)) + if (!bch2_dev_exists(c, k.k->p.inode)) continue; - struct bch_dev *ca = bch_dev_bkey_exists(c, k.k->p.inode); + struct bch_dev *ca = bch2_dev_bkey_exists(c, k.k->p.inode); for (u64 b = max_t(u64, ca->mi.first_bucket, start); b < min_t(u64, ca->mi.nbuckets, end); @@ -597,7 +597,7 @@ int bch2_alloc_read(struct bch_fs *c) if (!bch2_dev_bucket_exists(c, k.k->p)) continue; - struct bch_dev *ca = bch_dev_bkey_exists(c, k.k->p.inode); + struct bch_dev *ca = bch2_dev_bkey_exists(c, k.k->p.inode); struct bch_alloc_v4 a; *bucket_gen(ca, k.k->p.offset) = bch2_alloc_to_v4(k, &a)->gen; @@ -620,7 +620,7 @@ static int bch2_bucket_do_index(struct btree_trans *trans, bool set) { struct bch_fs *c = trans->c; - struct bch_dev *ca = bch_dev_bkey_exists(c, alloc_k.k->p.inode); + struct bch_dev *ca = bch2_dev_bkey_exists(c, alloc_k.k->p.inode); struct btree_iter iter; struct bkey_s_c old; struct bkey_i *k; @@ -733,7 +733,7 @@ int bch2_trigger_alloc(struct btree_trans *trans, "alloc key for invalid device or bucket")) return -EIO; - struct bch_dev *ca = bch_dev_bkey_exists(c, new.k->p.inode); + struct bch_dev *ca = bch2_dev_bkey_exists(c, new.k->p.inode); struct bch_alloc_v4 old_a_convert; const struct bch_alloc_v4 *old_a = bch2_alloc_to_v4(old, &old_a_convert); @@ -781,7 +781,7 @@ int bch2_trigger_alloc(struct btree_trans *trans, } new_a->fragmentation_lru = alloc_lru_idx_fragmentation(*new_a, - bch_dev_bkey_exists(c, new.k->p.inode)); + bch2_dev_bkey_exists(c, new.k->p.inode)); if (old_a->fragmentation_lru != new_a->fragmentation_lru) { ret = bch2_lru_change(trans, BCH_LRU_FRAGMENTATION_START, @@ -955,8 +955,8 @@ static bool next_bucket(struct bch_fs *c, struct bpos *bucket) if (bch2_dev_bucket_exists(c, *bucket)) return true; - if (bch2_dev_exists2(c, bucket->inode)) { - ca = bch_dev_bkey_exists(c, bucket->inode); + if (bch2_dev_exists(c, bucket->inode)) { + ca = bch2_dev_bkey_exists(c, bucket->inode); if (bucket->offset < ca->mi.first_bucket) { bucket->offset = ca->mi.first_bucket; @@ -997,7 +997,7 @@ static struct bkey_s_c bch2_get_key_or_real_bucket_hole(struct btree_iter *iter, } if (!bch2_dev_bucket_exists(c, k.k->p)) { - struct bch_dev *ca = bch_dev_bkey_exists(c, bucket.inode); + struct bch_dev *ca = bch2_dev_bkey_exists(c, bucket.inode); bch2_key_resize(hole, ca->mi.nbuckets - bucket.offset); } @@ -1030,7 +1030,7 @@ int bch2_check_alloc_key(struct btree_trans *trans, alloc_k.k->p.inode, alloc_k.k->p.offset)) return bch2_btree_delete_at(trans, alloc_iter, 0); - ca = bch_dev_bkey_exists(c, alloc_k.k->p.inode); + ca = bch2_dev_bkey_exists(c, alloc_k.k->p.inode); if (!ca->mi.freespace_initialized) return 0; @@ -1149,7 +1149,7 @@ int bch2_check_alloc_hole_freespace(struct btree_trans *trans, struct printbuf buf = PRINTBUF; int ret; - ca = bch_dev_bkey_exists(c, start.inode); + ca = bch2_dev_bkey_exists(c, start.inode); if (!ca->mi.freespace_initialized) return 0; @@ -1339,7 +1339,7 @@ int bch2_check_bucket_gens_key(struct btree_trans *trans, bkey_reassemble(&g.k_i, k); /* if no bch_dev, skip out whether we repair or not */ - dev_exists = bch2_dev_exists2(c, k.k->p.inode); + dev_exists = bch2_dev_exists(c, k.k->p.inode); if (!dev_exists) { if (fsck_err_on(!dev_exists, c, bucket_gens_to_invalid_dev, @@ -1350,7 +1350,7 @@ int bch2_check_bucket_gens_key(struct btree_trans *trans, goto out; } - ca = bch_dev_bkey_exists(c, k.k->p.inode); + ca = bch2_dev_bkey_exists(c, k.k->p.inode); if (fsck_err_on(end <= ca->mi.first_bucket || start >= ca->mi.nbuckets, c, bucket_gens_to_invalid_buckets, @@ -1669,7 +1669,7 @@ static int bch2_discard_one_bucket(struct btree_trans *trans, bool discard_locked = false; int ret = 0; - ca = bch_dev_bkey_exists(c, pos.inode); + ca = bch2_dev_bkey_exists(c, pos.inode); if (!percpu_ref_tryget(&ca->io_ref)) { bch2_btree_iter_set_pos(need_discard_iter, POS(pos.inode + 1, 0)); @@ -1852,7 +1852,7 @@ static void bch2_do_discards_fast_work(struct work_struct *work) if (i->snapshot) continue; - ca = bch_dev_bkey_exists(c, i->inode); + ca = bch2_dev_bkey_exists(c, i->inode); if (!percpu_ref_tryget(&ca->io_ref)) { darray_remove_item(&c->discard_buckets_in_flight, i); @@ -1893,7 +1893,7 @@ static void bch2_do_discards_fast_work(struct work_struct *work) static void bch2_discard_one_bucket_fast(struct bch_fs *c, struct bpos bucket) { - struct bch_dev *ca = bch_dev_bkey_exists(c, bucket.inode); + struct bch_dev *ca = bch2_dev_bkey_exists(c, bucket.inode); if (!percpu_ref_is_dying(&ca->io_ref) && !discard_in_flight_add(c, bucket) && diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h index 2bcc648081c38..44db71ccc953a 100644 --- a/fs/bcachefs/alloc_background.h +++ b/fs/bcachefs/alloc_background.h @@ -17,10 +17,10 @@ static inline bool bch2_dev_bucket_exists(struct bch_fs *c, struct bpos pos) { struct bch_dev *ca; - if (!bch2_dev_exists2(c, pos.inode)) + if (!bch2_dev_exists(c, pos.inode)) return false; - ca = bch_dev_bkey_exists(c, pos.inode); + ca = bch2_dev_bkey_exists(c, pos.inode); return bucket_valid(ca, pos.offset); } diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index b68e1fd782f33..df4439a38df5a 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -100,7 +100,7 @@ static void bch2_open_bucket_hash_remove(struct bch_fs *c, struct open_bucket *o void __bch2_open_bucket_put(struct bch_fs *c, struct open_bucket *ob) { - struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ob->dev); if (ob->ec) { ec_stripe_new_put(c, ob->ec, STRIPE_REF_io); @@ -679,7 +679,7 @@ static int add_new_bucket(struct bch_fs *c, struct open_bucket *ob) { unsigned durability = - bch_dev_bkey_exists(c, ob->dev)->mi.durability; + bch2_dev_bkey_exists(c, ob->dev)->mi.durability; BUG_ON(*nr_effective >= nr_replicas); @@ -836,7 +836,7 @@ static bool want_bucket(struct bch_fs *c, bool *have_cache, bool ec, struct open_bucket *ob) { - struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ob->dev); if (!test_bit(ob->dev, devs_may_alloc->d)) return false; @@ -906,7 +906,7 @@ static int bucket_alloc_set_partial(struct bch_fs *c, struct open_bucket *ob = c->open_buckets + c->open_buckets_partial[i]; if (want_bucket(c, wp, devs_may_alloc, have_cache, ec, ob)) { - struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ob->dev); struct bch_dev_usage usage; u64 avail; @@ -1291,7 +1291,7 @@ deallocate_extra_replicas(struct bch_fs *c, unsigned i; open_bucket_for_each(c, ptrs, ob, i) { - unsigned d = bch_dev_bkey_exists(c, ob->dev)->mi.durability; + unsigned d = bch2_dev_bkey_exists(c, ob->dev)->mi.durability; if (d && d <= extra_replicas) { extra_replicas -= d; @@ -1444,7 +1444,7 @@ int bch2_alloc_sectors_start_trans(struct btree_trans *trans, struct bch_extent_ptr bch2_ob_ptr(struct bch_fs *c, struct open_bucket *ob) { - struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ob->dev); return (struct bch_extent_ptr) { .type = 1 << BCH_EXTENT_ENTRY_ptr, @@ -1520,7 +1520,7 @@ void bch2_fs_allocator_foreground_init(struct bch_fs *c) static void bch2_open_bucket_to_text(struct printbuf *out, struct bch_fs *c, struct open_bucket *ob) { - struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ob->dev); unsigned data_type = ob->data_type; barrier(); /* READ_ONCE() doesn't work on bitfields */ diff --git a/fs/bcachefs/alloc_foreground.h b/fs/bcachefs/alloc_foreground.h index 7aaeec44c7466..c101b2c367439 100644 --- a/fs/bcachefs/alloc_foreground.h +++ b/fs/bcachefs/alloc_foreground.h @@ -184,7 +184,7 @@ bch2_alloc_sectors_append_ptrs_inlined(struct bch_fs *c, struct write_point *wp, wp->sectors_allocated += sectors; open_bucket_for_each(c, &wp->ptrs, ob, i) { - struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ob->dev); struct bch_extent_ptr ptr = bch2_ob_ptr(c, ob); ptr.cached = cached || diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c index 5586820fcf2c4..36e5e63ec3d5f 100644 --- a/fs/bcachefs/backpointers.c +++ b/fs/bcachefs/backpointers.c @@ -46,10 +46,10 @@ int bch2_backpointer_invalid(struct bch_fs *c, struct bkey_s_c k, struct bkey_s_c_backpointer bp = bkey_s_c_to_backpointer(k); /* these will be caught by fsck */ - if (!bch2_dev_exists2(c, bp.k->p.inode)) + if (!bch2_dev_exists(c, bp.k->p.inode)) return 0; - struct bch_dev *ca = bch_dev_bkey_exists(c, bp.k->p.inode); + struct bch_dev *ca = bch2_dev_bkey_exists(c, bp.k->p.inode); struct bpos bucket = bp_pos_to_bucket(c, bp.k->p); int ret = 0; @@ -75,7 +75,7 @@ void bch2_backpointer_to_text(struct printbuf *out, const struct bch_backpointer void bch2_backpointer_k_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k) { - if (bch2_dev_exists2(c, k.k->p.inode)) { + if (bch2_dev_exists(c, k.k->p.inode)) { prt_str(out, "bucket="); bch2_bpos_to_text(out, bp_pos_to_bucket(c, k.k->p)); prt_str(out, " "); @@ -366,7 +366,7 @@ static int bch2_check_btree_backpointer(struct btree_trans *trans, struct btree_ struct printbuf buf = PRINTBUF; int ret = 0; - if (fsck_err_on(!bch2_dev_exists2(c, k.k->p.inode), c, + if (fsck_err_on(!bch2_dev_exists(c, k.k->p.inode), c, backpointer_to_missing_device, "backpointer for missing device:\n%s", (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { @@ -459,7 +459,7 @@ static int check_extent_checksum(struct btree_trans *trans, bytes = p.crc.compressed_size << 9; - struct bch_dev *ca = bch_dev_bkey_exists(c, dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, dev); if (!bch2_dev_get_ioref(ca, READ)) return false; diff --git a/fs/bcachefs/backpointers.h b/fs/bcachefs/backpointers.h index c1b274eadda14..e7f1eddbf4f48 100644 --- a/fs/bcachefs/backpointers.h +++ b/fs/bcachefs/backpointers.h @@ -39,7 +39,7 @@ void bch2_backpointer_swab(struct bkey_s); static inline struct bpos bp_pos_to_bucket(const struct bch_fs *c, struct bpos bp_pos) { - struct bch_dev *ca = bch_dev_bkey_exists(c, bp_pos.inode); + struct bch_dev *ca = bch2_dev_bkey_exists(c, bp_pos.inode); u64 bucket_sector = bp_pos.offset >> MAX_EXTENT_COMPRESS_RATIO_SHIFT; return POS(bp_pos.inode, sector_to_bucket(ca, bucket_sector)); @@ -61,7 +61,7 @@ static inline struct bpos bucket_pos_to_bp(const struct bch_fs *c, struct bpos bucket, u64 bucket_offset) { - struct bch_dev *ca = bch_dev_bkey_exists(c, bucket.inode); + struct bch_dev *ca = bch2_dev_bkey_exists(c, bucket.inode); struct bpos ret = bucket_pos_to_bp_noerror(ca, bucket, bucket_offset); EBUG_ON(!bkey_eq(bucket, bp_pos_to_bucket(c, ret))); return ret; diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index 3dad475334918..7562446f2d2af 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -1253,11 +1253,6 @@ static inline s64 bch2_current_time(const struct bch_fs *c) return timespec_to_bch2_time(c, now); } -static inline bool bch2_dev_exists2(const struct bch_fs *c, unsigned dev) -{ - return dev < c->sb.nr_devices && c->devs[dev]; -} - static inline struct stdio_redirect *bch2_fs_stdio_redirect(struct bch_fs *c) { struct stdio_redirect *stdio = c->stdio; diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index c98b0ce455243..4c5bd256d2511 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -597,7 +597,7 @@ static int bch2_check_fix_ptrs(struct btree_trans *trans, enum btree_id btree_id * use check_bucket_ref here */ bkey_for_each_ptr_decode(k->k, ptrs_c, p, entry_c) { - struct bch_dev *ca = bch_dev_bkey_exists(c, p.ptr.dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr); enum bch_data_type data_type = bch2_bkey_ptr_data_type(*k, p, entry_c); @@ -730,7 +730,7 @@ static int bch2_check_fix_ptrs(struct btree_trans *trans, enum btree_id btree_id */ struct bkey_ptrs ptrs = bch2_bkey_ptrs(bkey_i_to_s(new)); bkey_for_each_ptr(ptrs, ptr) { - struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); struct bucket *g = PTR_GC_BUCKET(ca, ptr); ptr->gen = g->gen; @@ -741,7 +741,7 @@ static int bch2_check_fix_ptrs(struct btree_trans *trans, enum btree_id btree_id restart_drop_ptrs: ptrs = bch2_bkey_ptrs(bkey_i_to_s(new)); bkey_for_each_ptr_decode(bkey_i_to_s(new).k, ptrs, p, entry) { - struct bch_dev *ca = bch_dev_bkey_exists(c, p.ptr.dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr); enum bch_data_type data_type = bch2_bkey_ptr_data_type(bkey_i_to_s_c(new), p, entry); @@ -1215,7 +1215,7 @@ static int bch2_alloc_write_key(struct btree_trans *trans, struct bkey_s_c k) { struct bch_fs *c = trans->c; - struct bch_dev *ca = bch_dev_bkey_exists(c, iter->pos.inode); + struct bch_dev *ca = bch2_dev_bkey_exists(c, iter->pos.inode); struct bucket old_gc, gc, *b; struct bkey_i_alloc_v4 *a; struct bch_alloc_v4 old_convert, new; @@ -1351,7 +1351,7 @@ static int bch2_gc_alloc_start(struct bch_fs *c) int ret = bch2_trans_run(c, for_each_btree_key(trans, iter, BTREE_ID_alloc, POS_MIN, BTREE_ITER_prefetch, k, ({ - struct bch_dev *ca = bch_dev_bkey_exists(c, k.k->p.inode); + struct bch_dev *ca = bch2_dev_bkey_exists(c, k.k->p.inode); struct bucket *g = gc_bucket(ca, k.k->p.offset); struct bch_alloc_v4 a_convert; @@ -1671,7 +1671,7 @@ static int gc_btree_gens_key(struct btree_trans *trans, percpu_down_read(&c->mark_lock); bkey_for_each_ptr(ptrs, ptr) { - struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); if (ptr_stale(ca, ptr) > 16) { percpu_up_read(&c->mark_lock); @@ -1680,7 +1680,7 @@ static int gc_btree_gens_key(struct btree_trans *trans, } bkey_for_each_ptr(ptrs, ptr) { - struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); u8 *gen = &ca->oldest_gen[PTR_BUCKET_NR(ca, ptr)]; if (gen_after(*gen, ptr->gen)) @@ -1701,7 +1701,7 @@ static int gc_btree_gens_key(struct btree_trans *trans, static int bch2_alloc_write_oldest_gen(struct btree_trans *trans, struct btree_iter *iter, struct bkey_s_c k) { - struct bch_dev *ca = bch_dev_bkey_exists(trans->c, iter->pos.inode); + struct bch_dev *ca = bch2_dev_bkey_exists(trans->c, iter->pos.inode); struct bch_alloc_v4 a_convert; const struct bch_alloc_v4 *a = bch2_alloc_to_v4(k, &a_convert); struct bkey_i_alloc_v4 *a_mut; diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c index ed80556b80fe2..37f00d844edd7 100644 --- a/fs/bcachefs/btree_io.c +++ b/fs/bcachefs/btree_io.c @@ -1263,7 +1263,7 @@ int bch2_btree_node_read_done(struct bch_fs *c, struct bch_dev *ca, btree_node_reset_sib_u64s(b); bkey_for_each_ptr(bch2_bkey_ptrs(bkey_i_to_s(&b->key)), ptr) { - struct bch_dev *ca2 = bch_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca2 = bch2_dev_bkey_exists(c, ptr->dev); if (ca2->mi.state != BCH_MEMBER_STATE_rw) set_btree_node_need_rewrite(b); @@ -1293,7 +1293,7 @@ static void btree_node_read_work(struct work_struct *work) container_of(work, struct btree_read_bio, work); struct bch_fs *c = rb->c; struct btree *b = rb->b; - struct bch_dev *ca = bch_dev_bkey_exists(c, rb->pick.ptr.dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, rb->pick.ptr.dev); struct bio *bio = &rb->bio; struct bch_io_failures failed = { .nr = 0 }; struct printbuf buf = PRINTBUF; @@ -1305,7 +1305,7 @@ static void btree_node_read_work(struct work_struct *work) while (1) { retry = true; bch_info(c, "retrying read"); - ca = bch_dev_bkey_exists(c, rb->pick.ptr.dev); + ca = bch2_dev_bkey_exists(c, rb->pick.ptr.dev); rb->have_ioref = bch2_dev_get_ioref(ca, READ); bio_reset(bio, NULL, REQ_OP_READ|REQ_SYNC|REQ_META); bio->bi_iter.bi_sector = rb->pick.ptr.offset; @@ -1376,7 +1376,7 @@ static void btree_node_read_endio(struct bio *bio) struct bch_fs *c = rb->c; if (rb->have_ioref) { - struct bch_dev *ca = bch_dev_bkey_exists(c, rb->pick.ptr.dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, rb->pick.ptr.dev); bch2_latency_acct(ca, rb->start_time, READ); } @@ -1573,7 +1573,7 @@ static void btree_node_read_all_replicas_endio(struct bio *bio) struct btree_node_read_all *ra = rb->ra; if (rb->have_ioref) { - struct bch_dev *ca = bch_dev_bkey_exists(c, rb->pick.ptr.dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, rb->pick.ptr.dev); bch2_latency_acct(ca, rb->start_time, READ); } @@ -1615,7 +1615,7 @@ static int btree_node_read_all_replicas(struct bch_fs *c, struct btree *b, bool i = 0; bkey_for_each_ptr_decode(k.k, ptrs, pick, entry) { - struct bch_dev *ca = bch_dev_bkey_exists(c, pick.ptr.dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, pick.ptr.dev); struct btree_read_bio *rb = container_of(ra->bio[i], struct btree_read_bio, bio); rb->c = c; @@ -1692,7 +1692,7 @@ void bch2_btree_node_read(struct btree_trans *trans, struct btree *b, return; } - ca = bch_dev_bkey_exists(c, pick.ptr.dev); + ca = bch2_dev_bkey_exists(c, pick.ptr.dev); bio = bio_alloc_bioset(NULL, buf_pages(b->data, btree_buf_bytes(b)), @@ -1909,7 +1909,7 @@ static void btree_node_write_endio(struct bio *bio) struct btree_write_bio *wb = container_of(orig, struct btree_write_bio, wbio); struct bch_fs *c = wbio->c; struct btree *b = wbio->bio.bi_private; - struct bch_dev *ca = bch_dev_bkey_exists(c, wbio->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, wbio->dev); unsigned long flags; if (wbio->have_ioref) diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 20f471c08b2e0..74e2098dfde51 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -493,7 +493,7 @@ int bch2_check_bucket_ref(struct btree_trans *trans, u32 bucket_sectors) { struct bch_fs *c = trans->c; - struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); size_t bucket_nr = PTR_BUCKET_NR(ca, ptr); struct printbuf buf = PRINTBUF; int ret = 0; @@ -787,7 +787,7 @@ static int bch2_trigger_pointer(struct btree_trans *trans, if (flags & BTREE_TRIGGER_gc) { struct bch_fs *c = trans->c; - struct bch_dev *ca = bch_dev_bkey_exists(c, p.ptr.dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); enum bch_data_type data_type = bch2_bkey_ptr_data_type(k, p, entry); percpu_down_read(&c->mark_lock); diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h index 97b4bf8961e2b..a88b9033349fa 100644 --- a/fs/bcachefs/buckets.h +++ b/fs/bcachefs/buckets.h @@ -128,7 +128,7 @@ static inline size_t PTR_BUCKET_NR(const struct bch_dev *ca, static inline struct bpos PTR_BUCKET_POS(const struct bch_fs *c, const struct bch_extent_ptr *ptr) { - struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); return POS(ptr->dev, PTR_BUCKET_NR(ca, ptr)); } @@ -137,7 +137,7 @@ static inline struct bpos PTR_BUCKET_POS_OFFSET(const struct bch_fs *c, const struct bch_extent_ptr *ptr, u32 *bucket_offset) { - struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); return POS(ptr->dev, sector_to_bucket_and_offset(ca, ptr->offset, bucket_offset)); } diff --git a/fs/bcachefs/data_update.c b/fs/bcachefs/data_update.c index 3c0f0801468eb..cae4e4eb5329a 100644 --- a/fs/bcachefs/data_update.c +++ b/fs/bcachefs/data_update.c @@ -360,7 +360,7 @@ void bch2_data_update_exit(struct data_update *update) if (c->opts.nocow_enabled) bch2_bucket_nocow_unlock(&c->nocow_locks, PTR_BUCKET_POS(c, ptr), 0); - percpu_ref_put(&bch_dev_bkey_exists(c, ptr->dev)->ref); + percpu_ref_put(&bch2_dev_bkey_exists(c, ptr->dev)->ref); } bch2_bkey_buf_exit(&update->k, c); @@ -540,7 +540,7 @@ int bch2_data_update_init(struct btree_trans *trans, m->op.watermark = m->data_opts.btree_insert_flags & BCH_WATERMARK_MASK; bkey_for_each_ptr(ptrs, ptr) - percpu_ref_get(&bch_dev_bkey_exists(c, ptr->dev)->ref); + percpu_ref_get(&bch2_dev_bkey_exists(c, ptr->dev)->ref); unsigned durability_have = 0, durability_removing = 0; @@ -652,7 +652,7 @@ int bch2_data_update_init(struct btree_trans *trans, if ((1U << i) & ptrs_locked) bch2_bucket_nocow_unlock(&c->nocow_locks, PTR_BUCKET_POS(c, &p.ptr), 0); - percpu_ref_put(&bch_dev_bkey_exists(c, p.ptr.dev)->ref); + percpu_ref_put(&bch2_dev_bkey_exists(c, p.ptr.dev)->ref); i++; } diff --git a/fs/bcachefs/debug.c b/fs/bcachefs/debug.c index 8310eff12a172..b5aa4e5b224b6 100644 --- a/fs/bcachefs/debug.c +++ b/fs/bcachefs/debug.c @@ -37,7 +37,7 @@ static bool bch2_btree_verify_replica(struct bch_fs *c, struct btree *b, struct btree_node *n_ondisk = c->verify_ondisk; struct btree_node *n_sorted = c->verify_data->data; struct bset *sorted, *inmemory = &b->data->keys; - struct bch_dev *ca = bch_dev_bkey_exists(c, pick.ptr.dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, pick.ptr.dev); struct bio *bio; bool failed = false, saw_error = false; @@ -194,7 +194,7 @@ void bch2_btree_node_ondisk_to_text(struct printbuf *out, struct bch_fs *c, return; } - ca = bch_dev_bkey_exists(c, pick.ptr.dev); + ca = bch2_dev_bkey_exists(c, pick.ptr.dev); if (!bch2_dev_get_ioref(ca, READ)) { prt_printf(out, "error getting device to read from: not online\n"); return; diff --git a/fs/bcachefs/disk_groups.c b/fs/bcachefs/disk_groups.c index 06a7df529b401..04b6aa4144765 100644 --- a/fs/bcachefs/disk_groups.c +++ b/fs/bcachefs/disk_groups.c @@ -177,7 +177,7 @@ int bch2_sb_disk_groups_to_cpu(struct bch_fs *c) struct bch_member m = bch2_sb_member_get(c->disk_sb.sb, i); struct bch_disk_group_cpu *dst; - if (!bch2_member_exists(&m)) + if (!bch2_member_alive(&m)) continue; g = BCH_MEMBER_GROUP(&m); @@ -588,7 +588,7 @@ static void bch2_target_to_text_sb(struct printbuf *out, struct bch_sb *sb, unsi case TARGET_DEV: { struct bch_member m = bch2_sb_member_get(sb, t.dev); - if (bch2_dev_exists(sb, t.dev)) { + if (bch2_member_exists(sb, t.dev)) { prt_printf(out, "Device "); pr_uuid(out, m.uuid.b); prt_printf(out, " (%u)", t.dev); diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 86999ed80439b..97ed598a55a52 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -253,7 +253,7 @@ static int mark_stripe_bucket(struct btree_trans *trans, enum bch_data_type data_type = parity ? BCH_DATA_parity : BCH_DATA_stripe; s64 sectors = parity ? le16_to_cpu(s->sectors) : 0; const struct bch_extent_ptr *ptr = s->ptrs + ptr_idx; - struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); struct bucket old, new, *g; struct printbuf buf = PRINTBUF; int ret = 0; @@ -609,7 +609,7 @@ static void ec_validate_checksums(struct bch_fs *c, struct ec_stripe_buf *buf) if (bch2_crc_cmp(want, got)) { struct printbuf err = PRINTBUF; - struct bch_dev *ca = bch_dev_bkey_exists(c, v->ptrs[i].dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, v->ptrs[i].dev); prt_str(&err, "stripe "); bch2_csum_err_msg(&err, v->csum_type, want, got); @@ -705,7 +705,7 @@ static void ec_block_io(struct bch_fs *c, struct ec_stripe_buf *buf, struct bch_stripe *v = &bkey_i_to_stripe(&buf->key)->v; unsigned offset = 0, bytes = buf->size << 9; struct bch_extent_ptr *ptr = &v->ptrs[idx]; - struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); enum bch_data_type data_type = idx < v->nr_blocks - v->nr_redundant ? BCH_DATA_user : BCH_DATA_parity; @@ -1321,7 +1321,7 @@ static void zero_out_rest_of_ec_bucket(struct bch_fs *c, unsigned block, struct open_bucket *ob) { - struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ob->dev); unsigned offset = ca->mi.bucket_size - ob->sectors_free; int ret; @@ -1527,7 +1527,7 @@ void *bch2_writepoint_ec_buf(struct bch_fs *c, struct write_point *wp) BUG_ON(!ob->ec->new_stripe.data[ob->ec_idx]); - ca = bch_dev_bkey_exists(c, ob->dev); + ca = bch2_dev_bkey_exists(c, ob->dev); offset = ca->mi.bucket_size - ob->sectors_free; return ob->ec->new_stripe.data[ob->ec_idx] + (offset << 9); diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index e9db8f6d6bc11..cb8d52436ab15 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -79,8 +79,8 @@ static inline bool ptr_better(struct bch_fs *c, const struct extent_ptr_decoded p2) { if (likely(!p1.idx && !p2.idx)) { - struct bch_dev *dev1 = bch_dev_bkey_exists(c, p1.ptr.dev); - struct bch_dev *dev2 = bch_dev_bkey_exists(c, p2.ptr.dev); + struct bch_dev *dev1 = bch2_dev_bkey_exists(c, p1.ptr.dev); + struct bch_dev *dev2 = bch2_dev_bkey_exists(c, p2.ptr.dev); u64 l1 = atomic64_read(&dev1->cur_latency[READ]); u64 l2 = atomic64_read(&dev2->cur_latency[READ]); @@ -123,7 +123,7 @@ int bch2_bkey_pick_read_device(struct bch_fs *c, struct bkey_s_c k, if (p.ptr.unwritten) return 0; - ca = bch_dev_bkey_exists(c, p.ptr.dev); + ca = bch2_dev_bkey_exists(c, p.ptr.dev); /* * If there are any dirty pointers it's an error if we can't @@ -278,7 +278,7 @@ bool bch2_extent_merge(struct bch_fs *c, struct bkey_s l, struct bkey_s_c r) return false; /* Extents may not straddle buckets: */ - ca = bch_dev_bkey_exists(c, lp.ptr.dev); + ca = bch2_dev_bkey_exists(c, lp.ptr.dev); if (PTR_BUCKET_NR(ca, &lp.ptr) != PTR_BUCKET_NR(ca, &rp.ptr)) return false; @@ -667,14 +667,14 @@ static inline unsigned __extent_ptr_durability(struct bch_dev *ca, struct extent unsigned bch2_extent_ptr_desired_durability(struct bch_fs *c, struct extent_ptr_decoded *p) { - struct bch_dev *ca = bch_dev_bkey_exists(c, p->ptr.dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, p->ptr.dev); return __extent_ptr_durability(ca, p); } unsigned bch2_extent_ptr_durability(struct bch_fs *c, struct extent_ptr_decoded *p) { - struct bch_dev *ca = bch_dev_bkey_exists(c, p->ptr.dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, p->ptr.dev); if (ca->mi.state == BCH_MEMBER_STATE_failed) return 0; @@ -864,7 +864,7 @@ bool bch2_bkey_has_target(struct bch_fs *c, struct bkey_s_c k, unsigned target) bkey_for_each_ptr(ptrs, ptr) if (bch2_dev_in_target(c, ptr->dev, target) && (!ptr->cached || - !ptr_stale(bch_dev_bkey_exists(c, ptr->dev), ptr))) + !ptr_stale(bch2_dev_bkey_exists(c, ptr->dev), ptr))) return true; return false; @@ -973,17 +973,16 @@ bool bch2_extent_normalize(struct bch_fs *c, struct bkey_s k) bch2_bkey_drop_ptrs(k, ptr, ptr->cached && - ptr_stale(bch_dev_bkey_exists(c, ptr->dev), ptr)); + ptr_stale(bch2_dev_bkey_exists(c, ptr->dev), ptr)); return bkey_deleted(k.k); } void bch2_extent_ptr_to_text(struct printbuf *out, struct bch_fs *c, const struct bch_extent_ptr *ptr) { - struct bch_dev *ca = c && ptr->dev < c->sb.nr_devices && c->devs[ptr->dev] - ? bch_dev_bkey_exists(c, ptr->dev) - : NULL; - + out->atomic++; + rcu_read_lock(); + struct bch_dev *ca = bch2_dev_safe(c, ptr->dev); if (!ca) { prt_printf(out, "ptr: %u:%llu gen %u%s", ptr->dev, (u64) ptr->offset, ptr->gen, @@ -1001,6 +1000,8 @@ void bch2_extent_ptr_to_text(struct printbuf *out, struct bch_fs *c, const struc if (bucket_valid(ca, b) && ptr_stale(ca, ptr)) prt_printf(out, " stale"); } + rcu_read_unlock(); + --out->atomic; } void bch2_bkey_ptrs_to_text(struct printbuf *out, struct bch_fs *c, @@ -1079,7 +1080,7 @@ static int extent_ptr_invalid(struct bch_fs *c, struct bch_dev *ca; int ret = 0; - if (!bch2_dev_exists2(c, ptr->dev)) { + if (!bch2_dev_exists(c, ptr->dev)) { /* * If we're in the write path this key might have already been * overwritten, and we could be seeing a device that doesn't @@ -1092,7 +1093,7 @@ static int extent_ptr_invalid(struct bch_fs *c, "pointer to invalid device (%u)", ptr->dev); } - ca = bch_dev_bkey_exists(c, ptr->dev); + ca = bch2_dev_bkey_exists(c, ptr->dev); bkey_for_each_ptr(ptrs, ptr2) bkey_fsck_err_on(ptr != ptr2 && ptr->dev == ptr2->dev, c, err, ptr_to_duplicate_device, diff --git a/fs/bcachefs/io_read.c b/fs/bcachefs/io_read.c index 475ab6c02dd16..af9bd7c45f21e 100644 --- a/fs/bcachefs/io_read.c +++ b/fs/bcachefs/io_read.c @@ -541,7 +541,7 @@ static void __bch2_read_endio(struct work_struct *work) struct bch_read_bio *rbio = container_of(work, struct bch_read_bio, work); struct bch_fs *c = rbio->c; - struct bch_dev *ca = bch_dev_bkey_exists(c, rbio->pick.ptr.dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, rbio->pick.ptr.dev); struct bio *src = &rbio->bio; struct bio *dst = &bch2_rbio_parent(rbio)->bio; struct bvec_iter dst_iter = rbio->bvec_iter; @@ -675,7 +675,7 @@ static void bch2_read_endio(struct bio *bio) struct bch_read_bio *rbio = container_of(bio, struct bch_read_bio, bio); struct bch_fs *c = rbio->c; - struct bch_dev *ca = bch_dev_bkey_exists(c, rbio->pick.ptr.dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, rbio->pick.ptr.dev); struct workqueue_struct *wq = NULL; enum rbio_context context = RBIO_CONTEXT_NULL; @@ -762,7 +762,7 @@ static noinline void read_from_stale_dirty_pointer(struct btree_trans *trans, struct bch_extent_ptr ptr) { struct bch_fs *c = trans->c; - struct bch_dev *ca = bch_dev_bkey_exists(c, ptr.dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr.dev); struct btree_iter iter; struct printbuf buf = PRINTBUF; int ret; @@ -831,7 +831,7 @@ int __bch2_read_extent(struct btree_trans *trans, struct bch_read_bio *orig, goto err; } - ca = bch_dev_bkey_exists(c, pick.ptr.dev); + ca = bch2_dev_bkey_exists(c, pick.ptr.dev); /* * Stale dirty pointers are treated as IO errors, but @failed isn't diff --git a/fs/bcachefs/io_write.c b/fs/bcachefs/io_write.c index 9826772c0eb4c..217cb98ed707c 100644 --- a/fs/bcachefs/io_write.c +++ b/fs/bcachefs/io_write.c @@ -407,9 +407,9 @@ void bch2_submit_wbio_replicas(struct bch_write_bio *wbio, struct bch_fs *c, BUG_ON(c->opts.nochanges); bkey_for_each_ptr(ptrs, ptr) { - BUG_ON(!bch2_dev_exists2(c, ptr->dev)); + BUG_ON(!bch2_dev_exists(c, ptr->dev)); - struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); if (to_entry(ptr + 1) < ptrs.end) { n = to_wbio(bio_alloc_clone(NULL, &wbio->bio, @@ -650,7 +650,7 @@ static void bch2_write_endio(struct bio *bio) struct bch_write_bio *wbio = to_wbio(bio); struct bch_write_bio *parent = wbio->split ? wbio->parent : NULL; struct bch_fs *c = wbio->c; - struct bch_dev *ca = bch_dev_bkey_exists(c, wbio->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, wbio->dev); if (bch2_dev_inum_io_err_on(bio->bi_status, ca, BCH_MEMBER_ERROR_write, op->pos.inode, @@ -1272,7 +1272,7 @@ static void bch2_nocow_write(struct bch_write_op *op) bucket_nocow_lock(&c->nocow_locks, bucket_to_u64(b)); prefetch(l); - if (unlikely(!bch2_dev_get_ioref(bch_dev_bkey_exists(c, ptr->dev), WRITE))) + if (unlikely(!bch2_dev_get_ioref(bch2_dev_bkey_exists(c, ptr->dev), WRITE))) goto err_get_ioref; /* XXX allocating memory with btree locks held - rare */ @@ -1293,7 +1293,7 @@ static void bch2_nocow_write(struct bch_write_op *op) bch2_cut_back(POS(op->pos.inode, op->pos.offset + bio_sectors(bio)), op->insert_keys.top); darray_for_each(buckets, i) { - struct bch_dev *ca = bch_dev_bkey_exists(c, i->b.inode); + struct bch_dev *ca = bch2_dev_bkey_exists(c, i->b.inode); __bch2_bucket_nocow_lock(&c->nocow_locks, i->l, bucket_to_u64(i->b), @@ -1370,7 +1370,7 @@ static void bch2_nocow_write(struct bch_write_op *op) return; err_get_ioref: darray_for_each(buckets, i) - percpu_ref_put(&bch_dev_bkey_exists(c, i->b.inode)->io_ref); + percpu_ref_put(&bch2_dev_bkey_exists(c, i->b.inode)->io_ref); /* Fall back to COW path: */ goto out; diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c index cd14636bc2ab4..1d37eda8d7f00 100644 --- a/fs/bcachefs/journal_io.c +++ b/fs/bcachefs/journal_io.c @@ -21,7 +21,7 @@ void bch2_journal_ptrs_to_text(struct printbuf *out, struct bch_fs *c, struct journal_replay *j) { darray_for_each(j->ptrs, i) { - struct bch_dev *ca = bch_dev_bkey_exists(c, i->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, i->dev); u64 offset; div64_u64_rem(i->sector, ca->mi.bucket_size, &offset); @@ -677,7 +677,7 @@ static int journal_entry_dev_usage_validate(struct bch_fs *c, dev = le32_to_cpu(u->dev); - if (journal_entry_err_on(!bch2_dev_exists2(c, dev), + if (journal_entry_err_on(!bch2_dev_exists(c, dev), c, version, jset, entry, journal_entry_dev_usage_bad_dev, "bad dev")) { @@ -1390,7 +1390,7 @@ int bch2_journal_read(struct bch_fs *c, continue; darray_for_each(i->ptrs, ptr) { - struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); if (!ptr->csum_good) bch_err_dev_offset(ca, ptr->sector, @@ -1400,7 +1400,7 @@ int bch2_journal_read(struct bch_fs *c, } ret = jset_validate(c, - bch_dev_bkey_exists(c, i->ptrs.data[0].dev), + bch2_dev_bkey_exists(c, i->ptrs.data[0].dev), &i->j, i->ptrs.data[0].sector, READ); @@ -1731,7 +1731,7 @@ static CLOSURE_CALLBACK(journal_write_submit) unsigned sectors = vstruct_sectors(w->data, c->block_bits); extent_for_each_ptr(bkey_i_to_s_extent(&w->key), ptr) { - struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); struct journal_device *ja = &ca->journal; if (!percpu_ref_tryget(&ca->io_ref)) { diff --git a/fs/bcachefs/move.c b/fs/bcachefs/move.c index db383dc239bb0..12bc577354ff0 100644 --- a/fs/bcachefs/move.c +++ b/fs/bcachefs/move.c @@ -711,7 +711,7 @@ int bch2_evacuate_bucket(struct moving_context *ctxt, a = bch2_alloc_to_v4(k, &a_convert); dirty_sectors = bch2_bucket_sectors_dirty(*a); - bucket_size = bch_dev_bkey_exists(c, bucket.inode)->mi.bucket_size; + bucket_size = bch2_dev_bkey_exists(c, bucket.inode)->mi.bucket_size; fragmentation = a->fragmentation_lru; ret = bch2_btree_write_buffer_tryflush(trans); diff --git a/fs/bcachefs/recovery.c b/fs/bcachefs/recovery.c index e997217374c64..cc603276f917d 100644 --- a/fs/bcachefs/recovery.c +++ b/fs/bcachefs/recovery.c @@ -372,7 +372,7 @@ static int journal_replay_entry_early(struct bch_fs *c, case BCH_JSET_ENTRY_dev_usage: { struct jset_entry_dev_usage *u = container_of(entry, struct jset_entry_dev_usage, entry); - struct bch_dev *ca = bch_dev_bkey_exists(c, le32_to_cpu(u->dev)); + struct bch_dev *ca = bch2_dev_bkey_exists(c, le32_to_cpu(u->dev)); unsigned i, nr_types = jset_entry_dev_usage_nr_types(u); for (i = 0; i < min_t(unsigned, nr_types, BCH_DATA_NR); i++) { diff --git a/fs/bcachefs/replicas.c b/fs/bcachefs/replicas.c index 678b9c20e2514..f2c99703c7306 100644 --- a/fs/bcachefs/replicas.c +++ b/fs/bcachefs/replicas.c @@ -84,7 +84,7 @@ int bch2_replicas_entry_validate(struct bch_replicas_entry_v1 *r, } for (unsigned i = 0; i < r->nr_devs; i++) - if (!bch2_dev_exists(sb, r->devs[i])) { + if (!bch2_member_exists(sb, r->devs[i])) { prt_printf(err, "invalid device %u in entry ", r->devs[i]); goto bad; } @@ -200,7 +200,7 @@ cpu_replicas_add_entry(struct bch_fs *c, }; for (i = 0; i < new_entry->nr_devs; i++) - BUG_ON(!bch2_dev_exists2(c, new_entry->devs[i])); + BUG_ON(!bch2_dev_exists(c, new_entry->devs[i])); BUG_ON(!new_entry->data_type); verify_replicas_entry(new_entry); @@ -954,7 +954,7 @@ bool bch2_have_enough_devs(struct bch_fs *c, struct bch_devs_mask devs, continue; for (i = 0; i < e->nr_devs; i++) { - struct bch_dev *ca = bch_dev_bkey_exists(c, e->devs[i]); + struct bch_dev *ca = bch2_dev_bkey_exists(c, e->devs[i]); nr_online += test_bit(e->devs[i], devs.d); nr_failed += ca->mi.state == BCH_MEMBER_STATE_failed; diff --git a/fs/bcachefs/sb-members.c b/fs/bcachefs/sb-members.c index 0007fcad5baa8..80bf09cfc0212 100644 --- a/fs/bcachefs/sb-members.c +++ b/fs/bcachefs/sb-members.c @@ -164,7 +164,7 @@ static void member_to_text(struct printbuf *out, u64 bucket_size = le16_to_cpu(m.bucket_size); u64 device_size = le64_to_cpu(m.nbuckets) * bucket_size; - if (!bch2_member_exists(&m)) + if (!bch2_member_alive(&m)) return; prt_printf(out, "Device:\t%u\n", i); @@ -390,7 +390,7 @@ void bch2_dev_errors_reset(struct bch_dev *ca) bool bch2_dev_btree_bitmap_marked(struct bch_fs *c, struct bkey_s_c k) { bkey_for_each_ptr(bch2_bkey_ptrs_c(k), ptr) - if (!bch2_dev_btree_bitmap_marked_sectors(bch_dev_bkey_exists(c, ptr->dev), + if (!bch2_dev_btree_bitmap_marked_sectors(bch2_dev_bkey_exists(c, ptr->dev), ptr->offset, btree_sectors(c))) return false; return true; diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index fa55b96662413..53034cea5843b 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -158,26 +158,38 @@ static inline struct bch_dev *bch2_get_next_online_dev(struct bch_fs *c, #define for_each_readable_member(c, ca) \ __for_each_online_member(c, ca, BIT( BCH_MEMBER_STATE_rw)|BIT(BCH_MEMBER_STATE_ro)) +static inline bool bch2_dev_exists(const struct bch_fs *c, unsigned dev) +{ + return dev < c->sb.nr_devices && c->devs[dev]; +} + /* * If a key exists that references a device, the device won't be going away and * we can omit rcu_read_lock(): */ -static inline struct bch_dev *bch_dev_bkey_exists(const struct bch_fs *c, unsigned idx) +static inline struct bch_dev *bch2_dev_bkey_exists(const struct bch_fs *c, unsigned dev) { - EBUG_ON(idx >= c->sb.nr_devices || !c->devs[idx]); + EBUG_ON(!bch2_dev_exists(c, dev)); - return rcu_dereference_check(c->devs[idx], 1); + return rcu_dereference_check(c->devs[dev], 1); } -static inline struct bch_dev *bch_dev_locked(struct bch_fs *c, unsigned idx) +static inline struct bch_dev *bch2_dev_locked(struct bch_fs *c, unsigned dev) { - EBUG_ON(idx >= c->sb.nr_devices || !c->devs[idx]); + EBUG_ON(!bch2_dev_exists(c, dev)); - return rcu_dereference_protected(c->devs[idx], + return rcu_dereference_protected(c->devs[dev], lockdep_is_held(&c->sb_lock) || lockdep_is_held(&c->state_lock)); } +static inline struct bch_dev *bch2_dev_safe(struct bch_fs *c, unsigned dev) +{ + return c && dev < c->sb.nr_devices + ? rcu_dereference(c->devs[dev]) + : NULL; +} + /* XXX kill, move to struct bch_fs */ static inline struct bch_devs_mask bch2_online_devs(struct bch_fs *c) { @@ -192,16 +204,16 @@ static inline struct bch_devs_mask bch2_online_devs(struct bch_fs *c) extern const struct bch_sb_field_ops bch_sb_field_ops_members_v1; extern const struct bch_sb_field_ops bch_sb_field_ops_members_v2; -static inline bool bch2_member_exists(struct bch_member *m) +static inline bool bch2_member_alive(struct bch_member *m) { return !bch2_is_zero(&m->uuid, sizeof(m->uuid)); } -static inline bool bch2_dev_exists(struct bch_sb *sb, unsigned dev) +static inline bool bch2_member_exists(struct bch_sb *sb, unsigned dev) { if (dev < sb->nr_devices) { struct bch_member m = bch2_sb_member_get(sb, dev); - return bch2_member_exists(&m); + return bch2_member_alive(&m); } return false; } @@ -222,7 +234,7 @@ static inline struct bch_member_cpu bch2_mi_to_cpu(struct bch_member *mi) ? BCH_MEMBER_DURABILITY(mi) - 1 : 1, .freespace_initialized = BCH_MEMBER_FREESPACE_INITIALIZED(mi), - .valid = bch2_member_exists(mi), + .valid = bch2_member_alive(mi), .btree_bitmap_shift = mi->btree_bitmap_shift, .btree_allocated_bitmap = le64_to_cpu(mi->btree_allocated_bitmap), }; diff --git a/fs/bcachefs/super-io.c b/fs/bcachefs/super-io.c index 52f0ffd4a0d27..fb45749586811 100644 --- a/fs/bcachefs/super-io.c +++ b/fs/bcachefs/super-io.c @@ -1300,7 +1300,7 @@ void bch2_sb_to_text(struct printbuf *out, struct bch_sb *sb, printbuf_tabstop_push(out, 44); for (int i = 0; i < sb->nr_devices; i++) - nr_devices += bch2_dev_exists(sb, i); + nr_devices += bch2_member_exists(sb, i); prt_printf(out, "External UUID:\t"); pr_uuid(out, sb->user_uuid.b); diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index c161313913001..64fa76788ed1e 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -941,7 +941,7 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) goto err; for (i = 0; i < c->sb.nr_devices; i++) - if (bch2_dev_exists(c->disk_sb.sb, i) && + if (bch2_member_exists(c->disk_sb.sb, i) && bch2_dev_alloc(c, i)) { ret = -EEXIST; goto err; @@ -1102,7 +1102,7 @@ static int bch2_dev_in_fs(struct bch_sb_handle *fs, if (!uuid_equal(&fs->sb->uuid, &sb->sb->uuid)) return -BCH_ERR_device_not_a_member_of_filesystem; - if (!bch2_dev_exists(fs->sb, sb->sb->dev_idx)) + if (!bch2_member_exists(fs->sb, sb->sb->dev_idx)) return -BCH_ERR_device_has_been_removed; if (fs->sb->block_size != sb->sb->block_size) @@ -1412,10 +1412,9 @@ static int bch2_dev_attach_bdev(struct bch_fs *c, struct bch_sb_handle *sb) le64_to_cpu(c->disk_sb.sb->seq)) bch2_sb_to_fs(c, sb->sb); - BUG_ON(sb->sb->dev_idx >= c->sb.nr_devices || - !c->devs[sb->sb->dev_idx]); + BUG_ON(!bch2_dev_exists(c, sb->sb->dev_idx)); - ca = bch_dev_locked(c, sb->sb->dev_idx); + ca = bch2_dev_locked(c, sb->sb->dev_idx); ret = __bch2_dev_attach_bdev(ca, sb); if (ret) @@ -1507,10 +1506,10 @@ static bool bch2_fs_may_start(struct bch_fs *c) mutex_lock(&c->sb_lock); for (i = 0; i < c->disk_sb.sb->nr_devices; i++) { - if (!bch2_dev_exists(c->disk_sb.sb, i)) + if (!bch2_member_exists(c->disk_sb.sb, i)) continue; - ca = bch_dev_locked(c, i); + ca = bch2_dev_locked(c, i); if (!bch2_dev_is_online(ca) && (ca->mi.state == BCH_MEMBER_STATE_rw || @@ -1779,7 +1778,7 @@ int bch2_dev_add(struct bch_fs *c, const char *path) goto no_slot; for (dev_idx = 0; dev_idx < BCH_SB_MEMBERS_MAX; dev_idx++) - if (!bch2_dev_exists(c->disk_sb.sb, dev_idx)) + if (!bch2_member_exists(c->disk_sb.sb, dev_idx)) goto have_slot; no_slot: ret = -BCH_ERR_ENOSPC_sb_members; @@ -1885,7 +1884,7 @@ int bch2_dev_online(struct bch_fs *c, const char *path) if (ret) goto err; - ca = bch_dev_locked(c, dev_idx); + ca = bch2_dev_locked(c, dev_idx); ret = bch2_trans_mark_dev_sb(c, ca, BTREE_TRIGGER_transactional); bch_err_msg(c, ret, "bringing %s online: error from bch2_trans_mark_dev_sb", path); From af3b39b4c64198a08271485c6c2214ff637c3cbd Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Fri, 8 Mar 2024 09:12:47 -0300 Subject: [PATCH 1385/1702] bcachefs: chardev: make bch_chardev_class constant Since commit 43a7206b0963 ("driver core: class: make class_register() take a const *"), the driver core allows for struct class to be in read-only memory, so move the bch_chardev_class structure to be declared at build time placing it into read-only memory, instead of having to be dynamically allocated at boot time. Also, correctly clean up after failing paths in bch2_chardev_init(). Cc: Hongbo Li Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: Ricardo B. Marliere Signed-off-by: Kent Overstreet --- fs/bcachefs/chardev.c | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/fs/bcachefs/chardev.c b/fs/bcachefs/chardev.c index 4d14f19f51850..dbcd433db0049 100644 --- a/fs/bcachefs/chardev.c +++ b/fs/bcachefs/chardev.c @@ -961,7 +961,9 @@ static const struct file_operations bch_chardev_fops = { }; static int bch_chardev_major; -static struct class *bch_chardev_class; +static const struct class bch_chardev_class = { + .name = "bcachefs", +}; static struct device *bch_chardev; void bch2_fs_chardev_exit(struct bch_fs *c) @@ -978,7 +980,7 @@ int bch2_fs_chardev_init(struct bch_fs *c) if (c->minor < 0) return c->minor; - c->chardev = device_create(bch_chardev_class, NULL, + c->chardev = device_create(&bch_chardev_class, NULL, MKDEV(bch_chardev_major, c->minor), c, "bcachefs%u-ctl", c->minor); if (IS_ERR(c->chardev)) @@ -989,32 +991,39 @@ int bch2_fs_chardev_init(struct bch_fs *c) void bch2_chardev_exit(void) { - if (!IS_ERR_OR_NULL(bch_chardev_class)) - device_destroy(bch_chardev_class, - MKDEV(bch_chardev_major, U8_MAX)); - if (!IS_ERR_OR_NULL(bch_chardev_class)) - class_destroy(bch_chardev_class); + device_destroy(&bch_chardev_class, MKDEV(bch_chardev_major, U8_MAX)); + class_unregister(&bch_chardev_class); if (bch_chardev_major > 0) unregister_chrdev(bch_chardev_major, "bcachefs"); } int __init bch2_chardev_init(void) { + int ret; + bch_chardev_major = register_chrdev(0, "bcachefs-ctl", &bch_chardev_fops); if (bch_chardev_major < 0) return bch_chardev_major; - bch_chardev_class = class_create("bcachefs"); - if (IS_ERR(bch_chardev_class)) - return PTR_ERR(bch_chardev_class); + ret = class_register(&bch_chardev_class); + if (ret) + goto major_out; - bch_chardev = device_create(bch_chardev_class, NULL, + bch_chardev = device_create(&bch_chardev_class, NULL, MKDEV(bch_chardev_major, U8_MAX), NULL, "bcachefs-ctl"); - if (IS_ERR(bch_chardev)) - return PTR_ERR(bch_chardev); + if (IS_ERR(bch_chardev)) { + ret = PTR_ERR(bch_chardev); + goto class_out; + } return 0; + +class_out: + class_unregister(&bch_chardev_class); +major_out: + unregister_chrdev(bch_chardev_major, "bcachefs-ctl"); + return ret; } #endif /* NO_BCACHEFS_CHARDEV */ From d434c2398fe30c8e7d8784410c0a9273af0f2f79 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Thu, 11 Apr 2024 10:29:31 +0200 Subject: [PATCH 1386/1702] bcachefs: fix typo in reference to BCACHEFS_DEBUG Commit ec9cc18fc2e6 ("bcachefs: Add checks for invalid snapshot IDs") intends to check the sanity of a snapshot and panic when BCACHEFS_DEBUG is set, but that conditional has a typo. Fix the typo to refer to the actual existing Kconfig symbol. This was found with ./scripts/checkkconfigsymbols.py. Signed-off-by: Lukas Bulwahn Signed-off-by: Kent Overstreet --- fs/bcachefs/snapshot.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/bcachefs/snapshot.h b/fs/bcachefs/snapshot.h index b7d2fed37c4f3..3fdb41b33d2d2 100644 --- a/fs/bcachefs/snapshot.h +++ b/fs/bcachefs/snapshot.h @@ -77,7 +77,7 @@ static inline u32 __bch2_snapshot_parent(struct bch_fs *c, u32 id) return 0; u32 parent = s->parent; - if (IS_ENABLED(CONFIG_BCACHEFS_DEBU) && + if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG) && parent && s->depth != snapshot_t(c, parent)->depth + 1) panic("id %u depth=%u parent %u depth=%u\n", From 91b5d97fdf3ca307b7aec1b3c0dcbaa0473cf43d Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 12 Apr 2024 15:44:09 -0400 Subject: [PATCH 1387/1702] bcachefs: get_unlocked_mut_path -> bch2_path_get_unlocked_mut Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.c | 16 +++++++++++++++ fs/bcachefs/btree_iter.h | 3 +++ fs/bcachefs/btree_update_interior.c | 30 +++++++---------------------- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index db47063e5844e..e08167e3559d3 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -1729,6 +1729,22 @@ btree_path_idx_t bch2_path_get(struct btree_trans *trans, return path_idx; } +btree_path_idx_t bch2_path_get_unlocked_mut(struct btree_trans *trans, + enum btree_id btree_id, + unsigned level, + struct bpos pos) +{ + btree_path_idx_t path_idx = bch2_path_get(trans, btree_id, pos, level + 1, level, + BTREE_ITER_nopreserve| + BTREE_ITER_intent, _RET_IP_); + path_idx = bch2_btree_path_make_mut(trans, path_idx, true, _RET_IP_); + + struct btree_path *path = trans->paths + path_idx; + bch2_btree_path_downgrade(trans, path); + __bch2_btree_path_unlock(trans, path); + return path_idx; +} + struct bkey_s_c bch2_btree_path_peek_slot(struct btree_path *path, struct bkey *u) { diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index abbf7bc8d9c53..3bef5b1af16d0 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -227,6 +227,9 @@ static inline int __must_check bch2_btree_path_traverse(struct btree_trans *tran btree_path_idx_t bch2_path_get(struct btree_trans *, enum btree_id, struct bpos, unsigned, unsigned, unsigned, unsigned long); +btree_path_idx_t bch2_path_get_unlocked_mut(struct btree_trans *, enum btree_id, + unsigned, struct bpos); + struct bkey_s_c bch2_btree_path_peek_slot(struct btree_path *, struct bkey *); /* diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index 0b6645c2266fb..6427328bcf177 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -38,22 +38,6 @@ static int bch2_btree_insert_node(struct btree_update *, struct btree_trans *, btree_path_idx_t, struct btree *, struct keylist *); static void bch2_btree_update_add_new_node(struct btree_update *, struct btree *); -static btree_path_idx_t get_unlocked_mut_path(struct btree_trans *trans, - enum btree_id btree_id, - unsigned level, - struct bpos pos) -{ - btree_path_idx_t path_idx = bch2_path_get(trans, btree_id, pos, level + 1, level, - BTREE_ITER_nopreserve| - BTREE_ITER_intent, _RET_IP_); - path_idx = bch2_btree_path_make_mut(trans, path_idx, true, _RET_IP_); - - struct btree_path *path = trans->paths + path_idx; - bch2_btree_path_downgrade(trans, path); - __bch2_btree_path_unlock(trans, path); - return path_idx; -} - /* * Verify that child nodes correctly span parent node's range: */ @@ -753,7 +737,7 @@ static void btree_update_nodes_written(struct btree_update *as) */ b = READ_ONCE(as->b); if (b) { - btree_path_idx_t path_idx = get_unlocked_mut_path(trans, + btree_path_idx_t path_idx = bch2_path_get_unlocked_mut(trans, as->btree_id, b->c.level, b->key.k.p); struct btree_path *path = trans->paths + path_idx; /* @@ -773,7 +757,7 @@ static void btree_update_nodes_written(struct btree_update *as) * btree_node_lock_nopath() (the use of which is always suspect, * we need to work on removing this in the future) * - * It should be, but get_unlocked_mut_path() -> bch2_path_get() + * It should be, but bch2_path_get_unlocked_mut() -> bch2_path_get() * calls bch2_path_upgrade(), before we call path_make_mut(), so * we may rarely end up with a locked path besides the one we * have here: @@ -1637,12 +1621,12 @@ static int btree_split(struct btree_update *as, struct btree_trans *trans, six_unlock_write(&n2->c.lock); six_unlock_write(&n1->c.lock); - path1 = get_unlocked_mut_path(trans, as->btree_id, n1->c.level, n1->key.k.p); + path1 = bch2_path_get_unlocked_mut(trans, as->btree_id, n1->c.level, n1->key.k.p); six_lock_increment(&n1->c.lock, SIX_LOCK_intent); mark_btree_node_locked(trans, trans->paths + path1, n1->c.level, BTREE_NODE_INTENT_LOCKED); bch2_btree_path_level_init(trans, trans->paths + path1, n1); - path2 = get_unlocked_mut_path(trans, as->btree_id, n2->c.level, n2->key.k.p); + path2 = bch2_path_get_unlocked_mut(trans, as->btree_id, n2->c.level, n2->key.k.p); six_lock_increment(&n2->c.lock, SIX_LOCK_intent); mark_btree_node_locked(trans, trans->paths + path2, n2->c.level, BTREE_NODE_INTENT_LOCKED); bch2_btree_path_level_init(trans, trans->paths + path2, n2); @@ -1687,7 +1671,7 @@ static int btree_split(struct btree_update *as, struct btree_trans *trans, bch2_btree_update_add_new_node(as, n1); six_unlock_write(&n1->c.lock); - path1 = get_unlocked_mut_path(trans, as->btree_id, n1->c.level, n1->key.k.p); + path1 = bch2_path_get_unlocked_mut(trans, as->btree_id, n1->c.level, n1->key.k.p); six_lock_increment(&n1->c.lock, SIX_LOCK_intent); mark_btree_node_locked(trans, trans->paths + path1, n1->c.level, BTREE_NODE_INTENT_LOCKED); bch2_btree_path_level_init(trans, trans->paths + path1, n1); @@ -2090,7 +2074,7 @@ int __bch2_foreground_maybe_merge(struct btree_trans *trans, bch2_btree_update_add_new_node(as, n); six_unlock_write(&n->c.lock); - new_path = get_unlocked_mut_path(trans, btree, n->c.level, n->key.k.p); + new_path = bch2_path_get_unlocked_mut(trans, btree, n->c.level, n->key.k.p); six_lock_increment(&n->c.lock, SIX_LOCK_intent); mark_btree_node_locked(trans, trans->paths + new_path, n->c.level, BTREE_NODE_INTENT_LOCKED); bch2_btree_path_level_init(trans, trans->paths + new_path, n); @@ -2168,7 +2152,7 @@ int bch2_btree_node_rewrite(struct btree_trans *trans, bch2_btree_update_add_new_node(as, n); six_unlock_write(&n->c.lock); - new_path = get_unlocked_mut_path(trans, iter->btree_id, n->c.level, n->key.k.p); + new_path = bch2_path_get_unlocked_mut(trans, iter->btree_id, n->c.level, n->key.k.p); six_lock_increment(&n->c.lock, SIX_LOCK_intent); mark_btree_node_locked(trans, trans->paths + new_path, n->c.level, BTREE_NODE_INTENT_LOCKED); bch2_btree_path_level_init(trans, trans->paths + new_path, n); From b7f10636d51a8f07fdf569c2de5cbfabae549c91 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 13 Apr 2024 17:40:06 -0400 Subject: [PATCH 1388/1702] bcachefs: prefer drop_locks_do() Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_update_interior.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index 6427328bcf177..ce9712d7c764f 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -1160,9 +1160,8 @@ bch2_btree_update_start(struct btree_trans *trans, struct btree_path *path, if (flags & BCH_TRANS_COMMIT_journal_reclaim) return ERR_PTR(-BCH_ERR_journal_reclaim_would_deadlock); - bch2_trans_unlock(trans); - wait_event(c->journal.wait, !test_bit(JOURNAL_SPACE_LOW, &c->journal.flags)); - ret = bch2_trans_relock(trans); + ret = drop_locks_do(trans, + ({ wait_event(c->journal.wait, !test_bit(JOURNAL_SPACE_LOW, &c->journal.flags)); 0; })); if (ret) return ERR_PTR(ret); } From 449ceafb49e43012a76ac26a2c12aa94bdb50176 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 13 Apr 2024 17:49:23 -0400 Subject: [PATCH 1389/1702] bcachefs: bch2_trans_commit_flags_to_text() Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_trans_commit.c | 21 +++++++++++++++++++++ fs/bcachefs/btree_update.h | 2 ++ fs/bcachefs/btree_update_interior.c | 9 +++++---- fs/bcachefs/btree_update_interior.h | 2 +- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/fs/bcachefs/btree_trans_commit.c b/fs/bcachefs/btree_trans_commit.c index c5317e74ee069..04831c9e603cc 100644 --- a/fs/bcachefs/btree_trans_commit.c +++ b/fs/bcachefs/btree_trans_commit.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include "bcachefs.h" +#include "alloc_foreground.h" #include "btree_gc.h" #include "btree_io.h" #include "btree_iter.h" @@ -19,6 +20,26 @@ #include +static const char * const trans_commit_flags_strs[] = { +#define x(n, ...) #n, + BCH_TRANS_COMMIT_FLAGS() +#undef x + NULL +}; + +void bch2_trans_commit_flags_to_text(struct printbuf *out, enum bch_trans_commit_flags flags) +{ + enum bch_watermark watermark = flags & BCH_WATERMARK_MASK; + + prt_printf(out, "watermark=%s", bch2_watermarks[watermark]); + + flags >>= BCH_WATERMARK_BITS; + if (flags) { + prt_char(out, ' '); + bch2_prt_bitflags(out, trans_commit_flags_strs, flags); + } +} + static void verify_update_old_key(struct btree_trans *trans, struct btree_insert_entry *i) { #ifdef CONFIG_BCACHEFS_DEBUG diff --git a/fs/bcachefs/btree_update.h b/fs/bcachefs/btree_update.h index 3a04ede35475f..b4894e4d5447f 100644 --- a/fs/bcachefs/btree_update.h +++ b/fs/bcachefs/btree_update.h @@ -44,6 +44,8 @@ enum bch_trans_commit_flags { #undef x }; +void bch2_trans_commit_flags_to_text(struct printbuf *, enum bch_trans_commit_flags); + int bch2_btree_delete_extent_at(struct btree_trans *, struct btree_iter *, unsigned, unsigned); int bch2_btree_delete_at(struct btree_trans *, struct btree_iter *, unsigned); diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index ce9712d7c764f..4aa62a74a59bf 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -1207,7 +1207,7 @@ bch2_btree_update_start(struct btree_trans *trans, struct btree_path *path, as->start_time = start_time; as->ip_started = _RET_IP_; as->mode = BTREE_UPDATE_none; - as->watermark = watermark; + as->flags = flags; as->took_gc_lock = true; as->btree_id = path->btree_id; as->update_level_start = level_start; @@ -2565,12 +2565,13 @@ void bch2_btree_root_alloc_fake(struct bch_fs *c, enum btree_id id, unsigned lev static void bch2_btree_update_to_text(struct printbuf *out, struct btree_update *as) { - prt_printf(out, "%ps: btree=%s l=%u-%u watermark=%s mode=%s nodes_written=%u cl.remaining=%u journal_seq=%llu\n", - (void *) as->ip_started, + prt_printf(out, "%ps: ", (void *) as->ip_started); + bch2_trans_commit_flags_to_text(out, as->flags); + + prt_printf(out, " btree=%s l=%u-%u mode=%s nodes_written=%u cl.remaining=%u journal_seq=%llu\n", bch2_btree_id_str(as->btree_id), as->update_level_start, as->update_level_end, - bch2_watermarks[as->watermark], bch2_btree_update_modes[as->mode], as->nodes_written, closure_nr_remaining(&as->cl), diff --git a/fs/bcachefs/btree_update_interior.h b/fs/bcachefs/btree_update_interior.h index ca1a3be43af8b..b2f7f8110c50d 100644 --- a/fs/bcachefs/btree_update_interior.h +++ b/fs/bcachefs/btree_update_interior.h @@ -52,7 +52,7 @@ struct btree_update { struct list_head unwritten_list; enum btree_update_mode mode; - enum bch_watermark watermark; + enum bch_trans_commit_flags flags; unsigned nodes_written:1; unsigned took_gc_lock:1; From f2d9823f46b4aebbe2d6a7f1a76158536c6291a8 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 11 Apr 2024 18:04:00 -0400 Subject: [PATCH 1390/1702] bcachefs: maintain lock invariants in btree_iter_next_node() Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index e08167e3559d3..bb037e4da2990 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -1902,6 +1902,9 @@ struct btree *bch2_btree_iter_next_node(struct btree_iter *iter) if (bpos_eq(iter->pos, b->key.k.p)) { __btree_path_set_level_up(trans, path, path->level++); } else { + if (btree_lock_want(path, path->level + 1) == BTREE_NODE_UNLOCKED) + btree_node_unlock(trans, path, path->level + 1); + /* * Haven't gotten to the end of the parent node: go back down to * the next child node From 5d8c9d94283ff170e74f13b501c3f264365621b5 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 10 Apr 2024 22:19:40 -0400 Subject: [PATCH 1391/1702] bcachefs: bch2_btree_path_upgrade() checks nodes_locked, not uptodate In the key cache fill path, we use path_upgrade() on a path that isn't uptodate yet but should be locked. This change makes bch2_btree_path_upgrade() slightly looser so we can use it in key cache upgrade, instead of the __ version. Also, make the related assert - that path->uptodate implies nodes_locked - slightly clearer. Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_locking.c | 13 ++++++------- fs/bcachefs/btree_locking.h | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/fs/bcachefs/btree_locking.c b/fs/bcachefs/btree_locking.c index e6adb2df3a571..9a3ab56f6cfcb 100644 --- a/fs/bcachefs/btree_locking.c +++ b/fs/bcachefs/btree_locking.c @@ -840,20 +840,19 @@ int __bch2_trans_mutex_lock(struct btree_trans *trans, void bch2_btree_path_verify_locks(struct btree_path *path) { - unsigned l; - /* * A path may be uptodate and yet have nothing locked if and only if * there is no node at path->level, which generally means we were * iterating over all nodes and got to the end of the btree */ - if (!path->nodes_locked) { - BUG_ON(path->uptodate == BTREE_ITER_UPTODATE && - btree_path_node(path, path->level)); + BUG_ON(path->uptodate == BTREE_ITER_UPTODATE && + btree_path_node(path, path->level) && + !path->nodes_locked); + + if (!path->nodes_locked) return; - } - for (l = 0; l < BTREE_MAX_DEPTH; l++) { + for (unsigned l = 0; l < BTREE_MAX_DEPTH; l++) { int want = btree_lock_want(path, l); int have = btree_node_locked_type(path, l); diff --git a/fs/bcachefs/btree_locking.h b/fs/bcachefs/btree_locking.h index 4bd72c855da1a..7f41545b9147f 100644 --- a/fs/bcachefs/btree_locking.h +++ b/fs/bcachefs/btree_locking.h @@ -364,14 +364,14 @@ static inline int bch2_btree_path_upgrade(struct btree_trans *trans, struct btree_path *path, unsigned new_locks_want) { - struct get_locks_fail f; + struct get_locks_fail f = {}; unsigned old_locks_want = path->locks_want; new_locks_want = min(new_locks_want, BTREE_MAX_DEPTH); if (path->locks_want < new_locks_want ? __bch2_btree_path_upgrade(trans, path, new_locks_want, &f) - : path->uptodate == BTREE_ITER_UPTODATE) + : path->nodes_locked) return 0; trace_and_count(trans->c, trans_restart_upgrade, trans, _THIS_IP_, path, From 4984faff5d42f8069ab9223dd80dabd4f73469f5 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 10 Apr 2024 00:35:24 -0400 Subject: [PATCH 1392/1702] bcachefs: Use bch2_btree_path_upgrade() in key cache traverse Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_key_cache.c | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/fs/bcachefs/btree_key_cache.c b/fs/bcachefs/btree_key_cache.c index 547ba9329678e..6645264a481bd 100644 --- a/fs/bcachefs/btree_key_cache.c +++ b/fs/bcachefs/btree_key_cache.c @@ -516,22 +516,9 @@ bch2_btree_path_traverse_cached_slowpath(struct btree_trans *trans, struct btree path->uptodate = BTREE_ITER_UPTODATE; if (!ck->valid && !(flags & BTREE_ITER_cached_nofill)) { - /* - * Using the underscore version because we haven't set - * path->uptodate yet: - */ - if (!path->locks_want && - !__bch2_btree_path_upgrade(trans, path, 1, NULL)) { - trace_and_count(trans->c, trans_restart_key_cache_upgrade, trans, _THIS_IP_); - ret = btree_trans_restart(trans, BCH_ERR_transaction_restart_key_cache_upgrade); - goto err; - } - - ret = btree_key_cache_fill(trans, path, ck); - if (ret) - goto err; - - ret = bch2_btree_path_relock(trans, path, _THIS_IP_); + ret = bch2_btree_path_upgrade(trans, path, 1) ?: + btree_key_cache_fill(trans, path, ck) ?: + bch2_btree_path_relock(trans, path, _THIS_IP_); if (ret) goto err; From ca563dccb2f0fb9b77971dcbead37b7e5f75723d Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 9 Apr 2024 23:23:08 -0400 Subject: [PATCH 1393/1702] bcachefs: bch2_trans_unlock() must always be followed by relock() or begin() We're about to add new asserts for btree_trans locking consistency, and part of that requires that aren't using the btree_trans while it's unlocked. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 3 +++ fs/bcachefs/alloc_foreground.c | 4 ++++ fs/bcachefs/btree_iter.h | 2 ++ fs/bcachefs/btree_update_interior.c | 7 ++++--- fs/bcachefs/data_update.c | 2 ++ fs/bcachefs/fs.c | 4 ++++ fs/bcachefs/io_write.c | 4 ++++ fs/bcachefs/movinggc.c | 2 ++ fs/bcachefs/rebalance.c | 2 ++ fs/bcachefs/recovery.c | 3 ++- 10 files changed, 29 insertions(+), 4 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 2f91ff67453f2..f07373b781745 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -2172,6 +2172,9 @@ int bch2_bucket_io_time_reset(struct btree_trans *trans, unsigned dev, u64 now; int ret = 0; + if (bch2_trans_relock(trans)) + bch2_trans_begin(trans); + a = bch2_trans_start_alloc_update(trans, &iter, POS(dev, bucket_nr)); ret = PTR_ERR_OR_ZERO(a); if (ret) diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index df4439a38df5a..fb8825c4e7ad8 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -1342,6 +1342,10 @@ int bch2_alloc_sectors_start_trans(struct btree_trans *trans, *wp_ret = wp = writepoint_find(trans, write_point.v); + ret = bch2_trans_relock(trans); + if (ret) + goto err; + /* metadata may not allocate on cache devices: */ if (wp->data_type != BCH_DATA_user) have_cache = true; diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index 3bef5b1af16d0..0d44d613e4ac1 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -729,6 +729,8 @@ transaction_restart: \ #define for_each_btree_key_upto(_trans, _iter, _btree_id, \ _start, _end, _flags, _k, _do) \ ({ \ + bch2_trans_begin(trans); \ + \ struct btree_iter _iter; \ bch2_trans_iter_init((_trans), &(_iter), (_btree_id), \ (_start), (_flags)); \ diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index 4aa62a74a59bf..35f80c97b9737 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -737,9 +737,6 @@ static void btree_update_nodes_written(struct btree_update *as) */ b = READ_ONCE(as->b); if (b) { - btree_path_idx_t path_idx = bch2_path_get_unlocked_mut(trans, - as->btree_id, b->c.level, b->key.k.p); - struct btree_path *path = trans->paths + path_idx; /* * @b is the node we did the final insert into: * @@ -763,6 +760,10 @@ static void btree_update_nodes_written(struct btree_update *as) * have here: */ bch2_trans_unlock(trans); + bch2_trans_begin(trans); + btree_path_idx_t path_idx = bch2_path_get_unlocked_mut(trans, + as->btree_id, b->c.level, b->key.k.p); + struct btree_path *path = trans->paths + path_idx; btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_intent); mark_btree_node_locked(trans, path, b->c.level, BTREE_NODE_INTENT_LOCKED); path->l[b->c.level].lock_seq = six_lock_seq(&b->c.lock); diff --git a/fs/bcachefs/data_update.c b/fs/bcachefs/data_update.c index cae4e4eb5329a..e8c7c5cec03fe 100644 --- a/fs/bcachefs/data_update.c +++ b/fs/bcachefs/data_update.c @@ -386,6 +386,8 @@ static void bch2_update_unwritten_extent(struct btree_trans *trans, while (bio_sectors(bio)) { unsigned sectors = bio_sectors(bio); + bch2_trans_begin(trans); + bch2_trans_iter_init(trans, &iter, update->btree_id, update->op.pos, BTREE_ITER_slots); ret = lockrestart_do(trans, ({ diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c index 81cfc74828fd7..841bb92e53dfd 100644 --- a/fs/bcachefs/fs.c +++ b/fs/bcachefs/fs.c @@ -1036,6 +1036,10 @@ static int bch2_fiemap(struct inode *vinode, struct fiemap_extent_info *info, bch2_btree_iter_set_pos(&iter, POS(iter.pos.inode, iter.pos.offset + sectors)); + + ret = bch2_trans_relock(trans); + if (ret) + break; } start = iter.pos.offset; bch2_trans_iter_exit(trans, &iter); diff --git a/fs/bcachefs/io_write.c b/fs/bcachefs/io_write.c index 217cb98ed707c..73e25250de754 100644 --- a/fs/bcachefs/io_write.c +++ b/fs/bcachefs/io_write.c @@ -1248,6 +1248,10 @@ static void bch2_nocow_write(struct bch_write_op *op) buckets.nr = 0; + ret = bch2_trans_relock(trans); + if (ret) + break; + k = bch2_btree_iter_peek_slot(&iter); ret = bkey_err(k); if (ret) diff --git a/fs/bcachefs/movinggc.c b/fs/bcachefs/movinggc.c index d0afd8bc0193b..10bfb31c151b2 100644 --- a/fs/bcachefs/movinggc.c +++ b/fs/bcachefs/movinggc.c @@ -158,6 +158,8 @@ static int bch2_copygc_get_buckets(struct moving_context *ctxt, if (bch2_fs_fatal_err_on(ret, c, "%s: from bch2_btree_write_buffer_tryflush()", bch2_err_str(ret))) return ret; + bch2_trans_begin(trans); + ret = for_each_btree_key_upto(trans, iter, BTREE_ID_lru, lru_pos(BCH_LRU_FRAGMENTATION_START, 0, 0), lru_pos(BCH_LRU_FRAGMENTATION_START, U64_MAX, LRU_TIME_MAX), diff --git a/fs/bcachefs/rebalance.c b/fs/bcachefs/rebalance.c index c640c0f13976c..cf81e5128c3ab 100644 --- a/fs/bcachefs/rebalance.c +++ b/fs/bcachefs/rebalance.c @@ -323,6 +323,8 @@ static int do_rebalance(struct moving_context *ctxt) struct bkey_s_c k; int ret = 0; + bch2_trans_begin(trans); + bch2_move_stats_init(&r->work_stats, "rebalance_work"); bch2_move_stats_init(&r->scan_stats, "rebalance_scan"); diff --git a/fs/bcachefs/recovery.c b/fs/bcachefs/recovery.c index cc603276f917d..27ee27b285bde 100644 --- a/fs/bcachefs/recovery.c +++ b/fs/bcachefs/recovery.c @@ -202,7 +202,7 @@ int bch2_journal_replay(struct bch_fs *c) struct journal *j = &c->journal; u64 start_seq = c->journal_replay_seq_start; u64 end_seq = c->journal_replay_seq_start; - struct btree_trans *trans = bch2_trans_get(c); + struct btree_trans *trans = NULL; bool immediate_flush = false; int ret = 0; @@ -216,6 +216,7 @@ int bch2_journal_replay(struct bch_fs *c) BUG_ON(!atomic_read(&keys->ref)); move_gap(keys, keys->nr); + trans = bch2_trans_get(c); /* * First, attempt to replay keys in sorted order. This is more From e2e568bd9775aa86196ea89b9bacc17c04c652f5 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 14 Apr 2024 18:42:42 -0400 Subject: [PATCH 1394/1702] bcachefs: bch2_btree_root_alloc_fake_trans() We're starting to be more strict about transaction locked state, and multiple transactions in a task. Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_gc.c | 6 +++--- fs/bcachefs/btree_update_interior.c | 4 ++-- fs/bcachefs/btree_update_interior.h | 2 ++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 4c5bd256d2511..7549e806a6f48 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -540,9 +540,9 @@ int bch2_check_topology(struct bch_fs *c) if (!bch2_btree_has_scanned_nodes(c, i)) { mustfix_fsck_err(c, btree_root_unreadable_and_scan_found_nothing, "no nodes found for btree %s, continue?", bch2_btree_id_str(i)); - bch2_btree_root_alloc_fake(c, i, 0); + bch2_btree_root_alloc_fake_trans(trans, i, 0); } else { - bch2_btree_root_alloc_fake(c, i, 1); + bch2_btree_root_alloc_fake_trans(trans, i, 1); bch2_shoot_down_journal_keys(c, i, 1, BTREE_MAX_DEPTH, POS_MIN, SPOS_MAX); ret = bch2_get_scanned_nodes(c, i, 0, POS_MIN, SPOS_MAX); if (ret) @@ -570,7 +570,7 @@ int bch2_check_topology(struct bch_fs *c) goto reconstruct_root; bch_err(c, "empty btree root %s", bch2_btree_id_str(i)); - bch2_btree_root_alloc_fake(c, i, 0); + bch2_btree_root_alloc_fake_trans(trans, i, 0); r->alive = false; ret = 0; } diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index 35f80c97b9737..a87608dd74c6d 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -2513,7 +2513,7 @@ void bch2_btree_set_root_for_read(struct bch_fs *c, struct btree *b) bch2_btree_set_root_inmem(c, b); } -static int __bch2_btree_root_alloc_fake(struct btree_trans *trans, enum btree_id id, unsigned level) +int bch2_btree_root_alloc_fake_trans(struct btree_trans *trans, enum btree_id id, unsigned level) { struct bch_fs *c = trans->c; struct closure cl; @@ -2561,7 +2561,7 @@ static int __bch2_btree_root_alloc_fake(struct btree_trans *trans, enum btree_id void bch2_btree_root_alloc_fake(struct bch_fs *c, enum btree_id id, unsigned level) { - bch2_trans_run(c, __bch2_btree_root_alloc_fake(trans, id, level)); + bch2_trans_run(c, bch2_btree_root_alloc_fake_trans(trans, id, level)); } static void bch2_btree_update_to_text(struct printbuf *out, struct btree_update *as) diff --git a/fs/bcachefs/btree_update_interior.h b/fs/bcachefs/btree_update_interior.h index b2f7f8110c50d..b5b76ce01cfcb 100644 --- a/fs/bcachefs/btree_update_interior.h +++ b/fs/bcachefs/btree_update_interior.h @@ -175,6 +175,8 @@ int bch2_btree_node_update_key_get_iter(struct btree_trans *, struct btree *, struct bkey_i *, unsigned, bool); void bch2_btree_set_root_for_read(struct bch_fs *, struct btree *); + +int bch2_btree_root_alloc_fake_trans(struct btree_trans *, enum btree_id, unsigned); void bch2_btree_root_alloc_fake(struct bch_fs *, enum btree_id, unsigned); static inline unsigned btree_update_reserve_required(struct bch_fs *c, From 650db8a87c343c856dcbaed2eb8d184df0308c42 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 9 Apr 2024 19:57:08 -0400 Subject: [PATCH 1395/1702] bcachefs: trans->locked Add a field for tracking whether a transaction object holds btree locks, and assertions to verify state. Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.c | 9 +++++-- fs/bcachefs/btree_iter.h | 1 - fs/bcachefs/btree_locking.c | 52 +++++++++++++++++++++++-------------- fs/bcachefs/btree_types.h | 4 ++- 4 files changed, 43 insertions(+), 23 deletions(-) diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index bb037e4da2990..224001d8470d6 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -999,6 +999,7 @@ static int bch2_btree_path_traverse_all(struct btree_trans *trans) bch2_trans_unlock(trans); cond_resched(); + trans->locked = true; if (unlikely(trans->memory_allocation_failure)) { struct closure cl; @@ -3023,7 +3024,8 @@ u32 bch2_trans_begin(struct btree_trans *trans) if (!trans->restarted && (need_resched() || time_after64(now, trans->last_begin_time + BTREE_TRANS_MAX_LOCK_HOLD_TIME_NS))) { - drop_locks_do(trans, (cond_resched(), 0)); + bch2_trans_unlock(trans); + cond_resched(); now = local_clock(); } trans->last_begin_time = now; @@ -3033,6 +3035,8 @@ u32 bch2_trans_begin(struct btree_trans *trans) bch2_trans_srcu_unlock(trans); trans->last_begin_ip = _RET_IP_; + trans->locked = true; + if (trans->restarted) { bch2_btree_path_traverse_all(trans); trans->notrace_relock_fail = false; @@ -3090,7 +3094,7 @@ struct btree_trans *__bch2_trans_get(struct bch_fs *c, unsigned fn_idx) */ BUG_ON(pos_task && pid == pos_task->pid && - bch2_trans_locked(pos)); + pos->locked); if (pos_task && pid < pos_task->pid) { list_add_tail(&trans->list, &pos->list); @@ -3106,6 +3110,7 @@ struct btree_trans *__bch2_trans_get(struct bch_fs *c, unsigned fn_idx) trans->last_begin_time = local_clock(); trans->fn_idx = fn_idx; trans->locking_wait.task = current; + trans->locked = true; trans->journal_replay_not_finished = unlikely(!test_bit(JOURNAL_REPLAY_DONE, &c->journal.flags)) && atomic_inc_not_zero(&c->journal_keys.ref); diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index 0d44d613e4ac1..e478661053902 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -286,7 +286,6 @@ int bch2_trans_relock(struct btree_trans *); int bch2_trans_relock_notrace(struct btree_trans *); void bch2_trans_unlock(struct btree_trans *); void bch2_trans_unlock_long(struct btree_trans *); -bool bch2_trans_locked(struct btree_trans *); static inline int trans_was_restarted(struct btree_trans *trans, u32 restart_count) { diff --git a/fs/bcachefs/btree_locking.c b/fs/bcachefs/btree_locking.c index 9a3ab56f6cfcb..7824a7a2e50f7 100644 --- a/fs/bcachefs/btree_locking.c +++ b/fs/bcachefs/btree_locking.c @@ -490,8 +490,6 @@ static inline bool btree_path_get_locks(struct btree_trans *trans, if (path->uptodate == BTREE_ITER_NEED_RELOCK) path->uptodate = BTREE_ITER_UPTODATE; - bch2_trans_verify_locks(trans); - return path->uptodate < BTREE_ITER_NEED_RELOCK; } @@ -607,7 +605,9 @@ bool bch2_btree_path_relock_norestart(struct btree_trans *trans, struct btree_pa { struct get_locks_fail f; - return btree_path_get_locks(trans, path, false, &f); + bool ret = btree_path_get_locks(trans, path, false, &f); + bch2_trans_verify_locks(trans); + return ret; } int __bch2_btree_path_relock(struct btree_trans *trans, @@ -630,7 +630,9 @@ bool bch2_btree_path_upgrade_noupgrade_sibs(struct btree_trans *trans, path->locks_want = new_locks_want; - return btree_path_get_locks(trans, path, true, f); + bool ret = btree_path_get_locks(trans, path, true, f); + bch2_trans_verify_locks(trans); + return ret; } bool __bch2_btree_path_upgrade(struct btree_trans *trans, @@ -638,8 +640,9 @@ bool __bch2_btree_path_upgrade(struct btree_trans *trans, unsigned new_locks_want, struct get_locks_fail *f) { - if (bch2_btree_path_upgrade_noupgrade_sibs(trans, path, new_locks_want, f)) - return true; + bool ret = bch2_btree_path_upgrade_noupgrade_sibs(trans, path, new_locks_want, f); + if (ret) + goto out; /* * XXX: this is ugly - we'd prefer to not be mucking with other @@ -673,8 +676,9 @@ bool __bch2_btree_path_upgrade(struct btree_trans *trans, btree_path_get_locks(trans, linked, true, NULL); } } - - return false; +out: + bch2_trans_verify_locks(trans); + return ret; } void __bch2_btree_path_downgrade(struct btree_trans *trans, @@ -774,6 +778,8 @@ static inline int __bch2_trans_relock(struct btree_trans *trans, bool trace) if (unlikely(trans->restarted)) return -((int) trans->restarted); + if (unlikely(trans->locked)) + goto out; trans_for_each_path(trans, path, i) { struct get_locks_fail f; @@ -783,6 +789,8 @@ static inline int __bch2_trans_relock(struct btree_trans *trans, bool trace) return bch2_trans_relock_fail(trans, path, &f, trace); } + trans->locked = true; +out: bch2_trans_verify_locks(trans); return 0; } @@ -800,11 +808,17 @@ int bch2_trans_relock_notrace(struct btree_trans *trans) void bch2_trans_unlock_noassert(struct btree_trans *trans) { __bch2_trans_unlock(trans); + + trans->locked = false; + trans->last_unlock_ip = _RET_IP_; } void bch2_trans_unlock(struct btree_trans *trans) { __bch2_trans_unlock(trans); + + trans->locked = false; + trans->last_unlock_ip = _RET_IP_; } void bch2_trans_unlock_long(struct btree_trans *trans) @@ -813,17 +827,6 @@ void bch2_trans_unlock_long(struct btree_trans *trans) bch2_trans_srcu_unlock(trans); } -bool bch2_trans_locked(struct btree_trans *trans) -{ - struct btree_path *path; - unsigned i; - - trans_for_each_path(trans, path, i) - if (path->nodes_locked) - return true; - return false; -} - int __bch2_trans_mutex_lock(struct btree_trans *trans, struct mutex *lock) { @@ -865,6 +868,17 @@ void bch2_btree_path_verify_locks(struct btree_path *path) } } +static bool bch2_trans_locked(struct btree_trans *trans) +{ + struct btree_path *path; + unsigned i; + + trans_for_each_path(trans, path, i) + if (path->nodes_locked) + return true; + return false; +} + void bch2_trans_verify_locks(struct btree_trans *trans) { struct btree_path *path; diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index 123abeec4ce9c..a973ba6264d3b 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -469,6 +469,8 @@ struct btree_trans { u8 lock_must_abort; bool lock_may_not_fail:1; bool srcu_held:1; + bool locked:1; + bool write_locked:1; bool used_mempool:1; bool in_traverse_all:1; bool paths_sorted:1; @@ -476,13 +478,13 @@ struct btree_trans { bool journal_transaction_names:1; bool journal_replay_not_finished:1; bool notrace_relock_fail:1; - bool write_locked:1; enum bch_errcode restarted:16; u32 restart_count; u64 last_begin_time; unsigned long last_begin_ip; unsigned long last_restarted_ip; + unsigned long last_unlock_ip; unsigned long srcu_lock_time; const char *fn; From e590e4e2229409d0e5dc6575a75488d5ebc33b45 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 10 Apr 2024 21:51:37 -0400 Subject: [PATCH 1396/1702] bcachefs: bch2_btree_path_can_relock() With the new assertions, we shouldn't be holding locks when trans->locked is false, thus, we shouldn't use relock when we just want to check if we can relock. Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.c | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index 224001d8470d6..7718630625ed7 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -1346,6 +1346,26 @@ static inline void __bch2_path_free(struct btree_trans *trans, btree_path_idx_t __clear_bit(path, trans->paths_allocated); } +static bool bch2_btree_path_can_relock(struct btree_trans *trans, struct btree_path *path) +{ + unsigned l = path->level; + + do { + if (!btree_path_node(path, l)) + break; + + if (!is_btree_node(path, l)) + return false; + + if (path->l[l].lock_seq != path->l[l].b->c.lock.seq) + return false; + + l++; + } while (l < path->locks_want); + + return true; +} + void bch2_path_put(struct btree_trans *trans, btree_path_idx_t path_idx, bool intent) { struct btree_path *path = trans->paths + path_idx, *dup; @@ -1360,10 +1380,15 @@ void bch2_path_put(struct btree_trans *trans, btree_path_idx_t path_idx, bool in if (!dup && !(!path->preserve && !is_btree_node(path, path->level))) return; - if (path->should_be_locked && - !trans->restarted && - (!dup || !bch2_btree_path_relock_norestart(trans, dup))) - return; + if (path->should_be_locked && !trans->restarted) { + if (!dup) + return; + + if (!(trans->locked + ? bch2_btree_path_relock_norestart(trans, dup) + : bch2_btree_path_can_relock(trans, dup))) + return; + } if (dup) { dup->preserve |= path->preserve; From fd104e2967b766f1151b4c58daa67cbde620b376 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 9 Apr 2024 20:14:21 -0400 Subject: [PATCH 1397/1702] bcachefs: bch2_trans_verify_not_unlocked() Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.c | 16 ++++++++++++++++ fs/bcachefs/btree_iter.h | 12 ++++++++++++ fs/bcachefs/btree_locking.c | 11 +++++++++-- fs/bcachefs/btree_trans_commit.c | 7 +++++++ fs/bcachefs/btree_update_interior.c | 2 ++ 5 files changed, 46 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index 7718630625ed7..27f3935d67cb5 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -1421,6 +1421,12 @@ void __noreturn bch2_trans_in_restart_error(struct btree_trans *trans) (void *) trans->last_restarted_ip); } +void __noreturn bch2_trans_unlocked_error(struct btree_trans *trans) +{ + panic("trans should be locked, unlocked by %pS\n", + (void *) trans->last_unlock_ip); +} + noinline __cold void bch2_trans_updates_to_text(struct printbuf *buf, struct btree_trans *trans) { @@ -1690,6 +1696,7 @@ btree_path_idx_t bch2_path_get(struct btree_trans *trans, struct trans_for_each_path_inorder_iter iter; btree_path_idx_t path_pos = 0, path_idx; + bch2_trans_verify_not_unlocked(trans); bch2_trans_verify_not_in_restart(trans); bch2_trans_verify_locks(trans); @@ -1826,6 +1833,8 @@ bch2_btree_iter_traverse(struct btree_iter *iter) struct btree_trans *trans = iter->trans; int ret; + bch2_trans_verify_not_unlocked(trans); + iter->path = bch2_btree_path_set_pos(trans, iter->path, btree_iter_search_key(iter), iter->flags & BTREE_ITER_intent, @@ -2102,6 +2111,9 @@ struct bkey_s_c btree_trans_peek_key_cache(struct btree_iter *iter, struct bpos struct bkey_s_c k; int ret; + bch2_trans_verify_not_in_restart(trans); + bch2_trans_verify_not_unlocked(trans); + if ((iter->flags & BTREE_ITER_key_cache_fill) && bpos_eq(iter->pos, pos)) return bkey_s_c_null; @@ -2240,6 +2252,7 @@ struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *iter, struct bpos e struct bpos iter_pos; int ret; + bch2_trans_verify_not_unlocked(trans); EBUG_ON((iter->flags & BTREE_ITER_filter_snapshots) && bkey_eq(end, POS_MAX)); if (iter->update_path) { @@ -2412,6 +2425,7 @@ struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_iter *iter) btree_path_idx_t saved_path = 0; int ret; + bch2_trans_verify_not_unlocked(trans); EBUG_ON(btree_iter_path(trans, iter)->cached || btree_iter_path(trans, iter)->level); @@ -2548,6 +2562,7 @@ struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_iter *iter) struct bkey_s_c k; int ret; + bch2_trans_verify_not_unlocked(trans); bch2_btree_iter_verify(iter); bch2_btree_iter_verify_entry_exit(iter); EBUG_ON(btree_iter_path(trans, iter)->level && (iter->flags & BTREE_ITER_with_key_cache)); @@ -3067,6 +3082,7 @@ u32 bch2_trans_begin(struct btree_trans *trans) trans->notrace_relock_fail = false; } + bch2_trans_verify_not_unlocked(trans); return trans->restart_count; } diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index e478661053902..430b0d4848744 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -216,9 +216,13 @@ int __must_check bch2_btree_path_traverse_one(struct btree_trans *, btree_path_idx_t, unsigned, unsigned long); +static inline void bch2_trans_verify_not_unlocked(struct btree_trans *); + static inline int __must_check bch2_btree_path_traverse(struct btree_trans *trans, btree_path_idx_t path, unsigned flags) { + bch2_trans_verify_not_unlocked(trans); + if (trans->paths[path].uptodate < BTREE_ITER_NEED_RELOCK) return 0; @@ -311,6 +315,14 @@ static inline void bch2_trans_verify_not_in_restart(struct btree_trans *trans) bch2_trans_in_restart_error(trans); } +void __noreturn bch2_trans_unlocked_error(struct btree_trans *); + +static inline void bch2_trans_verify_not_unlocked(struct btree_trans *trans) +{ + if (!trans->locked) + bch2_trans_unlocked_error(trans); +} + __always_inline static int btree_trans_restart_nounlock(struct btree_trans *trans, int err) { diff --git a/fs/bcachefs/btree_locking.c b/fs/bcachefs/btree_locking.c index 7824a7a2e50f7..c3e9b0cc7bbdc 100644 --- a/fs/bcachefs/btree_locking.c +++ b/fs/bcachefs/btree_locking.c @@ -773,14 +773,16 @@ static noinline __cold int bch2_trans_relock_fail(struct btree_trans *trans, str static inline int __bch2_trans_relock(struct btree_trans *trans, bool trace) { - struct btree_path *path; - unsigned i; + bch2_trans_verify_locks(trans); if (unlikely(trans->restarted)) return -((int) trans->restarted); if (unlikely(trans->locked)) goto out; + struct btree_path *path; + unsigned i; + trans_for_each_path(trans, path, i) { struct get_locks_fail f; @@ -881,6 +883,11 @@ static bool bch2_trans_locked(struct btree_trans *trans) void bch2_trans_verify_locks(struct btree_trans *trans) { + if (!trans->locked) { + BUG_ON(bch2_trans_locked(trans)); + return; + } + struct btree_path *path; unsigned i; diff --git a/fs/bcachefs/btree_trans_commit.c b/fs/bcachefs/btree_trans_commit.c index 04831c9e603cc..0bf7c70da4175 100644 --- a/fs/bcachefs/btree_trans_commit.c +++ b/fs/bcachefs/btree_trans_commit.c @@ -630,6 +630,9 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags, unsigned u64s = 0; int ret; + bch2_trans_verify_not_unlocked(trans); + bch2_trans_verify_not_in_restart(trans); + if (race_fault()) { trace_and_count(c, trans_restart_fault_inject, trans, trace_ip); return btree_trans_restart_nounlock(trans, BCH_ERR_transaction_restart_fault_inject); @@ -1007,6 +1010,9 @@ int __bch2_trans_commit(struct btree_trans *trans, unsigned flags) struct bch_fs *c = trans->c; int ret = 0; + bch2_trans_verify_not_unlocked(trans); + bch2_trans_verify_not_in_restart(trans); + if (!trans->nr_updates && !trans->journal_entries_u64s) goto out_reset; @@ -1105,6 +1111,7 @@ int __bch2_trans_commit(struct btree_trans *trans, unsigned flags) } retry: errored_at = NULL; + bch2_trans_verify_not_unlocked(trans); bch2_trans_verify_not_in_restart(trans); if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res))) memset(&trans->journal_res, 0, sizeof(trans->journal_res)); diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index a87608dd74c6d..1febab152bfad 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -1949,6 +1949,8 @@ int __bch2_foreground_maybe_merge(struct btree_trans *trans, u64 start_time = local_clock(); int ret = 0; + bch2_trans_verify_not_in_restart(trans); + bch2_trans_verify_not_unlocked(trans); BUG_ON(!trans->paths[path].should_be_locked); BUG_ON(!btree_node_locked(&trans->paths[path], level)); From feb255537d1e67ec7e08104683658dc5106457fe Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 14 Apr 2024 19:43:12 -0400 Subject: [PATCH 1398/1702] bcachefs: assert that online_reserved == 0 on shutdown Signed-off-by: Kent Overstreet --- fs/bcachefs/super.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index 64fa76788ed1e..adad2a7036295 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -573,6 +573,7 @@ static void __bch2_fs_free(struct bch_fs *c) BUG_ON(atomic_read(&c->journal_keys.ref)); bch2_fs_btree_write_buffer_exit(c); percpu_free_rwsem(&c->mark_lock); + EBUG_ON(percpu_u64_get(c->online_reserved)); free_percpu(c->online_reserved); darray_exit(&c->btree_roots_extra); From 427e1bb8381899084fc316cce92dbc7508d9353b Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 16 Apr 2024 00:11:33 -0400 Subject: [PATCH 1399/1702] bcachefs: fs_alloc_debug_to_text() Signed-off-by: Kent Overstreet --- fs/bcachefs/sysfs.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index 607c930831bfb..24d4c8ef25a59 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -362,6 +362,37 @@ static void bch2_btree_wakeup_all(struct bch_fs *c) seqmutex_unlock(&c->btree_trans_lock); } +static void fs_alloc_debug_to_text(struct printbuf *out, struct bch_fs *c) +{ + unsigned nr[BCH_DATA_NR]; + + memset(nr, 0, sizeof(nr)); + + for (unsigned i = 0; i < ARRAY_SIZE(c->open_buckets); i++) + nr[c->open_buckets[i].data_type]++; + + printbuf_tabstop_push(out, 24); + + percpu_down_read(&c->mark_lock); + prt_printf(out, "hidden\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.hidden)); + prt_printf(out, "btree\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.btree)); + prt_printf(out, "data\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.data)); + prt_printf(out, "cached\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.cached)); + prt_printf(out, "reserved\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.reserved)); + prt_printf(out, "online_reserved\t%llu\n", percpu_u64_get(c->online_reserved)); + prt_printf(out, "nr_inodes\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.nr_inodes)); + percpu_up_read(&c->mark_lock); + + prt_newline(out); + prt_printf(out, "freelist_wait\t%s\n", c->freelist_wait.list.first ? "waiting" : "empty"); + prt_printf(out, "open buckets allocated\t%i\n", OPEN_BUCKETS_COUNT - c->open_buckets_nr_free); + prt_printf(out, "open buckets total\t%u\n", OPEN_BUCKETS_COUNT); + prt_printf(out, "open_buckets_wait\t%s\n", c->open_buckets_wait.list.first ? "waiting" : "empty"); + prt_printf(out, "open_buckets_btree\t%u\n", nr[BCH_DATA_btree]); + prt_printf(out, "open_buckets_user\t%u\n", nr[BCH_DATA_user]); + prt_printf(out, "btree reserve cache\t%u\n", c->btree_reserve_cache_nr); +} + SHOW(bch2_fs) { struct bch_fs *c = container_of(kobj, struct bch_fs, kobj); @@ -444,6 +475,9 @@ SHOW(bch2_fs) if (attr == &sysfs_disk_groups) bch2_disk_groups_to_text(out, c); + if (attr == &sysfs_alloc_debug) + fs_alloc_debug_to_text(out, c); + return 0; } @@ -650,6 +684,7 @@ struct attribute *bch2_fs_internal_files[] = { &sysfs_internal_uuid, &sysfs_disk_groups, + &sysfs_alloc_debug, NULL }; From 60f2b1bcf519416dbffee219132aa949d0c39d0e Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 16 Apr 2024 19:33:51 -0400 Subject: [PATCH 1400/1702] bcachefs: Add asserts to bch2_dev_btree_bitmap_marked_sectors() Signed-off-by: Kent Overstreet --- fs/bcachefs/sb-members.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/bcachefs/sb-members.c b/fs/bcachefs/sb-members.c index 80bf09cfc0212..50dc9f937c45d 100644 --- a/fs/bcachefs/sb-members.c +++ b/fs/bcachefs/sb-members.c @@ -415,6 +415,9 @@ static void __bch2_dev_btree_bitmap_mark(struct bch_sb_field_members_v2 *mi, uns m->btree_bitmap_shift += resize; } + BUG_ON(m->btree_bitmap_shift > 57); + BUG_ON(end > 64ULL << m->btree_bitmap_shift); + for (unsigned bit = start >> m->btree_bitmap_shift; (u64) bit << m->btree_bitmap_shift < end; bit++) From 9de40d77f0a070814cc1107f6794b219f10d8e0b Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 16 Apr 2024 19:33:56 -0400 Subject: [PATCH 1401/1702] bcachefs: Check for writing btree_ptr_v2.sectors_written == 0 Signed-off-by: Kent Overstreet --- fs/bcachefs/extents.c | 5 +++++ fs/bcachefs/sb-errors_types.h | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index cb8d52436ab15..1143288d79408 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -201,6 +201,11 @@ int bch2_btree_ptr_v2_invalid(struct bch_fs *c, struct bkey_s_c k, c, err, btree_ptr_v2_min_key_bad, "min_key > key"); + if (flags & BKEY_INVALID_WRITE) + bkey_fsck_err_on(!bp.v->sectors_written, + c, err, btree_ptr_v2_written_0, + "sectors_written == 0"); + ret = bch2_bkey_ptrs_invalid(c, k, flags, err); fsck_err: return ret; diff --git a/fs/bcachefs/sb-errors_types.h b/fs/bcachefs/sb-errors_types.h index 06c7a644f4a44..87324747351aa 100644 --- a/fs/bcachefs/sb-errors_types.h +++ b/fs/bcachefs/sb-errors_types.h @@ -272,7 +272,8 @@ x(snapshot_node_missing, 264) \ x(dup_backpointer_to_bad_csum_extent, 265) \ x(btree_bitmap_not_marked, 266) \ - x(sb_clean_entry_overrun, 267) + x(sb_clean_entry_overrun, 267) \ + x(btree_ptr_v2_written_0, 268) enum bch_sb_error_id { #define x(t, n) BCH_FSCK_ERR_##t = n, From 5a2d15213d3187ed3b059a2ec8865aa9172fd3a2 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 16 Apr 2024 23:08:39 -0400 Subject: [PATCH 1402/1702] bcachefs: Rip bch2_snapshot_equiv() out of fsck Originally, when deleting snapshots we didn't collapse redundant snapshot nodes; thus, the notion of a class of equivalent snapshot nodes leaked into fsck. Now we do, so snapshot ID equivalence classes are purely local to snapshot deletion. Signed-off-by: Kent Overstreet --- fs/bcachefs/fsck.c | 120 +++++++++++------------------------------ fs/bcachefs/snapshot.h | 5 -- 2 files changed, 30 insertions(+), 95 deletions(-) diff --git a/fs/bcachefs/fsck.c b/fs/bcachefs/fsck.c index d166c17109dd4..4adb96965e111 100644 --- a/fs/bcachefs/fsck.c +++ b/fs/bcachefs/fsck.c @@ -486,14 +486,9 @@ static int reconstruct_reg_inode(struct btree_trans *trans, u32 snapshot, u64 in return reconstruct_inode(trans, snapshot, inum, k.k->p.offset << 9, S_IFREG); } -struct snapshots_seen_entry { - u32 id; - u32 equiv; -}; - struct snapshots_seen { struct bpos pos; - DARRAY(struct snapshots_seen_entry) ids; + snapshot_id_list ids; }; static inline void snapshots_seen_exit(struct snapshots_seen *s) @@ -508,20 +503,15 @@ static inline void snapshots_seen_init(struct snapshots_seen *s) static int snapshots_seen_add_inorder(struct bch_fs *c, struct snapshots_seen *s, u32 id) { - struct snapshots_seen_entry *i, n = { - .id = id, - .equiv = bch2_snapshot_equiv(c, id), - }; - int ret = 0; - + u32 *i; __darray_for_each(s->ids, i) { - if (i->id == id) + if (*i == id) return 0; - if (i->id > id) + if (*i > id) break; } - ret = darray_insert_item(&s->ids, i - s->ids.data, n); + int ret = darray_insert_item(&s->ids, i - s->ids.data, id); if (ret) bch_err(c, "error reallocating snapshots_seen table (size %zu)", s->ids.size); @@ -531,42 +521,11 @@ static int snapshots_seen_add_inorder(struct bch_fs *c, struct snapshots_seen *s static int snapshots_seen_update(struct bch_fs *c, struct snapshots_seen *s, enum btree_id btree_id, struct bpos pos) { - struct snapshots_seen_entry n = { - .id = pos.snapshot, - .equiv = bch2_snapshot_equiv(c, pos.snapshot), - }; - int ret = 0; - if (!bkey_eq(s->pos, pos)) s->ids.nr = 0; - s->pos = pos; - s->pos.snapshot = n.equiv; - - darray_for_each(s->ids, i) { - if (i->id == n.id) - return 0; - - /* - * We currently don't rigorously track for snapshot cleanup - * needing to be run, so it shouldn't be a fsck error yet: - */ - if (i->equiv == n.equiv) { - bch_err(c, "snapshot deletion did not finish:\n" - " duplicate keys in btree %s at %llu:%llu snapshots %u, %u (equiv %u)\n", - bch2_btree_id_str(btree_id), - pos.inode, pos.offset, - i->id, n.id, n.equiv); - set_bit(BCH_FS_need_delete_dead_snapshots, &c->flags); - return bch2_run_explicit_recovery_pass(c, BCH_RECOVERY_PASS_delete_dead_snapshots); - } - } - ret = darray_push(&s->ids, n); - if (ret) - bch_err(c, "error reallocating snapshots_seen table (size %zu)", - s->ids.size); - return ret; + return snapshot_list_add_nodup(c, &s->ids, pos.snapshot); } /** @@ -586,12 +545,10 @@ static bool key_visible_in_snapshot(struct bch_fs *c, struct snapshots_seen *see ssize_t i; EBUG_ON(id > ancestor); - EBUG_ON(!bch2_snapshot_is_equiv(c, id)); - EBUG_ON(!bch2_snapshot_is_equiv(c, ancestor)); /* @ancestor should be the snapshot most recently added to @seen */ EBUG_ON(ancestor != seen->pos.snapshot); - EBUG_ON(ancestor != seen->ids.data[seen->ids.nr - 1].equiv); + EBUG_ON(ancestor != darray_last(seen->ids)); if (id == ancestor) return true; @@ -610,9 +567,9 @@ static bool key_visible_in_snapshot(struct bch_fs *c, struct snapshots_seen *see */ for (i = seen->ids.nr - 2; - i >= 0 && seen->ids.data[i].equiv >= id; + i >= 0 && seen->ids.data[i] >= id; --i) - if (bch2_snapshot_is_ancestor(c, id, seen->ids.data[i].equiv)) + if (bch2_snapshot_is_ancestor(c, id, seen->ids.data[i])) return false; return true; @@ -643,9 +600,6 @@ static int ref_visible2(struct bch_fs *c, u32 src, struct snapshots_seen *src_seen, u32 dst, struct snapshots_seen *dst_seen) { - src = bch2_snapshot_equiv(c, src); - dst = bch2_snapshot_equiv(c, dst); - if (dst > src) { swap(dst, src); swap(dst_seen, src_seen); @@ -692,7 +646,7 @@ static int add_inode(struct bch_fs *c, struct inode_walker *w, return darray_push(&w->inodes, ((struct inode_walker_entry) { .inode = u, - .snapshot = bch2_snapshot_equiv(c, inode.k->p.snapshot), + .snapshot = inode.k->p.snapshot, })); } @@ -728,21 +682,20 @@ static struct inode_walker_entry * lookup_inode_for_snapshot(struct bch_fs *c, struct inode_walker *w, struct bkey_s_c k) { bool is_whiteout = k.k->type == KEY_TYPE_whiteout; - u32 snapshot = bch2_snapshot_equiv(c, k.k->p.snapshot); struct inode_walker_entry *i; __darray_for_each(w->inodes, i) - if (bch2_snapshot_is_ancestor(c, snapshot, i->snapshot)) + if (bch2_snapshot_is_ancestor(c, k.k->p.snapshot, i->snapshot)) goto found; return NULL; found: - BUG_ON(snapshot > i->snapshot); + BUG_ON(k.k->p.snapshot > i->snapshot); - if (snapshot != i->snapshot && !is_whiteout) { + if (k.k->p.snapshot != i->snapshot && !is_whiteout) { struct inode_walker_entry new = *i; - new.snapshot = snapshot; + new.snapshot = k.k->p.snapshot; new.count = 0; struct printbuf buf = PRINTBUF; @@ -751,10 +704,10 @@ lookup_inode_for_snapshot(struct bch_fs *c, struct inode_walker *w, struct bkey_ bch_info(c, "have key for inode %llu:%u but have inode in ancestor snapshot %u\n" "unexpected because we should always update the inode when we update a key in that inode\n" "%s", - w->last_pos.inode, snapshot, i->snapshot, buf.buf); + w->last_pos.inode, k.k->p.snapshot, i->snapshot, buf.buf); printbuf_exit(&buf); - while (i > w->inodes.data && i[-1].snapshot > snapshot) + while (i > w->inodes.data && i[-1].snapshot > k.k->p.snapshot) --i; size_t pos = i - w->inodes.data; @@ -786,10 +739,10 @@ static struct inode_walker_entry *walk_inode(struct btree_trans *trans, return lookup_inode_for_snapshot(trans->c, w, k); } -static int __get_visible_inodes(struct btree_trans *trans, - struct inode_walker *w, - struct snapshots_seen *s, - u64 inum) +static int get_visible_inodes(struct btree_trans *trans, + struct inode_walker *w, + struct snapshots_seen *s, + u64 inum) { struct bch_fs *c = trans->c; struct btree_iter iter; @@ -800,18 +753,16 @@ static int __get_visible_inodes(struct btree_trans *trans, for_each_btree_key_norestart(trans, iter, BTREE_ID_inodes, POS(0, inum), BTREE_ITER_all_snapshots, k, ret) { - u32 equiv = bch2_snapshot_equiv(c, k.k->p.snapshot); - if (k.k->p.offset != inum) break; - if (!ref_visible(c, s, s->pos.snapshot, equiv)) + if (!ref_visible(c, s, s->pos.snapshot, k.k->p.snapshot)) continue; if (bkey_is_inode(k.k)) add_inode(c, w, k); - if (equiv >= s->pos.snapshot) + if (k.k->p.snapshot >= s->pos.snapshot) break; } bch2_trans_iter_exit(trans, &iter); @@ -1466,7 +1417,6 @@ static int check_overlapping_extents(struct btree_trans *trans, struct snapshots_seen *seen, struct extent_ends *extent_ends, struct bkey_s_c k, - u32 equiv, struct btree_iter *iter, bool *fixed) { @@ -1535,11 +1485,8 @@ static int check_extent(struct btree_trans *trans, struct btree_iter *iter, struct bch_fs *c = trans->c; struct inode_walker_entry *i; struct printbuf buf = PRINTBUF; - struct bpos equiv = k.k->p; int ret = 0; - equiv.snapshot = bch2_snapshot_equiv(c, k.k->p.snapshot); - ret = check_key_has_snapshot(trans, iter, k); if (ret) { ret = ret < 0 ? ret : 0; @@ -1589,8 +1536,7 @@ static int check_extent(struct btree_trans *trans, struct btree_iter *iter, bch2_bkey_val_to_text(&buf, c, k), buf.buf))) goto delete; - ret = check_overlapping_extents(trans, s, extent_ends, k, - equiv.snapshot, iter, + ret = check_overlapping_extents(trans, s, extent_ends, k, iter, &inode->recalculate_sums); if (ret) goto err; @@ -1607,8 +1553,8 @@ static int check_extent(struct btree_trans *trans, struct btree_iter *iter, for (; inode->inodes.data && i >= inode->inodes.data; --i) { - if (i->snapshot > equiv.snapshot || - !key_visible_in_snapshot(c, s, i->snapshot, equiv.snapshot)) + if (i->snapshot > k.k->p.snapshot || + !key_visible_in_snapshot(c, s, i->snapshot, k.k->p.snapshot)) continue; if (k.k->type != KEY_TYPE_whiteout) { @@ -2052,7 +1998,6 @@ static int check_dirent(struct btree_trans *trans, struct btree_iter *iter, struct bch_fs *c = trans->c; struct inode_walker_entry *i; struct printbuf buf = PRINTBUF; - struct bpos equiv; int ret = 0; ret = check_key_has_snapshot(trans, iter, k); @@ -2061,9 +2006,6 @@ static int check_dirent(struct btree_trans *trans, struct btree_iter *iter, goto out; } - equiv = k.k->p; - equiv.snapshot = bch2_snapshot_equiv(c, k.k->p.snapshot); - ret = snapshots_seen_update(c, s, iter->btree_id, k.k->p); if (ret) goto err; @@ -2140,14 +2082,13 @@ static int check_dirent(struct btree_trans *trans, struct btree_iter *iter, if (ret) goto err; } else { - ret = __get_visible_inodes(trans, target, s, le64_to_cpu(d.v->d_inum)); + ret = get_visible_inodes(trans, target, s, le64_to_cpu(d.v->d_inum)); if (ret) goto err; if (fsck_err_on(!target->inodes.nr, c, dirent_to_missing_inode, - "dirent points to missing inode: (equiv %u)\n%s", - equiv.snapshot, + "dirent points to missing inode:\n%s", (printbuf_reset(&buf), bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { @@ -2164,7 +2105,7 @@ static int check_dirent(struct btree_trans *trans, struct btree_iter *iter, } if (d.v->d_type == DT_DIR) - for_each_visible_inode(c, s, dir, equiv.snapshot, i) + for_each_visible_inode(c, s, dir, d.k->p.snapshot, i) i->count++; } out: @@ -2457,7 +2398,7 @@ static int check_path(struct btree_trans *trans, pathbuf *p, struct bkey_s_c ino struct btree_iter inode_iter = {}; struct bch_inode_unpacked inode; struct printbuf buf = PRINTBUF; - u32 snapshot = bch2_snapshot_equiv(c, inode_k.k->p.snapshot); + u32 snapshot = inode_k.k->p.snapshot; int ret = 0; p->nr = 0; @@ -2717,8 +2658,7 @@ static int check_nlinks_walk_dirents(struct bch_fs *c, struct nlink_table *links if (d.v->d_type != DT_DIR && d.v->d_type != DT_SUBVOL) inc_link(c, &s, links, range_start, range_end, - le64_to_cpu(d.v->d_inum), - bch2_snapshot_equiv(c, d.k->p.snapshot)); + le64_to_cpu(d.v->d_inum), d.k->p.snapshot); } 0; }))); diff --git a/fs/bcachefs/snapshot.h b/fs/bcachefs/snapshot.h index 3fdb41b33d2d2..ccb1f309b8d16 100644 --- a/fs/bcachefs/snapshot.h +++ b/fs/bcachefs/snapshot.h @@ -135,11 +135,6 @@ static inline u32 bch2_snapshot_equiv(struct bch_fs *c, u32 id) return id; } -static inline bool bch2_snapshot_is_equiv(struct bch_fs *c, u32 id) -{ - return id == bch2_snapshot_equiv(c, id); -} - static inline int bch2_snapshot_is_internal_node(struct bch_fs *c, u32 id) { rcu_read_lock(); From f7643bc9749f270d487c32dc35b578575bf1adb0 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 17 Apr 2024 01:26:02 -0400 Subject: [PATCH 1403/1702] bcachefs: make btree read errors silent during scan Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_io.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c index 37f00d844edd7..c111976fd7182 100644 --- a/fs/bcachefs/btree_io.c +++ b/fs/bcachefs/btree_io.c @@ -555,6 +555,7 @@ static int __btree_err(int ret, const char *fmt, ...) { struct printbuf out = PRINTBUF; + bool silent = c->curr_recovery_pass == BCH_RECOVERY_PASS_scan_for_btree_nodes; va_list args; btree_err_msg(&out, c, ca, b, i, b->written, write); @@ -576,12 +577,14 @@ static int __btree_err(int ret, if (!have_retry && ret == -BCH_ERR_btree_node_read_err_must_retry) ret = -BCH_ERR_btree_node_read_err_bad_node; - if (ret != -BCH_ERR_btree_node_read_err_fixable) + if (!silent && ret != -BCH_ERR_btree_node_read_err_fixable) bch2_sb_error_count(c, err_type); switch (ret) { case -BCH_ERR_btree_node_read_err_fixable: - ret = bch2_fsck_err(c, FSCK_CAN_FIX, err_type, "%s", out.buf); + ret = !silent + ? bch2_fsck_err(c, FSCK_CAN_FIX, err_type, "%s", out.buf) + : -BCH_ERR_fsck_fix; if (ret != -BCH_ERR_fsck_fix && ret != -BCH_ERR_fsck_ignore) goto fsck_err; @@ -589,14 +592,17 @@ static int __btree_err(int ret, break; case -BCH_ERR_btree_node_read_err_want_retry: case -BCH_ERR_btree_node_read_err_must_retry: - bch2_print_string_as_lines(KERN_ERR, out.buf); + if (!silent) + bch2_print_string_as_lines(KERN_ERR, out.buf); break; case -BCH_ERR_btree_node_read_err_bad_node: - bch2_print_string_as_lines(KERN_ERR, out.buf); + if (!silent) + bch2_print_string_as_lines(KERN_ERR, out.buf); ret = bch2_topology_error(c); break; case -BCH_ERR_btree_node_read_err_incompatible: - bch2_print_string_as_lines(KERN_ERR, out.buf); + if (!silent) + bch2_print_string_as_lines(KERN_ERR, out.buf); ret = -BCH_ERR_fsck_errors_not_fixed; break; default: From aef7eecb5711f8bcbaf0709793ff950c290e3493 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 17 Apr 2024 02:03:22 -0400 Subject: [PATCH 1404/1702] bcachefs: Sync journal when we complete a recovery pass Make things easier when we're debugging long fsck runs - persist the work that successful recovery passes did. Signed-off-by: Kent Overstreet --- fs/bcachefs/recovery_passes.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/bcachefs/recovery_passes.c b/fs/bcachefs/recovery_passes.c index c065c0d012125..4a9eb9582b6e5 100644 --- a/fs/bcachefs/recovery_passes.c +++ b/fs/bcachefs/recovery_passes.c @@ -222,7 +222,8 @@ int bch2_run_recovery_passes(struct bch_fs *c) if (should_run_recovery_pass(c, c->curr_recovery_pass)) { unsigned pass = c->curr_recovery_pass; - ret = bch2_run_recovery_pass(c, c->curr_recovery_pass); + ret = bch2_run_recovery_pass(c, c->curr_recovery_pass) ?: + bch2_journal_flush(&c->journal); if (bch2_err_matches(ret, BCH_ERR_restart_recovery) || (ret && c->curr_recovery_pass < pass)) continue; From b25fd02ab4edb0193e60754626704adacbfbeefb Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 19 Apr 2024 20:23:32 -0400 Subject: [PATCH 1405/1702] bcachefs: fix flag printing in journal_buf_to_text() Signed-off-by: Kent Overstreet --- fs/bcachefs/journal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/journal.c b/fs/bcachefs/journal.c index 1b62034e197d6..17ceddbd88582 100644 --- a/fs/bcachefs/journal.c +++ b/fs/bcachefs/journal.c @@ -77,9 +77,9 @@ static void bch2_journal_buf_to_text(struct printbuf *out, struct journal *j, u6 if (buf->write_started) prt_str(out, "write_started "); if (buf->write_allocated) - prt_str(out, "write allocated "); + prt_str(out, "write_allocated "); if (buf->write_done) - prt_str(out, "write done"); + prt_str(out, "write_done"); prt_newline(out); printbuf_indent_sub(out, 2); From 103304021e54bfb5cab9ba04cd5ef0dc2bf33888 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 19 Apr 2024 22:44:12 -0400 Subject: [PATCH 1406/1702] bcachefs: Move gc of bucket.oldest_gen to workqueue This is a nice cleanup - and we've also been having problems with kthread creation in the mount path. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 2 +- fs/bcachefs/alloc_foreground.c | 2 +- fs/bcachefs/bcachefs.h | 5 +-- fs/bcachefs/btree_gc.c | 82 ++++++---------------------------- fs/bcachefs/btree_gc.h | 12 ++--- fs/bcachefs/super.c | 10 +---- fs/bcachefs/sysfs.c | 11 ----- 7 files changed, 22 insertions(+), 102 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index f07373b781745..6fa51ee16cc33 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -874,7 +874,7 @@ int bch2_trigger_alloc(struct btree_trans *trans, bch2_do_invalidates(c); if (statechange(a->data_type == BCH_DATA_need_gc_gens)) - bch2_do_gc_gens(c); + bch2_gc_gens_async(c); } if ((flags & BTREE_TRIGGER_gc) && diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index fb8825c4e7ad8..6cb878f5e5ebb 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -541,7 +541,7 @@ static struct open_bucket *bch2_bucket_alloc_trans(struct btree_trans *trans, bch2_do_discards(c); if (usage->d[BCH_DATA_need_gc_gens].buckets > avail) - bch2_do_gc_gens(c); + bch2_gc_gens_async(c); if (should_invalidate_buckets(ca, *usage)) bch2_do_invalidates(c); diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index 7562446f2d2af..6e324c1fe9245 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -718,6 +718,7 @@ struct btree_trans_buf { x(discard_fast) \ x(invalidate) \ x(delete_dead_snapshots) \ + x(gc_gens) \ x(snapshot_delete_pagecache) \ x(sysfs) \ x(btree_write_buffer) @@ -960,8 +961,7 @@ struct bch_fs { struct work_struct discard_fast_work; /* GARBAGE COLLECTION */ - struct task_struct *gc_thread; - atomic_t kick_gc; + struct work_struct gc_gens_work; unsigned long gc_count; enum btree_id gc_gens_btree; @@ -1118,7 +1118,6 @@ struct bch_fs { u64 counters_on_mount[BCH_COUNTER_NR]; u64 __percpu *counters; - unsigned btree_gc_periodic:1; unsigned copy_gc_enabled:1; bool promote_whole_extents; diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 7549e806a6f48..919fa1b9fc2e2 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -1669,6 +1669,9 @@ static int gc_btree_gens_key(struct btree_trans *trans, struct bkey_i *u; int ret; + if (unlikely(test_bit(BCH_FS_going_ro, &c->flags))) + return -EROFS; + percpu_down_read(&c->mark_lock); bkey_for_each_ptr(ptrs, ptr) { struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); @@ -1802,80 +1805,23 @@ int bch2_gc_gens(struct bch_fs *c) return ret; } -static int bch2_gc_thread(void *arg) +static void bch2_gc_gens_work(struct work_struct *work) { - struct bch_fs *c = arg; - struct io_clock *clock = &c->io_clock[WRITE]; - unsigned long last = atomic64_read(&clock->now); - unsigned last_kick = atomic_read(&c->kick_gc); - - set_freezable(); - - while (1) { - while (1) { - set_current_state(TASK_INTERRUPTIBLE); - - if (kthread_should_stop()) { - __set_current_state(TASK_RUNNING); - return 0; - } - - if (atomic_read(&c->kick_gc) != last_kick) - break; - - if (c->btree_gc_periodic) { - unsigned long next = last + c->capacity / 16; - - if (atomic64_read(&clock->now) >= next) - break; - - bch2_io_clock_schedule_timeout(clock, next); - } else { - schedule(); - } - - try_to_freeze(); - } - __set_current_state(TASK_RUNNING); - - last = atomic64_read(&clock->now); - last_kick = atomic_read(&c->kick_gc); - - bch2_gc_gens(c); - debug_check_no_locks_held(); - } - - return 0; + struct bch_fs *c = container_of(work, struct bch_fs, gc_gens_work); + bch2_gc_gens(c); + bch2_write_ref_put(c, BCH_WRITE_REF_gc_gens); } -void bch2_gc_thread_stop(struct bch_fs *c) +void bch2_gc_gens_async(struct bch_fs *c) { - struct task_struct *p; - - p = c->gc_thread; - c->gc_thread = NULL; - - if (p) { - kthread_stop(p); - put_task_struct(p); - } + if (bch2_write_ref_tryget(c, BCH_WRITE_REF_gc_gens) && + !queue_work(c->write_ref_wq, &c->gc_gens_work)) + bch2_write_ref_put(c, BCH_WRITE_REF_gc_gens); } -int bch2_gc_thread_start(struct bch_fs *c) +void bch2_fs_gc_init(struct bch_fs *c) { - struct task_struct *p; - - if (c->gc_thread) - return 0; + seqcount_init(&c->gc_pos_lock); - p = kthread_create(bch2_gc_thread, c, "bch-gc/%s", c->name); - if (IS_ERR(p)) { - bch_err_fn(c, PTR_ERR(p)); - return PTR_ERR(p); - } - - get_task_struct(p); - c->gc_thread = p; - wake_up_process(p); - return 0; + INIT_WORK(&c->gc_gens_work, bch2_gc_gens_work); } diff --git a/fs/bcachefs/btree_gc.h b/fs/bcachefs/btree_gc.h index 0d6c0a2df613e..15315aab93bd8 100644 --- a/fs/bcachefs/btree_gc.h +++ b/fs/bcachefs/btree_gc.h @@ -7,9 +7,6 @@ int bch2_check_topology(struct bch_fs *); int bch2_check_allocations(struct bch_fs *); -int bch2_gc_gens(struct bch_fs *); -void bch2_gc_thread_stop(struct bch_fs *); -int bch2_gc_thread_start(struct bch_fs *); /* * For concurrent mark and sweep (with other index updates), we define a total @@ -104,11 +101,8 @@ static inline bool gc_visited(struct bch_fs *c, struct gc_pos pos) return ret; } -static inline void bch2_do_gc_gens(struct bch_fs *c) -{ - atomic_inc(&c->kick_gc); - if (c->gc_thread) - wake_up_process(c->gc_thread); -} +int bch2_gc_gens(struct bch_fs *); +void bch2_gc_gens_async(struct bch_fs *); +void bch2_fs_gc_init(struct bch_fs *); #endif /* _BCACHEFS_BTREE_GC_H */ diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index adad2a7036295..36833f86d50da 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -264,7 +264,6 @@ static void __bch2_fs_read_only(struct bch_fs *c) bch2_open_buckets_stop(c, NULL, true); bch2_rebalance_stop(c); bch2_copygc_stop(c); - bch2_gc_thread_stop(c); bch2_fs_ec_flush(c); bch_verbose(c, "flushing journal and stopping allocators, journal seq %llu", @@ -486,12 +485,6 @@ static int __bch2_fs_read_write(struct bch_fs *c, bool early) } #endif - ret = bch2_gc_thread_start(c); - if (ret) { - bch_err(c, "error starting gc thread"); - return ret; - } - ret = bch2_journal_reclaim_start(&c->journal); if (ret) goto err; @@ -780,6 +773,7 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) for (i = 0; i < BCH_TIME_STAT_NR; i++) bch2_time_stats_init(&c->times[i]); + bch2_fs_gc_init(c); bch2_fs_copygc_init(c); bch2_fs_btree_key_cache_init_early(&c->btree_key_cache); bch2_fs_btree_iter_init_early(c); @@ -810,8 +804,6 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) INIT_LIST_HEAD(&c->fsck_error_msgs); mutex_init(&c->fsck_error_msgs_lock); - seqcount_init(&c->gc_pos_lock); - seqcount_init(&c->usage_lock); sema_init(&c->io_in_flight, 128); diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index 24d4c8ef25a59..43edda74d3cbc 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -142,7 +142,6 @@ write_attribute(trigger_invalidates); write_attribute(trigger_journal_flush); write_attribute(prune_cache); write_attribute(btree_wakeup); -rw_attribute(btree_gc_periodic); rw_attribute(gc_gens_pos); read_attribute(uuid); @@ -408,8 +407,6 @@ SHOW(bch2_fs) if (attr == &sysfs_btree_write_stats) bch2_btree_write_stats_to_text(out, c); - sysfs_printf(btree_gc_periodic, "%u", (int) c->btree_gc_periodic); - if (attr == &sysfs_gc_gens_pos) bch2_gc_gens_pos_to_text(out, c); @@ -485,14 +482,6 @@ STORE(bch2_fs) { struct bch_fs *c = container_of(kobj, struct bch_fs, kobj); - if (attr == &sysfs_btree_gc_periodic) { - ssize_t ret = strtoul_safe(buf, c->btree_gc_periodic) - ?: (ssize_t) size; - - wake_up_process(c->gc_thread); - return ret; - } - if (attr == &sysfs_copy_gc_enabled) { ssize_t ret = strtoul_safe(buf, c->copy_gc_enabled) ?: (ssize_t) size; From e4f2c4dfeeaeb70164967aa4abfa1190f7e61781 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sat, 20 Apr 2024 03:50:01 +0100 Subject: [PATCH 1407/1702] bcachefs: Remove calls to folio_set_error Common code doesn't test the error flag, so we don't need to set it in bcachefs. We can use folio_end_read() to combine the setting (or not) of the uptodate flag and clearing the lock flag. Cc: Kent Overstreet Cc: Brian Foster Cc: linux-bcachefs@vger.kernel.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Kent Overstreet --- fs/bcachefs/fs-io-buffered.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/fs/bcachefs/fs-io-buffered.c b/fs/bcachefs/fs-io-buffered.c index 120430abdae74..b0a33fabadf81 100644 --- a/fs/bcachefs/fs-io-buffered.c +++ b/fs/bcachefs/fs-io-buffered.c @@ -30,15 +30,8 @@ static void bch2_readpages_end_io(struct bio *bio) { struct folio_iter fi; - bio_for_each_folio_all(fi, bio) { - if (!bio->bi_status) { - folio_mark_uptodate(fi.folio); - } else { - folio_clear_uptodate(fi.folio); - folio_set_error(fi.folio); - } - folio_unlock(fi.folio); - } + bio_for_each_folio_all(fi, bio) + folio_end_read(fi.folio, bio->bi_status == BLK_STS_OK); bio_put(bio); } @@ -408,7 +401,6 @@ static void bch2_writepage_io_done(struct bch_write_op *op) bio_for_each_folio_all(fi, bio) { struct bch_folio *s; - folio_set_error(fi.folio); mapping_set_error(fi.folio->mapping, -EIO); s = __bch2_folio(fi.folio); From 5147b9ae768758982b196d1b259e6372e328955e Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 20 Apr 2024 14:49:22 -0400 Subject: [PATCH 1408/1702] bcachefs: Btree key cache instrumentation It turns out the btree key cache shrinker wasn't actually reclaiming anything, prior to the previous patch. This adds instrumentation so that if we have further issues we can see what's going on. Specifically, sysfs internal/btree_key_cache is greatly expanded with new counters, and the SRCU sequence numbers of the first 10 entries on each pending freelist, and we also add trigger_btree_key_cache_shrink for testing without having to prune all the system caches. Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_key_cache.c | 53 ++++++++++++++++++++++++++--- fs/bcachefs/btree_key_cache_types.h | 8 +++++ fs/bcachefs/sysfs.c | 34 +++++++----------- 3 files changed, 69 insertions(+), 26 deletions(-) diff --git a/fs/bcachefs/btree_key_cache.c b/fs/bcachefs/btree_key_cache.c index 6645264a481bd..203fbb38e9d47 100644 --- a/fs/bcachefs/btree_key_cache.c +++ b/fs/bcachefs/btree_key_cache.c @@ -822,6 +822,8 @@ static unsigned long bch2_btree_key_cache_scan(struct shrinker *shrink, int srcu_idx; mutex_lock(&bc->lock); + bc->requested_to_free += sc->nr_to_scan; + srcu_idx = srcu_read_lock(&c->btree_trans_barrier); flags = memalloc_nofs_save(); @@ -840,6 +842,7 @@ static unsigned long bch2_btree_key_cache_scan(struct shrinker *shrink, atomic_long_dec(&bc->nr_freed); freed++; bc->nr_freed_nonpcpu--; + bc->freed++; } list_for_each_entry_safe(ck, t, &bc->freed_pcpu, list) { @@ -853,6 +856,7 @@ static unsigned long bch2_btree_key_cache_scan(struct shrinker *shrink, atomic_long_dec(&bc->nr_freed); freed++; bc->nr_freed_pcpu--; + bc->freed++; } rcu_read_lock(); @@ -871,13 +875,18 @@ static unsigned long bch2_btree_key_cache_scan(struct shrinker *shrink, ck = container_of(pos, struct bkey_cached, hash); if (test_bit(BKEY_CACHED_DIRTY, &ck->flags)) { + bc->skipped_dirty++; goto next; } else if (test_bit(BKEY_CACHED_ACCESSED, &ck->flags)) { clear_bit(BKEY_CACHED_ACCESSED, &ck->flags); + bc->skipped_accessed++; goto next; } else if (bkey_cached_lock_for_evict(ck)) { bkey_cached_evict(bc, ck); bkey_cached_free(bc, ck); + bc->moved_to_freelist++; + } else { + bc->skipped_lock_fail++; } scanned++; @@ -1024,11 +1033,47 @@ int bch2_fs_btree_key_cache_init(struct btree_key_cache *bc) return 0; } -void bch2_btree_key_cache_to_text(struct printbuf *out, struct btree_key_cache *c) +void bch2_btree_key_cache_to_text(struct printbuf *out, struct btree_key_cache *bc) { - prt_printf(out, "nr_freed:\t%lu\n", atomic_long_read(&c->nr_freed)); - prt_printf(out, "nr_keys:\t%lu\n", atomic_long_read(&c->nr_keys)); - prt_printf(out, "nr_dirty:\t%lu\n", atomic_long_read(&c->nr_dirty)); + struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache); + + printbuf_tabstop_push(out, 24); + printbuf_tabstop_push(out, 12); + + unsigned flags = memalloc_nofs_save(); + mutex_lock(&bc->lock); + prt_printf(out, "keys:\t%lu\r\n", atomic_long_read(&bc->nr_keys)); + prt_printf(out, "dirty:\t%lu\r\n", atomic_long_read(&bc->nr_dirty)); + prt_printf(out, "freelist:\t%lu\r\n", atomic_long_read(&bc->nr_freed)); + prt_printf(out, "nonpcpu freelist:\t%lu\r\n", bc->nr_freed_nonpcpu); + prt_printf(out, "pcpu freelist:\t%lu\r\n", bc->nr_freed_pcpu); + + prt_printf(out, "\nshrinker:\n"); + prt_printf(out, "requested_to_free:\t%lu\r\n", bc->requested_to_free); + prt_printf(out, "freed:\t%lu\r\n", bc->freed); + prt_printf(out, "moved_to_freelist:\t%lu\r\n", bc->moved_to_freelist); + prt_printf(out, "skipped_dirty:\t%lu\r\n", bc->skipped_dirty); + prt_printf(out, "skipped_accessed:\t%lu\r\n", bc->skipped_accessed); + prt_printf(out, "skipped_lock_fail:\t%lu\r\n", bc->skipped_lock_fail); + + prt_printf(out, "srcu seq:\t%lu\r\n", get_state_synchronize_srcu(&c->btree_trans_barrier)); + + struct bkey_cached *ck; + unsigned iter = 0; + list_for_each_entry(ck, &bc->freed_nonpcpu, list) { + prt_printf(out, "freed_nonpcpu:\t%lu\r\n", ck->btree_trans_barrier_seq); + if (++iter > 10) + break; + } + + iter = 0; + list_for_each_entry(ck, &bc->freed_pcpu, list) { + prt_printf(out, "freed_pcpu:\t%lu\r\n", ck->btree_trans_barrier_seq); + if (++iter > 10) + break; + } + mutex_unlock(&bc->lock); + memalloc_flags_restore(flags); } void bch2_btree_key_cache_exit(void) diff --git a/fs/bcachefs/btree_key_cache_types.h b/fs/bcachefs/btree_key_cache_types.h index 290e4e57df5bb..237e8bb3ac407 100644 --- a/fs/bcachefs/btree_key_cache_types.h +++ b/fs/bcachefs/btree_key_cache_types.h @@ -24,6 +24,14 @@ struct btree_key_cache { atomic_long_t nr_freed; atomic_long_t nr_keys; atomic_long_t nr_dirty; + + /* shrinker stats */ + unsigned long requested_to_free; + unsigned long freed; + unsigned long moved_to_freelist; + unsigned long skipped_dirty; + unsigned long skipped_accessed; + unsigned long skipped_lock_fail; }; struct bkey_cached_key { diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index 43edda74d3cbc..4627b0ba179e7 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -140,8 +140,8 @@ write_attribute(trigger_gc); write_attribute(trigger_discards); write_attribute(trigger_invalidates); write_attribute(trigger_journal_flush); -write_attribute(prune_cache); -write_attribute(btree_wakeup); +write_attribute(trigger_btree_cache_shrink); +write_attribute(trigger_btree_key_cache_shrink); rw_attribute(gc_gens_pos); read_attribute(uuid); @@ -346,21 +346,6 @@ static void bch2_gc_gens_pos_to_text(struct printbuf *out, struct bch_fs *c) prt_printf(out, "\n"); } -static void bch2_btree_wakeup_all(struct bch_fs *c) -{ - struct btree_trans *trans; - - seqmutex_lock(&c->btree_trans_lock); - list_for_each_entry(trans, &c->btree_trans_list, list) { - struct btree_bkey_cached_common *b = READ_ONCE(trans->locking); - - if (b) - six_lock_wakeup_all(&b->lock); - - } - seqmutex_unlock(&c->btree_trans_lock); -} - static void fs_alloc_debug_to_text(struct printbuf *out, struct bch_fs *c) { unsigned nr[BCH_DATA_NR]; @@ -513,7 +498,7 @@ STORE(bch2_fs) if (!bch2_write_ref_tryget(c, BCH_WRITE_REF_sysfs)) return -EROFS; - if (attr == &sysfs_prune_cache) { + if (attr == &sysfs_trigger_btree_cache_shrink) { struct shrink_control sc; sc.gfp_mask = GFP_KERNEL; @@ -521,8 +506,13 @@ STORE(bch2_fs) c->btree_cache.shrink->scan_objects(c->btree_cache.shrink, &sc); } - if (attr == &sysfs_btree_wakeup) - bch2_btree_wakeup_all(c); + if (attr == &sysfs_trigger_btree_key_cache_shrink) { + struct shrink_control sc; + + sc.gfp_mask = GFP_KERNEL; + sc.nr_to_scan = strtoul_or_return(buf); + c->btree_key_cache.shrink->scan_objects(c->btree_cache.shrink, &sc); + } if (attr == &sysfs_trigger_gc) bch2_gc_gens(c); @@ -656,8 +646,8 @@ struct attribute *bch2_fs_internal_files[] = { &sysfs_trigger_discards, &sysfs_trigger_invalidates, &sysfs_trigger_journal_flush, - &sysfs_prune_cache, - &sysfs_btree_wakeup, + &sysfs_trigger_btree_cache_shrink, + &sysfs_trigger_btree_key_cache_shrink, &sysfs_gc_gens_pos, From 018b32a63fabbc4c3a69c350f7ef922f1a3ac5f4 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 20 Apr 2024 16:50:46 -0400 Subject: [PATCH 1409/1702] bcachefs: Add btree_allocated_bitmap to member_to_text() Signed-off-by: Kent Overstreet --- fs/bcachefs/sb-members.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fs/bcachefs/sb-members.c b/fs/bcachefs/sb-members.c index 50dc9f937c45d..52054f26982f6 100644 --- a/fs/bcachefs/sb-members.c +++ b/fs/bcachefs/sb-members.c @@ -234,6 +234,14 @@ static void member_to_text(struct printbuf *out, prt_printf(out, "(none)"); prt_newline(out); + prt_printf(out, "Btree allocated bitmap blocksize:\t"); + prt_units_u64(out, 1ULL << m.btree_bitmap_shift); + prt_newline(out); + + prt_printf(out, "Btree allocated bitmap:\t"); + bch2_prt_u64_base2_nbits(out, le64_to_cpu(m.btree_allocated_bitmap), 64); + prt_newline(out); + prt_printf(out, "Durability:\t%llu\n", BCH_MEMBER_DURABILITY(&m) ? BCH_MEMBER_DURABILITY(&m) - 1 : 1); prt_printf(out, "Discard:\t%llu\n", BCH_MEMBER_DISCARD(&m)); From e7f63c67fcb4a479651ed8c50306bb654749faab Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 20 Apr 2024 16:10:40 -0400 Subject: [PATCH 1410/1702] bcachefs: plumb data_type into bch2_bucket_alloc_trans() prep work for making the allocator try to keep btree nodes within the existing member info btree allocated bitmap Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_foreground.c | 12 ++++++++---- fs/bcachefs/alloc_foreground.h | 3 ++- fs/bcachefs/journal.c | 3 ++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index 6cb878f5e5ebb..847000a7ead2b 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -516,6 +516,7 @@ static struct open_bucket *bch2_bucket_alloc_freelist(struct btree_trans *trans, * @trans: transaction object * @ca: device to allocate from * @watermark: how important is this allocation? + * @data_type: BCH_DATA_journal, btree, user... * @cl: if not NULL, closure to be used to wait if buckets not available * @usage: for secondarily also returning the current device usage * @@ -524,6 +525,7 @@ static struct open_bucket *bch2_bucket_alloc_freelist(struct btree_trans *trans, static struct open_bucket *bch2_bucket_alloc_trans(struct btree_trans *trans, struct bch_dev *ca, enum bch_watermark watermark, + enum bch_data_type data_type, struct closure *cl, struct bch_dev_usage *usage) { @@ -577,6 +579,9 @@ static struct open_bucket *bch2_bucket_alloc_trans(struct btree_trans *trans, if (!ob) ob = ERR_PTR(-BCH_ERR_no_buckets_found); + if (!IS_ERR(ob)) + ob->data_type = data_type; + if (!IS_ERR(ob)) trace_and_count(c, bucket_alloc, ca, bch2_watermarks[watermark], @@ -605,6 +610,7 @@ static struct open_bucket *bch2_bucket_alloc_trans(struct btree_trans *trans, struct open_bucket *bch2_bucket_alloc(struct bch_fs *c, struct bch_dev *ca, enum bch_watermark watermark, + enum bch_data_type data_type, struct closure *cl) { struct bch_dev_usage usage; @@ -612,7 +618,7 @@ struct open_bucket *bch2_bucket_alloc(struct bch_fs *c, struct bch_dev *ca, bch2_trans_do(c, NULL, NULL, 0, PTR_ERR_OR_ZERO(ob = bch2_bucket_alloc_trans(trans, ca, watermark, - cl, &usage))); + data_type, cl, &usage))); return ob; } @@ -738,7 +744,7 @@ int bch2_bucket_alloc_set_trans(struct btree_trans *trans, continue; } - ob = bch2_bucket_alloc_trans(trans, ca, watermark, cl, &usage); + ob = bch2_bucket_alloc_trans(trans, ca, watermark, data_type, cl, &usage); if (!IS_ERR(ob)) bch2_dev_stripe_increment_inlined(ca, stripe, &usage); percpu_ref_put(&ca->ref); @@ -750,8 +756,6 @@ int bch2_bucket_alloc_set_trans(struct btree_trans *trans, continue; } - ob->data_type = data_type; - if (add_new_bucket(c, ptrs, devs_may_alloc, nr_replicas, nr_effective, have_cache, flags, ob)) { diff --git a/fs/bcachefs/alloc_foreground.h b/fs/bcachefs/alloc_foreground.h index c101b2c367439..f0339eb7e30e9 100644 --- a/fs/bcachefs/alloc_foreground.h +++ b/fs/bcachefs/alloc_foreground.h @@ -31,7 +31,8 @@ void bch2_dev_stripe_increment(struct bch_dev *, struct dev_stripe_state *); long bch2_bucket_alloc_new_fs(struct bch_dev *); struct open_bucket *bch2_bucket_alloc(struct bch_fs *, struct bch_dev *, - enum bch_watermark, struct closure *); + enum bch_watermark, enum bch_data_type, + struct closure *); static inline void ob_push(struct bch_fs *c, struct open_buckets *obs, struct open_bucket *ob) diff --git a/fs/bcachefs/journal.c b/fs/bcachefs/journal.c index 17ceddbd88582..4d03a77f1e837 100644 --- a/fs/bcachefs/journal.c +++ b/fs/bcachefs/journal.c @@ -938,7 +938,8 @@ static int __bch2_set_nr_journal_buckets(struct bch_dev *ca, unsigned nr, break; } } else { - ob[nr_got] = bch2_bucket_alloc(c, ca, BCH_WATERMARK_normal, cl); + ob[nr_got] = bch2_bucket_alloc(c, ca, BCH_WATERMARK_normal, + BCH_DATA_journal, cl); ret = PTR_ERR_OR_ZERO(ob[nr_got]); if (ret) break; From f04158290d8bdd282899c6dc1539300df40b77d1 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 20 Apr 2024 22:19:48 -0400 Subject: [PATCH 1411/1702] bcachefs: journal seq blacklist gc no longer has to walk btree Since btree_ptr_v2, we no longer require the journal seq blacklist table for skipping blacklisted bsets (btree node entries); the pointer to a given node indicates how much data is present. Therefore there's no longer any need for journal seq blacklist gc to walk the btree - we can prune entries older than journal last_seq. Signed-off-by: Kent Overstreet --- fs/bcachefs/bcachefs.h | 1 - fs/bcachefs/journal_io.c | 4 ++ fs/bcachefs/journal_seq_blacklist.c | 72 +++++++---------------------- fs/bcachefs/journal_seq_blacklist.h | 2 +- fs/bcachefs/journal_types.h | 1 + fs/bcachefs/recovery.c | 7 ++- fs/bcachefs/super.c | 5 -- 7 files changed, 26 insertions(+), 66 deletions(-) diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index 6e324c1fe9245..d8eb5beb5977d 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -930,7 +930,6 @@ struct bch_fs { /* JOURNAL SEQ BLACKLIST */ struct journal_seq_blacklist_table * journal_seq_blacklist_table; - struct work_struct journal_seq_blacklist_gc_work; /* ALLOCATOR */ spinlock_t freelist_lock; diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c index 1d37eda8d7f00..98cf9a65216f6 100644 --- a/fs/bcachefs/journal_io.c +++ b/fs/bcachefs/journal_io.c @@ -122,6 +122,10 @@ static int journal_entry_add(struct bch_fs *c, struct bch_dev *ca, struct printbuf buf = PRINTBUF; int ret = JOURNAL_ENTRY_ADD_OK; + if (!c->journal.oldest_seq_found_ondisk || + le64_to_cpu(j->seq) < c->journal.oldest_seq_found_ondisk) + c->journal.oldest_seq_found_ondisk = le64_to_cpu(j->seq); + /* Is this entry older than the range we need? */ if (!c->opts.read_entire_journal && le64_to_cpu(j->seq) < jlist->last_seq) diff --git a/fs/bcachefs/journal_seq_blacklist.c b/fs/bcachefs/journal_seq_blacklist.c index c87c68ae27f43..d8d40d46a27a2 100644 --- a/fs/bcachefs/journal_seq_blacklist.c +++ b/fs/bcachefs/journal_seq_blacklist.c @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 #include "bcachefs.h" -#include "btree_iter.h" #include "eytzinger.h" +#include "journal.h" #include "journal_seq_blacklist.h" #include "super-io.h" @@ -217,78 +217,40 @@ const struct bch_sb_field_ops bch_sb_field_ops_journal_seq_blacklist = { .to_text = bch2_sb_journal_seq_blacklist_to_text }; -void bch2_blacklist_entries_gc(struct work_struct *work) +bool bch2_blacklist_entries_gc(struct bch_fs *c) { - struct bch_fs *c = container_of(work, struct bch_fs, - journal_seq_blacklist_gc_work); - struct journal_seq_blacklist_table *t; - struct bch_sb_field_journal_seq_blacklist *bl; struct journal_seq_blacklist_entry *src, *dst; - struct btree_trans *trans = bch2_trans_get(c); - unsigned i, nr, new_nr; - int ret; - - for (i = 0; i < BTREE_ID_NR; i++) { - struct btree_iter iter; - struct btree *b; - - bch2_trans_node_iter_init(trans, &iter, i, POS_MIN, - 0, 0, BTREE_ITER_prefetch); -retry: - bch2_trans_begin(trans); - - b = bch2_btree_iter_peek_node(&iter); - - while (!(ret = PTR_ERR_OR_ZERO(b)) && - b && - !test_bit(BCH_FS_stopping, &c->flags)) - b = bch2_btree_iter_next_node(&iter); - - if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) - goto retry; - bch2_trans_iter_exit(trans, &iter); - } - - bch2_trans_put(trans); - if (ret) - return; - - mutex_lock(&c->sb_lock); - bl = bch2_sb_field_get(c->disk_sb.sb, journal_seq_blacklist); + struct bch_sb_field_journal_seq_blacklist *bl = + bch2_sb_field_get(c->disk_sb.sb, journal_seq_blacklist); if (!bl) - goto out; + return false; - nr = blacklist_nr_entries(bl); + unsigned nr = blacklist_nr_entries(bl); dst = bl->start; - t = c->journal_seq_blacklist_table; + struct journal_seq_blacklist_table *t = c->journal_seq_blacklist_table; BUG_ON(nr != t->nr); + unsigned i; for (src = bl->start, i = eytzinger0_first(t->nr); src < bl->start + nr; src++, i = eytzinger0_next(i, nr)) { BUG_ON(t->entries[i].start != le64_to_cpu(src->start)); BUG_ON(t->entries[i].end != le64_to_cpu(src->end)); - if (t->entries[i].dirty) + if (t->entries[i].dirty || t->entries[i].end >= c->journal.oldest_seq_found_ondisk) *dst++ = *src; } - new_nr = dst - bl->start; - - bch_info(c, "nr blacklist entries was %u, now %u", nr, new_nr); - - if (new_nr != nr) { - bl = bch2_sb_field_resize(&c->disk_sb, journal_seq_blacklist, - new_nr ? sb_blacklist_u64s(new_nr) : 0); - BUG_ON(new_nr && !bl); + unsigned new_nr = dst - bl->start; + if (new_nr == nr) + return false; - if (!new_nr) - c->disk_sb.sb->features[0] &= cpu_to_le64(~(1ULL << BCH_FEATURE_journal_seq_blacklist_v3)); + bch_verbose(c, "nr blacklist entries was %u, now %u", nr, new_nr); - bch2_write_super(c); - } -out: - mutex_unlock(&c->sb_lock); + bl = bch2_sb_field_resize(&c->disk_sb, journal_seq_blacklist, + new_nr ? sb_blacklist_u64s(new_nr) : 0); + BUG_ON(new_nr && !bl); + return true; } diff --git a/fs/bcachefs/journal_seq_blacklist.h b/fs/bcachefs/journal_seq_blacklist.h index afb886ec8e254..d47636f96fdc6 100644 --- a/fs/bcachefs/journal_seq_blacklist.h +++ b/fs/bcachefs/journal_seq_blacklist.h @@ -17,6 +17,6 @@ int bch2_blacklist_table_initialize(struct bch_fs *); extern const struct bch_sb_field_ops bch_sb_field_ops_journal_seq_blacklist; -void bch2_blacklist_entries_gc(struct work_struct *); +bool bch2_blacklist_entries_gc(struct bch_fs *); #endif /* _BCACHEFS_JOURNAL_SEQ_BLACKLIST_H */ diff --git a/fs/bcachefs/journal_types.h b/fs/bcachefs/journal_types.h index 08debe6bfeeff..3d2135e1d7a1c 100644 --- a/fs/bcachefs/journal_types.h +++ b/fs/bcachefs/journal_types.h @@ -229,6 +229,7 @@ struct journal { u64 last_seq_ondisk; u64 err_seq; u64 last_empty_seq; + u64 oldest_seq_found_ondisk; /* * FIFO of journal entries whose btree updates have not yet been diff --git a/fs/bcachefs/recovery.c b/fs/bcachefs/recovery.c index 27ee27b285bde..c33e20aa56a28 100644 --- a/fs/bcachefs/recovery.c +++ b/fs/bcachefs/recovery.c @@ -878,6 +878,9 @@ int bch2_fs_recovery(struct bch_fs *c) write_sb = true; } + if (bch2_blacklist_entries_gc(c)) + write_sb = true; + if (write_sb) bch2_write_super(c); mutex_unlock(&c->sb_lock); @@ -900,10 +903,6 @@ int bch2_fs_recovery(struct bch_fs *c) bch_info(c, "scanning for old btree nodes done"); } - if (c->journal_seq_blacklist_table && - c->journal_seq_blacklist_table->nr > 128) - queue_work(system_long_wq, &c->journal_seq_blacklist_gc_work); - ret = 0; out: bch2_flush_fsck_errs(c); diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index 36833f86d50da..72dde1f1a3e67 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -611,8 +611,6 @@ void __bch2_fs_stop(struct bch_fs *c) set_bit(BCH_FS_stopping, &c->flags); - cancel_work_sync(&c->journal_seq_blacklist_gc_work); - down_write(&c->state_lock); bch2_fs_read_only(c); up_write(&c->state_lock); @@ -796,9 +794,6 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) spin_lock_init(&c->btree_write_error_lock); - INIT_WORK(&c->journal_seq_blacklist_gc_work, - bch2_blacklist_entries_gc); - INIT_LIST_HEAD(&c->journal_iters); INIT_LIST_HEAD(&c->fsck_error_msgs); From b769590f33dcb04cb8efc4905030f4a41df08e2b Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 20 Apr 2024 22:03:09 -0400 Subject: [PATCH 1412/1702] bcachefs: Clean up inode alloc There's no need to be using new_inode(); we can skip all that indirection and make the code easier to follow. Signed-off-by: Kent Overstreet --- fs/bcachefs/fs.c | 51 +++++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c index 841bb92e53dfd..5624c2ea8d55a 100644 --- a/fs/bcachefs/fs.c +++ b/fs/bcachefs/fs.c @@ -213,19 +213,43 @@ static struct bch_inode_info *bch2_inode_insert(struct bch_fs *c, struct bch_ino _ret; \ }) +static struct inode *bch2_alloc_inode(struct super_block *sb) +{ + BUG(); +} + +static struct bch_inode_info *__bch2_new_inode(struct bch_fs *c) +{ + struct bch_inode_info *inode = kmem_cache_alloc(bch2_inode_cache, GFP_NOFS); + if (!inode) + return NULL; + + inode_init_once(&inode->v); + mutex_init(&inode->ei_update_lock); + two_state_lock_init(&inode->ei_pagecache_lock); + INIT_LIST_HEAD(&inode->ei_vfs_inode_list); + mutex_init(&inode->ei_quota_lock); + inode->v.i_state = 0; + + if (unlikely(inode_init_always(c->vfs_sb, &inode->v))) { + kmem_cache_free(bch2_inode_cache, inode); + return NULL; + } + + return inode; +} + /* * Allocate a new inode, dropping/retaking btree locks if necessary: */ static struct bch_inode_info *bch2_new_inode(struct btree_trans *trans) { - struct bch_fs *c = trans->c; - struct bch_inode_info *inode = memalloc_flags_do(PF_MEMALLOC_NORECLAIM|PF_MEMALLOC_NOWARN, - to_bch_ei(new_inode(c->vfs_sb))); + __bch2_new_inode(trans->c)); if (unlikely(!inode)) { - int ret = drop_locks_do(trans, (inode = to_bch_ei(new_inode(c->vfs_sb))) ? 0 : -ENOMEM); + int ret = drop_locks_do(trans, (inode = __bch2_new_inode(trans->c)) ? 0 : -ENOMEM); if (ret && inode) { __destroy_inode(&inode->v); kmem_cache_free(bch2_inode_cache, inode); @@ -290,7 +314,7 @@ __bch2_create(struct mnt_idmap *idmap, if (ret) return ERR_PTR(ret); #endif - inode = to_bch_ei(new_inode(c->vfs_sb)); + inode = __bch2_new_inode(c); if (unlikely(!inode)) { inode = ERR_PTR(-ENOMEM); goto err; @@ -1487,23 +1511,6 @@ static void bch2_vfs_inode_init(struct btree_trans *trans, subvol_inum inum, mapping_set_large_folios(inode->v.i_mapping); } -static struct inode *bch2_alloc_inode(struct super_block *sb) -{ - struct bch_inode_info *inode; - - inode = kmem_cache_alloc(bch2_inode_cache, GFP_NOFS); - if (!inode) - return NULL; - - inode_init_once(&inode->v); - mutex_init(&inode->ei_update_lock); - two_state_lock_init(&inode->ei_pagecache_lock); - INIT_LIST_HEAD(&inode->ei_vfs_inode_list); - mutex_init(&inode->ei_quota_lock); - - return &inode->v; -} - static void bch2_i_callback(struct rcu_head *head) { struct inode *vinode = container_of(head, struct inode, i_rcu); From c4e8db2b5d31fc488c644019b99bf41fd616895f Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 22 Apr 2024 17:21:35 -0400 Subject: [PATCH 1413/1702] bcachefs: bucket_data_type_mismatch() We're working on potentially unifying bch2_check_bucket_ref() and bch2_check_fix_ptrs() - or at least eliminating gratuitious differences. Most immediately, there's a bunch of cleanups to be done regarding BCH_DATA_stripe. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.h | 15 ++++++++++++++- fs/bcachefs/btree_gc.c | 6 ++---- fs/bcachefs/buckets.c | 11 +---------- fs/bcachefs/ec.c | 19 +++++++++---------- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h index 44db71ccc953a..e94432ced6a66 100644 --- a/fs/bcachefs/alloc_background.h +++ b/fs/bcachefs/alloc_background.h @@ -67,7 +67,20 @@ static inline enum bch_data_type alloc_data_type(struct bch_alloc_v4 a, static inline enum bch_data_type bucket_data_type(enum bch_data_type data_type) { - return data_type == BCH_DATA_stripe ? BCH_DATA_user : data_type; + switch (data_type) { + case BCH_DATA_cached: + case BCH_DATA_stripe: + return BCH_DATA_user; + default: + return data_type; + } +} + +static inline bool bucket_data_type_mismatch(enum bch_data_type bucket, + enum bch_data_type ptr) +{ + return !data_type_is_empty(bucket) && + bucket_data_type(bucket) != bucket_data_type(ptr); } static inline unsigned bch2_bucket_sectors(struct bch_alloc_v4 a) diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 919fa1b9fc2e2..f0adc060e4e25 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -664,10 +664,8 @@ static int bch2_check_fix_ptrs(struct btree_trans *trans, enum btree_id btree_id if (data_type != BCH_DATA_btree && p.ptr.gen != g->gen) continue; - if (fsck_err_on(bucket_data_type(g->data_type) && - bucket_data_type(g->data_type) != - bucket_data_type(data_type), c, - ptr_bucket_data_type_mismatch, + if (fsck_err_on(bucket_data_type_mismatch(g->data_type, data_type), + c, ptr_bucket_data_type_mismatch, "bucket %u:%zu different types of data in same bucket: %s, %s\n" "while marking %s", p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 74e2098dfde51..36d13819a27dc 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -498,13 +498,6 @@ int bch2_check_bucket_ref(struct btree_trans *trans, struct printbuf buf = PRINTBUF; int ret = 0; - if (bucket_data_type == BCH_DATA_cached) - bucket_data_type = BCH_DATA_user; - - if ((bucket_data_type == BCH_DATA_stripe && ptr_data_type == BCH_DATA_user) || - (bucket_data_type == BCH_DATA_user && ptr_data_type == BCH_DATA_stripe)) - bucket_data_type = ptr_data_type = BCH_DATA_stripe; - if (gen_after(ptr->gen, b_gen)) { bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, BCH_FSCK_ERR_ptr_gen_newer_than_bucket_gen, @@ -552,9 +545,7 @@ int bch2_check_bucket_ref(struct btree_trans *trans, goto out; } - if (!data_type_is_empty(bucket_data_type) && - ptr_data_type && - bucket_data_type != ptr_data_type) { + if (bucket_data_type_mismatch(bucket_data_type, ptr_data_type)) { bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, BCH_FSCK_ERR_ptr_bucket_data_type_mismatch, "bucket %u:%zu gen %u different types of data in same bucket: %s, %s\n" diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 97ed598a55a52..4ccf715659614 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -165,15 +165,16 @@ void bch2_stripe_to_text(struct printbuf *out, struct bch_fs *c, static int bch2_trans_mark_stripe_bucket(struct btree_trans *trans, struct bkey_s_c_stripe s, - unsigned idx, bool deleting) + unsigned ptr_idx, bool deleting) { struct bch_fs *c = trans->c; - const struct bch_extent_ptr *ptr = &s.v->ptrs[idx]; + const struct bch_extent_ptr *ptr = s.v->ptrs + ptr_idx; struct btree_iter iter; struct bkey_i_alloc_v4 *a; - enum bch_data_type data_type = idx >= s.v->nr_blocks - s.v->nr_redundant - ? BCH_DATA_parity : 0; - s64 sectors = data_type ? le16_to_cpu(s.v->sectors) : 0; + unsigned nr_data = s.v->nr_blocks - s.v->nr_redundant; + bool parity = ptr_idx >= nr_data; + enum bch_data_type data_type = parity ? BCH_DATA_parity : BCH_DATA_stripe; + s64 sectors = parity ? le16_to_cpu(s.v->sectors) : 0; int ret = 0; if (deleting) @@ -201,8 +202,8 @@ static int bch2_trans_mark_stripe_bucket(struct btree_trans *trans, goto err; } - if (bch2_trans_inconsistent_on(data_type && a->v.dirty_sectors, trans, - "bucket %llu:%llu gen %u data type %s dirty_sectors %u: data already in stripe bucket %llu", + if (bch2_trans_inconsistent_on(parity && a->v.dirty_sectors, trans, + "bucket %llu:%llu gen %u data type %s dirty_sectors %u: data already in parity bucket %llu", iter.pos.inode, iter.pos.offset, a->v.gen, bch2_data_type_str(a->v.data_type), a->v.dirty_sectors, @@ -213,7 +214,7 @@ static int bch2_trans_mark_stripe_bucket(struct btree_trans *trans, a->v.stripe = s.k->p.offset; a->v.stripe_redundancy = s.v->nr_redundant; - a->v.data_type = BCH_DATA_stripe; + a->v.data_type = data_type; } else { if (bch2_trans_inconsistent_on(a->v.stripe != s.k->p.offset || a->v.stripe_redundancy != s.v->nr_redundant, trans, @@ -230,8 +231,6 @@ static int bch2_trans_mark_stripe_bucket(struct btree_trans *trans, } a->v.dirty_sectors += sectors; - if (data_type) - a->v.data_type = !deleting ? data_type : 0; ret = bch2_trans_update(trans, &iter, &a->k_i, 0); if (ret) From d9307646505e8e81a24ea5c07ae2bfc85e13f65d Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 20 Apr 2024 00:04:07 -0400 Subject: [PATCH 1414/1702] bcachefs: mark_stripe_bucket cleanup Start to work on unifying mark_stripe_bucket() and trans_mark_stripe_bucket(); first, clean up all the unnecessary and gratuitious differences. Signed-off-by: Kent Overstreet --- fs/bcachefs/ec.c | 100 ++++++++++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 35 deletions(-) diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 4ccf715659614..1848335707c1b 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -169,91 +169,118 @@ static int bch2_trans_mark_stripe_bucket(struct btree_trans *trans, { struct bch_fs *c = trans->c; const struct bch_extent_ptr *ptr = s.v->ptrs + ptr_idx; - struct btree_iter iter; - struct bkey_i_alloc_v4 *a; unsigned nr_data = s.v->nr_blocks - s.v->nr_redundant; bool parity = ptr_idx >= nr_data; enum bch_data_type data_type = parity ? BCH_DATA_parity : BCH_DATA_stripe; s64 sectors = parity ? le16_to_cpu(s.v->sectors) : 0; + struct printbuf buf = PRINTBUF; int ret = 0; if (deleting) sectors = -sectors; - a = bch2_trans_start_alloc_update(trans, &iter, PTR_BUCKET_POS(c, ptr)); - if (IS_ERR(a)) - return PTR_ERR(a); - - ret = bch2_check_bucket_ref(trans, s.s_c, ptr, sectors, data_type, - a->v.gen, a->v.data_type, - a->v.dirty_sectors); + struct btree_iter iter; + struct bkey_i_alloc_v4 *a = + bch2_trans_start_alloc_update(trans, &iter, PTR_BUCKET_POS(c, ptr)); + ret = PTR_ERR_OR_ZERO(a); if (ret) goto err; if (!deleting) { if (bch2_trans_inconsistent_on(a->v.stripe || a->v.stripe_redundancy, trans, - "bucket %llu:%llu gen %u data type %s dirty_sectors %u: multiple stripes using same bucket (%u, %llu)", + "bucket %llu:%llu gen %u data type %s dirty_sectors %u: multiple stripes using same bucket (%u, %llu)\n%s", iter.pos.inode, iter.pos.offset, a->v.gen, bch2_data_type_str(a->v.data_type), a->v.dirty_sectors, - a->v.stripe, s.k->p.offset)) { + a->v.stripe, s.k->p.offset, + (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { ret = -EIO; goto err; } - if (bch2_trans_inconsistent_on(parity && a->v.dirty_sectors, trans, - "bucket %llu:%llu gen %u data type %s dirty_sectors %u: data already in parity bucket %llu", + if (bch2_trans_inconsistent_on(parity && (a->v.dirty_sectors || a->v.cached_sectors), trans, + "bucket %llu:%llu gen %u data type %s dirty_sectors %u cached_sectors %u: data already in parity bucket\n%s", iter.pos.inode, iter.pos.offset, a->v.gen, bch2_data_type_str(a->v.data_type), a->v.dirty_sectors, - s.k->p.offset)) { + a->v.cached_sectors, + (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { ret = -EIO; goto err; } - - a->v.stripe = s.k->p.offset; - a->v.stripe_redundancy = s.v->nr_redundant; - a->v.data_type = data_type; } else { if (bch2_trans_inconsistent_on(a->v.stripe != s.k->p.offset || a->v.stripe_redundancy != s.v->nr_redundant, trans, - "bucket %llu:%llu gen %u: not marked as stripe when deleting stripe %llu (got %u)", + "bucket %llu:%llu gen %u: not marked as stripe when deleting stripe (got %u)\n%s", iter.pos.inode, iter.pos.offset, a->v.gen, - s.k->p.offset, a->v.stripe)) { + a->v.stripe, + (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { ret = -EIO; goto err; } + if (bch2_trans_inconsistent_on(a->v.data_type != data_type, trans, + "bucket %llu:%llu gen %u data type %s: wrong data type when stripe, should be %s\n%s", + iter.pos.inode, iter.pos.offset, a->v.gen, + bch2_data_type_str(a->v.data_type), + bch2_data_type_str(data_type), + (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { + ret = -EIO; + goto err; + } + + if (bch2_trans_inconsistent_on(parity && + (a->v.dirty_sectors != -sectors || + a->v.cached_sectors), trans, + "bucket %llu:%llu gen %u dirty_sectors %u cached_sectors %u: wrong sectors when deleting parity block of stripe\n%s", + iter.pos.inode, iter.pos.offset, a->v.gen, + a->v.dirty_sectors, + a->v.cached_sectors, + (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { + ret = -EIO; + goto err; + } + } + + ret = bch2_check_bucket_ref(trans, s.s_c, ptr, sectors, data_type, + a->v.gen, a->v.data_type, + a->v.dirty_sectors); + if (ret) + goto err; + + a->v.dirty_sectors += sectors; + + if (!deleting) { + a->v.stripe = s.k->p.offset; + a->v.stripe_redundancy = s.v->nr_redundant; + a->v.data_type = data_type; + } else { a->v.stripe = 0; a->v.stripe_redundancy = 0; a->v.data_type = alloc_data_type(a->v, BCH_DATA_user); } - a->v.dirty_sectors += sectors; - ret = bch2_trans_update(trans, &iter, &a->k_i, 0); if (ret) goto err; err: bch2_trans_iter_exit(trans, &iter); + printbuf_exit(&buf); return ret; } static int mark_stripe_bucket(struct btree_trans *trans, - struct bkey_s_c k, + struct bkey_s_c_stripe s, unsigned ptr_idx, enum btree_iter_update_trigger_flags flags) { struct bch_fs *c = trans->c; - const struct bch_stripe *s = bkey_s_c_to_stripe(k).v; - unsigned nr_data = s->nr_blocks - s->nr_redundant; + const struct bch_extent_ptr *ptr = s.v->ptrs + ptr_idx; + unsigned nr_data = s.v->nr_blocks - s.v->nr_redundant; bool parity = ptr_idx >= nr_data; enum bch_data_type data_type = parity ? BCH_DATA_parity : BCH_DATA_stripe; - s64 sectors = parity ? le16_to_cpu(s->sectors) : 0; - const struct bch_extent_ptr *ptr = s->ptrs + ptr_idx; - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); - struct bucket old, new, *g; + s64 sectors = parity ? le16_to_cpu(s.v->sectors) : 0; struct printbuf buf = PRINTBUF; int ret = 0; @@ -261,15 +288,18 @@ static int mark_stripe_bucket(struct btree_trans *trans, /* * XXX doesn't handle deletion */ + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); + struct bucket old, new, *g; + percpu_down_read(&c->mark_lock); g = PTR_GC_BUCKET(ca, ptr); if (g->dirty_sectors || - (g->stripe && g->stripe != k.k->p.offset)) { + (g->stripe && g->stripe != s.k->p.offset)) { bch2_fs_inconsistent(c, "bucket %u:%zu gen %u: multiple stripes using same bucket\n%s", ptr->dev, PTR_BUCKET_NR(ca, ptr), g->gen, - (bch2_bkey_val_to_text(&buf, c, k), buf.buf)); + (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf)); ret = -EINVAL; goto err; } @@ -277,7 +307,7 @@ static int mark_stripe_bucket(struct btree_trans *trans, bucket_lock(g); old = *g; - ret = bch2_check_bucket_ref(trans, k, ptr, sectors, data_type, + ret = bch2_check_bucket_ref(trans, s.s_c, ptr, sectors, data_type, g->gen, g->data_type, g->dirty_sectors); if (ret) @@ -286,8 +316,8 @@ static int mark_stripe_bucket(struct btree_trans *trans, g->data_type = data_type; g->dirty_sectors += sectors; - g->stripe = k.k->p.offset; - g->stripe_redundancy = s->nr_redundant; + g->stripe = s.k->p.offset; + g->stripe_redundancy = s.v->nr_redundant; new = *g; err: bucket_unlock(g); @@ -439,7 +469,7 @@ int bch2_trigger_stripe(struct btree_trans *trans, memset(m->block_sectors, 0, sizeof(m->block_sectors)); for (unsigned i = 0; i < new_s->nr_blocks; i++) { - int ret = mark_stripe_bucket(trans, new, i, flags); + int ret = mark_stripe_bucket(trans, bkey_s_c_to_stripe(new), i, flags); if (ret) return ret; } From 9cc455d1bcd3bd44d1f103bf3f2e58293976a006 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 22 Apr 2024 19:01:40 -0400 Subject: [PATCH 1415/1702] bcachefs: Consolidate mark_stripe_bucket() and trans_mark_stripe_bucket() This eliminates some duplicated logic, and the gc path now handles stripe updates and deletions - we need this since soon we're bringing back runtime gc. Signed-off-by: Kent Overstreet --- fs/bcachefs/ec.c | 251 +++++++++++++++++++++++++---------------------- 1 file changed, 135 insertions(+), 116 deletions(-) diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 1848335707c1b..2d1b20856841f 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -163,9 +163,16 @@ void bch2_stripe_to_text(struct printbuf *out, struct bch_fs *c, /* Triggers: */ -static int bch2_trans_mark_stripe_bucket(struct btree_trans *trans, - struct bkey_s_c_stripe s, - unsigned ptr_idx, bool deleting) +static int __mark_stripe_bucket(struct btree_trans *trans, + struct bkey_s_c_stripe s, + unsigned ptr_idx, bool deleting, + struct bpos bucket, + u8 bucket_gen, + u8 *bucket_data_type, + u32 *bucket_dirty_sectors, + u32 *bucket_cached_sectors, + u32 *bucket_stripe, + u8 *bucket_stripe_redundancy) { struct bch_fs *c = trans->c; const struct bch_extent_ptr *ptr = s.v->ptrs + ptr_idx; @@ -179,51 +186,44 @@ static int bch2_trans_mark_stripe_bucket(struct btree_trans *trans, if (deleting) sectors = -sectors; - struct btree_iter iter; - struct bkey_i_alloc_v4 *a = - bch2_trans_start_alloc_update(trans, &iter, PTR_BUCKET_POS(c, ptr)); - ret = PTR_ERR_OR_ZERO(a); - if (ret) - goto err; - if (!deleting) { - if (bch2_trans_inconsistent_on(a->v.stripe || - a->v.stripe_redundancy, trans, + if (bch2_trans_inconsistent_on(*bucket_stripe || + *bucket_stripe_redundancy, trans, "bucket %llu:%llu gen %u data type %s dirty_sectors %u: multiple stripes using same bucket (%u, %llu)\n%s", - iter.pos.inode, iter.pos.offset, a->v.gen, - bch2_data_type_str(a->v.data_type), - a->v.dirty_sectors, - a->v.stripe, s.k->p.offset, + bucket.inode, bucket.offset, bucket_gen, + bch2_data_type_str(*bucket_data_type), + *bucket_dirty_sectors, + *bucket_stripe, s.k->p.offset, (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { ret = -EIO; goto err; } - if (bch2_trans_inconsistent_on(parity && (a->v.dirty_sectors || a->v.cached_sectors), trans, + if (bch2_trans_inconsistent_on(parity && (*bucket_dirty_sectors || *bucket_cached_sectors), trans, "bucket %llu:%llu gen %u data type %s dirty_sectors %u cached_sectors %u: data already in parity bucket\n%s", - iter.pos.inode, iter.pos.offset, a->v.gen, - bch2_data_type_str(a->v.data_type), - a->v.dirty_sectors, - a->v.cached_sectors, + bucket.inode, bucket.offset, bucket_gen, + bch2_data_type_str(*bucket_data_type), + *bucket_dirty_sectors, + *bucket_cached_sectors, (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { ret = -EIO; goto err; } } else { - if (bch2_trans_inconsistent_on(a->v.stripe != s.k->p.offset || - a->v.stripe_redundancy != s.v->nr_redundant, trans, + if (bch2_trans_inconsistent_on(*bucket_stripe != s.k->p.offset || + *bucket_stripe_redundancy != s.v->nr_redundant, trans, "bucket %llu:%llu gen %u: not marked as stripe when deleting stripe (got %u)\n%s", - iter.pos.inode, iter.pos.offset, a->v.gen, - a->v.stripe, + bucket.inode, bucket.offset, bucket_gen, + *bucket_stripe, (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { ret = -EIO; goto err; } - if (bch2_trans_inconsistent_on(a->v.data_type != data_type, trans, + if (bch2_trans_inconsistent_on(*bucket_data_type != data_type, trans, "bucket %llu:%llu gen %u data type %s: wrong data type when stripe, should be %s\n%s", - iter.pos.inode, iter.pos.offset, a->v.gen, - bch2_data_type_str(a->v.data_type), + bucket.inode, bucket.offset, bucket_gen, + bch2_data_type_str(*bucket_data_type), bch2_data_type_str(data_type), (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { ret = -EIO; @@ -231,12 +231,12 @@ static int bch2_trans_mark_stripe_bucket(struct btree_trans *trans, } if (bch2_trans_inconsistent_on(parity && - (a->v.dirty_sectors != -sectors || - a->v.cached_sectors), trans, + (*bucket_dirty_sectors != -sectors || + *bucket_cached_sectors), trans, "bucket %llu:%llu gen %u dirty_sectors %u cached_sectors %u: wrong sectors when deleting parity block of stripe\n%s", - iter.pos.inode, iter.pos.offset, a->v.gen, - a->v.dirty_sectors, - a->v.cached_sectors, + bucket.inode, bucket.offset, bucket_gen, + *bucket_dirty_sectors, + *bucket_cached_sectors, (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { ret = -EIO; goto err; @@ -244,88 +244,128 @@ static int bch2_trans_mark_stripe_bucket(struct btree_trans *trans, } ret = bch2_check_bucket_ref(trans, s.s_c, ptr, sectors, data_type, - a->v.gen, a->v.data_type, - a->v.dirty_sectors); + bucket_gen, *bucket_data_type, + *bucket_dirty_sectors); if (ret) goto err; - a->v.dirty_sectors += sectors; + *bucket_dirty_sectors += sectors; if (!deleting) { - a->v.stripe = s.k->p.offset; - a->v.stripe_redundancy = s.v->nr_redundant; - a->v.data_type = data_type; + *bucket_stripe = s.k->p.offset; + *bucket_stripe_redundancy = s.v->nr_redundant; + *bucket_data_type = data_type; } else { - a->v.stripe = 0; - a->v.stripe_redundancy = 0; - a->v.data_type = alloc_data_type(a->v, BCH_DATA_user); + *bucket_stripe = 0; + *bucket_stripe_redundancy = 0; + *bucket_data_type = BCH_DATA_free; } - - ret = bch2_trans_update(trans, &iter, &a->k_i, 0); - if (ret) - goto err; err: - bch2_trans_iter_exit(trans, &iter); printbuf_exit(&buf); return ret; } static int mark_stripe_bucket(struct btree_trans *trans, struct bkey_s_c_stripe s, - unsigned ptr_idx, + unsigned ptr_idx, bool deleting, enum btree_iter_update_trigger_flags flags) { struct bch_fs *c = trans->c; const struct bch_extent_ptr *ptr = s.v->ptrs + ptr_idx; - unsigned nr_data = s.v->nr_blocks - s.v->nr_redundant; - bool parity = ptr_idx >= nr_data; - enum bch_data_type data_type = parity ? BCH_DATA_parity : BCH_DATA_stripe; - s64 sectors = parity ? le16_to_cpu(s.v->sectors) : 0; - struct printbuf buf = PRINTBUF; - int ret = 0; + struct bpos bucket = PTR_BUCKET_POS(c, ptr); - BUG_ON(!(flags & BTREE_TRIGGER_gc)); - - /* * XXX doesn't handle deletion */ + if (flags & BTREE_TRIGGER_transactional) { + struct btree_iter iter; + struct bkey_i_alloc_v4 *a = + bch2_trans_start_alloc_update(trans, &iter, bucket); + int ret = PTR_ERR_OR_ZERO(a) ?: + __mark_stripe_bucket(trans, s, ptr_idx, deleting, iter.pos, + a->v.gen, + &a->v.data_type, + &a->v.dirty_sectors, + &a->v.cached_sectors, + &a->v.stripe, + &a->v.stripe_redundancy); + if (ret) + goto err; - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); - struct bucket old, new, *g; + if (deleting) + a->v.data_type = alloc_data_type(a->v, BCH_DATA_user); - percpu_down_read(&c->mark_lock); - g = PTR_GC_BUCKET(ca, ptr); + ret = bch2_trans_update(trans, &iter, &a->k_i, 0); + if (ret) + goto err; +err: + bch2_trans_iter_exit(trans, &iter); + return ret; + } - if (g->dirty_sectors || - (g->stripe && g->stripe != s.k->p.offset)) { - bch2_fs_inconsistent(c, - "bucket %u:%zu gen %u: multiple stripes using same bucket\n%s", - ptr->dev, PTR_BUCKET_NR(ca, ptr), g->gen, - (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf)); - ret = -EINVAL; - goto err; + if (flags & BTREE_TRIGGER_gc) { + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); + + percpu_down_read(&c->mark_lock); + struct bucket *g = gc_bucket(ca, bucket.offset); + + bucket_lock(g); + struct bucket old = *g; + u8 data_type = g->data_type; + + int ret = __mark_stripe_bucket(trans, s, ptr_idx, deleting, bucket, + g->gen, + &data_type, + &g->dirty_sectors, + &g->cached_sectors, + &g->stripe, + &g->stripe_redundancy); + g->data_type = data_type; + struct bucket new = *g; + bucket_unlock(g); + if (!ret) + bch2_dev_usage_update_m(c, ca, &old, &new); + percpu_up_read(&c->mark_lock); + return ret; } - bucket_lock(g); - old = *g; + BUG(); + return 0; +} - ret = bch2_check_bucket_ref(trans, s.s_c, ptr, sectors, data_type, - g->gen, g->data_type, - g->dirty_sectors); - if (ret) - goto err; +static int mark_stripe_buckets(struct btree_trans *trans, + struct bkey_s_c old, struct bkey_s_c new, + enum btree_iter_update_trigger_flags flags) +{ + const struct bch_stripe *old_s = old.k->type == KEY_TYPE_stripe + ? bkey_s_c_to_stripe(old).v : NULL; + const struct bch_stripe *new_s = new.k->type == KEY_TYPE_stripe + ? bkey_s_c_to_stripe(new).v : NULL; - g->data_type = data_type; - g->dirty_sectors += sectors; + BUG_ON(old_s && new_s && old_s->nr_blocks != new_s->nr_blocks); - g->stripe = s.k->p.offset; - g->stripe_redundancy = s.v->nr_redundant; - new = *g; -err: - bucket_unlock(g); - if (!ret) - bch2_dev_usage_update_m(c, ca, &old, &new); - percpu_up_read(&c->mark_lock); - printbuf_exit(&buf); - return ret; + unsigned nr_blocks = new_s ? new_s->nr_blocks : old_s->nr_blocks; + + for (unsigned i = 0; i < nr_blocks; i++) { + if (new_s && old_s && + !memcmp(&new_s->ptrs[i], + &old_s->ptrs[i], + sizeof(new_s->ptrs[i]))) + continue; + + if (new_s) { + int ret = mark_stripe_bucket(trans, + bkey_s_c_to_stripe(new), i, false, flags); + if (ret) + return ret; + } + + if (old_s) { + int ret = mark_stripe_bucket(trans, + bkey_s_c_to_stripe(old), i, true, flags); + if (ret) + return ret; + } + } + + return 0; } int bch2_trigger_stripe(struct btree_trans *trans, @@ -376,28 +416,9 @@ int bch2_trigger_stripe(struct btree_trans *trans, return ret; } - unsigned nr_blocks = new_s ? new_s->nr_blocks : old_s->nr_blocks; - for (unsigned i = 0; i < nr_blocks; i++) { - if (new_s && old_s && - !memcmp(&new_s->ptrs[i], - &old_s->ptrs[i], - sizeof(new_s->ptrs[i]))) - continue; - - if (new_s) { - int ret = bch2_trans_mark_stripe_bucket(trans, - bkey_s_c_to_stripe(new), i, false); - if (ret) - return ret; - } - - if (old_s) { - int ret = bch2_trans_mark_stripe_bucket(trans, - bkey_s_c_to_stripe(old), i, true); - if (ret) - return ret; - } - } + int ret = mark_stripe_buckets(trans, old, new, flags); + if (ret) + return ret; } if (flags & BTREE_TRIGGER_atomic) { @@ -468,13 +489,11 @@ int bch2_trigger_stripe(struct btree_trans *trans, */ memset(m->block_sectors, 0, sizeof(m->block_sectors)); - for (unsigned i = 0; i < new_s->nr_blocks; i++) { - int ret = mark_stripe_bucket(trans, bkey_s_c_to_stripe(new), i, flags); - if (ret) - return ret; - } + int ret = mark_stripe_buckets(trans, old, new, flags); + if (ret) + return ret; - int ret = bch2_update_replicas(c, new, &m->r.e, + ret = bch2_update_replicas(c, new, &m->r.e, ((s64) m->sectors * m->nr_redundant), 0, true); if (ret) { From 70e3e039cf65f67fa3c41b51cb00a58f6cd48886 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 19 Apr 2024 19:03:58 -0400 Subject: [PATCH 1416/1702] bcachefs: bch2_bucket_ref_update() If we hit an inconsistency when updating allocation information, we don't want to fail the update if it's for a deletion - only if it's for a new key. Rename check_bucket_ref() -> bucket_ref_update() so we can centralize the logic to do this. Signed-off-by: Kent Overstreet --- fs/bcachefs/buckets.c | 55 +++++++++++++++++++++++++------------------ fs/bcachefs/buckets.h | 6 ++--- fs/bcachefs/ec.c | 14 +++++------ 3 files changed, 42 insertions(+), 33 deletions(-) diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 36d13819a27dc..adaa66f7764a2 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -485,19 +485,22 @@ int bch2_update_cached_sectors_list(struct btree_trans *trans, unsigned dev, s64 return bch2_update_replicas_list(trans, &r.e, sectors); } -int bch2_check_bucket_ref(struct btree_trans *trans, +int bch2_bucket_ref_update(struct btree_trans *trans, struct bkey_s_c k, const struct bch_extent_ptr *ptr, s64 sectors, enum bch_data_type ptr_data_type, u8 b_gen, u8 bucket_data_type, - u32 bucket_sectors) + u32 *bucket_sectors) { struct bch_fs *c = trans->c; struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); size_t bucket_nr = PTR_BUCKET_NR(ca, ptr); struct printbuf buf = PRINTBUF; + bool inserting = sectors > 0; int ret = 0; + BUG_ON(!sectors); + if (gen_after(ptr->gen, b_gen)) { bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, BCH_FSCK_ERR_ptr_gen_newer_than_bucket_gen, @@ -507,8 +510,9 @@ int bch2_check_bucket_ref(struct btree_trans *trans, bch2_data_type_str(bucket_data_type ?: ptr_data_type), ptr->gen, (bch2_bkey_val_to_text(&buf, c, k), buf.buf)); - ret = -EIO; - goto err; + if (inserting) + goto err; + goto out; } if (gen_cmp(b_gen, ptr->gen) > BUCKET_GC_GEN_MAX) { @@ -521,11 +525,17 @@ int bch2_check_bucket_ref(struct btree_trans *trans, ptr->gen, (printbuf_reset(&buf), bch2_bkey_val_to_text(&buf, c, k), buf.buf)); - ret = -EIO; - goto err; + if (inserting) + goto err; + goto out; } - if (b_gen != ptr->gen && !ptr->cached) { + if (b_gen != ptr->gen && ptr->cached) { + ret = 1; + goto out; + } + + if (b_gen != ptr->gen) { bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, BCH_FSCK_ERR_stale_dirty_ptr, "bucket %u:%zu gen %u (mem gen %u) data type %s: stale dirty ptr (gen %u)\n" @@ -536,12 +546,8 @@ int bch2_check_bucket_ref(struct btree_trans *trans, ptr->gen, (printbuf_reset(&buf), bch2_bkey_val_to_text(&buf, c, k), buf.buf)); - ret = -EIO; - goto err; - } - - if (b_gen != ptr->gen) { - ret = 1; + if (inserting) + goto err; goto out; } @@ -555,28 +561,33 @@ int bch2_check_bucket_ref(struct btree_trans *trans, bch2_data_type_str(ptr_data_type), (printbuf_reset(&buf), bch2_bkey_val_to_text(&buf, c, k), buf.buf)); - ret = -EIO; - goto err; + if (inserting) + goto err; + goto out; } - if ((u64) bucket_sectors + sectors > U32_MAX) { + if ((u64) *bucket_sectors + sectors > U32_MAX) { bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK, BCH_FSCK_ERR_bucket_sector_count_overflow, "bucket %u:%zu gen %u data type %s sector count overflow: %u + %lli > U32_MAX\n" "while marking %s", ptr->dev, bucket_nr, b_gen, bch2_data_type_str(bucket_data_type ?: ptr_data_type), - bucket_sectors, sectors, + *bucket_sectors, sectors, (printbuf_reset(&buf), bch2_bkey_val_to_text(&buf, c, k), buf.buf)); - ret = -EIO; - goto err; + if (inserting) + goto err; + sectors = -*bucket_sectors; } + + *bucket_sectors += sectors; out: printbuf_exit(&buf); return ret; err: bch2_dump_trans_updates(trans); + ret = -EIO; goto out; } @@ -723,14 +734,12 @@ static int __mark_pointer(struct btree_trans *trans, u32 *dst_sectors = !ptr->cached ? dirty_sectors : cached_sectors; - int ret = bch2_check_bucket_ref(trans, k, ptr, sectors, ptr_data_type, - bucket_gen, *bucket_data_type, *dst_sectors); + int ret = bch2_bucket_ref_update(trans, k, ptr, sectors, ptr_data_type, + bucket_gen, *bucket_data_type, dst_sectors); if (ret) return ret; - *dst_sectors += sectors; - if (!*dirty_sectors && !*cached_sectors) *bucket_data_type = 0; else if (*bucket_data_type != BCH_DATA_stripe) diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h index a88b9033349fa..c2c2ac36bee5b 100644 --- a/fs/bcachefs/buckets.h +++ b/fs/bcachefs/buckets.h @@ -337,9 +337,9 @@ int bch2_replicas_deltas_realloc(struct btree_trans *, unsigned); void bch2_fs_usage_initialize(struct bch_fs *); -int bch2_check_bucket_ref(struct btree_trans *, struct bkey_s_c, - const struct bch_extent_ptr *, - s64, enum bch_data_type, u8, u8, u32); +int bch2_bucket_ref_update(struct btree_trans *, struct bkey_s_c, + const struct bch_extent_ptr *, + s64, enum bch_data_type, u8, u8, u32 *); int bch2_trigger_extent(struct btree_trans *, enum btree_id, unsigned, struct bkey_s_c, struct bkey_s, diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 2d1b20856841f..31db3c8fce50b 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -243,13 +243,13 @@ static int __mark_stripe_bucket(struct btree_trans *trans, } } - ret = bch2_check_bucket_ref(trans, s.s_c, ptr, sectors, data_type, - bucket_gen, *bucket_data_type, - *bucket_dirty_sectors); - if (ret) - goto err; - - *bucket_dirty_sectors += sectors; + if (sectors) { + ret = bch2_bucket_ref_update(trans, s.s_c, ptr, sectors, data_type, + bucket_gen, *bucket_data_type, + bucket_dirty_sectors); + if (ret) + goto err; + } if (!deleting) { *bucket_stripe = s.k->p.offset; From 930e1a92d62d3f0dbf8d27002c539819738ef6bb Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 16 Apr 2024 22:35:02 -0400 Subject: [PATCH 1417/1702] bcachefs: kill gc looping for bucket gens looping when we change a bucket gen is not ideal - it means we risk failing if we'd go into an infinite loop, and it's better to make forward progress even if fsck doesn't fix everything. Signed-off-by: Kent Overstreet --- fs/bcachefs/bcachefs.h | 1 - fs/bcachefs/btree_gc.c | 103 +++++++---------------------------------- 2 files changed, 18 insertions(+), 86 deletions(-) diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index d8eb5beb5977d..2345e090d8a1f 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -630,7 +630,6 @@ struct bch_dev { x(clean_shutdown) \ x(fsck_running) \ x(initial_gc_unfixed) \ - x(need_another_gc) \ x(need_delete_dead_snapshots) \ x(error) \ x(topology_error) \ diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index f0adc060e4e25..7fdaf236e8d54 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -627,13 +627,14 @@ static int bch2_check_fix_ptrs(struct btree_trans *trans, enum btree_id btree_id p.ptr.gen, g->gen, (printbuf_reset(&buf), bch2_bkey_val_to_text(&buf, c, *k), buf.buf))) { - if (!p.ptr.cached) { + if (!p.ptr.cached && + (g->data_type != BCH_DATA_btree || + data_type == BCH_DATA_btree)) { g->gen_valid = true; g->gen = p.ptr.gen; g->data_type = 0; g->dirty_sectors = 0; g->cached_sectors = 0; - set_bit(BCH_FS_need_another_gc, &c->flags); } else { do_update = true; } @@ -666,16 +667,19 @@ static int bch2_check_fix_ptrs(struct btree_trans *trans, enum btree_id btree_id if (fsck_err_on(bucket_data_type_mismatch(g->data_type, data_type), c, ptr_bucket_data_type_mismatch, - "bucket %u:%zu different types of data in same bucket: %s, %s\n" + "bucket %u:%zu gen %u different types of data in same bucket: %s, %s\n" "while marking %s", - p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), + p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), g->gen, bch2_data_type_str(g->data_type), bch2_data_type_str(data_type), (printbuf_reset(&buf), bch2_bkey_val_to_text(&buf, c, *k), buf.buf))) { if (data_type == BCH_DATA_btree) { - g->data_type = data_type; - set_bit(BCH_FS_need_another_gc, &c->flags); + g->gen_valid = true; + g->gen = p.ptr.gen; + g->data_type = data_type; + g->dirty_sectors = 0; + g->cached_sectors = 0; } else { do_update = true; } @@ -749,7 +753,7 @@ static int bch2_check_fix_ptrs(struct btree_trans *trans, enum btree_id btree_id gen_cmp(p.ptr.gen, g->gen) < 0) || gen_cmp(g->gen, p.ptr.gen) > BUCKET_GC_GEN_MAX || (g->data_type && - g->data_type != data_type)) { + bucket_data_type(g->data_type) != data_type)) { bch2_bkey_drop_ptr(bkey_i_to_s(new), &entry->ptr); goto restart_drop_ptrs; } @@ -1182,19 +1186,6 @@ static int bch2_gc_start(struct bch_fs *c) return 0; } -static int bch2_gc_reset(struct bch_fs *c) -{ - for_each_member_device(c, ca) { - free_percpu(ca->usage_gc); - ca->usage_gc = NULL; - } - - free_percpu(c->usage_gc); - c->usage_gc = NULL; - - return bch2_gc_start(c); -} - /* returns true if not equal */ static inline bool bch2_alloc_v4_cmp(struct bch_alloc_v4 l, struct bch_alloc_v4 r) @@ -1363,20 +1354,6 @@ static int bch2_gc_alloc_start(struct bch_fs *c) return ret; } -static void bch2_gc_alloc_reset(struct bch_fs *c) -{ - for_each_member_device(c, ca) { - struct bucket_array *buckets = gc_bucket_array(ca); - struct bucket *g; - - for_each_bucket(g, buckets) { - g->data_type = 0; - g->dirty_sectors = 0; - g->cached_sectors = 0; - } - } -} - static int bch2_gc_write_reflink_key(struct btree_trans *trans, struct btree_iter *iter, struct bkey_s_c k, @@ -1469,15 +1446,6 @@ static int bch2_gc_reflink_start(struct bch_fs *c) return ret; } -static void bch2_gc_reflink_reset(struct bch_fs *c) -{ - struct genradix_iter iter; - struct reflink_gc *r; - - genradix_for_each(&c->reflink_gc_table, iter, r) - r->refcount = 0; -} - static int bch2_gc_write_stripes_key(struct btree_trans *trans, struct btree_iter *iter, struct bkey_s_c k) @@ -1541,11 +1509,6 @@ static int bch2_gc_stripes_done(struct bch_fs *c) bch2_gc_write_stripes_key(trans, &iter, k))); } -static void bch2_gc_stripes_reset(struct bch_fs *c) -{ - genradix_free(&c->gc_stripes); -} - /** * bch2_gc - walk _all_ references to buckets, and recompute them: * @@ -1571,7 +1534,6 @@ static void bch2_gc_stripes_reset(struct bch_fs *c) */ static int bch2_gc(struct bch_fs *c, bool initial) { - unsigned iter = 0; int ret; lockdep_assert_held(&c->state_lock); @@ -1585,7 +1547,7 @@ static int bch2_gc(struct bch_fs *c, bool initial) bch2_gc_reflink_start(c); if (ret) goto out; -again: + gc_pos_set(c, gc_phase(GC_PHASE_START)); ret = bch2_mark_superblocks(c); @@ -1597,43 +1559,14 @@ static int bch2_gc(struct bch_fs *c, bool initial) c->gc_count++; - if (test_bit(BCH_FS_need_another_gc, &c->flags) || - (!iter && bch2_test_restart_gc)) { - if (iter++ > 2) { - bch_info(c, "Unable to fix bucket gens, looping"); - ret = -EINVAL; - goto out; - } - - /* - * XXX: make sure gens we fixed got saved - */ - bch_info(c, "Second GC pass needed, restarting:"); - clear_bit(BCH_FS_need_another_gc, &c->flags); - __gc_pos_set(c, gc_phase(GC_PHASE_NOT_RUNNING)); - - bch2_gc_stripes_reset(c); - bch2_gc_alloc_reset(c); - bch2_gc_reflink_reset(c); - ret = bch2_gc_reset(c); - if (ret) - goto out; - - /* flush fsck errors, reset counters */ - bch2_flush_fsck_errs(c); - goto again; - } + bch2_journal_block(&c->journal); out: - if (!ret) { - bch2_journal_block(&c->journal); - - ret = bch2_gc_alloc_done(c) ?: - bch2_gc_done(c) ?: - bch2_gc_stripes_done(c) ?: - bch2_gc_reflink_done(c); + ret = bch2_gc_alloc_done(c) ?: + bch2_gc_done(c) ?: + bch2_gc_stripes_done(c) ?: + bch2_gc_reflink_done(c); - bch2_journal_unblock(&c->journal); - } + bch2_journal_unblock(&c->journal); percpu_down_write(&c->mark_lock); /* Indicates that gc is no longer in progress: */ From f40d13f94df744185f99732e89bc8d085729ecc9 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 7 Apr 2024 19:07:09 -0400 Subject: [PATCH 1418/1702] bcachefs: Run bch2_check_fix_ptrs() via triggers Currently, the reflink_p gc trigger does repair as well - turning a reflink_p key into an error key if the reflink_v it points to doesn't exist. This won't work with online check/repair, because the repair path once online will be subject to transaction restarts, but BTREE_TRIGGER_gc is not idempotant - we can't run it multiple times if we get a transaction restart. So we need to split these paths; to do so this patch calls check_fix_ptrs() by a new general path - a new trigger type, BTREE_TRIGGER_check_repair. Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_cache.c | 29 ++++ fs/bcachefs/btree_cache.h | 3 + fs/bcachefs/btree_gc.c | 337 +++++--------------------------------- fs/bcachefs/btree_types.h | 2 + fs/bcachefs/buckets.c | 242 ++++++++++++++++++++++++++- fs/bcachefs/buckets.h | 4 + fs/bcachefs/ec.c | 5 +- 7 files changed, 326 insertions(+), 296 deletions(-) diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c index 02c70e813face..debdd7dc04778 100644 --- a/fs/bcachefs/btree_cache.c +++ b/fs/bcachefs/btree_cache.c @@ -190,6 +190,35 @@ int bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b, return ret; } +void bch2_btree_node_update_key_early(struct btree_trans *trans, + enum btree_id btree, unsigned level, + struct bkey_s_c old, struct bkey_i *new) +{ + struct bch_fs *c = trans->c; + struct btree *b; + struct bkey_buf tmp; + int ret; + + bch2_bkey_buf_init(&tmp); + bch2_bkey_buf_reassemble(&tmp, c, old); + + b = bch2_btree_node_get_noiter(trans, tmp.k, btree, level, true); + if (!IS_ERR_OR_NULL(b)) { + mutex_lock(&c->btree_cache.lock); + + bch2_btree_node_hash_remove(&c->btree_cache, b); + + bkey_copy(&b->key, new); + ret = __bch2_btree_node_hash_insert(&c->btree_cache, b); + BUG_ON(ret); + + mutex_unlock(&c->btree_cache.lock); + six_unlock_read(&b->c.lock); + } + + bch2_bkey_buf_exit(&tmp, c); +} + __flatten static inline struct btree *btree_cache_find(struct btree_cache *bc, const struct bkey_i *k) diff --git a/fs/bcachefs/btree_cache.h b/fs/bcachefs/btree_cache.h index 6d33885fdbde0..6fe91d1c0fd4c 100644 --- a/fs/bcachefs/btree_cache.h +++ b/fs/bcachefs/btree_cache.h @@ -17,6 +17,9 @@ int __bch2_btree_node_hash_insert(struct btree_cache *, struct btree *); int bch2_btree_node_hash_insert(struct btree_cache *, struct btree *, unsigned, enum btree_id); +void bch2_btree_node_update_key_early(struct btree_trans *, enum btree_id, unsigned, + struct bkey_s_c, struct bkey_i *); + void bch2_btree_cache_cannibalize_unlock(struct btree_trans *); int bch2_btree_cache_cannibalize_lock(struct btree_trans *, struct closure *); diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 7fdaf236e8d54..e30fa3c47553b 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -91,35 +91,6 @@ static void btree_ptr_to_v2(struct btree *b, struct bkey_i_btree_ptr_v2 *dst) } } -static void bch2_btree_node_update_key_early(struct btree_trans *trans, - enum btree_id btree, unsigned level, - struct bkey_s_c old, struct bkey_i *new) -{ - struct bch_fs *c = trans->c; - struct btree *b; - struct bkey_buf tmp; - int ret; - - bch2_bkey_buf_init(&tmp); - bch2_bkey_buf_reassemble(&tmp, c, old); - - b = bch2_btree_node_get_noiter(trans, tmp.k, btree, level, true); - if (!IS_ERR_OR_NULL(b)) { - mutex_lock(&c->btree_cache.lock); - - bch2_btree_node_hash_remove(&c->btree_cache, b); - - bkey_copy(&b->key, new); - ret = __bch2_btree_node_hash_insert(&c->btree_cache, b); - BUG_ON(ret); - - mutex_unlock(&c->btree_cache.lock); - six_unlock_read(&b->c.lock); - } - - bch2_bkey_buf_exit(&tmp, c); -} - static int set_node_min(struct bch_fs *c, struct btree *b, struct bpos new_min) { struct bkey_i_btree_ptr_v2 *new; @@ -580,245 +551,11 @@ int bch2_check_topology(struct bch_fs *c) return ret; } -static int bch2_check_fix_ptrs(struct btree_trans *trans, enum btree_id btree_id, - unsigned level, bool is_root, - struct bkey_s_c *k) -{ - struct bch_fs *c = trans->c; - struct bkey_ptrs_c ptrs_c = bch2_bkey_ptrs_c(*k); - const union bch_extent_entry *entry_c; - struct extent_ptr_decoded p = { 0 }; - bool do_update = false; - struct printbuf buf = PRINTBUF; - int ret = 0; - - /* - * XXX - * use check_bucket_ref here - */ - bkey_for_each_ptr_decode(k->k, ptrs_c, p, entry_c) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); - struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr); - enum bch_data_type data_type = bch2_bkey_ptr_data_type(*k, p, entry_c); - - if (fsck_err_on(!g->gen_valid, - c, ptr_to_missing_alloc_key, - "bucket %u:%zu data type %s ptr gen %u missing in alloc btree\n" - "while marking %s", - p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), - bch2_data_type_str(ptr_data_type(k->k, &p.ptr)), - p.ptr.gen, - (printbuf_reset(&buf), - bch2_bkey_val_to_text(&buf, c, *k), buf.buf))) { - if (!p.ptr.cached) { - g->gen_valid = true; - g->gen = p.ptr.gen; - } else { - do_update = true; - } - } - - if (fsck_err_on(gen_cmp(p.ptr.gen, g->gen) > 0, - c, ptr_gen_newer_than_bucket_gen, - "bucket %u:%zu data type %s ptr gen in the future: %u > %u\n" - "while marking %s", - p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), - bch2_data_type_str(ptr_data_type(k->k, &p.ptr)), - p.ptr.gen, g->gen, - (printbuf_reset(&buf), - bch2_bkey_val_to_text(&buf, c, *k), buf.buf))) { - if (!p.ptr.cached && - (g->data_type != BCH_DATA_btree || - data_type == BCH_DATA_btree)) { - g->gen_valid = true; - g->gen = p.ptr.gen; - g->data_type = 0; - g->dirty_sectors = 0; - g->cached_sectors = 0; - } else { - do_update = true; - } - } - - if (fsck_err_on(gen_cmp(g->gen, p.ptr.gen) > BUCKET_GC_GEN_MAX, - c, ptr_gen_newer_than_bucket_gen, - "bucket %u:%zu gen %u data type %s: ptr gen %u too stale\n" - "while marking %s", - p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), g->gen, - bch2_data_type_str(ptr_data_type(k->k, &p.ptr)), - p.ptr.gen, - (printbuf_reset(&buf), - bch2_bkey_val_to_text(&buf, c, *k), buf.buf))) - do_update = true; - - if (fsck_err_on(!p.ptr.cached && gen_cmp(p.ptr.gen, g->gen) < 0, - c, stale_dirty_ptr, - "bucket %u:%zu data type %s stale dirty ptr: %u < %u\n" - "while marking %s", - p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), - bch2_data_type_str(ptr_data_type(k->k, &p.ptr)), - p.ptr.gen, g->gen, - (printbuf_reset(&buf), - bch2_bkey_val_to_text(&buf, c, *k), buf.buf))) - do_update = true; - - if (data_type != BCH_DATA_btree && p.ptr.gen != g->gen) - continue; - - if (fsck_err_on(bucket_data_type_mismatch(g->data_type, data_type), - c, ptr_bucket_data_type_mismatch, - "bucket %u:%zu gen %u different types of data in same bucket: %s, %s\n" - "while marking %s", - p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), g->gen, - bch2_data_type_str(g->data_type), - bch2_data_type_str(data_type), - (printbuf_reset(&buf), - bch2_bkey_val_to_text(&buf, c, *k), buf.buf))) { - if (data_type == BCH_DATA_btree) { - g->gen_valid = true; - g->gen = p.ptr.gen; - g->data_type = data_type; - g->dirty_sectors = 0; - g->cached_sectors = 0; - } else { - do_update = true; - } - } - - if (p.has_ec) { - struct gc_stripe *m = genradix_ptr(&c->gc_stripes, p.ec.idx); - - if (fsck_err_on(!m || !m->alive, c, - ptr_to_missing_stripe, - "pointer to nonexistent stripe %llu\n" - "while marking %s", - (u64) p.ec.idx, - (printbuf_reset(&buf), - bch2_bkey_val_to_text(&buf, c, *k), buf.buf))) - do_update = true; - - if (fsck_err_on(m && m->alive && !bch2_ptr_matches_stripe_m(m, p), c, - ptr_to_incorrect_stripe, - "pointer does not match stripe %llu\n" - "while marking %s", - (u64) p.ec.idx, - (printbuf_reset(&buf), - bch2_bkey_val_to_text(&buf, c, *k), buf.buf))) - do_update = true; - } - } - - if (do_update) { - if (is_root) { - bch_err(c, "cannot update btree roots yet"); - ret = -EINVAL; - goto err; - } - - struct bkey_i *new = kmalloc(bkey_bytes(k->k), GFP_KERNEL); - if (!new) { - ret = -BCH_ERR_ENOMEM_gc_repair_key; - bch_err_msg(c, ret, "allocating new key"); - goto err; - } - - bkey_reassemble(new, *k); - - if (level) { - /* - * We don't want to drop btree node pointers - if the - * btree node isn't there anymore, the read path will - * sort it out: - */ - struct bkey_ptrs ptrs = bch2_bkey_ptrs(bkey_i_to_s(new)); - bkey_for_each_ptr(ptrs, ptr) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); - struct bucket *g = PTR_GC_BUCKET(ca, ptr); - - ptr->gen = g->gen; - } - } else { - struct bkey_ptrs ptrs; - union bch_extent_entry *entry; -restart_drop_ptrs: - ptrs = bch2_bkey_ptrs(bkey_i_to_s(new)); - bkey_for_each_ptr_decode(bkey_i_to_s(new).k, ptrs, p, entry) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); - struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr); - enum bch_data_type data_type = bch2_bkey_ptr_data_type(bkey_i_to_s_c(new), p, entry); - - if ((p.ptr.cached && - (!g->gen_valid || gen_cmp(p.ptr.gen, g->gen) > 0)) || - (!p.ptr.cached && - gen_cmp(p.ptr.gen, g->gen) < 0) || - gen_cmp(g->gen, p.ptr.gen) > BUCKET_GC_GEN_MAX || - (g->data_type && - bucket_data_type(g->data_type) != data_type)) { - bch2_bkey_drop_ptr(bkey_i_to_s(new), &entry->ptr); - goto restart_drop_ptrs; - } - } -again: - ptrs = bch2_bkey_ptrs(bkey_i_to_s(new)); - bkey_extent_entry_for_each(ptrs, entry) { - if (extent_entry_type(entry) == BCH_EXTENT_ENTRY_stripe_ptr) { - struct gc_stripe *m = genradix_ptr(&c->gc_stripes, - entry->stripe_ptr.idx); - union bch_extent_entry *next_ptr; - - bkey_extent_entry_for_each_from(ptrs, next_ptr, entry) - if (extent_entry_type(next_ptr) == BCH_EXTENT_ENTRY_ptr) - goto found; - next_ptr = NULL; -found: - if (!next_ptr) { - bch_err(c, "aieee, found stripe ptr with no data ptr"); - continue; - } - - if (!m || !m->alive || - !__bch2_ptr_matches_stripe(&m->ptrs[entry->stripe_ptr.block], - &next_ptr->ptr, - m->sectors)) { - bch2_bkey_extent_entry_drop(new, entry); - goto again; - } - } - } - } - - if (level) - bch2_btree_node_update_key_early(trans, btree_id, level - 1, *k, new); - - if (0) { - printbuf_reset(&buf); - bch2_bkey_val_to_text(&buf, c, *k); - bch_info(c, "updated %s", buf.buf); - - printbuf_reset(&buf); - bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(new)); - bch_info(c, "new key %s", buf.buf); - } - - ret = bch2_journal_key_insert_take(c, btree_id, level, new); - if (ret) { - kfree(new); - goto err; - } - - *k = bkey_i_to_s_c(new); - } -err: -fsck_err: - printbuf_exit(&buf); - return ret; -} - /* marking of btree keys/nodes: */ static int bch2_gc_mark_key(struct btree_trans *trans, enum btree_id btree_id, unsigned level, bool is_root, - struct bkey_s_c *k, + struct bkey_s_c k, bool initial) { struct bch_fs *c = trans->c; @@ -827,40 +564,53 @@ static int bch2_gc_mark_key(struct btree_trans *trans, enum btree_id btree_id, struct printbuf buf = PRINTBUF; int ret = 0; - deleted.p = k->k->p; + deleted.p = k.k->p; if (initial) { BUG_ON(bch2_journal_seq_verify && - k->k->version.lo > atomic64_read(&c->journal.seq)); + k.k->version.lo > atomic64_read(&c->journal.seq)); - if (fsck_err_on(k->k->version.lo > atomic64_read(&c->key_version), c, + if (fsck_err_on(k.k->version.lo > atomic64_read(&c->key_version), c, bkey_version_in_future, "key version number higher than recorded: %llu > %llu", - k->k->version.lo, + k.k->version.lo, atomic64_read(&c->key_version))) - atomic64_set(&c->key_version, k->k->version.lo); + atomic64_set(&c->key_version, k.k->version.lo); } - ret = bch2_check_fix_ptrs(trans, btree_id, level, is_root, k); - if (ret) - goto err; - - if (mustfix_fsck_err_on(level && !bch2_dev_btree_bitmap_marked(c, *k), + if (mustfix_fsck_err_on(level && !bch2_dev_btree_bitmap_marked(c, k), c, btree_bitmap_not_marked, "btree ptr not marked in member info btree allocated bitmap\n %s", - (bch2_bkey_val_to_text(&buf, c, *k), + (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { mutex_lock(&c->sb_lock); - bch2_dev_btree_bitmap_mark(c, *k); + bch2_dev_btree_bitmap_mark(c, k); bch2_write_super(c); mutex_unlock(&c->sb_lock); } - ret = commit_do(trans, NULL, NULL, 0, - bch2_key_trigger(trans, btree_id, level, old, - unsafe_bkey_s_c_to_s(*k), BTREE_TRIGGER_gc)); + /* + * We require a commit before key_trigger() because + * key_trigger(BTREE_TRIGGER_GC) is not idempotant; we'll calculate the + * wrong result if we run it multiple times. + */ + unsigned flags = is_root ? BTREE_TRIGGER_is_root : 0; + + ret = bch2_key_trigger(trans, btree_id, level, old, unsafe_bkey_s_c_to_s(k), + BTREE_TRIGGER_check_repair|flags); + if (ret) + goto out; + + if (trans->nr_updates) { + ret = bch2_trans_commit(trans, NULL, NULL, 0) ?: + -BCH_ERR_transaction_restart_nested; + goto out; + } + + ret = bch2_key_trigger(trans, btree_id, level, old, unsafe_bkey_s_c_to_s(k), + BTREE_TRIGGER_gc|flags); +out: fsck_err: -err: printbuf_exit(&buf); bch_err_fn(c, ret); return ret; @@ -879,7 +629,12 @@ static int btree_gc_mark_node(struct btree_trans *trans, struct btree *b, bool i bch2_btree_and_journal_iter_init_node_iter(trans, &iter, b); while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) { - ret = bch2_gc_mark_key(trans, b->c.btree_id, b->c.level, false, &k, initial); + bch2_trans_begin(trans); + ret = bch2_gc_mark_key(trans, b->c.btree_id, b->c.level, false, k, initial); + if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) { + ret = 0; + continue; + } if (ret) break; @@ -918,12 +673,10 @@ static int bch2_gc_btree(struct btree_trans *trans, enum btree_id btree_id, mutex_lock(&c->btree_root_lock); b = bch2_btree_id_root(c, btree_id)->b; - if (!btree_node_fake(b)) { - struct bkey_s_c k = bkey_i_to_s_c(&b->key); - - ret = bch2_gc_mark_key(trans, b->c.btree_id, b->c.level + 1, - true, &k, initial); - } + if (!btree_node_fake(b)) + ret = lockrestart_do(trans, + bch2_gc_mark_key(trans, b->c.btree_id, b->c.level + 1, + true, bkey_i_to_s_c(&b->key), initial)); gc_pos_set(c, gc_pos_btree_root(b->c.btree_id)); mutex_unlock(&c->btree_root_lock); @@ -991,12 +744,10 @@ static int bch2_gc_btree_init(struct btree_trans *trans, if (b->c.level >= target_depth) ret = bch2_gc_btree_init_recurse(trans, b, target_depth); - if (!ret) { - struct bkey_s_c k = bkey_i_to_s_c(&b->key); - - ret = bch2_gc_mark_key(trans, b->c.btree_id, b->c.level + 1, true, - &k, true); - } + if (!ret) + ret = lockrestart_do(trans, + bch2_gc_mark_key(trans, b->c.btree_id, b->c.level + 1, true, + bkey_i_to_s_c(&b->key), true)); six_unlock_read(&b->c.lock); bch_err_fn(c, ret); diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index a973ba6264d3b..4ff5213219a50 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -237,9 +237,11 @@ struct btree_node_iter { x(norun) \ x(transactional) \ x(atomic) \ + x(check_repair) \ x(gc) \ x(insert) \ x(overwrite) \ + x(is_root) \ x(bucket_invalidate) enum { diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index adaa66f7764a2..e0e991043aa51 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -485,6 +485,241 @@ int bch2_update_cached_sectors_list(struct btree_trans *trans, unsigned dev, s64 return bch2_update_replicas_list(trans, &r.e, sectors); } +int bch2_check_fix_ptrs(struct btree_trans *trans, + enum btree_id btree, unsigned level, struct bkey_s_c k, + enum btree_iter_update_trigger_flags flags) +{ + struct bch_fs *c = trans->c; + struct bkey_ptrs_c ptrs_c = bch2_bkey_ptrs_c(k); + const union bch_extent_entry *entry_c; + struct extent_ptr_decoded p = { 0 }; + bool do_update = false; + struct printbuf buf = PRINTBUF; + int ret = 0; + + percpu_down_read(&c->mark_lock); + + bkey_for_each_ptr_decode(k.k, ptrs_c, p, entry_c) { + struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); + struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr); + enum bch_data_type data_type = bch2_bkey_ptr_data_type(k, p, entry_c); + + if (fsck_err_on(!g->gen_valid, + c, ptr_to_missing_alloc_key, + "bucket %u:%zu data type %s ptr gen %u missing in alloc btree\n" + "while marking %s", + p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), + bch2_data_type_str(ptr_data_type(k.k, &p.ptr)), + p.ptr.gen, + (printbuf_reset(&buf), + bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { + if (!p.ptr.cached) { + g->gen_valid = true; + g->gen = p.ptr.gen; + } else { + do_update = true; + } + } + + if (fsck_err_on(gen_cmp(p.ptr.gen, g->gen) > 0, + c, ptr_gen_newer_than_bucket_gen, + "bucket %u:%zu data type %s ptr gen in the future: %u > %u\n" + "while marking %s", + p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), + bch2_data_type_str(ptr_data_type(k.k, &p.ptr)), + p.ptr.gen, g->gen, + (printbuf_reset(&buf), + bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { + if (!p.ptr.cached && + (g->data_type != BCH_DATA_btree || + data_type == BCH_DATA_btree)) { + g->gen_valid = true; + g->gen = p.ptr.gen; + g->data_type = 0; + g->dirty_sectors = 0; + g->cached_sectors = 0; + } else { + do_update = true; + } + } + + if (fsck_err_on(gen_cmp(g->gen, p.ptr.gen) > BUCKET_GC_GEN_MAX, + c, ptr_gen_newer_than_bucket_gen, + "bucket %u:%zu gen %u data type %s: ptr gen %u too stale\n" + "while marking %s", + p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), g->gen, + bch2_data_type_str(ptr_data_type(k.k, &p.ptr)), + p.ptr.gen, + (printbuf_reset(&buf), + bch2_bkey_val_to_text(&buf, c, k), buf.buf))) + do_update = true; + + if (fsck_err_on(!p.ptr.cached && gen_cmp(p.ptr.gen, g->gen) < 0, + c, stale_dirty_ptr, + "bucket %u:%zu data type %s stale dirty ptr: %u < %u\n" + "while marking %s", + p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), + bch2_data_type_str(ptr_data_type(k.k, &p.ptr)), + p.ptr.gen, g->gen, + (printbuf_reset(&buf), + bch2_bkey_val_to_text(&buf, c, k), buf.buf))) + do_update = true; + + if (data_type != BCH_DATA_btree && p.ptr.gen != g->gen) + continue; + + if (fsck_err_on(bucket_data_type_mismatch(g->data_type, data_type), + c, ptr_bucket_data_type_mismatch, + "bucket %u:%zu gen %u different types of data in same bucket: %s, %s\n" + "while marking %s", + p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), g->gen, + bch2_data_type_str(g->data_type), + bch2_data_type_str(data_type), + (printbuf_reset(&buf), + bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { + if (data_type == BCH_DATA_btree) { + g->gen_valid = true; + g->gen = p.ptr.gen; + g->data_type = data_type; + g->dirty_sectors = 0; + g->cached_sectors = 0; + } else { + do_update = true; + } + } + + if (p.has_ec) { + struct gc_stripe *m = genradix_ptr(&c->gc_stripes, p.ec.idx); + + if (fsck_err_on(!m || !m->alive, c, + ptr_to_missing_stripe, + "pointer to nonexistent stripe %llu\n" + "while marking %s", + (u64) p.ec.idx, + (printbuf_reset(&buf), + bch2_bkey_val_to_text(&buf, c, k), buf.buf))) + do_update = true; + + if (fsck_err_on(m && m->alive && !bch2_ptr_matches_stripe_m(m, p), c, + ptr_to_incorrect_stripe, + "pointer does not match stripe %llu\n" + "while marking %s", + (u64) p.ec.idx, + (printbuf_reset(&buf), + bch2_bkey_val_to_text(&buf, c, k), buf.buf))) + do_update = true; + } + } + + if (do_update) { + if (flags & BTREE_TRIGGER_is_root) { + bch_err(c, "cannot update btree roots yet"); + ret = -EINVAL; + goto err; + } + + struct bkey_i *new = bch2_bkey_make_mut_noupdate(trans, k); + ret = PTR_ERR_OR_ZERO(new); + if (ret) + goto err; + + if (level) { + /* + * We don't want to drop btree node pointers - if the + * btree node isn't there anymore, the read path will + * sort it out: + */ + struct bkey_ptrs ptrs = bch2_bkey_ptrs(bkey_i_to_s(new)); + bkey_for_each_ptr(ptrs, ptr) { + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); + struct bucket *g = PTR_GC_BUCKET(ca, ptr); + + ptr->gen = g->gen; + } + } else { + struct bkey_ptrs ptrs; + union bch_extent_entry *entry; +restart_drop_ptrs: + ptrs = bch2_bkey_ptrs(bkey_i_to_s(new)); + bkey_for_each_ptr_decode(bkey_i_to_s(new).k, ptrs, p, entry) { + struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); + struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr); + enum bch_data_type data_type = bch2_bkey_ptr_data_type(bkey_i_to_s_c(new), p, entry); + + if ((p.ptr.cached && + (!g->gen_valid || gen_cmp(p.ptr.gen, g->gen) > 0)) || + (!p.ptr.cached && + gen_cmp(p.ptr.gen, g->gen) < 0) || + gen_cmp(g->gen, p.ptr.gen) > BUCKET_GC_GEN_MAX || + (g->data_type && + g->data_type != data_type)) { + bch2_bkey_drop_ptr(bkey_i_to_s(new), &entry->ptr); + goto restart_drop_ptrs; + } + } +again: + ptrs = bch2_bkey_ptrs(bkey_i_to_s(new)); + bkey_extent_entry_for_each(ptrs, entry) { + if (extent_entry_type(entry) == BCH_EXTENT_ENTRY_stripe_ptr) { + struct gc_stripe *m = genradix_ptr(&c->gc_stripes, + entry->stripe_ptr.idx); + union bch_extent_entry *next_ptr; + + bkey_extent_entry_for_each_from(ptrs, next_ptr, entry) + if (extent_entry_type(next_ptr) == BCH_EXTENT_ENTRY_ptr) + goto found; + next_ptr = NULL; +found: + if (!next_ptr) { + bch_err(c, "aieee, found stripe ptr with no data ptr"); + continue; + } + + if (!m || !m->alive || + !__bch2_ptr_matches_stripe(&m->ptrs[entry->stripe_ptr.block], + &next_ptr->ptr, + m->sectors)) { + bch2_bkey_extent_entry_drop(new, entry); + goto again; + } + } + } + } + + if (0) { + printbuf_reset(&buf); + bch2_bkey_val_to_text(&buf, c, k); + bch_info(c, "updated %s", buf.buf); + + printbuf_reset(&buf); + bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(new)); + bch_info(c, "new key %s", buf.buf); + } + + percpu_up_read(&c->mark_lock); + struct btree_iter iter; + bch2_trans_node_iter_init(trans, &iter, btree, new->k.p, 0, level, + BTREE_ITER_intent|BTREE_ITER_all_snapshots); + ret = bch2_btree_iter_traverse(&iter) ?: + bch2_trans_update(trans, &iter, new, + BTREE_UPDATE_internal_snapshot_node| + BTREE_TRIGGER_norun); + bch2_trans_iter_exit(trans, &iter); + percpu_down_read(&c->mark_lock); + + if (ret) + goto err; + + if (level) + bch2_btree_node_update_key_early(trans, btree, level - 1, k, new); + } +err: +fsck_err: + percpu_up_read(&c->mark_lock); + printbuf_exit(&buf); + return ret; +} + int bch2_bucket_ref_update(struct btree_trans *trans, struct bkey_s_c k, const struct bch_extent_ptr *ptr, @@ -970,7 +1205,7 @@ static int __trigger_extent(struct btree_trans *trans, } int bch2_trigger_extent(struct btree_trans *trans, - enum btree_id btree_id, unsigned level, + enum btree_id btree, unsigned level, struct bkey_s_c old, struct bkey_s new, enum btree_iter_update_trigger_flags flags) { @@ -979,6 +1214,9 @@ int bch2_trigger_extent(struct btree_trans *trans, unsigned new_ptrs_bytes = (void *) new_ptrs.end - (void *) new_ptrs.start; unsigned old_ptrs_bytes = (void *) old_ptrs.end - (void *) old_ptrs.start; + if (unlikely(flags & BTREE_TRIGGER_check_repair)) + return bch2_check_fix_ptrs(trans, btree, level, new.s_c, flags); + /* if pointers aren't changing - nothing to do: */ if (new_ptrs_bytes == old_ptrs_bytes && !memcmp(new_ptrs.start, @@ -1000,7 +1238,7 @@ int bch2_trigger_extent(struct btree_trans *trans, } if (flags & (BTREE_TRIGGER_transactional|BTREE_TRIGGER_gc)) - return trigger_run_overwrite_then_insert(__trigger_extent, trans, btree_id, level, old, new, flags); + return trigger_run_overwrite_then_insert(__trigger_extent, trans, btree, level, old, new, flags); return 0; } diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h index c2c2ac36bee5b..df10f390101a2 100644 --- a/fs/bcachefs/buckets.h +++ b/fs/bcachefs/buckets.h @@ -341,6 +341,10 @@ int bch2_bucket_ref_update(struct btree_trans *, struct bkey_s_c, const struct bch_extent_ptr *, s64, enum bch_data_type, u8, u8, u32 *); +int bch2_check_fix_ptrs(struct btree_trans *, + enum btree_id, unsigned, struct bkey_s_c, + enum btree_iter_update_trigger_flags); + int bch2_trigger_extent(struct btree_trans *, enum btree_id, unsigned, struct bkey_s_c, struct bkey_s, enum btree_iter_update_trigger_flags); diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 31db3c8fce50b..d6fa7e0bb2666 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -369,7 +369,7 @@ static int mark_stripe_buckets(struct btree_trans *trans, } int bch2_trigger_stripe(struct btree_trans *trans, - enum btree_id btree_id, unsigned level, + enum btree_id btree, unsigned level, struct bkey_s_c old, struct bkey_s _new, enum btree_iter_update_trigger_flags flags) { @@ -381,6 +381,9 @@ int bch2_trigger_stripe(struct btree_trans *trans, const struct bch_stripe *new_s = new.k->type == KEY_TYPE_stripe ? bkey_s_c_to_stripe(new).v : NULL; + if (unlikely(flags & BTREE_TRIGGER_check_repair)) + return bch2_check_fix_ptrs(trans, btree, level, _new.s_c, flags); + if (flags & BTREE_TRIGGER_transactional) { /* * If the pointers aren't changing, we don't need to do anything: From c451986bf4c64e1f21932117ec6ade269ca825db Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 7 Apr 2024 19:47:31 -0400 Subject: [PATCH 1419/1702] bcachefs: do reflink_p repair from BTREE_TRIGGER_check_repair Signed-off-by: Kent Overstreet --- fs/bcachefs/reflink.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/reflink.c b/fs/bcachefs/reflink.c index a0cf5d97513f1..a8a761d59a5da 100644 --- a/fs/bcachefs/reflink.c +++ b/fs/bcachefs/reflink.c @@ -164,10 +164,13 @@ static s64 gc_trigger_reflink_p_segment(struct btree_trans *trans, BUG_ON((s64) r->refcount + add < 0); - r->refcount += add; + if (flags & BTREE_TRIGGER_gc) + r->refcount += add; *idx = r->offset; return 0; not_found: + BUG_ON(!(flags & BTREE_TRIGGER_check_repair)); + if (fsck_err(c, reflink_p_to_missing_reflink_v, "pointer to missing indirect extent\n" " %s\n" @@ -216,7 +219,7 @@ static int __trigger_reflink_p(struct btree_trans *trans, ret = trans_trigger_reflink_p_segment(trans, p, &idx, flags); } - if (flags & BTREE_TRIGGER_gc) { + if (flags & (BTREE_TRIGGER_check_repair|BTREE_TRIGGER_gc)) { size_t l = 0, r = c->reflink_gc_nr; while (l < r) { From 24b27975a9866f32abb46b74834e963700624fcd Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 6 Apr 2024 23:58:01 -0400 Subject: [PATCH 1420/1702] bcachefs: Kill gc_init_recurse() This unifies the online and offline btree gc passes; we're not yet running it online. We now iterate over one level of the btree at a time - the same as check_extents_to_backpointers(); this ordering preserves order of keys regardless of btree splits and merges, which will be important when we re-enable online gc. Signed-off-by: Kent Overstreet --- fs/bcachefs/bcachefs.h | 2 +- fs/bcachefs/btree_gc.c | 181 ++++++++++------------------------------- fs/bcachefs/btree_gc.h | 30 ++----- 3 files changed, 55 insertions(+), 158 deletions(-) diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index 2345e090d8a1f..82544f826c588 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -519,8 +519,8 @@ enum gc_phase { struct gc_pos { enum gc_phase phase; + u16 level; struct bpos pos; - unsigned level; }; struct reflink_gc { diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index e30fa3c47553b..9694ffb0b098a 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -63,7 +63,7 @@ static inline void __gc_pos_set(struct bch_fs *c, struct gc_pos new_pos) static inline void gc_pos_set(struct bch_fs *c, struct gc_pos new_pos) { - BUG_ON(gc_pos_cmp(new_pos, c->gc_pos) <= 0); + BUG_ON(gc_pos_cmp(new_pos, c->gc_pos) < 0); __gc_pos_set(c, new_pos); } @@ -554,11 +554,24 @@ int bch2_check_topology(struct bch_fs *c) /* marking of btree keys/nodes: */ static int bch2_gc_mark_key(struct btree_trans *trans, enum btree_id btree_id, - unsigned level, bool is_root, - struct bkey_s_c k, + unsigned level, struct btree **prev, + struct btree_iter *iter, struct bkey_s_c k, bool initial) { struct bch_fs *c = trans->c; + + if (iter) { + struct btree_path *path = btree_iter_path(trans, iter); + struct btree *b = path_l(path)->b; + + if (*prev != b) { + int ret = bch2_btree_node_check_topology(trans, b); + if (ret) + return ret; + } + *prev = b; + } + struct bkey deleted = KEY(0, 0, 0); struct bkey_s_c old = (struct bkey_s_c) { &deleted, NULL }; struct printbuf buf = PRINTBUF; @@ -594,7 +607,7 @@ static int bch2_gc_mark_key(struct btree_trans *trans, enum btree_id btree_id, * key_trigger(BTREE_TRIGGER_GC) is not idempotant; we'll calculate the * wrong result if we run it multiple times. */ - unsigned flags = is_root ? BTREE_TRIGGER_is_root : 0; + unsigned flags = !iter ? BTREE_TRIGGER_is_root : 0; ret = bch2_key_trigger(trans, btree_id, level, old, unsafe_bkey_s_c_to_s(k), BTREE_TRIGGER_check_repair|flags); @@ -616,151 +629,55 @@ static int bch2_gc_mark_key(struct btree_trans *trans, enum btree_id btree_id, return ret; } -static int btree_gc_mark_node(struct btree_trans *trans, struct btree *b, bool initial) -{ - struct btree_and_journal_iter iter; - struct bkey_s_c k; - int ret = 0; - - ret = bch2_btree_node_check_topology(trans, b); - if (ret) - return ret; - - bch2_btree_and_journal_iter_init_node_iter(trans, &iter, b); - - while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) { - bch2_trans_begin(trans); - ret = bch2_gc_mark_key(trans, b->c.btree_id, b->c.level, false, k, initial); - if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) { - ret = 0; - continue; - } - if (ret) - break; - - bch2_btree_and_journal_iter_advance(&iter); - } - - bch2_btree_and_journal_iter_exit(&iter); - return ret; -} - -static int bch2_gc_btree(struct btree_trans *trans, enum btree_id btree_id, - bool initial) +static int bch2_gc_btree(struct btree_trans *trans, enum btree_id btree, bool initial) { struct bch_fs *c = trans->c; - struct btree_iter iter; - struct btree *b; - unsigned target_depth = btree_type_has_ptrs(btree_id) ? 0 : 1; + int level = 0, target_depth = btree_node_type_needs_gc(__btree_node_type(0, btree)) ? 0 : 1; int ret = 0; - gc_pos_set(c, gc_pos_btree(btree_id, POS_MIN, 0)); - - __for_each_btree_node(trans, iter, btree_id, POS_MIN, - 0, target_depth, BTREE_ITER_prefetch, b, ret) { - bch2_verify_btree_nr_keys(b); - - gc_pos_set(c, gc_pos_btree_node(b)); - - ret = btree_gc_mark_node(trans, b, initial); - if (ret) - break; - } - bch2_trans_iter_exit(trans, &iter); - - if (ret) - return ret; + /* We need to make sure every leaf node is readable before going RW */ + if (initial) + target_depth = 0; + /* root */ mutex_lock(&c->btree_root_lock); - b = bch2_btree_id_root(c, btree_id)->b; - if (!btree_node_fake(b)) + struct btree *b = bch2_btree_id_root(c, btree)->b; + if (!btree_node_fake(b)) { + gc_pos_set(c, gc_pos_btree(btree, b->c.level + 1, SPOS_MAX)); ret = lockrestart_do(trans, bch2_gc_mark_key(trans, b->c.btree_id, b->c.level + 1, - true, bkey_i_to_s_c(&b->key), initial)); - gc_pos_set(c, gc_pos_btree_root(b->c.btree_id)); + NULL, NULL, bkey_i_to_s_c(&b->key), initial)); + level = b->c.level; + } mutex_unlock(&c->btree_root_lock); - return ret; -} - -static int bch2_gc_btree_init_recurse(struct btree_trans *trans, struct btree *b, - unsigned target_depth) -{ - int ret = btree_gc_mark_node(trans, b, true); if (ret) return ret; - if (b->c.level > target_depth) { - struct bch_fs *c = trans->c; - struct btree_and_journal_iter iter; - struct bkey_s_c k; - struct bkey_buf cur; - - bch2_bkey_buf_init(&cur); - bch2_btree_and_journal_iter_init_node_iter(trans, &iter, b); - iter.prefetch = true; - - while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) { - bch2_bkey_buf_reassemble(&cur, c, k); - bch2_btree_and_journal_iter_advance(&iter); - - struct btree *child = - bch2_btree_node_get_noiter(trans, cur.k, - b->c.btree_id, b->c.level - 1, - false); - ret = PTR_ERR_OR_ZERO(child); - bch_err_msg(c, ret, "getting btree node"); - if (ret) - break; - - ret = bch2_gc_btree_init_recurse(trans, child, target_depth); - six_unlock_read(&child->c.lock); + for (; level >= target_depth; --level) { + struct btree *prev = NULL; + struct btree_iter iter; + bch2_trans_node_iter_init(trans, &iter, btree, POS_MIN, 0, level, + BTREE_ITER_prefetch); - if (ret) - break; - } - - bch2_bkey_buf_exit(&cur, c); - bch2_btree_and_journal_iter_exit(&iter); + ret = for_each_btree_key_continue(trans, iter, 0, k, ({ + gc_pos_set(c, gc_pos_btree(btree, level, k.k->p)); + bch2_gc_mark_key(trans, btree, level, &prev, &iter, k, initial); + })); + if (ret) + break; } return ret; } -static int bch2_gc_btree_init(struct btree_trans *trans, - enum btree_id btree_id) -{ - struct bch_fs *c = trans->c; - /* - * We need to make sure every leaf node is readable before going RW - unsigned target_depth = btree_node_type_needs_gc(__btree_node_type(0, btree_id)) ? 0 : 1; - */ - unsigned target_depth = 0; - int ret = 0; - - struct btree *b = bch2_btree_id_root(c, btree_id)->b; - - six_lock_read(&b->c.lock, NULL, NULL); - if (b->c.level >= target_depth) - ret = bch2_gc_btree_init_recurse(trans, b, target_depth); - - if (!ret) - ret = lockrestart_do(trans, - bch2_gc_mark_key(trans, b->c.btree_id, b->c.level + 1, true, - bkey_i_to_s_c(&b->key), true)); - six_unlock_read(&b->c.lock); - - bch_err_fn(c, ret); - return ret; -} - static inline int btree_id_gc_phase_cmp(enum btree_id l, enum btree_id r) { return (int) btree_id_to_gc_phase(l) - (int) btree_id_to_gc_phase(r); } -static int bch2_gc_btrees(struct bch_fs *c, bool initial) +static int bch2_gc_btrees(struct bch_fs *c) { struct btree_trans *trans = bch2_trans_get(c); enum btree_id ids[BTREE_ID_NR]; @@ -777,9 +694,7 @@ static int bch2_gc_btrees(struct bch_fs *c, bool initial) if (IS_ERR_OR_NULL(bch2_btree_id_root(c, btree)->b)) continue; - ret = initial - ? bch2_gc_btree_init(trans, btree) - : bch2_gc_btree(trans, btree, initial); + ret = bch2_gc_btree(trans, btree, true); if (mustfix_fsck_err_on(bch2_err_matches(ret, EIO), c, btree_node_read_error, @@ -1261,10 +1176,9 @@ static int bch2_gc_stripes_done(struct bch_fs *c) } /** - * bch2_gc - walk _all_ references to buckets, and recompute them: + * bch2_check_allocations - walk all references to buckets, and recompute them: * * @c: filesystem object - * @initial: are we in recovery? * * Returns: 0 on success, or standard errcode on failure * @@ -1283,7 +1197,7 @@ static int bch2_gc_stripes_done(struct bch_fs *c) * move around - if references move backwards in the ordering GC * uses, GC could skip past them */ -static int bch2_gc(struct bch_fs *c, bool initial) +int bch2_check_allocations(struct bch_fs *c) { int ret; @@ -1304,7 +1218,7 @@ static int bch2_gc(struct bch_fs *c, bool initial) ret = bch2_mark_superblocks(c); BUG_ON(ret); - ret = bch2_gc_btrees(c, initial); + ret = bch2_gc_btrees(c); if (ret) goto out; @@ -1337,11 +1251,6 @@ static int bch2_gc(struct bch_fs *c, bool initial) return ret; } -int bch2_check_allocations(struct bch_fs *c) -{ - return bch2_gc(c, true); -} - static int gc_btree_gens_key(struct btree_trans *trans, struct btree_iter *iter, struct bkey_s_c k) diff --git a/fs/bcachefs/btree_gc.h b/fs/bcachefs/btree_gc.h index 15315aab93bd8..1b6489d8e0f4f 100644 --- a/fs/bcachefs/btree_gc.h +++ b/fs/bcachefs/btree_gc.h @@ -34,16 +34,16 @@ static inline struct gc_pos gc_phase(enum gc_phase phase) { return (struct gc_pos) { .phase = phase, - .pos = POS_MIN, .level = 0, + .pos = POS_MIN, }; } static inline int gc_pos_cmp(struct gc_pos l, struct gc_pos r) { - return cmp_int(l.phase, r.phase) ?: - bpos_cmp(l.pos, r.pos) ?: - cmp_int(l.level, r.level); + return cmp_int(l.phase, r.phase) ?: + -cmp_int(l.level, r.level) ?: + bpos_cmp(l.pos, r.pos); } static inline enum gc_phase btree_id_to_gc_phase(enum btree_id id) @@ -57,13 +57,13 @@ static inline enum gc_phase btree_id_to_gc_phase(enum btree_id id) } } -static inline struct gc_pos gc_pos_btree(enum btree_id id, - struct bpos pos, unsigned level) +static inline struct gc_pos gc_pos_btree(enum btree_id btree, unsigned level, + struct bpos pos) { return (struct gc_pos) { - .phase = btree_id_to_gc_phase(id), - .pos = pos, + .phase = btree_id_to_gc_phase(btree), .level = level, + .pos = pos, }; } @@ -73,19 +73,7 @@ static inline struct gc_pos gc_pos_btree(enum btree_id id, */ static inline struct gc_pos gc_pos_btree_node(struct btree *b) { - return gc_pos_btree(b->c.btree_id, b->key.k.p, b->c.level); -} - -/* - * GC position of the pointer to a btree root: we don't use - * gc_pos_pointer_to_btree_node() here to avoid a potential race with - * btree_split() increasing the tree depth - the new root will have level > the - * old root and thus have a greater gc position than the old root, but that - * would be incorrect since once gc has marked the root it's not coming back. - */ -static inline struct gc_pos gc_pos_btree_root(enum btree_id id) -{ - return gc_pos_btree(id, SPOS_MAX, BTREE_MAX_DEPTH); + return gc_pos_btree(b->c.btree_id, b->c.level, b->key.k.p); } static inline bool gc_visited(struct bch_fs *c, struct gc_pos pos) From 2d288745ebc202d7ac4905cba4a540549dc8044e Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 23 Apr 2024 11:58:09 -0700 Subject: [PATCH 1421/1702] bcachefs: Fix type of flags parameter for some ->trigger() implementations When building with clang's -Wincompatible-function-pointer-types-strict (a warning designed to catch potential kCFI failures at build time), there are several warnings along the lines of: fs/bcachefs/bkey_methods.c:118:2: error: incompatible function pointer types initializing 'int (*)(struct btree_trans *, enum btree_id, unsigned int, struct bkey_s_c, struct bkey_s, enum btree_iter_update_trigger_flags)' with an expression of type 'int (struct btree_trans *, enum btree_id, unsigned int, struct bkey_s_c, struct bkey_s, unsigned int)' [-Werror,-Wincompatible-function-pointer-types-strict] 118 | BCH_BKEY_TYPES() | ^~~~~~~~~~~~~~~~ fs/bcachefs/bcachefs_format.h:394:2: note: expanded from macro 'BCH_BKEY_TYPES' 394 | x(inode, 8) \ | ^~~~~~~~~~~~~~~~~~~~~~~~~~ fs/bcachefs/bkey_methods.c:117:41: note: expanded from macro 'x' 117 | #define x(name, nr) [KEY_TYPE_##name] = bch2_bkey_ops_##name, | ^~~~~~~~~~~~~~~~~~~~ :277:1: note: expanded from here 277 | bch2_bkey_ops_inode | ^~~~~~~~~~~~~~~~~~~ fs/bcachefs/inode.h:26:13: note: expanded from macro 'bch2_bkey_ops_inode' 26 | .trigger = bch2_trigger_inode, \ | ^~~~~~~~~~~~~~~~~~ There are several functions that did not have their flags parameter converted to 'enum btree_iter_update_trigger_flags' in the recent unification, which will cause kCFI failures at runtime because the types, while ABI compatible (hence no warning from the non-strict version of this warning), do not match exactly. Fix up these functions (as well as a few other obvious functions that should have it, even if there are no warnings currently) to resolve the warnings and potential kCFI runtime failures. Fixes: 31e4ef3280c8 ("bcachefs: iter/update/trigger/str_hash flag cleanup") Signed-off-by: Nathan Chancellor Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 2 +- fs/bcachefs/alloc_background.h | 3 ++- fs/bcachefs/inode.c | 2 +- fs/bcachefs/inode.h | 3 ++- fs/bcachefs/reflink.c | 6 ++++-- fs/bcachefs/reflink.h | 2 +- fs/bcachefs/snapshot.c | 4 ++-- fs/bcachefs/snapshot.h | 3 ++- fs/bcachefs/subvolume.c | 2 +- fs/bcachefs/subvolume.h | 3 ++- 10 files changed, 18 insertions(+), 12 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 6fa51ee16cc33..63bcba995ca2d 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -724,7 +724,7 @@ static noinline int bch2_bucket_gen_update(struct btree_trans *trans, int bch2_trigger_alloc(struct btree_trans *trans, enum btree_id btree, unsigned level, struct bkey_s_c old, struct bkey_s new, - unsigned flags) + enum btree_iter_update_trigger_flags flags) { struct bch_fs *c = trans->c; int ret = 0; diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h index e94432ced6a66..e0e0d72db03bc 100644 --- a/fs/bcachefs/alloc_background.h +++ b/fs/bcachefs/alloc_background.h @@ -245,7 +245,8 @@ static inline bool bkey_is_alloc(const struct bkey *k) int bch2_alloc_read(struct bch_fs *); int bch2_trigger_alloc(struct btree_trans *, enum btree_id, unsigned, - struct bkey_s_c, struct bkey_s, unsigned); + struct bkey_s_c, struct bkey_s, + enum btree_iter_update_trigger_flags); int bch2_check_alloc_info(struct bch_fs *); int bch2_check_alloc_to_lru_refs(struct bch_fs *); void bch2_do_discards(struct bch_fs *); diff --git a/fs/bcachefs/inode.c b/fs/bcachefs/inode.c index fc8b30441bbc0..91572e05d5f06 100644 --- a/fs/bcachefs/inode.c +++ b/fs/bcachefs/inode.c @@ -594,7 +594,7 @@ int bch2_trigger_inode(struct btree_trans *trans, enum btree_id btree_id, unsigned level, struct bkey_s_c old, struct bkey_s new, - unsigned flags) + enum btree_iter_update_trigger_flags flags) { s64 nr = (s64) bkey_is_inode(new.k) - (s64) bkey_is_inode(old.k); diff --git a/fs/bcachefs/inode.h b/fs/bcachefs/inode.h index 08e8505dfa33c..46477a40846c4 100644 --- a/fs/bcachefs/inode.h +++ b/fs/bcachefs/inode.h @@ -18,7 +18,8 @@ int bch2_inode_v3_invalid(struct bch_fs *, struct bkey_s_c, void bch2_inode_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); int bch2_trigger_inode(struct btree_trans *, enum btree_id, unsigned, - struct bkey_s_c, struct bkey_s, unsigned); + struct bkey_s_c, struct bkey_s, + enum btree_iter_update_trigger_flags); #define bch2_bkey_ops_inode ((struct bkey_ops) { \ .key_invalid = bch2_inode_invalid, \ diff --git a/fs/bcachefs/reflink.c b/fs/bcachefs/reflink.c index a8a761d59a5da..fb31a68148ff9 100644 --- a/fs/bcachefs/reflink.c +++ b/fs/bcachefs/reflink.c @@ -285,7 +285,9 @@ bool bch2_reflink_v_merge(struct bch_fs *c, struct bkey_s _l, struct bkey_s_c _r } #endif -static inline void check_indirect_extent_deleting(struct bkey_s new, unsigned *flags) +static inline void +check_indirect_extent_deleting(struct bkey_s new, + enum btree_iter_update_trigger_flags *flags) { if ((*flags & BTREE_TRIGGER_insert) && !*bkey_refcount(new)) { new.k->type = KEY_TYPE_deleted; @@ -330,7 +332,7 @@ void bch2_indirect_inline_data_to_text(struct printbuf *out, int bch2_trigger_indirect_inline_data(struct btree_trans *trans, enum btree_id btree_id, unsigned level, struct bkey_s_c old, struct bkey_s new, - unsigned flags) + enum btree_iter_update_trigger_flags flags) { check_indirect_extent_deleting(new, &flags); diff --git a/fs/bcachefs/reflink.h b/fs/bcachefs/reflink.h index e34c4bdf7a76b..59d4dbae3fb1f 100644 --- a/fs/bcachefs/reflink.h +++ b/fs/bcachefs/reflink.h @@ -44,7 +44,7 @@ void bch2_indirect_inline_data_to_text(struct printbuf *, int bch2_trigger_indirect_inline_data(struct btree_trans *, enum btree_id, unsigned, struct bkey_s_c, struct bkey_s, - unsigned); + enum btree_iter_update_trigger_flags); #define bch2_bkey_ops_indirect_inline_data ((struct bkey_ops) { \ .key_invalid = bch2_indirect_inline_data_invalid, \ diff --git a/fs/bcachefs/snapshot.c b/fs/bcachefs/snapshot.c index 0b26dee17a5ab..2368218070d45 100644 --- a/fs/bcachefs/snapshot.c +++ b/fs/bcachefs/snapshot.c @@ -298,7 +298,7 @@ static void set_is_ancestor_bitmap(struct bch_fs *c, u32 id) static int __bch2_mark_snapshot(struct btree_trans *trans, enum btree_id btree, unsigned level, struct bkey_s_c old, struct bkey_s_c new, - unsigned flags) + enum btree_iter_update_trigger_flags flags) { struct bch_fs *c = trans->c; struct snapshot_t *t; @@ -352,7 +352,7 @@ static int __bch2_mark_snapshot(struct btree_trans *trans, int bch2_mark_snapshot(struct btree_trans *trans, enum btree_id btree, unsigned level, struct bkey_s_c old, struct bkey_s new, - unsigned flags) + enum btree_iter_update_trigger_flags flags) { return __bch2_mark_snapshot(trans, btree, level, old, new.s_c, flags); } diff --git a/fs/bcachefs/snapshot.h b/fs/bcachefs/snapshot.h index ccb1f309b8d16..b3c43d5795402 100644 --- a/fs/bcachefs/snapshot.h +++ b/fs/bcachefs/snapshot.h @@ -22,7 +22,8 @@ void bch2_snapshot_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); int bch2_snapshot_invalid(struct bch_fs *, struct bkey_s_c, enum bkey_invalid_flags, struct printbuf *); int bch2_mark_snapshot(struct btree_trans *, enum btree_id, unsigned, - struct bkey_s_c, struct bkey_s, unsigned); + struct bkey_s_c, struct bkey_s, + enum btree_iter_update_trigger_flags); #define bch2_bkey_ops_snapshot ((struct bkey_ops) { \ .key_invalid = bch2_snapshot_invalid, \ diff --git a/fs/bcachefs/subvolume.c b/fs/bcachefs/subvolume.c index 33292dae4d509..3341285e75317 100644 --- a/fs/bcachefs/subvolume.c +++ b/fs/bcachefs/subvolume.c @@ -245,7 +245,7 @@ static int subvolume_children_mod(struct btree_trans *trans, struct bpos pos, bo int bch2_subvolume_trigger(struct btree_trans *trans, enum btree_id btree_id, unsigned level, struct bkey_s_c old, struct bkey_s new, - unsigned flags) + enum btree_iter_update_trigger_flags flags) { if (flags & BTREE_TRIGGER_transactional) { struct bpos children_pos_old = subvolume_children_pos(old); diff --git a/fs/bcachefs/subvolume.h b/fs/bcachefs/subvolume.h index d2015d549bd2a..1530a10327ab3 100644 --- a/fs/bcachefs/subvolume.h +++ b/fs/bcachefs/subvolume.h @@ -14,7 +14,8 @@ int bch2_subvolume_invalid(struct bch_fs *, struct bkey_s_c, enum bkey_invalid_flags, struct printbuf *); void bch2_subvolume_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); int bch2_subvolume_trigger(struct btree_trans *, enum btree_id, unsigned, - struct bkey_s_c, struct bkey_s, unsigned); + struct bkey_s_c, struct bkey_s, + enum btree_iter_update_trigger_flags); #define bch2_bkey_ops_subvolume ((struct bkey_ops) { \ .key_invalid = bch2_subvolume_invalid, \ From 8bb0eddbbcc99be5551521a089662f3b5b5a3204 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 23 Apr 2024 13:12:27 -0700 Subject: [PATCH 1422/1702] bcachefs: Fix format specifiers in bch2_btree_key_cache_to_text() When building for a 32-bit target, for which 'size_t' is 'unsigned int', there are two warnings around mismatched format specifiers and argument types: In file included from fs/bcachefs/vstructs.h:5, from fs/bcachefs/bcachefs_format.h:79, from fs/bcachefs/bcachefs.h:207, from fs/bcachefs/btree_key_cache.c:3: fs/bcachefs/btree_key_cache.c: In function 'bch2_btree_key_cache_to_text': fs/bcachefs/btree_key_cache.c:1046:25: error: format '%lu' expects argument of type 'long unsigned int', but argument 3 has type 'size_t' {aka 'unsigned int'} [-Werror=format=] 1046 | prt_printf(out, "nonpcpu freelist:\t%lu\r\n", bc->nr_freed_nonpcpu); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~ | | | size_t {aka unsigned int} fs/bcachefs/util.h:192:63: note: in definition of macro 'prt_printf' 192 | #define prt_printf(_out, ...) bch2_prt_printf(_out, __VA_ARGS__) | ^~~~~~~~~~~ fs/bcachefs/btree_key_cache.c:1046:47: note: format string is defined here 1046 | prt_printf(out, "nonpcpu freelist:\t%lu\r\n", bc->nr_freed_nonpcpu); | ~~^ | | | long unsigned int | %u fs/bcachefs/btree_key_cache.c:1047:25: error: format '%lu' expects argument of type 'long unsigned int', but argument 3 has type 'size_t' {aka 'unsigned int'} [-Werror=format=] 1047 | prt_printf(out, "pcpu freelist:\t%lu\r\n", bc->nr_freed_pcpu); | ^~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~ | | | size_t {aka unsigned int} fs/bcachefs/util.h:192:63: note: in definition of macro 'prt_printf' 192 | #define prt_printf(_out, ...) bch2_prt_printf(_out, __VA_ARGS__) | ^~~~~~~~~~~ fs/bcachefs/btree_key_cache.c:1047:44: note: format string is defined here 1047 | prt_printf(out, "pcpu freelist:\t%lu\r\n", bc->nr_freed_pcpu); | ~~^ | | | long unsigned int | %u cc1: all warnings being treated as error Use the proper 'size_t' specifier, '%zu', to clear up the warnings for these platforms. Fixes: f2d47ec26af5 ("bcachefs: Btree key cache instrumentation") Signed-off-by: Nathan Chancellor Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_key_cache.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/btree_key_cache.c b/fs/bcachefs/btree_key_cache.c index 203fbb38e9d47..e2eca7dab70e6 100644 --- a/fs/bcachefs/btree_key_cache.c +++ b/fs/bcachefs/btree_key_cache.c @@ -1045,8 +1045,8 @@ void bch2_btree_key_cache_to_text(struct printbuf *out, struct btree_key_cache * prt_printf(out, "keys:\t%lu\r\n", atomic_long_read(&bc->nr_keys)); prt_printf(out, "dirty:\t%lu\r\n", atomic_long_read(&bc->nr_dirty)); prt_printf(out, "freelist:\t%lu\r\n", atomic_long_read(&bc->nr_freed)); - prt_printf(out, "nonpcpu freelist:\t%lu\r\n", bc->nr_freed_nonpcpu); - prt_printf(out, "pcpu freelist:\t%lu\r\n", bc->nr_freed_pcpu); + prt_printf(out, "nonpcpu freelist:\t%zu\r\n", bc->nr_freed_nonpcpu); + prt_printf(out, "pcpu freelist:\t%zu\r\n", bc->nr_freed_pcpu); prt_printf(out, "\nshrinker:\n"); prt_printf(out, "requested_to_free:\t%lu\r\n", bc->requested_to_free); From 56522d7276746bdeeea095883ba7022f0cb4daf0 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 25 Apr 2024 02:15:08 -0400 Subject: [PATCH 1423/1702] bcachefs: fix btree_path_clone() ip_allocated Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index 27f3935d67cb5..ea70e9efb91cd 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -1233,11 +1233,14 @@ static inline void btree_path_copy(struct btree_trans *trans, struct btree_path } static btree_path_idx_t btree_path_clone(struct btree_trans *trans, btree_path_idx_t src, - bool intent) + bool intent, unsigned long ip) { btree_path_idx_t new = btree_path_alloc(trans, src); btree_path_copy(trans, trans->paths + new, trans->paths + src); __btree_path_get(trans->paths + new, intent); +#ifdef TRACK_PATH_ALLOCATED + trans->paths[new].ip_allocated = ip; +#endif return new; } @@ -1246,7 +1249,7 @@ btree_path_idx_t __bch2_btree_path_make_mut(struct btree_trans *trans, btree_path_idx_t path, bool intent, unsigned long ip) { __btree_path_put(trans->paths + path, intent); - path = btree_path_clone(trans, path, intent); + path = btree_path_clone(trans, path, intent, ip); trans->paths[path].preserve = false; return path; } @@ -2491,7 +2494,8 @@ struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_iter *iter) bch2_path_put_nokeep(trans, saved_path, iter->flags & BTREE_ITER_intent); saved_path = btree_path_clone(trans, iter->path, - iter->flags & BTREE_ITER_intent); + iter->flags & BTREE_ITER_intent, + _THIS_IP_); path = btree_iter_path(trans, iter); saved_k = *k.k; saved_v = k.v; From 0af0b963b52b45513dcdabae9b77575befea738d Mon Sep 17 00:00:00 2001 From: Hongbo Li Date: Fri, 26 Apr 2024 11:21:35 +0800 Subject: [PATCH 1424/1702] bcachefs: eliminate the uninitialized compilation warning in bch2_reconstruct_snapshots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When compiling the bcachefs-tools, the following compilation warning is reported: libbcachefs/snapshot.c: In function ‘bch2_reconstruct_snapshots’: libbcachefs/snapshot.c:915:19: warning: ‘tree_id’ may be used uninitialized in this function [-Wmaybe-uninitialized] 915 | snapshot->v.tree = cpu_to_le32(tree_id); libbcachefs/snapshot.c:903:6: note: ‘tree_id’ was declared here 903 | u32 tree_id; | ^~~~~~~ This is a false alert, because @tree_id is changed in bch2_snapshot_tree_create after it returns 0. And if this function returns other value, @tree_id wouldn't be used. Thus there should be nothing wrong in logical. Although the report itself is a false alert, we can still make it more explicit by setting the initial value of @tree_id to 0 (an invalid tree ID). Fixes: a292be3b68f3 ("bcachefs: Reconstruct missing snapshot nodes") Signed-off-by: Hongbo Li Signed-off-by: Kent Overstreet --- fs/bcachefs/snapshot.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/bcachefs/snapshot.c b/fs/bcachefs/snapshot.c index 2368218070d45..3aa4686cc71fc 100644 --- a/fs/bcachefs/snapshot.c +++ b/fs/bcachefs/snapshot.c @@ -900,7 +900,8 @@ static int check_snapshot_exists(struct btree_trans *trans, u32 id) if (bch2_snapshot_equiv(c, id)) return 0; - u32 tree_id; + /* 0 is an invalid tree ID */ + u32 tree_id = 0; int ret = bch2_snapshot_tree_create(trans, id, 0, &tree_id); if (ret) return ret; From c74954135353fac7c206ec15d71c95f981fd0e9d Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 25 Apr 2024 02:20:20 -0400 Subject: [PATCH 1425/1702] bcachefs: uninline set_btree_iter_dontneed() Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 2 +- fs/bcachefs/alloc_foreground.c | 8 ++++---- fs/bcachefs/btree_iter.c | 13 +++++++++++++ fs/bcachefs/btree_iter.h | 13 +------------ fs/bcachefs/btree_key_cache.c | 2 +- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 63bcba995ca2d..37e8446b30395 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -1303,7 +1303,7 @@ static noinline_for_stack int bch2_check_discard_freespace_key(struct btree_tran goto delete; out: fsck_err: - set_btree_iter_dontneed(&alloc_iter); + bch2_set_btree_iter_dontneed(&alloc_iter); bch2_trans_iter_exit(trans, &alloc_iter); printbuf_exit(&buf); return ret; diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index 847000a7ead2b..438e00c8316bd 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -363,10 +363,10 @@ static struct open_bucket *try_alloc_bucket(struct btree_trans *trans, struct bc ob = __try_alloc_bucket(c, ca, b, watermark, a, s, cl); if (!ob) - set_btree_iter_dontneed(&iter); + bch2_set_btree_iter_dontneed(&iter); err: if (iter.path) - set_btree_iter_dontneed(&iter); + bch2_set_btree_iter_dontneed(&iter); bch2_trans_iter_exit(trans, &iter); printbuf_exit(&buf); return ob; @@ -433,7 +433,7 @@ bch2_bucket_alloc_early(struct btree_trans *trans, ob = __try_alloc_bucket(trans->c, ca, k.k->p.offset, watermark, a, s, cl); next: - set_btree_iter_dontneed(&citer); + bch2_set_btree_iter_dontneed(&citer); bch2_trans_iter_exit(trans, &citer); if (ob) break; @@ -488,7 +488,7 @@ static struct open_bucket *bch2_bucket_alloc_freelist(struct btree_trans *trans, ob = try_alloc_bucket(trans, ca, watermark, alloc_cursor, s, k, cl); if (ob) { - set_btree_iter_dontneed(&iter); + bch2_set_btree_iter_dontneed(&iter); break; } } diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index ea70e9efb91cd..7f0eb5d396a23 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -1822,6 +1822,19 @@ struct bkey_s_c bch2_btree_path_peek_slot(struct btree_path *path, struct bkey * return (struct bkey_s_c) { u, NULL }; } + +void bch2_set_btree_iter_dontneed(struct btree_iter *iter) +{ + struct btree_trans *trans = iter->trans; + + if (!iter->path || trans->restarted) + return; + + struct btree_path *path = btree_iter_path(trans, iter); + path->preserve = false; + if (path->ref == 1) + path->should_be_locked = false; +} /* Btree iterators: */ int __must_check diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index 430b0d4848744..eab2a25bdc7a2 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -508,18 +508,7 @@ void bch2_trans_node_iter_init(struct btree_trans *, struct btree_iter *, unsigned, unsigned, unsigned); void bch2_trans_copy_iter(struct btree_iter *, struct btree_iter *); -static inline void set_btree_iter_dontneed(struct btree_iter *iter) -{ - struct btree_trans *trans = iter->trans; - - if (!iter->path || trans->restarted) - return; - - struct btree_path *path = btree_iter_path(trans, iter); - path->preserve = false; - if (path->ref == 1) - path->should_be_locked = false; -} +void bch2_set_btree_iter_dontneed(struct btree_iter *); void *__bch2_trans_kmalloc(struct btree_trans *, size_t); diff --git a/fs/bcachefs/btree_key_cache.c b/fs/bcachefs/btree_key_cache.c index e2eca7dab70e6..69e5f08db374c 100644 --- a/fs/bcachefs/btree_key_cache.c +++ b/fs/bcachefs/btree_key_cache.c @@ -456,7 +456,7 @@ static int btree_key_cache_fill(struct btree_trans *trans, bch2_btree_node_unlock_write(trans, ck_path, ck_path->l[0].b); /* We're not likely to need this iterator again: */ - set_btree_iter_dontneed(&iter); + bch2_set_btree_iter_dontneed(&iter); err: bch2_trans_iter_exit(trans, &iter); return ret; From 45150765d307c1043baa68aff53cb6fa5ba34603 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 26 Apr 2024 00:32:56 -0400 Subject: [PATCH 1426/1702] bcachefs: bch_member.last_journal_bucket On recovery from clean shutdown we don't typically read the journal, but we still want to avoid overwriting existing entries in the journal for list_journal debugging. Thus, add some fields to the member info section so we can remember where we left off. Signed-off-by: Kent Overstreet --- fs/bcachefs/bcachefs_format.h | 7 +++++++ fs/bcachefs/journal_io.c | 28 ++++++++++++++++++++++++++++ fs/bcachefs/journal_io.h | 3 +++ fs/bcachefs/recovery.c | 2 ++ fs/bcachefs/sb-clean.c | 2 ++ 5 files changed, 42 insertions(+) diff --git a/fs/bcachefs/bcachefs_format.h b/fs/bcachefs/bcachefs_format.h index 2e8b1a489c209..d5b90439e581f 100644 --- a/fs/bcachefs/bcachefs_format.h +++ b/fs/bcachefs/bcachefs_format.h @@ -589,6 +589,13 @@ struct bch_member { __le64 errors_reset_time; __le64 seq; __le64 btree_allocated_bitmap; + /* + * On recovery from a clean shutdown we don't normally read the journal, + * but we still want to resume writing from where we left off so we + * don't overwrite more than is necessary, for list journal debugging: + */ + __le32 last_journal_bucket; + __le32 last_journal_bucket_offset; }; /* diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c index 98cf9a65216f6..f3c5e9a423fd1 100644 --- a/fs/bcachefs/journal_io.c +++ b/fs/bcachefs/journal_io.c @@ -17,6 +17,34 @@ #include "sb-clean.h" #include "trace.h" +void bch2_journal_pos_from_member_info_set(struct bch_fs *c) +{ + lockdep_assert_held(&c->sb_lock); + + for_each_member_device(c, ca) { + struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx); + + m->last_journal_bucket = cpu_to_le32(ca->journal.cur_idx); + m->last_journal_bucket_offset = cpu_to_le32(ca->mi.bucket_size - ca->journal.sectors_free); + } +} + +void bch2_journal_pos_from_member_info_resume(struct bch_fs *c) +{ + mutex_lock(&c->sb_lock); + for_each_member_device(c, ca) { + struct bch_member m = bch2_sb_member_get(c->disk_sb.sb, ca->dev_idx); + + unsigned idx = le32_to_cpu(m.last_journal_bucket); + if (idx < ca->journal.nr) + ca->journal.cur_idx = idx; + unsigned offset = le32_to_cpu(m.last_journal_bucket_offset); + if (offset <= ca->mi.bucket_size) + ca->journal.sectors_free = ca->mi.bucket_size - offset; + } + mutex_unlock(&c->sb_lock); +} + void bch2_journal_ptrs_to_text(struct printbuf *out, struct bch_fs *c, struct journal_replay *j) { diff --git a/fs/bcachefs/journal_io.h b/fs/bcachefs/journal_io.h index 4f1e763ab5060..43cfed58ef33f 100644 --- a/fs/bcachefs/journal_io.h +++ b/fs/bcachefs/journal_io.h @@ -4,6 +4,9 @@ #include "darray.h" +void bch2_journal_pos_from_member_info_set(struct bch_fs *); +void bch2_journal_pos_from_member_info_resume(struct bch_fs *); + struct journal_ptr { bool csum_good; u8 dev; diff --git a/fs/bcachefs/recovery.c b/fs/bcachefs/recovery.c index c33e20aa56a28..a6447ffd336e3 100644 --- a/fs/bcachefs/recovery.c +++ b/fs/bcachefs/recovery.c @@ -670,6 +670,8 @@ int bch2_fs_recovery(struct bch_fs *c) goto err; } + bch2_journal_pos_from_member_info_resume(c); + if (!c->sb.clean || c->opts.retain_recovery_info) { struct genradix_iter iter; struct journal_replay **i; diff --git a/fs/bcachefs/sb-clean.c b/fs/bcachefs/sb-clean.c index c2d4be52c8c52..e2beb2308c70c 100644 --- a/fs/bcachefs/sb-clean.c +++ b/fs/bcachefs/sb-clean.c @@ -390,6 +390,8 @@ void bch2_fs_mark_clean(struct bch_fs *c) goto out; } + bch2_journal_pos_from_member_info_set(c); + bch2_write_super(c); out: mutex_unlock(&c->sb_lock); From 4da1713a8db5b6d5e421b20687ecde9354ee85cf Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 25 Apr 2024 22:11:49 -0400 Subject: [PATCH 1427/1702] bcachefs: check for inodes that should have backpointers in fsck Signed-off-by: Kent Overstreet --- fs/bcachefs/fsck.c | 10 ++++++++++ fs/bcachefs/inode.h | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/fs/bcachefs/fsck.c b/fs/bcachefs/fsck.c index 4adb96965e111..c8f57465131c5 100644 --- a/fs/bcachefs/fsck.c +++ b/fs/bcachefs/fsck.c @@ -1713,6 +1713,15 @@ static int check_dirent_inode_dirent(struct btree_trans *trans, if (inode_points_to_dirent(target, d)) return 0; + if (bch2_inode_should_have_bp(target) && + !fsck_err(c, inode_wrong_backpointer, + "dirent points to inode that does not point back:\n %s", + (bch2_bkey_val_to_text(&buf, c, d.s_c), + prt_printf(&buf, "\n "), + bch2_inode_unpacked_to_text(&buf, target), + buf.buf))) + goto out_noiter; + if (!target->bi_dir && !target->bi_dir_offset) { target->bi_dir = d.k->p.inode; @@ -1781,6 +1790,7 @@ static int check_dirent_inode_dirent(struct btree_trans *trans, err: fsck_err: bch2_trans_iter_exit(trans, &bp_iter); +out_noiter: printbuf_exit(&buf); bch_err_fn(c, ret); return ret; diff --git a/fs/bcachefs/inode.h b/fs/bcachefs/inode.h index 46477a40846c4..abdfb1f3e4d08 100644 --- a/fs/bcachefs/inode.h +++ b/fs/bcachefs/inode.h @@ -221,6 +221,14 @@ static inline void bch2_inode_nlink_set(struct bch_inode_unpacked *bi, int bch2_inode_nlink_inc(struct bch_inode_unpacked *); void bch2_inode_nlink_dec(struct btree_trans *, struct bch_inode_unpacked *); +static inline bool bch2_inode_should_have_bp(struct bch_inode_unpacked *inode) +{ + bool inode_has_bp = inode->bi_dir || inode->bi_dir_offset; + + return S_ISDIR(inode->bi_mode) || + (!inode->bi_nlink && inode_has_bp); +} + struct bch_opts bch2_inode_opts_to_opts(struct bch_inode_unpacked *); void bch2_inode_opts_get(struct bch_io_opts *, struct bch_fs *, struct bch_inode_unpacked *); From 83c38e3ef8214cb5b80c113ac39e855548f79a44 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 25 Apr 2024 19:54:03 -0400 Subject: [PATCH 1428/1702] bcachefs: check inode backpointer in bch2_lookup() Signed-off-by: Kent Overstreet --- fs/bcachefs/fs.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c index 5624c2ea8d55a..138d416a64d5a 100644 --- a/fs/bcachefs/fs.c +++ b/fs/bcachefs/fs.c @@ -400,6 +400,7 @@ static struct bch_inode_info *bch2_lookup_trans(struct btree_trans *trans, struct bch_fs *c = trans->c; struct btree_iter dirent_iter = {}; subvol_inum inum = {}; + struct printbuf buf = PRINTBUF; struct bkey_s_c k = bch2_hash_lookup(trans, &dirent_iter, bch2_dirent_hash_desc, dir_hash_info, dir, name, 0); @@ -426,20 +427,31 @@ static struct bch_inode_info *bch2_lookup_trans(struct btree_trans *trans, ret = bch2_subvolume_get(trans, inum.subvol, true, 0, &subvol) ?: bch2_inode_find_by_inum_nowarn_trans(trans, inum, &inode_u) ?: PTR_ERR_OR_ZERO(inode = bch2_new_inode(trans)); - if (bch2_err_matches(ret, ENOENT)) { - struct printbuf buf = PRINTBUF; - bch2_bkey_val_to_text(&buf, c, k); - bch_err(c, "%s points to missing inode", buf.buf); - printbuf_exit(&buf); - } + bch2_fs_inconsistent_on(bch2_err_matches(ret, ENOENT), + c, "dirent to missing inode:\n %s", + (bch2_bkey_val_to_text(&buf, c, k), buf.buf)); if (ret) goto err; + /* regular files may have hardlinks: */ + if (bch2_fs_inconsistent_on(bch2_inode_should_have_bp(&inode_u) && + !bkey_eq(k.k->p, POS(inode_u.bi_dir, inode_u.bi_dir_offset)), + c, + "dirent points to inode that does not point back:\n %s", + (bch2_bkey_val_to_text(&buf, c, k), + prt_printf(&buf, "\n "), + bch2_inode_unpacked_to_text(&buf, &inode_u), + buf.buf))) { + ret = -ENOENT; + goto err; + } + bch2_vfs_inode_init(trans, inum, inode, &inode_u, &subvol); inode = bch2_inode_insert(c, inode); out: bch2_trans_iter_exit(trans, &dirent_iter); + printbuf_exit(&buf); return inode; err: inode = ERR_PTR(ret); From c8bda9f20a01cef7e12fd230ada83d53f7cdc884 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 26 Apr 2024 00:31:14 -0400 Subject: [PATCH 1429/1702] bcachefs: Simplify resuming of journal position Signed-off-by: Kent Overstreet --- fs/bcachefs/journal_io.c | 50 ++++++------------------------------- fs/bcachefs/journal_types.h | 1 + 2 files changed, 8 insertions(+), 43 deletions(-) diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c index f3c5e9a423fd1..dec50e7a8cb41 100644 --- a/fs/bcachefs/journal_io.c +++ b/fs/bcachefs/journal_io.c @@ -1089,6 +1089,13 @@ static int journal_read_bucket(struct bch_dev *ca, goto err; } + if (le64_to_cpu(j->seq) > ja->highest_seq_found) { + ja->highest_seq_found = le64_to_cpu(j->seq); + ja->cur_idx = bucket; + ja->sectors_free = ca->mi.bucket_size - + bucket_remainder(ca, offset) - sectors; + } + /* * This happens sometimes if we don't have discards on - * when we've partially overwritten a bucket with new @@ -1157,8 +1164,6 @@ static CLOSURE_CALLBACK(bch2_journal_read_device) struct bch_fs *c = ca->fs; struct journal_list *jlist = container_of(cl->parent, struct journal_list, cl); - struct journal_replay *r, **_r; - struct genradix_iter iter; struct journal_read_buf buf = { NULL, 0 }; unsigned i; int ret = 0; @@ -1178,47 +1183,6 @@ static CLOSURE_CALLBACK(bch2_journal_read_device) goto err; } - ja->sectors_free = ca->mi.bucket_size; - - mutex_lock(&jlist->lock); - genradix_for_each_reverse(&c->journal_entries, iter, _r) { - r = *_r; - - if (!r) - continue; - - darray_for_each(r->ptrs, i) - if (i->dev == ca->dev_idx) { - unsigned wrote = bucket_remainder(ca, i->sector) + - vstruct_sectors(&r->j, c->block_bits); - - ja->cur_idx = i->bucket; - ja->sectors_free = ca->mi.bucket_size - wrote; - goto found; - } - } -found: - mutex_unlock(&jlist->lock); - - if (ja->bucket_seq[ja->cur_idx] && - ja->sectors_free == ca->mi.bucket_size) { -#if 0 - /* - * Debug code for ZNS support, where we (probably) want to be - * correlated where we stopped in the journal to the zone write - * points: - */ - bch_err(c, "ja->sectors_free == ca->mi.bucket_size"); - bch_err(c, "cur_idx %u/%u", ja->cur_idx, ja->nr); - for (i = 0; i < 3; i++) { - unsigned idx = (ja->cur_idx + ja->nr - 1 + i) % ja->nr; - - bch_err(c, "bucket_seq[%u] = %llu", idx, ja->bucket_seq[idx]); - } -#endif - ja->sectors_free = 0; - } - /* * Set dirty_idx to indicate the entire journal is full and needs to be * reclaimed - journal reclaim will immediately reclaim whatever isn't diff --git a/fs/bcachefs/journal_types.h b/fs/bcachefs/journal_types.h index 3d2135e1d7a1c..45c1c48f49179 100644 --- a/fs/bcachefs/journal_types.h +++ b/fs/bcachefs/journal_types.h @@ -327,6 +327,7 @@ struct journal_device { /* for bch_journal_read_device */ struct closure read; + u64 highest_seq_found; }; /* From 36aa49d33e8d59246bd338064d6a516f693954e5 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Fri, 26 Apr 2024 15:29:56 +0800 Subject: [PATCH 1430/1702] bcachefs: Change destroy_inode to free_inode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vfs[1] documentation describes free_inode as follows: ``` free_inode this method is called from RCU callback. If you use call_rcu() in ->destroy_inode to free ‘struct inode’ memory, then it’s better to release memory in this method. ``` free_inode will be called by the RCU callback, so it might be better to move the inode free operation to destroy_inode. Similar to commit ae6b47b5653e ("fs/ntfs3: Change destroy_inode to free_inode"). Link: [1]: https://www.kernel.org/doc/html/latest/filesystems/vfs.html Signed-off-by: Youling Tang Signed-off-by: Kent Overstreet --- fs/bcachefs/fs.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c index 138d416a64d5a..a8d71cec1c174 100644 --- a/fs/bcachefs/fs.c +++ b/fs/bcachefs/fs.c @@ -1523,17 +1523,9 @@ static void bch2_vfs_inode_init(struct btree_trans *trans, subvol_inum inum, mapping_set_large_folios(inode->v.i_mapping); } -static void bch2_i_callback(struct rcu_head *head) +static void bch2_free_inode(struct inode *vinode) { - struct inode *vinode = container_of(head, struct inode, i_rcu); - struct bch_inode_info *inode = to_bch_ei(vinode); - - kmem_cache_free(bch2_inode_cache, inode); -} - -static void bch2_destroy_inode(struct inode *vinode) -{ - call_rcu(&vinode->i_rcu, bch2_i_callback); + kmem_cache_free(bch2_inode_cache, to_bch_ei(vinode)); } static int inode_update_times_fn(struct btree_trans *trans, @@ -1841,7 +1833,7 @@ static int bch2_unfreeze(struct super_block *sb) static const struct super_operations bch_super_operations = { .alloc_inode = bch2_alloc_inode, - .destroy_inode = bch2_destroy_inode, + .free_inode = bch2_free_inode, .write_inode = bch2_vfs_write_inode, .evict_inode = bch2_evict_inode, .sync_fs = bch2_sync_fs, From 9862022d09f785088619fd3c14063592fddc2aa5 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Tue, 30 Apr 2024 11:28:39 +0800 Subject: [PATCH 1431/1702] bcachefs: Fix error path of bch2_link_trans() In bch2_link_trans(), if bch2_inode_nlink_inc() fails, it needs to call bch2_trans_iter_exit() in the error path. Signed-off-by: Youling Tang Signed-off-by: Kent Overstreet --- fs/bcachefs/fs-common.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/fs-common.c b/fs/bcachefs/fs-common.c index 6c737def357e3..508d029ac53d0 100644 --- a/fs/bcachefs/fs-common.c +++ b/fs/bcachefs/fs-common.c @@ -200,12 +200,12 @@ int bch2_link_trans(struct btree_trans *trans, ret = bch2_inode_peek(trans, &inode_iter, inode_u, inum, BTREE_ITER_intent); if (ret) - goto err; + return ret; inode_u->bi_ctime = now; ret = bch2_inode_nlink_inc(inode_u); if (ret) - return ret; + goto err; ret = bch2_inode_peek(trans, &dir_iter, dir_u, dir, BTREE_ITER_intent); if (ret) From 75a53a0a230529425fa27bdfd5b7d9f95736488e Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Tue, 30 Apr 2024 11:27:44 +0800 Subject: [PATCH 1432/1702] bcachefs: Correct the FS_IOC_GETFLAGS to FS_IOC32_GETFLAGS in bch2_compat_fs_ioctl() It should be FS_IOC32_GETFLAGS instead of FS_IOC_GETFLAGS in compat ioctl. Signed-off-by: Youling Tang Signed-off-by: Kent Overstreet --- fs/bcachefs/fs-ioctl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/bcachefs/fs-ioctl.c b/fs/bcachefs/fs-ioctl.c index 3dc8630ff9fe1..205a323ffc6dd 100644 --- a/fs/bcachefs/fs-ioctl.c +++ b/fs/bcachefs/fs-ioctl.c @@ -548,7 +548,7 @@ long bch2_compat_fs_ioctl(struct file *file, unsigned cmd, unsigned long arg) { /* These are just misnamed, they actually get/put from/to user an int */ switch (cmd) { - case FS_IOC_GETFLAGS: + case FS_IOC32_GETFLAGS: cmd = FS_IOC_GETFLAGS; break; case FS_IOC32_SETFLAGS: From d3c44cfd5e98605f707a8eaaa096c0d17b2211fb Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 00:41:42 -0400 Subject: [PATCH 1433/1702] bcachefs: delete old gen check bch2_alloc_write_key() this was from metadata only gc - we don't need it anymore Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_gc.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 9694ffb0b098a..c9dfcf3cd199f 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -907,9 +907,6 @@ static int bch2_alloc_write_key(struct btree_trans *trans, bch2_dev_usage_update_m(c, ca, &old_gc, &gc); percpu_up_read(&c->mark_lock); - if (gen_after(old->gen, gc.gen)) - return 0; - if (fsck_err_on(new.data_type != gc.data_type, c, alloc_key_data_type_wrong, "bucket %llu:%llu gen %u has wrong data_type" From 2685c67d128339db01f8af0798c8a9c3ad04c5ba Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 01:49:59 -0400 Subject: [PATCH 1434/1702] bcachefs: dirty_sectors -> replicas_sectors Signed-off-by: Kent Overstreet --- fs/bcachefs/buckets.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index e0e991043aa51..93ce0eb350fe8 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -1144,7 +1144,7 @@ static int __trigger_extent(struct btree_trans *trans, enum bch_data_type data_type = bkey_is_btree_ptr(k.k) ? BCH_DATA_btree : BCH_DATA_user; - s64 dirty_sectors = 0; + s64 replicas_sectors = 0; int ret = 0; r.e.data_type = data_type; @@ -1170,7 +1170,7 @@ static int __trigger_extent(struct btree_trans *trans, return ret; } } else if (!p.has_ec) { - dirty_sectors += disk_sectors; + replicas_sectors += disk_sectors; r.e.devs[r.e.nr_devs++] = p.ptr.dev; } else { ret = bch2_trigger_stripe_ptr(trans, k, p, data_type, disk_sectors, flags); @@ -1188,8 +1188,8 @@ static int __trigger_extent(struct btree_trans *trans, if (r.e.nr_devs) { ret = !gc - ? bch2_update_replicas_list(trans, &r.e, dirty_sectors) - : bch2_update_replicas(c, k, &r.e, dirty_sectors, 0, true); + ? bch2_update_replicas_list(trans, &r.e, replicas_sectors) + : bch2_update_replicas(c, k, &r.e, replicas_sectors, 0, true); if (unlikely(ret && gc)) { struct printbuf buf = PRINTBUF; From fa9bb741fea1173f94f4fbdc7523744cde6dcdb8 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 02:47:30 -0400 Subject: [PATCH 1435/1702] bcachefs: alloc_data_type_set() Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 6 ++-- fs/bcachefs/alloc_background.h | 57 ++++++++++++++++++---------------- fs/bcachefs/btree_gc.c | 2 +- fs/bcachefs/ec.c | 2 +- 4 files changed, 36 insertions(+), 31 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 37e8446b30395..bbd85f1e59735 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -741,7 +741,7 @@ int bch2_trigger_alloc(struct btree_trans *trans, if (flags & BTREE_TRIGGER_transactional) { struct bch_alloc_v4 *new_a = bkey_s_to_alloc_v4(new).v; - new_a->data_type = alloc_data_type(*new_a, new_a->data_type); + alloc_data_type_set(new_a, new_a->data_type); if (bch2_bucket_sectors(*new_a) > bch2_bucket_sectors(*old_a)) { new_a->io_time[READ] = max_t(u64, 1, atomic64_read(&c->io_clock[READ].now)); @@ -1761,7 +1761,7 @@ static int bch2_discard_one_bucket(struct btree_trans *trans, } SET_BCH_ALLOC_V4_NEED_DISCARD(&a->v, false); - a->v.data_type = alloc_data_type(a->v, a->v.data_type); + alloc_data_type_set(&a->v, a->v.data_type); write: ret = bch2_trans_update(trans, &iter, &a->k_i, 0) ?: bch2_trans_commit(trans, NULL, NULL, @@ -1830,7 +1830,7 @@ static int bch2_clear_bucket_needs_discard(struct btree_trans *trans, struct bpo BUG_ON(a->v.dirty_sectors); SET_BCH_ALLOC_V4_NEED_DISCARD(&a->v, false); - a->v.data_type = alloc_data_type(a->v, a->v.data_type); + alloc_data_type_set(&a->v, a->v.data_type); ret = bch2_trans_update(trans, &iter, &a->k_i, 0); err: diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h index e0e0d72db03bc..98ba934c352f7 100644 --- a/fs/bcachefs/alloc_background.h +++ b/fs/bcachefs/alloc_background.h @@ -39,32 +39,6 @@ static inline u8 alloc_gc_gen(struct bch_alloc_v4 a) return a.gen - a.oldest_gen; } -static inline enum bch_data_type __alloc_data_type(u32 dirty_sectors, - u32 cached_sectors, - u32 stripe, - struct bch_alloc_v4 a, - enum bch_data_type data_type) -{ - if (stripe) - return data_type == BCH_DATA_parity ? data_type : BCH_DATA_stripe; - if (dirty_sectors) - return data_type; - if (cached_sectors) - return BCH_DATA_cached; - if (BCH_ALLOC_V4_NEED_DISCARD(&a)) - return BCH_DATA_need_discard; - if (alloc_gc_gen(a) >= BUCKET_GC_GEN_MAX) - return BCH_DATA_need_gc_gens; - return BCH_DATA_free; -} - -static inline enum bch_data_type alloc_data_type(struct bch_alloc_v4 a, - enum bch_data_type data_type) -{ - return __alloc_data_type(a.dirty_sectors, a.cached_sectors, - a.stripe, a, data_type); -} - static inline enum bch_data_type bucket_data_type(enum bch_data_type data_type) { switch (data_type) { @@ -101,6 +75,37 @@ static inline unsigned bch2_bucket_sectors_fragmented(struct bch_dev *ca, return d ? max(0, ca->mi.bucket_size - d) : 0; } +static inline enum bch_data_type __alloc_data_type(u32 dirty_sectors, + u32 cached_sectors, + u32 stripe, + struct bch_alloc_v4 a, + enum bch_data_type data_type) +{ + if (stripe) + return data_type == BCH_DATA_parity ? data_type : BCH_DATA_stripe; + if (dirty_sectors) + return data_type; + if (cached_sectors) + return BCH_DATA_cached; + if (BCH_ALLOC_V4_NEED_DISCARD(&a)) + return BCH_DATA_need_discard; + if (alloc_gc_gen(a) >= BUCKET_GC_GEN_MAX) + return BCH_DATA_need_gc_gens; + return BCH_DATA_free; +} + +static inline enum bch_data_type alloc_data_type(struct bch_alloc_v4 a, + enum bch_data_type data_type) +{ + return __alloc_data_type(a.dirty_sectors, a.cached_sectors, + a.stripe, a, data_type); +} + +static inline void alloc_data_type_set(struct bch_alloc_v4 *a, enum bch_data_type data_type) +{ + a->data_type = alloc_data_type(*a, data_type); +} + static inline u64 alloc_lru_idx_read(struct bch_alloc_v4 a) { return a.data_type == BCH_DATA_cached ? a.io_time[READ] : 0; diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index c9dfcf3cd199f..ef325ff71f89c 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -1307,7 +1307,7 @@ static int bch2_alloc_write_oldest_gen(struct btree_trans *trans, struct btree_i return ret; a_mut->v.oldest_gen = ca->oldest_gen[iter->pos.offset]; - a_mut->v.data_type = alloc_data_type(a_mut->v, a_mut->v.data_type); + alloc_data_type_set(&a_mut->v, a_mut->v.data_type); return bch2_trans_update(trans, iter, &a_mut->k_i, 0); } diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index d6fa7e0bb2666..a4ae39b10e98e 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -290,7 +290,7 @@ static int mark_stripe_bucket(struct btree_trans *trans, goto err; if (deleting) - a->v.data_type = alloc_data_type(a->v, BCH_DATA_user); + alloc_data_type_set(&a->v, BCH_DATA_user); ret = bch2_trans_update(trans, &iter, &a->k_i, 0); if (ret) From c02eb9e89184bd016d3c907d1287d05430b2016b Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 00:29:24 -0400 Subject: [PATCH 1436/1702] bcachefs: kill bch2_dev_usage_update_m() by using bucket_m_to_alloc() more, we can get some nice code cleanup. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.h | 36 ++++++++++++++++++++-------------- fs/bcachefs/btree_gc.c | 25 ++++++++++------------- fs/bcachefs/buckets.c | 35 +++++++-------------------------- fs/bcachefs/buckets.h | 2 -- fs/bcachefs/ec.c | 6 +++--- 5 files changed, 41 insertions(+), 63 deletions(-) diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h index 98ba934c352f7..7d747ad0de668 100644 --- a/fs/bcachefs/alloc_background.h +++ b/fs/bcachefs/alloc_background.h @@ -39,6 +39,22 @@ static inline u8 alloc_gc_gen(struct bch_alloc_v4 a) return a.gen - a.oldest_gen; } +static inline void __bucket_m_to_alloc(struct bch_alloc_v4 *dst, struct bucket b) +{ + dst->gen = b.gen; + dst->data_type = b.data_type; + dst->dirty_sectors = b.dirty_sectors; + dst->cached_sectors = b.cached_sectors; + dst->stripe = b.stripe; +} + +static inline struct bch_alloc_v4 bucket_m_to_alloc(struct bucket b) +{ + struct bch_alloc_v4 ret = {}; + __bucket_m_to_alloc(&ret, b); + return ret; +} + static inline enum bch_data_type bucket_data_type(enum bch_data_type data_type) { switch (data_type) { @@ -75,17 +91,14 @@ static inline unsigned bch2_bucket_sectors_fragmented(struct bch_dev *ca, return d ? max(0, ca->mi.bucket_size - d) : 0; } -static inline enum bch_data_type __alloc_data_type(u32 dirty_sectors, - u32 cached_sectors, - u32 stripe, - struct bch_alloc_v4 a, - enum bch_data_type data_type) +static inline enum bch_data_type alloc_data_type(struct bch_alloc_v4 a, + enum bch_data_type data_type) { - if (stripe) + if (a.stripe) return data_type == BCH_DATA_parity ? data_type : BCH_DATA_stripe; - if (dirty_sectors) + if (a.dirty_sectors) return data_type; - if (cached_sectors) + if (a.cached_sectors) return BCH_DATA_cached; if (BCH_ALLOC_V4_NEED_DISCARD(&a)) return BCH_DATA_need_discard; @@ -94,13 +107,6 @@ static inline enum bch_data_type __alloc_data_type(u32 dirty_sectors, return BCH_DATA_free; } -static inline enum bch_data_type alloc_data_type(struct bch_alloc_v4 a, - enum bch_data_type data_type) -{ - return __alloc_data_type(a.dirty_sectors, a.cached_sectors, - a.stripe, a, data_type); -} - static inline void alloc_data_type_set(struct bch_alloc_v4 *a, enum bch_data_type data_type) { a->data_type = alloc_data_type(*a, data_type); diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index ef325ff71f89c..e5289b53d1759 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -871,40 +871,35 @@ static int bch2_alloc_write_key(struct btree_trans *trans, { struct bch_fs *c = trans->c; struct bch_dev *ca = bch2_dev_bkey_exists(c, iter->pos.inode); - struct bucket old_gc, gc, *b; struct bkey_i_alloc_v4 *a; - struct bch_alloc_v4 old_convert, new; + struct bch_alloc_v4 old_gc, gc, old_convert, new; const struct bch_alloc_v4 *old; int ret; old = bch2_alloc_to_v4(k, &old_convert); - new = *old; + gc = new = *old; percpu_down_read(&c->mark_lock); - b = gc_bucket(ca, iter->pos.offset); - old_gc = *b; + __bucket_m_to_alloc(&gc, *gc_bucket(ca, iter->pos.offset)); + + old_gc = gc; if ((old->data_type == BCH_DATA_sb || old->data_type == BCH_DATA_journal) && !bch2_dev_is_online(ca)) { - b->data_type = old->data_type; - b->dirty_sectors = old->dirty_sectors; + gc.data_type = old->data_type; + gc.dirty_sectors = old->dirty_sectors; } /* - * b->data_type doesn't yet include need_discard & need_gc_gen states - + * gc.data_type doesn't yet include need_discard & need_gc_gen states - * fix that here: */ - b->data_type = __alloc_data_type(b->dirty_sectors, - b->cached_sectors, - b->stripe, - *old, - b->data_type); - gc = *b; + alloc_data_type_set(&gc, gc.data_type); if (gc.data_type != old_gc.data_type || gc.dirty_sectors != old_gc.dirty_sectors) - bch2_dev_usage_update_m(c, ca, &old_gc, &gc); + bch2_dev_usage_update(c, ca, &old_gc, &gc, 0, true); percpu_up_read(&c->mark_lock); if (fsck_err_on(new.data_type != gc.data_type, c, diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 93ce0eb350fe8..3acd35b771faa 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -318,26 +318,6 @@ void bch2_dev_usage_update(struct bch_fs *c, struct bch_dev *ca, preempt_enable(); } -static inline struct bch_alloc_v4 bucket_m_to_alloc(struct bucket b) -{ - return (struct bch_alloc_v4) { - .gen = b.gen, - .data_type = b.data_type, - .dirty_sectors = b.dirty_sectors, - .cached_sectors = b.cached_sectors, - .stripe = b.stripe, - }; -} - -void bch2_dev_usage_update_m(struct bch_fs *c, struct bch_dev *ca, - struct bucket *old, struct bucket *new) -{ - struct bch_alloc_v4 old_a = bucket_m_to_alloc(*old); - struct bch_alloc_v4 new_a = bucket_m_to_alloc(*new); - - bch2_dev_usage_update(c, ca, &old_a, &new_a, 0, true); -} - static inline int __update_replicas(struct bch_fs *c, struct bch_fs_usage *fs_usage, struct bch_replicas_entry_v1 *r, @@ -1028,7 +1008,7 @@ static int bch2_trigger_pointer(struct btree_trans *trans, percpu_down_read(&c->mark_lock); struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr); bucket_lock(g); - struct bucket old = *g; + struct bch_alloc_v4 old = bucket_m_to_alloc(*g); u8 bucket_data_type = g->data_type; int ret = __mark_pointer(trans, k, &p.ptr, *sectors, @@ -1043,9 +1023,9 @@ static int bch2_trigger_pointer(struct btree_trans *trans, } g->data_type = bucket_data_type; - struct bucket new = *g; + struct bch_alloc_v4 new = bucket_m_to_alloc(*g); bucket_unlock(g); - bch2_dev_usage_update_m(c, ca, &old, &new); + bch2_dev_usage_update(c, ca, &old, &new, 0, true); percpu_up_read(&c->mark_lock); } @@ -1336,14 +1316,13 @@ static int bch2_mark_metadata_bucket(struct bch_fs *c, struct bch_dev *ca, u64 b, enum bch_data_type data_type, unsigned sectors, enum btree_iter_update_trigger_flags flags) { - struct bucket old, new, *g; int ret = 0; percpu_down_read(&c->mark_lock); - g = gc_bucket(ca, b); + struct bucket *g = gc_bucket(ca, b); bucket_lock(g); - old = *g; + struct bch_alloc_v4 old = bucket_m_to_alloc(*g); if (bch2_fs_inconsistent_on(g->data_type && g->data_type != data_type, c, @@ -1365,11 +1344,11 @@ static int bch2_mark_metadata_bucket(struct bch_fs *c, struct bch_dev *ca, g->data_type = data_type; g->dirty_sectors += sectors; - new = *g; + struct bch_alloc_v4 new = bucket_m_to_alloc(*g); err: bucket_unlock(g); if (!ret) - bch2_dev_usage_update_m(c, ca, &old, &new); + bch2_dev_usage_update(c, ca, &old, &new, 0, true); percpu_up_read(&c->mark_lock); return ret; } diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h index df10f390101a2..f328ddb82dd97 100644 --- a/fs/bcachefs/buckets.h +++ b/fs/bcachefs/buckets.h @@ -310,8 +310,6 @@ bch2_fs_usage_read_short(struct bch_fs *); void bch2_dev_usage_update(struct bch_fs *, struct bch_dev *, const struct bch_alloc_v4 *, const struct bch_alloc_v4 *, u64, bool); -void bch2_dev_usage_update_m(struct bch_fs *, struct bch_dev *, - struct bucket *, struct bucket *); /* key/bucket marking: */ diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index a4ae39b10e98e..ce82cef1267d1 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -307,7 +307,7 @@ static int mark_stripe_bucket(struct btree_trans *trans, struct bucket *g = gc_bucket(ca, bucket.offset); bucket_lock(g); - struct bucket old = *g; + struct bch_alloc_v4 old = bucket_m_to_alloc(*g); u8 data_type = g->data_type; int ret = __mark_stripe_bucket(trans, s, ptr_idx, deleting, bucket, @@ -318,10 +318,10 @@ static int mark_stripe_bucket(struct btree_trans *trans, &g->stripe, &g->stripe_redundancy); g->data_type = data_type; - struct bucket new = *g; + struct bch_alloc_v4 new = bucket_m_to_alloc(*g); bucket_unlock(g); if (!ret) - bch2_dev_usage_update_m(c, ca, &old, &new); + bch2_dev_usage_update(c, ca, &old, &new, 0, true); percpu_up_read(&c->mark_lock); return ret; } From be11ae16c4907fc9ede68cf5589d0bdd2b195d01 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 02:10:57 -0400 Subject: [PATCH 1437/1702] bcachefs: __mark_pointer now takes bch_alloc_v4 Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 6 ++--- fs/bcachefs/alloc_background.h | 23 +++++++++++------ fs/bcachefs/buckets.c | 45 +++++++++++----------------------- 3 files changed, 33 insertions(+), 41 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index bbd85f1e59735..9122e886bbf24 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -263,7 +263,7 @@ int bch2_alloc_v4_invalid(struct bch_fs *c, struct bkey_s_c k, case BCH_DATA_free: case BCH_DATA_need_gc_gens: case BCH_DATA_need_discard: - bkey_fsck_err_on(bch2_bucket_sectors(*a.v) || a.v->stripe, + bkey_fsck_err_on(bch2_bucket_sectors_total(*a.v) || a.v->stripe, c, err, alloc_key_empty_but_have_data, "empty data type free but have data"); break; @@ -743,7 +743,7 @@ int bch2_trigger_alloc(struct btree_trans *trans, alloc_data_type_set(new_a, new_a->data_type); - if (bch2_bucket_sectors(*new_a) > bch2_bucket_sectors(*old_a)) { + if (bch2_bucket_sectors_total(*new_a) > bch2_bucket_sectors_total(*old_a)) { new_a->io_time[READ] = max_t(u64, 1, atomic64_read(&c->io_clock[READ].now)); new_a->io_time[WRITE]= max_t(u64, 1, atomic64_read(&c->io_clock[WRITE].now)); SET_BCH_ALLOC_V4_NEED_INC_GEN(new_a, true); @@ -1703,7 +1703,7 @@ static int bch2_discard_one_bucket(struct btree_trans *trans, if (ret) goto out; - if (a->v.dirty_sectors) { + if (bch2_bucket_sectors_total(a->v)) { if (bch2_trans_inconsistent_on(c->curr_recovery_pass > BCH_RECOVERY_PASS_check_alloc_info, trans, "attempting to discard bucket with dirty data\n%s", (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h index 7d747ad0de668..fe72e8a53bdb6 100644 --- a/fs/bcachefs/alloc_background.h +++ b/fs/bcachefs/alloc_background.h @@ -39,13 +39,22 @@ static inline u8 alloc_gc_gen(struct bch_alloc_v4 a) return a.gen - a.oldest_gen; } -static inline void __bucket_m_to_alloc(struct bch_alloc_v4 *dst, struct bucket b) +static inline void alloc_to_bucket(struct bucket *dst, struct bch_alloc_v4 src) { - dst->gen = b.gen; - dst->data_type = b.data_type; - dst->dirty_sectors = b.dirty_sectors; - dst->cached_sectors = b.cached_sectors; - dst->stripe = b.stripe; + dst->gen = src.gen; + dst->data_type = src.data_type; + dst->dirty_sectors = src.dirty_sectors; + dst->cached_sectors = src.cached_sectors; + dst->stripe = src.stripe; +} + +static inline void __bucket_m_to_alloc(struct bch_alloc_v4 *dst, struct bucket src) +{ + dst->gen = src.gen; + dst->data_type = src.data_type; + dst->dirty_sectors = src.dirty_sectors; + dst->cached_sectors = src.cached_sectors; + dst->stripe = src.stripe; } static inline struct bch_alloc_v4 bucket_m_to_alloc(struct bucket b) @@ -73,7 +82,7 @@ static inline bool bucket_data_type_mismatch(enum bch_data_type bucket, bucket_data_type(bucket) != bucket_data_type(ptr); } -static inline unsigned bch2_bucket_sectors(struct bch_alloc_v4 a) +static inline unsigned bch2_bucket_sectors_total(struct bch_alloc_v4 a) { return a.dirty_sectors + a.cached_sectors; } diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 3acd35b771faa..bb54931fcd697 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -943,23 +943,18 @@ static int __mark_pointer(struct btree_trans *trans, struct bkey_s_c k, const struct bch_extent_ptr *ptr, s64 sectors, enum bch_data_type ptr_data_type, - u8 bucket_gen, u8 *bucket_data_type, - u32 *dirty_sectors, u32 *cached_sectors) + struct bch_alloc_v4 *a) { u32 *dst_sectors = !ptr->cached - ? dirty_sectors - : cached_sectors; + ? &a->dirty_sectors + : &a->cached_sectors; int ret = bch2_bucket_ref_update(trans, k, ptr, sectors, ptr_data_type, - bucket_gen, *bucket_data_type, dst_sectors); + a->gen, a->data_type, dst_sectors); if (ret) return ret; - if (!*dirty_sectors && !*cached_sectors) - *bucket_data_type = 0; - else if (*bucket_data_type != BCH_DATA_stripe) - *bucket_data_type = ptr_data_type; - + alloc_data_type_set(a, ptr_data_type); return 0; } @@ -984,9 +979,7 @@ static int bch2_trigger_pointer(struct btree_trans *trans, if (ret) return ret; - ret = __mark_pointer(trans, k, &p.ptr, *sectors, bp.data_type, - a->v.gen, &a->v.data_type, - &a->v.dirty_sectors, &a->v.cached_sectors) ?: + ret = __mark_pointer(trans, k, &p.ptr, *sectors, bp.data_type, &a->v) ?: bch2_trans_update(trans, &iter, &a->k_i, 0); bch2_trans_iter_exit(trans, &iter); @@ -1003,30 +996,20 @@ static int bch2_trigger_pointer(struct btree_trans *trans, if (flags & BTREE_TRIGGER_gc) { struct bch_fs *c = trans->c; struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); - enum bch_data_type data_type = bch2_bkey_ptr_data_type(k, p, entry); percpu_down_read(&c->mark_lock); - struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr); + struct bucket *g = gc_bucket(ca, bucket.offset); bucket_lock(g); - struct bch_alloc_v4 old = bucket_m_to_alloc(*g); - - u8 bucket_data_type = g->data_type; - int ret = __mark_pointer(trans, k, &p.ptr, *sectors, - data_type, g->gen, - &bucket_data_type, - &g->dirty_sectors, - &g->cached_sectors); - if (ret) { - bucket_unlock(g); - percpu_up_read(&c->mark_lock); - return ret; - } + struct bch_alloc_v4 old = bucket_m_to_alloc(*g), new = old; - g->data_type = bucket_data_type; - struct bch_alloc_v4 new = bucket_m_to_alloc(*g); + int ret = __mark_pointer(trans, k, &p.ptr, *sectors, bp.data_type, &new); + if (!ret) { + alloc_to_bucket(g, new); + bch2_dev_usage_update(c, ca, &old, &new, 0, true); + } bucket_unlock(g); - bch2_dev_usage_update(c, ca, &old, &new, 0, true); percpu_up_read(&c->mark_lock); + return ret; } return 0; From 0acf2169a5e9b1ce286c44a740536e1cccb18db7 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 02:41:16 -0400 Subject: [PATCH 1438/1702] bcachefs: __mark_stripe_bucket() now takes bch_alloc_v4 Signed-off-by: Kent Overstreet --- fs/bcachefs/buckets.c | 1 - fs/bcachefs/ec.c | 100 ++++++++++++++++-------------------------- 2 files changed, 38 insertions(+), 63 deletions(-) diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index bb54931fcd697..9de96539817b2 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -1001,7 +1001,6 @@ static int bch2_trigger_pointer(struct btree_trans *trans, struct bucket *g = gc_bucket(ca, bucket.offset); bucket_lock(g); struct bch_alloc_v4 old = bucket_m_to_alloc(*g), new = old; - int ret = __mark_pointer(trans, k, &p.ptr, *sectors, bp.data_type, &new); if (!ret) { alloc_to_bucket(g, new); diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index ce82cef1267d1..1a5608488441f 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -167,12 +167,7 @@ static int __mark_stripe_bucket(struct btree_trans *trans, struct bkey_s_c_stripe s, unsigned ptr_idx, bool deleting, struct bpos bucket, - u8 bucket_gen, - u8 *bucket_data_type, - u32 *bucket_dirty_sectors, - u32 *bucket_cached_sectors, - u32 *bucket_stripe, - u8 *bucket_stripe_redundancy) + struct bch_alloc_v4 *a) { struct bch_fs *c = trans->c; const struct bch_extent_ptr *ptr = s.v->ptrs + ptr_idx; @@ -187,43 +182,43 @@ static int __mark_stripe_bucket(struct btree_trans *trans, sectors = -sectors; if (!deleting) { - if (bch2_trans_inconsistent_on(*bucket_stripe || - *bucket_stripe_redundancy, trans, + if (bch2_trans_inconsistent_on(a->stripe || + a->stripe_redundancy, trans, "bucket %llu:%llu gen %u data type %s dirty_sectors %u: multiple stripes using same bucket (%u, %llu)\n%s", - bucket.inode, bucket.offset, bucket_gen, - bch2_data_type_str(*bucket_data_type), - *bucket_dirty_sectors, - *bucket_stripe, s.k->p.offset, + bucket.inode, bucket.offset, a->gen, + bch2_data_type_str(a->data_type), + a->dirty_sectors, + a->stripe, s.k->p.offset, (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { ret = -EIO; goto err; } - if (bch2_trans_inconsistent_on(parity && (*bucket_dirty_sectors || *bucket_cached_sectors), trans, + if (bch2_trans_inconsistent_on(parity && bch2_bucket_sectors_total(*a), trans, "bucket %llu:%llu gen %u data type %s dirty_sectors %u cached_sectors %u: data already in parity bucket\n%s", - bucket.inode, bucket.offset, bucket_gen, - bch2_data_type_str(*bucket_data_type), - *bucket_dirty_sectors, - *bucket_cached_sectors, + bucket.inode, bucket.offset, a->gen, + bch2_data_type_str(a->data_type), + a->dirty_sectors, + a->cached_sectors, (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { ret = -EIO; goto err; } } else { - if (bch2_trans_inconsistent_on(*bucket_stripe != s.k->p.offset || - *bucket_stripe_redundancy != s.v->nr_redundant, trans, + if (bch2_trans_inconsistent_on(a->stripe != s.k->p.offset || + a->stripe_redundancy != s.v->nr_redundant, trans, "bucket %llu:%llu gen %u: not marked as stripe when deleting stripe (got %u)\n%s", - bucket.inode, bucket.offset, bucket_gen, - *bucket_stripe, + bucket.inode, bucket.offset, a->gen, + a->stripe, (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { ret = -EIO; goto err; } - if (bch2_trans_inconsistent_on(*bucket_data_type != data_type, trans, + if (bch2_trans_inconsistent_on(a->data_type != data_type, trans, "bucket %llu:%llu gen %u data type %s: wrong data type when stripe, should be %s\n%s", - bucket.inode, bucket.offset, bucket_gen, - bch2_data_type_str(*bucket_data_type), + bucket.inode, bucket.offset, a->gen, + bch2_data_type_str(a->data_type), bch2_data_type_str(data_type), (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { ret = -EIO; @@ -231,12 +226,12 @@ static int __mark_stripe_bucket(struct btree_trans *trans, } if (bch2_trans_inconsistent_on(parity && - (*bucket_dirty_sectors != -sectors || - *bucket_cached_sectors), trans, + (a->dirty_sectors != -sectors || + a->cached_sectors), trans, "bucket %llu:%llu gen %u dirty_sectors %u cached_sectors %u: wrong sectors when deleting parity block of stripe\n%s", - bucket.inode, bucket.offset, bucket_gen, - *bucket_dirty_sectors, - *bucket_cached_sectors, + bucket.inode, bucket.offset, a->gen, + a->dirty_sectors, + a->cached_sectors, (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) { ret = -EIO; goto err; @@ -245,21 +240,20 @@ static int __mark_stripe_bucket(struct btree_trans *trans, if (sectors) { ret = bch2_bucket_ref_update(trans, s.s_c, ptr, sectors, data_type, - bucket_gen, *bucket_data_type, - bucket_dirty_sectors); + a->gen, a->data_type, &a->dirty_sectors); if (ret) goto err; } if (!deleting) { - *bucket_stripe = s.k->p.offset; - *bucket_stripe_redundancy = s.v->nr_redundant; - *bucket_data_type = data_type; + a->stripe = s.k->p.offset; + a->stripe_redundancy = s.v->nr_redundant; } else { - *bucket_stripe = 0; - *bucket_stripe_redundancy = 0; - *bucket_data_type = BCH_DATA_free; + a->stripe = 0; + a->stripe_redundancy = 0; } + + alloc_data_type_set(a, data_type); err: printbuf_exit(&buf); return ret; @@ -279,19 +273,10 @@ static int mark_stripe_bucket(struct btree_trans *trans, struct bkey_i_alloc_v4 *a = bch2_trans_start_alloc_update(trans, &iter, bucket); int ret = PTR_ERR_OR_ZERO(a) ?: - __mark_stripe_bucket(trans, s, ptr_idx, deleting, iter.pos, - a->v.gen, - &a->v.data_type, - &a->v.dirty_sectors, - &a->v.cached_sectors, - &a->v.stripe, - &a->v.stripe_redundancy); + __mark_stripe_bucket(trans, s, ptr_idx, deleting, iter.pos, &a->v); if (ret) goto err; - if (deleting) - alloc_data_type_set(&a->v, BCH_DATA_user); - ret = bch2_trans_update(trans, &iter, &a->k_i, 0); if (ret) goto err; @@ -305,23 +290,14 @@ static int mark_stripe_bucket(struct btree_trans *trans, percpu_down_read(&c->mark_lock); struct bucket *g = gc_bucket(ca, bucket.offset); - bucket_lock(g); - struct bch_alloc_v4 old = bucket_m_to_alloc(*g); - u8 data_type = g->data_type; - - int ret = __mark_stripe_bucket(trans, s, ptr_idx, deleting, bucket, - g->gen, - &data_type, - &g->dirty_sectors, - &g->cached_sectors, - &g->stripe, - &g->stripe_redundancy); - g->data_type = data_type; - struct bch_alloc_v4 new = bucket_m_to_alloc(*g); - bucket_unlock(g); - if (!ret) + struct bch_alloc_v4 old = bucket_m_to_alloc(*g), new = old; + int ret = __mark_stripe_bucket(trans, s, ptr_idx, deleting, bucket, &new); + if (!ret) { + alloc_to_bucket(g, new); bch2_dev_usage_update(c, ca, &old, &new, 0, true); + } + bucket_unlock(g); percpu_up_read(&c->mark_lock); return ret; } From abe2f470bc18a8ba50ca0a4c73327d993f6678c5 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 03:27:30 -0400 Subject: [PATCH 1439/1702] bcachefs: simplify bch2_trans_start_alloc_update() Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 48 +++++++++++++++++++--------------- fs/bcachefs/alloc_background.h | 4 ++- fs/bcachefs/buckets.c | 15 +++-------- fs/bcachefs/ec.c | 16 +++--------- 4 files changed, 37 insertions(+), 46 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 9122e886bbf24..6b3bad3858120 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -429,22 +429,18 @@ struct bkey_i_alloc_v4 *bch2_alloc_to_v4_mut(struct btree_trans *trans, struct b } struct bkey_i_alloc_v4 * -bch2_trans_start_alloc_update(struct btree_trans *trans, struct btree_iter *iter, - struct bpos pos) +bch2_trans_start_alloc_update_noupdate(struct btree_trans *trans, struct btree_iter *iter, + struct bpos pos) { - struct bkey_s_c k; - struct bkey_i_alloc_v4 *a; - int ret; - - k = bch2_bkey_get_iter(trans, iter, BTREE_ID_alloc, pos, - BTREE_ITER_with_updates| - BTREE_ITER_cached| - BTREE_ITER_intent); - ret = bkey_err(k); + struct bkey_s_c k = bch2_bkey_get_iter(trans, iter, BTREE_ID_alloc, pos, + BTREE_ITER_with_updates| + BTREE_ITER_cached| + BTREE_ITER_intent); + int ret = bkey_err(k); if (unlikely(ret)) return ERR_PTR(ret); - a = bch2_alloc_to_v4_mut_inlined(trans, k); + struct bkey_i_alloc_v4 *a = bch2_alloc_to_v4_mut_inlined(trans, k); ret = PTR_ERR_OR_ZERO(a); if (unlikely(ret)) goto err; @@ -454,6 +450,20 @@ bch2_trans_start_alloc_update(struct btree_trans *trans, struct btree_iter *iter return ERR_PTR(ret); } +__flatten +struct bkey_i_alloc_v4 *bch2_trans_start_alloc_update(struct btree_trans *trans, struct bpos pos) +{ + struct btree_iter iter; + struct bkey_i_alloc_v4 *a = bch2_trans_start_alloc_update_noupdate(trans, &iter, pos); + int ret = PTR_ERR_OR_ZERO(a); + if (ret) + return ERR_PTR(ret); + + ret = bch2_trans_update(trans, &iter, &a->k_i, 0); + bch2_trans_iter_exit(trans, &iter); + return unlikely(ret) ? ERR_PTR(ret) : a; +} + static struct bpos alloc_gens_pos(struct bpos pos, unsigned *offset) { *offset = pos.offset & KEY_TYPE_BUCKET_GENS_MASK; @@ -1908,7 +1918,6 @@ static int invalidate_one_bucket(struct btree_trans *trans, s64 *nr_to_invalidate) { struct bch_fs *c = trans->c; - struct btree_iter alloc_iter = { NULL }; struct bkey_i_alloc_v4 *a = NULL; struct printbuf buf = PRINTBUF; struct bpos bucket = u64_to_bucket(lru_k.k->p.offset); @@ -1926,7 +1935,7 @@ static int invalidate_one_bucket(struct btree_trans *trans, if (bch2_bucket_is_open_safe(c, bucket.inode, bucket.offset)) return 0; - a = bch2_trans_start_alloc_update(trans, &alloc_iter, bucket); + a = bch2_trans_start_alloc_update(trans, bucket); ret = PTR_ERR_OR_ZERO(a); if (ret) goto out; @@ -1951,18 +1960,15 @@ static int invalidate_one_bucket(struct btree_trans *trans, a->v.io_time[READ] = atomic64_read(&c->io_clock[READ].now); a->v.io_time[WRITE] = atomic64_read(&c->io_clock[WRITE].now); - ret = bch2_trans_update(trans, &alloc_iter, &a->k_i, - BTREE_TRIGGER_bucket_invalidate) ?: - bch2_trans_commit(trans, NULL, NULL, - BCH_WATERMARK_btree| - BCH_TRANS_COMMIT_no_enospc); + ret = bch2_trans_commit(trans, NULL, NULL, + BCH_WATERMARK_btree| + BCH_TRANS_COMMIT_no_enospc); if (ret) goto out; trace_and_count(c, bucket_invalidate, c, bucket.inode, bucket.offset, cached_sectors); --*nr_to_invalidate; out: - bch2_trans_iter_exit(trans, &alloc_iter); printbuf_exit(&buf); return ret; err: @@ -2175,7 +2181,7 @@ int bch2_bucket_io_time_reset(struct btree_trans *trans, unsigned dev, if (bch2_trans_relock(trans)) bch2_trans_begin(trans); - a = bch2_trans_start_alloc_update(trans, &iter, POS(dev, bucket_nr)); + a = bch2_trans_start_alloc_update_noupdate(trans, &iter, POS(dev, bucket_nr)); ret = PTR_ERR_OR_ZERO(a); if (ret) return ret; diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h index fe72e8a53bdb6..a75eba6d95860 100644 --- a/fs/bcachefs/alloc_background.h +++ b/fs/bcachefs/alloc_background.h @@ -179,7 +179,9 @@ static inline void set_alloc_v4_u64s(struct bkey_i_alloc_v4 *a) } struct bkey_i_alloc_v4 * -bch2_trans_start_alloc_update(struct btree_trans *, struct btree_iter *, struct bpos); +bch2_trans_start_alloc_update_noupdate(struct btree_trans *, struct btree_iter *, struct bpos); +struct bkey_i_alloc_v4 * +bch2_trans_start_alloc_update(struct btree_trans *, struct bpos); void __bch2_alloc_to_v4(struct bkey_s_c, struct bch_alloc_v4 *); diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 9de96539817b2..1f810cfb35413 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -973,16 +973,9 @@ static int bch2_trigger_pointer(struct btree_trans *trans, *sectors = insert ? bp.bucket_len : -((s64) bp.bucket_len); if (flags & BTREE_TRIGGER_transactional) { - struct btree_iter iter; - struct bkey_i_alloc_v4 *a = bch2_trans_start_alloc_update(trans, &iter, bucket); - int ret = PTR_ERR_OR_ZERO(a); - if (ret) - return ret; - - ret = __mark_pointer(trans, k, &p.ptr, *sectors, bp.data_type, &a->v) ?: - bch2_trans_update(trans, &iter, &a->k_i, 0); - bch2_trans_iter_exit(trans, &iter); - + struct bkey_i_alloc_v4 *a = bch2_trans_start_alloc_update(trans, bucket); + int ret = PTR_ERR_OR_ZERO(a) ?: + __mark_pointer(trans, k, &p.ptr, *sectors, bp.data_type, &a->v); if (ret) return ret; @@ -1266,7 +1259,7 @@ static int __bch2_trans_mark_metadata_bucket(struct btree_trans *trans, int ret = 0; struct bkey_i_alloc_v4 *a = - bch2_trans_start_alloc_update(trans, &iter, POS(ca->dev_idx, b)); + bch2_trans_start_alloc_update_noupdate(trans, &iter, POS(ca->dev_idx, b)); if (IS_ERR(a)) return PTR_ERR(a); diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 1a5608488441f..64dcf852daedd 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -269,20 +269,10 @@ static int mark_stripe_bucket(struct btree_trans *trans, struct bpos bucket = PTR_BUCKET_POS(c, ptr); if (flags & BTREE_TRIGGER_transactional) { - struct btree_iter iter; struct bkey_i_alloc_v4 *a = - bch2_trans_start_alloc_update(trans, &iter, bucket); - int ret = PTR_ERR_OR_ZERO(a) ?: - __mark_stripe_bucket(trans, s, ptr_idx, deleting, iter.pos, &a->v); - if (ret) - goto err; - - ret = bch2_trans_update(trans, &iter, &a->k_i, 0); - if (ret) - goto err; -err: - bch2_trans_iter_exit(trans, &iter); - return ret; + bch2_trans_start_alloc_update(trans, bucket); + return PTR_ERR_OR_ZERO(a) ?: + __mark_stripe_bucket(trans, s, ptr_idx, deleting, bucket, &a->v); } if (flags & BTREE_TRIGGER_gc) { From 706833dbe3feb1e3b094885652e16eb74aa3f886 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 29 Apr 2024 03:15:21 -0400 Subject: [PATCH 1440/1702] bcachefs: CodingStyle Signed-off-by: Kent Overstreet --- .../filesystems/bcachefs/CodingStyle.rst | 186 ++++++++++++++++++ Documentation/filesystems/bcachefs/index.rst | 1 + 2 files changed, 187 insertions(+) create mode 100644 Documentation/filesystems/bcachefs/CodingStyle.rst diff --git a/Documentation/filesystems/bcachefs/CodingStyle.rst b/Documentation/filesystems/bcachefs/CodingStyle.rst new file mode 100644 index 0000000000000..0c45829a4899a --- /dev/null +++ b/Documentation/filesystems/bcachefs/CodingStyle.rst @@ -0,0 +1,186 @@ +.. SPDX-License-Identifier: GPL-2.0 + +bcachefs coding style +===================== + +Good development is like gardening, and codebases are our gardens. Tend to them +every day; look for little things that are out of place or in need of tidying. +A little weeding here and there goes a long way; don't wait until things have +spiraled out of control. + +Things don't always have to be perfect - nitpicking often does more harm than +good. But appreciate beauty when you see it - and let people know. + +The code that you are afraid to touch is the code most in need of refactoring. + +A little organizing here and there goes a long way. + +Put real thought into how you organize things. + +Good code is readable code, where the structure is simple and leaves nowhere +for bugs to hide. + +Assertions are one of our most important tools for writing reliable code. If in +the course of writing a patchset you encounter a condition that shouldn't +happen (and will have unpredictable or undefined behaviour if it does), or +you're not sure if it can happen and not sure how to handle it yet - make it a +BUG_ON(). Don't leave undefined or unspecified behavior lurking in the codebase. + +By the time you finish the patchset, you should understand better which +assertions need to be handled and turned into checks with error paths, and +which should be logically impossible. Leave the BUG_ON()s in for the ones which +are logically impossible. (Or, make them debug mode assertions if they're +expensive - but don't turn everything into a debug mode assertion, so that +we're not stuck debugging undefined behaviour should it turn out that you were +wrong). + +Assertions are documentation that can't go out of date. Good assertions are +wonderful. + +Good assertions drastically and dramatically reduce the amount of testing +required to shake out bugs. + +Good assertions are based on state, not logic. To write good assertions, you +have to think about what the invariants on your state are. + +Good invariants and assertions will hold everywhere in your codebase. This +means that you can run them in only a few places in the checked in version, but +should you need to debug something that caused the assertion to fail, you can +quickly shotgun them everywhere to find the codepath that broke the invariant. + +A good assertion checks something that the compiler could check for us, and +elide - if we were working in a language with embedded correctness proofs that +the compiler could check. This is something that exists today, but it'll likely +still be a few decades before it comes to systems programming languages. But we +can still incorporate that kind of thinking into our code and document the +invariants with runtime checks - much like the way people working in +dynamically typed languages may add type annotations, gradually making their +code statically typed. + +Looking for ways to make your assertions simpler - and higher level - will +often nudge you towards making the entire system simpler and more robust. + +Good code is code where you can poke around and see what it's doing - +introspection. We can't debug anything if we can't see what's going on. + +Whenever we're debugging, and the solution isn't immediately obvious, if the +issue is that we don't know where the issue is because we can't see what's +going on - fix that first. + +We have the tools to make anything visible at runtime, efficiently - RCU and +percpu data structures among them. Don't let things stay hidden. + +The most important tool for introspection is the humble pretty printer - in +bcachefs, this means `*_to_text()` functions, which output to printbufs. + +Pretty printers are wonderful, because they compose and you can use them +everywhere. Having functions to print whatever object you're working with will +make your error messages much easier to write (therefore they will actually +exist) and much more informative. And they can be used from sysfs/debugfs, as +well as tracepoints. + +Runtime info and debugging tools should come with clear descriptions and +labels, and good structure - we don't want files with a list of bare integers, +like in procfs. Part of the job of the debugging tools is to educate users and +new developers as to how the system works. + +Error messages should, whenever possible, tell you everything you need to debug +the issue. It's worth putting effort into them. + +Tracepoints shouldn't be the first thing you reach for. They're an important +tool, but always look for more immediate ways to make things visible. When we +have to rely on tracing, we have to know which tracepoints we're looking for, +and then we have to run the troublesome workload, and then we have to sift +through logs. This is a lot of steps to go through when a user is hitting +something, and if it's intermittent it may not even be possible. + +The humble counter is an incredibly useful tool. They're cheap and simple to +use, and many complicated internal operations with lots of things that can +behave weirdly (anything involving memory reclaim, for example) become +shockingly easy to debug once you have counters on every distinct codepath. + +Persistent counters are even better. + +When debugging, try to get the most out of every bug you come across; don't +rush to fix the initial issue. Look for things that will make related bugs +easier the next time around - introspection, new assertions, better error +messages, new debug tools, and do those first. Look for ways to make the system +better behaved; often one bug will uncover several other bugs through +downstream effects. + +Fix all that first, and then the original bug last - even if that means keeping +a user waiting. They'll thank you in the long run, and when they understand +what you're doing you'll be amazed at how patient they're happy to be. Users +like to help - otherwise they wouldn't be reporting the bug in the first place. + +Talk to your users. Don't isolate yourself. + +Users notice all sorts of interesting things, and by just talking to them and +interacting with them you can benefit from their experience. + +Spend time doing support and helpdesk stuff. Don't just write code - code isn't +finished until it's being used trouble free. + +This will also motivate you to make your debugging tools as good as possible, +and perhaps even your documentation, too. Like anything else in life, the more +time you spend at it the better you'll get, and you the developer are the +person most able to improve the tools to make debugging quick and easy. + +Be wary of how you take on and commit to big projects. Don't let development +become product-manager focused. Often time an idea is a good one but needs to +wait for its proper time - but you won't know if it's the proper time for an +idea until you start writing code. + +Expect to throw a lot of things away, or leave them half finished for later. +Nobody writes all perfect code that all gets shipped, and you'll be much more +productive in the long run if you notice this early and shift to something +else. The experience gained and lessons learned will be valuable for all the +other work you do. + +But don't be afraid to tackle projects that require significant rework of +existing code. Sometimes these can be the best projects, because they can lead +us to make existing code more general, more flexible, more multipurpose and +perhaps more robust. Just don't hesitate to abandon the idea if it looks like +it's going to make a mess of things. + +Complicated features can often be done as a series of refactorings, with the +final change that actually implements the feature as a quite small patch at the +end. It's wonderful when this happens, especially when those refactorings are +things that improve the codebase in their own right. When that happens there's +much less risk of wasted effort if the feature you were going for doesn't work +out. + +Always strive to work incrementally. Always strive to turn the big projects +into little bite sized projects that can prove their own merits. + +Instead of always tackling those big projects, look for little things that +will be useful, and make the big projects easier. + +The question of what's likely to be useful is where junior developers most +often go astray - doing something because it seems like it'll be useful often +leads to overengineering. Knowing what's useful comes from many years of +experience, or talking with people who have that experience - or from simply +reading lots of code and looking for common patterns and issues. Don't be +afraid to throw things away and do something simpler. + +Talk about your ideas with your fellow developers; often times the best things +come from relaxed conversations where people aren't afraid to say "what if?". + +Don't neglect your tools. + +The most important tools (besides the compiler and our text editor) are the +tools we use for testing. The shortest possible edit/test/debug cycle is +essential for working productively. We learn, gain experience, and discover the +errors in our thinking by running our code and seeing what happens. If your +time is being wasted because your tools are bad or too slow - don't accept it, +fix it. + +Put effort into your documentation, commmit messages, and code comments - but +don't go overboard. A good commit message is wonderful - but if the information +was important enough to go in a commit message, ask yourself if it would be +even better as a code comment. + +A good code comment is wonderful, but even better is the comment that didn't +need to exist because the code was so straightforward as to be obvious; +organized into small clean and tidy modules, with clear and descriptive names +for functions and variable, where every line of code has a clear purpose. diff --git a/Documentation/filesystems/bcachefs/index.rst b/Documentation/filesystems/bcachefs/index.rst index e2bd61ccd96ff..95fc4b90739ed 100644 --- a/Documentation/filesystems/bcachefs/index.rst +++ b/Documentation/filesystems/bcachefs/index.rst @@ -8,4 +8,5 @@ bcachefs Documentation :maxdepth: 2 :numbered: + CodingStyle errorcodes From ffcbec607613bf8f83c06ae2d3cfe017de52f917 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 6 Apr 2024 00:07:46 -0400 Subject: [PATCH 1441/1702] bcachefs: Kill opts.buckets_nouse Now explicitly allocate and free the buckets_nouse bitmap - this is going to be used for online fsck. To go RW when we haven't check allocations, we'll do a much slimmed down version that just initializes the buckets_nouse bitmaps. Signed-off-by: Kent Overstreet --- fs/bcachefs/buckets.c | 43 +++++++++++++++++++++++++++---------------- fs/bcachefs/buckets.h | 3 +++ fs/bcachefs/opts.h | 5 ----- fs/bcachefs/super.c | 5 ++--- 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 1f810cfb35413..525c72c0277c9 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -1519,6 +1519,31 @@ int __bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res, /* Startup/shutdown: */ +void bch2_buckets_nouse_free(struct bch_fs *c) +{ + for_each_member_device(c, ca) { + kvfree_rcu_mightsleep(ca->buckets_nouse); + ca->buckets_nouse = NULL; + } +} + +int bch2_buckets_nouse_alloc(struct bch_fs *c) +{ + for_each_member_device(c, ca) { + BUG_ON(ca->buckets_nouse); + + ca->buckets_nouse = kvmalloc(BITS_TO_LONGS(ca->mi.nbuckets) * + sizeof(unsigned long), + GFP_KERNEL|__GFP_ZERO); + if (!ca->buckets_nouse) { + percpu_ref_put(&ca->ref); + return -BCH_ERR_ENOMEM_buckets_nouse; + } + } + + return 0; +} + static void bucket_gens_free_rcu(struct rcu_head *rcu) { struct bucket_gens *buckets = @@ -1530,24 +1555,17 @@ static void bucket_gens_free_rcu(struct rcu_head *rcu) int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets) { struct bucket_gens *bucket_gens = NULL, *old_bucket_gens = NULL; - unsigned long *buckets_nouse = NULL; bool resize = ca->bucket_gens != NULL; int ret; + BUG_ON(resize && ca->buckets_nouse); + if (!(bucket_gens = kvmalloc(sizeof(struct bucket_gens) + nbuckets, GFP_KERNEL|__GFP_ZERO))) { ret = -BCH_ERR_ENOMEM_bucket_gens; goto err; } - if ((c->opts.buckets_nouse && - !(buckets_nouse = kvmalloc(BITS_TO_LONGS(nbuckets) * - sizeof(unsigned long), - GFP_KERNEL|__GFP_ZERO)))) { - ret = -BCH_ERR_ENOMEM_buckets_nouse; - goto err; - } - bucket_gens->first_bucket = ca->mi.first_bucket; bucket_gens->nbuckets = nbuckets; @@ -1565,17 +1583,11 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets) memcpy(bucket_gens->b, old_bucket_gens->b, n); - if (buckets_nouse) - memcpy(buckets_nouse, - ca->buckets_nouse, - BITS_TO_LONGS(n) * sizeof(unsigned long)); } rcu_assign_pointer(ca->bucket_gens, bucket_gens); bucket_gens = old_bucket_gens; - swap(ca->buckets_nouse, buckets_nouse); - nbuckets = ca->mi.nbuckets; if (resize) { @@ -1586,7 +1598,6 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets) ret = 0; err: - kvfree(buckets_nouse); if (bucket_gens) call_rcu(&bucket_gens->rcu, bucket_gens_free_rcu); diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h index f328ddb82dd97..f352d88d9b8e5 100644 --- a/fs/bcachefs/buckets.h +++ b/fs/bcachefs/buckets.h @@ -472,6 +472,9 @@ static inline u64 avail_factor(u64 r) return div_u64(r << RESERVE_FACTOR, (1 << RESERVE_FACTOR) + 1); } +void bch2_buckets_nouse_free(struct bch_fs *); +int bch2_buckets_nouse_alloc(struct bch_fs *); + int bch2_dev_buckets_resize(struct bch_fs *, struct bch_dev *, u64); void bch2_dev_buckets_free(struct bch_dev *); int bch2_dev_buckets_alloc(struct bch_fs *, struct bch_dev *); diff --git a/fs/bcachefs/opts.h b/fs/bcachefs/opts.h index 74f4d92849bb6..25530e0bb2f38 100644 --- a/fs/bcachefs/opts.h +++ b/fs/bcachefs/opts.h @@ -426,11 +426,6 @@ enum fsck_err_opts { BCH_SB_VERSION_UPGRADE, BCH_VERSION_UPGRADE_compatible, \ NULL, "Set superblock to latest version,\n" \ "allowing any new features to be used") \ - x(buckets_nouse, u8, \ - 0, \ - OPT_BOOL(), \ - BCH2_NO_SB_OPT, false, \ - NULL, "Allocate the buckets_nouse bitmap") \ x(stdio, u64, \ 0, \ OPT_UINT(0, S64_MAX), \ diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index 72dde1f1a3e67..1cbb2b3f47405 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -531,9 +531,7 @@ int bch2_fs_read_write_early(struct bch_fs *c) static void __bch2_fs_free(struct bch_fs *c) { - unsigned i; - - for (i = 0; i < BCH_TIME_STAT_NR; i++) + for (unsigned i = 0; i < BCH_TIME_STAT_NR; i++) bch2_time_stats_exit(&c->times[i]); bch2_find_btree_nodes_exit(&c->found_btree_nodes); @@ -1189,6 +1187,7 @@ static void bch2_dev_free(struct bch_dev *ca) if (ca->kobj.state_in_sysfs) kobject_del(&ca->kobj); + kfree(ca->buckets_nouse); bch2_free_super(&ca->disk_sb); bch2_dev_journal_exit(ca); From 3a718c0647680965171a19c2524c137be8f287b9 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 04:24:58 -0400 Subject: [PATCH 1442/1702] bcachefs: On device add, prefer unused slots We can't strictly guarantee that no pointers refer to nonexistent devices - we attempt to, but we need to be safe when the filesystem is corrupt. Therefore, change device_add to try to pick a slot that's never been used, or the slot that's been unused the longest. Signed-off-by: Kent Overstreet --- fs/bcachefs/super.c | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index 1cbb2b3f47405..6c6087039781e 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -1764,9 +1764,28 @@ int bch2_dev_add(struct bch_fs *c, const char *path) if (dynamic_fault("bcachefs:add:no_slot")) goto no_slot; - for (dev_idx = 0; dev_idx < BCH_SB_MEMBERS_MAX; dev_idx++) - if (!bch2_member_exists(c->disk_sb.sb, dev_idx)) - goto have_slot; + if (c->sb.nr_devices < BCH_SB_MEMBERS_MAX) { + dev_idx = c->sb.nr_devices; + goto have_slot; + } + + int best = -1; + u64 best_last_mount = 0; + for (dev_idx = 0; dev_idx < BCH_SB_MEMBERS_MAX; dev_idx++) { + struct bch_member m = bch2_sb_member_get(c->disk_sb.sb, dev_idx); + if (bch2_member_alive(&m)) + continue; + + u64 last_mount = le64_to_cpu(m.last_mount); + if (best < 0 || last_mount < best_last_mount) { + best = dev_idx; + best_last_mount = last_mount; + } + } + if (best >= 0) { + dev_idx = best; + goto have_slot; + } no_slot: ret = -BCH_ERR_ENOSPC_sb_members; bch_err_msg(c, ret, "setting up new superblock"); From b895c70326aa43b48d9b862e5e017de5fa83d844 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 06:20:37 -0400 Subject: [PATCH 1443/1702] bcachefs: x-macroize journal flags enums Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_iter.c | 2 +- fs/bcachefs/btree_key_cache.c | 2 +- fs/bcachefs/btree_trans_commit.c | 2 +- fs/bcachefs/btree_update.c | 2 +- fs/bcachefs/btree_update_interior.c | 6 +++--- fs/bcachefs/journal.c | 20 +++++++++++++------- fs/bcachefs/journal.h | 6 +++--- fs/bcachefs/journal_io.c | 6 +++--- fs/bcachefs/journal_reclaim.c | 10 +++++----- fs/bcachefs/journal_types.h | 15 ++++++++++----- fs/bcachefs/super.c | 6 +++--- 11 files changed, 44 insertions(+), 33 deletions(-) diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index 7f0eb5d396a23..5bf98cb8b15da 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -3170,7 +3170,7 @@ struct btree_trans *__bch2_trans_get(struct bch_fs *c, unsigned fn_idx) trans->locking_wait.task = current; trans->locked = true; trans->journal_replay_not_finished = - unlikely(!test_bit(JOURNAL_REPLAY_DONE, &c->journal.flags)) && + unlikely(!test_bit(JOURNAL_replay_done, &c->journal.flags)) && atomic_inc_not_zero(&c->journal_keys.ref); trans->nr_paths = ARRAY_SIZE(trans->_paths); trans->paths_allocated = trans->_paths_allocated; diff --git a/fs/bcachefs/btree_key_cache.c b/fs/bcachefs/btree_key_cache.c index 69e5f08db374c..75f5e6fe46349 100644 --- a/fs/bcachefs/btree_key_cache.c +++ b/fs/bcachefs/btree_key_cache.c @@ -648,7 +648,7 @@ static int btree_key_cache_flush_pos(struct btree_trans *trans, commit_flags |= BCH_WATERMARK_reclaim; if (ck->journal.seq != journal_last_seq(j) || - !test_bit(JOURNAL_SPACE_LOW, &c->journal.flags)) + !test_bit(JOURNAL_space_low, &c->journal.flags)) commit_flags |= BCH_TRANS_COMMIT_no_journal_res; ret = bch2_btree_iter_traverse(&b_iter) ?: diff --git a/fs/bcachefs/btree_trans_commit.c b/fs/bcachefs/btree_trans_commit.c index 0bf7c70da4175..a219a87e9ef07 100644 --- a/fs/bcachefs/btree_trans_commit.c +++ b/fs/bcachefs/btree_trans_commit.c @@ -337,7 +337,7 @@ static inline void btree_insert_entry_checks(struct btree_trans *trans, EBUG_ON(!i->level && btree_type_has_snapshots(i->btree_id) && !(i->flags & BTREE_UPDATE_internal_snapshot_node) && - test_bit(JOURNAL_REPLAY_DONE, &trans->c->journal.flags) && + test_bit(JOURNAL_replay_done, &trans->c->journal.flags) && i->k->k.p.snapshot && bch2_snapshot_is_internal_node(trans->c, i->k->k.p.snapshot) > 0); } diff --git a/fs/bcachefs/btree_update.c b/fs/bcachefs/btree_update.c index ee10c1f845b80..f3c645a43dcba 100644 --- a/fs/bcachefs/btree_update.c +++ b/fs/bcachefs/btree_update.c @@ -849,7 +849,7 @@ __bch2_fs_log_msg(struct bch_fs *c, unsigned commit_flags, const char *fmt, if (ret) goto err; - if (!test_bit(JOURNAL_RUNNING, &c->journal.flags)) { + if (!test_bit(JOURNAL_running, &c->journal.flags)) { ret = darray_make_room(&c->journal.early_journal_entries, jset_u64s(u64s)); if (ret) goto err; diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index 1febab152bfad..2e8b092a18206 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -1157,12 +1157,12 @@ bch2_btree_update_start(struct btree_trans *trans, struct btree_path *path, flags |= watermark; if (watermark < BCH_WATERMARK_reclaim && - test_bit(JOURNAL_SPACE_LOW, &c->journal.flags)) { + test_bit(JOURNAL_space_low, &c->journal.flags)) { if (flags & BCH_TRANS_COMMIT_journal_reclaim) return ERR_PTR(-BCH_ERR_journal_reclaim_would_deadlock); ret = drop_locks_do(trans, - ({ wait_event(c->journal.wait, !test_bit(JOURNAL_SPACE_LOW, &c->journal.flags)); 0; })); + ({ wait_event(c->journal.wait, !test_bit(JOURNAL_space_low, &c->journal.flags)); 0; })); if (ret) return ERR_PTR(ret); } @@ -1362,7 +1362,7 @@ static void bch2_insert_fixup_btree_ptr(struct btree_update *as, BUG_ON(insert->k.type == KEY_TYPE_btree_ptr_v2 && !btree_ptr_sectors_written(insert)); - if (unlikely(!test_bit(JOURNAL_REPLAY_DONE, &c->journal.flags))) + if (unlikely(!test_bit(JOURNAL_replay_done, &c->journal.flags))) bch2_journal_key_overwritten(c, b->c.btree_id, b->c.level, insert->k.p); if (bch2_bkey_invalid(c, bkey_i_to_s_c(insert), diff --git a/fs/bcachefs/journal.c b/fs/bcachefs/journal.c index 4d03a77f1e837..c984cdc2795d3 100644 --- a/fs/bcachefs/journal.c +++ b/fs/bcachefs/journal.c @@ -1182,11 +1182,11 @@ void bch2_fs_journal_stop(struct journal *j) cancel_delayed_work_sync(&j->write_work); BUG_ON(!bch2_journal_error(j) && - test_bit(JOURNAL_REPLAY_DONE, &j->flags) && + test_bit(JOURNAL_replay_done, &j->flags) && j->last_empty_seq != journal_cur_seq(j)); if (!bch2_journal_error(j)) - clear_bit(JOURNAL_RUNNING, &j->flags); + clear_bit(JOURNAL_running, &j->flags); } int bch2_fs_journal_start(struct journal *j, u64 cur_seq) @@ -1260,7 +1260,7 @@ int bch2_fs_journal_start(struct journal *j, u64 cur_seq) spin_lock(&j->lock); - set_bit(JOURNAL_RUNNING, &j->flags); + set_bit(JOURNAL_running, &j->flags); j->last_flush_write = jiffies; j->reservations.idx = j->reservations.unwritten_idx = journal_cur_seq(j); @@ -1401,6 +1401,13 @@ int bch2_fs_journal_init(struct journal *j) /* debug: */ +static const char * const bch2_journal_flags_strs[] = { +#define x(n) #n, + JOURNAL_FLAGS() +#undef x + NULL +}; + void __bch2_journal_debug_to_text(struct printbuf *out, struct journal *j) { struct bch_fs *c = container_of(j, struct bch_fs, journal); @@ -1415,6 +1422,9 @@ void __bch2_journal_debug_to_text(struct printbuf *out, struct journal *j) rcu_read_lock(); s = READ_ONCE(j->reservations); + prt_printf(out, "flags:\t"); + prt_bitflags(out, bch2_journal_flags_strs, j->flags); + prt_newline(out); prt_printf(out, "dirty journal entries:\t%llu/%llu\n", fifo_used(&j->pin), j->pin.size); prt_printf(out, "seq:\t%llu\n", journal_cur_seq(j)); prt_printf(out, "seq_ondisk:\t%llu\n", j->seq_ondisk); @@ -1453,10 +1463,6 @@ void __bch2_journal_debug_to_text(struct printbuf *out, struct journal *j) prt_printf(out, "unwritten entries:\n"); bch2_journal_bufs_to_text(out, j); - prt_printf(out, - "replay done:\t%i\n", - test_bit(JOURNAL_REPLAY_DONE, &j->flags)); - prt_printf(out, "space:\n"); printbuf_indent_add(out, 2); prt_printf(out, "discarded\t%u:%u\n", diff --git a/fs/bcachefs/journal.h b/fs/bcachefs/journal.h index 4b8c709bc317b..fd1f7cdaa8bc6 100644 --- a/fs/bcachefs/journal.h +++ b/fs/bcachefs/journal.h @@ -372,7 +372,7 @@ static inline int bch2_journal_res_get(struct journal *j, struct journal_res *re int ret; EBUG_ON(res->ref); - EBUG_ON(!test_bit(JOURNAL_RUNNING, &j->flags)); + EBUG_ON(!test_bit(JOURNAL_running, &j->flags)); res->u64s = u64s; @@ -418,8 +418,8 @@ struct bch_dev; static inline void bch2_journal_set_replay_done(struct journal *j) { - BUG_ON(!test_bit(JOURNAL_RUNNING, &j->flags)); - set_bit(JOURNAL_REPLAY_DONE, &j->flags); + BUG_ON(!test_bit(JOURNAL_running, &j->flags)); + set_bit(JOURNAL_replay_done, &j->flags); } void bch2_journal_unblock(struct journal *); diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c index dec50e7a8cb41..e76eb98af74a2 100644 --- a/fs/bcachefs/journal_io.c +++ b/fs/bcachefs/journal_io.c @@ -1954,14 +1954,14 @@ static int bch2_journal_write_pick_flush(struct journal *j, struct journal_buf * * So if we're in an error state, and we're still starting up, we don't * write anything at all. */ - if (error && test_bit(JOURNAL_NEED_FLUSH_WRITE, &j->flags)) + if (error && test_bit(JOURNAL_need_flush_write, &j->flags)) return -EIO; if (error || w->noflush || (!w->must_flush && (jiffies - j->last_flush_write) < msecs_to_jiffies(c->opts.journal_flush_delay) && - test_bit(JOURNAL_MAY_SKIP_FLUSH, &j->flags))) { + test_bit(JOURNAL_may_skip_flush, &j->flags))) { w->noflush = true; SET_JSET_NO_FLUSH(w->data, true); w->data->last_seq = 0; @@ -1972,7 +1972,7 @@ static int bch2_journal_write_pick_flush(struct journal *j, struct journal_buf * w->must_flush = true; j->last_flush_write = jiffies; j->nr_flush_writes++; - clear_bit(JOURNAL_NEED_FLUSH_WRITE, &j->flags); + clear_bit(JOURNAL_need_flush_write, &j->flags); } return 0; diff --git a/fs/bcachefs/journal_reclaim.c b/fs/bcachefs/journal_reclaim.c index a0c9f7ac611df..79be0eaddfa0d 100644 --- a/fs/bcachefs/journal_reclaim.c +++ b/fs/bcachefs/journal_reclaim.c @@ -67,7 +67,7 @@ void bch2_journal_set_watermark(struct journal *j) track_event_change(&c->times[BCH_TIME_blocked_write_buffer_full], low_on_wb)) trace_and_count(c, journal_full, c); - mod_bit(JOURNAL_SPACE_LOW, &j->flags, low_on_space || low_on_pin); + mod_bit(JOURNAL_space_low, &j->flags, low_on_space || low_on_pin); swap(watermark, j->watermark); if (watermark > j->watermark) @@ -225,9 +225,9 @@ void bch2_journal_space_available(struct journal *j) j->space[journal_space_clean_ondisk].total) && (clean - clean_ondisk <= total / 8) && (clean_ondisk * 2 > clean)) - set_bit(JOURNAL_MAY_SKIP_FLUSH, &j->flags); + set_bit(JOURNAL_may_skip_flush, &j->flags); else - clear_bit(JOURNAL_MAY_SKIP_FLUSH, &j->flags); + clear_bit(JOURNAL_may_skip_flush, &j->flags); bch2_journal_set_watermark(j); out: @@ -818,7 +818,7 @@ static int journal_flush_done(struct journal *j, u64 seq_to_flush, * If journal replay hasn't completed, the unreplayed journal entries * hold refs on their corresponding sequence numbers */ - ret = !test_bit(JOURNAL_REPLAY_DONE, &j->flags) || + ret = !test_bit(JOURNAL_replay_done, &j->flags) || journal_last_seq(j) > seq_to_flush || !fifo_used(&j->pin); @@ -833,7 +833,7 @@ bool bch2_journal_flush_pins(struct journal *j, u64 seq_to_flush) /* time_stats this */ bool did_work = false; - if (!test_bit(JOURNAL_RUNNING, &j->flags)) + if (!test_bit(JOURNAL_running, &j->flags)) return false; closure_wait_event(&j->async_wait, diff --git a/fs/bcachefs/journal_types.h b/fs/bcachefs/journal_types.h index 45c1c48f49179..19183fcf7ad7f 100644 --- a/fs/bcachefs/journal_types.h +++ b/fs/bcachefs/journal_types.h @@ -129,12 +129,17 @@ enum journal_space_from { journal_space_nr, }; +#define JOURNAL_FLAGS() \ + x(replay_done) \ + x(running) \ + x(may_skip_flush) \ + x(need_flush_write) \ + x(space_low) + enum journal_flags { - JOURNAL_REPLAY_DONE, - JOURNAL_RUNNING, - JOURNAL_MAY_SKIP_FLUSH, - JOURNAL_NEED_FLUSH_WRITE, - JOURNAL_SPACE_LOW, +#define x(n) JOURNAL_##n, + JOURNAL_FLAGS() +#undef x }; /* Reasons we may fail to get a journal reservation: */ diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index 6c6087039781e..5af8450098fdc 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -284,7 +284,7 @@ static void __bch2_fs_read_only(struct bch_fs *c) bch_verbose(c, "flushing journal and stopping allocators complete, journal seq %llu", journal_cur_seq(&c->journal)); - if (test_bit(JOURNAL_REPLAY_DONE, &c->journal.flags) && + if (test_bit(JOURNAL_replay_done, &c->journal.flags) && !test_bit(BCH_FS_emergency_ro, &c->flags)) set_bit(BCH_FS_clean_shutdown, &c->flags); @@ -466,8 +466,8 @@ static int __bch2_fs_read_write(struct bch_fs *c, bool early) * overwriting whatever was there previously, and there must always be * at least one non-flush write in the journal or recovery will fail: */ - set_bit(JOURNAL_NEED_FLUSH_WRITE, &c->journal.flags); - set_bit(JOURNAL_RUNNING, &c->journal.flags); + set_bit(JOURNAL_need_flush_write, &c->journal.flags); + set_bit(JOURNAL_running, &c->journal.flags); for_each_rw_member(c, ca) bch2_dev_allocator_add(c, ca); From 9a768ab75bef37eac23022f296e9be63b5f85565 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 1 May 2024 19:04:13 -0400 Subject: [PATCH 1444/1702] bcachefs: bch2_bkey_drop_ptrs() declares loop iter Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_io.c | 1 - fs/bcachefs/btree_update_interior.c | 1 - fs/bcachefs/data_update.c | 1 - fs/bcachefs/ec.c | 2 +- fs/bcachefs/extents.c | 4 ---- fs/bcachefs/extents.h | 2 +- fs/bcachefs/io_write.c | 1 - 7 files changed, 2 insertions(+), 10 deletions(-) diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c index c111976fd7182..9dcaccda7b935 100644 --- a/fs/bcachefs/btree_io.c +++ b/fs/bcachefs/btree_io.c @@ -1865,7 +1865,6 @@ static void btree_node_write_work(struct work_struct *work) container_of(work, struct btree_write_bio, work); struct bch_fs *c = wbio->wbio.c; struct btree *b = wbio->wbio.bio.bi_private; - struct bch_extent_ptr *ptr; int ret = 0; btree_bounce_free(c, diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index 2e8b092a18206..80eb862a89620 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -2491,7 +2491,6 @@ int bch2_btree_node_update_key_get_iter(struct btree_trans *trans, BUG_ON(!btree_node_hashed(b)); - struct bch_extent_ptr *ptr; bch2_bkey_drop_ptrs(bkey_i_to_s(new_key), ptr, !bch2_bkey_has_device(bkey_i_to_s(&b->key), ptr->dev)); diff --git a/fs/bcachefs/data_update.c b/fs/bcachefs/data_update.c index e8c7c5cec03fe..88f49e7d2a16d 100644 --- a/fs/bcachefs/data_update.c +++ b/fs/bcachefs/data_update.c @@ -467,7 +467,6 @@ int bch2_extent_drop_ptrs(struct btree_trans *trans, while (data_opts.kill_ptrs) { unsigned i = 0, drop = __fls(data_opts.kill_ptrs); - struct bch_extent_ptr *ptr; bch2_bkey_drop_ptrs(bkey_i_to_s(n), ptr, i++ == drop); data_opts.kill_ptrs ^= 1U << drop; diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 64dcf852daedd..330c3da70b96f 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -1200,7 +1200,7 @@ static int ec_stripe_update_extent(struct btree_trans *trans, struct btree_iter iter; struct bkey_s_c k; const struct bch_extent_ptr *ptr_c; - struct bch_extent_ptr *ptr, *ec_ptr = NULL; + struct bch_extent_ptr *ec_ptr = NULL; struct bch_extent_stripe_ptr stripe_ptr; struct bkey_i *n; int ret, dev, block; diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index 1143288d79408..0d1473ba584a4 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -838,8 +838,6 @@ union bch_extent_entry *bch2_bkey_drop_ptr(struct bkey_s k, void bch2_bkey_drop_device(struct bkey_s k, unsigned dev) { - struct bch_extent_ptr *ptr; - bch2_bkey_drop_ptrs(k, ptr, ptr->dev == dev); } @@ -974,8 +972,6 @@ void bch2_extent_ptr_set_cached(struct bkey_s k, struct bch_extent_ptr *ptr) */ bool bch2_extent_normalize(struct bch_fs *c, struct bkey_s k) { - struct bch_extent_ptr *ptr; - bch2_bkey_drop_ptrs(k, ptr, ptr->cached && ptr_stale(bch2_dev_bkey_exists(c, ptr->dev), ptr)); diff --git a/fs/bcachefs/extents.h b/fs/bcachefs/extents.h index 528e817eacbda..10e1817e83e3b 100644 --- a/fs/bcachefs/extents.h +++ b/fs/bcachefs/extents.h @@ -654,7 +654,7 @@ union bch_extent_entry *bch2_bkey_drop_ptr(struct bkey_s, do { \ struct bkey_ptrs _ptrs = bch2_bkey_ptrs(_k); \ \ - _ptr = &_ptrs.start->ptr; \ + struct bch_extent_ptr *_ptr = &_ptrs.start->ptr; \ \ while ((_ptr = bkey_ptr_next(_ptrs, _ptr))) { \ if (_cond) { \ diff --git a/fs/bcachefs/io_write.c b/fs/bcachefs/io_write.c index 73e25250de754..e96dad0dd60d4 100644 --- a/fs/bcachefs/io_write.c +++ b/fs/bcachefs/io_write.c @@ -481,7 +481,6 @@ static void bch2_write_done(struct closure *cl) static noinline int bch2_write_drop_io_error_ptrs(struct bch_write_op *op) { struct keylist *keys = &op->insert_keys; - struct bch_extent_ptr *ptr; struct bkey_i *src, *dst = keys->keys, *n; for (src = keys->keys; src != keys->top; src = n) { From 4c5b7294dedbdfae46f77262c0744879bb9b7221 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 3 May 2024 14:43:54 -0400 Subject: [PATCH 1445/1702] closures: closure_sync_timeout() Add a new variant of closure_sync_timeout() that takes a timeout. Note that when this returns -ETIME the closure will still be waiting on something, i.e. it's not safe to return if you've got a stack allocated closure. Signed-off-by: Kent Overstreet --- include/linux/closure.h | 12 ++++++++++++ lib/closure.c | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/include/linux/closure.h b/include/linux/closure.h index c554c6a08768a..99155df162d03 100644 --- a/include/linux/closure.h +++ b/include/linux/closure.h @@ -194,6 +194,18 @@ static inline void closure_sync(struct closure *cl) __closure_sync(cl); } +int __closure_sync_timeout(struct closure *cl, unsigned long timeout); + +static inline int closure_sync_timeout(struct closure *cl, unsigned long timeout) +{ +#ifdef CONFIG_DEBUG_CLOSURES + BUG_ON(closure_nr_remaining(cl) != 1 && !cl->closure_get_happened); +#endif + return cl->closure_get_happened + ? __closure_sync_timeout(cl, timeout) + : 0; +} + #ifdef CONFIG_DEBUG_CLOSURES void closure_debug_create(struct closure *cl); diff --git a/lib/closure.c b/lib/closure.c index c16540552d61b..07409e9e35a53 100644 --- a/lib/closure.c +++ b/lib/closure.c @@ -139,6 +139,43 @@ void __sched __closure_sync(struct closure *cl) } EXPORT_SYMBOL(__closure_sync); +int __sched __closure_sync_timeout(struct closure *cl, unsigned long timeout) +{ + struct closure_syncer s = { .task = current }; + int ret = 0; + + cl->s = &s; + continue_at(cl, closure_sync_fn, NULL); + + while (1) { + set_current_state(TASK_UNINTERRUPTIBLE); + if (s.done) + break; + if (!timeout) { + /* + * Carefully undo the continue_at() - but only if it + * hasn't completed, i.e. the final closure_put() hasn't + * happened yet: + */ + unsigned old, new, v = atomic_read(&cl->remaining); + do { + old = v; + if (!old || (old & CLOSURE_RUNNING)) + goto success; + + new = old + CLOSURE_REMAINING_INITIALIZER; + } while ((v = atomic_cmpxchg(&cl->remaining, old, new)) != old); + ret = -ETIME; + } + + timeout = schedule_timeout(timeout); + } +success: + __set_current_state(TASK_RUNNING); + return ret; +} +EXPORT_SYMBOL(__closure_sync_timeout); + #ifdef CONFIG_DEBUG_CLOSURES static LIST_HEAD(closure_list); From e98786ea855cb28176e27ffce23fb163a36ed32e Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 3 May 2024 14:49:23 -0400 Subject: [PATCH 1446/1702] bcachefs: bch2_print_allocator_stuck() If we block on the allocator for more than 10 seconds, print out some useful debugging info. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_foreground.c | 101 +++++++++++++++++++++++++++++++++ fs/bcachefs/alloc_foreground.h | 5 ++ fs/bcachefs/io_write.c | 6 +- fs/bcachefs/journal.c | 2 +- fs/bcachefs/sysfs.c | 76 +------------------------ 5 files changed, 114 insertions(+), 76 deletions(-) diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index 438e00c8316bd..13460aab3cae6 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -1630,3 +1630,104 @@ void bch2_write_points_to_text(struct printbuf *out, struct bch_fs *c) prt_str(out, "Btree write point\n"); bch2_write_point_to_text(out, c, &c->btree_write_point); } + +void bch2_fs_alloc_debug_to_text(struct printbuf *out, struct bch_fs *c) +{ + unsigned nr[BCH_DATA_NR]; + + memset(nr, 0, sizeof(nr)); + + for (unsigned i = 0; i < ARRAY_SIZE(c->open_buckets); i++) + nr[c->open_buckets[i].data_type]++; + + printbuf_tabstop_push(out, 24); + + percpu_down_read(&c->mark_lock); + prt_printf(out, "hidden\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.hidden)); + prt_printf(out, "btree\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.btree)); + prt_printf(out, "data\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.data)); + prt_printf(out, "cached\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.cached)); + prt_printf(out, "reserved\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.reserved)); + prt_printf(out, "online_reserved\t%llu\n", percpu_u64_get(c->online_reserved)); + prt_printf(out, "nr_inodes\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.nr_inodes)); + percpu_up_read(&c->mark_lock); + + prt_newline(out); + prt_printf(out, "freelist_wait\t%s\n", c->freelist_wait.list.first ? "waiting" : "empty"); + prt_printf(out, "open buckets allocated\t%i\n", OPEN_BUCKETS_COUNT - c->open_buckets_nr_free); + prt_printf(out, "open buckets total\t%u\n", OPEN_BUCKETS_COUNT); + prt_printf(out, "open_buckets_wait\t%s\n", c->open_buckets_wait.list.first ? "waiting" : "empty"); + prt_printf(out, "open_buckets_btree\t%u\n", nr[BCH_DATA_btree]); + prt_printf(out, "open_buckets_user\t%u\n", nr[BCH_DATA_user]); + prt_printf(out, "btree reserve cache\t%u\n", c->btree_reserve_cache_nr); +} + +void bch2_dev_alloc_debug_to_text(struct printbuf *out, struct bch_dev *ca) +{ + struct bch_fs *c = ca->fs; + struct bch_dev_usage stats = bch2_dev_usage_read(ca); + unsigned nr[BCH_DATA_NR]; + + memset(nr, 0, sizeof(nr)); + + for (unsigned i = 0; i < ARRAY_SIZE(c->open_buckets); i++) + nr[c->open_buckets[i].data_type]++; + + printbuf_tabstop_push(out, 12); + printbuf_tabstop_push(out, 16); + printbuf_tabstop_push(out, 16); + printbuf_tabstop_push(out, 16); + printbuf_tabstop_push(out, 16); + + bch2_dev_usage_to_text(out, &stats); + + prt_newline(out); + + prt_printf(out, "reserves:\n"); + for (unsigned i = 0; i < BCH_WATERMARK_NR; i++) + prt_printf(out, "%s\t%llu\r\n", bch2_watermarks[i], bch2_dev_buckets_reserved(ca, i)); + + prt_newline(out); + + printbuf_tabstops_reset(out); + printbuf_tabstop_push(out, 12); + printbuf_tabstop_push(out, 16); + + prt_printf(out, "open buckets\t%i\r\n", ca->nr_open_buckets); + prt_printf(out, "buckets to invalidate\t%llu\r\n", should_invalidate_buckets(ca, stats)); +} + +void bch2_print_allocator_stuck(struct bch_fs *c) +{ + struct printbuf buf = PRINTBUF; + + prt_printf(&buf, "Allocator stuck? Waited for 10 seconds\n"); + + prt_printf(&buf, "Allocator debug:\n"); + printbuf_indent_add(&buf, 2); + bch2_fs_alloc_debug_to_text(&buf, c); + printbuf_indent_sub(&buf, 2); + prt_newline(&buf); + + for_each_online_member(c, ca) { + prt_printf(&buf, "Dev %u:\n", ca->dev_idx); + printbuf_indent_add(&buf, 2); + bch2_dev_alloc_debug_to_text(&buf, ca); + printbuf_indent_sub(&buf, 2); + prt_newline(&buf); + } + + prt_printf(&buf, "Copygc debug:\n"); + printbuf_indent_add(&buf, 2); + bch2_copygc_wait_to_text(&buf, c); + printbuf_indent_sub(&buf, 2); + prt_newline(&buf); + + prt_printf(&buf, "Journal debug:\n"); + printbuf_indent_add(&buf, 2); + bch2_journal_debug_to_text(&buf, &c->journal); + printbuf_indent_sub(&buf, 2); + + bch2_print_string_as_lines(KERN_ERR, buf.buf); + printbuf_exit(&buf); +} diff --git a/fs/bcachefs/alloc_foreground.h b/fs/bcachefs/alloc_foreground.h index f0339eb7e30e9..96f24b67d0663 100644 --- a/fs/bcachefs/alloc_foreground.h +++ b/fs/bcachefs/alloc_foreground.h @@ -222,4 +222,9 @@ void bch2_open_buckets_partial_to_text(struct printbuf *, struct bch_fs *); void bch2_write_points_to_text(struct printbuf *, struct bch_fs *); +void bch2_fs_alloc_debug_to_text(struct printbuf *, struct bch_fs *); +void bch2_dev_alloc_debug_to_text(struct printbuf *, struct bch_dev *); + +void bch2_print_allocator_stuck(struct bch_fs *); + #endif /* _BCACHEFS_ALLOC_FOREGROUND_H */ diff --git a/fs/bcachefs/io_write.c b/fs/bcachefs/io_write.c index e96dad0dd60d4..55e24c83fb197 100644 --- a/fs/bcachefs/io_write.c +++ b/fs/bcachefs/io_write.c @@ -1494,7 +1494,11 @@ static void __bch2_write(struct bch_write_op *op) if ((op->flags & BCH_WRITE_SYNC) || (!(op->flags & BCH_WRITE_DONE) && !(op->flags & BCH_WRITE_IN_WORKER))) { - closure_sync(&op->cl); + if (closure_sync_timeout(&op->cl, HZ * 10)) { + bch2_print_allocator_stuck(c); + closure_sync(&op->cl); + } + __bch2_write_index(op); if (!(op->flags & BCH_WRITE_DONE)) diff --git a/fs/bcachefs/journal.c b/fs/bcachefs/journal.c index c984cdc2795d3..adec8e1ea73ea 100644 --- a/fs/bcachefs/journal.c +++ b/fs/bcachefs/journal.c @@ -1416,7 +1416,7 @@ void __bch2_journal_debug_to_text(struct printbuf *out, struct journal *j) u64 nr_writes = j->nr_flush_writes + j->nr_noflush_writes; if (!out->nr_tabstops) - printbuf_tabstop_push(out, 24); + printbuf_tabstop_push(out, 28); out->atomic++; rcu_read_lock(); diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index 4627b0ba179e7..df3d28659bfd4 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -346,37 +346,6 @@ static void bch2_gc_gens_pos_to_text(struct printbuf *out, struct bch_fs *c) prt_printf(out, "\n"); } -static void fs_alloc_debug_to_text(struct printbuf *out, struct bch_fs *c) -{ - unsigned nr[BCH_DATA_NR]; - - memset(nr, 0, sizeof(nr)); - - for (unsigned i = 0; i < ARRAY_SIZE(c->open_buckets); i++) - nr[c->open_buckets[i].data_type]++; - - printbuf_tabstop_push(out, 24); - - percpu_down_read(&c->mark_lock); - prt_printf(out, "hidden\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.hidden)); - prt_printf(out, "btree\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.btree)); - prt_printf(out, "data\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.data)); - prt_printf(out, "cached\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.cached)); - prt_printf(out, "reserved\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.reserved)); - prt_printf(out, "online_reserved\t%llu\n", percpu_u64_get(c->online_reserved)); - prt_printf(out, "nr_inodes\t%llu\n", bch2_fs_usage_read_one(c, &c->usage_base->b.nr_inodes)); - percpu_up_read(&c->mark_lock); - - prt_newline(out); - prt_printf(out, "freelist_wait\t%s\n", c->freelist_wait.list.first ? "waiting" : "empty"); - prt_printf(out, "open buckets allocated\t%i\n", OPEN_BUCKETS_COUNT - c->open_buckets_nr_free); - prt_printf(out, "open buckets total\t%u\n", OPEN_BUCKETS_COUNT); - prt_printf(out, "open_buckets_wait\t%s\n", c->open_buckets_wait.list.first ? "waiting" : "empty"); - prt_printf(out, "open_buckets_btree\t%u\n", nr[BCH_DATA_btree]); - prt_printf(out, "open_buckets_user\t%u\n", nr[BCH_DATA_user]); - prt_printf(out, "btree reserve cache\t%u\n", c->btree_reserve_cache_nr); -} - SHOW(bch2_fs) { struct bch_fs *c = container_of(kobj, struct bch_fs, kobj); @@ -458,7 +427,7 @@ SHOW(bch2_fs) bch2_disk_groups_to_text(out, c); if (attr == &sysfs_alloc_debug) - fs_alloc_debug_to_text(out, c); + bch2_fs_alloc_debug_to_text(out, c); return 0; } @@ -779,47 +748,6 @@ struct attribute *bch2_fs_time_stats_files[] = { NULL }; -static void dev_alloc_debug_to_text(struct printbuf *out, struct bch_dev *ca) -{ - struct bch_fs *c = ca->fs; - struct bch_dev_usage stats = bch2_dev_usage_read(ca); - unsigned nr[BCH_DATA_NR]; - - memset(nr, 0, sizeof(nr)); - - for (unsigned i = 0; i < ARRAY_SIZE(c->open_buckets); i++) - nr[c->open_buckets[i].data_type]++; - - printbuf_tabstop_push(out, 8); - printbuf_tabstop_push(out, 16); - printbuf_tabstop_push(out, 16); - printbuf_tabstop_push(out, 16); - printbuf_tabstop_push(out, 16); - - bch2_dev_usage_to_text(out, &stats); - - prt_newline(out); - - prt_printf(out, "reserves:\n"); - for (unsigned i = 0; i < BCH_WATERMARK_NR; i++) - prt_printf(out, "%s\t%llu\r\n", bch2_watermarks[i], bch2_dev_buckets_reserved(ca, i)); - - prt_newline(out); - - printbuf_tabstops_reset(out); - printbuf_tabstop_push(out, 24); - - prt_printf(out, "freelist_wait\t%s\n", c->freelist_wait.list.first ? "waiting" : "empty"); - prt_printf(out, "open buckets allocated\t%i\n", OPEN_BUCKETS_COUNT - c->open_buckets_nr_free); - prt_printf(out, "open buckets this dev\t%i\n", ca->nr_open_buckets); - prt_printf(out, "open buckets total\t%u\n", OPEN_BUCKETS_COUNT); - prt_printf(out, "open_buckets_wait\t%s\n", c->open_buckets_wait.list.first ? "waiting" : "empty"); - prt_printf(out, "open_buckets_btree\t%u\n", nr[BCH_DATA_btree]); - prt_printf(out, "open_buckets_user\t%u\n", nr[BCH_DATA_user]); - prt_printf(out, "buckets_to_invalidate\t%llu\n", should_invalidate_buckets(ca, stats)); - prt_printf(out, "btree reserve cache\t%u\n", c->btree_reserve_cache_nr); -} - static const char * const bch2_rw[] = { "read", "write", @@ -889,7 +817,7 @@ SHOW(bch2_dev) * 100 / CONGESTED_MAX); if (attr == &sysfs_alloc_debug) - dev_alloc_debug_to_text(out, ca); + bch2_dev_alloc_debug_to_text(out, ca); return 0; } From f295298b8c6413f0ed2a5a69dd7f32409cc54f1d Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 3 May 2024 17:39:16 -0400 Subject: [PATCH 1447/1702] bcachefs: New helpers for device refcounts This will be used in the next patch for adding some new debug mode asserts. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 9 ++++----- fs/bcachefs/alloc_foreground.c | 6 +++--- fs/bcachefs/btree_gc.c | 12 +++++------- fs/bcachefs/buckets.c | 4 ++-- fs/bcachefs/chardev.c | 26 ++++++++++++-------------- fs/bcachefs/data_update.c | 6 +++--- fs/bcachefs/disk_groups.c | 2 +- fs/bcachefs/sb-members.h | 22 ++++++++++++++++++---- fs/bcachefs/super.c | 4 ++-- 9 files changed, 50 insertions(+), 41 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 6b3bad3858120..e24656f4b7103 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -1656,10 +1656,9 @@ static void discard_buckets_next_dev(struct bch_fs *c, struct discard_buckets_st bch2_dev_usage_read(s->ca).d[BCH_DATA_free].buckets) bch2_journal_flush_async(&c->journal, NULL); - if (s->ca) - percpu_ref_put(&s->ca->ref); + bch2_dev_put(s->ca); if (ca) - percpu_ref_get(&ca->ref); + bch2_dev_get(ca); s->ca = ca; s->need_journal_commit_this_dev = 0; } @@ -2014,7 +2013,7 @@ static void bch2_do_invalidates_work(struct work_struct *work) invalidate_one_bucket(trans, &iter, k, &nr_to_invalidate)); if (ret < 0) { - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); break; } } @@ -2151,7 +2150,7 @@ int bch2_fs_freespace_init(struct bch_fs *c) ret = bch2_dev_freespace_init(c, ca, 0, ca->mi.nbuckets); if (ret) { - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); bch_err_fn(c, ret); return ret; } diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index 13460aab3cae6..92cafde165c0c 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -733,21 +733,21 @@ int bch2_bucket_alloc_set_trans(struct btree_trans *trans, rcu_read_lock(); ca = rcu_dereference(c->devs[dev]); if (ca) - percpu_ref_get(&ca->ref); + bch2_dev_get(ca); rcu_read_unlock(); if (!ca) continue; if (!ca->mi.durability && *have_cache) { - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); continue; } ob = bch2_bucket_alloc_trans(trans, ca, watermark, data_type, cl, &usage); if (!IS_ERR(ob)) bch2_dev_stripe_increment_inlined(ca, stripe, &usage); - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); if (IS_ERR(ob)) { ret = PTR_ERR(ob); diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index e5289b53d1759..133c48c4047da 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -815,10 +815,8 @@ static int bch2_gc_done(struct bch_fs *c) #undef copy_stripe_field #undef copy_field fsck_err: - if (ca) - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); bch_err_fn(c, ret); - percpu_up_write(&c->mark_lock); printbuf_exit(&buf); return ret; @@ -841,7 +839,7 @@ static int bch2_gc_start(struct bch_fs *c) ca->usage_gc = alloc_percpu(struct bch_dev_usage); if (!ca->usage_gc) { bch_err(c, "error allocating ca->usage_gc"); - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); return -BCH_ERR_ENOMEM_gc_start; } @@ -969,7 +967,7 @@ static int bch2_gc_alloc_done(struct bch_fs *c) NULL, NULL, BCH_TRANS_COMMIT_lazy_rw, bch2_alloc_write_key(trans, &iter, k))); if (ret) { - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); break; } } @@ -985,7 +983,7 @@ static int bch2_gc_alloc_start(struct bch_fs *c) ca->mi.nbuckets * sizeof(struct bucket), GFP_KERNEL|__GFP_ZERO); if (!buckets) { - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); bch_err(c, "error allocating ca->buckets[gc]"); return -BCH_ERR_ENOMEM_gc_alloc_start; } @@ -1330,7 +1328,7 @@ int bch2_gc_gens(struct bch_fs *c) ca->oldest_gen = kvmalloc(gens->nbuckets, GFP_KERNEL); if (!ca->oldest_gen) { - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); ret = -BCH_ERR_ENOMEM_gc_gens; goto err; } diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 525c72c0277c9..525a7548fab7a 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -1438,7 +1438,7 @@ int bch2_trans_mark_dev_sbs_flags(struct bch_fs *c, for_each_online_member(c, ca) { int ret = bch2_trans_mark_dev_sb(c, ca, flags); if (ret) { - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); return ret; } } @@ -1536,7 +1536,7 @@ int bch2_buckets_nouse_alloc(struct bch_fs *c) sizeof(unsigned long), GFP_KERNEL|__GFP_ZERO); if (!ca->buckets_nouse) { - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); return -BCH_ERR_ENOMEM_buckets_nouse; } } diff --git a/fs/bcachefs/chardev.c b/fs/bcachefs/chardev.c index dbcd433db0049..ed1fe771a4266 100644 --- a/fs/bcachefs/chardev.c +++ b/fs/bcachefs/chardev.c @@ -35,7 +35,7 @@ static struct bch_dev *bch2_device_lookup(struct bch_fs *c, u64 dev, rcu_read_lock(); ca = rcu_dereference(c->devs[dev]); if (ca) - percpu_ref_get(&ca->ref); + bch2_dev_get(ca); rcu_read_unlock(); if (!ca) @@ -391,7 +391,7 @@ static long bch2_ioctl_disk_offline(struct bch_fs *c, struct bch_ioctl_disk arg) return PTR_ERR(ca); ret = bch2_dev_offline(c, ca, arg.flags); - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); return ret; } @@ -420,7 +420,7 @@ static long bch2_ioctl_disk_set_state(struct bch_fs *c, if (ret) bch_err(c, "Error setting device state: %s", bch2_err_str(ret)); - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); return ret; } @@ -615,7 +615,7 @@ static long bch2_ioctl_dev_usage(struct bch_fs *c, arg.d[i].fragmented = src.d[i].fragmented; } - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); return copy_to_user_errcode(user_arg, &arg, sizeof(arg)); } @@ -667,7 +667,7 @@ static long bch2_ioctl_dev_usage_v2(struct bch_fs *c, goto err; } err: - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); return ret; } @@ -689,11 +689,9 @@ static long bch2_ioctl_read_super(struct bch_fs *c, if (arg.flags & BCH_READ_DEV) { ca = bch2_device_lookup(c, arg.dev, arg.flags); - - if (IS_ERR(ca)) { - ret = PTR_ERR(ca); - goto err; - } + ret = PTR_ERR_OR_ZERO(ca); + if (ret) + goto err_unlock; sb = ca->disk_sb.sb; } else { @@ -708,8 +706,8 @@ static long bch2_ioctl_read_super(struct bch_fs *c, ret = copy_to_user_errcode((void __user *)(unsigned long)arg.sb, sb, vstruct_bytes(sb)); err: - if (!IS_ERR_OR_NULL(ca)) - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); +err_unlock: mutex_unlock(&c->sb_lock); return ret; } @@ -753,7 +751,7 @@ static long bch2_ioctl_disk_resize(struct bch_fs *c, ret = bch2_dev_resize(c, ca, arg.nbuckets); - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); return ret; } @@ -779,7 +777,7 @@ static long bch2_ioctl_disk_resize_journal(struct bch_fs *c, ret = bch2_set_nr_journal_buckets(c, ca, arg.nbuckets); - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); return ret; } diff --git a/fs/bcachefs/data_update.c b/fs/bcachefs/data_update.c index 88f49e7d2a16d..c252f0619b736 100644 --- a/fs/bcachefs/data_update.c +++ b/fs/bcachefs/data_update.c @@ -360,7 +360,7 @@ void bch2_data_update_exit(struct data_update *update) if (c->opts.nocow_enabled) bch2_bucket_nocow_unlock(&c->nocow_locks, PTR_BUCKET_POS(c, ptr), 0); - percpu_ref_put(&bch2_dev_bkey_exists(c, ptr->dev)->ref); + bch2_dev_put(bch2_dev_bkey_exists(c, ptr->dev)); } bch2_bkey_buf_exit(&update->k, c); @@ -541,7 +541,7 @@ int bch2_data_update_init(struct btree_trans *trans, m->op.watermark = m->data_opts.btree_insert_flags & BCH_WATERMARK_MASK; bkey_for_each_ptr(ptrs, ptr) - percpu_ref_get(&bch2_dev_bkey_exists(c, ptr->dev)->ref); + bch2_dev_get(bch2_dev_bkey_exists(c, ptr->dev)); unsigned durability_have = 0, durability_removing = 0; @@ -653,7 +653,7 @@ int bch2_data_update_init(struct btree_trans *trans, if ((1U << i) & ptrs_locked) bch2_bucket_nocow_unlock(&c->nocow_locks, PTR_BUCKET_POS(c, &p.ptr), 0); - percpu_ref_put(&bch2_dev_bkey_exists(c, p.ptr.dev)->ref); + bch2_dev_put(bch2_dev_bkey_exists(c, p.ptr.dev)); i++; } diff --git a/fs/bcachefs/disk_groups.c b/fs/bcachefs/disk_groups.c index 04b6aa4144765..1cb563dfa0176 100644 --- a/fs/bcachefs/disk_groups.c +++ b/fs/bcachefs/disk_groups.c @@ -523,7 +523,7 @@ int bch2_opt_target_parse(struct bch_fs *c, const char *val, u64 *res, ca = bch2_dev_lookup(c, val); if (!IS_ERR(ca)) { *res = dev_to_target(ca->dev_idx); - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); return 0; } diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index 53034cea5843b..0aeb7285dc8c7 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -105,14 +105,28 @@ static inline struct bch_dev *__bch2_next_dev(struct bch_fs *c, struct bch_dev * for (struct bch_dev *_ca = NULL; \ (_ca = __bch2_next_dev((_c), _ca, (_mask)));) -static inline struct bch_dev *bch2_get_next_dev(struct bch_fs *c, struct bch_dev *ca) +static inline void bch2_dev_get(struct bch_dev *ca) +{ + percpu_ref_get(&ca->ref); +} + +static inline void __bch2_dev_put(struct bch_dev *ca) +{ + percpu_ref_put(&ca->ref); +} + +static inline void bch2_dev_put(struct bch_dev *ca) { - rcu_read_lock(); if (ca) - percpu_ref_put(&ca->ref); + __bch2_dev_put(ca); +} +static inline struct bch_dev *bch2_get_next_dev(struct bch_fs *c, struct bch_dev *ca) +{ + rcu_read_lock(); + bch2_dev_put(ca); if ((ca = __bch2_next_dev(c, ca, NULL))) - percpu_ref_get(&ca->ref); + bch2_dev_get(ca); rcu_read_unlock(); return ca; diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index 5af8450098fdc..8182cc971e7c6 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -710,7 +710,7 @@ static int bch2_fs_online(struct bch_fs *c) ret = bch2_dev_sysfs_online(c, ca); if (ret) { bch_err(c, "error creating sysfs objects"); - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); goto err; } } @@ -1613,7 +1613,7 @@ int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags) * We consume a reference to ca->ref, regardless of whether we succeed * or fail: */ - percpu_ref_put(&ca->ref); + bch2_dev_put(ca); if (!bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_failed, flags)) { bch_err(ca, "Cannot remove without losing data"); From 552aa5486579c18b4f8e7ca03ef88fa573c517b5 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 3 May 2024 18:07:40 -0400 Subject: [PATCH 1448/1702] bcachefs: Debug asserts for ca->ref Signed-off-by: Kent Overstreet --- fs/bcachefs/bcachefs.h | 6 ++++++ fs/bcachefs/sb-members.h | 13 +++++++++++++ fs/bcachefs/super.c | 21 ++++++++++++++++++--- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index 82544f826c588..f5ba8c6a34d73 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -537,7 +537,13 @@ struct io_count { struct bch_dev { struct kobject kobj; +#ifdef CONFIG_BCACHEFS_DEBUG + atomic_long_t ref; + bool dying; + unsigned long last_put; +#else struct percpu_ref ref; +#endif struct completion ref_completion; struct percpu_ref io_ref; struct completion io_ref_completion; diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index 0aeb7285dc8c7..ecb8284af0ded 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -107,12 +107,25 @@ static inline struct bch_dev *__bch2_next_dev(struct bch_fs *c, struct bch_dev * static inline void bch2_dev_get(struct bch_dev *ca) { +#ifdef CONFIG_BCACHEFS_DEBUG + BUG_ON(atomic_long_inc_return(&ca->ref) <= 1L); +#else percpu_ref_get(&ca->ref); +#endif } static inline void __bch2_dev_put(struct bch_dev *ca) { +#ifdef CONFIG_BCACHEFS_DEBUG + long r = atomic_long_dec_return(&ca->ref); + if (r < (long) !ca->dying) + panic("bch_dev->ref underflow, last put: %pS\n", (void *) ca->last_put); + ca->last_put = _THIS_IP_; + if (!r) + complete(&ca->ref_completion); +#else percpu_ref_put(&ca->ref); +#endif } static inline void bch2_dev_put(struct bch_dev *ca) diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index 8182cc971e7c6..ed135166ab86e 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -656,6 +656,7 @@ void bch2_fs_free(struct bch_fs *c) struct bch_dev *ca = rcu_dereference_protected(c->devs[i], true); if (ca) { + EBUG_ON(atomic_long_read(&ca->ref) != 1); bch2_free_super(&ca->disk_sb); bch2_dev_free(ca); } @@ -1200,7 +1201,9 @@ static void bch2_dev_free(struct bch_dev *ca) bch2_time_stats_quantiles_exit(&ca->io_latency[READ]); percpu_ref_exit(&ca->io_ref); +#ifndef CONFIG_BCACHEFS_DEBUG percpu_ref_exit(&ca->ref); +#endif kobject_put(&ca->kobj); } @@ -1227,12 +1230,14 @@ static void __bch2_dev_offline(struct bch_fs *c, struct bch_dev *ca) bch2_dev_journal_exit(ca); } +#ifndef CONFIG_BCACHEFS_DEBUG static void bch2_dev_ref_complete(struct percpu_ref *ref) { struct bch_dev *ca = container_of(ref, struct bch_dev, ref); complete(&ca->ref_completion); } +#endif static void bch2_dev_io_ref_complete(struct percpu_ref *ref) { @@ -1301,9 +1306,14 @@ static struct bch_dev *__bch2_dev_alloc(struct bch_fs *c, ca->nr_btree_reserve = DIV_ROUND_UP(BTREE_NODE_RESERVE, ca->mi.bucket_size / btree_sectors(c)); - if (percpu_ref_init(&ca->ref, bch2_dev_ref_complete, - 0, GFP_KERNEL) || - percpu_ref_init(&ca->io_ref, bch2_dev_io_ref_complete, +#ifndef CONFIG_BCACHEFS_DEBUG + if (percpu_ref_init(&ca->ref, bch2_dev_ref_complete, 0, GFP_KERNEL)) + goto err; +#else + atomic_long_set(&ca->ref, 1); +#endif + + if (percpu_ref_init(&ca->io_ref, bch2_dev_io_ref_complete, PERCPU_REF_INIT_DEAD, GFP_KERNEL) || !(ca->sb_read_scratch = (void *) __get_free_page(GFP_KERNEL)) || bch2_dev_buckets_alloc(c, ca) || @@ -1665,7 +1675,12 @@ int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags) rcu_assign_pointer(c->devs[ca->dev_idx], NULL); mutex_unlock(&c->sb_lock); +#ifndef CONFIG_BCACHEFS_DEBUG percpu_ref_kill(&ca->ref); +#else + ca->dying = true; + bch2_dev_put(ca); +#endif wait_for_completion(&ca->ref_completion); bch2_dev_free(ca); From 23f308ae19d345c8fc022edc5aae9a0af172ad73 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 15:30:35 -0400 Subject: [PATCH 1449/1702] bcachefs: bch2_dev_safe() -> bch2_dev_rcu() Signed-off-by: Kent Overstreet --- fs/bcachefs/extents.c | 2 +- fs/bcachefs/sb-members.c | 1 + fs/bcachefs/sb-members.h | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index 0d1473ba584a4..6376440f6f30f 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -983,7 +983,7 @@ void bch2_extent_ptr_to_text(struct printbuf *out, struct bch_fs *c, const struc { out->atomic++; rcu_read_lock(); - struct bch_dev *ca = bch2_dev_safe(c, ptr->dev); + struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev); if (!ca) { prt_printf(out, "ptr: %u:%llu gen %u%s", ptr->dev, (u64) ptr->offset, ptr->gen, diff --git a/fs/bcachefs/sb-members.c b/fs/bcachefs/sb-members.c index 52054f26982f6..96a49ad0525cc 100644 --- a/fs/bcachefs/sb-members.c +++ b/fs/bcachefs/sb-members.c @@ -3,6 +3,7 @@ #include "bcachefs.h" #include "btree_cache.h" #include "disk_groups.h" +#include "error.h" #include "opts.h" #include "replicas.h" #include "sb-members.h" diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index ecb8284af0ded..bea4c2efbb6e8 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -210,7 +210,7 @@ static inline struct bch_dev *bch2_dev_locked(struct bch_fs *c, unsigned dev) lockdep_is_held(&c->state_lock)); } -static inline struct bch_dev *bch2_dev_safe(struct bch_fs *c, unsigned dev) +static inline struct bch_dev *bch2_dev_rcu(struct bch_fs *c, unsigned dev) { return c && dev < c->sb.nr_devices ? rcu_dereference(c->devs[dev]) From f5faf43f853abb40255059d2b5133699c8528eaa Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 15:44:24 -0400 Subject: [PATCH 1450/1702] bcachefs: Pass device to bch2_alloc_write_key() More elimating bch2_dev_bkey_exists() Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_gc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 133c48c4047da..07b343e7ddeb9 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -865,10 +865,10 @@ static inline bool bch2_alloc_v4_cmp(struct bch_alloc_v4 l, static int bch2_alloc_write_key(struct btree_trans *trans, struct btree_iter *iter, + struct bch_dev *ca, struct bkey_s_c k) { struct bch_fs *c = trans->c; - struct bch_dev *ca = bch2_dev_bkey_exists(c, iter->pos.inode); struct bkey_i_alloc_v4 *a; struct bch_alloc_v4 old_gc, gc, old_convert, new; const struct bch_alloc_v4 *old; @@ -965,7 +965,7 @@ static int bch2_gc_alloc_done(struct bch_fs *c) POS(ca->dev_idx, ca->mi.nbuckets - 1), BTREE_ITER_slots|BTREE_ITER_prefetch, k, NULL, NULL, BCH_TRANS_COMMIT_lazy_rw, - bch2_alloc_write_key(trans, &iter, k))); + bch2_alloc_write_key(trans, &iter, ca, k))); if (ret) { bch2_dev_put(ca); break; From 267039d0fc78d6f9689b7314ba4b9efce95b37f5 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 15:41:48 -0400 Subject: [PATCH 1451/1702] bcachefs: Pass device to bch2_bucket_do_index() Eliminating bch2_dev_bkey_exists() uses and replacing them with proper checks; this one was unnecessary since the caller already has it. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index e24656f4b7103..97d70def49b31 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -625,12 +625,12 @@ int bch2_alloc_read(struct bch_fs *c) /* Free space/discard btree: */ static int bch2_bucket_do_index(struct btree_trans *trans, + struct bch_dev *ca, struct bkey_s_c alloc_k, const struct bch_alloc_v4 *a, bool set) { struct bch_fs *c = trans->c; - struct bch_dev *ca = bch2_dev_bkey_exists(c, alloc_k.k->p.inode); struct btree_iter iter; struct bkey_s_c old; struct bkey_i *k; @@ -770,8 +770,8 @@ int bch2_trigger_alloc(struct btree_trans *trans, if (old_a->data_type != new_a->data_type || (new_a->data_type == BCH_DATA_free && alloc_freespace_genbits(*old_a) != alloc_freespace_genbits(*new_a))) { - ret = bch2_bucket_do_index(trans, old, old_a, false) ?: - bch2_bucket_do_index(trans, new.s_c, new_a, true); + ret = bch2_bucket_do_index(trans, ca, old, old_a, false) ?: + bch2_bucket_do_index(trans, ca, new.s_c, new_a, true); if (ret) return ret; } @@ -790,8 +790,7 @@ int bch2_trigger_alloc(struct btree_trans *trans, return ret; } - new_a->fragmentation_lru = alloc_lru_idx_fragmentation(*new_a, - bch2_dev_bkey_exists(c, new.k->p.inode)); + new_a->fragmentation_lru = alloc_lru_idx_fragmentation(*new_a, ca); if (old_a->fragmentation_lru != new_a->fragmentation_lru) { ret = bch2_lru_change(trans, BCH_LRU_FRAGMENTATION_START, @@ -2078,7 +2077,7 @@ int bch2_dev_freespace_init(struct bch_fs *c, struct bch_dev *ca, struct bch_alloc_v4 a_convert; const struct bch_alloc_v4 *a = bch2_alloc_to_v4(k, &a_convert); - ret = bch2_bucket_do_index(trans, k, a, true) ?: + ret = bch2_bucket_do_index(trans, ca, k, a, true) ?: bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc); if (ret) From 13a16dabde3394350a877a6a11a90887ca899668 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 16:11:15 -0400 Subject: [PATCH 1452/1702] bcachefs: bch2_dev_btree_bitmap_marked() -> bch2_dev_rcu() Signed-off-by: Kent Overstreet --- fs/bcachefs/sb-members.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/fs/bcachefs/sb-members.c b/fs/bcachefs/sb-members.c index 96a49ad0525cc..e4a9c858ca8e2 100644 --- a/fs/bcachefs/sb-members.c +++ b/fs/bcachefs/sb-members.c @@ -398,11 +398,20 @@ void bch2_dev_errors_reset(struct bch_dev *ca) bool bch2_dev_btree_bitmap_marked(struct bch_fs *c, struct bkey_s_c k) { - bkey_for_each_ptr(bch2_bkey_ptrs_c(k), ptr) - if (!bch2_dev_btree_bitmap_marked_sectors(bch2_dev_bkey_exists(c, ptr->dev), - ptr->offset, btree_sectors(c))) - return false; - return true; + bool ret = true; + rcu_read_lock(); + bkey_for_each_ptr(bch2_bkey_ptrs_c(k), ptr) { + struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev); + if (!ca) + continue; + + if (!bch2_dev_btree_bitmap_marked_sectors(ca, ptr->offset, btree_sectors(c))) { + ret = false; + break; + } + } + rcu_read_unlock(); + return ret; } static void __bch2_dev_btree_bitmap_mark(struct bch_sb_field_members_v2 *mi, unsigned dev, @@ -440,6 +449,10 @@ void bch2_dev_btree_bitmap_mark(struct bch_fs *c, struct bkey_s_c k) lockdep_assert_held(&c->sb_lock); struct bch_sb_field_members_v2 *mi = bch2_sb_field_get(c->disk_sb.sb, members_v2); - bkey_for_each_ptr(bch2_bkey_ptrs_c(k), ptr) + bkey_for_each_ptr(bch2_bkey_ptrs_c(k), ptr) { + if (!bch2_member_exists(c->disk_sb.sb, ptr->dev)) + continue; + __bch2_dev_btree_bitmap_mark(mi, ptr->dev, ptr->offset, btree_sectors(c)); + } } From adf81796ee9cf2dd46214962a18015bffddbdbd2 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 15:45:26 -0400 Subject: [PATCH 1453/1702] bcachefs: journal_replay_entry_early() checks for nonexistent device Signed-off-by: Kent Overstreet --- fs/bcachefs/recovery.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/fs/bcachefs/recovery.c b/fs/bcachefs/recovery.c index a6447ffd336e3..1266916ac03f0 100644 --- a/fs/bcachefs/recovery.c +++ b/fs/bcachefs/recovery.c @@ -373,14 +373,17 @@ static int journal_replay_entry_early(struct bch_fs *c, case BCH_JSET_ENTRY_dev_usage: { struct jset_entry_dev_usage *u = container_of(entry, struct jset_entry_dev_usage, entry); - struct bch_dev *ca = bch2_dev_bkey_exists(c, le32_to_cpu(u->dev)); - unsigned i, nr_types = jset_entry_dev_usage_nr_types(u); - - for (i = 0; i < min_t(unsigned, nr_types, BCH_DATA_NR); i++) { - ca->usage_base->d[i].buckets = le64_to_cpu(u->d[i].buckets); - ca->usage_base->d[i].sectors = le64_to_cpu(u->d[i].sectors); - ca->usage_base->d[i].fragmented = le64_to_cpu(u->d[i].fragmented); - } + unsigned nr_types = jset_entry_dev_usage_nr_types(u); + + rcu_read_lock(); + struct bch_dev *ca = bch2_dev_rcu(c, le32_to_cpu(u->dev)); + if (ca) + for (unsigned i = 0; i < min_t(unsigned, nr_types, BCH_DATA_NR); i++) { + ca->usage_base->d[i].buckets = le64_to_cpu(u->d[i].buckets); + ca->usage_base->d[i].sectors = le64_to_cpu(u->d[i].sectors); + ca->usage_base->d[i].fragmented = le64_to_cpu(u->d[i].fragmented); + } + rcu_read_unlock(); break; } From 6349b07c25a03dbad3ff0e9a20a27a367ddd2997 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 15:45:05 -0400 Subject: [PATCH 1454/1702] bcachefs: bch2_have_enough_devs() checks for nonexistent device Signed-off-by: Kent Overstreet --- fs/bcachefs/replicas.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/fs/bcachefs/replicas.c b/fs/bcachefs/replicas.c index f2c99703c7306..f8ff7c8bb05ee 100644 --- a/fs/bcachefs/replicas.c +++ b/fs/bcachefs/replicas.c @@ -947,18 +947,20 @@ bool bch2_have_enough_devs(struct bch_fs *c, struct bch_devs_mask devs, percpu_down_read(&c->mark_lock); for_each_cpu_replicas_entry(&c->replicas, e) { - unsigned i, nr_online = 0, nr_failed = 0, dflags = 0; + unsigned nr_online = 0, nr_failed = 0, dflags = 0; bool metadata = e->data_type < BCH_DATA_user; if (e->data_type == BCH_DATA_cached) continue; - for (i = 0; i < e->nr_devs; i++) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, e->devs[i]); - + rcu_read_lock(); + for (unsigned i = 0; i < e->nr_devs; i++) { nr_online += test_bit(e->devs[i], devs.d); - nr_failed += ca->mi.state == BCH_MEMBER_STATE_failed; + + struct bch_dev *ca = bch2_dev_rcu(c, e->devs[i]); + nr_failed += ca && ca->mi.state == BCH_MEMBER_STATE_failed; } + rcu_read_unlock(); if (nr_failed == e->nr_devs) continue; From b07eb8252fb1455cc54bb306e253e193a7a39228 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 15:30:35 -0400 Subject: [PATCH 1455/1702] bcachefs: bch2_dev_tryget() Most uses of bch2_dev_bkey_exists() are going away, where we assume that because a key references a device the device most exist - instead, we'll be explicitly checking if the device exists and getting a reference to it. This adds the new helpers. Signed-off-by: Kent Overstreet --- fs/bcachefs/sb-members.c | 5 +++++ fs/bcachefs/sb-members.h | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/fs/bcachefs/sb-members.c b/fs/bcachefs/sb-members.c index e4a9c858ca8e2..2ca557bffc0e7 100644 --- a/fs/bcachefs/sb-members.c +++ b/fs/bcachefs/sb-members.c @@ -9,6 +9,11 @@ #include "sb-members.h" #include "super-io.h" +void bch2_dev_missing(struct bch_fs *c, unsigned dev) +{ + bch2_fs_inconsistent(c, "pointer to nonexistent device %u", dev); +} + #define x(t, n, ...) [n] = #t, static const char * const bch2_iops_measurements[] = { BCH_IOPS_MEASUREMENTS() diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index bea4c2efbb6e8..714559e2ef593 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -217,6 +217,26 @@ static inline struct bch_dev *bch2_dev_rcu(struct bch_fs *c, unsigned dev) : NULL; } +static inline struct bch_dev *bch2_dev_tryget_noerror(struct bch_fs *c, unsigned dev) +{ + rcu_read_lock(); + struct bch_dev *ca = bch2_dev_rcu(c, dev); + if (ca) + bch2_dev_get(ca); + rcu_read_unlock(); + return ca; +} + +void bch2_dev_missing(struct bch_fs *, unsigned); + +static inline struct bch_dev *bch2_dev_tryget(struct bch_fs *c, unsigned dev) +{ + struct bch_dev *ca = bch2_dev_tryget_noerror(c, dev); + if (!ca) + bch2_dev_missing(c, dev); + return ca; +} + /* XXX kill, move to struct bch_fs */ static inline struct bch_devs_mask bch2_online_devs(struct bch_fs *c) { From 4cd91e2f87a6b1f28806fc0082a9b31ce23d28b8 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 3 May 2024 17:48:23 -0400 Subject: [PATCH 1456/1702] bcachefs: Convert to bch2_dev_tryget_noerror() Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_foreground.c | 15 +++------------ fs/bcachefs/chardev.c | 7 +------ 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index 92cafde165c0c..f54d908683147 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -717,25 +717,16 @@ int bch2_bucket_alloc_set_trans(struct btree_trans *trans, struct bch_fs *c = trans->c; struct dev_alloc_list devs_sorted = bch2_dev_alloc_list(c, stripe, devs_may_alloc); - unsigned dev; - struct bch_dev *ca; int ret = -BCH_ERR_insufficient_devices; - unsigned i; BUG_ON(*nr_effective >= nr_replicas); - for (i = 0; i < devs_sorted.nr; i++) { + for (unsigned i = 0; i < devs_sorted.nr; i++) { struct bch_dev_usage usage; struct open_bucket *ob; - dev = devs_sorted.devs[i]; - - rcu_read_lock(); - ca = rcu_dereference(c->devs[dev]); - if (ca) - bch2_dev_get(ca); - rcu_read_unlock(); - + unsigned dev = devs_sorted.devs[i]; + struct bch_dev *ca = bch2_dev_tryget_noerror(c, dev); if (!ca) continue; diff --git a/fs/bcachefs/chardev.c b/fs/bcachefs/chardev.c index ed1fe771a4266..9e54323f0f5fc 100644 --- a/fs/bcachefs/chardev.c +++ b/fs/bcachefs/chardev.c @@ -32,12 +32,7 @@ static struct bch_dev *bch2_device_lookup(struct bch_fs *c, u64 dev, if (dev >= c->sb.nr_devices) return ERR_PTR(-EINVAL); - rcu_read_lock(); - ca = rcu_dereference(c->devs[dev]); - if (ca) - bch2_dev_get(ca); - rcu_read_unlock(); - + ca = bch2_dev_tryget_noerror(c, dev); if (!ca) return ERR_PTR(-EINVAL); } else { From 9b3059a1b3e9a71e345edf52f8487fc881d9f414 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 15:43:20 -0400 Subject: [PATCH 1457/1702] bcachefs: bch2_check_alloc_key() -> bch2_dev_tryget_noerror() More elimination of bch2_dev_bkey_exists() usage. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 17 ++++++++++------- fs/bcachefs/buckets.h | 5 ----- fs/bcachefs/sb-members.c | 5 +++++ fs/bcachefs/sb-members.h | 25 +++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 12 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 97d70def49b31..e61f00efe619b 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -1024,24 +1024,25 @@ int bch2_check_alloc_key(struct btree_trans *trans, struct btree_iter *bucket_gens_iter) { struct bch_fs *c = trans->c; - struct bch_dev *ca; struct bch_alloc_v4 a_convert; const struct bch_alloc_v4 *a; unsigned discard_key_type, freespace_key_type; unsigned gens_offset; struct bkey_s_c k; struct printbuf buf = PRINTBUF; - int ret; + int ret = 0; - if (fsck_err_on(!bch2_dev_bucket_exists(c, alloc_k.k->p), c, - alloc_key_to_missing_dev_bucket, + struct bch_dev *ca = bch2_dev_bucket_tryget_noerror(c, alloc_k.k->p); + if (fsck_err_on(!ca, + c, alloc_key_to_missing_dev_bucket, "alloc key for invalid device:bucket %llu:%llu", alloc_k.k->p.inode, alloc_k.k->p.offset)) - return bch2_btree_delete_at(trans, alloc_iter, 0); + ret = bch2_btree_delete_at(trans, alloc_iter, 0); + if (!ca) + return ret; - ca = bch2_dev_bkey_exists(c, alloc_k.k->p.inode); if (!ca->mi.freespace_initialized) - return 0; + goto out; a = bch2_alloc_to_v4(alloc_k, &a_convert); @@ -1140,8 +1141,10 @@ int bch2_check_alloc_key(struct btree_trans *trans, if (ret) goto err; } +out: err: fsck_err: + bch2_dev_put(ca); printbuf_exit(&buf); return ret; } diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h index f352d88d9b8e5..95cf3c2907344 100644 --- a/fs/bcachefs/buckets.h +++ b/fs/bcachefs/buckets.h @@ -35,11 +35,6 @@ static inline u64 sector_to_bucket_and_offset(const struct bch_dev *ca, sector_t return div_u64_rem(s, ca->mi.bucket_size, offset); } -static inline bool bucket_valid(const struct bch_dev *ca, u64 b) -{ - return b - ca->mi.first_bucket < ca->mi.nbuckets_minus_first; -} - #define for_each_bucket(_b, _buckets) \ for (_b = (_buckets)->b + (_buckets)->first_bucket; \ _b < (_buckets)->b + (_buckets)->nbuckets; _b++) diff --git a/fs/bcachefs/sb-members.c b/fs/bcachefs/sb-members.c index 2ca557bffc0e7..8f197bb088a06 100644 --- a/fs/bcachefs/sb-members.c +++ b/fs/bcachefs/sb-members.c @@ -14,6 +14,11 @@ void bch2_dev_missing(struct bch_fs *c, unsigned dev) bch2_fs_inconsistent(c, "pointer to nonexistent device %u", dev); } +void bch2_dev_bucket_missing(struct bch_fs *c, struct bpos bucket) +{ + bch2_fs_inconsistent(c, "pointer to nonexistent bucket %llu:%llu", bucket.inode, bucket.offset); +} + #define x(t, n, ...) [n] = #t, static const char * const bch2_iops_measurements[] = { BCH_IOPS_MEASUREMENTS() diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index 714559e2ef593..503b56051d7eb 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -190,6 +190,11 @@ static inline bool bch2_dev_exists(const struct bch_fs *c, unsigned dev) return dev < c->sb.nr_devices && c->devs[dev]; } +static inline bool bucket_valid(const struct bch_dev *ca, u64 b) +{ + return b - ca->mi.first_bucket < ca->mi.nbuckets_minus_first; +} + /* * If a key exists that references a device, the device won't be going away and * we can omit rcu_read_lock(): @@ -237,6 +242,26 @@ static inline struct bch_dev *bch2_dev_tryget(struct bch_fs *c, unsigned dev) return ca; } +static inline struct bch_dev *bch2_dev_bucket_tryget_noerror(struct bch_fs *c, struct bpos bucket) +{ + struct bch_dev *ca = bch2_dev_tryget_noerror(c, bucket.inode); + if (ca && !bucket_valid(ca, bucket.offset)) { + bch2_dev_put(ca); + ca = NULL; + } + return ca; +} + +void bch2_dev_bucket_missing(struct bch_fs *, struct bpos); + +static inline struct bch_dev *bch2_dev_bucket_tryget(struct bch_fs *c, struct bpos bucket) +{ + struct bch_dev *ca = bch2_dev_bucket_tryget_noerror(c, bucket); + if (!ca) + bch2_dev_bucket_missing(c, bucket); + return ca; +} + /* XXX kill, move to struct bch_fs */ static inline struct bch_devs_mask bch2_online_devs(struct bch_fs *c) { From a7f1c26f5907dde57f58c329ef0c532643092e71 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 15:53:03 -0400 Subject: [PATCH 1458/1702] bcachefs: bch2_trigger_alloc() -> bch2_dev_tryget() Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index e61f00efe619b..41f2e19aa8fe6 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -739,12 +739,10 @@ int bch2_trigger_alloc(struct btree_trans *trans, struct bch_fs *c = trans->c; int ret = 0; - if (bch2_trans_inconsistent_on(!bch2_dev_bucket_exists(c, new.k->p), trans, - "alloc key for invalid device or bucket")) + struct bch_dev *ca = bch2_dev_bucket_tryget(c, new.k->p); + if (!ca) return -EIO; - struct bch_dev *ca = bch2_dev_bkey_exists(c, new.k->p.inode); - struct bch_alloc_v4 old_a_convert; const struct bch_alloc_v4 *old_a = bch2_alloc_to_v4(old, &old_a_convert); @@ -773,7 +771,7 @@ int bch2_trigger_alloc(struct btree_trans *trans, ret = bch2_bucket_do_index(trans, ca, old, old_a, false) ?: bch2_bucket_do_index(trans, ca, new.s_c, new_a, true); if (ret) - return ret; + goto err; } if (new_a->data_type == BCH_DATA_cached && @@ -787,7 +785,7 @@ int bch2_trigger_alloc(struct btree_trans *trans, bucket_to_u64(new.k->p), old_lru, new_lru); if (ret) - return ret; + goto err; } new_a->fragmentation_lru = alloc_lru_idx_fragmentation(*new_a, ca); @@ -797,13 +795,13 @@ int bch2_trigger_alloc(struct btree_trans *trans, bucket_to_u64(new.k->p), old_a->fragmentation_lru, new_a->fragmentation_lru); if (ret) - return ret; + goto err; } if (old_a->gen != new_a->gen) { ret = bch2_bucket_gen_update(trans, new.k->p, new_a->gen); if (ret) - return ret; + goto err; } /* @@ -816,7 +814,7 @@ int bch2_trigger_alloc(struct btree_trans *trans, ret = bch2_update_cached_sectors_list(trans, new.k->p.inode, -((s64) old_a->cached_sectors)); if (ret) - return ret; + goto err; } } @@ -853,7 +851,7 @@ int bch2_trigger_alloc(struct btree_trans *trans, if (ret) { bch2_fs_fatal_error(c, "setting bucket_needs_journal_commit: %s", bch2_err_str(ret)); - return ret; + goto err; } } @@ -907,8 +905,9 @@ int bch2_trigger_alloc(struct btree_trans *trans, bucket_unlock(g); percpu_up_read(&c->mark_lock); } - - return 0; +err: + bch2_dev_put(ca); + return ret; } /* From 07d7c4da7bd1faefd338f4073846632fbac54880 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 16:20:49 -0400 Subject: [PATCH 1459/1702] bcachefs: bch2_bucket_ref_update() now takes bch_dev Signed-off-by: Kent Overstreet --- fs/bcachefs/buckets.c | 47 ++++++++++++++++++++++++------------------- fs/bcachefs/buckets.h | 4 ++-- fs/bcachefs/ec.c | 19 ++++++++++++----- 3 files changed, 42 insertions(+), 28 deletions(-) diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 525a7548fab7a..68f2d5952abdc 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -700,15 +700,14 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, return ret; } -int bch2_bucket_ref_update(struct btree_trans *trans, - struct bkey_s_c k, - const struct bch_extent_ptr *ptr, - s64 sectors, enum bch_data_type ptr_data_type, - u8 b_gen, u8 bucket_data_type, - u32 *bucket_sectors) +int bch2_bucket_ref_update(struct btree_trans *trans, struct bch_dev *ca, + struct bkey_s_c k, + const struct bch_extent_ptr *ptr, + s64 sectors, enum bch_data_type ptr_data_type, + u8 b_gen, u8 bucket_data_type, + u32 *bucket_sectors) { struct bch_fs *c = trans->c; - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); size_t bucket_nr = PTR_BUCKET_NR(ca, ptr); struct printbuf buf = PRINTBUF; bool inserting = sectors > 0; @@ -939,7 +938,7 @@ int bch2_trans_fs_usage_apply(struct btree_trans *trans, /* KEY_TYPE_extent: */ -static int __mark_pointer(struct btree_trans *trans, +static int __mark_pointer(struct btree_trans *trans, struct bch_dev *ca, struct bkey_s_c k, const struct bch_extent_ptr *ptr, s64 sectors, enum bch_data_type ptr_data_type, @@ -948,7 +947,7 @@ static int __mark_pointer(struct btree_trans *trans, u32 *dst_sectors = !ptr->cached ? &a->dirty_sectors : &a->cached_sectors; - int ret = bch2_bucket_ref_update(trans, k, ptr, sectors, ptr_data_type, + int ret = bch2_bucket_ref_update(trans, ca, k, ptr, sectors, ptr_data_type, a->gen, a->data_type, dst_sectors); if (ret) @@ -966,45 +965,51 @@ static int bch2_trigger_pointer(struct btree_trans *trans, enum btree_iter_update_trigger_flags flags) { bool insert = !(flags & BTREE_TRIGGER_overwrite); + int ret = 0; + + struct bch_fs *c = trans->c; + struct bch_dev *ca = bch2_dev_tryget(c, p.ptr.dev); + if (unlikely(!ca)) { + if (insert) + ret = -EIO; + goto err; + } + struct bpos bucket; struct bch_backpointer bp; - bch2_extent_ptr_to_bp(trans->c, btree_id, level, k, p, entry, &bucket, &bp); *sectors = insert ? bp.bucket_len : -((s64) bp.bucket_len); if (flags & BTREE_TRIGGER_transactional) { struct bkey_i_alloc_v4 *a = bch2_trans_start_alloc_update(trans, bucket); - int ret = PTR_ERR_OR_ZERO(a) ?: - __mark_pointer(trans, k, &p.ptr, *sectors, bp.data_type, &a->v); + ret = PTR_ERR_OR_ZERO(a) ?: + __mark_pointer(trans, ca, k, &p.ptr, *sectors, bp.data_type, &a->v); if (ret) - return ret; + goto err; if (!p.ptr.cached) { ret = bch2_bucket_backpointer_mod(trans, bucket, bp, k, insert); if (ret) - return ret; + goto err; } } if (flags & BTREE_TRIGGER_gc) { - struct bch_fs *c = trans->c; - struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); - percpu_down_read(&c->mark_lock); struct bucket *g = gc_bucket(ca, bucket.offset); bucket_lock(g); struct bch_alloc_v4 old = bucket_m_to_alloc(*g), new = old; - int ret = __mark_pointer(trans, k, &p.ptr, *sectors, bp.data_type, &new); + ret = __mark_pointer(trans, ca, k, &p.ptr, *sectors, bp.data_type, &new); if (!ret) { alloc_to_bucket(g, new); bch2_dev_usage_update(c, ca, &old, &new, 0, true); } bucket_unlock(g); percpu_up_read(&c->mark_lock); - return ret; } - - return 0; +err: + bch2_dev_put(ca); + return ret; } static int bch2_trigger_stripe_ptr(struct btree_trans *trans, diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h index 95cf3c2907344..46bf66d6b2540 100644 --- a/fs/bcachefs/buckets.h +++ b/fs/bcachefs/buckets.h @@ -330,8 +330,8 @@ int bch2_replicas_deltas_realloc(struct btree_trans *, unsigned); void bch2_fs_usage_initialize(struct bch_fs *); -int bch2_bucket_ref_update(struct btree_trans *, struct bkey_s_c, - const struct bch_extent_ptr *, +int bch2_bucket_ref_update(struct btree_trans *, struct bch_dev *, + struct bkey_s_c, const struct bch_extent_ptr *, s64, enum bch_data_type, u8, u8, u32 *); int bch2_check_fix_ptrs(struct btree_trans *, diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 330c3da70b96f..ea10f838d9ccb 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -167,9 +167,9 @@ static int __mark_stripe_bucket(struct btree_trans *trans, struct bkey_s_c_stripe s, unsigned ptr_idx, bool deleting, struct bpos bucket, - struct bch_alloc_v4 *a) + struct bch_alloc_v4 *a, + enum btree_iter_update_trigger_flags flags) { - struct bch_fs *c = trans->c; const struct bch_extent_ptr *ptr = s.v->ptrs + ptr_idx; unsigned nr_data = s.v->nr_blocks - s.v->nr_redundant; bool parity = ptr_idx >= nr_data; @@ -178,6 +178,14 @@ static int __mark_stripe_bucket(struct btree_trans *trans, struct printbuf buf = PRINTBUF; int ret = 0; + struct bch_fs *c = trans->c; + struct bch_dev *ca = bch2_dev_tryget(c, ptr->dev); + if (unlikely(!ca)) { + if (!(flags & BTREE_TRIGGER_overwrite)) + ret = -EIO; + goto err; + } + if (deleting) sectors = -sectors; @@ -239,7 +247,7 @@ static int __mark_stripe_bucket(struct btree_trans *trans, } if (sectors) { - ret = bch2_bucket_ref_update(trans, s.s_c, ptr, sectors, data_type, + ret = bch2_bucket_ref_update(trans, ca, s.s_c, ptr, sectors, data_type, a->gen, a->data_type, &a->dirty_sectors); if (ret) goto err; @@ -255,6 +263,7 @@ static int __mark_stripe_bucket(struct btree_trans *trans, alloc_data_type_set(a, data_type); err: + bch2_dev_put(ca); printbuf_exit(&buf); return ret; } @@ -272,7 +281,7 @@ static int mark_stripe_bucket(struct btree_trans *trans, struct bkey_i_alloc_v4 *a = bch2_trans_start_alloc_update(trans, bucket); return PTR_ERR_OR_ZERO(a) ?: - __mark_stripe_bucket(trans, s, ptr_idx, deleting, bucket, &a->v); + __mark_stripe_bucket(trans, s, ptr_idx, deleting, bucket, &a->v, flags); } if (flags & BTREE_TRIGGER_gc) { @@ -282,7 +291,7 @@ static int mark_stripe_bucket(struct btree_trans *trans, struct bucket *g = gc_bucket(ca, bucket.offset); bucket_lock(g); struct bch_alloc_v4 old = bucket_m_to_alloc(*g), new = old; - int ret = __mark_stripe_bucket(trans, s, ptr_idx, deleting, bucket, &new); + int ret = __mark_stripe_bucket(trans, s, ptr_idx, deleting, bucket, &new, flags); if (!ret) { alloc_to_bucket(g, new); bch2_dev_usage_update(c, ca, &old, &new, 0, true); From cb4d340a10295fcea7e4363f1b95c1a8c6c9bed4 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 15:46:45 -0400 Subject: [PATCH 1460/1702] bcachefs: bch2_evacuate_bucket() -> bch2_dev_tryget() Signed-off-by: Kent Overstreet --- fs/bcachefs/move.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/bcachefs/move.c b/fs/bcachefs/move.c index 12bc577354ff0..c441ce7c92eca 100644 --- a/fs/bcachefs/move.c +++ b/fs/bcachefs/move.c @@ -690,6 +690,10 @@ int bch2_evacuate_bucket(struct moving_context *ctxt, struct bpos bp_pos = POS_MIN; int ret = 0; + struct bch_dev *ca = bch2_dev_tryget(c, bucket.inode); + if (!ca) + return 0; + trace_bucket_evacuate(c, &bucket); bch2_bkey_buf_init(&sk); @@ -711,7 +715,7 @@ int bch2_evacuate_bucket(struct moving_context *ctxt, a = bch2_alloc_to_v4(k, &a_convert); dirty_sectors = bch2_bucket_sectors_dirty(*a); - bucket_size = bch2_dev_bkey_exists(c, bucket.inode)->mi.bucket_size; + bucket_size = ca->mi.bucket_size; fragmentation = a->fragmentation_lru; ret = bch2_btree_write_buffer_tryflush(trans); @@ -823,6 +827,7 @@ int bch2_evacuate_bucket(struct moving_context *ctxt, trace_evacuate_bucket(c, &bucket, dirty_sectors, bucket_size, fragmentation, ret); err: + bch2_dev_put(ca); bch2_bkey_buf_exit(&sk, c); return ret; } From fa6cce09f070990dfe384ef4ddfefdea73970abe Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 15:37:25 -0400 Subject: [PATCH 1461/1702] bcachefs: bch2_dev_iterate() New helper for getting refs to devices as we iterate. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 18 +++++++++++------- fs/bcachefs/sb-members.h | 8 ++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 41f2e19aa8fe6..69a77d268edf1 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -567,6 +567,7 @@ int bch2_bucket_gens_init(struct bch_fs *c) int bch2_alloc_read(struct bch_fs *c) { struct btree_trans *trans = bch2_trans_get(c); + struct bch_dev *ca = NULL; int ret; down_read(&c->gc_lock); @@ -580,16 +581,17 @@ int bch2_alloc_read(struct bch_fs *c) if (k.k->type != KEY_TYPE_bucket_gens) continue; - const struct bch_bucket_gens *g = bkey_s_c_to_bucket_gens(k).v; - + ca = bch2_dev_iterate(c, ca, k.k->p.inode); /* * Not a fsck error because this is checked/repaired by * bch2_check_alloc_key() which runs later: */ - if (!bch2_dev_exists(c, k.k->p.inode)) + if (!ca) { + bch2_btree_iter_set_pos(&iter, POS(k.k->p.inode + 1, 0)); continue; + } - struct bch_dev *ca = bch2_dev_bkey_exists(c, k.k->p.inode); + const struct bch_bucket_gens *g = bkey_s_c_to_bucket_gens(k).v; for (u64 b = max_t(u64, ca->mi.first_bucket, start); b < min_t(u64, ca->mi.nbuckets, end); @@ -600,14 +602,15 @@ int bch2_alloc_read(struct bch_fs *c) } else { ret = for_each_btree_key(trans, iter, BTREE_ID_alloc, POS_MIN, BTREE_ITER_prefetch, k, ({ + ca = bch2_dev_iterate(c, ca, k.k->p.inode); /* * Not a fsck error because this is checked/repaired by * bch2_check_alloc_key() which runs later: */ - if (!bch2_dev_bucket_exists(c, k.k->p)) + if (!ca) { + bch2_btree_iter_set_pos(&iter, POS(k.k->p.inode + 1, 0)); continue; - - struct bch_dev *ca = bch2_dev_bkey_exists(c, k.k->p.inode); + } struct bch_alloc_v4 a; *bucket_gen(ca, k.k->p.offset) = bch2_alloc_to_v4(k, &a)->gen; @@ -615,6 +618,7 @@ int bch2_alloc_read(struct bch_fs *c) })); } + bch2_dev_put(ca); bch2_trans_put(trans); up_read(&c->gc_lock); diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index 503b56051d7eb..d6d391dc21ca7 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -262,6 +262,14 @@ static inline struct bch_dev *bch2_dev_bucket_tryget(struct bch_fs *c, struct bp return ca; } +static inline struct bch_dev *bch2_dev_iterate(struct bch_fs *c, struct bch_dev *ca, unsigned dev_idx) +{ + if (ca && ca->dev_idx == dev_idx) + return ca; + bch2_dev_put(ca); + return bch2_dev_tryget(c, dev_idx); +} + /* XXX kill, move to struct bch_fs */ static inline struct bch_devs_mask bch2_online_devs(struct bch_fs *c) { From 1f2f92ec3f2e5802c2870d0d34ccfa66ae9216ad Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 19:34:28 -0400 Subject: [PATCH 1462/1702] bcachefs: PTR_BUCKET_POS() now takes bch_dev Signed-off-by: Kent Overstreet --- fs/bcachefs/backpointers.c | 7 ++++-- fs/bcachefs/backpointers.h | 4 ++-- fs/bcachefs/buckets.c | 2 +- fs/bcachefs/buckets.h | 10 +++----- fs/bcachefs/data_update.c | 22 +++++++++-------- fs/bcachefs/ec.c | 48 +++++++++++++++++++++----------------- fs/bcachefs/io_read.c | 2 +- fs/bcachefs/io_write.c | 11 +++++---- 8 files changed, 57 insertions(+), 49 deletions(-) diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c index 36e5e63ec3d5f..282aee70a51f8 100644 --- a/fs/bcachefs/backpointers.c +++ b/fs/bcachefs/backpointers.c @@ -30,7 +30,9 @@ static bool extent_matches_bp(struct bch_fs *c, if (p.ptr.cached) continue; - bch2_extent_ptr_to_bp(c, btree_id, level, k, p, entry, &bucket2, &bp2); + struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); + + bch2_extent_ptr_to_bp(c, ca, btree_id, level, k, p, entry, &bucket2, &bp2); if (bpos_eq(bucket, bucket2) && !memcmp(&bp, &bp2, sizeof(bp))) return true; @@ -666,7 +668,8 @@ static int check_extent_to_backpointers(struct btree_trans *trans, if (p.ptr.cached) continue; - bch2_extent_ptr_to_bp(c, btree, level, k, p, entry, &bucket_pos, &bp); + struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); + bch2_extent_ptr_to_bp(c, ca, btree, level, k, p, entry, &bucket_pos, &bp); ret = check_bp_exists(trans, s, bucket_pos, bp, k); if (ret) diff --git a/fs/bcachefs/backpointers.h b/fs/bcachefs/backpointers.h index e7f1eddbf4f48..1d270c40fce9a 100644 --- a/fs/bcachefs/backpointers.h +++ b/fs/bcachefs/backpointers.h @@ -120,7 +120,7 @@ static inline enum bch_data_type bch2_bkey_ptr_data_type(struct bkey_s_c k, } } -static inline void bch2_extent_ptr_to_bp(struct bch_fs *c, +static inline void bch2_extent_ptr_to_bp(struct bch_fs *c, struct bch_dev *ca, enum btree_id btree_id, unsigned level, struct bkey_s_c k, struct extent_ptr_decoded p, const union bch_extent_entry *entry, @@ -130,7 +130,7 @@ static inline void bch2_extent_ptr_to_bp(struct bch_fs *c, s64 sectors = level ? btree_sectors(c) : k.k->size; u32 bucket_offset; - *bucket_pos = PTR_BUCKET_POS_OFFSET(c, &p.ptr, &bucket_offset); + *bucket_pos = PTR_BUCKET_POS_OFFSET(ca, &p.ptr, &bucket_offset); *bp = (struct bch_backpointer) { .btree_id = btree_id, .level = level, diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 68f2d5952abdc..4774f9c7f060e 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -977,7 +977,7 @@ static int bch2_trigger_pointer(struct btree_trans *trans, struct bpos bucket; struct bch_backpointer bp; - bch2_extent_ptr_to_bp(trans->c, btree_id, level, k, p, entry, &bucket, &bp); + bch2_extent_ptr_to_bp(trans->c, ca, btree_id, level, k, p, entry, &bucket, &bp); *sectors = insert ? bp.bucket_len : -((s64) bp.bucket_len); if (flags & BTREE_TRIGGER_transactional) { diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h index 46bf66d6b2540..0a54faf7c50a6 100644 --- a/fs/bcachefs/buckets.h +++ b/fs/bcachefs/buckets.h @@ -120,20 +120,16 @@ static inline size_t PTR_BUCKET_NR(const struct bch_dev *ca, return sector_to_bucket(ca, ptr->offset); } -static inline struct bpos PTR_BUCKET_POS(const struct bch_fs *c, - const struct bch_extent_ptr *ptr) +static inline struct bpos PTR_BUCKET_POS(const struct bch_dev *ca, + const struct bch_extent_ptr *ptr) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); - return POS(ptr->dev, PTR_BUCKET_NR(ca, ptr)); } -static inline struct bpos PTR_BUCKET_POS_OFFSET(const struct bch_fs *c, +static inline struct bpos PTR_BUCKET_POS_OFFSET(const struct bch_dev *ca, const struct bch_extent_ptr *ptr, u32 *bucket_offset) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); - return POS(ptr->dev, sector_to_bucket_and_offset(ca, ptr->offset, bucket_offset)); } diff --git a/fs/bcachefs/data_update.c b/fs/bcachefs/data_update.c index c252f0619b736..1cf92bea7f9f7 100644 --- a/fs/bcachefs/data_update.c +++ b/fs/bcachefs/data_update.c @@ -357,10 +357,11 @@ void bch2_data_update_exit(struct data_update *update) bch2_bkey_ptrs_c(bkey_i_to_s_c(update->k.k)); bkey_for_each_ptr(ptrs, ptr) { + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); if (c->opts.nocow_enabled) bch2_bucket_nocow_unlock(&c->nocow_locks, - PTR_BUCKET_POS(c, ptr), 0); - bch2_dev_put(bch2_dev_bkey_exists(c, ptr->dev)); + PTR_BUCKET_POS(ca, ptr), 0); + bch2_dev_put(ca); } bch2_bkey_buf_exit(&update->k, c); @@ -547,6 +548,8 @@ int bch2_data_update_init(struct btree_trans *trans, i = 0; bkey_for_each_ptr_decode(k.k, ptrs, p, entry) { + struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); + struct bpos bucket = PTR_BUCKET_POS(ca, &p.ptr); bool locked; if (((1U << i) & m->data_opts.rewrite_ptrs)) { @@ -580,15 +583,13 @@ int bch2_data_update_init(struct btree_trans *trans, if (ctxt) { move_ctxt_wait_event(ctxt, (locked = bch2_bucket_nocow_trylock(&c->nocow_locks, - PTR_BUCKET_POS(c, &p.ptr), 0)) || + bucket, 0)) || list_empty(&ctxt->ios)); if (!locked) - bch2_bucket_nocow_lock(&c->nocow_locks, - PTR_BUCKET_POS(c, &p.ptr), 0); + bch2_bucket_nocow_lock(&c->nocow_locks, bucket, 0); } else { - if (!bch2_bucket_nocow_trylock(&c->nocow_locks, - PTR_BUCKET_POS(c, &p.ptr), 0)) { + if (!bch2_bucket_nocow_trylock(&c->nocow_locks, bucket, 0)) { ret = -BCH_ERR_nocow_lock_blocked; goto err; } @@ -650,10 +651,11 @@ int bch2_data_update_init(struct btree_trans *trans, err: i = 0; bkey_for_each_ptr_decode(k.k, ptrs, p, entry) { + struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); + struct bpos bucket = PTR_BUCKET_POS(ca, &p.ptr); if ((1U << i) & ptrs_locked) - bch2_bucket_nocow_unlock(&c->nocow_locks, - PTR_BUCKET_POS(c, &p.ptr), 0); - bch2_dev_put(bch2_dev_bkey_exists(c, p.ptr.dev)); + bch2_bucket_nocow_unlock(&c->nocow_locks, bucket, 0); + bch2_dev_put(ca); i++; } diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index ea10f838d9ccb..f09df7f9036a0 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -164,6 +164,7 @@ void bch2_stripe_to_text(struct printbuf *out, struct bch_fs *c, /* Triggers: */ static int __mark_stripe_bucket(struct btree_trans *trans, + struct bch_dev *ca, struct bkey_s_c_stripe s, unsigned ptr_idx, bool deleting, struct bpos bucket, @@ -179,13 +180,6 @@ static int __mark_stripe_bucket(struct btree_trans *trans, int ret = 0; struct bch_fs *c = trans->c; - struct bch_dev *ca = bch2_dev_tryget(c, ptr->dev); - if (unlikely(!ca)) { - if (!(flags & BTREE_TRIGGER_overwrite)) - ret = -EIO; - goto err; - } - if (deleting) sectors = -sectors; @@ -263,7 +257,6 @@ static int __mark_stripe_bucket(struct btree_trans *trans, alloc_data_type_set(a, data_type); err: - bch2_dev_put(ca); printbuf_exit(&buf); return ret; } @@ -275,34 +268,40 @@ static int mark_stripe_bucket(struct btree_trans *trans, { struct bch_fs *c = trans->c; const struct bch_extent_ptr *ptr = s.v->ptrs + ptr_idx; - struct bpos bucket = PTR_BUCKET_POS(c, ptr); + int ret = 0; + + struct bch_dev *ca = bch2_dev_tryget(c, ptr->dev); + if (unlikely(!ca)) { + if (!(flags & BTREE_TRIGGER_overwrite)) + ret = -EIO; + goto err; + } + + struct bpos bucket = PTR_BUCKET_POS(ca, ptr); if (flags & BTREE_TRIGGER_transactional) { struct bkey_i_alloc_v4 *a = bch2_trans_start_alloc_update(trans, bucket); - return PTR_ERR_OR_ZERO(a) ?: - __mark_stripe_bucket(trans, s, ptr_idx, deleting, bucket, &a->v, flags); + ret = PTR_ERR_OR_ZERO(a) ?: + __mark_stripe_bucket(trans, ca, s, ptr_idx, deleting, bucket, &a->v, flags); } if (flags & BTREE_TRIGGER_gc) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); - percpu_down_read(&c->mark_lock); struct bucket *g = gc_bucket(ca, bucket.offset); bucket_lock(g); struct bch_alloc_v4 old = bucket_m_to_alloc(*g), new = old; - int ret = __mark_stripe_bucket(trans, s, ptr_idx, deleting, bucket, &new, flags); + ret = __mark_stripe_bucket(trans, ca, s, ptr_idx, deleting, bucket, &new, flags); if (!ret) { alloc_to_bucket(g, new); bch2_dev_usage_update(c, ca, &old, &new, 0, true); } bucket_unlock(g); percpu_up_read(&c->mark_lock); - return ret; } - - BUG(); - return 0; +err: + bch2_dev_put(ca); + return ret; } static int mark_stripe_buckets(struct btree_trans *trans, @@ -1298,17 +1297,21 @@ static int ec_stripe_update_bucket(struct btree_trans *trans, struct ec_stripe_b { struct bch_fs *c = trans->c; struct bch_stripe *v = &bkey_i_to_stripe(&s->key)->v; - struct bch_extent_ptr bucket = v->ptrs[block]; - struct bpos bucket_pos = PTR_BUCKET_POS(c, &bucket); + struct bch_extent_ptr ptr = v->ptrs[block]; struct bpos bp_pos = POS_MIN; int ret = 0; + struct bch_dev *ca = bch2_dev_tryget(c, ptr.dev); + if (!ca) + return -EIO; + + struct bpos bucket_pos = PTR_BUCKET_POS(ca, &ptr); + while (1) { ret = commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_check_rw| BCH_TRANS_COMMIT_no_enospc, - ec_stripe_update_extent(trans, bucket_pos, bucket.gen, - s, &bp_pos)); + ec_stripe_update_extent(trans, bucket_pos, ptr.gen, s, &bp_pos)); if (ret) break; if (bkey_eq(bp_pos, POS_MAX)) @@ -1317,6 +1320,7 @@ static int ec_stripe_update_bucket(struct btree_trans *trans, struct ec_stripe_b bp_pos = bpos_nosnap_successor(bp_pos); } + bch2_dev_put(ca); return ret; } diff --git a/fs/bcachefs/io_read.c b/fs/bcachefs/io_read.c index af9bd7c45f21e..1091b066639e4 100644 --- a/fs/bcachefs/io_read.c +++ b/fs/bcachefs/io_read.c @@ -768,7 +768,7 @@ static noinline void read_from_stale_dirty_pointer(struct btree_trans *trans, int ret; bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc, - PTR_BUCKET_POS(c, &ptr), + PTR_BUCKET_POS(ca, &ptr), BTREE_ITER_cached); prt_printf(&buf, "Attempting to read from stale dirty pointer:\n"); diff --git a/fs/bcachefs/io_write.c b/fs/bcachefs/io_write.c index 55e24c83fb197..40c8f88442241 100644 --- a/fs/bcachefs/io_write.c +++ b/fs/bcachefs/io_write.c @@ -1117,10 +1117,12 @@ static inline void bch2_nocow_write_unlock(struct bch_write_op *op) for_each_keylist_key(&op->insert_keys, k) { struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(bkey_i_to_s_c(k)); - bkey_for_each_ptr(ptrs, ptr) + bkey_for_each_ptr(ptrs, ptr) { + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); bch2_bucket_nocow_unlock(&c->nocow_locks, - PTR_BUCKET_POS(c, ptr), + PTR_BUCKET_POS(ca, ptr), BUCKET_NOCOW_LOCK_UPDATE); + } } } @@ -1270,12 +1272,13 @@ static void bch2_nocow_write(struct bch_write_op *op) /* Get iorefs before dropping btree locks: */ struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); bkey_for_each_ptr(ptrs, ptr) { - struct bpos b = PTR_BUCKET_POS(c, ptr); + struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); + struct bpos b = PTR_BUCKET_POS(ca, ptr); struct nocow_lock_bucket *l = bucket_nocow_lock(&c->nocow_locks, bucket_to_u64(b)); prefetch(l); - if (unlikely(!bch2_dev_get_ioref(bch2_dev_bkey_exists(c, ptr->dev), WRITE))) + if (unlikely(!bch2_dev_get_ioref(ca, WRITE))) goto err_get_ioref; /* XXX allocating memory with btree locks held - rare */ From 633cf069445d95384d88e9ff40f395c001f8b2b7 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 16:50:28 -0400 Subject: [PATCH 1463/1702] bcachefs: Kill bch2_dev_bkey_exists() in backpointer code Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_foreground.c | 2 +- fs/bcachefs/backpointers.c | 106 ++++++++++++++++++++++----------- fs/bcachefs/backpointers.h | 37 ++++++++---- fs/bcachefs/buckets.c | 2 +- fs/bcachefs/ec.c | 5 +- fs/bcachefs/move.c | 2 +- 6 files changed, 102 insertions(+), 52 deletions(-) diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index f54d908683147..cc182b2630665 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -342,7 +342,7 @@ static struct open_bucket *try_alloc_bucket(struct btree_trans *trans, struct bc struct bch_backpointer bp; struct bpos bp_pos = POS_MIN; - ret = bch2_get_next_backpointer(trans, POS(ca->dev_idx, b), -1, + ret = bch2_get_next_backpointer(trans, ca, POS(ca->dev_idx, b), -1, &bp_pos, &bp, BTREE_ITER_nopreserve); if (ret) { diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c index 282aee70a51f8..3f5333b4c3dee 100644 --- a/fs/bcachefs/backpointers.c +++ b/fs/bcachefs/backpointers.c @@ -23,6 +23,7 @@ static bool extent_matches_bp(struct bch_fs *c, const union bch_extent_entry *entry; struct extent_ptr_decoded p; + rcu_read_lock(); bkey_for_each_ptr_decode(k.k, ptrs, p, entry) { struct bpos bucket2; struct bch_backpointer bp2; @@ -30,13 +31,18 @@ static bool extent_matches_bp(struct bch_fs *c, if (p.ptr.cached) continue; - struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); + struct bch_dev *ca = bch2_dev_rcu(c, p.ptr.dev); + if (!ca) + continue; bch2_extent_ptr_to_bp(c, ca, btree_id, level, k, p, entry, &bucket2, &bp2); if (bpos_eq(bucket, bucket2) && - !memcmp(&bp, &bp2, sizeof(bp))) + !memcmp(&bp, &bp2, sizeof(bp))) { + rcu_read_unlock(); return true; + } } + rcu_read_unlock(); return false; } @@ -47,16 +53,21 @@ int bch2_backpointer_invalid(struct bch_fs *c, struct bkey_s_c k, { struct bkey_s_c_backpointer bp = bkey_s_c_to_backpointer(k); - /* these will be caught by fsck */ - if (!bch2_dev_exists(c, bp.k->p.inode)) + rcu_read_lock(); + struct bch_dev *ca = bch2_dev_rcu(c, bp.k->p.inode); + if (!ca) { + /* these will be caught by fsck */ + rcu_read_unlock(); return 0; + } - struct bch_dev *ca = bch2_dev_bkey_exists(c, bp.k->p.inode); - struct bpos bucket = bp_pos_to_bucket(c, bp.k->p); + struct bpos bucket = bp_pos_to_bucket(ca, bp.k->p); + struct bpos bp_pos = bucket_pos_to_bp_noerror(ca, bucket, bp.v->bucket_offset); + rcu_read_unlock(); int ret = 0; bkey_fsck_err_on((bp.v->bucket_offset >> MAX_EXTENT_COMPRESS_RATIO_SHIFT) >= ca->mi.bucket_size || - !bpos_eq(bp.k->p, bucket_pos_to_bp_noerror(ca, bucket, bp.v->bucket_offset)), + !bpos_eq(bp.k->p, bp_pos), c, err, backpointer_bucket_offset_wrong, "backpointer bucket_offset wrong"); @@ -77,10 +88,16 @@ void bch2_backpointer_to_text(struct printbuf *out, const struct bch_backpointer void bch2_backpointer_k_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k) { - if (bch2_dev_exists(c, k.k->p.inode)) { + rcu_read_lock(); + struct bch_dev *ca = bch2_dev_rcu(c, k.k->p.inode); + if (ca) { + struct bpos bucket = bp_pos_to_bucket(ca, k.k->p); + rcu_read_unlock(); prt_str(out, "bucket="); - bch2_bpos_to_text(out, bp_pos_to_bucket(c, k.k->p)); + bch2_bpos_to_text(out, bucket); prt_str(out, " "); + } else { + rcu_read_unlock(); } bch2_backpointer_to_text(out, bkey_s_c_to_backpointer(k).v); @@ -146,6 +163,7 @@ static noinline int backpointer_mod_err(struct btree_trans *trans, } int bch2_bucket_backpointer_mod_nowritebuffer(struct btree_trans *trans, + struct bch_dev *ca, struct bpos bucket, struct bch_backpointer bp, struct bkey_s_c orig_k, @@ -162,7 +180,7 @@ int bch2_bucket_backpointer_mod_nowritebuffer(struct btree_trans *trans, return ret; bkey_backpointer_init(&bp_k->k_i); - bp_k->k.p = bucket_pos_to_bp(trans->c, bucket, bp.bucket_offset); + bp_k->k.p = bucket_pos_to_bp(ca, bucket, bp.bucket_offset); bp_k->v = bp; if (!insert) { @@ -198,13 +216,13 @@ int bch2_bucket_backpointer_mod_nowritebuffer(struct btree_trans *trans, * Find the next backpointer >= *bp_offset: */ int bch2_get_next_backpointer(struct btree_trans *trans, + struct bch_dev *ca, struct bpos bucket, int gen, struct bpos *bp_pos, struct bch_backpointer *bp, unsigned iter_flags) { - struct bch_fs *c = trans->c; - struct bpos bp_end_pos = bucket_pos_to_bp(c, bpos_nosnap_successor(bucket), 0); + struct bpos bp_end_pos = bucket_pos_to_bp(ca, bpos_nosnap_successor(bucket), 0); struct btree_iter alloc_iter = { NULL }, bp_iter = { NULL }; struct bkey_s_c k; int ret = 0; @@ -224,7 +242,7 @@ int bch2_get_next_backpointer(struct btree_trans *trans, goto done; } - *bp_pos = bpos_max(*bp_pos, bucket_pos_to_bp(c, bucket, 0)); + *bp_pos = bpos_max(*bp_pos, bucket_pos_to_bp(ca, bucket, 0)); for_each_btree_key_norestart(trans, bp_iter, BTREE_ID_backpointers, *bp_pos, iter_flags, k, ret) { @@ -250,7 +268,6 @@ static void backpointer_not_found(struct btree_trans *trans, { struct bch_fs *c = trans->c; struct printbuf buf = PRINTBUF; - struct bpos bucket = bp_pos_to_bucket(c, bp_pos); /* * If we're using the btree write buffer, the backpointer we were @@ -260,6 +277,10 @@ static void backpointer_not_found(struct btree_trans *trans, if (likely(!bch2_backpointers_no_use_write_buffer)) return; + struct bpos bucket; + if (!bp_pos_to_bucket_nodev(c, bp_pos, &bucket)) + return; + prt_printf(&buf, "backpointer doesn't match %s it points to:\n ", bp.level ? "btree node" : "extent"); prt_printf(&buf, "bucket: "); @@ -289,15 +310,17 @@ struct bkey_s_c bch2_backpointer_get_key(struct btree_trans *trans, { if (likely(!bp.level)) { struct bch_fs *c = trans->c; - struct bpos bucket = bp_pos_to_bucket(c, bp_pos); - struct bkey_s_c k; + + struct bpos bucket; + if (!bp_pos_to_bucket_nodev(c, bp_pos, &bucket)) + return bkey_s_c_err(-EIO); bch2_trans_node_iter_init(trans, iter, bp.btree_id, bp.pos, 0, 0, iter_flags); - k = bch2_btree_iter_peek_slot(iter); + struct bkey_s_c k = bch2_btree_iter_peek_slot(iter); if (bkey_err(k)) { bch2_trans_iter_exit(trans, iter); return k; @@ -326,18 +349,20 @@ struct btree *bch2_backpointer_get_node(struct btree_trans *trans, struct bch_backpointer bp) { struct bch_fs *c = trans->c; - struct bpos bucket = bp_pos_to_bucket(c, bp_pos); - struct btree *b; BUG_ON(!bp.level); + struct bpos bucket; + if (!bp_pos_to_bucket_nodev(c, bp_pos, &bucket)) + return ERR_PTR(-EIO); + bch2_trans_node_iter_init(trans, iter, bp.btree_id, bp.pos, 0, bp.level - 1, 0); - b = bch2_btree_iter_peek_node(iter); + struct btree *b = bch2_btree_iter_peek_node(iter); if (IS_ERR_OR_NULL(b)) goto err; @@ -368,16 +393,16 @@ static int bch2_check_btree_backpointer(struct btree_trans *trans, struct btree_ struct printbuf buf = PRINTBUF; int ret = 0; - if (fsck_err_on(!bch2_dev_exists(c, k.k->p.inode), c, - backpointer_to_missing_device, - "backpointer for missing device:\n%s", - (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { - ret = bch2_btree_delete_at(trans, bp_iter, 0); + struct bpos bucket; + if (!bp_pos_to_bucket_nodev_noerror(c, k.k->p, &bucket)) { + if (fsck_err(c, backpointer_to_missing_device, + "backpointer for missing device:\n%s", + (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) + ret = bch2_btree_delete_at(trans, bp_iter, 0); goto out; } - alloc_k = bch2_bkey_get_iter(trans, &alloc_iter, BTREE_ID_alloc, - bp_pos_to_bucket(c, k.k->p), 0); + alloc_k = bch2_bkey_get_iter(trans, &alloc_iter, BTREE_ID_alloc, bucket, 0); ret = bkey_err(alloc_k); if (ret) goto out; @@ -512,25 +537,27 @@ static int check_bp_exists(struct btree_trans *trans, struct printbuf buf = PRINTBUF; struct bkey_s_c bp_k; struct bkey_buf tmp; - int ret; + int ret = 0; bch2_bkey_buf_init(&tmp); - if (!bch2_dev_bucket_exists(c, bucket)) { + struct bch_dev *ca = bch2_dev_bucket_tryget(c, bucket); + if (!ca) { prt_str(&buf, "extent for nonexistent device:bucket "); bch2_bpos_to_text(&buf, bucket); prt_str(&buf, "\n "); bch2_bkey_val_to_text(&buf, c, orig_k); bch_err(c, "%s", buf.buf); - return -BCH_ERR_fsck_repair_unimplemented; + ret = -BCH_ERR_fsck_repair_unimplemented; + goto err; } if (bpos_lt(bucket, s->bucket_start) || bpos_gt(bucket, s->bucket_end)) - return 0; + goto out; bp_k = bch2_bkey_get_iter(trans, &bp_iter, BTREE_ID_backpointers, - bucket_pos_to_bp(c, bucket, bp.bucket_offset), + bucket_pos_to_bp(ca, bucket, bp.bucket_offset), 0); ret = bkey_err(bp_k); if (ret) @@ -563,6 +590,7 @@ static int check_bp_exists(struct btree_trans *trans, bch2_trans_iter_exit(trans, &other_extent_iter); bch2_trans_iter_exit(trans, &bp_iter); bch2_bkey_buf_exit(&tmp, c); + bch2_dev_put(ca); printbuf_exit(&buf); return ret; check_existing_bp: @@ -638,13 +666,13 @@ static int check_bp_exists(struct btree_trans *trans, struct bkey_i_backpointer n_bp_k; bkey_backpointer_init(&n_bp_k.k_i); - n_bp_k.k.p = bucket_pos_to_bp(trans->c, bucket, bp.bucket_offset); + n_bp_k.k.p = bucket_pos_to_bp(ca, bucket, bp.bucket_offset); n_bp_k.v = bp; prt_printf(&buf, "\n want: "); bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&n_bp_k.k_i)); if (fsck_err(c, ptr_to_missing_backpointer, "%s", buf.buf)) - ret = bch2_bucket_backpointer_mod(trans, bucket, bp, orig_k, true); + ret = bch2_bucket_backpointer_mod(trans, ca, bucket, bp, orig_k, true); goto out; } @@ -668,8 +696,14 @@ static int check_extent_to_backpointers(struct btree_trans *trans, if (p.ptr.cached) continue; - struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); - bch2_extent_ptr_to_bp(c, ca, btree, level, k, p, entry, &bucket_pos, &bp); + rcu_read_lock(); + struct bch_dev *ca = bch2_dev_rcu(c, p.ptr.dev); + if (ca) + bch2_extent_ptr_to_bp(c, ca, btree, level, k, p, entry, &bucket_pos, &bp); + rcu_read_unlock(); + + if (!ca) + continue; ret = check_bp_exists(trans, s, bucket_pos, bp, k); if (ret) diff --git a/fs/bcachefs/backpointers.h b/fs/bcachefs/backpointers.h index 1d270c40fce9a..929f4a068a0a0 100644 --- a/fs/bcachefs/backpointers.h +++ b/fs/bcachefs/backpointers.h @@ -6,6 +6,7 @@ #include "btree_iter.h" #include "btree_update.h" #include "buckets.h" +#include "error.h" #include "super.h" static inline u64 swab40(u64 x) @@ -36,15 +37,29 @@ void bch2_backpointer_swab(struct bkey_s); * Convert from pos in backpointer btree to pos of corresponding bucket in alloc * btree: */ -static inline struct bpos bp_pos_to_bucket(const struct bch_fs *c, - struct bpos bp_pos) +static inline struct bpos bp_pos_to_bucket(const struct bch_dev *ca, struct bpos bp_pos) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, bp_pos.inode); u64 bucket_sector = bp_pos.offset >> MAX_EXTENT_COMPRESS_RATIO_SHIFT; return POS(bp_pos.inode, sector_to_bucket(ca, bucket_sector)); } +static inline bool bp_pos_to_bucket_nodev_noerror(struct bch_fs *c, struct bpos bp_pos, struct bpos *bucket) +{ + rcu_read_lock(); + struct bch_dev *ca = bch2_dev_rcu(c, bp_pos.inode); + if (ca) + *bucket = bp_pos_to_bucket(ca, bp_pos); + rcu_read_unlock(); + return ca != NULL; +} + +static inline bool bp_pos_to_bucket_nodev(struct bch_fs *c, struct bpos bp_pos, struct bpos *bucket) +{ + return !bch2_fs_inconsistent_on(!bp_pos_to_bucket_nodev_noerror(c, bp_pos, bucket), + c, "backpointer for missing device %llu", bp_pos.inode); +} + static inline struct bpos bucket_pos_to_bp_noerror(const struct bch_dev *ca, struct bpos bucket, u64 bucket_offset) @@ -57,32 +72,32 @@ static inline struct bpos bucket_pos_to_bp_noerror(const struct bch_dev *ca, /* * Convert from pos in alloc btree + bucket offset to pos in backpointer btree: */ -static inline struct bpos bucket_pos_to_bp(const struct bch_fs *c, +static inline struct bpos bucket_pos_to_bp(const struct bch_dev *ca, struct bpos bucket, u64 bucket_offset) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, bucket.inode); struct bpos ret = bucket_pos_to_bp_noerror(ca, bucket, bucket_offset); - EBUG_ON(!bkey_eq(bucket, bp_pos_to_bucket(c, ret))); + EBUG_ON(!bkey_eq(bucket, bp_pos_to_bucket(ca, ret))); return ret; } -int bch2_bucket_backpointer_mod_nowritebuffer(struct btree_trans *, struct bpos bucket, - struct bch_backpointer, struct bkey_s_c, bool); +int bch2_bucket_backpointer_mod_nowritebuffer(struct btree_trans *, struct bch_dev *, + struct bpos bucket, struct bch_backpointer, struct bkey_s_c, bool); static inline int bch2_bucket_backpointer_mod(struct btree_trans *trans, + struct bch_dev *ca, struct bpos bucket, struct bch_backpointer bp, struct bkey_s_c orig_k, bool insert) { if (unlikely(bch2_backpointers_no_use_write_buffer)) - return bch2_bucket_backpointer_mod_nowritebuffer(trans, bucket, bp, orig_k, insert); + return bch2_bucket_backpointer_mod_nowritebuffer(trans, ca, bucket, bp, orig_k, insert); struct bkey_i_backpointer bp_k; bkey_backpointer_init(&bp_k.k_i); - bp_k.k.p = bucket_pos_to_bp(trans->c, bucket, bp.bucket_offset); + bp_k.k.p = bucket_pos_to_bp(ca, bucket, bp.bucket_offset); bp_k.v = bp; if (!insert) { @@ -142,7 +157,7 @@ static inline void bch2_extent_ptr_to_bp(struct bch_fs *c, struct bch_dev *ca, }; } -int bch2_get_next_backpointer(struct btree_trans *, struct bpos, int, +int bch2_get_next_backpointer(struct btree_trans *, struct bch_dev *ca, struct bpos, int, struct bpos *, struct bch_backpointer *, unsigned); struct bkey_s_c bch2_backpointer_get_key(struct btree_trans *, struct btree_iter *, struct bpos, struct bch_backpointer, diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 4774f9c7f060e..ae6d6879e2e02 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -988,7 +988,7 @@ static int bch2_trigger_pointer(struct btree_trans *trans, goto err; if (!p.ptr.cached) { - ret = bch2_bucket_backpointer_mod(trans, bucket, bp, k, insert); + ret = bch2_bucket_backpointer_mod(trans, ca, bucket, bp, k, insert); if (ret) goto err; } diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index f09df7f9036a0..6d41530615cbd 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -1198,6 +1198,7 @@ static int ec_stripe_key_update(struct btree_trans *trans, } static int ec_stripe_update_extent(struct btree_trans *trans, + struct bch_dev *ca, struct bpos bucket, u8 gen, struct ec_stripe_buf *s, struct bpos *bp_pos) @@ -1213,7 +1214,7 @@ static int ec_stripe_update_extent(struct btree_trans *trans, struct bkey_i *n; int ret, dev, block; - ret = bch2_get_next_backpointer(trans, bucket, gen, + ret = bch2_get_next_backpointer(trans, ca, bucket, gen, bp_pos, &bp, BTREE_ITER_cached); if (ret) return ret; @@ -1311,7 +1312,7 @@ static int ec_stripe_update_bucket(struct btree_trans *trans, struct ec_stripe_b ret = commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_check_rw| BCH_TRANS_COMMIT_no_enospc, - ec_stripe_update_extent(trans, bucket_pos, ptr.gen, s, &bp_pos)); + ec_stripe_update_extent(trans, ca, bucket_pos, ptr.gen, s, &bp_pos)); if (ret) break; if (bkey_eq(bp_pos, POS_MAX)) diff --git a/fs/bcachefs/move.c b/fs/bcachefs/move.c index c441ce7c92eca..0e3203de1cd64 100644 --- a/fs/bcachefs/move.c +++ b/fs/bcachefs/move.c @@ -729,7 +729,7 @@ int bch2_evacuate_bucket(struct moving_context *ctxt, bch2_trans_begin(trans); - ret = bch2_get_next_backpointer(trans, bucket, gen, + ret = bch2_get_next_backpointer(trans, ca, bucket, gen, &bp_pos, &bp, BTREE_ITER_cached); if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) From dbd0408087853c7842aa21f58f99d395eff02544 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 20:32:44 -0400 Subject: [PATCH 1464/1702] bcachefs: move replica_set from bch_dev to bch_fs This is needed for the next patch - the write submit path has to be able to allocate a replica bio even when we weren't able to get a ref on the device. Signed-off-by: Kent Overstreet --- fs/bcachefs/bcachefs.h | 4 +--- fs/bcachefs/io_write.c | 8 ++++---- fs/bcachefs/super.c | 3 --- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index f5ba8c6a34d73..22d303813d31f 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -569,9 +569,6 @@ struct bch_dev { struct bch_devs_mask self; - /* biosets used in cloned bios for writing multiple replicas */ - struct bio_set replica_set; - /* * Buckets: * Per-bucket arrays are protected by c->mark_lock, bucket_lock and @@ -995,6 +992,7 @@ struct bch_fs { struct bio_set bio_read; struct bio_set bio_read_split; struct bio_set bio_write; + struct bio_set replica_set; struct mutex bio_bounce_pages_lock; mempool_t bio_bounce_pages; struct bucket_nocow_lock_table diff --git a/fs/bcachefs/io_write.c b/fs/bcachefs/io_write.c index 40c8f88442241..5d9f349a15043 100644 --- a/fs/bcachefs/io_write.c +++ b/fs/bcachefs/io_write.c @@ -412,8 +412,7 @@ void bch2_submit_wbio_replicas(struct bch_write_bio *wbio, struct bch_fs *c, struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); if (to_entry(ptr + 1) < ptrs.end) { - n = to_wbio(bio_alloc_clone(NULL, &wbio->bio, - GFP_NOFS, &ca->replica_set)); + n = to_wbio(bio_alloc_clone(NULL, &wbio->bio, GFP_NOFS, &c->replica_set)); n->bio.bi_end_io = wbio->bio.bi_end_io; n->bio.bi_private = wbio->bio.bi_private; @@ -1667,13 +1666,14 @@ void bch2_write_op_to_text(struct printbuf *out, struct bch_write_op *op) void bch2_fs_io_write_exit(struct bch_fs *c) { mempool_exit(&c->bio_bounce_pages); + bioset_exit(&c->replica_set); bioset_exit(&c->bio_write); } int bch2_fs_io_write_init(struct bch_fs *c) { - if (bioset_init(&c->bio_write, 1, offsetof(struct bch_write_bio, bio), - BIOSET_NEED_BVECS)) + if (bioset_init(&c->bio_write, 1, offsetof(struct bch_write_bio, bio), BIOSET_NEED_BVECS) || + bioset_init(&c->replica_set, 4, offsetof(struct bch_write_bio, bio), 0)) return -BCH_ERR_ENOMEM_bio_write_init; if (mempool_init_page_pool(&c->bio_bounce_pages, diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index ed135166ab86e..294a9d35a9f25 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -1193,7 +1193,6 @@ static void bch2_dev_free(struct bch_dev *ca) bch2_dev_journal_exit(ca); free_percpu(ca->io_done); - bioset_exit(&ca->replica_set); bch2_dev_buckets_free(ca); free_page((unsigned long) ca->sb_read_scratch); @@ -1317,8 +1316,6 @@ static struct bch_dev *__bch2_dev_alloc(struct bch_fs *c, PERCPU_REF_INIT_DEAD, GFP_KERNEL) || !(ca->sb_read_scratch = (void *) __get_free_page(GFP_KERNEL)) || bch2_dev_buckets_alloc(c, ca) || - bioset_init(&ca->replica_set, 4, - offsetof(struct bch_write_bio, bio), 0) || !(ca->io_done = alloc_percpu(*ca->io_done))) goto err; From 8783856ab15e9ad6faea8df3b72658f4cd2325c3 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 20:38:05 -0400 Subject: [PATCH 1465/1702] bcachefs: ob_dev() Wrapper around bch2_dev_have_ref() for open_buckets; we do guarantee that the device an open_bucket points to exists. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_foreground.c | 15 +++++++-------- fs/bcachefs/alloc_foreground.h | 7 ++++++- fs/bcachefs/ec.c | 7 ++----- fs/bcachefs/sb-members.h | 7 +++++++ 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index cc182b2630665..77f3723e0fc0f 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -100,7 +100,7 @@ static void bch2_open_bucket_hash_remove(struct bch_fs *c, struct open_bucket *o void __bch2_open_bucket_put(struct bch_fs *c, struct open_bucket *ob) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ob->dev); + struct bch_dev *ca = ob_dev(c, ob); if (ob->ec) { ec_stripe_new_put(c, ob->ec, STRIPE_REF_io); @@ -684,8 +684,7 @@ static int add_new_bucket(struct bch_fs *c, unsigned flags, struct open_bucket *ob) { - unsigned durability = - bch2_dev_bkey_exists(c, ob->dev)->mi.durability; + unsigned durability = ob_dev(c, ob)->mi.durability; BUG_ON(*nr_effective >= nr_replicas); @@ -831,7 +830,7 @@ static bool want_bucket(struct bch_fs *c, bool *have_cache, bool ec, struct open_bucket *ob) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ob->dev); + struct bch_dev *ca = ob_dev(c, ob); if (!test_bit(ob->dev, devs_may_alloc->d)) return false; @@ -901,7 +900,7 @@ static int bucket_alloc_set_partial(struct bch_fs *c, struct open_bucket *ob = c->open_buckets + c->open_buckets_partial[i]; if (want_bucket(c, wp, devs_may_alloc, have_cache, ec, ob)) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ob->dev); + struct bch_dev *ca = ob_dev(c, ob); struct bch_dev_usage usage; u64 avail; @@ -1286,7 +1285,7 @@ deallocate_extra_replicas(struct bch_fs *c, unsigned i; open_bucket_for_each(c, ptrs, ob, i) { - unsigned d = bch2_dev_bkey_exists(c, ob->dev)->mi.durability; + unsigned d = ob_dev(c, ob)->mi.durability; if (d && d <= extra_replicas) { extra_replicas -= d; @@ -1443,7 +1442,7 @@ int bch2_alloc_sectors_start_trans(struct btree_trans *trans, struct bch_extent_ptr bch2_ob_ptr(struct bch_fs *c, struct open_bucket *ob) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ob->dev); + struct bch_dev *ca = ob_dev(c, ob); return (struct bch_extent_ptr) { .type = 1 << BCH_EXTENT_ENTRY_ptr, @@ -1519,7 +1518,7 @@ void bch2_fs_allocator_foreground_init(struct bch_fs *c) static void bch2_open_bucket_to_text(struct printbuf *out, struct bch_fs *c, struct open_bucket *ob) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ob->dev); + struct bch_dev *ca = ob_dev(c, ob); unsigned data_type = ob->data_type; barrier(); /* READ_ONCE() doesn't work on bitfields */ diff --git a/fs/bcachefs/alloc_foreground.h b/fs/bcachefs/alloc_foreground.h index 96f24b67d0663..a42c9730d32a8 100644 --- a/fs/bcachefs/alloc_foreground.h +++ b/fs/bcachefs/alloc_foreground.h @@ -30,6 +30,11 @@ void bch2_dev_stripe_increment(struct bch_dev *, struct dev_stripe_state *); long bch2_bucket_alloc_new_fs(struct bch_dev *); +static inline struct bch_dev *ob_dev(struct bch_fs *c, struct open_bucket *ob) +{ + return bch2_dev_have_ref(c, ob->dev); +} + struct open_bucket *bch2_bucket_alloc(struct bch_fs *, struct bch_dev *, enum bch_watermark, enum bch_data_type, struct closure *); @@ -185,7 +190,7 @@ bch2_alloc_sectors_append_ptrs_inlined(struct bch_fs *c, struct write_point *wp, wp->sectors_allocated += sectors; open_bucket_for_each(c, &wp->ptrs, ob, i) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ob->dev); + struct bch_dev *ca = ob_dev(c, ob); struct bch_extent_ptr ptr = bch2_ob_ptr(c, ob); ptr.cached = cached || diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 6d41530615cbd..fb5aa2b763ca3 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -1550,16 +1550,13 @@ void bch2_ec_bucket_cancel(struct bch_fs *c, struct open_bucket *ob) void *bch2_writepoint_ec_buf(struct bch_fs *c, struct write_point *wp) { struct open_bucket *ob = ec_open_bucket(c, &wp->ptrs); - struct bch_dev *ca; - unsigned offset; - if (!ob) return NULL; BUG_ON(!ob->ec->new_stripe.data[ob->ec_idx]); - ca = bch2_dev_bkey_exists(c, ob->dev); - offset = ca->mi.bucket_size - ob->sectors_free; + struct bch_dev *ca = ob_dev(c, ob); + unsigned offset = ca->mi.bucket_size - ob->sectors_free; return ob->ec->new_stripe.data[ob->ec_idx] + (offset << 9); } diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index d6d391dc21ca7..a3ba1ddebe4d3 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -195,6 +195,13 @@ static inline bool bucket_valid(const struct bch_dev *ca, u64 b) return b - ca->mi.first_bucket < ca->mi.nbuckets_minus_first; } +static inline struct bch_dev *bch2_dev_have_ref(const struct bch_fs *c, unsigned dev) +{ + EBUG_ON(!bch2_dev_exists(c, dev)); + + return rcu_dereference_check(c->devs[dev], 1); +} + /* * If a key exists that references a device, the device won't be going away and * we can omit rcu_read_lock(): From c387d84413711388a1bc5bf551c38f4b02529cc4 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 20:42:50 -0400 Subject: [PATCH 1466/1702] bcachefs: ec_validate_checksums() -> bch2_dev_tryget() Signed-off-by: Kent Overstreet --- fs/bcachefs/ec.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index fb5aa2b763ca3..be8bddb38957a 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -633,19 +633,21 @@ static void ec_validate_checksums(struct bch_fs *c, struct ec_stripe_buf *buf) struct bch_csum got = ec_block_checksum(buf, i, offset); if (bch2_crc_cmp(want, got)) { - struct printbuf err = PRINTBUF; - struct bch_dev *ca = bch2_dev_bkey_exists(c, v->ptrs[i].dev); + struct bch_dev *ca = bch2_dev_tryget(c, v->ptrs[i].dev); + if (ca) { + struct printbuf err = PRINTBUF; - prt_str(&err, "stripe "); - bch2_csum_err_msg(&err, v->csum_type, want, got); - prt_printf(&err, " for %ps at %u of\n ", (void *) _RET_IP_, i); - bch2_bkey_val_to_text(&err, c, bkey_i_to_s_c(&buf->key)); - bch_err_ratelimited(ca, "%s", err.buf); - printbuf_exit(&err); + prt_str(&err, "stripe "); + bch2_csum_err_msg(&err, v->csum_type, want, got); + prt_printf(&err, " for %ps at %u of\n ", (void *) _RET_IP_, i); + bch2_bkey_val_to_text(&err, c, bkey_i_to_s_c(&buf->key)); + bch_err_ratelimited(ca, "%s", err.buf); + printbuf_exit(&err); - clear_bit(i, buf->valid); + bch2_io_error(ca, BCH_MEMBER_ERROR_checksum); + } - bch2_io_error(ca, BCH_MEMBER_ERROR_checksum); + clear_bit(i, buf->valid); break; } From 3793b3f91f88269ff8ad8ed2b95c6779415e7fd5 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 20:49:22 -0400 Subject: [PATCH 1467/1702] bcachefs: bch2_extent_merge() -> bch2_dev_rcu() Signed-off-by: Kent Overstreet --- fs/bcachefs/extents.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index 6376440f6f30f..46f8164b1a49d 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -252,7 +252,6 @@ bool bch2_extent_merge(struct bch_fs *c, struct bkey_s l, struct bkey_s_c r) const union bch_extent_entry *en_r; struct extent_ptr_decoded lp, rp; bool use_right_ptr; - struct bch_dev *ca; en_l = l_ptrs.start; en_r = r_ptrs.start; @@ -283,8 +282,12 @@ bool bch2_extent_merge(struct bch_fs *c, struct bkey_s l, struct bkey_s_c r) return false; /* Extents may not straddle buckets: */ - ca = bch2_dev_bkey_exists(c, lp.ptr.dev); - if (PTR_BUCKET_NR(ca, &lp.ptr) != PTR_BUCKET_NR(ca, &rp.ptr)) + rcu_read_lock(); + struct bch_dev *ca = bch2_dev_rcu(c, lp.ptr.dev); + bool same_bucket = ca && PTR_BUCKET_NR(ca, &lp.ptr) == PTR_BUCKET_NR(ca, &rp.ptr); + rcu_read_unlock(); + + if (!same_bucket) return false; if (lp.has_ec != rp.has_ec || From 302c980a818763d53dfa8fcd8549b04a5c7b8b17 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 20:54:20 -0400 Subject: [PATCH 1468/1702] bcachefs: extent_ptr_durability() -> bch2_dev_rcu() Signed-off-by: Kent Overstreet --- fs/bcachefs/data_update.c | 5 +++++ fs/bcachefs/extents.c | 12 ++++++++---- fs/bcachefs/io_write.c | 7 ++++++- fs/bcachefs/move.c | 2 ++ 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/fs/bcachefs/data_update.c b/fs/bcachefs/data_update.c index 1cf92bea7f9f7..7aad6085ef53e 100644 --- a/fs/bcachefs/data_update.c +++ b/fs/bcachefs/data_update.c @@ -203,6 +203,8 @@ static int __bch2_data_update_index_update(struct btree_trans *trans, /* Now, drop excess replicas: */ restart_drop_extra_replicas: + + rcu_read_lock(); bkey_for_each_ptr_decode(old.k, bch2_bkey_ptrs(bkey_i_to_s(insert)), p, entry) { unsigned ptr_durability = bch2_extent_ptr_durability(c, &p); @@ -214,6 +216,7 @@ static int __bch2_data_update_index_update(struct btree_trans *trans, goto restart_drop_extra_replicas; } } + rcu_read_unlock(); /* Finally, add the pointers we just wrote: */ extent_for_each_ptr_decode(extent_i_to_s(new), p, entry) @@ -552,6 +555,7 @@ int bch2_data_update_init(struct btree_trans *trans, struct bpos bucket = PTR_BUCKET_POS(ca, &p.ptr); bool locked; + rcu_read_lock(); if (((1U << i) & m->data_opts.rewrite_ptrs)) { BUG_ON(p.ptr.cached); @@ -565,6 +569,7 @@ int bch2_data_update_init(struct btree_trans *trans, bch2_dev_list_add_dev(&m->op.devs_have, p.ptr.dev); durability_have += bch2_extent_ptr_durability(c, &p); } + rcu_read_unlock(); /* * op->csum_type is normally initialized from the fs/file's diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index 46f8164b1a49d..dc78ace561810 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -675,16 +675,16 @@ static inline unsigned __extent_ptr_durability(struct bch_dev *ca, struct extent unsigned bch2_extent_ptr_desired_durability(struct bch_fs *c, struct extent_ptr_decoded *p) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, p->ptr.dev); + struct bch_dev *ca = bch2_dev_rcu(c, p->ptr.dev); - return __extent_ptr_durability(ca, p); + return ca ? __extent_ptr_durability(ca, p) : 0; } unsigned bch2_extent_ptr_durability(struct bch_fs *c, struct extent_ptr_decoded *p) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, p->ptr.dev); + struct bch_dev *ca = bch2_dev_rcu(c, p->ptr.dev); - if (ca->mi.state == BCH_MEMBER_STATE_failed) + if (!ca || ca->mi.state == BCH_MEMBER_STATE_failed) return 0; return __extent_ptr_durability(ca, p); @@ -697,8 +697,10 @@ unsigned bch2_bkey_durability(struct bch_fs *c, struct bkey_s_c k) struct extent_ptr_decoded p; unsigned durability = 0; + rcu_read_lock(); bkey_for_each_ptr_decode(k.k, ptrs, p, entry) durability += bch2_extent_ptr_durability(c, &p); + rcu_read_unlock(); return durability; } @@ -710,9 +712,11 @@ static unsigned bch2_bkey_durability_safe(struct bch_fs *c, struct bkey_s_c k) struct extent_ptr_decoded p; unsigned durability = 0; + rcu_read_lock(); bkey_for_each_ptr_decode(k.k, ptrs, p, entry) if (p.ptr.dev < c->sb.nr_devices && c->devs[p.ptr.dev]) durability += bch2_extent_ptr_durability(c, &p); + rcu_read_unlock(); return durability; } diff --git a/fs/bcachefs/io_write.c b/fs/bcachefs/io_write.c index 5d9f349a15043..285e1a23bde5b 100644 --- a/fs/bcachefs/io_write.c +++ b/fs/bcachefs/io_write.c @@ -1099,12 +1099,17 @@ static bool bch2_extent_is_writeable(struct bch_write_op *op, return false; e = bkey_s_c_to_extent(k); + + rcu_read_lock(); extent_for_each_ptr_decode(e, p, entry) { - if (crc_is_encoded(p.crc) || p.has_ec) + if (crc_is_encoded(p.crc) || p.has_ec) { + rcu_read_unlock(); return false; + } replicas += bch2_extent_ptr_durability(c, &p); } + rcu_read_unlock(); return replicas >= op->opts.data_replicas; } diff --git a/fs/bcachefs/move.c b/fs/bcachefs/move.c index 0e3203de1cd64..8171f947fac8f 100644 --- a/fs/bcachefs/move.c +++ b/fs/bcachefs/move.c @@ -1033,6 +1033,7 @@ static bool drop_extra_replicas_pred(struct bch_fs *c, void *arg, struct extent_ptr_decoded p; unsigned i = 0; + rcu_read_lock(); bkey_for_each_ptr_decode(k.k, bch2_bkey_ptrs_c(k), p, entry) { unsigned d = bch2_extent_ptr_durability(c, &p); @@ -1043,6 +1044,7 @@ static bool drop_extra_replicas_pred(struct bch_fs *c, void *arg, i++; } + rcu_read_unlock(); return data_opts->kill_ptrs != 0; } From 3858aa4268b2f71827097f548879e4405b4cc995 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 20:56:54 -0400 Subject: [PATCH 1469/1702] bcachefs: ptr_stale() -> dev_ptr_stale() Signed-off-by: Kent Overstreet --- fs/bcachefs/bcachefs.h | 2 +- fs/bcachefs/btree_gc.c | 2 +- fs/bcachefs/buckets.h | 14 ++++++++------ fs/bcachefs/ec.c | 4 ++-- fs/bcachefs/extents.c | 8 ++++---- fs/bcachefs/io_read.c | 4 ++-- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index 22d303813d31f..ab415f0dd14ee 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -573,7 +573,7 @@ struct bch_dev { * Buckets: * Per-bucket arrays are protected by c->mark_lock, bucket_lock and * gc_lock, for device resize - holding any is sufficient for access: - * Or rcu_read_lock(), but only for ptr_stale(): + * Or rcu_read_lock(), but only for dev_ptr_stale(): */ struct bucket_array __rcu *buckets_gc; struct bucket_gens __rcu *bucket_gens; diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 07b343e7ddeb9..05ebf1186be10 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -1257,7 +1257,7 @@ static int gc_btree_gens_key(struct btree_trans *trans, bkey_for_each_ptr(ptrs, ptr) { struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); - if (ptr_stale(ca, ptr) > 16) { + if (dev_ptr_stale(ca, ptr) > 16) { percpu_up_read(&c->mark_lock); goto update; } diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h index 0a54faf7c50a6..617ffde2fb7ad 100644 --- a/fs/bcachefs/buckets.h +++ b/fs/bcachefs/buckets.h @@ -170,17 +170,19 @@ static inline int gen_after(u8 a, u8 b) return r > 0 ? r : 0; } +static inline u8 dev_ptr_stale_rcu(struct bch_dev *ca, const struct bch_extent_ptr *ptr) +{ + return gen_after(*bucket_gen(ca, PTR_BUCKET_NR(ca, ptr)), ptr->gen); +} + /** - * ptr_stale() - check if a pointer points into a bucket that has been + * dev_ptr_stale() - check if a pointer points into a bucket that has been * invalidated. */ -static inline u8 ptr_stale(struct bch_dev *ca, - const struct bch_extent_ptr *ptr) +static inline u8 dev_ptr_stale(struct bch_dev *ca, const struct bch_extent_ptr *ptr) { - u8 ret; - rcu_read_lock(); - ret = gen_after(*bucket_gen(ca, PTR_BUCKET_NR(ca, ptr)), ptr->gen); + u8 ret = dev_ptr_stale_rcu(ca, ptr); rcu_read_unlock(); return ret; diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index be8bddb38957a..99a8c8fe8e3b4 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -714,7 +714,7 @@ static void ec_block_endio(struct bio *bio) bch2_blk_status_to_str(bio->bi_status))) clear_bit(ec_bio->idx, ec_bio->buf->valid); - if (ptr_stale(ca, ptr)) { + if (dev_ptr_stale(ca, ptr)) { bch_err_ratelimited(ca->fs, "error %s stripe: stale pointer after io", bio_data_dir(bio) == READ ? "reading from" : "writing to"); @@ -738,7 +738,7 @@ static void ec_block_io(struct bch_fs *c, struct ec_stripe_buf *buf, : BCH_DATA_parity; int rw = op_is_write(opf); - if (ptr_stale(ca, ptr)) { + if (dev_ptr_stale(ca, ptr)) { bch_err_ratelimited(c, "error %s stripe: stale pointer", rw == READ ? "reading from" : "writing to"); diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index dc78ace561810..a12fffb32f61b 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -132,7 +132,7 @@ int bch2_bkey_pick_read_device(struct bch_fs *c, struct bkey_s_c k, if (!ret && !p.ptr.cached) ret = -EIO; - if (p.ptr.cached && ptr_stale(ca, &p.ptr)) + if (p.ptr.cached && dev_ptr_stale(ca, &p.ptr)) continue; f = failed ? dev_io_failures(failed, p.ptr.dev) : NULL; @@ -874,7 +874,7 @@ bool bch2_bkey_has_target(struct bch_fs *c, struct bkey_s_c k, unsigned target) bkey_for_each_ptr(ptrs, ptr) if (bch2_dev_in_target(c, ptr->dev, target) && (!ptr->cached || - !ptr_stale(bch2_dev_bkey_exists(c, ptr->dev), ptr))) + !dev_ptr_stale(bch2_dev_bkey_exists(c, ptr->dev), ptr))) return true; return false; @@ -981,7 +981,7 @@ bool bch2_extent_normalize(struct bch_fs *c, struct bkey_s k) { bch2_bkey_drop_ptrs(k, ptr, ptr->cached && - ptr_stale(bch2_dev_bkey_exists(c, ptr->dev), ptr)); + dev_ptr_stale(bch2_dev_bkey_exists(c, ptr->dev), ptr)); return bkey_deleted(k.k); } @@ -1005,7 +1005,7 @@ void bch2_extent_ptr_to_text(struct printbuf *out, struct bch_fs *c, const struc prt_str(out, " cached"); if (ptr->unwritten) prt_str(out, " unwritten"); - if (bucket_valid(ca, b) && ptr_stale(ca, ptr)) + if (bucket_valid(ca, b) && dev_ptr_stale_rcu(ca, ptr)) prt_printf(out, " stale"); } rcu_read_unlock(); diff --git a/fs/bcachefs/io_read.c b/fs/bcachefs/io_read.c index 1091b066639e4..0107b2fb4d333 100644 --- a/fs/bcachefs/io_read.c +++ b/fs/bcachefs/io_read.c @@ -697,7 +697,7 @@ static void bch2_read_endio(struct bio *bio) } if (((rbio->flags & BCH_READ_RETRY_IF_STALE) && race_fault()) || - ptr_stale(ca, &rbio->pick.ptr)) { + dev_ptr_stale(ca, &rbio->pick.ptr)) { trace_and_count(c, read_reuse_race, &rbio->bio); if (rbio->flags & BCH_READ_RETRY_IF_STALE) @@ -841,7 +841,7 @@ int __bch2_read_extent(struct btree_trans *trans, struct bch_read_bio *orig, */ if ((flags & BCH_READ_IN_RETRY) && !pick.ptr.cached && - unlikely(ptr_stale(ca, &pick.ptr))) { + unlikely(dev_ptr_stale(ca, &pick.ptr))) { read_from_stale_dirty_pointer(trans, k, pick.ptr); bch2_mark_io_failure(failed, &pick); goto retry_pick; From 8feecbed241b96957cd286e68bbcc113d2e45750 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 21:09:45 -0400 Subject: [PATCH 1470/1702] bcachefs: extent_ptr_invalid() -> bch2_dev_rcu() Signed-off-by: Kent Overstreet --- fs/bcachefs/extents.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index a12fffb32f61b..5f5038f0ca7cf 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -1083,9 +1083,6 @@ static int extent_ptr_invalid(struct bch_fs *c, struct printbuf *err) { struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); - u64 bucket; - u32 bucket_offset; - struct bch_dev *ca; int ret = 0; if (!bch2_dev_exists(c, ptr->dev)) { @@ -1101,24 +1098,35 @@ static int extent_ptr_invalid(struct bch_fs *c, "pointer to invalid device (%u)", ptr->dev); } - ca = bch2_dev_bkey_exists(c, ptr->dev); + rcu_read_lock(); + struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev); + if (!ca) { + rcu_read_unlock(); + return 0; + } + u32 bucket_offset; + u64 bucket = sector_to_bucket_and_offset(ca, ptr->offset, &bucket_offset); + unsigned first_bucket = ca->mi.first_bucket; + u64 nbuckets = ca->mi.nbuckets; + unsigned bucket_size = ca->mi.bucket_size; + rcu_read_unlock(); + bkey_for_each_ptr(ptrs, ptr2) bkey_fsck_err_on(ptr != ptr2 && ptr->dev == ptr2->dev, c, err, ptr_to_duplicate_device, "multiple pointers to same device (%u)", ptr->dev); - bucket = sector_to_bucket_and_offset(ca, ptr->offset, &bucket_offset); - bkey_fsck_err_on(bucket >= ca->mi.nbuckets, c, err, + bkey_fsck_err_on(bucket >= nbuckets, c, err, ptr_after_last_bucket, - "pointer past last bucket (%llu > %llu)", bucket, ca->mi.nbuckets); - bkey_fsck_err_on(ptr->offset < bucket_to_sector(ca, ca->mi.first_bucket), c, err, + "pointer past last bucket (%llu > %llu)", bucket, nbuckets); + bkey_fsck_err_on(bucket < first_bucket, c, err, ptr_before_first_bucket, - "pointer before first bucket (%llu < %u)", bucket, ca->mi.first_bucket); - bkey_fsck_err_on(bucket_offset + size_ondisk > ca->mi.bucket_size, c, err, + "pointer before first bucket (%llu < %u)", bucket, first_bucket); + bkey_fsck_err_on(bucket_offset + size_ondisk > bucket_size, c, err, ptr_spans_multiple_buckets, "pointer spans multiple buckets (%u + %u > %u)", - bucket_offset, size_ondisk, ca->mi.bucket_size); + bucket_offset, size_ondisk, bucket_size); fsck_err: return ret; } From 8e3cc2003fd55f32b82c2a06ae7f0bd28b938bd7 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 21:12:31 -0400 Subject: [PATCH 1471/1702] bcachefs: bch2_bkey_has_target() -> bch2_dev_rcu() Signed-off-by: Kent Overstreet --- fs/bcachefs/extents.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index 5f5038f0ca7cf..52e79b87b4853 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -870,14 +870,21 @@ const struct bch_extent_ptr *bch2_bkey_has_device_c(struct bkey_s_c k, unsigned bool bch2_bkey_has_target(struct bch_fs *c, struct bkey_s_c k, unsigned target) { struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); + struct bch_dev *ca; + bool ret = false; + rcu_read_lock(); bkey_for_each_ptr(ptrs, ptr) if (bch2_dev_in_target(c, ptr->dev, target) && + (ca = bch2_dev_rcu(c, ptr->dev)) && (!ptr->cached || - !dev_ptr_stale(bch2_dev_bkey_exists(c, ptr->dev), ptr))) - return true; + !dev_ptr_stale_rcu(ca, ptr))) { + ret = true; + break; + } + rcu_read_unlock(); - return false; + return ret; } bool bch2_bkey_matches_ptr(struct bch_fs *c, struct bkey_s_c k, From 9cadb4ea56d4d961ba0efd47321905b3fda1c664 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 21:12:46 -0400 Subject: [PATCH 1472/1702] bcachefs: bch2_extent_normalize() -> bch2_dev_rcu() Signed-off-by: Kent Overstreet --- fs/bcachefs/extents.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index 52e79b87b4853..7b889f93878bb 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -986,9 +986,14 @@ void bch2_extent_ptr_set_cached(struct bkey_s k, struct bch_extent_ptr *ptr) */ bool bch2_extent_normalize(struct bch_fs *c, struct bkey_s k) { + struct bch_dev *ca; + + rcu_read_lock(); bch2_bkey_drop_ptrs(k, ptr, ptr->cached && - dev_ptr_stale(bch2_dev_bkey_exists(c, ptr->dev), ptr)); + (ca = bch2_dev_rcu(c, ptr->dev)) && + dev_ptr_stale_rcu(ca, ptr)); + rcu_read_unlock(); return bkey_deleted(k.k); } From ad897d241b703a3e81fd4785a4b007168471e9e2 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 22:54:50 -0400 Subject: [PATCH 1473/1702] bcachefs: kill bch2_dev_bkey_exists() in btree_gc.c Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_gc.c | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index 05ebf1186be10..8035c8b797ab3 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -993,19 +993,25 @@ static int bch2_gc_alloc_start(struct bch_fs *c) rcu_assign_pointer(ca->buckets_gc, buckets); } + struct bch_dev *ca = NULL; int ret = bch2_trans_run(c, for_each_btree_key(trans, iter, BTREE_ID_alloc, POS_MIN, BTREE_ITER_prefetch, k, ({ - struct bch_dev *ca = bch2_dev_bkey_exists(c, k.k->p.inode); - struct bucket *g = gc_bucket(ca, k.k->p.offset); + ca = bch2_dev_iterate(c, ca, k.k->p.inode); + if (!ca) { + bch2_btree_iter_set_pos(&iter, POS(k.k->p.inode + 1, 0)); + continue; + } struct bch_alloc_v4 a_convert; const struct bch_alloc_v4 *a = bch2_alloc_to_v4(k, &a_convert); + struct bucket *g = gc_bucket(ca, k.k->p.offset); g->gen_valid = 1; g->gen = a->gen; 0; }))); + bch2_dev_put(ca); bch_err_fn(c, ret); return ret; } @@ -1254,22 +1260,29 @@ static int gc_btree_gens_key(struct btree_trans *trans, return -EROFS; percpu_down_read(&c->mark_lock); + rcu_read_lock(); bkey_for_each_ptr(ptrs, ptr) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev); + if (!ca) + continue; if (dev_ptr_stale(ca, ptr) > 16) { + rcu_read_unlock(); percpu_up_read(&c->mark_lock); goto update; } } bkey_for_each_ptr(ptrs, ptr) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); - u8 *gen = &ca->oldest_gen[PTR_BUCKET_NR(ca, ptr)]; + struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev); + if (!ca) + continue; + u8 *gen = &ca->oldest_gen[PTR_BUCKET_NR(ca, ptr)]; if (gen_after(*gen, ptr->gen)) *gen = ptr->gen; } + rcu_read_unlock(); percpu_up_read(&c->mark_lock); return 0; update: @@ -1282,10 +1295,9 @@ static int gc_btree_gens_key(struct btree_trans *trans, return 0; } -static int bch2_alloc_write_oldest_gen(struct btree_trans *trans, struct btree_iter *iter, - struct bkey_s_c k) +static int bch2_alloc_write_oldest_gen(struct btree_trans *trans, struct bch_dev *ca, + struct btree_iter *iter, struct bkey_s_c k) { - struct bch_dev *ca = bch2_dev_bkey_exists(trans->c, iter->pos.inode); struct bch_alloc_v4 a_convert; const struct bch_alloc_v4 *a = bch2_alloc_to_v4(k, &a_convert); struct bkey_i_alloc_v4 *a_mut; @@ -1355,14 +1367,23 @@ int bch2_gc_gens(struct bch_fs *c) goto err; } + struct bch_dev *ca = NULL; ret = bch2_trans_run(c, for_each_btree_key_commit(trans, iter, BTREE_ID_alloc, POS_MIN, BTREE_ITER_prefetch, k, NULL, NULL, - BCH_TRANS_COMMIT_no_enospc, - bch2_alloc_write_oldest_gen(trans, &iter, k))); + BCH_TRANS_COMMIT_no_enospc, ({ + ca = bch2_dev_iterate(c, ca, k.k->p.inode); + if (!ca) { + bch2_btree_iter_set_pos(&iter, POS(k.k->p.inode + 1, 0)); + continue; + } + bch2_alloc_write_oldest_gen(trans, ca, &iter, k); + }))); + bch2_dev_put(ca); + if (ret) goto err; From 78e9b548f37b92ef81e1bc8cb4dfd4165c5a97d4 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 1 May 2024 03:26:37 -0400 Subject: [PATCH 1474/1702] bcachefs: bch2_dev_bucket_exists() uses bch2_dev_rcu() Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.h | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h index a75eba6d95860..5f42dc1e7fcc8 100644 --- a/fs/bcachefs/alloc_background.h +++ b/fs/bcachefs/alloc_background.h @@ -15,13 +15,11 @@ enum bkey_invalid_flags; static inline bool bch2_dev_bucket_exists(struct bch_fs *c, struct bpos pos) { - struct bch_dev *ca; - - if (!bch2_dev_exists(c, pos.inode)) - return false; - - ca = bch2_dev_bkey_exists(c, pos.inode); - return bucket_valid(ca, pos.offset); + rcu_read_lock(); + struct bch_dev *ca = bch2_dev_rcu(c, pos.inode); + bool ret = ca && bucket_valid(ca, pos.offset); + rcu_read_unlock(); + return ret; } static inline u64 bucket_to_u64(struct bpos bucket) From db39a35dde9dcf77cf599aab5f5c2be35abdcff5 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 1 May 2024 03:58:13 -0400 Subject: [PATCH 1475/1702] bcachefs: pass bch_dev to read_from_stale_dirty_pointer() Signed-off-by: Kent Overstreet --- fs/bcachefs/io_read.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/io_read.c b/fs/bcachefs/io_read.c index 0107b2fb4d333..91260d3dbf503 100644 --- a/fs/bcachefs/io_read.c +++ b/fs/bcachefs/io_read.c @@ -758,11 +758,11 @@ int __bch2_read_indirect_extent(struct btree_trans *trans, } static noinline void read_from_stale_dirty_pointer(struct btree_trans *trans, + struct bch_dev *ca, struct bkey_s_c k, struct bch_extent_ptr ptr) { struct bch_fs *c = trans->c; - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr.dev); struct btree_iter iter; struct printbuf buf = PRINTBUF; int ret; @@ -842,7 +842,7 @@ int __bch2_read_extent(struct btree_trans *trans, struct bch_read_bio *orig, if ((flags & BCH_READ_IN_RETRY) && !pick.ptr.cached && unlikely(dev_ptr_stale(ca, &pick.ptr))) { - read_from_stale_dirty_pointer(trans, k, pick.ptr); + read_from_stale_dirty_pointer(trans, ca, k, pick.ptr); bch2_mark_io_failure(failed, &pick); goto retry_pick; } From a9422fd40410842b51496a00297ab1dabde57af0 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 20:49:13 -0400 Subject: [PATCH 1476/1702] bcachefs: kill bch2_dev_bkey_exists() in bkey_pick_read_device() Signed-off-by: Kent Overstreet --- fs/bcachefs/extents.c | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index 7b889f93878bb..b785712ae9b26 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -71,6 +71,12 @@ void bch2_mark_io_failure(struct bch_io_failures *failed, } } +static inline u64 dev_latency(struct bch_fs *c, unsigned dev) +{ + struct bch_dev *ca = bch2_dev_rcu(c, dev); + return ca ? atomic64_read(&ca->cur_latency[READ]) : S64_MAX; +} + /* * returns true if p1 is better than p2: */ @@ -79,11 +85,8 @@ static inline bool ptr_better(struct bch_fs *c, const struct extent_ptr_decoded p2) { if (likely(!p1.idx && !p2.idx)) { - struct bch_dev *dev1 = bch2_dev_bkey_exists(c, p1.ptr.dev); - struct bch_dev *dev2 = bch2_dev_bkey_exists(c, p2.ptr.dev); - - u64 l1 = atomic64_read(&dev1->cur_latency[READ]); - u64 l2 = atomic64_read(&dev2->cur_latency[READ]); + u64 l1 = dev_latency(c, p1.ptr.dev); + u64 l2 = dev_latency(c, p2.ptr.dev); /* Pick at random, biased in favor of the faster device: */ @@ -109,21 +112,21 @@ int bch2_bkey_pick_read_device(struct bch_fs *c, struct bkey_s_c k, const union bch_extent_entry *entry; struct extent_ptr_decoded p; struct bch_dev_io_failures *f; - struct bch_dev *ca; int ret = 0; if (k.k->type == KEY_TYPE_error) return -EIO; + rcu_read_lock(); bkey_for_each_ptr_decode(k.k, ptrs, p, entry) { /* * Unwritten extent: no need to actually read, treat it as a * hole and return 0s: */ - if (p.ptr.unwritten) - return 0; - - ca = bch2_dev_bkey_exists(c, p.ptr.dev); + if (p.ptr.unwritten) { + ret = 0; + break; + } /* * If there are any dirty pointers it's an error if we can't @@ -132,7 +135,9 @@ int bch2_bkey_pick_read_device(struct bch_fs *c, struct bkey_s_c k, if (!ret && !p.ptr.cached) ret = -EIO; - if (p.ptr.cached && dev_ptr_stale(ca, &p.ptr)) + struct bch_dev *ca = bch2_dev_rcu(c, p.ptr.dev); + + if (p.ptr.cached && (!ca || dev_ptr_stale(ca, &p.ptr))) continue; f = failed ? dev_io_failures(failed, p.ptr.dev) : NULL; @@ -141,12 +146,13 @@ int bch2_bkey_pick_read_device(struct bch_fs *c, struct bkey_s_c k, ? f->idx : f->idx + 1; - if (!p.idx && - !bch2_dev_is_readable(ca)) + if (!p.idx && !ca) p.idx++; - if (bch2_force_reconstruct_read && - !p.idx && p.has_ec) + if (!p.idx && p.has_ec && bch2_force_reconstruct_read) + p.idx++; + + if (!p.idx && !bch2_dev_is_readable(ca)) p.idx++; if (p.idx >= (unsigned) p.has_ec + 1) @@ -158,6 +164,7 @@ int bch2_bkey_pick_read_device(struct bch_fs *c, struct bkey_s_c k, *pick = p; ret = 1; } + rcu_read_unlock(); return ret; } From 222eacabc17f360ede4efddda50534f828228ed0 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 1 May 2024 18:06:35 -0400 Subject: [PATCH 1477/1702] bcachefs: kill bch2_dev_bkey_exists() in data_update_init() Signed-off-by: Kent Overstreet --- fs/bcachefs/data_update.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/data_update.c b/fs/bcachefs/data_update.c index 7aad6085ef53e..1dee93eee7add 100644 --- a/fs/bcachefs/data_update.c +++ b/fs/bcachefs/data_update.c @@ -544,8 +544,16 @@ int bch2_data_update_init(struct btree_trans *trans, m->op.compression_opt = background_compression(io_opts); m->op.watermark = m->data_opts.btree_insert_flags & BCH_WATERMARK_MASK; - bkey_for_each_ptr(ptrs, ptr) - bch2_dev_get(bch2_dev_bkey_exists(c, ptr->dev)); + bkey_for_each_ptr(ptrs, ptr) { + if (!bch2_dev_tryget(c, ptr->dev)) { + bkey_for_each_ptr(ptrs, ptr2) { + if (ptr2 == ptr) + break; + bch2_dev_put(bch2_dev_have_ref(c, ptr2->dev)); + } + return -BCH_ERR_data_update_done; + } + } unsigned durability_have = 0, durability_removing = 0; From d8585a79bebcfb3315f23c955e36ab8d8a7749d5 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 1 May 2024 03:59:45 -0400 Subject: [PATCH 1478/1702] bcachefs: bch2_dev_have_ref() bch2_dev_bkey_exists() is going away; bch2_dev_have_ref() documents that we're looking up a device without checking if it's present because we have a reference to it already. Signed-off-by: Kent Overstreet --- fs/bcachefs/data_update.c | 6 +++--- fs/bcachefs/io_write.c | 4 ++-- fs/bcachefs/journal_io.c | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fs/bcachefs/data_update.c b/fs/bcachefs/data_update.c index 1dee93eee7add..0d807c2ce9c67 100644 --- a/fs/bcachefs/data_update.c +++ b/fs/bcachefs/data_update.c @@ -360,7 +360,7 @@ void bch2_data_update_exit(struct data_update *update) bch2_bkey_ptrs_c(bkey_i_to_s_c(update->k.k)); bkey_for_each_ptr(ptrs, ptr) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_have_ref(c, ptr->dev); if (c->opts.nocow_enabled) bch2_bucket_nocow_unlock(&c->nocow_locks, PTR_BUCKET_POS(ca, ptr), 0); @@ -559,7 +559,7 @@ int bch2_data_update_init(struct btree_trans *trans, i = 0; bkey_for_each_ptr_decode(k.k, ptrs, p, entry) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); + struct bch_dev *ca = bch2_dev_have_ref(c, p.ptr.dev); struct bpos bucket = PTR_BUCKET_POS(ca, &p.ptr); bool locked; @@ -664,7 +664,7 @@ int bch2_data_update_init(struct btree_trans *trans, err: i = 0; bkey_for_each_ptr_decode(k.k, ptrs, p, entry) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); + struct bch_dev *ca = bch2_dev_have_ref(c, p.ptr.dev); struct bpos bucket = PTR_BUCKET_POS(ca, &p.ptr); if ((1U << i) & ptrs_locked) bch2_bucket_nocow_unlock(&c->nocow_locks, bucket, 0); diff --git a/fs/bcachefs/io_write.c b/fs/bcachefs/io_write.c index 285e1a23bde5b..33202b77f6cf0 100644 --- a/fs/bcachefs/io_write.c +++ b/fs/bcachefs/io_write.c @@ -1303,7 +1303,7 @@ static void bch2_nocow_write(struct bch_write_op *op) bch2_cut_back(POS(op->pos.inode, op->pos.offset + bio_sectors(bio)), op->insert_keys.top); darray_for_each(buckets, i) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, i->b.inode); + struct bch_dev *ca = bch2_dev_have_ref(c, i->b.inode); __bch2_bucket_nocow_lock(&c->nocow_locks, i->l, bucket_to_u64(i->b), @@ -1380,7 +1380,7 @@ static void bch2_nocow_write(struct bch_write_op *op) return; err_get_ioref: darray_for_each(buckets, i) - percpu_ref_put(&bch2_dev_bkey_exists(c, i->b.inode)->io_ref); + percpu_ref_put(&bch2_dev_have_ref(c, i->b.inode)->io_ref); /* Fall back to COW path: */ goto out; diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c index e76eb98af74a2..9561f1c2ce1ac 100644 --- a/fs/bcachefs/journal_io.c +++ b/fs/bcachefs/journal_io.c @@ -1386,7 +1386,7 @@ int bch2_journal_read(struct bch_fs *c, continue; darray_for_each(i->ptrs, ptr) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_have_ref(c, ptr->dev); if (!ptr->csum_good) bch_err_dev_offset(ca, ptr->sector, @@ -1396,7 +1396,7 @@ int bch2_journal_read(struct bch_fs *c, } ret = jset_validate(c, - bch2_dev_bkey_exists(c, i->ptrs.data[0].dev), + bch2_dev_have_ref(c, i->ptrs.data[0].dev), &i->j, i->ptrs.data[0].sector, READ); From bc3204c80ab69ac64a03b070623b9e7b084e2b2c Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 1 May 2024 17:04:08 -0400 Subject: [PATCH 1479/1702] bcachefs: kill bch2_dev_bkey_exists() in check_alloc_info() Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 76 ++++++++++++++++------------------ fs/bcachefs/sb-members.h | 8 ++++ 2 files changed, 44 insertions(+), 40 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 69a77d268edf1..ef1d17fbd750c 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -960,35 +960,34 @@ static struct bkey_s_c bch2_get_key_or_hole(struct btree_iter *iter, struct bpos } } -static bool next_bucket(struct bch_fs *c, struct bpos *bucket) +static bool next_bucket(struct bch_fs *c, struct bch_dev **ca, struct bpos *bucket) { - struct bch_dev *ca; - - if (bch2_dev_bucket_exists(c, *bucket)) - return true; + if (*ca) { + if (bucket->offset < (*ca)->mi.first_bucket) + bucket->offset = (*ca)->mi.first_bucket; - if (bch2_dev_exists(c, bucket->inode)) { - ca = bch2_dev_bkey_exists(c, bucket->inode); - - if (bucket->offset < ca->mi.first_bucket) { - bucket->offset = ca->mi.first_bucket; + if (bucket->offset < (*ca)->mi.nbuckets) return true; - } + bch2_dev_put(*ca); + *ca = NULL; bucket->inode++; bucket->offset = 0; } rcu_read_lock(); - ca = __bch2_next_dev_idx(c, bucket->inode, NULL); - if (ca) - *bucket = POS(ca->dev_idx, ca->mi.first_bucket); + *ca = __bch2_next_dev_idx(c, bucket->inode, NULL); + if (*ca) { + *bucket = POS((*ca)->dev_idx, (*ca)->mi.first_bucket); + bch2_dev_get(*ca); + } rcu_read_unlock(); - return ca != NULL; + return *ca != NULL; } -static struct bkey_s_c bch2_get_key_or_real_bucket_hole(struct btree_iter *iter, struct bkey *hole) +static struct bkey_s_c bch2_get_key_or_real_bucket_hole(struct btree_iter *iter, + struct bch_dev **ca, struct bkey *hole) { struct bch_fs *c = iter->trans->c; struct bkey_s_c k; @@ -997,22 +996,21 @@ static struct bkey_s_c bch2_get_key_or_real_bucket_hole(struct btree_iter *iter, if (bkey_err(k)) return k; + *ca = bch2_dev_iterate_noerror(c, *ca, k.k->p.inode); + if (!k.k->type) { - struct bpos bucket = bkey_start_pos(k.k); + struct bpos hole_start = bkey_start_pos(k.k); - if (!bch2_dev_bucket_exists(c, bucket)) { - if (!next_bucket(c, &bucket)) + if (!*ca || !bucket_valid(*ca, hole_start.offset)) { + if (!next_bucket(c, ca, &hole_start)) return bkey_s_c_null; - bch2_btree_iter_set_pos(iter, bucket); + bch2_btree_iter_set_pos(iter, hole_start); goto again; } - if (!bch2_dev_bucket_exists(c, k.k->p)) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, bucket.inode); - - bch2_key_resize(hole, ca->mi.nbuckets - bucket.offset); - } + if (k.k->p.offset > (*ca)->mi.nbuckets) + bch2_key_resize(hole, (*ca)->mi.nbuckets - hole_start.offset); } return k; @@ -1154,17 +1152,16 @@ int bch2_check_alloc_key(struct btree_trans *trans, static noinline_for_stack int bch2_check_alloc_hole_freespace(struct btree_trans *trans, + struct bch_dev *ca, struct bpos start, struct bpos *end, struct btree_iter *freespace_iter) { struct bch_fs *c = trans->c; - struct bch_dev *ca; struct bkey_s_c k; struct printbuf buf = PRINTBUF; int ret; - ca = bch2_dev_bkey_exists(c, start.inode); if (!ca->mi.freespace_initialized) return 0; @@ -1342,30 +1339,25 @@ int bch2_check_bucket_gens_key(struct btree_trans *trans, { struct bch_fs *c = trans->c; struct bkey_i_bucket_gens g; - struct bch_dev *ca; u64 start = bucket_gens_pos_to_alloc(k.k->p, 0).offset; u64 end = bucket_gens_pos_to_alloc(bpos_nosnap_successor(k.k->p), 0).offset; u64 b; - bool need_update = false, dev_exists; + bool need_update = false; struct printbuf buf = PRINTBUF; int ret = 0; BUG_ON(k.k->type != KEY_TYPE_bucket_gens); bkey_reassemble(&g.k_i, k); - /* if no bch_dev, skip out whether we repair or not */ - dev_exists = bch2_dev_exists(c, k.k->p.inode); - if (!dev_exists) { - if (fsck_err_on(!dev_exists, c, - bucket_gens_to_invalid_dev, - "bucket_gens key for invalid device:\n %s", - (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { + struct bch_dev *ca = bch2_dev_tryget_noerror(c, k.k->p.inode); + if (!ca) { + if (fsck_err(c, bucket_gens_to_invalid_dev, + "bucket_gens key for invalid device:\n %s", + (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) ret = bch2_btree_delete_at(trans, iter, 0); - } goto out; } - ca = bch2_dev_bkey_exists(c, k.k->p.inode); if (fsck_err_on(end <= ca->mi.first_bucket || start >= ca->mi.nbuckets, c, bucket_gens_to_invalid_buckets, @@ -1403,6 +1395,7 @@ int bch2_check_bucket_gens_key(struct btree_trans *trans, } out: fsck_err: + bch2_dev_put(ca); printbuf_exit(&buf); return ret; } @@ -1411,6 +1404,7 @@ int bch2_check_alloc_info(struct bch_fs *c) { struct btree_trans *trans = bch2_trans_get(c); struct btree_iter iter, discard_iter, freespace_iter, bucket_gens_iter; + struct bch_dev *ca = NULL; struct bkey hole; struct bkey_s_c k; int ret = 0; @@ -1429,7 +1423,7 @@ int bch2_check_alloc_info(struct bch_fs *c) bch2_trans_begin(trans); - k = bch2_get_key_or_real_bucket_hole(&iter, &hole); + k = bch2_get_key_or_real_bucket_hole(&iter, &ca, &hole); ret = bkey_err(k); if (ret) goto bkey_err; @@ -1450,7 +1444,7 @@ int bch2_check_alloc_info(struct bch_fs *c) } else { next = k.k->p; - ret = bch2_check_alloc_hole_freespace(trans, + ret = bch2_check_alloc_hole_freespace(trans, ca, bkey_start_pos(k.k), &next, &freespace_iter) ?: @@ -1478,6 +1472,8 @@ int bch2_check_alloc_info(struct bch_fs *c) bch2_trans_iter_exit(trans, &freespace_iter); bch2_trans_iter_exit(trans, &discard_iter); bch2_trans_iter_exit(trans, &iter); + bch2_dev_put(ca); + ca = NULL; if (ret < 0) goto err; diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index a3ba1ddebe4d3..7cecac44ab80a 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -269,6 +269,14 @@ static inline struct bch_dev *bch2_dev_bucket_tryget(struct bch_fs *c, struct bp return ca; } +static inline struct bch_dev *bch2_dev_iterate_noerror(struct bch_fs *c, struct bch_dev *ca, unsigned dev_idx) +{ + if (ca && ca->dev_idx == dev_idx) + return ca; + bch2_dev_put(ca); + return bch2_dev_tryget_noerror(c, dev_idx); +} + static inline struct bch_dev *bch2_dev_iterate(struct bch_fs *c, struct bch_dev *ca, unsigned dev_idx) { if (ca && ca->dev_idx == dev_idx) From f4301b635a2ee6588d61d03fe261a2bba9225eb7 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 1 May 2024 17:08:40 -0400 Subject: [PATCH 1480/1702] bcachefs: kill bch2_dev_bkey_exists() in discard_one_bucket_fast() Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index ef1d17fbd750c..71b727f5d432c 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -1903,9 +1903,12 @@ static void bch2_do_discards_fast_work(struct work_struct *work) static void bch2_discard_one_bucket_fast(struct bch_fs *c, struct bpos bucket) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, bucket.inode); + rcu_read_lock(); + struct bch_dev *ca = bch2_dev_rcu(c, bucket.inode); + bool dead = !ca || percpu_ref_is_dying(&ca->io_ref); + rcu_read_unlock(); - if (!percpu_ref_is_dying(&ca->io_ref) && + if (!dead && !discard_in_flight_add(c, bucket) && bch2_write_ref_tryget(c, BCH_WRITE_REF_discard_fast) && !queue_work(c->write_ref_wq, &c->discard_fast_work)) From b6d29b586920daf9f1c5e59ffea91ed162842781 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 1 May 2024 18:20:52 -0400 Subject: [PATCH 1481/1702] bcachefs: kill bch2_dev_bkey_exists() in journal_ptrs_to_text() Signed-off-by: Kent Overstreet --- fs/bcachefs/journal_io.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c index 9561f1c2ce1ac..c9744546d48d9 100644 --- a/fs/bcachefs/journal_io.c +++ b/fs/bcachefs/journal_io.c @@ -49,11 +49,6 @@ void bch2_journal_ptrs_to_text(struct printbuf *out, struct bch_fs *c, struct journal_replay *j) { darray_for_each(j->ptrs, i) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, i->dev); - u64 offset; - - div64_u64_rem(i->sector, ca->mi.bucket_size, &offset); - if (i != j->ptrs.data) prt_printf(out, " "); prt_printf(out, "%u:%u:%u (sector %llu)", From 62025697778cb5ca6173b3e52d7e7270edc65a2c Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 1 May 2024 18:56:40 -0400 Subject: [PATCH 1482/1702] bcachefs: Move nocow unlock to bch2_write_endio() This fixes a lifetime issue; bch2_nocow_write_unlock() uses PTR_BUCKET_POS(), which needs the device - but we drop our ref to the device in bch2_write_endio(). Signed-off-by: Kent Overstreet --- fs/bcachefs/io_write.c | 26 +++++++------------------- fs/bcachefs/io_write_types.h | 1 + 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/fs/bcachefs/io_write.c b/fs/bcachefs/io_write.c index 33202b77f6cf0..fa3413d19b6ac 100644 --- a/fs/bcachefs/io_write.c +++ b/fs/bcachefs/io_write.c @@ -434,6 +434,8 @@ void bch2_submit_wbio_replicas(struct bch_write_bio *wbio, struct bch_fs *c, n->nocow = nocow; n->submit_time = local_clock(); n->inode_offset = bkey_start_offset(&k->k); + if (nocow) + n->nocow_bucket = PTR_BUCKET_NR(ca, ptr); n->bio.bi_iter.bi_sector = ptr->offset; if (likely(n->have_ioref)) { @@ -659,8 +661,12 @@ static void bch2_write_endio(struct bio *bio) op->flags |= BCH_WRITE_IO_ERROR; } - if (wbio->nocow) + if (wbio->nocow) { + bch2_bucket_nocow_unlock(&c->nocow_locks, + POS(ca->dev_idx, wbio->nocow_bucket), + BUCKET_NOCOW_LOCK_UPDATE); set_bit(wbio->dev, op->devs_need_flush->d); + } if (wbio->have_ioref) { bch2_latency_acct(ca, wbio->submit_time, WRITE); @@ -1114,22 +1120,6 @@ static bool bch2_extent_is_writeable(struct bch_write_op *op, return replicas >= op->opts.data_replicas; } -static inline void bch2_nocow_write_unlock(struct bch_write_op *op) -{ - struct bch_fs *c = op->c; - - for_each_keylist_key(&op->insert_keys, k) { - struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(bkey_i_to_s_c(k)); - - bkey_for_each_ptr(ptrs, ptr) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); - bch2_bucket_nocow_unlock(&c->nocow_locks, - PTR_BUCKET_POS(ca, ptr), - BUCKET_NOCOW_LOCK_UPDATE); - } - } -} - static int bch2_nocow_write_convert_one_unwritten(struct btree_trans *trans, struct btree_iter *iter, struct bkey_i *orig, @@ -1200,8 +1190,6 @@ static void bch2_nocow_write_convert_unwritten(struct bch_write_op *op) static void __bch2_nocow_write_done(struct bch_write_op *op) { - bch2_nocow_write_unlock(op); - if (unlikely(op->flags & BCH_WRITE_IO_ERROR)) { op->error = -EIO; } else if (unlikely(op->flags & BCH_WRITE_CONVERT_UNWRITTEN)) diff --git a/fs/bcachefs/io_write_types.h b/fs/bcachefs/io_write_types.h index c7f97c2c4805f..6e878a6f2f0b2 100644 --- a/fs/bcachefs/io_write_types.h +++ b/fs/bcachefs/io_write_types.h @@ -20,6 +20,7 @@ struct bch_write_bio { u64 submit_time; u64 inode_offset; + u64 nocow_bucket; struct bch_devs_list failed; u8 dev; From 40574946b8a5179ed811bc9783d21bed302eb949 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 4 May 2024 16:46:29 -0400 Subject: [PATCH 1483/1702] bcachefs: Better bucket alloc tracepoints Tracepoints are garbage, and perf trace even cuts off some of our fields. Much nicer to just trace a string, and then we can build nicely formatted output with printbufs. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_foreground.c | 65 ++++++++++++++++------- fs/bcachefs/trace.h | 97 +++------------------------------- 2 files changed, 51 insertions(+), 111 deletions(-) diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index 77f3723e0fc0f..9fbd0d97f68cc 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -511,6 +511,44 @@ static struct open_bucket *bch2_bucket_alloc_freelist(struct btree_trans *trans, return ob; } +static noinline void trace_bucket_alloc2(struct bch_fs *c, struct bch_dev *ca, + enum bch_watermark watermark, + enum bch_data_type data_type, + struct closure *cl, + struct bch_dev_usage *usage, + struct bucket_alloc_state *s, + struct open_bucket *ob) +{ + struct printbuf buf = PRINTBUF; + + printbuf_tabstop_push(&buf, 24); + + prt_printf(&buf, "dev\t%s (%u)\n", ca->name, ca->dev_idx); + prt_printf(&buf, "watermark\t%s\n", bch2_watermarks[watermark]); + prt_printf(&buf, "data type\t%s\n", __bch2_data_types[data_type]); + prt_printf(&buf, "blocking\t%u\n", cl != NULL); + prt_printf(&buf, "free\t%llu\n", usage->d[BCH_DATA_free].buckets); + prt_printf(&buf, "avail\t%llu\n", dev_buckets_free(ca, *usage, watermark)); + prt_printf(&buf, "copygc_wait\t%lu/%lli\n", + bch2_copygc_wait_amount(c), + c->copygc_wait - atomic64_read(&c->io_clock[WRITE].now)); + prt_printf(&buf, "seen\t%llu\n", s->buckets_seen); + prt_printf(&buf, "open\t%llu\n", s->skipped_open); + prt_printf(&buf, "need journal commit\t%llu\n", s->skipped_need_journal_commit); + prt_printf(&buf, "nocow\t%llu\n", s->skipped_nocow); + prt_printf(&buf, "nouse\t%llu\n", s->skipped_nouse); + + if (!IS_ERR(ob)) { + prt_printf(&buf, "allocated\t%llu\n", ob->bucket); + trace_bucket_alloc(c, buf.buf); + } else { + prt_printf(&buf, "err\t%s\n", bch2_err_str(PTR_ERR(ob))); + trace_bucket_alloc_fail(c, buf.buf); + } + + printbuf_exit(&buf); +} + /** * bch2_bucket_alloc_trans - allocate a single bucket from a specific device * @trans: transaction object @@ -583,27 +621,14 @@ static struct open_bucket *bch2_bucket_alloc_trans(struct btree_trans *trans, ob->data_type = data_type; if (!IS_ERR(ob)) - trace_and_count(c, bucket_alloc, ca, - bch2_watermarks[watermark], - ob->bucket, - usage->d[BCH_DATA_free].buckets, - avail, - bch2_copygc_wait_amount(c), - c->copygc_wait - atomic64_read(&c->io_clock[WRITE].now), - &s, - cl == NULL, - ""); + count_event(c, bucket_alloc); else if (!bch2_err_matches(PTR_ERR(ob), BCH_ERR_transaction_restart)) - trace_and_count(c, bucket_alloc_fail, ca, - bch2_watermarks[watermark], - 0, - usage->d[BCH_DATA_free].buckets, - avail, - bch2_copygc_wait_amount(c), - c->copygc_wait - atomic64_read(&c->io_clock[WRITE].now), - &s, - cl == NULL, - bch2_err_str(PTR_ERR(ob))); + count_event(c, bucket_alloc_fail); + + if (!IS_ERR(ob) + ? trace_bucket_alloc_enabled() + : trace_bucket_alloc_fail_enabled()) + trace_bucket_alloc2(c, ca, watermark, data_type, cl, usage, &s, ob); return ob; } diff --git a/fs/bcachefs/trace.h b/fs/bcachefs/trace.h index 6aa81d1e6d36a..362e1fc7ef6a4 100644 --- a/fs/bcachefs/trace.h +++ b/fs/bcachefs/trace.h @@ -638,99 +638,14 @@ DEFINE_EVENT(bch_fs, gc_gens_end, /* Allocator */ -DECLARE_EVENT_CLASS(bucket_alloc, - TP_PROTO(struct bch_dev *ca, const char *alloc_reserve, - u64 bucket, - u64 free, - u64 avail, - u64 copygc_wait_amount, - s64 copygc_waiting_for, - struct bucket_alloc_state *s, - bool nonblocking, - const char *err), - TP_ARGS(ca, alloc_reserve, bucket, free, avail, - copygc_wait_amount, copygc_waiting_for, - s, nonblocking, err), - - TP_STRUCT__entry( - __field(u8, dev ) - __array(char, reserve, 16 ) - __field(u64, bucket ) - __field(u64, free ) - __field(u64, avail ) - __field(u64, copygc_wait_amount ) - __field(s64, copygc_waiting_for ) - __field(u64, seen ) - __field(u64, open ) - __field(u64, need_journal_commit ) - __field(u64, nouse ) - __field(bool, nonblocking ) - __field(u64, nocow ) - __array(char, err, 32 ) - ), - - TP_fast_assign( - __entry->dev = ca->dev_idx; - strscpy(__entry->reserve, alloc_reserve, sizeof(__entry->reserve)); - __entry->bucket = bucket; - __entry->free = free; - __entry->avail = avail; - __entry->copygc_wait_amount = copygc_wait_amount; - __entry->copygc_waiting_for = copygc_waiting_for; - __entry->seen = s->buckets_seen; - __entry->open = s->skipped_open; - __entry->need_journal_commit = s->skipped_need_journal_commit; - __entry->nouse = s->skipped_nouse; - __entry->nonblocking = nonblocking; - __entry->nocow = s->skipped_nocow; - strscpy(__entry->err, err, sizeof(__entry->err)); - ), - - TP_printk("reserve %s bucket %u:%llu free %llu avail %llu copygc_wait %llu/%lli seen %llu open %llu need_journal_commit %llu nouse %llu nocow %llu nonblocking %u err %s", - __entry->reserve, - __entry->dev, - __entry->bucket, - __entry->free, - __entry->avail, - __entry->copygc_wait_amount, - __entry->copygc_waiting_for, - __entry->seen, - __entry->open, - __entry->need_journal_commit, - __entry->nouse, - __entry->nocow, - __entry->nonblocking, - __entry->err) +DEFINE_EVENT(fs_str, bucket_alloc, + TP_PROTO(struct bch_fs *c, const char *str), + TP_ARGS(c, str) ); -DEFINE_EVENT(bucket_alloc, bucket_alloc, - TP_PROTO(struct bch_dev *ca, const char *alloc_reserve, - u64 bucket, - u64 free, - u64 avail, - u64 copygc_wait_amount, - s64 copygc_waiting_for, - struct bucket_alloc_state *s, - bool nonblocking, - const char *err), - TP_ARGS(ca, alloc_reserve, bucket, free, avail, - copygc_wait_amount, copygc_waiting_for, - s, nonblocking, err) -); - -DEFINE_EVENT(bucket_alloc, bucket_alloc_fail, - TP_PROTO(struct bch_dev *ca, const char *alloc_reserve, - u64 bucket, - u64 free, - u64 avail, - u64 copygc_wait_amount, - s64 copygc_waiting_for, - struct bucket_alloc_state *s, - bool nonblocking, - const char *err), - TP_ARGS(ca, alloc_reserve, bucket, free, avail, - copygc_wait_amount, copygc_waiting_for, - s, nonblocking, err) +DEFINE_EVENT(fs_str, bucket_alloc_fail, + TP_PROTO(struct bch_fs *c, const char *str), + TP_ARGS(c, str) ); TRACE_EVENT(discard_buckets, From c6705091342c06c963015dac07ede417d2e0ad04 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 20 Apr 2024 16:25:34 -0400 Subject: [PATCH 1484/1702] bcachefs: Allocator prefers not to expand mi.btree_allocated bitmap We now have a small bitmap in the member info section of the superblock for "regions that have btree nodes", so that if we ever have to scan for btree nodes in repair we don't have to scan the whole device(s). This tweaks the allocator to prefer allocating from regions that are already marked in this bitmap. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_foreground.c | 68 +++++++++++++++++++++++++++++----- fs/bcachefs/alloc_types.h | 7 ++++ fs/bcachefs/bcachefs.h | 2 +- 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c index 9fbd0d97f68cc..927a5f300b30e 100644 --- a/fs/bcachefs/alloc_foreground.c +++ b/fs/bcachefs/alloc_foreground.c @@ -71,7 +71,7 @@ void bch2_reset_alloc_cursors(struct bch_fs *c) { rcu_read_lock(); for_each_member_device_rcu(c, ca, NULL) - ca->alloc_cursor = 0; + memset(ca->alloc_cursor, 0, sizeof(ca->alloc_cursor)); rcu_read_unlock(); } @@ -389,7 +389,8 @@ bch2_bucket_alloc_early(struct btree_trans *trans, struct bkey_s_c k, ck; struct open_bucket *ob = NULL; u64 first_bucket = max_t(u64, ca->mi.first_bucket, ca->new_fs_bucket_idx); - u64 alloc_start = max(first_bucket, READ_ONCE(ca->alloc_cursor)); + u64 *dev_alloc_cursor = &ca->alloc_cursor[s->btree_bitmap]; + u64 alloc_start = max(first_bucket, *dev_alloc_cursor); u64 alloc_cursor = alloc_start; int ret; @@ -405,8 +406,7 @@ bch2_bucket_alloc_early(struct btree_trans *trans, again: for_each_btree_key_norestart(trans, iter, BTREE_ID_alloc, POS(ca->dev_idx, alloc_cursor), BTREE_ITER_slots, k, ret) { - struct bch_alloc_v4 a_convert; - const struct bch_alloc_v4 *a; + u64 bucket = k.k->p.offset; if (bkey_ge(k.k->p, POS(ca->dev_idx, ca->mi.nbuckets))) break; @@ -415,7 +415,24 @@ bch2_bucket_alloc_early(struct btree_trans *trans, is_superblock_bucket(ca, k.k->p.offset)) continue; - a = bch2_alloc_to_v4(k, &a_convert); + if (s->btree_bitmap != BTREE_BITMAP_ANY && + s->btree_bitmap != bch2_dev_btree_bitmap_marked_sectors(ca, + bucket_to_sector(ca, bucket), ca->mi.bucket_size)) { + if (s->btree_bitmap == BTREE_BITMAP_YES && + bucket_to_sector(ca, bucket) > 64ULL << ca->mi.btree_bitmap_shift) + break; + + bucket = sector_to_bucket(ca, + round_up(bucket_to_sector(ca, bucket) + 1, + 1ULL << ca->mi.btree_bitmap_shift)); + bch2_btree_iter_set_pos(&iter, POS(ca->dev_idx, bucket)); + s->buckets_seen++; + s->skipped_mi_btree_bitmap++; + continue; + } + + struct bch_alloc_v4 a_convert; + const struct bch_alloc_v4 *a = bch2_alloc_to_v4(k, &a_convert); if (a->data_type != BCH_DATA_free) continue; @@ -441,7 +458,6 @@ bch2_bucket_alloc_early(struct btree_trans *trans, bch2_trans_iter_exit(trans, &iter); alloc_cursor = iter.pos.offset; - ca->alloc_cursor = alloc_cursor; if (!ob && ret) ob = ERR_PTR(ret); @@ -451,6 +467,8 @@ bch2_bucket_alloc_early(struct btree_trans *trans, goto again; } + *dev_alloc_cursor = alloc_cursor; + return ob; } @@ -463,7 +481,8 @@ static struct open_bucket *bch2_bucket_alloc_freelist(struct btree_trans *trans, struct btree_iter iter; struct bkey_s_c k; struct open_bucket *ob = NULL; - u64 alloc_start = max_t(u64, ca->mi.first_bucket, READ_ONCE(ca->alloc_cursor)); + u64 *dev_alloc_cursor = &ca->alloc_cursor[s->btree_bitmap]; + u64 alloc_start = max_t(u64, ca->mi.first_bucket, READ_ONCE(*dev_alloc_cursor)); u64 alloc_cursor = alloc_start; int ret; @@ -485,6 +504,26 @@ static struct open_bucket *bch2_bucket_alloc_freelist(struct btree_trans *trans, s->buckets_seen++; + u64 bucket = alloc_cursor & ~(~0ULL << 56); + if (s->btree_bitmap != BTREE_BITMAP_ANY && + s->btree_bitmap != bch2_dev_btree_bitmap_marked_sectors(ca, + bucket_to_sector(ca, bucket), ca->mi.bucket_size)) { + if (s->btree_bitmap == BTREE_BITMAP_YES && + bucket_to_sector(ca, bucket) > 64ULL << ca->mi.btree_bitmap_shift) + goto fail; + + bucket = sector_to_bucket(ca, + round_up(bucket_to_sector(ca, bucket) + 1, + 1ULL << ca->mi.btree_bitmap_shift)); + u64 genbits = alloc_cursor >> 56; + alloc_cursor = bucket | (genbits << 56); + + if (alloc_cursor > k.k->p.offset) + bch2_btree_iter_set_pos(&iter, POS(ca->dev_idx, alloc_cursor)); + s->skipped_mi_btree_bitmap++; + continue; + } + ob = try_alloc_bucket(trans, ca, watermark, alloc_cursor, s, k, cl); if (ob) { @@ -496,10 +535,9 @@ static struct open_bucket *bch2_bucket_alloc_freelist(struct btree_trans *trans, if (ob || ret) break; } +fail: bch2_trans_iter_exit(trans, &iter); - ca->alloc_cursor = alloc_cursor; - if (!ob && ret) ob = ERR_PTR(ret); @@ -508,6 +546,8 @@ static struct open_bucket *bch2_bucket_alloc_freelist(struct btree_trans *trans, goto again; } + *dev_alloc_cursor = alloc_cursor; + return ob; } @@ -537,6 +577,7 @@ static noinline void trace_bucket_alloc2(struct bch_fs *c, struct bch_dev *ca, prt_printf(&buf, "need journal commit\t%llu\n", s->skipped_need_journal_commit); prt_printf(&buf, "nocow\t%llu\n", s->skipped_nocow); prt_printf(&buf, "nouse\t%llu\n", s->skipped_nouse); + prt_printf(&buf, "mi_btree_bitmap\t%llu\n", s->skipped_mi_btree_bitmap); if (!IS_ERR(ob)) { prt_printf(&buf, "allocated\t%llu\n", ob->bucket); @@ -571,7 +612,9 @@ static struct open_bucket *bch2_bucket_alloc_trans(struct btree_trans *trans, struct open_bucket *ob = NULL; bool freespace = READ_ONCE(ca->mi.freespace_initialized); u64 avail; - struct bucket_alloc_state s = { 0 }; + struct bucket_alloc_state s = { + .btree_bitmap = data_type == BCH_DATA_btree, + }; bool waiting = false; again: bch2_dev_usage_read_fast(ca, usage); @@ -609,6 +652,11 @@ static struct open_bucket *bch2_bucket_alloc_trans(struct btree_trans *trans, if (s.skipped_need_journal_commit * 2 > avail) bch2_journal_flush_async(&c->journal, NULL); + if (!ob && s.btree_bitmap != BTREE_BITMAP_ANY) { + s.btree_bitmap = BTREE_BITMAP_ANY; + goto alloc; + } + if (!ob && freespace && c->curr_recovery_pass <= BCH_RECOVERY_PASS_check_alloc_info) { freespace = false; goto alloc; diff --git a/fs/bcachefs/alloc_types.h b/fs/bcachefs/alloc_types.h index c2226e947c41f..9bbb28e90b934 100644 --- a/fs/bcachefs/alloc_types.h +++ b/fs/bcachefs/alloc_types.h @@ -9,11 +9,18 @@ #include "fifo.h" struct bucket_alloc_state { + enum { + BTREE_BITMAP_NO, + BTREE_BITMAP_YES, + BTREE_BITMAP_ANY, + } btree_bitmap; + u64 buckets_seen; u64 skipped_open; u64 skipped_need_journal_commit; u64 skipped_nocow; u64 skipped_nouse; + u64 skipped_mi_btree_bitmap; }; #define BCH_WATERMARKS() \ diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index ab415f0dd14ee..ab2dc2c70da21 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -587,7 +587,7 @@ struct bch_dev { /* Allocator: */ u64 new_fs_bucket_idx; - u64 alloc_cursor; + u64 alloc_cursor[3]; unsigned nr_open_buckets; unsigned nr_btree_reserve; From e11ecc6133b345a6c5919ae35920c384c6ff6f95 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sun, 5 May 2024 09:47:53 -0400 Subject: [PATCH 1485/1702] bcachefs: Improve sysfs internal/btree_cache Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_cache.c | 33 ++++++++++++++++++++++++++++----- fs/bcachefs/btree_types.h | 2 ++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c index debdd7dc04778..2c226e0cc2630 100644 --- a/fs/bcachefs/btree_cache.c +++ b/fs/bcachefs/btree_cache.c @@ -162,6 +162,9 @@ void bch2_btree_node_hash_remove(struct btree_cache *bc, struct btree *b) /* Cause future lookups for this node to fail: */ b->hash_val = 0; + + if (b->c.btree_id < BTREE_ID_NR) + --bc->used_by_btree[b->c.btree_id]; } int __bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b) @@ -169,8 +172,11 @@ int __bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b) BUG_ON(b->hash_val); b->hash_val = btree_ptr_hash_val(&b->key); - return rhashtable_lookup_insert_fast(&bc->table, &b->hash, - bch_btree_cache_params); + int ret = rhashtable_lookup_insert_fast(&bc->table, &b->hash, + bch_btree_cache_params); + if (!ret && b->c.btree_id < BTREE_ID_NR) + bc->used_by_btree[b->c.btree_id]++; + return ret; } int bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b, @@ -1269,9 +1275,26 @@ void bch2_btree_node_to_text(struct printbuf *out, struct bch_fs *c, const struc stats.failed); } +static void prt_btree_cache_line(struct printbuf *out, const struct bch_fs *c, + const char *label, unsigned nr) +{ + prt_printf(out, "%s\t", label); + prt_human_readable_u64(out, nr * c->opts.btree_node_size); + prt_printf(out, " (%u)\n", nr); +} + void bch2_btree_cache_to_text(struct printbuf *out, const struct bch_fs *c) { - prt_printf(out, "nr nodes:\t\t%u\n", c->btree_cache.used); - prt_printf(out, "nr dirty:\t\t%u\n", atomic_read(&c->btree_cache.dirty)); - prt_printf(out, "cannibalize lock:\t%p\n", c->btree_cache.alloc_lock); + const struct btree_cache *bc = &c->btree_cache; + + if (!out->nr_tabstops) + printbuf_tabstop_push(out, 24); + + prt_btree_cache_line(out, c, "total:", bc->used); + prt_btree_cache_line(out, c, "nr dirty:", atomic_read(&bc->dirty)); + prt_printf(out, "cannibalize lock:\t%p\n", bc->alloc_lock); + prt_newline(out); + + for (unsigned i = 0; i < ARRAY_SIZE(bc->used_by_btree); i++) + prt_btree_cache_line(out, c, bch2_btree_id_str(i), bc->used_by_btree[i]); } diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index 4ff5213219a50..76364bd4347e3 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -166,6 +166,8 @@ struct btree_cache { atomic_t dirty; struct shrinker *shrink; + unsigned used_by_btree[BTREE_ID_NR]; + /* * If we need to allocate memory for a new btree node and that * allocation fails, we can cannibalize another node in the btree cache From e2f48c48090dea172c0c571101041de64634dae5 Mon Sep 17 00:00:00 2001 From: Petr Vorel Date: Tue, 7 May 2024 17:37:57 +0200 Subject: [PATCH 1486/1702] bcachefs: Move BCACHEFS_STATFS_MAGIC value to UAPI magic.h Move BCACHEFS_STATFS_MAGIC value to UAPI under BCACHEFS_SUPER_MAGIC definition (use common approach for name) and reuse the definition in bcachefs_format.h BCACHEFS_STATFS_MAGIC. There are other bcachefs magic definitions: BCACHE_MAGIC, BCHFS_MAGIC, which use UUID_INIT() and are used only in libbcachefs. Therefore move only BCACHEFS_STATFS_MAGIC value, which can be used outside of libbcachefs for f_type field in struct statfs in statfs() or fstatfs(). Suggested-by: Su Yue Signed-off-by: Petr Vorel Acked-by: Brian Foster Signed-off-by: Kent Overstreet --- fs/bcachefs/bcachefs_format.h | 3 ++- include/uapi/linux/magic.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/bcachefs/bcachefs_format.h b/fs/bcachefs/bcachefs_format.h index d5b90439e581f..1bebba881d899 100644 --- a/fs/bcachefs/bcachefs_format.h +++ b/fs/bcachefs/bcachefs_format.h @@ -76,6 +76,7 @@ #include #include #include +#include #include "vstructs.h" #ifdef __KERNEL__ @@ -1290,7 +1291,7 @@ enum bch_compression_opts { UUID_INIT(0xc68573f6, 0x66ce, 0x90a9, \ 0xd9, 0x6a, 0x60, 0xcf, 0x80, 0x3d, 0xf7, 0xef) -#define BCACHEFS_STATFS_MAGIC 0xca451a4e +#define BCACHEFS_STATFS_MAGIC BCACHEFS_SUPER_MAGIC #define JSET_MAGIC __cpu_to_le64(0x245235c1a3625032ULL) #define BSET_MAGIC __cpu_to_le64(0x90135c78b99e07f5ULL) diff --git a/include/uapi/linux/magic.h b/include/uapi/linux/magic.h index 1b40a968ba91f..bb575f3ab45e5 100644 --- a/include/uapi/linux/magic.h +++ b/include/uapi/linux/magic.h @@ -37,6 +37,7 @@ #define HOSTFS_SUPER_MAGIC 0x00c0ffee #define OVERLAYFS_SUPER_MAGIC 0x794c7630 #define FUSE_SUPER_MAGIC 0x65735546 +#define BCACHEFS_SUPER_MAGIC 0xca451a4e #define MINIX_SUPER_MAGIC 0x137F /* minix v1 fs, 14 char names */ #define MINIX_SUPER_MAGIC2 0x138F /* minix v1 fs, 30 char names */ From aa4074e8fec4d2e686daee627fcafb3503efe365 Mon Sep 17 00:00:00 2001 From: Wu Bo Date: Sun, 28 Apr 2024 21:51:42 -0600 Subject: [PATCH 1487/1702] f2fs: fix block migration when section is not aligned to pow2 As for zoned-UFS, f2fs section size is forced to zone size. And zone size may not aligned to pow2. Fixes: 859fca6b706e ("f2fs: swap: support migrating swapfile in aligned write mode") Signed-off-by: Liao Yuanhong Signed-off-by: Wu Bo Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/data.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index a990236f137ea..bdbd4195cadc7 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -3917,15 +3917,14 @@ static int check_swap_activate(struct swap_info_struct *sis, struct address_space *mapping = swap_file->f_mapping; struct inode *inode = mapping->host; struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - sector_t cur_lblock; - sector_t last_lblock; - sector_t pblock; - sector_t lowest_pblock = -1; - sector_t highest_pblock = 0; + block_t cur_lblock; + block_t last_lblock; + block_t pblock; + block_t lowest_pblock = -1; + block_t highest_pblock = 0; int nr_extents = 0; - unsigned long nr_pblocks; + unsigned int nr_pblocks; unsigned int blks_per_sec = BLKS_PER_SEC(sbi); - unsigned int sec_blks_mask = BLKS_PER_SEC(sbi) - 1; unsigned int not_aligned = 0; int ret = 0; @@ -3963,8 +3962,8 @@ static int check_swap_activate(struct swap_info_struct *sis, pblock = map.m_pblk; nr_pblocks = map.m_len; - if ((pblock - SM_I(sbi)->main_blkaddr) & sec_blks_mask || - nr_pblocks & sec_blks_mask || + if ((pblock - SM_I(sbi)->main_blkaddr) % blks_per_sec || + nr_pblocks % blks_per_sec || !f2fs_valid_pinned_area(sbi, pblock)) { bool last_extent = false; From a78118406d52dde495311c0c4917613868b53169 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Mon, 6 May 2024 18:47:42 +0800 Subject: [PATCH 1488/1702] f2fs: use f2fs_{err,info}_ratelimited() for cleanup Commit b1c9d3f833ba ("f2fs: support printk_ratelimited() in f2fs_printk()") missed some cases, cover all remains for cleanup. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/compress.c | 54 +++++++++++++++++++++------------------------- fs/f2fs/segment.c | 5 ++--- 2 files changed, 26 insertions(+), 33 deletions(-) diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index b12d3a49bfdac..1ef82a5463910 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -198,8 +198,8 @@ static int lzo_compress_pages(struct compress_ctx *cc) ret = lzo1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata, &cc->clen, cc->private); if (ret != LZO_E_OK) { - printk_ratelimited("%sF2FS-fs (%s): lzo compress failed, ret:%d\n", - KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, ret); + f2fs_err_ratelimited(F2FS_I_SB(cc->inode), + "lzo compress failed, ret:%d", ret); return -EIO; } return 0; @@ -212,17 +212,15 @@ static int lzo_decompress_pages(struct decompress_io_ctx *dic) ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen, dic->rbuf, &dic->rlen); if (ret != LZO_E_OK) { - printk_ratelimited("%sF2FS-fs (%s): lzo decompress failed, ret:%d\n", - KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret); + f2fs_err_ratelimited(F2FS_I_SB(dic->inode), + "lzo decompress failed, ret:%d", ret); return -EIO; } if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) { - printk_ratelimited("%sF2FS-fs (%s): lzo invalid rlen:%zu, " - "expected:%lu\n", KERN_ERR, - F2FS_I_SB(dic->inode)->sb->s_id, - dic->rlen, - PAGE_SIZE << dic->log_cluster_size); + f2fs_err_ratelimited(F2FS_I_SB(dic->inode), + "lzo invalid rlen:%zu, expected:%lu", + dic->rlen, PAGE_SIZE << dic->log_cluster_size); return -EIO; } return 0; @@ -294,16 +292,15 @@ static int lz4_decompress_pages(struct decompress_io_ctx *dic) ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf, dic->clen, dic->rlen); if (ret < 0) { - printk_ratelimited("%sF2FS-fs (%s): lz4 decompress failed, ret:%d\n", - KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret); + f2fs_err_ratelimited(F2FS_I_SB(dic->inode), + "lz4 decompress failed, ret:%d", ret); return -EIO; } if (ret != PAGE_SIZE << dic->log_cluster_size) { - printk_ratelimited("%sF2FS-fs (%s): lz4 invalid ret:%d, " - "expected:%lu\n", KERN_ERR, - F2FS_I_SB(dic->inode)->sb->s_id, ret, - PAGE_SIZE << dic->log_cluster_size); + f2fs_err_ratelimited(F2FS_I_SB(dic->inode), + "lz4 invalid ret:%d, expected:%lu", + ret, PAGE_SIZE << dic->log_cluster_size); return -EIO; } return 0; @@ -350,9 +347,8 @@ static int zstd_init_compress_ctx(struct compress_ctx *cc) stream = zstd_init_cstream(¶ms, 0, workspace, workspace_size); if (!stream) { - printk_ratelimited("%sF2FS-fs (%s): %s zstd_init_cstream failed\n", - KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, - __func__); + f2fs_err_ratelimited(F2FS_I_SB(cc->inode), + "%s zstd_init_cstream failed", __func__); kvfree(workspace); return -EIO; } @@ -390,16 +386,16 @@ static int zstd_compress_pages(struct compress_ctx *cc) ret = zstd_compress_stream(stream, &outbuf, &inbuf); if (zstd_is_error(ret)) { - printk_ratelimited("%sF2FS-fs (%s): %s zstd_compress_stream failed, ret: %d\n", - KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, + f2fs_err_ratelimited(F2FS_I_SB(cc->inode), + "%s zstd_compress_stream failed, ret: %d", __func__, zstd_get_error_code(ret)); return -EIO; } ret = zstd_end_stream(stream, &outbuf); if (zstd_is_error(ret)) { - printk_ratelimited("%sF2FS-fs (%s): %s zstd_end_stream returned %d\n", - KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, + f2fs_err_ratelimited(F2FS_I_SB(cc->inode), + "%s zstd_end_stream returned %d", __func__, zstd_get_error_code(ret)); return -EIO; } @@ -432,9 +428,8 @@ static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic) stream = zstd_init_dstream(max_window_size, workspace, workspace_size); if (!stream) { - printk_ratelimited("%sF2FS-fs (%s): %s zstd_init_dstream failed\n", - KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, - __func__); + f2fs_err_ratelimited(F2FS_I_SB(dic->inode), + "%s zstd_init_dstream failed", __func__); kvfree(workspace); return -EIO; } @@ -469,16 +464,15 @@ static int zstd_decompress_pages(struct decompress_io_ctx *dic) ret = zstd_decompress_stream(stream, &outbuf, &inbuf); if (zstd_is_error(ret)) { - printk_ratelimited("%sF2FS-fs (%s): %s zstd_decompress_stream failed, ret: %d\n", - KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, + f2fs_err_ratelimited(F2FS_I_SB(dic->inode), + "%s zstd_decompress_stream failed, ret: %d", __func__, zstd_get_error_code(ret)); return -EIO; } if (dic->rlen != outbuf.pos) { - printk_ratelimited("%sF2FS-fs (%s): %s ZSTD invalid rlen:%zu, " - "expected:%lu\n", KERN_ERR, - F2FS_I_SB(dic->inode)->sb->s_id, + f2fs_err_ratelimited(F2FS_I_SB(dic->inode), + "%s ZSTD invalid rlen:%zu, expected:%lu", __func__, dic->rlen, PAGE_SIZE << dic->log_cluster_size); return -EIO; diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 8ddd4f1c665a0..ab3e9a6e9c96c 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -1109,9 +1109,8 @@ static void __remove_discard_cmd(struct f2fs_sb_info *sbi, dc->error = 0; if (dc->error) - printk_ratelimited( - "%sF2FS-fs (%s): Issue discard(%u, %u, %u) failed, ret: %d", - KERN_INFO, sbi->sb->s_id, + f2fs_info_ratelimited(sbi, + "Issue discard(%u, %u, %u) failed, ret: %d", dc->di.lstart, dc->di.start, dc->di.len, dc->error); __detach_discard_cmd(dcc, dc); } From 968c4f72b23c0c8f1e94e942eab89b8c5a3022e7 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Mon, 6 May 2024 18:45:37 +0800 Subject: [PATCH 1489/1702] f2fs: remove unused GC_FAILURE_PIN After commit 3db1de0e582c ("f2fs: change the current atomic write way"), we removed all GC_FAILURE_ATOMIC usage, let's change i_gc_failures[] array to i_pin_failure for cleanup. Meanwhile, let's define i_current_depth and i_gc_failures as union variable due to they won't be valid at the same time. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/f2fs.h | 14 +++++--------- fs/f2fs/file.c | 12 +++++------- fs/f2fs/inode.c | 6 ++---- fs/f2fs/recovery.c | 3 +-- 4 files changed, 13 insertions(+), 22 deletions(-) diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 3a95c8ccb9e0b..a4ebf3e91b68f 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -765,11 +765,6 @@ enum { #define DEF_DIR_LEVEL 0 -enum { - GC_FAILURE_PIN, - MAX_GC_FAILURE -}; - /* used for f2fs_inode_info->flags */ enum { FI_NEW_INODE, /* indicate newly allocated inode */ @@ -816,9 +811,10 @@ struct f2fs_inode_info { unsigned long i_flags; /* keep an inode flags for ioctl */ unsigned char i_advise; /* use to give file attribute hints */ unsigned char i_dir_level; /* use for dentry level for large dir */ - unsigned int i_current_depth; /* only for directory depth */ - /* for gc failure statistic */ - unsigned int i_gc_failures[MAX_GC_FAILURE]; + union { + unsigned int i_current_depth; /* only for directory depth */ + unsigned int i_gc_failures; /* for gc failure statistic */ + }; unsigned int i_pino; /* parent inode number */ umode_t i_acl_mode; /* keep file acl mode temporarily */ @@ -3133,7 +3129,7 @@ static inline void f2fs_i_depth_write(struct inode *inode, unsigned int depth) static inline void f2fs_i_gc_failures_write(struct inode *inode, unsigned int count) { - F2FS_I(inode)->i_gc_failures[GC_FAILURE_PIN] = count; + F2FS_I(inode)->i_gc_failures = count; f2fs_mark_inode_dirty_sync(inode, true); } diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index f1a6b4da57466..92aaa6986f993 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -3217,13 +3217,11 @@ int f2fs_pin_file_control(struct inode *inode, bool inc) /* Use i_gc_failures for normal file as a risk signal. */ if (inc) - f2fs_i_gc_failures_write(inode, - fi->i_gc_failures[GC_FAILURE_PIN] + 1); + f2fs_i_gc_failures_write(inode, fi->i_gc_failures + 1); - if (fi->i_gc_failures[GC_FAILURE_PIN] > sbi->gc_pin_file_threshold) { + if (fi->i_gc_failures > sbi->gc_pin_file_threshold) { f2fs_warn(sbi, "%s: Enable GC = ino %lx after %x GC trials", - __func__, inode->i_ino, - fi->i_gc_failures[GC_FAILURE_PIN]); + __func__, inode->i_ino, fi->i_gc_failures); clear_inode_flag(inode, FI_PIN_FILE); return -EAGAIN; } @@ -3287,7 +3285,7 @@ static int f2fs_ioc_set_pin_file(struct file *filp, unsigned long arg) } set_inode_flag(inode, FI_PIN_FILE); - ret = F2FS_I(inode)->i_gc_failures[GC_FAILURE_PIN]; + ret = F2FS_I(inode)->i_gc_failures; done: f2fs_update_time(sbi, REQ_TIME); out: @@ -3302,7 +3300,7 @@ static int f2fs_ioc_get_pin_file(struct file *filp, unsigned long arg) __u32 pin = 0; if (is_inode_flag_set(inode, FI_PIN_FILE)) - pin = F2FS_I(inode)->i_gc_failures[GC_FAILURE_PIN]; + pin = F2FS_I(inode)->i_gc_failures; return put_user(pin, (u32 __user *)arg); } diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index 7968b14d49f40..005dde72aff3d 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -415,8 +415,7 @@ static int do_read_inode(struct inode *inode) if (S_ISDIR(inode->i_mode)) fi->i_current_depth = le32_to_cpu(ri->i_current_depth); else if (S_ISREG(inode->i_mode)) - fi->i_gc_failures[GC_FAILURE_PIN] = - le16_to_cpu(ri->i_gc_failures); + fi->i_gc_failures = le16_to_cpu(ri->i_gc_failures); fi->i_xattr_nid = le32_to_cpu(ri->i_xattr_nid); fi->i_flags = le32_to_cpu(ri->i_flags); if (S_ISREG(inode->i_mode)) @@ -686,8 +685,7 @@ void f2fs_update_inode(struct inode *inode, struct page *node_page) ri->i_current_depth = cpu_to_le32(F2FS_I(inode)->i_current_depth); else if (S_ISREG(inode->i_mode)) - ri->i_gc_failures = - cpu_to_le16(F2FS_I(inode)->i_gc_failures[GC_FAILURE_PIN]); + ri->i_gc_failures = cpu_to_le16(F2FS_I(inode)->i_gc_failures); ri->i_xattr_nid = cpu_to_le32(F2FS_I(inode)->i_xattr_nid); ri->i_flags = cpu_to_le32(F2FS_I(inode)->i_flags); ri->i_pino = cpu_to_le32(F2FS_I(inode)->i_pino); diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index e7bf15b8240ae..496aee53c38a2 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -330,8 +330,7 @@ static int recover_inode(struct inode *inode, struct page *page) F2FS_I(inode)->i_advise = raw->i_advise; F2FS_I(inode)->i_flags = le32_to_cpu(raw->i_flags); f2fs_set_inode_flags(inode); - F2FS_I(inode)->i_gc_failures[GC_FAILURE_PIN] = - le16_to_cpu(raw->i_gc_failures); + F2FS_I(inode)->i_gc_failures = le16_to_cpu(raw->i_gc_failures); recover_inline_flags(inode, raw); From c521a6ab4ad75fe0755f4cd923a84e1289153aa2 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Mon, 6 May 2024 18:45:38 +0800 Subject: [PATCH 1490/1702] f2fs: fix to limit gc_pin_file_threshold type of f2fs_inode.i_gc_failures, f2fs_inode_info.i_gc_failures, and f2fs_sb_info.gc_pin_file_threshold is __le16, unsigned int, and u64, so it will cause truncation during comparison and persistence. Unifying variable of these three variables to unsigned short, and add an upper boundary limitation for gc_pin_file_threshold. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- Documentation/ABI/testing/sysfs-fs-f2fs | 2 +- fs/f2fs/f2fs.h | 4 ++-- fs/f2fs/file.c | 11 ++++++----- fs/f2fs/gc.h | 1 + fs/f2fs/sysfs.c | 7 +++++++ 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs index 1a4d83953379e..cad6c3dc1f9c1 100644 --- a/Documentation/ABI/testing/sysfs-fs-f2fs +++ b/Documentation/ABI/testing/sysfs-fs-f2fs @@ -331,7 +331,7 @@ Date: January 2018 Contact: Jaegeuk Kim Description: This indicates how many GC can be failed for the pinned file. If it exceeds this, F2FS doesn't guarantee its pinning - state. 2048 trials is set by default. + state. 2048 trials is set by default, and 65535 as maximum. What: /sys/fs/f2fs//extension_list Date: February 2018 diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index a4ebf3e91b68f..1352da2dfaa33 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -813,7 +813,7 @@ struct f2fs_inode_info { unsigned char i_dir_level; /* use for dentry level for large dir */ union { unsigned int i_current_depth; /* only for directory depth */ - unsigned int i_gc_failures; /* for gc failure statistic */ + unsigned short i_gc_failures; /* for gc failure statistic */ }; unsigned int i_pino; /* parent inode number */ umode_t i_acl_mode; /* keep file acl mode temporarily */ @@ -1673,7 +1673,7 @@ struct f2fs_sb_info { unsigned long long skipped_gc_rwsem; /* FG_GC only */ /* threshold for gc trials on pinned files */ - u64 gc_pin_file_threshold; + unsigned short gc_pin_file_threshold; struct f2fs_rwsem pin_sem; /* maximum # of trials to find a victim segment for SSR and GC */ diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 92aaa6986f993..72ce1a522fb26 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -3215,16 +3215,17 @@ int f2fs_pin_file_control(struct inode *inode, bool inc) struct f2fs_inode_info *fi = F2FS_I(inode); struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - /* Use i_gc_failures for normal file as a risk signal. */ - if (inc) - f2fs_i_gc_failures_write(inode, fi->i_gc_failures + 1); - - if (fi->i_gc_failures > sbi->gc_pin_file_threshold) { + if (fi->i_gc_failures >= sbi->gc_pin_file_threshold) { f2fs_warn(sbi, "%s: Enable GC = ino %lx after %x GC trials", __func__, inode->i_ino, fi->i_gc_failures); clear_inode_flag(inode, FI_PIN_FILE); return -EAGAIN; } + + /* Use i_gc_failures for normal file as a risk signal. */ + if (inc) + f2fs_i_gc_failures_write(inode, fi->i_gc_failures + 1); + return 0; } diff --git a/fs/f2fs/gc.h b/fs/f2fs/gc.h index 9c0d06c4d19a9..a8ea3301b815a 100644 --- a/fs/f2fs/gc.h +++ b/fs/f2fs/gc.h @@ -26,6 +26,7 @@ #define LIMIT_FREE_BLOCK 40 /* percentage over invalid + free space */ #define DEF_GC_FAILED_PINNED_FILES 2048 +#define MAX_GC_FAILED_PINNED_FILES USHRT_MAX /* Search max. number of dirty segments to select a victim segment */ #define DEF_MAX_VICTIM_SEARCH 4096 /* covers 8GB */ diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index a568ce96cf563..531fc58001095 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -675,6 +675,13 @@ static ssize_t __sbi_store(struct f2fs_attr *a, return count; } + if (!strcmp(a->attr.name, "gc_pin_file_threshold")) { + if (t > MAX_GC_FAILED_PINNED_FILES) + return -EINVAL; + sbi->gc_pin_file_threshold = t; + return count; + } + if (!strcmp(a->attr.name, "gc_reclaimed_segments")) { if (t != 0) return -EINVAL; From 4ed886b187f47447ad559619c48c086f432d2b77 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Tue, 7 May 2024 11:38:47 +0800 Subject: [PATCH 1491/1702] f2fs: check validation of fault attrs in f2fs_build_fault_attr() - It missed to check validation of fault attrs in parse_options(), let's fix to add check condition in f2fs_build_fault_attr(). - Use f2fs_build_fault_attr() in __sbi_store() to clean up code. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/f2fs.h | 12 ++++++++---- fs/f2fs/super.c | 27 ++++++++++++++++++++------- fs/f2fs/sysfs.c | 14 ++++++++++---- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 1352da2dfaa33..30058e16a5d0d 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -72,7 +72,7 @@ enum { struct f2fs_fault_info { atomic_t inject_ops; - unsigned int inject_rate; + int inject_rate; unsigned int inject_type; }; @@ -4598,10 +4598,14 @@ static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx) } #ifdef CONFIG_F2FS_FAULT_INJECTION -extern void f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned int rate, - unsigned int type); +extern int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned long rate, + unsigned long type); #else -#define f2fs_build_fault_attr(sbi, rate, type) do { } while (0) +static int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned long rate, + unsigned long type) +{ + return 0; +} #endif static inline bool is_journalled_quota(struct f2fs_sb_info *sbi) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index ef673f853366b..daf2c4dbe1506 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -66,21 +66,31 @@ const char *f2fs_fault_name[FAULT_MAX] = { [FAULT_NO_SEGMENT] = "no free segment", }; -void f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned int rate, - unsigned int type) +int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned long rate, + unsigned long type) { struct f2fs_fault_info *ffi = &F2FS_OPTION(sbi).fault_info; if (rate) { + if (rate > INT_MAX) + return -EINVAL; atomic_set(&ffi->inject_ops, 0); - ffi->inject_rate = rate; + ffi->inject_rate = (int)rate; } - if (type) - ffi->inject_type = type; + if (type) { + if (type >= BIT(FAULT_MAX)) + return -EINVAL; + ffi->inject_type = (unsigned int)type; + } if (!rate && !type) memset(ffi, 0, sizeof(struct f2fs_fault_info)); + else + f2fs_info(sbi, + "build fault injection attr: rate: %lu, type: 0x%lx", + rate, type); + return 0; } #endif @@ -886,14 +896,17 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) case Opt_fault_injection: if (args->from && match_int(args, &arg)) return -EINVAL; - f2fs_build_fault_attr(sbi, arg, F2FS_ALL_FAULT_TYPE); + if (f2fs_build_fault_attr(sbi, arg, + F2FS_ALL_FAULT_TYPE)) + return -EINVAL; set_opt(sbi, FAULT_INJECTION); break; case Opt_fault_type: if (args->from && match_int(args, &arg)) return -EINVAL; - f2fs_build_fault_attr(sbi, 0, arg); + if (f2fs_build_fault_attr(sbi, 0, arg)) + return -EINVAL; set_opt(sbi, FAULT_INJECTION); break; #else diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index 531fc58001095..09d3ecfaa4f1a 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -484,10 +484,16 @@ static ssize_t __sbi_store(struct f2fs_attr *a, if (ret < 0) return ret; #ifdef CONFIG_F2FS_FAULT_INJECTION - if (a->struct_type == FAULT_INFO_TYPE && t >= BIT(FAULT_MAX)) - return -EINVAL; - if (a->struct_type == FAULT_INFO_RATE && t >= UINT_MAX) - return -EINVAL; + if (a->struct_type == FAULT_INFO_TYPE) { + if (f2fs_build_fault_attr(sbi, 0, t)) + return -EINVAL; + return count; + } + if (a->struct_type == FAULT_INFO_RATE) { + if (f2fs_build_fault_attr(sbi, t, 0)) + return -EINVAL; + return count; + } #endif if (a->struct_type == RESERVED_BLOCKS) { spin_lock(&sbi->stat_lock); From bc2da26599ed800357f24fca4b7f6a8f35d87dcb Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Wed, 8 May 2024 22:33:24 +0100 Subject: [PATCH 1492/1702] clk, reset: microchip: mpfs: fix incorrect preprocessor conditions While moving all the reset code in the PolarFire SoC clock driver to the reset subsystem, I removed an `#if IS_ENABLED(RESET_CONTROLLER)` from the driver and moved it to the header, however this was not the correct thing to do. In the driver such a condition over-eagerly provided a complete implementation for mpfs_reset_{read,write}() when the reset subsystem was enabled without the PolarFire SoC reset driver, but in the header it meant that when the subsystem was enabled and the driver was not, no implementation for mpfs_reset_controller_register() was provided. Fix the condition so that the stub implementation of mpfs_reset_controller_register() is used when the reset driver is disabled. Fixes: 098c290a490d ("clock, reset: microchip: move all mpfs reset code to the reset subsystem") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202405082259.44DzHvaN-lkp@intel.com/ Closes: https://lore.kernel.org/oe-kbuild-all/202405082200.tBrEs5CZ-lkp@intel.com/ Signed-off-by: Conor Dooley Link: https://lore.kernel.org/r/20240508-unabashed-cheese-8f645b4f69ba@spud Signed-off-by: Stephen Boyd --- include/soc/microchip/mpfs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/soc/microchip/mpfs.h b/include/soc/microchip/mpfs.h index d7e612b5e22ef..0bd67e10b7049 100644 --- a/include/soc/microchip/mpfs.h +++ b/include/soc/microchip/mpfs.h @@ -43,11 +43,11 @@ struct mtd_info *mpfs_sys_controller_get_flash(struct mpfs_sys_controller *mpfs_ #endif /* if IS_ENABLED(CONFIG_POLARFIRE_SOC_SYS_CTRL) */ #if IS_ENABLED(CONFIG_MCHP_CLK_MPFS) -#if IS_ENABLED(CONFIG_RESET_CONTROLLER) +#if IS_ENABLED(CONFIG_RESET_POLARFIRE_SOC) int mpfs_reset_controller_register(struct device *clk_dev, void __iomem *base); #else static inline int mpfs_reset_controller_register(struct device *clk_dev, void __iomem *base) { return 0; } -#endif /* if IS_ENABLED(CONFIG_RESET_CONTROLLER) */ +#endif /* if IS_ENABLED(CONFIG_RESET_POLARFIRE_SOC) */ #endif /* if IS_ENABLED(CONFIG_MCHP_CLK_MPFS) */ #endif /* __SOC_MPFS_H__ */ From ea4fd933ab4310822e244af28d22ff63785dea0e Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sat, 20 Apr 2024 03:50:05 +0100 Subject: [PATCH 1493/1702] ext4: remove calls to to set/clear the folio error flag Nobody checks this flag on ext4 folios, stop setting and clearing it. Cc: Theodore Ts'o Cc: Andreas Dilger Cc: linux-ext4@vger.kernel.org Signed-off-by: Matthew Wilcox (Oracle) Link: https://lore.kernel.org/r/20240420025029.2166544-11-willy@infradead.org Signed-off-by: Theodore Ts'o --- fs/ext4/move_extent.c | 4 +--- fs/ext4/page-io.c | 3 --- fs/ext4/readpage.c | 1 - 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/fs/ext4/move_extent.c b/fs/ext4/move_extent.c index 7cd4afa4de1d3..204f53b236229 100644 --- a/fs/ext4/move_extent.c +++ b/fs/ext4/move_extent.c @@ -199,10 +199,8 @@ mext_page_mkuptodate(struct folio *folio, unsigned from, unsigned to) continue; if (!buffer_mapped(bh)) { err = ext4_get_block(inode, block, bh, 0); - if (err) { - folio_set_error(folio); + if (err) return err; - } if (!buffer_mapped(bh)) { folio_zero_range(folio, block_start, blocksize); set_buffer_uptodate(bh); diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index 312bc68133574..ad5543866d215 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -117,7 +117,6 @@ static void ext4_finish_bio(struct bio *bio) if (bio->bi_status) { int err = blk_status_to_errno(bio->bi_status); - folio_set_error(folio); mapping_set_error(folio->mapping, err); } bh = head = folio_buffers(folio); @@ -441,8 +440,6 @@ int ext4_bio_write_folio(struct ext4_io_submit *io, struct folio *folio, BUG_ON(!folio_test_locked(folio)); BUG_ON(folio_test_writeback(folio)); - folio_clear_error(folio); - /* * Comments copied from block_write_full_folio: * diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c index 21e8f0aebb3c6..8494492582abe 100644 --- a/fs/ext4/readpage.c +++ b/fs/ext4/readpage.c @@ -289,7 +289,6 @@ int ext4_mpage_readpages(struct inode *inode, if (ext4_map_blocks(NULL, inode, &map, 0) < 0) { set_error_page: - folio_set_error(folio); folio_zero_segment(folio, 0, folio_size(folio)); folio_unlock(folio); From 78cfd17142ef70599d6409cbd709d94b3da58659 Mon Sep 17 00:00:00 2001 From: Michal Schmidt Date: Tue, 7 May 2024 12:39:28 +0200 Subject: [PATCH 1494/1702] bnxt_re: avoid shift undefined behavior in bnxt_qplib_alloc_init_hwq Undefined behavior is triggered when bnxt_qplib_alloc_init_hwq is called with hwq_attr->aux_depth != 0 and hwq_attr->aux_stride == 0. In that case, "roundup_pow_of_two(hwq_attr->aux_stride)" gets called. roundup_pow_of_two is documented as undefined for 0. Fix it in the one caller that had this combination. The undefined behavior was detected by UBSAN: UBSAN: shift-out-of-bounds in ./include/linux/log2.h:57:13 shift exponent 64 is too large for 64-bit type 'long unsigned int' CPU: 24 PID: 1075 Comm: (udev-worker) Not tainted 6.9.0-rc6+ #4 Hardware name: Abacus electric, s.r.o. - servis@abacus.cz Super Server/H12SSW-iN, BIOS 2.7 10/25/2023 Call Trace: dump_stack_lvl+0x5d/0x80 ubsan_epilogue+0x5/0x30 __ubsan_handle_shift_out_of_bounds.cold+0x61/0xec __roundup_pow_of_two+0x25/0x35 [bnxt_re] bnxt_qplib_alloc_init_hwq+0xa1/0x470 [bnxt_re] bnxt_qplib_create_qp+0x19e/0x840 [bnxt_re] bnxt_re_create_qp+0x9b1/0xcd0 [bnxt_re] ? srso_alias_return_thunk+0x5/0xfbef5 ? srso_alias_return_thunk+0x5/0xfbef5 ? __kmalloc+0x1b6/0x4f0 ? create_qp.part.0+0x128/0x1c0 [ib_core] ? __pfx_bnxt_re_create_qp+0x10/0x10 [bnxt_re] create_qp.part.0+0x128/0x1c0 [ib_core] ib_create_qp_kernel+0x50/0xd0 [ib_core] create_mad_qp+0x8e/0xe0 [ib_core] ? __pfx_qp_event_handler+0x10/0x10 [ib_core] ib_mad_init_device+0x2be/0x680 [ib_core] add_client_context+0x10d/0x1a0 [ib_core] enable_device_and_get+0xe0/0x1d0 [ib_core] ib_register_device+0x53c/0x630 [ib_core] ? srso_alias_return_thunk+0x5/0xfbef5 bnxt_re_probe+0xbd8/0xe50 [bnxt_re] ? __pfx_bnxt_re_probe+0x10/0x10 [bnxt_re] auxiliary_bus_probe+0x49/0x80 ? driver_sysfs_add+0x57/0xc0 really_probe+0xde/0x340 ? pm_runtime_barrier+0x54/0x90 ? __pfx___driver_attach+0x10/0x10 __driver_probe_device+0x78/0x110 driver_probe_device+0x1f/0xa0 __driver_attach+0xba/0x1c0 bus_for_each_dev+0x8f/0xe0 bus_add_driver+0x146/0x220 driver_register+0x72/0xd0 __auxiliary_driver_register+0x6e/0xd0 ? __pfx_bnxt_re_mod_init+0x10/0x10 [bnxt_re] bnxt_re_mod_init+0x3e/0xff0 [bnxt_re] ? __pfx_bnxt_re_mod_init+0x10/0x10 [bnxt_re] do_one_initcall+0x5b/0x310 do_init_module+0x90/0x250 init_module_from_file+0x86/0xc0 idempotent_init_module+0x121/0x2b0 __x64_sys_finit_module+0x5e/0xb0 do_syscall_64+0x82/0x160 ? srso_alias_return_thunk+0x5/0xfbef5 ? syscall_exit_to_user_mode_prepare+0x149/0x170 ? srso_alias_return_thunk+0x5/0xfbef5 ? syscall_exit_to_user_mode+0x75/0x230 ? srso_alias_return_thunk+0x5/0xfbef5 ? do_syscall_64+0x8e/0x160 ? srso_alias_return_thunk+0x5/0xfbef5 ? __count_memcg_events+0x69/0x100 ? srso_alias_return_thunk+0x5/0xfbef5 ? count_memcg_events.constprop.0+0x1a/0x30 ? srso_alias_return_thunk+0x5/0xfbef5 ? handle_mm_fault+0x1f0/0x300 ? srso_alias_return_thunk+0x5/0xfbef5 ? do_user_addr_fault+0x34e/0x640 ? srso_alias_return_thunk+0x5/0xfbef5 ? srso_alias_return_thunk+0x5/0xfbef5 entry_SYSCALL_64_after_hwframe+0x76/0x7e RIP: 0033:0x7f4e5132821d Code: ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d e3 db 0c 00 f7 d8 64 89 01 48 RSP: 002b:00007ffca9c906a8 EFLAGS: 00000246 ORIG_RAX: 0000000000000139 RAX: ffffffffffffffda RBX: 0000563ec8a8f130 RCX: 00007f4e5132821d RDX: 0000000000000000 RSI: 00007f4e518fa07d RDI: 000000000000003b RBP: 00007ffca9c90760 R08: 00007f4e513f6b20 R09: 00007ffca9c906f0 R10: 0000563ec8a8faa0 R11: 0000000000000246 R12: 00007f4e518fa07d R13: 0000000000020000 R14: 0000563ec8409e90 R15: 0000563ec8a8fa60 ---[ end trace ]--- Fixes: 0c4dcd602817 ("RDMA/bnxt_re: Refactor hardware queue memory allocation") Signed-off-by: Michal Schmidt Link: https://lore.kernel.org/r/20240507103929.30003-1-mschmidt@redhat.com Acked-by: Selvin Xavier Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/bnxt_re/qplib_fp.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c index 439d0c7c5d0ca..04258676d0726 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -1013,7 +1013,8 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) hwq_attr.stride = sizeof(struct sq_sge); hwq_attr.depth = bnxt_qplib_get_depth(sq); hwq_attr.aux_stride = psn_sz; - hwq_attr.aux_depth = bnxt_qplib_set_sq_size(sq, qp->wqe_mode); + hwq_attr.aux_depth = psn_sz ? bnxt_qplib_set_sq_size(sq, qp->wqe_mode) + : 0; /* Update msn tbl size */ if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) { hwq_attr.aux_depth = roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, qp->wqe_mode)); From bafa6b4d95d97877baa61883ff90f7e374427fae Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 7 May 2024 09:10:41 -0400 Subject: [PATCH 1495/1702] SUNRPC: Fix gss_free_in_token_pages() Dan Carpenter says: > Commit 5866efa8cbfb ("SUNRPC: Fix svcauth_gss_proxy_init()") from Oct > 24, 2019 (linux-next), leads to the following Smatch static checker > warning: > > net/sunrpc/auth_gss/svcauth_gss.c:1039 gss_free_in_token_pages() > warn: iterator 'i' not incremented > > net/sunrpc/auth_gss/svcauth_gss.c > 1034 static void gss_free_in_token_pages(struct gssp_in_token *in_token) > 1035 { > 1036 u32 inlen; > 1037 int i; > 1038 > --> 1039 i = 0; > 1040 inlen = in_token->page_len; > 1041 while (inlen) { > 1042 if (in_token->pages[i]) > 1043 put_page(in_token->pages[i]); > ^ > This puts page zero over and over. > > 1044 inlen -= inlen > PAGE_SIZE ? PAGE_SIZE : inlen; > 1045 } > 1046 > 1047 kfree(in_token->pages); > 1048 in_token->pages = NULL; > 1049 } Based on the way that the ->pages[] array is constructed in gss_read_proxy_verf(), we know that once the loop encounters a NULL page pointer, the remaining array elements must also be NULL. Reported-by: Dan Carpenter Suggested-by: Trond Myklebust Fixes: 5866efa8cbfb ("SUNRPC: Fix svcauth_gss_proxy_init()") Signed-off-by: Chuck Lever --- net/sunrpc/auth_gss/svcauth_gss.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c index 24de941847003..96ab50eda9c2e 100644 --- a/net/sunrpc/auth_gss/svcauth_gss.c +++ b/net/sunrpc/auth_gss/svcauth_gss.c @@ -1033,17 +1033,11 @@ svcauth_gss_proc_init_verf(struct cache_detail *cd, struct svc_rqst *rqstp, static void gss_free_in_token_pages(struct gssp_in_token *in_token) { - u32 inlen; int i; i = 0; - inlen = in_token->page_len; - while (inlen) { - if (in_token->pages[i]) - put_page(in_token->pages[i]); - inlen -= inlen > PAGE_SIZE ? PAGE_SIZE : inlen; - } - + while (in_token->pages[i]) + put_page(in_token->pages[i++]); kfree(in_token->pages); in_token->pages = NULL; } From 8d915bbf39266bb66082c1e4980e123883f19830 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 7 May 2024 09:37:14 -0400 Subject: [PATCH 1496/1702] NFSD: Force all NFSv4.2 COPY requests to be synchronous We've discovered that delivering a CB_OFFLOAD operation can be unreliable in some pretty unremarkable situations. Examples include: - The server dropped the connection because it lost a forechannel NFSv4 request and wishes to force the client to retransmit - The GSS sequence number window under-flowed - A network partition occurred When that happens, all pending callback operations, including CB_OFFLOAD, are lost. NFSD does not retransmit them. Moreover, the Linux NFS client does not yet support sending an OFFLOAD_STATUS operation to probe whether an asynchronous COPY operation has finished. Thus, on Linux NFS clients, when a CB_OFFLOAD is lost, asynchronous COPY can hang until manually interrupted. I've tried a couple of remedies, but so far the side-effects are worse than the disease and they have had to be reverted. So temporarily force COPY operations to be synchronous so that the use of CB_OFFLOAD is avoided entirely. This is a fix that can easily be backported to LTS kernels. I am working on client patches that introduce an implementation of OFFLOAD_STATUS. Note that NFSD arbitrarily limits the size of a copy_file_range to 4MB to avoid indefinitely blocking an nfsd thread. A short COPY result is returned in that case, and the client can present a fresh COPY request for the remainder. Signed-off-by: Chuck Lever --- fs/nfsd/nfs4proc.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index ea3cc3e870a7f..46bd20fe5c0f4 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -1807,6 +1807,13 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, __be32 status; struct nfsd4_copy *async_copy = NULL; + /* + * Currently, async COPY is not reliable. Force all COPY + * requests to be synchronous to avoid client application + * hangs waiting for COPY completion. + */ + nfsd4_copy_set_sync(copy, true); + copy->cp_clp = cstate->clp; if (nfsd4_ssc_is_inter(copy)) { trace_nfsd_copy_inter(copy); From b4b4fda34e535756f9e774fb2d09c4537b7dfd1c Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Tue, 2 Jan 2024 21:37:30 +0800 Subject: [PATCH 1497/1702] ext4: fix uninitialized ratelimit_state->lock access in __ext4_fill_super() In the following concurrency we will access the uninitialized rs->lock: ext4_fill_super ext4_register_sysfs // sysfs registered msg_ratelimit_interval_ms // Other processes modify rs->interval to // non-zero via msg_ratelimit_interval_ms ext4_orphan_cleanup ext4_msg(sb, KERN_INFO, "Errors on filesystem, " __ext4_msg ___ratelimit(&(EXT4_SB(sb)->s_msg_ratelimit_state) if (!rs->interval) // do nothing if interval is 0 return 1; raw_spin_trylock_irqsave(&rs->lock, flags) raw_spin_trylock(lock) _raw_spin_trylock __raw_spin_trylock spin_acquire(&lock->dep_map, 0, 1, _RET_IP_) lock_acquire __lock_acquire register_lock_class assign_lock_key dump_stack(); ratelimit_state_init(&sbi->s_msg_ratelimit_state, 5 * HZ, 10); raw_spin_lock_init(&rs->lock); // init rs->lock here and get the following dump_stack: ========================================================= INFO: trying to register non-static key. The code is fine but needs lockdep annotation, or maybe you didn't initialize this object before use? turning off the locking correctness validator. CPU: 12 PID: 753 Comm: mount Tainted: G E 6.7.0-rc6-next-20231222 #504 [...] Call Trace: dump_stack_lvl+0xc5/0x170 dump_stack+0x18/0x30 register_lock_class+0x740/0x7c0 __lock_acquire+0x69/0x13a0 lock_acquire+0x120/0x450 _raw_spin_trylock+0x98/0xd0 ___ratelimit+0xf6/0x220 __ext4_msg+0x7f/0x160 [ext4] ext4_orphan_cleanup+0x665/0x740 [ext4] __ext4_fill_super+0x21ea/0x2b10 [ext4] ext4_fill_super+0x14d/0x360 [ext4] [...] ========================================================= Normally interval is 0 until s_msg_ratelimit_state is initialized, so ___ratelimit() does nothing. But registering sysfs precedes initializing rs->lock, so it is possible to change rs->interval to a non-zero value via the msg_ratelimit_interval_ms interface of sysfs while rs->lock is uninitialized, and then a call to ext4_msg triggers the problem by accessing an uninitialized rs->lock. Therefore register sysfs after all initializations are complete to avoid such problems. Signed-off-by: Baokun Li Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20240102133730.1098120-1-libaokun1@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/super.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/fs/ext4/super.c b/fs/ext4/super.c index f9b2b18374f3d..f9a4a4e89dac1 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -5550,19 +5550,15 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) if (err) goto failed_mount6; - err = ext4_register_sysfs(sb); - if (err) - goto failed_mount7; - err = ext4_init_orphan_info(sb); if (err) - goto failed_mount8; + goto failed_mount7; #ifdef CONFIG_QUOTA /* Enable quota usage during mount. */ if (ext4_has_feature_quota(sb) && !sb_rdonly(sb)) { err = ext4_enable_quotas(sb); if (err) - goto failed_mount9; + goto failed_mount8; } #endif /* CONFIG_QUOTA */ @@ -5588,7 +5584,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) ext4_msg(sb, KERN_INFO, "recovery complete"); err = ext4_mark_recovery_complete(sb, es); if (err) - goto failed_mount10; + goto failed_mount9; } if (test_opt(sb, DISCARD) && !bdev_max_discard_sectors(sb->s_bdev)) @@ -5605,15 +5601,17 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) atomic_set(&sbi->s_warning_count, 0); atomic_set(&sbi->s_msg_count, 0); + /* Register sysfs after all initializations are complete. */ + err = ext4_register_sysfs(sb); + if (err) + goto failed_mount9; + return 0; -failed_mount10: +failed_mount9: ext4_quotas_off(sb, EXT4_MAXQUOTAS); -failed_mount9: __maybe_unused +failed_mount8: __maybe_unused ext4_release_orphan_info(sb); -failed_mount8: - ext4_unregister_sysfs(sb); - kobject_put(&sbi->s_kobj); failed_mount7: ext4_unregister_li_request(sb); failed_mount6: From 078760d950016f5982751f5512e69f26ad8feb31 Mon Sep 17 00:00:00 2001 From: Ye Bin Date: Sun, 7 Apr 2024 14:53:54 +0800 Subject: [PATCH 1498/1702] jbd2: use shrink_type type instead of bool type for __jbd2_journal_clean_checkpoint_list() "enum shrink_type" can clearly express the meaning of the parameter of __jbd2_journal_clean_checkpoint_list(), and there is no need to use the bool type. Signed-off-by: Ye Bin Reviewed-by: Jan Kara Reviewed-by: Zhang Yi Link: https://lore.kernel.org/r/20240407065355.1528580-2-yebin10@huawei.com Signed-off-by: Theodore Ts'o --- fs/jbd2/checkpoint.c | 16 +++++++++------- fs/jbd2/commit.c | 2 +- include/linux/jbd2.h | 4 +++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c index 1c97e64c47849..80c0ab98bc639 100644 --- a/fs/jbd2/checkpoint.c +++ b/fs/jbd2/checkpoint.c @@ -337,8 +337,6 @@ int jbd2_cleanup_journal_tail(journal_t *journal) /* Checkpoint list management */ -enum shrink_type {SHRINK_DESTROY, SHRINK_BUSY_STOP, SHRINK_BUSY_SKIP}; - /* * journal_shrink_one_cp_list * @@ -472,21 +470,25 @@ unsigned long jbd2_journal_shrink_checkpoint_list(journal_t *journal, * journal_clean_checkpoint_list * * Find all the written-back checkpoint buffers in the journal and release them. - * If 'destroy' is set, release all buffers unconditionally. + * If 'type' is SHRINK_DESTROY, release all buffers unconditionally. If 'type' + * is SHRINK_BUSY_STOP, will stop release buffers if encounters a busy buffer. + * To avoid wasting CPU cycles scanning the buffer list in some cases, don't + * pass SHRINK_BUSY_SKIP 'type' for this function. * * Called with j_list_lock held. */ -void __jbd2_journal_clean_checkpoint_list(journal_t *journal, bool destroy) +void __jbd2_journal_clean_checkpoint_list(journal_t *journal, + enum shrink_type type) { transaction_t *transaction, *last_transaction, *next_transaction; - enum shrink_type type; bool released; + WARN_ON_ONCE(type == SHRINK_BUSY_SKIP); + transaction = journal->j_checkpoint_transactions; if (!transaction) return; - type = destroy ? SHRINK_DESTROY : SHRINK_BUSY_STOP; last_transaction = transaction->t_cpprev; next_transaction = transaction; do { @@ -527,7 +529,7 @@ void jbd2_journal_destroy_checkpoint(journal_t *journal) spin_unlock(&journal->j_list_lock); break; } - __jbd2_journal_clean_checkpoint_list(journal, true); + __jbd2_journal_clean_checkpoint_list(journal, SHRINK_DESTROY); spin_unlock(&journal->j_list_lock); cond_resched(); } diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c index 78a9d08ae9f85..b341c396311b4 100644 --- a/fs/jbd2/commit.c +++ b/fs/jbd2/commit.c @@ -501,7 +501,7 @@ void jbd2_journal_commit_transaction(journal_t *journal) * frees some memory */ spin_lock(&journal->j_list_lock); - __jbd2_journal_clean_checkpoint_list(journal, false); + __jbd2_journal_clean_checkpoint_list(journal, SHRINK_BUSY_STOP); spin_unlock(&journal->j_list_lock); jbd2_debug(3, "JBD2: commit phase 1\n"); diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h index 971f3e826e152..58a961999d704 100644 --- a/include/linux/jbd2.h +++ b/include/linux/jbd2.h @@ -1434,7 +1434,9 @@ void jbd2_update_log_tail(journal_t *journal, tid_t tid, unsigned long block); extern void jbd2_journal_commit_transaction(journal_t *); /* Checkpoint list management */ -void __jbd2_journal_clean_checkpoint_list(journal_t *journal, bool destroy); +enum shrink_type {SHRINK_DESTROY, SHRINK_BUSY_STOP, SHRINK_BUSY_SKIP}; + +void __jbd2_journal_clean_checkpoint_list(journal_t *journal, enum shrink_type type); unsigned long jbd2_journal_shrink_checkpoint_list(journal_t *journal, unsigned long *nr_to_scan); int __jbd2_journal_remove_checkpoint(struct journal_head *); int jbd2_journal_try_remove_checkpoint(struct journal_head *jh); From 26770a717cac57041d9414725e3e01dd19b08dd2 Mon Sep 17 00:00:00 2001 From: Ye Bin Date: Sun, 7 Apr 2024 14:53:55 +0800 Subject: [PATCH 1499/1702] jbd2: add prefix 'jbd2' for 'shrink_type' As 'shrink_type' is exported. The module prefix 'jbd2' is added to distinguish from memory reclamation. Signed-off-by: Ye Bin Reviewed-by: Jan Kara Reviewed-by: Zhang Yi Link: https://lore.kernel.org/r/20240407065355.1528580-3-yebin10@huawei.com Signed-off-by: Theodore Ts'o --- fs/jbd2/checkpoint.c | 22 +++++++++++----------- fs/jbd2/commit.c | 2 +- include/linux/jbd2.h | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c index 80c0ab98bc639..951f78634adfa 100644 --- a/fs/jbd2/checkpoint.c +++ b/fs/jbd2/checkpoint.c @@ -348,7 +348,7 @@ int jbd2_cleanup_journal_tail(journal_t *journal) * Called with j_list_lock held. */ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, - enum shrink_type type, + enum jbd2_shrink_type type, bool *released) { struct journal_head *last_jh; @@ -365,12 +365,12 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, jh = next_jh; next_jh = jh->b_cpnext; - if (type == SHRINK_DESTROY) { + if (type == JBD2_SHRINK_DESTROY) { ret = __jbd2_journal_remove_checkpoint(jh); } else { ret = jbd2_journal_try_remove_checkpoint(jh); if (ret < 0) { - if (type == SHRINK_BUSY_SKIP) + if (type == JBD2_SHRINK_BUSY_SKIP) continue; break; } @@ -437,7 +437,7 @@ unsigned long jbd2_journal_shrink_checkpoint_list(journal_t *journal, tid = transaction->t_tid; freed = journal_shrink_one_cp_list(transaction->t_checkpoint_list, - SHRINK_BUSY_SKIP, &released); + JBD2_SHRINK_BUSY_SKIP, &released); nr_freed += freed; (*nr_to_scan) -= min(*nr_to_scan, freed); if (*nr_to_scan == 0) @@ -470,20 +470,20 @@ unsigned long jbd2_journal_shrink_checkpoint_list(journal_t *journal, * journal_clean_checkpoint_list * * Find all the written-back checkpoint buffers in the journal and release them. - * If 'type' is SHRINK_DESTROY, release all buffers unconditionally. If 'type' - * is SHRINK_BUSY_STOP, will stop release buffers if encounters a busy buffer. - * To avoid wasting CPU cycles scanning the buffer list in some cases, don't - * pass SHRINK_BUSY_SKIP 'type' for this function. + * If 'type' is JBD2_SHRINK_DESTROY, release all buffers unconditionally. If + * 'type' is JBD2_SHRINK_BUSY_STOP, will stop release buffers if encounters a + * busy buffer. To avoid wasting CPU cycles scanning the buffer list in some + * cases, don't pass JBD2_SHRINK_BUSY_SKIP 'type' for this function. * * Called with j_list_lock held. */ void __jbd2_journal_clean_checkpoint_list(journal_t *journal, - enum shrink_type type) + enum jbd2_shrink_type type) { transaction_t *transaction, *last_transaction, *next_transaction; bool released; - WARN_ON_ONCE(type == SHRINK_BUSY_SKIP); + WARN_ON_ONCE(type == JBD2_SHRINK_BUSY_SKIP); transaction = journal->j_checkpoint_transactions; if (!transaction) @@ -529,7 +529,7 @@ void jbd2_journal_destroy_checkpoint(journal_t *journal) spin_unlock(&journal->j_list_lock); break; } - __jbd2_journal_clean_checkpoint_list(journal, SHRINK_DESTROY); + __jbd2_journal_clean_checkpoint_list(journal, JBD2_SHRINK_DESTROY); spin_unlock(&journal->j_list_lock); cond_resched(); } diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c index b341c396311b4..75ea4e9a5cabd 100644 --- a/fs/jbd2/commit.c +++ b/fs/jbd2/commit.c @@ -501,7 +501,7 @@ void jbd2_journal_commit_transaction(journal_t *journal) * frees some memory */ spin_lock(&journal->j_list_lock); - __jbd2_journal_clean_checkpoint_list(journal, SHRINK_BUSY_STOP); + __jbd2_journal_clean_checkpoint_list(journal, JBD2_SHRINK_BUSY_STOP); spin_unlock(&journal->j_list_lock); jbd2_debug(3, "JBD2: commit phase 1\n"); diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h index 58a961999d704..7479f64c09395 100644 --- a/include/linux/jbd2.h +++ b/include/linux/jbd2.h @@ -1434,9 +1434,9 @@ void jbd2_update_log_tail(journal_t *journal, tid_t tid, unsigned long block); extern void jbd2_journal_commit_transaction(journal_t *); /* Checkpoint list management */ -enum shrink_type {SHRINK_DESTROY, SHRINK_BUSY_STOP, SHRINK_BUSY_SKIP}; +enum jbd2_shrink_type {JBD2_SHRINK_DESTROY, JBD2_SHRINK_BUSY_STOP, JBD2_SHRINK_BUSY_SKIP}; -void __jbd2_journal_clean_checkpoint_list(journal_t *journal, enum shrink_type type); +void __jbd2_journal_clean_checkpoint_list(journal_t *journal, enum jbd2_shrink_type type); unsigned long jbd2_journal_shrink_checkpoint_list(journal_t *journal, unsigned long *nr_to_scan); int __jbd2_journal_remove_checkpoint(struct journal_head *); int jbd2_journal_try_remove_checkpoint(struct journal_head *jh); From 1dd719a959794467c2a75b3813df86cc6f55f5e3 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 8 May 2024 18:21:11 +0300 Subject: [PATCH 1500/1702] isofs: Use *-y instead of *-objs in Makefile *-objs suffix is reserved rather for (user-space) host programs while usually *-y suffix is used for kernel drivers (although *-objs works for that purpose for now). Let's correct the old usages of *-objs in Makefiles. Signed-off-by: Andy Shevchenko Signed-off-by: Jan Kara Message-Id: <20240508152129.1445372-1-andriy.shevchenko@linux.intel.com> --- fs/isofs/Makefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fs/isofs/Makefile b/fs/isofs/Makefile index 6498fd2b0f607..b25bc542a22bf 100644 --- a/fs/isofs/Makefile +++ b/fs/isofs/Makefile @@ -5,7 +5,6 @@ obj-$(CONFIG_ISO9660_FS) += isofs.o -isofs-objs-y := namei.o inode.o dir.o util.o rock.o export.o -isofs-objs-$(CONFIG_JOLIET) += joliet.o -isofs-objs-$(CONFIG_ZISOFS) += compress.o -isofs-objs := $(isofs-objs-y) +isofs-y := namei.o inode.o dir.o util.o rock.o export.o +isofs-$(CONFIG_JOLIET) += joliet.o +isofs-$(CONFIG_ZISOFS) += compress.o From a6016aac5252da9d22a4dc0b98121b0acdf6d2f5 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Thu, 9 May 2024 16:46:16 +0200 Subject: [PATCH 1501/1702] dma: fix DMA sync for drivers not calling dma_set_mask*() There are several reports that the DMA sync shortcut broke non-coherent devices. dev->dma_need_sync is false after the &device allocation and if a driver didn't call dma_set_mask*(), it will still be false even if the device is not DMA-coherent and thus needs synchronizing. Due to historical reasons, there's still a lot of drivers not calling it. Invert the boolean, so that the sync will be performed by default and the shortcut will be enabled only when calling dma_set_mask*(). Reported-by: Steven Price Closes: https://lore.kernel.org/lkml/010686f5-3049-46a1-8230-7752a1b433ff@arm.com Reported-by: Marek Szyprowski Closes: https://lore.kernel.org/lkml/46160534-5003-4809-a408-6b3a3f4921e9@samsung.com Fixes: f406c8e4b770. ("dma: avoid redundant calls for sync operations") Signed-off-by: Alexander Lobakin Signed-off-by: Christoph Hellwig Tested-by: Steven Price Tested-by: Marek Szyprowski --- include/linux/device.h | 4 ++-- include/linux/dma-map-ops.h | 4 ++-- include/linux/dma-mapping.h | 2 +- kernel/dma/mapping.c | 10 +++++----- kernel/dma/swiotlb.c | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/include/linux/device.h b/include/linux/device.h index ed95b829f05bb..d4b50accff26c 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -691,7 +691,7 @@ struct device_physical_location { * and optionall (if the coherent mask is large enough) also * for dma allocations. This flag is managed by the dma ops * instance from ->dma_supported. - * @dma_need_sync: The device needs performing DMA sync operations. + * @dma_skip_sync: DMA sync operations can be skipped for coherent buffers. * * At the lowest level, every device in a Linux system is represented by an * instance of struct device. The device structure contains the information @@ -805,7 +805,7 @@ struct device { bool dma_ops_bypass : 1; #endif #ifdef CONFIG_DMA_NEED_SYNC - bool dma_need_sync:1; + bool dma_skip_sync:1; #endif }; diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 4893cb89cb524..5217b922d29f7 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -280,8 +280,8 @@ static inline void dma_reset_need_sync(struct device *dev) { #ifdef CONFIG_DMA_NEED_SYNC /* Reset it only once so that the function can be called on hotpath */ - if (unlikely(!dev->dma_need_sync)) - dev->dma_need_sync = true; + if (unlikely(dev->dma_skip_sync)) + dev->dma_skip_sync = false; #endif } diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index eb4e15893b6c4..f693aafe221f2 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -295,7 +295,7 @@ bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr); static inline bool dma_dev_need_sync(const struct device *dev) { /* Always call DMA sync operations when debugging is enabled */ - return dev->dma_need_sync || IS_ENABLED(CONFIG_DMA_API_DEBUG); + return !dev->dma_skip_sync || IS_ENABLED(CONFIG_DMA_API_DEBUG); } static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 3524bc92c37fd..3f77c3f8d16d6 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -392,7 +392,7 @@ bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr) if (dma_map_direct(dev, ops)) /* - * dma_need_sync could've been reset on first SWIOTLB buffer + * dma_skip_sync could've been reset on first SWIOTLB buffer * mapping, but @dma_addr is not necessary an SWIOTLB buffer. * In this case, fall back to more granular check. */ @@ -407,20 +407,20 @@ static void dma_setup_need_sync(struct device *dev) if (dma_map_direct(dev, ops) || (ops->flags & DMA_F_CAN_SKIP_SYNC)) /* - * dma_need_sync will be reset to %true on first SWIOTLB buffer + * dma_skip_sync will be reset to %false on first SWIOTLB buffer * mapping, if any. During the device initialization, it's * enough to check only for the DMA coherence. */ - dev->dma_need_sync = !dev_is_dma_coherent(dev); + dev->dma_skip_sync = dev_is_dma_coherent(dev); else if (!ops->sync_single_for_device && !ops->sync_single_for_cpu && !ops->sync_sg_for_device && !ops->sync_sg_for_cpu) /* * Synchronization is not possible when none of DMA sync ops * is set. */ - dev->dma_need_sync = false; + dev->dma_skip_sync = true; else - dev->dma_need_sync = true; + dev->dma_skip_sync = false; } #else /* !CONFIG_DMA_NEED_SYNC */ static inline void dma_setup_need_sync(struct device *dev) { } diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index ae3e593eaadb9..068134697cf1b 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -1409,7 +1409,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr, } /* - * If dma_need_sync wasn't set, reset it on first SWIOTLB buffer + * If dma_skip_sync was set, reset it on first SWIOTLB buffer * mapping to always sync SWIOTLB buffers. */ dma_reset_need_sync(dev); From 9a0ebe5011f49e932bb0a2cea2034fd65e6e567e Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 27 Apr 2024 23:55:01 +0900 Subject: [PATCH 1502/1702] kbuild: use $(obj)/ instead of $(src)/ for common pattern rules Kbuild conventionally uses $(obj)/ for generated files, and $(src)/ for checked-in source files. It is merely a convention without any functional difference. In fact, $(obj) and $(src) are exactly the same, as defined in scripts/Makefile.build: src := $(obj) Before changing the semantics of $(src) in the next commit, this commit replaces $(obj)/ with $(src)/ in pattern rules where the prerequisite might be a generated file. C, assembly, Rust, and DTS files are sometimes generated by tools, so they could be either generated files or real sources. The $(obj)/ prefix works for both cases with the help of VPATH. As mentioned above, $(obj) and $(src) are the same at this point, hence this commit has no functional change. I did not modify scripts/Makefile.userprogs because there is no use case where userspace C files are generated. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Makefile.build | 26 +++++++++++++------------- scripts/Makefile.host | 4 ++-- scripts/Makefile.lib | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/Makefile.build b/scripts/Makefile.build index baf86c0880b6d..41138346afb01 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -113,13 +113,13 @@ endif quiet_cmd_cc_s_c = CC $(quiet_modtag) $@ cmd_cc_s_c = $(CC) $(filter-out $(DEBUG_CFLAGS) $(CC_FLAGS_LTO), $(c_flags)) -fverbose-asm -S -o $@ $< -$(obj)/%.s: $(src)/%.c FORCE +$(obj)/%.s: $(obj)/%.c FORCE $(call if_changed_dep,cc_s_c) quiet_cmd_cpp_i_c = CPP $(quiet_modtag) $@ cmd_cpp_i_c = $(CPP) $(c_flags) -o $@ $< -$(obj)/%.i: $(src)/%.c FORCE +$(obj)/%.i: $(obj)/%.c FORCE $(call if_changed_dep,cpp_i_c) genksyms = scripts/genksyms/genksyms \ @@ -133,7 +133,7 @@ cmd_gensymtypes_c = $(CPP) -D__GENKSYMS__ $(c_flags) $< | $(genksyms) quiet_cmd_cc_symtypes_c = SYM $(quiet_modtag) $@ cmd_cc_symtypes_c = $(call cmd_gensymtypes_c,true,$@) >/dev/null -$(obj)/%.symtypes : $(src)/%.c FORCE +$(obj)/%.symtypes : $(obj)/%.c FORCE $(call cmd,cc_symtypes_c) # LLVM assembly @@ -141,7 +141,7 @@ $(obj)/%.symtypes : $(src)/%.c FORCE quiet_cmd_cc_ll_c = CC $(quiet_modtag) $@ cmd_cc_ll_c = $(CC) $(c_flags) -emit-llvm -S -fno-discard-value-names -o $@ $< -$(obj)/%.ll: $(src)/%.c FORCE +$(obj)/%.ll: $(obj)/%.c FORCE $(call if_changed_dep,cc_ll_c) # C (.c) files @@ -240,7 +240,7 @@ define rule_as_o_S endef # Built-in and composite module parts -$(obj)/%.o: $(src)/%.c $(recordmcount_source) FORCE +$(obj)/%.o: $(obj)/%.c $(recordmcount_source) FORCE $(call if_changed_rule,cc_o_c) $(call cmd,force_checksrc) @@ -257,7 +257,7 @@ quiet_cmd_cc_lst_c = MKLST $@ $(CONFIG_SHELL) $(srctree)/scripts/makelst $*.o \ System.map $(OBJDUMP) > $@ -$(obj)/%.lst: $(src)/%.c FORCE +$(obj)/%.lst: $(obj)/%.c FORCE $(call if_changed_dep,cc_lst_c) # Compile Rust sources (.rs) @@ -290,7 +290,7 @@ rust_common_cmd = \ quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_o_rs = $(rust_common_cmd) --emit=obj=$@ $< -$(obj)/%.o: $(src)/%.rs FORCE +$(obj)/%.o: $(obj)/%.rs FORCE +$(call if_changed_dep,rustc_o_rs) quiet_cmd_rustc_rsi_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ @@ -298,19 +298,19 @@ quiet_cmd_rustc_rsi_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ $(rust_common_cmd) -Zunpretty=expanded $< >$@; \ command -v $(RUSTFMT) >/dev/null && $(RUSTFMT) $@ -$(obj)/%.rsi: $(src)/%.rs FORCE +$(obj)/%.rsi: $(obj)/%.rs FORCE +$(call if_changed_dep,rustc_rsi_rs) quiet_cmd_rustc_s_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_s_rs = $(rust_common_cmd) --emit=asm=$@ $< -$(obj)/%.s: $(src)/%.rs FORCE +$(obj)/%.s: $(obj)/%.rs FORCE +$(call if_changed_dep,rustc_s_rs) quiet_cmd_rustc_ll_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_ll_rs = $(rust_common_cmd) --emit=llvm-ir=$@ $< -$(obj)/%.ll: $(src)/%.rs FORCE +$(obj)/%.ll: $(obj)/%.rs FORCE +$(call if_changed_dep,rustc_ll_rs) # Compile assembler sources (.S) @@ -336,14 +336,14 @@ cmd_gensymtypes_S = \ quiet_cmd_cc_symtypes_S = SYM $(quiet_modtag) $@ cmd_cc_symtypes_S = $(call cmd_gensymtypes_S,true,$@) >/dev/null -$(obj)/%.symtypes : $(src)/%.S FORCE +$(obj)/%.symtypes : $(obj)/%.S FORCE $(call cmd,cc_symtypes_S) quiet_cmd_cpp_s_S = CPP $(quiet_modtag) $@ cmd_cpp_s_S = $(CPP) $(a_flags) -o $@ $< -$(obj)/%.s: $(src)/%.S FORCE +$(obj)/%.s: $(obj)/%.S FORCE $(call if_changed_dep,cpp_s_S) quiet_cmd_as_o_S = AS $(quiet_modtag) $@ @@ -358,7 +358,7 @@ cmd_gen_symversions_S = $(call gen_symversions,S) endif -$(obj)/%.o: $(src)/%.S FORCE +$(obj)/%.o: $(obj)/%.S FORCE $(call if_changed_rule,as_o_S) targets += $(filter-out $(subdir-builtin), $(real-obj-y)) diff --git a/scripts/Makefile.host b/scripts/Makefile.host index 3c17e6ba421ce..d35f55e0d141e 100644 --- a/scripts/Makefile.host +++ b/scripts/Makefile.host @@ -112,7 +112,7 @@ endif quiet_cmd_host-csingle = HOSTCC $@ cmd_host-csingle = $(HOSTCC) $(hostc_flags) $(KBUILD_HOSTLDFLAGS) -o $@ $< \ $(KBUILD_HOSTLDLIBS) $(HOSTLDLIBS_$(target-stem)) -$(host-csingle): $(obj)/%: $(src)/%.c FORCE +$(host-csingle): $(obj)/%: $(obj)/%.c FORCE $(call if_changed_dep,host-csingle) # Link an executable based on list of .o files, all plain c @@ -129,7 +129,7 @@ $(call multi_depend, $(host-cmulti), , -objs) # host-cobjs -> .o quiet_cmd_host-cobjs = HOSTCC $@ cmd_host-cobjs = $(HOSTCC) $(hostc_flags) -c -o $@ $< -$(host-cobjs): $(obj)/%.o: $(src)/%.c FORCE +$(host-cobjs): $(obj)/%.o: $(obj)/%.c FORCE $(call if_changed_dep,host-cobjs) # Link an executable based on list of .o files, a mixture of .c and .cc diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index e67f066c0cea7..42d3c772c8e06 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -423,7 +423,7 @@ quiet_cmd_dtb = $(quiet_cmd_dtc) cmd_dtb = $(cmd_dtc) endif -$(obj)/%.dtb: $(src)/%.dts $(DTC) $(DT_TMP_SCHEMA) FORCE +$(obj)/%.dtb: $(obj)/%.dts $(DTC) $(DT_TMP_SCHEMA) FORCE $(call if_changed_dep,dtb) $(obj)/%.dtbo: $(src)/%.dtso $(DTC) FORCE From b1992c3772e69a6fd0e3fc81cd4d2820c8b6eca0 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 27 Apr 2024 23:55:02 +0900 Subject: [PATCH 1503/1702] kbuild: use $(src) instead of $(srctree)/$(src) for source directory Kbuild conventionally uses $(obj)/ for generated files, and $(src)/ for checked-in source files. It is merely a convention without any functional difference. In fact, $(obj) and $(src) are exactly the same, as defined in scripts/Makefile.build: src := $(obj) When the kernel is built in a separate output directory, $(src) does not accurately reflect the source directory location. While Kbuild resolves this discrepancy by specifying VPATH=$(srctree) to search for source files, it does not cover all cases. For example, when adding a header search path for local headers, -I$(srctree)/$(src) is typically passed to the compiler. This introduces inconsistency between upstream and downstream Makefiles because $(src) is used instead of $(srctree)/$(src) for the latter. To address this inconsistency, this commit changes the semantics of $(src) so that it always points to the directory in the source tree. Going forward, the variables used in Makefiles will have the following meanings: $(obj) - directory in the object tree $(src) - directory in the source tree (changed by this commit) $(objtree) - the top of the kernel object tree $(srctree) - the top of the kernel source tree Consequently, $(srctree)/$(src) in upstream Makefiles need to be replaced with $(src). Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- Documentation/Makefile | 8 ++++---- Documentation/devicetree/bindings/Makefile | 6 +++--- Documentation/kbuild/makefiles.rst | 12 ++++++------ Makefile | 7 +++++++ arch/arc/boot/dts/Makefile | 3 +-- arch/arm/Kbuild | 2 +- arch/arm/boot/Makefile | 3 +-- arch/arm/mach-s3c/Makefile | 2 +- arch/arm/plat-orion/Makefile | 2 +- arch/arm/tools/Makefile | 2 +- arch/arm64/kernel/vdso/Makefile | 2 +- arch/arm64/kvm/Makefile | 4 ++-- arch/arm64/kvm/hyp/Makefile | 2 +- arch/csky/boot/dts/Makefile | 4 +--- arch/csky/kernel/vdso/Makefile | 2 +- arch/loongarch/kvm/Makefile | 2 +- arch/loongarch/vdso/Makefile | 2 +- arch/mips/kernel/syscalls/Makefile | 2 +- arch/mips/vdso/Makefile | 4 ++-- arch/nios2/boot/dts/Makefile | 3 +-- arch/parisc/kernel/vdso32/Makefile | 2 +- arch/parisc/kernel/vdso64/Makefile | 2 +- arch/powerpc/boot/Makefile | 6 +++--- arch/powerpc/boot/dts/Makefile | 3 +-- arch/powerpc/boot/dts/fsl/Makefile | 3 +-- arch/powerpc/kernel/vdso/Makefile | 4 ++-- arch/riscv/kernel/compat_vdso/Makefile | 2 +- arch/riscv/kernel/vdso/Makefile | 2 +- arch/riscv/kvm/Makefile | 2 +- arch/s390/kernel/syscalls/Makefile | 4 ++-- arch/s390/kernel/vdso32/Makefile | 2 +- arch/s390/kernel/vdso64/Makefile | 2 +- arch/sparc/vdso/Makefile | 2 +- arch/um/kernel/Makefile | 2 +- arch/x86/boot/Makefile | 2 +- arch/x86/entry/vdso/Makefile | 2 +- arch/x86/kernel/Makefile | 2 +- arch/x86/kernel/cpu/Makefile | 2 +- arch/x86/mm/Makefile | 2 +- arch/x86/um/vdso/Makefile | 2 +- arch/xtensa/boot/dts/Makefile | 3 +-- certs/Makefile | 4 ++-- drivers/Makefile | 5 ----- drivers/crypto/intel/qat/qat_420xx/Makefile | 2 +- drivers/crypto/intel/qat/qat_4xxx/Makefile | 2 +- drivers/crypto/intel/qat/qat_c3xxx/Makefile | 2 +- drivers/crypto/intel/qat/qat_c3xxxvf/Makefile | 2 +- drivers/crypto/intel/qat/qat_c62x/Makefile | 2 +- drivers/crypto/intel/qat/qat_c62xvf/Makefile | 2 +- drivers/crypto/intel/qat/qat_dh895xcc/Makefile | 2 +- drivers/crypto/intel/qat/qat_dh895xccvf/Makefile | 2 +- drivers/gpu/drm/amd/amdgpu/Makefile | 2 +- drivers/gpu/drm/arm/display/komeda/Makefile | 4 ++-- drivers/gpu/drm/i915/Makefile | 4 ++-- drivers/gpu/drm/imagination/Makefile | 2 +- drivers/gpu/drm/msm/Makefile | 8 ++++---- drivers/gpu/drm/nouveau/Kbuild | 10 ++++------ drivers/gpu/drm/xe/Makefile | 10 +++++----- drivers/hid/amd-sfh-hid/Makefile | 2 +- drivers/hid/intel-ish-hid/Makefile | 2 +- drivers/md/dm-vdo/Makefile | 2 +- drivers/net/ethernet/aquantia/atlantic/Makefile | 2 +- drivers/net/ethernet/chelsio/libcxgb/Makefile | 2 +- drivers/net/ethernet/fungible/funeth/Makefile | 2 +- drivers/net/ethernet/hisilicon/hns3/Makefile | 2 +- .../wireless/broadcom/brcm80211/brcmfmac/Makefile | 4 ++-- .../broadcom/brcm80211/brcmfmac/bca/Makefile | 6 +++--- .../broadcom/brcm80211/brcmfmac/cyw/Makefile | 6 +++--- .../broadcom/brcm80211/brcmfmac/wcc/Makefile | 6 +++--- .../wireless/broadcom/brcm80211/brcmsmac/Makefile | 6 +++--- .../wireless/broadcom/brcm80211/brcmutil/Makefile | 2 +- drivers/net/wireless/intel/iwlwifi/dvm/Makefile | 2 +- drivers/net/wireless/intel/iwlwifi/mei/Makefile | 2 +- drivers/net/wireless/intel/iwlwifi/mvm/Makefile | 2 +- drivers/net/wireless/intel/iwlwifi/tests/Makefile | 2 +- .../net/wireless/realtek/rtl818x/rtl8180/Makefile | 2 +- .../net/wireless/realtek/rtl818x/rtl8187/Makefile | 2 +- drivers/scsi/aic7xxx/Makefile | 12 ++++++------ drivers/staging/rtl8723bs/Makefile | 2 +- fs/iomap/Makefile | 2 +- fs/unicode/Makefile | 14 +++++++------- fs/xfs/Makefile | 4 ++-- init/Makefile | 2 +- lib/Makefile | 6 +++--- lib/raid6/Makefile | 2 +- net/wireless/Makefile | 2 +- rust/Makefile | 6 +++--- samples/bpf/Makefile | 2 +- samples/hid/Makefile | 2 +- scripts/Kbuild.include | 3 +-- scripts/Makefile.asm-generic | 6 +++--- scripts/Makefile.build | 2 +- scripts/Makefile.clean | 2 +- scripts/Makefile.lib | 10 +++++----- scripts/Makefile.modpost | 2 +- scripts/dtc/Makefile | 6 +++--- scripts/gdb/linux/Makefile | 2 +- scripts/genksyms/Makefile | 4 ++-- scripts/kconfig/Makefile | 8 ++++---- security/tomoyo/Makefile | 2 +- usr/Makefile | 2 +- usr/include/Makefile | 2 +- 102 files changed, 173 insertions(+), 182 deletions(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index b68f8c816897b..66d15fff20893 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -76,22 +76,22 @@ loop_cmd = $(echo-cmd) $(cmd_$(1)) || exit; # * dest folder relative to $(BUILDDIR) and # * cache folder relative to $(BUILDDIR)/.doctrees # $4 dest subfolder e.g. "man" for man pages at userspace-api/media/man -# $5 reST source folder relative to $(srctree)/$(src), +# $5 reST source folder relative to $(src), # e.g. "userspace-api/media" for the linux-tv book-set at ./Documentation/userspace-api/media quiet_cmd_sphinx = SPHINX $@ --> file://$(abspath $(BUILDDIR)/$3/$4) cmd_sphinx = $(MAKE) BUILDDIR=$(abspath $(BUILDDIR)) $(build)=Documentation/userspace-api/media $2 && \ PYTHONDONTWRITEBYTECODE=1 \ - BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(srctree)/$(src)/$5/$(SPHINX_CONF)) \ + BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(src)/$5/$(SPHINX_CONF)) \ $(PYTHON3) $(srctree)/scripts/jobserver-exec \ $(CONFIG_SHELL) $(srctree)/Documentation/sphinx/parallel-wrapper.sh \ $(SPHINXBUILD) \ -b $2 \ - -c $(abspath $(srctree)/$(src)) \ + -c $(abspath $(src)) \ -d $(abspath $(BUILDDIR)/.doctrees/$3) \ -D version=$(KERNELVERSION) -D release=$(KERNELRELEASE) \ $(ALLSPHINXOPTS) \ - $(abspath $(srctree)/$(src)/$5) \ + $(abspath $(src)/$5) \ $(abspath $(BUILDDIR)/$3/$4) && \ if [ "x$(DOCS_CSS)" != "x" ]; then \ cp $(if $(patsubst /%,,$(DOCS_CSS)),$(abspath $(srctree)/$(DOCS_CSS)),$(DOCS_CSS)) $(BUILDDIR)/$3/_static/; \ diff --git a/Documentation/devicetree/bindings/Makefile b/Documentation/devicetree/bindings/Makefile index 8cdda477987fd..bf7d64632e20a 100644 --- a/Documentation/devicetree/bindings/Makefile +++ b/Documentation/devicetree/bindings/Makefile @@ -25,7 +25,7 @@ quiet_cmd_extract_ex = DTEX $@ $(obj)/%.example.dts: $(src)/%.yaml check_dtschema_version FORCE $(call if_changed,extract_ex) -find_all_cmd = find $(srctree)/$(src) \( -name '*.yaml' ! \ +find_all_cmd = find $(src) \( -name '*.yaml' ! \ -name 'processed-schema*' \) find_cmd = $(find_all_cmd) | \ @@ -37,12 +37,12 @@ CHK_DT_EXAMPLES := $(patsubst $(srctree)/%.yaml,%.example.dtb, $(shell $(find_cm quiet_cmd_yamllint = LINT $(src) cmd_yamllint = ($(find_cmd) | \ xargs -n200 -P$$(nproc) \ - $(DT_SCHEMA_LINT) -f parsable -c $(srctree)/$(src)/.yamllint >&2) \ + $(DT_SCHEMA_LINT) -f parsable -c $(src)/.yamllint >&2) \ && touch $@ || true quiet_cmd_chk_bindings = CHKDT $(src) cmd_chk_bindings = ($(find_cmd) | \ - xargs -n200 -P$$(nproc) $(DT_DOC_CHECKER) -u $(srctree)/$(src)) \ + xargs -n200 -P$$(nproc) $(DT_DOC_CHECKER) -u $(src)) \ && touch $@ || true quiet_cmd_mk_schema = SCHEMA $@ diff --git a/Documentation/kbuild/makefiles.rst b/Documentation/kbuild/makefiles.rst index ad118b7a18063..991ce6081e357 100644 --- a/Documentation/kbuild/makefiles.rst +++ b/Documentation/kbuild/makefiles.rst @@ -346,7 +346,7 @@ ccflags-y, asflags-y and ldflags-y Example:: #arch/cris/boot/compressed/Makefile - ldflags-y += -T $(srctree)/$(src)/decompress_$(arch-y).lds + ldflags-y += -T $(src)/decompress_$(arch-y).lds subdir-ccflags-y, subdir-asflags-y The two flags listed above are similar to ccflags-y and asflags-y. @@ -426,14 +426,14 @@ path to prerequisite files and target files. Two variables are used when defining custom rules: $(src) - $(src) is a relative path which points to the directory - where the Makefile is located. Always use $(src) when + $(src) is the directory where the Makefile is located. Always use $(src) when referring to files located in the src tree. $(obj) - $(obj) is a relative path which points to the directory - where the target is saved. Always use $(obj) when - referring to generated files. + $(obj) is the directory where the target is saved. Always use $(obj) when + referring to generated files. Use $(obj) for pattern rules that need to work + for both generated files and real sources (VPATH will help to find the + prerequisites not only in the object tree but also in the source tree). Example:: diff --git a/Makefile b/Makefile index 12e1a792b8de4..c97c8d96061c5 100644 --- a/Makefile +++ b/Makefile @@ -263,7 +263,14 @@ srctree := $(abs_srctree) endif objtree := . + +VPATH := + +ifeq ($(KBUILD_EXTMOD),) +ifdef building_out_of_srctree VPATH := $(srctree) +endif +endif export building_out_of_srctree srctree objtree VPATH diff --git a/arch/arc/boot/dts/Makefile b/arch/arc/boot/dts/Makefile index 4237aa5de3a37..48704dfdf75cb 100644 --- a/arch/arc/boot/dts/Makefile +++ b/arch/arc/boot/dts/Makefile @@ -10,8 +10,7 @@ obj-y += $(builtindtb-y).dtb.o dtb-y := $(builtindtb-y).dtb # for CONFIG_OF_ALL_DTBS test -dtstree := $(srctree)/$(src) -dtb- := $(patsubst $(dtstree)/%.dts,%.dtb, $(wildcard $(dtstree)/*.dts)) +dtb- := $(patsubst $(src)/%.dts,%.dtb, $(wildcard $(src)/*.dts)) # board-specific dtc flags DTC_FLAGS_hsdk += --pad 20 diff --git a/arch/arm/Kbuild b/arch/arm/Kbuild index b506622e7e23a..69de6b6243c70 100644 --- a/arch/arm/Kbuild +++ b/arch/arm/Kbuild @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_FPE_NWFPE) += nwfpe/ # Put arch/arm/fastfpe/ to use this. -obj-$(CONFIG_FPE_FASTFPE) += $(patsubst $(srctree)/$(src)/%,%,$(wildcard $(srctree)/$(src)/fastfpe/)) +obj-$(CONFIG_FPE_FASTFPE) += $(patsubst $(src)/%,%,$(wildcard $(src)/fastfpe/)) obj-$(CONFIG_VFP) += vfp/ obj-$(CONFIG_XEN) += xen/ obj-$(CONFIG_VDSO) += vdso/ diff --git a/arch/arm/boot/Makefile b/arch/arm/boot/Makefile index abd6a2889fd0e..ba9b9a802469f 100644 --- a/arch/arm/boot/Makefile +++ b/arch/arm/boot/Makefile @@ -25,8 +25,7 @@ targets := Image zImage xipImage bootpImage uImage ifeq ($(CONFIG_XIP_KERNEL),y) -cmd_deflate_xip_data = $(CONFIG_SHELL) -c \ - '$(srctree)/$(src)/deflate_xip_data.sh $< $@' +cmd_deflate_xip_data = $(CONFIG_SHELL) -c '$(src)/deflate_xip_data.sh $< $@' ifeq ($(CONFIG_XIP_DEFLATED_DATA),y) quiet_cmd_mkxip = XIPZ $@ diff --git a/arch/arm/mach-s3c/Makefile b/arch/arm/mach-s3c/Makefile index 713827bef8311..988c496727158 100644 --- a/arch/arm/mach-s3c/Makefile +++ b/arch/arm/mach-s3c/Makefile @@ -2,7 +2,7 @@ # # Copyright 2009 Simtec Electronics -include $(srctree)/$(src)/Makefile.s3c64xx +include $(src)/Makefile.s3c64xx # Objects we always build independent of SoC choice diff --git a/arch/arm/plat-orion/Makefile b/arch/arm/plat-orion/Makefile index 830b0be038c6b..e8c7580df8caf 100644 --- a/arch/arm/plat-orion/Makefile +++ b/arch/arm/plat-orion/Makefile @@ -2,7 +2,7 @@ # # Makefile for the linux kernel. # -ccflags-y := -I$(srctree)/$(src)/include +ccflags-y := -I$(src)/include orion-gpio-$(CONFIG_GPIOLIB) += gpio.o obj-$(CONFIG_PLAT_ORION_LEGACY) += irq.o pcie.o time.o common.o mpp.o diff --git a/arch/arm/tools/Makefile b/arch/arm/tools/Makefile index 81f13bdf32f2b..28b6da8ac5f64 100644 --- a/arch/arm/tools/Makefile +++ b/arch/arm/tools/Makefile @@ -9,7 +9,7 @@ gen := arch/$(ARCH)/include/generated kapi := $(gen)/asm uapi := $(gen)/uapi/asm syshdr := $(srctree)/scripts/syscallhdr.sh -sysnr := $(srctree)/$(src)/syscallnr.sh +sysnr := $(src)/syscallnr.sh systbl := $(srctree)/scripts/syscalltbl.sh syscall := $(src)/syscall.tbl diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile index 8818287f10955..53e86d3bc159a 100644 --- a/arch/arm64/kernel/vdso/Makefile +++ b/arch/arm64/kernel/vdso/Makefile @@ -68,7 +68,7 @@ $(obj)/%.so: $(obj)/%.so.dbg FORCE $(call if_changed,objcopy) # Generate VDSO offsets using helper script -gen-vdsosym := $(srctree)/$(src)/gen_vdso_offsets.sh +gen-vdsosym := $(src)/gen_vdso_offsets.sh quiet_cmd_vdsosym = VDSOSYM $@ cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index c0c050e53157d..ab474d444cef8 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -3,7 +3,7 @@ # Makefile for Kernel-based Virtual Machine module # -ccflags-y += -I $(srctree)/$(src) +ccflags-y += -I $(src) include $(srctree)/virt/kvm/Makefile.kvm @@ -30,7 +30,7 @@ define rule_gen_hyp_constants $(call filechk,offsets,__HYP_CONSTANTS_H__) endef -CFLAGS_hyp-constants.o = -I $(srctree)/$(src)/hyp/include +CFLAGS_hyp-constants.o = -I $(src)/hyp/include $(obj)/hyp-constants.s: $(src)/hyp/hyp-constants.c FORCE $(call if_changed_dep,cc_s_c) diff --git a/arch/arm64/kvm/hyp/Makefile b/arch/arm64/kvm/hyp/Makefile index a38dea6186c90..d61e44642f980 100644 --- a/arch/arm64/kvm/hyp/Makefile +++ b/arch/arm64/kvm/hyp/Makefile @@ -3,7 +3,7 @@ # Makefile for Kernel-based Virtual Machine module, HYP part # -incdir := $(srctree)/$(src)/include +incdir := $(src)/include subdir-asflags-y := -I$(incdir) subdir-ccflags-y := -I$(incdir) diff --git a/arch/csky/boot/dts/Makefile b/arch/csky/boot/dts/Makefile index 5f1f55e911adf..aabffc1912e8c 100644 --- a/arch/csky/boot/dts/Makefile +++ b/arch/csky/boot/dts/Makefile @@ -1,4 +1,2 @@ # SPDX-License-Identifier: GPL-2.0-only -dtstree := $(srctree)/$(src) - -dtb-y := $(patsubst $(dtstree)/%.dts,%.dtb, $(wildcard $(dtstree)/*.dts)) +dtb-y := $(patsubst $(src)/%.dts,%.dtb, $(wildcard $(src)/*.dts)) diff --git a/arch/csky/kernel/vdso/Makefile b/arch/csky/kernel/vdso/Makefile index 24a01c22293ec..e79a725f50758 100644 --- a/arch/csky/kernel/vdso/Makefile +++ b/arch/csky/kernel/vdso/Makefile @@ -57,4 +57,4 @@ quiet_cmd_vdsold = VDSOLD $@ # Extracts symbol offsets from the VDSO, converting them into an assembly file # that contains the same symbols at the same offsets. quiet_cmd_so2s = SO2S $@ - cmd_so2s = $(NM) -D $< | $(srctree)/$(src)/so2s.sh > $@ + cmd_so2s = $(NM) -D $< | $(src)/so2s.sh > $@ diff --git a/arch/loongarch/kvm/Makefile b/arch/loongarch/kvm/Makefile index 244467d7792a9..b2f4cbe01ae80 100644 --- a/arch/loongarch/kvm/Makefile +++ b/arch/loongarch/kvm/Makefile @@ -3,7 +3,7 @@ # Makefile for LoongArch KVM support # -ccflags-y += -I $(srctree)/$(src) +ccflags-y += -I $(src) include $(srctree)/virt/kvm/Makefile.kvm diff --git a/arch/loongarch/vdso/Makefile b/arch/loongarch/vdso/Makefile index 75c6726382c34..cdfc4c793e2c3 100644 --- a/arch/loongarch/vdso/Makefile +++ b/arch/loongarch/vdso/Makefile @@ -52,7 +52,7 @@ quiet_cmd_vdsoas_o_S = AS $@ cmd_vdsoas_o_S = $(CC) $(a_flags) -c -o $@ $< # Generate VDSO offsets using helper script -gen-vdsosym := $(srctree)/$(src)/gen_vdso_offsets.sh +gen-vdsosym := $(src)/gen_vdso_offsets.sh quiet_cmd_vdsosym = VDSOSYM $@ cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ diff --git a/arch/mips/kernel/syscalls/Makefile b/arch/mips/kernel/syscalls/Makefile index e6b21de65cca4..56f6f093bb88e 100644 --- a/arch/mips/kernel/syscalls/Makefile +++ b/arch/mips/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm $(shell mkdir -p $(uapi) $(kapi)) syshdr := $(srctree)/scripts/syscallhdr.sh -sysnr := $(srctree)/$(src)/syscallnr.sh +sysnr := $(src)/syscallnr.sh systbl := $(srctree)/scripts/syscalltbl.sh quiet_cmd_syshdr = SYSHDR $@ diff --git a/arch/mips/vdso/Makefile b/arch/mips/vdso/Makefile index eb56581f6d734..40b839e918060 100644 --- a/arch/mips/vdso/Makefile +++ b/arch/mips/vdso/Makefile @@ -43,8 +43,8 @@ CFLAGS_vgettimeofday.o = -include $(c-gettimeofday-y) # config-n32-o32-env.c prepares the environment to build a 32bit vDSO # library on a 64bit kernel. # Note: Needs to be included before than the generic library. -CFLAGS_vgettimeofday-o32.o = -include $(srctree)/$(src)/config-n32-o32-env.c -include $(c-gettimeofday-y) -CFLAGS_vgettimeofday-n32.o = -include $(srctree)/$(src)/config-n32-o32-env.c -include $(c-gettimeofday-y) +CFLAGS_vgettimeofday-o32.o = -include $(src)/config-n32-o32-env.c -include $(c-gettimeofday-y) +CFLAGS_vgettimeofday-n32.o = -include $(src)/config-n32-o32-env.c -include $(c-gettimeofday-y) endif CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) diff --git a/arch/nios2/boot/dts/Makefile b/arch/nios2/boot/dts/Makefile index e9e31bb40df85..1a2e8996bec77 100644 --- a/arch/nios2/boot/dts/Makefile +++ b/arch/nios2/boot/dts/Makefile @@ -2,5 +2,4 @@ obj-y := $(patsubst %.dts,%.dtb.o,$(CONFIG_NIOS2_DTB_SOURCE)) -dtstree := $(srctree)/$(src) -dtb-$(CONFIG_OF_ALL_DTBS) := $(patsubst $(dtstree)/%.dts,%.dtb, $(wildcard $(dtstree)/*.dts)) +dtb-$(CONFIG_OF_ALL_DTBS) := $(patsubst $(src)/%.dts,%.dtb, $(wildcard $(src)/*.dts)) diff --git a/arch/parisc/kernel/vdso32/Makefile b/arch/parisc/kernel/vdso32/Makefile index 60dc708a0f807..1350d50c63068 100644 --- a/arch/parisc/kernel/vdso32/Makefile +++ b/arch/parisc/kernel/vdso32/Makefile @@ -40,7 +40,7 @@ quiet_cmd_vdso32as = VDSO32A $@ cmd_vdso32as = $(CROSS32CC) $(a_flags) -c -o $@ $< # Generate VDSO offsets using helper script -gen-vdsosym := $(srctree)/$(src)/gen_vdso_offsets.sh +gen-vdsosym := $(src)/gen_vdso_offsets.sh quiet_cmd_vdsosym = VDSOSYM $@ cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ diff --git a/arch/parisc/kernel/vdso64/Makefile b/arch/parisc/kernel/vdso64/Makefile index 0982f3099ae25..0b1c1cc4c2c7c 100644 --- a/arch/parisc/kernel/vdso64/Makefile +++ b/arch/parisc/kernel/vdso64/Makefile @@ -40,7 +40,7 @@ quiet_cmd_vdso64as = VDSO64A $@ cmd_vdso64as = $(CC) $(a_flags) -c -o $@ $< # Generate VDSO offsets using helper script -gen-vdsosym := $(srctree)/$(src)/gen_vdso_offsets.sh +gen-vdsosym := $(src)/gen_vdso_offsets.sh quiet_cmd_vdsosym = VDSOSYM $@ cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile index 968aee2025b81..a0b3b5cc14cdc 100644 --- a/arch/powerpc/boot/Makefile +++ b/arch/powerpc/boot/Makefile @@ -218,7 +218,7 @@ $(addprefix $(obj)/,$(libfdt) $(libfdtheader)): $(obj)/%: $(srctree)/scripts/dtc $(obj)/empty.c: $(Q)touch $@ -$(obj)/zImage.coff.lds $(obj)/zImage.ps3.lds : $(obj)/%: $(srctree)/$(src)/%.S +$(obj)/zImage.coff.lds $(obj)/zImage.ps3.lds : $(obj)/%: $(src)/%.S $(Q)cp $< $@ clean-files := $(zlib-) $(zlibheader-) $(zliblinuxheader-) \ @@ -252,9 +252,9 @@ targets += $(patsubst $(obj)/%,%,$(obj-boot) wrapper.a) zImage.lds extra-y := $(obj)/wrapper.a $(obj-plat) $(obj)/empty.o \ $(obj)/zImage.lds $(obj)/zImage.coff.lds $(obj)/zImage.ps3.lds -dtstree := $(srctree)/$(src)/dts +dtstree := $(src)/dts -wrapper :=$(srctree)/$(src)/wrapper +wrapper := $(src)/wrapper wrapperbits := $(extra-y) $(addprefix $(obj)/,addnote hack-coff mktree) \ $(wrapper) FORCE diff --git a/arch/powerpc/boot/dts/Makefile b/arch/powerpc/boot/dts/Makefile index fb335d05aae8f..0cd0d8558b475 100644 --- a/arch/powerpc/boot/dts/Makefile +++ b/arch/powerpc/boot/dts/Makefile @@ -2,5 +2,4 @@ subdir-y += fsl -dtstree := $(srctree)/$(src) -dtb-$(CONFIG_OF_ALL_DTBS) := $(patsubst $(dtstree)/%.dts,%.dtb, $(wildcard $(dtstree)/*.dts)) +dtb-$(CONFIG_OF_ALL_DTBS) := $(patsubst $(src)/%.dts,%.dtb, $(wildcard $(src)/*.dts)) diff --git a/arch/powerpc/boot/dts/fsl/Makefile b/arch/powerpc/boot/dts/fsl/Makefile index 3bae982641e92..d3ecdf14bc42e 100644 --- a/arch/powerpc/boot/dts/fsl/Makefile +++ b/arch/powerpc/boot/dts/fsl/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 -dtstree := $(srctree)/$(src) -dtb-$(CONFIG_OF_ALL_DTBS) := $(patsubst $(dtstree)/%.dts,%.dtb, $(wildcard $(dtstree)/*.dts)) +dtb-$(CONFIG_OF_ALL_DTBS) := $(patsubst $(src)/%.dts,%.dtb, $(wildcard $(src)/*.dts)) diff --git a/arch/powerpc/kernel/vdso/Makefile b/arch/powerpc/kernel/vdso/Makefile index a14eab3669939..7d66b6e079938 100644 --- a/arch/powerpc/kernel/vdso/Makefile +++ b/arch/powerpc/kernel/vdso/Makefile @@ -90,10 +90,10 @@ $(obj)/vgettimeofday-64.o: %-64.o: %.c FORCE $(call if_changed_dep,cc_o_c) # Generate VDSO offsets using helper script -gen-vdso32sym := $(srctree)/$(src)/gen_vdso32_offsets.sh +gen-vdso32sym := $(src)/gen_vdso32_offsets.sh quiet_cmd_vdso32sym = VDSO32SYM $@ cmd_vdso32sym = $(NM) $< | $(gen-vdso32sym) | LC_ALL=C sort > $@ -gen-vdso64sym := $(srctree)/$(src)/gen_vdso64_offsets.sh +gen-vdso64sym := $(src)/gen_vdso64_offsets.sh quiet_cmd_vdso64sym = VDSO64SYM $@ cmd_vdso64sym = $(NM) $< | $(gen-vdso64sym) | LC_ALL=C sort > $@ diff --git a/arch/riscv/kernel/compat_vdso/Makefile b/arch/riscv/kernel/compat_vdso/Makefile index 3df4cb788c1fa..362208dfa7eec 100644 --- a/arch/riscv/kernel/compat_vdso/Makefile +++ b/arch/riscv/kernel/compat_vdso/Makefile @@ -58,7 +58,7 @@ $(obj)/%.so: $(obj)/%.so.dbg FORCE $(call if_changed,objcopy) # Generate VDSO offsets using helper script -gen-compat_vdsosym := $(srctree)/$(src)/gen_compat_vdso_offsets.sh +gen-compat_vdsosym := $(src)/gen_compat_vdso_offsets.sh quiet_cmd_compat_vdsosym = VDSOSYM $@ cmd_compat_vdsosym = $(NM) $< | $(gen-compat_vdsosym) | LC_ALL=C sort > $@ diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 272c431ac5b9f..25f0ee629971f 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -60,7 +60,7 @@ $(obj)/%.so: $(obj)/%.so.dbg FORCE $(call if_changed,objcopy) # Generate VDSO offsets using helper script -gen-vdsosym := $(srctree)/$(src)/gen_vdso_offsets.sh +gen-vdsosym := $(src)/gen_vdso_offsets.sh quiet_cmd_vdsosym = VDSOSYM $@ cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ diff --git a/arch/riscv/kvm/Makefile b/arch/riscv/kvm/Makefile index c9646521f1132..c2cacfbc06a06 100644 --- a/arch/riscv/kvm/Makefile +++ b/arch/riscv/kvm/Makefile @@ -3,7 +3,7 @@ # Makefile for RISC-V KVM support # -ccflags-y += -I $(srctree)/$(src) +ccflags-y += -I $(src) include $(srctree)/virt/kvm/Makefile.kvm diff --git a/arch/s390/kernel/syscalls/Makefile b/arch/s390/kernel/syscalls/Makefile index fb85e797946db..1bb78b9468e8a 100644 --- a/arch/s390/kernel/syscalls/Makefile +++ b/arch/s390/kernel/syscalls/Makefile @@ -4,8 +4,8 @@ gen := arch/$(ARCH)/include/generated kapi := $(gen)/asm uapi := $(gen)/uapi/asm -syscall := $(srctree)/$(src)/syscall.tbl -systbl := $(srctree)/$(src)/syscalltbl +syscall := $(src)/syscall.tbl +systbl := $(src)/syscalltbl gen-y := $(kapi)/syscall_table.h kapi-hdrs-y := $(kapi)/unistd_nr.h diff --git a/arch/s390/kernel/vdso32/Makefile b/arch/s390/kernel/vdso32/Makefile index 70e9949c26128..c263b91adfb15 100644 --- a/arch/s390/kernel/vdso32/Makefile +++ b/arch/s390/kernel/vdso32/Makefile @@ -62,7 +62,7 @@ quiet_cmd_vdso32cc = VDSO32C $@ cmd_vdso32cc = $(CC) $(c_flags) -c -o $@ $< # Generate VDSO offsets using helper script -gen-vdsosym := $(srctree)/$(src)/gen_vdso_offsets.sh +gen-vdsosym := $(src)/gen_vdso_offsets.sh quiet_cmd_vdsosym = VDSOSYM $@ cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile index 2b3617b6d1624..9566bed7d5b28 100644 --- a/arch/s390/kernel/vdso64/Makefile +++ b/arch/s390/kernel/vdso64/Makefile @@ -72,7 +72,7 @@ quiet_cmd_vdso64cc = VDSO64C $@ cmd_vdso64cc = $(CC) $(c_flags) -c -o $@ $< # Generate VDSO offsets using helper script -gen-vdsosym := $(srctree)/$(src)/gen_vdso_offsets.sh +gen-vdsosym := $(src)/gen_vdso_offsets.sh quiet_cmd_vdsosym = VDSOSYM $@ cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ diff --git a/arch/sparc/vdso/Makefile b/arch/sparc/vdso/Makefile index e8aef2c8ae99b..0fe134abbcf17 100644 --- a/arch/sparc/vdso/Makefile +++ b/arch/sparc/vdso/Makefile @@ -103,7 +103,7 @@ quiet_cmd_vdso = VDSO $@ cmd_vdso = $(LD) -nostdlib -o $@ \ $(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \ -T $(filter %.lds,$^) $(filter %.o,$^) && \ - sh $(srctree)/$(src)/checkundef.sh '$(OBJDUMP)' '$@' + sh $(src)/checkundef.sh '$(OBJDUMP)' '$@' VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 -Bsymbolic GCOV_PROFILE := n diff --git a/arch/um/kernel/Makefile b/arch/um/kernel/Makefile index 811188be954ca..f8567b933ffaa 100644 --- a/arch/um/kernel/Makefile +++ b/arch/um/kernel/Makefile @@ -47,7 +47,7 @@ $(obj)/config.c: $(src)/config.c.in $(obj)/config.tmp FORCE $(call if_changed,quote2) quiet_cmd_mkcapflags = MKCAP $@ - cmd_mkcapflags = $(CONFIG_SHELL) $(srctree)/$(src)/../../x86/kernel/cpu/mkcapflags.sh $@ $^ + cmd_mkcapflags = $(CONFIG_SHELL) $(src)/../../x86/kernel/cpu/mkcapflags.sh $@ $^ cpufeature = $(src)/../../x86/include/asm/cpufeatures.h vmxfeature = $(src)/../../x86/include/asm/vmxfeatures.h diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile index 3cece19b74732..29cda98c65f8b 100644 --- a/arch/x86/boot/Makefile +++ b/arch/x86/boot/Makefile @@ -129,7 +129,7 @@ targets += mtools.conf # genimage.sh requires bash, but it also has a bunch of other # external dependencies. quiet_cmd_genimage = GENIMAGE $3 -cmd_genimage = $(BASH) $(srctree)/$(src)/genimage.sh $2 $3 $(obj)/bzImage \ + cmd_genimage = $(BASH) $(src)/genimage.sh $2 $3 $(obj)/bzImage \ $(obj)/mtools.conf '$(FDARGS)' $(FDINITRD) PHONY += bzdisk fdimage fdimage144 fdimage288 hdimage isoimage diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index 3d64bcc403cfb..c003452dba8c5 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -176,7 +176,7 @@ quiet_cmd_vdso = VDSO $@ cmd_vdso = $(LD) -o $@ \ $(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \ -T $(filter %.lds,$^) $(filter %.o,$^) && \ - sh $(srctree)/$(src)/checkundef.sh '$(NM)' '$@' + sh $(src)/checkundef.sh '$(NM)' '$@' VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 \ $(call ld-option, --eh-frame-hdr) -Bsymbolic -z noexecstack diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 74077694da7d9..8a9fc9ec69d3d 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -40,7 +40,7 @@ KMSAN_SANITIZE_sev.o := n KCOV_INSTRUMENT_head$(BITS).o := n KCOV_INSTRUMENT_sev.o := n -CFLAGS_irq.o := -I $(srctree)/$(src)/../include/asm/trace +CFLAGS_irq.o := -I $(src)/../include/asm/trace obj-y += head_$(BITS).o obj-y += head$(BITS).o diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index eb4dbcdf41f1d..a02bba0ed6b95 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -60,7 +60,7 @@ obj-$(CONFIG_ACRN_GUEST) += acrn.o obj-$(CONFIG_DEBUG_FS) += debugfs.o quiet_cmd_mkcapflags = MKCAP $@ - cmd_mkcapflags = $(CONFIG_SHELL) $(srctree)/$(src)/mkcapflags.sh $@ $^ + cmd_mkcapflags = $(CONFIG_SHELL) $(src)/mkcapflags.sh $@ $^ cpufeature = $(src)/../../include/asm/cpufeatures.h vmxfeature = $(src)/../../include/asm/vmxfeatures.h diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile index 428048e73bd2e..8d3a00e5c528e 100644 --- a/arch/x86/mm/Makefile +++ b/arch/x86/mm/Makefile @@ -34,7 +34,7 @@ obj-y += pat/ CFLAGS_physaddr.o := -fno-stack-protector CFLAGS_mem_encrypt_identity.o := -fno-stack-protector -CFLAGS_fault.o := -I $(srctree)/$(src)/../include/asm/trace +CFLAGS_fault.o := -I $(src)/../include/asm/trace obj-$(CONFIG_X86_32) += pgtable_32.o iomap_32.o diff --git a/arch/x86/um/vdso/Makefile b/arch/x86/um/vdso/Makefile index b86d634730b2d..2303fa59971ce 100644 --- a/arch/x86/um/vdso/Makefile +++ b/arch/x86/um/vdso/Makefile @@ -63,7 +63,7 @@ quiet_cmd_vdso = VDSO $@ cmd_vdso = $(CC) -nostdlib -o $@ \ $(CC_FLAGS_LTO) $(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \ -Wl,-T,$(filter %.lds,$^) $(filter %.o,$^) && \ - sh $(srctree)/$(src)/checkundef.sh '$(NM)' '$@' + sh $(src)/checkundef.sh '$(NM)' '$@' VDSO_LDFLAGS = -fPIC -shared -Wl,--hash-style=sysv -z noexecstack GCOV_PROFILE := n diff --git a/arch/xtensa/boot/dts/Makefile b/arch/xtensa/boot/dts/Makefile index 720628c0d8b94..d6408c16d74e4 100644 --- a/arch/xtensa/boot/dts/Makefile +++ b/arch/xtensa/boot/dts/Makefile @@ -10,5 +10,4 @@ obj-$(CONFIG_OF) += $(addsuffix .dtb.o, $(CONFIG_BUILTIN_DTB_SOURCE)) # for CONFIG_OF_ALL_DTBS test -dtstree := $(srctree)/$(src) -dtb- := $(patsubst $(dtstree)/%.dts,%.dtb, $(wildcard $(dtstree)/*.dts)) +dtb- := $(patsubst $(src)/%.dts,%.dtb, $(wildcard $(src)/*.dts)) diff --git a/certs/Makefile b/certs/Makefile index 799ad7b9e68a0..1094e3860c2a7 100644 --- a/certs/Makefile +++ b/certs/Makefile @@ -13,7 +13,7 @@ CFLAGS_blacklist_hashes.o := -I $(obj) quiet_cmd_check_and_copy_blacklist_hash_list = GEN $@ cmd_check_and_copy_blacklist_hash_list = \ $(if $(CONFIG_SYSTEM_BLACKLIST_HASH_LIST), \ - $(AWK) -f $(srctree)/$(src)/check-blacklist-hashes.awk $(CONFIG_SYSTEM_BLACKLIST_HASH_LIST) >&2; \ + $(AWK) -f $(src)/check-blacklist-hashes.awk $(CONFIG_SYSTEM_BLACKLIST_HASH_LIST) >&2; \ { cat $(CONFIG_SYSTEM_BLACKLIST_HASH_LIST); echo $(comma) NULL; } > $@, \ echo NULL > $@) @@ -55,7 +55,7 @@ $(obj)/signing_key.pem: $(obj)/x509.genkey FORCE targets += signing_key.pem quiet_cmd_copy_x509_config = COPY $@ - cmd_copy_x509_config = cat $(srctree)/$(src)/default_x509.genkey > $@ + cmd_copy_x509_config = cat $(src)/default_x509.genkey > $@ # You can provide your own config file. If not present, copy the default one. $(obj)/x509.genkey: diff --git a/drivers/Makefile b/drivers/Makefile index 3bf5cab4b4519..fe9ceb0d2288a 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -6,11 +6,6 @@ # Rewritten to use lists instead of if-statements. # -# Some driver Makefiles miss $(srctree)/ for include directive. -ifdef building_out_of_srctree -MAKEFLAGS += --include-dir=$(srctree) -endif - obj-y += cache/ obj-y += irqchip/ obj-y += bus/ diff --git a/drivers/crypto/intel/qat/qat_420xx/Makefile b/drivers/crypto/intel/qat/qat_420xx/Makefile index a90fbe00b3c88..45728659fbc44 100644 --- a/drivers/crypto/intel/qat/qat_420xx/Makefile +++ b/drivers/crypto/intel/qat/qat_420xx/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_420XX) += qat_420xx.o qat_420xx-objs := adf_drv.o adf_420xx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_4xxx/Makefile b/drivers/crypto/intel/qat/qat_4xxx/Makefile index ff9c8b5897ea7..9ba202079a22f 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/Makefile +++ b/drivers/crypto/intel/qat/qat_4xxx/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx.o qat_4xxx-objs := adf_drv.o adf_4xxx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c3xxx/Makefile b/drivers/crypto/intel/qat/qat_c3xxx/Makefile index 92ef416ccc78b..7a06ad519bfc8 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/Makefile +++ b/drivers/crypto/intel/qat/qat_c3xxx/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx.o qat_c3xxx-objs := adf_drv.o adf_c3xxx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile b/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile index b6d76825a92c3..7ef633058c4f4 100644 --- a/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf.o qat_c3xxxvf-objs := adf_drv.o adf_c3xxxvf_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c62x/Makefile b/drivers/crypto/intel/qat/qat_c62x/Makefile index d581f7c87d6cf..cc9255b3b198d 100644 --- a/drivers/crypto/intel/qat/qat_c62x/Makefile +++ b/drivers/crypto/intel/qat/qat_c62x/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x.o qat_c62x-objs := adf_drv.o adf_c62x_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c62xvf/Makefile b/drivers/crypto/intel/qat/qat_c62xvf/Makefile index 446c3d6386059..256786662d60e 100644 --- a/drivers/crypto/intel/qat/qat_c62xvf/Makefile +++ b/drivers/crypto/intel/qat/qat_c62xvf/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf.o qat_c62xvf-objs := adf_drv.o adf_c62xvf_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/Makefile b/drivers/crypto/intel/qat/qat_dh895xcc/Makefile index 38d6f8e1624a3..cfd3bd7577158 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/Makefile +++ b/drivers/crypto/intel/qat/qat_dh895xcc/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc.o qat_dh895xcc-objs := adf_drv.o adf_dh895xcc_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile b/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile index 0153c85ce7435..64b54e92b2b4f 100644 --- a/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../qat_common +ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf.o qat_dh895xccvf-objs := adf_drv.o adf_dh895xccvf_hw_data.o diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index 4536c8ad0e117..e568e89283c01 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -23,7 +23,7 @@ # Makefile for the drm device driver. This driver provides support for the # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher. -FULL_AMD_PATH=$(srctree)/$(src)/.. +FULL_AMD_PATH=$(src)/.. DISPLAY_FOLDER_NAME=display FULL_AMD_DISPLAY_PATH = $(FULL_AMD_PATH)/$(DISPLAY_FOLDER_NAME) diff --git a/drivers/gpu/drm/arm/display/komeda/Makefile b/drivers/gpu/drm/arm/display/komeda/Makefile index 1931a7fa1a144..cf5287fcbbc2c 100644 --- a/drivers/gpu/drm/arm/display/komeda/Makefile +++ b/drivers/gpu/drm/arm/display/komeda/Makefile @@ -1,8 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 ccflags-y := \ - -I $(srctree)/$(src)/../include \ - -I $(srctree)/$(src) + -I $(src)/../include \ + -I $(src) komeda-y := \ komeda_drv.o \ diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index fba73c38e2356..2220448ed07f1 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -41,7 +41,7 @@ CFLAGS_display/intel_fbdev.o = -Wno-override-init # drivers. Define I915 when building i915. subdir-ccflags-y += -DI915 -subdir-ccflags-y += -I$(srctree)/$(src) +subdir-ccflags-y += -I$(src) # Please keep these build lists sorted! @@ -434,7 +434,7 @@ no-header-test := \ always-$(CONFIG_DRM_I915_WERROR) += \ $(patsubst %.h,%.hdrtest, $(filter-out $(no-header-test), \ - $(shell cd $(srctree)/$(src) && find * -name '*.h'))) + $(shell cd $(src) && find * -name '*.h'))) quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@) cmd_hdrtest = $(CC) $(filter-out $(CFLAGS_GCOV), $(c_flags)) -S -o /dev/null -x c /dev/null -include $<; \ diff --git a/drivers/gpu/drm/imagination/Makefile b/drivers/gpu/drm/imagination/Makefile index ec6db8e9b4037..9bc6a3884c223 100644 --- a/drivers/gpu/drm/imagination/Makefile +++ b/drivers/gpu/drm/imagination/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT # Copyright (c) 2023 Imagination Technologies Ltd. -subdir-ccflags-y := -I$(srctree)/$(src) +subdir-ccflags-y := -I$(src) powervr-y := \ pvr_ccb.o \ diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile index b21ae2880c715..b8cc007fc1b97 100644 --- a/drivers/gpu/drm/msm/Makefile +++ b/drivers/gpu/drm/msm/Makefile @@ -1,8 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 -ccflags-y := -I $(srctree)/$(src) -ccflags-y += -I $(srctree)/$(src)/disp/dpu1 -ccflags-$(CONFIG_DRM_MSM_DSI) += -I $(srctree)/$(src)/dsi -ccflags-$(CONFIG_DRM_MSM_DP) += -I $(srctree)/$(src)/dp +ccflags-y := -I $(src) +ccflags-y += -I $(src)/disp/dpu1 +ccflags-$(CONFIG_DRM_MSM_DSI) += -I $(src)/dsi +ccflags-$(CONFIG_DRM_MSM_DP) += -I $(src)/dp msm-y := \ adreno/adreno_device.o \ diff --git a/drivers/gpu/drm/nouveau/Kbuild b/drivers/gpu/drm/nouveau/Kbuild index cf6b3a80c0c8a..c32c01827c1d2 100644 --- a/drivers/gpu/drm/nouveau/Kbuild +++ b/drivers/gpu/drm/nouveau/Kbuild @@ -1,10 +1,8 @@ -NOUVEAU_PATH ?= $(srctree) - # SPDX-License-Identifier: MIT -ccflags-y += -I $(NOUVEAU_PATH)/$(src)/include -ccflags-y += -I $(NOUVEAU_PATH)/$(src)/include/nvkm -ccflags-y += -I $(NOUVEAU_PATH)/$(src)/nvkm -ccflags-y += -I $(NOUVEAU_PATH)/$(src) +ccflags-y += -I $(src)/include +ccflags-y += -I $(src)/include/nvkm +ccflags-y += -I $(src)/nvkm +ccflags-y += -I $(src) # NVKM - HW resource manager #- code also used by various userspace tools/tests diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index c29a850859ad5..8162a522222b6 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -32,7 +32,7 @@ endif # Enable -Werror in CI and development subdir-ccflags-$(CONFIG_DRM_XE_WERROR) += -Werror -subdir-ccflags-y += -I$(obj) -I$(srctree)/$(src) +subdir-ccflags-y += -I$(obj) -I$(src) # generated sources hostprogs := xe_gen_wa_oob @@ -43,7 +43,7 @@ quiet_cmd_wa_oob = GEN $(notdir $(generated_oob)) cmd_wa_oob = mkdir -p $(@D); $^ $(generated_oob) $(obj)/generated/%_wa_oob.c $(obj)/generated/%_wa_oob.h: $(obj)/xe_gen_wa_oob \ - $(srctree)/$(src)/xe_wa_oob.rules + $(src)/xe_wa_oob.rules $(call cmd,wa_oob) uses_generated_oob := \ @@ -166,8 +166,8 @@ endif # i915 Display compat #defines and #includes subdir-ccflags-$(CONFIG_DRM_XE_DISPLAY) += \ - -I$(srctree)/$(src)/display/ext \ - -I$(srctree)/$(src)/compat-i915-headers \ + -I$(src)/display/ext \ + -I$(src)/compat-i915-headers \ -I$(srctree)/drivers/gpu/drm/i915/display/ \ -Ddrm_i915_gem_object=xe_bo \ -Ddrm_i915_private=xe_device @@ -310,7 +310,7 @@ ifneq ($(CONFIG_DRM_XE_DISPLAY),y) endif always-$(CONFIG_DRM_XE_WERROR) += \ - $(patsubst %.h,%.hdrtest, $(shell cd $(srctree)/$(src) && find * -name '*.h' $(hdrtest_find_args))) + $(patsubst %.h,%.hdrtest, $(shell cd $(src) && find * -name '*.h' $(hdrtest_find_args))) quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@) cmd_hdrtest = $(CC) -DHDRTEST $(filter-out $(CFLAGS_GCOV), $(c_flags)) -S -o /dev/null -x c /dev/null -include $<; touch $@ diff --git a/drivers/hid/amd-sfh-hid/Makefile b/drivers/hid/amd-sfh-hid/Makefile index 0222170ab7ad7..106514b54d16c 100644 --- a/drivers/hid/amd-sfh-hid/Makefile +++ b/drivers/hid/amd-sfh-hid/Makefile @@ -13,4 +13,4 @@ amd_sfh-objs += sfh1_1/amd_sfh_init.o amd_sfh-objs += sfh1_1/amd_sfh_interface.o amd_sfh-objs += sfh1_1/amd_sfh_desc.o -ccflags-y += -I $(srctree)/$(src)/ +ccflags-y += -I $(src)/ diff --git a/drivers/hid/intel-ish-hid/Makefile b/drivers/hid/intel-ish-hid/Makefile index f0a82b1c7cb93..db4974c43431c 100644 --- a/drivers/hid/intel-ish-hid/Makefile +++ b/drivers/hid/intel-ish-hid/Makefile @@ -23,4 +23,4 @@ intel-ishtp-hid-objs += ishtp-hid-client.o obj-$(CONFIG_INTEL_ISH_FIRMWARE_DOWNLOADER) += intel-ishtp-loader.o intel-ishtp-loader-objs += ishtp-fw-loader.o -ccflags-y += -I $(srctree)/$(src)/ishtp +ccflags-y += -I $(src)/ishtp diff --git a/drivers/md/dm-vdo/Makefile b/drivers/md/dm-vdo/Makefile index 33e09abc6acd0..9476957bfbf4e 100644 --- a/drivers/md/dm-vdo/Makefile +++ b/drivers/md/dm-vdo/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I$(srctree)/$(src) -I$(srctree)/$(src)/indexer +ccflags-y := -I$(src) -I$(src)/indexer obj-$(CONFIG_DM_VDO) += dm-vdo.o diff --git a/drivers/net/ethernet/aquantia/atlantic/Makefile b/drivers/net/ethernet/aquantia/atlantic/Makefile index 8ebcc68e807fc..f6a96931c89a5 100644 --- a/drivers/net/ethernet/aquantia/atlantic/Makefile +++ b/drivers/net/ethernet/aquantia/atlantic/Makefile @@ -8,7 +8,7 @@ obj-$(CONFIG_AQTION) += atlantic.o -ccflags-y += -I$(srctree)/$(src) +ccflags-y += -I$(src) atlantic-objs := aq_main.o \ aq_nic.o \ diff --git a/drivers/net/ethernet/chelsio/libcxgb/Makefile b/drivers/net/ethernet/chelsio/libcxgb/Makefile index aa79264e72ba1..fbedc31674b3e 100644 --- a/drivers/net/ethernet/chelsio/libcxgb/Makefile +++ b/drivers/net/ethernet/chelsio/libcxgb/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(srctree)/$(src)/../cxgb4 +ccflags-y := -I $(src)/../cxgb4 obj-$(CONFIG_CHELSIO_LIB) += libcxgb.o diff --git a/drivers/net/ethernet/fungible/funeth/Makefile b/drivers/net/ethernet/fungible/funeth/Makefile index 646d69595b4f0..d51e4c2b4a1a4 100644 --- a/drivers/net/ethernet/fungible/funeth/Makefile +++ b/drivers/net/ethernet/fungible/funeth/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) -ccflags-y += -I$(srctree)/$(src)/../funcore -I$(srctree)/$(src) +ccflags-y += -I$(src)/../funcore -I$(src) obj-$(CONFIG_FUN_ETH) += funeth.o diff --git a/drivers/net/ethernet/hisilicon/hns3/Makefile b/drivers/net/ethernet/hisilicon/hns3/Makefile index e214bfaece1f3..8e9293e57bfd5 100644 --- a/drivers/net/ethernet/hisilicon/hns3/Makefile +++ b/drivers/net/ethernet/hisilicon/hns3/Makefile @@ -3,7 +3,7 @@ # Makefile for the HISILICON network device drivers. # -ccflags-y += -I$(srctree)/$(src) +ccflags-y += -I$(src) ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/hns3pf ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/hns3vf ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/hns3_common diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile index dc6d27a36faa9..e5ca0f5118227 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile @@ -6,8 +6,8 @@ # ccflags-y += \ - -I $(srctree)/$(src) \ - -I $(srctree)/$(src)/../include + -I $(src) \ + -I $(src)/../include obj-$(CONFIG_BRCMFMAC) += brcmfmac.o brcmfmac-objs += \ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bca/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bca/Makefile index 46098705e2367..5e37c638f9661 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bca/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bca/Makefile @@ -3,9 +3,9 @@ # Copyright (c) 2022 Broadcom Corporation ccflags-y += \ - -I $(srctree)/$(src) \ - -I $(srctree)/$(src)/.. \ - -I $(srctree)/$(src)/../../include + -I $(src) \ + -I $(src)/.. \ + -I $(src)/../../include obj-m += brcmfmac-bca.o brcmfmac-bca-objs += \ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/Makefile index 5e1fddaff79e2..33e86724ba14e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/Makefile @@ -3,9 +3,9 @@ # Copyright (c) 2022 Broadcom Corporation ccflags-y += \ - -I $(srctree)/$(src) \ - -I $(srctree)/$(src)/.. \ - -I $(srctree)/$(src)/../../include + -I $(src) \ + -I $(src)/.. \ + -I $(src)/../../include obj-m += brcmfmac-cyw.o brcmfmac-cyw-objs += \ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/wcc/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/wcc/Makefile index 7f455a19a2b12..3db4cce44f420 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/wcc/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/wcc/Makefile @@ -3,9 +3,9 @@ # Copyright (c) 2022 Broadcom Corporation ccflags-y += \ - -I $(srctree)/$(src) \ - -I $(srctree)/$(src)/.. \ - -I $(srctree)/$(src)/../../include + -I $(src) \ + -I $(src)/.. \ + -I $(src)/../../include obj-m += brcmfmac-wcc.o brcmfmac-wcc-objs += \ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/Makefile index 090757730ba60..ffdd17eabe1df 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/Makefile @@ -16,9 +16,9 @@ # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ccflags-y := \ - -I $(srctree)/$(src) \ - -I $(srctree)/$(src)/phy \ - -I $(srctree)/$(src)/../include + -I $(src) \ + -I $(src)/phy \ + -I $(src)/../include brcmsmac-y := \ mac80211_if.o \ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmutil/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmutil/Makefile index 7a82d615ba2ae..f9b40cab766c6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmutil/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmutil/Makefile @@ -4,7 +4,7 @@ # # Copyright (c) 2011 Broadcom Corporation # -ccflags-y := -I $(srctree)/$(src)/../include +ccflags-y := -I $(src)/../include obj-$(CONFIG_BRCMUTIL) += brcmutil.o brcmutil-objs = utils.o d11.o diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/Makefile b/drivers/net/wireless/intel/iwlwifi/dvm/Makefile index 0486b17d7c41f..6109d64006dbc 100644 --- a/drivers/net/wireless/intel/iwlwifi/dvm/Makefile +++ b/drivers/net/wireless/intel/iwlwifi/dvm/Makefile @@ -11,4 +11,4 @@ iwldvm-objs += rxon.o devices.o iwldvm-$(CONFIG_IWLWIFI_LEDS) += led.o iwldvm-$(CONFIG_IWLWIFI_DEBUGFS) += debugfs.o -ccflags-y += -I $(srctree)/$(src)/../ +ccflags-y += -I $(src)/../ diff --git a/drivers/net/wireless/intel/iwlwifi/mei/Makefile b/drivers/net/wireless/intel/iwlwifi/mei/Makefile index 8e3ef0347db7e..e05f9421be4ad 100644 --- a/drivers/net/wireless/intel/iwlwifi/mei/Makefile +++ b/drivers/net/wireless/intel/iwlwifi/mei/Makefile @@ -5,4 +5,4 @@ iwlmei-y += net.o iwlmei-$(CONFIG_IWLWIFI_DEVICE_TRACING) += trace.o CFLAGS_trace.o := -I$(src) -ccflags-y += -I $(srctree)/$(src)/../ +ccflags-y += -I $(src)/../ diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/Makefile b/drivers/net/wireless/intel/iwlwifi/mvm/Makefile index 593fe28d89cf7..764ba73cde1ee 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/Makefile +++ b/drivers/net/wireless/intel/iwlwifi/mvm/Makefile @@ -15,4 +15,4 @@ iwlmvm-$(CONFIG_IWLWIFI_LEDS) += led.o iwlmvm-$(CONFIG_PM) += d3.o iwlmvm-$(CONFIG_IWLMEI) += vendor-cmd.o -ccflags-y += -I $(srctree)/$(src)/../ +ccflags-y += -I $(src)/../ diff --git a/drivers/net/wireless/intel/iwlwifi/tests/Makefile b/drivers/net/wireless/intel/iwlwifi/tests/Makefile index 5658471bdf0a3..84491488f5892 100644 --- a/drivers/net/wireless/intel/iwlwifi/tests/Makefile +++ b/drivers/net/wireless/intel/iwlwifi/tests/Makefile @@ -2,6 +2,6 @@ iwlwifi-tests-y += module.o devinfo.o -ccflags-y += -I$(srctree)/$(src)/../ +ccflags-y += -I$(src)/../ obj-$(CONFIG_IWLWIFI_KUNIT_TESTS) += iwlwifi-tests.o diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8180/Makefile b/drivers/net/wireless/realtek/rtl818x/rtl8180/Makefile index 565a9a1141344..1044105c2557d 100644 --- a/drivers/net/wireless/realtek/rtl818x/rtl8180/Makefile +++ b/drivers/net/wireless/realtek/rtl818x/rtl8180/Makefile @@ -3,4 +3,4 @@ rtl818x_pci-objs := dev.o rtl8225.o sa2400.o max2820.o grf5101.o rtl8225se.o obj-$(CONFIG_RTL8180) += rtl818x_pci.o -ccflags-y += -I $(srctree)/$(src)/.. +ccflags-y += -I $(src)/.. diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8187/Makefile b/drivers/net/wireless/realtek/rtl818x/rtl8187/Makefile index 0bf64dfb233ae..3deded2c05b6b 100644 --- a/drivers/net/wireless/realtek/rtl818x/rtl8187/Makefile +++ b/drivers/net/wireless/realtek/rtl818x/rtl8187/Makefile @@ -3,4 +3,4 @@ rtl8187-objs := dev.o rtl8225.o leds.o rfkill.o obj-$(CONFIG_RTL8187) += rtl8187.o -ccflags-y += -I $(srctree)/$(src)/.. +ccflags-y += -I $(src)/.. diff --git a/drivers/scsi/aic7xxx/Makefile b/drivers/scsi/aic7xxx/Makefile index e0188ecd85b22..853c72a81ae01 100644 --- a/drivers/scsi/aic7xxx/Makefile +++ b/drivers/scsi/aic7xxx/Makefile @@ -55,9 +55,9 @@ aicasm-7xxx-opts-$(CONFIG_AIC7XXX_REG_PRETTY_PRINT) := \ ifeq ($(CONFIG_AIC7XXX_BUILD_FIRMWARE),y) $(obj)/aic7xxx_seq.h: $(src)/aic7xxx.seq $(src)/aic7xxx.reg $(obj)/aicasm/aicasm - $(obj)/aicasm/aicasm -I$(srctree)/$(src) -r $(obj)/aic7xxx_reg.h \ + $(obj)/aicasm/aicasm -I $(src) -r $(obj)/aic7xxx_reg.h \ $(aicasm-7xxx-opts-y) -o $(obj)/aic7xxx_seq.h \ - $(srctree)/$(src)/aic7xxx.seq + $(src)/aic7xxx.seq $(aic7xxx-gen-y): $(objtree)/$(obj)/aic7xxx_seq.h @true @@ -73,9 +73,9 @@ aicasm-79xx-opts-$(CONFIG_AIC79XX_REG_PRETTY_PRINT) := \ ifeq ($(CONFIG_AIC79XX_BUILD_FIRMWARE),y) $(obj)/aic79xx_seq.h: $(src)/aic79xx.seq $(src)/aic79xx.reg $(obj)/aicasm/aicasm - $(obj)/aicasm/aicasm -I$(srctree)/$(src) -r $(obj)/aic79xx_reg.h \ + $(obj)/aicasm/aicasm -I $(src) -r $(obj)/aic79xx_reg.h \ $(aicasm-79xx-opts-y) -o $(obj)/aic79xx_seq.h \ - $(srctree)/$(src)/aic79xx.seq + $(src)/aic79xx.seq $(aic79xx-gen-y): $(objtree)/$(obj)/aic79xx_seq.h @true @@ -83,5 +83,5 @@ else $(obj)/aic79xx_reg_print.c: $(src)/aic79xx_reg_print.c_shipped endif -$(obj)/aicasm/aicasm: $(srctree)/$(src)/aicasm/*.[chyl] - $(MAKE) -C $(srctree)/$(src)/aicasm OUTDIR=$(shell pwd)/$(obj)/aicasm/ +$(obj)/aicasm/aicasm: $(src)/aicasm/*.[chyl] + $(MAKE) -C $(src)/aicasm OUTDIR=$(shell pwd)/$(obj)/aicasm/ diff --git a/drivers/staging/rtl8723bs/Makefile b/drivers/staging/rtl8723bs/Makefile index 590bde02058c7..7f5067e89295e 100644 --- a/drivers/staging/rtl8723bs/Makefile +++ b/drivers/staging/rtl8723bs/Makefile @@ -62,4 +62,4 @@ r8723bs-y = \ obj-$(CONFIG_RTL8723BS) := r8723bs.o -ccflags-y += -I$(srctree)/$(src)/include -I$(srctree)/$(src)/hal +ccflags-y += -I$(src)/include -I$(src)/hal diff --git a/fs/iomap/Makefile b/fs/iomap/Makefile index fc070184b7faa..381d76c5c2325 100644 --- a/fs/iomap/Makefile +++ b/fs/iomap/Makefile @@ -4,7 +4,7 @@ # All Rights Reserved. # -ccflags-y += -I $(srctree)/$(src) # needed for trace events +ccflags-y += -I $(src) # needed for trace events obj-$(CONFIG_FS_IOMAP) += iomap.o diff --git a/fs/unicode/Makefile b/fs/unicode/Makefile index 0e51c0025a16f..e309afe2b2bb7 100644 --- a/fs/unicode/Makefile +++ b/fs/unicode/Makefile @@ -18,13 +18,13 @@ ifdef REGENERATE_UTF8DATA quiet_cmd_utf8data = GEN $@ cmd_utf8data = $< \ - -a $(srctree)/$(src)/DerivedAge.txt \ - -c $(srctree)/$(src)/DerivedCombiningClass.txt \ - -p $(srctree)/$(src)/DerivedCoreProperties.txt \ - -d $(srctree)/$(src)/UnicodeData.txt \ - -f $(srctree)/$(src)/CaseFolding.txt \ - -n $(srctree)/$(src)/NormalizationCorrections.txt \ - -t $(srctree)/$(src)/NormalizationTest.txt \ + -a $(src)/DerivedAge.txt \ + -c $(src)/DerivedCombiningClass.txt \ + -p $(src)/DerivedCoreProperties.txt \ + -d $(src)/UnicodeData.txt \ + -f $(src)/CaseFolding.txt \ + -n $(src)/NormalizationCorrections.txt \ + -t $(src)/NormalizationTest.txt \ -o $@ $(obj)/utf8data.c: $(obj)/mkutf8data $(filter %.txt, $(cmd_utf8data)) FORCE diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index 76674ad5833e3..c5a35e32adf00 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -4,8 +4,8 @@ # All Rights Reserved. # -ccflags-y += -I $(srctree)/$(src) # needed for trace events -ccflags-y += -I $(srctree)/$(src)/libxfs +ccflags-y += -I $(src) # needed for trace events +ccflags-y += -I $(src)/libxfs obj-$(CONFIG_XFS_FS) += xfs.o diff --git a/init/Makefile b/init/Makefile index cbac576c57d63..3c48d97538c10 100644 --- a/init/Makefile +++ b/init/Makefile @@ -52,7 +52,7 @@ CFLAGS_version.o := -include $(obj)/utsversion-tmp.h # Build version-timestamp.c with final UTS_VERSION # -include/generated/utsversion.h: build-version-auto = $(shell $(srctree)/$(src)/build-version) +include/generated/utsversion.h: build-version-auto = $(shell $(src)/build-version) include/generated/utsversion.h: build-timestamp-auto = $(shell LC_ALL=C date) include/generated/utsversion.h: FORCE $(call filechk,uts_version) diff --git a/lib/Makefile b/lib/Makefile index ffc6b2341b45a..27baa6e6daa80 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -352,7 +352,7 @@ $(obj)/oid_registry_data.c: $(srctree)/include/linux/oid_registry.h \ $(call cmd,build_OID_registry) quiet_cmd_build_OID_registry = GEN $@ - cmd_build_OID_registry = perl $(srctree)/$(src)/build_OID_registry $< $@ + cmd_build_OID_registry = perl $(src)/build_OID_registry $< $@ clean-files += oid_registry_data.c @@ -412,8 +412,8 @@ obj-$(CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED) += devmem_is_allowed.o obj-$(CONFIG_FIRMWARE_TABLE) += fw_table.o # FORTIFY_SOURCE compile-time behavior tests -TEST_FORTIFY_SRCS = $(wildcard $(srctree)/$(src)/test_fortify/*-*.c) -TEST_FORTIFY_LOGS = $(patsubst $(srctree)/$(src)/%.c, %.log, $(TEST_FORTIFY_SRCS)) +TEST_FORTIFY_SRCS = $(wildcard $(src)/test_fortify/*-*.c) +TEST_FORTIFY_LOGS = $(patsubst $(src)/%.c, %.log, $(TEST_FORTIFY_SRCS)) TEST_FORTIFY_LOG = test_fortify.log quiet_cmd_test_fortify = TEST $@ diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile index 385a94aa0b999..8785353c6140b 100644 --- a/lib/raid6/Makefile +++ b/lib/raid6/Makefile @@ -53,7 +53,7 @@ endif endif quiet_cmd_unroll = UNROLL $@ - cmd_unroll = $(AWK) -v N=$* -f $(srctree)/$(src)/unroll.awk < $< > $@ + cmd_unroll = $(AWK) -v N=$* -f $(src)/unroll.awk < $< > $@ targets += int1.c int2.c int4.c int8.c $(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE diff --git a/net/wireless/Makefile b/net/wireless/Makefile index 72074fd36df4c..1d49cc8b6da15 100644 --- a/net/wireless/Makefile +++ b/net/wireless/Makefile @@ -25,7 +25,7 @@ ifneq ($(CONFIG_CFG80211_EXTRA_REGDB_KEYDIR),) cfg80211-y += extra-certs.o endif -$(obj)/shipped-certs.c: $(sort $(wildcard $(srctree)/$(src)/certs/*.hex)) +$(obj)/shipped-certs.c: $(sort $(wildcard $(src)/certs/*.hex)) @$(kecho) " GEN $@" $(Q)(echo '#include "reg.h"'; \ echo 'const u8 shipped_regdb_certs[] = {'; \ diff --git a/rust/Makefile b/rust/Makefile index 846e6ab9d5a9b..85e46dde2a5bd 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -239,7 +239,7 @@ quiet_cmd_rustsysroot = RUSTSYSROOT rm -rf $(objtree)/$(obj)/test; \ mkdir -p $(objtree)/$(obj)/test; \ cp -a $(rustc_sysroot) $(objtree)/$(obj)/test/sysroot; \ - cp -r $(srctree)/$(src)/alloc/* \ + cp -r $(src)/alloc/* \ $(objtree)/$(obj)/test/sysroot/lib/rustlib/src/rust/library/alloc/src; \ echo '\#!/bin/sh' > $(objtree)/$(obj)/test/rustc_sysroot; \ echo "$(RUSTC) --sysroot=$(abspath $(objtree)/$(obj)/test/sysroot) \"\$$@\"" \ @@ -340,7 +340,7 @@ quiet_cmd_bindgen = BINDGEN $@ $(bindgen_target_cflags) $(bindgen_target_extra) $(obj)/bindings/bindings_generated.rs: private bindgen_target_flags = \ - $(shell grep -Ev '^#|^$$' $(srctree)/$(src)/bindgen_parameters) + $(shell grep -Ev '^#|^$$' $(src)/bindgen_parameters) $(obj)/bindings/bindings_generated.rs: private bindgen_target_extra = ; \ sed -Ei 's/pub const RUST_CONST_HELPER_([a-zA-Z0-9_]*)/pub const \1/g' $@ $(obj)/bindings/bindings_generated.rs: $(src)/bindings/bindings_helper.h \ @@ -348,7 +348,7 @@ $(obj)/bindings/bindings_generated.rs: $(src)/bindings/bindings_helper.h \ $(call if_changed_dep,bindgen) $(obj)/uapi/uapi_generated.rs: private bindgen_target_flags = \ - $(shell grep -Ev '^#|^$$' $(srctree)/$(src)/bindgen_parameters) + $(shell grep -Ev '^#|^$$' $(src)/bindgen_parameters) $(obj)/uapi/uapi_generated.rs: $(src)/uapi/uapi_helper.h \ $(src)/bindgen_parameters FORCE $(call if_changed_dep,bindgen) diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile index 933f6c3fe6b04..f054e0828f137 100644 --- a/samples/bpf/Makefile +++ b/samples/bpf/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -BPF_SAMPLES_PATH ?= $(abspath $(srctree)/$(src)) +BPF_SAMPLES_PATH ?= $(abspath $(src)) TOOLS_PATH := $(BPF_SAMPLES_PATH)/../../tools pound := \# diff --git a/samples/hid/Makefile b/samples/hid/Makefile index 9f7fe29dd7497..c128ccd499745 100644 --- a/samples/hid/Makefile +++ b/samples/hid/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -HID_SAMPLES_PATH ?= $(abspath $(srctree)/$(src)) +HID_SAMPLES_PATH ?= $(abspath $(src)) TOOLS_PATH := $(HID_SAMPLES_PATH)/../../tools pound := \# diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 2f331879816b8..faf37bafa3f81 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -62,8 +62,7 @@ stringify = $(squote)$(quote)$1$(quote)$(squote) ### # The path to Kbuild or Makefile. Kbuild has precedence over Makefile. -kbuild-dir = $(if $(filter /%,$(src)),$(src),$(srctree)/$(src)) -kbuild-file = $(or $(wildcard $(kbuild-dir)/Kbuild),$(kbuild-dir)/Makefile) +kbuild-file = $(or $(wildcard $(src)/Kbuild),$(src)/Makefile) ### # Read a file, replacing newlines with spaces diff --git a/scripts/Makefile.asm-generic b/scripts/Makefile.asm-generic index 8d01b37b76775..1486abf34c7c9 100644 --- a/scripts/Makefile.asm-generic +++ b/scripts/Makefile.asm-generic @@ -9,7 +9,7 @@ PHONY := all all: -src := $(subst /generated,,$(obj)) +src := $(srctree)/$(subst /generated,,$(obj)) include $(srctree)/scripts/Kbuild.include -include $(kbuild-file) @@ -20,14 +20,14 @@ include $(srctree)/$(generic)/Kbuild endif redundant := $(filter $(mandatory-y) $(generated-y), $(generic-y)) -redundant += $(foreach f, $(generic-y), $(if $(wildcard $(srctree)/$(src)/$(f)),$(f))) +redundant += $(foreach f, $(generic-y), $(if $(wildcard $(src)/$(f)),$(f))) redundant := $(sort $(redundant)) $(if $(redundant),\ $(warning redundant generic-y found in $(src)/Kbuild: $(redundant))) # If arch does not implement mandatory headers, fallback to asm-generic ones. mandatory-y := $(filter-out $(generated-y), $(mandatory-y)) -generic-y += $(foreach f, $(mandatory-y), $(if $(wildcard $(srctree)/$(src)/$(f)),,$(f))) +generic-y += $(foreach f, $(mandatory-y), $(if $(wildcard $(src)/$(f)),,$(f))) generic-y := $(addprefix $(obj)/, $(generic-y)) generated-y := $(addprefix $(obj)/, $(generated-y)) diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 41138346afb01..43b71c3d0ff62 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -3,7 +3,7 @@ # Building # ========================================================================== -src := $(obj) +src := $(if $(VPATH),$(VPATH)/)$(obj) PHONY := $(obj)/ $(obj)/: diff --git a/scripts/Makefile.clean b/scripts/Makefile.clean index f2cb4d7ffd962..4fcfab40ed61a 100644 --- a/scripts/Makefile.clean +++ b/scripts/Makefile.clean @@ -3,7 +3,7 @@ # Cleaning up # ========================================================================== -src := $(obj) +src := $(if $(VPATH),$(VPATH)/)$(obj) PHONY := __clean __clean: diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 42d3c772c8e06..08d42e93bea02 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -209,13 +209,13 @@ _c_flags += $(if $(patsubst n%,, \ -D__KCSAN_INSTRUMENT_BARRIERS__) endif -# $(srctree)/$(src) for including checkin headers from generated source files -# $(objtree)/$(obj) for including generated headers from checkin source files +# $(src) for including checkin headers from generated source files +# $(obj) for including generated headers from checkin source files ifeq ($(KBUILD_EXTMOD),) ifdef building_out_of_srctree -_c_flags += $(addprefix -I $(srctree)/,$(src)) $(addprefix -I $(objtree)/,$(obj)) -_a_flags += $(addprefix -I $(srctree)/,$(src)) $(addprefix -I $(objtree)/,$(obj)) -_cpp_flags += $(addprefix -I $(srctree)/,$(src)) $(addprefix -I $(objtree)/,$(obj)) +_c_flags += $(addprefix -I, $(src) $(obj)) +_a_flags += $(addprefix -I, $(src) $(obj)) +_cpp_flags += $(addprefix -I, $(src) $(obj)) endif endif diff --git a/scripts/Makefile.modpost b/scripts/Makefile.modpost index 739402f455097..68c4f9ee0c909 100644 --- a/scripts/Makefile.modpost +++ b/scripts/Makefile.modpost @@ -112,7 +112,7 @@ else # set src + obj - they may be used in the modules's Makefile obj := $(KBUILD_EXTMOD) -src := $(obj) +src := $(if $(VPATH),$(VPATH)/)$(obj) # Include the module's Makefile to find KBUILD_EXTRA_SYMBOLS include $(kbuild-file) diff --git a/scripts/dtc/Makefile b/scripts/dtc/Makefile index 4d32b9497da94..a186570725412 100644 --- a/scripts/dtc/Makefile +++ b/scripts/dtc/Makefile @@ -16,12 +16,12 @@ libfdt = $(addprefix libfdt/,$(libfdt-objs)) fdtoverlay-objs := $(libfdt) fdtoverlay.o util.o # Source files need to get at the userspace version of libfdt_env.h to compile -HOST_EXTRACFLAGS += -I $(srctree)/$(src)/libfdt +HOST_EXTRACFLAGS += -I $(src)/libfdt HOST_EXTRACFLAGS += -DNO_YAML # Generated files need one more search path to include headers in source tree -HOSTCFLAGS_dtc-lexer.lex.o := -I $(srctree)/$(src) -HOSTCFLAGS_dtc-parser.tab.o := -I $(srctree)/$(src) +HOSTCFLAGS_dtc-lexer.lex.o := -I $(src) +HOSTCFLAGS_dtc-parser.tab.o := -I $(src) # dependencies on generated files need to be listed explicitly $(obj)/dtc-lexer.lex.o: $(obj)/dtc-parser.tab.h diff --git a/scripts/gdb/linux/Makefile b/scripts/gdb/linux/Makefile index 48941faa6ea69..d77ad9079d0f9 100644 --- a/scripts/gdb/linux/Makefile +++ b/scripts/gdb/linux/Makefile @@ -2,7 +2,7 @@ ifdef building_out_of_srctree -symlinks := $(patsubst $(srctree)/$(src)/%,%,$(wildcard $(srctree)/$(src)/*.py)) +symlinks := $(patsubst $(src)/%,%,$(wildcard $(src)/*.py)) quiet_cmd_symlink = SYMLINK $@ cmd_symlink = ln -fsn $(patsubst $(obj)/%,$(abspath $(srctree))/$(src)/%,$@) $@ diff --git a/scripts/genksyms/Makefile b/scripts/genksyms/Makefile index d6a422a63b6ad..312edccda7363 100644 --- a/scripts/genksyms/Makefile +++ b/scripts/genksyms/Makefile @@ -23,8 +23,8 @@ $(obj)/pars%.tab.c $(obj)/pars%.tab.h: $(src)/pars%.y FORCE endif # -I needed for generated C source to include headers in source tree -HOSTCFLAGS_parse.tab.o := -I $(srctree)/$(src) -HOSTCFLAGS_lex.lex.o := -I $(srctree)/$(src) +HOSTCFLAGS_parse.tab.o := -I $(src) +HOSTCFLAGS_lex.lex.o := -I $(src) # dependencies on generated files need to be listed explicitly $(obj)/lex.lex.o: $(obj)/parse.tab.h diff --git a/scripts/kconfig/Makefile b/scripts/kconfig/Makefile index ea1bf3b3dbde1..a0a0be38cbdc1 100644 --- a/scripts/kconfig/Makefile +++ b/scripts/kconfig/Makefile @@ -57,7 +57,7 @@ $(foreach c, config menuconfig nconfig gconfig xconfig, $(eval $(call config_rul PHONY += localmodconfig localyesconfig localyesconfig localmodconfig: $(obj)/conf - $(Q)$(PERL) $(srctree)/$(src)/streamline_config.pl --$@ $(srctree) $(Kconfig) > .tmp.config + $(Q)$(PERL) $(src)/streamline_config.pl --$@ $(srctree) $(Kconfig) > .tmp.config $(Q)if [ -f .config ]; then \ cmp -s .tmp.config .config || \ (mv -f .config .config.old.1; \ @@ -118,7 +118,7 @@ tinyconfig: # CHECK: -o cache_dir= working? PHONY += testconfig testconfig: $(obj)/conf - $(Q)$(PYTHON3) -B -m pytest $(srctree)/$(src)/tests \ + $(Q)$(PYTHON3) -B -m pytest $(src)/tests \ -o cache_dir=$(abspath $(obj)/tests/.cache) \ $(if $(findstring 1,$(KBUILD_VERBOSE)),--capture=no) clean-files += tests/.cache @@ -165,8 +165,8 @@ common-objs := confdata.o expr.o lexer.lex.o menu.o parser.tab.o \ preprocess.o symbol.o util.o $(obj)/lexer.lex.o: $(obj)/parser.tab.h -HOSTCFLAGS_lexer.lex.o := -I $(srctree)/$(src) -HOSTCFLAGS_parser.tab.o := -I $(srctree)/$(src) +HOSTCFLAGS_lexer.lex.o := -I $(src) +HOSTCFLAGS_parser.tab.o := -I $(src) # conf: Used for defconfig, oldconfig and related targets hostprogs += conf diff --git a/security/tomoyo/Makefile b/security/tomoyo/Makefile index 884ff155edc39..55c67b9846a93 100644 --- a/security/tomoyo/Makefile +++ b/security/tomoyo/Makefile @@ -11,7 +11,7 @@ quiet_cmd_policy = POLICY $@ printf '\t"";\n';) \ } > $@ -$(obj)/builtin-policy.h: $(wildcard $(obj)/policy/*.conf $(srctree)/$(src)/policy/*.conf.default) FORCE +$(obj)/builtin-policy.h: $(wildcard $(obj)/policy/*.conf $(src)/policy/*.conf.default) FORCE $(call if_changed,policy) ifndef CONFIG_SECURITY_TOMOYO_INSECURE_BUILTIN_SETTING diff --git a/usr/Makefile b/usr/Makefile index f8e1ad19e05c4..132ef7e96e6d5 100644 --- a/usr/Makefile +++ b/usr/Makefile @@ -22,7 +22,7 @@ cpio-data := # If CONFIG_INITRAMFS_SOURCE is empty, generate a small initramfs with the # default contents. ifeq ($(ramfs-input),) -ramfs-input := $(srctree)/$(src)/default_cpio_list +ramfs-input := $(src)/default_cpio_list endif ifeq ($(words $(ramfs-input)),1) diff --git a/usr/include/Makefile b/usr/include/Makefile index 338c81f1fcf31..771e32872b2ab 100644 --- a/usr/include/Makefile +++ b/usr/include/Makefile @@ -78,7 +78,7 @@ quiet_cmd_hdrtest = HDRTEST $< cmd_hdrtest = \ $(CC) $(c_flags) -fsyntax-only -x c /dev/null \ $(if $(filter-out $(no-header-test), $*.h), -include $< -include $<); \ - $(PERL) $(srctree)/$(src)/headers_check.pl $(obj) $(SRCARCH) $<; \ + $(PERL) $(src)/headers_check.pl $(obj) $(SRCARCH) $<; \ touch $@ $(obj)/%.hdrtest: $(obj)/%.h FORCE From 770202a2233f819bfc33844d860e7282e696a91c Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 28 Apr 2024 00:32:52 +0900 Subject: [PATCH 1504/1702] kbuild: remove redundant $(wildcard ) for rm-files The $(wildcard ) is called in quiet_cmd_rmfiles. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c97c8d96061c5..b73519492887f 100644 --- a/Makefile +++ b/Makefile @@ -1515,7 +1515,7 @@ clean: archclean vmlinuxclean resolve_btfids_clean # mrproper - Delete all generated files, including .config # -mrproper: rm-files := $(wildcard $(MRPROPER_FILES)) +mrproper: rm-files := $(MRPROPER_FILES) mrproper-dirs := $(addprefix _mrproper_,scripts) PHONY += $(mrproper-dirs) mrproper From d98dba8852592402b67b643015f64b394760daa9 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 28 Apr 2024 00:32:53 +0900 Subject: [PATCH 1505/1702] kbuild: add 'private' to target-specific variables Currently, Kbuild produces inconsistent results in some cases. You can do an interesting experiment using the --shuffle option, which is supported by GNU Make 4.4 or later. Set CONFIG_KVM_INTEL=y and CONFIG_KVM_AMD=m (or vice versa), and repeat incremental builds w/wo --shuffle=reverse. $ make [ snip ] CC arch/x86/kvm/kvm-asm-offsets.s $ make --shuffle=reverse [ snip ] CC [M] arch/x86/kvm/kvm-asm-offsets.s $ make [ snip ] CC arch/x86/kvm/kvm-asm-offsets.s arch/x86/kvm/kvm-asm-offsets.s is rebuilt every time w/wo the [M] marker. arch/x86/kvm/kvm-asm-offsets.s is built as built-in when it is built as a prerequisite of arch/x86/kvm/kvm-intel.o, which is built-in. arch/x86/kvm/kvm-asm-offsets.s is built as modular when it is built as a prerequisite of arch/x86/kvm/kvm-amd.o, which is a module. Another odd example is single target builds. When CONFIG_LKDTM=m, drivers/misc/lkdtm/rodata.o can be built as built-in or modular, depending on how it is built. $ make drivers/misc/lkdtm/lkdtm.o [ snip ] CC [M] drivers/misc/lkdtm/rodata.o $ make drivers/misc/lkdtm/rodata.o [ snip ] CC drivers/misc/lkdtm/rodata.o drivers/misc/lkdtm/rodata.o is built as modular when it is built as a prerequisite of another, but built as built-in when it is a final target. The same thing happens to drivers/memory/emif-asm-offsets.s when CONFIG_TI_EMIF_SRAM=m. $ make drivers/memory/ti-emif-sram.o [ snip ] CC [M] drivers/memory/emif-asm-offsets.s $ make drivers/memory/emif-asm-offsets.s [ snip ] CC drivers/memory/emif-asm-offsets.s This is because the part-of-module=y flag defined for the modules is inherited by its prerequisites. Target-specific variables are likely intended only for local use. This commit adds 'private' to them. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- Makefile | 10 +++++----- scripts/Makefile.build | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index b73519492887f..81cbe8f0659c0 100644 --- a/Makefile +++ b/Makefile @@ -1254,8 +1254,8 @@ define filechk_version.h echo \#define LINUX_VERSION_SUBLEVEL $(SUBLEVEL) endef -$(version_h): PATCHLEVEL := $(or $(PATCHLEVEL), 0) -$(version_h): SUBLEVEL := $(or $(SUBLEVEL), 0) +$(version_h): private PATCHLEVEL := $(or $(PATCHLEVEL), 0) +$(version_h): private SUBLEVEL := $(or $(SUBLEVEL), 0) $(version_h): FORCE $(call filechk,version.h) @@ -1503,7 +1503,7 @@ MRPROPER_FILES += include/config include/generated \ # clean - Delete most, but leave enough to build external modules # -clean: rm-files := $(CLEAN_FILES) +clean: private rm-files := $(CLEAN_FILES) PHONY += archclean vmlinuxclean @@ -1515,7 +1515,7 @@ clean: archclean vmlinuxclean resolve_btfids_clean # mrproper - Delete all generated files, including .config # -mrproper: rm-files := $(MRPROPER_FILES) +mrproper: private rm-files := $(MRPROPER_FILES) mrproper-dirs := $(addprefix _mrproper_,scripts) PHONY += $(mrproper-dirs) mrproper @@ -1792,7 +1792,7 @@ compile_commands.json: $(extmod_prefix)compile_commands.json PHONY += compile_commands.json clean-dirs := $(KBUILD_EXTMOD) -clean: rm-files := $(KBUILD_EXTMOD)/Module.symvers $(KBUILD_EXTMOD)/modules.nsdeps \ +clean: private rm-files := $(KBUILD_EXTMOD)/Module.symvers $(KBUILD_EXTMOD)/modules.nsdeps \ $(KBUILD_EXTMOD)/compile_commands.json PHONY += prepare diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 43b71c3d0ff62..c9c07a6144eb7 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -216,7 +216,7 @@ endif # CONFIG_FTRACE_MCOUNT_USE_RECORDMCOUNT is-standard-object = $(if $(filter-out y%, $(OBJECT_FILES_NON_STANDARD_$(target-stem).o)$(OBJECT_FILES_NON_STANDARD)n),y) -$(obj)/%.o: objtool-enabled = $(if $(is-standard-object),$(if $(delay-objtool),$(is-single-obj-m),y)) +$(obj)/%.o: private objtool-enabled = $(if $(is-standard-object),$(if $(delay-objtool),$(is-single-obj-m),y)) ifneq ($(findstring 1, $(KBUILD_EXTRA_WARN)),) cmd_warn_shared_object = $(if $(word 2, $(modname-multi)),$(warning $(kbuild-file): $*.o is added to multiple modules: $(modname-multi))) @@ -437,8 +437,8 @@ define rule_ld_multi_m $(call cmd,gen_objtooldep) endef -$(multi-obj-m): objtool-enabled := $(delay-objtool) -$(multi-obj-m): part-of-module := y +$(multi-obj-m): private objtool-enabled := $(delay-objtool) +$(multi-obj-m): private part-of-module := y $(multi-obj-m): %.o: %.mod FORCE $(call if_changed_rule,ld_multi_m) $(call multi_depend, $(multi-obj-m), .o, -objs -y -m) From 1c369b6c9492756c58ad4abb92a9433a59f31f7d Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Tue, 30 Apr 2024 00:07:54 +0900 Subject: [PATCH 1506/1702] kbuild: simplify generic vdso installation code With commit 4b0bf9a01270 ("riscv: compat_vdso: install compat_vdso.so.dbg to /lib/modules/*/vdso/") applied, all debug VDSO files are installed in $(MODLIB)/vdso/. Simplify the installation rule. Signed-off-by: Masahiro Yamada --- scripts/Makefile.vdsoinst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/Makefile.vdsoinst b/scripts/Makefile.vdsoinst index c477d17b0aa5b..bf72880c50d00 100644 --- a/scripts/Makefile.vdsoinst +++ b/scripts/Makefile.vdsoinst @@ -13,16 +13,15 @@ install-dir := $(MODLIB)/vdso define gen_install_rules -src := $$(firstword $$(subst :,$(space),$(1))) -dest := $(install-dir)/$$(or $$(word 2,$$(subst :,$(space),$(1))),$$(patsubst %.dbg,%,$$(notdir $(1)))) +dest := $(install-dir)/$$(patsubst %.dbg,%,$$(notdir $(1))) __default: $$(dest) -$$(dest): $$(src) FORCE +$$(dest): $(1) FORCE $$(call cmd,install) # Some architectures create .build-id symlinks ifneq ($(filter arm sparc x86, $(SRCARCH)),) -link := $(install-dir)/.build-id/$$(shell $(READELF) -n $$(src) | sed -n 's@^.*Build ID: \(..\)\(.*\)@\1/\2@p').debug +link := $(install-dir)/.build-id/$$(shell $(READELF) -n $(1) | sed -n 's@^.*Build ID: \(..\)\(.*\)@\1/\2@p').debug __default: $$(link) $$(link): $$(dest) FORCE From a0b49a9102019c790cbe3f102b1f4361342f8c70 Mon Sep 17 00:00:00 2001 From: Emil Renner Berthing Date: Thu, 2 May 2024 13:16:08 +0200 Subject: [PATCH 1507/1702] kbuild: buildtar: install riscv compressed images as vmlinuz Use the KBUILD_IMAGE variable to determine the right kernel image to install and install compressed images to /boot/vmlinuz-$version like the 'make install' target already does. Signed-off-by: Emil Renner Berthing Signed-off-by: Masahiro Yamada --- scripts/package/buildtar | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/scripts/package/buildtar b/scripts/package/buildtar index fe816f62a290c..eb67787f86733 100755 --- a/scripts/package/buildtar +++ b/scripts/package/buildtar @@ -54,9 +54,8 @@ cp -v -- "${objtree}/vmlinux" "${tmpdir}/boot/vmlinux-${KERNELRELEASE}" # Install arch-specific kernel image(s) # # Note: -# mips, arm64, and riscv copy the first image found. This may not produce -# the desired outcome because it may pick up a stale file remaining in the -# build tree. +# mips and arm64 copy the first image found. This may not produce the desired +# outcome because it may pick up a stale file remaining in the build tree. # case "${ARCH}" in alpha) @@ -98,13 +97,12 @@ case "${ARCH}" in done ;; riscv) - # Please note the following code may copy a stale file. - for i in Image.bz2 Image.gz Image; do - if [ -f "${objtree}/arch/riscv/boot/${i}" ] ; then - cp -v -- "${objtree}/arch/riscv/boot/${i}" "${tmpdir}/boot/vmlinux-${KERNELRELEASE}" - break - fi - done + case "${KBUILD_IMAGE##*/}" in + Image.*|vmlinuz.efi) + cp -v -- "${KBUILD_IMAGE}" "${tmpdir}/boot/vmlinuz-${KERNELRELEASE}";; + *) + cp -v -- "${KBUILD_IMAGE}" "${tmpdir}/boot/vmlinux-${KERNELRELEASE}";; + esac ;; *) cp -v -- "${KBUILD_IMAGE}" "${tmpdir}/boot/vmlinuz-${KERNELRELEASE}" From 4763175ad2be3395bb23880a4cc9c48cf823dc40 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 5 May 2024 03:33:24 +0900 Subject: [PATCH 1508/1702] kconfig: gconf: update pane correctly after loading a config file Every time a config file is loaded (either by clicking the "Load" button or selecting "File" -> "Load" from the menu), a new list is appended to the pane. The current tree needs to be cleared by calling gtk_tree_store_clear(). Signed-off-by: Masahiro Yamada --- scripts/kconfig/gconf.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/kconfig/gconf.c b/scripts/kconfig/gconf.c index 67a27c497c403..12b55f2e369bf 100644 --- a/scripts/kconfig/gconf.c +++ b/scripts/kconfig/gconf.c @@ -487,7 +487,7 @@ load_filename(GtkFileSelection * file_selector, gpointer user_data) if (conf_read(fn)) text_insert_msg("Error", "Unable to load configuration !"); else - display_tree(&rootmenu); + display_tree_part(); } void on_load1_activate(GtkMenuItem * menuitem, gpointer user_data) @@ -1399,6 +1399,8 @@ static void display_tree_part(void) display_tree(current); else if (view_mode == SPLIT_VIEW) display_tree(browsed); + else if (view_mode == FULL_VIEW) + display_tree(&rootmenu); gtk_tree_view_expand_all(GTK_TREE_VIEW(tree2_w)); } From a7efb160f6c11ec1a72b8f69bd844e01753c769d Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 5 May 2024 03:33:25 +0900 Subject: [PATCH 1509/1702] kconfig: gconf: remove debug code This is not so useful. If necessary, you can insert printf() or whatever during debugging. Signed-off-by: Masahiro Yamada --- scripts/kconfig/gconf.c | 49 +---------------------------------------- 1 file changed, 1 insertion(+), 48 deletions(-) diff --git a/scripts/kconfig/gconf.c b/scripts/kconfig/gconf.c index 12b55f2e369bf..89614f1e49e65 100644 --- a/scripts/kconfig/gconf.c +++ b/scripts/kconfig/gconf.c @@ -18,8 +18,6 @@ #include #include -//#define DEBUG - enum { SINGLE_VIEW, SPLIT_VIEW, FULL_VIEW }; @@ -71,33 +69,6 @@ static void set_node(GtkTreeIter * node, struct menu *menu, gchar ** row); static gchar **fill_row(struct menu *menu); static void conf_changed(void); -/* Helping/Debugging Functions */ -#ifdef DEBUG -static const char *dbg_sym_flags(int val) -{ - static char buf[256]; - - bzero(buf, 256); - - if (val & SYMBOL_CONST) - strcat(buf, "const/"); - if (val & SYMBOL_CHECK) - strcat(buf, "check/"); - if (val & SYMBOL_CHOICEVAL) - strcat(buf, "choiceval/"); - if (val & SYMBOL_VALID) - strcat(buf, "valid/"); - if (val & SYMBOL_WRITE) - strcat(buf, "write/"); - if (val & SYMBOL_CHANGED) - strcat(buf, "changed/"); - - buf[strlen(buf) - 1] = '\0'; - - return buf; -} -#endif - static void replace_button_icon(GladeXML *xml, GdkDrawable *window, GtkStyle *style, gchar *btn_name, gchar **xpm) { @@ -1262,12 +1233,6 @@ static void update_tree(struct menu *src, GtkTreeIter * dst) else menu2 = NULL; // force adding of a first child -#ifdef DEBUG - printf("%*c%s | %s\n", indent, ' ', - menu1 ? menu_get_prompt(menu1) : "nil", - menu2 ? menu_get_prompt(menu2) : "nil"); -#endif - if ((opt_mode == OPT_NORMAL && !menu_is_visible(child1)) || (opt_mode == OPT_PROMPT && !menu_has_prompt(child1)) || (opt_mode == OPT_ALL && !menu_get_prompt(child1))) { @@ -1354,19 +1319,7 @@ static void display_tree(struct menu *menu) (opt_mode == OPT_PROMPT && menu_has_prompt(child)) || (opt_mode == OPT_ALL && menu_get_prompt(child))) place_node(child, fill_row(child)); -#ifdef DEBUG - printf("%*c%s: ", indent, ' ', menu_get_prompt(child)); - printf("%s", child->flags & MENU_ROOT ? "rootmenu | " : ""); - printf("%s", prop_get_type_name(ptype)); - printf(" | "); - if (sym) { - printf("%s", sym_type_name(sym->type)); - printf(" | "); - printf("%s", dbg_sym_flags(sym->flags)); - printf("\n"); - } else - printf("\n"); -#endif + if ((view_mode != FULL_VIEW) && (ptype == P_MENU) && (tree == tree2)) continue; From 4cc7e6cef3461d7921a6dcf8873f98501209c3a9 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 5 May 2024 03:33:26 +0900 Subject: [PATCH 1510/1702] kconfig: gconf: use MENU_CHANGED instead of SYMBOL_CHANGED SYMBOL_CHANGED and MENU_CHANGED are used to update GUI frontends when the symbol value is changed. These are used inconsistently: SYMBOL_CHANGED in gconf.c and MENU_CHANGE in qconf.cc. MENU_CHANGED works more properly when a symbol has multiple prompts (although such code is not ideal). [test code] config FOO bool "foo prompt 1" config FOO bool "foo prompt 2" In gconfig, if one of the two checkboxes is clicked, only the first one is toggled. In xconfig, the two checkboxes work in sync. Replace SYMBOL_CHANGED in gconf.c with MENU_CHANGED to align with the xconfig behavior. Signed-off-by: Masahiro Yamada --- scripts/kconfig/gconf.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/kconfig/gconf.c b/scripts/kconfig/gconf.c index 89614f1e49e65..10d602faa51e1 100644 --- a/scripts/kconfig/gconf.c +++ b/scripts/kconfig/gconf.c @@ -1047,7 +1047,7 @@ static gchar **fill_row(struct menu *menu) row[COL_NAME] = g_strdup(sym->name); sym_calc_value(sym); - sym->flags &= ~SYMBOL_CHANGED; + menu->flags &= ~MENU_CHANGED; if (sym_is_choice(sym)) { // parse childs for getting final value struct menu *child; @@ -1273,7 +1273,7 @@ static void update_tree(struct menu *src, GtkTreeIter * dst) else goto reparse; // next child } - } else if (sym && (sym->flags & SYMBOL_CHANGED)) { + } else if (sym && (child1->flags & MENU_CHANGED)) { set_node(child2, menu1, fill_row(menu1)); } @@ -1289,7 +1289,6 @@ static void update_tree(struct menu *src, GtkTreeIter * dst) /* Display the whole tree (single/split/full view) */ static void display_tree(struct menu *menu) { - struct symbol *sym; struct property *prop; struct menu *child; enum prop_type ptype; @@ -1301,11 +1300,9 @@ static void display_tree(struct menu *menu) for (child = menu->list; child; child = child->next) { prop = child->prompt; - sym = child->sym; ptype = prop ? prop->type : P_UNKNOWN; - if (sym) - sym->flags &= ~SYMBOL_CHANGED; + menu->flags &= ~MENU_CHANGED; if ((view_mode == SPLIT_VIEW) && !(child->flags & MENU_ROOT) && (tree == tree1)) From 7d2806746d4636320e8e061a68eed32493e7c284 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 5 May 2024 03:33:27 +0900 Subject: [PATCH 1511/1702] kconfig: use linked list in sym_set_changed() Following the approach employed in commit bedf92362317 ("kconfig: use linked list in get_symbol_str() to iterate over menus"), simplify the iteration on the menus of the specified symbol. Signed-off-by: Masahiro Yamada --- scripts/kconfig/symbol.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/kconfig/symbol.c b/scripts/kconfig/symbol.c index b909c64f3bac8..8512c29c84bb8 100644 --- a/scripts/kconfig/symbol.c +++ b/scripts/kconfig/symbol.c @@ -152,13 +152,11 @@ static void sym_validate_range(struct symbol *sym) static void sym_set_changed(struct symbol *sym) { - struct property *prop; + struct menu *menu; sym->flags |= SYMBOL_CHANGED; - for (prop = sym->prop; prop; prop = prop->next) { - if (prop->menu) - prop->menu->flags |= MENU_CHANGED; - } + list_for_each_entry(menu, &sym->menus, link) + menu->flags |= MENU_CHANGED; } static void sym_set_all_changed(void) From 8c00e58005e33068f29b8f9c84436ccbe73bef7c Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 5 May 2024 03:33:31 +0900 Subject: [PATCH 1512/1702] kconfig: turn conf_choice() into void function The return value of conf_choice() is not used. Signed-off-by: Masahiro Yamada --- scripts/kconfig/conf.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/kconfig/conf.c b/scripts/kconfig/conf.c index 0156ca13f949c..8ad2c52d9b1f9 100644 --- a/scripts/kconfig/conf.c +++ b/scripts/kconfig/conf.c @@ -446,7 +446,7 @@ static int conf_sym(struct menu *menu) } } -static int conf_choice(struct menu *menu) +static void conf_choice(struct menu *menu) { struct symbol *sym, *def_sym; struct menu *child; @@ -459,19 +459,18 @@ static int conf_choice(struct menu *menu) sym_calc_value(sym); switch (sym_get_tristate_value(sym)) { case no: - return 1; case mod: - return 0; + return; case yes: break; } } else { switch (sym_get_tristate_value(sym)) { case no: - return 1; + return; case mod: printf("%*s%s\n", indent - 1, "", menu_get_prompt(menu)); - return 0; + return; case yes: break; } @@ -551,7 +550,7 @@ static int conf_choice(struct menu *menu) continue; } sym_set_tristate_value(child->sym, yes); - return 1; + return; } } From 700e7a8d05ea690308131f69ffc597dfab6db838 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 5 May 2024 03:33:32 +0900 Subject: [PATCH 1513/1702] kconfig: turn missing prompt for choice members into error Choice members must have a prompt; hence make it an error. While I was here, I moved the check to the parser to slim down _menu_finalize(). Signed-off-by: Masahiro Yamada --- scripts/kconfig/menu.c | 2 -- scripts/kconfig/parser.y | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/kconfig/menu.c b/scripts/kconfig/menu.c index e01b9ee87c05b..a9b1e451dfe77 100644 --- a/scripts/kconfig/menu.c +++ b/scripts/kconfig/menu.c @@ -507,8 +507,6 @@ static void _menu_finalize(struct menu *parent, bool inside_choice) menu->sym && !sym_is_choice_value(menu->sym)) { current_entry = menu; menu->sym->flags |= SYMBOL_CHOICEVAL; - if (!menu->prompt) - menu_warn(menu, "choice value must have a prompt"); for (prop = menu->sym->prop; prop; prop = prop->next) { if (prop->type == P_DEFAULT) prop_warn(prop, "defaults for choice " diff --git a/scripts/kconfig/parser.y b/scripts/kconfig/parser.y index 613fa8c9c2d04..ed86869e5ed00 100644 --- a/scripts/kconfig/parser.y +++ b/scripts/kconfig/parser.y @@ -30,6 +30,8 @@ static bool zconf_endtoken(const char *tokenname, struct menu *current_menu, *current_entry; +static bool inside_choice = false; + %} %union @@ -145,6 +147,14 @@ config_entry_start: T_CONFIG nonconst_symbol T_EOL config_stmt: config_entry_start config_option_list { + if (inside_choice) { + if (!current_entry->prompt) { + fprintf(stderr, "%s:%d: error: choice member must have a prompt\n", + current_entry->filename, current_entry->lineno); + yynerrs++; + } + } + printd(DEBUG_PARSE, "%s:%d:endconfig\n", cur_filename, cur_lineno); }; @@ -237,10 +247,14 @@ choice_entry: choice choice_option_list } $$ = menu_add_menu(); + + inside_choice = true; }; choice_end: end { + inside_choice = false; + if (zconf_endtoken($1, "choice")) { menu_end_menu(); printd(DEBUG_PARSE, "%s:%d:endchoice\n", cur_filename, cur_lineno); From 8a22f867e330965539b9cb5ccc42c2b470330b43 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 5 May 2024 03:33:33 +0900 Subject: [PATCH 1514/1702] kconfig: turn defaults and additional prompt for choice members into error menu_finalize() warns default properties for choice members and prompts outside the choice block. These should be hard errors. While I was here, I moved the checks to slim down menu_finalize(). Signed-off-by: Masahiro Yamada --- scripts/kconfig/menu.c | 10 ---------- scripts/kconfig/parser.y | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/scripts/kconfig/menu.c b/scripts/kconfig/menu.c index a9b1e451dfe77..bee96c9964fd8 100644 --- a/scripts/kconfig/menu.c +++ b/scripts/kconfig/menu.c @@ -507,16 +507,6 @@ static void _menu_finalize(struct menu *parent, bool inside_choice) menu->sym && !sym_is_choice_value(menu->sym)) { current_entry = menu; menu->sym->flags |= SYMBOL_CHOICEVAL; - for (prop = menu->sym->prop; prop; prop = prop->next) { - if (prop->type == P_DEFAULT) - prop_warn(prop, "defaults for choice " - "values not supported"); - if (prop->menu == menu) - continue; - if (prop->type == P_PROMPT && - prop->menu->parent->sym != sym) - prop_warn(prop, "choice value used outside its choice group"); - } /* Non-tristate choice values of tristate choices must * depend on the choice being set to Y. The choice * values' dependencies were propagated to their diff --git a/scripts/kconfig/parser.y b/scripts/kconfig/parser.y index ed86869e5ed00..ff709001b1f02 100644 --- a/scripts/kconfig/parser.y +++ b/scripts/kconfig/parser.y @@ -476,6 +476,38 @@ assign_val: %% +/** + * choice_check_sanity - check sanity of a choice member + * + * @menu: menu of the choice member + * + * Return: -1 if an error is found, 0 otherwise. + */ +static int choice_check_sanity(struct menu *menu) +{ + struct property *prop; + int ret = 0; + + for (prop = menu->sym->prop; prop; prop = prop->next) { + if (prop->type == P_DEFAULT) { + fprintf(stderr, "%s:%d: error: %s", + prop->filename, prop->lineno, + "defaults for choice values not supported\n"); + ret = -1; + } + + if (prop->menu != menu && prop->type == P_PROMPT && + prop->menu->parent != menu->parent) { + fprintf(stderr, "%s:%d: error: %s", + prop->filename, prop->lineno, + "choice value has a prompt outside its choice group\n"); + ret = -1; + } + } + + return ret; +} + void conf_parse(const char *name) { struct menu *menu; @@ -523,8 +555,16 @@ void conf_parse(const char *name) menu_finalize(); menu_for_each_entry(menu) { + struct menu *child; + if (menu->sym && sym_check_deps(menu->sym)) yynerrs++; + + if (menu->sym && sym_is_choice(menu->sym)) { + menu_for_each_sub_entry(child, menu) + if (child->sym && choice_check_sanity(child)) + yynerrs++; + } } if (yynerrs) From 7bcf2e03b50256d13bcc1b08a43af9762bafbc0e Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 5 May 2024 03:33:28 +0900 Subject: [PATCH 1515/1702] kconfig: add sym_get_choice_menu() helper Choices and their members are associated via the P_CHOICE property. Currently, prop_get_symbol(sym_get_choice_prop()) is used to obtain the choice of the given choice member. We can do this without relying on P_CHOICE by checking the parent in the menu structure. Introduce a new helper to retrieve the choice if the given symbol is a choice member. This is intended to replace prop_get_symbol(sym_get_choice_prop()) and deprecate P_CHOICE eventually. Signed-off-by: Masahiro Yamada --- scripts/kconfig/lkc_proto.h | 1 + scripts/kconfig/symbol.c | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/scripts/kconfig/lkc_proto.h b/scripts/kconfig/lkc_proto.h index 2807fa584c2b0..d76aaf4ea1176 100644 --- a/scripts/kconfig/lkc_proto.h +++ b/scripts/kconfig/lkc_proto.h @@ -34,6 +34,7 @@ bool sym_string_within_range(struct symbol *sym, const char *str); bool sym_set_string_value(struct symbol *sym, const char *newval); bool sym_is_changeable(struct symbol *sym); struct property * sym_get_choice_prop(struct symbol *sym); +struct menu *sym_get_choice_menu(struct symbol *sym); const char * sym_get_string_value(struct symbol *sym); const char * prop_get_type_name(enum prop_type type); diff --git a/scripts/kconfig/symbol.c b/scripts/kconfig/symbol.c index 8512c29c84bb8..23829f44b8f89 100644 --- a/scripts/kconfig/symbol.c +++ b/scripts/kconfig/symbol.c @@ -78,6 +78,41 @@ struct property *sym_get_choice_prop(struct symbol *sym) return NULL; } +/** + * sym_get_choice_menu - get the parent choice menu if present + * + * @sym: a symbol pointer + * + * Return: a choice menu if this function is called against a choice member. + */ +struct menu *sym_get_choice_menu(struct symbol *sym) +{ + struct menu *menu = NULL; + struct menu *m; + + /* + * Choice members must have a prompt. Find a menu entry with a prompt, + * and assume it resides inside a choice block. + */ + list_for_each_entry(m, &sym->menus, link) + if (m->prompt) { + menu = m; + break; + } + + if (!menu) + return NULL; + + do { + menu = menu->parent; + } while (menu && !menu->sym); + + if (menu && menu->sym && sym_is_choice(menu->sym)) + return menu; + + return NULL; +} + static struct property *sym_get_default_prop(struct symbol *sym) { struct property *prop; From fb8dd48214b0234ba03a808742f69690d9a9f500 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 5 May 2024 03:33:29 +0900 Subject: [PATCH 1516/1702] kconfig: use sym_get_choice_menu() in conf_write_defconfig() Choices and their members are associated via the P_CHOICE property. Currently, prop_get_symbol(sym_get_choice_prop()) is used to obtain the choice of the given choice member. Replace it with sym_get_choice_menu(), which retrieves the choice without relying on P_CHOICE. Signed-off-by: Masahiro Yamada --- scripts/kconfig/confdata.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/kconfig/confdata.c b/scripts/kconfig/confdata.c index 5caec434e6f4a..387503daf0f73 100644 --- a/scripts/kconfig/confdata.c +++ b/scripts/kconfig/confdata.c @@ -794,6 +794,8 @@ int conf_write_defconfig(const char *filename) sym_clear_all_valid(); menu_for_each_entry(menu) { + struct menu *choice; + sym = menu->sym; if (sym && !sym_is_choice(sym)) { sym_calc_value(sym); @@ -811,12 +813,11 @@ int conf_write_defconfig(const char *filename) * If symbol is a choice value and equals to the * default for a choice - skip. */ - if (sym_is_choice_value(sym)) { - struct symbol *cs; + choice = sym_get_choice_menu(sym); + if (choice) { struct symbol *ds; - cs = prop_get_symbol(sym_get_choice_prop(sym)); - ds = sym_choice_default(cs); + ds = sym_choice_default(choice->sym); if (sym == ds) { if ((sym->type == S_BOOLEAN) && sym_get_tristate_value(sym) == yes) From dfff05cc10052b4c36a11d268aeaab2ed6ca66a0 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 5 May 2024 03:33:30 +0900 Subject: [PATCH 1517/1702] kconfig: use menu_list_for_each_sym() in sym_check_choice_deps() Choices and their members are associated via the P_CHOICE property. Currently, sym_get_choice_prop() and expr_list_for_each_sym() are used to iterate on choice members. Replace them with menu_for_each_sub_entry(), which achieves the same without relying on P_CHOICE. Signed-off-by: Masahiro Yamada --- scripts/kconfig/symbol.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/scripts/kconfig/symbol.c b/scripts/kconfig/symbol.c index 23829f44b8f89..aa0e25ee5119e 100644 --- a/scripts/kconfig/symbol.c +++ b/scripts/kconfig/symbol.c @@ -1204,16 +1204,18 @@ static struct symbol *sym_check_sym_deps(struct symbol *sym) static struct symbol *sym_check_choice_deps(struct symbol *choice) { - struct symbol *sym, *sym2; - struct property *prop; - struct expr *e; + struct menu *choice_menu, *menu; + struct symbol *sym2; struct dep_stack stack; dep_stack_insert(&stack, choice); - prop = sym_get_choice_prop(choice); - expr_list_for_each_sym(prop->expr, e, sym) - sym->flags |= (SYMBOL_CHECK | SYMBOL_CHECKED); + choice_menu = list_first_entry(&choice->menus, struct menu, link); + + menu_for_each_sub_entry(menu, choice_menu) { + if (menu->sym) + menu->sym->flags |= SYMBOL_CHECK | SYMBOL_CHECKED; + } choice->flags |= (SYMBOL_CHECK | SYMBOL_CHECKED); sym2 = sym_check_sym_deps(choice); @@ -1221,14 +1223,17 @@ static struct symbol *sym_check_choice_deps(struct symbol *choice) if (sym2) goto out; - expr_list_for_each_sym(prop->expr, e, sym) { - sym2 = sym_check_sym_deps(sym); + menu_for_each_sub_entry(menu, choice_menu) { + if (!menu->sym) + continue; + sym2 = sym_check_sym_deps(menu->sym); if (sym2) break; } out: - expr_list_for_each_sym(prop->expr, e, sym) - sym->flags &= ~SYMBOL_CHECK; + menu_for_each_sub_entry(menu, choice_menu) + if (menu->sym) + menu->sym->flags &= ~SYMBOL_CHECK; if (sym2 && sym_is_choice_value(sym2) && prop_get_symbol(sym_get_choice_prop(sym2)) == choice) From b6fb4269e707bb410e202fb160c8d7a188ef60f8 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 8 May 2024 01:42:10 -0400 Subject: [PATCH 1518/1702] bcachefs: for_each_bset() declares loop iter Signed-off-by: Kent Overstreet --- fs/bcachefs/bset.c | 29 ++++++++++------------------- fs/bcachefs/bset.h | 6 ++++-- fs/bcachefs/btree_cache.c | 3 --- fs/bcachefs/btree_io.c | 7 ------- fs/bcachefs/btree_io.h | 2 -- fs/bcachefs/btree_update_interior.c | 1 - 6 files changed, 14 insertions(+), 34 deletions(-) diff --git a/fs/bcachefs/bset.c b/fs/bcachefs/bset.c index 3bb477840eab6..575e1d0b6eeb2 100644 --- a/fs/bcachefs/bset.c +++ b/fs/bcachefs/bset.c @@ -103,8 +103,6 @@ void bch2_dump_bset(struct bch_fs *c, struct btree *b, void bch2_dump_btree_node(struct bch_fs *c, struct btree *b) { - struct bset_tree *t; - console_lock(); for_each_bset(b, t) bch2_dump_bset(c, b, bset(b, t), t - b->set); @@ -136,7 +134,6 @@ void bch2_dump_btree_node_iter(struct btree *b, struct btree_nr_keys bch2_btree_node_count_keys(struct btree *b) { - struct bset_tree *t; struct bkey_packed *k; struct btree_nr_keys nr = {}; @@ -198,7 +195,6 @@ void bch2_btree_node_iter_verify(struct btree_node_iter *iter, { struct btree_node_iter_set *set, *s2; struct bkey_packed *k, *p; - struct bset_tree *t; if (bch2_btree_node_iter_end(iter)) return; @@ -213,12 +209,14 @@ void bch2_btree_node_iter_verify(struct btree_node_iter *iter, /* Verify that set->end is correct: */ btree_node_iter_for_each(iter, set) { for_each_bset(b, t) - if (set->end == t->end_offset) + if (set->end == t->end_offset) { + BUG_ON(set->k < btree_bkey_first_offset(t) || + set->k >= t->end_offset); goto found; + } BUG(); found: - BUG_ON(set->k < btree_bkey_first_offset(t) || - set->k >= t->end_offset); + do {} while (0); } /* Verify iterator is sorted: */ @@ -377,11 +375,9 @@ static struct bkey_float *bkey_float(const struct btree *b, return ro_aux_tree_base(b, t)->f + idx; } -static void bset_aux_tree_verify(const struct btree *b) +static void bset_aux_tree_verify(struct btree *b) { #ifdef CONFIG_BCACHEFS_DEBUG - const struct bset_tree *t; - for_each_bset(b, t) { if (t->aux_data_offset == U16_MAX) continue; @@ -685,20 +681,20 @@ static __always_inline void make_bfloat(struct btree *b, struct bset_tree *t, } /* bytes remaining - only valid for last bset: */ -static unsigned __bset_tree_capacity(const struct btree *b, const struct bset_tree *t) +static unsigned __bset_tree_capacity(struct btree *b, const struct bset_tree *t) { bset_aux_tree_verify(b); return btree_aux_data_bytes(b) - t->aux_data_offset * sizeof(u64); } -static unsigned bset_ro_tree_capacity(const struct btree *b, const struct bset_tree *t) +static unsigned bset_ro_tree_capacity(struct btree *b, const struct bset_tree *t) { return __bset_tree_capacity(b, t) / (sizeof(struct bkey_float) + sizeof(u8)); } -static unsigned bset_rw_tree_capacity(const struct btree *b, const struct bset_tree *t) +static unsigned bset_rw_tree_capacity(struct btree *b, const struct bset_tree *t) { return __bset_tree_capacity(b, t) / sizeof(struct rw_aux_tree); } @@ -1374,8 +1370,6 @@ void bch2_btree_node_iter_init(struct btree_node_iter *iter, void bch2_btree_node_iter_init_from_start(struct btree_node_iter *iter, struct btree *b) { - struct bset_tree *t; - memset(iter, 0, sizeof(*iter)); for_each_bset(b, t) @@ -1481,7 +1475,6 @@ struct bkey_packed *bch2_btree_node_iter_prev_all(struct btree_node_iter *iter, { struct bkey_packed *k, *prev = NULL; struct btree_node_iter_set *set; - struct bset_tree *t; unsigned end = 0; if (bch2_expensive_debug_checks) @@ -1550,9 +1543,7 @@ struct bkey_s_c bch2_btree_node_iter_peek_unpack(struct btree_node_iter *iter, void bch2_btree_keys_stats(const struct btree *b, struct bset_stats *stats) { - const struct bset_tree *t; - - for_each_bset(b, t) { + for_each_bset_c(b, t) { enum bset_aux_tree_type type = bset_aux_tree_type(t); size_t j; diff --git a/fs/bcachefs/bset.h b/fs/bcachefs/bset.h index 120a79fd456bd..5c6c7a14fa0f4 100644 --- a/fs/bcachefs/bset.h +++ b/fs/bcachefs/bset.h @@ -206,7 +206,10 @@ static inline size_t btree_aux_data_u64s(const struct btree *b) } #define for_each_bset(_b, _t) \ - for (_t = (_b)->set; _t < (_b)->set + (_b)->nsets; _t++) + for (struct bset_tree *_t = (_b)->set; _t < (_b)->set + (_b)->nsets; _t++) + +#define for_each_bset_c(_b, _t) \ + for (const struct bset_tree *_t = (_b)->set; _t < (_b)->set + (_b)->nsets; _t++) #define bset_tree_for_each_key(_b, _t, _k) \ for (_k = btree_bkey_first(_b, _t); \ @@ -294,7 +297,6 @@ static inline struct bset_tree * bch2_bkey_to_bset_inlined(struct btree *b, struct bkey_packed *k) { unsigned offset = __btree_node_key_to_offset(b, k); - struct bset_tree *t; for_each_bset(b, t) if (offset <= t->end_offset) { diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c index 2c226e0cc2630..7dd35d6224d94 100644 --- a/fs/bcachefs/btree_cache.c +++ b/fs/bcachefs/btree_cache.c @@ -881,7 +881,6 @@ static struct btree *__bch2_btree_node_get(struct btree_trans *trans, struct btr struct bch_fs *c = trans->c; struct btree_cache *bc = &c->btree_cache; struct btree *b; - struct bset_tree *t; bool need_relock = false; int ret; @@ -1001,7 +1000,6 @@ struct btree *bch2_btree_node_get(struct btree_trans *trans, struct btree_path * { struct bch_fs *c = trans->c; struct btree *b; - struct bset_tree *t; int ret; EBUG_ON(level >= BTREE_MAX_DEPTH); @@ -1078,7 +1076,6 @@ struct btree *bch2_btree_node_get_noiter(struct btree_trans *trans, struct bch_fs *c = trans->c; struct btree_cache *bc = &c->btree_cache; struct btree *b; - struct bset_tree *t; int ret; EBUG_ON(level >= BTREE_MAX_DEPTH); diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c index 9dcaccda7b935..e14c404d25c12 100644 --- a/fs/bcachefs/btree_io.c +++ b/fs/bcachefs/btree_io.c @@ -229,7 +229,6 @@ static bool should_compact_bset(struct btree *b, struct bset_tree *t, static bool bch2_drop_whiteouts(struct btree *b, enum compact_mode mode) { - struct bset_tree *t; bool ret = false; for_each_bset(b, t) { @@ -451,8 +450,6 @@ static bool btree_node_compact(struct bch_fs *c, struct btree *b) void bch2_btree_build_aux_trees(struct btree *b) { - struct bset_tree *t; - for_each_bset(b, t) bch2_bset_build_aux_tree(b, t, !bset_written(b, bset(b, t)) && @@ -637,8 +634,6 @@ static int __btree_err(int ret, __cold void bch2_btree_node_drop_keys_outside_node(struct btree *b) { - struct bset_tree *t; - for_each_bset(b, t) { struct bset *i = bset(b, t); struct bkey_packed *k; @@ -1987,7 +1982,6 @@ static void btree_write_submit(struct work_struct *work) void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, unsigned flags) { struct btree_write_bio *wbio; - struct bset_tree *t; struct bset *i; struct btree_node *bn = NULL; struct btree_node_entry *bne = NULL; @@ -2244,7 +2238,6 @@ bool bch2_btree_post_write_cleanup(struct bch_fs *c, struct btree *b) { bool invalidated_iter = false; struct btree_node_entry *bne; - struct bset_tree *t; if (!btree_node_just_written(b)) return false; diff --git a/fs/bcachefs/btree_io.h b/fs/bcachefs/btree_io.h index e251cb6b965ff..2b8b564fc5608 100644 --- a/fs/bcachefs/btree_io.h +++ b/fs/bcachefs/btree_io.h @@ -81,8 +81,6 @@ static inline bool should_compact_bset_lazy(struct btree *b, static inline bool bch2_maybe_compact_whiteouts(struct bch_fs *c, struct btree *b) { - struct bset_tree *t; - for_each_bset(b, t) if (should_compact_bset_lazy(b, t)) return bch2_compact_whiteouts(c, b, COMPACT_LAZY); diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index 80eb862a89620..60b8544cea48c 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -160,7 +160,6 @@ int bch2_btree_node_check_topology(struct btree_trans *trans, struct btree *b) static void __bch2_btree_calc_format(struct bkey_format_state *s, struct btree *b) { struct bkey_packed *k; - struct bset_tree *t; struct bkey uk; for_each_bset(b, t) From 0e57996c6964027e36a854247bfd266d207314c7 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 3 May 2024 12:43:31 -0400 Subject: [PATCH 1519/1702] bcachefs: bch2_dev_get_ioref2(); alloc_background.c Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 19 ++++++++----------- fs/bcachefs/sb-members.h | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 71b727f5d432c..b9ded84d2d94e 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -1657,9 +1657,8 @@ static void discard_buckets_next_dev(struct bch_fs *c, struct discard_buckets_st bch2_dev_usage_read(s->ca).d[BCH_DATA_free].buckets) bch2_journal_flush_async(&c->journal, NULL); - bch2_dev_put(s->ca); - if (ca) - bch2_dev_get(ca); + if (s->ca) + percpu_ref_put(&s->ca->io_ref); s->ca = ca; s->need_journal_commit_this_dev = 0; } @@ -1673,15 +1672,15 @@ static int bch2_discard_one_bucket(struct btree_trans *trans, struct bpos pos = need_discard_iter->pos; struct btree_iter iter = { NULL }; struct bkey_s_c k; - struct bch_dev *ca; struct bkey_i_alloc_v4 *a; struct printbuf buf = PRINTBUF; bool discard_locked = false; int ret = 0; - ca = bch2_dev_bkey_exists(c, pos.inode); - - if (!percpu_ref_tryget(&ca->io_ref)) { + struct bch_dev *ca = s->ca && s->ca->dev_idx == pos.inode + ? s->ca + : bch2_dev_get_ioref2(c, pos.inode, WRITE); + if (!ca) { bch2_btree_iter_set_pos(need_discard_iter, POS(pos.inode + 1, 0)); return 0; } @@ -1787,7 +1786,6 @@ static int bch2_discard_one_bucket(struct btree_trans *trans, discard_in_flight_remove(c, iter.pos); s->seen++; bch2_trans_iter_exit(trans, &iter); - percpu_ref_put(&ca->io_ref); printbuf_exit(&buf); return ret; } @@ -1862,9 +1860,8 @@ static void bch2_do_discards_fast_work(struct work_struct *work) if (i->snapshot) continue; - ca = bch2_dev_bkey_exists(c, i->inode); - - if (!percpu_ref_tryget(&ca->io_ref)) { + ca = bch2_dev_get_ioref2(c, i->inode, WRITE); + if (!ca) { darray_remove_item(&c->discard_buckets_in_flight, i); continue; } diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index 7cecac44ab80a..f2a7794b0b500 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -285,6 +285,24 @@ static inline struct bch_dev *bch2_dev_iterate(struct bch_fs *c, struct bch_dev return bch2_dev_tryget(c, dev_idx); } +static inline struct bch_dev *bch2_dev_get_ioref2(struct bch_fs *c, unsigned dev, int rw) +{ + rcu_read_lock(); + struct bch_dev *ca = bch2_dev_rcu(c, dev); + if (ca && !percpu_ref_tryget(&ca->io_ref)) + ca = NULL; + rcu_read_unlock(); + + if (ca && + (ca->mi.state == BCH_MEMBER_STATE_rw || + (ca->mi.state == BCH_MEMBER_STATE_ro && rw == READ))) + return ca; + + if (ca) + percpu_ref_put(&ca->io_ref); + return NULL; +} + /* XXX kill, move to struct bch_fs */ static inline struct bch_devs_mask bch2_online_devs(struct bch_fs *c) { From 466298e2f6df31b5d1f7eebb1479ed4e154c75bf Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 3 May 2024 12:44:07 -0400 Subject: [PATCH 1520/1702] bcachefs: bch2_dev_get_ioref2(); backpointers.c Signed-off-by: Kent Overstreet --- fs/bcachefs/backpointers.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c index 3f5333b4c3dee..9ac46f868bb45 100644 --- a/fs/bcachefs/backpointers.c +++ b/fs/bcachefs/backpointers.c @@ -486,8 +486,8 @@ static int check_extent_checksum(struct btree_trans *trans, bytes = p.crc.compressed_size << 9; - struct bch_dev *ca = bch2_dev_bkey_exists(c, dev); - if (!bch2_dev_get_ioref(ca, READ)) + struct bch_dev *ca = bch2_dev_get_ioref2(c, dev, READ); + if (!ca) return false; data_buf = kvmalloc(bytes, GFP_KERNEL); From 690f7cdf732a032388e26c5daed6ad46c9e48256 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 3 May 2024 12:45:16 -0400 Subject: [PATCH 1521/1702] bcachefs: bch2_dev_get_ioref2(); btree_io.c Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_io.c | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c index e14c404d25c12..876a26ae0497c 100644 --- a/fs/bcachefs/btree_io.c +++ b/fs/bcachefs/btree_io.c @@ -1109,7 +1109,7 @@ int bch2_btree_node_read_done(struct bch_fs *c, struct bch_dev *ca, nonce = btree_nonce(i, b->written << 9); struct bch_csum csum = csum_vstruct(c, BSET_CSUM_TYPE(i), nonce, bne); csum_bad = bch2_crc_cmp(bne->csum, csum); - if (csum_bad) + if (ca && csum_bad) bch2_io_error(ca, BCH_MEMBER_ERROR_checksum); btree_err_on(csum_bad, @@ -1263,12 +1263,14 @@ int bch2_btree_node_read_done(struct bch_fs *c, struct bch_dev *ca, btree_node_reset_sib_u64s(b); + rcu_read_lock(); bkey_for_each_ptr(bch2_bkey_ptrs(bkey_i_to_s(&b->key)), ptr) { - struct bch_dev *ca2 = bch2_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca2 = bch2_dev_rcu(c, ptr->dev); - if (ca2->mi.state != BCH_MEMBER_STATE_rw) + if (!ca2 || ca2->mi.state != BCH_MEMBER_STATE_rw) set_btree_node_need_rewrite(b); } + rcu_read_unlock(); if (!ptr_written) set_btree_node_need_rewrite(b); @@ -1293,8 +1295,8 @@ static void btree_node_read_work(struct work_struct *work) struct btree_read_bio *rb = container_of(work, struct btree_read_bio, work); struct bch_fs *c = rb->c; + struct bch_dev *ca = rb->have_ioref ? bch2_dev_have_ref(c, rb->pick.ptr.dev) : NULL; struct btree *b = rb->b; - struct bch_dev *ca = bch2_dev_bkey_exists(c, rb->pick.ptr.dev); struct bio *bio = &rb->bio; struct bch_io_failures failed = { .nr = 0 }; struct printbuf buf = PRINTBUF; @@ -1306,8 +1308,8 @@ static void btree_node_read_work(struct work_struct *work) while (1) { retry = true; bch_info(c, "retrying read"); - ca = bch2_dev_bkey_exists(c, rb->pick.ptr.dev); - rb->have_ioref = bch2_dev_get_ioref(ca, READ); + ca = bch2_dev_get_ioref2(c, rb->pick.ptr.dev, READ); + rb->have_ioref = ca != NULL; bio_reset(bio, NULL, REQ_OP_READ|REQ_SYNC|REQ_META); bio->bi_iter.bi_sector = rb->pick.ptr.offset; bio->bi_iter.bi_size = btree_buf_bytes(b); @@ -1321,7 +1323,7 @@ static void btree_node_read_work(struct work_struct *work) start: printbuf_reset(&buf); bch2_btree_pos_to_text(&buf, c, b); - bch2_dev_io_err_on(bio->bi_status, ca, BCH_MEMBER_ERROR_read, + bch2_dev_io_err_on(ca && bio->bi_status, ca, BCH_MEMBER_ERROR_read, "btree read error %s for %s", bch2_blk_status_to_str(bio->bi_status), buf.buf); if (rb->have_ioref) @@ -1377,7 +1379,7 @@ static void btree_node_read_endio(struct bio *bio) struct bch_fs *c = rb->c; if (rb->have_ioref) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, rb->pick.ptr.dev); + struct bch_dev *ca = bch2_dev_have_ref(c, rb->pick.ptr.dev); bch2_latency_acct(ca, rb->start_time, READ); } @@ -1574,7 +1576,7 @@ static void btree_node_read_all_replicas_endio(struct bio *bio) struct btree_node_read_all *ra = rb->ra; if (rb->have_ioref) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, rb->pick.ptr.dev); + struct bch_dev *ca = bch2_dev_have_ref(c, rb->pick.ptr.dev); bch2_latency_acct(ca, rb->start_time, READ); } @@ -1616,14 +1618,14 @@ static int btree_node_read_all_replicas(struct bch_fs *c, struct btree *b, bool i = 0; bkey_for_each_ptr_decode(k.k, ptrs, pick, entry) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, pick.ptr.dev); + struct bch_dev *ca = bch2_dev_get_ioref2(c, pick.ptr.dev, READ); struct btree_read_bio *rb = container_of(ra->bio[i], struct btree_read_bio, bio); rb->c = c; rb->b = b; rb->ra = ra; rb->start_time = local_clock(); - rb->have_ioref = bch2_dev_get_ioref(ca, READ); + rb->have_ioref = ca != NULL; rb->idx = i; rb->pick = pick; rb->bio.bi_iter.bi_sector = pick.ptr.offset; @@ -1693,7 +1695,7 @@ void bch2_btree_node_read(struct btree_trans *trans, struct btree *b, return; } - ca = bch2_dev_bkey_exists(c, pick.ptr.dev); + ca = bch2_dev_get_ioref2(c, pick.ptr.dev, READ); bio = bio_alloc_bioset(NULL, buf_pages(b->data, btree_buf_bytes(b)), @@ -1705,7 +1707,7 @@ void bch2_btree_node_read(struct btree_trans *trans, struct btree *b, rb->b = b; rb->ra = NULL; rb->start_time = local_clock(); - rb->have_ioref = bch2_dev_get_ioref(ca, READ); + rb->have_ioref = ca != NULL; rb->pick = pick; INIT_WORK(&rb->work, btree_node_read_work); bio->bi_iter.bi_sector = pick.ptr.offset; @@ -1909,13 +1911,14 @@ static void btree_node_write_endio(struct bio *bio) struct btree_write_bio *wb = container_of(orig, struct btree_write_bio, wbio); struct bch_fs *c = wbio->c; struct btree *b = wbio->bio.bi_private; - struct bch_dev *ca = bch2_dev_bkey_exists(c, wbio->dev); + struct bch_dev *ca = wbio->have_ioref ? bch2_dev_have_ref(c, wbio->dev) : NULL; unsigned long flags; if (wbio->have_ioref) bch2_latency_acct(ca, wbio->submit_time, WRITE); - if (bch2_dev_io_err_on(bio->bi_status, ca, BCH_MEMBER_ERROR_write, + if (!ca || + bch2_dev_io_err_on(bio->bi_status, ca, BCH_MEMBER_ERROR_write, "btree write error: %s", bch2_blk_status_to_str(bio->bi_status)) || bch2_meta_write_fault("btree")) { From 48af853925998c04a4be1473b32101ed4ba367e6 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 3 May 2024 12:53:27 -0400 Subject: [PATCH 1522/1702] bcachefs: bch2_dev_get_ioref2(); io_write.c Signed-off-by: Kent Overstreet --- fs/bcachefs/io_write.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/fs/bcachefs/io_write.c b/fs/bcachefs/io_write.c index fa3413d19b6ac..d87c0d2f99021 100644 --- a/fs/bcachefs/io_write.c +++ b/fs/bcachefs/io_write.c @@ -407,9 +407,9 @@ void bch2_submit_wbio_replicas(struct bch_write_bio *wbio, struct bch_fs *c, BUG_ON(c->opts.nochanges); bkey_for_each_ptr(ptrs, ptr) { - BUG_ON(!bch2_dev_exists(c, ptr->dev)); - - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = nocow + ? bch2_dev_have_ref(c, ptr->dev) + : bch2_dev_get_ioref2(c, ptr->dev, type == BCH_DATA_btree ? READ : WRITE); if (to_entry(ptr + 1) < ptrs.end) { n = to_wbio(bio_alloc_clone(NULL, &wbio->bio, GFP_NOFS, &c->replica_set)); @@ -429,8 +429,7 @@ void bch2_submit_wbio_replicas(struct bch_write_bio *wbio, struct bch_fs *c, n->c = c; n->dev = ptr->dev; - n->have_ioref = nocow || bch2_dev_get_ioref(ca, - type == BCH_DATA_btree ? READ : WRITE); + n->have_ioref = ca != NULL; n->nocow = nocow; n->submit_time = local_clock(); n->inode_offset = bkey_start_offset(&k->k); @@ -650,7 +649,9 @@ static void bch2_write_endio(struct bio *bio) struct bch_write_bio *wbio = to_wbio(bio); struct bch_write_bio *parent = wbio->split ? wbio->parent : NULL; struct bch_fs *c = wbio->c; - struct bch_dev *ca = bch2_dev_bkey_exists(c, wbio->dev); + struct bch_dev *ca = wbio->have_ioref + ? bch2_dev_have_ref(c, wbio->dev) + : NULL; if (bch2_dev_inum_io_err_on(bio->bi_status, ca, BCH_MEMBER_ERROR_write, op->pos.inode, @@ -1264,15 +1265,15 @@ static void bch2_nocow_write(struct bch_write_op *op) /* Get iorefs before dropping btree locks: */ struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); bkey_for_each_ptr(ptrs, ptr) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_get_ioref2(c, ptr->dev, WRITE); + if (unlikely(!ca)) + goto err_get_ioref; + struct bpos b = PTR_BUCKET_POS(ca, ptr); struct nocow_lock_bucket *l = bucket_nocow_lock(&c->nocow_locks, bucket_to_u64(b)); prefetch(l); - if (unlikely(!bch2_dev_get_ioref(ca, WRITE))) - goto err_get_ioref; - /* XXX allocating memory with btree locks held - rare */ darray_push_gfp(&buckets, ((struct bucket_to_lock) { .b = b, .gen = ptr->gen, .l = l, From 6212ea24975faf47f7c0f402b46b3a5c7ecc8053 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 3 May 2024 12:54:25 -0400 Subject: [PATCH 1523/1702] bcachefs: bch2_dev_get_ioref2(); journal_io.c Signed-off-by: Kent Overstreet --- fs/bcachefs/journal_io.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c index c9744546d48d9..f6f9d03bb8a3a 100644 --- a/fs/bcachefs/journal_io.c +++ b/fs/bcachefs/journal_io.c @@ -1722,10 +1722,8 @@ static CLOSURE_CALLBACK(journal_write_submit) unsigned sectors = vstruct_sectors(w->data, c->block_bits); extent_for_each_ptr(bkey_i_to_s_extent(&w->key), ptr) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); - struct journal_device *ja = &ca->journal; - - if (!percpu_ref_tryget(&ca->io_ref)) { + struct bch_dev *ca = bch2_dev_get_ioref2(c, ptr->dev, WRITE); + if (!ca) { /* XXX: fix this */ bch_err(c, "missing device for journal write\n"); continue; @@ -1734,6 +1732,7 @@ static CLOSURE_CALLBACK(journal_write_submit) this_cpu_add(ca->io_done->sectors[WRITE][BCH_DATA_journal], sectors); + struct journal_device *ja = &ca->journal; struct bio *bio = &ja->bio[w->idx]->bio; bio_reset(bio, ca->disk_sb.bdev, REQ_OP_WRITE|REQ_SYNC|REQ_META); bio->bi_iter.bi_sector = ptr->offset; From 91ffdecfc796e79ce9d6ae1cf052afebbda8b01e Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 3 May 2024 12:55:55 -0400 Subject: [PATCH 1524/1702] bcachefs: bch2_dev_get_ioref2(); debug.c Signed-off-by: Kent Overstreet --- fs/bcachefs/debug.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/bcachefs/debug.c b/fs/bcachefs/debug.c index b5aa4e5b224b6..d75735aad27e2 100644 --- a/fs/bcachefs/debug.c +++ b/fs/bcachefs/debug.c @@ -37,11 +37,11 @@ static bool bch2_btree_verify_replica(struct bch_fs *c, struct btree *b, struct btree_node *n_ondisk = c->verify_ondisk; struct btree_node *n_sorted = c->verify_data->data; struct bset *sorted, *inmemory = &b->data->keys; - struct bch_dev *ca = bch2_dev_bkey_exists(c, pick.ptr.dev); struct bio *bio; bool failed = false, saw_error = false; - if (!bch2_dev_get_ioref(ca, READ)) + struct bch_dev *ca = bch2_dev_get_ioref2(c, pick.ptr.dev, READ); + if (!ca) return false; bio = bio_alloc_bioset(ca->disk_sb.bdev, @@ -194,8 +194,8 @@ void bch2_btree_node_ondisk_to_text(struct printbuf *out, struct bch_fs *c, return; } - ca = bch2_dev_bkey_exists(c, pick.ptr.dev); - if (!bch2_dev_get_ioref(ca, READ)) { + ca = bch2_dev_get_ioref2(c, pick.ptr.dev, READ); + if (!ca) { prt_printf(out, "error getting device to read from: not online\n"); return; } From 465bf6f42aac1474c6d58258501e55e8760c0a34 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Fri, 3 May 2024 12:50:22 -0400 Subject: [PATCH 1525/1702] bcachefs: bch2_dev_get_ioref2(); io_read.c Signed-off-by: Kent Overstreet --- fs/bcachefs/io_read.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/fs/bcachefs/io_read.c b/fs/bcachefs/io_read.c index 91260d3dbf503..4aa7c6d8d6939 100644 --- a/fs/bcachefs/io_read.c +++ b/fs/bcachefs/io_read.c @@ -800,7 +800,6 @@ int __bch2_read_extent(struct btree_trans *trans, struct bch_read_bio *orig, struct bch_fs *c = trans->c; struct extent_ptr_decoded pick; struct bch_read_bio *rbio = NULL; - struct bch_dev *ca = NULL; struct promote_op *promote = NULL; bool bounce = false, read_full = false, narrow_crcs = false; struct bpos data_pos = bkey_start_pos(k.k); @@ -831,7 +830,7 @@ int __bch2_read_extent(struct btree_trans *trans, struct bch_read_bio *orig, goto err; } - ca = bch2_dev_bkey_exists(c, pick.ptr.dev); + struct bch_dev *ca = bch2_dev_get_ioref2(c, pick.ptr.dev, READ); /* * Stale dirty pointers are treated as IO errors, but @failed isn't @@ -841,9 +840,11 @@ int __bch2_read_extent(struct btree_trans *trans, struct bch_read_bio *orig, */ if ((flags & BCH_READ_IN_RETRY) && !pick.ptr.cached && + ca && unlikely(dev_ptr_stale(ca, &pick.ptr))) { read_from_stale_dirty_pointer(trans, ca, k, pick.ptr); bch2_mark_io_failure(failed, &pick); + percpu_ref_put(&ca->io_ref); goto retry_pick; } @@ -858,8 +859,11 @@ int __bch2_read_extent(struct btree_trans *trans, struct bch_read_bio *orig, * can happen if we retry, and the extent we were going to read * has been merged in the meantime: */ - if (pick.crc.compressed_size > orig->bio.bi_vcnt * PAGE_SECTORS) + if (pick.crc.compressed_size > orig->bio.bi_vcnt * PAGE_SECTORS) { + if (ca) + percpu_ref_put(&ca->io_ref); goto hole; + } iter.bi_size = pick.crc.compressed_size << 9; goto get_bio; @@ -964,7 +968,7 @@ int __bch2_read_extent(struct btree_trans *trans, struct bch_read_bio *orig, rbio->bvec_iter = iter; rbio->offset_into_extent= offset_into_extent; rbio->flags = flags; - rbio->have_ioref = pick_ret > 0 && bch2_dev_get_ioref(ca, READ); + rbio->have_ioref = ca != NULL; rbio->narrow_crcs = narrow_crcs; rbio->hole = 0; rbio->retry = 0; @@ -994,7 +998,7 @@ int __bch2_read_extent(struct btree_trans *trans, struct bch_read_bio *orig, * If it's being moved internally, we don't want to flag it as a cache * hit: */ - if (pick.ptr.cached && !(flags & BCH_READ_NODECODE)) + if (ca && pick.ptr.cached && !(flags & BCH_READ_NODECODE)) bch2_bucket_io_time_reset(trans, pick.ptr.dev, PTR_BUCKET_NR(ca, &pick.ptr), READ); From 2c91ab7262e6b08bbc76b83ed1981602fa6ef835 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 30 Apr 2024 15:37:51 -0400 Subject: [PATCH 1526/1702] bcachefs: bch2_dev_get_ioref() checks for device not present Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 4 ++-- fs/bcachefs/backpointers.c | 2 +- fs/bcachefs/btree_io.c | 6 +++--- fs/bcachefs/debug.c | 4 ++-- fs/bcachefs/ec.c | 21 ++++++++++----------- fs/bcachefs/io_read.c | 2 +- fs/bcachefs/io_write.c | 4 ++-- fs/bcachefs/journal_io.c | 2 +- fs/bcachefs/sb-members.h | 15 +-------------- 9 files changed, 23 insertions(+), 37 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index b9ded84d2d94e..66a1d57db25d4 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -1679,7 +1679,7 @@ static int bch2_discard_one_bucket(struct btree_trans *trans, struct bch_dev *ca = s->ca && s->ca->dev_idx == pos.inode ? s->ca - : bch2_dev_get_ioref2(c, pos.inode, WRITE); + : bch2_dev_get_ioref(c, pos.inode, WRITE); if (!ca) { bch2_btree_iter_set_pos(need_discard_iter, POS(pos.inode + 1, 0)); return 0; @@ -1860,7 +1860,7 @@ static void bch2_do_discards_fast_work(struct work_struct *work) if (i->snapshot) continue; - ca = bch2_dev_get_ioref2(c, i->inode, WRITE); + ca = bch2_dev_get_ioref(c, i->inode, WRITE); if (!ca) { darray_remove_item(&c->discard_buckets_in_flight, i); continue; diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c index 9ac46f868bb45..921d95c46cf7e 100644 --- a/fs/bcachefs/backpointers.c +++ b/fs/bcachefs/backpointers.c @@ -486,7 +486,7 @@ static int check_extent_checksum(struct btree_trans *trans, bytes = p.crc.compressed_size << 9; - struct bch_dev *ca = bch2_dev_get_ioref2(c, dev, READ); + struct bch_dev *ca = bch2_dev_get_ioref(c, dev, READ); if (!ca) return false; diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c index 876a26ae0497c..cbf8f5d906021 100644 --- a/fs/bcachefs/btree_io.c +++ b/fs/bcachefs/btree_io.c @@ -1308,7 +1308,7 @@ static void btree_node_read_work(struct work_struct *work) while (1) { retry = true; bch_info(c, "retrying read"); - ca = bch2_dev_get_ioref2(c, rb->pick.ptr.dev, READ); + ca = bch2_dev_get_ioref(c, rb->pick.ptr.dev, READ); rb->have_ioref = ca != NULL; bio_reset(bio, NULL, REQ_OP_READ|REQ_SYNC|REQ_META); bio->bi_iter.bi_sector = rb->pick.ptr.offset; @@ -1618,7 +1618,7 @@ static int btree_node_read_all_replicas(struct bch_fs *c, struct btree *b, bool i = 0; bkey_for_each_ptr_decode(k.k, ptrs, pick, entry) { - struct bch_dev *ca = bch2_dev_get_ioref2(c, pick.ptr.dev, READ); + struct bch_dev *ca = bch2_dev_get_ioref(c, pick.ptr.dev, READ); struct btree_read_bio *rb = container_of(ra->bio[i], struct btree_read_bio, bio); rb->c = c; @@ -1695,7 +1695,7 @@ void bch2_btree_node_read(struct btree_trans *trans, struct btree *b, return; } - ca = bch2_dev_get_ioref2(c, pick.ptr.dev, READ); + ca = bch2_dev_get_ioref(c, pick.ptr.dev, READ); bio = bio_alloc_bioset(NULL, buf_pages(b->data, btree_buf_bytes(b)), diff --git a/fs/bcachefs/debug.c b/fs/bcachefs/debug.c index d75735aad27e2..51cbf39283612 100644 --- a/fs/bcachefs/debug.c +++ b/fs/bcachefs/debug.c @@ -40,7 +40,7 @@ static bool bch2_btree_verify_replica(struct bch_fs *c, struct btree *b, struct bio *bio; bool failed = false, saw_error = false; - struct bch_dev *ca = bch2_dev_get_ioref2(c, pick.ptr.dev, READ); + struct bch_dev *ca = bch2_dev_get_ioref(c, pick.ptr.dev, READ); if (!ca) return false; @@ -194,7 +194,7 @@ void bch2_btree_node_ondisk_to_text(struct printbuf *out, struct bch_fs *c, return; } - ca = bch2_dev_get_ioref2(c, pick.ptr.dev, READ); + ca = bch2_dev_get_ioref(c, pick.ptr.dev, READ); if (!ca) { prt_printf(out, "error getting device to read from: not online\n"); return; diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 99a8c8fe8e3b4..055be0ca92507 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -732,12 +732,17 @@ static void ec_block_io(struct bch_fs *c, struct ec_stripe_buf *buf, struct bch_stripe *v = &bkey_i_to_stripe(&buf->key)->v; unsigned offset = 0, bytes = buf->size << 9; struct bch_extent_ptr *ptr = &v->ptrs[idx]; - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); enum bch_data_type data_type = idx < v->nr_blocks - v->nr_redundant ? BCH_DATA_user : BCH_DATA_parity; int rw = op_is_write(opf); + struct bch_dev *ca = bch2_dev_get_ioref(c, ptr->dev, rw); + if (!ca) { + clear_bit(idx, buf->valid); + return; + } + if (dev_ptr_stale(ca, ptr)) { bch_err_ratelimited(c, "error %s stripe: stale pointer", @@ -746,10 +751,6 @@ static void ec_block_io(struct bch_fs *c, struct ec_stripe_buf *buf, return; } - if (!bch2_dev_get_ioref(ca, rw)) { - clear_bit(idx, buf->valid); - return; - } this_cpu_add(ca->io_done->sectors[rw][data_type], buf->size); @@ -1354,20 +1355,18 @@ static void zero_out_rest_of_ec_bucket(struct bch_fs *c, unsigned block, struct open_bucket *ob) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ob->dev); - unsigned offset = ca->mi.bucket_size - ob->sectors_free; - int ret; - - if (!bch2_dev_get_ioref(ca, WRITE)) { + struct bch_dev *ca = bch2_dev_get_ioref(c, ob->dev, WRITE); + if (!ca) { s->err = -BCH_ERR_erofs_no_writes; return; } + unsigned offset = ca->mi.bucket_size - ob->sectors_free; memset(s->new_stripe.data[block] + (offset << 9), 0, ob->sectors_free << 9); - ret = blkdev_issue_zeroout(ca->disk_sb.bdev, + int ret = blkdev_issue_zeroout(ca->disk_sb.bdev, ob->bucket * ca->mi.bucket_size + offset, ob->sectors_free, GFP_KERNEL, 0); diff --git a/fs/bcachefs/io_read.c b/fs/bcachefs/io_read.c index 4aa7c6d8d6939..3e7dceef4cb5a 100644 --- a/fs/bcachefs/io_read.c +++ b/fs/bcachefs/io_read.c @@ -830,7 +830,7 @@ int __bch2_read_extent(struct btree_trans *trans, struct bch_read_bio *orig, goto err; } - struct bch_dev *ca = bch2_dev_get_ioref2(c, pick.ptr.dev, READ); + struct bch_dev *ca = bch2_dev_get_ioref(c, pick.ptr.dev, READ); /* * Stale dirty pointers are treated as IO errors, but @failed isn't diff --git a/fs/bcachefs/io_write.c b/fs/bcachefs/io_write.c index d87c0d2f99021..9401d13e31bb6 100644 --- a/fs/bcachefs/io_write.c +++ b/fs/bcachefs/io_write.c @@ -409,7 +409,7 @@ void bch2_submit_wbio_replicas(struct bch_write_bio *wbio, struct bch_fs *c, bkey_for_each_ptr(ptrs, ptr) { struct bch_dev *ca = nocow ? bch2_dev_have_ref(c, ptr->dev) - : bch2_dev_get_ioref2(c, ptr->dev, type == BCH_DATA_btree ? READ : WRITE); + : bch2_dev_get_ioref(c, ptr->dev, type == BCH_DATA_btree ? READ : WRITE); if (to_entry(ptr + 1) < ptrs.end) { n = to_wbio(bio_alloc_clone(NULL, &wbio->bio, GFP_NOFS, &c->replica_set)); @@ -1265,7 +1265,7 @@ static void bch2_nocow_write(struct bch_write_op *op) /* Get iorefs before dropping btree locks: */ struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); bkey_for_each_ptr(ptrs, ptr) { - struct bch_dev *ca = bch2_dev_get_ioref2(c, ptr->dev, WRITE); + struct bch_dev *ca = bch2_dev_get_ioref(c, ptr->dev, WRITE); if (unlikely(!ca)) goto err_get_ioref; diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c index f6f9d03bb8a3a..d3bc5df0f3de5 100644 --- a/fs/bcachefs/journal_io.c +++ b/fs/bcachefs/journal_io.c @@ -1722,7 +1722,7 @@ static CLOSURE_CALLBACK(journal_write_submit) unsigned sectors = vstruct_sectors(w->data, c->block_bits); extent_for_each_ptr(bkey_i_to_s_extent(&w->key), ptr) { - struct bch_dev *ca = bch2_dev_get_ioref2(c, ptr->dev, WRITE); + struct bch_dev *ca = bch2_dev_get_ioref(c, ptr->dev, WRITE); if (!ca) { /* XXX: fix this */ bch_err(c, "missing device for journal write\n"); diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index f2a7794b0b500..ae28b5d401e33 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -29,19 +29,6 @@ static inline bool bch2_dev_is_readable(struct bch_dev *ca) ca->mi.state != BCH_MEMBER_STATE_failed; } -static inline bool bch2_dev_get_ioref(struct bch_dev *ca, int rw) -{ - if (!percpu_ref_tryget(&ca->io_ref)) - return false; - - if (ca->mi.state == BCH_MEMBER_STATE_rw || - (ca->mi.state == BCH_MEMBER_STATE_ro && rw == READ)) - return true; - - percpu_ref_put(&ca->io_ref); - return false; -} - static inline unsigned dev_mask_nr(const struct bch_devs_mask *devs) { return bitmap_weight(devs->d, BCH_SB_MEMBERS_MAX); @@ -285,7 +272,7 @@ static inline struct bch_dev *bch2_dev_iterate(struct bch_fs *c, struct bch_dev return bch2_dev_tryget(c, dev_idx); } -static inline struct bch_dev *bch2_dev_get_ioref2(struct bch_fs *c, unsigned dev, int rw) +static inline struct bch_dev *bch2_dev_get_ioref(struct bch_fs *c, unsigned dev, int rw) { rcu_read_lock(); struct bch_dev *ca = bch2_dev_rcu(c, dev); From 02b7fa4fe5307b1de74e734e28192623b7af7020 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 1 May 2024 03:59:52 -0400 Subject: [PATCH 1527/1702] bcachefs: kill bch2_dev_bkey_exists() in bch2_read_endio() Signed-off-by: Kent Overstreet --- fs/bcachefs/io_read.c | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/fs/bcachefs/io_read.c b/fs/bcachefs/io_read.c index 3e7dceef4cb5a..f57486794484d 100644 --- a/fs/bcachefs/io_read.c +++ b/fs/bcachefs/io_read.c @@ -541,7 +541,6 @@ static void __bch2_read_endio(struct work_struct *work) struct bch_read_bio *rbio = container_of(work, struct bch_read_bio, work); struct bch_fs *c = rbio->c; - struct bch_dev *ca = bch2_dev_bkey_exists(c, rbio->pick.ptr.dev); struct bio *src = &rbio->bio; struct bio *dst = &bch2_rbio_parent(rbio)->bio; struct bvec_iter dst_iter = rbio->bvec_iter; @@ -647,13 +646,15 @@ static void __bch2_read_endio(struct work_struct *work) prt_str(&buf, "data "); bch2_csum_err_msg(&buf, crc.csum_type, rbio->pick.crc.csum, csum); - bch_err_inum_offset_ratelimited(ca, - rbio->read_pos.inode, - rbio->read_pos.offset << 9, - "data %s", buf.buf); + struct bch_dev *ca = rbio->have_ioref ? bch2_dev_have_ref(c, rbio->pick.ptr.dev) : NULL; + if (ca) { + bch_err_inum_offset_ratelimited(ca, + rbio->read_pos.inode, + rbio->read_pos.offset << 9, + "data %s", buf.buf); + bch2_io_error(ca, BCH_MEMBER_ERROR_checksum); + } printbuf_exit(&buf); - - bch2_io_error(ca, BCH_MEMBER_ERROR_checksum); bch2_rbio_error(rbio, READ_RETRY_AVOID, BLK_STS_IOERR); goto out; decompression_err: @@ -675,7 +676,7 @@ static void bch2_read_endio(struct bio *bio) struct bch_read_bio *rbio = container_of(bio, struct bch_read_bio, bio); struct bch_fs *c = rbio->c; - struct bch_dev *ca = bch2_dev_bkey_exists(c, rbio->pick.ptr.dev); + struct bch_dev *ca = rbio->have_ioref ? bch2_dev_have_ref(c, rbio->pick.ptr.dev) : NULL; struct workqueue_struct *wq = NULL; enum rbio_context context = RBIO_CONTEXT_NULL; @@ -687,17 +688,21 @@ static void bch2_read_endio(struct bio *bio) if (!rbio->split) rbio->bio.bi_end_io = rbio->end_io; - if (bch2_dev_inum_io_err_on(bio->bi_status, ca, BCH_MEMBER_ERROR_read, - rbio->read_pos.inode, - rbio->read_pos.offset, - "data read error: %s", - bch2_blk_status_to_str(bio->bi_status))) { + if (bio->bi_status) { + if (ca) { + bch_err_inum_offset_ratelimited(ca, + rbio->read_pos.inode, + rbio->read_pos.offset, + "data read error: %s", + bch2_blk_status_to_str(bio->bi_status)); + bch2_io_error(ca, BCH_MEMBER_ERROR_read); + } bch2_rbio_error(rbio, READ_RETRY_AVOID, bio->bi_status); return; } if (((rbio->flags & BCH_READ_RETRY_IF_STALE) && race_fault()) || - dev_ptr_stale(ca, &rbio->pick.ptr)) { + (ca && dev_ptr_stale(ca, &rbio->pick.ptr))) { trace_and_count(c, read_reuse_race, &rbio->bio); if (rbio->flags & BCH_READ_RETRY_IF_STALE) From 2f4b4a3b44755d8c86eff64b9c54e3242a9b659c Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 1 May 2024 19:10:17 -0400 Subject: [PATCH 1528/1702] bcachefs: kill bch2_dev_bkey_exists() in bch2_check_fix_ptrs() Signed-off-by: Kent Overstreet --- fs/bcachefs/buckets.c | 27 ++++++++++++++++++++++++--- fs/bcachefs/sb-members.h | 11 ----------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index ae6d6879e2e02..e28d28ac2a13d 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -479,8 +479,20 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, percpu_down_read(&c->mark_lock); + rcu_read_lock(); bkey_for_each_ptr_decode(k.k, ptrs_c, p, entry_c) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); + struct bch_dev *ca = bch2_dev_rcu(c, p.ptr.dev); + if (!ca) { + if (fsck_err(c, ptr_to_invalid_device, + "pointer to missing device %u\n" + "while marking %s", + p.ptr.dev, + (printbuf_reset(&buf), + bch2_bkey_val_to_text(&buf, c, k), buf.buf))) + do_update = true; + continue; + } + struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr); enum bch_data_type data_type = bch2_bkey_ptr_data_type(k, p, entry_c); @@ -590,6 +602,7 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, do_update = true; } } + rcu_read_unlock(); if (do_update) { if (flags & BTREE_TRIGGER_is_root) { @@ -603,6 +616,10 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, if (ret) goto err; + rcu_read_lock(); + bch2_bkey_drop_ptrs(bkey_i_to_s(new), ptr, !bch2_dev_rcu(c, ptr->dev)); + rcu_read_unlock(); + if (level) { /* * We don't want to drop btree node pointers - if the @@ -610,19 +627,22 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, * sort it out: */ struct bkey_ptrs ptrs = bch2_bkey_ptrs(bkey_i_to_s(new)); + rcu_read_lock(); bkey_for_each_ptr(ptrs, ptr) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, ptr->dev); + struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev); struct bucket *g = PTR_GC_BUCKET(ca, ptr); ptr->gen = g->gen; } + rcu_read_unlock(); } else { struct bkey_ptrs ptrs; union bch_extent_entry *entry; restart_drop_ptrs: ptrs = bch2_bkey_ptrs(bkey_i_to_s(new)); + rcu_read_lock(); bkey_for_each_ptr_decode(bkey_i_to_s(new).k, ptrs, p, entry) { - struct bch_dev *ca = bch2_dev_bkey_exists(c, p.ptr.dev); + struct bch_dev *ca = bch2_dev_rcu(c, p.ptr.dev); struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr); enum bch_data_type data_type = bch2_bkey_ptr_data_type(bkey_i_to_s_c(new), p, entry); @@ -637,6 +657,7 @@ int bch2_check_fix_ptrs(struct btree_trans *trans, goto restart_drop_ptrs; } } + rcu_read_unlock(); again: ptrs = bch2_bkey_ptrs(bkey_i_to_s(new)); bkey_extent_entry_for_each(ptrs, entry) { diff --git a/fs/bcachefs/sb-members.h b/fs/bcachefs/sb-members.h index ae28b5d401e33..dd93192ec065e 100644 --- a/fs/bcachefs/sb-members.h +++ b/fs/bcachefs/sb-members.h @@ -189,17 +189,6 @@ static inline struct bch_dev *bch2_dev_have_ref(const struct bch_fs *c, unsigned return rcu_dereference_check(c->devs[dev], 1); } -/* - * If a key exists that references a device, the device won't be going away and - * we can omit rcu_read_lock(): - */ -static inline struct bch_dev *bch2_dev_bkey_exists(const struct bch_fs *c, unsigned dev) -{ - EBUG_ON(!bch2_dev_exists(c, dev)); - - return rcu_dereference_check(c->devs[dev], 1); -} - static inline struct bch_dev *bch2_dev_locked(struct bch_fs *c, unsigned dev) { EBUG_ON(!bch2_dev_exists(c, dev)); From 99179fb89847f312229bf816200e1e6e5038e3a3 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 1 May 2024 19:15:29 -0400 Subject: [PATCH 1529/1702] bcachefs: Invalid devices are now checked for by fsck, not .invalid methods Signed-off-by: Kent Overstreet --- fs/bcachefs/extents.c | 15 +-------------- fs/bcachefs/journal_io.c | 11 ----------- 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index b785712ae9b26..094caf1cdb35b 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -1101,22 +1101,8 @@ static int extent_ptr_invalid(struct bch_fs *c, bool metadata, struct printbuf *err) { - struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); int ret = 0; - if (!bch2_dev_exists(c, ptr->dev)) { - /* - * If we're in the write path this key might have already been - * overwritten, and we could be seeing a device that doesn't - * exist anymore due to racing with device removal: - */ - if (flags & BKEY_INVALID_WRITE) - return 0; - - bkey_fsck_err(c, err, ptr_to_invalid_device, - "pointer to invalid device (%u)", ptr->dev); - } - rcu_read_lock(); struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev); if (!ca) { @@ -1130,6 +1116,7 @@ static int extent_ptr_invalid(struct bch_fs *c, unsigned bucket_size = ca->mi.bucket_size; rcu_read_unlock(); + struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); bkey_for_each_ptr(ptrs, ptr2) bkey_fsck_err_on(ptr != ptr2 && ptr->dev == ptr2->dev, c, err, ptr_to_duplicate_device, diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c index d3bc5df0f3de5..56f4d46ef0941 100644 --- a/fs/bcachefs/journal_io.c +++ b/fs/bcachefs/journal_io.c @@ -690,7 +690,6 @@ static int journal_entry_dev_usage_validate(struct bch_fs *c, container_of(entry, struct jset_entry_dev_usage, entry); unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64); unsigned expected = sizeof(*u); - unsigned dev; int ret = 0; if (journal_entry_err_on(bytes < expected, @@ -702,16 +701,6 @@ static int journal_entry_dev_usage_validate(struct bch_fs *c, return ret; } - dev = le32_to_cpu(u->dev); - - if (journal_entry_err_on(!bch2_dev_exists(c, dev), - c, version, jset, entry, - journal_entry_dev_usage_bad_dev, - "bad dev")) { - journal_entry_null_range(entry, vstruct_next(entry)); - return ret; - } - if (journal_entry_err_on(u->pad, c, version, jset, entry, journal_entry_dev_usage_bad_pad, From d09a8468d915850709ae5f34c23e2b24cb5c3c62 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 8 May 2024 17:33:29 -0400 Subject: [PATCH 1530/1702] bcachefs: fsync() should not return -EROFS fsync has a slightly odd usage of -EROFS, where it means "does not support fsync". I didn't choose it... Signed-off-by: Kent Overstreet --- fs/bcachefs/fs-io.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fs/bcachefs/fs-io.c b/fs/bcachefs/fs-io.c index 442bcb0793c41..ef20b64033e09 100644 --- a/fs/bcachefs/fs-io.c +++ b/fs/bcachefs/fs-io.c @@ -202,7 +202,10 @@ int bch2_fsync(struct file *file, loff_t start, loff_t end, int datasync) goto out; ret = bch2_flush_inode(c, inode); out: - return bch2_err_class(ret); + ret = bch2_err_class(ret); + if (ret == -EROFS) + ret = -EIO; + return ret; } /* truncate: */ From 65eaf4e24ab6b4809491248e1fed36b8d49c1ea9 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 8 May 2024 18:40:42 -0400 Subject: [PATCH 1531/1702] bcachefs: s/bkey_invalid_flags/bch_validate_flags We're about to start using bch_validate_flags for superblock section validation - it's no longer bkey specific. Signed-off-by: Kent Overstreet --- fs/bcachefs/alloc_background.c | 10 +++---- fs/bcachefs/alloc_background.h | 12 ++++----- fs/bcachefs/backpointers.c | 2 +- fs/bcachefs/backpointers.h | 2 +- fs/bcachefs/bkey.c | 2 +- fs/bcachefs/bkey.h | 10 +++---- fs/bcachefs/bkey_methods.c | 16 +++++------ fs/bcachefs/bkey_methods.h | 8 +++--- fs/bcachefs/btree_trans_commit.c | 10 +++---- fs/bcachefs/dirent.c | 4 +-- fs/bcachefs/dirent.h | 4 +-- fs/bcachefs/ec.c | 2 +- fs/bcachefs/ec.h | 4 +-- fs/bcachefs/extents.c | 14 +++++----- fs/bcachefs/extents.h | 10 +++---- fs/bcachefs/inode.c | 8 +++--- fs/bcachefs/inode.h | 10 +++---- fs/bcachefs/journal_io.c | 46 ++++++++++++++++---------------- fs/bcachefs/journal_io.h | 2 +- fs/bcachefs/lru.c | 2 +- fs/bcachefs/lru.h | 2 +- fs/bcachefs/quota.c | 2 +- fs/bcachefs/quota.h | 4 +-- fs/bcachefs/reflink.c | 6 ++--- fs/bcachefs/reflink.h | 8 +++--- fs/bcachefs/snapshot.c | 4 +-- fs/bcachefs/snapshot.h | 6 ++--- fs/bcachefs/subvolume.c | 2 +- fs/bcachefs/subvolume.h | 4 +-- fs/bcachefs/xattr.c | 2 +- fs/bcachefs/xattr.h | 2 +- 31 files changed, 110 insertions(+), 110 deletions(-) diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c index 66a1d57db25d4..346cd91f91f99 100644 --- a/fs/bcachefs/alloc_background.c +++ b/fs/bcachefs/alloc_background.c @@ -195,7 +195,7 @@ static unsigned bch_alloc_v1_val_u64s(const struct bch_alloc *a) } int bch2_alloc_v1_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { struct bkey_s_c_alloc a = bkey_s_c_to_alloc(k); @@ -211,7 +211,7 @@ int bch2_alloc_v1_invalid(struct bch_fs *c, struct bkey_s_c k, } int bch2_alloc_v2_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { struct bkey_alloc_unpacked u; @@ -225,7 +225,7 @@ int bch2_alloc_v2_invalid(struct bch_fs *c, struct bkey_s_c k, } int bch2_alloc_v3_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { struct bkey_alloc_unpacked u; @@ -239,7 +239,7 @@ int bch2_alloc_v3_invalid(struct bch_fs *c, struct bkey_s_c k, } int bch2_alloc_v4_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, struct printbuf *err) + enum bch_validate_flags flags, struct printbuf *err) { struct bkey_s_c_alloc_v4 a = bkey_s_c_to_alloc_v4(k); int ret = 0; @@ -487,7 +487,7 @@ static unsigned alloc_gen(struct bkey_s_c k, unsigned offset) } int bch2_bucket_gens_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { int ret = 0; diff --git a/fs/bcachefs/alloc_background.h b/fs/bcachefs/alloc_background.h index 5f42dc1e7fcc8..ae31a94be6f91 100644 --- a/fs/bcachefs/alloc_background.h +++ b/fs/bcachefs/alloc_background.h @@ -8,7 +8,7 @@ #include "debug.h" #include "super.h" -enum bkey_invalid_flags; +enum bch_validate_flags; /* How out of date a pointer gen is allowed to be: */ #define BUCKET_GC_GEN_MAX 96U @@ -205,13 +205,13 @@ struct bkey_i_alloc_v4 *bch2_alloc_to_v4_mut(struct btree_trans *, struct bkey_s int bch2_bucket_io_time_reset(struct btree_trans *, unsigned, size_t, int); int bch2_alloc_v1_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); int bch2_alloc_v2_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); int bch2_alloc_v3_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); int bch2_alloc_v4_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_alloc_v4_swab(struct bkey_s); void bch2_alloc_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); @@ -245,7 +245,7 @@ void bch2_alloc_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); }) int bch2_bucket_gens_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_bucket_gens_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); #define bch2_bkey_ops_bucket_gens ((struct bkey_ops) { \ diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c index 921d95c46cf7e..692b1c7d5018c 100644 --- a/fs/bcachefs/backpointers.c +++ b/fs/bcachefs/backpointers.c @@ -48,7 +48,7 @@ static bool extent_matches_bp(struct bch_fs *c, } int bch2_backpointer_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { struct bkey_s_c_backpointer bp = bkey_s_c_to_backpointer(k); diff --git a/fs/bcachefs/backpointers.h b/fs/bcachefs/backpointers.h index 929f4a068a0a0..6021de1c5e98d 100644 --- a/fs/bcachefs/backpointers.h +++ b/fs/bcachefs/backpointers.h @@ -19,7 +19,7 @@ static inline u64 swab40(u64 x) } int bch2_backpointer_invalid(struct bch_fs *, struct bkey_s_c k, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_backpointer_to_text(struct printbuf *, const struct bch_backpointer *); void bch2_backpointer_k_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); void bch2_backpointer_swab(struct bkey_s); diff --git a/fs/bcachefs/bkey.c b/fs/bcachefs/bkey.c index 952299a2e4163..f46978e5cb7c6 100644 --- a/fs/bcachefs/bkey.c +++ b/fs/bcachefs/bkey.c @@ -640,7 +640,7 @@ struct bkey_format bch2_bkey_format_done(struct bkey_format_state *s) int bch2_bkey_format_invalid(struct bch_fs *c, struct bkey_format *f, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { unsigned i, bits = KEY_PACKED_BITS_START; diff --git a/fs/bcachefs/bkey.h b/fs/bcachefs/bkey.h index 1bb0bd4371b04..fcd43915df079 100644 --- a/fs/bcachefs/bkey.h +++ b/fs/bcachefs/bkey.h @@ -9,10 +9,10 @@ #include "util.h" #include "vstructs.h" -enum bkey_invalid_flags { - BKEY_INVALID_WRITE = (1U << 0), - BKEY_INVALID_COMMIT = (1U << 1), - BKEY_INVALID_JOURNAL = (1U << 2), +enum bch_validate_flags { + BCH_VALIDATE_write = (1U << 0), + BCH_VALIDATE_commit = (1U << 1), + BCH_VALIDATE_journal = (1U << 2), }; #if 0 @@ -598,7 +598,7 @@ static inline bool bch2_bkey_format_field_overflows(struct bkey_format *f, unsig } int bch2_bkey_format_invalid(struct bch_fs *, struct bkey_format *, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_bkey_format_to_text(struct printbuf *, const struct bkey_format *); #endif /* _BCACHEFS_BKEY_H */ diff --git a/fs/bcachefs/bkey_methods.c b/fs/bcachefs/bkey_methods.c index a275a9e8e341a..f692f9d8f00da 100644 --- a/fs/bcachefs/bkey_methods.c +++ b/fs/bcachefs/bkey_methods.c @@ -27,7 +27,7 @@ const char * const bch2_bkey_types[] = { }; static int deleted_key_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, struct printbuf *err) + enum bch_validate_flags flags, struct printbuf *err) { return 0; } @@ -41,7 +41,7 @@ static int deleted_key_invalid(struct bch_fs *c, struct bkey_s_c k, }) static int empty_val_key_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, struct printbuf *err) + enum bch_validate_flags flags, struct printbuf *err) { int ret = 0; @@ -58,7 +58,7 @@ static int empty_val_key_invalid(struct bch_fs *c, struct bkey_s_c k, }) static int key_type_cookie_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, struct printbuf *err) + enum bch_validate_flags flags, struct printbuf *err) { return 0; } @@ -82,7 +82,7 @@ static void key_type_cookie_to_text(struct printbuf *out, struct bch_fs *c, }) static int key_type_inline_data_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, struct printbuf *err) + enum bch_validate_flags flags, struct printbuf *err) { return 0; } @@ -123,7 +123,7 @@ const struct bkey_ops bch2_bkey_null_ops = { }; int bch2_bkey_val_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { const struct bkey_ops *ops = bch2_bkey_type_ops(k.k->type); @@ -159,7 +159,7 @@ const char *bch2_btree_node_type_str(enum btree_node_type type) int __bch2_bkey_invalid(struct bch_fs *c, struct bkey_s_c k, enum btree_node_type type, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { int ret = 0; @@ -172,7 +172,7 @@ int __bch2_bkey_invalid(struct bch_fs *c, struct bkey_s_c k, return 0; bkey_fsck_err_on(k.k->type < KEY_TYPE_MAX && - (type == BKEY_TYPE_btree || (flags & BKEY_INVALID_COMMIT)) && + (type == BKEY_TYPE_btree || (flags & BCH_VALIDATE_commit)) && !(bch2_key_types_allowed[type] & BIT_ULL(k.k->type)), c, err, bkey_invalid_type_for_btree, "invalid key type for btree %s (%s)", @@ -224,7 +224,7 @@ int __bch2_bkey_invalid(struct bch_fs *c, struct bkey_s_c k, int bch2_bkey_invalid(struct bch_fs *c, struct bkey_s_c k, enum btree_node_type type, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { return __bch2_bkey_invalid(c, k, type, flags, err) ?: diff --git a/fs/bcachefs/bkey_methods.h b/fs/bcachefs/bkey_methods.h index 7351c0d38534f..726ef74837638 100644 --- a/fs/bcachefs/bkey_methods.h +++ b/fs/bcachefs/bkey_methods.h @@ -22,7 +22,7 @@ extern const struct bkey_ops bch2_bkey_null_ops; */ struct bkey_ops { int (*key_invalid)(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, struct printbuf *err); + enum bch_validate_flags flags, struct printbuf *err); void (*val_to_text)(struct printbuf *, struct bch_fs *, struct bkey_s_c); void (*swab)(struct bkey_s); @@ -49,11 +49,11 @@ static inline const struct bkey_ops *bch2_bkey_type_ops(enum bch_bkey_type type) } int bch2_bkey_val_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); int __bch2_bkey_invalid(struct bch_fs *, struct bkey_s_c, enum btree_node_type, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); int bch2_bkey_invalid(struct bch_fs *, struct bkey_s_c, enum btree_node_type, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); int bch2_bkey_in_btree_node(struct bch_fs *, struct btree *, struct bkey_s_c, struct printbuf *); diff --git a/fs/bcachefs/btree_trans_commit.c b/fs/bcachefs/btree_trans_commit.c index a219a87e9ef07..74e1ff225674c 100644 --- a/fs/bcachefs/btree_trans_commit.c +++ b/fs/bcachefs/btree_trans_commit.c @@ -790,7 +790,7 @@ static noinline void bch2_drop_overwrites_from_journal(struct btree_trans *trans } static noinline int bch2_trans_commit_bkey_invalid(struct btree_trans *trans, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct btree_insert_entry *i, struct printbuf *err) { @@ -1025,10 +1025,10 @@ int __bch2_trans_commit(struct btree_trans *trans, unsigned flags) trans_for_each_update(trans, i) { struct printbuf buf = PRINTBUF; - enum bkey_invalid_flags invalid_flags = 0; + enum bch_validate_flags invalid_flags = 0; if (!(flags & BCH_TRANS_COMMIT_no_journal_res)) - invalid_flags |= BKEY_INVALID_WRITE|BKEY_INVALID_COMMIT; + invalid_flags |= BCH_VALIDATE_write|BCH_VALIDATE_commit; if (unlikely(bch2_bkey_invalid(c, bkey_i_to_s_c(i->k), i->bkey_type, invalid_flags, &buf))) @@ -1043,10 +1043,10 @@ int __bch2_trans_commit(struct btree_trans *trans, unsigned flags) for (struct jset_entry *i = trans->journal_entries; i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s); i = vstruct_next(i)) { - enum bkey_invalid_flags invalid_flags = 0; + enum bch_validate_flags invalid_flags = 0; if (!(flags & BCH_TRANS_COMMIT_no_journal_res)) - invalid_flags |= BKEY_INVALID_WRITE|BKEY_INVALID_COMMIT; + invalid_flags |= BCH_VALIDATE_write|BCH_VALIDATE_commit; if (unlikely(bch2_journal_entry_validate(c, NULL, i, bcachefs_metadata_version_current, diff --git a/fs/bcachefs/dirent.c b/fs/bcachefs/dirent.c index dd9de8e5a28a9..6bbf9a7d9e4d2 100644 --- a/fs/bcachefs/dirent.c +++ b/fs/bcachefs/dirent.c @@ -98,7 +98,7 @@ const struct bch_hash_desc bch2_dirent_hash_desc = { }; int bch2_dirent_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { struct bkey_s_c_dirent d = bkey_s_c_to_dirent(k); @@ -118,7 +118,7 @@ int bch2_dirent_invalid(struct bch_fs *c, struct bkey_s_c k, * Check new keys don't exceed the max length * (older keys may be larger.) */ - bkey_fsck_err_on((flags & BKEY_INVALID_COMMIT) && d_name.len > BCH_NAME_MAX, c, err, + bkey_fsck_err_on((flags & BCH_VALIDATE_commit) && d_name.len > BCH_NAME_MAX, c, err, dirent_name_too_long, "dirent name too big (%u > %u)", d_name.len, BCH_NAME_MAX); diff --git a/fs/bcachefs/dirent.h b/fs/bcachefs/dirent.h index 635ceb3c07fbc..24037e6e0a094 100644 --- a/fs/bcachefs/dirent.h +++ b/fs/bcachefs/dirent.h @@ -4,11 +4,11 @@ #include "str_hash.h" -enum bkey_invalid_flags; +enum bch_validate_flags; extern const struct bch_hash_desc bch2_dirent_hash_desc; int bch2_dirent_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_dirent_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); #define bch2_bkey_ops_dirent ((struct bkey_ops) { \ diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c index 055be0ca92507..b26dc74246623 100644 --- a/fs/bcachefs/ec.c +++ b/fs/bcachefs/ec.c @@ -107,7 +107,7 @@ struct ec_bio { /* Stripes btree keys: */ int bch2_stripe_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { const struct bch_stripe *s = bkey_s_c_to_stripe(k).v; diff --git a/fs/bcachefs/ec.h b/fs/bcachefs/ec.h index a1c2bc74ef458..84a23eeb62495 100644 --- a/fs/bcachefs/ec.h +++ b/fs/bcachefs/ec.h @@ -6,10 +6,10 @@ #include "buckets_types.h" #include "extents_types.h" -enum bkey_invalid_flags; +enum bch_validate_flags; int bch2_stripe_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_stripe_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); int bch2_trigger_stripe(struct btree_trans *, enum btree_id, unsigned, diff --git a/fs/bcachefs/extents.c b/fs/bcachefs/extents.c index 094caf1cdb35b..469037929685b 100644 --- a/fs/bcachefs/extents.c +++ b/fs/bcachefs/extents.c @@ -172,7 +172,7 @@ int bch2_bkey_pick_read_device(struct bch_fs *c, struct bkey_s_c k, /* KEY_TYPE_btree_ptr: */ int bch2_btree_ptr_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { int ret = 0; @@ -193,7 +193,7 @@ void bch2_btree_ptr_to_text(struct printbuf *out, struct bch_fs *c, } int bch2_btree_ptr_v2_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { struct bkey_s_c_btree_ptr_v2 bp = bkey_s_c_to_btree_ptr_v2(k); @@ -208,7 +208,7 @@ int bch2_btree_ptr_v2_invalid(struct bch_fs *c, struct bkey_s_c k, c, err, btree_ptr_v2_min_key_bad, "min_key > key"); - if (flags & BKEY_INVALID_WRITE) + if (flags & BCH_VALIDATE_write) bkey_fsck_err_on(!bp.v->sectors_written, c, err, btree_ptr_v2_written_0, "sectors_written == 0"); @@ -400,7 +400,7 @@ bool bch2_extent_merge(struct bch_fs *c, struct bkey_s l, struct bkey_s_c r) /* KEY_TYPE_reservation: */ int bch2_reservation_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { struct bkey_s_c_reservation r = bkey_s_c_to_reservation(k); @@ -1095,7 +1095,7 @@ void bch2_bkey_ptrs_to_text(struct printbuf *out, struct bch_fs *c, static int extent_ptr_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, const struct bch_extent_ptr *ptr, unsigned size_ondisk, bool metadata, @@ -1138,7 +1138,7 @@ static int extent_ptr_invalid(struct bch_fs *c, } int bch2_bkey_ptrs_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); @@ -1214,7 +1214,7 @@ int bch2_bkey_ptrs_invalid(struct bch_fs *c, struct bkey_s_c k, bkey_fsck_err_on(crc_is_encoded(crc) && (crc.uncompressed_size > c->opts.encoded_extent_max >> 9) && - (flags & (BKEY_INVALID_WRITE|BKEY_INVALID_COMMIT)), c, err, + (flags & (BCH_VALIDATE_write|BCH_VALIDATE_commit)), c, err, ptr_crc_uncompressed_size_too_big, "too large encoded extent"); diff --git a/fs/bcachefs/extents.h b/fs/bcachefs/extents.h index 10e1817e83e3b..1ade959652b2e 100644 --- a/fs/bcachefs/extents.h +++ b/fs/bcachefs/extents.h @@ -8,7 +8,7 @@ struct bch_fs; struct btree_trans; -enum bkey_invalid_flags; +enum bch_validate_flags; /* extent entries: */ @@ -406,12 +406,12 @@ int bch2_bkey_pick_read_device(struct bch_fs *, struct bkey_s_c, /* KEY_TYPE_btree_ptr: */ int bch2_btree_ptr_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_btree_ptr_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); int bch2_btree_ptr_v2_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_btree_ptr_v2_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); void bch2_btree_ptr_v2_compat(enum btree_id, unsigned, unsigned, int, struct bkey_s); @@ -448,7 +448,7 @@ bool bch2_extent_merge(struct bch_fs *, struct bkey_s, struct bkey_s_c); /* KEY_TYPE_reservation: */ int bch2_reservation_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_reservation_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); bool bch2_reservation_merge(struct bch_fs *, struct bkey_s, struct bkey_s_c); @@ -680,7 +680,7 @@ void bch2_extent_ptr_to_text(struct printbuf *out, struct bch_fs *, const struct void bch2_bkey_ptrs_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); int bch2_bkey_ptrs_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_ptr_swab(struct bkey_s); diff --git a/fs/bcachefs/inode.c b/fs/bcachefs/inode.c index 91572e05d5f06..aafa79fa63514 100644 --- a/fs/bcachefs/inode.c +++ b/fs/bcachefs/inode.c @@ -473,7 +473,7 @@ static int __bch2_inode_invalid(struct bch_fs *c, struct bkey_s_c k, struct prin } int bch2_inode_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { struct bkey_s_c_inode inode = bkey_s_c_to_inode(k); @@ -490,7 +490,7 @@ int bch2_inode_invalid(struct bch_fs *c, struct bkey_s_c k, } int bch2_inode_v2_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { struct bkey_s_c_inode_v2 inode = bkey_s_c_to_inode_v2(k); @@ -507,7 +507,7 @@ int bch2_inode_v2_invalid(struct bch_fs *c, struct bkey_s_c k, } int bch2_inode_v3_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { struct bkey_s_c_inode_v3 inode = bkey_s_c_to_inode_v3(k); @@ -635,7 +635,7 @@ int bch2_trigger_inode(struct btree_trans *trans, } int bch2_inode_generation_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { int ret = 0; diff --git a/fs/bcachefs/inode.h b/fs/bcachefs/inode.h index abdfb1f3e4d08..679f5f5e5d154 100644 --- a/fs/bcachefs/inode.h +++ b/fs/bcachefs/inode.h @@ -6,15 +6,15 @@ #include "bkey_methods.h" #include "opts.h" -enum bkey_invalid_flags; +enum bch_validate_flags; extern const char * const bch2_inode_opts[]; int bch2_inode_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); int bch2_inode_v2_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); int bch2_inode_v3_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_inode_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); int bch2_trigger_inode(struct btree_trans *, enum btree_id, unsigned, @@ -50,7 +50,7 @@ static inline bool bkey_is_inode(const struct bkey *k) } int bch2_inode_generation_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_inode_generation_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); #define bch2_bkey_ops_inode_generation ((struct bkey_ops) { \ diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c index 56f4d46ef0941..cdcb1ad49af42 100644 --- a/fs/bcachefs/journal_io.c +++ b/fs/bcachefs/journal_io.c @@ -299,7 +299,7 @@ static void journal_entry_err_msg(struct printbuf *out, journal_entry_err_msg(&_buf, version, jset, entry); \ prt_printf(&_buf, msg, ##__VA_ARGS__); \ \ - switch (flags & BKEY_INVALID_WRITE) { \ + switch (flags & BCH_VALIDATE_write) { \ case READ: \ mustfix_fsck_err(c, _err, "%s", _buf.buf); \ break; \ @@ -328,9 +328,9 @@ static int journal_validate_key(struct bch_fs *c, unsigned level, enum btree_id btree_id, struct bkey_i *k, unsigned version, int big_endian, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { - int write = flags & BKEY_INVALID_WRITE; + int write = flags & BCH_VALIDATE_write; void *next = vstruct_next(entry); struct printbuf buf = PRINTBUF; int ret = 0; @@ -403,7 +403,7 @@ static int journal_entry_btree_keys_validate(struct bch_fs *c, struct jset *jset, struct jset_entry *entry, unsigned version, int big_endian, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { struct bkey_i *k = entry->start; @@ -412,7 +412,7 @@ static int journal_entry_btree_keys_validate(struct bch_fs *c, entry->level, entry->btree_id, k, version, big_endian, - flags|BKEY_INVALID_JOURNAL); + flags|BCH_VALIDATE_journal); if (ret == FSCK_DELETED_KEY) continue; @@ -443,7 +443,7 @@ static int journal_entry_btree_root_validate(struct bch_fs *c, struct jset *jset, struct jset_entry *entry, unsigned version, int big_endian, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { struct bkey_i *k = entry->start; int ret = 0; @@ -482,7 +482,7 @@ static int journal_entry_prio_ptrs_validate(struct bch_fs *c, struct jset *jset, struct jset_entry *entry, unsigned version, int big_endian, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { /* obsolete, don't care: */ return 0; @@ -497,7 +497,7 @@ static int journal_entry_blacklist_validate(struct bch_fs *c, struct jset *jset, struct jset_entry *entry, unsigned version, int big_endian, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { int ret = 0; @@ -524,7 +524,7 @@ static int journal_entry_blacklist_v2_validate(struct bch_fs *c, struct jset *jset, struct jset_entry *entry, unsigned version, int big_endian, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { struct jset_entry_blacklist_v2 *bl_entry; int ret = 0; @@ -566,7 +566,7 @@ static int journal_entry_usage_validate(struct bch_fs *c, struct jset *jset, struct jset_entry *entry, unsigned version, int big_endian, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { struct jset_entry_usage *u = container_of(entry, struct jset_entry_usage, entry); @@ -600,7 +600,7 @@ static int journal_entry_data_usage_validate(struct bch_fs *c, struct jset *jset, struct jset_entry *entry, unsigned version, int big_endian, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { struct jset_entry_data_usage *u = container_of(entry, struct jset_entry_data_usage, entry); @@ -644,7 +644,7 @@ static int journal_entry_clock_validate(struct bch_fs *c, struct jset *jset, struct jset_entry *entry, unsigned version, int big_endian, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { struct jset_entry_clock *clock = container_of(entry, struct jset_entry_clock, entry); @@ -684,7 +684,7 @@ static int journal_entry_dev_usage_validate(struct bch_fs *c, struct jset *jset, struct jset_entry *entry, unsigned version, int big_endian, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { struct jset_entry_dev_usage *u = container_of(entry, struct jset_entry_dev_usage, entry); @@ -735,7 +735,7 @@ static int journal_entry_log_validate(struct bch_fs *c, struct jset *jset, struct jset_entry *entry, unsigned version, int big_endian, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { return 0; } @@ -753,7 +753,7 @@ static int journal_entry_overwrite_validate(struct bch_fs *c, struct jset *jset, struct jset_entry *entry, unsigned version, int big_endian, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { return journal_entry_btree_keys_validate(c, jset, entry, version, big_endian, READ); @@ -769,7 +769,7 @@ static int journal_entry_write_buffer_keys_validate(struct bch_fs *c, struct jset *jset, struct jset_entry *entry, unsigned version, int big_endian, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { return journal_entry_btree_keys_validate(c, jset, entry, version, big_endian, READ); @@ -785,7 +785,7 @@ static int journal_entry_datetime_validate(struct bch_fs *c, struct jset *jset, struct jset_entry *entry, unsigned version, int big_endian, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { unsigned bytes = vstruct_bytes(entry); unsigned expected = 16; @@ -815,7 +815,7 @@ static void journal_entry_datetime_to_text(struct printbuf *out, struct bch_fs * struct jset_entry_ops { int (*validate)(struct bch_fs *, struct jset *, struct jset_entry *, unsigned, int, - enum bkey_invalid_flags); + enum bch_validate_flags); void (*to_text)(struct printbuf *, struct bch_fs *, struct jset_entry *); }; @@ -833,7 +833,7 @@ int bch2_journal_entry_validate(struct bch_fs *c, struct jset *jset, struct jset_entry *entry, unsigned version, int big_endian, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { return entry->type < BCH_JSET_ENTRY_NR ? bch2_jset_entry_ops[entry->type].validate(c, jset, entry, @@ -853,7 +853,7 @@ void bch2_journal_entry_to_text(struct printbuf *out, struct bch_fs *c, } static int jset_validate_entries(struct bch_fs *c, struct jset *jset, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { unsigned version = le32_to_cpu(jset->version); int ret = 0; @@ -879,7 +879,7 @@ static int jset_validate_entries(struct bch_fs *c, struct jset *jset, static int jset_validate(struct bch_fs *c, struct bch_dev *ca, struct jset *jset, u64 sector, - enum bkey_invalid_flags flags) + enum bch_validate_flags flags) { unsigned version; int ret = 0; @@ -934,7 +934,7 @@ static int jset_validate_early(struct bch_fs *c, { size_t bytes = vstruct_bytes(jset); unsigned version; - enum bkey_invalid_flags flags = BKEY_INVALID_JOURNAL; + enum bch_validate_flags flags = BCH_VALIDATE_journal; int ret = 0; if (le64_to_cpu(jset->magic) != jset_magic(c)) @@ -1235,7 +1235,7 @@ int bch2_journal_read(struct bch_fs *c, * those entries will be blacklisted: */ genradix_for_each_reverse(&c->journal_entries, radix_iter, _i) { - enum bkey_invalid_flags flags = BKEY_INVALID_JOURNAL; + enum bch_validate_flags flags = BCH_VALIDATE_journal; i = *_i; diff --git a/fs/bcachefs/journal_io.h b/fs/bcachefs/journal_io.h index 43cfed58ef33f..2ca9cde30ea8d 100644 --- a/fs/bcachefs/journal_io.h +++ b/fs/bcachefs/journal_io.h @@ -63,7 +63,7 @@ static inline struct jset_entry *__jset_entry_type_next(struct jset *jset, int bch2_journal_entry_validate(struct bch_fs *, struct jset *, struct jset_entry *, unsigned, int, - enum bkey_invalid_flags); + enum bch_validate_flags); void bch2_journal_entry_to_text(struct printbuf *, struct bch_fs *, struct jset_entry *); diff --git a/fs/bcachefs/lru.c b/fs/bcachefs/lru.c index 19285fcd252c5..a40d116224edd 100644 --- a/fs/bcachefs/lru.c +++ b/fs/bcachefs/lru.c @@ -11,7 +11,7 @@ /* KEY_TYPE_lru is obsolete: */ int bch2_lru_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { int ret = 0; diff --git a/fs/bcachefs/lru.h b/fs/bcachefs/lru.h index 429dca816df5c..fb11ab0dd00ea 100644 --- a/fs/bcachefs/lru.h +++ b/fs/bcachefs/lru.h @@ -49,7 +49,7 @@ static inline enum bch_lru_type lru_type(struct bkey_s_c l) } int bch2_lru_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_lru_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); void bch2_lru_pos_to_text(struct printbuf *, struct bpos); diff --git a/fs/bcachefs/quota.c b/fs/bcachefs/quota.c index 9d7498e774e20..d86381e9ac869 100644 --- a/fs/bcachefs/quota.c +++ b/fs/bcachefs/quota.c @@ -60,7 +60,7 @@ const struct bch_sb_field_ops bch_sb_field_ops_quota = { }; int bch2_quota_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { int ret = 0; diff --git a/fs/bcachefs/quota.h b/fs/bcachefs/quota.h index 884f601f41c42..02d37a332218a 100644 --- a/fs/bcachefs/quota.h +++ b/fs/bcachefs/quota.h @@ -5,11 +5,11 @@ #include "inode.h" #include "quota_types.h" -enum bkey_invalid_flags; +enum bch_validate_flags; extern const struct bch_sb_field_ops bch_sb_field_ops_quota; int bch2_quota_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_quota_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); #define bch2_bkey_ops_quota ((struct bkey_ops) { \ diff --git a/fs/bcachefs/reflink.c b/fs/bcachefs/reflink.c index fb31a68148ff9..9ac6cf21cfbf7 100644 --- a/fs/bcachefs/reflink.c +++ b/fs/bcachefs/reflink.c @@ -30,7 +30,7 @@ static inline unsigned bkey_type_to_indirect(const struct bkey *k) /* reflink pointers */ int bch2_reflink_p_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { struct bkey_s_c_reflink_p p = bkey_s_c_to_reflink_p(k); @@ -257,7 +257,7 @@ int bch2_trigger_reflink_p(struct btree_trans *trans, /* indirect extents */ int bch2_reflink_v_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { return bch2_bkey_ptrs_invalid(c, k, flags, err); @@ -312,7 +312,7 @@ int bch2_trigger_reflink_v(struct btree_trans *trans, /* indirect inline data */ int bch2_indirect_inline_data_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { return 0; diff --git a/fs/bcachefs/reflink.h b/fs/bcachefs/reflink.h index 59d4dbae3fb1f..e894f3a2c67ac 100644 --- a/fs/bcachefs/reflink.h +++ b/fs/bcachefs/reflink.h @@ -2,10 +2,10 @@ #ifndef _BCACHEFS_REFLINK_H #define _BCACHEFS_REFLINK_H -enum bkey_invalid_flags; +enum bch_validate_flags; int bch2_reflink_p_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_reflink_p_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); bool bch2_reflink_p_merge(struct bch_fs *, struct bkey_s, struct bkey_s_c); @@ -22,7 +22,7 @@ int bch2_trigger_reflink_p(struct btree_trans *, enum btree_id, unsigned, }) int bch2_reflink_v_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_reflink_v_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); int bch2_trigger_reflink_v(struct btree_trans *, enum btree_id, unsigned, @@ -38,7 +38,7 @@ int bch2_trigger_reflink_v(struct btree_trans *, enum btree_id, unsigned, }) int bch2_indirect_inline_data_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_indirect_inline_data_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); int bch2_trigger_indirect_inline_data(struct btree_trans *, diff --git a/fs/bcachefs/snapshot.c b/fs/bcachefs/snapshot.c index 3aa4686cc71fc..629900a5e6411 100644 --- a/fs/bcachefs/snapshot.c +++ b/fs/bcachefs/snapshot.c @@ -32,7 +32,7 @@ void bch2_snapshot_tree_to_text(struct printbuf *out, struct bch_fs *c, } int bch2_snapshot_tree_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { int ret = 0; @@ -223,7 +223,7 @@ void bch2_snapshot_to_text(struct printbuf *out, struct bch_fs *c, } int bch2_snapshot_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { struct bkey_s_c_snapshot s; diff --git a/fs/bcachefs/snapshot.h b/fs/bcachefs/snapshot.h index b3c43d5795402..bd5d74269d15a 100644 --- a/fs/bcachefs/snapshot.h +++ b/fs/bcachefs/snapshot.h @@ -2,11 +2,11 @@ #ifndef _BCACHEFS_SNAPSHOT_H #define _BCACHEFS_SNAPSHOT_H -enum bkey_invalid_flags; +enum bch_validate_flags; void bch2_snapshot_tree_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); int bch2_snapshot_tree_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); #define bch2_bkey_ops_snapshot_tree ((struct bkey_ops) { \ .key_invalid = bch2_snapshot_tree_invalid, \ @@ -20,7 +20,7 @@ int bch2_snapshot_tree_lookup(struct btree_trans *, u32, struct bch_snapshot_tre void bch2_snapshot_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); int bch2_snapshot_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); int bch2_mark_snapshot(struct btree_trans *, enum btree_id, unsigned, struct bkey_s_c, struct bkey_s, enum btree_iter_update_trigger_flags); diff --git a/fs/bcachefs/subvolume.c b/fs/bcachefs/subvolume.c index 3341285e75317..132213761ef64 100644 --- a/fs/bcachefs/subvolume.c +++ b/fs/bcachefs/subvolume.c @@ -208,7 +208,7 @@ int bch2_check_subvol_children(struct bch_fs *c) /* Subvolumes: */ int bch2_subvolume_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, struct printbuf *err) + enum bch_validate_flags flags, struct printbuf *err) { int ret = 0; diff --git a/fs/bcachefs/subvolume.h b/fs/bcachefs/subvolume.h index 1530a10327ab3..afa5e871efb25 100644 --- a/fs/bcachefs/subvolume.h +++ b/fs/bcachefs/subvolume.h @@ -5,13 +5,13 @@ #include "darray.h" #include "subvolume_types.h" -enum bkey_invalid_flags; +enum bch_validate_flags; int bch2_check_subvols(struct bch_fs *); int bch2_check_subvol_children(struct bch_fs *); int bch2_subvolume_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_subvolume_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); int bch2_subvolume_trigger(struct btree_trans *, enum btree_id, unsigned, struct bkey_s_c, struct bkey_s, diff --git a/fs/bcachefs/xattr.c b/fs/bcachefs/xattr.c index 786ccd7a7662d..c11bf6dacc2c7 100644 --- a/fs/bcachefs/xattr.c +++ b/fs/bcachefs/xattr.c @@ -71,7 +71,7 @@ const struct bch_hash_desc bch2_xattr_hash_desc = { }; int bch2_xattr_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bkey_invalid_flags flags, + enum bch_validate_flags flags, struct printbuf *err) { struct bkey_s_c_xattr xattr = bkey_s_c_to_xattr(k); diff --git a/fs/bcachefs/xattr.h b/fs/bcachefs/xattr.h index 1337f31a5c492..1574b9eb4c850 100644 --- a/fs/bcachefs/xattr.h +++ b/fs/bcachefs/xattr.h @@ -7,7 +7,7 @@ extern const struct bch_hash_desc bch2_xattr_hash_desc; int bch2_xattr_invalid(struct bch_fs *, struct bkey_s_c, - enum bkey_invalid_flags, struct printbuf *); + enum bch_validate_flags, struct printbuf *); void bch2_xattr_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); #define bch2_bkey_ops_xattr ((struct bkey_ops) { \ From a5c3e265d3b61ab661df4f259a97b57840cb041e Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 8 May 2024 18:49:14 -0400 Subject: [PATCH 1532/1702] bcachefs: Plumb bch_validate_flags to sb_field_ops.validate() Signed-off-by: Kent Overstreet --- fs/bcachefs/checksum.c | 5 ++--- fs/bcachefs/disk_groups.c | 5 ++--- fs/bcachefs/journal_sb.c | 10 ++++------ fs/bcachefs/journal_seq_blacklist.c | 5 ++--- fs/bcachefs/quota.c | 5 ++--- fs/bcachefs/replicas.c | 4 ++-- fs/bcachefs/sb-clean.c | 5 ++--- fs/bcachefs/sb-counters.c | 5 ++--- fs/bcachefs/sb-downgrade.c | 2 +- fs/bcachefs/sb-errors.c | 2 +- fs/bcachefs/sb-members.c | 10 ++++------ fs/bcachefs/super-io.c | 24 ++++++++++++------------ fs/bcachefs/super-io.h | 3 ++- 13 files changed, 38 insertions(+), 47 deletions(-) diff --git a/fs/bcachefs/checksum.c b/fs/bcachefs/checksum.c index 2b3677ae09425..85198f391e9c8 100644 --- a/fs/bcachefs/checksum.c +++ b/fs/bcachefs/checksum.c @@ -469,9 +469,8 @@ int bch2_rechecksum_bio(struct bch_fs *c, struct bio *bio, /* BCH_SB_FIELD_crypt: */ -static int bch2_sb_crypt_validate(struct bch_sb *sb, - struct bch_sb_field *f, - struct printbuf *err) +static int bch2_sb_crypt_validate(struct bch_sb *sb, struct bch_sb_field *f, + enum bch_validate_flags flags, struct printbuf *err) { struct bch_sb_field_crypt *crypt = field_to_type(f, crypt); diff --git a/fs/bcachefs/disk_groups.c b/fs/bcachefs/disk_groups.c index 1cb563dfa0176..521a86df5e52a 100644 --- a/fs/bcachefs/disk_groups.c +++ b/fs/bcachefs/disk_groups.c @@ -18,9 +18,8 @@ static int group_cmp(const void *_l, const void *_r) strncmp(l->label, r->label, sizeof(l->label)); } -static int bch2_sb_disk_groups_validate(struct bch_sb *sb, - struct bch_sb_field *f, - struct printbuf *err) +static int bch2_sb_disk_groups_validate(struct bch_sb *sb, struct bch_sb_field *f, + enum bch_validate_flags flags, struct printbuf *err) { struct bch_sb_field_disk_groups *groups = field_to_type(f, disk_groups); diff --git a/fs/bcachefs/journal_sb.c b/fs/bcachefs/journal_sb.c index ae4fb8c3a2bc2..db80e506e3abe 100644 --- a/fs/bcachefs/journal_sb.c +++ b/fs/bcachefs/journal_sb.c @@ -16,9 +16,8 @@ static int u64_cmp(const void *_l, const void *_r) return cmp_int(*l, *r); } -static int bch2_sb_journal_validate(struct bch_sb *sb, - struct bch_sb_field *f, - struct printbuf *err) +static int bch2_sb_journal_validate(struct bch_sb *sb, struct bch_sb_field *f, + enum bch_validate_flags flags, struct printbuf *err) { struct bch_sb_field_journal *journal = field_to_type(f, journal); struct bch_member m = bch2_sb_member_get(sb, sb->dev_idx); @@ -99,9 +98,8 @@ static int u64_range_cmp(const void *_l, const void *_r) return cmp_int(l->start, r->start); } -static int bch2_sb_journal_v2_validate(struct bch_sb *sb, - struct bch_sb_field *f, - struct printbuf *err) +static int bch2_sb_journal_v2_validate(struct bch_sb *sb, struct bch_sb_field *f, + enum bch_validate_flags flags, struct printbuf *err) { struct bch_sb_field_journal_v2 *journal = field_to_type(f, journal_v2); struct bch_member m = bch2_sb_member_get(sb, sb->dev_idx); diff --git a/fs/bcachefs/journal_seq_blacklist.c b/fs/bcachefs/journal_seq_blacklist.c index d8d40d46a27a2..ed48467096115 100644 --- a/fs/bcachefs/journal_seq_blacklist.c +++ b/fs/bcachefs/journal_seq_blacklist.c @@ -162,9 +162,8 @@ int bch2_blacklist_table_initialize(struct bch_fs *c) return 0; } -static int bch2_sb_journal_seq_blacklist_validate(struct bch_sb *sb, - struct bch_sb_field *f, - struct printbuf *err) +static int bch2_sb_journal_seq_blacklist_validate(struct bch_sb *sb, struct bch_sb_field *f, + enum bch_validate_flags flags, struct printbuf *err) { struct bch_sb_field_journal_seq_blacklist *bl = field_to_type(f, journal_seq_blacklist); diff --git a/fs/bcachefs/quota.c b/fs/bcachefs/quota.c index d86381e9ac869..a0cca8b70e0ae 100644 --- a/fs/bcachefs/quota.c +++ b/fs/bcachefs/quota.c @@ -20,7 +20,7 @@ static const char * const bch2_quota_counters[] = { }; static int bch2_sb_quota_validate(struct bch_sb *sb, struct bch_sb_field *f, - struct printbuf *err) + enum bch_validate_flags flags, struct printbuf *err) { struct bch_sb_field_quota *q = field_to_type(f, quota); @@ -60,8 +60,7 @@ const struct bch_sb_field_ops bch_sb_field_ops_quota = { }; int bch2_quota_invalid(struct bch_fs *c, struct bkey_s_c k, - enum bch_validate_flags flags, - struct printbuf *err) + enum bch_validate_flags flags, struct printbuf *err) { int ret = 0; diff --git a/fs/bcachefs/replicas.c b/fs/bcachefs/replicas.c index f8ff7c8bb05ee..bd1d5d085e233 100644 --- a/fs/bcachefs/replicas.c +++ b/fs/bcachefs/replicas.c @@ -860,7 +860,7 @@ static int bch2_cpu_replicas_validate(struct bch_replicas_cpu *cpu_r, } static int bch2_sb_replicas_validate(struct bch_sb *sb, struct bch_sb_field *f, - struct printbuf *err) + enum bch_validate_flags flags, struct printbuf *err) { struct bch_sb_field_replicas *sb_r = field_to_type(f, replicas); struct bch_replicas_cpu cpu_r; @@ -899,7 +899,7 @@ const struct bch_sb_field_ops bch_sb_field_ops_replicas = { }; static int bch2_sb_replicas_v0_validate(struct bch_sb *sb, struct bch_sb_field *f, - struct printbuf *err) + enum bch_validate_flags flags, struct printbuf *err) { struct bch_sb_field_replicas_v0 *sb_r = field_to_type(f, replicas_v0); struct bch_replicas_cpu cpu_r; diff --git a/fs/bcachefs/sb-clean.c b/fs/bcachefs/sb-clean.c index e2beb2308c70c..47f10ab57f40c 100644 --- a/fs/bcachefs/sb-clean.c +++ b/fs/bcachefs/sb-clean.c @@ -266,9 +266,8 @@ void bch2_journal_super_entries_add_common(struct bch_fs *c, } } -static int bch2_sb_clean_validate(struct bch_sb *sb, - struct bch_sb_field *f, - struct printbuf *err) +static int bch2_sb_clean_validate(struct bch_sb *sb, struct bch_sb_field *f, + enum bch_validate_flags flags, struct printbuf *err) { struct bch_sb_field_clean *clean = field_to_type(f, clean); diff --git a/fs/bcachefs/sb-counters.c b/fs/bcachefs/sb-counters.c index 7282f78861926..6992e7469112c 100644 --- a/fs/bcachefs/sb-counters.c +++ b/fs/bcachefs/sb-counters.c @@ -20,9 +20,8 @@ static size_t bch2_sb_counter_nr_entries(struct bch_sb_field_counters *ctrs) return (__le64 *) vstruct_end(&ctrs->field) - &ctrs->d[0]; }; -static int bch2_sb_counters_validate(struct bch_sb *sb, - struct bch_sb_field *f, - struct printbuf *err) +static int bch2_sb_counters_validate(struct bch_sb *sb, struct bch_sb_field *f, + enum bch_validate_flags flags, struct printbuf *err) { return 0; }; diff --git a/fs/bcachefs/sb-downgrade.c b/fs/bcachefs/sb-downgrade.c index 90b06f8aa1df0..7d2abadbd6e39 100644 --- a/fs/bcachefs/sb-downgrade.c +++ b/fs/bcachefs/sb-downgrade.c @@ -138,7 +138,7 @@ downgrade_entry_next_c(const struct bch_sb_field_downgrade_entry *e) _i = downgrade_entry_next_c(_i)) static int bch2_sb_downgrade_validate(struct bch_sb *sb, struct bch_sb_field *f, - struct printbuf *err) + enum bch_validate_flags flags, struct printbuf *err) { struct bch_sb_field_downgrade *e = field_to_type(f, downgrade); diff --git a/fs/bcachefs/sb-errors.c b/fs/bcachefs/sb-errors.c index 5f5bcae391fb9..bda33e59e2264 100644 --- a/fs/bcachefs/sb-errors.c +++ b/fs/bcachefs/sb-errors.c @@ -30,7 +30,7 @@ static inline unsigned bch2_sb_field_errors_u64s(unsigned nr) } static int bch2_sb_errors_validate(struct bch_sb *sb, struct bch_sb_field *f, - struct printbuf *err) + enum bch_validate_flags flags, struct printbuf *err) { struct bch_sb_field_errors *e = field_to_type(f, errors); unsigned i, nr = bch2_sb_field_errors_nr_entries(e); diff --git a/fs/bcachefs/sb-members.c b/fs/bcachefs/sb-members.c index 8f197bb088a06..39196f2a41974 100644 --- a/fs/bcachefs/sb-members.c +++ b/fs/bcachefs/sb-members.c @@ -261,9 +261,8 @@ static void member_to_text(struct printbuf *out, printbuf_indent_sub(out, 2); } -static int bch2_sb_members_v1_validate(struct bch_sb *sb, - struct bch_sb_field *f, - struct printbuf *err) +static int bch2_sb_members_v1_validate(struct bch_sb *sb, struct bch_sb_field *f, + enum bch_validate_flags flags, struct printbuf *err) { struct bch_sb_field_members_v1 *mi = field_to_type(f, members_v1); unsigned i; @@ -311,9 +310,8 @@ static void bch2_sb_members_v2_to_text(struct printbuf *out, struct bch_sb *sb, member_to_text(out, members_v2_get(mi, i), gi, sb, i); } -static int bch2_sb_members_v2_validate(struct bch_sb *sb, - struct bch_sb_field *f, - struct printbuf *err) +static int bch2_sb_members_v2_validate(struct bch_sb *sb, struct bch_sb_field *f, + enum bch_validate_flags flags, struct printbuf *err) { struct bch_sb_field_members_v2 *mi = field_to_type(f, members_v2); size_t mi_bytes = (void *) __bch2_members_v2_get_mut(mi, sb->nr_devices) - diff --git a/fs/bcachefs/super-io.c b/fs/bcachefs/super-io.c index fb45749586811..f1bee6c5222d2 100644 --- a/fs/bcachefs/super-io.c +++ b/fs/bcachefs/super-io.c @@ -76,7 +76,7 @@ const char * const bch2_sb_fields[] = { }; static int bch2_sb_field_validate(struct bch_sb *, struct bch_sb_field *, - struct printbuf *); + enum bch_validate_flags, struct printbuf *); struct bch_sb_field *bch2_sb_field_get_id(struct bch_sb *sb, enum bch_sb_field_type type) @@ -344,8 +344,8 @@ static int bch2_sb_compatible(struct bch_sb *sb, struct printbuf *out) return 0; } -static int bch2_sb_validate(struct bch_sb_handle *disk_sb, struct printbuf *out, - int rw) +static int bch2_sb_validate(struct bch_sb_handle *disk_sb, + enum bch_validate_flags flags, struct printbuf *out) { struct bch_sb *sb = disk_sb->sb; struct bch_sb_field_members_v1 *mi; @@ -401,7 +401,7 @@ static int bch2_sb_validate(struct bch_sb_handle *disk_sb, struct printbuf *out, return -BCH_ERR_invalid_sb_time_precision; } - if (rw == READ) { + if (!flags) { /* * Been seeing a bug where these are getting inexplicably * zeroed, so we're now validating them, but we have to be @@ -457,7 +457,7 @@ static int bch2_sb_validate(struct bch_sb_handle *disk_sb, struct printbuf *out, return -BCH_ERR_invalid_sb_members_missing; } - ret = bch2_sb_field_validate(sb, &mi->field, out); + ret = bch2_sb_field_validate(sb, &mi->field, flags, out); if (ret) return ret; @@ -465,12 +465,12 @@ static int bch2_sb_validate(struct bch_sb_handle *disk_sb, struct printbuf *out, if (le32_to_cpu(f->type) == BCH_SB_FIELD_members_v1) continue; - ret = bch2_sb_field_validate(sb, f, out); + ret = bch2_sb_field_validate(sb, f, flags, out); if (ret) return ret; } - if (rw == WRITE && + if ((flags & BCH_VALIDATE_write) && bch2_sb_member_get(sb, sb->dev_idx).seq != sb->seq) { prt_printf(out, "Invalid superblock: member seq %llu != sb seq %llu", le64_to_cpu(bch2_sb_member_get(sb, sb->dev_idx).seq), @@ -819,7 +819,7 @@ static int __bch2_read_super(const char *path, struct bch_opts *opts, sb->have_layout = true; - ret = bch2_sb_validate(sb, &err, READ); + ret = bch2_sb_validate(sb, 0, &err); if (ret) { bch2_print_opts(opts, KERN_ERR "bcachefs (%s): error validating superblock: %s\n", path, err.buf); @@ -975,7 +975,7 @@ int bch2_write_super(struct bch_fs *c) darray_for_each(online_devices, ca) { printbuf_reset(&err); - ret = bch2_sb_validate(&(*ca)->disk_sb, &err, WRITE); + ret = bch2_sb_validate(&(*ca)->disk_sb, BCH_VALIDATE_write, &err); if (ret) { bch2_fs_inconsistent(c, "sb invalid before write: %s", err.buf); goto out; @@ -1161,7 +1161,7 @@ void bch2_sb_upgrade(struct bch_fs *c, unsigned new_version) } static int bch2_sb_ext_validate(struct bch_sb *sb, struct bch_sb_field *f, - struct printbuf *err) + enum bch_validate_flags flags, struct printbuf *err) { if (vstruct_bytes(f) < 88) { prt_printf(err, "field too small (%zu < %u)", vstruct_bytes(f), 88); @@ -1219,14 +1219,14 @@ static const struct bch_sb_field_ops *bch2_sb_field_type_ops(unsigned type) } static int bch2_sb_field_validate(struct bch_sb *sb, struct bch_sb_field *f, - struct printbuf *err) + enum bch_validate_flags flags, struct printbuf *err) { unsigned type = le32_to_cpu(f->type); struct printbuf field_err = PRINTBUF; const struct bch_sb_field_ops *ops = bch2_sb_field_type_ops(type); int ret; - ret = ops->validate ? ops->validate(sb, f, &field_err) : 0; + ret = ops->validate ? ops->validate(sb, f, flags, &field_err) : 0; if (ret) { prt_printf(err, "Invalid superblock section %s: %s", bch2_sb_fields[type], field_err.buf); diff --git a/fs/bcachefs/super-io.h b/fs/bcachefs/super-io.h index 95e80e06316bf..fadd364e2802a 100644 --- a/fs/bcachefs/super-io.h +++ b/fs/bcachefs/super-io.h @@ -51,7 +51,8 @@ void bch2_sb_field_delete(struct bch_sb_handle *, enum bch_sb_field_type); extern const char * const bch2_sb_fields[]; struct bch_sb_field_ops { - int (*validate)(struct bch_sb *, struct bch_sb_field *, struct printbuf *); + int (*validate)(struct bch_sb *, struct bch_sb_field *, + enum bch_validate_flags, struct printbuf *); void (*to_text)(struct printbuf *, struct bch_sb *, struct bch_sb_field *); }; From 692aa7a54b2b28d59f24b3bf8250837805484b99 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 6 May 2024 09:16:33 -0400 Subject: [PATCH 1533/1702] bcachefs: Fix sb_field_downgrade validation - bch2_sb_downgrade_validate() wasn't checking for a downgrade entry extending past the end of the superblock section - for_each_downgrade_entry() is used in to_text() and needs to work on malformed input; it also was missing a check for a field extending past the end of the section Reported-by: syzbot+e49ccab73449180bc9be@syzkaller.appspotmail.com Fixes: 84f1638795da ("bcachefs: bch_sb_field_downgrade") Signed-off-by: Kent Overstreet --- fs/bcachefs/sb-downgrade.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/fs/bcachefs/sb-downgrade.c b/fs/bcachefs/sb-downgrade.c index 7d2abadbd6e39..390a1bbd2567a 100644 --- a/fs/bcachefs/sb-downgrade.c +++ b/fs/bcachefs/sb-downgrade.c @@ -134,7 +134,8 @@ downgrade_entry_next_c(const struct bch_sb_field_downgrade_entry *e) #define for_each_downgrade_entry(_d, _i) \ for (const struct bch_sb_field_downgrade_entry *_i = (_d)->entries; \ (void *) _i < vstruct_end(&(_d)->field) && \ - (void *) &_i->errors[0] < vstruct_end(&(_d)->field); \ + (void *) &_i->errors[0] <= vstruct_end(&(_d)->field) && \ + (void *) downgrade_entry_next_c(_i) <= vstruct_end(&(_d)->field); \ _i = downgrade_entry_next_c(_i)) static int bch2_sb_downgrade_validate(struct bch_sb *sb, struct bch_sb_field *f, @@ -142,7 +143,16 @@ static int bch2_sb_downgrade_validate(struct bch_sb *sb, struct bch_sb_field *f, { struct bch_sb_field_downgrade *e = field_to_type(f, downgrade); - for_each_downgrade_entry(e, i) { + for (const struct bch_sb_field_downgrade_entry *i = e->entries; + (void *) i < vstruct_end(&e->field); + i = downgrade_entry_next_c(i)) { + if (flags & BCH_VALIDATE_write && + ((void *) &i->errors[0] > vstruct_end(&e->field) || + (void *) downgrade_entry_next_c(i) > vstruct_end(&e->field))) { + prt_printf(err, "downgrade entry overruns end of superblock section)"); + return -BCH_ERR_invalid_sb_downgrade; + } + if (BCH_VERSION_MAJOR(le16_to_cpu(i->version)) != BCH_VERSION_MAJOR(le16_to_cpu(sb->version))) { prt_printf(err, "downgrade entry with mismatched major version (%u != %u)", From bceacfa97ec8b67a76efad2f95899434230b317c Mon Sep 17 00:00:00 2001 From: Daniel Hill Date: Fri, 30 Sep 2022 16:37:15 +1300 Subject: [PATCH 1534/1702] bcachefs: add counters for failed shrinker reclaim This adds distinct counters for every reason the btree node shrinker can fail to free an object - if our shrinker isn't making progress, this will tell us why. Signed-off-by: Daniel Hill Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_cache.c | 88 ++++++++++++++++++++++++++++++--------- fs/bcachefs/btree_cache.h | 2 +- fs/bcachefs/btree_types.h | 10 +++++ fs/bcachefs/sysfs.c | 2 +- 4 files changed, 80 insertions(+), 22 deletions(-) diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c index 7dd35d6224d94..9e4ed75d36756 100644 --- a/fs/bcachefs/btree_cache.c +++ b/fs/bcachefs/btree_cache.c @@ -16,6 +16,12 @@ #include #include +#define BTREE_CACHE_NOT_FREED_INCREMENT(counter) \ +do { \ + if (shrinker_counter) \ + bc->not_freed_##counter++; \ +} while (0) + const char * const bch2_btree_node_flags[] = { #define x(f) #f, BTREE_FLAGS() @@ -238,7 +244,7 @@ static inline struct btree *btree_cache_find(struct btree_cache *bc, * this version is for btree nodes that have already been freed (we're not * reaping a real btree node) */ -static int __btree_node_reclaim(struct bch_fs *c, struct btree *b, bool flush) +static int __btree_node_reclaim(struct bch_fs *c, struct btree *b, bool flush, bool shrinker_counter) { struct btree_cache *bc = &c->btree_cache; int ret = 0; @@ -260,38 +266,64 @@ static int __btree_node_reclaim(struct bch_fs *c, struct btree *b, bool flush) if (b->flags & ((1U << BTREE_NODE_dirty)| (1U << BTREE_NODE_read_in_flight)| (1U << BTREE_NODE_write_in_flight))) { - if (!flush) + if (!flush) { + if (btree_node_dirty(b)) + BTREE_CACHE_NOT_FREED_INCREMENT(dirty); + else if (btree_node_read_in_flight(b)) + BTREE_CACHE_NOT_FREED_INCREMENT(read_in_flight); + else if (btree_node_write_in_flight(b)) + BTREE_CACHE_NOT_FREED_INCREMENT(write_in_flight); return -BCH_ERR_ENOMEM_btree_node_reclaim; + } /* XXX: waiting on IO with btree cache lock held */ bch2_btree_node_wait_on_read(b); bch2_btree_node_wait_on_write(b); } - if (!six_trylock_intent(&b->c.lock)) + if (!six_trylock_intent(&b->c.lock)) { + BTREE_CACHE_NOT_FREED_INCREMENT(lock_intent); return -BCH_ERR_ENOMEM_btree_node_reclaim; + } - if (!six_trylock_write(&b->c.lock)) + if (!six_trylock_write(&b->c.lock)) { + BTREE_CACHE_NOT_FREED_INCREMENT(lock_write); goto out_unlock_intent; + } /* recheck under lock */ if (b->flags & ((1U << BTREE_NODE_read_in_flight)| (1U << BTREE_NODE_write_in_flight))) { - if (!flush) + if (!flush) { + if (btree_node_read_in_flight(b)) + BTREE_CACHE_NOT_FREED_INCREMENT(read_in_flight); + else if (btree_node_write_in_flight(b)) + BTREE_CACHE_NOT_FREED_INCREMENT(write_in_flight); goto out_unlock; + } six_unlock_write(&b->c.lock); six_unlock_intent(&b->c.lock); goto wait_on_io; } - if (btree_node_noevict(b) || - btree_node_write_blocked(b) || - btree_node_will_make_reachable(b)) + if (btree_node_noevict(b)) { + BTREE_CACHE_NOT_FREED_INCREMENT(noevict); + goto out_unlock; + } + if (btree_node_write_blocked(b)) { + BTREE_CACHE_NOT_FREED_INCREMENT(write_blocked); goto out_unlock; + } + if (btree_node_will_make_reachable(b)) { + BTREE_CACHE_NOT_FREED_INCREMENT(will_make_reachable); + goto out_unlock; + } if (btree_node_dirty(b)) { - if (!flush) + if (!flush) { + BTREE_CACHE_NOT_FREED_INCREMENT(dirty); goto out_unlock; + } /* * Using the underscore version because we don't want to compact * bsets after the write, since this node is about to be evicted @@ -321,14 +353,14 @@ static int __btree_node_reclaim(struct bch_fs *c, struct btree *b, bool flush) goto out; } -static int btree_node_reclaim(struct bch_fs *c, struct btree *b) +static int btree_node_reclaim(struct bch_fs *c, struct btree *b, bool shrinker_counter) { - return __btree_node_reclaim(c, b, false); + return __btree_node_reclaim(c, b, false, shrinker_counter); } static int btree_node_write_and_reclaim(struct bch_fs *c, struct btree *b) { - return __btree_node_reclaim(c, b, true); + return __btree_node_reclaim(c, b, true, false); } static unsigned long bch2_btree_cache_scan(struct shrinker *shrink, @@ -376,11 +408,12 @@ static unsigned long bch2_btree_cache_scan(struct shrinker *shrink, if (touched >= nr) goto out; - if (!btree_node_reclaim(c, b)) { + if (!btree_node_reclaim(c, b, true)) { btree_node_data_free(c, b); six_unlock_write(&b->c.lock); six_unlock_intent(&b->c.lock); freed++; + bc->freed++; } } restart: @@ -389,9 +422,11 @@ static unsigned long bch2_btree_cache_scan(struct shrinker *shrink, if (btree_node_accessed(b)) { clear_btree_node_accessed(b); - } else if (!btree_node_reclaim(c, b)) { + bc->not_freed_access_bit++; + } else if (!btree_node_reclaim(c, b, true)) { freed++; btree_node_data_free(c, b); + bc->freed++; bch2_btree_node_hash_remove(bc, b); six_unlock_write(&b->c.lock); @@ -599,7 +634,7 @@ static struct btree *btree_node_cannibalize(struct bch_fs *c) struct btree *b; list_for_each_entry_reverse(b, &bc->live, list) - if (!btree_node_reclaim(c, b)) + if (!btree_node_reclaim(c, b, false)) return b; while (1) { @@ -635,7 +670,7 @@ struct btree *bch2_btree_node_mem_alloc(struct btree_trans *trans, bool pcpu_rea * disk node. Check the freed list before allocating a new one: */ list_for_each_entry(b, freed, list) - if (!btree_node_reclaim(c, b)) { + if (!btree_node_reclaim(c, b, false)) { list_del_init(&b->list); goto got_node; } @@ -661,7 +696,7 @@ struct btree *bch2_btree_node_mem_alloc(struct btree_trans *trans, bool pcpu_rea * the list. Check if there's any freed nodes there: */ list_for_each_entry(b2, &bc->freeable, list) - if (!btree_node_reclaim(c, b2)) { + if (!btree_node_reclaim(c, b2, false)) { swap(b->data, b2->data); swap(b->aux_data, b2->aux_data); btree_node_to_freedlist(bc, b2); @@ -1280,12 +1315,12 @@ static void prt_btree_cache_line(struct printbuf *out, const struct bch_fs *c, prt_printf(out, " (%u)\n", nr); } -void bch2_btree_cache_to_text(struct printbuf *out, const struct bch_fs *c) +void bch2_btree_cache_to_text(struct printbuf *out, const struct btree_cache *bc) { - const struct btree_cache *bc = &c->btree_cache; + struct bch_fs *c = container_of(bc, struct bch_fs, btree_cache); if (!out->nr_tabstops) - printbuf_tabstop_push(out, 24); + printbuf_tabstop_push(out, 32); prt_btree_cache_line(out, c, "total:", bc->used); prt_btree_cache_line(out, c, "nr dirty:", atomic_read(&bc->dirty)); @@ -1294,4 +1329,17 @@ void bch2_btree_cache_to_text(struct printbuf *out, const struct bch_fs *c) for (unsigned i = 0; i < ARRAY_SIZE(bc->used_by_btree); i++) prt_btree_cache_line(out, c, bch2_btree_id_str(i), bc->used_by_btree[i]); + + prt_newline(out); + prt_printf(out, "freed:\t%u\n", bc->freed); + prt_printf(out, "not freed:\n"); + prt_printf(out, " dirty\t%u\n", bc->not_freed_dirty); + prt_printf(out, " write in flight\t%u\n", bc->not_freed_write_in_flight); + prt_printf(out, " read in flight\t%u\n", bc->not_freed_read_in_flight); + prt_printf(out, " lock intent failed\t%u\n", bc->not_freed_lock_intent); + prt_printf(out, " lock write failed\t%u\n", bc->not_freed_lock_write); + prt_printf(out, " access bit\t%u\n", bc->not_freed_access_bit); + prt_printf(out, " no evict failed\t%u\n", bc->not_freed_noevict); + prt_printf(out, " write blocked\t%u\n", bc->not_freed_write_blocked); + prt_printf(out, " will make reachable\t%u\n", bc->not_freed_will_make_reachable); } diff --git a/fs/bcachefs/btree_cache.h b/fs/bcachefs/btree_cache.h index 6fe91d1c0fd4c..fed35de3e4de7 100644 --- a/fs/bcachefs/btree_cache.h +++ b/fs/bcachefs/btree_cache.h @@ -134,6 +134,6 @@ static inline struct btree *btree_node_root(struct bch_fs *c, struct btree *b) const char *bch2_btree_id_str(enum btree_id); void bch2_btree_pos_to_text(struct printbuf *, struct bch_fs *, const struct btree *); void bch2_btree_node_to_text(struct printbuf *, struct bch_fs *, const struct btree *); -void bch2_btree_cache_to_text(struct printbuf *, const struct bch_fs *); +void bch2_btree_cache_to_text(struct printbuf *, const struct btree_cache *); #endif /* _BCACHEFS_BTREE_CACHE_H */ diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index 76364bd4347e3..d63db4fefe734 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -163,6 +163,16 @@ struct btree_cache { /* Number of elements in live + freeable lists */ unsigned used; unsigned reserve; + unsigned freed; + unsigned not_freed_lock_intent; + unsigned not_freed_lock_write; + unsigned not_freed_dirty; + unsigned not_freed_read_in_flight; + unsigned not_freed_write_in_flight; + unsigned not_freed_noevict; + unsigned not_freed_write_blocked; + unsigned not_freed_will_make_reachable; + unsigned not_freed_access_bit; atomic_t dirty; struct shrinker *shrink; diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c index df3d28659bfd4..93ca74d108b17 100644 --- a/fs/bcachefs/sysfs.c +++ b/fs/bcachefs/sysfs.c @@ -383,7 +383,7 @@ SHOW(bch2_fs) bch2_journal_debug_to_text(out, &c->journal); if (attr == &sysfs_btree_cache) - bch2_btree_cache_to_text(out, c); + bch2_btree_cache_to_text(out, &c->btree_cache); if (attr == &sysfs_btree_key_cache) bch2_btree_key_cache_to_text(out, &c->btree_key_cache); From 07f9a27f1969764d11374942961d51fee0ab628f Mon Sep 17 00:00:00 2001 From: Thomas Bertschinger Date: Thu, 9 May 2024 12:37:24 -0600 Subject: [PATCH 1535/1702] bcachefs: add no_invalid_checks flag Setting this flag on a filesystem results in validity checks being skipped when writing bkeys. This flag will be used by tooling that deliberately injects corruption into a filesystem in order to exercise fsck. It shouldn't be set outside of testing/debugging code. Signed-off-by: Thomas Bertschinger Signed-off-by: Kent Overstreet --- fs/bcachefs/bcachefs.h | 3 ++- fs/bcachefs/bkey_methods.c | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index ab2dc2c70da21..bc0ea2c4efef2 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -637,7 +637,8 @@ struct bch_dev { x(error) \ x(topology_error) \ x(errors_fixed) \ - x(errors_not_fixed) + x(errors_not_fixed) \ + x(no_invalid_checks) enum bch_fs_flags { #define x(n) BCH_FS_##n, diff --git a/fs/bcachefs/bkey_methods.c b/fs/bcachefs/bkey_methods.c index f692f9d8f00da..c2c3dae521865 100644 --- a/fs/bcachefs/bkey_methods.c +++ b/fs/bcachefs/bkey_methods.c @@ -126,6 +126,9 @@ int bch2_bkey_val_invalid(struct bch_fs *c, struct bkey_s_c k, enum bch_validate_flags flags, struct printbuf *err) { + if (test_bit(BCH_FS_no_invalid_checks, &c->flags)) + return 0; + const struct bkey_ops *ops = bch2_bkey_type_ops(k.k->type); int ret = 0; @@ -162,6 +165,9 @@ int __bch2_bkey_invalid(struct bch_fs *c, struct bkey_s_c k, enum bch_validate_flags flags, struct printbuf *err) { + if (test_bit(BCH_FS_no_invalid_checks, &c->flags)) + return 0; + int ret = 0; bkey_fsck_err_on(k.k->u64s < BKEY_U64s, c, err, From 186e7d71534df4589405925caca5597af7626c12 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Mon, 6 May 2024 18:41:36 +0800 Subject: [PATCH 1536/1702] f2fs: compress: fix to update i_compr_blocks correctly Previously, we account reserved blocks and compressed blocks into @compr_blocks, then, f2fs_i_compr_blocks_update(,compr_blocks) will update i_compr_blocks incorrectly, fix it. Meanwhile, for the case all blocks in cluster were reserved, fix to update dn->ofs_in_node correctly. Fixes: eb8fbaa53374 ("f2fs: compress: fix to check unreleased compressed cluster") Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 72ce1a522fb26..13b430b54299d 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -3666,7 +3666,8 @@ static int reserve_compress_blocks(struct dnode_of_data *dn, pgoff_t count, while (count) { int compr_blocks = 0; - blkcnt_t reserved; + blkcnt_t reserved = 0; + blkcnt_t to_reserved; int ret; for (i = 0; i < cluster_size; i++) { @@ -3686,20 +3687,26 @@ static int reserve_compress_blocks(struct dnode_of_data *dn, pgoff_t count, * fails in release_compress_blocks(), so NEW_ADDR * is a possible case. */ - if (blkaddr == NEW_ADDR || - __is_valid_data_blkaddr(blkaddr)) { + if (blkaddr == NEW_ADDR) { + reserved++; + continue; + } + if (__is_valid_data_blkaddr(blkaddr)) { compr_blocks++; continue; } } - reserved = cluster_size - compr_blocks; + to_reserved = cluster_size - compr_blocks - reserved; /* for the case all blocks in cluster were reserved */ - if (reserved == 1) + if (to_reserved == 1) { + dn->ofs_in_node += cluster_size; goto next; + } - ret = inc_valid_block_count(sbi, dn->inode, &reserved, false); + ret = inc_valid_block_count(sbi, dn->inode, + &to_reserved, false); if (unlikely(ret)) return ret; @@ -3710,7 +3717,7 @@ static int reserve_compress_blocks(struct dnode_of_data *dn, pgoff_t count, f2fs_i_compr_blocks_update(dn->inode, compr_blocks, true); - *reserved_blocks += reserved; + *reserved_blocks += to_reserved; next: count -= cluster_size; } From a3a0bc6c223908a2a06736bcd43db962e0657c67 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Mon, 6 May 2024 18:41:38 +0800 Subject: [PATCH 1537/1702] f2fs: compress: fix typo in f2fs_reserve_compress_blocks() s/released/reserved. Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 13b430b54299d..f03ae3d952cc0 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -3810,7 +3810,7 @@ static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg) } else if (reserved_blocks && atomic_read(&F2FS_I(inode)->i_compr_blocks)) { set_sbi_flag(sbi, SBI_NEED_FSCK); - f2fs_warn(sbi, "%s: partial blocks were released i_ino=%lx " + f2fs_warn(sbi, "%s: partial blocks were reserved i_ino=%lx " "iblocks=%llu, reserved=%u, compr_blocks=%u, " "run fsck to fix.", __func__, inode->i_ino, inode->i_blocks, From 043c832371cd9023fbd725138ddc6c7f288dc469 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Mon, 6 May 2024 18:41:37 +0800 Subject: [PATCH 1538/1702] f2fs: compress: fix error path of inc_valid_block_count() If inc_valid_block_count() can not allocate all requested blocks, it needs to release block count in .total_valid_block_count and resevation blocks in inode. Fixes: 54607494875e ("f2fs: compress: fix to avoid inconsistence bewteen i_blocks and dnode") Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/f2fs.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 30058e16a5d0d..ef7de97be6474 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -2306,7 +2306,7 @@ static inline void f2fs_i_blocks_write(struct inode *, block_t, bool, bool); static inline int inc_valid_block_count(struct f2fs_sb_info *sbi, struct inode *inode, blkcnt_t *count, bool partial) { - blkcnt_t diff = 0, release = 0; + long long diff = 0, release = 0; block_t avail_user_block_count; int ret; @@ -2326,26 +2326,27 @@ static inline int inc_valid_block_count(struct f2fs_sb_info *sbi, percpu_counter_add(&sbi->alloc_valid_block_count, (*count)); spin_lock(&sbi->stat_lock); - sbi->total_valid_block_count += (block_t)(*count); - avail_user_block_count = get_available_block_count(sbi, inode, true); - if (unlikely(sbi->total_valid_block_count > avail_user_block_count)) { + avail_user_block_count = get_available_block_count(sbi, inode, true); + diff = (long long)sbi->total_valid_block_count + *count - + avail_user_block_count; + if (unlikely(diff > 0)) { if (!partial) { spin_unlock(&sbi->stat_lock); + release = *count; goto enospc; } - - diff = sbi->total_valid_block_count - avail_user_block_count; if (diff > *count) diff = *count; *count -= diff; release = diff; - sbi->total_valid_block_count -= diff; if (!*count) { spin_unlock(&sbi->stat_lock); goto enospc; } } + sbi->total_valid_block_count += (block_t)(*count); + spin_unlock(&sbi->stat_lock); if (unlikely(release)) { From 0a4ed2d97cb6d044196cc3e726b6699222b41019 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Mon, 6 May 2024 18:41:39 +0800 Subject: [PATCH 1539/1702] f2fs: compress: fix to cover {reserve,release}_compress_blocks() w/ cp_rwsem lock It needs to cover {reserve,release}_compress_blocks() w/ cp_rwsem lock to avoid racing with checkpoint, otherwise, filesystem metadata including blkaddr in dnode, inode fields and .total_valid_block_count may be corrupted after SPO case. Fixes: ef8d563f184e ("f2fs: introduce F2FS_IOC_RELEASE_COMPRESS_BLOCKS") Fixes: c75488fb4d82 ("f2fs: introduce F2FS_IOC_RESERVE_COMPRESS_BLOCKS") Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index f03ae3d952cc0..2a9de84d2a717 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -3593,9 +3593,12 @@ static int f2fs_release_compress_blocks(struct file *filp, unsigned long arg) struct dnode_of_data dn; pgoff_t end_offset, count; + f2fs_lock_op(sbi); + set_new_dnode(&dn, inode, NULL, NULL, 0); ret = f2fs_get_dnode_of_data(&dn, page_idx, LOOKUP_NODE); if (ret) { + f2fs_unlock_op(sbi); if (ret == -ENOENT) { page_idx = f2fs_get_next_page_offset(&dn, page_idx); @@ -3613,6 +3616,8 @@ static int f2fs_release_compress_blocks(struct file *filp, unsigned long arg) f2fs_put_dnode(&dn); + f2fs_unlock_op(sbi); + if (ret < 0) break; @@ -3765,9 +3770,12 @@ static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg) struct dnode_of_data dn; pgoff_t end_offset, count; + f2fs_lock_op(sbi); + set_new_dnode(&dn, inode, NULL, NULL, 0); ret = f2fs_get_dnode_of_data(&dn, page_idx, LOOKUP_NODE); if (ret) { + f2fs_unlock_op(sbi); if (ret == -ENOENT) { page_idx = f2fs_get_next_page_offset(&dn, page_idx); @@ -3785,6 +3793,8 @@ static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg) f2fs_put_dnode(&dn); + f2fs_unlock_op(sbi); + if (ret < 0) break; From 0fa4e57c1db263effd72d2149d4e21da0055c316 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Tue, 7 May 2024 11:31:00 +0800 Subject: [PATCH 1540/1702] f2fs: fix to release node block count in error path of f2fs_new_node_page() It missed to call dec_valid_node_count() to release node block count in error path, fix it. Fixes: 141170b759e0 ("f2fs: fix to avoid use f2fs_bug_on() in f2fs_new_node_page()") Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/node.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 95cecf08cb377..b72ef96f7e33a 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -1329,6 +1329,7 @@ struct page *f2fs_new_node_page(struct dnode_of_data *dn, unsigned int ofs) } if (unlikely(new_ni.blk_addr != NULL_ADDR)) { err = -EFSCORRUPTED; + dec_valid_node_count(sbi, dn->inode, !ofs); set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); goto fail; @@ -1355,7 +1356,6 @@ struct page *f2fs_new_node_page(struct dnode_of_data *dn, unsigned int ofs) if (ofs == 0) inc_valid_inode_count(sbi); return page; - fail: clear_node_page_dirty(page); f2fs_put_page(page, 1); From 29ed2b5dd521ce7c5d8466cd70bf0cc9d07afeee Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Fri, 10 May 2024 11:33:39 +0800 Subject: [PATCH 1541/1702] f2fs: compress: don't allow unaligned truncation on released compress inode f2fs image may be corrupted after below testcase: - mkfs.f2fs -O extra_attr,compression -f /dev/vdb - mount /dev/vdb /mnt/f2fs - touch /mnt/f2fs/file - f2fs_io setflags compression /mnt/f2fs/file - dd if=/dev/zero of=/mnt/f2fs/file bs=4k count=4 - f2fs_io release_cblocks /mnt/f2fs/file - truncate -s 8192 /mnt/f2fs/file - umount /mnt/f2fs - fsck.f2fs /dev/vdb [ASSERT] (fsck_chk_inode_blk:1256) --> ino: 0x5 has i_blocks: 0x00000002, but has 0x3 blocks [FSCK] valid_block_count matching with CP [Fail] [0x4, 0x5] [FSCK] other corrupted bugs [Fail] The reason is: partial truncation assume compressed inode has reserved blocks, after partial truncation, valid block count may change w/o .i_blocks and .total_valid_block_count update, result in corruption. This patch only allow cluster size aligned truncation on released compress inode for fixing. Fixes: c61404153eb6 ("f2fs: introduce FI_COMPRESS_RELEASED instead of using IMMUTABLE bit") Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/file.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 2a9de84d2a717..ef4cfb4436efd 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -952,9 +952,14 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, ATTR_GID | ATTR_TIMES_SET)))) return -EPERM; - if ((attr->ia_valid & ATTR_SIZE) && - !f2fs_is_compress_backend_ready(inode)) - return -EOPNOTSUPP; + if ((attr->ia_valid & ATTR_SIZE)) { + if (!f2fs_is_compress_backend_ready(inode)) + return -EOPNOTSUPP; + if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED) && + !IS_ALIGNED(attr->ia_size, + F2FS_BLK_TO_BYTES(F2FS_I(inode)->i_cluster_size))) + return -EINVAL; + } err = setattr_prepare(idmap, dentry, attr); if (err) From b7bd96ec1b709f5079fd203b680261dabc0050aa Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Sat, 4 May 2024 09:36:56 +0900 Subject: [PATCH 1542/1702] selftests/ftrace: Fix required features for VFS type test case Since the VFS type argument test case uses fprobe events, it must check the availablity of dynamic_events file and fprobe events syntax in README. Without this fix, the test fails if CONFIG_FPROBE_EVENTS=n. Link: https://lore.kernel.org/all/171478301645.110267.464634740467398506.stgit@devnote2/ Fixes: ee97e5e135c6 ("selftests/ftrace: add fprobe test cases for VFS type "%pd" and "%pD"") Signed-off-by: Masami Hiramatsu (Google) --- .../selftests/ftrace/test.d/dynevent/fprobe_args_vfs.tc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_args_vfs.tc b/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_args_vfs.tc index 49a833bf334ce..c6a9d2466a719 100644 --- a/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_args_vfs.tc +++ b/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_args_vfs.tc @@ -1,7 +1,8 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0 # description: Fprobe event VFS type argument -# requires: kprobe_events "%pd/%pD":README +# requires: dynamic_events "%pd/%pD":README "f[:[/][]] [%return] []":README + : "Test argument %pd with name for fprobe" echo 'f:testprobe dput name=$arg1:%pd' > dynamic_events From 8b80549f1bc692cf9130af8555b6c89cec24e1a6 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 30 Apr 2024 11:22:53 +0100 Subject: [PATCH 1543/1702] arm64: Properly clean up iommu-dma remnants Thanks to the somewhat asymmetrical nature, while removing iommu_setup_dma_ops() from the arch_setup_dma_ops() flow, I managed to forget that arm64's teardown path was also specific to iommu-dma. Clean that up to match, otherwise probe deferral will lead to the arch code erroneously removing DMA ops set elsewhere. Reported-by: Dmitry Baryshkov Link: https://lore.kernel.org/linux-iommu/Zi_LV28TR-P-PzXi@eriador.lumag.spb.ru/ Fixes: b67483b3c44e ("iommu/dma: Centralise iommu_setup_dma_ops()") Signed-off-by: Robin Murphy Tested-by: Dmitry Baryshkov Acked-by: Catalin Marinas Reviewed-by: Konrad Dybcio Acked-by: Will Deacon Tested-by: Nicolin Chen Link: https://lore.kernel.org/r/d4cc20cbb0c45175e98dd76bf187e2ad6421296d.1714472573.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- arch/arm64/Kconfig | 1 - arch/arm64/mm/dma-mapping.c | 8 -------- 2 files changed, 9 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 7b11c98b3e84b..8fe59fb9cb357 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -46,7 +46,6 @@ config ARM64 select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYSCALL_WRAPPER - select ARCH_HAS_TEARDOWN_DMA_OPS if IOMMU_SUPPORT select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_ZONE_DMA_SET if EXPERT select ARCH_HAVE_ELF_PROT diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 0b320a25a471e..b2b5792b2caaf 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include @@ -39,13 +38,6 @@ void arch_dma_prep_coherent(struct page *page, size_t size) dcache_clean_poc(start, start + size); } -#ifdef CONFIG_IOMMU_DMA -void arch_teardown_dma_ops(struct device *dev) -{ - dev->dma_ops = NULL; -} -#endif - void arch_setup_dma_ops(struct device *dev, bool coherent) { int cls = cache_line_size_of_cpu(); From da55da5a42d4247d7a48b843fa5fcd9a4a10f4fe Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 7 May 2024 10:21:10 -0300 Subject: [PATCH 1544/1702] iommu/arm-smmu-v3: Make the kunit into a module It turns out kconfig has problems ensuring the SMMU module and the KUNIT module are consistently y/m to allow linking. It will permit KUNIT to be a module while SMMU is built in. Also, Fedora apparently enables kunit on production kernels. So, put the entire kunit in its own module using the VISIBLE_IF_KUNIT/EXPORT_SYMBOL_IF_KUNIT machinery. This keeps it out of vmlinus on Fedora and makes the kconfig work in the normal way. There is no cost if kunit is disabled. Fixes: 56e1a4cc2588 ("iommu/arm-smmu-v3: Add unit tests for arm_smmu_write_entry") Reported-by: Thorsten Leemhuis Link: https://lore.kernel.org/all/aeea8546-5bce-4c51-b506-5d2008e52fef@leemhuis.info Signed-off-by: Jason Gunthorpe Tested-by: Thorsten Leemhuis Acked-by: Will Deacon Link: https://lore.kernel.org/r/0-v1-24cba6c0f404+2ae-smmu_kunit_module_jgg@nvidia.com Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 2 +- drivers/iommu/arm/arm-smmu-v3/Makefile | 3 ++- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 1 + drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 3 +++ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 8 ++++++++ 5 files changed, 15 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 66325210c8c98..c04584be30893 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -415,7 +415,7 @@ config ARM_SMMU_V3_SVA and PRI. config ARM_SMMU_V3_KUNIT_TEST - bool "KUnit tests for arm-smmu-v3 driver" if !KUNIT_ALL_TESTS + tristate "KUnit tests for arm-smmu-v3 driver" if !KUNIT_ALL_TESTS depends on KUNIT depends on ARM_SMMU_V3_SVA default KUNIT_ALL_TESTS diff --git a/drivers/iommu/arm/arm-smmu-v3/Makefile b/drivers/iommu/arm/arm-smmu-v3/Makefile index 0b97054b3929b..014a997753a8a 100644 --- a/drivers/iommu/arm/arm-smmu-v3/Makefile +++ b/drivers/iommu/arm/arm-smmu-v3/Makefile @@ -2,5 +2,6 @@ obj-$(CONFIG_ARM_SMMU_V3) += arm_smmu_v3.o arm_smmu_v3-objs-y += arm-smmu-v3.o arm_smmu_v3-objs-$(CONFIG_ARM_SMMU_V3_SVA) += arm-smmu-v3-sva.o -arm_smmu_v3-objs-$(CONFIG_ARM_SMMU_V3_KUNIT_TEST) += arm-smmu-v3-test.o arm_smmu_v3-objs := $(arm_smmu_v3-objs-y) + +obj-$(CONFIG_ARM_SMMU_V3_KUNIT_TEST) += arm-smmu-v3-test.o diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 34a977a0767d4..e490ffb380154 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -185,6 +185,7 @@ void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, */ target->data[3] = cpu_to_le64(read_sysreg(mair_el1)); } +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_sva_cd); static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm) { diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index 417804392ff08..315e487fd990e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -463,3 +463,6 @@ static struct kunit_suite arm_smmu_v3_test_module = { .test_cases = arm_smmu_v3_test_cases, }; kunit_test_suites(&arm_smmu_v3_test_module); + +MODULE_IMPORT_NS(EXPORTED_FOR_KUNIT_TESTING); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 1ad0937760c69..aa62f0ecd0533 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1007,6 +1007,7 @@ void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits) if (cfg == STRTAB_STE_0_CFG_BYPASS) used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG); } +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_used); /* * Figure out if we can do a hitless update of entry to become target. Returns a @@ -1141,6 +1142,7 @@ void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry, entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS)); } } +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_write_entry); static void arm_smmu_sync_cd(struct arm_smmu_master *master, int ssid, bool leaf) @@ -1268,6 +1270,7 @@ void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits) used_bits[1] &= ~cpu_to_le64(CTXDESC_CD_1_TTB0_MASK); } } +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_cd_used); static void arm_smmu_cd_writer_sync_entry(struct arm_smmu_entry_writer *writer) { @@ -1332,6 +1335,7 @@ void arm_smmu_make_s1_cd(struct arm_smmu_cd *target, CTXDESC_CD_1_TTB0_MASK); target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.mair); } +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s1_cd); void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid) { @@ -1515,6 +1519,7 @@ void arm_smmu_make_abort_ste(struct arm_smmu_ste *target) STRTAB_STE_0_V | FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT)); } +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_abort_ste); VISIBLE_IF_KUNIT void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu, @@ -1529,6 +1534,7 @@ void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu, target->data[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING)); } +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_bypass_ste); VISIBLE_IF_KUNIT void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, @@ -1580,6 +1586,7 @@ void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, cpu_to_le64(FIELD_PREP(STRTAB_STE_2_S2VMID, 0)); } } +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_cdtable_ste); VISIBLE_IF_KUNIT void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, @@ -1627,6 +1634,7 @@ void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s2_cfg.vttbr & STRTAB_STE_3_S2TTB_MASK); } +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s2_domain_ste); /* * This can safely directly manipulate the STE memory without a sync sequence From 0ca4f51fa522df285ebff76ec8bd2bde7d1c080d Mon Sep 17 00:00:00 2001 From: "Dr. David Alan Gilbert" Date: Mon, 6 May 2024 00:10:59 +0100 Subject: [PATCH 1545/1702] parisc/math-emu: Remove unused struct 'exc_reg' This has been here since pre-git. Build tested. Signed-off-by: Dr. David Alan Gilbert Signed-off-by: Helge Deller --- arch/parisc/math-emu/driver.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/arch/parisc/math-emu/driver.c b/arch/parisc/math-emu/driver.c index 6ce427b58836c..34495446e051c 100644 --- a/arch/parisc/math-emu/driver.c +++ b/arch/parisc/math-emu/driver.c @@ -26,12 +26,6 @@ #define FPUDEBUG 0 -/* Format of the floating-point exception registers. */ -struct exc_reg { - unsigned int exception : 6; - unsigned int ei : 26; -}; - /* Macros for grabbing bits of the instruction format from the 'ei' field above. */ /* Major opcode 0c and 0e */ From f2526c5cf1d94359467d9472387363d57c6b3e6d Mon Sep 17 00:00:00 2001 From: Daeho Jeong Date: Fri, 10 May 2024 06:49:08 -0700 Subject: [PATCH 1546/1702] f2fs: allow dirty sections with zero valid block for checkpoint disabled Following the semantic for dirty segments in checkpoint disabled mode, apply the same rule to dirty sections. Signed-off-by: Daeho Jeong Signed-off-by: Jaegeuk Kim --- fs/f2fs/segment.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index ab3e9a6e9c96c..a0ce3d080f80a 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -771,8 +771,10 @@ static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, block_t valid_blocks = get_valid_blocks(sbi, segno, true); - f2fs_bug_on(sbi, unlikely(!valid_blocks || - valid_blocks == CAP_BLKS_PER_SEC(sbi))); + f2fs_bug_on(sbi, + (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) && + !valid_blocks) || + valid_blocks == CAP_BLKS_PER_SEC(sbi)); if (!IS_CURSEC(sbi, secno)) set_bit(secno, dirty_i->dirty_secmap); From a798ff17cd2dabe47d5d4ed3d509631793c36e19 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Fri, 10 May 2024 11:43:33 +0800 Subject: [PATCH 1547/1702] f2fs: fix to add missing iput() in gc_data_segment() During gc_data_segment(), if inode state is abnormal, it missed to call iput(), fix it. Fixes: b73e52824c89 ("f2fs: reposition unlock_new_inode to prevent accessing invalid inode") Fixes: 9056d6489f5a ("f2fs: fix to do sanity check on inode type during garbage collection") Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/gc.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index ac4cbbe50c2fc..6066c6eecf41d 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -1554,10 +1554,15 @@ static int gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, int err; inode = f2fs_iget(sb, dni.ino); - if (IS_ERR(inode) || is_bad_inode(inode) || - special_file(inode->i_mode)) + if (IS_ERR(inode)) continue; + if (is_bad_inode(inode) || + special_file(inode->i_mode)) { + iput(inode); + continue; + } + err = f2fs_gc_pinned_control(inode, gc_type, segno); if (err == -EAGAIN) { iput(inode); From 991b6bdf1b009832256f8bc3035d4bcba664657b Mon Sep 17 00:00:00 2001 From: Zhiguo Niu Date: Fri, 26 Apr 2024 20:01:29 +0800 Subject: [PATCH 1548/1702] f2fs: fix some ambiguous comments After commit d7e9a9037de2 ("f2fs: Support Block Size == Page Size"), Some comments are confused and just correct with block size is 4KB. Signed-off-by: Zhiguo Niu Signed-off-by: Jaegeuk Kim --- include/linux/f2fs_fs.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index a357287eac1ea..41d1d71c36ff5 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -394,7 +394,8 @@ struct f2fs_nat_block { /* * F2FS uses 4 bytes to represent block address. As a result, supported size of - * disk is 16 TB and it equals to 16 * 1024 * 1024 / 2 segments. + * disk is 16 TB for a 4K page size and 64 TB for a 16K page size and it equals + * to 16 * 1024 * 1024 / 2 segments. */ #define F2FS_MAX_SEGMENT ((16 * 1024 * 1024) / 2) @@ -424,8 +425,10 @@ struct f2fs_sit_block { /* * For segment summary * - * One summary block contains exactly 512 summary entries, which represents - * exactly one segment by default. Not allow to change the basic units. + * One summary block with 4KB size contains exactly 512 summary entries, which + * represents exactly one segment with 2MB size. + * Similarly, in the case of block with 16KB size, it represents one segment with 8MB size. + * Not allow to change the basic units. * * NOTE: For initializing fields, you must use set_summary * @@ -556,6 +559,7 @@ typedef __le32 f2fs_hash_t; /* * space utilization of regular dentry and inline dentry (w/o extra reservation) + * when block size is 4KB. * regular dentry inline dentry (def) inline dentry (min) * bitmap 1 * 27 = 27 1 * 23 = 23 1 * 1 = 1 * reserved 1 * 3 = 3 1 * 7 = 7 1 * 1 = 1 From c45b8cfc6d5c12fbbc4d89b24b59402df99c1ecb Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Tue, 7 May 2024 12:07:56 +0200 Subject: [PATCH 1549/1702] watchdog: LENOVO_SE10_WDT should depend on X86 && DMI The Lenovo SE10 watchdog is only present on Lenovo ThinkEdge SE10 platforms, which are based on Intel Atom SoCs, and its driver relies on DMI tables. Hence add dependencies on X86 && DMI, to prevent asking the user about this driver when configuring a kernel without Intel Atom or DMI support. While at it, fix the odd indentation (spaces instead of TABs). Fixes: 1f6602c8ed1eccac ("watchdog: lenovo_se10_wdt: Watchdog driver for Lenovo SE10 platform") Signed-off-by: Geert Uytterhoeven Reviewed-by: Mark Pearson Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/58005595a05ef803b454b78d3ae9b8ee0675bd5d.1715076440.git.geert+renesas@glider.be Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/Kconfig | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig index f99880a3b976a..85eea38dbdf4b 100644 --- a/drivers/watchdog/Kconfig +++ b/drivers/watchdog/Kconfig @@ -255,14 +255,15 @@ config GPIO_WATCHDOG_ARCH_INITCALL If in doubt, say N. config LENOVO_SE10_WDT - tristate "Lenovo SE10 Watchdog" - select WATCHDOG_CORE - help - If you say yes here you get support for the watchdog - functionality for the Lenovo SE10 platform. - - This driver can also be built as a module. If so, the module - will be called lenovo-se10-wdt. + tristate "Lenovo SE10 Watchdog" + depends on (X86 && DMI) || COMPILE_TEST + select WATCHDOG_CORE + help + If you say yes here you get support for the watchdog + functionality for the Lenovo SE10 platform. + + This driver can also be built as a module. If so, the module + will be called lenovo-se10-wdt. config MENF21BMC_WATCHDOG tristate "MEN 14F021P00 BMC Watchdog" From d14d6b0e7da3c2d768418fc7e5ad65c359c35962 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Thu, 2 May 2024 10:27:17 -0700 Subject: [PATCH 1550/1702] selftests/damon/_damon_sysfs: support quota goals Patch series "selftests/damon: add DAMOS quota goal test". Extend DAMON selftest-purpose sysfs wrapper to support DAMOS quota goal, and implement a simple selftest for the feature using it. This patch (of 2): The DAMON sysfs test purpose wrapper, _damon_sysfs.py, is not supporting quota goals. Implement the support for testing the feature. The test will be implemented and added by the following commit. Link: https://lkml.kernel.org/r/20240502172718.74166-1-sj@kernel.org Link: https://lkml.kernel.org/r/20240502172718.74166-2-sj@kernel.org Signed-off-by: SeongJae Park Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/_damon_sysfs.py | 84 ++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/damon/_damon_sysfs.py b/tools/testing/selftests/damon/_damon_sysfs.py index d23d7398a27a8..036e64070fdbd 100644 --- a/tools/testing/selftests/damon/_damon_sysfs.py +++ b/tools/testing/selftests/damon/_damon_sysfs.py @@ -70,16 +70,56 @@ def stage(self): if err != None: return err +qgoal_metric_user_input = 'user_input' +qgoal_metric_some_mem_psi_us = 'some_mem_psi_us' +qgoal_metrics = [qgoal_metric_user_input, qgoal_metric_some_mem_psi_us] + +class DamosQuotaGoal: + metric = None + target_value = None + current_value = None + effective_bytes = None + quota = None # owner quota + idx = None + + def __init__(self, metric, target_value=10000, current_value=0): + self.metric = metric + self.target_value = target_value + self.current_value = current_value + + def sysfs_dir(self): + return os.path.join(self.quota.sysfs_dir(), 'goals', '%d' % self.idx) + + def stage(self): + err = write_file(os.path.join(self.sysfs_dir(), 'target_metric'), + self.metric) + if err is not None: + return err + err = write_file(os.path.join(self.sysfs_dir(), 'target_value'), + self.target_value) + if err is not None: + return err + err = write_file(os.path.join(self.sysfs_dir(), 'current_value'), + self.current_value) + if err is not None: + return err + return None + class DamosQuota: sz = None # size quota, in bytes ms = None # time quota + goals = None # quota goals reset_interval_ms = None # quota reset interval scheme = None # owner scheme - def __init__(self, sz=0, ms=0, reset_interval_ms=0): + def __init__(self, sz=0, ms=0, goals=None, reset_interval_ms=0): self.sz = sz self.ms = ms self.reset_interval_ms = reset_interval_ms + self.goals = goals if goals is not None else [] + for idx, goal in enumerate(self.goals): + goal.idx = idx + goal.quota = self def sysfs_dir(self): return os.path.join(self.scheme.sysfs_dir(), 'quotas') @@ -96,6 +136,20 @@ def stage(self): if err != None: return err + nr_goals_file = os.path.join(self.sysfs_dir(), 'goals', 'nr_goals') + content, err = read_file(nr_goals_file) + if err is not None: + return err + if int(content) != len(self.goals): + err = write_file(nr_goals_file, len(self.goals)) + if err is not None: + return err + for goal in self.goals: + err = goal.stage() + if err is not None: + return err + return None + class DamosStats: nr_tried = None sz_tried = None @@ -361,6 +415,34 @@ def update_schemes_stats(self): stat_values.append(int(content)) scheme.stats = DamosStats(*stat_values) + def update_schemes_effective_quotas(self): + err = write_file(os.path.join(self.sysfs_dir(), 'state'), + 'update_schemes_effective_quotas') + if err is not None: + return err + for context in self.contexts: + for scheme in context.schemes: + for goal in scheme.quota.goals: + content, err = read_file( + os.path.join(scheme.quota.sysfs_dir(), + 'effective_bytes')) + if err is not None: + return err + goal.effective_bytes = int(content) + return None + + def commit_schemes_quota_goals(self): + for context in self.contexts: + for scheme in context.schemes: + for goal in scheme.quota.goals: + err = goal.stage() + if err is not None: + print('commit_schemes_quota_goals failed stagign: %s'% + err) + exit(1) + return write_file(os.path.join(self.sysfs_dir(), 'state'), + 'commit_schemes_quota_goals') + class Kdamonds: kdamonds = [] From f1c07c0a1662b4f8aa7dae381013729aac6ba691 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Thu, 2 May 2024 10:27:18 -0700 Subject: [PATCH 1551/1702] selftests/damon: add a test for DAMOS quota goal Add a selftest for DAMOS quota goal. It tests the feature by setting a user_input metric based goal, change the current feedback, and check if the effective quota size is increased and decreased as expected. Link: https://lkml.kernel.org/r/20240502172718.74166-3-sj@kernel.org Signed-off-by: SeongJae Park Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/Makefile | 2 +- .../selftests/damon/damos_quota_goal.py | 77 +++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/damon/damos_quota_goal.py diff --git a/tools/testing/selftests/damon/Makefile b/tools/testing/selftests/damon/Makefile index 789d6949c2471..06c248880172d 100644 --- a/tools/testing/selftests/damon/Makefile +++ b/tools/testing/selftests/damon/Makefile @@ -16,7 +16,7 @@ TEST_PROGS += debugfs_target_ids_pid_leak.sh TEST_PROGS += sysfs.sh sysfs_update_removed_scheme_dir.sh TEST_PROGS += sysfs_update_schemes_tried_regions_hang.py TEST_PROGS += sysfs_update_schemes_tried_regions_wss_estimation.py -TEST_PROGS += damos_quota.py damos_apply_interval.py +TEST_PROGS += damos_quota.py damos_quota_goal.py damos_apply_interval.py TEST_PROGS += reclaim.sh lru_sort.sh include ../lib.mk diff --git a/tools/testing/selftests/damon/damos_quota_goal.py b/tools/testing/selftests/damon/damos_quota_goal.py new file mode 100644 index 0000000000000..18246f3b62f7e --- /dev/null +++ b/tools/testing/selftests/damon/damos_quota_goal.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +import subprocess +import time + +import _damon_sysfs + +def main(): + # access two 10 MiB memory regions, 2 second per each + sz_region = 10 * 1024 * 1024 + proc = subprocess.Popen(['./access_memory', '2', '%d' % sz_region, '2000']) + + goal = _damon_sysfs.DamosQuotaGoal( + metric=_damon_sysfs.qgoal_metric_user_input, target_value=10000) + kdamonds = _damon_sysfs.Kdamonds([_damon_sysfs.Kdamond( + contexts=[_damon_sysfs.DamonCtx( + ops='vaddr', + targets=[_damon_sysfs.DamonTarget(pid=proc.pid)], + schemes=[_damon_sysfs.Damos( + action='stat', + quota=_damon_sysfs.DamosQuota( + goals=[goal], reset_interval_ms=100), + )] # schemes + )] # contexts + )]) # kdamonds + + err = kdamonds.start() + if err != None: + print('kdamond start failed: %s' % err) + exit(1) + + score_values_to_test = [0, 15000, 5000, 18000] + while proc.poll() == None: + if len(score_values_to_test) == 0: + time.sleep(0.1) + continue + + goal.current_value = score_values_to_test.pop(0) + expect_increase = goal.current_value < goal.target_value + + err = kdamonds.kdamonds[0].commit_schemes_quota_goals() + if err is not None: + print('commit_schemes_quota_goals failed: %s' % err) + exit(1) + + err = kdamonds.kdamonds[0].update_schemes_effective_quotas() + if err is not None: + print('before-update_schemes_effective_quotas failed: %s' % err) + exit(1) + last_effective_bytes = goal.effective_bytes + + time.sleep(0.5) + + err = kdamonds.kdamonds[0].update_schemes_effective_quotas() + if err is not None: + print('after-update_schemes_effective_quotas failed: %s' % err) + exit(1) + + print('score: %s, effective quota: %d -> %d (%.3fx)' % ( + goal.current_value, last_effective_bytes, goal.effective_bytes, + goal.effective_bytes / last_effective_bytes + if last_effective_bytes != 0 else -1.0)) + + if last_effective_bytes == goal.effective_bytes: + print('efective bytes not changed: %d' % goal.effective_bytes) + exit(1) + + increased = last_effective_bytes < goal.effective_bytes + if expect_increase != increased: + print('expectation of increase (%s) != increased (%s)' % + (expect_increase, increased)) + exit(1) + last_effective_bytes = goal.effective_bytes + +if __name__ == '__main__': + main() From b96a303b68dd3fe86f739fd34bd7cf194118ae89 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 3 May 2024 11:03:09 -0700 Subject: [PATCH 1552/1702] mm/damon/core: initialize ->esz_bp from damos_quota_init_priv() Patch series "mm/damon: misc fixes and improvements". Add miscelleneous and non-urgent fixes and improvements for DAMON code, selftests, and documents. This patch (of 10): damos_quota_init_priv() function should initialize all private fields of struct damos_quota. However, it is not initializing ->esz_bp field. This could result in use of uninitialized variable from damon_feed_loop_next_input() function. There is no such issue at the moment because every caller of the function is passing damos_quota object that already having the field zero value. But we cannot guarantee the future, and the function is not doing what it is promising. A bug is a bug. This fix is for preventing possible future issues. Link: https://lkml.kernel.org/r/20240503180318.72798-1-sj@kernel.org Link: https://lkml.kernel.org/r/20240503180318.72798-2-sj@kernel.org Fixes: 9294a037c015 ("mm/damon/core: implement goal-oriented feedback-driven quota auto-tuning") Signed-off-by: SeongJae Park Cc: Jonathan Corbet Cc: Shuah Khan Signed-off-by: Andrew Morton --- mm/damon/core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/damon/core.c b/mm/damon/core.c index 6d503c1c125ef..939ecfcd4641f 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -346,6 +346,7 @@ static struct damos_quota *damos_quota_init(struct damos_quota *quota) quota->charged_from = 0; quota->charge_target_from = NULL; quota->charge_addr_from = 0; + quota->esz_bp = 0; return quota; } From 732b8815c079199d29b0426d9372bb098c63cdc7 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 3 May 2024 11:03:10 -0700 Subject: [PATCH 1553/1702] selftests/damon/_damon_sysfs: check errors from nr_schemes file reads DAMON context staging method in _damon_sysfs.py is not checking the returned error from nr_schemes file read. Check it. Link: https://lkml.kernel.org/r/20240503180318.72798-3-sj@kernel.org Fixes: f5f0e5a2bef9 ("selftests/damon/_damon_sysfs: implement kdamonds start function") Signed-off-by: SeongJae Park Cc: Jonathan Corbet Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/_damon_sysfs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/testing/selftests/damon/_damon_sysfs.py b/tools/testing/selftests/damon/_damon_sysfs.py index 036e64070fdbd..43239e5b38b26 100644 --- a/tools/testing/selftests/damon/_damon_sysfs.py +++ b/tools/testing/selftests/damon/_damon_sysfs.py @@ -341,6 +341,8 @@ def stage(self): nr_schemes_file = os.path.join( self.sysfs_dir(), 'schemes', 'nr_schemes') content, err = read_file(nr_schemes_file) + if err is not None: + return err if int(content) != len(self.schemes): err = write_file(nr_schemes_file, '%d' % len(self.schemes)) if err != None: From e799fda6926ec01f8488023d4f891289996322aa Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 3 May 2024 11:03:11 -0700 Subject: [PATCH 1554/1702] selftests/damon/_damon_sysfs: find sysfs mount point from /proc/mounts _damon_sysfs.py assumes sysfs is mounted at /sys. In some systems, that might not be true. Find the mount point from /proc/mounts file content. Link: https://lkml.kernel.org/r/20240503180318.72798-4-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Corbet Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/_damon_sysfs.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/damon/_damon_sysfs.py b/tools/testing/selftests/damon/_damon_sysfs.py index 43239e5b38b26..9682e18dd4503 100644 --- a/tools/testing/selftests/damon/_damon_sysfs.py +++ b/tools/testing/selftests/damon/_damon_sysfs.py @@ -2,7 +2,18 @@ import os -sysfs_root = '/sys/kernel/mm/damon/admin' +ksft_skip=4 + +sysfs_root = None +with open('/proc/mounts', 'r') as f: + for line in f: + dev_name, mount_point, dev_fs = line.split()[:3] + if dev_fs == 'sysfs': + sysfs_root = '%s/kernel/mm/damon/admin' % mount_point + break +if sysfs_root is None: + print('Seems sysfs not mounted?') + exit(ksft_skip) def write_file(path, string): "Returns error string if failed, or None otherwise" From 06cf8ce12cd659a3064c2e7b0208a2c0a3426838 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 3 May 2024 11:03:12 -0700 Subject: [PATCH 1555/1702] selftests/damon/_damon_sysfs: use 'is' instead of '==' for 'None' _damon_sysfs.py is using '==' or '!=' for 'None'. Since 'None' is a singleton, using 'is' or 'is not' is more efficient. Use the more efficient one. Link: https://lkml.kernel.org/r/20240503180318.72798-5-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Corbet Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/_damon_sysfs.py | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/tools/testing/selftests/damon/_damon_sysfs.py b/tools/testing/selftests/damon/_damon_sysfs.py index 9682e18dd4503..2bd44c32be1bf 100644 --- a/tools/testing/selftests/damon/_damon_sysfs.py +++ b/tools/testing/selftests/damon/_damon_sysfs.py @@ -45,11 +45,11 @@ def __init__(self, size=None, nr_accesses=None, age=None): self.nr_accesses = nr_accesses self.age = age - if self.size == None: + if self.size is None: self.size = [0, 2**64 - 1] - if self.nr_accesses == None: + if self.nr_accesses is None: self.nr_accesses = [0, 2**64 - 1] - if self.age == None: + if self.age is None: self.age = [0, 2**64 - 1] def sysfs_dir(self): @@ -58,27 +58,27 @@ def sysfs_dir(self): def stage(self): err = write_file( os.path.join(self.sysfs_dir(), 'sz', 'min'), self.size[0]) - if err != None: + if err is not None: return err err = write_file( os.path.join(self.sysfs_dir(), 'sz', 'max'), self.size[1]) - if err != None: + if err is not None: return err err = write_file(os.path.join(self.sysfs_dir(), 'nr_accesses', 'min'), self.nr_accesses[0]) - if err != None: + if err is not None: return err err = write_file(os.path.join(self.sysfs_dir(), 'nr_accesses', 'max'), self.nr_accesses[1]) - if err != None: + if err is not None: return err err = write_file( os.path.join(self.sysfs_dir(), 'age', 'min'), self.age[0]) - if err != None: + if err is not None: return err err = write_file( os.path.join(self.sysfs_dir(), 'age', 'max'), self.age[1]) - if err != None: + if err is not None: return err qgoal_metric_user_input = 'user_input' @@ -137,14 +137,14 @@ def sysfs_dir(self): def stage(self): err = write_file(os.path.join(self.sysfs_dir(), 'bytes'), self.sz) - if err != None: + if err is not None: return err err = write_file(os.path.join(self.sysfs_dir(), 'ms'), self.ms) - if err != None: + if err is not None: return err err = write_file(os.path.join(self.sysfs_dir(), 'reset_interval_ms'), self.reset_interval_ms) - if err != None: + if err is not None: return err nr_goals_file = os.path.join(self.sysfs_dir(), 'goals', 'nr_goals') @@ -201,30 +201,30 @@ def sysfs_dir(self): def stage(self): err = write_file(os.path.join(self.sysfs_dir(), 'action'), self.action) - if err != None: + if err is not None: return err err = self.access_pattern.stage() - if err != None: + if err is not None: return err err = write_file(os.path.join(self.sysfs_dir(), 'apply_interval_us'), '%d' % self.apply_interval_us) - if err != None: + if err is not None: return err err = self.quota.stage() - if err != None: + if err is not None: return err # disable watermarks err = write_file( os.path.join(self.sysfs_dir(), 'watermarks', 'metric'), 'none') - if err != None: + if err is not None: return err # disable filters err = write_file( os.path.join(self.sysfs_dir(), 'filters', 'nr_filters'), '0') - if err != None: + if err is not None: return err class DamonTarget: @@ -243,7 +243,7 @@ def sysfs_dir(self): def stage(self): err = write_file( os.path.join(self.sysfs_dir(), 'regions', 'nr_regions'), '0') - if err != None: + if err is not None: return err return write_file( os.path.join(self.sysfs_dir(), 'pid_target'), self.pid) @@ -275,27 +275,27 @@ def nr_regions_range_sysfs_dir(self): def stage(self): err = write_file(os.path.join(self.interval_sysfs_dir(), 'sample_us'), self.sample_us) - if err != None: + if err is not None: return err err = write_file(os.path.join(self.interval_sysfs_dir(), 'aggr_us'), self.aggr_us) - if err != None: + if err is not None: return err err = write_file(os.path.join(self.interval_sysfs_dir(), 'update_us'), self.update_us) - if err != None: + if err is not None: return err err = write_file( os.path.join(self.nr_regions_range_sysfs_dir(), 'min'), self.min_nr_regions) - if err != None: + if err is not None: return err err = write_file( os.path.join(self.nr_regions_range_sysfs_dir(), 'max'), self.max_nr_regions) - if err != None: + if err is not None: return err class DamonCtx: @@ -329,24 +329,24 @@ def sysfs_dir(self): def stage(self): err = write_file( os.path.join(self.sysfs_dir(), 'operations'), self.ops) - if err != None: + if err is not None: return err err = self.monitoring_attrs.stage() - if err != None: + if err is not None: return err nr_targets_file = os.path.join( self.sysfs_dir(), 'targets', 'nr_targets') content, err = read_file(nr_targets_file) - if err != None: + if err is not None: return err if int(content) != len(self.targets): err = write_file(nr_targets_file, '%d' % len(self.targets)) - if err != None: + if err is not None: return err for target in self.targets: err = target.stage() - if err != None: + if err is not None: return err nr_schemes_file = os.path.join( @@ -356,11 +356,11 @@ def stage(self): return err if int(content) != len(self.schemes): err = write_file(nr_schemes_file, '%d' % len(self.schemes)) - if err != None: + if err is not None: return err for scheme in self.schemes: err = scheme.stage() - if err != None: + if err is not None: return err return None @@ -384,16 +384,16 @@ def start(self): nr_contexts_file = os.path.join(self.sysfs_dir(), 'contexts', 'nr_contexts') content, err = read_file(nr_contexts_file) - if err != None: + if err is not None: return err if int(content) != len(self.contexts): err = write_file(nr_contexts_file, '%d' % len(self.contexts)) - if err != None: + if err is not None: return err for context in self.contexts: err = context.stage() - if err != None: + if err is not None: return err err = write_file(os.path.join(self.sysfs_dir(), 'state'), 'on') return err @@ -401,20 +401,20 @@ def start(self): def update_schemes_tried_bytes(self): err = write_file(os.path.join(self.sysfs_dir(), 'state'), 'update_schemes_tried_bytes') - if err != None: + if err is not None: return err for context in self.contexts: for scheme in context.schemes: content, err = read_file(os.path.join(scheme.sysfs_dir(), 'tried_regions', 'total_bytes')) - if err != None: + if err is not None: return err scheme.tried_bytes = int(content) def update_schemes_stats(self): err = write_file(os.path.join(self.sysfs_dir(), 'state'), 'update_schemes_stats') - if err != None: + if err is not None: return err for context in self.contexts: for scheme in context.schemes: @@ -423,7 +423,7 @@ def update_schemes_stats(self): 'sz_applied', 'qt_exceeds']: content, err = read_file( os.path.join(scheme.sysfs_dir(), 'stats', stat)) - if err != None: + if err is not None: return err stat_values.append(int(content)) scheme.stats = DamosStats(*stat_values) @@ -471,10 +471,10 @@ def sysfs_dir(self): def start(self): err = write_file(os.path.join(self.sysfs_dir(), 'nr_kdamonds'), '%s' % len(self.kdamonds)) - if err != None: + if err is not None: return err for kdamond in self.kdamonds: err = kdamond.start() - if err != None: + if err is not None: return err return None From 5965d5615b75bbc770a7f36895f91cc72cfd4118 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 3 May 2024 11:03:13 -0700 Subject: [PATCH 1556/1702] selftests/damon: classify tests for functionalities and regressions DAMON selftests can be classified into two categories: functionalities and regressions. Functionality tests are for checking if the function is working as specified, while the regression tests are basically reproducers of previously reported and fixed bugs. The tests of the categories are mixed in the selftests Makefile. Separate those for easier understanding of the types of tests. Link: https://lkml.kernel.org/r/20240503180318.72798-6-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Corbet Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/damon/Makefile | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/damon/Makefile b/tools/testing/selftests/damon/Makefile index 06c248880172d..29a22f50e762a 100644 --- a/tools/testing/selftests/damon/Makefile +++ b/tools/testing/selftests/damon/Makefile @@ -7,16 +7,21 @@ TEST_GEN_FILES += debugfs_target_ids_pid_leak TEST_GEN_FILES += access_memory TEST_FILES = _chk_dependency.sh _debugfs_common.sh + +# functionality tests TEST_PROGS = debugfs_attrs.sh debugfs_schemes.sh debugfs_target_ids.sh +TEST_PROGS += sysfs.sh +TEST_PROGS += sysfs_update_schemes_tried_regions_wss_estimation.py +TEST_PROGS += damos_quota.py damos_quota_goal.py damos_apply_interval.py +TEST_PROGS += reclaim.sh lru_sort.sh + +# regression tests (reproducers of previously found bugs) TEST_PROGS += debugfs_empty_targets.sh debugfs_huge_count_read_write.sh TEST_PROGS += debugfs_duplicate_context_creation.sh TEST_PROGS += debugfs_rm_non_contexts.sh TEST_PROGS += debugfs_target_ids_read_before_terminate_race.sh TEST_PROGS += debugfs_target_ids_pid_leak.sh -TEST_PROGS += sysfs.sh sysfs_update_removed_scheme_dir.sh +TEST_PROGS += sysfs_update_removed_scheme_dir.sh TEST_PROGS += sysfs_update_schemes_tried_regions_hang.py -TEST_PROGS += sysfs_update_schemes_tried_regions_wss_estimation.py -TEST_PROGS += damos_quota.py damos_quota_goal.py damos_apply_interval.py -TEST_PROGS += reclaim.sh lru_sort.sh include ../lib.mk From da2a061888883e067e8e649d086df35c92c760a7 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 3 May 2024 11:03:14 -0700 Subject: [PATCH 1557/1702] Docs/admin-guide/mm/damon/usage: fix wrong example of DAMOS filter matching sysfs file The example usage of DAMOS filter sysfs files, specifically the part of 'matching' file writing for memcg type filter, is wrong. The intention is to exclude pages of a memcg that already getting enough care from a given scheme, but the example is setting the filter to apply the scheme to only the pages of the memcg. Fix it. Link: https://lkml.kernel.org/r/20240503180318.72798-7-sj@kernel.org Fixes: 9b7f9322a530 ("Docs/admin-guide/mm/damon/usage: document DAMOS filters of sysfs") Closes: https://lore.kernel.org/r/20240317191358.97578-1-sj@kernel.org Signed-off-by: SeongJae Park Cc: [6.3.x] Cc: Jonathan Corbet Cc: Shuah Khan Signed-off-by: Andrew Morton --- Documentation/admin-guide/mm/damon/usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst index 69bc8fabf3781..3ce3f0aaa1d5b 100644 --- a/Documentation/admin-guide/mm/damon/usage.rst +++ b/Documentation/admin-guide/mm/damon/usage.rst @@ -434,7 +434,7 @@ pages of all memory cgroups except ``/having_care_already``.:: # # further filter out all cgroups except one at '/having_care_already' echo memcg > 1/type echo /having_care_already > 1/memcg_path - echo N > 1/matching + echo Y > 1/matching Note that ``anon`` and ``memcg`` filters are currently supported only when ``paddr`` :ref:`implementation ` is being used. From 14e70e4660d6ecaf503c461f072949ef8758e4a1 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 3 May 2024 11:03:15 -0700 Subject: [PATCH 1558/1702] Docs/admin-guide/mm/damon/usage: fix wrong schemes effective quota update command To update effective size quota of DAMOS schemes on DAMON sysfs file interface, user should write 'update_schemes_effective_quotas' to the kdamond 'state' file. But the document is mistakenly saying the input string as 'update_schemes_effective_bytes'. Fix it (s/bytes/quotas/). Link: https://lkml.kernel.org/r/20240503180318.72798-8-sj@kernel.org Fixes: a6068d6dfa2f ("Docs/admin-guide/mm/damon/usage: document effective_bytes file") Signed-off-by: SeongJae Park Cc: [6.9.x] Cc: Jonathan Corbet Cc: Shuah Khan Signed-off-by: Andrew Morton --- Documentation/admin-guide/mm/damon/usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst index 3ce3f0aaa1d5b..e58ceb89ea2a7 100644 --- a/Documentation/admin-guide/mm/damon/usage.rst +++ b/Documentation/admin-guide/mm/damon/usage.rst @@ -153,7 +153,7 @@ Users can write below commands for the kdamond to the ``state`` file. - ``clear_schemes_tried_regions``: Clear the DAMON-based operating scheme action tried regions directory for each DAMON-based operation scheme of the kdamond. -- ``update_schemes_effective_bytes``: Update the contents of +- ``update_schemes_effective_quotas``: Update the contents of ``effective_bytes`` files for each DAMON-based operation scheme of the kdamond. For more details, refer to :ref:`quotas directory `. @@ -342,7 +342,7 @@ Based on the user-specified :ref:`goal `, the effective size quota is further adjusted. Reading ``effective_bytes`` returns the current effective size quota. The file is not updated in real time, so users should ask DAMON sysfs interface to update the content of the file for -the stats by writing a special keyword, ``update_schemes_effective_bytes`` to +the stats by writing a special keyword, ``update_schemes_effective_quotas`` to the relevant ``kdamonds//state`` file. Under ``weights`` directory, three files (``sz_permil``, From 2f02fbba35454dd05838d0c1552fcd39e1ab45f1 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 3 May 2024 11:03:16 -0700 Subject: [PATCH 1559/1702] Docs/mm/damon/design: use a list for supported filters Filters section is listing currently supported filter types in a normal paragraph. Since the number of types are higher than four, it is not easy to read for only specific types. Use a list for easier finding of specific types. [sj@kernel.org: fix build warning] Link: https://lkml.kernel.org/r/20240507161747.52430-1-sj@kernel.org Link: https://lkml.kernel.org/r/20240503180318.72798-9-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Corbet Cc: Shuah Khan Signed-off-by: Andrew Morton --- Documentation/mm/damon/design.rst | 46 +++++++++++++++++-------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index f2baf617184d0..3df387249937b 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -461,26 +461,32 @@ number of filters for each scheme. Each filter specifies the type of target memory, and whether it should exclude the memory of the type (filter-out), or all except the memory of the type (filter-in). -Currently, anonymous page, memory cgroup, young page, address range, and DAMON -monitoring target type filters are supported by the feature. Some filter -target types require additional arguments. The memory cgroup filter type asks -users to specify the file path of the memory cgroup for the filter. The -address range type asks the start and end addresses of the range. The DAMON -monitoring target type asks the index of the target from the context's -monitoring targets list. Hence, users can apply specific schemes to only -anonymous pages, non-anonymous pages, pages of specific cgroups, all pages -excluding those of specific cgroups, pages that not accessed after the last -access check from the scheme, pages that accessed after the last access check -from the scheme, pages in specific address range, pages in specific DAMON -monitoring targets, and any combination of those. - -To handle filters efficiently, the address range and DAMON monitoring target -type filters are handled by the core layer, while others are handled by -operations set. If a memory region is filtered by a core layer-handled filter, -it is not counted as the scheme has tried to the region. In contrast, if a -memory regions is filtered by an operations set layer-handled filter, it is -counted as the scheme has tried. The difference in accounting leads to changes -in the statistics. +For efficient handling of filters, some types of filters are handled by the +core layer, while others are handled by operations set. In the latter case, +hence, support of the filter types depends on the DAMON operations set. In +case of the core layer-handled filters, the memory regions that excluded by the +filter are not counted as the scheme has tried to the region. In contrast, if +a memory regions is filtered by an operations set layer-handled filter, it is +counted as the scheme has tried. This difference affects the statistics. + +Below types of filters are currently supported. + +- anonymous page + - Applied to pages that containing data that not stored in files. + - Handled by operations set layer. Supported by only ``paddr`` set. +- memory cgroup + - Applied to pages that belonging to a given cgroup. + - Handled by operations set layer. Supported by only ``paddr`` set. +- young page + - Applied to pages that are accessed after the last access check from the + scheme. + - Handled by operations set layer. Supported by only ``paddr`` set. +- address range + - Applied to pages that belonging to a given address range. + - Handled by the core logic. +- DAMON monitoring target + - Applied to pages that belonging to a given DAMON monitoring target. + - Handled by the core logic. Application Programming Interface From 437f7960d043bdce0eb6abe1c57232bc90a55fa9 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 3 May 2024 11:03:17 -0700 Subject: [PATCH 1560/1702] Docs/mm/damon/maintainer-profile: change the maintainer's timezone from PST to PT The document says the maintainer is working on only PST. The maintainer respects daylight saving system, though. Update the time zone to PT. Link: https://lkml.kernel.org/r/20240503180318.72798-10-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Corbet Cc: Shuah Khan Signed-off-by: Andrew Morton --- Documentation/mm/damon/maintainer-profile.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/mm/damon/maintainer-profile.rst b/Documentation/mm/damon/maintainer-profile.rst index 5a306e4de22e5..ea42f57cf9dcc 100644 --- a/Documentation/mm/damon/maintainer-profile.rst +++ b/Documentation/mm/damon/maintainer-profile.rst @@ -48,9 +48,9 @@ Review cadence -------------- The DAMON maintainer does the work on the usual work hour (09:00 to 17:00, -Mon-Fri) in PST. The response to patches will occasionally be slow. Do not -hesitate to send a ping if you have not heard back within a week of sending a -patch. +Mon-Fri) in PT (Pacific Time). The response to patches will occasionally be +slow. Do not hesitate to send a ping if you have not heard back within a week +of sending a patch. .. [1] https://git.kernel.org/akpm/mm/h/mm-unstable From 3974345f269c2701d4cad7bd3f2ed4445bdbbfbb Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 3 May 2024 11:03:18 -0700 Subject: [PATCH 1561/1702] Docs/mm/damon/maintainer-profile: allow posting patches based on damon/next tree The document mentions any patches for review should based on mm-unstable instead of damon/next. It should be the recommended process, but sometimes patches based on damon/next could be posted for some reasons. Actually, the DAMON-based tiered memory management patchset[1] was written on top of 'young page' DAMOS filter patchset, which was in damon/next tree as of the writing. Allow such case and just ask such things to be clearly specified. [1] https://lore.kernel.org/20240405060858.2818-1-honggyu.kim@sk.com Link: https://lkml.kernel.org/r/20240503180318.72798-11-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Corbet Cc: Shuah Khan Signed-off-by: Andrew Morton --- Documentation/mm/damon/maintainer-profile.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Documentation/mm/damon/maintainer-profile.rst b/Documentation/mm/damon/maintainer-profile.rst index ea42f57cf9dcc..8213cf61d38a5 100644 --- a/Documentation/mm/damon/maintainer-profile.rst +++ b/Documentation/mm/damon/maintainer-profile.rst @@ -20,9 +20,10 @@ management subsystem maintainer. After more sufficient tests, the patches will be queued in mm-stable [3]_ , and finally pull-requested to the mainline by the memory management subsystem maintainer. -Note again the patches for review should be made against the mm-unstable -tree [1]_ whenever possible. damon/next is only for preview of others' works -in progress. +Note again the patches for mm-unstable tree [1]_ are queued by the memory +management subsystem maintainer. If the patches requires some patches in +damon/next tree [2]_ which not yet merged in mm-unstable, please make sure the +requirement is clearly specified. Submit checklist addendum ------------------------- From e6b331ab0a716c1d9273383ae59c5b2eea5ac00d Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Thu, 2 May 2024 21:04:26 +0100 Subject: [PATCH 1562/1702] selftests: cgroup: remove redundant enabling of memory controller Memory controller is already enabled in main which invokes the test, hence this does not need to be done in test_no_kmem_bypass. Link: https://lkml.kernel.org/r/20240502200529.4193651-2-usamaarif642@gmail.com Signed-off-by: Usama Arif Acked-by: Yosry Ahmed Cc: Chengming Zhou Cc: Johannes Weiner Cc: Nhat Pham Signed-off-by: Andrew Morton --- tools/testing/selftests/cgroup/test_zswap.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/testing/selftests/cgroup/test_zswap.c b/tools/testing/selftests/cgroup/test_zswap.c index f0e488ed90d89..0f1023813c034 100644 --- a/tools/testing/selftests/cgroup/test_zswap.c +++ b/tools/testing/selftests/cgroup/test_zswap.c @@ -364,8 +364,6 @@ static int test_no_kmem_bypass(const char *root) trigger_allocation_size = sys_info.totalram / 20; /* Set up test memcg */ - if (cg_write(root, "cgroup.subtree_control", "+memory")) - goto out; test_group = cg_name(root, "kmem_bypass_test"); if (!test_group) goto out; From 4f687281012e2da1a34cfe08ab10ea1361c600d2 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Mon, 6 May 2024 19:29:24 +0000 Subject: [PATCH 1563/1702] mm: do not update memcg stats for NR_{FILE/SHMEM}_PMDMAPPED Previously, all NR_VM_EVENT_ITEMS stats were maintained per-memcg, although some of those fields are not exposed anywhere. Commit 14e0f6c957e39 ("memcg: reduce memory for the lruvec and memcg stats") changed this such that we only maintain the stats we actually expose per-memcg via a translation table. Additionally, commit 514462bbe927b ("memcg: warn for unexpected events and stats") added a warning if a per-memcg stat update is attempted for a stat that is not in the translation table. The warning started firing for the NR_{FILE/SHMEM}_PMDMAPPED stat updates in the rmap code. These stats are not maintained per-memcg, and hence are not in the translation table. Do not use __lruvec_stat_mod_folio() when updating NR_FILE_PMDMAPPED and NR_SHMEM_PMDMAPPED. Use __mod_node_page_state() instead, which updates the global per-node stats only. Link: https://lkml.kernel.org/r/20240506192924.271999-1-yosryahmed@google.com Fixes: 514462bbe927 ("memcg: warn for unexpected events and stats") Signed-off-by: Yosry Ahmed Reported-by: syzbot+9319a4268a640e26b72b@syzkaller.appspotmail.com Closes: https://lore.kernel.org/lkml/0000000000001b9d500617c8b23c@google.com Acked-by: Shakeel Butt Acked-by: David Hildenbrand Reviewed-by: Roman Gushchin Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Signed-off-by: Andrew Morton --- mm/rmap.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index fd021efe02a12..e8fc5ecb59b2f 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1435,13 +1435,14 @@ static __always_inline void __folio_add_file_rmap(struct folio *folio, struct page *page, int nr_pages, struct vm_area_struct *vma, enum rmap_level level) { + pg_data_t *pgdat = folio_pgdat(folio); int nr, nr_pmdmapped = 0; VM_WARN_ON_FOLIO(folio_test_anon(folio), folio); nr = __folio_add_rmap(folio, page, nr_pages, level, &nr_pmdmapped); if (nr_pmdmapped) - __lruvec_stat_mod_folio(folio, folio_test_swapbacked(folio) ? + __mod_node_page_state(pgdat, folio_test_swapbacked(folio) ? NR_SHMEM_PMDMAPPED : NR_FILE_PMDMAPPED, nr_pmdmapped); if (nr) __lruvec_stat_mod_folio(folio, NR_FILE_MAPPED, nr); @@ -1493,6 +1494,7 @@ static __always_inline void __folio_remove_rmap(struct folio *folio, enum rmap_level level) { atomic_t *mapped = &folio->_nr_pages_mapped; + pg_data_t *pgdat = folio_pgdat(folio); int last, nr = 0, nr_pmdmapped = 0; bool partially_mapped = false; enum node_stat_item idx; @@ -1540,13 +1542,14 @@ static __always_inline void __folio_remove_rmap(struct folio *folio, } if (nr_pmdmapped) { + /* NR_{FILE/SHMEM}_PMDMAPPED are not maintained per-memcg */ if (folio_test_anon(folio)) - idx = NR_ANON_THPS; - else if (folio_test_swapbacked(folio)) - idx = NR_SHMEM_PMDMAPPED; + __lruvec_stat_mod_folio(folio, NR_ANON_THPS, -nr_pmdmapped); else - idx = NR_FILE_PMDMAPPED; - __lruvec_stat_mod_folio(folio, idx, -nr_pmdmapped); + __mod_node_page_state(pgdat, + folio_test_swapbacked(folio) ? + NR_SHMEM_PMDMAPPED : NR_FILE_PMDMAPPED, + -nr_pmdmapped); } if (nr) { idx = folio_test_anon(folio) ? NR_ANON_MAPPED : NR_FILE_MAPPED; From 3b15f9d1c22dfe82efd03cb7acc2eeb557c735b5 Mon Sep 17 00:00:00 2001 From: Alex Rusuf Date: Mon, 6 May 2024 11:02:38 -0700 Subject: [PATCH 1564/1702] mm/damon/core: fix return value from damos_wmark_metric_value damos_wmark_metric_value's return value is 'unsigned long', so returning -EINVAL as 'unsigned long' may turn out to be very different from the expected one (using 2's complement) and treat as usual matric's value. So, fix that, checking if returned value is not 0. Link: https://lkml.kernel.org/r/20240506180238.53842-1-sj@kernel.org Fixes: ee801b7dd782 ("mm/damon/schemes: activate schemes based on a watermarks mechanism") Signed-off-by: Alex Rusuf Reviewed-by: SeongJae Park Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton --- mm/damon/core.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 939ecfcd4641f..6392f1cc97a3e 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1481,12 +1481,14 @@ static bool kdamond_need_stop(struct damon_ctx *ctx) return true; } -static unsigned long damos_wmark_metric_value(enum damos_wmark_metric metric) +static int damos_get_wmark_metric_value(enum damos_wmark_metric metric, + unsigned long *metric_value) { switch (metric) { case DAMOS_WMARK_FREE_MEM_RATE: - return global_zone_page_state(NR_FREE_PAGES) * 1000 / + *metric_value = global_zone_page_state(NR_FREE_PAGES) * 1000 / totalram_pages(); + return 0; default: break; } @@ -1501,10 +1503,9 @@ static unsigned long damos_wmark_wait_us(struct damos *scheme) { unsigned long metric; - if (scheme->wmarks.metric == DAMOS_WMARK_NONE) + if (damos_get_wmark_metric_value(scheme->wmarks.metric, &metric)) return 0; - metric = damos_wmark_metric_value(scheme->wmarks.metric); /* higher than high watermark or lower than low watermark */ if (metric > scheme->wmarks.high || scheme->wmarks.low > metric) { if (scheme->wmarks.activated) From a8248bb72fed5888bc3a0c7a4c97eb358906d7ea Mon Sep 17 00:00:00 2001 From: Xiu Jianfeng Date: Tue, 7 May 2024 13:23:24 +0000 Subject: [PATCH 1565/1702] mm: memcg: make alloc_mem_cgroup_per_node_info() return bool alloc_mem_cgroup_per_node_info() returns int that doesn't map to any errno error code. The only existing caller doesn't really need an error code so change the function to return bool (true on success) because this is slightly less confusing and more consistent with the other code. Link: https://lkml.kernel.org/r/20240507132324.1158510-1-xiujianfeng@huawei.com Signed-off-by: Xiu Jianfeng Acked-by: Michal Hocko Acked-by: Shakeel Butt Acked-by: Johannes Weiner Cc: Muchun Song Cc: Roman Gushchin Signed-off-by: Andrew Morton --- mm/memcontrol.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 2103ae77cf8d4..eef02a59b8c9c 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5637,13 +5637,13 @@ struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino) } #endif -static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) +static bool alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) { struct mem_cgroup_per_node *pn; pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, node); if (!pn) - return 1; + return false; pn->lruvec_stats = kzalloc_node(sizeof(struct lruvec_stats), GFP_KERNEL_ACCOUNT, node); @@ -5659,11 +5659,11 @@ static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) pn->memcg = memcg; memcg->nodeinfo[node] = pn; - return 0; + return true; fail: kfree(pn->lruvec_stats); kfree(pn); - return 1; + return false; } static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) @@ -5736,7 +5736,7 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent) } for_each_node(node) - if (alloc_mem_cgroup_per_node_info(memcg, node)) + if (!alloc_mem_cgroup_per_node_info(memcg, node)) goto fail; if (memcg_wb_domain_init(memcg, GFP_KERNEL)) From 158863e5d7cc6c9ee7e6c998ef84625caa2e88b3 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Wed, 8 May 2024 18:13:59 +0100 Subject: [PATCH 1566/1702] selftests: cgroup: add tests to verify the zswap writeback path Attempt writeback with the below steps and check using memory.stat.zswpwb if zswap writeback occurred: 1. Allocate memory. 2. Reclaim memory equal to the amount that was allocated in step 1. This will move it into zswap. 3. Save current zswap usage. 4. Move the memory allocated in step 1 back in from zswap. 5. Set zswap.max to half the amount that was recorded in step 3. 6. Attempt to reclaim memory equal to the amount that was allocated, this will either trigger writeback if it's enabled, or reclamation will fail if writeback is disabled as there isn't enough zswap space. Link: https://lkml.kernel.org/r/20240508171359.1545744-1-usamaarif642@gmail.com Signed-off-by: Usama Arif Suggested-by: Nhat Pham Acked-by: Yosry Ahmed Acked-by: Nhat Pham Cc: Chengming Zhou Cc: Johannes Weiner Signed-off-by: Andrew Morton --- tools/testing/selftests/cgroup/test_zswap.c | 130 +++++++++++++++++++- 1 file changed, 129 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/cgroup/test_zswap.c b/tools/testing/selftests/cgroup/test_zswap.c index 0f1023813c034..e0aed703f3da3 100644 --- a/tools/testing/selftests/cgroup/test_zswap.c +++ b/tools/testing/selftests/cgroup/test_zswap.c @@ -50,7 +50,7 @@ static int get_zswap_stored_pages(size_t *value) return read_int("/sys/kernel/debug/zswap/stored_pages", value); } -static int get_cg_wb_count(const char *cg) +static long get_cg_wb_count(const char *cg) { return cg_read_key_long(cg, "memory.stat", "zswpwb"); } @@ -248,6 +248,132 @@ static int test_zswapin(const char *root) return ret; } +/* + * Attempt writeback with the following steps: + * 1. Allocate memory. + * 2. Reclaim memory equal to the amount that was allocated in step 1. + This will move it into zswap. + * 3. Save current zswap usage. + * 4. Move the memory allocated in step 1 back in from zswap. + * 5. Set zswap.max to half the amount that was recorded in step 3. + * 6. Attempt to reclaim memory equal to the amount that was allocated, + this will either trigger writeback if it's enabled, or reclamation + will fail if writeback is disabled as there isn't enough zswap space. + */ +static int attempt_writeback(const char *cgroup, void *arg) +{ + long pagesize = sysconf(_SC_PAGESIZE); + char *test_group = arg; + size_t memsize = MB(4); + char buf[pagesize]; + long zswap_usage; + bool wb_enabled; + int ret = -1; + char *mem; + + wb_enabled = cg_read_long(test_group, "memory.zswap.writeback"); + mem = (char *)malloc(memsize); + if (!mem) + return ret; + + /* + * Fill half of each page with increasing data, and keep other + * half empty, this will result in data that is still compressible + * and ends up in zswap, with material zswap usage. + */ + for (int i = 0; i < pagesize; i++) + buf[i] = i < pagesize/2 ? (char) i : 0; + + for (int i = 0; i < memsize; i += pagesize) + memcpy(&mem[i], buf, pagesize); + + /* Try and reclaim allocated memory */ + if (cg_write_numeric(test_group, "memory.reclaim", memsize)) { + ksft_print_msg("Failed to reclaim all of the requested memory\n"); + goto out; + } + + zswap_usage = cg_read_long(test_group, "memory.zswap.current"); + + /* zswpin */ + for (int i = 0; i < memsize; i += pagesize) { + if (memcmp(&mem[i], buf, pagesize)) { + ksft_print_msg("invalid memory\n"); + goto out; + } + } + + if (cg_write_numeric(test_group, "memory.zswap.max", zswap_usage/2)) + goto out; + + /* + * If writeback is enabled, trying to reclaim memory now will trigger a + * writeback as zswap.max is half of what was needed when reclaim ran the first time. + * If writeback is disabled, memory reclaim will fail as zswap is limited and + * it can't writeback to swap. + */ + ret = cg_write_numeric(test_group, "memory.reclaim", memsize); + if (!wb_enabled) + ret = (ret == -EAGAIN) ? 0 : -1; + +out: + free(mem); + return ret; +} + +/* Test to verify the zswap writeback path */ +static int test_zswap_writeback(const char *root, bool wb) +{ + long zswpwb_before, zswpwb_after; + int ret = KSFT_FAIL; + char *test_group; + + test_group = cg_name(root, "zswap_writeback_test"); + if (!test_group) + goto out; + if (cg_create(test_group)) + goto out; + if (cg_write(test_group, "memory.zswap.writeback", wb ? "1" : "0")) + goto out; + + zswpwb_before = get_cg_wb_count(test_group); + if (zswpwb_before != 0) { + ksft_print_msg("zswpwb_before = %ld instead of 0\n", zswpwb_before); + goto out; + } + + if (cg_run(test_group, attempt_writeback, (void *) test_group)) + goto out; + + /* Verify that zswap writeback occurred only if writeback was enabled */ + zswpwb_after = get_cg_wb_count(test_group); + if (zswpwb_after < 0) + goto out; + + if (wb != !!zswpwb_after) { + ksft_print_msg("zswpwb_after is %ld while wb is %s", + zswpwb_after, wb ? "enabled" : "disabled"); + goto out; + } + + ret = KSFT_PASS; + +out: + cg_destroy(test_group); + free(test_group); + return ret; +} + +static int test_zswap_writeback_enabled(const char *root) +{ + return test_zswap_writeback(root, true); +} + +static int test_zswap_writeback_disabled(const char *root) +{ + return test_zswap_writeback(root, false); +} + /* * When trying to store a memcg page in zswap, if the memcg hits its memory * limit in zswap, writeback should affect only the zswapped pages of that @@ -423,6 +549,8 @@ struct zswap_test { T(test_zswap_usage), T(test_swapin_nozswap), T(test_zswapin), + T(test_zswap_writeback_enabled), + T(test_zswap_writeback_disabled), T(test_no_kmem_bypass), T(test_no_invasive_cgroup_shrink), }; From 8e34419f4de3eaeed57aee69076105e5d88fbcbe Mon Sep 17 00:00:00 2001 From: Oscar Salvador Date: Thu, 9 May 2024 12:01:47 +0200 Subject: [PATCH 1567/1702] mm/hugetlb: add missing VM_FAULT_SET_HINDEX in hugetlb_fault Patch series "Minor fixups for hugetlb fault path". This series contains a couple of fixups for hugetlb_fault and hugetlb_wp respectively, where a VM_FAULT_SET_HINDEX call was missing. I did not bother with a Fixes tag because the missing piece here is that we will not report to userspace the right extension of the faulty area by adjusting struct kernel_siginfo.si_addr_lsb, but I do not consider that to be a big issue because I assume that userspace already knows the size of the mapping anyway. This patch (of 2): commit af19487f00f3 ("mm: make PTE_MARKER_SWAPIN_ERROR more general") added the code to handle pte_markers in hugetlb faulting path. In case of an UFFD_POISON event, a PTE_MARKER_POISONED will be created and we will return VM_FAULT_HWPOISON_LARGE upon detecting that in the fault path. Add the missing VM_FAULT_SET_HINDEX, so the right si_addr_lsb will be passed to userspace to report the extension of the faulty area. Link: https://lkml.kernel.org/r/20240509100148.22384-1-osalvador@suse.de Link: https://lkml.kernel.org/r/20240509100148.22384-2-osalvador@suse.de Signed-off-by: Oscar Salvador Acked-by: Peter Xu Acked-by: Axel Rasmussen Cc: Liu Shixin Cc: Muchun Song Signed-off-by: Andrew Morton --- mm/hugetlb.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 33d175add0447..262456daf3274 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6485,7 +6485,8 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, pte_marker_get(pte_to_swp_entry(vmf.orig_pte)); if (marker & PTE_MARKER_POISONED) { - ret = VM_FAULT_HWPOISON_LARGE; + ret = VM_FAULT_HWPOISON_LARGE | + VM_FAULT_SET_HINDEX(hstate_index(h)); goto out_mutex; } } From 88e4f525002bd3c28fe7a272ffcce743d07c02bd Mon Sep 17 00:00:00 2001 From: Oscar Salvador Date: Thu, 9 May 2024 12:01:48 +0200 Subject: [PATCH 1568/1702] mm/hugetlb: add missing VM_FAULT_SET_HINDEX in hugetlb_wp commit 1cb9dc4b475c ("mm: hwpoison: support recovery from HugePage copy-on-write faults") added support to use the mc variants when coping hugetlb pages on CoW faults. Add the missing VM_FAULT_SET_HINDEX, so the right si_addr_lsb will be passed to userspace to report the extension of the faulty area. Link: https://lkml.kernel.org/r/20240509100148.22384-3-osalvador@suse.de Signed-off-by: Oscar Salvador Acked-by: Peter Xu Acked-by: Axel Rasmussen Cc: Liu Shixin Cc: Muchun Song Signed-off-by: Andrew Morton --- mm/hugetlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 262456daf3274..6be78e7d4f6e0 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6067,7 +6067,7 @@ static vm_fault_t hugetlb_wp(struct folio *pagecache_folio, goto out_release_all; if (copy_user_large_folio(new_folio, old_folio, vmf->real_address, vma)) { - ret = VM_FAULT_HWPOISON_LARGE; + ret = VM_FAULT_HWPOISON_LARGE | VM_FAULT_SET_HINDEX(hstate_index(h)); goto out_release_all; } __folio_mark_uptodate(new_folio); From b665eed25fed247510486353985060a9ba50c6a3 Mon Sep 17 00:00:00 2001 From: Dev Jain Date: Thu, 9 May 2024 15:24:47 +0530 Subject: [PATCH 1569/1702] selftests/mm: hugetlb_madv_vs_map: avoid test skipping by querying hugepage size at runtime Currently, the size used in mmap() is statically defined, leading to skipping of the test on a hugepage size other than 2 MB, since munmap() won't free the hugepage for a size greater than 2 MB. Hence, query the size at runtime. Also, there is no reason why a hugepage allocation should fail, since we are using a simple mmap() using MAP_HUGETLB; hence, instead of skipping the test, make it fail. Link: https://lkml.kernel.org/r/20240509095447.3791573-1-dev.jain@arm.com Signed-off-by: Dev Jain Reviewed-by: Muhammad Usama Anjum Cc: Anshuman Khandual Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/hugetlb_madv_vs_map.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/mm/hugetlb_madv_vs_map.c b/tools/testing/selftests/mm/hugetlb_madv_vs_map.c index d01e8d4901d0b..8f122a0f08281 100644 --- a/tools/testing/selftests/mm/hugetlb_madv_vs_map.c +++ b/tools/testing/selftests/mm/hugetlb_madv_vs_map.c @@ -27,9 +27,9 @@ #include "vm_util.h" #include "../kselftest.h" -#define MMAP_SIZE (1 << 21) #define INLOOP_ITER 100 +size_t mmap_size; char *huge_ptr; /* Touch the memory while it is being madvised() */ @@ -44,7 +44,7 @@ void *touch(void *unused) void *madv(void *unused) { for (int i = 0; i < INLOOP_ITER; i++) - madvise(huge_ptr, MMAP_SIZE, MADV_DONTNEED); + madvise(huge_ptr, mmap_size, MADV_DONTNEED); return NULL; } @@ -59,7 +59,7 @@ void *map_extra(void *unused) void *ptr; for (int i = 0; i < INLOOP_ITER; i++) { - ptr = mmap(NULL, MMAP_SIZE, PROT_READ | PROT_WRITE, + ptr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); @@ -93,14 +93,16 @@ int main(void) free_hugepages); } + mmap_size = default_huge_page_size(); + while (max--) { - huge_ptr = mmap(NULL, MMAP_SIZE, PROT_READ | PROT_WRITE, + huge_ptr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); if ((unsigned long)huge_ptr == -1) { - ksft_exit_skip("Failed to allocated huge page\n"); - return KSFT_SKIP; + ksft_test_result_fail("Failed to allocate huge page\n"); + return KSFT_FAIL; } pthread_create(&thread1, NULL, madv, NULL); @@ -117,7 +119,7 @@ int main(void) } /* Unmap and restart */ - munmap(huge_ptr, MMAP_SIZE); + munmap(huge_ptr, mmap_size); } return KSFT_PASS; From 76edc534cc289308130272a2ac28694fc9b72a03 Mon Sep 17 00:00:00 2001 From: Xiu Jianfeng Date: Thu, 9 May 2024 03:26:28 +0000 Subject: [PATCH 1570/1702] memcg, oom: cleanup unused memcg_oom_gfp_mask and memcg_oom_order Since commit 857f21397f71 ("memcg, oom: remove unnecessary check in mem_cgroup_oom_synchronize()"), memcg_oom_gfp_mask and memcg_oom_order are no longer used any more. Link: https://lkml.kernel.org/r/20240509032628.1217652-1-xiujianfeng@huawei.com Signed-off-by: Xiu Jianfeng Acked-by: Michal Hocko Acked-by: Shakeel Butt Reviewed-by: Roman Gushchin Cc: Johannes Weiner Cc: Muchun Song Cc: Benjamin Segall Cc: Daniel Bristot de Oliveira Cc: Dietmar Eggemann Cc: Ingo Molnar Cc: Juri Lelli Cc: Mel Gorman Cc: Peter Zijlstra Cc: Steven Rostedt (Google) Cc: Valentin Schneider Cc: Vincent Guittot Signed-off-by: Andrew Morton --- include/linux/sched.h | 2 -- mm/memcontrol.c | 2 -- 2 files changed, 4 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 4118b3f959c32..427de5e4754b5 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1448,8 +1448,6 @@ struct task_struct { #ifdef CONFIG_MEMCG struct mem_cgroup *memcg_in_oom; - gfp_t memcg_oom_gfp_mask; - int memcg_oom_order; /* Number of pages to reclaim on returning to userland: */ unsigned int memcg_nr_pages_over_high; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index eef02a59b8c9c..7fad15b2290c7 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2188,8 +2188,6 @@ static bool mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order) if (current->in_user_fault) { css_get(&memcg->css); current->memcg_in_oom = memcg; - current->memcg_oom_gfp_mask = mask; - current->memcg_oom_order = order; } return false; } From a7ac59f4f23660473e6350306f9b88f24fcc38f1 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 30 Apr 2024 14:09:01 +0900 Subject: [PATCH 1571/1702] nilfs2: remove calls to folio_set_error() and folio_clear_error() Nobody checks this flag on nilfs2 folios, stop setting and clearing it. That lets us simplify nilfs_end_folio_io() slightly. Link: https://lkml.kernel.org/r/20240420025029.2166544-17-willy@infradead.org Link: https://lkml.kernel.org/r/20240430050901.3239-1-konishi.ryusuke@gmail.com Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Ryusuke Konishi Cc: kernel test robot Cc: Peter Zijlstra Cc: Song Liu Signed-off-by: Andrew Morton --- fs/nilfs2/dir.c | 1 - fs/nilfs2/segment.c | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/fs/nilfs2/dir.c b/fs/nilfs2/dir.c index bc846b904b68d..0900fcad2d0c2 100644 --- a/fs/nilfs2/dir.c +++ b/fs/nilfs2/dir.c @@ -174,7 +174,6 @@ static bool nilfs_check_folio(struct folio *folio, char *kaddr) dir->i_ino, (folio->index << PAGE_SHIFT) + offs, (unsigned long)le64_to_cpu(p->inode)); fail: - folio_set_error(folio); return false; } diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c index aa5290cb7467c..8654ab8ad5340 100644 --- a/fs/nilfs2/segment.c +++ b/fs/nilfs2/segment.c @@ -1725,14 +1725,8 @@ static void nilfs_end_folio_io(struct folio *folio, int err) return; } - if (!err) { - if (!nilfs_folio_buffers_clean(folio)) - filemap_dirty_folio(folio->mapping, folio); - folio_clear_error(folio); - } else { + if (err || !nilfs_folio_buffers_clean(folio)) filemap_dirty_folio(folio->mapping, folio); - folio_set_error(folio); - } folio_end_writeback(folio); } From eb59a58113717df04b8a8229befd8ab1e5dbf86e Mon Sep 17 00:00:00 2001 From: Edward Liaw Date: Mon, 29 Apr 2024 23:46:09 +0000 Subject: [PATCH 1572/1702] selftests/kcmp: remove unused open mode Android bionic warns that open modes are ignored if O_CREAT or O_TMPFILE aren't specified. The permissions for the file are set above: fd1 = open(kpath, O_RDWR | O_CREAT | O_TRUNC, 0644); Link: https://lkml.kernel.org/r/20240429234610.191144-1-edliaw@google.com Fixes: d97b46a64674 ("syscalls, x86: add __NR_kcmp syscall") Signed-off-by: Edward Liaw Reviewed-by: Cyrill Gorcunov Cc: Eric Biederman Cc: Shuah Khan Signed-off-by: Andrew Morton --- tools/testing/selftests/kcmp/kcmp_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kcmp/kcmp_test.c b/tools/testing/selftests/kcmp/kcmp_test.c index 25110c7c0b3ed..d7a8e321bb16b 100644 --- a/tools/testing/selftests/kcmp/kcmp_test.c +++ b/tools/testing/selftests/kcmp/kcmp_test.c @@ -91,7 +91,7 @@ int main(int argc, char **argv) ksft_print_header(); ksft_set_plan(3); - fd2 = open(kpath, O_RDWR, 0644); + fd2 = open(kpath, O_RDWR); if (fd2 < 0) { perror("Can't open file"); ksft_exit_fail(); From 33580d667bb20e00356fd06500f5197ef1baa1f5 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Tue, 7 May 2024 23:24:54 +0900 Subject: [PATCH 1573/1702] nilfs2: use __field_struct() for a bitwise field As one can see in include/trace/stages/stage4_event_fields.h, the implementation of __field() uses the is_signed_type() macro. As one can see in commit dcf8e5633e2e ("tracing: Define the is_signed_type() macro once"), there has been an attempt to not make is_signed_type() trigger sparse warnings for bitwise types. Despite that change, sparse complains when passing a bitwise type to is_signed_type(). The reason is that in its definition below, an inequality comparison will be made against bitwise types, which are random collections of bits (the casts to bitwise types themselves are semantically valid and not problematic): #define is_signed_type(type) (((type)(-1)) < (__force type)1) So, as a workaround, follow the example of and suppress the following sparse warnings by changing __field() into __field_struct() that doesn't use is_signed_type(): fs/nilfs2/segment.c: note: in included file (through include/trace/trace_events.h, include/trace/define_trace.h, include/trace/events/nilfs2.h): ./include/trace/events/nilfs2.h:191:1: warning: cast to restricted blk_opf_t ./include/trace/events/nilfs2.h:191:1: warning: restricted blk_opf_t degrades to integer ./include/trace/events/nilfs2.h:191:1: warning: restricted blk_opf_t degrades to integer [konishi.ryusuke: describe the reason for the warnings per Linus's explanation] Link: https://lkml.kernel.org/r/20240507222041.4876-1-konishi.ryusuke@gmail.com Link: https://lkml.kernel.org/r/20240507142454.3344-1-konishi.ryusuke@gmail.com Signed-off-by: Bart Van Assche Signed-off-by: Ryusuke Konishi Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202401092241.I4mm9OWl-lkp@intel.com/ Reported-by: Ryusuke Konishi Closes: https://lore.kernel.org/all/20240430080019.4242-2-konishi.ryusuke@gmail.com/ Cc: Linus Torvalds Cc: Rasmus Villemoes Signed-off-by: Andrew Morton --- include/trace/events/nilfs2.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/trace/events/nilfs2.h b/include/trace/events/nilfs2.h index 8efc6236f57c3..8880c11733dd3 100644 --- a/include/trace/events/nilfs2.h +++ b/include/trace/events/nilfs2.h @@ -200,7 +200,11 @@ TRACE_EVENT(nilfs2_mdt_submit_block, __field(struct inode *, inode) __field(unsigned long, ino) __field(unsigned long, blkoff) - __field(enum req_op, mode) + /* + * Use field_struct() to avoid is_signed_type() on the + * bitwise type enum req_op. + */ + __field_struct(enum req_op, mode) ), TP_fast_assign( From 6813216bbdba18e182759d949589be95ebef290f Mon Sep 17 00:00:00 2001 From: Barry Song Date: Tue, 7 May 2024 15:27:56 +1200 Subject: [PATCH 1574/1702] Documentation: coding-style: ask function-like macros to evaluate parameters Patch series "codingstyle: avoid unused parameters for a function-like macro", v7. A function-like macro could result in build warnings such as "unused variable." This patchset updates the guidance to recommend always using a static inline function instead and also provides checkpatch support for this new rule. This patch (of 2): Recent commit 77292bb8ca69c80 ("crypto: scomp - remove memcpy if sg_nents is 1 and pages are lowmem") leads to warnings on xtensa and loongarch, In file included from crypto/scompress.c:12: include/crypto/scatterwalk.h: In function 'scatterwalk_pagedone': include/crypto/scatterwalk.h:76:30: warning: variable 'page' set but not used [-Wunused-but-set-variable] 76 | struct page *page; | ^~~~ crypto/scompress.c: In function 'scomp_acomp_comp_decomp': >> crypto/scompress.c:174:38: warning: unused variable 'dst_page' [-Wunused-variable] 174 | struct page *dst_page = sg_page(req->dst); | The reason is that flush_dcache_page() is implemented as a noop macro on these platforms as below, #define flush_dcache_page(page) do { } while (0) The driver code, for itself, seems be quite innocent and placing maybe_unused seems pointless, struct page *dst_page = sg_page(req->dst); for (i = 0; i < nr_pages; i++) flush_dcache_page(dst_page + i); And it should be independent of architectural implementation differences. Let's provide guidance on coding style for requesting parameter evaluation or proposing the migration to a static inline function. Link: https://lkml.kernel.org/r/20240507032757.146386-1-21cnbao@gmail.com Link: https://lkml.kernel.org/r/20240507032757.146386-2-21cnbao@gmail.com Signed-off-by: Barry Song Suggested-by: Max Filippov Reviewed-by: Mark Brown Acked-by: Joe Perches Cc: Chris Zankel Cc: Huacai Chen Cc: Herbert Xu Cc: Guenter Roeck Cc: Stephen Rothwell Cc: Andy Whitcroft Cc: Dwaipayan Ray Cc: Joe Perches Cc: Jonathan Corbet Cc: Lukas Bulwahn Cc: Xining Xu Cc: Charlemagne Lasse Cc: Jeff Johnson Signed-off-by: Andrew Morton --- Documentation/process/coding-style.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Documentation/process/coding-style.rst b/Documentation/process/coding-style.rst index 9c7cf73473943..7e768c65aa926 100644 --- a/Documentation/process/coding-style.rst +++ b/Documentation/process/coding-style.rst @@ -827,6 +827,29 @@ Macros with multiple statements should be enclosed in a do - while block: do_this(b, c); \ } while (0) +Function-like macros with unused parameters should be replaced by static +inline functions to avoid the issue of unused variables: + +.. code-block:: c + + static inline void fun(struct foo *foo) + { + } + +Due to historical practices, many files still employ the "cast to (void)" +approach to evaluate parameters. However, this method is not advisable. +Inline functions address the issue of "expression with side effects +evaluated more than once", circumvent unused-variable problems, and +are generally better documented than macros for some reason. + +.. code-block:: c + + /* + * Avoid doing this whenever possible and instead opt for static + * inline functions + */ + #define macrofun(foo) do { (void) (foo); } while (0) + Things to avoid when using macros: 1) macros that affect control flow: From b1be5844c1a0124a49a30a20a189d0a53aa10578 Mon Sep 17 00:00:00 2001 From: Xining Xu Date: Tue, 7 May 2024 15:27:57 +1200 Subject: [PATCH 1575/1702] scripts: checkpatch: check unused parameters for function-like macro If function-like macros do not utilize a parameter, it might result in a build warning. In our coding style guidelines, we advocate for utilizing static inline functions to replace such macros. This patch verifies compliance with the new rule. For a macro such as the one below, #define test(a) do { } while (0) The test result is as follows. WARNING: Argument 'a' is not used in function-like macro #21: FILE: mm/init-mm.c:20: +#define test(a) do { } while (0) total: 0 errors, 1 warnings, 8 lines checked Link: https://lkml.kernel.org/r/20240507032757.146386-3-21cnbao@gmail.com Signed-off-by: Xining Xu Tested-by: Barry Song Signed-off-by: Barry Song Acked-by: Joe Perches Cc: Chris Zankel Cc: Huacai Chen Cc: Herbert Xu Cc: Guenter Roeck Cc: Stephen Rothwell Cc: Mark Brown Cc: Andy Whitcroft Cc: Dwaipayan Ray Cc: Jonathan Corbet Cc: Lukas Bulwahn Cc: Max Filippov Cc: Jeff Johnson Cc: Charlemagne Lasse Signed-off-by: Andrew Morton --- Documentation/dev-tools/checkpatch.rst | 14 ++++++++++++++ scripts/checkpatch.pl | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/Documentation/dev-tools/checkpatch.rst b/Documentation/dev-tools/checkpatch.rst index 1279689958473..a9fac978a5251 100644 --- a/Documentation/dev-tools/checkpatch.rst +++ b/Documentation/dev-tools/checkpatch.rst @@ -906,6 +906,20 @@ Macros, Attributes and Symbols See: https://lore.kernel.org/lkml/1399671106.2912.21.camel@joe-AO725/ + **MACRO_ARG_UNUSED** + If function-like macros do not utilize a parameter, it might result + in a build warning. We advocate for utilizing static inline functions + to replace such macros. + For example, for a macro such as the one below:: + + #define test(a) do { } while (0) + + there would be a warning like below:: + + WARNING: Argument 'a' is not used in function-like macro. + + See: https://www.kernel.org/doc/html/latest/process/coding-style.html#macros-enums-and-rtl + **SINGLE_STATEMENT_DO_WHILE_MACRO** For the multi-statement macros, it is necessary to use the do-while loop to avoid unpredictable code paths. The do-while loop helps to diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 9c4c4a61bc832..2b812210b412b 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -6040,6 +6040,12 @@ sub process { CHK("MACRO_ARG_PRECEDENCE", "Macro argument '$arg' may be better as '($arg)' to avoid precedence issues\n" . "$herectx"); } + +# check if this is an unused argument + if ($define_stmt !~ /\b$arg\b/) { + WARN("MACRO_ARG_UNUSED", + "Argument '$arg' is not used in function-like macro\n" . "$herectx"); + } } # check for macros with flow control, but without ## concatenation From 0a73eac1ed10097d1799c10dff2172605fd40c75 Mon Sep 17 00:00:00 2001 From: Ryusuke Konishi Date: Thu, 9 May 2024 07:14:29 +0900 Subject: [PATCH 1576/1702] nilfs2: convert BUG_ON() in nilfs_finish_roll_forward() to WARN_ON() The BUG_ON check performed on the return value of __getblk() in nilfs_finish_roll_forward() assumes that a buffer that has been successfully read once is retrieved with the same parameters and does not fail (__getblk() does not return an error due to memory allocation failure). Also, nilfs_finish_roll_forward() is called at most once during mount. Taking these into consideration, rewrite the check to use WARN_ON() to avoid using BUG_ON(). Link: https://lkml.kernel.org/r/20240508221429.7559-1-konishi.ryusuke@gmail.com Signed-off-by: Ryusuke Konishi Signed-off-by: Andrew Morton --- fs/nilfs2/recovery.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/nilfs2/recovery.c b/fs/nilfs2/recovery.c index e48372618ac40..020f304c600e1 100644 --- a/fs/nilfs2/recovery.c +++ b/fs/nilfs2/recovery.c @@ -699,7 +699,9 @@ static void nilfs_finish_roll_forward(struct the_nilfs *nilfs, return; bh = __getblk(nilfs->ns_bdev, ri->ri_lsegs_start, nilfs->ns_blocksize); - BUG_ON(!bh); + if (WARN_ON(!bh)) + return; /* should never happen */ + memset(bh->b_data, 0, bh->b_size); set_buffer_dirty(bh); err = sync_dirty_buffer(bh); From 5cbcb62dddf5346077feb82b7b0c9254222d3445 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Tue, 7 May 2024 09:18:58 -0400 Subject: [PATCH 1577/1702] fs/proc: fix softlockup in __read_vmcore While taking a kernel core dump with makedumpfile on a larger system, softlockup messages often appear. While softlockup warnings can be harmless, they can also interfere with things like RCU freeing memory, which can be problematic when the kdump kexec image is configured with as little memory as possible. Avoid the softlockup, and give things like work items and RCU a chance to do their thing during __read_vmcore by adding a cond_resched. Link: https://lkml.kernel.org/r/20240507091858.36ff767f@imladris.surriel.com Signed-off-by: Rik van Riel Acked-by: Baoquan He Cc: Dave Young Cc: Vivek Goyal Cc: Signed-off-by: Andrew Morton --- fs/proc/vmcore.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/proc/vmcore.c b/fs/proc/vmcore.c index 5d08d4d159d30..b52d85f8ad592 100644 --- a/fs/proc/vmcore.c +++ b/fs/proc/vmcore.c @@ -383,6 +383,8 @@ static ssize_t __read_vmcore(struct iov_iter *iter, loff_t *fpos) /* leave now if filled buffer already */ if (!iov_iter_count(iter)) return acc; + + cond_resched(); } list_for_each_entry(m, &vmcore_list, list) { From 49ca2b2ef3d003402584c68ae7b3055ba72e750a Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Thu, 9 May 2024 10:39:33 +0300 Subject: [PATCH 1578/1702] RDMA/IPoIB: Fix format truncation compilation errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Truncate the device name to store IPoIB VLAN name. [leonro@5b4e8fba4ddd kernel]$ make -s -j 20 allmodconfig [leonro@5b4e8fba4ddd kernel]$ make -s -j 20 W=1 drivers/infiniband/ulp/ipoib/ drivers/infiniband/ulp/ipoib/ipoib_vlan.c: In function ‘ipoib_vlan_add’: drivers/infiniband/ulp/ipoib/ipoib_vlan.c:187:52: error: ‘%04x’ directive output may be truncated writing 4 bytes into a region of size between 0 and 15 [-Werror=format-truncation=] 187 | snprintf(intf_name, sizeof(intf_name), "%s.%04x", | ^~~~ drivers/infiniband/ulp/ipoib/ipoib_vlan.c:187:48: note: directive argument in the range [0, 65535] 187 | snprintf(intf_name, sizeof(intf_name), "%s.%04x", | ^~~~~~~~~ drivers/infiniband/ulp/ipoib/ipoib_vlan.c:187:9: note: ‘snprintf’ output between 6 and 21 bytes into a destination of size 16 187 | snprintf(intf_name, sizeof(intf_name), "%s.%04x", | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 188 | ppriv->dev->name, pkey); | ~~~~~~~~~~~~~~~~~~~~~~~ cc1: all warnings being treated as errors make[6]: *** [scripts/Makefile.build:244: drivers/infiniband/ulp/ipoib/ipoib_vlan.o] Error 1 make[6]: *** Waiting for unfinished jobs.... Fixes: 9baa0b036410 ("IB/ipoib: Add rtnl_link_ops support") Link: https://lore.kernel.org/r/e9d3e1fef69df4c9beaf402cc3ac342bad680791.1715240029.git.leon@kernel.org Signed-off-by: Leon Romanovsky --- drivers/infiniband/ulp/ipoib/ipoib_vlan.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c index 4bd161e86f8dd..562df2b3ef187 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c @@ -184,8 +184,12 @@ int ipoib_vlan_add(struct net_device *pdev, unsigned short pkey) ppriv = ipoib_priv(pdev); - snprintf(intf_name, sizeof(intf_name), "%s.%04x", - ppriv->dev->name, pkey); + /* If you increase IFNAMSIZ, update snprintf below + * to allow longer names. + */ + BUILD_BUG_ON(IFNAMSIZ != 16); + snprintf(intf_name, sizeof(intf_name), "%.10s.%04x", ppriv->dev->name, + pkey); ndev = ipoib_intf_alloc(ppriv->ca, ppriv->port, intf_name); if (IS_ERR(ndev)) { From 9c0731832d3b7420cbadba6a7f334363bc8dfb15 Mon Sep 17 00:00:00 2001 From: Zhu Yanjun Date: Fri, 10 May 2024 23:12:47 +0200 Subject: [PATCH 1579/1702] RDMA/cma: Fix kmemleak in rdma_core observed during blktests nvme/rdma use siw When running blktests nvme/rdma, the following kmemleak issue will appear. kmemleak: Kernel memory leak detector initialized (mempool available:36041) kmemleak: Automatic memory scanning thread started kmemleak: 2 new suspected memory leaks (see /sys/kernel/debug/kmemleak) kmemleak: 8 new suspected memory leaks (see /sys/kernel/debug/kmemleak) kmemleak: 17 new suspected memory leaks (see /sys/kernel/debug/kmemleak) kmemleak: 4 new suspected memory leaks (see /sys/kernel/debug/kmemleak) unreferenced object 0xffff88855da53400 (size 192): comm "rdma", pid 10630, jiffies 4296575922 hex dump (first 32 bytes): 37 00 00 00 00 00 00 00 c0 ff ff ff 1f 00 00 00 7............... 10 34 a5 5d 85 88 ff ff 10 34 a5 5d 85 88 ff ff .4.].....4.].... backtrace (crc 47f66721): [] kmalloc_trace+0x30d/0x3b0 [] alloc_gid_entry+0x47/0x380 [ib_core] [] add_modify_gid+0x166/0x930 [ib_core] [] ib_cache_update.part.0+0x6d8/0x910 [ib_core] [] ib_cache_setup_one+0x24a/0x350 [ib_core] [] ib_register_device+0x9e/0x3a0 [ib_core] [] 0xffffffffc2a3d389 [] nldev_newlink+0x2b8/0x520 [ib_core] [] rdma_nl_rcv_msg+0x2c3/0x520 [ib_core] [] rdma_nl_rcv_skb.constprop.0.isra.0+0x23c/0x3a0 [ib_core] [] netlink_unicast+0x445/0x710 [] netlink_sendmsg+0x761/0xc40 [] __sys_sendto+0x3a9/0x420 [] __x64_sys_sendto+0xdc/0x1b0 [] do_syscall_64+0x93/0x180 [] entry_SYSCALL_64_after_hwframe+0x71/0x79 The root cause: rdma_put_gid_attr is not called when sgid_attr is set to ERR_PTR(-ENODEV). Reported-and-tested-by: Yi Zhang Closes: https://lore.kernel.org/all/19bf5745-1b3b-4b8a-81c2-20d945943aaf@linux.dev/T/ Fixes: f8ef1be816bf ("RDMA/cma: Avoid GID lookups on iWARP devices") Reviewed-by: Chuck Lever Signed-off-by: Zhu Yanjun Link: https://lore.kernel.org/r/20240510211247.31345-1-yanjun.zhu@linux.dev Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/cma.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 1e2cd7c8716e8..64ace0b968f07 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -715,8 +715,10 @@ cma_validate_port(struct ib_device *device, u32 port, rcu_read_lock(); ndev = rcu_dereference(sgid_attr->ndev); if (!net_eq(dev_net(ndev), dev_addr->net) || - ndev->ifindex != bound_if_index) + ndev->ifindex != bound_if_index) { + rdma_put_gid_attr(sgid_attr); sgid_attr = ERR_PTR(-ENODEV); + } rcu_read_unlock(); goto out; } From 42212936d9d811c7cf6efc4804747a6c417aafd4 Mon Sep 17 00:00:00 2001 From: Ivanov Mikhail Date: Tue, 26 Mar 2024 17:56:25 +0800 Subject: [PATCH 1580/1702] samples/landlock: Fix incorrect free in populate_ruleset_net MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pointer env_port_name changes after strsep(). Memory allocated via strdup() will not be freed if landlock_add_rule() returns non-zero value. Fixes: 5e990dcef12e ("samples/landlock: Support TCP restrictions") Signed-off-by: Ivanov Mikhail Reviewed-by: Konstantin Meskhidze Link: https://lore.kernel.org/r/20240326095625.3576164-1-ivanov.mikhail1@huawei-partners.com Signed-off-by: Mickaël Salaün --- samples/landlock/sandboxer.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/samples/landlock/sandboxer.c b/samples/landlock/sandboxer.c index 32e930c853bba..8b8ecd65c28c4 100644 --- a/samples/landlock/sandboxer.c +++ b/samples/landlock/sandboxer.c @@ -153,7 +153,7 @@ static int populate_ruleset_net(const char *const env_var, const int ruleset_fd, const __u64 allowed_access) { int ret = 1; - char *env_port_name, *strport; + char *env_port_name, *env_port_name_next, *strport; struct landlock_net_port_attr net_port = { .allowed_access = allowed_access, .port = 0, @@ -165,7 +165,8 @@ static int populate_ruleset_net(const char *const env_var, const int ruleset_fd, env_port_name = strdup(env_port_name); unsetenv(env_var); - while ((strport = strsep(&env_port_name, ENV_DELIMITER))) { + env_port_name_next = env_port_name; + while ((strport = strsep(&env_port_name_next, ENV_DELIMITER))) { net_port.port = atoi(strport); if (landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, &net_port, 0)) { From b25f7415eb4108aa32dd3e74289d7f997090708f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Noack?= Date: Fri, 19 Apr 2024 16:11:12 +0000 Subject: [PATCH 1581/1702] landlock: Add IOCTL access right for character and block devices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces the LANDLOCK_ACCESS_FS_IOCTL_DEV right and increments the Landlock ABI version to 5. This access right applies to device-custom IOCTL commands when they are invoked on block or character device files. Like the truncate right, this right is associated with a file descriptor at the time of open(2), and gets respected even when the file descriptor is used outside of the thread which it was originally opened in. Therefore, a newly enabled Landlock policy does not apply to file descriptors which are already open. If the LANDLOCK_ACCESS_FS_IOCTL_DEV right is handled, only a small number of safe IOCTL commands will be permitted on newly opened device files. These include FIOCLEX, FIONCLEX, FIONBIO and FIOASYNC, as well as other IOCTL commands for regular files which are implemented in fs/ioctl.c. Noteworthy scenarios which require special attention: TTY devices are often passed into a process from the parent process, and so a newly enabled Landlock policy does not retroactively apply to them automatically. In the past, TTY devices have often supported IOCTL commands like TIOCSTI and some TIOCLINUX subcommands, which were letting callers control the TTY input buffer (and simulate keypresses). This should be restricted to CAP_SYS_ADMIN programs on modern kernels though. Known limitations: The LANDLOCK_ACCESS_FS_IOCTL_DEV access right is a coarse-grained control over IOCTL commands. Landlock users may use path-based restrictions in combination with their knowledge about the file system layout to control what IOCTLs can be done. Cc: Paul Moore Cc: Christian Brauner Cc: Arnd Bergmann Signed-off-by: Günther Noack Link: https://lore.kernel.org/r/20240419161122.2023765-2-gnoack@google.com Signed-off-by: Mickaël Salaün --- include/uapi/linux/landlock.h | 38 +++- security/landlock/fs.c | 225 ++++++++++++++++++- security/landlock/limits.h | 2 +- security/landlock/syscalls.c | 2 +- tools/testing/selftests/landlock/base_test.c | 2 +- tools/testing/selftests/landlock/fs_test.c | 5 +- 6 files changed, 258 insertions(+), 16 deletions(-) diff --git a/include/uapi/linux/landlock.h b/include/uapi/linux/landlock.h index 25c8d76775393..68625e728f43d 100644 --- a/include/uapi/linux/landlock.h +++ b/include/uapi/linux/landlock.h @@ -128,7 +128,7 @@ struct landlock_net_port_attr { * files and directories. Files or directories opened before the sandboxing * are not subject to these restrictions. * - * A file can only receive these access rights: + * The following access rights apply only to files: * * - %LANDLOCK_ACCESS_FS_EXECUTE: Execute a file. * - %LANDLOCK_ACCESS_FS_WRITE_FILE: Open a file with write access. Note that @@ -138,12 +138,13 @@ struct landlock_net_port_attr { * - %LANDLOCK_ACCESS_FS_READ_FILE: Open a file with read access. * - %LANDLOCK_ACCESS_FS_TRUNCATE: Truncate a file with :manpage:`truncate(2)`, * :manpage:`ftruncate(2)`, :manpage:`creat(2)`, or :manpage:`open(2)` with - * ``O_TRUNC``. Whether an opened file can be truncated with - * :manpage:`ftruncate(2)` is determined during :manpage:`open(2)`, in the - * same way as read and write permissions are checked during - * :manpage:`open(2)` using %LANDLOCK_ACCESS_FS_READ_FILE and - * %LANDLOCK_ACCESS_FS_WRITE_FILE. This access right is available since the - * third version of the Landlock ABI. + * ``O_TRUNC``. This access right is available since the third version of the + * Landlock ABI. + * + * Whether an opened file can be truncated with :manpage:`ftruncate(2)` or used + * with `ioctl(2)` is determined during :manpage:`open(2)`, in the same way as + * read and write permissions are checked during :manpage:`open(2)` using + * %LANDLOCK_ACCESS_FS_READ_FILE and %LANDLOCK_ACCESS_FS_WRITE_FILE. * * A directory can receive access rights related to files or directories. The * following access right is applied to the directory itself, and the @@ -198,13 +199,33 @@ struct landlock_net_port_attr { * If multiple requirements are not met, the ``EACCES`` error code takes * precedence over ``EXDEV``. * + * The following access right applies both to files and directories: + * + * - %LANDLOCK_ACCESS_FS_IOCTL_DEV: Invoke :manpage:`ioctl(2)` commands on an opened + * character or block device. + * + * This access right applies to all `ioctl(2)` commands implemented by device + * drivers. However, the following common IOCTL commands continue to be + * invokable independent of the %LANDLOCK_ACCESS_FS_IOCTL_DEV right: + * + * * IOCTL commands targeting file descriptors (``FIOCLEX``, ``FIONCLEX``), + * * IOCTL commands targeting file descriptions (``FIONBIO``, ``FIOASYNC``), + * * IOCTL commands targeting file systems (``FIFREEZE``, ``FITHAW``, + * ``FIGETBSZ``, ``FS_IOC_GETFSUUID``, ``FS_IOC_GETFSSYSFSPATH``) + * * Some IOCTL commands which do not make sense when used with devices, but + * whose implementations are safe and return the right error codes + * (``FS_IOC_FIEMAP``, ``FICLONE``, ``FICLONERANGE``, ``FIDEDUPERANGE``) + * + * This access right is available since the fifth version of the Landlock + * ABI. + * * .. warning:: * * It is currently not possible to restrict some file-related actions * accessible through these syscall families: :manpage:`chdir(2)`, * :manpage:`stat(2)`, :manpage:`flock(2)`, :manpage:`chmod(2)`, * :manpage:`chown(2)`, :manpage:`setxattr(2)`, :manpage:`utime(2)`, - * :manpage:`ioctl(2)`, :manpage:`fcntl(2)`, :manpage:`access(2)`. + * :manpage:`fcntl(2)`, :manpage:`access(2)`. * Future Landlock evolutions will enable to restrict them. */ /* clang-format off */ @@ -223,6 +244,7 @@ struct landlock_net_port_attr { #define LANDLOCK_ACCESS_FS_MAKE_SYM (1ULL << 12) #define LANDLOCK_ACCESS_FS_REFER (1ULL << 13) #define LANDLOCK_ACCESS_FS_TRUNCATE (1ULL << 14) +#define LANDLOCK_ACCESS_FS_IOCTL_DEV (1ULL << 15) /* clang-format on */ /** diff --git a/security/landlock/fs.c b/security/landlock/fs.c index c15559432d3d5..22d8b7c28074e 100644 --- a/security/landlock/fs.c +++ b/security/landlock/fs.c @@ -5,8 +5,11 @@ * Copyright © 2016-2020 Mickaël Salaün * Copyright © 2018-2020 ANSSI * Copyright © 2021-2022 Microsoft Corporation + * Copyright © 2022 Günther Noack + * Copyright © 2023-2024 Google LLC */ +#include #include #include #include @@ -14,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +33,7 @@ #include #include #include +#include #include #include "common.h" @@ -84,6 +89,160 @@ static const struct landlock_object_underops landlock_fs_underops = { .release = release_inode }; +/* IOCTL helpers */ + +/** + * is_masked_device_ioctl - Determine whether an IOCTL command is always + * permitted with Landlock for device files. These commands can not be + * restricted on device files by enforcing a Landlock policy. + * + * @cmd: The IOCTL command that is supposed to be run. + * + * By default, any IOCTL on a device file requires the + * LANDLOCK_ACCESS_FS_IOCTL_DEV right. However, we blanket-permit some + * commands, if: + * + * 1. The command is implemented in fs/ioctl.c's do_vfs_ioctl(), + * not in f_ops->unlocked_ioctl() or f_ops->compat_ioctl(). + * + * 2. The command is harmless when invoked on devices. + * + * We also permit commands that do not make sense for devices, but where the + * do_vfs_ioctl() implementation returns a more conventional error code. + * + * Any new IOCTL commands that are implemented in fs/ioctl.c's do_vfs_ioctl() + * should be considered for inclusion here. + * + * Returns: true if the IOCTL @cmd can not be restricted with Landlock for + * device files. + */ +static __attribute_const__ bool is_masked_device_ioctl(const unsigned int cmd) +{ + switch (cmd) { + /* + * FIOCLEX, FIONCLEX, FIONBIO and FIOASYNC manipulate the FD's + * close-on-exec and the file's buffered-IO and async flags. These + * operations are also available through fcntl(2), and are + * unconditionally permitted in Landlock. + */ + case FIOCLEX: + case FIONCLEX: + case FIONBIO: + case FIOASYNC: + /* + * FIOQSIZE queries the size of a regular file, directory, or link. + * + * We still permit it, because it always returns -ENOTTY for + * other file types. + */ + case FIOQSIZE: + /* + * FIFREEZE and FITHAW freeze and thaw the file system which the + * given file belongs to. Requires CAP_SYS_ADMIN. + * + * These commands operate on the file system's superblock rather + * than on the file itself. The same operations can also be + * done through any other file or directory on the same file + * system, so it is safe to permit these. + */ + case FIFREEZE: + case FITHAW: + /* + * FS_IOC_FIEMAP queries information about the allocation of + * blocks within a file. + * + * This IOCTL command only makes sense for regular files and is + * not implemented by devices. It is harmless to permit. + */ + case FS_IOC_FIEMAP: + /* + * FIGETBSZ queries the file system's block size for a file or + * directory. + * + * This command operates on the file system's superblock rather + * than on the file itself. The same operation can also be done + * through any other file or directory on the same file system, + * so it is safe to permit it. + */ + case FIGETBSZ: + /* + * FICLONE, FICLONERANGE and FIDEDUPERANGE make files share + * their underlying storage ("reflink") between source and + * destination FDs, on file systems which support that. + * + * These IOCTL commands only apply to regular files + * and are harmless to permit for device files. + */ + case FICLONE: + case FICLONERANGE: + case FIDEDUPERANGE: + /* + * FS_IOC_GETFSUUID and FS_IOC_GETFSSYSFSPATH both operate on + * the file system superblock, not on the specific file, so + * these operations are available through any other file on the + * same file system as well. + */ + case FS_IOC_GETFSUUID: + case FS_IOC_GETFSSYSFSPATH: + return true; + + /* + * FIONREAD, FS_IOC_GETFLAGS, FS_IOC_SETFLAGS, FS_IOC_FSGETXATTR and + * FS_IOC_FSSETXATTR are forwarded to device implementations. + */ + + /* + * file_ioctl() commands (FIBMAP, FS_IOC_RESVSP, FS_IOC_RESVSP64, + * FS_IOC_UNRESVSP, FS_IOC_UNRESVSP64 and FS_IOC_ZERO_RANGE) are + * forwarded to device implementations, so not permitted. + */ + + /* Other commands are guarded by the access right. */ + default: + return false; + } +} + +/* + * is_masked_device_ioctl_compat - same as the helper above, but checking the + * "compat" IOCTL commands. + * + * The IOCTL commands with special handling in compat-mode should behave the + * same as their non-compat counterparts. + */ +static __attribute_const__ bool +is_masked_device_ioctl_compat(const unsigned int cmd) +{ + switch (cmd) { + /* FICLONE is permitted, same as in the non-compat variant. */ + case FICLONE: + return true; + +#if defined(CONFIG_X86_64) + /* + * FS_IOC_RESVSP_32, FS_IOC_RESVSP64_32, FS_IOC_UNRESVSP_32, + * FS_IOC_UNRESVSP64_32, FS_IOC_ZERO_RANGE_32: not blanket-permitted, + * for consistency with their non-compat variants. + */ + case FS_IOC_RESVSP_32: + case FS_IOC_RESVSP64_32: + case FS_IOC_UNRESVSP_32: + case FS_IOC_UNRESVSP64_32: + case FS_IOC_ZERO_RANGE_32: +#endif + + /* + * FS_IOC32_GETFLAGS, FS_IOC32_SETFLAGS are forwarded to their device + * implementations. + */ + case FS_IOC32_GETFLAGS: + case FS_IOC32_SETFLAGS: + return false; + default: + return is_masked_device_ioctl(cmd); + } +} + /* Ruleset management */ static struct landlock_object *get_inode_object(struct inode *const inode) @@ -148,7 +307,8 @@ static struct landlock_object *get_inode_object(struct inode *const inode) LANDLOCK_ACCESS_FS_EXECUTE | \ LANDLOCK_ACCESS_FS_WRITE_FILE | \ LANDLOCK_ACCESS_FS_READ_FILE | \ - LANDLOCK_ACCESS_FS_TRUNCATE) + LANDLOCK_ACCESS_FS_TRUNCATE | \ + LANDLOCK_ACCESS_FS_IOCTL_DEV) /* clang-format on */ /* @@ -1332,11 +1492,18 @@ static int hook_file_alloc_security(struct file *const file) return 0; } +static bool is_device(const struct file *const file) +{ + const struct inode *inode = file_inode(file); + + return S_ISBLK(inode->i_mode) || S_ISCHR(inode->i_mode); +} + static int hook_file_open(struct file *const file) { layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_FS] = {}; - access_mask_t open_access_request, full_access_request, allowed_access; - const access_mask_t optional_access = LANDLOCK_ACCESS_FS_TRUNCATE; + access_mask_t open_access_request, full_access_request, allowed_access, + optional_access; const struct landlock_ruleset *const dom = get_fs_domain(landlock_cred(file->f_cred)->domain); @@ -1354,6 +1521,10 @@ static int hook_file_open(struct file *const file) * We look up more access than what we immediately need for open(), so * that we can later authorize operations on opened files. */ + optional_access = LANDLOCK_ACCESS_FS_TRUNCATE; + if (is_device(file)) + optional_access |= LANDLOCK_ACCESS_FS_IOCTL_DEV; + full_access_request = open_access_request | optional_access; if (is_access_to_paths_allowed( @@ -1410,6 +1581,52 @@ static int hook_file_truncate(struct file *const file) return -EACCES; } +static int hook_file_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + access_mask_t allowed_access = landlock_file(file)->allowed_access; + + /* + * It is the access rights at the time of opening the file which + * determine whether IOCTL can be used on the opened file later. + * + * The access right is attached to the opened file in hook_file_open(). + */ + if (allowed_access & LANDLOCK_ACCESS_FS_IOCTL_DEV) + return 0; + + if (!is_device(file)) + return 0; + + if (is_masked_device_ioctl(cmd)) + return 0; + + return -EACCES; +} + +static int hook_file_ioctl_compat(struct file *file, unsigned int cmd, + unsigned long arg) +{ + access_mask_t allowed_access = landlock_file(file)->allowed_access; + + /* + * It is the access rights at the time of opening the file which + * determine whether IOCTL can be used on the opened file later. + * + * The access right is attached to the opened file in hook_file_open(). + */ + if (allowed_access & LANDLOCK_ACCESS_FS_IOCTL_DEV) + return 0; + + if (!is_device(file)) + return 0; + + if (is_masked_device_ioctl_compat(cmd)) + return 0; + + return -EACCES; +} + static struct security_hook_list landlock_hooks[] __ro_after_init = { LSM_HOOK_INIT(inode_free_security, hook_inode_free_security), @@ -1432,6 +1649,8 @@ static struct security_hook_list landlock_hooks[] __ro_after_init = { LSM_HOOK_INIT(file_alloc_security, hook_file_alloc_security), LSM_HOOK_INIT(file_open, hook_file_open), LSM_HOOK_INIT(file_truncate, hook_file_truncate), + LSM_HOOK_INIT(file_ioctl, hook_file_ioctl), + LSM_HOOK_INIT(file_ioctl_compat, hook_file_ioctl_compat), }; __init void landlock_add_fs_hooks(void) diff --git a/security/landlock/limits.h b/security/landlock/limits.h index 93c9c6f915567..20fdb5ff35148 100644 --- a/security/landlock/limits.h +++ b/security/landlock/limits.h @@ -18,7 +18,7 @@ #define LANDLOCK_MAX_NUM_LAYERS 16 #define LANDLOCK_MAX_NUM_RULES U32_MAX -#define LANDLOCK_LAST_ACCESS_FS LANDLOCK_ACCESS_FS_TRUNCATE +#define LANDLOCK_LAST_ACCESS_FS LANDLOCK_ACCESS_FS_IOCTL_DEV #define LANDLOCK_MASK_ACCESS_FS ((LANDLOCK_LAST_ACCESS_FS << 1) - 1) #define LANDLOCK_NUM_ACCESS_FS __const_hweight64(LANDLOCK_MASK_ACCESS_FS) #define LANDLOCK_SHIFT_ACCESS_FS 0 diff --git a/security/landlock/syscalls.c b/security/landlock/syscalls.c index 6788e73b6681b..03b470f5a85a3 100644 --- a/security/landlock/syscalls.c +++ b/security/landlock/syscalls.c @@ -149,7 +149,7 @@ static const struct file_operations ruleset_fops = { .write = fop_dummy_write, }; -#define LANDLOCK_ABI_VERSION 4 +#define LANDLOCK_ABI_VERSION 5 /** * sys_landlock_create_ruleset - Create a new ruleset diff --git a/tools/testing/selftests/landlock/base_test.c b/tools/testing/selftests/landlock/base_test.c index a6f89aaea77d1..3c1e9f35b5312 100644 --- a/tools/testing/selftests/landlock/base_test.c +++ b/tools/testing/selftests/landlock/base_test.c @@ -75,7 +75,7 @@ TEST(abi_version) const struct landlock_ruleset_attr ruleset_attr = { .handled_access_fs = LANDLOCK_ACCESS_FS_READ_FILE, }; - ASSERT_EQ(4, landlock_create_ruleset(NULL, 0, + ASSERT_EQ(5, landlock_create_ruleset(NULL, 0, LANDLOCK_CREATE_RULESET_VERSION)); ASSERT_EQ(-1, landlock_create_ruleset(&ruleset_attr, 0, diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c index 27744524df519..17b00e6c778d3 100644 --- a/tools/testing/selftests/landlock/fs_test.c +++ b/tools/testing/selftests/landlock/fs_test.c @@ -536,9 +536,10 @@ TEST_F_FORK(layout1, inval) LANDLOCK_ACCESS_FS_EXECUTE | \ LANDLOCK_ACCESS_FS_WRITE_FILE | \ LANDLOCK_ACCESS_FS_READ_FILE | \ - LANDLOCK_ACCESS_FS_TRUNCATE) + LANDLOCK_ACCESS_FS_TRUNCATE | \ + LANDLOCK_ACCESS_FS_IOCTL_DEV) -#define ACCESS_LAST LANDLOCK_ACCESS_FS_TRUNCATE +#define ACCESS_LAST LANDLOCK_ACCESS_FS_IOCTL_DEV #define ACCESS_ALL ( \ ACCESS_FILE | \ From 3ecf19e56843a6fe65f109c773728c36d220f947 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Noack?= Date: Fri, 19 Apr 2024 16:11:13 +0000 Subject: [PATCH 1582/1702] selftests/landlock: Test IOCTL support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exercises Landlock's IOCTL feature in different combinations of handling and permitting the LANDLOCK_ACCESS_FS_IOCTL_DEV right, and in different combinations of using files and directories. Signed-off-by: Günther Noack Link: https://lore.kernel.org/r/20240419161122.2023765-3-gnoack@google.com Signed-off-by: Mickaël Salaün --- tools/testing/selftests/landlock/fs_test.c | 192 ++++++++++++++++++++- 1 file changed, 189 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c index 17b00e6c778d3..fd7793b413d16 100644 --- a/tools/testing/selftests/landlock/fs_test.c +++ b/tools/testing/selftests/landlock/fs_test.c @@ -8,6 +8,7 @@ */ #define _GNU_SOURCE +#include #include #include #include @@ -16,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +26,12 @@ #include #include +/* + * Intentionally included last to work around header conflict. + * See https://sourceware.org/glibc/wiki/Synchronizing_Headers. + */ +#include + #include "common.h" #ifndef renameat2 @@ -744,6 +752,9 @@ static int create_ruleset(struct __test_metadata *const _metadata, } for (i = 0; rules[i].path; i++) { + if (!rules[i].access) + continue; + add_path_beneath(_metadata, ruleset_fd, rules[i].access, rules[i].path); } @@ -3452,7 +3463,7 @@ TEST_F_FORK(layout1, truncate_unhandled) LANDLOCK_ACCESS_FS_WRITE_FILE; int ruleset_fd; - /* Enable Landlock. */ + /* Enables Landlock. */ ruleset_fd = create_ruleset(_metadata, handled, rules); ASSERT_LE(0, ruleset_fd); @@ -3535,7 +3546,7 @@ TEST_F_FORK(layout1, truncate) LANDLOCK_ACCESS_FS_TRUNCATE; int ruleset_fd; - /* Enable Landlock. */ + /* Enables Landlock. */ ruleset_fd = create_ruleset(_metadata, handled, rules); ASSERT_LE(0, ruleset_fd); @@ -3761,7 +3772,7 @@ TEST_F_FORK(ftruncate, open_and_ftruncate) }; int fd, ruleset_fd; - /* Enable Landlock. */ + /* Enables Landlock. */ ruleset_fd = create_ruleset(_metadata, variant->handled, rules); ASSERT_LE(0, ruleset_fd); enforce_ruleset(_metadata, ruleset_fd); @@ -3854,6 +3865,181 @@ TEST(memfd_ftruncate) ASSERT_EQ(0, close(fd)); } +static int test_fionread_ioctl(int fd) +{ + size_t sz = 0; + + if (ioctl(fd, FIONREAD, &sz) < 0 && errno == EACCES) + return errno; + return 0; +} + +/* clang-format off */ +FIXTURE(ioctl) {}; + +FIXTURE_SETUP(ioctl) {}; + +FIXTURE_TEARDOWN(ioctl) {}; +/* clang-format on */ + +FIXTURE_VARIANT(ioctl) +{ + const __u64 handled; + const __u64 allowed; + const mode_t open_mode; + /* + * FIONREAD is used as a characteristic device-specific IOCTL command. + * It is implemented in fs/ioctl.c for regular files, + * but we do not blanket-permit it for devices. + */ + const int expected_fionread_result; +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(ioctl, handled_i_allowed_none) { + /* clang-format on */ + .handled = LANDLOCK_ACCESS_FS_IOCTL_DEV, + .allowed = 0, + .open_mode = O_RDWR, + .expected_fionread_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(ioctl, handled_i_allowed_i) { + /* clang-format on */ + .handled = LANDLOCK_ACCESS_FS_IOCTL_DEV, + .allowed = LANDLOCK_ACCESS_FS_IOCTL_DEV, + .open_mode = O_RDWR, + .expected_fionread_result = 0, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(ioctl, unhandled) { + /* clang-format on */ + .handled = LANDLOCK_ACCESS_FS_EXECUTE, + .allowed = LANDLOCK_ACCESS_FS_EXECUTE, + .open_mode = O_RDWR, + .expected_fionread_result = 0, +}; + +TEST_F_FORK(ioctl, handle_dir_access_file) +{ + const int flag = 0; + const struct rule rules[] = { + { + .path = "/dev", + .access = variant->allowed, + }, + {}, + }; + int file_fd, ruleset_fd; + + /* Enables Landlock. */ + ruleset_fd = create_ruleset(_metadata, variant->handled, rules); + ASSERT_LE(0, ruleset_fd); + enforce_ruleset(_metadata, ruleset_fd); + ASSERT_EQ(0, close(ruleset_fd)); + + file_fd = open("/dev/zero", variant->open_mode); + ASSERT_LE(0, file_fd); + + /* Checks that IOCTL commands return the expected errors. */ + EXPECT_EQ(variant->expected_fionread_result, + test_fionread_ioctl(file_fd)); + + /* Checks that unrestrictable commands are unrestricted. */ + EXPECT_EQ(0, ioctl(file_fd, FIOCLEX)); + EXPECT_EQ(0, ioctl(file_fd, FIONCLEX)); + EXPECT_EQ(0, ioctl(file_fd, FIONBIO, &flag)); + EXPECT_EQ(0, ioctl(file_fd, FIOASYNC, &flag)); + EXPECT_EQ(0, ioctl(file_fd, FIGETBSZ, &flag)); + + ASSERT_EQ(0, close(file_fd)); +} + +TEST_F_FORK(ioctl, handle_dir_access_dir) +{ + const int flag = 0; + const struct rule rules[] = { + { + .path = "/dev", + .access = variant->allowed, + }, + {}, + }; + int dir_fd, ruleset_fd; + + /* Enables Landlock. */ + ruleset_fd = create_ruleset(_metadata, variant->handled, rules); + ASSERT_LE(0, ruleset_fd); + enforce_ruleset(_metadata, ruleset_fd); + ASSERT_EQ(0, close(ruleset_fd)); + + /* + * Ignore variant->open_mode for this test, as we intend to open a + * directory. If the directory can not be opened, the variant is + * infeasible to test with an opened directory. + */ + dir_fd = open("/dev", O_RDONLY); + if (dir_fd < 0) + return; + + /* + * Checks that IOCTL commands return the expected errors. + * We do not use the expected values from the fixture here. + * + * When using IOCTL on a directory, no Landlock restrictions apply. + */ + EXPECT_EQ(0, test_fionread_ioctl(dir_fd)); + + /* Checks that unrestrictable commands are unrestricted. */ + EXPECT_EQ(0, ioctl(dir_fd, FIOCLEX)); + EXPECT_EQ(0, ioctl(dir_fd, FIONCLEX)); + EXPECT_EQ(0, ioctl(dir_fd, FIONBIO, &flag)); + EXPECT_EQ(0, ioctl(dir_fd, FIOASYNC, &flag)); + EXPECT_EQ(0, ioctl(dir_fd, FIGETBSZ, &flag)); + + ASSERT_EQ(0, close(dir_fd)); +} + +TEST_F_FORK(ioctl, handle_file_access_file) +{ + const int flag = 0; + const struct rule rules[] = { + { + .path = "/dev/zero", + .access = variant->allowed, + }, + {}, + }; + int file_fd, ruleset_fd; + + /* Enables Landlock. */ + ruleset_fd = create_ruleset(_metadata, variant->handled, rules); + ASSERT_LE(0, ruleset_fd); + enforce_ruleset(_metadata, ruleset_fd); + ASSERT_EQ(0, close(ruleset_fd)); + + file_fd = open("/dev/zero", variant->open_mode); + ASSERT_LE(0, file_fd) + { + TH_LOG("Failed to open /dev/zero: %s", strerror(errno)); + } + + /* Checks that IOCTL commands return the expected errors. */ + EXPECT_EQ(variant->expected_fionread_result, + test_fionread_ioctl(file_fd)); + + /* Checks that unrestrictable commands are unrestricted. */ + EXPECT_EQ(0, ioctl(file_fd, FIOCLEX)); + EXPECT_EQ(0, ioctl(file_fd, FIONCLEX)); + EXPECT_EQ(0, ioctl(file_fd, FIONBIO, &flag)); + EXPECT_EQ(0, ioctl(file_fd, FIOASYNC, &flag)); + EXPECT_EQ(0, ioctl(file_fd, FIGETBSZ, &flag)); + + ASSERT_EQ(0, close(file_fd)); +} + /* clang-format off */ FIXTURE(layout1_bind) {}; /* clang-format on */ From dd6d32afdf5f1869a8f543e8efdd191f7e4b0368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Noack?= Date: Fri, 19 Apr 2024 16:11:14 +0000 Subject: [PATCH 1583/1702] selftests/landlock: Test IOCTL with memfds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Because the LANDLOCK_ACCESS_FS_IOCTL_DEV right is associated with the opened file during open(2), IOCTLs are supposed to work with files which are opened by means other than open(2). Signed-off-by: Günther Noack Link: https://lore.kernel.org/r/20240419161122.2023765-4-gnoack@google.com Signed-off-by: Mickaël Salaün --- tools/testing/selftests/landlock/fs_test.c | 44 ++++++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c index fd7793b413d16..193dc58bac7a9 100644 --- a/tools/testing/selftests/landlock/fs_test.c +++ b/tools/testing/selftests/landlock/fs_test.c @@ -3849,20 +3849,48 @@ TEST_F_FORK(ftruncate, open_and_ftruncate_in_different_processes) ASSERT_EQ(0, close(socket_fds[1])); } -TEST(memfd_ftruncate) +/* Invokes the FS_IOC_GETFLAGS IOCTL and returns its errno or 0. */ +static int test_fs_ioc_getflags_ioctl(int fd) { - int fd; + uint32_t flags; + + if (ioctl(fd, FS_IOC_GETFLAGS, &flags) < 0) + return errno; + return 0; +} - fd = memfd_create("name", MFD_CLOEXEC); - ASSERT_LE(0, fd); +TEST(memfd_ftruncate_and_ioctl) +{ + const struct landlock_ruleset_attr attr = { + .handled_access_fs = ACCESS_ALL, + }; + int ruleset_fd, fd, i; /* - * Checks that ftruncate is permitted on file descriptors that are - * created in ways other than open(2). + * We exercise the same test both with and without Landlock enabled, to + * ensure that it behaves the same in both cases. */ - EXPECT_EQ(0, test_ftruncate(fd)); + for (i = 0; i < 2; i++) { + /* Creates a new memfd. */ + fd = memfd_create("name", MFD_CLOEXEC); + ASSERT_LE(0, fd); - ASSERT_EQ(0, close(fd)); + /* + * Checks that operations associated with the opened file + * (ftruncate, ioctl) are permitted on file descriptors that are + * created in ways other than open(2). + */ + EXPECT_EQ(0, test_ftruncate(fd)); + EXPECT_EQ(0, test_fs_ioc_getflags_ioctl(fd)); + + ASSERT_EQ(0, close(fd)); + + /* Enables Landlock. */ + ruleset_fd = landlock_create_ruleset(&attr, sizeof(attr), 0); + ASSERT_LE(0, ruleset_fd); + enforce_ruleset(_metadata, ruleset_fd); + ASSERT_EQ(0, close(ruleset_fd)); + } } static int test_fionread_ioctl(int fd) From 7954a1d155975a3fbc9570151358738f59605121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Noack?= Date: Fri, 19 Apr 2024 16:11:15 +0000 Subject: [PATCH 1584/1702] selftests/landlock: Test ioctl(2) and ftruncate(2) with open(O_PATH) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ioctl(2) and ftruncate(2) operations on files opened with O_PATH should always return EBADF, independent of the LANDLOCK_ACCESS_FS_TRUNCATE and LANDLOCK_ACCESS_FS_IOCTL_DEV access rights in that file hierarchy. Suggested-by: Mickaël Salaün Signed-off-by: Günther Noack Link: https://lore.kernel.org/r/20240419161122.2023765-5-gnoack@google.com Signed-off-by: Mickaël Salaün --- tools/testing/selftests/landlock/fs_test.c | 40 ++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c index 193dc58bac7a9..cb6e9330fcf56 100644 --- a/tools/testing/selftests/landlock/fs_test.c +++ b/tools/testing/selftests/landlock/fs_test.c @@ -3902,6 +3902,46 @@ static int test_fionread_ioctl(int fd) return 0; } +TEST_F_FORK(layout1, o_path_ftruncate_and_ioctl) +{ + const struct landlock_ruleset_attr attr = { + .handled_access_fs = ACCESS_ALL, + }; + int ruleset_fd, fd; + + /* + * Checks that for files opened with O_PATH, both ioctl(2) and + * ftruncate(2) yield EBADF, as it is documented in open(2) for the + * O_PATH flag. + */ + fd = open(dir_s1d1, O_PATH | O_CLOEXEC); + ASSERT_LE(0, fd); + + EXPECT_EQ(EBADF, test_ftruncate(fd)); + EXPECT_EQ(EBADF, test_fs_ioc_getflags_ioctl(fd)); + + ASSERT_EQ(0, close(fd)); + + /* Enables Landlock. */ + ruleset_fd = landlock_create_ruleset(&attr, sizeof(attr), 0); + ASSERT_LE(0, ruleset_fd); + enforce_ruleset(_metadata, ruleset_fd); + ASSERT_EQ(0, close(ruleset_fd)); + + /* + * Checks that after enabling Landlock, + * - the file can still be opened with O_PATH + * - both ioctl and truncate still yield EBADF (not EACCES). + */ + fd = open(dir_s1d1, O_PATH | O_CLOEXEC); + ASSERT_LE(0, fd); + + EXPECT_EQ(EBADF, test_ftruncate(fd)); + EXPECT_EQ(EBADF, test_fs_ioc_getflags_ioctl(fd)); + + ASSERT_EQ(0, close(fd)); +} + /* clang-format off */ FIXTURE(ioctl) {}; From 56ffd377c7abe670ba5f024328e0c0fc0f49920c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Noack?= Date: Fri, 19 Apr 2024 16:11:16 +0000 Subject: [PATCH 1585/1702] selftests/landlock: Test IOCTLs on named pipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Named pipes should behave like pipes created with pipe(2), so we don't want to restrict IOCTLs on them. Suggested-by: Mickaël Salaün Signed-off-by: Günther Noack Link: https://lore.kernel.org/r/20240419161122.2023765-6-gnoack@google.com Signed-off-by: Mickaël Salaün --- tools/testing/selftests/landlock/fs_test.c | 43 ++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c index cb6e9330fcf56..b133020c7761c 100644 --- a/tools/testing/selftests/landlock/fs_test.c +++ b/tools/testing/selftests/landlock/fs_test.c @@ -3942,6 +3942,49 @@ TEST_F_FORK(layout1, o_path_ftruncate_and_ioctl) ASSERT_EQ(0, close(fd)); } +/* + * Named pipes are not governed by the LANDLOCK_ACCESS_FS_IOCTL_DEV right, + * because they are not character or block devices. + */ +TEST_F_FORK(layout1, named_pipe_ioctl) +{ + pid_t child_pid; + int fd, ruleset_fd; + const char *const path = file1_s1d1; + const struct landlock_ruleset_attr attr = { + .handled_access_fs = LANDLOCK_ACCESS_FS_IOCTL_DEV, + }; + + ASSERT_EQ(0, unlink(path)); + ASSERT_EQ(0, mkfifo(path, 0600)); + + /* Enables Landlock. */ + ruleset_fd = landlock_create_ruleset(&attr, sizeof(attr), 0); + ASSERT_LE(0, ruleset_fd); + enforce_ruleset(_metadata, ruleset_fd); + ASSERT_EQ(0, close(ruleset_fd)); + + /* The child process opens the pipe for writing. */ + child_pid = fork(); + ASSERT_NE(-1, child_pid); + if (child_pid == 0) { + fd = open(path, O_WRONLY); + close(fd); + exit(0); + } + + fd = open(path, O_RDONLY); + ASSERT_LE(0, fd); + + /* FIONREAD is implemented by pipefifo_fops. */ + EXPECT_EQ(0, test_fionread_ioctl(fd)); + + ASSERT_EQ(0, close(fd)); + ASSERT_EQ(0, unlink(path)); + + ASSERT_EQ(child_pid, waitpid(child_pid, NULL, 0)); +} + /* clang-format off */ FIXTURE(ioctl) {}; From f83d51a5bdfe5ab56c27fd5dfe01c613b37e2dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Noack?= Date: Fri, 19 Apr 2024 16:11:17 +0000 Subject: [PATCH 1586/1702] selftests/landlock: Check IOCTL restrictions for named UNIX domain sockets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The LANDLOCK_ACCESS_FS_IOCTL_DEV right should have no effect on the use of named UNIX domain sockets. Suggested-by: Mickaël Salaün Signed-off-by: Günther Noack Link: https://lore.kernel.org/r/20240419161122.2023765-7-gnoack@google.com [mic: Add missing stddef.h for offsetof()] Signed-off-by: Mickaël Salaün --- tools/testing/selftests/landlock/fs_test.c | 53 ++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c index b133020c7761c..8443f329ac7cc 100644 --- a/tools/testing/selftests/landlock/fs_test.c +++ b/tools/testing/selftests/landlock/fs_test.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -21,8 +22,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -3985,6 +3988,56 @@ TEST_F_FORK(layout1, named_pipe_ioctl) ASSERT_EQ(child_pid, waitpid(child_pid, NULL, 0)); } +/* For named UNIX domain sockets, no IOCTL restrictions apply. */ +TEST_F_FORK(layout1, named_unix_domain_socket_ioctl) +{ + const char *const path = file1_s1d1; + int srv_fd, cli_fd, ruleset_fd; + socklen_t size; + struct sockaddr_un srv_un, cli_un; + const struct landlock_ruleset_attr attr = { + .handled_access_fs = LANDLOCK_ACCESS_FS_IOCTL_DEV, + }; + + /* Sets up a server */ + srv_un.sun_family = AF_UNIX; + strncpy(srv_un.sun_path, path, sizeof(srv_un.sun_path)); + + ASSERT_EQ(0, unlink(path)); + srv_fd = socket(AF_UNIX, SOCK_STREAM, 0); + ASSERT_LE(0, srv_fd); + + size = offsetof(struct sockaddr_un, sun_path) + strlen(srv_un.sun_path); + ASSERT_EQ(0, bind(srv_fd, (struct sockaddr *)&srv_un, size)); + ASSERT_EQ(0, listen(srv_fd, 10 /* qlen */)); + + /* Enables Landlock. */ + ruleset_fd = landlock_create_ruleset(&attr, sizeof(attr), 0); + ASSERT_LE(0, ruleset_fd); + enforce_ruleset(_metadata, ruleset_fd); + ASSERT_EQ(0, close(ruleset_fd)); + + /* Sets up a client connection to it */ + cli_un.sun_family = AF_UNIX; + cli_fd = socket(AF_UNIX, SOCK_STREAM, 0); + ASSERT_LE(0, cli_fd); + + size = offsetof(struct sockaddr_un, sun_path) + strlen(cli_un.sun_path); + ASSERT_EQ(0, bind(cli_fd, (struct sockaddr *)&cli_un, size)); + + bzero(&cli_un, sizeof(cli_un)); + cli_un.sun_family = AF_UNIX; + strncpy(cli_un.sun_path, path, sizeof(cli_un.sun_path)); + size = offsetof(struct sockaddr_un, sun_path) + strlen(cli_un.sun_path); + + ASSERT_EQ(0, connect(cli_fd, (struct sockaddr *)&cli_un, size)); + + /* FIONREAD and other IOCTLs should not be forbidden. */ + EXPECT_EQ(0, test_fionread_ioctl(cli_fd)); + + ASSERT_EQ(0, close(cli_fd)); +} + /* clang-format off */ FIXTURE(ioctl) {}; From bce605e0cfa55da513b672500dd838be1ef41de7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Noack?= Date: Fri, 19 Apr 2024 16:11:18 +0000 Subject: [PATCH 1587/1702] selftests/landlock: Exhaustive test for the IOCTL allow-list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This test checks all IOCTL commands implemented in do_vfs_ioctl(). Test coverage for security/landlock is 90.9% of 722 lines according to gcc/gcov-13. Suggested-by: Mickaël Salaün Signed-off-by: Günther Noack Link: https://lore.kernel.org/r/20240419161122.2023765-8-gnoack@google.com [mic: Add test coverage] Signed-off-by: Mickaël Salaün --- tools/testing/selftests/landlock/fs_test.c | 114 +++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c index 8443f329ac7cc..6b5a9ff88c3d7 100644 --- a/tools/testing/selftests/landlock/fs_test.c +++ b/tools/testing/selftests/landlock/fs_test.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -3945,6 +3946,119 @@ TEST_F_FORK(layout1, o_path_ftruncate_and_ioctl) ASSERT_EQ(0, close(fd)); } +/* + * ioctl_error - generically call the given ioctl with a pointer to a + * sufficiently large zeroed-out memory region. + * + * Returns the IOCTLs error, or 0. + */ +static int ioctl_error(struct __test_metadata *const _metadata, int fd, + unsigned int cmd) +{ + char buf[128]; /* sufficiently large */ + int res, stdinbak_fd; + + /* + * Depending on the IOCTL command, parts of the zeroed-out buffer might + * be interpreted as file descriptor numbers. We do not want to + * accidentally operate on file descriptor 0 (stdin), so we temporarily + * move stdin to a different FD and close FD 0 for the IOCTL call. + */ + stdinbak_fd = dup(0); + ASSERT_LT(0, stdinbak_fd); + ASSERT_EQ(0, close(0)); + + /* Invokes the IOCTL with a zeroed-out buffer. */ + bzero(&buf, sizeof(buf)); + res = ioctl(fd, cmd, &buf); + + /* Restores the old FD 0 and closes the backup FD. */ + ASSERT_EQ(0, dup2(stdinbak_fd, 0)); + ASSERT_EQ(0, close(stdinbak_fd)); + + if (res < 0) + return errno; + + return 0; +} + +/* Define some linux/falloc.h IOCTL commands which are not available in uapi headers. */ +struct space_resv { + __s16 l_type; + __s16 l_whence; + __s64 l_start; + __s64 l_len; /* len == 0 means until end of file */ + __s32 l_sysid; + __u32 l_pid; + __s32 l_pad[4]; /* reserved area */ +}; + +#define FS_IOC_RESVSP _IOW('X', 40, struct space_resv) +#define FS_IOC_UNRESVSP _IOW('X', 41, struct space_resv) +#define FS_IOC_RESVSP64 _IOW('X', 42, struct space_resv) +#define FS_IOC_UNRESVSP64 _IOW('X', 43, struct space_resv) +#define FS_IOC_ZERO_RANGE _IOW('X', 57, struct space_resv) + +/* + * Tests a series of blanket-permitted and denied IOCTLs. + */ +TEST_F_FORK(layout1, blanket_permitted_ioctls) +{ + const struct landlock_ruleset_attr attr = { + .handled_access_fs = LANDLOCK_ACCESS_FS_IOCTL_DEV, + }; + int ruleset_fd, fd; + + /* Enables Landlock. */ + ruleset_fd = landlock_create_ruleset(&attr, sizeof(attr), 0); + ASSERT_LE(0, ruleset_fd); + enforce_ruleset(_metadata, ruleset_fd); + ASSERT_EQ(0, close(ruleset_fd)); + + fd = open("/dev/null", O_RDWR | O_CLOEXEC); + ASSERT_LE(0, fd); + + /* + * Checks permitted commands. + * These ones may return errors, but should not be blocked by Landlock. + */ + EXPECT_NE(EACCES, ioctl_error(_metadata, fd, FIOCLEX)); + EXPECT_NE(EACCES, ioctl_error(_metadata, fd, FIONCLEX)); + EXPECT_NE(EACCES, ioctl_error(_metadata, fd, FIONBIO)); + EXPECT_NE(EACCES, ioctl_error(_metadata, fd, FIOASYNC)); + EXPECT_NE(EACCES, ioctl_error(_metadata, fd, FIOQSIZE)); + EXPECT_NE(EACCES, ioctl_error(_metadata, fd, FIFREEZE)); + EXPECT_NE(EACCES, ioctl_error(_metadata, fd, FITHAW)); + EXPECT_NE(EACCES, ioctl_error(_metadata, fd, FS_IOC_FIEMAP)); + EXPECT_NE(EACCES, ioctl_error(_metadata, fd, FIGETBSZ)); + EXPECT_NE(EACCES, ioctl_error(_metadata, fd, FICLONE)); + EXPECT_NE(EACCES, ioctl_error(_metadata, fd, FICLONERANGE)); + EXPECT_NE(EACCES, ioctl_error(_metadata, fd, FIDEDUPERANGE)); + EXPECT_NE(EACCES, ioctl_error(_metadata, fd, FS_IOC_GETFSUUID)); + EXPECT_NE(EACCES, ioctl_error(_metadata, fd, FS_IOC_GETFSSYSFSPATH)); + + /* + * Checks blocked commands. + * A call to a blocked IOCTL command always returns EACCES. + */ + EXPECT_EQ(EACCES, ioctl_error(_metadata, fd, FIONREAD)); + EXPECT_EQ(EACCES, ioctl_error(_metadata, fd, FS_IOC_GETFLAGS)); + EXPECT_EQ(EACCES, ioctl_error(_metadata, fd, FS_IOC_SETFLAGS)); + EXPECT_EQ(EACCES, ioctl_error(_metadata, fd, FS_IOC_FSGETXATTR)); + EXPECT_EQ(EACCES, ioctl_error(_metadata, fd, FS_IOC_FSSETXATTR)); + EXPECT_EQ(EACCES, ioctl_error(_metadata, fd, FIBMAP)); + EXPECT_EQ(EACCES, ioctl_error(_metadata, fd, FS_IOC_RESVSP)); + EXPECT_EQ(EACCES, ioctl_error(_metadata, fd, FS_IOC_RESVSP64)); + EXPECT_EQ(EACCES, ioctl_error(_metadata, fd, FS_IOC_UNRESVSP)); + EXPECT_EQ(EACCES, ioctl_error(_metadata, fd, FS_IOC_UNRESVSP64)); + EXPECT_EQ(EACCES, ioctl_error(_metadata, fd, FS_IOC_ZERO_RANGE)); + + /* Default case is also blocked. */ + EXPECT_EQ(EACCES, ioctl_error(_metadata, fd, 0xc00ffeee)); + + ASSERT_EQ(0, close(fd)); +} + /* * Named pipes are not governed by the LANDLOCK_ACCESS_FS_IOCTL_DEV right, * because they are not character or block devices. From cd13738d44c9863ce54243fdcc2d228233f23355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Noack?= Date: Fri, 19 Apr 2024 16:11:19 +0000 Subject: [PATCH 1588/1702] samples/landlock: Add support for LANDLOCK_ACCESS_FS_IOCTL_DEV MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add IOCTL support to the Landlock sample tool. The IOCTL right is grouped with the read-write rights in the sample tool, as some IOCTL requests provide features that mutate state. Signed-off-by: Günther Noack Link: https://lore.kernel.org/r/20240419161122.2023765-9-gnoack@google.com Signed-off-by: Mickaël Salaün --- samples/landlock/sandboxer.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/samples/landlock/sandboxer.c b/samples/landlock/sandboxer.c index 8b8ecd65c28c4..e8223c3e781ab 100644 --- a/samples/landlock/sandboxer.c +++ b/samples/landlock/sandboxer.c @@ -81,7 +81,8 @@ static int parse_path(char *env_path, const char ***const path_list) LANDLOCK_ACCESS_FS_EXECUTE | \ LANDLOCK_ACCESS_FS_WRITE_FILE | \ LANDLOCK_ACCESS_FS_READ_FILE | \ - LANDLOCK_ACCESS_FS_TRUNCATE) + LANDLOCK_ACCESS_FS_TRUNCATE | \ + LANDLOCK_ACCESS_FS_IOCTL_DEV) /* clang-format on */ @@ -202,11 +203,12 @@ static int populate_ruleset_net(const char *const env_var, const int ruleset_fd, LANDLOCK_ACCESS_FS_MAKE_BLOCK | \ LANDLOCK_ACCESS_FS_MAKE_SYM | \ LANDLOCK_ACCESS_FS_REFER | \ - LANDLOCK_ACCESS_FS_TRUNCATE) + LANDLOCK_ACCESS_FS_TRUNCATE | \ + LANDLOCK_ACCESS_FS_IOCTL_DEV) /* clang-format on */ -#define LANDLOCK_ABI_LAST 4 +#define LANDLOCK_ABI_LAST 5 int main(const int argc, char *const argv[], char *const *const envp) { @@ -320,6 +322,11 @@ int main(const int argc, char *const argv[], char *const *const envp) ruleset_attr.handled_access_net &= ~(LANDLOCK_ACCESS_NET_BIND_TCP | LANDLOCK_ACCESS_NET_CONNECT_TCP); + __attribute__((fallthrough)); + case 4: + /* Removes LANDLOCK_ACCESS_FS_IOCTL_DEV for ABI < 5 */ + ruleset_attr.handled_access_fs &= ~LANDLOCK_ACCESS_FS_IOCTL_DEV; + fprintf(stderr, "Hint: You should update the running kernel " "to leverage Landlock features " From a3746da89faf985132e428056d147b743b7174e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Noack?= Date: Fri, 19 Apr 2024 16:11:20 +0000 Subject: [PATCH 1589/1702] landlock: Document IOCTL support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the paragraph above the fallback logic, use the shorter phrasing from the landlock(7) man page. Signed-off-by: Günther Noack Link: https://lore.kernel.org/r/20240419161122.2023765-10-gnoack@google.com [mic: Update date, and fix redundant "access"] Signed-off-by: Mickaël Salaün --- Documentation/userspace-api/landlock.rst | 78 +++++++++++++++++++----- 1 file changed, 62 insertions(+), 16 deletions(-) diff --git a/Documentation/userspace-api/landlock.rst b/Documentation/userspace-api/landlock.rst index 9dd636aaa829f..07b63aec56fa3 100644 --- a/Documentation/userspace-api/landlock.rst +++ b/Documentation/userspace-api/landlock.rst @@ -8,7 +8,7 @@ Landlock: unprivileged access control ===================================== :Author: Mickaël Salaün -:Date: October 2023 +:Date: April 2024 The goal of Landlock is to enable to restrict ambient rights (e.g. global filesystem or network access) for a set of processes. Because Landlock @@ -76,7 +76,8 @@ to be explicit about the denied-by-default access rights. LANDLOCK_ACCESS_FS_MAKE_BLOCK | LANDLOCK_ACCESS_FS_MAKE_SYM | LANDLOCK_ACCESS_FS_REFER | - LANDLOCK_ACCESS_FS_TRUNCATE, + LANDLOCK_ACCESS_FS_TRUNCATE | + LANDLOCK_ACCESS_FS_IOCTL_DEV, .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP | LANDLOCK_ACCESS_NET_CONNECT_TCP, @@ -85,10 +86,10 @@ to be explicit about the denied-by-default access rights. Because we may not know on which kernel version an application will be executed, it is safer to follow a best-effort security approach. Indeed, we should try to protect users as much as possible whatever the kernel they are -using. To avoid binary enforcement (i.e. either all security features or -none), we can leverage a dedicated Landlock command to get the current version -of the Landlock ABI and adapt the handled accesses. Let's check if we should -remove access rights which are only supported in higher versions of the ABI. +using. + +To be compatible with older Linux versions, we detect the available Landlock ABI +version, and only use the available subset of access rights: .. code-block:: c @@ -114,6 +115,10 @@ remove access rights which are only supported in higher versions of the ABI. ruleset_attr.handled_access_net &= ~(LANDLOCK_ACCESS_NET_BIND_TCP | LANDLOCK_ACCESS_NET_CONNECT_TCP); + __attribute__((fallthrough)); + case 4: + /* Removes LANDLOCK_ACCESS_FS_IOCTL_DEV for ABI < 5 */ + ruleset_attr.handled_access_fs &= ~LANDLOCK_ACCESS_FS_IOCTL_DEV; } This enables to create an inclusive ruleset that will contain our rules. @@ -225,6 +230,7 @@ access rights per directory enables to change the location of such directory without relying on the destination directory access rights (except those that are required for this operation, see ``LANDLOCK_ACCESS_FS_REFER`` documentation). + Having self-sufficient hierarchies also helps to tighten the required access rights to the minimal set of data. This also helps avoid sinkhole directories, i.e. directories where data can be linked to but not linked from. However, @@ -318,18 +324,26 @@ It should also be noted that truncating files does not require the system call, this can also be done through :manpage:`open(2)` with the flags ``O_RDONLY | O_TRUNC``. -When opening a file, the availability of the ``LANDLOCK_ACCESS_FS_TRUNCATE`` -right is associated with the newly created file descriptor and will be used for -subsequent truncation attempts using :manpage:`ftruncate(2)`. The behavior is -similar to opening a file for reading or writing, where permissions are checked -during :manpage:`open(2)`, but not during the subsequent :manpage:`read(2)` and +The truncate right is associated with the opened file (see below). + +Rights associated with file descriptors +--------------------------------------- + +When opening a file, the availability of the ``LANDLOCK_ACCESS_FS_TRUNCATE`` and +``LANDLOCK_ACCESS_FS_IOCTL_DEV`` rights is associated with the newly created +file descriptor and will be used for subsequent truncation and ioctl attempts +using :manpage:`ftruncate(2)` and :manpage:`ioctl(2)`. The behavior is similar +to opening a file for reading or writing, where permissions are checked during +:manpage:`open(2)`, but not during the subsequent :manpage:`read(2)` and :manpage:`write(2)` calls. -As a consequence, it is possible to have multiple open file descriptors for the -same file, where one grants the right to truncate the file and the other does -not. It is also possible to pass such file descriptors between processes, -keeping their Landlock properties, even when these processes do not have an -enforced Landlock ruleset. +As a consequence, it is possible that a process has multiple open file +descriptors referring to the same file, but Landlock enforces different things +when operating with these file descriptors. This can happen when a Landlock +ruleset gets enforced and the process keeps file descriptors which were opened +both before and after the enforcement. It is also possible to pass such file +descriptors between processes, keeping their Landlock properties, even when some +of the involved processes do not have an enforced Landlock ruleset. Compatibility ============= @@ -458,6 +472,28 @@ Memory usage Kernel memory allocated to create rulesets is accounted and can be restricted by the Documentation/admin-guide/cgroup-v1/memory.rst. +IOCTL support +------------- + +The ``LANDLOCK_ACCESS_FS_IOCTL_DEV`` right restricts the use of +:manpage:`ioctl(2)`, but it only applies to *newly opened* device files. This +means specifically that pre-existing file descriptors like stdin, stdout and +stderr are unaffected. + +Users should be aware that TTY devices have traditionally permitted to control +other processes on the same TTY through the ``TIOCSTI`` and ``TIOCLINUX`` IOCTL +commands. Both of these require ``CAP_SYS_ADMIN`` on modern Linux systems, but +the behavior is configurable for ``TIOCSTI``. + +On older systems, it is therefore recommended to close inherited TTY file +descriptors, or to reopen them from ``/proc/self/fd/*`` without the +``LANDLOCK_ACCESS_FS_IOCTL_DEV`` right, if possible. + +Landlock's IOCTL support is coarse-grained at the moment, but may become more +fine-grained in the future. Until then, users are advised to establish the +guarantees that they need through the file hierarchy, by only allowing the +``LANDLOCK_ACCESS_FS_IOCTL_DEV`` right on files where it is really required. + Previous limitations ==================== @@ -495,6 +531,16 @@ bind and connect actions to only a set of allowed ports thanks to the new ``LANDLOCK_ACCESS_NET_BIND_TCP`` and ``LANDLOCK_ACCESS_NET_CONNECT_TCP`` access rights. +IOCTL (ABI < 5) +--------------- + +IOCTL operations could not be denied before the fifth Landlock ABI, so +:manpage:`ioctl(2)` is always allowed when using a kernel that only supports an +earlier ABI. + +Starting with the Landlock ABI version 5, it is possible to restrict the use of +:manpage:`ioctl(2)` using the new ``LANDLOCK_ACCESS_FS_IOCTL_DEV`` right. + .. _kernel_support: Kernel support From 943aa818cd6606a1385130bc72f5066b82fe4346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Noack?= Date: Fri, 19 Apr 2024 16:11:21 +0000 Subject: [PATCH 1590/1702] MAINTAINERS: Notify Landlock maintainers about changes to fs/ioctl.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Landlock needs to track changes to do_vfs_ioctl() when new IOCTL implementations are added to it. Suggested-by: Mickaël Salaün Signed-off-by: Günther Noack Link: https://lore.kernel.org/r/20240419161122.2023765-11-gnoack@google.com Signed-off-by: Mickaël Salaün --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 28e20975c26f5..b01345abadb26 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12220,6 +12220,7 @@ W: https://landlock.io T: git https://git.kernel.org/pub/scm/linux/kernel/git/mic/linux.git F: Documentation/security/landlock.rst F: Documentation/userspace-api/landlock.rst +F: fs/ioctl.c F: include/uapi/linux/landlock.h F: samples/landlock/ F: security/landlock/ From d1654fd98be731b08d2b7b857ef2026df6767e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnther=20Noack?= Date: Fri, 19 Apr 2024 16:11:22 +0000 Subject: [PATCH 1591/1702] fs/ioctl: Add a comment to keep the logic in sync with LSM policies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Landlock's IOCTL support needs to partially replicate the list of IOCTLs from do_vfs_ioctl(). The list of commands implemented in do_vfs_ioctl() should be kept in sync with Landlock's IOCTL policies. Suggested-by: Paul Moore Suggested-by: Mickaël Salaün Signed-off-by: Günther Noack Link: https://lore.kernel.org/r/20240419161122.2023765-12-gnoack@google.com Signed-off-by: Mickaël Salaün --- fs/ioctl.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/ioctl.c b/fs/ioctl.c index fb0628e680c40..64776891120c9 100644 --- a/fs/ioctl.c +++ b/fs/ioctl.c @@ -796,6 +796,9 @@ static int ioctl_get_fs_sysfs_path(struct file *file, void __user *argp) * * When you add any new common ioctls to the switches above and below, * please ensure they have compatible arguments in compat mode. + * + * The LSM mailing list should also be notified of any command additions or + * changes, as specific LSMs may be affected. */ static int do_vfs_ioctl(struct file *filp, unsigned int fd, unsigned int cmd, unsigned long arg) From 5bf9e57e634bd72a97b4b12c87186fc052a6a116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Sala=C3=BCn?= Date: Thu, 25 Apr 2024 11:21:21 +0200 Subject: [PATCH 1592/1702] =?UTF-8?q?MAINTAINERS:=20Add=20G=C3=BCnther=20N?= =?UTF-8?q?oack=20as=20Landlock=20reviewer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Günther is a major contributor to Landlock, both on the kernel and user space sides, and he is already reviewing Landlock changes. Thanks! Cc: James Morris Cc: Paul Moore Cc: Serge E. Hallyn Acked-by: Günther Noack Link: https://lore.kernel.org/r/20240425092126.975830-1-mic@digikod.net Signed-off-by: Mickaël Salaün --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index b01345abadb26..4b261c8971c7c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12214,6 +12214,7 @@ F: net/l3mdev LANDLOCK SECURITY MODULE M: Mickaël Salaün +R: Günther Noack L: linux-security-module@vger.kernel.org S: Supported W: https://landlock.io From fec1982d70721c0062758861fec7e4e9d1103fb6 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Thu, 18 Apr 2024 22:55:39 +0200 Subject: [PATCH 1593/1702] i2c: mux: Remove class argument from i2c_mux_add_adapter() 99a741aa7a2d ("i2c: mux: gpio: remove support for class-based device instantiation") removed the last call to i2c_mux_add_adapter() with a non-null class argument. Therefore the class argument can be removed. Note: Class-based device instantiation is a legacy mechanism which shouldn't be used in new code, so we can rule out that this argument may be needed again in the future. Signed-off-by: Heiner Kallweit Reviewed-by: Thomas Zimmermann Acked-by: Peter Rosin Reviewed-by: Laurent Pinchart Acked-by: Jonathan Cameron Acked-by: Rob Herring (Arm) Signed-off-by: Wolfram Sang --- drivers/gpu/drm/bridge/sii902x.c | 2 +- drivers/i2c/i2c-mux.c | 24 +--------------------- drivers/i2c/muxes/i2c-arb-gpio-challenge.c | 2 +- drivers/i2c/muxes/i2c-mux-gpio.c | 2 +- drivers/i2c/muxes/i2c-mux-gpmux.c | 2 +- drivers/i2c/muxes/i2c-mux-ltc4306.c | 2 +- drivers/i2c/muxes/i2c-mux-mlxcpld.c | 2 +- drivers/i2c/muxes/i2c-mux-pca9541.c | 2 +- drivers/i2c/muxes/i2c-mux-pca954x.c | 2 +- drivers/i2c/muxes/i2c-mux-pinctrl.c | 2 +- drivers/i2c/muxes/i2c-mux-reg.c | 2 +- drivers/iio/gyro/mpu3050-i2c.c | 2 +- drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c | 2 +- drivers/media/dvb-frontends/af9013.c | 2 +- drivers/media/dvb-frontends/lgdt3306a.c | 2 +- drivers/media/dvb-frontends/m88ds3103.c | 2 +- drivers/media/dvb-frontends/rtl2830.c | 2 +- drivers/media/dvb-frontends/rtl2832.c | 2 +- drivers/media/dvb-frontends/si2168.c | 2 +- drivers/media/i2c/max9286.c | 2 +- drivers/media/usb/cx231xx/cx231xx-i2c.c | 5 +---- drivers/of/unittest.c | 2 +- include/linux/i2c-mux.h | 3 +-- 23 files changed, 23 insertions(+), 49 deletions(-) diff --git a/drivers/gpu/drm/bridge/sii902x.c b/drivers/gpu/drm/bridge/sii902x.c index 8f84e98249c74..2fbeda9025bfc 100644 --- a/drivers/gpu/drm/bridge/sii902x.c +++ b/drivers/gpu/drm/bridge/sii902x.c @@ -1092,7 +1092,7 @@ static int sii902x_init(struct sii902x *sii902x) } sii902x->i2cmux->priv = sii902x; - ret = i2c_mux_add_adapter(sii902x->i2cmux, 0, 0, 0); + ret = i2c_mux_add_adapter(sii902x->i2cmux, 0, 0); if (ret) goto err_unreg_audio; diff --git a/drivers/i2c/i2c-mux.c b/drivers/i2c/i2c-mux.c index 57ff09f18c371..fda72e8be8850 100644 --- a/drivers/i2c/i2c-mux.c +++ b/drivers/i2c/i2c-mux.c @@ -127,19 +127,6 @@ static u32 i2c_mux_functionality(struct i2c_adapter *adap) return parent->algo->functionality(parent); } -/* Return all parent classes, merged */ -static unsigned int i2c_mux_parent_classes(struct i2c_adapter *parent) -{ - unsigned int class = 0; - - do { - class |= parent->class; - parent = i2c_parent_is_i2c_adapter(parent); - } while (parent); - - return class; -} - static void i2c_mux_lock_bus(struct i2c_adapter *adapter, unsigned int flags) { struct i2c_mux_priv *priv = adapter->algo_data; @@ -281,8 +268,7 @@ static const struct i2c_lock_operations i2c_parent_lock_ops = { }; int i2c_mux_add_adapter(struct i2c_mux_core *muxc, - u32 force_nr, u32 chan_id, - unsigned int class) + u32 force_nr, u32 chan_id) { struct i2c_adapter *parent = muxc->parent; struct i2c_mux_priv *priv; @@ -340,14 +326,6 @@ int i2c_mux_add_adapter(struct i2c_mux_core *muxc, else priv->adap.lock_ops = &i2c_parent_lock_ops; - /* Sanity check on class */ - if (i2c_mux_parent_classes(parent) & class & ~I2C_CLASS_DEPRECATED) - dev_err(&parent->dev, - "Segment %d behind mux can't share classes with ancestors\n", - chan_id); - else - priv->adap.class = class; - /* * Try to populate the mux adapter's of_node, expands to * nothing if !CONFIG_OF. diff --git a/drivers/i2c/muxes/i2c-arb-gpio-challenge.c b/drivers/i2c/muxes/i2c-arb-gpio-challenge.c index 24168e9f7df4c..7aa6e795d833d 100644 --- a/drivers/i2c/muxes/i2c-arb-gpio-challenge.c +++ b/drivers/i2c/muxes/i2c-arb-gpio-challenge.c @@ -167,7 +167,7 @@ static int i2c_arbitrator_probe(struct platform_device *pdev) } /* Actually add the mux adapter */ - ret = i2c_mux_add_adapter(muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(muxc, 0, 0); if (ret) i2c_put_adapter(muxc->parent); diff --git a/drivers/i2c/muxes/i2c-mux-gpio.c b/drivers/i2c/muxes/i2c-mux-gpio.c index 0fbb33a3d518c..d6bbb8b683333 100644 --- a/drivers/i2c/muxes/i2c-mux-gpio.c +++ b/drivers/i2c/muxes/i2c-mux-gpio.c @@ -207,7 +207,7 @@ static int i2c_mux_gpio_probe(struct platform_device *pdev) for (i = 0; i < mux->data.n_values; i++) { u32 nr = mux->data.base_nr ? (mux->data.base_nr + i) : 0; - ret = i2c_mux_add_adapter(muxc, nr, mux->data.values[i], 0); + ret = i2c_mux_add_adapter(muxc, nr, mux->data.values[i]); if (ret) goto add_adapter_failed; } diff --git a/drivers/i2c/muxes/i2c-mux-gpmux.c b/drivers/i2c/muxes/i2c-mux-gpmux.c index 8305661e12539..10d63307b14d0 100644 --- a/drivers/i2c/muxes/i2c-mux-gpmux.c +++ b/drivers/i2c/muxes/i2c-mux-gpmux.c @@ -124,7 +124,7 @@ static int i2c_mux_probe(struct platform_device *pdev) goto err_children; } - ret = i2c_mux_add_adapter(muxc, 0, chan, 0); + ret = i2c_mux_add_adapter(muxc, 0, chan); if (ret) goto err_children; } diff --git a/drivers/i2c/muxes/i2c-mux-ltc4306.c b/drivers/i2c/muxes/i2c-mux-ltc4306.c index 23766d853e767..19a7c370946d4 100644 --- a/drivers/i2c/muxes/i2c-mux-ltc4306.c +++ b/drivers/i2c/muxes/i2c-mux-ltc4306.c @@ -279,7 +279,7 @@ static int ltc4306_probe(struct i2c_client *client) /* Now create an adapter for each channel */ for (num = 0; num < chip->nchans; num++) { - ret = i2c_mux_add_adapter(muxc, 0, num, 0); + ret = i2c_mux_add_adapter(muxc, 0, num); if (ret) { i2c_mux_del_adapters(muxc); return ret; diff --git a/drivers/i2c/muxes/i2c-mux-mlxcpld.c b/drivers/i2c/muxes/i2c-mux-mlxcpld.c index 4c6ed1d58c79a..3f06aa3331a78 100644 --- a/drivers/i2c/muxes/i2c-mux-mlxcpld.c +++ b/drivers/i2c/muxes/i2c-mux-mlxcpld.c @@ -154,7 +154,7 @@ static int mlxcpld_mux_probe(struct platform_device *pdev) /* Create an adapter for each channel. */ for (num = 0; num < pdata->num_adaps; num++) { - err = i2c_mux_add_adapter(muxc, 0, pdata->chan_ids[num], 0); + err = i2c_mux_add_adapter(muxc, 0, pdata->chan_ids[num]); if (err) goto virt_reg_failed; } diff --git a/drivers/i2c/muxes/i2c-mux-pca9541.c b/drivers/i2c/muxes/i2c-mux-pca9541.c index ce0fb69249a8f..e28694d991fb0 100644 --- a/drivers/i2c/muxes/i2c-mux-pca9541.c +++ b/drivers/i2c/muxes/i2c-mux-pca9541.c @@ -314,7 +314,7 @@ static int pca9541_probe(struct i2c_client *client) i2c_set_clientdata(client, muxc); - ret = i2c_mux_add_adapter(muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(muxc, 0, 0); if (ret) return ret; diff --git a/drivers/i2c/muxes/i2c-mux-pca954x.c b/drivers/i2c/muxes/i2c-mux-pca954x.c index c3f4ff08ac385..6f84018258c45 100644 --- a/drivers/i2c/muxes/i2c-mux-pca954x.c +++ b/drivers/i2c/muxes/i2c-mux-pca954x.c @@ -644,7 +644,7 @@ static int pca954x_probe(struct i2c_client *client) /* Now create an adapter for each channel */ for (num = 0; num < data->chip->nchans; num++) { - ret = i2c_mux_add_adapter(muxc, 0, num, 0); + ret = i2c_mux_add_adapter(muxc, 0, num); if (ret) goto fail_cleanup; } diff --git a/drivers/i2c/muxes/i2c-mux-pinctrl.c b/drivers/i2c/muxes/i2c-mux-pinctrl.c index 6ebca7bfd8a26..02aaf0781e9c8 100644 --- a/drivers/i2c/muxes/i2c-mux-pinctrl.c +++ b/drivers/i2c/muxes/i2c-mux-pinctrl.c @@ -151,7 +151,7 @@ static int i2c_mux_pinctrl_probe(struct platform_device *pdev) /* Do not add any adapter for the idle state (if it's there at all). */ for (i = 0; i < num_names - !!muxc->deselect; i++) { - ret = i2c_mux_add_adapter(muxc, 0, i, 0); + ret = i2c_mux_add_adapter(muxc, 0, i); if (ret) goto err_del_adapter; } diff --git a/drivers/i2c/muxes/i2c-mux-reg.c b/drivers/i2c/muxes/i2c-mux-reg.c index 8489971babd37..ef765fcd33f54 100644 --- a/drivers/i2c/muxes/i2c-mux-reg.c +++ b/drivers/i2c/muxes/i2c-mux-reg.c @@ -213,7 +213,7 @@ static int i2c_mux_reg_probe(struct platform_device *pdev) for (i = 0; i < mux->data.n_values; i++) { nr = mux->data.base_nr ? (mux->data.base_nr + i) : 0; - ret = i2c_mux_add_adapter(muxc, nr, mux->data.values[i], 0); + ret = i2c_mux_add_adapter(muxc, nr, mux->data.values[i]); if (ret) goto err_del_mux_adapters; } diff --git a/drivers/iio/gyro/mpu3050-i2c.c b/drivers/iio/gyro/mpu3050-i2c.c index 52b6feed26379..29ecfa6fd6335 100644 --- a/drivers/iio/gyro/mpu3050-i2c.c +++ b/drivers/iio/gyro/mpu3050-i2c.c @@ -72,7 +72,7 @@ static int mpu3050_i2c_probe(struct i2c_client *client) else { mpu3050->i2cmux->priv = mpu3050; /* Ignore failure, not critical */ - i2c_mux_add_adapter(mpu3050->i2cmux, 0, 0, 0); + i2c_mux_add_adapter(mpu3050->i2cmux, 0, 0); } return 0; diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c index 410ea39fd495a..0e03137fb3d40 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c @@ -142,7 +142,7 @@ static int inv_mpu_probe(struct i2c_client *client) if (!st->muxc) return -ENOMEM; st->muxc->priv = dev_get_drvdata(&client->dev); - result = i2c_mux_add_adapter(st->muxc, 0, 0, 0); + result = i2c_mux_add_adapter(st->muxc, 0, 0); if (result) return result; result = inv_mpu_acpi_create_mux_client(client); diff --git a/drivers/media/dvb-frontends/af9013.c b/drivers/media/dvb-frontends/af9013.c index a829c89792a43..5afdbe244596b 100644 --- a/drivers/media/dvb-frontends/af9013.c +++ b/drivers/media/dvb-frontends/af9013.c @@ -1480,7 +1480,7 @@ static int af9013_probe(struct i2c_client *client) goto err_regmap_exit; } state->muxc->priv = state; - ret = i2c_mux_add_adapter(state->muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(state->muxc, 0, 0); if (ret) goto err_regmap_exit; diff --git a/drivers/media/dvb-frontends/lgdt3306a.c b/drivers/media/dvb-frontends/lgdt3306a.c index 2638875924153..91c71b24cacbc 100644 --- a/drivers/media/dvb-frontends/lgdt3306a.c +++ b/drivers/media/dvb-frontends/lgdt3306a.c @@ -2203,7 +2203,7 @@ static int lgdt3306a_probe(struct i2c_client *client) goto err_kfree; } state->muxc->priv = client; - ret = i2c_mux_add_adapter(state->muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(state->muxc, 0, 0); if (ret) goto err_kfree; diff --git a/drivers/media/dvb-frontends/m88ds3103.c b/drivers/media/dvb-frontends/m88ds3103.c index e0272054fca51..6c69bcc7a1bc4 100644 --- a/drivers/media/dvb-frontends/m88ds3103.c +++ b/drivers/media/dvb-frontends/m88ds3103.c @@ -1866,7 +1866,7 @@ static int m88ds3103_probe(struct i2c_client *client) goto err_kfree; } dev->muxc->priv = dev; - ret = i2c_mux_add_adapter(dev->muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(dev->muxc, 0, 0); if (ret) goto err_kfree; diff --git a/drivers/media/dvb-frontends/rtl2830.c b/drivers/media/dvb-frontends/rtl2830.c index 35c969fd2cb5e..30d10fe4b33e3 100644 --- a/drivers/media/dvb-frontends/rtl2830.c +++ b/drivers/media/dvb-frontends/rtl2830.c @@ -838,7 +838,7 @@ static int rtl2830_probe(struct i2c_client *client) goto err_regmap_exit; } dev->muxc->priv = client; - ret = i2c_mux_add_adapter(dev->muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(dev->muxc, 0, 0); if (ret) goto err_regmap_exit; diff --git a/drivers/media/dvb-frontends/rtl2832.c b/drivers/media/dvb-frontends/rtl2832.c index 601cf45c39358..5142820b1b3d9 100644 --- a/drivers/media/dvb-frontends/rtl2832.c +++ b/drivers/media/dvb-frontends/rtl2832.c @@ -1082,7 +1082,7 @@ static int rtl2832_probe(struct i2c_client *client) goto err_regmap_exit; } dev->muxc->priv = dev; - ret = i2c_mux_add_adapter(dev->muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(dev->muxc, 0, 0); if (ret) goto err_regmap_exit; diff --git a/drivers/media/dvb-frontends/si2168.c b/drivers/media/dvb-frontends/si2168.c index dae1f2153e8be..26828fd41e684 100644 --- a/drivers/media/dvb-frontends/si2168.c +++ b/drivers/media/dvb-frontends/si2168.c @@ -744,7 +744,7 @@ static int si2168_probe(struct i2c_client *client) goto err_kfree; } dev->muxc->priv = client; - ret = i2c_mux_add_adapter(dev->muxc, 0, 0, 0); + ret = i2c_mux_add_adapter(dev->muxc, 0, 0); if (ret) goto err_kfree; diff --git a/drivers/media/i2c/max9286.c b/drivers/media/i2c/max9286.c index d685d445cf23a..dfcb3fc033500 100644 --- a/drivers/media/i2c/max9286.c +++ b/drivers/media/i2c/max9286.c @@ -383,7 +383,7 @@ static int max9286_i2c_mux_init(struct max9286_priv *priv) for_each_source(priv, source) { unsigned int index = to_index(priv, source); - ret = i2c_mux_add_adapter(priv->mux, 0, index, 0); + ret = i2c_mux_add_adapter(priv->mux, 0, index); if (ret < 0) goto error; } diff --git a/drivers/media/usb/cx231xx/cx231xx-i2c.c b/drivers/media/usb/cx231xx/cx231xx-i2c.c index c6659253c6fbc..6da8e7943d94a 100644 --- a/drivers/media/usb/cx231xx/cx231xx-i2c.c +++ b/drivers/media/usb/cx231xx/cx231xx-i2c.c @@ -567,10 +567,7 @@ int cx231xx_i2c_mux_create(struct cx231xx *dev) int cx231xx_i2c_mux_register(struct cx231xx *dev, int mux_no) { - return i2c_mux_add_adapter(dev->muxc, - 0, - mux_no /* chan_id */, - 0 /* class */); + return i2c_mux_add_adapter(dev->muxc, 0, mux_no); } void cx231xx_i2c_mux_unregister(struct cx231xx *dev) diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c index 6b5c36b6a7586..c8ee866d76d4e 100644 --- a/drivers/of/unittest.c +++ b/drivers/of/unittest.c @@ -2815,7 +2815,7 @@ static int unittest_i2c_mux_probe(struct i2c_client *client) if (!muxc) return -ENOMEM; for (i = 0; i < nchans; i++) { - if (i2c_mux_add_adapter(muxc, 0, i, 0)) { + if (i2c_mux_add_adapter(muxc, 0, i)) { dev_err(dev, "Failed to register mux #%d\n", i); i2c_mux_del_adapters(muxc); return -ENODEV; diff --git a/include/linux/i2c-mux.h b/include/linux/i2c-mux.h index 98ef73b7c8fd9..1784ac7afb116 100644 --- a/include/linux/i2c-mux.h +++ b/include/linux/i2c-mux.h @@ -56,8 +56,7 @@ struct i2c_adapter *i2c_root_adapter(struct device *dev); * callback functions to perform hardware-specific mux control. */ int i2c_mux_add_adapter(struct i2c_mux_core *muxc, - u32 force_nr, u32 chan_id, - unsigned int class); + u32 force_nr, u32 chan_id); void i2c_mux_del_adapters(struct i2c_mux_core *muxc); From 2ca99e6efc91ee7c58caab3a94e9cef7a75d8ba7 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 9 May 2024 11:42:51 +0200 Subject: [PATCH 1594/1702] dt-bindings: display: samsung,ams495qa01: add missing SPI properties ref Samsung AMS495QA01 panel is a SPI device, so it should reference spi-peripheral-props.yaml schema to allow and validate the SPI device properties. Fixes: 92be07c65b22 ("dt-bindings: display: panel: Add Samsung AMS495QA01") Signed-off-by: Krzysztof Kozlowski Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20240509-dt-bindings-dsi-panel-reg-v1-1-8b2443705be0@linaro.org Signed-off-by: Rob Herring (Arm) --- .../devicetree/bindings/display/panel/samsung,ams495qa01.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/display/panel/samsung,ams495qa01.yaml b/Documentation/devicetree/bindings/display/panel/samsung,ams495qa01.yaml index 58fa073ce2585..a5ff4de74a158 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,ams495qa01.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,ams495qa01.yaml @@ -11,6 +11,7 @@ maintainers: allOf: - $ref: panel-common.yaml# + - $ref: /schemas/spi/spi-peripheral-props.yaml# properties: compatible: From 6b2358ff9493db92c4fa6ed3ad7b66c02c66f440 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 9 May 2024 11:42:52 +0200 Subject: [PATCH 1595/1702] dt-bindings: display: panel: constrain 'reg' in SPI panels SPI-attached devices could have more than one chip-select, thus their bindings are supposed to constrain the 'reg' property to match hardware. Add missing 'reg' constrain for SPI-attached display panels. Signed-off-by: Krzysztof Kozlowski Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20240509-dt-bindings-dsi-panel-reg-v1-2-8b2443705be0@linaro.org Signed-off-by: Rob Herring (Arm) --- .../devicetree/bindings/display/panel/abt,y030xx067a.yaml | 4 +++- .../bindings/display/panel/fascontek,fs035vg158.yaml | 3 +++ .../devicetree/bindings/display/panel/ilitek,ili9163.yaml | 4 +++- .../devicetree/bindings/display/panel/ilitek,ili9322.yaml | 3 +++ .../devicetree/bindings/display/panel/ilitek,ili9341.yaml | 3 ++- .../devicetree/bindings/display/panel/innolux,ej030na.yaml | 4 +++- .../bindings/display/panel/kingdisplay,kd035g6-54nt.yaml | 4 +++- .../bindings/display/panel/leadtek,ltk035c5444t.yaml | 3 +++ .../devicetree/bindings/display/panel/lg,lg4573.yaml | 3 ++- .../devicetree/bindings/display/panel/lgphilips,lb035q02.yaml | 3 +++ .../devicetree/bindings/display/panel/nec,nl8048hl11.yaml | 4 +++- .../devicetree/bindings/display/panel/panel-mipi-dbi-spi.yaml | 3 +++ .../devicetree/bindings/display/panel/samsung,ams495qa01.yaml | 4 +++- .../devicetree/bindings/display/panel/samsung,ld9040.yaml | 4 +++- .../devicetree/bindings/display/panel/samsung,lms380kf01.yaml | 3 ++- .../devicetree/bindings/display/panel/samsung,lms397kf04.yaml | 3 ++- .../devicetree/bindings/display/panel/samsung,s6d27a1.yaml | 3 ++- .../devicetree/bindings/display/panel/samsung,s6e63m0.yaml | 4 +++- .../devicetree/bindings/display/panel/sitronix,st7789v.yaml | 4 +++- .../devicetree/bindings/display/panel/sony,acx565akm.yaml | 3 +++ Documentation/devicetree/bindings/display/panel/tpo,td.yaml | 4 +++- .../devicetree/bindings/display/panel/tpo,tpg110.yaml | 3 ++- 22 files changed, 60 insertions(+), 16 deletions(-) diff --git a/Documentation/devicetree/bindings/display/panel/abt,y030xx067a.yaml b/Documentation/devicetree/bindings/display/panel/abt,y030xx067a.yaml index acd2f3faa6b9b..0aa2d3fbadaa0 100644 --- a/Documentation/devicetree/bindings/display/panel/abt,y030xx067a.yaml +++ b/Documentation/devicetree/bindings/display/panel/abt,y030xx067a.yaml @@ -17,10 +17,12 @@ properties: compatible: const: abt,y030xx067a + reg: + maxItems: 1 + backlight: true port: true power-supply: true - reg: true reset-gpios: true required: diff --git a/Documentation/devicetree/bindings/display/panel/fascontek,fs035vg158.yaml b/Documentation/devicetree/bindings/display/panel/fascontek,fs035vg158.yaml index d13c4bd26de46..9847da784cc87 100644 --- a/Documentation/devicetree/bindings/display/panel/fascontek,fs035vg158.yaml +++ b/Documentation/devicetree/bindings/display/panel/fascontek,fs035vg158.yaml @@ -17,6 +17,9 @@ properties: compatible: const: fascontek,fs035vg158 + reg: + maxItems: 1 + spi-3wire: true required: diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili9163.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,ili9163.yaml index 3cabbba86581c..ef5a2240b684d 100644 --- a/Documentation/devicetree/bindings/display/panel/ilitek,ili9163.yaml +++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili9163.yaml @@ -24,6 +24,9 @@ properties: - newhaven,1.8-128160EF - const: ilitek,ili9163 + reg: + maxItems: 1 + spi-max-frequency: maximum: 32000000 @@ -32,7 +35,6 @@ properties: description: Display data/command selection (D/CX) backlight: true - reg: true reset-gpios: true rotation: true diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili9322.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,ili9322.yaml index 7d221ef354434..44423465f6e35 100644 --- a/Documentation/devicetree/bindings/display/panel/ilitek,ili9322.yaml +++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili9322.yaml @@ -26,6 +26,9 @@ properties: - dlink,dir-685-panel - const: ilitek,ili9322 + reg: + maxItems: 1 + reset-gpios: true port: true diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili9341.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,ili9341.yaml index 94f169ea065a0..5f41758c96d5c 100644 --- a/Documentation/devicetree/bindings/display/panel/ilitek,ili9341.yaml +++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili9341.yaml @@ -28,7 +28,8 @@ properties: - canaan,kd233-tft - const: ilitek,ili9341 - reg: true + reg: + maxItems: 1 dc-gpios: maxItems: 1 diff --git a/Documentation/devicetree/bindings/display/panel/innolux,ej030na.yaml b/Documentation/devicetree/bindings/display/panel/innolux,ej030na.yaml index 72788e3e6c590..c7df9a7f6589e 100644 --- a/Documentation/devicetree/bindings/display/panel/innolux,ej030na.yaml +++ b/Documentation/devicetree/bindings/display/panel/innolux,ej030na.yaml @@ -17,10 +17,12 @@ properties: compatible: const: innolux,ej030na + reg: + maxItems: 1 + backlight: true port: true power-supply: true - reg: true reset-gpios: true required: diff --git a/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml b/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml index b4be9bd8ddde7..d86c916f7b554 100644 --- a/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml +++ b/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml @@ -17,10 +17,12 @@ properties: compatible: const: kingdisplay,kd035g6-54nt + reg: + maxItems: 1 + backlight: true port: true power-supply: true - reg: true reset-gpios: true spi-3wire: true diff --git a/Documentation/devicetree/bindings/display/panel/leadtek,ltk035c5444t.yaml b/Documentation/devicetree/bindings/display/panel/leadtek,ltk035c5444t.yaml index 7a55961e1a3d3..b5dc02b272002 100644 --- a/Documentation/devicetree/bindings/display/panel/leadtek,ltk035c5444t.yaml +++ b/Documentation/devicetree/bindings/display/panel/leadtek,ltk035c5444t.yaml @@ -18,6 +18,9 @@ properties: compatible: const: leadtek,ltk035c5444t + reg: + maxItems: 1 + spi-3wire: true required: diff --git a/Documentation/devicetree/bindings/display/panel/lg,lg4573.yaml b/Documentation/devicetree/bindings/display/panel/lg,lg4573.yaml index ee357e139ac07..590ccc27d1044 100644 --- a/Documentation/devicetree/bindings/display/panel/lg,lg4573.yaml +++ b/Documentation/devicetree/bindings/display/panel/lg,lg4573.yaml @@ -21,7 +21,8 @@ properties: compatible: const: lg,lg4573 - reg: true + reg: + maxItems: 1 required: - compatible diff --git a/Documentation/devicetree/bindings/display/panel/lgphilips,lb035q02.yaml b/Documentation/devicetree/bindings/display/panel/lgphilips,lb035q02.yaml index 628c4b8981114..3de17fd8513bf 100644 --- a/Documentation/devicetree/bindings/display/panel/lgphilips,lb035q02.yaml +++ b/Documentation/devicetree/bindings/display/panel/lgphilips,lb035q02.yaml @@ -17,6 +17,9 @@ properties: compatible: const: lgphilips,lb035q02 + reg: + maxItems: 1 + label: true enable-gpios: true port: true diff --git a/Documentation/devicetree/bindings/display/panel/nec,nl8048hl11.yaml b/Documentation/devicetree/bindings/display/panel/nec,nl8048hl11.yaml index accf933d6e465..1cffe4d6d4984 100644 --- a/Documentation/devicetree/bindings/display/panel/nec,nl8048hl11.yaml +++ b/Documentation/devicetree/bindings/display/panel/nec,nl8048hl11.yaml @@ -21,9 +21,11 @@ properties: compatible: const: nec,nl8048hl11 + reg: + maxItems: 1 + label: true port: true - reg: true reset-gpios: true spi-max-frequency: diff --git a/Documentation/devicetree/bindings/display/panel/panel-mipi-dbi-spi.yaml b/Documentation/devicetree/bindings/display/panel/panel-mipi-dbi-spi.yaml index e808215cb39ee..d0ac31ab60cf3 100644 --- a/Documentation/devicetree/bindings/display/panel/panel-mipi-dbi-spi.yaml +++ b/Documentation/devicetree/bindings/display/panel/panel-mipi-dbi-spi.yaml @@ -71,6 +71,9 @@ properties: - shineworld,lh133k - const: panel-mipi-dbi-spi + reg: + maxItems: 1 + write-only: type: boolean description: diff --git a/Documentation/devicetree/bindings/display/panel/samsung,ams495qa01.yaml b/Documentation/devicetree/bindings/display/panel/samsung,ams495qa01.yaml index a5ff4de74a158..e081c84a932b6 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,ams495qa01.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,ams495qa01.yaml @@ -17,7 +17,9 @@ properties: compatible: const: samsung,ams495qa01 - reg: true + reg: + maxItems: 1 + reset-gpios: description: reset gpio, must be GPIO_ACTIVE_LOW elvdd-supply: diff --git a/Documentation/devicetree/bindings/display/panel/samsung,ld9040.yaml b/Documentation/devicetree/bindings/display/panel/samsung,ld9040.yaml index c0fabeb386288..bc92b16c95b9e 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,ld9040.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,ld9040.yaml @@ -17,9 +17,11 @@ properties: compatible: const: samsung,ld9040 + reg: + maxItems: 1 + display-timings: true port: true - reg: true reset-gpios: true vdd3-supply: diff --git a/Documentation/devicetree/bindings/display/panel/samsung,lms380kf01.yaml b/Documentation/devicetree/bindings/display/panel/samsung,lms380kf01.yaml index 70ffc88d2a081..7ce8540551f9e 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,lms380kf01.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,lms380kf01.yaml @@ -21,7 +21,8 @@ properties: compatible: const: samsung,lms380kf01 - reg: true + reg: + maxItems: 1 interrupts: description: provides an optional ESD (electrostatic discharge) diff --git a/Documentation/devicetree/bindings/display/panel/samsung,lms397kf04.yaml b/Documentation/devicetree/bindings/display/panel/samsung,lms397kf04.yaml index 5e77cee93f833..9363032883de4 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,lms397kf04.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,lms397kf04.yaml @@ -20,7 +20,8 @@ properties: compatible: const: samsung,lms397kf04 - reg: true + reg: + maxItems: 1 reset-gpios: true diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6d27a1.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6d27a1.yaml index d273faf4442ac..d74904164719d 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,s6d27a1.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,s6d27a1.yaml @@ -20,7 +20,8 @@ properties: compatible: const: samsung,s6d27a1 - reg: true + reg: + maxItems: 1 interrupts: description: provides an optional ESD (electrostatic discharge) diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml index 6f1fc7469f076..c47e2a1a30e5b 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml @@ -18,7 +18,9 @@ properties: compatible: const: samsung,s6e63m0 - reg: true + reg: + maxItems: 1 + reset-gpios: true port: true default-brightness: true diff --git a/Documentation/devicetree/bindings/display/panel/sitronix,st7789v.yaml b/Documentation/devicetree/bindings/display/panel/sitronix,st7789v.yaml index ef162b51d010d..0ce2ea13583dd 100644 --- a/Documentation/devicetree/bindings/display/panel/sitronix,st7789v.yaml +++ b/Documentation/devicetree/bindings/display/panel/sitronix,st7789v.yaml @@ -21,7 +21,9 @@ properties: - jasonic,jt240mhqs-hwt-ek-e3 - sitronix,st7789v - reg: true + reg: + maxItems: 1 + reset-gpios: true power-supply: true backlight: true diff --git a/Documentation/devicetree/bindings/display/panel/sony,acx565akm.yaml b/Documentation/devicetree/bindings/display/panel/sony,acx565akm.yaml index 98abdf4ddeac6..5a8260224b747 100644 --- a/Documentation/devicetree/bindings/display/panel/sony,acx565akm.yaml +++ b/Documentation/devicetree/bindings/display/panel/sony,acx565akm.yaml @@ -17,6 +17,9 @@ properties: compatible: const: sony,acx565akm + reg: + maxItems: 1 + label: true reset-gpios: true port: true diff --git a/Documentation/devicetree/bindings/display/panel/tpo,td.yaml b/Documentation/devicetree/bindings/display/panel/tpo,td.yaml index e8c8ee8d7c883..7edd29df4bbb5 100644 --- a/Documentation/devicetree/bindings/display/panel/tpo,td.yaml +++ b/Documentation/devicetree/bindings/display/panel/tpo,td.yaml @@ -22,7 +22,9 @@ properties: # Toppoly TD043MTEA1 Panel - tpo,td043mtea1 - reg: true + reg: + maxItems: 1 + label: true reset-gpios: true backlight: true diff --git a/Documentation/devicetree/bindings/display/panel/tpo,tpg110.yaml b/Documentation/devicetree/bindings/display/panel/tpo,tpg110.yaml index f0243d1961916..59a373728e628 100644 --- a/Documentation/devicetree/bindings/display/panel/tpo,tpg110.yaml +++ b/Documentation/devicetree/bindings/display/panel/tpo,tpg110.yaml @@ -52,7 +52,8 @@ properties: - const: tpo,tpg110 - const: tpo,tpg110 - reg: true + reg: + maxItems: 1 grestb-gpios: maxItems: 1 From 9fa6bcf23e4417127c8dca996fe7ba7a14a9b7de Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 9 May 2024 11:42:53 +0200 Subject: [PATCH 1596/1702] dt-bindings: display: panel: constrain 'reg' in DSI panels DSI-attached devices could respond to more than one virtual channel number, thus their bindings are supposed to constrain the 'reg' property to match hardware. Add missing 'reg' constrain for DSI-attached display panels, based on DTS sources in Linux kernel (assume all devices take only one channel number). Signed-off-by: Krzysztof Kozlowski Acked-by: Conor Dooley Link: https://lore.kernel.org/r/20240509-dt-bindings-dsi-panel-reg-v1-3-8b2443705be0@linaro.org Signed-off-by: Rob Herring (Arm) --- .../bindings/display/panel/asus,z00t-tm5p5-nt35596.yaml | 5 ++++- .../devicetree/bindings/display/panel/boe,bf060y8m-aj0.yaml | 4 +++- .../devicetree/bindings/display/panel/boe,himax8279d.yaml | 4 +++- .../bindings/display/panel/boe,th101mb31ig002-28a.yaml | 4 +++- .../devicetree/bindings/display/panel/boe,tv101wum-nl6.yaml | 2 +- .../devicetree/bindings/display/panel/elida,kd35t133.yaml | 5 ++++- .../bindings/display/panel/feixin,k101-im2ba02.yaml | 5 ++++- .../devicetree/bindings/display/panel/himax,hx83112a.yaml | 4 +++- .../devicetree/bindings/display/panel/himax,hx8394.yaml | 3 ++- .../devicetree/bindings/display/panel/ilitek,ili9805.yaml | 4 +++- .../devicetree/bindings/display/panel/ilitek,ili9881c.yaml | 4 +++- .../devicetree/bindings/display/panel/innolux,p097pfg.yaml | 4 +++- .../bindings/display/panel/jadard,jd9365da-h3.yaml | 3 ++- .../devicetree/bindings/display/panel/jdi,lpm102a188a.yaml | 4 +++- .../devicetree/bindings/display/panel/jdi,lt070me05000.yaml | 4 +++- .../bindings/display/panel/leadtek,ltk050h3146w.yaml | 5 ++++- .../bindings/display/panel/leadtek,ltk500hd1829.yaml | 5 ++++- .../bindings/display/panel/newvision,nv3051d.yaml | 4 +++- .../devicetree/bindings/display/panel/novatek,nt35510.yaml | 5 ++++- .../devicetree/bindings/display/panel/novatek,nt35950.yaml | 4 +++- .../devicetree/bindings/display/panel/novatek,nt36523.yaml | 4 +++- .../devicetree/bindings/display/panel/novatek,nt36672a.yaml | 4 +++- .../bindings/display/panel/olimex,lcd-olinuxino.yaml | 4 +++- .../devicetree/bindings/display/panel/raydium,rm67191.yaml | 4 +++- .../devicetree/bindings/display/panel/raydium,rm692e5.yaml | 4 +++- .../devicetree/bindings/display/panel/ronbo,rb070d30.yaml | 2 +- .../bindings/display/panel/samsung,amoled-mipi-dsi.yaml | 4 +++- .../devicetree/bindings/display/panel/samsung,s6d16d0.yaml | 4 +++- .../devicetree/bindings/display/panel/samsung,s6d7aa0.yaml | 3 ++- .../bindings/display/panel/samsung,s6e88a0-ams452ef01.yaml | 5 ++++- .../devicetree/bindings/display/panel/samsung,s6e8aa0.yaml | 4 +++- .../bindings/display/panel/sharp,lq101r1sx01.yaml | 4 +++- .../bindings/display/panel/sharp,ls043t1le01.yaml | 4 +++- .../bindings/display/panel/sharp,ls060t1sx01.yaml | 4 +++- .../devicetree/bindings/display/panel/sony,acx424akp.yaml | 5 ++++- .../devicetree/bindings/display/panel/sony,td4353-jdi.yaml | 3 ++- .../bindings/display/panel/sony,tulip-truly-nt35521.yaml | 3 ++- .../devicetree/bindings/display/panel/synaptics,r63353.yaml | 6 ++++-- .../devicetree/bindings/display/panel/visionox,rm69299.yaml | 3 ++- .../bindings/display/panel/xinpeng,xpp055c272.yaml | 5 ++++- 40 files changed, 120 insertions(+), 41 deletions(-) diff --git a/Documentation/devicetree/bindings/display/panel/asus,z00t-tm5p5-nt35596.yaml b/Documentation/devicetree/bindings/display/panel/asus,z00t-tm5p5-nt35596.yaml index 75a09df68ba00..2399cabf044c2 100644 --- a/Documentation/devicetree/bindings/display/panel/asus,z00t-tm5p5-nt35596.yaml +++ b/Documentation/devicetree/bindings/display/panel/asus,z00t-tm5p5-nt35596.yaml @@ -21,7 +21,10 @@ allOf: properties: compatible: const: asus,z00t-tm5p5-n35596 - reg: true + + reg: + maxItems: 1 + reset-gpios: true vdd-supply: description: core voltage supply diff --git a/Documentation/devicetree/bindings/display/panel/boe,bf060y8m-aj0.yaml b/Documentation/devicetree/bindings/display/panel/boe,bf060y8m-aj0.yaml index a8f3afa922c89..8b7448ad9138f 100644 --- a/Documentation/devicetree/bindings/display/panel/boe,bf060y8m-aj0.yaml +++ b/Documentation/devicetree/bindings/display/panel/boe,bf060y8m-aj0.yaml @@ -26,6 +26,9 @@ properties: compatible: const: boe,bf060y8m-aj0 + reg: + maxItems: 1 + elvdd-supply: description: EL Driving positive (VDD) supply (4.40-4.80V) elvss-supply: @@ -38,7 +41,6 @@ properties: description: I/O voltage supply (1.62-1.98V) port: true - reg: true reset-gpios: true required: diff --git a/Documentation/devicetree/bindings/display/panel/boe,himax8279d.yaml b/Documentation/devicetree/bindings/display/panel/boe,himax8279d.yaml index 272a3a018a33a..f2496cdd92602 100644 --- a/Documentation/devicetree/bindings/display/panel/boe,himax8279d.yaml +++ b/Documentation/devicetree/bindings/display/panel/boe,himax8279d.yaml @@ -18,9 +18,11 @@ properties: - const: boe,himax8279d8p - const: boe,himax8279d10p + reg: + maxItems: 1 + backlight: true enable-gpios: true - reg: true pp33-gpios: maxItems: 1 diff --git a/Documentation/devicetree/bindings/display/panel/boe,th101mb31ig002-28a.yaml b/Documentation/devicetree/bindings/display/panel/boe,th101mb31ig002-28a.yaml index 32df26cbfeed7..5eaccce13c216 100644 --- a/Documentation/devicetree/bindings/display/panel/boe,th101mb31ig002-28a.yaml +++ b/Documentation/devicetree/bindings/display/panel/boe,th101mb31ig002-28a.yaml @@ -18,7 +18,9 @@ properties: # BOE TH101MB31IG002-28A 10.1" WXGA TFT LCD panel - boe,th101mb31ig002-28a - reg: true + reg: + maxItems: 1 + backlight: true enable-gpios: true power-supply: true diff --git a/Documentation/devicetree/bindings/display/panel/boe,tv101wum-nl6.yaml b/Documentation/devicetree/bindings/display/panel/boe,tv101wum-nl6.yaml index 906ef62709b8a..9e603cad13480 100644 --- a/Documentation/devicetree/bindings/display/panel/boe,tv101wum-nl6.yaml +++ b/Documentation/devicetree/bindings/display/panel/boe,tv101wum-nl6.yaml @@ -38,7 +38,7 @@ properties: - starry,ili9882t reg: - description: the virtual channel number of a DSI peripheral + maxItems: 1 enable-gpios: description: a GPIO spec for the enable pin diff --git a/Documentation/devicetree/bindings/display/panel/elida,kd35t133.yaml b/Documentation/devicetree/bindings/display/panel/elida,kd35t133.yaml index 265ab6d305726..f4cb825d1e966 100644 --- a/Documentation/devicetree/bindings/display/panel/elida,kd35t133.yaml +++ b/Documentation/devicetree/bindings/display/panel/elida,kd35t133.yaml @@ -15,7 +15,10 @@ allOf: properties: compatible: const: elida,kd35t133 - reg: true + + reg: + maxItems: 1 + backlight: true port: true reset-gpios: true diff --git a/Documentation/devicetree/bindings/display/panel/feixin,k101-im2ba02.yaml b/Documentation/devicetree/bindings/display/panel/feixin,k101-im2ba02.yaml index 81adb82f061d2..0d8707a5844dd 100644 --- a/Documentation/devicetree/bindings/display/panel/feixin,k101-im2ba02.yaml +++ b/Documentation/devicetree/bindings/display/panel/feixin,k101-im2ba02.yaml @@ -15,7 +15,10 @@ allOf: properties: compatible: const: feixin,k101-im2ba02 - reg: true + + reg: + maxItems: 1 + backlight: true reset-gpios: true avdd-supply: diff --git a/Documentation/devicetree/bindings/display/panel/himax,hx83112a.yaml b/Documentation/devicetree/bindings/display/panel/himax,hx83112a.yaml index 174661d138111..56bcd152f43cb 100644 --- a/Documentation/devicetree/bindings/display/panel/himax,hx83112a.yaml +++ b/Documentation/devicetree/bindings/display/panel/himax,hx83112a.yaml @@ -21,6 +21,9 @@ properties: contains: const: djn,9a-3r063-1102b + reg: + maxItems: 1 + vdd1-supply: description: Digital voltage rail @@ -30,7 +33,6 @@ properties: vsp-supply: description: Negative source voltage rail - reg: true port: true required: diff --git a/Documentation/devicetree/bindings/display/panel/himax,hx8394.yaml b/Documentation/devicetree/bindings/display/panel/himax,hx8394.yaml index 916bb7f942062..644387e4fb6f9 100644 --- a/Documentation/devicetree/bindings/display/panel/himax,hx8394.yaml +++ b/Documentation/devicetree/bindings/display/panel/himax,hx8394.yaml @@ -26,7 +26,8 @@ properties: - powkiddy,x55-panel - const: himax,hx8394 - reg: true + reg: + maxItems: 1 reset-gpios: true diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili9805.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,ili9805.yaml index f4f91f93f4903..ff67129c94669 100644 --- a/Documentation/devicetree/bindings/display/panel/ilitek,ili9805.yaml +++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili9805.yaml @@ -20,9 +20,11 @@ properties: - tianma,tm041xdhg01 - const: ilitek,ili9805 + reg: + maxItems: 1 + avdd-supply: true dvdd-supply: true - reg: true required: - compatible diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili9881c.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,ili9881c.yaml index b1e624be3e334..ac226d5b2c1e9 100644 --- a/Documentation/devicetree/bindings/display/panel/ilitek,ili9881c.yaml +++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili9881c.yaml @@ -23,9 +23,11 @@ properties: - wanchanglong,w552946aba - const: ilitek,ili9881c + reg: + maxItems: 1 + backlight: true power-supply: true - reg: true reset-gpios: true rotation: true diff --git a/Documentation/devicetree/bindings/display/panel/innolux,p097pfg.yaml b/Documentation/devicetree/bindings/display/panel/innolux,p097pfg.yaml index 5a5f071627fb2..4164e3f7061d8 100644 --- a/Documentation/devicetree/bindings/display/panel/innolux,p097pfg.yaml +++ b/Documentation/devicetree/bindings/display/panel/innolux,p097pfg.yaml @@ -16,9 +16,11 @@ properties: compatible: const: innolux,p097pfg + reg: + maxItems: 1 + backlight: true enable-gpios: true - reg: true avdd-supply: description: The regulator that provides positive voltage diff --git a/Documentation/devicetree/bindings/display/panel/jadard,jd9365da-h3.yaml b/Documentation/devicetree/bindings/display/panel/jadard,jd9365da-h3.yaml index 41eb7fbf7715b..20afdb4568a26 100644 --- a/Documentation/devicetree/bindings/display/panel/jadard,jd9365da-h3.yaml +++ b/Documentation/devicetree/bindings/display/panel/jadard,jd9365da-h3.yaml @@ -21,7 +21,8 @@ properties: - radxa,display-8hd-ad002 - const: jadard,jd9365da-h3 - reg: true + reg: + maxItems: 1 vdd-supply: description: supply regulator for VDD, usually 3.3V diff --git a/Documentation/devicetree/bindings/display/panel/jdi,lpm102a188a.yaml b/Documentation/devicetree/bindings/display/panel/jdi,lpm102a188a.yaml index 2f4d27a309a78..a8621459005bf 100644 --- a/Documentation/devicetree/bindings/display/panel/jdi,lpm102a188a.yaml +++ b/Documentation/devicetree/bindings/display/panel/jdi,lpm102a188a.yaml @@ -26,7 +26,9 @@ properties: compatible: const: jdi,lpm102a188a - reg: true + reg: + maxItems: 1 + enable-gpios: true reset-gpios: true power-supply: true diff --git a/Documentation/devicetree/bindings/display/panel/jdi,lt070me05000.yaml b/Documentation/devicetree/bindings/display/panel/jdi,lt070me05000.yaml index 63c82a4378ff5..0c8b5cb78bfe4 100644 --- a/Documentation/devicetree/bindings/display/panel/jdi,lt070me05000.yaml +++ b/Documentation/devicetree/bindings/display/panel/jdi,lt070me05000.yaml @@ -16,8 +16,10 @@ properties: compatible: const: jdi,lt070me05000 + reg: + maxItems: 1 + enable-gpios: true - reg: true reset-gpios: true vddp-supply: diff --git a/Documentation/devicetree/bindings/display/panel/leadtek,ltk050h3146w.yaml b/Documentation/devicetree/bindings/display/panel/leadtek,ltk050h3146w.yaml index a40ab887ada7b..e2a2dd4ef5fa2 100644 --- a/Documentation/devicetree/bindings/display/panel/leadtek,ltk050h3146w.yaml +++ b/Documentation/devicetree/bindings/display/panel/leadtek,ltk050h3146w.yaml @@ -18,7 +18,10 @@ properties: - leadtek,ltk050h3146w - leadtek,ltk050h3146w-a2 - leadtek,ltk050h3148w - reg: true + + reg: + maxItems: 1 + backlight: true reset-gpios: true iovcc-supply: diff --git a/Documentation/devicetree/bindings/display/panel/leadtek,ltk500hd1829.yaml b/Documentation/devicetree/bindings/display/panel/leadtek,ltk500hd1829.yaml index d589f16772145..af9e0ea0e72f9 100644 --- a/Documentation/devicetree/bindings/display/panel/leadtek,ltk500hd1829.yaml +++ b/Documentation/devicetree/bindings/display/panel/leadtek,ltk500hd1829.yaml @@ -17,7 +17,10 @@ properties: enum: - leadtek,ltk101b4029w - leadtek,ltk500hd1829 - reg: true + + reg: + maxItems: 1 + backlight: true reset-gpios: true iovcc-supply: diff --git a/Documentation/devicetree/bindings/display/panel/newvision,nv3051d.yaml b/Documentation/devicetree/bindings/display/panel/newvision,nv3051d.yaml index 7a634fbc465e0..d3a25a8fd7e39 100644 --- a/Documentation/devicetree/bindings/display/panel/newvision,nv3051d.yaml +++ b/Documentation/devicetree/bindings/display/panel/newvision,nv3051d.yaml @@ -24,7 +24,9 @@ properties: - powkiddy,rk2023-panel - const: newvision,nv3051d - reg: true + reg: + maxItems: 1 + backlight: true port: true reset-gpios: diff --git a/Documentation/devicetree/bindings/display/panel/novatek,nt35510.yaml b/Documentation/devicetree/bindings/display/panel/novatek,nt35510.yaml index 91921f4b0e5fa..bb50fd5506c3d 100644 --- a/Documentation/devicetree/bindings/display/panel/novatek,nt35510.yaml +++ b/Documentation/devicetree/bindings/display/panel/novatek,nt35510.yaml @@ -24,7 +24,10 @@ properties: string determines how the NT35510 panel driver shall be configured to work with the indicated panel. The novatek,nt35510 compatible shall always be provided as a fallback. - reg: true + + reg: + maxItems: 1 + reset-gpios: true vdd-supply: description: regulator that supplies the vdd voltage diff --git a/Documentation/devicetree/bindings/display/panel/novatek,nt35950.yaml b/Documentation/devicetree/bindings/display/panel/novatek,nt35950.yaml index 377a05d48a02a..52cee4ee84b48 100644 --- a/Documentation/devicetree/bindings/display/panel/novatek,nt35950.yaml +++ b/Documentation/devicetree/bindings/display/panel/novatek,nt35950.yaml @@ -33,6 +33,9 @@ properties: to work with the indicated panel. The novatek,nt35950 compatible shall always be provided as a fallback. + reg: + maxItems: 1 + reset-gpios: maxItems: 1 description: phandle of gpio for reset line - This should be 8mA, gpio @@ -49,7 +52,6 @@ properties: backlight: true ports: true - reg: true required: - compatible diff --git a/Documentation/devicetree/bindings/display/panel/novatek,nt36523.yaml b/Documentation/devicetree/bindings/display/panel/novatek,nt36523.yaml index 5f7e4c4860944..0447ee7249474 100644 --- a/Documentation/devicetree/bindings/display/panel/novatek,nt36523.yaml +++ b/Documentation/devicetree/bindings/display/panel/novatek,nt36523.yaml @@ -30,6 +30,9 @@ properties: - lenovo,j606f-boe-nt36523w - const: novatek,nt36523w + reg: + maxItems: 1 + reset-gpios: maxItems: 1 description: phandle of gpio for reset line - This should be 8mA @@ -37,7 +40,6 @@ properties: vddio-supply: description: regulator that supplies the I/O voltage - reg: true ports: true rotation: true backlight: true diff --git a/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml b/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml index ae821f465e1c8..800a2f0a4dad9 100644 --- a/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml +++ b/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml @@ -29,6 +29,9 @@ properties: determines how the NT36672A panel driver is configured for the indicated panel. The novatek,nt36672a compatible shall always be provided as a fallback. + reg: + maxItems: 1 + reset-gpios: maxItems: 1 description: phandle of gpio for reset line - This should be 8mA, gpio @@ -44,7 +47,6 @@ properties: vddneg-supply: description: phandle of the negative boost supply regulator - reg: true port: true backlight: true diff --git a/Documentation/devicetree/bindings/display/panel/olimex,lcd-olinuxino.yaml b/Documentation/devicetree/bindings/display/panel/olimex,lcd-olinuxino.yaml index 72463795e4c6d..e5d8785fdf905 100644 --- a/Documentation/devicetree/bindings/display/panel/olimex,lcd-olinuxino.yaml +++ b/Documentation/devicetree/bindings/display/panel/olimex,lcd-olinuxino.yaml @@ -38,10 +38,12 @@ properties: compatible: const: olimex,lcd-olinuxino + reg: + maxItems: 1 + backlight: true enable-gpios: true power-supply: true - reg: true required: - compatible diff --git a/Documentation/devicetree/bindings/display/panel/raydium,rm67191.yaml b/Documentation/devicetree/bindings/display/panel/raydium,rm67191.yaml index d62fd692bf106..4825792bf712f 100644 --- a/Documentation/devicetree/bindings/display/panel/raydium,rm67191.yaml +++ b/Documentation/devicetree/bindings/display/panel/raydium,rm67191.yaml @@ -16,7 +16,9 @@ properties: compatible: const: raydium,rm67191 - reg: true + reg: + maxItems: 1 + port: true reset-gpios: true width-mm: true diff --git a/Documentation/devicetree/bindings/display/panel/raydium,rm692e5.yaml b/Documentation/devicetree/bindings/display/panel/raydium,rm692e5.yaml index f436ba6738caa..7ad223f98253a 100644 --- a/Documentation/devicetree/bindings/display/panel/raydium,rm692e5.yaml +++ b/Documentation/devicetree/bindings/display/panel/raydium,rm692e5.yaml @@ -22,6 +22,9 @@ properties: - const: fairphone,fp5-rm692e5-boe - const: raydium,rm692e5 + reg: + maxItems: 1 + dvdd-supply: description: Digital voltage rail @@ -31,7 +34,6 @@ properties: vddio-supply: description: I/O voltage rail - reg: true port: true required: diff --git a/Documentation/devicetree/bindings/display/panel/ronbo,rb070d30.yaml b/Documentation/devicetree/bindings/display/panel/ronbo,rb070d30.yaml index 95ce22c6787a7..04f86e0cbac91 100644 --- a/Documentation/devicetree/bindings/display/panel/ronbo,rb070d30.yaml +++ b/Documentation/devicetree/bindings/display/panel/ronbo,rb070d30.yaml @@ -14,7 +14,7 @@ properties: const: ronbo,rb070d30 reg: - description: MIPI-DSI virtual channel + maxItems: 1 power-gpios: description: GPIO used for the power pin diff --git a/Documentation/devicetree/bindings/display/panel/samsung,amoled-mipi-dsi.yaml b/Documentation/devicetree/bindings/display/panel/samsung,amoled-mipi-dsi.yaml index ccc482570d6a3..e8f9e9d06a292 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,amoled-mipi-dsi.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,amoled-mipi-dsi.yaml @@ -33,7 +33,9 @@ properties: # Samsung S6E3HF2 5.65" 1600x2560 AMOLED panel - samsung,s6e3hf2 - reg: true + reg: + maxItems: 1 + reset-gpios: true enable-gpios: true te-gpios: true diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6d16d0.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6d16d0.yaml index 66d147496bc3d..2af5bc47323f5 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,s6d16d0.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,s6d16d0.yaml @@ -16,8 +16,10 @@ properties: compatible: const: samsung,s6d16d0 + reg: + maxItems: 1 + port: true - reg: true reset-gpios: true vdd1-supply: diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6d7aa0.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6d7aa0.yaml index 45a236d2cc70a..939da65114bfc 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,s6d7aa0.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,s6d7aa0.yaml @@ -24,7 +24,8 @@ properties: - samsung,ltl101at01 - const: samsung,s6d7aa0 - reg: true + reg: + maxItems: 1 backlight: description: diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6e88a0-ams452ef01.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6e88a0-ams452ef01.yaml index b749e9e906b7e..42634fc3b5b20 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,s6e88a0-ams452ef01.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,s6e88a0-ams452ef01.yaml @@ -15,7 +15,10 @@ allOf: properties: compatible: const: samsung,s6e88a0-ams452ef01 - reg: true + + reg: + maxItems: 1 + port: true reset-gpios: true vdd3-supply: diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6e8aa0.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6e8aa0.yaml index 200fbf1c74a0b..4601fa4606807 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,s6e8aa0.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,s6e8aa0.yaml @@ -16,7 +16,9 @@ properties: compatible: const: samsung,s6e8aa0 - reg: true + reg: + maxItems: 1 + reset-gpios: true display-timings: true diff --git a/Documentation/devicetree/bindings/display/panel/sharp,lq101r1sx01.yaml b/Documentation/devicetree/bindings/display/panel/sharp,lq101r1sx01.yaml index 57b44a0e763de..ce820b96a7e2f 100644 --- a/Documentation/devicetree/bindings/display/panel/sharp,lq101r1sx01.yaml +++ b/Documentation/devicetree/bindings/display/panel/sharp,lq101r1sx01.yaml @@ -37,7 +37,9 @@ properties: - enum: - sharp,lq101r1sx01 - reg: true + reg: + maxItems: 1 + power-supply: true backlight: true diff --git a/Documentation/devicetree/bindings/display/panel/sharp,ls043t1le01.yaml b/Documentation/devicetree/bindings/display/panel/sharp,ls043t1le01.yaml index a90d0d8bf7c9e..b6ea246430cee 100644 --- a/Documentation/devicetree/bindings/display/panel/sharp,ls043t1le01.yaml +++ b/Documentation/devicetree/bindings/display/panel/sharp,ls043t1le01.yaml @@ -16,7 +16,9 @@ properties: compatible: const: sharp,ls043t1le01-qhd - reg: true + reg: + maxItems: 1 + backlight: true reset-gpios: true port: true diff --git a/Documentation/devicetree/bindings/display/panel/sharp,ls060t1sx01.yaml b/Documentation/devicetree/bindings/display/panel/sharp,ls060t1sx01.yaml index 271c097cc9a44..77a4fce129e75 100644 --- a/Documentation/devicetree/bindings/display/panel/sharp,ls060t1sx01.yaml +++ b/Documentation/devicetree/bindings/display/panel/sharp,ls060t1sx01.yaml @@ -16,7 +16,9 @@ properties: compatible: const: sharp,ls060t1sx01 - reg: true + reg: + maxItems: 1 + backlight: true reset-gpios: true port: true diff --git a/Documentation/devicetree/bindings/display/panel/sony,acx424akp.yaml b/Documentation/devicetree/bindings/display/panel/sony,acx424akp.yaml index 059cc6dbcfca9..fd778a20f7609 100644 --- a/Documentation/devicetree/bindings/display/panel/sony,acx424akp.yaml +++ b/Documentation/devicetree/bindings/display/panel/sony,acx424akp.yaml @@ -22,7 +22,10 @@ properties: enum: - sony,acx424akp - sony,acx424akm - reg: true + + reg: + maxItems: 1 + reset-gpios: true vddi-supply: description: regulator that supplies the vddi voltage diff --git a/Documentation/devicetree/bindings/display/panel/sony,td4353-jdi.yaml b/Documentation/devicetree/bindings/display/panel/sony,td4353-jdi.yaml index b6b885b4c22dc..feca40b0d7591 100644 --- a/Documentation/devicetree/bindings/display/panel/sony,td4353-jdi.yaml +++ b/Documentation/devicetree/bindings/display/panel/sony,td4353-jdi.yaml @@ -20,7 +20,8 @@ properties: compatible: const: sony,td4353-jdi-tama - reg: true + reg: + maxItems: 1 backlight: true diff --git a/Documentation/devicetree/bindings/display/panel/sony,tulip-truly-nt35521.yaml b/Documentation/devicetree/bindings/display/panel/sony,tulip-truly-nt35521.yaml index 9679729395988..a58a31349757d 100644 --- a/Documentation/devicetree/bindings/display/panel/sony,tulip-truly-nt35521.yaml +++ b/Documentation/devicetree/bindings/display/panel/sony,tulip-truly-nt35521.yaml @@ -21,7 +21,8 @@ properties: compatible: const: sony,tulip-truly-nt35521 - reg: true + reg: + maxItems: 1 positive5-supply: description: Positive 5V supply diff --git a/Documentation/devicetree/bindings/display/panel/synaptics,r63353.yaml b/Documentation/devicetree/bindings/display/panel/synaptics,r63353.yaml index e5617d125567d..2fd6e0ec36825 100644 --- a/Documentation/devicetree/bindings/display/panel/synaptics,r63353.yaml +++ b/Documentation/devicetree/bindings/display/panel/synaptics,r63353.yaml @@ -19,15 +19,17 @@ properties: - sharp,ls068b3sx02 - const: syna,r63353 + reg: + maxItems: 1 + avdd-supply: true dvdd-supply: true - reg: true required: - compatible + - reg - avdd-supply - dvdd-supply - - reg - reset-gpios - port - backlight diff --git a/Documentation/devicetree/bindings/display/panel/visionox,rm69299.yaml b/Documentation/devicetree/bindings/display/panel/visionox,rm69299.yaml index 7723990675158..30047a62fc111 100644 --- a/Documentation/devicetree/bindings/display/panel/visionox,rm69299.yaml +++ b/Documentation/devicetree/bindings/display/panel/visionox,rm69299.yaml @@ -20,7 +20,8 @@ properties: compatible: const: visionox,rm69299-1080p-display - reg: true + reg: + maxItems: 1 vdda-supply: description: | diff --git a/Documentation/devicetree/bindings/display/panel/xinpeng,xpp055c272.yaml b/Documentation/devicetree/bindings/display/panel/xinpeng,xpp055c272.yaml index c407deb6afb11..9c9743a23500d 100644 --- a/Documentation/devicetree/bindings/display/panel/xinpeng,xpp055c272.yaml +++ b/Documentation/devicetree/bindings/display/panel/xinpeng,xpp055c272.yaml @@ -15,7 +15,10 @@ allOf: properties: compatible: const: xinpeng,xpp055c272 - reg: true + + reg: + maxItems: 1 + backlight: true port: true reset-gpios: true From d976c6f4b32c2d273d44ff9ad7099efe16279a39 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Thu, 9 May 2024 17:38:20 +0530 Subject: [PATCH 1597/1702] of: property: Add fw_devlink support for interrupt-map property Some of the PCI host controllers (such as generic PCI host controller) use "interrupt-map" DT property to describe the mapping between PCI endpoints and PCI interrupt pins. This is the only case where the interrupts are not described in DT. Currently, there is no fw_devlink created based on "interrupt-map" DT property so interrupt controller is not guaranteed to be probed before the PCI host controller. This affects every platform where both PCI host controller and interrupt controllers are probed as regular platform devices. This creates fw_devlink between consumers (PCI host controller) and supplier (interrupt controller) based on "interrupt-map" DT property. Signed-off-by: Anup Patel Reviewed-by: Saravana Kannan Link: https://lore.kernel.org/r/20240509120820.1430587-1-apatel@ventanamicro.com Signed-off-by: Rob Herring (Arm) --- drivers/of/property.c | 52 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/drivers/of/property.c b/drivers/of/property.c index 8d2477ff89303..21f59e3cd6aa9 100644 --- a/drivers/of/property.c +++ b/drivers/of/property.c @@ -1301,6 +1301,57 @@ static struct device_node *parse_interrupts(struct device_node *np, return of_irq_parse_one(np, index, &sup_args) ? NULL : sup_args.np; } +static struct device_node *parse_interrupt_map(struct device_node *np, + const char *prop_name, int index) +{ + const __be32 *imap, *imap_end, *addr; + struct of_phandle_args sup_args; + u32 addrcells, intcells; + int i, imaplen; + + if (!IS_ENABLED(CONFIG_OF_IRQ)) + return NULL; + + if (strcmp(prop_name, "interrupt-map")) + return NULL; + + if (of_property_read_u32(np, "#interrupt-cells", &intcells)) + return NULL; + addrcells = of_bus_n_addr_cells(np); + + imap = of_get_property(np, "interrupt-map", &imaplen); + if (!imap || imaplen <= (addrcells + intcells)) + return NULL; + imap_end = imap + imaplen; + + while (imap < imap_end) { + addr = imap; + imap += addrcells; + + sup_args.np = np; + sup_args.args_count = intcells; + for (i = 0; i < intcells; i++) + sup_args.args[i] = be32_to_cpu(imap[i]); + imap += intcells; + + /* + * Upon success, the function of_irq_parse_raw() returns + * interrupt controller DT node pointer in sup_args.np. + */ + if (of_irq_parse_raw(addr, &sup_args)) + return NULL; + + if (!index) + return sup_args.np; + + of_node_put(sup_args.np); + imap += sup_args.args_count + 1; + index--; + } + + return NULL; +} + static struct device_node *parse_remote_endpoint(struct device_node *np, const char *prop_name, int index) @@ -1350,6 +1401,7 @@ static const struct supplier_bindings of_supplier_bindings[] = { { .parse_prop = parse_power_supplies, }, { .parse_prop = parse_gpio_compat, }, { .parse_prop = parse_interrupts, }, + { .parse_prop = parse_interrupt_map, }, { .parse_prop = parse_regulators, }, { .parse_prop = parse_gpio, }, { .parse_prop = parse_gpios, }, From bd125a084091396f3e796bb3dc009940d9771811 Mon Sep 17 00:00:00 2001 From: Beau Belgrave Date: Tue, 23 Apr 2024 16:23:37 +0000 Subject: [PATCH 1598/1702] tracing/user_events: Fix non-spaced field matching When the ABI was updated to prevent same name w/different args, it missed an important corner case when fields don't end with a space. Typically, space is used for fields to help separate them, like "u8 field1; u8 field2". If no spaces are used, like "u8 field1;u8 field2", then the parsing works for the first time. However, the match check fails on a subsequent register, leading to confusion. This is because the match check uses argv_split() and assumes that all fields will be split upon the space. When spaces are used, we get back { "u8", "field1;" }, without spaces we get back { "u8", "field1;u8" }. This causes a mismatch, and the user program gets back -EADDRINUSE. Add a method to detect this case before calling argv_split(). If found force a space after the field separator character ';'. This ensures all cases work properly for matching. With this fix, the following are all treated as matching: u8 field1;u8 field2 u8 field1; u8 field2 u8 field1;\tu8 field2 u8 field1;\nu8 field2 Link: https://lore.kernel.org/linux-trace-kernel/20240423162338.292-2-beaub@linux.microsoft.com Fixes: ba470eebc2f6 ("tracing/user_events: Prevent same name but different args event") Signed-off-by: Beau Belgrave Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events_user.c | 76 +++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index 70d428c394b68..82b191f33a28f 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -1989,6 +1989,80 @@ static int user_event_set_tp_name(struct user_event *user) return 0; } +/* + * Counts how many ';' without a trailing space are in the args. + */ +static int count_semis_no_space(char *args) +{ + int count = 0; + + while ((args = strchr(args, ';'))) { + args++; + + if (!isspace(*args)) + count++; + } + + return count; +} + +/* + * Copies the arguments while ensuring all ';' have a trailing space. + */ +static char *insert_space_after_semis(char *args, int count) +{ + char *fixed, *pos; + int len; + + len = strlen(args) + count; + fixed = kmalloc(len + 1, GFP_KERNEL); + + if (!fixed) + return NULL; + + pos = fixed; + + /* Insert a space after ';' if there is no trailing space. */ + while (*args) { + *pos = *args++; + + if (*pos++ == ';' && !isspace(*args)) + *pos++ = ' '; + } + + *pos = '\0'; + + return fixed; +} + +static char **user_event_argv_split(char *args, int *argc) +{ + char **split; + char *fixed; + int count; + + /* Count how many ';' without a trailing space */ + count = count_semis_no_space(args); + + /* No fixup is required */ + if (!count) + return argv_split(GFP_KERNEL, args, argc); + + /* We must fixup 'field;field' to 'field; field' */ + fixed = insert_space_after_semis(args, count); + + if (!fixed) + return NULL; + + /* We do a normal split afterwards */ + split = argv_split(GFP_KERNEL, fixed, argc); + + /* We can free since argv_split makes a copy */ + kfree(fixed); + + return split; +} + /* * Parses the event name, arguments and flags then registers if successful. * The name buffer lifetime is owned by this method for success cases only. @@ -2012,7 +2086,7 @@ static int user_event_parse(struct user_event_group *group, char *name, return -EPERM; if (args) { - argv = argv_split(GFP_KERNEL, args, &argc); + argv = user_event_argv_split(args, &argc); if (!argv) return -ENOMEM; From 78490b74435a8c738e91260e7df387e7cb6d6568 Mon Sep 17 00:00:00 2001 From: Beau Belgrave Date: Tue, 23 Apr 2024 16:23:38 +0000 Subject: [PATCH 1599/1702] selftests/user_events: Add non-spacing separator check The ABI documentation indicates that field separators do not need a space between them, only a ';'. When no spacing is used, the register must work. Any subsequent register, with or without spaces, must match and not return -EADDRINUSE. Add a non-spacing separator case to our self-test register case to ensure it works going forward. Link: https://lore.kernel.org/linux-trace-kernel/20240423162338.292-3-beaub@linux.microsoft.com Signed-off-by: Beau Belgrave Signed-off-by: Steven Rostedt (Google) --- tools/testing/selftests/user_events/ftrace_test.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/testing/selftests/user_events/ftrace_test.c b/tools/testing/selftests/user_events/ftrace_test.c index dcd7509fe2e05..0bb46793dcd46 100644 --- a/tools/testing/selftests/user_events/ftrace_test.c +++ b/tools/testing/selftests/user_events/ftrace_test.c @@ -261,6 +261,12 @@ TEST_F(user, register_events) { ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, ®)); ASSERT_EQ(0, reg.write_index); + /* Register without separator spacing should still match */ + reg.enable_bit = 29; + reg.name_args = (__u64)"__test_event u32 field1;u32 field2"; + ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, ®)); + ASSERT_EQ(0, reg.write_index); + /* Multiple registers to same name but different args should fail */ reg.enable_bit = 29; reg.name_args = (__u64)"__test_event u32 field1;"; @@ -288,6 +294,8 @@ TEST_F(user, register_events) { ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSUNREG, &unreg)); unreg.disable_bit = 30; ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSUNREG, &unreg)); + unreg.disable_bit = 29; + ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSUNREG, &unreg)); /* Delete should have been auto-done after close and unregister */ close(self->data_fd); From c09d4167b550f91ecf5c3db883eea314edc7f532 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Fri, 10 May 2024 15:04:30 +0100 Subject: [PATCH 1600/1702] ring-buffer: Allocate sub-buffers with __GFP_COMP In preparation for the ring-buffer memory mapping, allocate compound pages for the ring-buffer sub-buffers to enable us to map them to user-space with vm_insert_pages(). Link: https://lore.kernel.org/linux-trace-kernel/20240510140435.3550353-2-vdonnefort@google.com Acked-by: David Hildenbrand Signed-off-by: Vincent Donnefort Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ring_buffer.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 6511dc3a00da8..a1e011733f6a2 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1524,7 +1524,7 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, list_add(&bpage->list, pages); page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu), - mflags | __GFP_ZERO, + mflags | __GFP_COMP | __GFP_ZERO, cpu_buffer->buffer->subbuf_order); if (!page) goto free_pages; @@ -1609,7 +1609,7 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) cpu_buffer->reader_page = bpage; - page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL | __GFP_ZERO, + page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL | __GFP_COMP | __GFP_ZERO, cpu_buffer->buffer->subbuf_order); if (!page) goto fail_free_reader; @@ -5579,7 +5579,7 @@ ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) goto out; page = alloc_pages_node(cpu_to_node(cpu), - GFP_KERNEL | __GFP_NORETRY | __GFP_ZERO, + GFP_KERNEL | __GFP_NORETRY | __GFP_COMP | __GFP_ZERO, cpu_buffer->buffer->subbuf_order); if (!page) { kfree(bpage); From 117c39200d9d760cbd5944bb89efb7b9c51965aa Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Fri, 10 May 2024 15:04:31 +0100 Subject: [PATCH 1601/1702] ring-buffer: Introducing ring-buffer mapping functions In preparation for allowing the user-space to map a ring-buffer, add a set of mapping functions: ring_buffer_{map,unmap}() And controls on the ring-buffer: ring_buffer_map_get_reader() /* swap reader and head */ Mapping the ring-buffer also involves: A unique ID for each subbuf of the ring-buffer, currently they are only identified through their in-kernel VA. A meta-page, where are stored ring-buffer statistics and a description for the current reader The linear mapping exposes the meta-page, and each subbuf of the ring-buffer, ordered following their unique ID, assigned during the first mapping. Once mapped, no subbuf can get in or out of the ring-buffer: the buffer size will remain unmodified and the splice enabling functions will in reality simply memcpy the data instead of swapping subbufs. Link: https://lore.kernel.org/linux-trace-kernel/20240510140435.3550353-3-vdonnefort@google.com CC: Signed-off-by: Vincent Donnefort Acked-by: David Hildenbrand Signed-off-by: Steven Rostedt (Google) --- include/linux/ring_buffer.h | 6 + include/uapi/linux/trace_mmap.h | 46 ++++ kernel/trace/ring_buffer.c | 414 +++++++++++++++++++++++++++++++- 3 files changed, 463 insertions(+), 3 deletions(-) create mode 100644 include/uapi/linux/trace_mmap.h diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index dc5ae4e96aee0..96d2140b471ed 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -6,6 +6,8 @@ #include #include +#include + struct trace_buffer; struct ring_buffer_iter; @@ -223,4 +225,8 @@ int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node); #define trace_rb_cpu_prepare NULL #endif +int ring_buffer_map(struct trace_buffer *buffer, int cpu, + struct vm_area_struct *vma); +int ring_buffer_unmap(struct trace_buffer *buffer, int cpu); +int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu); #endif /* _LINUX_RING_BUFFER_H */ diff --git a/include/uapi/linux/trace_mmap.h b/include/uapi/linux/trace_mmap.h new file mode 100644 index 0000000000000..b682e9925539d --- /dev/null +++ b/include/uapi/linux/trace_mmap.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _TRACE_MMAP_H_ +#define _TRACE_MMAP_H_ + +#include + +/** + * struct trace_buffer_meta - Ring-buffer Meta-page description + * @meta_page_size: Size of this meta-page. + * @meta_struct_len: Size of this structure. + * @subbuf_size: Size of each sub-buffer. + * @nr_subbufs: Number of subbfs in the ring-buffer, including the reader. + * @reader.lost_events: Number of events lost at the time of the reader swap. + * @reader.id: subbuf ID of the current reader. ID range [0 : @nr_subbufs - 1] + * @reader.read: Number of bytes read on the reader subbuf. + * @flags: Placeholder for now, 0 until new features are supported. + * @entries: Number of entries in the ring-buffer. + * @overrun: Number of entries lost in the ring-buffer. + * @read: Number of entries that have been read. + * @Reserved1: Internal use only. + * @Reserved2: Internal use only. + */ +struct trace_buffer_meta { + __u32 meta_page_size; + __u32 meta_struct_len; + + __u32 subbuf_size; + __u32 nr_subbufs; + + struct { + __u64 lost_events; + __u32 id; + __u32 read; + } reader; + + __u64 flags; + + __u64 entries; + __u64 overrun; + __u64 read; + + __u64 Reserved1; + __u64 Reserved2; +}; + +#endif /* _TRACE_MMAP_H_ */ diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index a1e011733f6a2..cb303052bff22 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -26,6 +27,7 @@ #include #include #include +#include #include #include @@ -338,6 +340,7 @@ struct buffer_page { local_t entries; /* entries on this page */ unsigned long real_end; /* real end of data */ unsigned order; /* order of the page */ + u32 id; /* ID for external mapping */ struct buffer_data_page *page; /* Actual data page */ }; @@ -484,6 +487,12 @@ struct ring_buffer_per_cpu { u64 read_stamp; /* pages removed since last reset */ unsigned long pages_removed; + + unsigned int mapped; + struct mutex mapping_lock; + unsigned long *subbuf_ids; /* ID to subbuf VA */ + struct trace_buffer_meta *meta_page; + /* ring buffer pages to update, > 0 to add, < 0 to remove */ long nr_pages_to_update; struct list_head new_pages; /* new pages to add */ @@ -1599,6 +1608,7 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) init_irq_work(&cpu_buffer->irq_work.work, rb_wake_up_waiters); init_waitqueue_head(&cpu_buffer->irq_work.waiters); init_waitqueue_head(&cpu_buffer->irq_work.full_waiters); + mutex_init(&cpu_buffer->mapping_lock); bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); @@ -1789,8 +1799,6 @@ bool ring_buffer_time_stamp_abs(struct trace_buffer *buffer) return buffer->time_stamp_abs; } -static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer); - static inline unsigned long rb_page_entries(struct buffer_page *bpage) { return local_read(&bpage->entries) & RB_WRITE_MASK; @@ -5211,6 +5219,22 @@ static void rb_clear_buffer_page(struct buffer_page *page) page->read = 0; } +static void rb_update_meta_page(struct ring_buffer_per_cpu *cpu_buffer) +{ + struct trace_buffer_meta *meta = cpu_buffer->meta_page; + + meta->reader.read = cpu_buffer->reader_page->read; + meta->reader.id = cpu_buffer->reader_page->id; + meta->reader.lost_events = cpu_buffer->lost_events; + + meta->entries = local_read(&cpu_buffer->entries); + meta->overrun = local_read(&cpu_buffer->overrun); + meta->read = cpu_buffer->read; + + /* Some archs do not have data cache coherency between kernel and user-space */ + flush_dcache_folio(virt_to_folio(cpu_buffer->meta_page)); +} + static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer) { @@ -5255,6 +5279,9 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer) cpu_buffer->lost_events = 0; cpu_buffer->last_overrun = 0; + if (cpu_buffer->mapped) + rb_update_meta_page(cpu_buffer); + rb_head_page_activate(cpu_buffer); cpu_buffer->pages_removed = 0; } @@ -5469,6 +5496,12 @@ int ring_buffer_swap_cpu(struct trace_buffer *buffer_a, cpu_buffer_a = buffer_a->buffers[cpu]; cpu_buffer_b = buffer_b->buffers[cpu]; + /* It's up to the callers to not try to swap mapped buffers */ + if (WARN_ON_ONCE(cpu_buffer_a->mapped || cpu_buffer_b->mapped)) { + ret = -EBUSY; + goto out; + } + /* At least make sure the two buffers are somewhat the same */ if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages) goto out; @@ -5733,7 +5766,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * Otherwise, we can simply swap the page with the one passed in. */ if (read || (len < (commit - read)) || - cpu_buffer->reader_page == cpu_buffer->commit_page) { + cpu_buffer->reader_page == cpu_buffer->commit_page || + cpu_buffer->mapped) { struct buffer_data_page *rpage = cpu_buffer->reader_page->page; unsigned int rpos = read; unsigned int pos = 0; @@ -5956,6 +5990,11 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) cpu_buffer = buffer->buffers[cpu]; + if (cpu_buffer->mapped) { + err = -EBUSY; + goto error; + } + /* Update the number of pages to match the new size */ nr_pages = old_size * buffer->buffers[cpu]->nr_pages; nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size); @@ -6057,6 +6096,375 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) } EXPORT_SYMBOL_GPL(ring_buffer_subbuf_order_set); +static int rb_alloc_meta_page(struct ring_buffer_per_cpu *cpu_buffer) +{ + struct page *page; + + if (cpu_buffer->meta_page) + return 0; + + page = alloc_page(GFP_USER | __GFP_ZERO); + if (!page) + return -ENOMEM; + + cpu_buffer->meta_page = page_to_virt(page); + + return 0; +} + +static void rb_free_meta_page(struct ring_buffer_per_cpu *cpu_buffer) +{ + unsigned long addr = (unsigned long)cpu_buffer->meta_page; + + free_page(addr); + cpu_buffer->meta_page = NULL; +} + +static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer, + unsigned long *subbuf_ids) +{ + struct trace_buffer_meta *meta = cpu_buffer->meta_page; + unsigned int nr_subbufs = cpu_buffer->nr_pages + 1; + struct buffer_page *first_subbuf, *subbuf; + int id = 0; + + subbuf_ids[id] = (unsigned long)cpu_buffer->reader_page->page; + cpu_buffer->reader_page->id = id++; + + first_subbuf = subbuf = rb_set_head_page(cpu_buffer); + do { + if (WARN_ON(id >= nr_subbufs)) + break; + + subbuf_ids[id] = (unsigned long)subbuf->page; + subbuf->id = id; + + rb_inc_page(&subbuf); + id++; + } while (subbuf != first_subbuf); + + /* install subbuf ID to kern VA translation */ + cpu_buffer->subbuf_ids = subbuf_ids; + + meta->meta_page_size = PAGE_SIZE; + meta->meta_struct_len = sizeof(*meta); + meta->nr_subbufs = nr_subbufs; + meta->subbuf_size = cpu_buffer->buffer->subbuf_size + BUF_PAGE_HDR_SIZE; + + rb_update_meta_page(cpu_buffer); +} + +static struct ring_buffer_per_cpu * +rb_get_mapped_buffer(struct trace_buffer *buffer, int cpu) +{ + struct ring_buffer_per_cpu *cpu_buffer; + + if (!cpumask_test_cpu(cpu, buffer->cpumask)) + return ERR_PTR(-EINVAL); + + cpu_buffer = buffer->buffers[cpu]; + + mutex_lock(&cpu_buffer->mapping_lock); + + if (!cpu_buffer->mapped) { + mutex_unlock(&cpu_buffer->mapping_lock); + return ERR_PTR(-ENODEV); + } + + return cpu_buffer; +} + +static void rb_put_mapped_buffer(struct ring_buffer_per_cpu *cpu_buffer) +{ + mutex_unlock(&cpu_buffer->mapping_lock); +} + +/* + * Fast-path for rb_buffer_(un)map(). Called whenever the meta-page doesn't need + * to be set-up or torn-down. + */ +static int __rb_inc_dec_mapped(struct ring_buffer_per_cpu *cpu_buffer, + bool inc) +{ + unsigned long flags; + + lockdep_assert_held(&cpu_buffer->mapping_lock); + + if (inc && cpu_buffer->mapped == UINT_MAX) + return -EBUSY; + + if (WARN_ON(!inc && cpu_buffer->mapped == 0)) + return -EINVAL; + + mutex_lock(&cpu_buffer->buffer->mutex); + raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); + + if (inc) + cpu_buffer->mapped++; + else + cpu_buffer->mapped--; + + raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); + mutex_unlock(&cpu_buffer->buffer->mutex); + + return 0; +} + +/* + * +--------------+ pgoff == 0 + * | meta page | + * +--------------+ pgoff == 1 + * | subbuffer 0 | + * | | + * +--------------+ pgoff == (1 + (1 << subbuf_order)) + * | subbuffer 1 | + * | | + * ... + */ +#ifdef CONFIG_MMU +static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer, + struct vm_area_struct *vma) +{ + unsigned long nr_subbufs, nr_pages, vma_pages, pgoff = vma->vm_pgoff; + unsigned int subbuf_pages, subbuf_order; + struct page **pages; + int p = 0, s = 0; + int err; + + /* Refuse MP_PRIVATE or writable mappings */ + if (vma->vm_flags & VM_WRITE || vma->vm_flags & VM_EXEC || + !(vma->vm_flags & VM_MAYSHARE)) + return -EPERM; + + /* + * Make sure the mapping cannot become writable later. Also tell the VM + * to not touch these pages (VM_DONTCOPY | VM_DONTEXPAND). + */ + vm_flags_mod(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP, + VM_MAYWRITE); + + lockdep_assert_held(&cpu_buffer->mapping_lock); + + subbuf_order = cpu_buffer->buffer->subbuf_order; + subbuf_pages = 1 << subbuf_order; + + nr_subbufs = cpu_buffer->nr_pages + 1; /* + reader-subbuf */ + nr_pages = ((nr_subbufs) << subbuf_order) - pgoff + 1; /* + meta-page */ + + vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + if (!vma_pages || vma_pages > nr_pages) + return -EINVAL; + + nr_pages = vma_pages; + + pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL); + if (!pages) + return -ENOMEM; + + if (!pgoff) { + pages[p++] = virt_to_page(cpu_buffer->meta_page); + + /* + * TODO: Align sub-buffers on their size, once + * vm_insert_pages() supports the zero-page. + */ + } else { + /* Skip the meta-page */ + pgoff--; + + if (pgoff % subbuf_pages) { + err = -EINVAL; + goto out; + } + + s += pgoff / subbuf_pages; + } + + while (p < nr_pages) { + struct page *page = virt_to_page(cpu_buffer->subbuf_ids[s]); + int off = 0; + + if (WARN_ON_ONCE(s >= nr_subbufs)) { + err = -EINVAL; + goto out; + } + + for (; off < (1 << (subbuf_order)); off++, page++) { + if (p >= nr_pages) + break; + + pages[p++] = page; + } + s++; + } + + err = vm_insert_pages(vma, vma->vm_start, pages, &nr_pages); + +out: + kfree(pages); + + return err; +} +#else +static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer, + struct vm_area_struct *vma) +{ + return -EOPNOTSUPP; +} +#endif + +int ring_buffer_map(struct trace_buffer *buffer, int cpu, + struct vm_area_struct *vma) +{ + struct ring_buffer_per_cpu *cpu_buffer; + unsigned long flags, *subbuf_ids; + int err = 0; + + if (!cpumask_test_cpu(cpu, buffer->cpumask)) + return -EINVAL; + + cpu_buffer = buffer->buffers[cpu]; + + mutex_lock(&cpu_buffer->mapping_lock); + + if (cpu_buffer->mapped) { + err = __rb_map_vma(cpu_buffer, vma); + if (!err) + err = __rb_inc_dec_mapped(cpu_buffer, true); + mutex_unlock(&cpu_buffer->mapping_lock); + return err; + } + + /* prevent another thread from changing buffer/sub-buffer sizes */ + mutex_lock(&buffer->mutex); + + err = rb_alloc_meta_page(cpu_buffer); + if (err) + goto unlock; + + /* subbuf_ids include the reader while nr_pages does not */ + subbuf_ids = kcalloc(cpu_buffer->nr_pages + 1, sizeof(*subbuf_ids), GFP_KERNEL); + if (!subbuf_ids) { + rb_free_meta_page(cpu_buffer); + err = -ENOMEM; + goto unlock; + } + + atomic_inc(&cpu_buffer->resize_disabled); + + /* + * Lock all readers to block any subbuf swap until the subbuf IDs are + * assigned. + */ + raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); + rb_setup_ids_meta_page(cpu_buffer, subbuf_ids); + raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); + + err = __rb_map_vma(cpu_buffer, vma); + if (!err) { + raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); + cpu_buffer->mapped = 1; + raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); + } else { + kfree(cpu_buffer->subbuf_ids); + cpu_buffer->subbuf_ids = NULL; + rb_free_meta_page(cpu_buffer); + } + +unlock: + mutex_unlock(&buffer->mutex); + mutex_unlock(&cpu_buffer->mapping_lock); + + return err; +} + +int ring_buffer_unmap(struct trace_buffer *buffer, int cpu) +{ + struct ring_buffer_per_cpu *cpu_buffer; + unsigned long flags; + int err = 0; + + if (!cpumask_test_cpu(cpu, buffer->cpumask)) + return -EINVAL; + + cpu_buffer = buffer->buffers[cpu]; + + mutex_lock(&cpu_buffer->mapping_lock); + + if (!cpu_buffer->mapped) { + err = -ENODEV; + goto out; + } else if (cpu_buffer->mapped > 1) { + __rb_inc_dec_mapped(cpu_buffer, false); + goto out; + } + + mutex_lock(&buffer->mutex); + raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); + + cpu_buffer->mapped = 0; + + raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); + + kfree(cpu_buffer->subbuf_ids); + cpu_buffer->subbuf_ids = NULL; + rb_free_meta_page(cpu_buffer); + atomic_dec(&cpu_buffer->resize_disabled); + + mutex_unlock(&buffer->mutex); + +out: + mutex_unlock(&cpu_buffer->mapping_lock); + + return err; +} + +int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu) +{ + struct ring_buffer_per_cpu *cpu_buffer; + unsigned long reader_size; + unsigned long flags; + + cpu_buffer = rb_get_mapped_buffer(buffer, cpu); + if (IS_ERR(cpu_buffer)) + return (int)PTR_ERR(cpu_buffer); + + raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); + +consume: + if (rb_per_cpu_empty(cpu_buffer)) + goto out; + + reader_size = rb_page_size(cpu_buffer->reader_page); + + /* + * There are data to be read on the current reader page, we can + * return to the caller. But before that, we assume the latter will read + * everything. Let's update the kernel reader accordingly. + */ + if (cpu_buffer->reader_page->read < reader_size) { + while (cpu_buffer->reader_page->read < reader_size) + rb_advance_reader(cpu_buffer); + goto out; + } + + if (WARN_ON(!rb_get_reader_page(cpu_buffer))) + goto out; + + goto consume; + +out: + /* Some archs do not have data cache coherency between kernel and user-space */ + flush_dcache_folio(virt_to_folio(cpu_buffer->reader_page->page)); + + rb_update_meta_page(cpu_buffer); + + raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); + rb_put_mapped_buffer(cpu_buffer); + + return 0; +} + /* * We only allocate new buffers, never free them if the CPU goes down. * If we were to free the buffer, then the user would lose any trace that was in From cf9f0f7c4c5bb45e7bb270e48bab6f7837825a64 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Fri, 10 May 2024 15:04:32 +0100 Subject: [PATCH 1602/1702] tracing: Allow user-space mapping of the ring-buffer Currently, user-space extracts data from the ring-buffer via splice, which is handy for storage or network sharing. However, due to splice limitations, it is imposible to do real-time analysis without a copy. A solution for that problem is to let the user-space map the ring-buffer directly. The mapping is exposed via the per-CPU file trace_pipe_raw. The first element of the mapping is the meta-page. It is followed by each subbuffer constituting the ring-buffer, ordered by their unique page ID: * Meta-page -- include/uapi/linux/trace_mmap.h for a description * Subbuf ID 0 * Subbuf ID 1 ... It is therefore easy to translate a subbuf ID into an offset in the mapping: reader_id = meta->reader->id; reader_offset = meta->meta_page_size + reader_id * meta->subbuf_size; When new data is available, the mapper must call a newly introduced ioctl: TRACE_MMAP_IOCTL_GET_READER. This will update the Meta-page reader ID to point to the next reader containing unread data. Mapping will prevent snapshot and buffer size modifications. Link: https://lore.kernel.org/linux-trace-kernel/20240510140435.3550353-4-vdonnefort@google.com CC: Signed-off-by: Vincent Donnefort Signed-off-by: Steven Rostedt (Google) --- include/uapi/linux/trace_mmap.h | 2 + kernel/trace/trace.c | 104 ++++++++++++++++++++++++++++++-- kernel/trace/trace.h | 1 + 3 files changed, 102 insertions(+), 5 deletions(-) diff --git a/include/uapi/linux/trace_mmap.h b/include/uapi/linux/trace_mmap.h index b682e9925539d..bd10667542209 100644 --- a/include/uapi/linux/trace_mmap.h +++ b/include/uapi/linux/trace_mmap.h @@ -43,4 +43,6 @@ struct trace_buffer_meta { __u64 Reserved2; }; +#define TRACE_MMAP_IOCTL_GET_READER _IO('T', 0x1) + #endif /* _TRACE_MMAP_H_ */ diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 233d1af39fffe..a35e7f5982333 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -1191,6 +1191,12 @@ static void tracing_snapshot_instance_cond(struct trace_array *tr, return; } + if (tr->mapped) { + trace_array_puts(tr, "*** BUFFER MEMORY MAPPED ***\n"); + trace_array_puts(tr, "*** Can not use snapshot (sorry) ***\n"); + return; + } + local_irq_save(flags); update_max_tr(tr, current, smp_processor_id(), cond_data); local_irq_restore(flags); @@ -1323,7 +1329,7 @@ static int tracing_arm_snapshot_locked(struct trace_array *tr) lockdep_assert_held(&trace_types_lock); spin_lock(&tr->snapshot_trigger_lock); - if (tr->snapshot == UINT_MAX) { + if (tr->snapshot == UINT_MAX || tr->mapped) { spin_unlock(&tr->snapshot_trigger_lock); return -EBUSY; } @@ -6068,7 +6074,7 @@ static void tracing_set_nop(struct trace_array *tr) { if (tr->current_trace == &nop_trace) return; - + tr->current_trace->enabled--; if (tr->current_trace->reset) @@ -8194,15 +8200,32 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, return ret; } -/* An ioctl call with cmd 0 to the ring buffer file will wake up all waiters */ static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct ftrace_buffer_info *info = file->private_data; struct trace_iterator *iter = &info->iter; + int err; + + if (cmd == TRACE_MMAP_IOCTL_GET_READER) { + if (!(file->f_flags & O_NONBLOCK)) { + err = ring_buffer_wait(iter->array_buffer->buffer, + iter->cpu_file, + iter->tr->buffer_percent, + NULL, NULL); + if (err) + return err; + } - if (cmd) - return -ENOIOCTLCMD; + return ring_buffer_map_get_reader(iter->array_buffer->buffer, + iter->cpu_file); + } else if (cmd) { + return -ENOTTY; + } + /* + * An ioctl call with cmd 0 to the ring buffer file will wake up all + * waiters + */ mutex_lock(&trace_types_lock); /* Make sure the waiters see the new wait_index */ @@ -8214,6 +8237,76 @@ static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned return 0; } +#ifdef CONFIG_TRACER_MAX_TRACE +static int get_snapshot_map(struct trace_array *tr) +{ + int err = 0; + + /* + * Called with mmap_lock held. lockdep would be unhappy if we would now + * take trace_types_lock. Instead use the specific + * snapshot_trigger_lock. + */ + spin_lock(&tr->snapshot_trigger_lock); + + if (tr->snapshot || tr->mapped == UINT_MAX) + err = -EBUSY; + else + tr->mapped++; + + spin_unlock(&tr->snapshot_trigger_lock); + + /* Wait for update_max_tr() to observe iter->tr->mapped */ + if (tr->mapped == 1) + synchronize_rcu(); + + return err; + +} +static void put_snapshot_map(struct trace_array *tr) +{ + spin_lock(&tr->snapshot_trigger_lock); + if (!WARN_ON(!tr->mapped)) + tr->mapped--; + spin_unlock(&tr->snapshot_trigger_lock); +} +#else +static inline int get_snapshot_map(struct trace_array *tr) { return 0; } +static inline void put_snapshot_map(struct trace_array *tr) { } +#endif + +static void tracing_buffers_mmap_close(struct vm_area_struct *vma) +{ + struct ftrace_buffer_info *info = vma->vm_file->private_data; + struct trace_iterator *iter = &info->iter; + + WARN_ON(ring_buffer_unmap(iter->array_buffer->buffer, iter->cpu_file)); + put_snapshot_map(iter->tr); +} + +static const struct vm_operations_struct tracing_buffers_vmops = { + .close = tracing_buffers_mmap_close, +}; + +static int tracing_buffers_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct ftrace_buffer_info *info = filp->private_data; + struct trace_iterator *iter = &info->iter; + int ret = 0; + + ret = get_snapshot_map(iter->tr); + if (ret) + return ret; + + ret = ring_buffer_map(iter->array_buffer->buffer, iter->cpu_file, vma); + if (ret) + put_snapshot_map(iter->tr); + + vma->vm_ops = &tracing_buffers_vmops; + + return ret; +} + static const struct file_operations tracing_buffers_fops = { .open = tracing_buffers_open, .read = tracing_buffers_read, @@ -8223,6 +8316,7 @@ static const struct file_operations tracing_buffers_fops = { .splice_read = tracing_buffers_splice_read, .unlocked_ioctl = tracing_buffers_ioctl, .llseek = no_llseek, + .mmap = tracing_buffers_mmap, }; static ssize_t diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 64450615ca0cb..749a182dab480 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -336,6 +336,7 @@ struct trace_array { bool allocated_snapshot; spinlock_t snapshot_trigger_lock; unsigned int snapshot; + unsigned int mapped; unsigned long max_latency; #ifdef CONFIG_FSNOTIFY struct dentry *d_max_latency; From a1e0dd7ce38af3fb1a3bc54a222a7c5e4eaa4202 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Fri, 10 May 2024 15:04:33 +0100 Subject: [PATCH 1603/1702] Documentation: tracing: Add ring-buffer mapping It is now possible to mmap() a ring-buffer to stream its content. Add some documentation and a code example. Link: https://lore.kernel.org/linux-trace-kernel/20240510140435.3550353-5-vdonnefort@google.com Signed-off-by: Vincent Donnefort Signed-off-by: Steven Rostedt (Google) --- Documentation/trace/index.rst | 1 + Documentation/trace/ring-buffer-map.rst | 106 ++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 Documentation/trace/ring-buffer-map.rst diff --git a/Documentation/trace/index.rst b/Documentation/trace/index.rst index 5092d6c13af5e..0b300901fd750 100644 --- a/Documentation/trace/index.rst +++ b/Documentation/trace/index.rst @@ -29,6 +29,7 @@ Linux Tracing Technologies timerlat-tracer intel_th ring-buffer-design + ring-buffer-map stm sys-t coresight/index diff --git a/Documentation/trace/ring-buffer-map.rst b/Documentation/trace/ring-buffer-map.rst new file mode 100644 index 0000000000000..8e296bcc0d7f3 --- /dev/null +++ b/Documentation/trace/ring-buffer-map.rst @@ -0,0 +1,106 @@ +.. SPDX-License-Identifier: GPL-2.0 + +================================== +Tracefs ring-buffer memory mapping +================================== + +:Author: Vincent Donnefort + +Overview +======== +Tracefs ring-buffer memory map provides an efficient method to stream data +as no memory copy is necessary. The application mapping the ring-buffer becomes +then a consumer for that ring-buffer, in a similar fashion to trace_pipe. + +Memory mapping setup +==================== +The mapping works with a mmap() of the trace_pipe_raw interface. + +The first system page of the mapping contains ring-buffer statistics and +description. It is referred to as the meta-page. One of the most important +fields of the meta-page is the reader. It contains the sub-buffer ID which can +be safely read by the mapper (see ring-buffer-design.rst). + +The meta-page is followed by all the sub-buffers, ordered by ascending ID. It is +therefore effortless to know where the reader starts in the mapping: + +.. code-block:: c + + reader_id = meta->reader->id; + reader_offset = meta->meta_page_size + reader_id * meta->subbuf_size; + +When the application is done with the current reader, it can get a new one using +the trace_pipe_raw ioctl() TRACE_MMAP_IOCTL_GET_READER. This ioctl also updates +the meta-page fields. + +Limitations +=========== +When a mapping is in place on a Tracefs ring-buffer, it is not possible to +either resize it (either by increasing the entire size of the ring-buffer or +each subbuf). It is also not possible to use snapshot and causes splice to copy +the ring buffer data instead of using the copyless swap from the ring buffer. + +Concurrent readers (either another application mapping that ring-buffer or the +kernel with trace_pipe) are allowed but not recommended. They will compete for +the ring-buffer and the output is unpredictable, just like concurrent readers on +trace_pipe would be. + +Example +======= + +.. code-block:: c + + #include + #include + #include + #include + + #include + + #include + #include + + #define TRACE_PIPE_RAW "/sys/kernel/tracing/per_cpu/cpu0/trace_pipe_raw" + + int main(void) + { + int page_size = getpagesize(), fd, reader_id; + unsigned long meta_len, data_len; + struct trace_buffer_meta *meta; + void *map, *reader, *data; + + fd = open(TRACE_PIPE_RAW, O_RDONLY | O_NONBLOCK); + if (fd < 0) + exit(EXIT_FAILURE); + + map = mmap(NULL, page_size, PROT_READ, MAP_SHARED, fd, 0); + if (map == MAP_FAILED) + exit(EXIT_FAILURE); + + meta = (struct trace_buffer_meta *)map; + meta_len = meta->meta_page_size; + + printf("entries: %llu\n", meta->entries); + printf("overrun: %llu\n", meta->overrun); + printf("read: %llu\n", meta->read); + printf("nr_subbufs: %u\n", meta->nr_subbufs); + + data_len = meta->subbuf_size * meta->nr_subbufs; + data = mmap(NULL, data_len, PROT_READ, MAP_SHARED, fd, meta_len); + if (data == MAP_FAILED) + exit(EXIT_FAILURE); + + if (ioctl(fd, TRACE_MMAP_IOCTL_GET_READER) < 0) + exit(EXIT_FAILURE); + + reader_id = meta->reader.id; + reader = data + meta->subbuf_size * reader_id; + + printf("Current reader address: %p\n", reader); + + munmap(data, data_len); + munmap(meta, meta_len); + close (fd); + + return 0; + } From 75961e55415cdc368ca2d7f6203fb627c259d58a Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Fri, 10 May 2024 15:04:34 +0100 Subject: [PATCH 1604/1702] ring-buffer/selftest: Add ring-buffer mapping test Map a ring-buffer, validate the meta-page before and after emitting few events. Also check ring-buffer mapping boundaries and finally ensure the tracing snapshot is mutually exclusive. Link: https://lore.kernel.org/linux-trace-kernel/20240510140435.3550353-6-vdonnefort@google.com Cc: Shuah Khan Cc: Shuah Khan Cc: linux-kselftest@vger.kernel.org Acked-by: Muhammad Usama Anjum Signed-off-by: Vincent Donnefort Signed-off-by: Steven Rostedt (Google) --- .../testing/selftests/ring-buffer/.gitignore | 1 + tools/testing/selftests/ring-buffer/Makefile | 8 + tools/testing/selftests/ring-buffer/config | 2 + .../testing/selftests/ring-buffer/map_test.c | 294 ++++++++++++++++++ 4 files changed, 305 insertions(+) create mode 100644 tools/testing/selftests/ring-buffer/.gitignore create mode 100644 tools/testing/selftests/ring-buffer/Makefile create mode 100644 tools/testing/selftests/ring-buffer/config create mode 100644 tools/testing/selftests/ring-buffer/map_test.c diff --git a/tools/testing/selftests/ring-buffer/.gitignore b/tools/testing/selftests/ring-buffer/.gitignore new file mode 100644 index 0000000000000..3aed1a2a6c678 --- /dev/null +++ b/tools/testing/selftests/ring-buffer/.gitignore @@ -0,0 +1 @@ +map_test diff --git a/tools/testing/selftests/ring-buffer/Makefile b/tools/testing/selftests/ring-buffer/Makefile new file mode 100644 index 0000000000000..627c5fa6d1abe --- /dev/null +++ b/tools/testing/selftests/ring-buffer/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 +CFLAGS += -Wl,-no-as-needed -Wall +CFLAGS += $(KHDR_INCLUDES) +CFLAGS += -D_GNU_SOURCE + +TEST_GEN_PROGS = map_test + +include ../lib.mk diff --git a/tools/testing/selftests/ring-buffer/config b/tools/testing/selftests/ring-buffer/config new file mode 100644 index 0000000000000..d936f8f00e787 --- /dev/null +++ b/tools/testing/selftests/ring-buffer/config @@ -0,0 +1,2 @@ +CONFIG_FTRACE=y +CONFIG_TRACER_SNAPSHOT=y diff --git a/tools/testing/selftests/ring-buffer/map_test.c b/tools/testing/selftests/ring-buffer/map_test.c new file mode 100644 index 0000000000000..a9006fa7097ed --- /dev/null +++ b/tools/testing/selftests/ring-buffer/map_test.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Ring-buffer memory mapping tests + * + * Copyright (c) 2024 Vincent Donnefort + */ +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "../user_events/user_events_selftests.h" /* share tracefs setup */ +#include "../kselftest_harness.h" + +#define TRACEFS_ROOT "/sys/kernel/tracing" + +static int __tracefs_write(const char *path, const char *value) +{ + int fd, ret; + + fd = open(path, O_WRONLY | O_TRUNC); + if (fd < 0) + return fd; + + ret = write(fd, value, strlen(value)); + + close(fd); + + return ret == -1 ? -errno : 0; +} + +static int __tracefs_write_int(const char *path, int value) +{ + char *str; + int ret; + + if (asprintf(&str, "%d", value) < 0) + return -1; + + ret = __tracefs_write(path, str); + + free(str); + + return ret; +} + +#define tracefs_write_int(path, value) \ + ASSERT_EQ(__tracefs_write_int((path), (value)), 0) + +#define tracefs_write(path, value) \ + ASSERT_EQ(__tracefs_write((path), (value)), 0) + +static int tracefs_reset(void) +{ + if (__tracefs_write_int(TRACEFS_ROOT"/tracing_on", 0)) + return -1; + if (__tracefs_write(TRACEFS_ROOT"/trace", "")) + return -1; + if (__tracefs_write(TRACEFS_ROOT"/set_event", "")) + return -1; + if (__tracefs_write(TRACEFS_ROOT"/current_tracer", "nop")) + return -1; + + return 0; +} + +struct tracefs_cpu_map_desc { + struct trace_buffer_meta *meta; + int cpu_fd; +}; + +int tracefs_cpu_map(struct tracefs_cpu_map_desc *desc, int cpu) +{ + int page_size = getpagesize(); + char *cpu_path; + void *map; + + if (asprintf(&cpu_path, + TRACEFS_ROOT"/per_cpu/cpu%d/trace_pipe_raw", + cpu) < 0) + return -ENOMEM; + + desc->cpu_fd = open(cpu_path, O_RDONLY | O_NONBLOCK); + free(cpu_path); + if (desc->cpu_fd < 0) + return -ENODEV; + + map = mmap(NULL, page_size, PROT_READ, MAP_SHARED, desc->cpu_fd, 0); + if (map == MAP_FAILED) + return -errno; + + desc->meta = (struct trace_buffer_meta *)map; + + return 0; +} + +void tracefs_cpu_unmap(struct tracefs_cpu_map_desc *desc) +{ + munmap(desc->meta, desc->meta->meta_page_size); + close(desc->cpu_fd); +} + +FIXTURE(map) { + struct tracefs_cpu_map_desc map_desc; + bool umount; +}; + +FIXTURE_VARIANT(map) { + int subbuf_size; +}; + +FIXTURE_VARIANT_ADD(map, subbuf_size_4k) { + .subbuf_size = 4, +}; + +FIXTURE_VARIANT_ADD(map, subbuf_size_8k) { + .subbuf_size = 8, +}; + +FIXTURE_SETUP(map) +{ + int cpu = sched_getcpu(); + cpu_set_t cpu_mask; + bool fail, umount; + char *message; + + if (getuid() != 0) + SKIP(return, "Skipping: %s", "Please run the test as root"); + + if (!tracefs_enabled(&message, &fail, &umount)) { + if (fail) { + TH_LOG("Tracefs setup failed: %s", message); + ASSERT_FALSE(fail); + } + SKIP(return, "Skipping: %s", message); + } + + self->umount = umount; + + ASSERT_GE(cpu, 0); + + ASSERT_EQ(tracefs_reset(), 0); + + tracefs_write_int(TRACEFS_ROOT"/buffer_subbuf_size_kb", variant->subbuf_size); + + ASSERT_EQ(tracefs_cpu_map(&self->map_desc, cpu), 0); + + /* + * Ensure generated events will be found on this very same ring-buffer. + */ + CPU_ZERO(&cpu_mask); + CPU_SET(cpu, &cpu_mask); + ASSERT_EQ(sched_setaffinity(0, sizeof(cpu_mask), &cpu_mask), 0); +} + +FIXTURE_TEARDOWN(map) +{ + tracefs_reset(); + + if (self->umount) + tracefs_unmount(); + + tracefs_cpu_unmap(&self->map_desc); +} + +TEST_F(map, meta_page_check) +{ + struct tracefs_cpu_map_desc *desc = &self->map_desc; + int cnt = 0; + + ASSERT_EQ(desc->meta->entries, 0); + ASSERT_EQ(desc->meta->overrun, 0); + ASSERT_EQ(desc->meta->read, 0); + + ASSERT_EQ(desc->meta->reader.id, 0); + ASSERT_EQ(desc->meta->reader.read, 0); + + ASSERT_EQ(ioctl(desc->cpu_fd, TRACE_MMAP_IOCTL_GET_READER), 0); + ASSERT_EQ(desc->meta->reader.id, 0); + + tracefs_write_int(TRACEFS_ROOT"/tracing_on", 1); + for (int i = 0; i < 16; i++) + tracefs_write_int(TRACEFS_ROOT"/trace_marker", i); +again: + ASSERT_EQ(ioctl(desc->cpu_fd, TRACE_MMAP_IOCTL_GET_READER), 0); + + ASSERT_EQ(desc->meta->entries, 16); + ASSERT_EQ(desc->meta->overrun, 0); + ASSERT_EQ(desc->meta->read, 16); + + ASSERT_EQ(desc->meta->reader.id, 1); + + if (!(cnt++)) + goto again; +} + +TEST_F(map, data_mmap) +{ + struct tracefs_cpu_map_desc *desc = &self->map_desc; + unsigned long meta_len, data_len; + void *data; + + meta_len = desc->meta->meta_page_size; + data_len = desc->meta->subbuf_size * desc->meta->nr_subbufs; + + /* Map all the available subbufs */ + data = mmap(NULL, data_len, PROT_READ, MAP_SHARED, + desc->cpu_fd, meta_len); + ASSERT_NE(data, MAP_FAILED); + munmap(data, data_len); + + /* Map all the available subbufs - 1 */ + data_len -= desc->meta->subbuf_size; + data = mmap(NULL, data_len, PROT_READ, MAP_SHARED, + desc->cpu_fd, meta_len); + ASSERT_NE(data, MAP_FAILED); + munmap(data, data_len); + + /* Overflow the available subbufs by 1 */ + meta_len += desc->meta->subbuf_size * 2; + data = mmap(NULL, data_len, PROT_READ, MAP_SHARED, + desc->cpu_fd, meta_len); + ASSERT_EQ(data, MAP_FAILED); +} + +FIXTURE(snapshot) { + bool umount; +}; + +FIXTURE_SETUP(snapshot) +{ + bool fail, umount; + struct stat sb; + char *message; + + if (getuid() != 0) + SKIP(return, "Skipping: %s", "Please run the test as root"); + + if (stat(TRACEFS_ROOT"/snapshot", &sb)) + SKIP(return, "Skipping: %s", "snapshot not available"); + + if (!tracefs_enabled(&message, &fail, &umount)) { + if (fail) { + TH_LOG("Tracefs setup failed: %s", message); + ASSERT_FALSE(fail); + } + SKIP(return, "Skipping: %s", message); + } + + self->umount = umount; +} + +FIXTURE_TEARDOWN(snapshot) +{ + __tracefs_write(TRACEFS_ROOT"/events/sched/sched_switch/trigger", + "!snapshot"); + tracefs_reset(); + + if (self->umount) + tracefs_unmount(); +} + +TEST_F(snapshot, excludes_map) +{ + struct tracefs_cpu_map_desc map_desc; + int cpu = sched_getcpu(); + + ASSERT_GE(cpu, 0); + tracefs_write(TRACEFS_ROOT"/events/sched/sched_switch/trigger", + "snapshot"); + ASSERT_EQ(tracefs_cpu_map(&map_desc, cpu), -EBUSY); +} + +TEST_F(snapshot, excluded_by_map) +{ + struct tracefs_cpu_map_desc map_desc; + int cpu = sched_getcpu(); + + ASSERT_EQ(tracefs_cpu_map(&map_desc, cpu), 0); + + ASSERT_EQ(__tracefs_write(TRACEFS_ROOT"/events/sched/sched_switch/trigger", + "snapshot"), -EBUSY); + ASSERT_EQ(__tracefs_write(TRACEFS_ROOT"/snapshot", + "1"), -EBUSY); +} + +TEST_HARNESS_MAIN From c5963a0990d1da70d1bd399e6811887ff2231b0d Mon Sep 17 00:00:00 2001 From: Yuran Pereira Date: Mon, 20 Nov 2023 05:46:13 +0530 Subject: [PATCH 1605/1702] ftrace: Replaces simple_strtoul in ftrace The function simple_strtoul performs no error checking in scenarios where the input value overflows the intended output variable. This results in this function successfully returning, even when the output does not match the input string (aka the function returns successfully even when the result is wrong). Or as it was mentioned [1], "...simple_strtol(), simple_strtoll(), simple_strtoul(), and simple_strtoull() functions explicitly ignore overflows, which may lead to unexpected results in callers." Hence, the use of those functions is discouraged. This patch replaces all uses of the simple_strtoul with the safer alternatives kstrtoul and kstruint. Callers affected: - add_rec_by_index - set_graph_max_depth_function Side effects of this patch: - Since `fgraph_max_depth` is an `unsigned int`, this patch uses kstrtouint instead of kstrtoul to avoid any compiler warnings that could originate from calling the latter. - This patch ensures that the callers of kstrtou* return accordingly when kstrtoul and kstruint fail for some reason. In this case, both callers this patch is addressing return 0 on error. [1] https://www.kernel.org/doc/html/latest/process/deprecated.html#simple-strtol-simple-strtoll-simple-strtoul-simple-strtoull Link: https://lore.kernel.org/linux-trace-kernel/GV1PR10MB656333529A8D7B8AFB28D238E8B4A@GV1PR10MB6563.EURPRD10.PROD.OUTLOOK.COM Signed-off-by: Yuran Pereira Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ftrace.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index da1710499698b..3951c0d0ec631 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -4202,12 +4202,12 @@ static int add_rec_by_index(struct ftrace_hash *hash, struct ftrace_glob *func_g, int clear_filter) { - long index = simple_strtoul(func_g->search, NULL, 0); + long index; struct ftrace_page *pg; struct dyn_ftrace *rec; /* The index starts at 1 */ - if (--index < 0) + if (kstrtoul(func_g->search, 0, &index) || --index < 0) return 0; do_for_each_ftrace_rec(pg, rec) { @@ -5817,9 +5817,8 @@ __setup("ftrace_graph_notrace=", set_graph_notrace_function); static int __init set_graph_max_depth_function(char *str) { - if (!str) + if (!str || kstrtouint(str, 0, &fgraph_max_depth)) return 0; - fgraph_max_depth = simple_strtoul(str, NULL, 0); return 1; } __setup("ftrace_graph_max_depth=", set_graph_max_depth_function); From 33f137143e651321f10eb67ae6404a13bfbf69f8 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 1 May 2024 16:12:37 -0700 Subject: [PATCH 1606/1702] ftrace: Use asynchronous grace period for register_ftrace_direct() When running heavy test workloads with KASAN enabled, RCU Tasks grace periods can extend for many tens of seconds, significantly slowing trace registration. Therefore, make the registration-side RCU Tasks grace period be asynchronous via call_rcu_tasks(). Link: https://lore.kernel.org/linux-trace-kernel/ac05be77-2972-475b-9b57-56bef15aa00a@paulmck-laptop Reported-by: Jakub Kicinski Reported-by: Alexei Starovoitov Reported-by: Chris Mason Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Paul E. McKenney Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ftrace.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 3951c0d0ec631..f9223513414a3 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -5366,6 +5366,13 @@ static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long } } +static void register_ftrace_direct_cb(struct rcu_head *rhp) +{ + struct ftrace_hash *fhp = container_of(rhp, struct ftrace_hash, rcu); + + free_ftrace_hash(fhp); +} + /** * register_ftrace_direct - Call a custom trampoline directly * for multiple functions registered in @ops @@ -5464,10 +5471,8 @@ int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) out_unlock: mutex_unlock(&direct_mutex); - if (free_hash && free_hash != EMPTY_HASH) { - synchronize_rcu_tasks(); - free_ftrace_hash(free_hash); - } + if (free_hash && free_hash != EMPTY_HASH) + call_rcu_tasks(&free_hash->rcu, register_ftrace_direct_cb); if (new_hash) free_ftrace_hash(new_hash); From fe832be05a8eee5f1488cbcc2c562dd82d079fd6 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Tue, 12 Mar 2024 17:54:05 -0400 Subject: [PATCH 1607/1702] ring-buffer: Have mmapped ring buffer keep track of missed events While testing libtracefs on the mmapped ring buffer, the test that checks if missed events are accounted for failed when using the mapped buffer. This is because the mapped page does not update the missed events that were dropped because the writer filled up the ring buffer before the reader could catch it. Add the missed events to the reader page/sub-buffer when the IOCTL is done and a new reader page is acquired. Note that all accesses to the reader_page via rb_page_commit() had to be switched to rb_page_size(), and rb_page_size() which was just a copy of rb_page_commit() but now it masks out the RB_MISSED bits. This is needed as the mapped reader page is still active in the ring buffer code and where it reads the commit field of the bpage for the size, it now must mask it otherwise the missed bits that are now set will corrupt the size returned. Link: https://lore.kernel.org/linux-trace-kernel/20240312175405.12fb6726@gandalf.local.home Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Cc: Vincent Donnefort Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ring_buffer.c | 53 +++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index cb303052bff22..a02c7a52a0f5b 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -314,6 +314,8 @@ static u64 rb_event_time_stamp(struct ring_buffer_event *event) /* Missed count stored at end */ #define RB_MISSED_STORED (1 << 30) +#define RB_MISSED_MASK (3 << 30) + struct buffer_data_page { u64 time_stamp; /* page time stamp */ local_t commit; /* write committed index */ @@ -2326,7 +2328,7 @@ rb_iter_head_event(struct ring_buffer_iter *iter) /* Size is determined by what has been committed */ static __always_inline unsigned rb_page_size(struct buffer_page *bpage) { - return rb_page_commit(bpage); + return rb_page_commit(bpage) & ~RB_MISSED_MASK; } static __always_inline unsigned @@ -3953,7 +3955,7 @@ static bool rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer) return true; /* Reader should exhaust content in reader page */ - if (reader->read != rb_page_commit(reader)) + if (reader->read != rb_page_size(reader)) return false; /* @@ -4424,7 +4426,7 @@ int ring_buffer_iter_empty(struct ring_buffer_iter *iter) return ((iter->head_page == commit_page && iter->head >= commit) || (iter->head_page == reader && commit_page == head_page && head_page->read == commit && - iter->head == rb_page_commit(cpu_buffer->reader_page))); + iter->head == rb_page_size(cpu_buffer->reader_page))); } EXPORT_SYMBOL_GPL(ring_buffer_iter_empty); @@ -5753,7 +5755,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, event = rb_reader_event(cpu_buffer); read = reader->read; - commit = rb_page_commit(reader); + commit = rb_page_size(reader); /* Check if any events were dropped */ missed_events = cpu_buffer->lost_events; @@ -5830,7 +5832,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, } else { /* update the entry counter */ cpu_buffer->read += rb_page_entries(reader); - cpu_buffer->read_bytes += rb_page_commit(reader); + cpu_buffer->read_bytes += rb_page_size(reader); /* swap the pages */ rb_init_page(bpage); @@ -6422,6 +6424,8 @@ int ring_buffer_unmap(struct trace_buffer *buffer, int cpu) int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu) { struct ring_buffer_per_cpu *cpu_buffer; + struct buffer_page *reader; + unsigned long missed_events; unsigned long reader_size; unsigned long flags; @@ -6448,9 +6452,46 @@ int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu) goto out; } - if (WARN_ON(!rb_get_reader_page(cpu_buffer))) + reader = rb_get_reader_page(cpu_buffer); + if (WARN_ON(!reader)) goto out; + /* Check if any events were dropped */ + missed_events = cpu_buffer->lost_events; + + if (cpu_buffer->reader_page != cpu_buffer->commit_page) { + if (missed_events) { + struct buffer_data_page *bpage = reader->page; + unsigned int commit; + /* + * Use the real_end for the data size, + * This gives us a chance to store the lost events + * on the page. + */ + if (reader->real_end) + local_set(&bpage->commit, reader->real_end); + /* + * If there is room at the end of the page to save the + * missed events, then record it there. + */ + commit = rb_page_size(reader); + if (buffer->subbuf_size - commit >= sizeof(missed_events)) { + memcpy(&bpage->data[commit], &missed_events, + sizeof(missed_events)); + local_add(RB_MISSED_STORED, &bpage->commit); + } + local_add(RB_MISSED_EVENTS, &bpage->commit); + } + } else { + /* + * There really shouldn't be any missed events if the commit + * is on the reader page. + */ + WARN_ON_ONCE(missed_events); + } + + cpu_buffer->lost_events = 0; + goto consume; out: From 347bd7f072ea8c36e4becf32c76ee7e96bc7b1c3 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Fri, 29 Mar 2024 17:02:30 +0100 Subject: [PATCH 1608/1702] tracing: Improve benchmark test performance by using do_div() Partially revert commit d6cb38e10810 ("tracing: Use div64_u64() instead of do_div()") and use do_div() again to utilize its faster 64-by-32 division compared to the 64-by-64 division done by div64_u64(). Explicitly cast the divisor bm_cnt to u32 to prevent a Coccinelle warning reported by do_div.cocci. The warning was removed with commit d6cb38e10810 ("tracing: Use div64_u64() instead of do_div()"). Using the faster 64-by-32 division and casting bm_cnt to u32 is safe because we return early from trace_do_benchmark() if bm_cnt > UINT_MAX. This approach is already used twice in trace_do_benchmark() when calculating the standard deviation: do_div(stddev, (u32)bm_cnt); do_div(stddev, (u32)bm_cnt - 1); Link: https://lore.kernel.org/linux-trace-kernel/20240329160229.4874-2-thorsten.blum@toblux.com Signed-off-by: Thorsten Blum Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_benchmark.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/trace/trace_benchmark.c b/kernel/trace/trace_benchmark.c index 811b08439406a..e19c32f2a9381 100644 --- a/kernel/trace/trace_benchmark.c +++ b/kernel/trace/trace_benchmark.c @@ -104,7 +104,7 @@ static void trace_do_benchmark(void) stddev = 0; delta = bm_total; - delta = div64_u64(delta, bm_cnt); + do_div(delta, (u32)bm_cnt); avg = delta; if (stddev > 0) { From 92ef0fd55ac80dfc2e4654edfe5d1ddfa6e070fe Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 9 May 2024 09:20:08 -0600 Subject: [PATCH 1609/1702] net: change proto and proto_ops accept type Rather than pass in flags, error pointer, and whether this is a kernel invocation or not, add a struct proto_accept_arg struct as the argument. This then holds all of these arguments, and prepares accept for being able to pass back more information. No functional changes in this patch. Acked-by: Jakub Kicinski Signed-off-by: Jens Axboe --- crypto/af_alg.c | 11 ++++++----- crypto/algif_hash.c | 10 +++++----- drivers/xen/pvcalls-back.c | 6 +++++- fs/ocfs2/cluster/tcp.c | 5 ++++- include/crypto/if_alg.h | 3 ++- include/linux/net.h | 4 +++- include/net/inet_common.h | 4 ++-- include/net/inet_connection_sock.h | 2 +- include/net/sock.h | 12 +++++++++--- net/atm/svc.c | 8 ++++---- net/ax25/af_ax25.c | 6 +++--- net/bluetooth/iso.c | 4 ++-- net/bluetooth/l2cap_sock.c | 4 ++-- net/bluetooth/rfcomm/sock.c | 6 +++--- net/bluetooth/sco.c | 4 ++-- net/core/sock.c | 4 ++-- net/ipv4/af_inet.c | 10 +++++----- net/ipv4/inet_connection_sock.c | 6 +++--- net/iucv/af_iucv.c | 4 ++-- net/llc/af_llc.c | 7 +++---- net/mptcp/protocol.c | 11 +++++------ net/netrom/af_netrom.c | 6 +++--- net/nfc/llcp_sock.c | 4 ++-- net/phonet/pep.c | 12 ++++++------ net/phonet/socket.c | 7 +++---- net/rds/tcp_listen.c | 6 +++++- net/rose/af_rose.c | 6 +++--- net/sctp/socket.c | 8 ++++---- net/smc/af_smc.c | 6 +++--- net/socket.c | 13 ++++++++++--- net/tipc/socket.c | 13 +++++-------- net/unix/af_unix.c | 21 ++++++++++----------- net/vmw_vsock/af_vsock.c | 6 +++--- net/x25/af_x25.c | 4 ++-- 34 files changed, 132 insertions(+), 111 deletions(-) diff --git a/crypto/af_alg.c b/crypto/af_alg.c index 5bc6d0fa7498d..18cfead0081dc 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -407,7 +407,8 @@ static int alg_setsockopt(struct socket *sock, int level, int optname, return err; } -int af_alg_accept(struct sock *sk, struct socket *newsock, bool kern) +int af_alg_accept(struct sock *sk, struct socket *newsock, + struct proto_accept_arg *arg) { struct alg_sock *ask = alg_sk(sk); const struct af_alg_type *type; @@ -422,7 +423,7 @@ int af_alg_accept(struct sock *sk, struct socket *newsock, bool kern) if (!type) goto unlock; - sk2 = sk_alloc(sock_net(sk), PF_ALG, GFP_KERNEL, &alg_proto, kern); + sk2 = sk_alloc(sock_net(sk), PF_ALG, GFP_KERNEL, &alg_proto, arg->kern); err = -ENOMEM; if (!sk2) goto unlock; @@ -468,10 +469,10 @@ int af_alg_accept(struct sock *sk, struct socket *newsock, bool kern) } EXPORT_SYMBOL_GPL(af_alg_accept); -static int alg_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern) +static int alg_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg) { - return af_alg_accept(sock->sk, newsock, kern); + return af_alg_accept(sock->sk, newsock, arg); } static const struct proto_ops alg_proto_ops = { diff --git a/crypto/algif_hash.c b/crypto/algif_hash.c index e24c829d7a015..7c7394d46a235 100644 --- a/crypto/algif_hash.c +++ b/crypto/algif_hash.c @@ -223,8 +223,8 @@ static int hash_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, return err ?: len; } -static int hash_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern) +static int hash_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg) { struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); @@ -252,7 +252,7 @@ static int hash_accept(struct socket *sock, struct socket *newsock, int flags, if (err) goto out_free_state; - err = af_alg_accept(ask->parent, newsock, kern); + err = af_alg_accept(ask->parent, newsock, arg); if (err) goto out_free_state; @@ -355,7 +355,7 @@ static int hash_recvmsg_nokey(struct socket *sock, struct msghdr *msg, } static int hash_accept_nokey(struct socket *sock, struct socket *newsock, - int flags, bool kern) + struct proto_accept_arg *arg) { int err; @@ -363,7 +363,7 @@ static int hash_accept_nokey(struct socket *sock, struct socket *newsock, if (err) return err; - return hash_accept(sock, newsock, flags, kern); + return hash_accept(sock, newsock, arg); } static struct proto_ops algif_hash_ops_nokey = { diff --git a/drivers/xen/pvcalls-back.c b/drivers/xen/pvcalls-back.c index d52593466a792..fd7ed65e0197d 100644 --- a/drivers/xen/pvcalls-back.c +++ b/drivers/xen/pvcalls-back.c @@ -517,6 +517,10 @@ static void __pvcalls_back_accept(struct work_struct *work) { struct sockpass_mapping *mappass = container_of( work, struct sockpass_mapping, register_work); + struct proto_accept_arg arg = { + .flags = O_NONBLOCK, + .kern = true, + }; struct sock_mapping *map; struct pvcalls_ioworker *iow; struct pvcalls_fedata *fedata; @@ -548,7 +552,7 @@ static void __pvcalls_back_accept(struct work_struct *work) sock->type = mappass->sock->type; sock->ops = mappass->sock->ops; - ret = inet_accept(mappass->sock, sock, O_NONBLOCK, true); + ret = inet_accept(mappass->sock, sock, &arg); if (ret == -EAGAIN) { sock_release(sock); return; diff --git a/fs/ocfs2/cluster/tcp.c b/fs/ocfs2/cluster/tcp.c index 960080753d3bd..2b8fa3e782fb6 100644 --- a/fs/ocfs2/cluster/tcp.c +++ b/fs/ocfs2/cluster/tcp.c @@ -1784,6 +1784,9 @@ static int o2net_accept_one(struct socket *sock, int *more) struct o2nm_node *node = NULL; struct o2nm_node *local_node = NULL; struct o2net_sock_container *sc = NULL; + struct proto_accept_arg arg = { + .flags = O_NONBLOCK, + }; struct o2net_node *nn; unsigned int nofs_flag; @@ -1802,7 +1805,7 @@ static int o2net_accept_one(struct socket *sock, int *more) new_sock->type = sock->type; new_sock->ops = sock->ops; - ret = sock->ops->accept(sock, new_sock, O_NONBLOCK, false); + ret = sock->ops->accept(sock, new_sock, &arg); if (ret < 0) goto out; diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h index 78ecaf5db04c6..f7b3b93f3a49a 100644 --- a/include/crypto/if_alg.h +++ b/include/crypto/if_alg.h @@ -166,7 +166,8 @@ int af_alg_unregister_type(const struct af_alg_type *type); int af_alg_release(struct socket *sock); void af_alg_release_parent(struct sock *sk); -int af_alg_accept(struct sock *sk, struct socket *newsock, bool kern); +int af_alg_accept(struct sock *sk, struct socket *newsock, + struct proto_accept_arg *arg); void af_alg_free_sg(struct af_alg_sgl *sgl); diff --git a/include/linux/net.h b/include/linux/net.h index 15df6d5f27a7b..688320b79fcc6 100644 --- a/include/linux/net.h +++ b/include/linux/net.h @@ -153,6 +153,7 @@ struct sockaddr; struct msghdr; struct module; struct sk_buff; +struct proto_accept_arg; typedef int (*sk_read_actor_t)(read_descriptor_t *, struct sk_buff *, unsigned int, size_t); typedef int (*skb_read_actor_t)(struct sock *, struct sk_buff *); @@ -171,7 +172,8 @@ struct proto_ops { int (*socketpair)(struct socket *sock1, struct socket *sock2); int (*accept) (struct socket *sock, - struct socket *newsock, int flags, bool kern); + struct socket *newsock, + struct proto_accept_arg *arg); int (*getname) (struct socket *sock, struct sockaddr *addr, int peer); diff --git a/include/net/inet_common.h b/include/net/inet_common.h index f50a644d87a98..c17a6585d0b0b 100644 --- a/include/net/inet_common.h +++ b/include/net/inet_common.h @@ -29,8 +29,8 @@ int __inet_stream_connect(struct socket *sock, struct sockaddr *uaddr, int addr_len, int flags, int is_sendmsg); int inet_dgram_connect(struct socket *sock, struct sockaddr *uaddr, int addr_len, int flags); -int inet_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern); +int inet_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg); void __inet_accept(struct socket *sock, struct socket *newsock, struct sock *newsk); int inet_send_prepare(struct sock *sk); diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index 20e7b0c0b3d12..7d6b1254c92d5 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -250,7 +250,7 @@ inet_csk_rto_backoff(const struct inet_connection_sock *icsk, return (unsigned long)min_t(u64, when, max_when); } -struct sock *inet_csk_accept(struct sock *sk, int flags, int *err, bool kern); +struct sock *inet_csk_accept(struct sock *sk, struct proto_accept_arg *arg); int inet_csk_get_port(struct sock *sk, unsigned short snum); diff --git a/include/net/sock.h b/include/net/sock.h index 0450494a1766a..217079b3e3e82 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -1194,6 +1194,12 @@ static inline void sk_prot_clear_nulls(struct sock *sk, int size) size - offsetof(struct sock, sk_node.pprev)); } +struct proto_accept_arg { + int flags; + int err; + bool kern; +}; + /* Networking protocol blocks we attach to sockets. * socket layer -> transport layer interface */ @@ -1208,8 +1214,8 @@ struct proto { int addr_len); int (*disconnect)(struct sock *sk, int flags); - struct sock * (*accept)(struct sock *sk, int flags, int *err, - bool kern); + struct sock * (*accept)(struct sock *sk, + struct proto_accept_arg *arg); int (*ioctl)(struct sock *sk, int cmd, int *karg); @@ -1804,7 +1810,7 @@ int sock_cmsg_send(struct sock *sk, struct msghdr *msg, int sock_no_bind(struct socket *, struct sockaddr *, int); int sock_no_connect(struct socket *, struct sockaddr *, int, int); int sock_no_socketpair(struct socket *, struct socket *); -int sock_no_accept(struct socket *, struct socket *, int, bool); +int sock_no_accept(struct socket *, struct socket *, struct proto_accept_arg *); int sock_no_getname(struct socket *, struct sockaddr *, int); int sock_no_ioctl(struct socket *, unsigned int, unsigned long); int sock_no_listen(struct socket *, int); diff --git a/net/atm/svc.c b/net/atm/svc.c index 36a814f1fbd16..f8137ae693b08 100644 --- a/net/atm/svc.c +++ b/net/atm/svc.c @@ -324,8 +324,8 @@ static int svc_listen(struct socket *sock, int backlog) return error; } -static int svc_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern) +static int svc_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg) { struct sock *sk = sock->sk; struct sk_buff *skb; @@ -336,7 +336,7 @@ static int svc_accept(struct socket *sock, struct socket *newsock, int flags, lock_sock(sk); - error = svc_create(sock_net(sk), newsock, 0, kern); + error = svc_create(sock_net(sk), newsock, 0, arg->kern); if (error) goto out; @@ -355,7 +355,7 @@ static int svc_accept(struct socket *sock, struct socket *newsock, int flags, error = -sk->sk_err; break; } - if (flags & O_NONBLOCK) { + if (arg->flags & O_NONBLOCK) { error = -EAGAIN; break; } diff --git a/net/ax25/af_ax25.c b/net/ax25/af_ax25.c index 9169efb2f43aa..8077cf2ee4480 100644 --- a/net/ax25/af_ax25.c +++ b/net/ax25/af_ax25.c @@ -1373,8 +1373,8 @@ static int __must_check ax25_connect(struct socket *sock, return err; } -static int ax25_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern) +static int ax25_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg) { struct sk_buff *skb; struct sock *newsk; @@ -1409,7 +1409,7 @@ static int ax25_accept(struct socket *sock, struct socket *newsock, int flags, if (skb) break; - if (flags & O_NONBLOCK) { + if (arg->flags & O_NONBLOCK) { err = -EWOULDBLOCK; break; } diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index ef0cc80b4c0cc..2a075119d65d9 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -1186,7 +1186,7 @@ static int iso_sock_listen(struct socket *sock, int backlog) } static int iso_sock_accept(struct socket *sock, struct socket *newsock, - int flags, bool kern) + struct proto_accept_arg *arg) { DEFINE_WAIT_FUNC(wait, woken_wake_function); struct sock *sk = sock->sk, *ch; @@ -1195,7 +1195,7 @@ static int iso_sock_accept(struct socket *sock, struct socket *newsock, lock_sock(sk); - timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); + timeo = sock_rcvtimeo(sk, arg->flags & O_NONBLOCK); BT_DBG("sk %p timeo %ld", sk, timeo); diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 5cc83f906c123..125dddc77452b 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -327,7 +327,7 @@ static int l2cap_sock_listen(struct socket *sock, int backlog) } static int l2cap_sock_accept(struct socket *sock, struct socket *newsock, - int flags, bool kern) + struct proto_accept_arg *arg) { DEFINE_WAIT_FUNC(wait, woken_wake_function); struct sock *sk = sock->sk, *nsk; @@ -336,7 +336,7 @@ static int l2cap_sock_accept(struct socket *sock, struct socket *newsock, lock_sock_nested(sk, L2CAP_NESTING_PARENT); - timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); + timeo = sock_rcvtimeo(sk, arg->flags & O_NONBLOCK); BT_DBG("sk %p timeo %ld", sk, timeo); diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c index 29aa07e9db9d7..37d63d768afb8 100644 --- a/net/bluetooth/rfcomm/sock.c +++ b/net/bluetooth/rfcomm/sock.c @@ -468,8 +468,8 @@ static int rfcomm_sock_listen(struct socket *sock, int backlog) return err; } -static int rfcomm_sock_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern) +static int rfcomm_sock_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg) { DEFINE_WAIT_FUNC(wait, woken_wake_function); struct sock *sk = sock->sk, *nsk; @@ -483,7 +483,7 @@ static int rfcomm_sock_accept(struct socket *sock, struct socket *newsock, int f goto done; } - timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); + timeo = sock_rcvtimeo(sk, arg->flags & O_NONBLOCK); BT_DBG("sk %p timeo %ld", sk, timeo); diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c index e0ad30862ee41..94c6f2b462797 100644 --- a/net/bluetooth/sco.c +++ b/net/bluetooth/sco.c @@ -647,7 +647,7 @@ static int sco_sock_listen(struct socket *sock, int backlog) } static int sco_sock_accept(struct socket *sock, struct socket *newsock, - int flags, bool kern) + struct proto_accept_arg *arg) { DEFINE_WAIT_FUNC(wait, woken_wake_function); struct sock *sk = sock->sk, *ch; @@ -656,7 +656,7 @@ static int sco_sock_accept(struct socket *sock, struct socket *newsock, lock_sock(sk); - timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); + timeo = sock_rcvtimeo(sk, arg->flags & O_NONBLOCK); BT_DBG("sk %p timeo %ld", sk, timeo); diff --git a/net/core/sock.c b/net/core/sock.c index 8d6e638b5426d..8629f9aecf91a 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -3241,8 +3241,8 @@ int sock_no_socketpair(struct socket *sock1, struct socket *sock2) } EXPORT_SYMBOL(sock_no_socketpair); -int sock_no_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern) +int sock_no_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg) { return -EOPNOTSUPP; } diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index a7bad18bc8b58..de3449e16b896 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -771,16 +771,16 @@ void __inet_accept(struct socket *sock, struct socket *newsock, struct sock *new * Accept a pending connection. The TCP layer now gives BSD semantics. */ -int inet_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern) +int inet_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg) { struct sock *sk1 = sock->sk, *sk2; - int err = -EINVAL; /* IPV6_ADDRFORM can change sk->sk_prot under us. */ - sk2 = READ_ONCE(sk1->sk_prot)->accept(sk1, flags, &err, kern); + arg->err = -EINVAL; + sk2 = READ_ONCE(sk1->sk_prot)->accept(sk1, arg); if (!sk2) - return err; + return arg->err; lock_sock(sk2); __inet_accept(sock, newsock, sk2); diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 3b38610958ee4..7734d189c66b8 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -661,7 +661,7 @@ static int inet_csk_wait_for_connect(struct sock *sk, long timeo) /* * This will accept the next outstanding connection. */ -struct sock *inet_csk_accept(struct sock *sk, int flags, int *err, bool kern) +struct sock *inet_csk_accept(struct sock *sk, struct proto_accept_arg *arg) { struct inet_connection_sock *icsk = inet_csk(sk); struct request_sock_queue *queue = &icsk->icsk_accept_queue; @@ -680,7 +680,7 @@ struct sock *inet_csk_accept(struct sock *sk, int flags, int *err, bool kern) /* Find already established connection */ if (reqsk_queue_empty(queue)) { - long timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); + long timeo = sock_rcvtimeo(sk, arg->flags & O_NONBLOCK); /* If this is a non blocking socket don't sleep */ error = -EAGAIN; @@ -745,7 +745,7 @@ struct sock *inet_csk_accept(struct sock *sk, int flags, int *err, bool kern) out_err: newsk = NULL; req = NULL; - *err = error; + arg->err = error; goto out; } EXPORT_SYMBOL(inet_csk_accept); diff --git a/net/iucv/af_iucv.c b/net/iucv/af_iucv.c index c951bb9cc2e04..c3b0b610b0aa3 100644 --- a/net/iucv/af_iucv.c +++ b/net/iucv/af_iucv.c @@ -795,7 +795,7 @@ static int iucv_sock_listen(struct socket *sock, int backlog) /* Accept a pending connection */ static int iucv_sock_accept(struct socket *sock, struct socket *newsock, - int flags, bool kern) + struct proto_accept_arg *arg) { DECLARE_WAITQUEUE(wait, current); struct sock *sk = sock->sk, *nsk; @@ -809,7 +809,7 @@ static int iucv_sock_accept(struct socket *sock, struct socket *newsock, goto done; } - timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); + timeo = sock_rcvtimeo(sk, arg->flags & O_NONBLOCK); /* Wait for an incoming connection */ add_wait_queue_exclusive(sk_sleep(sk), &wait); diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c index fde1140d899ef..4eb52add7103b 100644 --- a/net/llc/af_llc.c +++ b/net/llc/af_llc.c @@ -688,14 +688,13 @@ static void llc_cmsg_rcv(struct msghdr *msg, struct sk_buff *skb) * llc_ui_accept - accept a new incoming connection. * @sock: Socket which connections arrive on. * @newsock: Socket to move incoming connection to. - * @flags: User specified operational flags. - * @kern: If the socket is kernel internal + * @arg: User specified arguments * * Accept a new incoming connection. * Returns 0 upon success, negative otherwise. */ -static int llc_ui_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern) +static int llc_ui_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg) { struct sock *sk = sock->sk, *newsk; struct llc_sock *llc, *newllc; diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index bb8f96f2b86fe..815ce439183c3 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -3881,11 +3881,10 @@ static int mptcp_listen(struct socket *sock, int backlog) } static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, - int flags, bool kern) + struct proto_accept_arg *arg) { struct mptcp_sock *msk = mptcp_sk(sock->sk); struct sock *ssk, *newsk; - int err; pr_debug("msk=%p", msk); @@ -3897,9 +3896,9 @@ static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, return -EINVAL; pr_debug("ssk=%p, listener=%p", ssk, mptcp_subflow_ctx(ssk)); - newsk = inet_csk_accept(ssk, flags, &err, kern); + newsk = inet_csk_accept(ssk, arg); if (!newsk) - return err; + return arg->err; pr_debug("newsk=%p, subflow is mptcp=%d", newsk, sk_is_mptcp(newsk)); if (sk_is_mptcp(newsk)) { @@ -3920,7 +3919,7 @@ static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, newsk = new_mptcp_sock; MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_MPCAPABLEPASSIVEACK); - newsk->sk_kern_sock = kern; + newsk->sk_kern_sock = arg->kern; lock_sock(newsk); __inet_accept(sock, newsock, newsk); @@ -3949,7 +3948,7 @@ static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, } } else { tcpfallback: - newsk->sk_kern_sock = kern; + newsk->sk_kern_sock = arg->kern; lock_sock(newsk); __inet_accept(sock, newsock, newsk); /* we are being invoked after accepting a non-mp-capable diff --git a/net/netrom/af_netrom.c b/net/netrom/af_netrom.c index 104a80b75477f..6ee148f0e6d04 100644 --- a/net/netrom/af_netrom.c +++ b/net/netrom/af_netrom.c @@ -772,8 +772,8 @@ static int nr_connect(struct socket *sock, struct sockaddr *uaddr, return err; } -static int nr_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern) +static int nr_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg) { struct sk_buff *skb; struct sock *newsk; @@ -805,7 +805,7 @@ static int nr_accept(struct socket *sock, struct socket *newsock, int flags, if (skb) break; - if (flags & O_NONBLOCK) { + if (arg->flags & O_NONBLOCK) { err = -EWOULDBLOCK; break; } diff --git a/net/nfc/llcp_sock.c b/net/nfc/llcp_sock.c index d5344563e525c..57a2f97004e17 100644 --- a/net/nfc/llcp_sock.c +++ b/net/nfc/llcp_sock.c @@ -447,7 +447,7 @@ struct sock *nfc_llcp_accept_dequeue(struct sock *parent, } static int llcp_sock_accept(struct socket *sock, struct socket *newsock, - int flags, bool kern) + struct proto_accept_arg *arg) { DECLARE_WAITQUEUE(wait, current); struct sock *sk = sock->sk, *new_sk; @@ -463,7 +463,7 @@ static int llcp_sock_accept(struct socket *sock, struct socket *newsock, goto error; } - timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); + timeo = sock_rcvtimeo(sk, arg->flags & O_NONBLOCK); /* Wait for an incoming connection. */ add_wait_queue_exclusive(sk_sleep(sk), &wait); diff --git a/net/phonet/pep.c b/net/phonet/pep.c index 3dd5f52bc1b58..53a858478e22f 100644 --- a/net/phonet/pep.c +++ b/net/phonet/pep.c @@ -759,8 +759,8 @@ static void pep_sock_close(struct sock *sk, long timeout) sock_put(sk); } -static struct sock *pep_sock_accept(struct sock *sk, int flags, int *errp, - bool kern) +static struct sock *pep_sock_accept(struct sock *sk, + struct proto_accept_arg *arg) { struct pep_sock *pn = pep_sk(sk), *newpn; struct sock *newsk = NULL; @@ -772,8 +772,8 @@ static struct sock *pep_sock_accept(struct sock *sk, int flags, int *errp, u8 pipe_handle, enabled, n_sb; u8 aligned = 0; - skb = skb_recv_datagram(sk, (flags & O_NONBLOCK) ? MSG_DONTWAIT : 0, - errp); + skb = skb_recv_datagram(sk, (arg->flags & O_NONBLOCK) ? MSG_DONTWAIT : 0, + &arg->err); if (!skb) return NULL; @@ -836,7 +836,7 @@ static struct sock *pep_sock_accept(struct sock *sk, int flags, int *errp, /* Create a new to-be-accepted sock */ newsk = sk_alloc(sock_net(sk), PF_PHONET, GFP_KERNEL, sk->sk_prot, - kern); + arg->kern); if (!newsk) { pep_reject_conn(sk, skb, PN_PIPE_ERR_OVERLOAD, GFP_KERNEL); err = -ENOBUFS; @@ -878,7 +878,7 @@ static struct sock *pep_sock_accept(struct sock *sk, int flags, int *errp, drop: release_sock(sk); kfree_skb(skb); - *errp = err; + arg->err = err; return newsk; } diff --git a/net/phonet/socket.c b/net/phonet/socket.c index 1018340d89a7d..5ce0b3ee5def8 100644 --- a/net/phonet/socket.c +++ b/net/phonet/socket.c @@ -292,18 +292,17 @@ static int pn_socket_connect(struct socket *sock, struct sockaddr *addr, } static int pn_socket_accept(struct socket *sock, struct socket *newsock, - int flags, bool kern) + struct proto_accept_arg *arg) { struct sock *sk = sock->sk; struct sock *newsk; - int err; if (unlikely(sk->sk_state != TCP_LISTEN)) return -EINVAL; - newsk = sk->sk_prot->accept(sk, flags, &err, kern); + newsk = sk->sk_prot->accept(sk, arg); if (!newsk) - return err; + return arg->err; lock_sock(newsk); sock_graft(newsk, newsock); diff --git a/net/rds/tcp_listen.c b/net/rds/tcp_listen.c index 05008ce5c4219..d89bd8d0c3545 100644 --- a/net/rds/tcp_listen.c +++ b/net/rds/tcp_listen.c @@ -105,6 +105,10 @@ int rds_tcp_accept_one(struct socket *sock) int conn_state; struct rds_conn_path *cp; struct in6_addr *my_addr, *peer_addr; + struct proto_accept_arg arg = { + .flags = O_NONBLOCK, + .kern = true, + }; #if !IS_ENABLED(CONFIG_IPV6) struct in6_addr saddr, daddr; #endif @@ -119,7 +123,7 @@ int rds_tcp_accept_one(struct socket *sock) if (ret) goto out; - ret = sock->ops->accept(sock, new_sock, O_NONBLOCK, true); + ret = sock->ops->accept(sock, new_sock, &arg); if (ret < 0) goto out; diff --git a/net/rose/af_rose.c b/net/rose/af_rose.c index ef81d019b20f4..59050caab65c8 100644 --- a/net/rose/af_rose.c +++ b/net/rose/af_rose.c @@ -919,8 +919,8 @@ static int rose_connect(struct socket *sock, struct sockaddr *uaddr, int addr_le return err; } -static int rose_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern) +static int rose_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg) { struct sk_buff *skb; struct sock *newsk; @@ -953,7 +953,7 @@ static int rose_accept(struct socket *sock, struct socket *newsock, int flags, if (skb) break; - if (flags & O_NONBLOCK) { + if (arg->flags & O_NONBLOCK) { err = -EWOULDBLOCK; break; } diff --git a/net/sctp/socket.c b/net/sctp/socket.c index 64196b1dce1d4..c009383369b26 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -4847,7 +4847,7 @@ static int sctp_disconnect(struct sock *sk, int flags) * descriptor will be returned from accept() to represent the newly * formed association. */ -static struct sock *sctp_accept(struct sock *sk, int flags, int *err, bool kern) +static struct sock *sctp_accept(struct sock *sk, struct proto_accept_arg *arg) { struct sctp_sock *sp; struct sctp_endpoint *ep; @@ -4871,7 +4871,7 @@ static struct sock *sctp_accept(struct sock *sk, int flags, int *err, bool kern) goto out; } - timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); + timeo = sock_rcvtimeo(sk, arg->flags & O_NONBLOCK); error = sctp_wait_for_accept(sk, timeo); if (error) @@ -4882,7 +4882,7 @@ static struct sock *sctp_accept(struct sock *sk, int flags, int *err, bool kern) */ asoc = list_entry(ep->asocs.next, struct sctp_association, asocs); - newsk = sp->pf->create_accept_sk(sk, asoc, kern); + newsk = sp->pf->create_accept_sk(sk, asoc, arg->kern); if (!newsk) { error = -ENOMEM; goto out; @@ -4899,7 +4899,7 @@ static struct sock *sctp_accept(struct sock *sk, int flags, int *err, bool kern) out: release_sock(sk); - *err = error; + arg->err = error; return newsk; } diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index 9389f0cfa374a..e50a286fd0fb7 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -2689,7 +2689,7 @@ static int smc_listen(struct socket *sock, int backlog) } static int smc_accept(struct socket *sock, struct socket *new_sock, - int flags, bool kern) + struct proto_accept_arg *arg) { struct sock *sk = sock->sk, *nsk; DECLARE_WAITQUEUE(wait, current); @@ -2708,7 +2708,7 @@ static int smc_accept(struct socket *sock, struct socket *new_sock, } /* Wait for an incoming connection */ - timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); + timeo = sock_rcvtimeo(sk, arg->flags & O_NONBLOCK); add_wait_queue_exclusive(sk_sleep(sk), &wait); while (!(nsk = smc_accept_dequeue(sk, new_sock))) { set_current_state(TASK_INTERRUPTIBLE); @@ -2735,7 +2735,7 @@ static int smc_accept(struct socket *sock, struct socket *new_sock, if (rc) goto out; - if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) { + if (lsmc->sockopt_defer_accept && !(arg->flags & O_NONBLOCK)) { /* wait till data arrives on the socket */ timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept * MSEC_PER_SEC); diff --git a/net/socket.c b/net/socket.c index 01a71ae10c35d..6ff5f21d96332 100644 --- a/net/socket.c +++ b/net/socket.c @@ -1898,6 +1898,9 @@ struct file *do_accept(struct file *file, unsigned file_flags, struct file *newfile; int err, len; struct sockaddr_storage address; + struct proto_accept_arg arg = { + .flags = file_flags, + }; const struct proto_ops *ops; sock = sock_from_file(file); @@ -1926,8 +1929,8 @@ struct file *do_accept(struct file *file, unsigned file_flags, if (err) goto out_fd; - err = ops->accept(sock, newsock, sock->file->f_flags | file_flags, - false); + arg.flags |= sock->file->f_flags; + err = ops->accept(sock, newsock, &arg); if (err < 0) goto out_fd; @@ -3580,6 +3583,10 @@ int kernel_accept(struct socket *sock, struct socket **newsock, int flags) { struct sock *sk = sock->sk; const struct proto_ops *ops = READ_ONCE(sock->ops); + struct proto_accept_arg arg = { + .flags = flags, + .kern = true, + }; int err; err = sock_create_lite(sk->sk_family, sk->sk_type, sk->sk_protocol, @@ -3587,7 +3594,7 @@ int kernel_accept(struct socket *sock, struct socket **newsock, int flags) if (err < 0) goto done; - err = ops->accept(sock, *newsock, flags, true); + err = ops->accept(sock, *newsock, &arg); if (err < 0) { sock_release(*newsock); *newsock = NULL; diff --git a/net/tipc/socket.c b/net/tipc/socket.c index 798397b6811e1..2d58ecae4e210 100644 --- a/net/tipc/socket.c +++ b/net/tipc/socket.c @@ -146,8 +146,6 @@ static void tipc_data_ready(struct sock *sk); static void tipc_write_space(struct sock *sk); static void tipc_sock_destruct(struct sock *sk); static int tipc_release(struct socket *sock); -static int tipc_accept(struct socket *sock, struct socket *new_sock, int flags, - bool kern); static void tipc_sk_timeout(struct timer_list *t); static int tipc_sk_publish(struct tipc_sock *tsk, struct tipc_uaddr *ua); static int tipc_sk_withdraw(struct tipc_sock *tsk, struct tipc_uaddr *ua); @@ -2711,13 +2709,12 @@ static int tipc_wait_for_accept(struct socket *sock, long timeo) * tipc_accept - wait for connection request * @sock: listening socket * @new_sock: new socket that is to be connected - * @flags: file-related flags associated with socket - * @kern: caused by kernel or by userspace? + * @arg: arguments for accept * * Return: 0 on success, errno otherwise */ -static int tipc_accept(struct socket *sock, struct socket *new_sock, int flags, - bool kern) +static int tipc_accept(struct socket *sock, struct socket *new_sock, + struct proto_accept_arg *arg) { struct sock *new_sk, *sk = sock->sk; struct tipc_sock *new_tsock; @@ -2733,14 +2730,14 @@ static int tipc_accept(struct socket *sock, struct socket *new_sock, int flags, res = -EINVAL; goto exit; } - timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); + timeo = sock_rcvtimeo(sk, arg->flags & O_NONBLOCK); res = tipc_wait_for_accept(sock, timeo); if (res) goto exit; buf = skb_peek(&sk->sk_receive_queue); - res = tipc_sk_create(sock_net(sock->sk), new_sock, 0, kern); + res = tipc_sk_create(sock_net(sock->sk), new_sock, 0, arg->kern); if (res) goto exit; security_sk_clone(sock->sk, new_sock->sk); diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index dc16515417232..26e3b5f1ee46e 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -755,7 +755,7 @@ static int unix_bind(struct socket *, struct sockaddr *, int); static int unix_stream_connect(struct socket *, struct sockaddr *, int addr_len, int flags); static int unix_socketpair(struct socket *, struct socket *); -static int unix_accept(struct socket *, struct socket *, int, bool); +static int unix_accept(struct socket *, struct socket *, struct proto_accept_arg *arg); static int unix_getname(struct socket *, struct sockaddr *, int); static __poll_t unix_poll(struct file *, struct socket *, poll_table *); static __poll_t unix_dgram_poll(struct file *, struct socket *, @@ -1689,19 +1689,18 @@ static void unix_sock_inherit_flags(const struct socket *old, set_bit(SOCK_PASSSEC, &new->flags); } -static int unix_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern) +static int unix_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg) { struct sock *sk = sock->sk; struct sk_buff *skb; struct sock *tsk; - int err; - err = -EOPNOTSUPP; + arg->err = -EOPNOTSUPP; if (sock->type != SOCK_STREAM && sock->type != SOCK_SEQPACKET) goto out; - err = -EINVAL; + arg->err = -EINVAL; if (sk->sk_state != TCP_LISTEN) goto out; @@ -1709,12 +1708,12 @@ static int unix_accept(struct socket *sock, struct socket *newsock, int flags, * so that no locks are necessary. */ - skb = skb_recv_datagram(sk, (flags & O_NONBLOCK) ? MSG_DONTWAIT : 0, - &err); + skb = skb_recv_datagram(sk, (arg->flags & O_NONBLOCK) ? MSG_DONTWAIT : 0, + &arg->err); if (!skb) { /* This means receive shutdown. */ - if (err == 0) - err = -EINVAL; + if (arg->err == 0) + arg->err = -EINVAL; goto out; } @@ -1732,7 +1731,7 @@ static int unix_accept(struct socket *sock, struct socket *newsock, int flags, return 0; out: - return err; + return arg->err; } diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index 54ba7316f8085..4b040285aa78c 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -1500,8 +1500,8 @@ static int vsock_connect(struct socket *sock, struct sockaddr *addr, return err; } -static int vsock_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern) +static int vsock_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg) { struct sock *listener; int err; @@ -1528,7 +1528,7 @@ static int vsock_accept(struct socket *sock, struct socket *newsock, int flags, /* Wait for children sockets to appear; these are the new sockets * created upon connection establishment. */ - timeout = sock_rcvtimeo(listener, flags & O_NONBLOCK); + timeout = sock_rcvtimeo(listener, arg->flags & O_NONBLOCK); prepare_to_wait(sk_sleep(listener), &wait, TASK_INTERRUPTIBLE); while ((connected = vsock_dequeue_accept(listener)) == NULL && diff --git a/net/x25/af_x25.c b/net/x25/af_x25.c index d18d51412cc00..8dda4178497c9 100644 --- a/net/x25/af_x25.c +++ b/net/x25/af_x25.c @@ -871,8 +871,8 @@ static int x25_wait_for_data(struct sock *sk, long timeout) return rc; } -static int x25_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern) +static int x25_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg) { struct sock *sk = sock->sk; struct sock *newsk; From 0645fbe760afcc5332c858d1cbf416bf77ef3c29 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 9 May 2024 09:31:05 -0600 Subject: [PATCH 1610/1702] net: have do_accept() take a struct proto_accept_arg argument In preparation for passing in more information via this API, change do_accept() to take a proto_accept_arg struct pointer rather than just the file flags separately. No functional changes in this patch. Acked-by: Jakub Kicinski Signed-off-by: Jens Axboe --- include/linux/socket.h | 3 ++- io_uring/net.c | 6 ++++-- net/socket.c | 12 +++++------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/include/linux/socket.h b/include/linux/socket.h index 139c330ccf2c3..89d16b90370bd 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -16,6 +16,7 @@ struct cred; struct socket; struct sock; struct sk_buff; +struct proto_accept_arg; #define __sockaddr_check_size(size) \ BUILD_BUG_ON(((size) > sizeof(struct __kernel_sockaddr_storage))) @@ -433,7 +434,7 @@ extern int __sys_recvfrom(int fd, void __user *ubuf, size_t size, extern int __sys_sendto(int fd, void __user *buff, size_t len, unsigned int flags, struct sockaddr __user *addr, int addr_len); -extern struct file *do_accept(struct file *file, unsigned file_flags, +extern struct file *do_accept(struct file *file, struct proto_accept_arg *arg, struct sockaddr __user *upeer_sockaddr, int __user *upeer_addrlen, int flags); extern int __sys_accept4(int fd, struct sockaddr __user *upeer_sockaddr, diff --git a/io_uring/net.c b/io_uring/net.c index 070dea9a4eda2..d4d1fc93635cb 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -1528,8 +1528,10 @@ int io_accept(struct io_kiocb *req, unsigned int issue_flags) { struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; - unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; bool fixed = !!accept->file_slot; + struct proto_accept_arg arg = { + .flags = force_nonblock ? O_NONBLOCK : 0, + }; struct file *file; int ret, fd; @@ -1543,7 +1545,7 @@ int io_accept(struct io_kiocb *req, unsigned int issue_flags) if (unlikely(fd < 0)) return fd; } - file = do_accept(req->file, file_flags, accept->addr, accept->addr_len, + file = do_accept(req->file, &arg, accept->addr, accept->addr_len, accept->flags); if (IS_ERR(file)) { if (!fixed) diff --git a/net/socket.c b/net/socket.c index 6ff5f21d96332..e416920e9399e 100644 --- a/net/socket.c +++ b/net/socket.c @@ -1890,7 +1890,7 @@ SYSCALL_DEFINE2(listen, int, fd, int, backlog) return __sys_listen(fd, backlog); } -struct file *do_accept(struct file *file, unsigned file_flags, +struct file *do_accept(struct file *file, struct proto_accept_arg *arg, struct sockaddr __user *upeer_sockaddr, int __user *upeer_addrlen, int flags) { @@ -1898,9 +1898,6 @@ struct file *do_accept(struct file *file, unsigned file_flags, struct file *newfile; int err, len; struct sockaddr_storage address; - struct proto_accept_arg arg = { - .flags = file_flags, - }; const struct proto_ops *ops; sock = sock_from_file(file); @@ -1929,8 +1926,8 @@ struct file *do_accept(struct file *file, unsigned file_flags, if (err) goto out_fd; - arg.flags |= sock->file->f_flags; - err = ops->accept(sock, newsock, &arg); + arg->flags |= sock->file->f_flags; + err = ops->accept(sock, newsock, arg); if (err < 0) goto out_fd; @@ -1956,6 +1953,7 @@ struct file *do_accept(struct file *file, unsigned file_flags, static int __sys_accept4_file(struct file *file, struct sockaddr __user *upeer_sockaddr, int __user *upeer_addrlen, int flags) { + struct proto_accept_arg arg = { }; struct file *newfile; int newfd; @@ -1969,7 +1967,7 @@ static int __sys_accept4_file(struct file *file, struct sockaddr __user *upeer_s if (unlikely(newfd < 0)) return newfd; - newfile = do_accept(file, 0, upeer_sockaddr, upeer_addrlen, + newfile = do_accept(file, &arg, upeer_sockaddr, upeer_addrlen, flags); if (IS_ERR(newfile)) { put_unused_fd(newfd); From 7951e36ac620a9ba1bae0ac0ddd62d2e82285725 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 9 May 2024 09:35:01 -0600 Subject: [PATCH 1611/1702] net: pass back whether socket was empty post accept This adds an 'is_empty' argument to struct proto_accept_arg, which can be used to pass back information on whether or not the given socket has more connections to accept post the one just accepted. To utilize this information, the caller should initialize the 'is_empty' field to, eg, -1 and then check for 0/1 after the accept. If the field has been set, the caller knows whether there are more pending connections or not. If the field remains -1 after the accept call, the protocol doesn't support passing back this information. This patch wires it up for ipv4/6 TCP. Acked-by: Jakub Kicinski Signed-off-by: Jens Axboe --- include/net/sock.h | 1 + net/ipv4/inet_connection_sock.c | 1 + 2 files changed, 2 insertions(+) diff --git a/include/net/sock.h b/include/net/sock.h index 217079b3e3e82..5f4d0629348f3 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -1197,6 +1197,7 @@ static inline void sk_prot_clear_nulls(struct sock *sk, int size) struct proto_accept_arg { int flags; int err; + int is_empty; bool kern; }; diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 7734d189c66b8..d81f74ce0f02e 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -692,6 +692,7 @@ struct sock *inet_csk_accept(struct sock *sk, struct proto_accept_arg *arg) goto out_err; } req = reqsk_queue_remove(queue, sk); + arg->is_empty = reqsk_queue_empty(queue); newsk = req->sk; if (sk->sk_protocol == IPPROTO_TCP && From ac287da2e0ea5be2523222981efec86f0ca977cd Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 9 May 2024 09:41:10 -0600 Subject: [PATCH 1612/1702] io_uring/net: wire up IORING_CQE_F_SOCK_NONEMPTY for accept If the given protocol supports passing back whether or not we had more pending accept post this one, pass back this information to userspace. This is done by setting IORING_CQE_F_SOCK_NONEMPTY in the CQE flags, just like we do for recv/recvmsg if there's more data available post a receive operation. We can also use this information to be smarter about multishot retry, as we don't need to do a pointless retry if we know for a fact that there aren't any more connections to accept. Suggested-by: Norman Maurer Acked-by: Jakub Kicinski Signed-off-by: Jens Axboe --- io_uring/net.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/io_uring/net.c b/io_uring/net.c index d4d1fc93635cb..0a48596429d9f 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -1533,6 +1533,7 @@ int io_accept(struct io_kiocb *req, unsigned int issue_flags) .flags = force_nonblock ? O_NONBLOCK : 0, }; struct file *file; + unsigned cflags; int ret, fd; if (!(req->flags & REQ_F_POLLED) && @@ -1545,6 +1546,8 @@ int io_accept(struct io_kiocb *req, unsigned int issue_flags) if (unlikely(fd < 0)) return fd; } + arg.err = 0; + arg.is_empty = -1; file = do_accept(req->file, &arg, accept->addr, accept->addr_len, accept->flags); if (IS_ERR(file)) { @@ -1573,17 +1576,26 @@ int io_accept(struct io_kiocb *req, unsigned int issue_flags) accept->file_slot); } + cflags = 0; + if (!arg.is_empty) + cflags |= IORING_CQE_F_SOCK_NONEMPTY; + if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { - io_req_set_res(req, ret, 0); + io_req_set_res(req, ret, cflags); return IOU_OK; } if (ret < 0) return ret; - if (io_req_post_cqe(req, ret, IORING_CQE_F_MORE)) - goto retry; + if (io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) { + if (cflags & IORING_CQE_F_SOCK_NONEMPTY || arg.is_empty == -1) + goto retry; + if (issue_flags & IO_URING_F_MULTISHOT) + return IOU_ISSUE_SKIP_COMPLETE; + return -EAGAIN; + } - io_req_set_res(req, ret, 0); + io_req_set_res(req, ret, cflags); return IOU_STOP_MULTISHOT; } From 8fe51b45c5645c259f759479c374648e9dfeaa03 Mon Sep 17 00:00:00 2001 From: Wang Yao Date: Wed, 17 Apr 2024 13:35:30 +0800 Subject: [PATCH 1613/1702] modules: Drop the .export_symbol section from the final modules Commit ddb5cdbafaaa ("kbuild: generate KSYMTAB entries by modpost") forget drop the .export_symbol section from the final modules. Fixes: ddb5cdbafaaa ("kbuild: generate KSYMTAB entries by modpost") Signed-off-by: Wang Yao Signed-off-by: Masahiro Yamada --- scripts/module.lds.S | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/module.lds.S b/scripts/module.lds.S index bf5bcf2836d81..89ff01a22634f 100644 --- a/scripts/module.lds.S +++ b/scripts/module.lds.S @@ -13,6 +13,7 @@ SECTIONS { /DISCARD/ : { *(.discard) *(.discard.*) + *(.export_symbol) } __ksymtab 0 : { *(SORT(___ksymtab+*)) } From c9d5b7b8264ba2ead8984a3a4a0c3233911342a4 Mon Sep 17 00:00:00 2001 From: "Dr. David Alan Gilbert" Date: Sat, 4 May 2024 14:23:03 +0100 Subject: [PATCH 1614/1702] ftrace: Remove unused list 'ftrace_direct_funcs' Commit 8788ca164eb4b ("ftrace: Remove the legacy _ftrace_direct API") stopped using 'ftrace_direct_funcs' (and the associated struct ftrace_direct_func). Remove them. Build tested only (on x86-64 with FTRACE and DYNAMIC_FTRACE enabled) Link: https://lore.kernel.org/linux-trace-kernel/20240504132303.67538-1-linux@treblig.org Signed-off-by: Dr. David Alan Gilbert Signed-off-by: Steven Rostedt (Google) --- include/linux/ftrace.h | 1 - kernel/trace/ftrace.c | 8 -------- 2 files changed, 9 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 54d53f345d149..b01cca36147ff 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -83,7 +83,6 @@ static inline void early_trace_init(void) { } struct module; struct ftrace_hash; -struct ftrace_direct_func; #if defined(CONFIG_FUNCTION_TRACER) && defined(CONFIG_MODULES) && \ defined(CONFIG_DYNAMIC_FTRACE) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index f9223513414a3..4613bf67ef2c0 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -5318,14 +5318,6 @@ ftrace_set_addr(struct ftrace_ops *ops, unsigned long *ips, unsigned int cnt, #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS -struct ftrace_direct_func { - struct list_head next; - unsigned long addr; - int count; -}; - -static LIST_HEAD(ftrace_direct_funcs); - static int register_ftrace_function_nolock(struct ftrace_ops *ops); /* From d2cc859cc8885e98907cffd47b990491c5321a80 Mon Sep 17 00:00:00 2001 From: "Dr. David Alan Gilbert" Date: Tue, 7 May 2024 00:33:05 +0100 Subject: [PATCH 1615/1702] ftrace: Remove unused global 'ftrace_direct_func_count' Commit 8788ca164eb4b ("ftrace: Remove the legacy _ftrace_direct API") stopped setting the 'ftrace_direct_func_count' variable, but left it around. Clean it up. Link: https://lore.kernel.org/linux-trace-kernel/20240506233305.215735-1-linux@treblig.org Signed-off-by: Dr. David Alan Gilbert Signed-off-by: Steven Rostedt (Google) --- include/linux/ftrace.h | 2 -- kernel/trace/fgraph.c | 11 ----------- kernel/trace/ftrace.c | 1 - 3 files changed, 14 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index b01cca36147ff..e3a83ebd1b333 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -413,7 +413,6 @@ struct ftrace_func_entry { }; #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS -extern int ftrace_direct_func_count; unsigned long ftrace_find_rec_direct(unsigned long ip); int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr); int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr, @@ -425,7 +424,6 @@ void ftrace_stub_direct_tramp(void); #else struct ftrace_ops; -# define ftrace_direct_func_count 0 static inline unsigned long ftrace_find_rec_direct(unsigned long ip) { return 0; diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index c83c005e654e3..a130b2d898f7c 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -125,17 +125,6 @@ int function_graph_enter(unsigned long ret, unsigned long func, { struct ftrace_graph_ent trace; -#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS - /* - * Skip graph tracing if the return location is served by direct trampoline, - * since call sequence and return addresses are unpredictable anyway. - * Ex: BPF trampoline may call original function and may skip frame - * depending on type of BPF programs attached. - */ - if (ftrace_direct_func_count && - ftrace_find_rec_direct(ret - MCOUNT_INSN_SIZE)) - return -EBUSY; -#endif trace.func = func; trace.depth = ++current->curr_ret_depth; diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 4613bf67ef2c0..5a01d72f66db5 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -2538,7 +2538,6 @@ ftrace_find_unique_ops(struct dyn_ftrace *rec) /* Protected by rcu_tasks for reading, and direct_mutex for writing */ static struct ftrace_hash __rcu *direct_functions = EMPTY_HASH; static DEFINE_MUTEX(direct_mutex); -int ftrace_direct_func_count; /* * Search the direct_functions hash to see if the given instruction pointer From 068a95ef3945033b5355e50fecea18737680d43d Mon Sep 17 00:00:00 2001 From: Wolfram Sang Date: Thu, 18 Apr 2024 22:55:39 +0200 Subject: [PATCH 1616/1702] power: supply: sbs-manager: Remove class argument from i2c_mux_add_adapter() Commit 99a741aa7a2d ("i2c: mux: gpio: remove support for class-based device instantiation") removed the last call to i2c_mux_add_adapter() with a non-null class argument. Therefore the class argument can be removed. Note: Class-based device instantiation is a legacy mechanism which shouldn't be used in new code, so we can rule out that this argument may be needed again in the future. This driver was forgotten by the patch in the Fixes tag. Fixes: fec1982d7072 ("i2c: mux: Remove class argument from i2c_mux_add_adapter()") Signed-off-by: Wolfram Sang Acked-by: Sebastian Reichel --- drivers/power/supply/sbs-manager.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/power/supply/sbs-manager.c b/drivers/power/supply/sbs-manager.c index 9e4141cffbf93..933b04806d10d 100644 --- a/drivers/power/supply/sbs-manager.c +++ b/drivers/power/supply/sbs-manager.c @@ -358,7 +358,7 @@ static int sbsm_probe(struct i2c_client *client) /* register muxed i2c channels. One for each supported battery */ for (i = 0; i < SBSM_MAX_BATS; ++i) { if (data->supported_bats & BIT(i)) { - ret = i2c_mux_add_adapter(data->muxc, 0, i + 1, 0); + ret = i2c_mux_add_adapter(data->muxc, 0, i + 1); if (ret) break; } From 4d1b28a8119c615f1e932520f9ee1f80bdda5204 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Tue, 14 May 2024 11:23:02 +0200 Subject: [PATCH 1617/1702] firmware: dmi: Add info message for number of populated and total memory slots As part of adding support for calling i2c_register_spd() on muxed SMBUS segments the same message has been removed from i2c_register_spd(). However users may find it useful, therefore reintroduce it as part of the DMI scan code. [JD: Static variable dmi_memdev_populated_nr is only used in __init functions, so it can be marked __initdata.] Signed-off-by: Heiner Kallweit Signed-off-by: Jean Delvare --- drivers/firmware/dmi_scan.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/firmware/dmi_scan.c b/drivers/firmware/dmi_scan.c index ac2a5d2d47463..68231c638c552 100644 --- a/drivers/firmware/dmi_scan.c +++ b/drivers/firmware/dmi_scan.c @@ -42,6 +42,7 @@ static struct dmi_memdev_info { u8 type; /* DDR2, DDR3, DDR4 etc */ } *dmi_memdev; static int dmi_memdev_nr; +static int dmi_memdev_populated_nr __initdata; static const char * __init dmi_string_nosave(const struct dmi_header *dm, u8 s) { @@ -459,6 +460,9 @@ static void __init save_mem_devices(const struct dmi_header *dm, void *v) else bytes = (u64)get_unaligned((u32 *)&d[0x1C]) << 20; + if (bytes) + dmi_memdev_populated_nr++; + dmi_memdev[nr].size = bytes; nr++; } @@ -835,6 +839,8 @@ void __init dmi_setup(void) return; dmi_memdev_walk(); + pr_info("DMI: Memory slots populated: %d/%d\n", + dmi_memdev_populated_nr, dmi_memdev_nr); dump_stack_set_arch_desc("%s", dmi_ids_string); } From 9c2d1328f88adb6cbfb218163623254b96f680d3 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 6 May 2024 22:35:42 +0900 Subject: [PATCH 1618/1702] kbuild: provide reasonable defaults for tool coverage The objtool, sanitizers (KASAN, UBSAN, etc.), and profilers (GCOV, etc.) are intended only for kernel space objects. For instance, the following are not kernel objects, and therefore should opt out of coverage: - vDSO - purgatory - bootloader (arch/*/boot/) However, to exclude these from coverage, you need to explicitly set OBJECT_FILES_NON_STNDARD=y, KASAN_SANITIZE=n, etc. Kbuild can achieve this without relying on such variables because objects not directly linked to vmlinux or modules are considered "non-standard objects". Detecting standard objects is straightforward: - objects added to obj-y or lib-y are linked to vmlinux - objects added to obj-m are linked to modules There are some exceptional Makefiles (e.g., arch/s390/boot/Makefile, arch/xtensa/boot/lib/Makefile) that use obj-y or lib-y for non-kernel space objects, but they can be fixed later if necessary. Going forward, objects that are not listed in obj-y, lib-y, or obj-m will opt out of objtool, sanitizers, and profilers by default. You can still override the Kbuild decision by explicitly specifying OBJECT_FILES_NON_STANDARD, KASAN_SANITIZE, etc. but most of such Make variables can be removed. The next commit will clean up redundant variables. Note: This commit changes the coverage for some objects: - exclude .vmlinux.export.o from UBSAN, KCOV - exclude arch/csky/kernel/vdso/vgettimeofday.o from UBSAN - exclude arch/parisc/kernel/vdso32/vdso32.so from UBSAN - exclude arch/parisc/kernel/vdso64/vdso64.so from UBSAN - exclude arch/x86/um/vdso/um_vdso.o from UBSAN - exclude drivers/misc/lkdtm/rodata.o from UBSAN, KCOV - exclude init/version-timestamp.o from UBSAN, KCOV - exclude lib/test_fortify/*.o from all santizers and profilers I believe these are positive effects. Signed-off-by: Masahiro Yamada Reviewed-by: Kees Cook Tested-by: Roberto Sassu --- scripts/Makefile.build | 2 +- scripts/Makefile.lib | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/scripts/Makefile.build b/scripts/Makefile.build index c9c07a6144eb7..56bacd992a094 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -214,7 +214,7 @@ endif # CONFIG_FTRACE_MCOUNT_USE_RECORDMCOUNT # 'OBJECT_FILES_NON_STANDARD_foo.o := 'y': skip objtool checking for a file # 'OBJECT_FILES_NON_STANDARD_foo.o := 'n': override directory skip for a file -is-standard-object = $(if $(filter-out y%, $(OBJECT_FILES_NON_STANDARD_$(target-stem).o)$(OBJECT_FILES_NON_STANDARD)n),y) +is-standard-object = $(if $(filter-out y%, $(OBJECT_FILES_NON_STANDARD_$(target-stem).o)$(OBJECT_FILES_NON_STANDARD)n),$(is-kernel-object)) $(obj)/%.o: private objtool-enabled = $(if $(is-standard-object),$(if $(delay-objtool),$(is-single-obj-m),y)) diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 08d42e93bea02..83635ec2bfe96 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -154,7 +154,7 @@ _cpp_flags = $(KBUILD_CPPFLAGS) $(cppflags-y) $(CPPFLAGS_$(target-stem).lds) # ifeq ($(CONFIG_GCOV_KERNEL),y) _c_flags += $(if $(patsubst n%,, \ - $(GCOV_PROFILE_$(target-stem).o)$(GCOV_PROFILE)$(CONFIG_GCOV_PROFILE_ALL)), \ + $(GCOV_PROFILE_$(target-stem).o)$(GCOV_PROFILE)$(if $(is-kernel-object),$(CONFIG_GCOV_PROFILE_ALL))), \ $(CFLAGS_GCOV)) endif @@ -165,32 +165,32 @@ endif ifeq ($(CONFIG_KASAN),y) ifneq ($(CONFIG_KASAN_HW_TAGS),y) _c_flags += $(if $(patsubst n%,, \ - $(KASAN_SANITIZE_$(target-stem).o)$(KASAN_SANITIZE)y), \ + $(KASAN_SANITIZE_$(target-stem).o)$(KASAN_SANITIZE)$(is-kernel-object)), \ $(CFLAGS_KASAN), $(CFLAGS_KASAN_NOSANITIZE)) endif endif ifeq ($(CONFIG_KMSAN),y) _c_flags += $(if $(patsubst n%,, \ - $(KMSAN_SANITIZE_$(target-stem).o)$(KMSAN_SANITIZE)y), \ + $(KMSAN_SANITIZE_$(target-stem).o)$(KMSAN_SANITIZE)$(is-kernel-object)), \ $(CFLAGS_KMSAN)) _c_flags += $(if $(patsubst n%,, \ - $(KMSAN_ENABLE_CHECKS_$(target-stem).o)$(KMSAN_ENABLE_CHECKS)y), \ + $(KMSAN_ENABLE_CHECKS_$(target-stem).o)$(KMSAN_ENABLE_CHECKS)$(is-kernel-object)), \ , -mllvm -msan-disable-checks=1) endif ifeq ($(CONFIG_UBSAN),y) _c_flags += $(if $(patsubst n%,, \ - $(UBSAN_SANITIZE_$(target-stem).o)$(UBSAN_SANITIZE)y), \ + $(UBSAN_SANITIZE_$(target-stem).o)$(UBSAN_SANITIZE)$(is-kernel-object)), \ $(CFLAGS_UBSAN)) _c_flags += $(if $(patsubst n%,, \ - $(UBSAN_SIGNED_WRAP_$(target-stem).o)$(UBSAN_SANITIZE_$(target-stem).o)$(UBSAN_SIGNED_WRAP)$(UBSAN_SANITIZE)y), \ + $(UBSAN_SIGNED_WRAP_$(target-stem).o)$(UBSAN_SANITIZE_$(target-stem).o)$(UBSAN_SIGNED_WRAP)$(UBSAN_SANITIZE)$(is-kernel-object)), \ $(CFLAGS_UBSAN_SIGNED_WRAP)) endif ifeq ($(CONFIG_KCOV),y) _c_flags += $(if $(patsubst n%,, \ - $(KCOV_INSTRUMENT_$(target-stem).o)$(KCOV_INSTRUMENT)$(CONFIG_KCOV_INSTRUMENT_ALL)), \ + $(KCOV_INSTRUMENT_$(target-stem).o)$(KCOV_INSTRUMENT)$(if $(is-kernel-object),$(CONFIG_KCOV_INSTRUMENT_ALL))), \ $(CFLAGS_KCOV)) endif @@ -200,7 +200,7 @@ endif # ifeq ($(CONFIG_KCSAN),y) _c_flags += $(if $(patsubst n%,, \ - $(KCSAN_SANITIZE_$(target-stem).o)$(KCSAN_SANITIZE)y), \ + $(KCSAN_SANITIZE_$(target-stem).o)$(KCSAN_SANITIZE)$(is-kernel-object)), \ $(CFLAGS_KCSAN)) # Some uninstrumented files provide implied barriers required to avoid false # positives: set KCSAN_INSTRUMENT_BARRIERS for barrier instrumentation only. @@ -219,6 +219,10 @@ _cpp_flags += $(addprefix -I, $(src) $(obj)) endif endif +# If $(is-kernel-object) is 'y', this object will be linked to vmlinux or modules +is-kernel-object = $(or $(part-of-builtin),$(part-of-module)) + +part-of-builtin = $(if $(filter $(basename $@).o, $(real-obj-y) $(lib-y)),y) part-of-module = $(if $(filter $(basename $@).o, $(real-obj-m)),y) quiet_modtag = $(if $(part-of-module),[M], ) From 7f7f6f7ad654b326897c9f54438a06f03454bd0d Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 6 May 2024 22:35:43 +0900 Subject: [PATCH 1619/1702] Makefile: remove redundant tool coverage variables Now Kbuild provides reasonable defaults for objtool, sanitizers, and profilers. Remove redundant variables. Note: This commit changes the coverage for some objects: - include arch/mips/vdso/vdso-image.o into UBSAN, GCOV, KCOV - include arch/sparc/vdso/vdso-image-*.o into UBSAN - include arch/sparc/vdso/vma.o into UBSAN - include arch/x86/entry/vdso/extable.o into KASAN, KCSAN, UBSAN, GCOV, KCOV - include arch/x86/entry/vdso/vdso-image-*.o into KASAN, KCSAN, UBSAN, GCOV, KCOV - include arch/x86/entry/vdso/vdso32-setup.o into KASAN, KCSAN, UBSAN, GCOV, KCOV - include arch/x86/entry/vdso/vma.o into GCOV, KCOV - include arch/x86/um/vdso/vma.o into KASAN, GCOV, KCOV I believe these are positive effects because all of them are kernel space objects. Signed-off-by: Masahiro Yamada Reviewed-by: Kees Cook Tested-by: Roberto Sassu --- arch/arm/boot/bootp/Makefile | 1 - arch/arm/boot/compressed/Makefile | 7 ------- arch/arm/vdso/Makefile | 9 --------- arch/arm64/kernel/pi/Makefile | 6 ------ arch/arm64/kernel/vdso/Makefile | 8 -------- arch/arm64/kvm/hyp/nvhe/Makefile | 13 ------------- arch/csky/kernel/vdso/Makefile | 4 ---- arch/loongarch/vdso/Makefile | 7 ------- arch/mips/boot/compressed/Makefile | 6 ------ arch/mips/vdso/Makefile | 7 ------- arch/parisc/boot/compressed/Makefile | 4 ---- arch/powerpc/kernel/vdso/Makefile | 8 -------- arch/powerpc/purgatory/Makefile | 3 --- arch/riscv/boot/Makefile | 2 -- arch/riscv/kernel/compat_vdso/Makefile | 6 ------ arch/riscv/kernel/pi/Makefile | 6 ------ arch/riscv/kernel/vdso/Makefile | 6 ------ arch/riscv/purgatory/Makefile | 8 -------- arch/s390/kernel/vdso32/Makefile | 8 -------- arch/s390/kernel/vdso64/Makefile | 8 -------- arch/s390/purgatory/Makefile | 8 -------- arch/sh/boot/compressed/Makefile | 3 --- arch/sparc/vdso/Makefile | 2 -- arch/x86/boot/Makefile | 15 --------------- arch/x86/boot/compressed/Makefile | 11 ----------- arch/x86/entry/vdso/Makefile | 26 -------------------------- arch/x86/purgatory/Makefile | 9 --------- arch/x86/realmode/rm/Makefile | 11 ----------- arch/x86/um/vdso/Makefile | 7 ------- drivers/firmware/efi/libstub/Makefile | 11 ----------- drivers/misc/lkdtm/Makefile | 4 ---- init/Makefile | 3 --- scripts/Makefile.vmlinux | 3 --- scripts/mod/Makefile | 1 - 34 files changed, 241 deletions(-) diff --git a/arch/arm/boot/bootp/Makefile b/arch/arm/boot/bootp/Makefile index a2934e6fd89aa..f3443f7d7b02f 100644 --- a/arch/arm/boot/bootp/Makefile +++ b/arch/arm/boot/bootp/Makefile @@ -5,7 +5,6 @@ # This file is included by the global makefile so that you can add your own # architecture-specific flags and dependencies. # -GCOV_PROFILE := n ifdef PHYS_OFFSET add_hex = $(shell printf 0x%x $$(( $(1) + $(2) )) ) diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile index 726ecabcef093..6bca03c0c7f0e 100644 --- a/arch/arm/boot/compressed/Makefile +++ b/arch/arm/boot/compressed/Makefile @@ -22,13 +22,6 @@ ifeq ($(CONFIG_ARM_VIRT_EXT),y) OBJS += hyp-stub.o endif -GCOV_PROFILE := n -KASAN_SANITIZE := n - -# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. -KCOV_INSTRUMENT := n -UBSAN_SANITIZE := n - # # Architecture dependencies # diff --git a/arch/arm/vdso/Makefile b/arch/arm/vdso/Makefile index d761bd2e2f407..01067a2bc43b7 100644 --- a/arch/arm/vdso/Makefile +++ b/arch/arm/vdso/Makefile @@ -33,15 +33,6 @@ else CFLAGS_vgettimeofday.o = -O2 -include $(c-gettimeofday-y) endif -# Disable gcov profiling for VDSO code -GCOV_PROFILE := n -UBSAN_SANITIZE := n - -# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. -KCOV_INSTRUMENT := n - -KASAN_SANITIZE := n - # Force dependency $(obj)/vdso.o : $(obj)/vdso.so diff --git a/arch/arm64/kernel/pi/Makefile b/arch/arm64/kernel/pi/Makefile index 4393b41f0b713..4d11a8c291816 100644 --- a/arch/arm64/kernel/pi/Makefile +++ b/arch/arm64/kernel/pi/Makefile @@ -19,12 +19,6 @@ KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS)) # disable LTO KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO), $(KBUILD_CFLAGS)) -GCOV_PROFILE := n -KASAN_SANITIZE := n -KCSAN_SANITIZE := n -UBSAN_SANITIZE := n -KCOV_INSTRUMENT := n - hostprogs := relacheck quiet_cmd_piobjcopy = $(quiet_cmd_objcopy) diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile index 53e86d3bc159a..d63930c828397 100644 --- a/arch/arm64/kernel/vdso/Makefile +++ b/arch/arm64/kernel/vdso/Makefile @@ -40,11 +40,6 @@ CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) \ $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) \ $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) \ -Wmissing-prototypes -Wmissing-declarations -KASAN_SANITIZE := n -KCSAN_SANITIZE := n -UBSAN_SANITIZE := n -OBJECT_FILES_NON_STANDARD := y -KCOV_INSTRUMENT := n CFLAGS_vgettimeofday.o = -O2 -mcmodel=tiny -fasynchronous-unwind-tables @@ -52,9 +47,6 @@ ifneq ($(c-gettimeofday-y),) CFLAGS_vgettimeofday.o += -include $(c-gettimeofday-y) endif -# Disable gcov profiling for VDSO code -GCOV_PROFILE := n - targets += vdso.lds CPPFLAGS_vdso.lds += -P -C -U$(ARCH) diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index 2250253a6429c..50fa0ffb6b7e2 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -97,16 +97,3 @@ KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_CFI) # causes a build failure. Remove profile optimization flags. KBUILD_CFLAGS := $(filter-out -fprofile-sample-use=% -fprofile-use=%, $(KBUILD_CFLAGS)) KBUILD_CFLAGS += -fno-asynchronous-unwind-tables -fno-unwind-tables - -# KVM nVHE code is run at a different exception code with a different map, so -# compiler instrumentation that inserts callbacks or checks into the code may -# cause crashes. Just disable it. -GCOV_PROFILE := n -KASAN_SANITIZE := n -KCSAN_SANITIZE := n -UBSAN_SANITIZE := n -KCOV_INSTRUMENT := n - -# Skip objtool checking for this directory because nVHE code is compiled with -# non-standard build rules. -OBJECT_FILES_NON_STANDARD := y diff --git a/arch/csky/kernel/vdso/Makefile b/arch/csky/kernel/vdso/Makefile index e79a725f50758..bc2261f5a8d43 100644 --- a/arch/csky/kernel/vdso/Makefile +++ b/arch/csky/kernel/vdso/Makefile @@ -23,10 +23,6 @@ obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) obj-y += vdso.o vdso-syms.o CPPFLAGS_vdso.lds += -P -C -U$(ARCH) -# Disable gcov profiling for VDSO code -GCOV_PROFILE := n -KCOV_INSTRUMENT := n - # Force dependency $(obj)/vdso.o: $(obj)/vdso.so diff --git a/arch/loongarch/vdso/Makefile b/arch/loongarch/vdso/Makefile index cdfc4c793e2c3..d724d46b07c84 100644 --- a/arch/loongarch/vdso/Makefile +++ b/arch/loongarch/vdso/Makefile @@ -1,11 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 # Objects to go into the VDSO. -KASAN_SANITIZE := n -UBSAN_SANITIZE := n -KCOV_INSTRUMENT := n -OBJECT_FILES_NON_STANDARD := y - # Include the generic Makefile to check the built vdso. include $(srctree)/lib/vdso/Makefile @@ -39,8 +34,6 @@ ldflags-y := -Bsymbolic --no-undefined -soname=linux-vdso.so.1 \ $(filter -E%,$(KBUILD_CFLAGS)) -nostdlib -shared \ --hash-style=sysv --build-id -T -GCOV_PROFILE := n - # # Shared build commands. # diff --git a/arch/mips/boot/compressed/Makefile b/arch/mips/boot/compressed/Makefile index 6cc28173bee89..e0b8ec9a95162 100644 --- a/arch/mips/boot/compressed/Makefile +++ b/arch/mips/boot/compressed/Makefile @@ -34,12 +34,6 @@ KBUILD_AFLAGS := $(KBUILD_AFLAGS) -D__ASSEMBLY__ \ -DBOOT_HEAP_SIZE=$(BOOT_HEAP_SIZE) \ -DKERNEL_ENTRY=$(VMLINUX_ENTRY_ADDRESS) -# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. -KCOV_INSTRUMENT := n -GCOV_PROFILE := n -UBSAN_SANITIZE := n -KCSAN_SANITIZE := n - # decompressor objects (linked with vmlinuz) vmlinuzobjs-y := $(obj)/head.o $(obj)/decompress.o $(obj)/string.o $(obj)/bswapsi.o diff --git a/arch/mips/vdso/Makefile b/arch/mips/vdso/Makefile index 40b839e918060..b289b2c1b2946 100644 --- a/arch/mips/vdso/Makefile +++ b/arch/mips/vdso/Makefile @@ -1,9 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 # Objects to go into the VDSO. -# Sanitizer runtimes are unavailable and cannot be linked here. - KCSAN_SANITIZE := n - # Include the generic Makefile to check the built vdso. include $(srctree)/lib/vdso/Makefile @@ -60,10 +57,6 @@ ldflags-y := -Bsymbolic --no-undefined -soname=linux-vdso.so.1 \ CFLAGS_REMOVE_vdso.o = $(CC_FLAGS_FTRACE) -GCOV_PROFILE := n -UBSAN_SANITIZE := n -KCOV_INSTRUMENT := n - # Check that we don't have PIC 'jalr t9' calls left quiet_cmd_vdso_mips_check = VDSOCHK $@ cmd_vdso_mips_check = if $(OBJDUMP) --disassemble $@ | grep -E -h "jalr.*t9" > /dev/null; \ diff --git a/arch/parisc/boot/compressed/Makefile b/arch/parisc/boot/compressed/Makefile index a294a1b58ee7e..92227fa813dc3 100644 --- a/arch/parisc/boot/compressed/Makefile +++ b/arch/parisc/boot/compressed/Makefile @@ -5,10 +5,6 @@ # create a compressed self-extracting vmlinux image from the original vmlinux # -KCOV_INSTRUMENT := n -GCOV_PROFILE := n -UBSAN_SANITIZE := n - OBJECTS := head.o real2.o firmware.o misc.o piggy.o targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 targets += vmlinux.bin.xz vmlinux.bin.lzma vmlinux.bin.lzo vmlinux.bin.lz4 diff --git a/arch/powerpc/kernel/vdso/Makefile b/arch/powerpc/kernel/vdso/Makefile index 7d66b6e079938..1425b6edc66b2 100644 --- a/arch/powerpc/kernel/vdso/Makefile +++ b/arch/powerpc/kernel/vdso/Makefile @@ -47,12 +47,6 @@ obj-vdso32 := $(addprefix $(obj)/, $(obj-vdso32)) targets += $(obj-vdso64) vdso64.so.dbg vgettimeofday-64.o obj-vdso64 := $(addprefix $(obj)/, $(obj-vdso64)) -GCOV_PROFILE := n -KCOV_INSTRUMENT := n -UBSAN_SANITIZE := n -KASAN_SANITIZE := n -KCSAN_SANITIZE := n - ccflags-y := -fno-common -fno-builtin ldflags-y := -Wl,--hash-style=both -nostdlib -shared -z noexecstack $(CLANG_FLAGS) ldflags-$(CONFIG_LD_IS_LLD) += $(call cc-option,--ld-path=$(LD),-fuse-ld=lld) @@ -114,5 +108,3 @@ quiet_cmd_vdso64ld_and_check = VDSO64L $@ cmd_vdso64ld_and_check = $(VDSOCC) $(ldflags-y) $(LD64FLAGS) -o $@ -Wl,-T$(filter %.lds,$^) $(filter %.o,$^); $(cmd_vdso_check) quiet_cmd_vdso64as = VDSO64A $@ cmd_vdso64as = $(VDSOCC) $(a_flags) $(AS64FLAGS) -c -o $@ $< - -OBJECT_FILES_NON_STANDARD := y diff --git a/arch/powerpc/purgatory/Makefile b/arch/powerpc/purgatory/Makefile index 78473d69cd2b6..e9890085953e4 100644 --- a/arch/powerpc/purgatory/Makefile +++ b/arch/powerpc/purgatory/Makefile @@ -1,8 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -KASAN_SANITIZE := n -KCSAN_SANITIZE := n - targets += trampoline_$(BITS).o purgatory.ro # When profile-guided optimization is enabled, llvm emits two different diff --git a/arch/riscv/boot/Makefile b/arch/riscv/boot/Makefile index 8e7fc0edf21d3..869c0345b9084 100644 --- a/arch/riscv/boot/Makefile +++ b/arch/riscv/boot/Makefile @@ -14,8 +14,6 @@ # Based on the ia64 and arm64 boot/Makefile. # -KCOV_INSTRUMENT := n - OBJCOPYFLAGS_Image :=-O binary -R .note -R .note.gnu.build-id -R .comment -S OBJCOPYFLAGS_loader.bin :=-O binary OBJCOPYFLAGS_xipImage :=-O binary -R .note -R .note.gnu.build-id -R .comment -S diff --git a/arch/riscv/kernel/compat_vdso/Makefile b/arch/riscv/kernel/compat_vdso/Makefile index 362208dfa7eec..24e37d1ef7ec0 100644 --- a/arch/riscv/kernel/compat_vdso/Makefile +++ b/arch/riscv/kernel/compat_vdso/Makefile @@ -34,12 +34,6 @@ obj-compat_vdso := $(addprefix $(obj)/, $(obj-compat_vdso)) obj-y += compat_vdso.o CPPFLAGS_compat_vdso.lds += -P -C -DCOMPAT_VDSO -U$(ARCH) -# Disable profiling and instrumentation for VDSO code -GCOV_PROFILE := n -KCOV_INSTRUMENT := n -KASAN_SANITIZE := n -UBSAN_SANITIZE := n - # Force dependency $(obj)/compat_vdso.o: $(obj)/compat_vdso.so diff --git a/arch/riscv/kernel/pi/Makefile b/arch/riscv/kernel/pi/Makefile index b75f150b923d6..50bc5ef7dd2fe 100644 --- a/arch/riscv/kernel/pi/Makefile +++ b/arch/riscv/kernel/pi/Makefile @@ -17,12 +17,6 @@ KBUILD_CFLAGS += -mcmodel=medany CFLAGS_cmdline_early.o += -D__NO_FORTIFY CFLAGS_lib-fdt_ro.o += -D__NO_FORTIFY -GCOV_PROFILE := n -KASAN_SANITIZE := n -KCSAN_SANITIZE := n -UBSAN_SANITIZE := n -KCOV_INSTRUMENT := n - $(obj)/%.pi.o: OBJCOPYFLAGS := --prefix-symbols=__pi_ \ --remove-section=.note.gnu.property \ --prefix-alloc-sections=.init.pi diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 25f0ee629971f..f7ef8ad9b550d 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -39,12 +39,6 @@ endif CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) CFLAGS_REMOVE_hwprobe.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) -# Disable profiling and instrumentation for VDSO code -GCOV_PROFILE := n -KCOV_INSTRUMENT := n -KASAN_SANITIZE := n -UBSAN_SANITIZE := n - # Force dependency $(obj)/vdso.o: $(obj)/vdso.so diff --git a/arch/riscv/purgatory/Makefile b/arch/riscv/purgatory/Makefile index 280b0eb352b8b..f11945ee24903 100644 --- a/arch/riscv/purgatory/Makefile +++ b/arch/riscv/purgatory/Makefile @@ -1,5 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -OBJECT_FILES_NON_STANDARD := y purgatory-y := purgatory.o sha256.o entry.o string.o ctype.o memcpy.o memset.o purgatory-y += strcmp.o strlen.o strncmp.o @@ -47,13 +46,6 @@ LDFLAGS_purgatory.ro := -r $(PURGATORY_LDFLAGS) LDFLAGS_purgatory.chk := $(PURGATORY_LDFLAGS) targets += purgatory.ro purgatory.chk -# Sanitizer, etc. runtimes are unavailable and cannot be linked here. -GCOV_PROFILE := n -KASAN_SANITIZE := n -UBSAN_SANITIZE := n -KCSAN_SANITIZE := n -KCOV_INSTRUMENT := n - # These are adjustments to the compiler flags used for objects that # make up the standalone purgatory.ro diff --git a/arch/s390/kernel/vdso32/Makefile b/arch/s390/kernel/vdso32/Makefile index c263b91adfb15..df928fee26b51 100644 --- a/arch/s390/kernel/vdso32/Makefile +++ b/arch/s390/kernel/vdso32/Makefile @@ -1,8 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 # List of files in the vdso -KCOV_INSTRUMENT := n - # Include the generic Makefile to check the built vdso. include $(srctree)/lib/vdso/Makefile obj-vdso32 = vdso_user_wrapper-32.o note-32.o @@ -32,12 +30,6 @@ obj-y += vdso32_wrapper.o targets += vdso32.lds CPPFLAGS_vdso32.lds += -P -C -U$(ARCH) -# Disable gcov profiling, ubsan and kasan for VDSO code -GCOV_PROFILE := n -UBSAN_SANITIZE := n -KASAN_SANITIZE := n -KCSAN_SANITIZE := n - # Force dependency (incbin is bad) $(obj)/vdso32_wrapper.o : $(obj)/vdso32.so diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile index 9566bed7d5b28..6da1b9ad8ab0c 100644 --- a/arch/s390/kernel/vdso64/Makefile +++ b/arch/s390/kernel/vdso64/Makefile @@ -1,8 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 # List of files in the vdso -KCOV_INSTRUMENT := n - # Include the generic Makefile to check the built vdso. include $(srctree)/lib/vdso/Makefile obj-vdso64 = vdso_user_wrapper.o note.o @@ -37,12 +35,6 @@ obj-y += vdso64_wrapper.o targets += vdso64.lds CPPFLAGS_vdso64.lds += -P -C -U$(ARCH) -# Disable gcov profiling, ubsan and kasan for VDSO code -GCOV_PROFILE := n -UBSAN_SANITIZE := n -KASAN_SANITIZE := n -KCSAN_SANITIZE := n - # Force dependency (incbin is bad) $(obj)/vdso64_wrapper.o : $(obj)/vdso64.so diff --git a/arch/s390/purgatory/Makefile b/arch/s390/purgatory/Makefile index 4e930f5668789..24eccaa293371 100644 --- a/arch/s390/purgatory/Makefile +++ b/arch/s390/purgatory/Makefile @@ -1,7 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -OBJECT_FILES_NON_STANDARD := y - purgatory-y := head.o purgatory.o string.o sha256.o mem.o targets += $(purgatory-y) purgatory.lds purgatory purgatory.chk purgatory.ro @@ -15,12 +13,6 @@ CFLAGS_sha256.o := -D__DISABLE_EXPORTS -D__NO_FORTIFY $(obj)/mem.o: $(srctree)/arch/s390/lib/mem.S FORCE $(call if_changed_rule,as_o_S) -KCOV_INSTRUMENT := n -GCOV_PROFILE := n -UBSAN_SANITIZE := n -KASAN_SANITIZE := n -KCSAN_SANITIZE := n - KBUILD_CFLAGS := -fno-strict-aliasing -Wall -Wstrict-prototypes KBUILD_CFLAGS += -Wno-pointer-sign -Wno-sign-compare KBUILD_CFLAGS += -fno-zero-initialized-in-bss -fno-builtin -ffreestanding diff --git a/arch/sh/boot/compressed/Makefile b/arch/sh/boot/compressed/Makefile index 6c6c791a1d063..3a46b871463d5 100644 --- a/arch/sh/boot/compressed/Makefile +++ b/arch/sh/boot/compressed/Makefile @@ -11,9 +11,6 @@ OBJECTS := head_32.o misc.o cache.o piggy.o \ targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 \ vmlinux.bin.lzma vmlinux.bin.xz vmlinux.bin.lzo $(OBJECTS) -GCOV_PROFILE := n -UBSAN_SANITIZE := n - # # IMAGE_OFFSET is the load offset of the compression loader # diff --git a/arch/sparc/vdso/Makefile b/arch/sparc/vdso/Makefile index 0fe134abbcf17..243dbfc4609d8 100644 --- a/arch/sparc/vdso/Makefile +++ b/arch/sparc/vdso/Makefile @@ -2,7 +2,6 @@ # # Building vDSO images for sparc. # -UBSAN_SANITIZE := n # files to link into the vdso vobjs-y := vdso-note.o vclock_gettime.o @@ -106,4 +105,3 @@ quiet_cmd_vdso = VDSO $@ sh $(src)/checkundef.sh '$(OBJDUMP)' '$@' VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 -Bsymbolic -GCOV_PROFILE := n diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile index 29cda98c65f8b..1cf24ff6acac0 100644 --- a/arch/x86/boot/Makefile +++ b/arch/x86/boot/Makefile @@ -9,19 +9,6 @@ # Changed by many, many contributors over the years. # -# Sanitizer runtimes are unavailable and cannot be linked for early boot code. -KASAN_SANITIZE := n -KCSAN_SANITIZE := n -KMSAN_SANITIZE := n -OBJECT_FILES_NON_STANDARD := y - -# Kernel does not boot with kcov instrumentation here. -# One of the problems observed was insertion of __sanitizer_cov_trace_pc() -# callback into middle of per-cpu data enabling code. Thus the callback observed -# inconsistent state and crashed. We are interested mostly in syscall coverage, -# so boot code is not interesting anyway. -KCOV_INSTRUMENT := n - # If you want to preset the SVGA mode, uncomment the next line and # set SVGA_MODE to whatever number you want. # Set it to -DSVGA_MODE=NORMAL_VGA if you just want the EGA/VGA mode. @@ -69,8 +56,6 @@ KBUILD_CFLAGS := $(REALMODE_CFLAGS) -D_SETUP KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ KBUILD_CFLAGS += $(call cc-option,-fmacro-prefix-map=$(srctree)/=) KBUILD_CFLAGS += -fno-asynchronous-unwind-tables -GCOV_PROFILE := n -UBSAN_SANITIZE := n $(obj)/bzImage: asflags-y := $(SVGA_MODE) diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index e9522c6893bee..243ee86cb1b1e 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -17,15 +17,6 @@ # (see scripts/Makefile.lib size_append) # compressed vmlinux.bin.all + u32 size of vmlinux.bin.all -# Sanitizer runtimes are unavailable and cannot be linked for early boot code. -KASAN_SANITIZE := n -KCSAN_SANITIZE := n -KMSAN_SANITIZE := n -OBJECT_FILES_NON_STANDARD := y - -# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. -KCOV_INSTRUMENT := n - targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \ vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4 vmlinux.bin.zst @@ -59,8 +50,6 @@ KBUILD_CFLAGS += -include $(srctree)/include/linux/hidden.h CFLAGS_sev.o += -I$(objtree)/arch/x86/lib/ KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ -GCOV_PROFILE := n -UBSAN_SANITIZE :=n KBUILD_LDFLAGS := -m elf_$(UTS_MACHINE) KBUILD_LDFLAGS += $(call ld-option,--no-ld-generated-unwind-info) diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index c003452dba8c5..215a1b202a918 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -6,20 +6,6 @@ # Include the generic Makefile to check the built vDSO: include $(srctree)/lib/vdso/Makefile -# Sanitizer runtimes are unavailable and cannot be linked here. -KASAN_SANITIZE := n -KMSAN_SANITIZE_vclock_gettime.o := n -KMSAN_SANITIZE_vdso32/vclock_gettime.o := n -KMSAN_SANITIZE_vgetcpu.o := n -KMSAN_SANITIZE_vdso32/vgetcpu.o := n - -UBSAN_SANITIZE := n -KCSAN_SANITIZE := n -OBJECT_FILES_NON_STANDARD := y - -# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. -KCOV_INSTRUMENT := n - # Files to link into the vDSO: vobjs-y := vdso-note.o vclock_gettime.o vgetcpu.o vobjs32-y := vdso32/note.o vdso32/system_call.o vdso32/sigreturn.o @@ -28,23 +14,12 @@ vobjs-$(CONFIG_X86_SGX) += vsgx.o # Files to link into the kernel: obj-y += vma.o extable.o -KASAN_SANITIZE_vma.o := y -UBSAN_SANITIZE_vma.o := y -KCSAN_SANITIZE_vma.o := y - -OBJECT_FILES_NON_STANDARD_vma.o := n -OBJECT_FILES_NON_STANDARD_extable.o := n # vDSO images to build: obj-$(CONFIG_X86_64) += vdso-image-64.o obj-$(CONFIG_X86_X32_ABI) += vdso-image-x32.o obj-$(CONFIG_COMPAT_32) += vdso-image-32.o vdso32-setup.o -OBJECT_FILES_NON_STANDARD_vdso-image-32.o := n -OBJECT_FILES_NON_STANDARD_vdso-image-x32.o := n -OBJECT_FILES_NON_STANDARD_vdso-image-64.o := n -OBJECT_FILES_NON_STANDARD_vdso32-setup.o := n - vobjs := $(addprefix $(obj)/, $(vobjs-y)) vobjs32 := $(addprefix $(obj)/, $(vobjs32-y)) @@ -180,7 +155,6 @@ quiet_cmd_vdso = VDSO $@ VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 \ $(call ld-option, --eh-frame-hdr) -Bsymbolic -z noexecstack -GCOV_PROFILE := n quiet_cmd_vdso_and_check = VDSO $@ cmd_vdso_and_check = $(cmd_vdso); $(cmd_vdso_check) diff --git a/arch/x86/purgatory/Makefile b/arch/x86/purgatory/Makefile index bc31863c5ee63..0a16f1373cf5f 100644 --- a/arch/x86/purgatory/Makefile +++ b/arch/x86/purgatory/Makefile @@ -1,5 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -OBJECT_FILES_NON_STANDARD := y purgatory-y := purgatory.o stack.o setup-x86_$(BITS).o sha256.o entry64.o string.o @@ -30,14 +29,6 @@ LDFLAGS_purgatory.ro := -r $(PURGATORY_LDFLAGS) LDFLAGS_purgatory.chk := $(PURGATORY_LDFLAGS) targets += purgatory.ro purgatory.chk -# Sanitizer, etc. runtimes are unavailable and cannot be linked here. -GCOV_PROFILE := n -KASAN_SANITIZE := n -UBSAN_SANITIZE := n -KCSAN_SANITIZE := n -KMSAN_SANITIZE := n -KCOV_INSTRUMENT := n - # These are adjustments to the compiler flags used for objects that # make up the standalone purgatory.ro diff --git a/arch/x86/realmode/rm/Makefile b/arch/x86/realmode/rm/Makefile index f614009d3e4e2..a0fb39abc5c86 100644 --- a/arch/x86/realmode/rm/Makefile +++ b/arch/x86/realmode/rm/Makefile @@ -7,15 +7,6 @@ # # -# Sanitizer runtimes are unavailable and cannot be linked here. -KASAN_SANITIZE := n -KCSAN_SANITIZE := n -KMSAN_SANITIZE := n -OBJECT_FILES_NON_STANDARD := y - -# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. -KCOV_INSTRUMENT := n - always-y := realmode.bin realmode.relocs wakeup-objs := wakeup_asm.o wakemain.o video-mode.o @@ -76,5 +67,3 @@ KBUILD_CFLAGS := $(REALMODE_CFLAGS) -D_SETUP -D_WAKEUP \ -I$(srctree)/arch/x86/boot KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ KBUILD_CFLAGS += -fno-asynchronous-unwind-tables -GCOV_PROFILE := n -UBSAN_SANITIZE := n diff --git a/arch/x86/um/vdso/Makefile b/arch/x86/um/vdso/Makefile index 2303fa59971ce..6a77ea6434ffd 100644 --- a/arch/x86/um/vdso/Makefile +++ b/arch/x86/um/vdso/Makefile @@ -3,12 +3,6 @@ # Building vDSO images for x86. # -# do not instrument on vdso because KASAN is not compatible with user mode -KASAN_SANITIZE := n - -# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. -KCOV_INSTRUMENT := n - VDSO64-y := y vdso-install-$(VDSO64-y) += vdso.so @@ -66,4 +60,3 @@ quiet_cmd_vdso = VDSO $@ sh $(src)/checkundef.sh '$(NM)' '$@' VDSO_LDFLAGS = -fPIC -shared -Wl,--hash-style=sysv -z noexecstack -GCOV_PROFILE := n diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index 31eb1e287ce16..06f0428a723cb 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -56,17 +56,6 @@ KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_CFI), $(KBUILD_CFLAGS)) # disable LTO KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO), $(KBUILD_CFLAGS)) -GCOV_PROFILE := n -# Sanitizer runtimes are unavailable and cannot be linked here. -KASAN_SANITIZE := n -KCSAN_SANITIZE := n -KMSAN_SANITIZE := n -UBSAN_SANITIZE := n -OBJECT_FILES_NON_STANDARD := y - -# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. -KCOV_INSTRUMENT := n - lib-y := efi-stub-helper.o gop.o secureboot.o tpm.o \ file.o mem.o random.o randomalloc.o pci.o \ skip_spaces.o lib-cmdline.o lib-ctype.o \ diff --git a/drivers/misc/lkdtm/Makefile b/drivers/misc/lkdtm/Makefile index 95ef971b5e1cb..33fe61152a158 100644 --- a/drivers/misc/lkdtm/Makefile +++ b/drivers/misc/lkdtm/Makefile @@ -15,10 +15,6 @@ lkdtm-$(CONFIG_PPC_64S_HASH_MMU) += powerpc.o KASAN_SANITIZE_stackleak.o := n -KASAN_SANITIZE_rodata.o := n -KCSAN_SANITIZE_rodata.o := n -KCOV_INSTRUMENT_rodata.o := n -OBJECT_FILES_NON_STANDARD_rodata.o := y CFLAGS_REMOVE_rodata.o += $(CC_FLAGS_LTO) $(RETHUNK_CFLAGS) OBJCOPYFLAGS := diff --git a/init/Makefile b/init/Makefile index 3c48d97538c10..ab71cedc5fd65 100644 --- a/init/Makefile +++ b/init/Makefile @@ -59,6 +59,3 @@ include/generated/utsversion.h: FORCE $(obj)/version-timestamp.o: include/generated/utsversion.h CFLAGS_version-timestamp.o := -include include/generated/utsversion.h -KASAN_SANITIZE_version-timestamp.o := n -KCSAN_SANITIZE_version-timestamp.o := n -GCOV_PROFILE_version-timestamp.o := n diff --git a/scripts/Makefile.vmlinux b/scripts/Makefile.vmlinux index c9f3e03124d7f..49946cb968440 100644 --- a/scripts/Makefile.vmlinux +++ b/scripts/Makefile.vmlinux @@ -18,9 +18,6 @@ quiet_cmd_cc_o_c = CC $@ $(call if_changed_dep,cc_o_c) ifdef CONFIG_MODULES -KASAN_SANITIZE_.vmlinux.export.o := n -KCSAN_SANITIZE_.vmlinux.export.o := n -GCOV_PROFILE_.vmlinux.export.o := n targets += .vmlinux.export.o vmlinux: .vmlinux.export.o endif diff --git a/scripts/mod/Makefile b/scripts/mod/Makefile index 3c54125eb3733..c729bc936bae1 100644 --- a/scripts/mod/Makefile +++ b/scripts/mod/Makefile @@ -1,5 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -OBJECT_FILES_NON_STANDARD := y CFLAGS_REMOVE_empty.o += $(CC_FLAGS_LTO) hostprogs-always-y += modpost mk_elfconfig From b4f944ba521485125206640f6e66dce3f350ad6b Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 6 May 2024 22:35:44 +0900 Subject: [PATCH 1620/1702] kbuild: use GCOV_PROFILE and KCSAN_SANITIZE in scripts/Makefile.modfinal Instead of filtering out the GCOV and KCSAN flags, let's set GCOV_PROFILE and KCSAN_SANITIZE to 'n', as in other Makefiles. Signed-off-by: Masahiro Yamada Reviewed-by: Kees Cook Tested-by: Roberto Sassu --- scripts/Makefile.modfinal | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/Makefile.modfinal b/scripts/Makefile.modfinal index 79fcf27316864..3bec9043e4f38 100644 --- a/scripts/Makefile.modfinal +++ b/scripts/Makefile.modfinal @@ -21,9 +21,11 @@ __modfinal: $(modules:%.o=%.ko) # modname and part-of-module are set to make c_flags define proper module flags modname = $(notdir $(@:.mod.o=)) part-of-module = y +GCOV_PROFILE := n +KCSAN_SANITIZE := n quiet_cmd_cc_o_c = CC [M] $@ - cmd_cc_o_c = $(CC) $(filter-out $(CC_FLAGS_CFI) $(CFLAGS_GCOV) $(CFLAGS_KCSAN), $(c_flags)) -c -o $@ $< + cmd_cc_o_c = $(CC) $(filter-out $(CC_FLAGS_CFI), $(c_flags)) -c -o $@ $< %.mod.o: %.mod.c FORCE $(call if_changed_dep,cc_o_c) From 6ba750ddebc90aa97be376e05bc093371934ddce Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 8 May 2024 00:14:50 +0900 Subject: [PATCH 1621/1702] kconfig: gconf: show checkbox for choice correctly Currently, bool choices have a checkbox, but tristate choices do not. It is opposite. Bool choices should not have a checkbox, as they are fixed to 'y' since commit 6a1215888e23 ("kconfig: remove 'optional' property support"). Tristate choices, however, should have a checkbox to allow users to toggle the value. Signed-off-by: Masahiro Yamada --- scripts/kconfig/gconf.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/kconfig/gconf.c b/scripts/kconfig/gconf.c index 10d602faa51e1..cc400ffe66150 100644 --- a/scripts/kconfig/gconf.c +++ b/scripts/kconfig/gconf.c @@ -1054,8 +1054,6 @@ static gchar **fill_row(struct menu *menu) struct symbol *def_sym = sym_get_choice_value(sym); struct menu *def_menu = NULL; - row[COL_BTNVIS] = GINT_TO_POINTER(FALSE); - for (child = menu->list; child; child = child->next) { if (menu_is_visible(child) && child->sym == def_sym) @@ -1065,6 +1063,11 @@ static gchar **fill_row(struct menu *menu) if (def_menu) row[COL_VALUE] = g_strdup(menu_get_prompt(def_menu)); + + if (sym_get_type(sym) == S_BOOLEAN) { + row[COL_BTNVIS] = GINT_TO_POINTER(FALSE); + return row; + } } if (sym->flags & SYMBOL_CHOICEVAL) row[COL_BTNRAD] = GINT_TO_POINTER(TRUE); @@ -1072,11 +1075,6 @@ static gchar **fill_row(struct menu *menu) stype = sym_get_type(sym); switch (stype) { case S_BOOLEAN: - if (GPOINTER_TO_INT(row[COL_PIXVIS]) == FALSE) - row[COL_BTNVIS] = GINT_TO_POINTER(TRUE); - if (sym_is_choice(sym)) - break; - /* fall through */ case S_TRISTATE: val = sym_get_tristate_value(sym); switch (val) { From e89b46159c510e2762ed39af65d530178122b6fb Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 10 May 2024 19:23:21 +0900 Subject: [PATCH 1622/1702] kconfig: m/nconf: remove dead code to display children of choice members This code previously displayed child symbols of the selected choice member. Since commit 7e3465f63a0a ("kconfig: do not reparent the menu inside a choice block"), choice members never have child symbols, therefore this is dead code. Signed-off-by: Masahiro Yamada --- scripts/kconfig/mconf.c | 81 +++++++++++++++++---------------------- scripts/kconfig/nconf.c | 85 +++++++++++++++++------------------------ 2 files changed, 71 insertions(+), 95 deletions(-) diff --git a/scripts/kconfig/mconf.c b/scripts/kconfig/mconf.c index c0969097447da..8de4af43e865d 100644 --- a/scripts/kconfig/mconf.c +++ b/scripts/kconfig/mconf.c @@ -551,11 +551,6 @@ static void build_conf(struct menu *menu) if (def_menu) { item_add_str(" (%s)", menu_get_prompt(def_menu)); item_add_str(" --->"); - if (def_menu->list) { - indent += 2; - build_conf(def_menu); - indent -= 2; - } } return; } @@ -568,49 +563,43 @@ static void build_conf(struct menu *menu) } child_count++; val = sym_get_tristate_value(sym); - if (sym_is_choice_value(sym) && val == yes) { - item_make(" "); - item_set_tag(':'); + switch (type) { + case S_BOOLEAN: + if (sym_is_changeable(sym)) + item_make("[%c]", val == no ? ' ' : '*'); + else + item_make("-%c-", val == no ? ' ' : '*'); + item_set_tag('t'); item_set_data(menu); - } else { - switch (type) { - case S_BOOLEAN: - if (sym_is_changeable(sym)) - item_make("[%c]", val == no ? ' ' : '*'); - else - item_make("-%c-", val == no ? ' ' : '*'); - item_set_tag('t'); - item_set_data(menu); - break; - case S_TRISTATE: - switch (val) { - case yes: ch = '*'; break; - case mod: ch = 'M'; break; - default: ch = ' '; break; - } - if (sym_is_changeable(sym)) { - if (sym->rev_dep.tri == mod) - item_make("{%c}", ch); - else - item_make("<%c>", ch); - } else - item_make("-%c-", ch); - item_set_tag('t'); - item_set_data(menu); - break; - default: - tmp = 2 + strlen(sym_get_string_value(sym)); /* () = 2 */ - item_make("(%s)", sym_get_string_value(sym)); - tmp = indent - tmp + 4; - if (tmp < 0) - tmp = 0; - item_add_str("%*c%s%s", tmp, ' ', menu_get_prompt(menu), - (sym_has_value(sym) || !sym_is_changeable(sym)) ? - "" : " (NEW)"); - item_set_tag('s'); - item_set_data(menu); - goto conf_childs; + break; + case S_TRISTATE: + switch (val) { + case yes: ch = '*'; break; + case mod: ch = 'M'; break; + default: ch = ' '; break; } + if (sym_is_changeable(sym)) { + if (sym->rev_dep.tri == mod) + item_make("{%c}", ch); + else + item_make("<%c>", ch); + } else + item_make("-%c-", ch); + item_set_tag('t'); + item_set_data(menu); + break; + default: + tmp = 2 + strlen(sym_get_string_value(sym)); /* () = 2 */ + item_make("(%s)", sym_get_string_value(sym)); + tmp = indent - tmp + 4; + if (tmp < 0) + tmp = 0; + item_add_str("%*c%s%s", tmp, ' ', menu_get_prompt(menu), + (sym_has_value(sym) || !sym_is_changeable(sym)) ? + "" : " (NEW)"); + item_set_tag('s'); + item_set_data(menu); + goto conf_childs; } item_add_str("%*c%s%s", indent + 1, ' ', menu_get_prompt(menu), (sym_has_value(sym) || !sym_is_changeable(sym)) ? diff --git a/scripts/kconfig/nconf.c b/scripts/kconfig/nconf.c index 9d22b0f3197b8..bf460233a4cca 100644 --- a/scripts/kconfig/nconf.c +++ b/scripts/kconfig/nconf.c @@ -857,11 +857,6 @@ static void build_conf(struct menu *menu) item_add_str(" (%s)", menu_get_prompt(def_menu)); item_add_str(" --->"); - if (def_menu->list) { - indent += 2; - build_conf(def_menu); - indent -= 2; - } } return; } @@ -874,54 +869,46 @@ static void build_conf(struct menu *menu) } child_count++; val = sym_get_tristate_value(sym); - if (sym_is_choice_value(sym) && val == yes) { - item_make(menu, ':', " "); - } else { - switch (type) { - case S_BOOLEAN: - if (sym_is_changeable(sym)) - item_make(menu, 't', "[%c]", - val == no ? ' ' : '*'); - else - item_make(menu, 't', "-%c-", - val == no ? ' ' : '*'); + switch (type) { + case S_BOOLEAN: + if (sym_is_changeable(sym)) + item_make(menu, 't', "[%c]", + val == no ? ' ' : '*'); + else + item_make(menu, 't', "-%c-", + val == no ? ' ' : '*'); + break; + case S_TRISTATE: + switch (val) { + case yes: + ch = '*'; break; - case S_TRISTATE: - switch (val) { - case yes: - ch = '*'; - break; - case mod: - ch = 'M'; - break; - default: - ch = ' '; - break; - } - if (sym_is_changeable(sym)) { - if (sym->rev_dep.tri == mod) - item_make(menu, - 't', "{%c}", ch); - else - item_make(menu, - 't', "<%c>", ch); - } else - item_make(menu, 't', "-%c-", ch); + case mod: + ch = 'M'; break; default: - tmp = 2 + strlen(sym_get_string_value(sym)); - item_make(menu, 's', " (%s)", - sym_get_string_value(sym)); - tmp = indent - tmp + 4; - if (tmp < 0) - tmp = 0; - item_add_str("%*c%s%s", tmp, ' ', - menu_get_prompt(menu), - (sym_has_value(sym) || - !sym_is_changeable(sym)) ? "" : - " (NEW)"); - goto conf_childs; + ch = ' '; + break; } + if (sym_is_changeable(sym)) { + if (sym->rev_dep.tri == mod) + item_make(menu, 't', "{%c}", ch); + else + item_make(menu, 't', "<%c>", ch); + } else + item_make(menu, 't', "-%c-", ch); + break; + default: + tmp = 2 + strlen(sym_get_string_value(sym)); + item_make(menu, 's', " (%s)", + sym_get_string_value(sym)); + tmp = indent - tmp + 4; + if (tmp < 0) + tmp = 0; + item_add_str("%*c%s%s", tmp, ' ', menu_get_prompt(menu), + (sym_has_value(sym) || + !sym_is_changeable(sym)) ? "" : " (NEW)"); + goto conf_childs; } item_add_str("%*c%s%s", indent + 1, ' ', menu_get_prompt(menu), From cc3e4e5e38f97da6085659fabbef2d560c05d1c3 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 10 May 2024 19:23:22 +0900 Subject: [PATCH 1623/1702] kconfig: m/nconf: remove dead code to display value of bool choice Previously, optional bool choices met the following conditions simultaneously: - sym_is_choice(sym) - sym_is_changeable(sym) - type == S_BOOLEAN It no longer occurs since 6a1215888e23 ("kconfig: remove 'optional' property support"). Remove the dead code. Signed-off-by: Masahiro Yamada --- scripts/kconfig/mconf.c | 17 +++++------------ scripts/kconfig/nconf.c | 26 +++++++++----------------- 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/scripts/kconfig/mconf.c b/scripts/kconfig/mconf.c index 8de4af43e865d..90cd59a96aacb 100644 --- a/scripts/kconfig/mconf.c +++ b/scripts/kconfig/mconf.c @@ -525,19 +525,12 @@ static void build_conf(struct menu *menu) val = sym_get_tristate_value(sym); if (sym_is_changeable(sym)) { - switch (type) { - case S_BOOLEAN: - item_make("[%c]", val == no ? ' ' : '*'); - break; - case S_TRISTATE: - switch (val) { - case yes: ch = '*'; break; - case mod: ch = 'M'; break; - default: ch = ' '; break; - } - item_make("<%c>", ch); - break; + switch (val) { + case yes: ch = '*'; break; + case mod: ch = 'M'; break; + default: ch = ' '; break; } + item_make("<%c>", ch); item_set_tag('t'); item_set_data(menu); } else { diff --git a/scripts/kconfig/nconf.c b/scripts/kconfig/nconf.c index bf460233a4cca..93047cd28f3fd 100644 --- a/scripts/kconfig/nconf.c +++ b/scripts/kconfig/nconf.c @@ -826,26 +826,18 @@ static void build_conf(struct menu *menu) val = sym_get_tristate_value(sym); if (sym_is_changeable(sym)) { - switch (type) { - case S_BOOLEAN: - item_make(menu, 't', "[%c]", - val == no ? ' ' : '*'); + switch (val) { + case yes: + ch = '*'; break; - case S_TRISTATE: - switch (val) { - case yes: - ch = '*'; - break; - case mod: - ch = 'M'; - break; - default: - ch = ' '; - break; - } - item_make(menu, 't', "<%c>", ch); + case mod: + ch = 'M'; + break; + default: + ch = ' '; break; } + item_make(menu, 't', "<%c>", ch); } else { item_make(menu, def_menu ? 't' : ':', " "); } From 648d82a984ba1c81b6cc2107917806864a9aa549 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 10 May 2024 19:23:23 +0900 Subject: [PATCH 1624/1702] kconfig: m/nconf: merge two item_add_str() calls Just trivial cleanups. Signed-off-by: Masahiro Yamada --- scripts/kconfig/mconf.c | 6 ++---- scripts/kconfig/nconf.c | 7 ++----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/scripts/kconfig/mconf.c b/scripts/kconfig/mconf.c index 90cd59a96aacb..d6a61ca1a9847 100644 --- a/scripts/kconfig/mconf.c +++ b/scripts/kconfig/mconf.c @@ -541,10 +541,8 @@ static void build_conf(struct menu *menu) item_add_str("%*c%s", indent + 1, ' ', menu_get_prompt(menu)); if (val == yes) { - if (def_menu) { - item_add_str(" (%s)", menu_get_prompt(def_menu)); - item_add_str(" --->"); - } + if (def_menu) + item_add_str(" (%s) --->", menu_get_prompt(def_menu)); return; } } else { diff --git a/scripts/kconfig/nconf.c b/scripts/kconfig/nconf.c index 93047cd28f3fd..e1cb09418cbe7 100644 --- a/scripts/kconfig/nconf.c +++ b/scripts/kconfig/nconf.c @@ -845,11 +845,8 @@ static void build_conf(struct menu *menu) item_add_str("%*c%s", indent + 1, ' ', menu_get_prompt(menu)); if (val == yes) { - if (def_menu) { - item_add_str(" (%s)", - menu_get_prompt(def_menu)); - item_add_str(" --->"); - } + if (def_menu) + item_add_str(" (%s) --->", menu_get_prompt(def_menu)); return; } } else { From 01b99162545b70656a486d34c5f39bbba75a5342 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 10 May 2024 21:08:09 +0900 Subject: [PATCH 1625/1702] kconfig: lxdialog: remove initialization with A_NORMAL A_NORMAL is zero, so the attribute is set to the default A_NORMAL without explicit assignment. Signed-off-by: Masahiro Yamada --- scripts/kconfig/lxdialog/util.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scripts/kconfig/lxdialog/util.c b/scripts/kconfig/lxdialog/util.c index f18e2a89f6135..964139c87fcb3 100644 --- a/scripts/kconfig/lxdialog/util.c +++ b/scripts/kconfig/lxdialog/util.c @@ -17,22 +17,13 @@ struct dialog_info dlg; static void set_mono_theme(void) { - dlg.screen.atr = A_NORMAL; - dlg.shadow.atr = A_NORMAL; - dlg.dialog.atr = A_NORMAL; dlg.title.atr = A_BOLD; - dlg.border.atr = A_NORMAL; dlg.button_active.atr = A_REVERSE; dlg.button_inactive.atr = A_DIM; dlg.button_key_active.atr = A_REVERSE; dlg.button_key_inactive.atr = A_BOLD; dlg.button_label_active.atr = A_REVERSE; - dlg.button_label_inactive.atr = A_NORMAL; - dlg.inputbox.atr = A_NORMAL; dlg.position_indicator.atr = A_BOLD; - dlg.menubox.atr = A_NORMAL; - dlg.menubox_border.atr = A_NORMAL; - dlg.item.atr = A_NORMAL; dlg.item_selected.atr = A_REVERSE; dlg.tag.atr = A_BOLD; dlg.tag_selected.atr = A_REVERSE; From bfb57ef0544ae6f67bf83430aa0bb877897da783 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 11 May 2024 09:09:53 +0900 Subject: [PATCH 1626/1702] rapidio: remove choice for enumeration This is the last use of the tristate choice. This choice was introduced a decade ago by commit a11650e11093 ("rapidio: make enumeration/discovery configurable"). Since then, RAPIDIO_ENUM_BASIC has always been the sole member. There was no need to have this choice block. Signed-off-by: Masahiro Yamada --- drivers/rapidio/Kconfig | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/drivers/rapidio/Kconfig b/drivers/rapidio/Kconfig index b9f8514909bf0..f1d742ac8f7c6 100644 --- a/drivers/rapidio/Kconfig +++ b/drivers/rapidio/Kconfig @@ -59,26 +59,13 @@ config RAPIDIO_DEBUG If you are unsure about this, say N here. -choice - prompt "Enumeration method" - depends on RAPIDIO - default RAPIDIO_ENUM_BASIC - help - There are different enumeration and discovery mechanisms offered - for RapidIO subsystem. You may select single built-in method or - or any number of methods to be built as modules. - Selecting a built-in method disables use of loadable methods. - - If unsure, select Basic built-in. - config RAPIDIO_ENUM_BASIC - tristate "Basic" + tristate "Basic Enumeration method" + depends on RAPIDIO help This option includes basic RapidIO fabric enumeration and discovery mechanism similar to one described in RapidIO specification Annex 1. -endchoice - config RAPIDIO_CHMAN tristate "RapidIO Channelized Messaging driver" depends on RAPIDIO From e60b613df8b6253def41215402f72986fee3fc8d Mon Sep 17 00:00:00 2001 From: Zheng Yejian Date: Fri, 10 May 2024 03:28:59 +0800 Subject: [PATCH 1627/1702] ftrace: Fix possible use-after-free issue in ftrace_location() KASAN reports a bug: BUG: KASAN: use-after-free in ftrace_location+0x90/0x120 Read of size 8 at addr ffff888141d40010 by task insmod/424 CPU: 8 PID: 424 Comm: insmod Tainted: G W 6.9.0-rc2+ [...] Call Trace: dump_stack_lvl+0x68/0xa0 print_report+0xcf/0x610 kasan_report+0xb5/0xe0 ftrace_location+0x90/0x120 register_kprobe+0x14b/0xa40 kprobe_init+0x2d/0xff0 [kprobe_example] do_one_initcall+0x8f/0x2d0 do_init_module+0x13a/0x3c0 load_module+0x3082/0x33d0 init_module_from_file+0xd2/0x130 __x64_sys_finit_module+0x306/0x440 do_syscall_64+0x68/0x140 entry_SYSCALL_64_after_hwframe+0x71/0x79 The root cause is that, in lookup_rec(), ftrace record of some address is being searched in ftrace pages of some module, but those ftrace pages at the same time is being freed in ftrace_release_mod() as the corresponding module is being deleted: CPU1 | CPU2 register_kprobes() { | delete_module() { check_kprobe_address_safe() { | arch_check_ftrace_location() { | ftrace_location() { | lookup_rec() // USE! | ftrace_release_mod() // Free! To fix this issue: 1. Hold rcu lock as accessing ftrace pages in ftrace_location_range(); 2. Use ftrace_location_range() instead of lookup_rec() in ftrace_location(); 3. Call synchronize_rcu() before freeing any ftrace pages both in ftrace_process_locs()/ftrace_release_mod()/ftrace_free_mem(). Link: https://lore.kernel.org/linux-trace-kernel/20240509192859.1273558-1-zhengyejian1@huawei.com Cc: stable@vger.kernel.org Cc: Cc: Cc: Fixes: ae6aa16fdc16 ("kprobes: introduce ftrace based optimization") Suggested-by: Steven Rostedt Signed-off-by: Zheng Yejian Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ftrace.c | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 5a01d72f66db5..2308c0a2fd29f 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1595,12 +1595,15 @@ static struct dyn_ftrace *lookup_rec(unsigned long start, unsigned long end) unsigned long ftrace_location_range(unsigned long start, unsigned long end) { struct dyn_ftrace *rec; + unsigned long ip = 0; + rcu_read_lock(); rec = lookup_rec(start, end); if (rec) - return rec->ip; + ip = rec->ip; + rcu_read_unlock(); - return 0; + return ip; } /** @@ -1614,25 +1617,22 @@ unsigned long ftrace_location_range(unsigned long start, unsigned long end) */ unsigned long ftrace_location(unsigned long ip) { - struct dyn_ftrace *rec; + unsigned long loc; unsigned long offset; unsigned long size; - rec = lookup_rec(ip, ip); - if (!rec) { + loc = ftrace_location_range(ip, ip); + if (!loc) { if (!kallsyms_lookup_size_offset(ip, &size, &offset)) goto out; /* map sym+0 to __fentry__ */ if (!offset) - rec = lookup_rec(ip, ip + size - 1); + loc = ftrace_location_range(ip, ip + size - 1); } - if (rec) - return rec->ip; - out: - return 0; + return loc; } /** @@ -6591,6 +6591,8 @@ static int ftrace_process_locs(struct module *mod, /* We should have used all pages unless we skipped some */ if (pg_unuse) { WARN_ON(!skipped); + /* Need to synchronize with ftrace_location_range() */ + synchronize_rcu(); ftrace_free_pages(pg_unuse); } return ret; @@ -6804,6 +6806,9 @@ void ftrace_release_mod(struct module *mod) out_unlock: mutex_unlock(&ftrace_lock); + /* Need to synchronize with ftrace_location_range() */ + if (tmp_page) + synchronize_rcu(); for (pg = tmp_page; pg; pg = tmp_page) { /* Needs to be called outside of ftrace_lock */ @@ -7137,6 +7142,7 @@ void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr) unsigned long start = (unsigned long)(start_ptr); unsigned long end = (unsigned long)(end_ptr); struct ftrace_page **last_pg = &ftrace_pages_start; + struct ftrace_page *tmp_page = NULL; struct ftrace_page *pg; struct dyn_ftrace *rec; struct dyn_ftrace key; @@ -7178,12 +7184,8 @@ void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr) ftrace_update_tot_cnt--; if (!pg->index) { *last_pg = pg->next; - if (pg->records) { - free_pages((unsigned long)pg->records, pg->order); - ftrace_number_of_pages -= 1 << pg->order; - } - ftrace_number_of_groups--; - kfree(pg); + pg->next = tmp_page; + tmp_page = pg; pg = container_of(last_pg, struct ftrace_page, next); if (!(*last_pg)) ftrace_pages = pg; @@ -7200,6 +7202,11 @@ void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr) clear_func_from_hashes(func); kfree(func); } + /* Need to synchronize with ftrace_location_range() */ + if (tmp_page) { + synchronize_rcu(); + ftrace_free_pages(tmp_page); + } } void __init ftrace_free_init_mem(void) From d4e9a968738bf66d3bb852dd5588d4c7afd6d7f4 Mon Sep 17 00:00:00 2001 From: Hao Ge Date: Mon, 13 May 2024 13:33:38 +0800 Subject: [PATCH 1628/1702] eventfs: Fix a possible null pointer dereference in eventfs_find_events() In function eventfs_find_events,there is a potential null pointer that may be caused by calling update_events_attr which will perform some operations on the members of the ei struct when ei is NULL. Hence,When ei->is_freed is set,return NULL directly. Link: https://lore.kernel.org/linux-trace-kernel/20240513053338.63017-1-hao.ge@linux.dev Cc: stable@vger.kernel.org Fixes: 8186fff7ab64 ("tracefs/eventfs: Use root and instance inodes as default ownership") Signed-off-by: Hao Ge Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index a878cea70f4c9..0256afdd4acfb 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -345,10 +345,9 @@ static struct eventfs_inode *eventfs_find_events(struct dentry *dentry) * If the ei is being freed, the ownership of the children * doesn't matter. */ - if (ei->is_freed) { - ei = NULL; - break; - } + if (ei->is_freed) + return NULL; + // Walk upwards until you find the events inode } while (!ei->is_events); From 0dc83ad8bfc9b5d40c1207674cecf77b85f54646 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Fri, 10 May 2024 08:59:01 +0200 Subject: [PATCH 1629/1702] clk: samsung: Don't register clkdev lookup for the fixed rate clocks Commit 4d11c62ca8d7 ("clkdev: report over-sized strings when creating clkdev entries") revealed that clock lookup is registered for all fixed clocks. The mentioned commit added a check if the registered name is not too long. This fails for some clocks registered for Exynos542x SoCs family. This lookup is a left-over from early common clock framework days, not really needed nowadays, so remove it to avoid further issues. Signed-off-by: Marek Szyprowski Reviewed-by: Sam Protsenko Tested-by: Sam Protsenko Link: https://lore.kernel.org/r/20240510065901.535124-1-m.szyprowski@samsung.com Signed-off-by: Stephen Boyd --- drivers/clk/samsung/clk.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/drivers/clk/samsung/clk.c b/drivers/clk/samsung/clk.c index b6701905f2546..afa5760ed3a11 100644 --- a/drivers/clk/samsung/clk.c +++ b/drivers/clk/samsung/clk.c @@ -139,7 +139,7 @@ void __init samsung_clk_register_fixed_rate(struct samsung_clk_provider *ctx, unsigned int nr_clk) { struct clk_hw *clk_hw; - unsigned int idx, ret; + unsigned int idx; for (idx = 0; idx < nr_clk; idx++, list++) { clk_hw = clk_hw_register_fixed_rate(ctx->dev, list->name, @@ -151,15 +151,6 @@ void __init samsung_clk_register_fixed_rate(struct samsung_clk_provider *ctx, } samsung_clk_add_lookup(ctx, clk_hw, list->id); - - /* - * Unconditionally add a clock lookup for the fixed rate clocks. - * There are not many of these on any of Samsung platforms. - */ - ret = clk_hw_register_clkdev(clk_hw, list->name, NULL); - if (ret) - pr_err("%s: failed to register clock lookup for %s", - __func__, list->name); } } From 0d8968287a1cf7b03d07387dc871de3861b9f6b9 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 13 May 2024 08:40:27 -0700 Subject: [PATCH 1630/1702] f2fs: Add inline to f2fs_build_fault_attr() stub When building without CONFIG_F2FS_FAULT_INJECTION, there is a warning from each file that includes f2fs.h because the stub for f2fs_build_fault_attr() is missing inline: In file included from fs/f2fs/segment.c:21: fs/f2fs/f2fs.h:4605:12: warning: 'f2fs_build_fault_attr' defined but not used [-Wunused-function] 4605 | static int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned long rate, | ^~~~~~~~~~~~~~~~~~~~~ Add the missing inline to resolve all of the warnings for this configuration. Fixes: 4ed886b187f4 ("f2fs: check validation of fault attrs in f2fs_build_fault_attr()") Signed-off-by: Nathan Chancellor Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/f2fs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index ef7de97be6474..1974b6aff397c 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -4602,8 +4602,8 @@ static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx) extern int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned long rate, unsigned long type); #else -static int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned long rate, - unsigned long type) +static inline int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, + unsigned long rate, unsigned long type) { return 0; } From 16409fdbb8828d7ae829bc4ac4e09e7ff02f8878 Mon Sep 17 00:00:00 2001 From: Wu Bo Date: Tue, 14 May 2024 05:35:29 -0600 Subject: [PATCH 1631/1702] f2fs: initialize last_block_in_bio variable Initialize last_block_in_bio of struct f2fs_bio_info and clean up code. Signed-off-by: Wu Bo Reviewed-by: Chao Yu Signed-off-by: Jaegeuk Kim --- fs/f2fs/data.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index bdbd4195cadc7..50ceb25b341de 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -595,17 +595,20 @@ int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi) return -ENOMEM; for (j = HOT; j < n; j++) { - init_f2fs_rwsem(&sbi->write_io[i][j].io_rwsem); - sbi->write_io[i][j].sbi = sbi; - sbi->write_io[i][j].bio = NULL; - spin_lock_init(&sbi->write_io[i][j].io_lock); - INIT_LIST_HEAD(&sbi->write_io[i][j].io_list); - INIT_LIST_HEAD(&sbi->write_io[i][j].bio_list); - init_f2fs_rwsem(&sbi->write_io[i][j].bio_list_lock); + struct f2fs_bio_info *io = &sbi->write_io[i][j]; + + init_f2fs_rwsem(&io->io_rwsem); + io->sbi = sbi; + io->bio = NULL; + io->last_block_in_bio = 0; + spin_lock_init(&io->io_lock); + INIT_LIST_HEAD(&io->io_list); + INIT_LIST_HEAD(&io->bio_list); + init_f2fs_rwsem(&io->bio_list_lock); #ifdef CONFIG_BLK_DEV_ZONED - init_completion(&sbi->write_io[i][j].zone_wait); - sbi->write_io[i][j].zone_pending_bio = NULL; - sbi->write_io[i][j].bi_private = NULL; + init_completion(&io->zone_wait); + io->zone_pending_bio = NULL; + io->bi_private = NULL; #endif } } From b9c6820f029abaabbc37646093866aa730ca0928 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 15 May 2024 01:05:58 -0400 Subject: [PATCH 1632/1702] ring-buffer: Add cast to unsigned long addr passed to virt_to_page() The sub-buffer pages are held in an unsigned long array, and when it is passed to virt_to_page() a cast is needed. Link: https://lore.kernel.org/all/20240515124808.06279d04@canb.auug.org.au/ Link: https://lore.kernel.org/linux-trace-kernel/20240515010558.4abaefdd@rorschach.local.home Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Fixes: 117c39200d9d ("ring-buffer: Introducing ring-buffer mapping functions") Reported-by: Stephen Rothwell Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ring_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index a02c7a52a0f5b..7345a8b625fb8 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -6283,7 +6283,7 @@ static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer, } while (p < nr_pages) { - struct page *page = virt_to_page(cpu_buffer->subbuf_ids[s]); + struct page *page = virt_to_page((void *)cpu_buffer->subbuf_ids[s]); int off = 0; if (WARN_ON_ONCE(s >= nr_subbufs)) { From e5bc44e47c531860be96ac615314b1ab23d5aa2b Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Thu, 25 Apr 2024 09:37:09 +0200 Subject: [PATCH 1633/1702] arch/topology: Fix variable naming to avoid shadowing Using 'hw_pressure' for local variable name is confusing in regard to the per-CPU 'hw_pressure' variable that uses the same name: include/linux/arch_topology.h:DECLARE_PER_CPU(unsigned long, hw_pressure); ... which puts it into a global scope for all code that includes , shadowing the local variable. Rename it to avoid compiler confusion & Sparse warnings. [ mingo: Expanded the changelog. ] Reported-by: kernel test robot Signed-off-by: Vincent Guittot Signed-off-by: Ingo Molnar Reviewed-by: Lukasz Luba Reviewed-by: Konrad Dybcio Acked-by: Sudeep Holla Cc: Linus Torvalds Link: https://lore.kernel.org/r/20240425073709.379016-1-vincent.guittot@linaro.org Closes: https://lore.kernel.org/oe-kbuild-all/202404250740.VhQQoD7N-lkp@intel.com/ Fixes: d4dbc991714e ("sched/cpufreq: Rename arch_update_thermal_pressure() => arch_update_hw_pressure()") Tested-by: Konrad Dybcio # QC SM8550 QRD --- drivers/base/arch_topology.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index 0248912ff6875..c66d070207a0e 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -179,7 +179,7 @@ DEFINE_PER_CPU(unsigned long, hw_pressure); void topology_update_hw_pressure(const struct cpumask *cpus, unsigned long capped_freq) { - unsigned long max_capacity, capacity, hw_pressure; + unsigned long max_capacity, capacity, pressure; u32 max_freq; int cpu; @@ -196,12 +196,12 @@ void topology_update_hw_pressure(const struct cpumask *cpus, else capacity = mult_frac(max_capacity, capped_freq, max_freq); - hw_pressure = max_capacity - capacity; + pressure = max_capacity - capacity; - trace_hw_pressure_update(cpu, hw_pressure); + trace_hw_pressure_update(cpu, pressure); for_each_cpu(cpu, cpus) - WRITE_ONCE(per_cpu(hw_pressure, cpu), hw_pressure); + WRITE_ONCE(per_cpu(hw_pressure, cpu), pressure); } EXPORT_SYMBOL_GPL(topology_update_hw_pressure); From 8bd67ebb50c0145fd2ca8681ab65eb7e8cde1afc Mon Sep 17 00:00:00 2001 From: Nikolay Aleksandrov Date: Mon, 13 May 2024 13:34:19 +0300 Subject: [PATCH 1634/1702] net: bridge: xmit: make sure we have at least eth header len bytes syzbot triggered an uninit value[1] error in bridge device's xmit path by sending a short (less than ETH_HLEN bytes) skb. To fix it check if we can actually pull that amount instead of assuming. Tested with dropwatch: drop at: br_dev_xmit+0xb93/0x12d0 [bridge] (0xffffffffc06739b3) origin: software timestamp: Mon May 13 11:31:53 2024 778214037 nsec protocol: 0x88a8 length: 2 original length: 2 drop reason: PKT_TOO_SMALL [1] BUG: KMSAN: uninit-value in br_dev_xmit+0x61d/0x1cb0 net/bridge/br_device.c:65 br_dev_xmit+0x61d/0x1cb0 net/bridge/br_device.c:65 __netdev_start_xmit include/linux/netdevice.h:4903 [inline] netdev_start_xmit include/linux/netdevice.h:4917 [inline] xmit_one net/core/dev.c:3531 [inline] dev_hard_start_xmit+0x247/0xa20 net/core/dev.c:3547 __dev_queue_xmit+0x34db/0x5350 net/core/dev.c:4341 dev_queue_xmit include/linux/netdevice.h:3091 [inline] __bpf_tx_skb net/core/filter.c:2136 [inline] __bpf_redirect_common net/core/filter.c:2180 [inline] __bpf_redirect+0x14a6/0x1620 net/core/filter.c:2187 ____bpf_clone_redirect net/core/filter.c:2460 [inline] bpf_clone_redirect+0x328/0x470 net/core/filter.c:2432 ___bpf_prog_run+0x13fe/0xe0f0 kernel/bpf/core.c:1997 __bpf_prog_run512+0xb5/0xe0 kernel/bpf/core.c:2238 bpf_dispatcher_nop_func include/linux/bpf.h:1234 [inline] __bpf_prog_run include/linux/filter.h:657 [inline] bpf_prog_run include/linux/filter.h:664 [inline] bpf_test_run+0x499/0xc30 net/bpf/test_run.c:425 bpf_prog_test_run_skb+0x14ea/0x1f20 net/bpf/test_run.c:1058 bpf_prog_test_run+0x6b7/0xad0 kernel/bpf/syscall.c:4269 __sys_bpf+0x6aa/0xd90 kernel/bpf/syscall.c:5678 __do_sys_bpf kernel/bpf/syscall.c:5767 [inline] __se_sys_bpf kernel/bpf/syscall.c:5765 [inline] __x64_sys_bpf+0xa0/0xe0 kernel/bpf/syscall.c:5765 x64_sys_call+0x96b/0x3b50 arch/x86/include/generated/asm/syscalls_64.h:322 do_syscall_x64 arch/x86/entry/common.c:52 [inline] do_syscall_64+0xcf/0x1e0 arch/x86/entry/common.c:83 entry_SYSCALL_64_after_hwframe+0x77/0x7f Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Reported-by: syzbot+a63a1f6a062033cf0f40@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=a63a1f6a062033cf0f40 Signed-off-by: Nikolay Aleksandrov Signed-off-by: David S. Miller --- net/bridge/br_device.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c index c366ccc8b3db7..ecac7886988b1 100644 --- a/net/bridge/br_device.c +++ b/net/bridge/br_device.c @@ -27,6 +27,7 @@ EXPORT_SYMBOL_GPL(nf_br_ops); /* net device transmit always called with BH disabled */ netdev_tx_t br_dev_xmit(struct sk_buff *skb, struct net_device *dev) { + enum skb_drop_reason reason = pskb_may_pull_reason(skb, ETH_HLEN); struct net_bridge_mcast_port *pmctx_null = NULL; struct net_bridge *br = netdev_priv(dev); struct net_bridge_mcast *brmctx = &br->multicast_ctx; @@ -38,6 +39,11 @@ netdev_tx_t br_dev_xmit(struct sk_buff *skb, struct net_device *dev) const unsigned char *dest; u16 vid = 0; + if (unlikely(reason != SKB_NOT_DROPPED_YET)) { + kfree_skb_reason(skb, reason); + return NETDEV_TX_OK; + } + memset(skb->cb, 0, sizeof(struct br_input_skb_cb)); br_tc_skb_miss_set(skb, false); From 06080ea23095afe04a2cb7a8d05fab4311782623 Mon Sep 17 00:00:00 2001 From: Nikolay Aleksandrov Date: Mon, 13 May 2024 13:52:57 +0300 Subject: [PATCH 1635/1702] selftests: net: bridge: increase IGMP/MLD exclude timeout membership interval When running the bridge IGMP/MLD selftests on debug kernels we can get spurious errors when setting up the IGMP/MLD exclude timeout tests because the membership interval is just 3 seconds and the setup has 2 seconds of sleep plus various validations, the one second that is left is not enough. Increase the membership interval from 3 to 5 seconds to make room for the setup validation and 2 seconds of sleep. Fixes: 34d7ecb3d4f7 ("selftests: net: bridge: update IGMP/MLD membership interval value") Reported-by: Jakub Kicinski Signed-off-by: Nikolay Aleksandrov Signed-off-by: David S. Miller --- tools/testing/selftests/net/forwarding/bridge_igmp.sh | 6 +++--- tools/testing/selftests/net/forwarding/bridge_mld.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/net/forwarding/bridge_igmp.sh b/tools/testing/selftests/net/forwarding/bridge_igmp.sh index 2aa66d2a1702b..e6a3e04fd83f3 100755 --- a/tools/testing/selftests/net/forwarding/bridge_igmp.sh +++ b/tools/testing/selftests/net/forwarding/bridge_igmp.sh @@ -478,10 +478,10 @@ v3exc_timeout_test() RET=0 local X=("192.0.2.20" "192.0.2.30") - # GMI should be 3 seconds + # GMI should be 5 seconds ip link set dev br0 type bridge mcast_query_interval 100 \ mcast_query_response_interval 100 \ - mcast_membership_interval 300 + mcast_membership_interval 500 v3exclude_prepare $h1 $ALL_MAC $ALL_GROUP ip link set dev br0 type bridge mcast_query_interval 500 \ @@ -489,7 +489,7 @@ v3exc_timeout_test() mcast_membership_interval 1500 $MZ $h1 -c 1 -b $ALL_MAC -B $ALL_GROUP -t ip "proto=2,p=$MZPKT_ALLOW2" -q - sleep 3 + sleep 5 bridge -j -d -s mdb show dev br0 \ | jq -e ".[].mdb[] | \ select(.grp == \"$TEST_GROUP\" and \ diff --git a/tools/testing/selftests/net/forwarding/bridge_mld.sh b/tools/testing/selftests/net/forwarding/bridge_mld.sh index e2b9ff773c6b6..f84ab2e657547 100755 --- a/tools/testing/selftests/net/forwarding/bridge_mld.sh +++ b/tools/testing/selftests/net/forwarding/bridge_mld.sh @@ -478,10 +478,10 @@ mldv2exc_timeout_test() RET=0 local X=("2001:db8:1::20" "2001:db8:1::30") - # GMI should be 3 seconds + # GMI should be 5 seconds ip link set dev br0 type bridge mcast_query_interval 100 \ mcast_query_response_interval 100 \ - mcast_membership_interval 300 + mcast_membership_interval 500 mldv2exclude_prepare $h1 ip link set dev br0 type bridge mcast_query_interval 500 \ @@ -489,7 +489,7 @@ mldv2exc_timeout_test() mcast_membership_interval 1500 $MZ $h1 -c 1 $MZPKT_ALLOW2 -q - sleep 3 + sleep 5 bridge -j -d -s mdb show dev br0 \ | jq -e ".[].mdb[] | \ select(.grp == \"$TEST_GROUP\" and \ From 3a7c1661ae1383364cd6092d851f5e5da64d476b Mon Sep 17 00:00:00 2001 From: Nikolay Aleksandrov Date: Mon, 13 May 2024 14:06:27 +0300 Subject: [PATCH 1636/1702] net: bridge: mst: fix vlan use-after-free syzbot reported a suspicious rcu usage[1] in bridge's mst code. While fixing it I noticed that nothing prevents a vlan to be freed while walking the list from the same path (br forward delay timer). Fix the rcu usage and also make sure we are not accessing freed memory by making br_mst_vlan_set_state use rcu read lock. [1] WARNING: suspicious RCU usage 6.9.0-rc6-syzkaller #0 Not tainted ----------------------------- net/bridge/br_private.h:1599 suspicious rcu_dereference_protected() usage! ... stack backtrace: CPU: 1 PID: 8017 Comm: syz-executor.1 Not tainted 6.9.0-rc6-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 03/27/2024 Call Trace: __dump_stack lib/dump_stack.c:88 [inline] dump_stack_lvl+0x241/0x360 lib/dump_stack.c:114 lockdep_rcu_suspicious+0x221/0x340 kernel/locking/lockdep.c:6712 nbp_vlan_group net/bridge/br_private.h:1599 [inline] br_mst_set_state+0x1ea/0x650 net/bridge/br_mst.c:105 br_set_state+0x28a/0x7b0 net/bridge/br_stp.c:47 br_forward_delay_timer_expired+0x176/0x440 net/bridge/br_stp_timer.c:88 call_timer_fn+0x18e/0x650 kernel/time/timer.c:1793 expire_timers kernel/time/timer.c:1844 [inline] __run_timers kernel/time/timer.c:2418 [inline] __run_timer_base+0x66a/0x8e0 kernel/time/timer.c:2429 run_timer_base kernel/time/timer.c:2438 [inline] run_timer_softirq+0xb7/0x170 kernel/time/timer.c:2448 __do_softirq+0x2c6/0x980 kernel/softirq.c:554 invoke_softirq kernel/softirq.c:428 [inline] __irq_exit_rcu+0xf2/0x1c0 kernel/softirq.c:633 irq_exit_rcu+0x9/0x30 kernel/softirq.c:645 instr_sysvec_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1043 [inline] sysvec_apic_timer_interrupt+0xa6/0xc0 arch/x86/kernel/apic/apic.c:1043 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:702 RIP: 0010:lock_acquire+0x264/0x550 kernel/locking/lockdep.c:5758 Code: 2b 00 74 08 4c 89 f7 e8 ba d1 84 00 f6 44 24 61 02 0f 85 85 01 00 00 41 f7 c7 00 02 00 00 74 01 fb 48 c7 44 24 40 0e 36 e0 45 <4b> c7 44 25 00 00 00 00 00 43 c7 44 25 09 00 00 00 00 43 c7 44 25 RSP: 0018:ffffc90013657100 EFLAGS: 00000206 RAX: 0000000000000001 RBX: 1ffff920026cae2c RCX: 0000000000000001 RDX: dffffc0000000000 RSI: ffffffff8bcaca00 RDI: ffffffff8c1eaa60 RBP: ffffc90013657260 R08: ffffffff92efe507 R09: 1ffffffff25dfca0 R10: dffffc0000000000 R11: fffffbfff25dfca1 R12: 1ffff920026cae28 R13: dffffc0000000000 R14: ffffc90013657160 R15: 0000000000000246 Fixes: ec7328b59176 ("net: bridge: mst: Multiple Spanning Tree (MST) mode") Reported-by: syzbot+fa04eb8a56fd923fc5d8@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=fa04eb8a56fd923fc5d8 Signed-off-by: Nikolay Aleksandrov Reviewed-by: Simon Horman Signed-off-by: David S. Miller --- net/bridge/br_mst.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/net/bridge/br_mst.c b/net/bridge/br_mst.c index ee680adcee179..3c66141d34d62 100644 --- a/net/bridge/br_mst.c +++ b/net/bridge/br_mst.c @@ -78,7 +78,7 @@ static void br_mst_vlan_set_state(struct net_bridge_port *p, struct net_bridge_v { struct net_bridge_vlan_group *vg = nbp_vlan_group(p); - if (v->state == state) + if (br_vlan_get_state(v) == state) return; br_vlan_set_state(v, state); @@ -100,11 +100,12 @@ int br_mst_set_state(struct net_bridge_port *p, u16 msti, u8 state, }; struct net_bridge_vlan_group *vg; struct net_bridge_vlan *v; - int err; + int err = 0; + rcu_read_lock(); vg = nbp_vlan_group(p); if (!vg) - return 0; + goto out; /* MSTI 0 (CST) state changes are notified via the regular * SWITCHDEV_ATTR_ID_PORT_STP_STATE. @@ -112,17 +113,20 @@ int br_mst_set_state(struct net_bridge_port *p, u16 msti, u8 state, if (msti) { err = switchdev_port_attr_set(p->dev, &attr, extack); if (err && err != -EOPNOTSUPP) - return err; + goto out; } - list_for_each_entry(v, &vg->vlan_list, vlist) { + err = 0; + list_for_each_entry_rcu(v, &vg->vlan_list, vlist) { if (v->brvlan->msti != msti) continue; br_mst_vlan_set_state(p, v, state); } - return 0; +out: + rcu_read_unlock(); + return err; } static void br_mst_vlan_sync_state(struct net_bridge_vlan *pv, u16 msti) From 317a215d493230da361028ea8a4675de334bfa1a Mon Sep 17 00:00:00 2001 From: Ronald Wahl Date: Mon, 13 May 2024 16:39:22 +0200 Subject: [PATCH 1637/1702] net: ks8851: Fix another TX stall caused by wrong ISR flag handling Under some circumstances it may happen that the ks8851 Ethernet driver stops sending data. Currently the interrupt handler resets the interrupt status flags in the hardware after handling TX. With this approach we may lose interrupts in the time window between handling the TX interrupt and resetting the TX interrupt status bit. When all of the three following conditions are true then transmitting data stops: - TX queue is stopped to wait for room in the hardware TX buffer - no queued SKBs in the driver (txq) that wait for being written to hw - hardware TX buffer is empty and the last TX interrupt was lost This is because reenabling the TX queue happens when handling the TX interrupt status but if the TX status bit has already been cleared then this interrupt will never come. With this commit the interrupt status flags will be cleared before they are handled. That way we stop losing interrupts. The wrong handling of the ISR flags was there from the beginning but with commit 3dc5d4454545 ("net: ks8851: Fix TX stall caused by TX buffer overrun") the issue becomes apparent. Fixes: 3dc5d4454545 ("net: ks8851: Fix TX stall caused by TX buffer overrun") Cc: "David S. Miller" Cc: Eric Dumazet Cc: Jakub Kicinski Cc: Paolo Abeni Cc: Simon Horman Cc: netdev@vger.kernel.org Cc: stable@vger.kernel.org # 5.10+ Signed-off-by: Ronald Wahl Reviewed-by: Simon Horman Signed-off-by: David S. Miller --- drivers/net/ethernet/micrel/ks8851_common.c | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/drivers/net/ethernet/micrel/ks8851_common.c b/drivers/net/ethernet/micrel/ks8851_common.c index 502518cdb4618..6453c92f0fa7c 100644 --- a/drivers/net/ethernet/micrel/ks8851_common.c +++ b/drivers/net/ethernet/micrel/ks8851_common.c @@ -328,7 +328,6 @@ static irqreturn_t ks8851_irq(int irq, void *_ks) { struct ks8851_net *ks = _ks; struct sk_buff_head rxq; - unsigned handled = 0; unsigned long flags; unsigned int status; struct sk_buff *skb; @@ -336,24 +335,17 @@ static irqreturn_t ks8851_irq(int irq, void *_ks) ks8851_lock(ks, &flags); status = ks8851_rdreg16(ks, KS_ISR); + ks8851_wrreg16(ks, KS_ISR, status); netif_dbg(ks, intr, ks->netdev, "%s: status 0x%04x\n", __func__, status); - if (status & IRQ_LCI) - handled |= IRQ_LCI; - if (status & IRQ_LDI) { u16 pmecr = ks8851_rdreg16(ks, KS_PMECR); pmecr &= ~PMECR_WKEVT_MASK; ks8851_wrreg16(ks, KS_PMECR, pmecr | PMECR_WKEVT_LINK); - - handled |= IRQ_LDI; } - if (status & IRQ_RXPSI) - handled |= IRQ_RXPSI; - if (status & IRQ_TXI) { unsigned short tx_space = ks8851_rdreg16(ks, KS_TXMIR); @@ -365,20 +357,12 @@ static irqreturn_t ks8851_irq(int irq, void *_ks) if (netif_queue_stopped(ks->netdev)) netif_wake_queue(ks->netdev); spin_unlock(&ks->statelock); - - handled |= IRQ_TXI; } - if (status & IRQ_RXI) - handled |= IRQ_RXI; - if (status & IRQ_SPIBEI) { netdev_err(ks->netdev, "%s: spi bus error\n", __func__); - handled |= IRQ_SPIBEI; } - ks8851_wrreg16(ks, KS_ISR, handled); - if (status & IRQ_RXI) { /* the datasheet says to disable the rx interrupt during * packet read-out, however we're masking the interrupt From 5f0769331a965675cdfec97c09f3f6e875d7c246 Mon Sep 17 00:00:00 2001 From: Daniel Bristot de Oliveira Date: Wed, 24 Apr 2024 16:36:50 +0200 Subject: [PATCH 1638/1702] rtla/timerlat: Simplify "no value" printing on top Instead of printing three times the same output, print it only once, reducing lines and being sure that all no values have the same length. It also fixes an extra '\n' when running the with kernel threads, like here: =============== %< ============== Timer Latency 0 00:00:01 | IRQ Timer Latency (us) | Thread Timer Latency (us) CPU COUNT | cur min avg max | cur min avg max 2 #0 | - - - - | 161 161 161 161 3 #0 | - - - - | 161 161 161 161 8 #1 | 54 54 54 54 | - - - -'\n' ---------------|----------------------------------------|--------------------------------------- ALL #1 e0 | 54 54 54 | 161 161 161 =============== %< ============== This '\n' should have been removed with the user-space support that added another '\n' if not running with kernel threads. Link: https://lkml.kernel.org/r/0a4d8085e7cd706733a5dc10a81ca38b82bd4992.1713968967.git.bristot@kernel.org Cc: stable@vger.kernel.org Cc: Jonathan Corbet Cc: Juri Lelli Fixes: cdca4f4e5e8e ("rtla/timerlat_top: Add timerlat user-space support") Signed-off-by: Daniel Bristot de Oliveira --- tools/tracing/rtla/src/timerlat_top.c | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index 8a3fa64319c6d..2665e0bb5f1ee 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -212,6 +212,8 @@ static void timerlat_top_header(struct osnoise_tool *top) trace_seq_printf(s, "\n"); } +static const char *no_value = " -"; + /* * timerlat_top_print - prints the output of a given CPU */ @@ -239,10 +241,7 @@ static void timerlat_top_print(struct osnoise_tool *top, int cpu) trace_seq_printf(s, "%3d #%-9d |", cpu, cpu_data->irq_count); if (!cpu_data->irq_count) { - trace_seq_printf(s, " - "); - trace_seq_printf(s, " - "); - trace_seq_printf(s, " - "); - trace_seq_printf(s, " - |"); + trace_seq_printf(s, "%s %s %s %s |", no_value, no_value, no_value, no_value); } else { trace_seq_printf(s, "%9llu ", cpu_data->cur_irq / params->output_divisor); trace_seq_printf(s, "%9llu ", cpu_data->min_irq / params->output_divisor); @@ -251,10 +250,7 @@ static void timerlat_top_print(struct osnoise_tool *top, int cpu) } if (!cpu_data->thread_count) { - trace_seq_printf(s, " - "); - trace_seq_printf(s, " - "); - trace_seq_printf(s, " - "); - trace_seq_printf(s, " -\n"); + trace_seq_printf(s, "%s %s %s %s", no_value, no_value, no_value, no_value); } else { trace_seq_printf(s, "%9llu ", cpu_data->cur_thread / divisor); trace_seq_printf(s, "%9llu ", cpu_data->min_thread / divisor); @@ -271,10 +267,7 @@ static void timerlat_top_print(struct osnoise_tool *top, int cpu) trace_seq_printf(s, " |"); if (!cpu_data->user_count) { - trace_seq_printf(s, " - "); - trace_seq_printf(s, " - "); - trace_seq_printf(s, " - "); - trace_seq_printf(s, " -\n"); + trace_seq_printf(s, "%s %s %s %s\n", no_value, no_value, no_value, no_value); } else { trace_seq_printf(s, "%9llu ", cpu_data->cur_user / divisor); trace_seq_printf(s, "%9llu ", cpu_data->min_user / divisor); From a40e5e4dd0207485dee75e2b8e860d5853bcc5f7 Mon Sep 17 00:00:00 2001 From: Daniel Bristot de Oliveira Date: Wed, 24 Apr 2024 16:36:51 +0200 Subject: [PATCH 1639/1702] rtla/auto-analysis: Replace \t with spaces When copying timerlat auto-analysis from a terminal to some web pages or chats, the \t are being replaced with a single ' ' or ' ', breaking the output. For example: ## CPU 3 hit stop tracing, analyzing it ## IRQ handler delay: 1.30 us (0.11 %) IRQ latency: 1.90 us Timerlat IRQ duration: 3.00 us (0.24 %) Blocking thread: 1223.16 us (99.00 %) insync:4048 1223.16 us IRQ interference 4.93 us (0.40 %) local_timer:236 4.93 us ------------------------------------------------------------------------ Thread latency: 1235.47 us (100%) Replace \t with spaces to avoid this problem. Link: https://lkml.kernel.org/r/ec7ed2b2809c22ab0dfc8eb7c805ab9cddc4254a.1713968967.git.bristot@kernel.org Cc: stable@vger.kernel.org Cc: Jonathan Corbet Cc: Juri Lelli Fixes: 27e348b221f6 ("rtla/timerlat: Add auto-analysis core") Signed-off-by: Daniel Bristot de Oliveira --- tools/tracing/rtla/src/timerlat_aa.c | 109 ++++++++++++++++----------- 1 file changed, 63 insertions(+), 46 deletions(-) diff --git a/tools/tracing/rtla/src/timerlat_aa.c b/tools/tracing/rtla/src/timerlat_aa.c index 7093fd5333beb..7bd80ee2a5b48 100644 --- a/tools/tracing/rtla/src/timerlat_aa.c +++ b/tools/tracing/rtla/src/timerlat_aa.c @@ -16,6 +16,9 @@ enum timelat_state { TIMERLAT_WAITING_THREAD, }; +/* Used to fill spaces in the output */ +static const char *spaces = " "; + #define MAX_COMM 24 /* @@ -274,14 +277,17 @@ static int timerlat_aa_nmi_handler(struct trace_seq *s, struct tep_record *recor taa_data->prev_irq_timstamp = start; trace_seq_reset(taa_data->prev_irqs_seq); - trace_seq_printf(taa_data->prev_irqs_seq, "\t%24s \t\t\t%9.2f us\n", - "nmi", ns_to_usf(duration)); + trace_seq_printf(taa_data->prev_irqs_seq, " %24s %.*s %9.2f us\n", + "nmi", + 24, spaces, + ns_to_usf(duration)); return 0; } taa_data->thread_nmi_sum += duration; - trace_seq_printf(taa_data->nmi_seq, " %24s \t\t\t%9.2f us\n", - "nmi", ns_to_usf(duration)); + trace_seq_printf(taa_data->nmi_seq, " %24s %.*s %9.2f us\n", + "nmi", + 24, spaces, ns_to_usf(duration)); return 0; } @@ -323,8 +329,10 @@ static int timerlat_aa_irq_handler(struct trace_seq *s, struct tep_record *recor taa_data->prev_irq_timstamp = start; trace_seq_reset(taa_data->prev_irqs_seq); - trace_seq_printf(taa_data->prev_irqs_seq, "\t%24s:%-3llu \t\t%9.2f us\n", - desc, vector, ns_to_usf(duration)); + trace_seq_printf(taa_data->prev_irqs_seq, " %24s:%-3llu %.*s %9.2f us\n", + desc, vector, + 15, spaces, + ns_to_usf(duration)); return 0; } @@ -372,8 +380,10 @@ static int timerlat_aa_irq_handler(struct trace_seq *s, struct tep_record *recor * IRQ interference. */ taa_data->thread_irq_sum += duration; - trace_seq_printf(taa_data->irqs_seq, " %24s:%-3llu \t %9.2f us\n", - desc, vector, ns_to_usf(duration)); + trace_seq_printf(taa_data->irqs_seq, " %24s:%-3llu %.*s %9.2f us\n", + desc, vector, + 24, spaces, + ns_to_usf(duration)); return 0; } @@ -408,8 +418,10 @@ static int timerlat_aa_softirq_handler(struct trace_seq *s, struct tep_record *r taa_data->thread_softirq_sum += duration; - trace_seq_printf(taa_data->softirqs_seq, "\t%24s:%-3llu \t %9.2f us\n", - softirq_name[vector], vector, ns_to_usf(duration)); + trace_seq_printf(taa_data->softirqs_seq, " %24s:%-3llu %.*s %9.2f us\n", + softirq_name[vector], vector, + 24, spaces, + ns_to_usf(duration)); return 0; } @@ -452,8 +464,10 @@ static int timerlat_aa_thread_handler(struct trace_seq *s, struct tep_record *re } else { taa_data->thread_thread_sum += duration; - trace_seq_printf(taa_data->threads_seq, "\t%24s:%-3llu \t\t%9.2f us\n", - comm, pid, ns_to_usf(duration)); + trace_seq_printf(taa_data->threads_seq, " %24s:%-12llu %.*s %9.2f us\n", + comm, pid, + 15, spaces, + ns_to_usf(duration)); } return 0; @@ -482,7 +496,8 @@ static int timerlat_aa_stack_handler(struct trace_seq *s, struct tep_record *rec function = tep_find_function(taa_ctx->tool->trace.tep, caller[i]); if (!function) break; - trace_seq_printf(taa_data->stack_seq, "\t\t-> %s\n", function); + trace_seq_printf(taa_data->stack_seq, " %.*s -> %s\n", + 14, spaces, function); } } return 0; @@ -568,23 +583,24 @@ static void timerlat_thread_analysis(struct timerlat_aa_data *taa_data, int cpu, exp_irq_ts = taa_data->timer_irq_start_time - taa_data->timer_irq_start_delay; if (exp_irq_ts < taa_data->prev_irq_timstamp + taa_data->prev_irq_duration) { if (taa_data->prev_irq_timstamp < taa_data->timer_irq_start_time) - printf(" Previous IRQ interference: \t\t up to %9.2f us\n", - ns_to_usf(taa_data->prev_irq_duration)); + printf(" Previous IRQ interference: %.*s up to %9.2f us\n", + 16, spaces, + ns_to_usf(taa_data->prev_irq_duration)); } /* * The delay that the IRQ suffered before starting. */ - printf(" IRQ handler delay: %16s %9.2f us (%.2f %%)\n", - (ns_to_usf(taa_data->timer_exit_from_idle) > 10) ? "(exit from idle)" : "", - ns_to_usf(taa_data->timer_irq_start_delay), - ns_to_per(total, taa_data->timer_irq_start_delay)); + printf(" IRQ handler delay: %.*s %16s %9.2f us (%.2f %%)\n", 16, spaces, + (ns_to_usf(taa_data->timer_exit_from_idle) > 10) ? "(exit from idle)" : "", + ns_to_usf(taa_data->timer_irq_start_delay), + ns_to_per(total, taa_data->timer_irq_start_delay)); /* * Timerlat IRQ. */ - printf(" IRQ latency: \t\t\t\t %9.2f us\n", - ns_to_usf(taa_data->tlat_irq_latency)); + printf(" IRQ latency: %.*s %9.2f us\n", 40, spaces, + ns_to_usf(taa_data->tlat_irq_latency)); if (irq) { /* @@ -595,15 +611,16 @@ static void timerlat_thread_analysis(struct timerlat_aa_data *taa_data, int cpu, * so it will be displayed, it is the key. */ printf(" Blocking thread:\n"); - printf(" %24s:%-9llu\n", - taa_data->run_thread_comm, taa_data->run_thread_pid); + printf(" %.*s %24s:%-9llu\n", 6, spaces, taa_data->run_thread_comm, + taa_data->run_thread_pid); } else { /* * The duration of the IRQ handler that handled the timerlat IRQ. */ - printf(" Timerlat IRQ duration: \t\t %9.2f us (%.2f %%)\n", - ns_to_usf(taa_data->timer_irq_duration), - ns_to_per(total, taa_data->timer_irq_duration)); + printf(" Timerlat IRQ duration: %.*s %9.2f us (%.2f %%)\n", + 30, spaces, + ns_to_usf(taa_data->timer_irq_duration), + ns_to_per(total, taa_data->timer_irq_duration)); /* * The amount of time that the current thread postponed the scheduler. @@ -611,13 +628,13 @@ static void timerlat_thread_analysis(struct timerlat_aa_data *taa_data, int cpu, * Recalling that it is net from NMI/IRQ/Softirq interference, so there * is no need to compute values here. */ - printf(" Blocking thread: \t\t\t %9.2f us (%.2f %%)\n", - ns_to_usf(taa_data->thread_blocking_duration), - ns_to_per(total, taa_data->thread_blocking_duration)); + printf(" Blocking thread: %.*s %9.2f us (%.2f %%)\n", 36, spaces, + ns_to_usf(taa_data->thread_blocking_duration), + ns_to_per(total, taa_data->thread_blocking_duration)); - printf(" %24s:%-9llu %9.2f us\n", - taa_data->run_thread_comm, taa_data->run_thread_pid, - ns_to_usf(taa_data->thread_blocking_duration)); + printf(" %.*s %24s:%-9llu %.*s %9.2f us\n", 6, spaces, + taa_data->run_thread_comm, taa_data->run_thread_pid, + 12, spaces, ns_to_usf(taa_data->thread_blocking_duration)); } /* @@ -629,9 +646,9 @@ static void timerlat_thread_analysis(struct timerlat_aa_data *taa_data, int cpu, * NMIs can happen during the IRQ, so they are always possible. */ if (taa_data->thread_nmi_sum) - printf(" NMI interference \t\t\t %9.2f us (%.2f %%)\n", - ns_to_usf(taa_data->thread_nmi_sum), - ns_to_per(total, taa_data->thread_nmi_sum)); + printf(" NMI interference %.*s %9.2f us (%.2f %%)\n", 36, spaces, + ns_to_usf(taa_data->thread_nmi_sum), + ns_to_per(total, taa_data->thread_nmi_sum)); /* * If it is an IRQ latency, the other factors can be skipped. @@ -643,9 +660,9 @@ static void timerlat_thread_analysis(struct timerlat_aa_data *taa_data, int cpu, * Prints the interference caused by IRQs to the thread latency. */ if (taa_data->thread_irq_sum) { - printf(" IRQ interference \t\t\t %9.2f us (%.2f %%)\n", - ns_to_usf(taa_data->thread_irq_sum), - ns_to_per(total, taa_data->thread_irq_sum)); + printf(" IRQ interference %.*s %9.2f us (%.2f %%)\n", 36, spaces, + ns_to_usf(taa_data->thread_irq_sum), + ns_to_per(total, taa_data->thread_irq_sum)); trace_seq_do_printf(taa_data->irqs_seq); } @@ -654,9 +671,9 @@ static void timerlat_thread_analysis(struct timerlat_aa_data *taa_data, int cpu, * Prints the interference caused by Softirqs to the thread latency. */ if (taa_data->thread_softirq_sum) { - printf(" Softirq interference \t\t\t %9.2f us (%.2f %%)\n", - ns_to_usf(taa_data->thread_softirq_sum), - ns_to_per(total, taa_data->thread_softirq_sum)); + printf(" Softirq interference %.*s %9.2f us (%.2f %%)\n", 32, spaces, + ns_to_usf(taa_data->thread_softirq_sum), + ns_to_per(total, taa_data->thread_softirq_sum)); trace_seq_do_printf(taa_data->softirqs_seq); } @@ -670,9 +687,9 @@ static void timerlat_thread_analysis(struct timerlat_aa_data *taa_data, int cpu, * timer handling latency. */ if (taa_data->thread_thread_sum) { - printf(" Thread interference \t\t\t %9.2f us (%.2f %%)\n", - ns_to_usf(taa_data->thread_thread_sum), - ns_to_per(total, taa_data->thread_thread_sum)); + printf(" Thread interference %.*s %9.2f us (%.2f %%)\n", 33, spaces, + ns_to_usf(taa_data->thread_thread_sum), + ns_to_per(total, taa_data->thread_thread_sum)); trace_seq_do_printf(taa_data->threads_seq); } @@ -682,8 +699,8 @@ static void timerlat_thread_analysis(struct timerlat_aa_data *taa_data, int cpu, */ print_total: printf("------------------------------------------------------------------------\n"); - printf(" %s latency: \t\t\t %9.2f us (100%%)\n", irq ? "IRQ" : "Thread", - ns_to_usf(total)); + printf(" %s latency: %.*s %9.2f us (100%%)\n", irq ? " IRQ" : "Thread", + 37, spaces, ns_to_usf(total)); } static int timerlat_auto_analysis_collect_trace(struct timerlat_aa_context *taa_ctx) From f5c0cdad6684aa4212346f48554636ec2ab98434 Mon Sep 17 00:00:00 2001 From: Daniel Bristot de Oliveira Date: Wed, 24 Apr 2024 16:36:52 +0200 Subject: [PATCH 1640/1702] rtla/timerlat: Use pretty formatting only on interactive tty timerlat top does some background/font color formatting. While useful on terminal, it breaks the output on other formats. For example, when piping the output for pastebin tools, the format strings are printed as characters. For instance: [2;37;40m Timer Latency [0;0;0m 0 00:00:01 | IRQ Timer Latency (us) | Thread Timer Latency (us) [2;30;47mCPU COUNT | cur min avg max | cur min avg max[0;0;0m 0 #1013 | 1 0 1 54 | 5 2 4 57 1 #1013 | 3 0 1 10 | 6 2 4 15 To avoid this problem, do the formatting only if running on a tty, and in !quiet mode. Link: https://lkml.kernel.org/r/8288e1544ceab21557d5dda93a0f00339497c649.1713968967.git.bristot@kernel.org Cc: Jonathan Corbet Cc: Juri Lelli Signed-off-by: Daniel Bristot de Oliveira --- tools/tracing/rtla/src/timerlat_top.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index 2665e0bb5f1ee..c9cf90ed4e6df 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -44,6 +44,7 @@ struct timerlat_top_params { int hk_cpus; int user_top; int user_workload; + int pretty_output; cpu_set_t hk_cpu_set; struct sched_attr sched_param; struct trace_events *events; @@ -179,19 +180,22 @@ timerlat_top_handler(struct trace_seq *s, struct tep_record *record, /* * timerlat_top_header - print the header of the tool output */ -static void timerlat_top_header(struct osnoise_tool *top) +static void timerlat_top_header(struct timerlat_top_params *params, struct osnoise_tool *top) { - struct timerlat_top_params *params = top->params; struct trace_seq *s = top->trace.seq; char duration[26]; get_duration(top->start_time, duration, sizeof(duration)); - trace_seq_printf(s, "\033[2;37;40m"); + if (params->pretty_output) + trace_seq_printf(s, "\033[2;37;40m"); + trace_seq_printf(s, " Timer Latency "); if (params->user_top) trace_seq_printf(s, " "); - trace_seq_printf(s, "\033[0;0;0m"); + + if (params->pretty_output) + trace_seq_printf(s, "\033[0;0;0m"); trace_seq_printf(s, "\n"); trace_seq_printf(s, "%-6s | IRQ Timer Latency (%s) | Thread Timer Latency (%s)", duration, @@ -204,11 +208,15 @@ static void timerlat_top_header(struct osnoise_tool *top) } trace_seq_printf(s, "\n"); - trace_seq_printf(s, "\033[2;30;47m"); + if (params->pretty_output) + trace_seq_printf(s, "\033[2;30;47m"); + trace_seq_printf(s, "CPU COUNT | cur min avg max | cur min avg max"); if (params->user_top) trace_seq_printf(s, " | cur min avg max"); - trace_seq_printf(s, "\033[0;0;0m"); + + if (params->pretty_output) + trace_seq_printf(s, "\033[0;0;0m"); trace_seq_printf(s, "\n"); } @@ -305,7 +313,7 @@ timerlat_print_stats(struct timerlat_top_params *params, struct osnoise_tool *to if (!params->quiet) clear_terminal(trace->seq); - timerlat_top_header(top); + timerlat_top_header(params, top); for (i = 0; i < nr_cpus; i++) { if (params->cpus && !CPU_ISSET(i, ¶ms->monitored_cpus)) @@ -693,6 +701,9 @@ timerlat_top_apply_config(struct osnoise_tool *top, struct timerlat_top_params * } } + if (isatty(1) && !params->quiet) + params->pretty_output = 1; + return 0; out_err: From 285dcb7665ae83d07f194a517ba290f02d4f5f73 Mon Sep 17 00:00:00 2001 From: Daniel Bristot de Oliveira Date: Wed, 24 Apr 2024 16:36:53 +0200 Subject: [PATCH 1641/1702] rtla/timerlat: Add a summary for top mode While the per-cpu values are the results to take into consideration, the overall system values are also useful. Add a summary at the bottom of rtla timerlat top showing the overall results. For instance: Timer Latency 0 00:00:10 | IRQ Timer Latency (us) | Thread Timer Latency (us) CPU COUNT | cur min avg max | cur min avg max 0 #10003 | 113 19 150 441 | 134 35 170 459 1 #10003 | 63 8 99 462 | 84 15 119 481 2 #10003 | 3 2 89 396 | 21 8 108 414 3 #10002 | 206 11 210 394 | 223 21 228 415 ---------------|----------------------------------------|--------------------------------------- ALL #40011 e0 | 2 137 462 | 8 156 481 Link: https://lkml.kernel.org/r/5eb510d6faeb4ce745e09395196752df75a2dd1a.1713968967.git.bristot@kernel.org Cc: Jonathan Corbet Suggested-by: Juri Lelli Signed-off-by: Daniel Bristot de Oliveira --- tools/tracing/rtla/src/timerlat_top.c | 108 ++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index c9cf90ed4e6df..bdcf8ef8f815e 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -119,6 +119,37 @@ static struct timerlat_top_data *timerlat_alloc_top(int nr_cpus) return NULL; } +static void +timerlat_top_reset_sum(struct timerlat_top_cpu *summary) +{ + memset(summary, 0, sizeof(*summary)); + summary->min_irq = ~0; + summary->min_thread = ~0; + summary->min_user = ~0; +} + +static void +timerlat_top_update_sum(struct osnoise_tool *tool, int cpu, struct timerlat_top_cpu *sum) +{ + struct timerlat_top_data *data = tool->data; + struct timerlat_top_cpu *cpu_data = &data->cpu_data[cpu]; + + sum->irq_count += cpu_data->irq_count; + update_min(&sum->min_irq, &cpu_data->min_irq); + update_sum(&sum->sum_irq, &cpu_data->sum_irq); + update_max(&sum->max_irq, &cpu_data->max_irq); + + sum->thread_count += cpu_data->thread_count; + update_min(&sum->min_thread, &cpu_data->min_thread); + update_sum(&sum->sum_thread, &cpu_data->sum_thread); + update_max(&sum->max_thread, &cpu_data->max_thread); + + sum->user_count += cpu_data->user_count; + update_min(&sum->min_user, &cpu_data->min_user); + update_sum(&sum->sum_user, &cpu_data->sum_user); + update_max(&sum->max_user, &cpu_data->max_user); +} + /* * timerlat_hist_update - record a new timerlat occurent on cpu, updating data */ @@ -285,6 +316,77 @@ static void timerlat_top_print(struct osnoise_tool *top, int cpu) } } +/* + * timerlat_top_print_sum - prints the summary output + */ +static void +timerlat_top_print_sum(struct osnoise_tool *top, struct timerlat_top_cpu *summary) +{ + const char *split = "----------------------------------------"; + struct timerlat_top_params *params = top->params; + unsigned long long count = summary->irq_count; + int divisor = params->output_divisor; + struct trace_seq *s = top->trace.seq; + int e = 0; + + if (divisor == 0) + return; + + /* + * Skip if no data is available: is this cpu offline? + */ + if (!summary->irq_count && !summary->thread_count) + return; + + while (count > 999999) { + e++; + count /= 10; + } + + trace_seq_printf(s, "%.*s|%.*s|%.*s", 15, split, 40, split, 39, split); + if (params->user_top) + trace_seq_printf(s, "-|%.*s", 39, split); + trace_seq_printf(s, "\n"); + + trace_seq_printf(s, "ALL #%-6llu e%d |", count, e); + + if (!summary->irq_count) { + trace_seq_printf(s, " %s %s %s |", no_value, no_value, no_value); + } else { + trace_seq_printf(s, " "); + trace_seq_printf(s, "%9llu ", summary->min_irq / params->output_divisor); + trace_seq_printf(s, "%9llu ", (summary->sum_irq / summary->irq_count) / divisor); + trace_seq_printf(s, "%9llu |", summary->max_irq / divisor); + } + + if (!summary->thread_count) { + trace_seq_printf(s, "%s %s %s %s", no_value, no_value, no_value, no_value); + } else { + trace_seq_printf(s, " "); + trace_seq_printf(s, "%9llu ", summary->min_thread / divisor); + trace_seq_printf(s, "%9llu ", + (summary->sum_thread / summary->thread_count) / divisor); + trace_seq_printf(s, "%9llu", summary->max_thread / divisor); + } + + if (!params->user_top) { + trace_seq_printf(s, "\n"); + return; + } + + trace_seq_printf(s, " |"); + + if (!summary->user_count) { + trace_seq_printf(s, " %s %s %s |", no_value, no_value, no_value); + } else { + trace_seq_printf(s, " "); + trace_seq_printf(s, "%9llu ", summary->min_user / divisor); + trace_seq_printf(s, "%9llu ", + (summary->sum_user / summary->user_count) / divisor); + trace_seq_printf(s, "%9llu\n", summary->max_user / divisor); + } +} + /* * clear_terminal - clears the output terminal */ @@ -301,6 +403,7 @@ static void timerlat_print_stats(struct timerlat_top_params *params, struct osnoise_tool *top) { struct trace_instance *trace = &top->trace; + struct timerlat_top_cpu summary; static int nr_cpus = -1; int i; @@ -313,14 +416,19 @@ timerlat_print_stats(struct timerlat_top_params *params, struct osnoise_tool *to if (!params->quiet) clear_terminal(trace->seq); + timerlat_top_reset_sum(&summary); + timerlat_top_header(params, top); for (i = 0; i < nr_cpus; i++) { if (params->cpus && !CPU_ISSET(i, ¶ms->monitored_cpus)) continue; timerlat_top_print(top, i); + timerlat_top_update_sum(top, i, &summary); } + timerlat_top_print_sum(top, &summary); + trace_seq_do_printf(trace->seq); trace_seq_reset(trace->seq); } From 1462501c7a8d565f5949d3d5635b2111d889aaaa Mon Sep 17 00:00:00 2001 From: Daniel Bristot de Oliveira Date: Wed, 24 Apr 2024 16:36:54 +0200 Subject: [PATCH 1642/1702] rtla/timerlat: Add a summary for hist mode Like on rtla timerlat top, add an overall summary at the bottom of timerlat hist. For instance: # timerlat hist -c 0-1 -d 10s -E 20 # RTLA timerlat histogram # Time unit is microseconds (us) # Duration: 0 00:00:10 Index IRQ-000 Thr-000 IRQ-001 Thr-001 6 1 0 0 0 7 1 0 0 0 8 1 0 1 0 9 7 0 0 0 10 16 0 0 0 11 1 0 3 0 15 0 0 3 0 16 0 0 12 0 17 0 0 28 0 18 0 2 26 0 19 1 1 80 1 over: 9973 9998 9848 10000 count: 10001 10001 10001 10001 min: 6 18 8 19 avg: 185 204 95 113 max: 428 450 341 371 ALL: IRQ Thr count: 20002 20002 min: 6 18 avg: 140 159 max: 428 450 Link: https://lkml.kernel.org/r/a6bc06c798f72127edc57d1f99da8d57e1187cee.1713968967.git.bristot@kernel.org Cc: Jonathan Corbet Suggested-by: Juri Lelli Signed-off-by: Daniel Bristot de Oliveira --- tools/tracing/rtla/src/timerlat_hist.c | 130 ++++++++++++++++++++++++- 1 file changed, 129 insertions(+), 1 deletion(-) diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c index 8bd51aab6513a..b12c6b571dd46 100644 --- a/tools/tracing/rtla/src/timerlat_hist.c +++ b/tools/tracing/rtla/src/timerlat_hist.c @@ -401,8 +401,135 @@ timerlat_print_summary(struct timerlat_hist_params *params, trace_seq_reset(trace->seq); } +static void +timerlat_print_stats_all(struct timerlat_hist_params *params, + struct trace_instance *trace, + struct timerlat_hist_data *data) +{ + struct timerlat_hist_cpu *cpu_data; + struct timerlat_hist_cpu sum; + int cpu; + + if (params->no_summary) + return; + + memset(&sum, 0, sizeof(sum)); + sum.min_irq = ~0; + sum.min_thread = ~0; + sum.min_user = ~0; + + for (cpu = 0; cpu < data->nr_cpus; cpu++) { + if (params->cpus && !CPU_ISSET(cpu, ¶ms->monitored_cpus)) + continue; + + if (!data->hist[cpu].irq_count && !data->hist[cpu].thread_count) + continue; + + cpu_data = &data->hist[cpu]; + + sum.irq_count += cpu_data->irq_count; + update_min(&sum.min_irq, &cpu_data->min_irq); + update_sum(&sum.sum_irq, &cpu_data->sum_irq); + update_max(&sum.max_irq, &cpu_data->max_irq); + + sum.thread_count += cpu_data->thread_count; + update_min(&sum.min_thread, &cpu_data->min_thread); + update_sum(&sum.sum_thread, &cpu_data->sum_thread); + update_max(&sum.max_thread, &cpu_data->max_thread); + + sum.user_count += cpu_data->user_count; + update_min(&sum.min_user, &cpu_data->min_user); + update_sum(&sum.sum_user, &cpu_data->sum_user); + update_max(&sum.max_user, &cpu_data->max_user); + } + + if (!params->no_index) + trace_seq_printf(trace->seq, "ALL: "); + + if (!params->no_irq) + trace_seq_printf(trace->seq, " IRQ"); + + if (!params->no_thread) + trace_seq_printf(trace->seq, " Thr"); + + if (params->user_hist) + trace_seq_printf(trace->seq, " Usr"); + + trace_seq_printf(trace->seq, "\n"); + + if (!params->no_index) + trace_seq_printf(trace->seq, "count:"); + + if (!params->no_irq) + trace_seq_printf(trace->seq, "%9d ", + sum.irq_count); + + if (!params->no_thread) + trace_seq_printf(trace->seq, "%9d ", + sum.thread_count); + + if (params->user_hist) + trace_seq_printf(trace->seq, "%9d ", + sum.user_count); + + trace_seq_printf(trace->seq, "\n"); + + if (!params->no_index) + trace_seq_printf(trace->seq, "min: "); + + if (!params->no_irq) + trace_seq_printf(trace->seq, "%9llu ", + sum.min_irq); + + if (!params->no_thread) + trace_seq_printf(trace->seq, "%9llu ", + sum.min_thread); + + if (params->user_hist) + trace_seq_printf(trace->seq, "%9llu ", + sum.min_user); + + trace_seq_printf(trace->seq, "\n"); + + if (!params->no_index) + trace_seq_printf(trace->seq, "avg: "); + + if (!params->no_irq) + trace_seq_printf(trace->seq, "%9llu ", + sum.sum_irq / sum.irq_count); + + if (!params->no_thread) + trace_seq_printf(trace->seq, "%9llu ", + sum.sum_thread / sum.thread_count); + + if (params->user_hist) + trace_seq_printf(trace->seq, "%9llu ", + sum.sum_user / sum.user_count); + + trace_seq_printf(trace->seq, "\n"); + + if (!params->no_index) + trace_seq_printf(trace->seq, "max: "); + + if (!params->no_irq) + trace_seq_printf(trace->seq, "%9llu ", + sum.max_irq); + + if (!params->no_thread) + trace_seq_printf(trace->seq, "%9llu ", + sum.max_thread); + + if (params->user_hist) + trace_seq_printf(trace->seq, "%9llu ", + sum.max_user); + + trace_seq_printf(trace->seq, "\n"); + trace_seq_do_printf(trace->seq); + trace_seq_reset(trace->seq); +} + /* - * timerlat_print_stats - print data for all CPUs + * timerlat_print_stats - print data for each CPUs */ static void timerlat_print_stats(struct timerlat_hist_params *params, struct osnoise_tool *tool) @@ -485,6 +612,7 @@ timerlat_print_stats(struct timerlat_hist_params *params, struct osnoise_tool *t trace_seq_reset(trace->seq); timerlat_print_summary(params, trace, data); + timerlat_print_stats_all(params, trace, data); } /* From cdbf71962bb07493d67fee34536a5724a8bb5886 Mon Sep 17 00:00:00 2001 From: Daniel Bristot de Oliveira Date: Wed, 24 Apr 2024 16:36:55 +0200 Subject: [PATCH 1643/1702] rtla: Add the --warm-up option On many cases, the results right after the startup are different from the rest of the execution, biasing the results. For example, on osnoise, the scheduler might take some time to adapt to the new busy-loop workload. Add the --warm-up option, adding a warm-up phase (in seconds) where the workload is set, but the results are discarded. Link: https://lkml.kernel.org/r/e682d5ce5af90f123bd13220f63d5c3d118a92be.1713968967.git.bristot@kernel.org Cc: Jonathan Corbet Cc: Juri Lelli Signed-off-by: Daniel Bristot de Oliveira --- Documentation/tools/rtla/common_options.rst | 4 ++ tools/tracing/rtla/src/osnoise_hist.c | 30 +++++++++++-- tools/tracing/rtla/src/osnoise_top.c | 29 +++++++++++- tools/tracing/rtla/src/timerlat_hist.c | 49 ++++++++++++++------- tools/tracing/rtla/src/timerlat_top.c | 47 ++++++++++++-------- 5 files changed, 119 insertions(+), 40 deletions(-) diff --git a/Documentation/tools/rtla/common_options.rst b/Documentation/tools/rtla/common_options.rst index aeb91ff3bd68c..a96ea0ed662ef 100644 --- a/Documentation/tools/rtla/common_options.rst +++ b/Documentation/tools/rtla/common_options.rst @@ -50,6 +50,10 @@ Set a *cgroup* to the tracer's threads. If the **-C** option is passed without arguments, the tracer's thread will inherit **rtla**'s *cgroup*. Otherwise, the threads will be placed on the *cgroup* passed to the option. +**--warm-up** *s* + + After starting the workload, let it run for *s* seconds before starting collecting the data, allowing the system to warm-up. Statistical data generated during warm-up is discarded. + **-h**, **--help** Print help menu. diff --git a/tools/tracing/rtla/src/osnoise_hist.c b/tools/tracing/rtla/src/osnoise_hist.c index 01870d50942a1..c6100ff46a7f0 100644 --- a/tools/tracing/rtla/src/osnoise_hist.c +++ b/tools/tracing/rtla/src/osnoise_hist.c @@ -36,13 +36,13 @@ struct osnoise_hist_params { cpu_set_t hk_cpu_set; struct sched_attr sched_param; struct trace_events *events; - char no_header; char no_summary; char no_index; char with_zeros; int bucket_size; int entries; + int warmup; }; struct osnoise_hist_cpu { @@ -438,7 +438,7 @@ static void osnoise_hist_usage(char *usage) " usage: rtla osnoise hist [-h] [-D] [-d s] [-a us] [-p us] [-r us] [-s us] [-S us] \\", " [-T us] [-t[=file]] [-e sys[:event]] [--filter ] [--trigger ] \\", " [-c cpu-list] [-H cpu-list] [-P priority] [-b N] [-E N] [--no-header] [--no-summary] \\", - " [--no-index] [--with-zeros] [-C[=cgroup_name]]", + " [--no-index] [--with-zeros] [-C[=cgroup_name]] [--warm-up]", "", " -h/--help: print this menu", " -a/--auto: set automatic trace mode, stopping the session if argument in us sample is hit", @@ -468,6 +468,7 @@ static void osnoise_hist_usage(char *usage) " f:prio - use SCHED_FIFO with prio", " d:runtime[us|ms|s]:period[us|ms|s] - use SCHED_DEADLINE with runtime and period", " in nanoseconds", + " --warm-up: let the workload run for s seconds before collecting data", NULL, }; @@ -531,13 +532,14 @@ static struct osnoise_hist_params {"with-zeros", no_argument, 0, '3'}, {"trigger", required_argument, 0, '4'}, {"filter", required_argument, 0, '5'}, + {"warm-up", required_argument, 0, '6'}, {0, 0, 0, 0} }; /* getopt_long stores the option index here. */ int option_index = 0; - c = getopt_long(argc, argv, "a:c:C::b:d:e:E:DhH:p:P:r:s:S:t::T:01234:5:", + c = getopt_long(argc, argv, "a:c:C::b:d:e:E:DhH:p:P:r:s:S:t::T:01234:5:6:", long_options, &option_index); /* detect the end of the options. */ @@ -680,6 +682,9 @@ static struct osnoise_hist_params osnoise_hist_usage("--filter requires a previous -e\n"); } break; + case '6': + params->warmup = get_llong_from_str(optarg); + break; default: osnoise_hist_usage("Invalid option"); } @@ -899,6 +904,25 @@ int osnoise_hist_main(int argc, char *argv[]) trace_instance_start(&record->trace); trace_instance_start(trace); + if (params->warmup > 0) { + debug_msg("Warming up for %d seconds\n", params->warmup); + sleep(params->warmup); + if (stop_tracing) + goto out_hist; + + /* + * Clean up the buffer. The osnoise workload do not run + * with tracing off to avoid creating a performance penalty + * when not needed. + */ + retval = tracefs_instance_file_write(trace->inst, "trace", ""); + if (retval < 0) { + debug_msg("Error cleaning up the buffer"); + goto out_hist; + } + + } + tool->start_time = time(NULL); osnoise_hist_set_signals(params); diff --git a/tools/tracing/rtla/src/osnoise_top.c b/tools/tracing/rtla/src/osnoise_top.c index 457360db07673..53a074c1222e3 100644 --- a/tools/tracing/rtla/src/osnoise_top.c +++ b/tools/tracing/rtla/src/osnoise_top.c @@ -40,6 +40,7 @@ struct osnoise_top_params { int set_sched; int cgroup; int hk_cpus; + int warmup; cpu_set_t hk_cpu_set; struct sched_attr sched_param; struct trace_events *events; @@ -282,7 +283,7 @@ static void osnoise_top_usage(struct osnoise_top_params *params, char *usage) static const char * const msg[] = { " [-h] [-q] [-D] [-d s] [-a us] [-p us] [-r us] [-s us] [-S us] \\", " [-T us] [-t[=file]] [-e sys[:event]] [--filter ] [--trigger ] \\", - " [-c cpu-list] [-H cpu-list] [-P priority] [-C[=cgroup_name]]", + " [-c cpu-list] [-H cpu-list] [-P priority] [-C[=cgroup_name]] [--warm-up s]", "", " -h/--help: print this menu", " -a/--auto: set automatic trace mode, stopping the session if argument in us sample is hit", @@ -307,6 +308,7 @@ static void osnoise_top_usage(struct osnoise_top_params *params, char *usage) " f:prio - use SCHED_FIFO with prio", " d:runtime[us|ms|s]:period[us|ms|s] - use SCHED_DEADLINE with runtime and period", " in nanoseconds", + " --warm-up s: let the workload run for s seconds before collecting data", NULL, }; @@ -381,13 +383,14 @@ struct osnoise_top_params *osnoise_top_parse_args(int argc, char **argv) {"trace", optional_argument, 0, 't'}, {"trigger", required_argument, 0, '0'}, {"filter", required_argument, 0, '1'}, + {"warm-up", required_argument, 0, '2'}, {0, 0, 0, 0} }; /* getopt_long stores the option index here. */ int option_index = 0; - c = getopt_long(argc, argv, "a:c:C::d:De:hH:p:P:qr:s:S:t::T:0:1:", + c = getopt_long(argc, argv, "a:c:C::d:De:hH:p:P:qr:s:S:t::T:0:1:2:", long_options, &option_index); /* Detect the end of the options. */ @@ -511,6 +514,9 @@ struct osnoise_top_params *osnoise_top_parse_args(int argc, char **argv) osnoise_top_usage(params, "--filter requires a previous -e\n"); } break; + case '2': + params->warmup = get_llong_from_str(optarg); + break; default: osnoise_top_usage(params, "Invalid option"); } @@ -732,6 +738,25 @@ int osnoise_top_main(int argc, char **argv) trace_instance_start(&record->trace); trace_instance_start(trace); + if (params->warmup > 0) { + debug_msg("Warming up for %d seconds\n", params->warmup); + sleep(params->warmup); + if (stop_tracing) + goto out_top; + + /* + * Clean up the buffer. The osnoise workload do not run + * with tracing off to avoid creating a performance penalty + * when not needed. + */ + retval = tracefs_instance_file_write(trace->inst, "trace", ""); + if (retval < 0) { + debug_msg("Error cleaning up the buffer"); + goto out_top; + } + + } + tool->start_time = time(NULL); osnoise_top_set_signals(params); diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c index b12c6b571dd46..da1c353bdac9d 100644 --- a/tools/tracing/rtla/src/timerlat_hist.c +++ b/tools/tracing/rtla/src/timerlat_hist.c @@ -52,6 +52,7 @@ struct timerlat_hist_params { char with_zeros; int bucket_size; int entries; + int warmup; }; struct timerlat_hist_cpu { @@ -628,6 +629,7 @@ static void timerlat_hist_usage(char *usage) " [-t[=file]] [-e sys[:event]] [--filter ] [--trigger ] [-c cpu-list] [-H cpu-list]\\", " [-P priority] [-E N] [-b N] [--no-irq] [--no-thread] [--no-header] [--no-summary] \\", " [--no-index] [--with-zeros] [--dma-latency us] [-C[=cgroup_name]] [--no-aa] [--dump-task] [-u]", + " [--warm-up s]", "", " -h/--help: print this menu", " -a/--auto: set automatic trace mode, stopping the session if argument in us latency is hit", @@ -664,6 +666,7 @@ static void timerlat_hist_usage(char *usage) " in nanoseconds", " -u/--user-threads: use rtla user-space threads instead of in-kernel timerlat threads", " -U/--user-load: enable timerlat for user-defined user-space workload", + " --warm-up s: let the workload run for s seconds before collecting data", NULL, }; @@ -738,13 +741,14 @@ static struct timerlat_hist_params {"dma-latency", required_argument, 0, '8'}, {"no-aa", no_argument, 0, '9'}, {"dump-task", no_argument, 0, '\1'}, + {"warm-up", required_argument, 0, '\2'}, {0, 0, 0, 0} }; /* getopt_long stores the option index here. */ int option_index = 0; - c = getopt_long(argc, argv, "a:c:C::b:d:e:E:DhH:i:np:P:s:t::T:uU0123456:7:8:9\1", + c = getopt_long(argc, argv, "a:c:C::b:d:e:E:DhH:i:np:P:s:t::T:uU0123456:7:8:9\1\2:", long_options, &option_index); /* detect the end of the options. */ @@ -913,6 +917,9 @@ static struct timerlat_hist_params case '\1': params->dump_tasks = 1; break; + case '\2': + params->warmup = get_llong_from_str(optarg); + break; default: timerlat_hist_usage("Invalid option"); } @@ -1167,22 +1174,6 @@ int timerlat_hist_main(int argc, char *argv[]) } } - /* - * Start the tracers here, after having set all instances. - * - * Let the trace instance start first for the case of hitting a stop - * tracing while enabling other instances. The trace instance is the - * one with most valuable information. - */ - if (params->trace_output) - trace_instance_start(&record->trace); - if (!params->no_aa) - trace_instance_start(&aa->trace); - trace_instance_start(trace); - - tool->start_time = time(NULL); - timerlat_hist_set_signals(params); - if (params->user_workload) { /* rtla asked to stop */ params_u.should_run = 1; @@ -1202,6 +1193,29 @@ int timerlat_hist_main(int argc, char *argv[]) err_msg("Error creating timerlat user-space threads\n"); } + if (params->warmup > 0) { + debug_msg("Warming up for %d seconds\n", params->warmup); + sleep(params->warmup); + if (stop_tracing) + goto out_hist; + } + + /* + * Start the tracers here, after having set all instances. + * + * Let the trace instance start first for the case of hitting a stop + * tracing while enabling other instances. The trace instance is the + * one with most valuable information. + */ + if (params->trace_output) + trace_instance_start(&record->trace); + if (!params->no_aa) + trace_instance_start(&aa->trace); + trace_instance_start(trace); + + tool->start_time = time(NULL); + timerlat_hist_set_signals(params); + while (!stop_tracing) { sleep(params->sleep_time); @@ -1227,6 +1241,7 @@ int timerlat_hist_main(int argc, char *argv[]) } } } + if (params->user_workload && !params_u.stopped_running) { params_u.should_run = 0; sleep(1); diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index bdcf8ef8f815e..410eb51749132 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -45,6 +45,7 @@ struct timerlat_top_params { int user_top; int user_workload; int pretty_output; + int warmup; cpu_set_t hk_cpu_set; struct sched_attr sched_param; struct trace_events *events; @@ -444,7 +445,7 @@ static void timerlat_top_usage(char *usage) "", " usage: rtla timerlat [top] [-h] [-q] [-a us] [-d s] [-D] [-n] [-p us] [-i us] [-T us] [-s us] \\", " [[-t[=file]] [-e sys[:event]] [--filter ] [--trigger ] [-c cpu-list] [-H cpu-list]\\", - " [-P priority] [--dma-latency us] [--aa-only us] [-C[=cgroup_name]] [-u]", + " [-P priority] [--dma-latency us] [--aa-only us] [-C[=cgroup_name]] [-u] [--warm-up s]", "", " -h/--help: print this menu", " -a/--auto: set automatic trace mode, stopping the session if argument in us latency is hit", @@ -475,6 +476,7 @@ static void timerlat_top_usage(char *usage) " in nanoseconds", " -u/--user-threads: use rtla user-space threads instead of in-kernel timerlat threads", " -U/--user-load: enable timerlat for user-defined user-space workload", + " --warm-up s: let the workload run for s seconds before collecting data", NULL, }; @@ -541,13 +543,14 @@ static struct timerlat_top_params {"no-aa", no_argument, 0, '3'}, {"dump-tasks", no_argument, 0, '4'}, {"aa-only", required_argument, 0, '5'}, + {"warm-up", required_argument, 0, '6'}, {0, 0, 0, 0} }; /* getopt_long stores the option index here. */ int option_index = 0; - c = getopt_long(argc, argv, "a:c:C::d:De:hH:i:np:P:qs:t::T:uU0:1:2:345:", + c = getopt_long(argc, argv, "a:c:C::d:De:hH:i:np:P:qs:t::T:uU0:1:2:345:6:", long_options, &option_index); /* detect the end of the options. */ @@ -704,6 +707,9 @@ static struct timerlat_top_params case '4': params->dump_tasks = 1; break; + case '6': + params->warmup = get_llong_from_str(optarg); + break; default: timerlat_top_usage("Invalid option"); } @@ -971,22 +977,6 @@ int timerlat_top_main(int argc, char *argv[]) } } - /* - * Start the tracers here, after having set all instances. - * - * Let the trace instance start first for the case of hitting a stop - * tracing while enabling other instances. The trace instance is the - * one with most valuable information. - */ - if (params->trace_output) - trace_instance_start(&record->trace); - if (!params->no_aa && aa != top) - trace_instance_start(&aa->trace); - trace_instance_start(trace); - - top->start_time = time(NULL); - timerlat_top_set_signals(params); - if (params->user_workload) { /* rtla asked to stop */ params_u.should_run = 1; @@ -1006,6 +996,27 @@ int timerlat_top_main(int argc, char *argv[]) err_msg("Error creating timerlat user-space threads\n"); } + if (params->warmup > 0) { + debug_msg("Warming up for %d seconds\n", params->warmup); + sleep(params->warmup); + } + + /* + * Start the tracers here, after having set all instances. + * + * Let the trace instance start first for the case of hitting a stop + * tracing while enabling other instances. The trace instance is the + * one with most valuable information. + */ + if (params->trace_output) + trace_instance_start(&record->trace); + if (!params->no_aa && aa != top) + trace_instance_start(&aa->trace); + trace_instance_start(trace); + + top->start_time = time(NULL); + timerlat_top_set_signals(params); + while (!stop_tracing) { sleep(params->sleep_time); From fb9e90a67ee9a42779a8ea296a4cf7734258b27d Mon Sep 17 00:00:00 2001 From: Daniel Bristot de Oliveira Date: Wed, 24 Apr 2024 16:36:56 +0200 Subject: [PATCH 1644/1702] rtla/timerlat: Make user-space threads the default After ther -u addition, most of the known users are setting it. And it makes sense, as it adds more information, and inherits the default setup for the threads - e.g., cgroups configs. Thus, if the user-space interface is available, enable -u. Otherwise, use the in-kernel thread. Add the -k option to allow the user to request kernel-threads. Link: https://lkml.kernel.org/r/9241d3089de4091b124f780ed832a0e6646cadaa.1713968967.git.bristot@kernel.org Cc: Jonathan Corbet Cc: Juri Lelli Signed-off-by: Daniel Bristot de Oliveira --- .../tools/rtla/common_timerlat_options.rst | 6 +++- tools/tracing/rtla/src/timerlat_hist.c | 31 +++++++++++++++++-- tools/tracing/rtla/src/timerlat_top.c | 31 +++++++++++++++++-- 3 files changed, 61 insertions(+), 7 deletions(-) diff --git a/Documentation/tools/rtla/common_timerlat_options.rst b/Documentation/tools/rtla/common_timerlat_options.rst index d3255ed70195f..090700a6ae9f0 100644 --- a/Documentation/tools/rtla/common_timerlat_options.rst +++ b/Documentation/tools/rtla/common_timerlat_options.rst @@ -27,12 +27,16 @@ *cyclictest* sets this value to *0* by default, use **--dma-latency** *0* to have similar results. +**-k**, **--kernel-threads** + + Use timerlat kernel-space threads, in contrast of **-u**. + **-u**, **--user-threads** Set timerlat to run without a workload, and then dispatches user-space workloads to wait on the timerlat_fd. Once the workload is awakes, it goes to sleep again adding so the measurement for the kernel-to-user and user-to-kernel to the tracer - output. + output. **--user-threads** will be used unless the user specify **-k**. **-U**, **--user-load** diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c index da1c353bdac9d..6eb6e38d4a05b 100644 --- a/tools/tracing/rtla/src/timerlat_hist.c +++ b/tools/tracing/rtla/src/timerlat_hist.c @@ -40,6 +40,7 @@ struct timerlat_hist_params { int no_aa; int dump_tasks; int user_workload; + int kernel_workload; int user_hist; cpu_set_t hk_cpu_set; struct sched_attr sched_param; @@ -628,7 +629,7 @@ static void timerlat_hist_usage(char *usage) " usage: [rtla] timerlat hist [-h] [-q] [-d s] [-D] [-n] [-a us] [-p us] [-i us] [-T us] [-s us] \\", " [-t[=file]] [-e sys[:event]] [--filter ] [--trigger ] [-c cpu-list] [-H cpu-list]\\", " [-P priority] [-E N] [-b N] [--no-irq] [--no-thread] [--no-header] [--no-summary] \\", - " [--no-index] [--with-zeros] [--dma-latency us] [-C[=cgroup_name]] [--no-aa] [--dump-task] [-u]", + " [--no-index] [--with-zeros] [--dma-latency us] [-C[=cgroup_name]] [--no-aa] [--dump-task] [-u|-k]", " [--warm-up s]", "", " -h/--help: print this menu", @@ -664,7 +665,8 @@ static void timerlat_hist_usage(char *usage) " f:prio - use SCHED_FIFO with prio", " d:runtime[us|ms|s]:period[us|ms|s] - use SCHED_DEADLINE with runtime and period", " in nanoseconds", - " -u/--user-threads: use rtla user-space threads instead of in-kernel timerlat threads", + " -u/--user-threads: use rtla user-space threads instead of kernel-space timerlat threads", + " -k/--kernel-threads: use timerlat kernel-space threads instead of rtla user-space threads", " -U/--user-load: enable timerlat for user-defined user-space workload", " --warm-up s: let the workload run for s seconds before collecting data", NULL, @@ -728,6 +730,7 @@ static struct timerlat_hist_params {"thread", required_argument, 0, 'T'}, {"trace", optional_argument, 0, 't'}, {"user-threads", no_argument, 0, 'u'}, + {"kernel-threads", no_argument, 0, 'k'}, {"user-load", no_argument, 0, 'U'}, {"event", required_argument, 0, 'e'}, {"no-irq", no_argument, 0, '0'}, @@ -748,7 +751,7 @@ static struct timerlat_hist_params /* getopt_long stores the option index here. */ int option_index = 0; - c = getopt_long(argc, argv, "a:c:C::b:d:e:E:DhH:i:np:P:s:t::T:uU0123456:7:8:9\1\2:", + c = getopt_long(argc, argv, "a:c:C::b:d:e:E:DhH:i:knp:P:s:t::T:uU0123456:7:8:9\1\2:", long_options, &option_index); /* detect the end of the options. */ @@ -831,6 +834,9 @@ static struct timerlat_hist_params case 'i': params->stop_us = get_llong_from_str(optarg); break; + case 'k': + params->kernel_workload = 1; + break; case 'n': params->output_divisor = 1; break; @@ -942,6 +948,9 @@ static struct timerlat_hist_params if (!params->stop_us && !params->stop_total_us) params->no_aa = 1; + if (params->kernel_workload && params->user_workload) + timerlat_hist_usage("--kernel-threads and --user-threads are mutually exclusive!"); + return params; } @@ -1017,6 +1026,22 @@ timerlat_hist_apply_config(struct osnoise_tool *tool, struct timerlat_hist_param auto_house_keeping(¶ms->monitored_cpus); } + /* + * If the user did not specify a type of thread, try user-threads first. + * Fall back to kernel threads otherwise. + */ + if (!params->kernel_workload && !params->user_workload) { + retval = tracefs_file_exists(NULL, "osnoise/per_cpu/cpu0/timerlat_fd"); + if (retval) { + debug_msg("User-space interface detected, setting user-threads\n"); + params->user_workload = 1; + params->user_hist = 1; + } else { + debug_msg("User-space interface not detected, setting kernel-threads\n"); + params->kernel_workload = 1; + } + } + if (params->user_hist) { retval = osnoise_set_workload(tool->context, 0); if (retval) { diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index 410eb51749132..0acfefe151f75 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -44,6 +44,7 @@ struct timerlat_top_params { int hk_cpus; int user_top; int user_workload; + int kernel_workload; int pretty_output; int warmup; cpu_set_t hk_cpu_set; @@ -445,7 +446,7 @@ static void timerlat_top_usage(char *usage) "", " usage: rtla timerlat [top] [-h] [-q] [-a us] [-d s] [-D] [-n] [-p us] [-i us] [-T us] [-s us] \\", " [[-t[=file]] [-e sys[:event]] [--filter ] [--trigger ] [-c cpu-list] [-H cpu-list]\\", - " [-P priority] [--dma-latency us] [--aa-only us] [-C[=cgroup_name]] [-u] [--warm-up s]", + " [-P priority] [--dma-latency us] [--aa-only us] [-C[=cgroup_name]] [-u|-k] [--warm-up s]", "", " -h/--help: print this menu", " -a/--auto: set automatic trace mode, stopping the session if argument in us latency is hit", @@ -474,7 +475,8 @@ static void timerlat_top_usage(char *usage) " f:prio - use SCHED_FIFO with prio", " d:runtime[us|ms|s]:period[us|ms|s] - use SCHED_DEADLINE with runtime and period", " in nanoseconds", - " -u/--user-threads: use rtla user-space threads instead of in-kernel timerlat threads", + " -u/--user-threads: use rtla user-space threads instead of kernel-space timerlat threads", + " -k/--kernel-threads: use timerlat kernel-space threads instead of rtla user-space threads", " -U/--user-load: enable timerlat for user-defined user-space workload", " --warm-up s: let the workload run for s seconds before collecting data", NULL, @@ -536,6 +538,7 @@ static struct timerlat_top_params {"thread", required_argument, 0, 'T'}, {"trace", optional_argument, 0, 't'}, {"user-threads", no_argument, 0, 'u'}, + {"kernel-threads", no_argument, 0, 'k'}, {"user-load", no_argument, 0, 'U'}, {"trigger", required_argument, 0, '0'}, {"filter", required_argument, 0, '1'}, @@ -550,7 +553,7 @@ static struct timerlat_top_params /* getopt_long stores the option index here. */ int option_index = 0; - c = getopt_long(argc, argv, "a:c:C::d:De:hH:i:np:P:qs:t::T:uU0:1:2:345:6:", + c = getopt_long(argc, argv, "a:c:C::d:De:hH:i:knp:P:qs:t::T:uU0:1:2:345:6:", long_options, &option_index); /* detect the end of the options. */ @@ -635,6 +638,9 @@ static struct timerlat_top_params case 'i': params->stop_us = get_llong_from_str(optarg); break; + case 'k': + params->kernel_workload = true; + break; case 'n': params->output_divisor = 1; break; @@ -729,6 +735,9 @@ static struct timerlat_top_params if (params->no_aa && params->aa_only) timerlat_top_usage("--no-aa and --aa-only are mutually exclusive!"); + if (params->kernel_workload && params->user_workload) + timerlat_top_usage("--kernel-threads and --user-threads are mutually exclusive!"); + return params; } @@ -807,6 +816,22 @@ timerlat_top_apply_config(struct osnoise_tool *top, struct timerlat_top_params * auto_house_keeping(¶ms->monitored_cpus); } + /* + * If the user did not specify a type of thread, try user-threads first. + * Fall back to kernel threads otherwise. + */ + if (!params->kernel_workload && !params->user_workload) { + retval = tracefs_file_exists(NULL, "osnoise/per_cpu/cpu0/timerlat_fd"); + if (retval) { + debug_msg("User-space interface detected, setting user-threads\n"); + params->user_workload = 1; + params->user_top = 1; + } else { + debug_msg("User-space interface not detected, setting kernel-threads\n"); + params->kernel_workload = 1; + } + } + if (params->user_top) { retval = osnoise_set_workload(top->context, 0); if (retval) { From d4a599910193b85f76c100e30d8551c8794f8c2a Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Wed, 15 May 2024 14:53:25 +0200 Subject: [PATCH 1645/1702] parisc: Define HAVE_ARCH_HUGETLB_UNMAPPED_AREA Define the HAVE_ARCH_HUGETLB_UNMAPPED_AREA macro like other platforms do in their page.h files to avoid this compile warning: arch/parisc/mm/hugetlbpage.c:25:1: warning: no previous prototype for 'hugetlb_get_unmapped_area' [-Wmissing-prototypes] Signed-off-by: Helge Deller Cc: stable@vger.kernel.org # 6.0+ Reported-by: John David Anglin Tested-by: John David Anglin --- arch/parisc/include/asm/page.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/parisc/include/asm/page.h b/arch/parisc/include/asm/page.h index ad4e15d12ed1c..4bea2e95798f0 100644 --- a/arch/parisc/include/asm/page.h +++ b/arch/parisc/include/asm/page.h @@ -8,6 +8,7 @@ #define PAGE_SIZE (_AC(1,UL) << PAGE_SHIFT) #define PAGE_MASK (~(PAGE_SIZE-1)) +#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA #ifndef __ASSEMBLY__ From 1de27bba6d50a909647f304eadc0f7c59a842a50 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 13 May 2024 11:08:03 -0700 Subject: [PATCH 1646/1702] libbpf: fix feature detectors when using token_fd Adjust `union bpf_attr` size passed to kernel in two feature-detecting functions to take into account prog_token_fd field. Libbpf is avoiding memset()'ing entire `union bpf_attr` by only using minimal set of bpf_attr's fields. Two places have been missed when wiring BPF token support in libbpf's feature detection logic. Fix them trivially. Fixes: f3dcee938f48 ("libbpf: Wire up token_fd into feature probing logic") Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20240513180804.403775-1-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- tools/lib/bpf/bpf.c | 2 +- tools/lib/bpf/features.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 466a29d801249..2a4c71501a17d 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -105,7 +105,7 @@ int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts) */ int probe_memcg_account(int token_fd) { - const size_t attr_sz = offsetofend(union bpf_attr, attach_btf_obj_fd); + const size_t attr_sz = offsetofend(union bpf_attr, prog_token_fd); struct bpf_insn insns[] = { BPF_EMIT_CALL(BPF_FUNC_ktime_get_coarse_ns), BPF_EXIT_INSN(), diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c index 4e783cc7fc4b5..a336786a22a38 100644 --- a/tools/lib/bpf/features.c +++ b/tools/lib/bpf/features.c @@ -22,7 +22,7 @@ int probe_fd(int fd) static int probe_kern_prog_name(int token_fd) { - const size_t attr_sz = offsetofend(union bpf_attr, prog_name); + const size_t attr_sz = offsetofend(union bpf_attr, prog_token_fd); struct bpf_insn insns[] = { BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), From 7a8030057f6791d35dd20987f9ff15855c01c1bb Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Tue, 14 May 2024 13:03:03 +0000 Subject: [PATCH 1647/1702] bpf, docs: Fix the description of 'src' in ALU instructions An ALU instruction's source operand can be the value in the source register or the 32-bit immediate value encoded in the instruction. This is controlled by the 's' bit of the 'opcode'. The current description explicitly uses the phrase 'value of the source register' when defining the meaning of 'src'. Change the description to use 'source operand' in place of 'value of the source register'. Signed-off-by: Puranjay Mohan Acked-by: Dave Thaler Link: https://lore.kernel.org/r/20240514130303.113607-1-puranjay@kernel.org Signed-off-by: Alexei Starovoitov --- Documentation/bpf/standardization/instruction-set.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Documentation/bpf/standardization/instruction-set.rst b/Documentation/bpf/standardization/instruction-set.rst index 997560abadabe..00c93eb42613e 100644 --- a/Documentation/bpf/standardization/instruction-set.rst +++ b/Documentation/bpf/standardization/instruction-set.rst @@ -301,8 +301,9 @@ Arithmetic instructions ``ALU`` uses 32-bit wide operands while ``ALU64`` uses 64-bit wide operands for otherwise identical operations. ``ALU64`` instructions belong to the base64 conformance group unless noted otherwise. -The 'code' field encodes the operation as below, where 'src' and 'dst' refer -to the values of the source and destination registers, respectively. +The 'code' field encodes the operation as below, where 'src' refers to the +the source operand and 'dst' refers to the value of the destination +register. ===== ===== ======= ========================================================== name code offset description From 325423cafc12031a69692363ddcabc63113bb3d6 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Tue, 14 May 2024 18:39:14 +0000 Subject: [PATCH 1648/1702] MAINTAINERS: Update ARM64 BPF JIT maintainer Zi Shen Lim is not actively doing kernel development and has decided to tranfer the responsibility of maintaining the JIT to me. Add myself as the maintainer for BPF JIT for ARM64 and remove Zi Shen Lim. Signed-off-by: Puranjay Mohan Acked-by: Zi Shen Lim Link: https://lore.kernel.org/r/20240514183914.27737-1-puranjay@kernel.org Signed-off-by: Alexei Starovoitov --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 94fddfcec2fb3..be7f72766dc65 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3777,7 +3777,7 @@ F: arch/arm/net/ BPF JIT for ARM64 M: Daniel Borkmann M: Alexei Starovoitov -M: Zi Shen Lim +M: Puranjay Mohan L: bpf@vger.kernel.org S: Supported F: arch/arm64/net/ From 9ee98229083186837199912a7debb666146b8c17 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 14 May 2024 23:24:39 -0700 Subject: [PATCH 1649/1702] bpf: save extended inner map info for percpu array maps as well ARRAY_OF_MAPS and HASH_OF_MAPS map types have special logic to save a few extra fields required for correct operations of ARRAY maps, when they are used as inner maps. PERCPU_ARRAY maps have similar requirements as they now support generating inline element lookup logic. So make sure that both classes of maps are handled correctly. Reported-by: Jakub Kicinski Fixes: db69718b8efa ("bpf: inline bpf_map_lookup_elem() for PERCPU_ARRAY maps") Signed-off-by: Andrii Nakryiko Acked-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/r/20240515062440.846086-1-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/map_in_map.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c index 8ef269e66ba50..b4f18c85d7bc7 100644 --- a/kernel/bpf/map_in_map.c +++ b/kernel/bpf/map_in_map.c @@ -32,7 +32,7 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd) inner_map_meta_size = sizeof(*inner_map_meta); /* In some cases verifier needs to access beyond just base map. */ - if (inner_map->ops == &array_map_ops) + if (inner_map->ops == &array_map_ops || inner_map->ops == &percpu_array_map_ops) inner_map_meta_size = sizeof(struct bpf_array); inner_map_meta = kzalloc(inner_map_meta_size, GFP_USER); @@ -68,7 +68,7 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd) /* Misc members not needed in bpf_map_meta_equal() check. */ inner_map_meta->ops = inner_map->ops; - if (inner_map->ops == &array_map_ops) { + if (inner_map->ops == &array_map_ops || inner_map->ops == &percpu_array_map_ops) { struct bpf_array *inner_array_meta = container_of(inner_map_meta, struct bpf_array, map); struct bpf_array *inner_array = container_of(inner_map, struct bpf_array, map); From 2322113ac9d0c5653017adbab504fb307b0e92e2 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 14 May 2024 23:24:40 -0700 Subject: [PATCH 1650/1702] selftests/bpf: add more variations of map-in-map situations Add test cases validating usage of PERCPU_ARRAY and PERCPU_HASH maps as inner maps. Signed-off-by: Andrii Nakryiko Acked-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/r/20240515062440.846086-2-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/progs/map_kptr.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/map_kptr.c b/tools/testing/selftests/bpf/progs/map_kptr.c index da30f0d593648..ab0ce1d01a4a7 100644 --- a/tools/testing/selftests/bpf/progs/map_kptr.c +++ b/tools/testing/selftests/bpf/progs/map_kptr.c @@ -110,10 +110,14 @@ DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_ARRAY_OF_MAPS, array_map, array_of_array_maps); DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_ARRAY_OF_MAPS, hash_map, array_of_hash_maps); DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_ARRAY_OF_MAPS, hash_malloc_map, array_of_hash_malloc_maps); DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_ARRAY_OF_MAPS, lru_hash_map, array_of_lru_hash_maps); +DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_ARRAY_OF_MAPS, pcpu_array_map, array_of_pcpu_array_maps); +DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_ARRAY_OF_MAPS, pcpu_hash_map, array_of_pcpu_hash_maps); DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_HASH_OF_MAPS, array_map, hash_of_array_maps); DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_HASH_OF_MAPS, hash_map, hash_of_hash_maps); DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_HASH_OF_MAPS, hash_malloc_map, hash_of_hash_malloc_maps); DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_HASH_OF_MAPS, lru_hash_map, hash_of_lru_hash_maps); +DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_HASH_OF_MAPS, pcpu_array_map, hash_of_pcpu_array_maps); +DEFINE_MAP_OF_MAP(BPF_MAP_TYPE_HASH_OF_MAPS, pcpu_hash_map, hash_of_pcpu_hash_maps); #define WRITE_ONCE(x, val) ((*(volatile typeof(x) *) &(x)) = (val)) @@ -204,6 +208,8 @@ int test_map_kptr(struct __sk_buff *ctx) TEST(hash_map); TEST(hash_malloc_map); TEST(lru_hash_map); + TEST(pcpu_array_map); + TEST(pcpu_hash_map); #undef TEST return 0; @@ -281,10 +287,14 @@ int test_map_in_map_kptr(struct __sk_buff *ctx) TEST(array_of_hash_maps); TEST(array_of_hash_malloc_maps); TEST(array_of_lru_hash_maps); + TEST(array_of_pcpu_array_maps); + TEST(array_of_pcpu_hash_maps); TEST(hash_of_array_maps); TEST(hash_of_hash_maps); TEST(hash_of_hash_malloc_maps); TEST(hash_of_lru_hash_maps); + TEST(hash_of_pcpu_array_maps); + TEST(hash_of_pcpu_hash_maps); #undef TEST return 0; From 6ffe4fdf8901dc0a15d7278531503ecd4522ae15 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Tue, 14 May 2024 08:31:42 +0900 Subject: [PATCH 1651/1702] kconfig: use sym_get_choice_menu() in sym_check_prop() Choices and their members are associated via the P_CHOICE property. Currently, prop_get_symbol(sym_get_choice_prop()) is used to obtain the choice of the given choice member. Replace it with sym_get_choice_menu(), which retrieves the choice without relying on P_CHOICE. Signed-off-by: Masahiro Yamada --- scripts/kconfig/menu.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/kconfig/menu.c b/scripts/kconfig/menu.c index bee96c9964fd8..53151c5a60281 100644 --- a/scripts/kconfig/menu.c +++ b/scripts/kconfig/menu.c @@ -263,11 +263,9 @@ static void sym_check_prop(struct symbol *sym) sym->name); } if (sym_is_choice(sym)) { - struct property *choice_prop = - sym_get_choice_prop(sym2); + struct menu *choice = sym_get_choice_menu(sym2); - if (!choice_prop || - prop_get_symbol(choice_prop) != sym) + if (!choice || choice->sym != sym) prop_warn(prop, "choice default symbol '%s' is not contained in the choice", sym2->name); From a395726cf823fe8f62f1b8c3829010e5652ce98c Mon Sep 17 00:00:00 2001 From: Steve French Date: Wed, 15 May 2024 16:59:01 -0500 Subject: [PATCH 1652/1702] cifs: fix data corruption in read after invalidate When invalidating a file as part of breaking a lease, the folios holding the file data are disposed of, and truncate calls ->invalidate_folio() to get rid of them rather than calling ->release_folio(). This means that the netfs_inode::zero_point value didn't get updated in current upstream code to reflect the point after which we can assume that the server will only return zeroes, and future reads will then return blocks of zeroes if the file got extended for any region beyond the old zero point. Fix this by updating zero_point before invalidating the inode in cifs_revalidate_mapping(). Suggested-by: David Howells Fixes: 3ee1a1fc3981 ("cifs: Cut over to using netfslib") Reviewed-by: David Howells Signed-off-by: Steve French --- fs/smb/client/inode.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c index 5d4b0fd3a59e4..262576573eb51 100644 --- a/fs/smb/client/inode.c +++ b/fs/smb/client/inode.c @@ -2465,7 +2465,8 @@ int cifs_revalidate_mapping(struct inode *inode) { int rc; - unsigned long *flags = &CIFS_I(inode)->flags; + struct cifsInodeInfo *cifs_inode = CIFS_I(inode); + unsigned long *flags = &cifs_inode->flags; struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); /* swapfiles are not supposed to be shared */ @@ -2482,6 +2483,7 @@ cifs_revalidate_mapping(struct inode *inode) if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_RW_CACHE) goto skip_invalidate; + cifs_inode->netfs.zero_point = cifs_inode->netfs.remote_i_size; rc = filemap_invalidate_inode(inode, true, 0, LLONG_MAX); if (rc) { cifs_dbg(VFS, "%s: invalidate inode %p failed with rc %d\n", From 1a7d0890dd4a502a202aaec792a6c04e6e049547 Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Wed, 1 May 2024 09:29:56 -0700 Subject: [PATCH 1653/1702] kprobe/ftrace: bail out if ftrace was killed If an error happens in ftrace, ftrace_kill() will prevent disarming kprobes. Eventually, the ftrace_ops associated with the kprobes will be freed, yet the kprobes will still be active, and when triggered, they will use the freed memory, likely resulting in a page fault and panic. This behavior can be reproduced quite easily, by creating a kprobe and then triggering a ftrace_kill(). For simplicity, we can simulate an ftrace error with a kernel module like [1]: [1]: https://github.com/brenns10/kernel_stuff/tree/master/ftrace_killer sudo perf probe --add commit_creds sudo perf trace -e probe:commit_creds # In another terminal make sudo insmod ftrace_killer.ko # calls ftrace_kill(), simulating bug # Back to perf terminal # ctrl-c sudo perf probe --del commit_creds After a short period, a page fault and panic would occur as the kprobe continues to execute and uses the freed ftrace_ops. While ftrace_kill() is supposed to be used only in extreme circumstances, it is invoked in FTRACE_WARN_ON() and so there are many places where an unexpected bug could be triggered, yet the system may continue operating, possibly without the administrator noticing. If ftrace_kill() does not panic the system, then we should do everything we can to continue operating, rather than leave a ticking time bomb. Link: https://lore.kernel.org/all/20240501162956.229427-1-stephen.s.brennan@oracle.com/ Signed-off-by: Stephen Brennan Acked-by: Masami Hiramatsu (Google) Acked-by: Guo Ren Reviewed-by: Steven Rostedt (Google) Signed-off-by: Masami Hiramatsu (Google) --- arch/csky/kernel/probes/ftrace.c | 3 +++ arch/loongarch/kernel/ftrace_dyn.c | 3 +++ arch/parisc/kernel/ftrace.c | 3 +++ arch/powerpc/kernel/kprobes-ftrace.c | 3 +++ arch/riscv/kernel/probes/ftrace.c | 3 +++ arch/s390/kernel/ftrace.c | 3 +++ arch/x86/kernel/kprobes/ftrace.c | 3 +++ include/linux/kprobes.h | 7 +++++++ kernel/kprobes.c | 6 ++++++ kernel/trace/ftrace.c | 1 + 10 files changed, 35 insertions(+) diff --git a/arch/csky/kernel/probes/ftrace.c b/arch/csky/kernel/probes/ftrace.c index 834cffcfbce32..7ba4b98076de1 100644 --- a/arch/csky/kernel/probes/ftrace.c +++ b/arch/csky/kernel/probes/ftrace.c @@ -12,6 +12,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, struct kprobe_ctlblk *kcb; struct pt_regs *regs; + if (unlikely(kprobe_ftrace_disabled)) + return; + bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return; diff --git a/arch/loongarch/kernel/ftrace_dyn.c b/arch/loongarch/kernel/ftrace_dyn.c index 73858c9029cc9..bff058317062e 100644 --- a/arch/loongarch/kernel/ftrace_dyn.c +++ b/arch/loongarch/kernel/ftrace_dyn.c @@ -287,6 +287,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, struct kprobe *p; struct kprobe_ctlblk *kcb; + if (unlikely(kprobe_ftrace_disabled)) + return; + bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return; diff --git a/arch/parisc/kernel/ftrace.c b/arch/parisc/kernel/ftrace.c index 621a4b386ae4f..c91f9c2e61ed2 100644 --- a/arch/parisc/kernel/ftrace.c +++ b/arch/parisc/kernel/ftrace.c @@ -206,6 +206,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, struct kprobe *p; int bit; + if (unlikely(kprobe_ftrace_disabled)) + return; + bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return; diff --git a/arch/powerpc/kernel/kprobes-ftrace.c b/arch/powerpc/kernel/kprobes-ftrace.c index 072ebe7f290ba..f8208c027148f 100644 --- a/arch/powerpc/kernel/kprobes-ftrace.c +++ b/arch/powerpc/kernel/kprobes-ftrace.c @@ -21,6 +21,9 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip, struct pt_regs *regs; int bit; + if (unlikely(kprobe_ftrace_disabled)) + return; + bit = ftrace_test_recursion_trylock(nip, parent_nip); if (bit < 0) return; diff --git a/arch/riscv/kernel/probes/ftrace.c b/arch/riscv/kernel/probes/ftrace.c index 7142ec42e889f..a69dfa610aa85 100644 --- a/arch/riscv/kernel/probes/ftrace.c +++ b/arch/riscv/kernel/probes/ftrace.c @@ -11,6 +11,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, struct kprobe_ctlblk *kcb; int bit; + if (unlikely(kprobe_ftrace_disabled)) + return; + bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return; diff --git a/arch/s390/kernel/ftrace.c b/arch/s390/kernel/ftrace.c index c46381ea04ecb..7f6f8c438c265 100644 --- a/arch/s390/kernel/ftrace.c +++ b/arch/s390/kernel/ftrace.c @@ -296,6 +296,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, struct kprobe *p; int bit; + if (unlikely(kprobe_ftrace_disabled)) + return; + bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return; diff --git a/arch/x86/kernel/kprobes/ftrace.c b/arch/x86/kernel/kprobes/ftrace.c index dd2ec14adb77b..15af7e98e161a 100644 --- a/arch/x86/kernel/kprobes/ftrace.c +++ b/arch/x86/kernel/kprobes/ftrace.c @@ -21,6 +21,9 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, struct kprobe_ctlblk *kcb; int bit; + if (unlikely(kprobe_ftrace_disabled)) + return; + bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return; diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h index 0ff44d6633e33..5fcbc254d1864 100644 --- a/include/linux/kprobes.h +++ b/include/linux/kprobes.h @@ -378,11 +378,15 @@ static inline void wait_for_kprobe_optimizer(void) { } extern void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *ops, struct ftrace_regs *fregs); extern int arch_prepare_kprobe_ftrace(struct kprobe *p); +/* Set when ftrace has been killed: kprobes on ftrace must be disabled for safety */ +extern bool kprobe_ftrace_disabled __read_mostly; +extern void kprobe_ftrace_kill(void); #else static inline int arch_prepare_kprobe_ftrace(struct kprobe *p) { return -EINVAL; } +static inline void kprobe_ftrace_kill(void) {} #endif /* CONFIG_KPROBES_ON_FTRACE */ /* Get the kprobe at this addr (if any) - called with preemption disabled */ @@ -495,6 +499,9 @@ static inline void kprobe_flush_task(struct task_struct *tk) static inline void kprobe_free_init_mem(void) { } +static inline void kprobe_ftrace_kill(void) +{ +} static inline int disable_kprobe(struct kprobe *kp) { return -EOPNOTSUPP; diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 65adc815fc6e6..166ebf81dc450 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -1068,6 +1068,7 @@ static struct ftrace_ops kprobe_ipmodify_ops __read_mostly = { static int kprobe_ipmodify_enabled; static int kprobe_ftrace_enabled; +bool kprobe_ftrace_disabled; static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops, int *cnt) @@ -1136,6 +1137,11 @@ static int disarm_kprobe_ftrace(struct kprobe *p) ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops, ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled); } + +void kprobe_ftrace_kill() +{ + kprobe_ftrace_disabled = true; +} #else /* !CONFIG_KPROBES_ON_FTRACE */ static inline int arm_kprobe_ftrace(struct kprobe *p) { diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index da1710499698b..96db99c347b3b 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -7895,6 +7895,7 @@ void ftrace_kill(void) ftrace_disabled = 1; ftrace_enabled = 0; ftrace_trace_function = ftrace_stub; + kprobe_ftrace_kill(); } /** From 78464d7681f79bb48995c3d29d7e93d27ba69bca Mon Sep 17 00:00:00 2001 From: Len Brown Date: Wed, 24 Apr 2024 21:12:18 -0400 Subject: [PATCH 1654/1702] tools/power turbostat: Add columns for clustered uncore frequency New machines have multiple uncore frequencies per package, visible in /sys/devices/system/cpu/intel_uncore_frequency/uncore##/ turbostat now samples these frequencies each measurement interval. For each package, turbostat now prints "UMHzX.Y" columns, where X = domain_id, and Y = fabric_cluster_id. The system summary for each UMHzX.Y column is the average value for across all of the packages in the system. Signed-off-by: Len Brown --- tools/power/x86/turbostat/turbostat.8 | 4 +- tools/power/x86/turbostat/turbostat.c | 342 +++++++++++++++++--------- 2 files changed, 226 insertions(+), 120 deletions(-) diff --git a/tools/power/x86/turbostat/turbostat.8 b/tools/power/x86/turbostat/turbostat.8 index 0d3672e5d9ed1..8d37acd392015 100644 --- a/tools/power/x86/turbostat/turbostat.8 +++ b/tools/power/x86/turbostat/turbostat.8 @@ -155,7 +155,9 @@ The system configuration dump (if --quiet is not used) is followed by statistics .PP \fBRAM_%\fP percent of the interval that RAPL throttling was active on DRAM. .PP -\fBUncMHz\fP uncore MHz, instantaneous sample. +\fBUncMHz\fP per-package uncore MHz, instantaneous sample. +.PP +\fBUMHz1.0\fP per-package uncore MHz for domain=1 and fabric_cluster=0, instantaneous sample. System summary is the average of all packages. .SH TOO MUCH INFORMATION EXAMPLE By default, turbostat dumps all possible information -- a system configuration header, followed by columns for all counters. This is ideal for remote debugging, use the "--out" option to save everything to a text file, and get that file to the expert helping you debug. diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index f92b46cfda31b..abfddff6aebd7 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -59,15 +59,21 @@ #define MAX_NOFILE 0x8000 enum counter_scope { SCOPE_CPU, SCOPE_CORE, SCOPE_PACKAGE }; -enum counter_type { COUNTER_ITEMS, COUNTER_CYCLES, COUNTER_SECONDS, COUNTER_USEC }; -enum counter_format { FORMAT_RAW, FORMAT_DELTA, FORMAT_PERCENT }; +enum counter_type { COUNTER_ITEMS, COUNTER_CYCLES, COUNTER_SECONDS, COUNTER_USEC, COUNTER_K2M }; +enum counter_format { FORMAT_RAW, FORMAT_DELTA, FORMAT_PERCENT, FORMAT_AVERAGE }; enum amperf_source { AMPERF_SOURCE_PERF, AMPERF_SOURCE_MSR }; enum rapl_source { RAPL_SOURCE_NONE, RAPL_SOURCE_PERF, RAPL_SOURCE_MSR }; +struct sysfs_path { + char path[PATH_BYTES]; + int id; + struct sysfs_path *next; +}; + struct msr_counter { unsigned int msr_num; char name[NAME_BYTES]; - char path[PATH_BYTES]; + struct sysfs_path *sp; unsigned int width; enum counter_type type; enum counter_format format; @@ -79,64 +85,64 @@ struct msr_counter { }; struct msr_counter bic[] = { - { 0x0, "usec", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Time_Of_Day_Seconds", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Package", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Node", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Avg_MHz", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Busy%", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Bzy_MHz", "", 0, 0, 0, NULL, 0 }, - { 0x0, "TSC_MHz", "", 0, 0, 0, NULL, 0 }, - { 0x0, "IRQ", "", 0, 0, 0, NULL, 0 }, - { 0x0, "SMI", "", 32, 0, FORMAT_DELTA, NULL, 0 }, - { 0x0, "sysfs", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CPU%c1", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CPU%c3", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CPU%c6", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CPU%c7", "", 0, 0, 0, NULL, 0 }, - { 0x0, "ThreadC", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CoreTmp", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CoreCnt", "", 0, 0, 0, NULL, 0 }, - { 0x0, "PkgTmp", "", 0, 0, 0, NULL, 0 }, - { 0x0, "GFX%rc6", "", 0, 0, 0, NULL, 0 }, - { 0x0, "GFXMHz", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pkg%pc2", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pkg%pc3", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pkg%pc6", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pkg%pc7", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pkg%pc8", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pkg%pc9", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pk%pc10", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CPU%LPI", "", 0, 0, 0, NULL, 0 }, - { 0x0, "SYS%LPI", "", 0, 0, 0, NULL, 0 }, - { 0x0, "PkgWatt", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CorWatt", "", 0, 0, 0, NULL, 0 }, - { 0x0, "GFXWatt", "", 0, 0, 0, NULL, 0 }, - { 0x0, "PkgCnt", "", 0, 0, 0, NULL, 0 }, - { 0x0, "RAMWatt", "", 0, 0, 0, NULL, 0 }, - { 0x0, "PKG_%", "", 0, 0, 0, NULL, 0 }, - { 0x0, "RAM_%", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pkg_J", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Cor_J", "", 0, 0, 0, NULL, 0 }, - { 0x0, "GFX_J", "", 0, 0, 0, NULL, 0 }, - { 0x0, "RAM_J", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Mod%c6", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Totl%C0", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Any%C0", "", 0, 0, 0, NULL, 0 }, - { 0x0, "GFX%C0", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CPUGFX%", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Core", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CPU", "", 0, 0, 0, NULL, 0 }, - { 0x0, "APIC", "", 0, 0, 0, NULL, 0 }, - { 0x0, "X2APIC", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Die", "", 0, 0, 0, NULL, 0 }, - { 0x0, "GFXAMHz", "", 0, 0, 0, NULL, 0 }, - { 0x0, "IPC", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CoreThr", "", 0, 0, 0, NULL, 0 }, - { 0x0, "UncMHz", "", 0, 0, 0, NULL, 0 }, - { 0x0, "SAM%mc6", "", 0, 0, 0, NULL, 0 }, - { 0x0, "SAMMHz", "", 0, 0, 0, NULL, 0 }, - { 0x0, "SAMAMHz", "", 0, 0, 0, NULL, 0 }, + { 0x0, "usec", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Time_Of_Day_Seconds", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Package", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Node", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Avg_MHz", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Busy%", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Bzy_MHz", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "TSC_MHz", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "IRQ", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "SMI", NULL, 32, 0, FORMAT_DELTA, NULL, 0 }, + { 0x0, "sysfs", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CPU%c1", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CPU%c3", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CPU%c6", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CPU%c7", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "ThreadC", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CoreTmp", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CoreCnt", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "PkgTmp", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "GFX%rc6", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "GFXMHz", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pkg%pc2", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pkg%pc3", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pkg%pc6", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pkg%pc7", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pkg%pc8", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pkg%pc9", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pk%pc10", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CPU%LPI", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "SYS%LPI", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "PkgWatt", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CorWatt", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "GFXWatt", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "PkgCnt", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "RAMWatt", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "PKG_%", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "RAM_%", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pkg_J", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Cor_J", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "GFX_J", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "RAM_J", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Mod%c6", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Totl%C0", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Any%C0", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "GFX%C0", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CPUGFX%", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Core", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CPU", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "APIC", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "X2APIC", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Die", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "GFXAMHz", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "IPC", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CoreThr", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "UncMHz", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "SAM%mc6", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "SAMMHz", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "SAMAMHz", NULL, 0, 0, 0, NULL, 0 }, }; #define MAX_BIC (sizeof(bic) / sizeof(struct msr_counter)) @@ -300,6 +306,9 @@ struct gfx_sysfs_info { static struct gfx_sysfs_info gfx_info[GFX_MAX]; int get_msr(int cpu, off_t offset, unsigned long long *msr); +int add_counter(unsigned int msr_num, char *path, char *name, + unsigned int width, enum counter_scope scope, + enum counter_type type, enum counter_format format, int flags, int package_num); /* Model specific support Start */ @@ -999,8 +1008,9 @@ char *progname; #define CPU_SUBSET_MAXCPUS 1024 /* need to use before probe... */ cpu_set_t *cpu_present_set, *cpu_effective_set, *cpu_allowed_set, *cpu_affinity_set, *cpu_subset; size_t cpu_present_setsize, cpu_effective_setsize, cpu_allowed_setsize, cpu_affinity_setsize, cpu_subset_size; -#define MAX_ADDED_COUNTERS 8 #define MAX_ADDED_THREAD_COUNTERS 24 +#define MAX_ADDED_CORE_COUNTERS 8 +#define MAX_ADDED_PACKAGE_COUNTERS 16 #define BITMASK_SIZE 32 /* Indexes used to map data read from perf and MSRs into global variables */ @@ -1201,7 +1211,7 @@ struct core_data { struct rapl_counter core_energy; /* MSR_CORE_ENERGY_STAT */ unsigned int core_id; unsigned long long core_throt_cnt; - unsigned long long counter[MAX_ADDED_COUNTERS]; + unsigned long long counter[MAX_ADDED_CORE_COUNTERS]; } *core_even, *core_odd; struct pkg_data { @@ -1234,7 +1244,7 @@ struct pkg_data { struct rapl_counter rapl_dram_perf_status; /* MSR_DRAM_PERF_STATUS */ unsigned int pkg_temp_c; unsigned int uncore_mhz; - unsigned long long counter[MAX_ADDED_COUNTERS]; + unsigned long long counter[MAX_ADDED_PACKAGE_COUNTERS]; } *package_even, *package_odd; #define ODD_COUNTERS thread_odd, core_odd, package_odd @@ -1955,13 +1965,15 @@ void print_header(char *delim) if (mp->format == FORMAT_RAW) { if (mp->width == 64) outp += sprintf(outp, "%s%18.18s", delim, mp->name); - else + else if (mp->width == 32) outp += sprintf(outp, "%s%10.10s", delim, mp->name); + else + outp += sprintf(outp, "%s%7.7s", delim, mp->name); } else { if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) outp += sprintf(outp, "%s%8s", delim, mp->name); else - outp += sprintf(outp, "%s%s", delim, mp->name); + outp += sprintf(outp, "%s%7.7s", delim, mp->name); } } @@ -1993,7 +2005,7 @@ int dump_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { outp += sprintf(outp, "tADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, - t->counter[i], mp->path); + t->counter[i], mp->sp->path); } } @@ -2014,7 +2026,7 @@ int dump_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { outp += sprintf(outp, "cADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, - c->counter[i], mp->path); + c->counter[i], mp->sp->path); } outp += sprintf(outp, "mc6_us: %016llX\n", c->mc6_us); } @@ -2050,7 +2062,7 @@ int dump_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { outp += sprintf(outp, "pADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, - p->counter[i], mp->path); + p->counter[i], mp->sp->path); } } @@ -2415,7 +2427,8 @@ int format_counters(struct thread_data *t, struct core_data *c, struct pkg_data outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), p->counter[i]); } else if (mp->format == FORMAT_PERCENT) { outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->counter[i] / tsc); - } + } else if (mp->type == COUNTER_K2M) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->counter[i] / 1000); } done: @@ -2525,6 +2538,8 @@ int delta_package(struct pkg_data *new, struct pkg_data *old) for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { if (mp->format == FORMAT_RAW) old->counter[i] = new->counter[i]; + else if (mp->format == FORMAT_AVERAGE) + old->counter[i] = new->counter[i]; else old->counter[i] = new->counter[i] - old->counter[i]; } @@ -2997,7 +3012,7 @@ unsigned long long snapshot_sysfs_counter(char *path) return counter; } -int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp) +int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp, char *counter_path) { if (mp->msr_num != 0) { assert(!no_msr); @@ -3007,11 +3022,11 @@ int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp) char path[128 + PATH_BYTES]; if (mp->flags & SYSFS_PERCPU) { - sprintf(path, "/sys/devices/system/cpu/cpu%d/%s", cpu, mp->path); + sprintf(path, "/sys/devices/system/cpu/cpu%d/%s", cpu, mp->sp->path); *counterp = snapshot_sysfs_counter(path); } else { - *counterp = snapshot_sysfs_counter(mp->path); + *counterp = snapshot_sysfs_counter(counter_path); } } @@ -3486,6 +3501,18 @@ int get_rapl_counters(int cpu, int domain, struct core_data *c, struct pkg_data return 0; } +char *find_sysfs_path_by_id(struct sysfs_path *sp, int id) +{ + while (sp) { + if (sp->id == id) + return (sp->path); + sp = sp->next; + } + if (debug) + warnx("%s: id%d not found", __func__, id); + return NULL; +} + /* * get_counters(...) * migrate to cpu @@ -3547,7 +3574,7 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) } for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { - if (get_mp(cpu, mp, &t->counter[i])) + if (get_mp(cpu, mp, &t->counter[i], mp->sp->path)) return -10; } @@ -3602,7 +3629,7 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) get_core_throt_cnt(cpu, &c->core_throt_cnt); for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { - if (get_mp(cpu, mp, &c->counter[i])) + if (get_mp(cpu, mp, &c->counter[i], mp->sp->path)) return -10; } @@ -3694,7 +3721,16 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) p->sam_act_mhz = gfx_info[SAM_ACTMHz].val; for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { - if (get_mp(cpu, mp, &p->counter[i])) + char *path = NULL; + + if (mp->msr_num == 0) { + path = find_sysfs_path_by_id(mp->sp, p->package_id); + if (path == NULL) { + warnx("%s: package_id %d not found", __func__, p->package_id); + return -10; + } + } + if (get_mp(cpu, mp, &p->counter[i], path)) return -10; } done: @@ -5377,8 +5413,9 @@ static void probe_intel_uncore_frequency_legacy(void) static void probe_intel_uncore_frequency_cluster(void) { - int i; + int i, uncore_max_id; char path[256]; + char path_base[128]; if (access("/sys/devices/system/cpu/intel_uncore_frequency/uncore00/current_freq_khz", R_OK)) return; @@ -5386,16 +5423,25 @@ static void probe_intel_uncore_frequency_cluster(void) if (quiet) return; - for (i = 0;; ++i) { + for (uncore_max_id = 0;; ++uncore_max_id) { + + sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", uncore_max_id); + + /* uncore## start at 00 and skips no numbers, so stop upon first missing */ + if (access(path_base, R_OK)) { + uncore_max_id -= 1; + break; + } + } + for (i = uncore_max_id; i >= 0; --i) { int k, l; - char path_base[128]; int package_id, domain_id, cluster_id; + char name_buf[16]; sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", i); - /* uncore## start at 00 and skip no numbers, so stop upon first missing */ if (access(path_base, R_OK)) - break; + err(1, "%s: %s\n", __func__, path_base); sprintf(path, "%s/package_id", path_base); package_id = read_sysfs_int(path); @@ -5422,6 +5468,11 @@ static void probe_intel_uncore_frequency_cluster(void) sprintf(path, "%s/current_freq_khz", path_base); k = read_sysfs_int(path); fprintf(outf, " %d MHz\n", k / 1000); + + sprintf(path, "%s/current_freq_khz", path_base); + sprintf(name_buf, "UMHz%d.%d", domain_id, cluster_id); + + add_counter(0, path, name_buf, 0, SCOPE_PACKAGE, COUNTER_K2M, FORMAT_AVERAGE, 0, package_id); } } @@ -7575,61 +7626,114 @@ void print_bootcmd(void) fclose(fp); } +struct msr_counter *find_msrp_by_name(struct msr_counter *head, char *name) +{ + struct msr_counter *mp; + + for (mp = head; mp; mp = mp->next) { + if (debug) + printf("%s: %s %s\n", __func__, name, mp->name); + if (!strncmp(name, mp->name, strlen(mp->name))) + return mp; + } + return NULL; +} + int add_counter(unsigned int msr_num, char *path, char *name, unsigned int width, enum counter_scope scope, - enum counter_type type, enum counter_format format, int flags) + enum counter_type type, enum counter_format format, int flags, int id) { struct msr_counter *msrp; if (no_msr && msr_num) errx(1, "Requested MSR counter 0x%x, but in --no-msr mode", msr_num); - msrp = calloc(1, sizeof(struct msr_counter)); - if (msrp == NULL) { - perror("calloc"); - exit(1); - } - - msrp->msr_num = msr_num; - strncpy(msrp->name, name, NAME_BYTES - 1); - if (path) - strncpy(msrp->path, path, PATH_BYTES - 1); - msrp->width = width; - msrp->type = type; - msrp->format = format; - msrp->flags = flags; + if (debug) + printf("%s(msr%d, %s, %s, width%d, scope%d, type%d, format%d, flags%x, id%d)\n", __func__, msr_num, + path, name, width, scope, type, format, flags, id); switch (scope) { case SCOPE_CPU: - msrp->next = sys.tp; - sys.tp = msrp; - sys.added_thread_counters++; - if (sys.added_thread_counters > MAX_ADDED_THREAD_COUNTERS) { - fprintf(stderr, "exceeded max %d added thread counters\n", MAX_ADDED_COUNTERS); - exit(-1); + msrp = find_msrp_by_name(sys.tp, name); + if (msrp) { + if (debug) + printf("%s: %s FOUND\n", __func__, name); + break; + } + if (sys.added_thread_counters++ >= MAX_ADDED_THREAD_COUNTERS) { + warnx("ignoring thread counter %s", name); + return -1; } break; - case SCOPE_CORE: - msrp->next = sys.cp; - sys.cp = msrp; - sys.added_core_counters++; - if (sys.added_core_counters > MAX_ADDED_COUNTERS) { - fprintf(stderr, "exceeded max %d added core counters\n", MAX_ADDED_COUNTERS); - exit(-1); + msrp = find_msrp_by_name(sys.cp, name); + if (msrp) { + if (debug) + printf("%s: %s FOUND\n", __func__, name); + break; + } + if (sys.added_core_counters++ >= MAX_ADDED_CORE_COUNTERS) { + warnx("ignoring core counter %s", name); + return -1; } break; - case SCOPE_PACKAGE: - msrp->next = sys.pp; - sys.pp = msrp; - sys.added_package_counters++; - if (sys.added_package_counters > MAX_ADDED_COUNTERS) { - fprintf(stderr, "exceeded max %d added package counters\n", MAX_ADDED_COUNTERS); - exit(-1); + msrp = find_msrp_by_name(sys.pp, name); + if (msrp) { + if (debug) + printf("%s: %s FOUND\n", __func__, name); + break; + } + if (sys.added_package_counters++ >= MAX_ADDED_PACKAGE_COUNTERS) { + warnx("ignoring package counter %s", name); + return -1; } break; + default: + warnx("ignoring counter %s with unknown scope", name); + return -1; + } + + if (msrp == NULL) { + msrp = calloc(1, sizeof(struct msr_counter)); + if (msrp == NULL) + err(-1, "calloc msr_counter"); + msrp->msr_num = msr_num; + strncpy(msrp->name, name, NAME_BYTES - 1); + msrp->width = width; + msrp->type = type; + msrp->format = format; + msrp->flags = flags; + + switch (scope) { + case SCOPE_CPU: + msrp->next = sys.tp; + sys.tp = msrp; + break; + case SCOPE_CORE: + msrp->next = sys.cp; + sys.cp = msrp; + break; + case SCOPE_PACKAGE: + msrp->next = sys.pp; + sys.pp = msrp; + break; + } + } + + if (path) { + struct sysfs_path *sp; + + sp = calloc(1, sizeof(struct sysfs_path)); + if (sp == NULL) { + perror("calloc"); + exit(1); + } + strncpy(sp->path, path, PATH_BYTES - 1); + sp->id = id; + sp->next = msrp->sp; + msrp->sp = sp; } return 0; @@ -7731,7 +7835,7 @@ void parse_add_command(char *add_command) sprintf(name_buffer, "M0X%x%s", msr_num, format == FORMAT_PERCENT ? "%" : ""); } - if (add_counter(msr_num, path, name_buffer, width, scope, type, format, 0)) + if (add_counter(msr_num, path, name_buffer, width, scope, type, format, 0, 0)) fail++; if (fail) { @@ -7796,7 +7900,7 @@ void probe_sysfs(void) if (is_deferred_skip(name_buf)) continue; - add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_USEC, FORMAT_PERCENT, SYSFS_PERCPU); + add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_USEC, FORMAT_PERCENT, SYSFS_PERCPU, 0); } for (state = 10; state >= 0; --state) { @@ -7824,7 +7928,7 @@ void probe_sysfs(void) if (is_deferred_skip(name_buf)) continue; - add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU); + add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); } } From 3559ea813ad3a9627934325c68ad05b18008a077 Mon Sep 17 00:00:00 2001 From: Patryk Wlazlyn Date: Mon, 6 May 2024 15:39:08 +0200 Subject: [PATCH 1655/1702] tools/power turbostat: Avoid possible memory corruption due to sparse topology IDs Save the highest core and package id when parsing topology to allocate enough memory when get_rapl_counters() is called with a core or a package id as a domain. Note that RAPL domains are per-package on Intel, but per-core on AMD. Thus, the RAPL code effectively runs in different modes on those two product lines. Signed-off-by: Patryk Wlazlyn Signed-off-by: Len Brown --- tools/power/x86/turbostat/turbostat.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index abfddff6aebd7..9ee9e4e5522c7 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -1052,6 +1052,7 @@ struct rapl_counter_info_t { /* struct rapl_counter_info_t for each RAPL domain */ struct rapl_counter_info_t *rapl_counter_info_perdomain; +unsigned int rapl_counter_info_perdomain_size; #define RAPL_COUNTER_FLAG_USE_MSR_SUM (1u << 1) @@ -1451,6 +1452,8 @@ struct topo_params { int allowed_cpus; int allowed_cores; int max_cpu_num; + int max_core_id; + int max_package_id; int max_die_id; int max_node_num; int nodes_per_pkg; @@ -3425,15 +3428,18 @@ void write_rapl_counter(struct rapl_counter *rc, struct rapl_counter_info_t *rci rc->scale = rci->scale[idx]; } -int get_rapl_counters(int cpu, int domain, struct core_data *c, struct pkg_data *p) +int get_rapl_counters(int cpu, unsigned int domain, struct core_data *c, struct pkg_data *p) { unsigned long long perf_data[NUM_RAPL_COUNTERS + 1]; - struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[domain]; + struct rapl_counter_info_t *rci; if (debug) fprintf(stderr, "%s: cpu%d domain%d\n", __func__, cpu, domain); assert(rapl_counter_info_perdomain); + assert(domain < rapl_counter_info_perdomain_size); + + rci = &rapl_counter_info_perdomain[domain]; /* * If we have any perf counters to read, read them all now, in bulk @@ -4257,7 +4263,7 @@ void free_fd_rapl_percpu(void) if (!rapl_counter_info_perdomain) return; - const int num_domains = platform->has_per_core_rapl ? topo.num_cores : topo.num_packages; + const int num_domains = rapl_counter_info_perdomain_size; for (int domain_id = 0; domain_id < num_domains; ++domain_id) { if (rapl_counter_info_perdomain[domain_id].fd_perf != -1) @@ -4265,6 +4271,8 @@ void free_fd_rapl_percpu(void) } free(rapl_counter_info_perdomain); + rapl_counter_info_perdomain = NULL; + rapl_counter_info_perdomain_size = 0; } void free_all_buffers(void) @@ -6582,17 +6590,18 @@ void linux_perf_init(void) void rapl_perf_init(void) { - const int num_domains = platform->has_per_core_rapl ? topo.num_cores : topo.num_packages; + const unsigned int num_domains = (platform->has_per_core_rapl ? topo.max_core_id : topo.max_package_id) + 1; bool *domain_visited = calloc(num_domains, sizeof(bool)); rapl_counter_info_perdomain = calloc(num_domains, sizeof(*rapl_counter_info_perdomain)); if (rapl_counter_info_perdomain == NULL) err(-1, "calloc rapl_counter_info_percpu"); + rapl_counter_info_perdomain_size = num_domains; /* * Initialize rapl_counter_info_percpu */ - for (int domain_id = 0; domain_id < num_domains; ++domain_id) { + for (unsigned int domain_id = 0; domain_id < num_domains; ++domain_id) { struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[domain_id]; rci->fd_perf = -1; @@ -6612,7 +6621,7 @@ void rapl_perf_init(void) bool has_counter = 0; double scale; enum rapl_unit unit; - int next_domain; + unsigned int next_domain; memset(domain_visited, 0, num_domains * sizeof(*domain_visited)); @@ -6625,6 +6634,8 @@ void rapl_perf_init(void) next_domain = platform->has_per_core_rapl ? cpus[cpu].physical_core_id : cpus[cpu].physical_package_id; + assert(next_domain < num_domains); + if (domain_visited[next_domain]) continue; @@ -7207,6 +7218,8 @@ void topology_probe(bool startup) if (cpus[i].thread_id == 0) topo.num_cores++; } + topo.max_core_id = max_core_id; + topo.max_package_id = max_package_id; topo.cores_per_node = max_core_id + 1; if (debug > 1) From 1f9e46da9cba54d12880948fd2adac31bb0eaadb Mon Sep 17 00:00:00 2001 From: Patryk Wlazlyn Date: Mon, 11 Mar 2024 18:06:16 +0100 Subject: [PATCH 1656/1702] tools/power turbostat: Read Core-cstates via perf Reading the counters via perf can be done in bulk with a single syscall, making the counter values more accurate with respect to one another by minimizing the time gap between individual counter reads. Signed-off-by: Patryk Wlazlyn Signed-off-by: Len Brown --- tools/power/x86/turbostat/turbostat.c | 379 +++++++++++++++++++++++--- 1 file changed, 335 insertions(+), 44 deletions(-) diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index 9ee9e4e5522c7..9d0278d179655 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -63,6 +63,7 @@ enum counter_type { COUNTER_ITEMS, COUNTER_CYCLES, COUNTER_SECONDS, COUNTER_USEC enum counter_format { FORMAT_RAW, FORMAT_DELTA, FORMAT_PERCENT, FORMAT_AVERAGE }; enum amperf_source { AMPERF_SOURCE_PERF, AMPERF_SOURCE_MSR }; enum rapl_source { RAPL_SOURCE_NONE, RAPL_SOURCE_PERF, RAPL_SOURCE_MSR }; +enum cstate_source { CSTATE_SOURCE_NONE, CSTATE_SOURCE_PERF, CSTATE_SOURCE_MSR }; struct sysfs_path { char path[PATH_BYTES]; @@ -1183,6 +1184,77 @@ struct rapl_counter { double scale; }; +/* Indexes used to map data read from perf and MSRs into global variables */ +enum ccstate_rci_index { + CCSTATE_RCI_INDEX_C1_RESIDENCY = 0, + CCSTATE_RCI_INDEX_C3_RESIDENCY = 1, + CCSTATE_RCI_INDEX_C6_RESIDENCY = 2, + CCSTATE_RCI_INDEX_C7_RESIDENCY = 3, + NUM_CCSTATE_COUNTERS, +}; + +struct cstate_counter_info_t { + unsigned long long data[NUM_CCSTATE_COUNTERS]; + enum cstate_source source[NUM_CCSTATE_COUNTERS]; + unsigned long long msr[NUM_CCSTATE_COUNTERS]; + int fd_perf; +}; + +struct cstate_counter_info_t *ccstate_counter_info; +unsigned int ccstate_counter_info_size; + +#define CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD (1u << 0) +#define CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY (1u << 1) + +struct cstate_counter_arch_info { + int feature_mask; /* Mask for testing if the counter is supported on host */ + const char *perf_subsys; + const char *perf_name; + unsigned long long msr; + unsigned int rci_index; /* Maps data from perf counters to global variables */ + unsigned long long bic; + unsigned long long flags; +}; + +static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { + { + .feature_mask = CC1, + .perf_subsys = "cstate_core", + .perf_name = "c1-residency", + .msr = MSR_CORE_C1_RES, + .rci_index = CCSTATE_RCI_INDEX_C1_RESIDENCY, + .bic = BIC_CPU_c1, + .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD, + }, + { + .feature_mask = CC3, + .perf_subsys = "cstate_core", + .perf_name = "c3-residency", + .msr = MSR_CORE_C3_RESIDENCY, + .rci_index = CCSTATE_RCI_INDEX_C3_RESIDENCY, + .bic = BIC_CPU_c3, + .flags = CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY, + }, + { + .feature_mask = CC6, + .perf_subsys = "cstate_core", + .perf_name = "c6-residency", + .msr = MSR_CORE_C6_RESIDENCY, + .rci_index = CCSTATE_RCI_INDEX_C6_RESIDENCY, + .bic = BIC_CPU_c6, + .flags = CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY, + }, + { + .feature_mask = CC7, + .perf_subsys = "cstate_core", + .perf_name = "c7-residency", + .msr = MSR_CORE_C7_RESIDENCY, + .rci_index = CCSTATE_RCI_INDEX_C7_RESIDENCY, + .bic = BIC_CPU_c7, + .flags = CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY, + }, +}; + struct thread_data { struct timeval tv_begin; struct timeval tv_end; @@ -1571,10 +1643,6 @@ static void bic_disable_msr_access(void) { const unsigned long bic_msrs = BIC_SMI | - BIC_CPU_c1 | - BIC_CPU_c3 | - BIC_CPU_c6 | - BIC_CPU_c7 | BIC_Mod_c6 | BIC_CoreTmp | BIC_Totl_c0 | @@ -3421,6 +3489,17 @@ size_t rapl_counter_info_count_perf(const struct rapl_counter_info_t *rci) return ret; } +static size_t cstate_counter_info_count_perf(const struct cstate_counter_info_t *cci) +{ + size_t ret = 0; + + for (int i = 0; i < NUM_CCSTATE_COUNTERS; ++i) + if (cci->source[i] == CSTATE_SOURCE_PERF) + ++ret; + + return ret; +} + void write_rapl_counter(struct rapl_counter *rc, struct rapl_counter_info_t *rci, unsigned int idx) { rc->raw_value = rci->data[idx]; @@ -3519,6 +3598,90 @@ char *find_sysfs_path_by_id(struct sysfs_path *sp, int id) return NULL; } +int get_cstate_counters(unsigned int cpu, struct thread_data *t, struct core_data *c) +{ + unsigned long long perf_data[NUM_CCSTATE_COUNTERS + 1]; + struct cstate_counter_info_t *cci; + + if (debug) + fprintf(stderr, "%s: cpu%d\n", __func__, cpu); + + assert(ccstate_counter_info); + assert(cpu <= ccstate_counter_info_size); + + cci = &ccstate_counter_info[cpu]; + + /* + * If we have any perf counters to read, read them all now, in bulk + */ + if (cci->fd_perf != -1) { + const size_t num_perf_counters = cstate_counter_info_count_perf(cci); + const ssize_t expected_read_size = + (num_perf_counters + 1) * sizeof(unsigned long long); + const ssize_t actual_read_size = + read(cci->fd_perf, &perf_data[0], sizeof(perf_data)); + + if (actual_read_size != expected_read_size) + err(-1, "%s: failed to read perf_data (%zu %zu)", + __func__, expected_read_size, actual_read_size); + } + + for (unsigned int i = 0, pi = 1; i < NUM_CCSTATE_COUNTERS; ++i) { + switch (cci->source[i]) { + case CSTATE_SOURCE_NONE: + break; + + case CSTATE_SOURCE_PERF: + assert(pi < ARRAY_SIZE(perf_data)); + assert(cci->fd_perf != -1); + + if (debug) { + fprintf(stderr, "cstate via %s %u: %llu\n", + "perf", i, perf_data[pi]); + } + + cci->data[i] = perf_data[pi]; + + ++pi; + break; + + case CSTATE_SOURCE_MSR: + assert(!no_msr); + if (get_msr(cpu, cci->msr[i], &cci->data[i])) + return -13 - i; + + if (debug) { + fprintf(stderr, "cstate via %s0x%llx %u: %llu\n", + "msr", cci->msr[i], i, cci->data[i]); + } + + break; + } + } + + /* + * Helper to write the data only if the source of + * the counter for the current cpu is not none. + * + * Otherwise we would overwrite core data with 0 (default value), + * when invoked for the thread sibling. + */ +#define PERF_COUNTER_WRITE_DATA(out_counter, index) do { \ + if (cci->source[index] != CSTATE_SOURCE_NONE) \ + out_counter = cci->data[index]; \ +} while (0) + + BUILD_BUG_ON(NUM_CCSTATE_COUNTERS != 4); + PERF_COUNTER_WRITE_DATA(t->c1, CCSTATE_RCI_INDEX_C1_RESIDENCY); + PERF_COUNTER_WRITE_DATA(c->c3, CCSTATE_RCI_INDEX_C3_RESIDENCY); + PERF_COUNTER_WRITE_DATA(c->c6, CCSTATE_RCI_INDEX_C6_RESIDENCY); + PERF_COUNTER_WRITE_DATA(c->c7, CCSTATE_RCI_INDEX_C7_RESIDENCY); + +#undef PERF_COUNTER_WRITE_DATA + + return 0; +} + /* * get_counters(...) * migrate to cpu @@ -3574,10 +3737,8 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) return -5; t->smi_count = msr & 0xFFFFFFFF; } - if (DO_BIC(BIC_CPU_c1) && platform->has_msr_core_c1_res) { - if (get_msr(cpu, MSR_CORE_C1_RES, &t->c1)) - return -6; - } + + get_cstate_counters(cpu, t, c); for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { if (get_mp(cpu, mp, &t->counter[i], mp->sp->path)) @@ -3594,31 +3755,14 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) return status; } - if (DO_BIC(BIC_CPU_c3) || soft_c1_residency_display(BIC_CPU_c3)) { - if (get_msr(cpu, MSR_CORE_C3_RESIDENCY, &c->c3)) - return -6; - } - - if ((DO_BIC(BIC_CPU_c6) || soft_c1_residency_display(BIC_CPU_c6)) && !platform->has_msr_knl_core_c6_residency) { - if (get_msr(cpu, MSR_CORE_C6_RESIDENCY, &c->c6)) - return -7; - } else if (platform->has_msr_knl_core_c6_residency && soft_c1_residency_display(BIC_CPU_c6)) { - if (get_msr(cpu, MSR_KNL_CORE_C6_RESIDENCY, &c->c6)) - return -7; - } - - if (DO_BIC(BIC_CPU_c7) || soft_c1_residency_display(BIC_CPU_c7)) { - if (get_msr(cpu, MSR_CORE_C7_RESIDENCY, &c->c7)) - return -8; - else if (t->is_atom) { - /* - * For Atom CPUs that has core cstate deeper than c6, - * MSR_CORE_C6_RESIDENCY returns residency of cc6 and deeper. - * Minus CC7 (and deeper cstates) residency to get - * accturate cc6 residency. - */ - c->c6 -= c->c7; - } + if (DO_BIC(BIC_CPU_c7) && t->is_atom) { + /* + * For Atom CPUs that has core cstate deeper than c6, + * MSR_CORE_C6_RESIDENCY returns residency of cc6 and deeper. + * Minus CC7 (and deeper cstates) residency to get + * accturate cc6 residency. + */ + c->c6 -= c->c7; } if (DO_BIC(BIC_Mod_c6)) @@ -4258,6 +4402,23 @@ void free_fd_instr_count_percpu(void) fd_instr_count_percpu = NULL; } +void free_fd_cstate(void) +{ + if (!ccstate_counter_info) + return; + + const int counter_info_num = ccstate_counter_info_size; + + for (int counter_id = 0; counter_id < counter_info_num; ++counter_id) { + if (ccstate_counter_info[counter_id].fd_perf != -1) + close(ccstate_counter_info[counter_id].fd_perf); + } + + free(ccstate_counter_info); + ccstate_counter_info = NULL; + ccstate_counter_info_size = 0; +} + void free_fd_rapl_percpu(void) { if (!rapl_counter_info_perdomain) @@ -4319,6 +4480,7 @@ void free_all_buffers(void) free_fd_instr_count_percpu(); free_fd_amperf_percpu(); free_fd_rapl_percpu(); + free_fd_cstate(); free(irq_column_2_cpu); free(irqs_per_cpu); @@ -4654,6 +4816,7 @@ static void update_effective_set(bool startup) void linux_perf_init(void); void rapl_perf_init(void); +void cstate_perf_init(void); void re_initialize(void) { @@ -4661,6 +4824,7 @@ void re_initialize(void) setup_all_buffers(false); linux_perf_init(); rapl_perf_init(); + cstate_perf_init(); fprintf(outf, "turbostat: re-initialized with num_cpus %d, allowed_cpus %d\n", topo.num_cpus, topo.allowed_cpus); } @@ -6517,7 +6681,8 @@ bool is_aperf_access_required(void) return BIC_IS_ENABLED(BIC_Avg_MHz) || BIC_IS_ENABLED(BIC_Busy) || BIC_IS_ENABLED(BIC_Bzy_MHz) - || BIC_IS_ENABLED(BIC_IPC); + || BIC_IS_ENABLED(BIC_IPC) + || BIC_IS_ENABLED(BIC_CPU_c1); } int add_rapl_perf_counter_(int cpu, struct rapl_counter_info_t *rci, const struct rapl_counter_arch_info *cai, @@ -6749,21 +6914,132 @@ static int has_amperf_access(void) return 0; } -void probe_cstates(void) +int add_cstate_perf_counter_(int cpu, struct cstate_counter_info_t *cci, + const struct cstate_counter_arch_info *cai) { - probe_cst_limit(); + if (no_perf) + return -1; - if (platform->supported_cstates & CC1) - BIC_PRESENT(BIC_CPU_c1); + const unsigned int type = read_perf_type(cai->perf_subsys); + const unsigned int config = read_rapl_config(cai->perf_subsys, cai->perf_name); + + const int fd_counter = + open_perf_counter(cpu, type, config, cci->fd_perf, PERF_FORMAT_GROUP); + + if (fd_counter == -1) + return -1; + + /* If it's the first counter opened, make it a group descriptor */ + if (cci->fd_perf == -1) + cci->fd_perf = fd_counter; + + return fd_counter; +} + +int add_cstate_perf_counter(int cpu, struct cstate_counter_info_t *cci, + const struct cstate_counter_arch_info *cai) +{ + int ret = add_cstate_perf_counter_(cpu, cci, cai); + + if (debug) + fprintf(stderr, "%s: %d (cpu: %d)\n", __func__, ret, cpu); + + return ret; +} + +void cstate_perf_init_(bool soft_c1) +{ + bool has_counter; + bool *cores_visited; + const int cores_visited_elems = topo.max_core_id + 1; + const int cci_num = topo.max_cpu_num + 1; + + ccstate_counter_info = calloc(cci_num, sizeof(*ccstate_counter_info)); + if (!ccstate_counter_info) + err(1, "calloc ccstate_counter_arch_info"); + ccstate_counter_info_size = cci_num; + + cores_visited = calloc(cores_visited_elems, sizeof(*cores_visited)); + if (!cores_visited) + err(1, "calloc cores_visited"); + + /* Initialize cstate_counter_info_percpu */ + for (int cpu = 0; cpu < cci_num; ++cpu) + ccstate_counter_info[cpu].fd_perf = -1; + + for (int cidx = 0; cidx < NUM_CCSTATE_COUNTERS; ++cidx) { + has_counter = false; + memset(cores_visited, 0, cores_visited_elems * sizeof(*cores_visited)); + + const struct cstate_counter_arch_info *cai = &ccstate_counter_arch_infos[cidx]; + + for (int cpu = 0; cpu < cci_num; ++cpu) { - if (platform->supported_cstates & CC3) - BIC_PRESENT(BIC_CPU_c3); + struct cstate_counter_info_t *const cci = &ccstate_counter_info[cpu]; - if (platform->supported_cstates & CC6) - BIC_PRESENT(BIC_CPU_c6); + if (cpu_is_not_allowed(cpu)) + continue; + + const int core_id = cpus[cpu].physical_core_id; + + assert(core_id < cores_visited_elems); + + const bool per_thread = cai->flags & CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD; + + if (!per_thread && cores_visited[core_id]) + continue; + + const bool counter_needed = BIC_IS_ENABLED(cai->bic) || + (soft_c1 && (cai->flags & CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY)); + const bool counter_supported = + platform->supported_cstates & cai->feature_mask; + + if (counter_needed && counter_supported) { + /* Use perf API for this counter */ + if (!no_perf && cai->perf_name + && add_cstate_perf_counter(cpu, cci, cai) != -1) { - if (platform->supported_cstates & CC7) - BIC_PRESENT(BIC_CPU_c7); + cci->source[cai->rci_index] = CSTATE_SOURCE_PERF; + + /* User MSR for this counter */ + } else if (!no_msr && cai->msr && probe_msr(cpu, cai->msr) == 0) { + cci->source[cai->rci_index] = CSTATE_SOURCE_MSR; + cci->msr[cai->rci_index] = cai->msr; + } + } + + if (cci->source[cai->rci_index] != CSTATE_SOURCE_NONE) { + has_counter = true; + cores_visited[core_id] = true; + } + } + + /* If any CPU has access to the counter, make it present */ + if (has_counter) + BIC_PRESENT(cai->bic); + } + + free(cores_visited); +} + +void cstate_perf_init(void) +{ + /* + * If we don't have a C1 residency MSR, we calculate it "in software", + * but we need APERF, MPERF too. + */ + const bool soft_c1 = !platform->has_msr_core_c1_res && has_amperf_access() + && platform->supported_cstates & CC1; + + if (soft_c1) + BIC_PRESENT(BIC_CPU_c1); + + cstate_perf_init_(soft_c1); +} + +void probe_cstates(void) +{ + probe_cst_limit(); if (platform->supported_cstates & PC2 && (pkg_cstate_limit >= PCL__2)) BIC_PRESENT(BIC_Pkgpc2); @@ -7042,6 +7318,19 @@ void process_cpuid() BIC_PRESENT(BIC_TSC_MHz); } +static void counter_info_init(void) +{ + for (int i = 0; i < NUM_CCSTATE_COUNTERS; ++i) { + struct cstate_counter_arch_info *const cai = &ccstate_counter_arch_infos[i]; + + if (platform->has_msr_knl_core_c6_residency && cai->msr == MSR_CORE_C6_RESIDENCY) + cai->msr = MSR_KNL_CORE_C6_RESIDENCY; + + if (!platform->has_msr_core_c1_res && cai->msr == MSR_CORE_C1_RES) + cai->msr = 0; + } +} + void probe_pm_features(void) { probe_pstates(); @@ -7519,10 +7808,12 @@ void turbostat_init() check_msr_access(); check_perf_access(); process_cpuid(); + counter_info_init(); probe_pm_features(); set_amperf_source(); linux_perf_init(); rapl_perf_init(); + cstate_perf_init(); for_all_cpus(get_cpu_type, ODD_COUNTERS); for_all_cpus(get_cpu_type, EVEN_COUNTERS); From 0451adf4d46d5df91f888a3d010a4109aa23a7ae Mon Sep 17 00:00:00 2001 From: Patryk Wlazlyn Date: Wed, 8 May 2024 15:00:14 +0200 Subject: [PATCH 1657/1702] tools/power turbostat: Read Package-cstates via perf Reading the counters via perf can be done in bulk with a single syscall, making the counter values more accurate with respect to one another by minimizing the time gap between individual counter reads. Signed-off-by: Patryk Wlazlyn Signed-off-by: Len Brown --- tools/power/x86/turbostat/turbostat.c | 374 +++++++++++++++++--------- 1 file changed, 244 insertions(+), 130 deletions(-) diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index 9d0278d179655..a3842e9277992 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -224,6 +224,28 @@ unsigned long long bic_present = BIC_USEC | BIC_TOD | BIC_sysfs | BIC_APIC | BIC #define BIC_NOT_PRESENT(COUNTER_BIT) (bic_present &= ~COUNTER_BIT) #define BIC_IS_ENABLED(COUNTER_BIT) (bic_enabled & COUNTER_BIT) +/* + * MSR_PKG_CST_CONFIG_CONTROL decoding for pkg_cstate_limit: + * If you change the values, note they are used both in comparisons + * (>= PCL__7) and to index pkg_cstate_limit_strings[]. + */ +#define PCLUKN 0 /* Unknown */ +#define PCLRSV 1 /* Reserved */ +#define PCL__0 2 /* PC0 */ +#define PCL__1 3 /* PC1 */ +#define PCL__2 4 /* PC2 */ +#define PCL__3 5 /* PC3 */ +#define PCL__4 6 /* PC4 */ +#define PCL__6 7 /* PC6 */ +#define PCL_6N 8 /* PC6 No Retention */ +#define PCL_6R 9 /* PC6 Retention */ +#define PCL__7 10 /* PC7 */ +#define PCL_7S 11 /* PC7 Shrink */ +#define PCL__8 12 /* PC8 */ +#define PCL__9 13 /* PC9 */ +#define PCL_10 14 /* PC10 */ +#define PCLUNL 15 /* Unlimited */ + struct amperf_group_fd; char *proc_stat = "/proc/stat"; @@ -1190,21 +1212,30 @@ enum ccstate_rci_index { CCSTATE_RCI_INDEX_C3_RESIDENCY = 1, CCSTATE_RCI_INDEX_C6_RESIDENCY = 2, CCSTATE_RCI_INDEX_C7_RESIDENCY = 3, - NUM_CCSTATE_COUNTERS, + PCSTATE_RCI_INDEX_C2_RESIDENCY = 4, + PCSTATE_RCI_INDEX_C3_RESIDENCY = 5, + PCSTATE_RCI_INDEX_C6_RESIDENCY = 6, + PCSTATE_RCI_INDEX_C7_RESIDENCY = 7, + PCSTATE_RCI_INDEX_C8_RESIDENCY = 8, + PCSTATE_RCI_INDEX_C9_RESIDENCY = 9, + PCSTATE_RCI_INDEX_C10_RESIDENCY = 10, + NUM_CSTATE_COUNTERS, }; struct cstate_counter_info_t { - unsigned long long data[NUM_CCSTATE_COUNTERS]; - enum cstate_source source[NUM_CCSTATE_COUNTERS]; - unsigned long long msr[NUM_CCSTATE_COUNTERS]; - int fd_perf; + unsigned long long data[NUM_CSTATE_COUNTERS]; + enum cstate_source source[NUM_CSTATE_COUNTERS]; + unsigned long long msr[NUM_CSTATE_COUNTERS]; + int fd_perf_core; + int fd_perf_pkg; }; struct cstate_counter_info_t *ccstate_counter_info; unsigned int ccstate_counter_info_size; -#define CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD (1u << 0) -#define CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY (1u << 1) +#define CSTATE_COUNTER_FLAG_COLLECT_PER_CORE (1u << 0) +#define CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD ((1u << 1) | CSTATE_COUNTER_FLAG_COLLECT_PER_CORE) +#define CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY (1u << 2) struct cstate_counter_arch_info { int feature_mask; /* Mask for testing if the counter is supported on host */ @@ -1214,6 +1245,7 @@ struct cstate_counter_arch_info { unsigned int rci_index; /* Maps data from perf counters to global variables */ unsigned long long bic; unsigned long long flags; + int pkg_cstate_limit; }; static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { @@ -1225,6 +1257,7 @@ static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { .rci_index = CCSTATE_RCI_INDEX_C1_RESIDENCY, .bic = BIC_CPU_c1, .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD, + .pkg_cstate_limit = 0, }, { .feature_mask = CC3, @@ -1233,7 +1266,8 @@ static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { .msr = MSR_CORE_C3_RESIDENCY, .rci_index = CCSTATE_RCI_INDEX_C3_RESIDENCY, .bic = BIC_CPU_c3, - .flags = CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY, + .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY, + .pkg_cstate_limit = 0, }, { .feature_mask = CC6, @@ -1242,7 +1276,8 @@ static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { .msr = MSR_CORE_C6_RESIDENCY, .rci_index = CCSTATE_RCI_INDEX_C6_RESIDENCY, .bic = BIC_CPU_c6, - .flags = CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY, + .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY, + .pkg_cstate_limit = 0, }, { .feature_mask = CC7, @@ -1251,7 +1286,78 @@ static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { .msr = MSR_CORE_C7_RESIDENCY, .rci_index = CCSTATE_RCI_INDEX_C7_RESIDENCY, .bic = BIC_CPU_c7, - .flags = CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY, + .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY, + .pkg_cstate_limit = 0, + }, + { + .feature_mask = PC2, + .perf_subsys = "cstate_pkg", + .perf_name = "c2-residency", + .msr = MSR_PKG_C2_RESIDENCY, + .rci_index = PCSTATE_RCI_INDEX_C2_RESIDENCY, + .bic = BIC_Pkgpc2, + .flags = 0, + .pkg_cstate_limit = PCL__2, + }, + { + .feature_mask = PC3, + .perf_subsys = "cstate_pkg", + .perf_name = "c3-residency", + .msr = MSR_PKG_C3_RESIDENCY, + .rci_index = PCSTATE_RCI_INDEX_C3_RESIDENCY, + .bic = BIC_Pkgpc3, + .flags = 0, + .pkg_cstate_limit = PCL__3, + }, + { + .feature_mask = PC6, + .perf_subsys = "cstate_pkg", + .perf_name = "c6-residency", + .msr = MSR_PKG_C6_RESIDENCY, + .rci_index = PCSTATE_RCI_INDEX_C6_RESIDENCY, + .bic = BIC_Pkgpc6, + .flags = 0, + .pkg_cstate_limit = PCL__6, + }, + { + .feature_mask = PC7, + .perf_subsys = "cstate_pkg", + .perf_name = "c7-residency", + .msr = MSR_PKG_C7_RESIDENCY, + .rci_index = PCSTATE_RCI_INDEX_C7_RESIDENCY, + .bic = BIC_Pkgpc7, + .flags = 0, + .pkg_cstate_limit = PCL__7, + }, + { + .feature_mask = PC8, + .perf_subsys = "cstate_pkg", + .perf_name = "c8-residency", + .msr = MSR_PKG_C8_RESIDENCY, + .rci_index = PCSTATE_RCI_INDEX_C8_RESIDENCY, + .bic = BIC_Pkgpc8, + .flags = 0, + .pkg_cstate_limit = PCL__8, + }, + { + .feature_mask = PC9, + .perf_subsys = "cstate_pkg", + .perf_name = "c9-residency", + .msr = MSR_PKG_C9_RESIDENCY, + .rci_index = PCSTATE_RCI_INDEX_C9_RESIDENCY, + .bic = BIC_Pkgpc9, + .flags = 0, + .pkg_cstate_limit = PCL__9, + }, + { + .feature_mask = PC10, + .perf_subsys = "cstate_pkg", + .perf_name = "c10-residency", + .msr = MSR_PKG_C10_RESIDENCY, + .rci_index = PCSTATE_RCI_INDEX_C10_RESIDENCY, + .bic = BIC_Pkgpc10, + .flags = 0, + .pkg_cstate_limit = PCL_10, }, }; @@ -1641,15 +1747,8 @@ int get_msr_fd(int cpu) static void bic_disable_msr_access(void) { - const unsigned long bic_msrs = - BIC_SMI | - BIC_Mod_c6 | - BIC_CoreTmp | - BIC_Totl_c0 | - BIC_Any_c0 | - BIC_GFX_c0 | - BIC_CPUGFX | - BIC_Pkgpc2 | BIC_Pkgpc3 | BIC_Pkgpc6 | BIC_Pkgpc7 | BIC_Pkgpc8 | BIC_Pkgpc9 | BIC_Pkgpc10 | BIC_PkgTmp; + const unsigned long bic_msrs = BIC_SMI | BIC_Mod_c6 | BIC_CoreTmp | + BIC_Totl_c0 | BIC_Any_c0 | BIC_GFX_c0 | BIC_CPUGFX | BIC_PkgTmp; bic_enabled &= ~bic_msrs; @@ -3493,7 +3592,7 @@ static size_t cstate_counter_info_count_perf(const struct cstate_counter_info_t { size_t ret = 0; - for (int i = 0; i < NUM_CCSTATE_COUNTERS; ++i) + for (int i = 0; i < NUM_CSTATE_COUNTERS; ++i) if (cci->source[i] == CSTATE_SOURCE_PERF) ++ret; @@ -3598,9 +3697,16 @@ char *find_sysfs_path_by_id(struct sysfs_path *sp, int id) return NULL; } -int get_cstate_counters(unsigned int cpu, struct thread_data *t, struct core_data *c) +int get_cstate_counters(unsigned int cpu, struct thread_data *t, struct core_data *c, struct pkg_data *p) { - unsigned long long perf_data[NUM_CCSTATE_COUNTERS + 1]; + /* + * Overcommit memory a little bit here, + * but skip calculating exact sizes for the buffers. + */ + unsigned long long perf_data[NUM_CSTATE_COUNTERS]; + unsigned long long perf_data_core[NUM_CSTATE_COUNTERS + 1]; + unsigned long long perf_data_pkg[NUM_CSTATE_COUNTERS + 1]; + struct cstate_counter_info_t *cci; if (debug) @@ -3609,35 +3715,72 @@ int get_cstate_counters(unsigned int cpu, struct thread_data *t, struct core_dat assert(ccstate_counter_info); assert(cpu <= ccstate_counter_info_size); + memset(perf_data, 0, sizeof(perf_data)); + memset(perf_data_core, 0, sizeof(perf_data_core)); + memset(perf_data_pkg, 0, sizeof(perf_data_pkg)); + cci = &ccstate_counter_info[cpu]; /* * If we have any perf counters to read, read them all now, in bulk */ - if (cci->fd_perf != -1) { - const size_t num_perf_counters = cstate_counter_info_count_perf(cci); - const ssize_t expected_read_size = - (num_perf_counters + 1) * sizeof(unsigned long long); - const ssize_t actual_read_size = - read(cci->fd_perf, &perf_data[0], sizeof(perf_data)); + const size_t num_perf_counters = cstate_counter_info_count_perf(cci); + ssize_t expected_read_size = num_perf_counters * sizeof(unsigned long long); + ssize_t actual_read_size_core = 0, actual_read_size_pkg = 0; - if (actual_read_size != expected_read_size) - err(-1, "%s: failed to read perf_data (%zu %zu)", - __func__, expected_read_size, actual_read_size); + if (cci->fd_perf_core != -1) { + /* Each descriptor read begins with number of counters read. */ + expected_read_size += sizeof(unsigned long long); + + actual_read_size_core = read(cci->fd_perf_core, &perf_data_core[0], sizeof(perf_data_core)); + + if (actual_read_size_core <= 0) + err(-1, "%s: read perf %s: %ld", __func__, "core", actual_read_size_core); + } + + if (cci->fd_perf_pkg != -1) { + /* Each descriptor read begins with number of counters read. */ + expected_read_size += sizeof(unsigned long long); + + actual_read_size_pkg = read(cci->fd_perf_pkg, &perf_data_pkg[0], sizeof(perf_data_pkg)); + + if (actual_read_size_pkg <= 0) + err(-1, "%s: read perf %s: %ld", __func__, "pkg", actual_read_size_pkg); } - for (unsigned int i = 0, pi = 1; i < NUM_CCSTATE_COUNTERS; ++i) { + const ssize_t actual_read_size_total = actual_read_size_core + actual_read_size_pkg; + + if (actual_read_size_total != expected_read_size) + err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size_total); + + /* + * Copy ccstate and pcstate data into unified buffer. + * + * Skip first element from core and pkg buffers. + * Kernel puts there how many counters were read. + */ + const size_t num_core_counters = perf_data_core[0]; + const size_t num_pkg_counters = perf_data_pkg[0]; + + assert(num_perf_counters == num_core_counters + num_pkg_counters); + + /* Copy ccstate perf data */ + memcpy(&perf_data[0], &perf_data_core[1], num_core_counters * sizeof(unsigned long long)); + + /* Copy pcstate perf data */ + memcpy(&perf_data[num_core_counters], &perf_data_pkg[1], num_pkg_counters * sizeof(unsigned long long)); + + for (unsigned int i = 0, pi = 0; i < NUM_CSTATE_COUNTERS; ++i) { switch (cci->source[i]) { case CSTATE_SOURCE_NONE: break; case CSTATE_SOURCE_PERF: assert(pi < ARRAY_SIZE(perf_data)); - assert(cci->fd_perf != -1); + assert(cci->fd_perf_core != -1 || cci->fd_perf_pkg != -1); if (debug) { - fprintf(stderr, "cstate via %s %u: %llu\n", - "perf", i, perf_data[pi]); + fprintf(stderr, "cstate via %s %u: %llu\n", "perf", i, perf_data[pi]); } cci->data[i] = perf_data[pi]; @@ -3651,8 +3794,7 @@ int get_cstate_counters(unsigned int cpu, struct thread_data *t, struct core_dat return -13 - i; if (debug) { - fprintf(stderr, "cstate via %s0x%llx %u: %llu\n", - "msr", cci->msr[i], i, cci->data[i]); + fprintf(stderr, "cstate via %s0x%llx %u: %llu\n", "msr", cci->msr[i], i, cci->data[i]); } break; @@ -3671,12 +3813,21 @@ int get_cstate_counters(unsigned int cpu, struct thread_data *t, struct core_dat out_counter = cci->data[index]; \ } while (0) - BUILD_BUG_ON(NUM_CCSTATE_COUNTERS != 4); + BUILD_BUG_ON(NUM_CSTATE_COUNTERS != 11); + PERF_COUNTER_WRITE_DATA(t->c1, CCSTATE_RCI_INDEX_C1_RESIDENCY); PERF_COUNTER_WRITE_DATA(c->c3, CCSTATE_RCI_INDEX_C3_RESIDENCY); PERF_COUNTER_WRITE_DATA(c->c6, CCSTATE_RCI_INDEX_C6_RESIDENCY); PERF_COUNTER_WRITE_DATA(c->c7, CCSTATE_RCI_INDEX_C7_RESIDENCY); + PERF_COUNTER_WRITE_DATA(p->pc2, PCSTATE_RCI_INDEX_C2_RESIDENCY); + PERF_COUNTER_WRITE_DATA(p->pc3, PCSTATE_RCI_INDEX_C3_RESIDENCY); + PERF_COUNTER_WRITE_DATA(p->pc6, PCSTATE_RCI_INDEX_C6_RESIDENCY); + PERF_COUNTER_WRITE_DATA(p->pc7, PCSTATE_RCI_INDEX_C7_RESIDENCY); + PERF_COUNTER_WRITE_DATA(p->pc8, PCSTATE_RCI_INDEX_C8_RESIDENCY); + PERF_COUNTER_WRITE_DATA(p->pc9, PCSTATE_RCI_INDEX_C9_RESIDENCY); + PERF_COUNTER_WRITE_DATA(p->pc10, PCSTATE_RCI_INDEX_C10_RESIDENCY); + #undef PERF_COUNTER_WRITE_DATA return 0; @@ -3738,7 +3889,7 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) t->smi_count = msr & 0xFFFFFFFF; } - get_cstate_counters(cpu, t, c); + get_cstate_counters(cpu, t, c, p); for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { if (get_mp(cpu, mp, &t->counter[i], mp->sp->path)) @@ -3803,34 +3954,6 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) if (get_msr(cpu, MSR_PKG_BOTH_CORE_GFXE_C0_RES, &p->pkg_both_core_gfxe_c0)) return -13; } - if (DO_BIC(BIC_Pkgpc3)) - if (get_msr(cpu, MSR_PKG_C3_RESIDENCY, &p->pc3)) - return -9; - if (DO_BIC(BIC_Pkgpc6)) { - if (platform->has_msr_atom_pkg_c6_residency) { - if (get_msr(cpu, MSR_ATOM_PKG_C6_RESIDENCY, &p->pc6)) - return -10; - } else { - if (get_msr(cpu, MSR_PKG_C6_RESIDENCY, &p->pc6)) - return -10; - } - } - - if (DO_BIC(BIC_Pkgpc2)) - if (get_msr(cpu, MSR_PKG_C2_RESIDENCY, &p->pc2)) - return -11; - if (DO_BIC(BIC_Pkgpc7)) - if (get_msr(cpu, MSR_PKG_C7_RESIDENCY, &p->pc7)) - return -12; - if (DO_BIC(BIC_Pkgpc8)) - if (get_msr(cpu, MSR_PKG_C8_RESIDENCY, &p->pc8)) - return -13; - if (DO_BIC(BIC_Pkgpc9)) - if (get_msr(cpu, MSR_PKG_C9_RESIDENCY, &p->pc9)) - return -13; - if (DO_BIC(BIC_Pkgpc10)) - if (get_msr(cpu, MSR_PKG_C10_RESIDENCY, &p->pc10)) - return -13; if (DO_BIC(BIC_CPU_LPI)) p->cpu_lpi = cpuidle_cur_cpu_lpi_us; @@ -3889,29 +4012,6 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) return 0; } -/* - * MSR_PKG_CST_CONFIG_CONTROL decoding for pkg_cstate_limit: - * If you change the values, note they are used both in comparisons - * (>= PCL__7) and to index pkg_cstate_limit_strings[]. - */ - -#define PCLUKN 0 /* Unknown */ -#define PCLRSV 1 /* Reserved */ -#define PCL__0 2 /* PC0 */ -#define PCL__1 3 /* PC1 */ -#define PCL__2 4 /* PC2 */ -#define PCL__3 5 /* PC3 */ -#define PCL__4 6 /* PC4 */ -#define PCL__6 7 /* PC6 */ -#define PCL_6N 8 /* PC6 No Retention */ -#define PCL_6R 9 /* PC6 Retention */ -#define PCL__7 10 /* PC7 */ -#define PCL_7S 11 /* PC7 Shrink */ -#define PCL__8 12 /* PC8 */ -#define PCL__9 13 /* PC9 */ -#define PCL_10 14 /* PC10 */ -#define PCLUNL 15 /* Unlimited */ - int pkg_cstate_limit = PCLUKN; char *pkg_cstate_limit_strings[] = { "reserved", "unknown", "pc0", "pc1", "pc2", "pc3", "pc4", "pc6", "pc6n", "pc6r", "pc7", "pc7s", "pc8", "pc9", "pc10", "unlimited" @@ -4410,8 +4510,11 @@ void free_fd_cstate(void) const int counter_info_num = ccstate_counter_info_size; for (int counter_id = 0; counter_id < counter_info_num; ++counter_id) { - if (ccstate_counter_info[counter_id].fd_perf != -1) - close(ccstate_counter_info[counter_id].fd_perf); + if (ccstate_counter_info[counter_id].fd_perf_core != -1) + close(ccstate_counter_info[counter_id].fd_perf_core); + + if (ccstate_counter_info[counter_id].fd_perf_pkg != -1) + close(ccstate_counter_info[counter_id].fd_perf_pkg); } free(ccstate_counter_info); @@ -6914,30 +7017,43 @@ static int has_amperf_access(void) return 0; } -int add_cstate_perf_counter_(int cpu, struct cstate_counter_info_t *cci, - const struct cstate_counter_arch_info *cai) +int *get_cstate_perf_group_fd(struct cstate_counter_info_t *cci, const char *group_name) +{ + if (strcmp(group_name, "cstate_core") == 0) + return &cci->fd_perf_core; + + if (strcmp(group_name, "cstate_pkg") == 0) + return &cci->fd_perf_pkg; + + return NULL; +} + +int add_cstate_perf_counter_(int cpu, struct cstate_counter_info_t *cci, const struct cstate_counter_arch_info *cai) { if (no_perf) return -1; + int *pfd_group = get_cstate_perf_group_fd(cci, cai->perf_subsys); + + if (pfd_group == NULL) + return -1; + const unsigned int type = read_perf_type(cai->perf_subsys); const unsigned int config = read_rapl_config(cai->perf_subsys, cai->perf_name); - const int fd_counter = - open_perf_counter(cpu, type, config, cci->fd_perf, PERF_FORMAT_GROUP); + const int fd_counter = open_perf_counter(cpu, type, config, *pfd_group, PERF_FORMAT_GROUP); if (fd_counter == -1) return -1; /* If it's the first counter opened, make it a group descriptor */ - if (cci->fd_perf == -1) - cci->fd_perf = fd_counter; + if (*pfd_group == -1) + *pfd_group = fd_counter; return fd_counter; } -int add_cstate_perf_counter(int cpu, struct cstate_counter_info_t *cci, - const struct cstate_counter_arch_info *cai) +int add_cstate_perf_counter(int cpu, struct cstate_counter_info_t *cci, const struct cstate_counter_arch_info *cai) { int ret = add_cstate_perf_counter_(cpu, cci, cai); @@ -6950,8 +7066,9 @@ int add_cstate_perf_counter(int cpu, struct cstate_counter_info_t *cci, void cstate_perf_init_(bool soft_c1) { bool has_counter; - bool *cores_visited; + bool *cores_visited = NULL, *pkg_visited = NULL; const int cores_visited_elems = topo.max_core_id + 1; + const int pkg_visited_elems = topo.max_package_id + 1; const int cci_num = topo.max_cpu_num + 1; ccstate_counter_info = calloc(cci_num, sizeof(*ccstate_counter_info)); @@ -6963,13 +7080,20 @@ void cstate_perf_init_(bool soft_c1) if (!cores_visited) err(1, "calloc cores_visited"); + pkg_visited = calloc(pkg_visited_elems, sizeof(*pkg_visited)); + if (!pkg_visited) + err(1, "calloc pkg_visited"); + /* Initialize cstate_counter_info_percpu */ - for (int cpu = 0; cpu < cci_num; ++cpu) - ccstate_counter_info[cpu].fd_perf = -1; + for (int cpu = 0; cpu < cci_num; ++cpu) { + ccstate_counter_info[cpu].fd_perf_core = -1; + ccstate_counter_info[cpu].fd_perf_pkg = -1; + } - for (int cidx = 0; cidx < NUM_CCSTATE_COUNTERS; ++cidx) { + for (int cidx = 0; cidx < NUM_CSTATE_COUNTERS; ++cidx) { has_counter = false; memset(cores_visited, 0, cores_visited_elems * sizeof(*cores_visited)); + memset(pkg_visited, 0, pkg_visited_elems * sizeof(*pkg_visited)); const struct cstate_counter_arch_info *cai = &ccstate_counter_arch_infos[cidx]; @@ -6981,23 +7105,29 @@ void cstate_perf_init_(bool soft_c1) continue; const int core_id = cpus[cpu].physical_core_id; + const int pkg_id = cpus[cpu].physical_package_id; assert(core_id < cores_visited_elems); + assert(pkg_id < pkg_visited_elems); const bool per_thread = cai->flags & CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD; + const bool per_core = cai->flags & CSTATE_COUNTER_FLAG_COLLECT_PER_CORE; if (!per_thread && cores_visited[core_id]) continue; + if (!per_core && pkg_visited[pkg_id]) + continue; + const bool counter_needed = BIC_IS_ENABLED(cai->bic) || (soft_c1 && (cai->flags & CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY)); const bool counter_supported = - platform->supported_cstates & cai->feature_mask; + (platform->supported_cstates & cai->feature_mask) && + (pkg_cstate_limit >= cai->pkg_cstate_limit); if (counter_needed && counter_supported) { /* Use perf API for this counter */ - if (!no_perf && cai->perf_name - && add_cstate_perf_counter(cpu, cci, cai) != -1) { + if (!no_perf && cai->perf_name && add_cstate_perf_counter(cpu, cci, cai) != -1) { cci->source[cai->rci_index] = CSTATE_SOURCE_PERF; @@ -7011,6 +7141,7 @@ void cstate_perf_init_(bool soft_c1) if (cci->source[cai->rci_index] != CSTATE_SOURCE_NONE) { has_counter = true; cores_visited[core_id] = true; + pkg_visited[pkg_id] = true; } } @@ -7020,6 +7151,7 @@ void cstate_perf_init_(bool soft_c1) } free(cores_visited); + free(pkg_visited); } void cstate_perf_init(void) @@ -7029,7 +7161,7 @@ void cstate_perf_init(void) * but we need APERF, MPERF too. */ const bool soft_c1 = !platform->has_msr_core_c1_res && has_amperf_access() - && platform->supported_cstates & CC1; + && platform->supported_cstates & CC1; if (soft_c1) BIC_PRESENT(BIC_CPU_c1); @@ -7041,27 +7173,6 @@ void probe_cstates(void) { probe_cst_limit(); - if (platform->supported_cstates & PC2 && (pkg_cstate_limit >= PCL__2)) - BIC_PRESENT(BIC_Pkgpc2); - - if (platform->supported_cstates & PC3 && (pkg_cstate_limit >= PCL__3)) - BIC_PRESENT(BIC_Pkgpc3); - - if (platform->supported_cstates & PC6 && (pkg_cstate_limit >= PCL__6)) - BIC_PRESENT(BIC_Pkgpc6); - - if (platform->supported_cstates & PC7 && (pkg_cstate_limit >= PCL__7)) - BIC_PRESENT(BIC_Pkgpc7); - - if (platform->supported_cstates & PC8 && (pkg_cstate_limit >= PCL__8)) - BIC_PRESENT(BIC_Pkgpc8); - - if (platform->supported_cstates & PC9 && (pkg_cstate_limit >= PCL__9)) - BIC_PRESENT(BIC_Pkgpc9); - - if (platform->supported_cstates & PC10 && (pkg_cstate_limit >= PCL_10)) - BIC_PRESENT(BIC_Pkgpc10); - if (platform->has_msr_module_c6_res_ms) BIC_PRESENT(BIC_Mod_c6); @@ -7320,7 +7431,7 @@ void process_cpuid() static void counter_info_init(void) { - for (int i = 0; i < NUM_CCSTATE_COUNTERS; ++i) { + for (int i = 0; i < NUM_CSTATE_COUNTERS; ++i) { struct cstate_counter_arch_info *const cai = &ccstate_counter_arch_infos[i]; if (platform->has_msr_knl_core_c6_residency && cai->msr == MSR_CORE_C6_RESIDENCY) @@ -7328,6 +7439,9 @@ static void counter_info_init(void) if (!platform->has_msr_core_c1_res && cai->msr == MSR_CORE_C1_RES) cai->msr = 0; + + if (platform->has_msr_atom_pkg_c6_residency && cai->msr == MSR_PKG_C6_RESIDENCY) + cai->msr = MSR_ATOM_PKG_C6_RESIDENCY; } } From 4e7ee02300805d26d9731fd24c4de8e10a43ffea Mon Sep 17 00:00:00 2001 From: Patryk Wlazlyn Date: Thu, 9 May 2024 12:24:02 +0200 Subject: [PATCH 1658/1702] tools/power turbostat: Fix order of strings in pkg_cstate_limit_strings Change the order so that it matches the indexes defined in: Signed-off-by: Patryk Wlazlyn Signed-off-by: Len Brown --- tools/power/x86/turbostat/turbostat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index a3842e9277992..b45b2f494416c 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -4013,7 +4013,7 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) } int pkg_cstate_limit = PCLUKN; -char *pkg_cstate_limit_strings[] = { "reserved", "unknown", "pc0", "pc1", "pc2", +char *pkg_cstate_limit_strings[] = { "unknown", "reserved", "pc0", "pc1", "pc2", "pc3", "pc4", "pc6", "pc6n", "pc6r", "pc7", "pc7s", "pc8", "pc9", "pc10", "unlimited" }; From 29fea61cd8d4d0e646022c0479aa35381cf1e990 Mon Sep 17 00:00:00 2001 From: Patryk Wlazlyn Date: Thu, 9 May 2024 12:39:47 +0200 Subject: [PATCH 1659/1702] tools/power turbostat: Ignore pkg_cstate_limit when it is not available When running in no-msr mode, the pkg_cstate_limit is not populated, thus we use perf to determine if given pcstate counter is present on the platform. Signed-off-by: Patryk Wlazlyn Signed-off-by: Len Brown --- tools/power/x86/turbostat/turbostat.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index b45b2f494416c..8cb18d42c189c 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -7121,9 +7121,7 @@ void cstate_perf_init_(bool soft_c1) const bool counter_needed = BIC_IS_ENABLED(cai->bic) || (soft_c1 && (cai->flags & CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY)); - const bool counter_supported = - (platform->supported_cstates & cai->feature_mask) && - (pkg_cstate_limit >= cai->pkg_cstate_limit); + const bool counter_supported = (platform->supported_cstates & cai->feature_mask); if (counter_needed && counter_supported) { /* Use perf API for this counter */ @@ -7132,7 +7130,8 @@ void cstate_perf_init_(bool soft_c1) cci->source[cai->rci_index] = CSTATE_SOURCE_PERF; /* User MSR for this counter */ - } else if (!no_msr && cai->msr && probe_msr(cpu, cai->msr) == 0) { + } else if (!no_msr && cai->msr && pkg_cstate_limit >= cai->pkg_cstate_limit + && probe_msr(cpu, cai->msr) == 0) { cci->source[cai->rci_index] = CSTATE_SOURCE_MSR; cci->msr[cai->rci_index] = cai->msr; } From 256d218ec6aea99855dc5c54af550fcff96fc732 Mon Sep 17 00:00:00 2001 From: Len Brown Date: Sat, 27 Apr 2024 22:15:48 -0400 Subject: [PATCH 1660/1702] tools/power turbostat: version 2024.05.10 New since 2024.04.08: Len Brown (6): tools/power turbostat: Add "snapshot:" Makefile target tools/power turbostat: Harden probe_intel_uncore_frequency() tools/power turbostat: Remember global max_die_id tools/power turbostat: Survive sparse die_id tools/power turbostat: Add columns for clustered uncore frequency tools/power turbostat: version 2024.05.10 Patryk Wlazlyn (7): tools/power turbostat: Replace _Static_assert with BUILD_BUG_ON tools/power turbostat: Enable non-privileged users to read sysfs counters tools/power turbostat: Avoid possible memory corruption due to sparse topology IDs tools/power turbostat: Read Core-cstates via perf tools/power turbostat: Read Package-cstates via perf tools/power turbostat: Fix order of strings in pkg_cstate_limit_strings tools/power turbostat: Ignore pkg_cstate_limit when it is not available Zhang Rui (2): tools/power turbostat: Enhance ARL/LNL support tools/power turbostat: Add ARL-H support Signed-off-by: Len Brown --- tools/power/x86/turbostat/turbostat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index 8cb18d42c189c..8cdf41906e981 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -8017,7 +8017,7 @@ int get_and_dump_counters(void) void print_version() { - fprintf(outf, "turbostat version 2024.04.08 - Len Brown \n"); + fprintf(outf, "turbostat version 2024.05.10 - Len Brown \n"); } #define COMMAND_LINE_SIZE 2048 From 99975ad644c7836414183fa7be4f883a4fb2bf64 Mon Sep 17 00:00:00 2001 From: Herve Codina Date: Mon, 13 May 2024 13:18:53 +0200 Subject: [PATCH 1661/1702] net: lan966x: remove debugfs directory in probe() error path A debugfs directory entry is create early during probe(). This entry is not removed on error path leading to some "already present" issues in case of EPROBE_DEFER. Create this entry later in the probe() code to avoid the need to change many 'return' in 'goto' and add the removal in the already present error path. Fixes: 942814840127 ("net: lan966x: Add VCAP debugFS support") Cc: Signed-off-by: Herve Codina Reviewed-by: Andrew Lunn Reviewed-by: Horatiu Vultur Signed-off-by: David S. Miller --- drivers/net/ethernet/microchip/lan966x/lan966x_main.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c index b12d3b8a64fd3..a3b919a3ce073 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c @@ -1087,8 +1087,6 @@ static int lan966x_probe(struct platform_device *pdev) platform_set_drvdata(pdev, lan966x); lan966x->dev = &pdev->dev; - lan966x->debugfs_root = debugfs_create_dir("lan966x", NULL); - if (!device_get_mac_address(&pdev->dev, mac_addr)) { ether_addr_copy(lan966x->base_mac, mac_addr); } else { @@ -1179,6 +1177,8 @@ static int lan966x_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, -ENODEV, "no ethernet-ports child found\n"); + lan966x->debugfs_root = debugfs_create_dir("lan966x", NULL); + /* init switch */ lan966x_init(lan966x); lan966x_stats_init(lan966x); @@ -1257,6 +1257,8 @@ static int lan966x_probe(struct platform_device *pdev) destroy_workqueue(lan966x->stats_queue); mutex_destroy(&lan966x->stats_lock); + debugfs_remove_recursive(lan966x->debugfs_root); + return err; } From fd76e5ccc48f9f54eb44909dd7c0b924005f1582 Mon Sep 17 00:00:00 2001 From: Chris Lew Date: Mon, 13 May 2024 10:31:46 -0700 Subject: [PATCH 1662/1702] net: qrtr: ns: Fix module refcnt The qrtr protocol core logic and the qrtr nameservice are combined into a single module. Neither the core logic or nameservice provide much functionality by themselves; combining the two into a single module also prevents any possible issues that may stem from client modules loading inbetween qrtr and the ns. Creating a socket takes two references to the module that owns the socket protocol. Since the ns needs to create the control socket, this creates a scenario where there are always two references to the qrtr module. This prevents the execution of 'rmmod' for qrtr. To resolve this, forcefully put the module refcount for the socket opened by the nameservice. Fixes: a365023a76f2 ("net: qrtr: combine nameservice into main module") Reported-by: Jeffrey Hugo Tested-by: Jeffrey Hugo Signed-off-by: Chris Lew Reviewed-by: Manivannan Sadhasivam Reviewed-by: Jeffrey Hugo Signed-off-by: David S. Miller --- net/qrtr/ns.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/net/qrtr/ns.c b/net/qrtr/ns.c index abb0c70ffc8b0..654a3cc0d3479 100644 --- a/net/qrtr/ns.c +++ b/net/qrtr/ns.c @@ -725,6 +725,24 @@ int qrtr_ns_init(void) if (ret < 0) goto err_wq; + /* As the qrtr ns socket owner and creator is the same module, we have + * to decrease the qrtr module reference count to guarantee that it + * remains zero after the ns socket is created, otherwise, executing + * "rmmod" command is unable to make the qrtr module deleted after the + * qrtr module is inserted successfully. + * + * However, the reference count is increased twice in + * sock_create_kern(): one is to increase the reference count of owner + * of qrtr socket's proto_ops struct; another is to increment the + * reference count of owner of qrtr proto struct. Therefore, we must + * decrement the module reference count twice to ensure that it keeps + * zero after server's listening socket is created. Of course, we + * must bump the module reference count twice as well before the socket + * is closed. + */ + module_put(qrtr_ns.sock->ops->owner); + module_put(qrtr_ns.sock->sk->sk_prot_creator->owner); + return 0; err_wq: @@ -739,6 +757,15 @@ void qrtr_ns_remove(void) { cancel_work_sync(&qrtr_ns.work); destroy_workqueue(qrtr_ns.workqueue); + + /* sock_release() expects the two references that were put during + * qrtr_ns_init(). This function is only called during module remove, + * so try_stop_module() has already set the refcnt to 0. Use + * __module_get() instead of try_module_get() to successfully take two + * references. + */ + __module_get(qrtr_ns.sock->ops->owner); + __module_get(qrtr_ns.sock->sk->sk_prot_creator->owner); sock_release(qrtr_ns.sock); } EXPORT_SYMBOL_GPL(qrtr_ns_remove); From 83e93942796db58652288f0391ac00072401816f Mon Sep 17 00:00:00 2001 From: Hangbin Liu Date: Tue, 14 May 2024 10:33:59 +0800 Subject: [PATCH 1663/1702] selftests/net/lib: no need to record ns name if it already exist There is no need to add the name to ns_list again if the netns already recoreded. Fixes: 25ae948b4478 ("selftests/net: add lib.sh") Signed-off-by: Hangbin Liu Reviewed-by: Simon Horman Signed-off-by: David S. Miller --- tools/testing/selftests/net/lib.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/net/lib.sh b/tools/testing/selftests/net/lib.sh index c868c0aec1216..72b191e4e064e 100644 --- a/tools/testing/selftests/net/lib.sh +++ b/tools/testing/selftests/net/lib.sh @@ -127,15 +127,17 @@ setup_ns() local ns="" local ns_name="" local ns_list="" + local ns_exist= for ns_name in "$@"; do # Some test may setup/remove same netns multi times if unset ${ns_name} 2> /dev/null; then ns="${ns_name,,}-$(mktemp -u XXXXXX)" eval readonly ${ns_name}="$ns" + ns_exist=false else eval ns='$'${ns_name} cleanup_ns "$ns" - + ns_exist=true fi if ! ip netns add "$ns"; then @@ -144,7 +146,7 @@ setup_ns() return $ksft_skip fi ip -n "$ns" link set lo up - ns_list="$ns_list $ns" + ! $ns_exist && ns_list="$ns_list $ns" done NS_LIST="$NS_LIST $ns_list" } From dd0716c2b87792ebea30864e7ad1df461d4c1525 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Thu, 16 May 2024 12:22:40 +0200 Subject: [PATCH 1664/1702] x86/boot: Add a fallthrough annotation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add implicit fallthrough checking to the decompressor code and fix this warning: arch/x86/boot/printf.c: In function ‘vsprintf’: arch/x86/boot/printf.c:248:10: warning: this statement may fall through [-Wimplicit-fallthrough=] 248 | flags |= SMALL; | ^ arch/x86/boot/printf.c:249:3: note: here 249 | case 'X': | ^~~~ This is a patch from three years ago which I found in my trees, thus the SUSE authorship still. Signed-off-by: Borislav Petkov Signed-off-by: Borislav Petkov (AMD) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20240516102240.16270-1-bp@kernel.org --- arch/x86/boot/Makefile | 1 + arch/x86/boot/printf.c | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile index 3cece19b74732..343aef6d752ff 100644 --- a/arch/x86/boot/Makefile +++ b/arch/x86/boot/Makefile @@ -69,6 +69,7 @@ KBUILD_CFLAGS := $(REALMODE_CFLAGS) -D_SETUP KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ KBUILD_CFLAGS += $(call cc-option,-fmacro-prefix-map=$(srctree)/=) KBUILD_CFLAGS += -fno-asynchronous-unwind-tables +KBUILD_CFLAGS += $(CONFIG_CC_IMPLICIT_FALLTHROUGH) GCOV_PROFILE := n UBSAN_SANITIZE := n diff --git a/arch/x86/boot/printf.c b/arch/x86/boot/printf.c index 1237beeb95400..c0ec1dc355abd 100644 --- a/arch/x86/boot/printf.c +++ b/arch/x86/boot/printf.c @@ -246,6 +246,7 @@ int vsprintf(char *buf, const char *fmt, va_list args) case 'x': flags |= SMALL; + fallthrough; case 'X': base = 16; break; From e9a4062e1527238c5649d0f4be794a8566fd77c9 Mon Sep 17 00:00:00 2001 From: Daniel Bristot de Oliveira Date: Thu, 16 May 2024 16:15:22 +0200 Subject: [PATCH 1665/1702] rtla: Add --trace-buffer-size option Add the option allow the users to set a different buffer size for the trace. For example, in large systems, the user might be interested on reducing the trace buffer to avoid large tracing files. The buffer size is specified in kB, and it is only affecting the tracing instance. The function trace_set_buffer_size() appears on libtracefs v1.6, so increase the minimum required version on Makefile.config. Link: https://lkml.kernel.org/r/e7c9ca5b3865f28e131a49ec3b984fadf2d056c6.1715860611.git.bristot@kernel.org Cc: Jonathan Corbet Cc: Juri Lelli Cc: John Kacur Signed-off-by: Daniel Bristot de Oliveira --- Documentation/tools/rtla/common_options.rst | 3 +++ tools/tracing/rtla/Makefile.config | 2 +- tools/tracing/rtla/src/osnoise_hist.c | 13 ++++++++++++- tools/tracing/rtla/src/osnoise_top.c | 14 +++++++++++++- tools/tracing/rtla/src/timerlat_hist.c | 14 +++++++++++++- tools/tracing/rtla/src/timerlat_top.c | 14 +++++++++++++- tools/tracing/rtla/src/trace.c | 15 +++++++++++++++ tools/tracing/rtla/src/trace.h | 1 + 8 files changed, 71 insertions(+), 5 deletions(-) diff --git a/Documentation/tools/rtla/common_options.rst b/Documentation/tools/rtla/common_options.rst index a96ea0ed662ef..7ac7b75814661 100644 --- a/Documentation/tools/rtla/common_options.rst +++ b/Documentation/tools/rtla/common_options.rst @@ -54,6 +54,9 @@ After starting the workload, let it run for *s* seconds before starting collecting the data, allowing the system to warm-up. Statistical data generated during warm-up is discarded. +**--trace-buffer-size** *kB* + Set the per-cpu trace buffer size in kB for the tracing output. + **-h**, **--help** Print help menu. diff --git a/tools/tracing/rtla/Makefile.config b/tools/tracing/rtla/Makefile.config index 6d4ba77847b68..0b7ecfb30d197 100644 --- a/tools/tracing/rtla/Makefile.config +++ b/tools/tracing/rtla/Makefile.config @@ -3,7 +3,7 @@ STOP_ERROR := LIBTRACEEVENT_MIN_VERSION = 1.5 -LIBTRACEFS_MIN_VERSION = 1.3 +LIBTRACEFS_MIN_VERSION = 1.6 define lib_setup $(eval LIB_INCLUDES += $(shell sh -c "$(PKG_CONFIG) --cflags lib$(1)")) diff --git a/tools/tracing/rtla/src/osnoise_hist.c b/tools/tracing/rtla/src/osnoise_hist.c index c6100ff46a7f0..198a17a3ea2e6 100644 --- a/tools/tracing/rtla/src/osnoise_hist.c +++ b/tools/tracing/rtla/src/osnoise_hist.c @@ -43,6 +43,7 @@ struct osnoise_hist_params { int bucket_size; int entries; int warmup; + int buffer_size; }; struct osnoise_hist_cpu { @@ -469,6 +470,7 @@ static void osnoise_hist_usage(char *usage) " d:runtime[us|ms|s]:period[us|ms|s] - use SCHED_DEADLINE with runtime and period", " in nanoseconds", " --warm-up: let the workload run for s seconds before collecting data", + " --trace-buffer-size kB: set the per-cpu trace buffer size in kB", NULL, }; @@ -533,13 +535,14 @@ static struct osnoise_hist_params {"trigger", required_argument, 0, '4'}, {"filter", required_argument, 0, '5'}, {"warm-up", required_argument, 0, '6'}, + {"trace-buffer-size", required_argument, 0, '7'}, {0, 0, 0, 0} }; /* getopt_long stores the option index here. */ int option_index = 0; - c = getopt_long(argc, argv, "a:c:C::b:d:e:E:DhH:p:P:r:s:S:t::T:01234:5:6:", + c = getopt_long(argc, argv, "a:c:C::b:d:e:E:DhH:p:P:r:s:S:t::T:01234:5:6:7:", long_options, &option_index); /* detect the end of the options. */ @@ -685,6 +688,9 @@ static struct osnoise_hist_params case '6': params->warmup = get_llong_from_str(optarg); break; + case '7': + params->buffer_size = get_llong_from_str(optarg); + break; default: osnoise_hist_usage("Invalid option"); } @@ -891,6 +897,11 @@ int osnoise_hist_main(int argc, char *argv[]) goto out_hist; } + if (params->buffer_size > 0) { + retval = trace_set_buffer_size(&record->trace, params->buffer_size); + if (retval) + goto out_hist; + } } /* diff --git a/tools/tracing/rtla/src/osnoise_top.c b/tools/tracing/rtla/src/osnoise_top.c index 53a074c1222e3..7e5aab22727da 100644 --- a/tools/tracing/rtla/src/osnoise_top.c +++ b/tools/tracing/rtla/src/osnoise_top.c @@ -41,6 +41,7 @@ struct osnoise_top_params { int cgroup; int hk_cpus; int warmup; + int buffer_size; cpu_set_t hk_cpu_set; struct sched_attr sched_param; struct trace_events *events; @@ -309,6 +310,7 @@ static void osnoise_top_usage(struct osnoise_top_params *params, char *usage) " d:runtime[us|ms|s]:period[us|ms|s] - use SCHED_DEADLINE with runtime and period", " in nanoseconds", " --warm-up s: let the workload run for s seconds before collecting data", + " --trace-buffer-size kB: set the per-cpu trace buffer size in kB", NULL, }; @@ -384,13 +386,14 @@ struct osnoise_top_params *osnoise_top_parse_args(int argc, char **argv) {"trigger", required_argument, 0, '0'}, {"filter", required_argument, 0, '1'}, {"warm-up", required_argument, 0, '2'}, + {"trace-buffer-size", required_argument, 0, '3'}, {0, 0, 0, 0} }; /* getopt_long stores the option index here. */ int option_index = 0; - c = getopt_long(argc, argv, "a:c:C::d:De:hH:p:P:qr:s:S:t::T:0:1:2:", + c = getopt_long(argc, argv, "a:c:C::d:De:hH:p:P:qr:s:S:t::T:0:1:2:3:", long_options, &option_index); /* Detect the end of the options. */ @@ -517,6 +520,9 @@ struct osnoise_top_params *osnoise_top_parse_args(int argc, char **argv) case '2': params->warmup = get_llong_from_str(optarg); break; + case '3': + params->buffer_size = get_llong_from_str(optarg); + break; default: osnoise_top_usage(params, "Invalid option"); } @@ -725,6 +731,12 @@ int osnoise_top_main(int argc, char **argv) if (retval) goto out_top; } + + if (params->buffer_size > 0) { + retval = trace_set_buffer_size(&record->trace, params->buffer_size); + if (retval) + goto out_top; + } } /* diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c index 6eb6e38d4a05b..d4bab86ca1b95 100644 --- a/tools/tracing/rtla/src/timerlat_hist.c +++ b/tools/tracing/rtla/src/timerlat_hist.c @@ -54,6 +54,7 @@ struct timerlat_hist_params { int bucket_size; int entries; int warmup; + int buffer_size; }; struct timerlat_hist_cpu { @@ -669,6 +670,7 @@ static void timerlat_hist_usage(char *usage) " -k/--kernel-threads: use timerlat kernel-space threads instead of rtla user-space threads", " -U/--user-load: enable timerlat for user-defined user-space workload", " --warm-up s: let the workload run for s seconds before collecting data", + " --trace-buffer-size kB: set the per-cpu trace buffer size in kB", NULL, }; @@ -745,13 +747,14 @@ static struct timerlat_hist_params {"no-aa", no_argument, 0, '9'}, {"dump-task", no_argument, 0, '\1'}, {"warm-up", required_argument, 0, '\2'}, + {"trace-buffer-size", required_argument, 0, '\3'}, {0, 0, 0, 0} }; /* getopt_long stores the option index here. */ int option_index = 0; - c = getopt_long(argc, argv, "a:c:C::b:d:e:E:DhH:i:knp:P:s:t::T:uU0123456:7:8:9\1\2:", + c = getopt_long(argc, argv, "a:c:C::b:d:e:E:DhH:i:knp:P:s:t::T:uU0123456:7:8:9\1\2:\3", long_options, &option_index); /* detect the end of the options. */ @@ -926,6 +929,9 @@ static struct timerlat_hist_params case '\2': params->warmup = get_llong_from_str(optarg); break; + case '\3': + params->buffer_size = get_llong_from_str(optarg); + break; default: timerlat_hist_usage("Invalid option"); } @@ -1179,6 +1185,12 @@ int timerlat_hist_main(int argc, char *argv[]) if (retval) goto out_hist; } + + if (params->buffer_size > 0) { + retval = trace_set_buffer_size(&record->trace, params->buffer_size); + if (retval) + goto out_hist; + } } if (!params->no_aa) { diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index 0acfefe151f75..3a23e8d481c66 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -47,6 +47,7 @@ struct timerlat_top_params { int kernel_workload; int pretty_output; int warmup; + int buffer_size; cpu_set_t hk_cpu_set; struct sched_attr sched_param; struct trace_events *events; @@ -479,6 +480,7 @@ static void timerlat_top_usage(char *usage) " -k/--kernel-threads: use timerlat kernel-space threads instead of rtla user-space threads", " -U/--user-load: enable timerlat for user-defined user-space workload", " --warm-up s: let the workload run for s seconds before collecting data", + " --trace-buffer-size kB: set the per-cpu trace buffer size in kB", NULL, }; @@ -547,13 +549,14 @@ static struct timerlat_top_params {"dump-tasks", no_argument, 0, '4'}, {"aa-only", required_argument, 0, '5'}, {"warm-up", required_argument, 0, '6'}, + {"trace-buffer-size", required_argument, 0, '7'}, {0, 0, 0, 0} }; /* getopt_long stores the option index here. */ int option_index = 0; - c = getopt_long(argc, argv, "a:c:C::d:De:hH:i:knp:P:qs:t::T:uU0:1:2:345:6:", + c = getopt_long(argc, argv, "a:c:C::d:De:hH:i:knp:P:qs:t::T:uU0:1:2:345:6:7:", long_options, &option_index); /* detect the end of the options. */ @@ -716,6 +719,9 @@ static struct timerlat_top_params case '6': params->warmup = get_llong_from_str(optarg); break; + case '7': + params->buffer_size = get_llong_from_str(optarg); + break; default: timerlat_top_usage("Invalid option"); } @@ -973,6 +979,12 @@ int timerlat_top_main(int argc, char *argv[]) if (retval) goto out_top; } + + if (params->buffer_size > 0) { + retval = trace_set_buffer_size(&record->trace, params->buffer_size); + if (retval) + goto out_top; + } } if (!params->no_aa) { diff --git a/tools/tracing/rtla/src/trace.c b/tools/tracing/rtla/src/trace.c index e1ba6d9f42658..170a706248abf 100644 --- a/tools/tracing/rtla/src/trace.c +++ b/tools/tracing/rtla/src/trace.c @@ -540,3 +540,18 @@ int trace_is_off(struct trace_instance *tool, struct trace_instance *trace) return 0; } + +/* + * trace_set_buffer_size - set the per-cpu tracing buffer size. + */ +int trace_set_buffer_size(struct trace_instance *trace, int size) +{ + int retval; + + debug_msg("Setting trace buffer size to %d Kb\n", size); + retval = tracefs_instance_set_buffer_size(trace->inst, size, -1); + if (retval) + err_msg("Error setting trace buffer size\n"); + + return retval; +} diff --git a/tools/tracing/rtla/src/trace.h b/tools/tracing/rtla/src/trace.h index 2e9a89a256150..c7c92dc9a18a6 100644 --- a/tools/tracing/rtla/src/trace.h +++ b/tools/tracing/rtla/src/trace.h @@ -48,3 +48,4 @@ int trace_events_enable(struct trace_instance *instance, int trace_event_add_filter(struct trace_events *event, char *filter); int trace_event_add_trigger(struct trace_events *event, char *trigger); int trace_is_off(struct trace_instance *tool, struct trace_instance *trace); +int trace_set_buffer_size(struct trace_instance *trace, int size); From 01b05fc0e5f3aec443a9a8ffa0022cbca2fd3608 Mon Sep 17 00:00:00 2001 From: John Kacur Date: Fri, 10 May 2024 15:03:18 -0400 Subject: [PATCH 1666/1702] rtla/timerlat: Fix histogram report when a cpu count is 0 On short runs it is possible to get no samples on a cpu, like this: # rtla timerlat hist -u -T50 Index IRQ-001 Thr-001 Usr-001 IRQ-002 Thr-002 Usr-002 2 1 0 0 0 0 0 33 0 1 0 0 0 0 36 0 0 1 0 0 0 49 0 0 0 1 0 0 52 0 0 0 0 1 0 over: 0 0 0 0 0 0 count: 1 1 1 1 1 0 min: 2 33 36 49 52 18446744073709551615 avg: 2 33 36 49 52 - max: 2 33 36 49 52 0 rtla timerlat hit stop tracing IRQ handler delay: (exit from idle) 48.21 us (91.09 %) IRQ latency: 49.11 us Timerlat IRQ duration: 2.17 us (4.09 %) Blocking thread: 1.01 us (1.90 %) swapper/2:0 1.01 us ------------------------------------------------------------------------ Thread latency: 52.93 us (100%) Max timerlat IRQ latency from idle: 49.11 us in cpu 2 Note, the value 18446744073709551615 is the same as ~0. Fix this by reporting no results for the min, avg and max if the count is 0. Link: https://lkml.kernel.org/r/20240510190318.44295-1-jkacur@redhat.com Cc: stable@vger.kernel.org Fixes: 1eeb6328e8b3 ("rtla/timerlat: Add timerlat hist mode") Suggested-by: Daniel Bristot de Oliveria Signed-off-by: John Kacur Signed-off-by: Daniel Bristot de Oliveira --- tools/tracing/rtla/src/timerlat_hist.c | 60 ++++++++++++++++++-------- 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c index d4bab86ca1b95..fbe2c6549bf94 100644 --- a/tools/tracing/rtla/src/timerlat_hist.c +++ b/tools/tracing/rtla/src/timerlat_hist.c @@ -327,17 +327,29 @@ timerlat_print_summary(struct timerlat_hist_params *params, if (!data->hist[cpu].irq_count && !data->hist[cpu].thread_count) continue; - if (!params->no_irq) - trace_seq_printf(trace->seq, "%9llu ", - data->hist[cpu].min_irq); + if (!params->no_irq) { + if (data->hist[cpu].irq_count) + trace_seq_printf(trace->seq, "%9llu ", + data->hist[cpu].min_irq); + else + trace_seq_printf(trace->seq, " - "); + } - if (!params->no_thread) - trace_seq_printf(trace->seq, "%9llu ", - data->hist[cpu].min_thread); + if (!params->no_thread) { + if (data->hist[cpu].thread_count) + trace_seq_printf(trace->seq, "%9llu ", + data->hist[cpu].min_thread); + else + trace_seq_printf(trace->seq, " - "); + } - if (params->user_hist) - trace_seq_printf(trace->seq, "%9llu ", - data->hist[cpu].min_user); + if (params->user_hist) { + if (data->hist[cpu].user_count) + trace_seq_printf(trace->seq, "%9llu ", + data->hist[cpu].min_user); + else + trace_seq_printf(trace->seq, " - "); + } } trace_seq_printf(trace->seq, "\n"); @@ -387,17 +399,29 @@ timerlat_print_summary(struct timerlat_hist_params *params, if (!data->hist[cpu].irq_count && !data->hist[cpu].thread_count) continue; - if (!params->no_irq) - trace_seq_printf(trace->seq, "%9llu ", - data->hist[cpu].max_irq); + if (!params->no_irq) { + if (data->hist[cpu].irq_count) + trace_seq_printf(trace->seq, "%9llu ", + data->hist[cpu].max_irq); + else + trace_seq_printf(trace->seq, " - "); + } - if (!params->no_thread) - trace_seq_printf(trace->seq, "%9llu ", - data->hist[cpu].max_thread); + if (!params->no_thread) { + if (data->hist[cpu].thread_count) + trace_seq_printf(trace->seq, "%9llu ", + data->hist[cpu].max_thread); + else + trace_seq_printf(trace->seq, " - "); + } - if (params->user_hist) - trace_seq_printf(trace->seq, "%9llu ", - data->hist[cpu].max_user); + if (params->user_hist) { + if (data->hist[cpu].user_count) + trace_seq_printf(trace->seq, "%9llu ", + data->hist[cpu].max_user); + else + trace_seq_printf(trace->seq, " - "); + } } trace_seq_printf(trace->seq, "\n"); trace_seq_do_printf(trace->seq); From 842fc5b87a5058d475b9411dbb94ed49f8d6bce3 Mon Sep 17 00:00:00 2001 From: John Kacur Date: Wed, 15 May 2024 14:30:23 -0400 Subject: [PATCH 1667/1702] rtla: Fix -t\--trace[=file] The -t option has an optional argument. The usual case is for a short option to be specified without an '=' and for the long version to be specified with an '=' Various forms of this do not work as expected. For example: rtla timerlat hist -T50 -tfile.txt will result in a truncated file name of "ile.txt" Another example is that the long form without the '=' will result in the default file name instead of the requested file name. This patch properly parses the optional argument with and without '=' and with and without spaces for the short form. This patch was also tested using -t and --trace without providing a file name both as the last requested option and with a following long and short option. For example: rtla timerlat hist -T50 -t -u rtla timerlat hist -T50 --trace -u This fix is applied to both timerlat top and hist and to osnoise top and hist. Here is the full testing for rtla timerlat hist. Before applying the patch rtla timerlat hist -T50 -t=file.txt Works as expected, "file.txt" rtla timerlat hist -T50 -tfile.txt Truncated file name "ile.txt" rtla timerlat hist -T50 -t file.txt Default file name instead of file.txt rtla timerlat hist -T50 --trace=file.txt Truncated file name "ile.txt" rtla timerlat hist -T50 --trace file.txt Default file name "timerlat_trace.txt" instead of "file.txt" After applying the patch: rtla timerlat hist -T50 -t=file.txt Works as expected, "file.txt" rtla timerlat hist -T50 -tfile.txt Works as expected, "file.txt" rtla timerlat hist -T50 -t file.txt Works as expected, "file.txt" rtla timerlat hist -T50 --trace=file.txt Works as expected, "file.txt" rtla timerlat hist -T50 --trace file.txt Works as expected, "file.txt" In addition the following tests were performed to make sure that the default file name worked as expected including with trailing options. rtla timerlat hist -T50 -t Works as expected "timerlat_trace.txt" rtla timerlat hist -T50 --trace Works as expected "timerlat_trace.txt" rtla timerlat hist -T50 -t -u Works as expected "timerlat_trace.txt" rtla timerlat hist -T50 --trace -u Works as expected "timerlat_trace.txt" Link: https://lkml.kernel.org/r/20240515183024.59985-1-jkacur@redhat.com Cc: Daniel Bristot de Oliveria Signed-off-by: John Kacur Signed-off-by: Daniel Bristot de Oliveira --- tools/tracing/rtla/src/osnoise_hist.c | 14 +++++++++----- tools/tracing/rtla/src/osnoise_top.c | 14 +++++++++----- tools/tracing/rtla/src/timerlat_hist.c | 14 +++++++++----- tools/tracing/rtla/src/timerlat_top.c | 14 +++++++++----- 4 files changed, 36 insertions(+), 20 deletions(-) diff --git a/tools/tracing/rtla/src/osnoise_hist.c b/tools/tracing/rtla/src/osnoise_hist.c index 198a17a3ea2e6..7be17d09f7e85 100644 --- a/tools/tracing/rtla/src/osnoise_hist.c +++ b/tools/tracing/rtla/src/osnoise_hist.c @@ -437,7 +437,7 @@ static void osnoise_hist_usage(char *usage) static const char * const msg[] = { "", " usage: rtla osnoise hist [-h] [-D] [-d s] [-a us] [-p us] [-r us] [-s us] [-S us] \\", - " [-T us] [-t[=file]] [-e sys[:event]] [--filter ] [--trigger ] \\", + " [-T us] [-t[file]] [-e sys[:event]] [--filter ] [--trigger ] \\", " [-c cpu-list] [-H cpu-list] [-P priority] [-b N] [-E N] [--no-header] [--no-summary] \\", " [--no-index] [--with-zeros] [-C[=cgroup_name]] [--warm-up]", "", @@ -453,7 +453,7 @@ static void osnoise_hist_usage(char *usage) " -C/--cgroup[=cgroup_name]: set cgroup, if no cgroup_name is passed, the rtla's cgroup will be inherited", " -d/--duration time[s|m|h|d]: duration of the session", " -D/--debug: print debug info", - " -t/--trace[=file]: save the stopped trace to [file|osnoise_trace.txt]", + " -t/--trace[file]: save the stopped trace to [file|osnoise_trace.txt]", " -e/--event : enable the in the trace instance, multiple -e are allowed", " --filter : enable a trace event filter to the previous -e event", " --trigger : enable a trace event trigger to the previous -e event", @@ -645,9 +645,13 @@ static struct osnoise_hist_params params->threshold = get_llong_from_str(optarg); break; case 't': - if (optarg) - /* skip = */ - params->trace_output = &optarg[1]; + if (optarg) { + if (optarg[0] == '=') + params->trace_output = &optarg[1]; + else + params->trace_output = &optarg[0]; + } else if (optind < argc && argv[optind][0] != '0') + params->trace_output = argv[optind]; else params->trace_output = "osnoise_trace.txt"; break; diff --git a/tools/tracing/rtla/src/osnoise_top.c b/tools/tracing/rtla/src/osnoise_top.c index 7e5aab22727da..07ba55d4ec06f 100644 --- a/tools/tracing/rtla/src/osnoise_top.c +++ b/tools/tracing/rtla/src/osnoise_top.c @@ -283,7 +283,7 @@ static void osnoise_top_usage(struct osnoise_top_params *params, char *usage) static const char * const msg[] = { " [-h] [-q] [-D] [-d s] [-a us] [-p us] [-r us] [-s us] [-S us] \\", - " [-T us] [-t[=file]] [-e sys[:event]] [--filter ] [--trigger ] \\", + " [-T us] [-t[file]] [-e sys[:event]] [--filter ] [--trigger ] \\", " [-c cpu-list] [-H cpu-list] [-P priority] [-C[=cgroup_name]] [--warm-up s]", "", " -h/--help: print this menu", @@ -298,7 +298,7 @@ static void osnoise_top_usage(struct osnoise_top_params *params, char *usage) " -C/--cgroup[=cgroup_name]: set cgroup, if no cgroup_name is passed, the rtla's cgroup will be inherited", " -d/--duration time[s|m|h|d]: duration of the session", " -D/--debug: print debug info", - " -t/--trace[=file]: save the stopped trace to [file|osnoise_trace.txt]", + " -t/--trace[file]: save the stopped trace to [file|osnoise_trace.txt]", " -e/--event : enable the in the trace instance, multiple -e are allowed", " --filter : enable a trace event filter to the previous -e event", " --trigger : enable a trace event trigger to the previous -e event", @@ -486,9 +486,13 @@ struct osnoise_top_params *osnoise_top_parse_args(int argc, char **argv) params->stop_total_us = get_llong_from_str(optarg); break; case 't': - if (optarg) - /* skip = */ - params->trace_output = &optarg[1]; + if (optarg) { + if (optarg[0] == '=') + params->trace_output = &optarg[1]; + else + params->trace_output = &optarg[0]; + } else if (optind < argc && argv[optind][0] != '-') + params->trace_output = argv[optind]; else params->trace_output = "osnoise_trace.txt"; break; diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c index fbe2c6549bf94..a3907c390d67a 100644 --- a/tools/tracing/rtla/src/timerlat_hist.c +++ b/tools/tracing/rtla/src/timerlat_hist.c @@ -652,7 +652,7 @@ static void timerlat_hist_usage(char *usage) char *msg[] = { "", " usage: [rtla] timerlat hist [-h] [-q] [-d s] [-D] [-n] [-a us] [-p us] [-i us] [-T us] [-s us] \\", - " [-t[=file]] [-e sys[:event]] [--filter ] [--trigger ] [-c cpu-list] [-H cpu-list]\\", + " [-t[file]] [-e sys[:event]] [--filter ] [--trigger ] [-c cpu-list] [-H cpu-list]\\", " [-P priority] [-E N] [-b N] [--no-irq] [--no-thread] [--no-header] [--no-summary] \\", " [--no-index] [--with-zeros] [--dma-latency us] [-C[=cgroup_name]] [--no-aa] [--dump-task] [-u|-k]", " [--warm-up s]", @@ -669,7 +669,7 @@ static void timerlat_hist_usage(char *usage) " -d/--duration time[m|h|d]: duration of the session in seconds", " --dump-tasks: prints the task running on all CPUs if stop conditions are met (depends on !--no-aa)", " -D/--debug: print debug info", - " -t/--trace[=file]: save the stopped trace to [file|timerlat_trace.txt]", + " -t/--trace[file]: save the stopped trace to [file|timerlat_trace.txt]", " -e/--event : enable the in the trace instance, multiple -e are allowed", " --filter : enable a trace event filter to the previous -e event", " --trigger : enable a trace event trigger to the previous -e event", @@ -885,9 +885,13 @@ static struct timerlat_hist_params params->stop_total_us = get_llong_from_str(optarg); break; case 't': - if (optarg) - /* skip = */ - params->trace_output = &optarg[1]; + if (optarg) { + if (optarg[0] == '=') + params->trace_output = &optarg[1]; + else + params->trace_output = &optarg[0]; + } else if (optind < argc && argv[optind][0] != '-') + params->trace_output = argv[optind]; else params->trace_output = "timerlat_trace.txt"; break; diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index 3a23e8d481c66..8c16419fe22aa 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -446,7 +446,7 @@ static void timerlat_top_usage(char *usage) static const char *const msg[] = { "", " usage: rtla timerlat [top] [-h] [-q] [-a us] [-d s] [-D] [-n] [-p us] [-i us] [-T us] [-s us] \\", - " [[-t[=file]] [-e sys[:event]] [--filter ] [--trigger ] [-c cpu-list] [-H cpu-list]\\", + " [[-t[file]] [-e sys[:event]] [--filter ] [--trigger ] [-c cpu-list] [-H cpu-list]\\", " [-P priority] [--dma-latency us] [--aa-only us] [-C[=cgroup_name]] [-u|-k] [--warm-up s]", "", " -h/--help: print this menu", @@ -462,7 +462,7 @@ static void timerlat_top_usage(char *usage) " -d/--duration time[m|h|d]: duration of the session in seconds", " -D/--debug: print debug info", " --dump-tasks: prints the task running on all CPUs if stop conditions are met (depends on !--no-aa)", - " -t/--trace[=file]: save the stopped trace to [file|timerlat_trace.txt]", + " -t/--trace[file]: save the stopped trace to [file|timerlat_trace.txt]", " -e/--event : enable the in the trace instance, multiple -e are allowed", " --filter : enable a trace event filter to the previous -e event", " --trigger : enable a trace event trigger to the previous -e event", @@ -668,9 +668,13 @@ static struct timerlat_top_params params->stop_total_us = get_llong_from_str(optarg); break; case 't': - if (optarg) - /* skip = */ - params->trace_output = &optarg[1]; + if (optarg) { + if (optarg[0] == '=') + params->trace_output = &optarg[1]; + else + params->trace_output = &optarg[0]; + } else if (optind < argc && argv[optind][0] != '-') + params->trace_output = argv[optind]; else params->trace_output = "timerlat_trace.txt"; From 59c22f70b2951d81de410d477ae536ba951b4f37 Mon Sep 17 00:00:00 2001 From: John Kacur Date: Thu, 16 May 2024 10:31:21 -0400 Subject: [PATCH 1668/1702] rtla: Documentation: Fix -t, --trace Move -t, --trace from common_options.rst to common_osnoise_options.rst and common_timerlat_options.rst so that it will appear in the man pages rtla-timerlat-hist.1 rtla-timerlat-top.1 rtla-osnoise-hist.1 rtla-osnoise-top.1 Remove the equals ('=') sign and add a space. Link: https://lkml.kernel.org/r/20240516143121.12614-1-jkacur@redhat.com Cc: Daniel Bristot de Oliveria Signed-off-by: John Kacur Signed-off-by: Daniel Bristot de Oliveira --- Documentation/tools/rtla/common_options.rst | 4 ---- Documentation/tools/rtla/common_osnoise_options.rst | 4 ++++ Documentation/tools/rtla/common_timerlat_options.rst | 4 ++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Documentation/tools/rtla/common_options.rst b/Documentation/tools/rtla/common_options.rst index 7ac7b75814661..2dc1575210aa1 100644 --- a/Documentation/tools/rtla/common_options.rst +++ b/Documentation/tools/rtla/common_options.rst @@ -14,10 +14,6 @@ Print debug info. -**-t**, **--trace**\[*=file*] - - Save the stopped trace to [*file|osnoise_trace.txt*]. - **-e**, **--event** *sys:event* Enable an event in the trace (**-t**) session. The argument can be a specific event, e.g., **-e** *sched:sched_switch*, or all events of a system group, e.g., **-e** *sched*. Multiple **-e** are allowed. It is only active when **-t** or **-a** are set. diff --git a/Documentation/tools/rtla/common_osnoise_options.rst b/Documentation/tools/rtla/common_osnoise_options.rst index f792ca58c211a..d73de2d58f5f3 100644 --- a/Documentation/tools/rtla/common_osnoise_options.rst +++ b/Documentation/tools/rtla/common_osnoise_options.rst @@ -25,3 +25,7 @@ Specify the minimum delta between two time reads to be considered noise. The default threshold is *5 us*. + +**-t**, **--trace** \[*file*] + + Save the stopped trace to [*file|osnoise_trace.txt*]. diff --git a/Documentation/tools/rtla/common_timerlat_options.rst b/Documentation/tools/rtla/common_timerlat_options.rst index 090700a6ae9f0..cef6651f14358 100644 --- a/Documentation/tools/rtla/common_timerlat_options.rst +++ b/Documentation/tools/rtla/common_timerlat_options.rst @@ -22,6 +22,10 @@ Save the stack trace at the *IRQ* if a *Thread* latency is higher than the argument in us. +**-t**, **--trace** \[*file*] + + Save the stopped trace to [*file|timerlat_trace.txt*]. + **--dma-latency** *us* Set the /dev/cpu_dma_latency to *us*, aiming to bound exit from idle latencies. *cyclictest* sets this value to *0* by default, use **--dma-latency** *0* to have From 5405807edd4168c2dc2f307f3c6b70e9579bf7be Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Thu, 16 May 2024 10:01:40 -0700 Subject: [PATCH 1669/1702] selftests/bpf: Adjust test_access_variable_array after a kernel function name change After commit 4c3e509ea9f2 ("sched/balancing: Rename load_balance() => sched_balance_rq()"), the load_balance kernel function is renamed to sched_balance_rq. This patch adjusts the fentry program in test_access_variable_array.c to reflect this kernel function name change. Signed-off-by: Martin KaFai Lau Signed-off-by: Daniel Borkmann Acked-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20240516170140.2689430-1-martin.lau@linux.dev --- tools/testing/selftests/bpf/progs/test_access_variable_array.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/progs/test_access_variable_array.c b/tools/testing/selftests/bpf/progs/test_access_variable_array.c index 808c49b798890..326b7d1f496ab 100644 --- a/tools/testing/selftests/bpf/progs/test_access_variable_array.c +++ b/tools/testing/selftests/bpf/progs/test_access_variable_array.c @@ -7,7 +7,7 @@ unsigned long span = 0; -SEC("fentry/load_balance") +SEC("fentry/sched_balance_rq") int BPF_PROG(fentry_fentry, int this_cpu, struct rq *this_rq, struct sched_domain *sd) { From 51e2b8d33199df9675d2a36ec6aad0c27e91c6fe Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Thu, 16 May 2024 09:43:10 -0700 Subject: [PATCH 1670/1702] selftests/bpf: Adjust btf_dump test to reflect recent change in file_operations The btf_dump test fails: test_btf_dump_struct_data:FAIL:file_operations unexpected file_operations: actual '(struct file_operations){ .owner = (struct module *)0xffffffffffffffff, .fop_flags = (fop_flags_t)4294967295, .llseek = (loff_t (*)(struct f' != expected '(struct file_operations){ .owner = (struct module *)0xffffffffffffffff, .llseek = (loff_t (*)(struct file *, loff_t, int))0xffffffffffffffff,' The "fop_flags" is a recent addition to the struct file_operations in commit 210a03c9d51a ("fs: claw back a few FMODE_* bits") This patch changes the test_btf_dump_struct_data() to reflect this change. Signed-off-by: Martin KaFai Lau Signed-off-by: Daniel Borkmann Reviewed-by: Alan Maguire Link: https://lore.kernel.org/bpf/20240516164310.2481460-1-martin.lau@linux.dev --- tools/testing/selftests/bpf/prog_tests/btf_dump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index e9ea38aa8248b..09a8e6f9b379d 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -653,7 +653,7 @@ static void test_btf_dump_struct_data(struct btf *btf, struct btf_dump *d, cmpstr = "(struct file_operations){\n" " .owner = (struct module *)0xffffffffffffffff,\n" -" .llseek = (loff_t (*)(struct file *, loff_t, int))0xffffffffffffffff,"; +" .fop_flags = (fop_flags_t)4294967295,"; ASSERT_STRNEQ(str, cmpstr, strlen(cmpstr), "file_operations"); } From 988af276360bc555c7685d5a607d4da0588616d9 Mon Sep 17 00:00:00 2001 From: Hangbin Liu Date: Tue, 14 May 2024 17:52:27 +0800 Subject: [PATCH 1671/1702] selftests/net: reduce xfrm_policy test time The check_random_order test add/get plenty of xfrm rules, which consume a lot time on debug kernel and always TIMEOUT. Let's reduce the test loop and see if it works. Signed-off-by: Hangbin Liu Link: https://lore.kernel.org/r/20240514095227.2597730-1-liuhangbin@gmail.com Signed-off-by: Jakub Kicinski --- tools/testing/selftests/net/xfrm_policy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/net/xfrm_policy.sh b/tools/testing/selftests/net/xfrm_policy.sh index 4577895306456..3eeeeffb4005b 100755 --- a/tools/testing/selftests/net/xfrm_policy.sh +++ b/tools/testing/selftests/net/xfrm_policy.sh @@ -293,7 +293,7 @@ check_random_order() local ns=$1 local log=$2 - for i in $(seq 100); do + for i in $(seq 50); do ip -net $ns xfrm policy flush for j in $(seq 0 16 255 | sort -R); do ip -net $ns xfrm policy add dst $j.0.0.0/24 dir out priority 10 action allow @@ -306,7 +306,7 @@ check_random_order() done done - for i in $(seq 100); do + for i in $(seq 50); do ip -net $ns xfrm policy flush for j in $(seq 0 16 255 | sort -R); do local addr=$(printf "e000:0000:%02x00::/56" $j) From bb487272380d120295e955ad8acfcbb281b57642 Mon Sep 17 00:00:00 2001 From: xu xin Date: Tue, 14 May 2024 20:11:02 +0800 Subject: [PATCH 1672/1702] net/ipv6: Fix route deleting failure when metric equals 0 Problem ========= After commit 67f695134703 ("ipv6: Move setting default metric for routes"), we noticed that the logic of assigning the default value of fc_metirc changed in the ioctl process. That is, when users use ioctl(fd, SIOCADDRT, rt) with a non-zero metric to add a route, then they may fail to delete a route with passing in a metric value of 0 to the kernel by ioctl(fd, SIOCDELRT, rt). But iproute can succeed in deleting it. As a reference, when using iproute tools by netlink to delete routes with a metric parameter equals 0, like the command as follows: ip -6 route del fe80::/64 via fe81::5054:ff:fe11:3451 dev eth0 metric 0 the user can still succeed in deleting the route entry with the smallest metric. Root Reason =========== After commit 67f695134703 ("ipv6: Move setting default metric for routes"), When ioctl() pass in SIOCDELRT with a zero metric, rtmsg_to_fib6_config() will set a defalut value (1024) to cfg->fc_metric in kernel, and in ip6_route_del() and the line 4074 at net/ipv3/route.c, it will check by if (cfg->fc_metric && cfg->fc_metric != rt->fib6_metric) continue; and the condition is true and skip the later procedure (deleting route) because cfg->fc_metric != rt->fib6_metric. But before that commit, cfg->fc_metric is still zero there, so the condition is false and it will do the following procedure (deleting). Solution ======== In order to keep a consistent behaviour across netlink() and ioctl(), we should allow to delete a route with a metric value of 0. So we only do the default setting of fc_metric in route adding. CC: stable@vger.kernel.org # 5.4+ Fixes: 67f695134703 ("ipv6: Move setting default metric for routes") Co-developed-by: Fan Yu Signed-off-by: Fan Yu Signed-off-by: xu xin Reviewed-by: David Ahern Link: https://lore.kernel.org/r/20240514201102055dD2Ba45qKbLlUMxu_DTHP@zte.com.cn Signed-off-by: Jakub Kicinski --- net/ipv6/route.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/net/ipv6/route.c b/net/ipv6/route.c index c43b0616742eb..bbc2a0dd93142 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -4445,7 +4445,7 @@ static void rtmsg_to_fib6_config(struct net *net, .fc_table = l3mdev_fib_table_by_index(net, rtmsg->rtmsg_ifindex) ? : RT6_TABLE_MAIN, .fc_ifindex = rtmsg->rtmsg_ifindex, - .fc_metric = rtmsg->rtmsg_metric ? : IP6_RT_PRIO_USER, + .fc_metric = rtmsg->rtmsg_metric, .fc_expires = rtmsg->rtmsg_info, .fc_dst_len = rtmsg->rtmsg_dst_len, .fc_src_len = rtmsg->rtmsg_src_len, @@ -4475,6 +4475,9 @@ int ipv6_route_ioctl(struct net *net, unsigned int cmd, struct in6_rtmsg *rtmsg) rtnl_lock(); switch (cmd) { case SIOCADDRT: + /* Only do the default setting of fc_metric in route adding */ + if (cfg.fc_metric == 0) + cfg.fc_metric = IP6_RT_PRIO_USER; err = ip6_route_add(&cfg, GFP_KERNEL, NULL); break; case SIOCDELRT: From a45835a0bb6ef7d5ddbc0714dd760de979cb6ece Mon Sep 17 00:00:00 2001 From: Tony Battersby Date: Tue, 14 May 2024 15:57:29 -0400 Subject: [PATCH 1673/1702] bonding: fix oops during rmmod "rmmod bonding" causes an oops ever since commit cc317ea3d927 ("bonding: remove redundant NULL check in debugfs function"). Here are the relevant functions being called: bonding_exit() bond_destroy_debugfs() debugfs_remove_recursive(bonding_debug_root); bonding_debug_root = NULL; <--------- SET TO NULL HERE bond_netlink_fini() rtnl_link_unregister() __rtnl_link_unregister() unregister_netdevice_many_notify() bond_uninit() bond_debug_unregister() (commit removed check for bonding_debug_root == NULL) debugfs_remove() simple_recursive_removal() down_write() -> OOPS However, reverting the bad commit does not solve the problem completely because the original code contains a race that could cause the same oops, although it was much less likely to be triggered unintentionally: CPU1 rmmod bonding bonding_exit() bond_destroy_debugfs() debugfs_remove_recursive(bonding_debug_root); CPU2 echo -bond0 > /sys/class/net/bonding_masters bond_uninit() bond_debug_unregister() if (!bonding_debug_root) CPU1 bonding_debug_root = NULL; So do NOT revert the bad commit (since the removed checks were racy anyway), and instead change the order of actions taken during module removal. The same oops can also happen if there is an error during module init, so apply the same fix there. Fixes: cc317ea3d927 ("bonding: remove redundant NULL check in debugfs function") Cc: stable@vger.kernel.org Signed-off-by: Tony Battersby Reviewed-by: Simon Horman Acked-by: Jay Vosburgh Link: https://lore.kernel.org/r/641f914f-3216-4eeb-87dd-91b78aa97773@cybernetics.com Signed-off-by: Jakub Kicinski --- drivers/net/bonding/bond_main.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 3b0c7758ea4f4..3c3fcce4acd48 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -6477,16 +6477,16 @@ static int __init bonding_init(void) if (res) goto out; + bond_create_debugfs(); + res = register_pernet_subsys(&bond_net_ops); if (res) - goto out; + goto err_net_ops; res = bond_netlink_init(); if (res) goto err_link; - bond_create_debugfs(); - for (i = 0; i < max_bonds; i++) { res = bond_create(&init_net, NULL); if (res) @@ -6501,10 +6501,11 @@ static int __init bonding_init(void) out: return res; err: - bond_destroy_debugfs(); bond_netlink_fini(); err_link: unregister_pernet_subsys(&bond_net_ops); +err_net_ops: + bond_destroy_debugfs(); goto out; } @@ -6513,11 +6514,11 @@ static void __exit bonding_exit(void) { unregister_netdevice_notifier(&bond_netdev_notifier); - bond_destroy_debugfs(); - bond_netlink_fini(); unregister_pernet_subsys(&bond_net_ops); + bond_destroy_debugfs(); + #ifdef CONFIG_NET_POLL_CONTROLLER /* Make sure we don't have an imbalance on our netpoll blocking */ WARN_ON(atomic_read(&netpoll_block_tx)); From fe32622763d8bc864231381f34f7521f8694748b Mon Sep 17 00:00:00 2001 From: Sagar Cheluvegowda Date: Tue, 14 May 2024 17:06:52 -0700 Subject: [PATCH 1674/1702] dt-bindings: net: qcom: ethernet: Allow dma-coherent On SA8775P, Ethernet DMA controller is coherent with the CPU. allow specifying that. Signed-off-by: Sagar Cheluvegowda Acked-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20240514-mark_ethernet_devices_dma_coherent-v4-2-04e1198858c5@quicinc.com Signed-off-by: Jakub Kicinski --- Documentation/devicetree/bindings/net/qcom,ethqos.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/net/qcom,ethqos.yaml b/Documentation/devicetree/bindings/net/qcom,ethqos.yaml index 69a337c7e345e..6672327358bc1 100644 --- a/Documentation/devicetree/bindings/net/qcom,ethqos.yaml +++ b/Documentation/devicetree/bindings/net/qcom,ethqos.yaml @@ -61,6 +61,8 @@ properties: iommus: maxItems: 1 + dma-coherent: true + phys: true phy-names: From 67708158e732bf03d076fba1e3d4453fbf8292a2 Mon Sep 17 00:00:00 2001 From: Michal Schmidt Date: Wed, 15 May 2024 11:24:14 +0200 Subject: [PATCH 1675/1702] idpf: don't skip over ethtool tcp-data-split setting Disabling tcp-data-split on idpf silently fails: # ethtool -G $NETDEV tcp-data-split off # ethtool -g $NETDEV | grep 'TCP data split' TCP data split: on But it works if you also change 'tx' or 'rx': # ethtool -G $NETDEV tcp-data-split off tx 256 # ethtool -g $NETDEV | grep 'TCP data split' TCP data split: off The bug is in idpf_set_ringparam, where it takes a shortcut out if the TX and RX sizes are not changing. Fix it by checking also if the tcp-data-split setting remains unchanged. Only then can the soft reset be skipped. Fixes: 9b1aa3ef2328 ("idpf: add get/set for Ethtool's header split ringparam") Reported-by: Xu Du Closes: https://issues.redhat.com/browse/RHEL-36182 Signed-off-by: Michal Schmidt Reviewed-by: Alexander Lobakin Link: https://lore.kernel.org/r/20240515092414.158079-1-mschmidt@redhat.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/intel/idpf/idpf_ethtool.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c index 986d429d11755..6972d728431cb 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c +++ b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c @@ -376,7 +376,8 @@ static int idpf_set_ringparam(struct net_device *netdev, new_tx_count); if (new_tx_count == vport->txq_desc_count && - new_rx_count == vport->rxq_desc_count) + new_rx_count == vport->rxq_desc_count && + kring->tcp_data_split == idpf_vport_get_hsplit(vport)) goto unlock_mutex; if (!idpf_vport_set_hsplit(vport, kring->tcp_data_split)) { From e03e7f20ebf7e1611d40d1fdc1bde900fd3335f6 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 15 May 2024 14:29:34 +0000 Subject: [PATCH 1676/1702] netrom: fix possible dead-lock in nr_rt_ioctl() syzbot loves netrom, and found a possible deadlock in nr_rt_ioctl [1] Make sure we always acquire nr_node_list_lock before nr_node_lock(nr_node) [1] WARNING: possible circular locking dependency detected 6.9.0-rc7-syzkaller-02147-g654de42f3fc6 #0 Not tainted ------------------------------------------------------ syz-executor350/5129 is trying to acquire lock: ffff8880186e2070 (&nr_node->node_lock){+...}-{2:2}, at: spin_lock_bh include/linux/spinlock.h:356 [inline] ffff8880186e2070 (&nr_node->node_lock){+...}-{2:2}, at: nr_node_lock include/net/netrom.h:152 [inline] ffff8880186e2070 (&nr_node->node_lock){+...}-{2:2}, at: nr_dec_obs net/netrom/nr_route.c:464 [inline] ffff8880186e2070 (&nr_node->node_lock){+...}-{2:2}, at: nr_rt_ioctl+0x1bb/0x1090 net/netrom/nr_route.c:697 but task is already holding lock: ffffffff8f7053b8 (nr_node_list_lock){+...}-{2:2}, at: spin_lock_bh include/linux/spinlock.h:356 [inline] ffffffff8f7053b8 (nr_node_list_lock){+...}-{2:2}, at: nr_dec_obs net/netrom/nr_route.c:462 [inline] ffffffff8f7053b8 (nr_node_list_lock){+...}-{2:2}, at: nr_rt_ioctl+0x10a/0x1090 net/netrom/nr_route.c:697 which lock already depends on the new lock. the existing dependency chain (in reverse order) is: -> #1 (nr_node_list_lock){+...}-{2:2}: lock_acquire+0x1ed/0x550 kernel/locking/lockdep.c:5754 __raw_spin_lock_bh include/linux/spinlock_api_smp.h:126 [inline] _raw_spin_lock_bh+0x35/0x50 kernel/locking/spinlock.c:178 spin_lock_bh include/linux/spinlock.h:356 [inline] nr_remove_node net/netrom/nr_route.c:299 [inline] nr_del_node+0x4b4/0x820 net/netrom/nr_route.c:355 nr_rt_ioctl+0xa95/0x1090 net/netrom/nr_route.c:683 sock_do_ioctl+0x158/0x460 net/socket.c:1222 sock_ioctl+0x629/0x8e0 net/socket.c:1341 vfs_ioctl fs/ioctl.c:51 [inline] __do_sys_ioctl fs/ioctl.c:904 [inline] __se_sys_ioctl+0xfc/0x170 fs/ioctl.c:890 do_syscall_x64 arch/x86/entry/common.c:52 [inline] do_syscall_64+0xf5/0x240 arch/x86/entry/common.c:83 entry_SYSCALL_64_after_hwframe+0x77/0x7f -> #0 (&nr_node->node_lock){+...}-{2:2}: check_prev_add kernel/locking/lockdep.c:3134 [inline] check_prevs_add kernel/locking/lockdep.c:3253 [inline] validate_chain+0x18cb/0x58e0 kernel/locking/lockdep.c:3869 __lock_acquire+0x1346/0x1fd0 kernel/locking/lockdep.c:5137 lock_acquire+0x1ed/0x550 kernel/locking/lockdep.c:5754 __raw_spin_lock_bh include/linux/spinlock_api_smp.h:126 [inline] _raw_spin_lock_bh+0x35/0x50 kernel/locking/spinlock.c:178 spin_lock_bh include/linux/spinlock.h:356 [inline] nr_node_lock include/net/netrom.h:152 [inline] nr_dec_obs net/netrom/nr_route.c:464 [inline] nr_rt_ioctl+0x1bb/0x1090 net/netrom/nr_route.c:697 sock_do_ioctl+0x158/0x460 net/socket.c:1222 sock_ioctl+0x629/0x8e0 net/socket.c:1341 vfs_ioctl fs/ioctl.c:51 [inline] __do_sys_ioctl fs/ioctl.c:904 [inline] __se_sys_ioctl+0xfc/0x170 fs/ioctl.c:890 do_syscall_x64 arch/x86/entry/common.c:52 [inline] do_syscall_64+0xf5/0x240 arch/x86/entry/common.c:83 entry_SYSCALL_64_after_hwframe+0x77/0x7f other info that might help us debug this: Possible unsafe locking scenario: CPU0 CPU1 ---- ---- lock(nr_node_list_lock); lock(&nr_node->node_lock); lock(nr_node_list_lock); lock(&nr_node->node_lock); *** DEADLOCK *** 1 lock held by syz-executor350/5129: #0: ffffffff8f7053b8 (nr_node_list_lock){+...}-{2:2}, at: spin_lock_bh include/linux/spinlock.h:356 [inline] #0: ffffffff8f7053b8 (nr_node_list_lock){+...}-{2:2}, at: nr_dec_obs net/netrom/nr_route.c:462 [inline] #0: ffffffff8f7053b8 (nr_node_list_lock){+...}-{2:2}, at: nr_rt_ioctl+0x10a/0x1090 net/netrom/nr_route.c:697 stack backtrace: CPU: 0 PID: 5129 Comm: syz-executor350 Not tainted 6.9.0-rc7-syzkaller-02147-g654de42f3fc6 #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 04/02/2024 Call Trace: __dump_stack lib/dump_stack.c:88 [inline] dump_stack_lvl+0x241/0x360 lib/dump_stack.c:114 check_noncircular+0x36a/0x4a0 kernel/locking/lockdep.c:2187 check_prev_add kernel/locking/lockdep.c:3134 [inline] check_prevs_add kernel/locking/lockdep.c:3253 [inline] validate_chain+0x18cb/0x58e0 kernel/locking/lockdep.c:3869 __lock_acquire+0x1346/0x1fd0 kernel/locking/lockdep.c:5137 lock_acquire+0x1ed/0x550 kernel/locking/lockdep.c:5754 __raw_spin_lock_bh include/linux/spinlock_api_smp.h:126 [inline] _raw_spin_lock_bh+0x35/0x50 kernel/locking/spinlock.c:178 spin_lock_bh include/linux/spinlock.h:356 [inline] nr_node_lock include/net/netrom.h:152 [inline] nr_dec_obs net/netrom/nr_route.c:464 [inline] nr_rt_ioctl+0x1bb/0x1090 net/netrom/nr_route.c:697 sock_do_ioctl+0x158/0x460 net/socket.c:1222 sock_ioctl+0x629/0x8e0 net/socket.c:1341 vfs_ioctl fs/ioctl.c:51 [inline] __do_sys_ioctl fs/ioctl.c:904 [inline] __se_sys_ioctl+0xfc/0x170 fs/ioctl.c:890 do_syscall_x64 arch/x86/entry/common.c:52 [inline] do_syscall_64+0xf5/0x240 arch/x86/entry/common.c:83 entry_SYSCALL_64_after_hwframe+0x77/0x7f Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Reported-by: syzbot Signed-off-by: Eric Dumazet Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/20240515142934.3708038-1-edumazet@google.com Signed-off-by: Jakub Kicinski --- net/netrom/nr_route.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/net/netrom/nr_route.c b/net/netrom/nr_route.c index 70480869ad1c5..bd2b17b219ae9 100644 --- a/net/netrom/nr_route.c +++ b/net/netrom/nr_route.c @@ -285,22 +285,14 @@ static int __must_check nr_add_node(ax25_address *nr, const char *mnemonic, return 0; } -static inline void __nr_remove_node(struct nr_node *nr_node) +static void nr_remove_node_locked(struct nr_node *nr_node) { + lockdep_assert_held(&nr_node_list_lock); + hlist_del_init(&nr_node->node_node); nr_node_put(nr_node); } -#define nr_remove_node_locked(__node) \ - __nr_remove_node(__node) - -static void nr_remove_node(struct nr_node *nr_node) -{ - spin_lock_bh(&nr_node_list_lock); - __nr_remove_node(nr_node); - spin_unlock_bh(&nr_node_list_lock); -} - static inline void __nr_remove_neigh(struct nr_neigh *nr_neigh) { hlist_del_init(&nr_neigh->neigh_node); @@ -339,6 +331,7 @@ static int nr_del_node(ax25_address *callsign, ax25_address *neighbour, struct n return -EINVAL; } + spin_lock_bh(&nr_node_list_lock); nr_node_lock(nr_node); for (i = 0; i < nr_node->count; i++) { if (nr_node->routes[i].neighbour == nr_neigh) { @@ -352,7 +345,7 @@ static int nr_del_node(ax25_address *callsign, ax25_address *neighbour, struct n nr_node->count--; if (nr_node->count == 0) { - nr_remove_node(nr_node); + nr_remove_node_locked(nr_node); } else { switch (i) { case 0: @@ -367,12 +360,14 @@ static int nr_del_node(ax25_address *callsign, ax25_address *neighbour, struct n nr_node_put(nr_node); } nr_node_unlock(nr_node); + spin_unlock_bh(&nr_node_list_lock); return 0; } } nr_neigh_put(nr_neigh); nr_node_unlock(nr_node); + spin_unlock_bh(&nr_node_list_lock); nr_node_put(nr_node); return -EINVAL; From fa033def4171d2e4e29d5e3714fb2a5b1fc077e8 Mon Sep 17 00:00:00 2001 From: Daniel Jurgens Date: Wed, 15 May 2024 11:31:25 -0500 Subject: [PATCH 1677/1702] virtio_net: Fix missed rtnl_unlock The rtnl_lock would stay locked if allocating promisc_allmulti failed. Also changed the allocation to GFP_KERNEL. Fixes: ff7c7d9f5261 ("virtio_net: Remove command data from control_buf") Reported-by: Eric Dumazet Link: https://lore.kernel.org/netdev/CANn89iLazVaUCvhPm6RPJJ0owra_oFnx7Fhc8d60gV-65ad3WQ@mail.gmail.com/ Signed-off-by: Daniel Jurgens Reviewed-by: Brett Creeley Reviewed-by: Eric Dumazet Acked-by: Michael S. Tsirkin Acked-by: Jason Wang Link: https://lore.kernel.org/r/20240515163125.569743-1-danielj@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/virtio_net.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 19a9b50646c75..4e1a0fc0d5558 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -2902,14 +2902,14 @@ static void virtnet_rx_mode_work(struct work_struct *work) if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX)) return; - rtnl_lock(); - - promisc_allmulti = kzalloc(sizeof(*promisc_allmulti), GFP_ATOMIC); + promisc_allmulti = kzalloc(sizeof(*promisc_allmulti), GFP_KERNEL); if (!promisc_allmulti) { dev_warn(&dev->dev, "Failed to set RX mode, no memory.\n"); return; } + rtnl_lock(); + *promisc_allmulti = !!(dev->flags & IFF_PROMISC); sg_init_one(sg, promisc_allmulti, sizeof(*promisc_allmulti)); From 581073f626e387d3e7eed55c48c8495584ead7ba Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 15 May 2024 16:33:58 +0000 Subject: [PATCH 1678/1702] af_packet: do not call packet_read_pending() from tpacket_destruct_skb() trafgen performance considerably sank on hosts with many cores after the blamed commit. packet_read_pending() is very expensive, and calling it in af_packet fast path defeats Daniel intent in commit b013840810c2 ("packet: use percpu mmap tx frame pending refcount") tpacket_destruct_skb() makes room for one packet, we can immediately wakeup a producer, no need to completely drain the tx ring. Fixes: 89ed5b519004 ("af_packet: Block execution of tasks waiting for transmit to complete in AF_PACKET") Signed-off-by: Eric Dumazet Cc: Neil Horman Cc: Daniel Borkmann Reviewed-by: Willem de Bruijn Link: https://lore.kernel.org/r/20240515163358.4105915-1-edumazet@google.com Signed-off-by: Jakub Kicinski --- net/packet/af_packet.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 8c6d3fbb4ed87..ea3ebc160e25c 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -2522,8 +2522,7 @@ static void tpacket_destruct_skb(struct sk_buff *skb) ts = __packet_set_timestamp(po, ph, skb); __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts); - if (!packet_read_pending(&po->tx_ring)) - complete(&po->skb_completion); + complete(&po->skb_completion); } sock_wfree(skb); From 82110ae235e0560d1f952f74f9fd991587b0e3a7 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Thu, 16 May 2024 07:03:41 -0700 Subject: [PATCH 1679/1702] x86/boot: Address clang -Wimplicit-fallthrough in vsprintf() After enabling -Wimplicit-fallthrough for the x86 boot code, clang warns: arch/x86/boot/printf.c:257:3: warning: unannotated fall-through between switch labels [-Wimplicit-fallthrough] 257 | case 'u': | ^ Clang is a little more pedantic than GCC, which does not warn when falling through to a case that is just break or return. Clang's version is more in line with the kernel's own stance in deprecated.rst, which states that all switch/case blocks must end in either break, fallthrough, continue, goto, or return. Add the missing break to silence the warning. Fixes: dd0716c2b877 ("x86/boot: Add a fallthrough annotation") Reported-by: kernel test robot Signed-off-by: Nathan Chancellor Signed-off-by: Ingo Molnar Acked-by: Justin Stitt Link: https://lore.kernel.org/r/20240516-x86-boot-fix-clang-implicit-fallthrough-v1-1-04dc320ca07c@kernel.org Closes: https://lore.kernel.org/oe-kbuild-all/202405162054.ryP73vy1-lkp@intel.com/ --- arch/x86/boot/printf.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/boot/printf.c b/arch/x86/boot/printf.c index c0ec1dc355abd..51dc14b714f64 100644 --- a/arch/x86/boot/printf.c +++ b/arch/x86/boot/printf.c @@ -254,6 +254,8 @@ int vsprintf(char *buf, const char *fmt, va_list args) case 'd': case 'i': flags |= SIGN; + break; + case 'u': break; From 9dba9c67e52dbe0978c0e86c994891eba480adf0 Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Wed, 15 May 2024 12:48:04 +0200 Subject: [PATCH 1680/1702] x86/alternatives: Use the correct length when optimizing NOPs Commit in Fixes moved the optimize_nops() call inside apply_relocation() and made it a second optimization pass after the relocations have been done. Since optimize_nops() works only on NOPs, that is fine and it'll simply jump over instructions which are not NOPs. However, it made that call with repl_len as the buffer length to optimize. However, it can happen that there are alternatives calls like this one: alternative("mfence; lfence", "", ALT_NOT(X86_FEATURE_APIC_MSRS_FENCE)); where the replacement length is 0. And using repl_len is wrong because apply_alternatives() expands the buffer size to the length of the source insn that is being patched, by padding it with one-byte NOPs: for (; insn_buff_sz < a->instrlen; insn_buff_sz++) insn_buff[insn_buff_sz] = 0x90; Long story short: pass the length of the original instruction(s) as the length of the temporary buffer which to optimize. Result: SMP alternatives: feat: 11*32+27, old: (lapic_next_deadline+0x9/0x50 (ffffffff81061829) len: 6), repl: (ffffffff89b1cc60, len: 0) flags: 0x1 SMP alternatives: ffffffff81061829: old_insn: 0f ae f0 0f ae e8 SMP alternatives: ffffffff81061829: final_insn: 90 90 90 90 90 90 => SMP alternatives: feat: 11*32+27, old: (lapic_next_deadline+0x9/0x50 (ffffffff81061839) len: 6), repl: (ffffffff89b1cc60, len: 0) flags: 0x1 SMP alternatives: ffffffff81061839: [0:6) optimized NOPs: 66 0f 1f 44 00 00 SMP alternatives: ffffffff81061839: old_insn: 0f ae f0 0f ae e8 SMP alternatives: ffffffff81061839: final_insn: 66 0f 1f 44 00 00 Fixes: da8f9cf7e721 ("x86/alternatives: Get rid of __optimize_nops()") Signed-off-by: Borislav Petkov (AMD) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20240515104804.32004-1-bp@kernel.org --- arch/x86/kernel/alternative.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 7555c15b71830..89de612432728 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -372,7 +372,7 @@ static void __apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, void apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len) { __apply_relocation(buf, instr, instrlen, repl, repl_len); - optimize_nops(instr, buf, repl_len); + optimize_nops(instr, buf, instrlen); } /* Low-level backend functions usable from alternative code replacements. */ From a1fd0b9d751f840df23ef0e75b691fc00cfd4743 Mon Sep 17 00:00:00 2001 From: Vitalii Bursov Date: Tue, 30 Apr 2024 18:05:23 +0300 Subject: [PATCH 1681/1702] sched/fair: Allow disabling sched_balance_newidle with sched_relax_domain_level Change relax_domain_level checks so that it would be possible to include or exclude all domains from newidle balancing. This matches the behavior described in the documentation: -1 no request. use system default or follow request of others. 0 no search. 1 search siblings (hyperthreads in a core). "2" enables levels 0 and 1, level_max excludes the last (level_max) level, and level_max+1 includes all levels. Fixes: 1d3504fcf560 ("sched, cpuset: customize sched domains, core") Signed-off-by: Vitalii Bursov Signed-off-by: Ingo Molnar Tested-by: Dietmar Eggemann Reviewed-by: Vincent Guittot Reviewed-by: Valentin Schneider Link: https://lore.kernel.org/r/bd6de28e80073c79466ec6401cdeae78f0d4423d.1714488502.git.vitaly@bursov.com --- kernel/cgroup/cpuset.c | 2 +- kernel/sched/topology.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 4237c8748715d..da24187c4e025 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -2948,7 +2948,7 @@ bool current_cpuset_is_being_rebound(void) static int update_relax_domain_level(struct cpuset *cs, s64 val) { #ifdef CONFIG_SMP - if (val < -1 || val >= sched_domain_level_max) + if (val < -1 || val > sched_domain_level_max + 1) return -EINVAL; #endif diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 63aecd2a7a9f3..67a777b317433 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -1475,7 +1475,7 @@ static void set_domain_attribute(struct sched_domain *sd, } else request = attr->relax_domain_level; - if (sd->level > request) { + if (sd->level >= request) { /* Turn off idle balance on this domain: */ sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE); } From 287372fa39f579a61e17b000aa74c8418d230528 Mon Sep 17 00:00:00 2001 From: Vitalii Bursov Date: Tue, 30 Apr 2024 18:05:24 +0300 Subject: [PATCH 1682/1702] sched/debug: Dump domains' level Knowing domain's level exactly can be useful when setting relax_domain_level or cpuset.sched_relax_domain_level Usage: cat /debug/sched/domains/cpu0/domain1/level to dump cpu0 domain1's level. SDM macro is not used because sd->level is 'int' and it would hide the type mismatch between 'int' and 'u32'. Signed-off-by: Vitalii Bursov Signed-off-by: Ingo Molnar Tested-by: Dietmar Eggemann Acked-by: Vincent Guittot Reviewed-by: Valentin Schneider Link: https://lore.kernel.org/r/9489b6475f6dd6fbc67c617752d4216fa094da53.1714488502.git.vitaly@bursov.com --- kernel/sched/debug.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 8d5d98a5834df..c1eb9a1afd13e 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -425,6 +425,7 @@ static void register_sd(struct sched_domain *sd, struct dentry *parent) debugfs_create_file("flags", 0444, parent, &sd->flags, &sd_flags_fops); debugfs_create_file("groups_flags", 0444, parent, &sd->groups->flags, &sd_flags_fops); + debugfs_create_u32("level", 0444, parent, (u32 *)&sd->level); } void update_sched_domain_debugfs(void) From 0f1c74befa656305ecc85c954dc31f84c1cc26e1 Mon Sep 17 00:00:00 2001 From: Vitalii Bursov Date: Tue, 30 Apr 2024 18:05:25 +0300 Subject: [PATCH 1683/1702] docs: cgroup-v1: Clarify that domain levels are system-specific Add a clarification that domain levels are system-specific and where to check for system details. Signed-off-by: Vitalii Bursov Signed-off-by: Ingo Molnar Reviewed-by: Valentin Schneider Acked-by: Vincent Guittot Link: https://lore.kernel.org/r/42b177a2e897cdf880caf9c2025f5b609e820334.1714488502.git.vitaly@bursov.com --- Documentation/admin-guide/cgroup-v1/cpusets.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Documentation/admin-guide/cgroup-v1/cpusets.rst b/Documentation/admin-guide/cgroup-v1/cpusets.rst index 7d3415eea05d0..f401af5e2f09a 100644 --- a/Documentation/admin-guide/cgroup-v1/cpusets.rst +++ b/Documentation/admin-guide/cgroup-v1/cpusets.rst @@ -568,7 +568,7 @@ on the next tick. For some applications in special situation, waiting The 'cpuset.sched_relax_domain_level' file allows you to request changing this searching range as you like. This file takes int value which -indicates size of searching range in levels ideally as follows, +indicates size of searching range in levels approximately as follows, otherwise initial value -1 that indicates the cpuset has no request. ====== =========================================================== @@ -581,6 +581,11 @@ otherwise initial value -1 that indicates the cpuset has no request. 5 search system wide [on NUMA system] ====== =========================================================== +Not all levels can be present and values can change depending on the +system architecture and kernel configuration. Check +/sys/kernel/debug/sched/domains/cpu*/domain*/ for system-specific +details. + The system default is architecture dependent. The system default can be changed using the relax_domain_level= boot parameter. From 72bffbf57c5247ac6146d1103ef42e9f8d094bc8 Mon Sep 17 00:00:00 2001 From: Dawei Li Date: Thu, 14 Mar 2024 18:59:16 -0700 Subject: [PATCH 1684/1702] sched/fair: Fix initial util_avg calculation Change se->load.weight to se_weight(se) in the calculation for the initial util_avg to avoid unnecessarily inflating the util_avg by 1024 times. The reason is that se->load.weight has the unit/scale as the scaled-up load, while cfs_rg->avg.load_avg has the unit/scale as the true task weight (as mapped directly from the task's nice/priority value). With CONFIG_32BIT, the scaled-up load is equal to the true task weight. With CONFIG_64BIT, the scaled-up load is 1024 times the true task weight. Thus, the current code may inflate the util_avg by 1024 times. The follow-up capping will not allow the util_avg value to go wild. But the calculation should have the correct logic. Signed-off-by: Dawei Li Signed-off-by: Ingo Molnar Reviewed-by: Vincent Guittot Reviewed-by: Vishal Chourasia Link: https://lore.kernel.org/r/20240315015916.21545-1-daweilics@gmail.com --- kernel/sched/fair.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 146ecf9cc3afe..900978741a81c 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1031,7 +1031,8 @@ void init_entity_runnable_average(struct sched_entity *se) * With new tasks being created, their initial util_avgs are extrapolated * based on the cfs_rq's current util_avg: * - * util_avg = cfs_rq->util_avg / (cfs_rq->load_avg + 1) * se.load.weight + * util_avg = cfs_rq->avg.util_avg / (cfs_rq->avg.load_avg + 1) + * * se_weight(se) * * However, in many cases, the above util_avg does not give a desired * value. Moreover, the sum of the util_avgs may be divergent, such @@ -1078,7 +1079,7 @@ void post_init_entity_util_avg(struct task_struct *p) if (cap > 0) { if (cfs_rq->avg.util_avg != 0) { - sa->util_avg = cfs_rq->avg.util_avg * se->load.weight; + sa->util_avg = cfs_rq->avg.util_avg * se_weight(se); sa->util_avg /= (cfs_rq->avg.load_avg + 1); if (sa->util_avg > cap) From 7cb7fb5b49399fc59f1c44686d82c0df0776c8c6 Mon Sep 17 00:00:00 2001 From: Christian Loehle Date: Tue, 5 Mar 2024 15:18:20 +0000 Subject: [PATCH 1685/1702] sched/fair: Remove stale FREQUENCY_UTIL comment On 05/03/2024 15:05, Vincent Guittot wrote: I'm fine with either and that was my first thought here, too, but it did seem like the comment was mostly placed there to justify the 'unexpected' high utilization when explicitly passing FREQUENCY_UTIL and the need to clamp it then. So removing did feel slightly more natural to me anyway. So alternatively: From: Christian Loehle Date: Tue, 5 Mar 2024 09:34:41 +0000 Subject: [PATCH] sched/fair: Remove stale FREQUENCY_UTIL mention effective_cpu_util() flags were removed, so remove mentioning of the flag. commit 9c0b4bb7f6303 ("sched/cpufreq: Rework schedutil governor performance estimation") reworked effective_cpu_util() removing enum cpu_util_type. Modify the comment accordingly. Signed-off-by: Christian Loehle Signed-off-by: Ingo Molnar Reviewed-by: Vincent Guittot Link: https://lore.kernel.org/r/0e2833ee-0939-44e0-82a2-520a585a0153@arm.com --- kernel/sched/fair.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 900978741a81c..9744b5036dd8e 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7900,8 +7900,8 @@ eenv_pd_max_util(struct energy_env *eenv, struct cpumask *pd_cpus, * Performance domain frequency: utilization clamping * must be considered since it affects the selection * of the performance domain frequency. - * NOTE: in case RT tasks are running, by default the - * FREQUENCY_UTIL's utilization can be max OPP. + * NOTE: in case RT tasks are running, by default the min + * utilization can be max OPP. */ eff_util = effective_cpu_util(cpu, util, &min, &max); From 49217ea147df7647cb89161b805c797487783fc0 Mon Sep 17 00:00:00 2001 From: Cheng Yu Date: Wed, 24 Apr 2024 21:24:38 +0800 Subject: [PATCH 1686/1702] sched/core: Fix incorrect initialization of the 'burst' parameter in cpu_max_write() In the cgroup v2 CPU subsystem, assuming we have a cgroup named 'test', and we set cpu.max and cpu.max.burst: # echo 1000000 > /sys/fs/cgroup/test/cpu.max # echo 1000000 > /sys/fs/cgroup/test/cpu.max.burst then we check cpu.max and cpu.max.burst: # cat /sys/fs/cgroup/test/cpu.max 1000000 100000 # cat /sys/fs/cgroup/test/cpu.max.burst 1000000 Next we set cpu.max again and check cpu.max and cpu.max.burst: # echo 2000000 > /sys/fs/cgroup/test/cpu.max # cat /sys/fs/cgroup/test/cpu.max 2000000 100000 # cat /sys/fs/cgroup/test/cpu.max.burst 1000 ... we find that the cpu.max.burst value changed unexpectedly. In cpu_max_write(), the unit of the burst value returned by tg_get_cfs_burst() is microseconds, while in cpu_max_write(), the burst unit used for calculation should be nanoseconds, which leads to the bug. To fix it, get the burst value directly from tg->cfs_bandwidth.burst. Fixes: f4183717b370 ("sched/fair: Introduce the burstable CFS controller") Reported-by: Qixin Liao Signed-off-by: Cheng Yu Signed-off-by: Zhang Qiao Signed-off-by: Ingo Molnar Reviewed-by: Vincent Guittot Tested-by: Vincent Guittot Link: https://lore.kernel.org/r/20240424132438.514720-1-serein.chengyu@huawei.com --- kernel/sched/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 1a914388144ae..f88f50552acbb 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -11402,7 +11402,7 @@ static ssize_t cpu_max_write(struct kernfs_open_file *of, { struct task_group *tg = css_tg(of_css(of)); u64 period = tg_get_cfs_period(tg); - u64 burst = tg_get_cfs_burst(tg); + u64 burst = tg->cfs_bandwidth.burst; u64 quota; int ret; From 68067f065ee730c7c67b361c3c81808d25d5a90b Mon Sep 17 00:00:00 2001 From: Jiawen Wu Date: Fri, 17 May 2024 14:51:38 +0800 Subject: [PATCH 1687/1702] net: wangxun: fix to change Rx features Fix the issue where some Rx features cannot be changed. When using ethtool -K to turn off rx offload, it returns error and displays "Could not change any device features". And netdev->features is not assigned a new value to actually configure the hardware. Fixes: 6dbedcffcf54 ("net: libwx: Implement xx_set_features ops") Signed-off-by: Jiawen Wu Reviewed-by: Simon Horman Signed-off-by: David S. Miller --- drivers/net/ethernet/wangxun/libwx/wx_lib.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/wangxun/libwx/wx_lib.c b/drivers/net/ethernet/wangxun/libwx/wx_lib.c index 6fae161cbcb82..667a5675998cb 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_lib.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_lib.c @@ -2690,12 +2690,14 @@ int wx_set_features(struct net_device *netdev, netdev_features_t features) wx->rss_enabled = false; } + netdev->features = features; + if (changed & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) wx_set_rx_mode(netdev); - return 1; + return 0; } EXPORT_SYMBOL(wx_set_features); From ac71ab7816b675f1c9614015bd87bfccb456c394 Mon Sep 17 00:00:00 2001 From: Jiawen Wu Date: Fri, 17 May 2024 14:51:39 +0800 Subject: [PATCH 1688/1702] net: wangxun: match VLAN CTAG and STAG features Hardware requires VLAN CTAG and STAG configuration always matches. And whether VLAN CTAG or STAG changes, the configuration needs to be changed as well. Fixes: 6670f1ece2c8 ("net: txgbe: Add netdev features support") Signed-off-by: Jiawen Wu Reviewed-by: Sai Krishna Reviewed-by: Simon Horman Signed-off-by: David S. Miller --- drivers/net/ethernet/wangxun/libwx/wx_lib.c | 46 +++++++++++++++++++ drivers/net/ethernet/wangxun/libwx/wx_lib.h | 2 + drivers/net/ethernet/wangxun/ngbe/ngbe_main.c | 1 + .../net/ethernet/wangxun/txgbe/txgbe_main.c | 1 + 4 files changed, 50 insertions(+) diff --git a/drivers/net/ethernet/wangxun/libwx/wx_lib.c b/drivers/net/ethernet/wangxun/libwx/wx_lib.c index 667a5675998cb..d0cb09a4bd3d4 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_lib.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_lib.c @@ -2701,6 +2701,52 @@ int wx_set_features(struct net_device *netdev, netdev_features_t features) } EXPORT_SYMBOL(wx_set_features); +#define NETIF_VLAN_STRIPPING_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \ + NETIF_F_HW_VLAN_STAG_RX) + +#define NETIF_VLAN_INSERTION_FEATURES (NETIF_F_HW_VLAN_CTAG_TX | \ + NETIF_F_HW_VLAN_STAG_TX) + +#define NETIF_VLAN_FILTERING_FEATURES (NETIF_F_HW_VLAN_CTAG_FILTER | \ + NETIF_F_HW_VLAN_STAG_FILTER) + +netdev_features_t wx_fix_features(struct net_device *netdev, + netdev_features_t features) +{ + netdev_features_t changed = netdev->features ^ features; + struct wx *wx = netdev_priv(netdev); + + if (changed & NETIF_VLAN_STRIPPING_FEATURES) { + if ((features & NETIF_VLAN_STRIPPING_FEATURES) != NETIF_VLAN_STRIPPING_FEATURES && + (features & NETIF_VLAN_STRIPPING_FEATURES) != 0) { + features &= ~NETIF_VLAN_STRIPPING_FEATURES; + features |= netdev->features & NETIF_VLAN_STRIPPING_FEATURES; + wx_err(wx, "802.1Q and 802.1ad VLAN stripping must be either both on or both off."); + } + } + + if (changed & NETIF_VLAN_INSERTION_FEATURES) { + if ((features & NETIF_VLAN_INSERTION_FEATURES) != NETIF_VLAN_INSERTION_FEATURES && + (features & NETIF_VLAN_INSERTION_FEATURES) != 0) { + features &= ~NETIF_VLAN_INSERTION_FEATURES; + features |= netdev->features & NETIF_VLAN_INSERTION_FEATURES; + wx_err(wx, "802.1Q and 802.1ad VLAN insertion must be either both on or both off."); + } + } + + if (changed & NETIF_VLAN_FILTERING_FEATURES) { + if ((features & NETIF_VLAN_FILTERING_FEATURES) != NETIF_VLAN_FILTERING_FEATURES && + (features & NETIF_VLAN_FILTERING_FEATURES) != 0) { + features &= ~NETIF_VLAN_FILTERING_FEATURES; + features |= netdev->features & NETIF_VLAN_FILTERING_FEATURES; + wx_err(wx, "802.1Q and 802.1ad VLAN filtering must be either both on or both off."); + } + } + + return features; +} +EXPORT_SYMBOL(wx_fix_features); + void wx_set_ring(struct wx *wx, u32 new_tx_count, u32 new_rx_count, struct wx_ring *temp_ring) { diff --git a/drivers/net/ethernet/wangxun/libwx/wx_lib.h b/drivers/net/ethernet/wangxun/libwx/wx_lib.h index ec909e876720c..c41b29ea812ff 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_lib.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_lib.h @@ -30,6 +30,8 @@ int wx_setup_resources(struct wx *wx); void wx_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats); int wx_set_features(struct net_device *netdev, netdev_features_t features); +netdev_features_t wx_fix_features(struct net_device *netdev, + netdev_features_t features); void wx_set_ring(struct wx *wx, u32 new_tx_count, u32 new_rx_count, struct wx_ring *temp_ring); diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c index fdd6b4f70b7a5..e894e01d030d1 100644 --- a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c +++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c @@ -499,6 +499,7 @@ static const struct net_device_ops ngbe_netdev_ops = { .ndo_start_xmit = wx_xmit_frame, .ndo_set_rx_mode = wx_set_rx_mode, .ndo_set_features = wx_set_features, + .ndo_fix_features = wx_fix_features, .ndo_validate_addr = eth_validate_addr, .ndo_set_mac_address = wx_set_mac, .ndo_get_stats64 = wx_get_stats64, diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c index bd4624d14ca03..b3c0058b045da 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c @@ -428,6 +428,7 @@ static const struct net_device_ops txgbe_netdev_ops = { .ndo_start_xmit = wx_xmit_frame, .ndo_set_rx_mode = wx_set_rx_mode, .ndo_set_features = wx_set_features, + .ndo_fix_features = wx_fix_features, .ndo_validate_addr = eth_validate_addr, .ndo_set_mac_address = wx_set_mac, .ndo_get_stats64 = wx_get_stats64, From 1d3c6414950badaa38002af3b5857e01a21f01e9 Mon Sep 17 00:00:00 2001 From: Jiawen Wu Date: Fri, 17 May 2024 14:51:40 +0800 Subject: [PATCH 1689/1702] net: txgbe: fix to control VLAN strip When VLAN tag strip is changed to enable or disable, the hardware requires the Rx ring to be in a disabled state, otherwise the feature cannot be changed. Fixes: f3b03c655f67 ("net: wangxun: Implement vlan add and kill functions") Signed-off-by: Jiawen Wu Reviewed-by: Simon Horman Signed-off-by: David S. Miller --- drivers/net/ethernet/wangxun/libwx/wx_hw.c | 2 ++ drivers/net/ethernet/wangxun/libwx/wx_lib.c | 6 ++-- drivers/net/ethernet/wangxun/libwx/wx_type.h | 22 ++++++++++++++ .../net/ethernet/wangxun/ngbe/ngbe_ethtool.c | 18 +++++++---- .../ethernet/wangxun/txgbe/txgbe_ethtool.c | 18 +++++++---- .../net/ethernet/wangxun/txgbe/txgbe_main.c | 30 +++++++++++++++++++ .../net/ethernet/wangxun/txgbe/txgbe_type.h | 1 + 7 files changed, 84 insertions(+), 13 deletions(-) diff --git a/drivers/net/ethernet/wangxun/libwx/wx_hw.c b/drivers/net/ethernet/wangxun/libwx/wx_hw.c index 3662483bfe2e7..7c4b6881a93fc 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_hw.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_hw.c @@ -1958,6 +1958,8 @@ int wx_sw_init(struct wx *wx) return -ENOMEM; } + bitmap_zero(wx->state, WX_STATE_NBITS); + return 0; } EXPORT_SYMBOL(wx_sw_init); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_lib.c b/drivers/net/ethernet/wangxun/libwx/wx_lib.c index d0cb09a4bd3d4..07ba3a270a14f 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_lib.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_lib.c @@ -2692,9 +2692,9 @@ int wx_set_features(struct net_device *netdev, netdev_features_t features) netdev->features = features; - if (changed & - (NETIF_F_HW_VLAN_CTAG_RX | - NETIF_F_HW_VLAN_STAG_RX)) + if (wx->mac.type == wx_mac_sp && changed & NETIF_F_HW_VLAN_CTAG_RX) + wx->do_reset(netdev); + else if (changed & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER)) wx_set_rx_mode(netdev); return 0; diff --git a/drivers/net/ethernet/wangxun/libwx/wx_type.h b/drivers/net/ethernet/wangxun/libwx/wx_type.h index 1fdeb464d5f4a..5aaf7b1fa2db9 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_type.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_type.h @@ -982,8 +982,13 @@ struct wx_hw_stats { u64 qmprc; }; +enum wx_state { + WX_STATE_RESETTING, + WX_STATE_NBITS, /* must be last */ +}; struct wx { unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; + DECLARE_BITMAP(state, WX_STATE_NBITS); void *priv; u8 __iomem *hw_addr; @@ -1071,6 +1076,8 @@ struct wx { u64 hw_csum_rx_good; u64 hw_csum_rx_error; u64 alloc_rx_buff_failed; + + void (*do_reset)(struct net_device *netdev); }; #define WX_INTR_ALL (~0ULL) @@ -1131,4 +1138,19 @@ static inline struct wx *phylink_to_wx(struct phylink_config *config) return container_of(config, struct wx, phylink_config); } +static inline int wx_set_state_reset(struct wx *wx) +{ + u8 timeout = 50; + + while (test_and_set_bit(WX_STATE_RESETTING, wx->state)) { + timeout--; + if (!timeout) + return -EBUSY; + + usleep_range(1000, 2000); + } + + return 0; +} + #endif /* _WX_TYPE_H_ */ diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c index 786a652ae64f3..46a5a3e952021 100644 --- a/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c +++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c @@ -52,7 +52,7 @@ static int ngbe_set_ringparam(struct net_device *netdev, struct wx *wx = netdev_priv(netdev); u32 new_rx_count, new_tx_count; struct wx_ring *temp_ring; - int i; + int i, err = 0; new_tx_count = clamp_t(u32, ring->tx_pending, WX_MIN_TXD, WX_MAX_TXD); new_tx_count = ALIGN(new_tx_count, WX_REQ_TX_DESCRIPTOR_MULTIPLE); @@ -64,6 +64,10 @@ static int ngbe_set_ringparam(struct net_device *netdev, new_rx_count == wx->rx_ring_count) return 0; + err = wx_set_state_reset(wx); + if (err) + return err; + if (!netif_running(wx->netdev)) { for (i = 0; i < wx->num_tx_queues; i++) wx->tx_ring[i]->count = new_tx_count; @@ -72,14 +76,16 @@ static int ngbe_set_ringparam(struct net_device *netdev, wx->tx_ring_count = new_tx_count; wx->rx_ring_count = new_rx_count; - return 0; + goto clear_reset; } /* allocate temporary buffer to store rings in */ i = max_t(int, wx->num_tx_queues, wx->num_rx_queues); temp_ring = kvmalloc_array(i, sizeof(struct wx_ring), GFP_KERNEL); - if (!temp_ring) - return -ENOMEM; + if (!temp_ring) { + err = -ENOMEM; + goto clear_reset; + } ngbe_down(wx); @@ -89,7 +95,9 @@ static int ngbe_set_ringparam(struct net_device *netdev, wx_configure(wx); ngbe_up(wx); - return 0; +clear_reset: + clear_bit(WX_STATE_RESETTING, wx->state); + return err; } static int ngbe_set_channels(struct net_device *dev, diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c index db675512ce4dc..31fde3fa7c6b5 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c @@ -19,7 +19,7 @@ static int txgbe_set_ringparam(struct net_device *netdev, struct wx *wx = netdev_priv(netdev); u32 new_rx_count, new_tx_count; struct wx_ring *temp_ring; - int i; + int i, err = 0; new_tx_count = clamp_t(u32, ring->tx_pending, WX_MIN_TXD, WX_MAX_TXD); new_tx_count = ALIGN(new_tx_count, WX_REQ_TX_DESCRIPTOR_MULTIPLE); @@ -31,6 +31,10 @@ static int txgbe_set_ringparam(struct net_device *netdev, new_rx_count == wx->rx_ring_count) return 0; + err = wx_set_state_reset(wx); + if (err) + return err; + if (!netif_running(wx->netdev)) { for (i = 0; i < wx->num_tx_queues; i++) wx->tx_ring[i]->count = new_tx_count; @@ -39,14 +43,16 @@ static int txgbe_set_ringparam(struct net_device *netdev, wx->tx_ring_count = new_tx_count; wx->rx_ring_count = new_rx_count; - return 0; + goto clear_reset; } /* allocate temporary buffer to store rings in */ i = max_t(int, wx->num_tx_queues, wx->num_rx_queues); temp_ring = kvmalloc_array(i, sizeof(struct wx_ring), GFP_KERNEL); - if (!temp_ring) - return -ENOMEM; + if (!temp_ring) { + err = -ENOMEM; + goto clear_reset; + } txgbe_down(wx); @@ -55,7 +61,9 @@ static int txgbe_set_ringparam(struct net_device *netdev, txgbe_up(wx); - return 0; +clear_reset: + clear_bit(WX_STATE_RESETTING, wx->state); + return err; } static int txgbe_set_channels(struct net_device *dev, diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c index b3c0058b045da..8c7a74981b907 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c @@ -269,6 +269,8 @@ static int txgbe_sw_init(struct wx *wx) wx->tx_work_limit = TXGBE_DEFAULT_TX_WORK; wx->rx_work_limit = TXGBE_DEFAULT_RX_WORK; + wx->do_reset = txgbe_do_reset; + return 0; } @@ -421,6 +423,34 @@ int txgbe_setup_tc(struct net_device *dev, u8 tc) return 0; } +static void txgbe_reinit_locked(struct wx *wx) +{ + int err = 0; + + netif_trans_update(wx->netdev); + + err = wx_set_state_reset(wx); + if (err) { + wx_err(wx, "wait device reset timeout\n"); + return; + } + + txgbe_down(wx); + txgbe_up(wx); + + clear_bit(WX_STATE_RESETTING, wx->state); +} + +void txgbe_do_reset(struct net_device *netdev) +{ + struct wx *wx = netdev_priv(netdev); + + if (netif_running(netdev)) + txgbe_reinit_locked(wx); + else + txgbe_reset(wx); +} + static const struct net_device_ops txgbe_netdev_ops = { .ndo_open = txgbe_open, .ndo_stop = txgbe_close, diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h index 1b4ff50d58571..f434a7865cb7b 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h @@ -134,6 +134,7 @@ extern char txgbe_driver_name[]; void txgbe_down(struct wx *wx); void txgbe_up(struct wx *wx); int txgbe_setup_tc(struct net_device *dev, u8 tc); +void txgbe_do_reset(struct net_device *netdev); #define NODE_PROP(_NAME, _PROP) \ (const struct software_node) { \ From d3b17c6d9dddc2db3670bc9be628b122416a3d26 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 8 May 2024 16:39:51 +0800 Subject: [PATCH 1690/1702] crypto: qat - Fix ADF_DEV_RESET_SYNC memory leak Using completion_done to determine whether the caller has gone away only works after a complete call. Furthermore it's still possible that the caller has not yet called wait_for_completion, resulting in another potential UAF. Fix this by making the caller use cancel_work_sync and then freeing the memory safely. Fixes: 7d42e097607c ("crypto: qat - resolve race condition during AER recovery") Cc: #6.8+ Signed-off-by: Herbert Xu Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_common/adf_aer.c | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c index 9da2278bd5b7d..04260f61d0429 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_aer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c @@ -130,8 +130,7 @@ static void adf_device_reset_worker(struct work_struct *work) if (adf_dev_restart(accel_dev)) { /* The device hanged and we can't restart it so stop here */ dev_err(&GET_DEV(accel_dev), "Restart device failed\n"); - if (reset_data->mode == ADF_DEV_RESET_ASYNC || - completion_done(&reset_data->compl)) + if (reset_data->mode == ADF_DEV_RESET_ASYNC) kfree(reset_data); WARN(1, "QAT: device restart failed. Device is unusable\n"); return; @@ -147,16 +146,8 @@ static void adf_device_reset_worker(struct work_struct *work) adf_dev_restarted_notify(accel_dev); clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); - /* - * The dev is back alive. Notify the caller if in sync mode - * - * If device restart will take a more time than expected, - * the schedule_reset() function can timeout and exit. This can be - * detected by calling the completion_done() function. In this case - * the reset_data structure needs to be freed here. - */ - if (reset_data->mode == ADF_DEV_RESET_ASYNC || - completion_done(&reset_data->compl)) + /* The dev is back alive. Notify the caller if in sync mode */ + if (reset_data->mode == ADF_DEV_RESET_ASYNC) kfree(reset_data); else complete(&reset_data->compl); @@ -191,10 +182,10 @@ static int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev, if (!timeout) { dev_err(&GET_DEV(accel_dev), "Reset device timeout expired\n"); + cancel_work_sync(&reset_data->reset_work); ret = -EFAULT; - } else { - kfree(reset_data); } + kfree(reset_data); return ret; } return 0; From c6ab5c915da460c0397960af3c308386c3f3247b Mon Sep 17 00:00:00 2001 From: Stefan Berger Date: Thu, 9 May 2024 21:59:21 -0400 Subject: [PATCH 1691/1702] crypto: ecc - Prevent ecc_digits_from_bytes from reading too many bytes Prevent ecc_digits_from_bytes from reading too many bytes from the input byte array in case an insufficient number of bytes is provided to fill the output digit array of ndigits. Therefore, initialize the most significant digits with 0 to avoid trying to read too many bytes later on. Convert the function into a regular function since it is getting too big for an inline function. If too many bytes are provided on the input byte array the extra bytes are ignored since the input variable 'ndigits' limits the number of digits that will be filled. Fixes: d67c96fb97b5 ("crypto: ecdsa - Convert byte arrays with key coordinates to digits") Reviewed-by: Jarkko Sakkinen Signed-off-by: Stefan Berger Signed-off-by: Herbert Xu --- crypto/ecc.c | 22 ++++++++++++++++++++++ include/crypto/internal/ecc.h | 15 ++------------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/crypto/ecc.c b/crypto/ecc.c index c1d2e884be1e9..fe761256e335b 100644 --- a/crypto/ecc.c +++ b/crypto/ecc.c @@ -68,6 +68,28 @@ const struct ecc_curve *ecc_get_curve(unsigned int curve_id) } EXPORT_SYMBOL(ecc_get_curve); +void ecc_digits_from_bytes(const u8 *in, unsigned int nbytes, + u64 *out, unsigned int ndigits) +{ + int diff = ndigits - DIV_ROUND_UP(nbytes, sizeof(u64)); + unsigned int o = nbytes & 7; + __be64 msd = 0; + + /* diff > 0: not enough input bytes: set most significant digits to 0 */ + if (diff > 0) { + ndigits -= diff; + memset(&out[ndigits - 1], 0, diff * sizeof(u64)); + } + + if (o) { + memcpy((u8 *)&msd + sizeof(msd) - o, in, o); + out[--ndigits] = be64_to_cpu(msd); + in += o; + } + ecc_swap_digits(in, out, ndigits); +} +EXPORT_SYMBOL(ecc_digits_from_bytes); + static u64 *ecc_alloc_digits_space(unsigned int ndigits) { size_t len = ndigits * sizeof(u64); diff --git a/include/crypto/internal/ecc.h b/include/crypto/internal/ecc.h index 7ca1f463d1ecf..f7e75e1e71f3c 100644 --- a/include/crypto/internal/ecc.h +++ b/include/crypto/internal/ecc.h @@ -64,19 +64,8 @@ static inline void ecc_swap_digits(const void *in, u64 *out, unsigned int ndigit * @out Output digits array * @ndigits: Number of digits to create from byte array */ -static inline void ecc_digits_from_bytes(const u8 *in, unsigned int nbytes, - u64 *out, unsigned int ndigits) -{ - unsigned int o = nbytes & 7; - __be64 msd = 0; - - if (o) { - memcpy((u8 *)&msd + sizeof(msd) - o, in, o); - out[--ndigits] = be64_to_cpu(msd); - in += o; - } - ecc_swap_digits(in, out, ndigits); -} +void ecc_digits_from_bytes(const u8 *in, unsigned int nbytes, + u64 *out, unsigned int ndigits); /** * ecc_is_key_valid() - Validate a given ECDH private key From c6a6c9694aadc4c3ab8d89bdd44aed3eab1e43c6 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 10 May 2024 18:22:53 +0300 Subject: [PATCH 1692/1702] ext4: fix error pointer dereference in ext4_mb_load_buddy_gfp() This code calls folio_put() on an error pointer which will lead to a crash. Check for both error pointers and NULL pointers before calling folio_put(). Fixes: 5eea586b47f0 ("ext4: convert bd_buddy_page to bd_buddy_folio") Signed-off-by: Dan Carpenter Link: https://lore.kernel.org/r/eaafa1d9-a61c-4af4-9f97-d3ad72c60200@moroto.mountain Signed-off-by: Theodore Ts'o --- fs/ext4/mballoc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 648989c125f2f..9dda9cd68ab2f 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -1717,7 +1717,7 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, return 0; err: - if (folio) + if (!IS_ERR_OR_NULL(folio)) folio_put(folio); if (e4b->bd_bitmap_folio) folio_put(e4b->bd_bitmap_folio); From 6e828dc60e509b79ef09882264952f341cb58425 Mon Sep 17 00:00:00 2001 From: Tom Parkin Date: Mon, 13 May 2024 18:22:47 +0100 Subject: [PATCH 1693/1702] l2tp: fix ICMP error handling for UDP-encap sockets Since commit a36e185e8c85 ("udp: Handle ICMP errors for tunnels with same destination port on both endpoints") UDP's handling of ICMP errors has allowed for UDP-encap tunnels to determine socket associations in scenarios where the UDP hash lookup could not. Subsequently, commit d26796ae58940 ("udp: check udp sock encap_type in __udp_lib_err") subtly tweaked the approach such that UDP ICMP error handling would be skipped for any UDP socket which has encapsulation enabled. In the case of L2TP tunnel sockets using UDP-encap, this latter modification effectively broke ICMP error reporting for the L2TP control plane. To a degree this isn't catastrophic inasmuch as the L2TP control protocol defines a reliable transport on top of the underlying packet switching network which will eventually detect errors and time out. However, paying attention to the ICMP error reporting allows for more timely detection of errors in L2TP userspace, and aids in debugging connectivity issues. Reinstate ICMP error handling for UDP encap L2TP tunnels: * implement struct udp_tunnel_sock_cfg .encap_err_rcv in order to allow the L2TP code to handle ICMP errors; * only implement error-handling for tunnels which have a managed socket: unmanaged tunnels using a kernel socket have no userspace to report errors back to; * flag the error on the socket, which allows for userspace to get an error such as -ECONNREFUSED back from sendmsg/recvmsg; * pass the error into ip[v6]_icmp_error() which allows for userspace to get extended error information via. MSG_ERRQUEUE. Fixes: d26796ae5894 ("udp: check udp sock encap_type in __udp_lib_err") Signed-off-by: Tom Parkin Link: https://lore.kernel.org/r/20240513172248.623261-1-tparkin@katalix.com Signed-off-by: Jakub Kicinski --- net/l2tp/l2tp_core.c | 44 +++++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/net/l2tp/l2tp_core.c b/net/l2tp/l2tp_core.c index 7d519a46a8447..88a34db265d86 100644 --- a/net/l2tp/l2tp_core.c +++ b/net/l2tp/l2tp_core.c @@ -910,22 +910,20 @@ static int l2tp_udp_recv_core(struct l2tp_tunnel *tunnel, struct sk_buff *skb) return 1; } -/* UDP encapsulation receive handler. See net/ipv4/udp.c. - * Return codes: - * 0 : success. - * <0: error - * >0: skb should be passed up to userspace as UDP. +/* UDP encapsulation receive and error receive handlers. + * See net/ipv4/udp.c for details. + * + * Note that these functions are called from inside an + * RCU-protected region, but without the socket being locked. + * + * Hence we use rcu_dereference_sk_user_data to access the + * tunnel data structure rather the usual l2tp_sk_to_tunnel + * accessor function. */ int l2tp_udp_encap_recv(struct sock *sk, struct sk_buff *skb) { struct l2tp_tunnel *tunnel; - /* Note that this is called from the encap_rcv hook inside an - * RCU-protected region, but without the socket being locked. - * Hence we use rcu_dereference_sk_user_data to access the - * tunnel data structure rather the usual l2tp_sk_to_tunnel - * accessor function. - */ tunnel = rcu_dereference_sk_user_data(sk); if (!tunnel) goto pass_up; @@ -942,6 +940,29 @@ int l2tp_udp_encap_recv(struct sock *sk, struct sk_buff *skb) } EXPORT_SYMBOL_GPL(l2tp_udp_encap_recv); +static void l2tp_udp_encap_err_recv(struct sock *sk, struct sk_buff *skb, int err, + __be16 port, u32 info, u8 *payload) +{ + struct l2tp_tunnel *tunnel; + + tunnel = rcu_dereference_sk_user_data(sk); + if (!tunnel || tunnel->fd < 0) + return; + + sk->sk_err = err; + sk_error_report(sk); + + if (ip_hdr(skb)->version == IPVERSION) { + if (inet_test_bit(RECVERR, sk)) + return ip_icmp_error(sk, skb, err, port, info, payload); +#if IS_ENABLED(CONFIG_IPV6) + } else { + if (inet6_test_bit(RECVERR6, sk)) + return ipv6_icmp_error(sk, skb, err, port, info, payload); +#endif + } +} + /************************************************************************ * Transmit handling ***********************************************************************/ @@ -1516,6 +1537,7 @@ int l2tp_tunnel_register(struct l2tp_tunnel *tunnel, struct net *net, .sk_user_data = tunnel, .encap_type = UDP_ENCAP_L2TPINUDP, .encap_rcv = l2tp_udp_encap_recv, + .encap_err_rcv = l2tp_udp_encap_err_recv, .encap_destroy = l2tp_udp_encap_destroy, }; From ce08eeb59df090ca74ab6c035d8636ca75680cb4 Mon Sep 17 00:00:00 2001 From: Ravi Gunasekaran Date: Thu, 16 May 2024 11:19:32 +0530 Subject: [PATCH 1694/1702] dt-bindings: net: ti: Update maintainers list Update the list with the current maintainers of TI's CPSW ethernet peripheral. Signed-off-by: Ravi Gunasekaran Acked-by: Conor Dooley Acked-by: Roger Quadros Link: https://lore.kernel.org/r/20240516054932.27597-1-r-gunasekaran@ti.com Signed-off-by: Jakub Kicinski --- Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml | 1 - Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml | 1 - Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml | 1 - 3 files changed, 3 deletions(-) diff --git a/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml b/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml index d5bd93ee4dbb1..d14ca81f70e08 100644 --- a/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml +++ b/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml @@ -8,7 +8,6 @@ title: TI SoC Ethernet Switch Controller (CPSW) maintainers: - Siddharth Vadapalli - - Ravi Gunasekaran - Roger Quadros description: diff --git a/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml b/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml index 73ed5951d2969..02b6d32003cc1 100644 --- a/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml +++ b/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml @@ -8,7 +8,6 @@ title: The TI AM654x/J721E/AM642x SoC Gigabit Ethernet MAC (Media Access Control maintainers: - Siddharth Vadapalli - - Ravi Gunasekaran - Roger Quadros description: diff --git a/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml b/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml index b1c875325776d..3888692275ad4 100644 --- a/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml +++ b/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml @@ -8,7 +8,6 @@ title: The TI AM654x/J721E Common Platform Time Sync (CPTS) module maintainers: - Siddharth Vadapalli - - Ravi Gunasekaran - Roger Quadros description: |+ From 31279b0cb45f2f6be1f0e7324aeec26578a6e65e Mon Sep 17 00:00:00 2001 From: Ravi Gunasekaran Date: Thu, 16 May 2024 13:55:45 +0530 Subject: [PATCH 1695/1702] MAINTAINERS: net: Update reviewers for TI's Ethernet drivers Remove myself as reviewer for TI's ethernet drivers Signed-off-by: Ravi Gunasekaran Reviewed-by: Roger Quadros Link: https://lore.kernel.org/r/20240516082545.6412-1-r-gunasekaran@ti.com Signed-off-by: Jakub Kicinski --- MAINTAINERS | 1 - 1 file changed, 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index be7f72766dc65..b76072a35a295 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -22275,7 +22275,6 @@ F: drivers/counter/ti-eqep.c TI ETHERNET SWITCH DRIVER (CPSW) R: Siddharth Vadapalli -R: Ravi Gunasekaran R: Roger Quadros L: linux-omap@vger.kernel.org L: netdev@vger.kernel.org From f0fa84116434b50a8d249d0da8852f410a21ba98 Mon Sep 17 00:00:00 2001 From: Oleksij Rempel Date: Fri, 17 May 2024 07:01:21 +0200 Subject: [PATCH 1696/1702] net: dsa: microchip: Correct initialization order for KSZ88x3 ports Adjust the initialization sequence of KSZ88x3 switches to enable 802.1p priority control on Port 2 before configuring Port 1. This change ensures the apptrust functionality on Port 1 operates correctly, as it depends on the priority settings of Port 2. The prior initialization sequence incorrectly configured Port 1 first, which could lead to functional discrepancies. Fixes: a1ea57710c9d ("net: dsa: microchip: dcb: add special handling for KSZ88X3 family") Signed-off-by: Oleksij Rempel Reviewed-by: Hariprasad Kelam Acked-by: Arun Ramadoss Link: https://lore.kernel.org/r/20240517050121.2174412-1-o.rempel@pengutronix.de Signed-off-by: Jakub Kicinski --- drivers/net/dsa/microchip/ksz_dcb.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/net/dsa/microchip/ksz_dcb.c b/drivers/net/dsa/microchip/ksz_dcb.c index a971063275629..086bc9b3cf536 100644 --- a/drivers/net/dsa/microchip/ksz_dcb.c +++ b/drivers/net/dsa/microchip/ksz_dcb.c @@ -805,5 +805,15 @@ int ksz_dcb_init(struct ksz_device *dev) if (ret) return ret; + /* Enable 802.1p priority control on Port 2 during switch initialization. + * This setup is critical for the apptrust functionality on Port 1, which + * relies on the priority settings of Port 2. Note: Port 1 is naturally + * configured before Port 2, necessitating this configuration order. + */ + if (ksz_is_ksz88x3(dev)) + return ksz_prmw8(dev, KSZ_PORT_2, KSZ8_REG_PORT_1_CTRL_0, + KSZ8_PORT_802_1P_ENABLE, + KSZ8_PORT_802_1P_ENABLE); + return 0; } From fe56d6e4a99a40f50e64d5a8043f1fa838b1f7a1 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Thu, 16 May 2024 08:25:13 -0700 Subject: [PATCH 1697/1702] selftests: net: local_termination: annotate the expected failures Vladimir said when adding this test: The bridge driver fares particularly badly [...] mainly because it does not implement IFF_UNICAST_FLT. See commit 90b9566aa5cd ("selftests: forwarding: add a test for local_termination.sh"). We don't want to hide the known gaps, but having a test which always fails prevents us from catching regressions. Report the cases we know may fail as XFAIL. Reviewed-by: Simon Horman Reviewed-by: Hangbin Liu Reviewed-by: Petr Machata Link: https://lore.kernel.org/r/20240516152513.1115270-1-kuba@kernel.org Signed-off-by: Jakub Kicinski --- .../net/forwarding/local_termination.sh | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tools/testing/selftests/net/forwarding/local_termination.sh b/tools/testing/selftests/net/forwarding/local_termination.sh index c5b0cbc85b3e0..4b364cdf3ef0c 100755 --- a/tools/testing/selftests/net/forwarding/local_termination.sh +++ b/tools/testing/selftests/net/forwarding/local_termination.sh @@ -155,25 +155,30 @@ run_test() "$smac > $MACVLAN_ADDR, ethertype IPv4 (0x0800)" \ true - check_rcv $rcv_if_name "Unicast IPv4 to unknown MAC address" \ - "$smac > $UNKNOWN_UC_ADDR1, ethertype IPv4 (0x0800)" \ - false + xfail_on_veth $h1 \ + check_rcv $rcv_if_name "Unicast IPv4 to unknown MAC address" \ + "$smac > $UNKNOWN_UC_ADDR1, ethertype IPv4 (0x0800)" \ + false check_rcv $rcv_if_name "Unicast IPv4 to unknown MAC address, promisc" \ "$smac > $UNKNOWN_UC_ADDR2, ethertype IPv4 (0x0800)" \ true - check_rcv $rcv_if_name "Unicast IPv4 to unknown MAC address, allmulti" \ - "$smac > $UNKNOWN_UC_ADDR3, ethertype IPv4 (0x0800)" \ - false + xfail_on_veth $h1 \ + check_rcv $rcv_if_name \ + "Unicast IPv4 to unknown MAC address, allmulti" \ + "$smac > $UNKNOWN_UC_ADDR3, ethertype IPv4 (0x0800)" \ + false check_rcv $rcv_if_name "Multicast IPv4 to joined group" \ "$smac > $JOINED_MACV4_MC_ADDR, ethertype IPv4 (0x0800)" \ true - check_rcv $rcv_if_name "Multicast IPv4 to unknown group" \ - "$smac > $UNKNOWN_MACV4_MC_ADDR1, ethertype IPv4 (0x0800)" \ - false + xfail_on_veth $h1 \ + check_rcv $rcv_if_name \ + "Multicast IPv4 to unknown group" \ + "$smac > $UNKNOWN_MACV4_MC_ADDR1, ethertype IPv4 (0x0800)" \ + false check_rcv $rcv_if_name "Multicast IPv4 to unknown group, promisc" \ "$smac > $UNKNOWN_MACV4_MC_ADDR2, ethertype IPv4 (0x0800)" \ @@ -187,9 +192,10 @@ run_test() "$smac > $JOINED_MACV6_MC_ADDR, ethertype IPv6 (0x86dd)" \ true - check_rcv $rcv_if_name "Multicast IPv6 to unknown group" \ - "$smac > $UNKNOWN_MACV6_MC_ADDR1, ethertype IPv6 (0x86dd)" \ - false + xfail_on_veth $h1 \ + check_rcv $rcv_if_name "Multicast IPv6 to unknown group" \ + "$smac > $UNKNOWN_MACV6_MC_ADDR1, ethertype IPv6 (0x86dd)" \ + false check_rcv $rcv_if_name "Multicast IPv6 to unknown group, promisc" \ "$smac > $UNKNOWN_MACV6_MC_ADDR2, ethertype IPv6 (0x86dd)" \ From 4b377b4868ef17b040065bd468668c707d2477a5 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Fri, 17 May 2024 19:17:55 -0700 Subject: [PATCH 1698/1702] kprobe/ftrace: fix build error due to bad function definition Commit 1a7d0890dd4a ("kprobe/ftrace: bail out if ftrace was killed") introduced a bad K&R function definition, which we haven't accepted in a long long time. Gcc seems to let it slide, but clang notices with the appropriate error: kernel/kprobes.c:1140:24: error: a function declaration without a prototype is deprecated in all > 1140 | void kprobe_ftrace_kill() | ^ | void but this commit was apparently never in linux-next before it was sent upstream, so it didn't get the appropriate build test coverage. Fixes: 1a7d0890dd4a kprobe/ftrace: bail out if ftrace was killed Cc: Stephen Brennan Cc: Masami Hiramatsu (Google) Cc: Guo Ren Cc: Steven Rostedt (Google) Signed-off-by: Linus Torvalds --- kernel/kprobes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/kprobes.c b/kernel/kprobes.c index f2cea3c80cb58..6a76a81000735 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -1137,7 +1137,7 @@ static int disarm_kprobe_ftrace(struct kprobe *p) ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled); } -void kprobe_ftrace_kill() +void kprobe_ftrace_kill(void) { kprobe_ftrace_disabled = true; } From cee27ae5f1fb8bc4762f5d5de19ec6de6c45e239 Mon Sep 17 00:00:00 2001 From: Shuah Khan Date: Thu, 16 May 2024 20:51:07 -0600 Subject: [PATCH 1699/1702] Revert "selftests: Compile kselftest headers with -D_GNU_SOURCE" This reverts commit daef47b89efd0b745e8478d69a3ad724bd8b4dc6. This framework change to add D_GNU_SOURCE to KHDR_INCLUDES to Makefile, lib.mk, and kselftest_harness.h is causing build failures and warnings. Revert this change. Reported-by: Mark Brown Signed-off-by: Shuah Khan --- tools/testing/selftests/Makefile | 4 ++-- tools/testing/selftests/kselftest_harness.h | 2 +- tools/testing/selftests/lib.mk | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index f0431e6cb67e0..9039f3709affb 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -170,11 +170,11 @@ ifneq ($(KBUILD_OUTPUT),) # $(realpath ...) resolves symlinks abs_objtree := $(realpath $(abs_objtree)) BUILD := $(abs_objtree)/kselftest - KHDR_INCLUDES := -D_GNU_SOURCE -isystem ${abs_objtree}/usr/include + KHDR_INCLUDES := -isystem ${abs_objtree}/usr/include else BUILD := $(CURDIR) abs_srctree := $(shell cd $(top_srcdir) && pwd) - KHDR_INCLUDES := -D_GNU_SOURCE -isystem ${abs_srctree}/usr/include + KHDR_INCLUDES := -isystem ${abs_srctree}/usr/include DEFAULT_INSTALL_HDR_PATH := 1 endif diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h index 37b03f1b8741d..3c8f2965c2850 100644 --- a/tools/testing/selftests/kselftest_harness.h +++ b/tools/testing/selftests/kselftest_harness.h @@ -51,7 +51,7 @@ #define __KSELFTEST_HARNESS_H #ifndef _GNU_SOURCE -static_assert(0, "kselftest harness requires _GNU_SOURCE to be defined"); +#define _GNU_SOURCE #endif #include #include diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk index 33c24ceddfb71..09e8a854f0fcf 100644 --- a/tools/testing/selftests/lib.mk +++ b/tools/testing/selftests/lib.mk @@ -67,7 +67,7 @@ MAKEFLAGS += --no-print-directory endif ifeq ($(KHDR_INCLUDES),) -KHDR_INCLUDES := -D_GNU_SOURCE -isystem $(top_srcdir)/usr/include +KHDR_INCLUDES := -isystem $(top_srcdir)/usr/include endif # The following are built by lib.mk common compile rules. From 3da164023582969280df17636a9d829752787b1c Mon Sep 17 00:00:00 2001 From: Shuah Khan Date: Thu, 16 May 2024 20:58:26 -0600 Subject: [PATCH 1700/1702] Revert "selftests/sgx: Include KHDR_INCLUDES in Makefile" This reverts commit 2c3b8f8f37c6c0c926d584cf4158db95e62b960c. The framework change to add D_GNU_SOURCE to KHDR_INCLUDES to Makefile, lib.mk, and kselftest_harness.h is reverted as it is causing build failures and warnings. Revert this change as this change depends on the framework change. Reported-by: Mark Brown Signed-off-by: Shuah Khan --- tools/testing/selftests/sgx/Makefile | 2 +- tools/testing/selftests/sgx/sigstruct.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/sgx/Makefile b/tools/testing/selftests/sgx/Makefile index 26ea30fae23ca..867f88ce2570a 100644 --- a/tools/testing/selftests/sgx/Makefile +++ b/tools/testing/selftests/sgx/Makefile @@ -12,7 +12,7 @@ OBJCOPY := $(CROSS_COMPILE)objcopy endif INCLUDES := -I$(top_srcdir)/tools/include -HOST_CFLAGS := -Wall -Werror $(KHDR_INCLUDES) -g $(INCLUDES) -fPIC +HOST_CFLAGS := -Wall -Werror -g $(INCLUDES) -fPIC HOST_LDFLAGS := -z noexecstack -lcrypto ENCL_CFLAGS += -Wall -Werror -static-pie -nostdlib -ffreestanding -fPIE \ -fno-stack-protector -mrdrnd $(INCLUDES) diff --git a/tools/testing/selftests/sgx/sigstruct.c b/tools/testing/selftests/sgx/sigstruct.c index 200034a0fee51..d73b29becf5b0 100644 --- a/tools/testing/selftests/sgx/sigstruct.c +++ b/tools/testing/selftests/sgx/sigstruct.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2016-20 Intel Corporation. */ +#define _GNU_SOURCE #include #include #include From a97853f25b06f71c23b2d7a59fbd40f3f42d55ac Mon Sep 17 00:00:00 2001 From: Shuah Khan Date: Thu, 16 May 2024 20:22:37 -0600 Subject: [PATCH 1701/1702] Revert "selftests/cgroup: Drop define _GNU_SOURCE" This reverts commit c1457d9aad5ee2feafcf85aa9a58ab50500159d2. The framework change to add D_GNU_SOURCE to KHDR_INCLUDES to Makefile, lib.mk, and kselftest_harness.h is reverted as it is causing build failures and warnings. Revert this change as this change depends on the framework change. Reported-by: Mark Brown Signed-off-by: Shuah Khan --- tools/testing/selftests/cgroup/cgroup_util.c | 3 +++ tools/testing/selftests/cgroup/test_core.c | 2 ++ tools/testing/selftests/cgroup/test_cpu.c | 2 ++ tools/testing/selftests/cgroup/test_hugetlb_memcg.c | 2 ++ tools/testing/selftests/cgroup/test_kmem.c | 2 ++ tools/testing/selftests/cgroup/test_memcontrol.c | 2 ++ tools/testing/selftests/cgroup/test_zswap.c | 2 ++ 7 files changed, 15 insertions(+) diff --git a/tools/testing/selftests/cgroup/cgroup_util.c b/tools/testing/selftests/cgroup/cgroup_util.c index ce16a50ecff87..432db923bced0 100644 --- a/tools/testing/selftests/cgroup/cgroup_util.c +++ b/tools/testing/selftests/cgroup/cgroup_util.c @@ -1,4 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ + +#define _GNU_SOURCE + #include #include #include diff --git a/tools/testing/selftests/cgroup/test_core.c b/tools/testing/selftests/cgroup/test_core.c index de8baad460222..a5672a91d273c 100644 --- a/tools/testing/selftests/cgroup/test_core.c +++ b/tools/testing/selftests/cgroup/test_core.c @@ -1,4 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ + +#define _GNU_SOURCE #include #include #include diff --git a/tools/testing/selftests/cgroup/test_cpu.c b/tools/testing/selftests/cgroup/test_cpu.c index 5a4a314f6af7e..dad2ed82f3ef7 100644 --- a/tools/testing/selftests/cgroup/test_cpu.c +++ b/tools/testing/selftests/cgroup/test_cpu.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE #include #include #include diff --git a/tools/testing/selftests/cgroup/test_hugetlb_memcg.c b/tools/testing/selftests/cgroup/test_hugetlb_memcg.c index 80d05d50a42db..856f9508ea562 100644 --- a/tools/testing/selftests/cgroup/test_hugetlb_memcg.c +++ b/tools/testing/selftests/cgroup/test_hugetlb_memcg.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE + #include #include #include diff --git a/tools/testing/selftests/cgroup/test_kmem.c b/tools/testing/selftests/cgroup/test_kmem.c index 2e453ac50c0d6..96693d8772be2 100644 --- a/tools/testing/selftests/cgroup/test_kmem.c +++ b/tools/testing/selftests/cgroup/test_kmem.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE + #include #include #include diff --git a/tools/testing/selftests/cgroup/test_memcontrol.c b/tools/testing/selftests/cgroup/test_memcontrol.c index c871630d62a3a..41ae8047b8895 100644 --- a/tools/testing/selftests/cgroup/test_memcontrol.c +++ b/tools/testing/selftests/cgroup/test_memcontrol.c @@ -1,4 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#define _GNU_SOURCE + #include #include #include diff --git a/tools/testing/selftests/cgroup/test_zswap.c b/tools/testing/selftests/cgroup/test_zswap.c index 8418a8d7439f5..fa571bfd24328 100644 --- a/tools/testing/selftests/cgroup/test_zswap.c +++ b/tools/testing/selftests/cgroup/test_zswap.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE + #include #include #include From d0e71e23ec5e71655e1a046c9be6f35f78b1d6bb Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Mon, 20 May 2024 12:43:58 -0700 Subject: [PATCH 1702/1702] Revert "fanotify: remove unneeded sub-zero check for unsigned value" This reverts commit e6595224464b692ddae193d783402130d1625147. These kinds of patches are only making the code worse. Compilers don't care about the unnecessary check, but removing it makes the code less obvious to a human. The declaration of 'len' is more than 80 lines earlier, so a human won't easily see that 'len' is of an unsigned type, so to a human the range check that checks against zero is much more explicit and obvious. Any tool that complains about a range check like this just because the variable is unsigned is actively detrimental, and should be ignored. Signed-off-by: Linus Torvalds --- fs/notify/fanotify/fanotify_user.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index 483a6a1255fb2..9ec313e9f6e19 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -502,7 +502,7 @@ static int copy_fid_info_to_user(__kernel_fsid_t *fsid, struct fanotify_fh *fh, } /* Pad with 0's */ - WARN_ON_ONCE(len >= FANOTIFY_EVENT_ALIGN); + WARN_ON_ONCE(len < 0 || len >= FANOTIFY_EVENT_ALIGN); if (len > 0 && clear_user(buf, len)) return -EFAULT;