From 8338a51be87c40a5aa5a28a72fbe8dffe5e75c8d Mon Sep 17 00:00:00 2001 From: Kamil Dudka Date: Mon, 21 Dec 2015 18:58:15 +0100 Subject: [PATCH 1/2] Resolves: #1252549 - enable leaf optimization for XFS and NFS --- findutils-4.5.15-leaf-opt.patch | 173 ++++++++++++++++++++++++++++++++ findutils.spec | 9 +- 2 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 findutils-4.5.15-leaf-opt.patch diff --git a/findutils-4.5.15-leaf-opt.patch b/findutils-4.5.15-leaf-opt.patch new file mode 100644 index 0000000..2ed898c --- /dev/null +++ b/findutils-4.5.15-leaf-opt.patch @@ -0,0 +1,173 @@ +From 1fc080b7c1eac7fa8063273aaae6b80165c17fe0 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?P=C3=A1draig=20Brady?= +Date: Sun, 20 Dec 2015 23:46:05 +0000 +Subject: [PATCH 1/4] fts: enable leaf optimization for XFS + +XFS provides usable dirent.d_type only for DT_DIR, +but the noleaf optimization still applies, as confirmed with: + + test $(($(find . -maxdepth 1 -type d | wc -l) + 1)) = $(stat -c %h .) + +Enabling this gives significant traversal speedup. +Testing with find(1) gives: + + $ time find/find-before /usr/share >/dev/null + real 0m0.410s + user 0m0.145s + sys 0m0.266s + + $ time find/find-after /usr/share >/dev/null + real 0m0.278s + user 0m0.147s + sys 0m0.131s + +* lib/fts.c (leaf_optimization_applies): Add XFS to the white list. + +Upstream-commit: d459ec6a4f97001a57d9299143ea9a5f6b1f313b +Signed-off-by: Kamil Dudka +--- + gl/lib/fts.c | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/gl/lib/fts.c b/gl/lib/fts.c +index f76c015..fb720c0 100644 +--- a/gl/lib/fts.c ++++ b/gl/lib/fts.c +@@ -663,6 +663,7 @@ fts_close (FTS *sp) + # define S_MAGIC_TMPFS 0x1021994 + # define S_MAGIC_NFS 0x6969 + # define S_MAGIC_REISERFS 0x52654973 ++# define S_MAGIC_XFS 0x58465342 + # define S_MAGIC_PROC 0x9FA0 + + /* Return false if it is easy to determine the file system type of +@@ -718,6 +719,7 @@ leaf_optimization_applies (int dir_fd) + /* List here the file system types that lack usable dirent.d_type + info, yet for which the optimization does apply. */ + case S_MAGIC_REISERFS: ++ case S_MAGIC_XFS: + return true; + + case S_MAGIC_PROC: +-- +2.5.0 + + +From 2f0dddbb01a70b2ae02797b315924de7e06c3d83 Mon Sep 17 00:00:00 2001 +From: Kamil Dudka +Date: Wed, 9 Dec 2015 07:34:56 +0100 +Subject: [PATCH 2/4] fts: ensure leaf optimization is used for NFS + +NFS provides usable dirent.d_type but not necessarily for all entries +of large directories. See +* lib/fts.c (leaf_optimization_applies): Append NFS on the white list. + +Upstream-commit: c97b8b9030de7c9a9f9f6d7dcdc3505c6b3f7f98 +Signed-off-by: Kamil Dudka +--- + gl/lib/fts.c | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/gl/lib/fts.c b/gl/lib/fts.c +index fb720c0..d2d404f 100644 +--- a/gl/lib/fts.c ++++ b/gl/lib/fts.c +@@ -716,6 +716,11 @@ leaf_optimization_applies (int dir_fd) + + switch (fs_buf.f_type) + { ++ case S_MAGIC_NFS: ++ /* NFS provides usable dirent.d_type but not necessarily for all entries ++ of large directories. See . */ ++ return true; ++ + /* List here the file system types that lack usable dirent.d_type + info, yet for which the optimization does apply. */ + case S_MAGIC_REISERFS: +-- +2.5.0 + + +From 1328926a705fdb4728c1f255dd368de928736d39 Mon Sep 17 00:00:00 2001 +From: Kamil Dudka +Date: Fri, 25 Sep 2015 16:09:39 +0200 +Subject: [PATCH 3/4] fts: introduce the FTS_NOLEAF flag + +The flag is needed to implement the -noleaf option of find. +* lib/fts.c (link_count_optimize_ok): Implement the FTS_NOLEAF flag. +* lib/fts_.h (FTS_NOLEAF): New macro, shifted conflicting constants. +--- + gl/lib/fts.c | 4 ++++ + gl/lib/fts_.h | 12 +++++++++--- + 2 files changed, 13 insertions(+), 3 deletions(-) + +diff --git a/gl/lib/fts.c b/gl/lib/fts.c +index d2d404f..808466f 100644 +--- a/gl/lib/fts.c ++++ b/gl/lib/fts.c +@@ -786,6 +786,10 @@ link_count_optimize_ok (FTSENT const *p) + bool opt_ok; + struct LCO_ent *t2; + ++ if (ISSET(FTS_NOLEAF)) ++ /* leaf optimization explicitly disabled by the FTS_NOLEAF flag */ ++ return false; ++ + /* If we're not in CWDFD mode, don't bother with this optimization, + since the caller is not serious about performance. */ + if (!ISSET(FTS_CWDFD)) +diff --git a/gl/lib/fts_.h b/gl/lib/fts_.h +index 63d4b74..f1d519b 100644 +--- a/gl/lib/fts_.h ++++ b/gl/lib/fts_.h +@@ -149,10 +149,16 @@ typedef struct { + from input path names during fts_open initialization. */ + # define FTS_VERBATIM 0x1000 + +-# define FTS_OPTIONMASK 0x1fff /* valid user option mask */ ++ /* Disable leaf optimization (which eliminates stat() calls during traversal, ++ based on the count of nested directories stored in stat.st_nlink of each ++ directory). Note that the optimization is by default enabled only for ++ selected file systems, and only if the FTS_CWDFD flag is set. */ ++# define FTS_NOLEAF 0x2000 + +-# define FTS_NAMEONLY 0x2000 /* (private) child names only */ +-# define FTS_STOP 0x4000 /* (private) unrecoverable error */ ++# define FTS_OPTIONMASK 0x3fff /* valid user option mask */ ++ ++# define FTS_NAMEONLY 0x4000 /* (private) child names only */ ++# define FTS_STOP 0x8000 /* (private) unrecoverable error */ + int fts_options; /* fts_open options, global flags */ + + /* Map a directory's device number to a boolean. The boolean is +-- +2.5.0 + + +From c186934e6e37ddadf7511abb9b1045192757618e Mon Sep 17 00:00:00 2001 +From: Kamil Dudka +Date: Fri, 25 Sep 2015 19:13:15 +0200 +Subject: [PATCH 4/4] ftsfind: propagate the -noleaf option to FTS + +* find/ftsfind.c (find): Propagate the -noleaf option to FTS. +--- + find/ftsfind.c | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/find/ftsfind.c b/find/ftsfind.c +index 5159470..e34b672 100644 +--- a/find/ftsfind.c ++++ b/find/ftsfind.c +@@ -559,6 +559,9 @@ find (char *arg) + if (options.stay_on_filesystem) + ftsoptions |= FTS_XDEV; + ++ if (options.no_leaf_check) ++ ftsoptions |= FTS_NOLEAF; ++ + p = fts_open (arglist, ftsoptions, NULL); + if (NULL == p) + { +-- +2.5.0 + diff --git a/findutils.spec b/findutils.spec index 139f6bf..189cf5c 100644 --- a/findutils.spec +++ b/findutils.spec @@ -1,7 +1,7 @@ Summary: The GNU versions of find utilities (find and xargs) Name: findutils Version: 4.5.14 -Release: 5%{?dist} +Release: 6%{?dist} Epoch: 1 License: GPLv3+ Group: Applications/File @@ -27,6 +27,9 @@ Patch5: findutils-4.5.12-ppc-gnulib-tests.patch # make the test-suite ready for Python 3 Patch6: findutils-4.5.14-python3.patch +# enable leaf optimization for XFS and NFS (#1252549) +Patch8: findutils-4.5.15-leaf-opt.patch + Requires(post): /sbin/install-info Requires(preun): /sbin/install-info Conflicts: filesystem < 3 @@ -59,6 +62,7 @@ rm -rf locate %patch4 -p1 %patch5 -p1 %patch6 -p1 +%patch8 -p1 # needed because of findutils-4.4.0-no-locate.patch autoreconf -iv @@ -107,6 +111,9 @@ fi %{_infodir}/find-maint.info.gz %changelog +* Tue Dec 29 2015 Kamil Dudka - 1:4.5.14-6 +- enable leaf optimization for XFS and NFS (#1252549) + * Wed Mar 18 2015 Kamil Dudka - 1:4.5.14-5 - make the test-suite ready for Python 3 From 5fa5d75808c3c54c8b4eda801f49426bd665f4b0 Mon Sep 17 00:00:00 2001 From: Kamil Dudka Date: Fri, 24 Jun 2016 13:24:07 +0200 Subject: [PATCH 2/2] Resolves: #1299169 - disable leaf optimization for NFS --- findutils-4.5.15-leaf-opt.patch | 66 +++++++++++++++++++++++++++++++-- findutils.spec | 5 ++- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/findutils-4.5.15-leaf-opt.patch b/findutils-4.5.15-leaf-opt.patch index 2ed898c..46b87f1 100644 --- a/findutils-4.5.15-leaf-opt.patch +++ b/findutils-4.5.15-leaf-opt.patch @@ -1,7 +1,7 @@ From 1fc080b7c1eac7fa8063273aaae6b80165c17fe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1draig=20Brady?= Date: Sun, 20 Dec 2015 23:46:05 +0000 -Subject: [PATCH 1/4] fts: enable leaf optimization for XFS +Subject: [PATCH 1/5] fts: enable leaf optimization for XFS XFS provides usable dirent.d_type only for DT_DIR, but the noleaf optimization still applies, as confirmed with: @@ -56,7 +56,7 @@ index f76c015..fb720c0 100644 From 2f0dddbb01a70b2ae02797b315924de7e06c3d83 Mon Sep 17 00:00:00 2001 From: Kamil Dudka Date: Wed, 9 Dec 2015 07:34:56 +0100 -Subject: [PATCH 2/4] fts: ensure leaf optimization is used for NFS +Subject: [PATCH 2/5] fts: ensure leaf optimization is used for NFS NFS provides usable dirent.d_type but not necessarily for all entries of large directories. See @@ -91,7 +91,7 @@ index fb720c0..d2d404f 100644 From 1328926a705fdb4728c1f255dd368de928736d39 Mon Sep 17 00:00:00 2001 From: Kamil Dudka Date: Fri, 25 Sep 2015 16:09:39 +0200 -Subject: [PATCH 3/4] fts: introduce the FTS_NOLEAF flag +Subject: [PATCH 3/5] fts: introduce the FTS_NOLEAF flag The flag is needed to implement the -noleaf option of find. * lib/fts.c (link_count_optimize_ok): Implement the FTS_NOLEAF flag. @@ -147,7 +147,7 @@ index 63d4b74..f1d519b 100644 From c186934e6e37ddadf7511abb9b1045192757618e Mon Sep 17 00:00:00 2001 From: Kamil Dudka Date: Fri, 25 Sep 2015 19:13:15 +0200 -Subject: [PATCH 4/4] ftsfind: propagate the -noleaf option to FTS +Subject: [PATCH 4/5] ftsfind: propagate the -noleaf option to FTS * find/ftsfind.c (find): Propagate the -noleaf option to FTS. --- @@ -171,3 +171,61 @@ index 5159470..e34b672 100644 -- 2.5.0 + +From 2edb6204bab0acb0ef8cdde7499396afd9c66131 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?P=C3=A1draig=20Brady?= +Date: Mon, 18 Jan 2016 17:29:28 +0000 +Subject: [PATCH 5/5] fts: don't unconditionally use leaf optimization for NFS + +NFS st_nlink are not accurate on all implementations, +leading to aborts() if that assumption is made. +See +* lib/fts.c (leaf_optimization_applies): Remove NFS from +the white list, and document the issue. + +Upstream-commit: 85717b68b03bf85016c5079fbbf0c8aa2b182ba6 +Signed-off-by: Kamil Dudka +--- + gl/lib/fts.c | 19 ++++++++++--------- + 1 file changed, 10 insertions(+), 9 deletions(-) + +diff --git a/gl/lib/fts.c b/gl/lib/fts.c +index 55cd554..6e1eaf5 100644 +--- a/gl/lib/fts.c ++++ b/gl/lib/fts.c +@@ -718,22 +718,23 @@ leaf_optimization_applies (int dir_fd) + + switch (fs_buf.f_type) + { +- case S_MAGIC_NFS: +- /* NFS provides usable dirent.d_type but not necessarily for all entries +- of large directories. See . */ +- return true; +- + /* List here the file system types that lack usable dirent.d_type + info, yet for which the optimization does apply. */ + case S_MAGIC_REISERFS: + case S_MAGIC_XFS: + return true; + ++ /* Explicitly list here any other file system type for which the ++ optimization is not applicable, but need documentation. */ ++ case S_MAGIC_NFS: ++ /* NFS provides usable dirent.d_type but not necessarily for all entries ++ of large directories, so as per ++ NFS should return true. However st_nlink values are not accurate on ++ all implementations as per . */ ++ /* fall through */ + case S_MAGIC_PROC: +- /* Explicitly listing this or any other file system type for which +- the optimization is not applicable is not necessary, but we leave +- it here to document the risk. Per http://bugs.debian.org/143111, +- /proc may have bogus stat.st_nlink values. */ ++ /* Per /proc may have ++ bogus stat.st_nlink values. */ + /* fall through */ + default: + return false; +-- +2.5.5 + diff --git a/findutils.spec b/findutils.spec index 189cf5c..b1b72cf 100644 --- a/findutils.spec +++ b/findutils.spec @@ -1,7 +1,7 @@ Summary: The GNU versions of find utilities (find and xargs) Name: findutils Version: 4.5.14 -Release: 6%{?dist} +Release: 7%{?dist} Epoch: 1 License: GPLv3+ Group: Applications/File @@ -111,6 +111,9 @@ fi %{_infodir}/find-maint.info.gz %changelog +* Fri Jun 24 2016 Kamil Dudka - 1:4.5.14-7 +- disable leaf optimization for NFS (#1299169) + * Tue Dec 29 2015 Kamil Dudka - 1:4.5.14-6 - enable leaf optimization for XFS and NFS (#1252549)