From: Ben Peart <peartben@gmail•com>
To: git@vger•kernel.org
Cc: gitster@pobox•com, benpeart@microsoft•com, pclouds@gmail•com,
johannes.schindelin@gmx•de, David.Turner@twosigma•com,
peff@peff•net, christian.couder@gmail•com, avarab@gmail•com
Subject: [PATCH v5 7/7] fsmonitor: add a performance test
Date: Sat, 10 Jun 2017 09:40:26 -0400 [thread overview]
Message-ID: <20170610134026.104552-8-benpeart@microsoft.com> (raw)
In-Reply-To: <20170610134026.104552-1-benpeart@microsoft.com>
Add a test utility (test-drop-caches) that enables dropping the file
system cache on Windows.
Add a perf test (p7519-fsmonitor.sh) for fsmonitor.
Signed-off-by: Ben Peart <benpeart@microsoft•com>
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail•com>
---
Makefile | 1 +
t/helper/test-drop-caches.c | 107 +++++++++++++++++++++++++++++
t/perf/p7519-fsmonitor.sh | 161 ++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 269 insertions(+)
create mode 100644 t/helper/test-drop-caches.c
create mode 100755 t/perf/p7519-fsmonitor.sh
diff --git a/Makefile b/Makefile
index 992dd58801..893947839f 100644
--- a/Makefile
+++ b/Makefile
@@ -648,6 +648,7 @@ TEST_PROGRAMS_NEED_X += test-subprocess
TEST_PROGRAMS_NEED_X += test-svn-fe
TEST_PROGRAMS_NEED_X += test-urlmatch-normalization
TEST_PROGRAMS_NEED_X += test-wildmatch
+TEST_PROGRAMS_NEED_X += test-drop-caches
TEST_PROGRAMS = $(patsubst %,t/helper/%$X,$(TEST_PROGRAMS_NEED_X))
diff --git a/t/helper/test-drop-caches.c b/t/helper/test-drop-caches.c
new file mode 100644
index 0000000000..80830d920b
--- /dev/null
+++ b/t/helper/test-drop-caches.c
@@ -0,0 +1,107 @@
+#include "git-compat-util.h"
+#include <stdio.h>
+
+typedef DWORD NTSTATUS;
+
+#ifdef GIT_WINDOWS_NATIVE
+#include <tchar.h>
+
+#define STATUS_SUCCESS (0x00000000L)
+#define STATUS_PRIVILEGE_NOT_HELD (0xC0000061L)
+
+typedef enum _SYSTEM_INFORMATION_CLASS {
+ SystemMemoryListInformation = 80, // 80, q: SYSTEM_MEMORY_LIST_INFORMATION; s: SYSTEM_MEMORY_LIST_COMMAND (requires SeProfileSingleProcessPrivilege)
+} SYSTEM_INFORMATION_CLASS;
+
+// private
+typedef enum _SYSTEM_MEMORY_LIST_COMMAND
+{
+ MemoryCaptureAccessedBits,
+ MemoryCaptureAndResetAccessedBits,
+ MemoryEmptyWorkingSets,
+ MemoryFlushModifiedList,
+ MemoryPurgeStandbyList,
+ MemoryPurgeLowPriorityStandbyList,
+ MemoryCommandMax
+} SYSTEM_MEMORY_LIST_COMMAND;
+
+BOOL GetPrivilege(HANDLE TokenHandle, LPCSTR lpName, int flags)
+{
+ BOOL bResult;
+ DWORD dwBufferLength;
+ LUID luid;
+ TOKEN_PRIVILEGES tpPreviousState;
+ TOKEN_PRIVILEGES tpNewState;
+
+ dwBufferLength = 16;
+ bResult = LookupPrivilegeValueA(0, lpName, &luid);
+ if (bResult)
+ {
+ tpNewState.PrivilegeCount = 1;
+ tpNewState.Privileges[0].Luid = luid;
+ tpNewState.Privileges[0].Attributes = 0;
+ bResult = AdjustTokenPrivileges(TokenHandle, 0, &tpNewState, (DWORD)((LPBYTE)&(tpNewState.Privileges[1]) - (LPBYTE)&tpNewState), &tpPreviousState, &dwBufferLength);
+ if (bResult)
+ {
+ tpPreviousState.PrivilegeCount = 1;
+ tpPreviousState.Privileges[0].Luid = luid;
+ tpPreviousState.Privileges[0].Attributes = flags != 0 ? 2 : 0;
+ bResult = AdjustTokenPrivileges(TokenHandle, 0, &tpPreviousState, dwBufferLength, 0, 0);
+ }
+ }
+ return bResult;
+}
+#endif
+
+int cmd_main(int argc, const char **argv)
+{
+ NTSTATUS status = 1;
+#ifdef GIT_WINDOWS_NATIVE
+ HANDLE hProcess = GetCurrentProcess();
+ HANDLE hToken;
+ if (!OpenProcessToken(hProcess, TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, &hToken))
+ {
+ _ftprintf(stderr, _T("Can't open current process token\n"));
+ return 1;
+ }
+
+ if (!GetPrivilege(hToken, "SeProfileSingleProcessPrivilege", 1))
+ {
+ _ftprintf(stderr, _T("Can't get SeProfileSingleProcessPrivilege\n"));
+ return 1;
+ }
+
+ CloseHandle(hToken);
+
+ HMODULE ntdll = LoadLibrary(_T("ntdll.dll"));
+ if (!ntdll)
+ {
+ _ftprintf(stderr, _T("Can't load ntdll.dll, wrong Windows version?\n"));
+ return 1;
+ }
+
+ NTSTATUS(WINAPI *NtSetSystemInformation)(INT, PVOID, ULONG) = (NTSTATUS(WINAPI *)(INT, PVOID, ULONG))GetProcAddress(ntdll, "NtSetSystemInformation");
+ if (!NtSetSystemInformation)
+ {
+ _ftprintf(stderr, _T("Can't get function addresses, wrong Windows version?\n"));
+ return 1;
+ }
+
+ SYSTEM_MEMORY_LIST_COMMAND command = MemoryPurgeStandbyList;
+ status = NtSetSystemInformation(
+ SystemMemoryListInformation,
+ &command,
+ sizeof(SYSTEM_MEMORY_LIST_COMMAND)
+ );
+ if (status == STATUS_PRIVILEGE_NOT_HELD)
+ {
+ _ftprintf(stderr, _T("Insufficient privileges to execute the memory list command"));
+ }
+ else if (status != STATUS_SUCCESS)
+ {
+ _ftprintf(stderr, _T("Unable to execute the memory list command %lX"), status);
+ }
+#endif
+
+ return status;
+}
diff --git a/t/perf/p7519-fsmonitor.sh b/t/perf/p7519-fsmonitor.sh
new file mode 100755
index 0000000000..e41905cb9b
--- /dev/null
+++ b/t/perf/p7519-fsmonitor.sh
@@ -0,0 +1,161 @@
+#!/bin/sh
+
+test_description="Test core.fsmonitor"
+
+. ./perf-lib.sh
+
+# This has to be run with GIT_PERF_REPEAT_COUNT=1 to generate valid results.
+# Otherwise the caching that happens for the nth run will negate the validity
+# of the comparisons.
+if [ "$GIT_PERF_REPEAT_COUNT" -ne 1 ]
+then
+ echo "warning: This test must be run with GIT_PERF_REPEAT_COUNT=1 to generate valid results." >&2
+ echo "warning: Setting GIT_PERF_REPEAT_COUNT=1" >&2
+ GIT_PERF_REPEAT_COUNT=1
+fi
+
+test_perf_large_repo
+test_checkout_worktree
+
+# Convert unix style paths to what Watchman expects
+case "$(uname -s)" in
+MINGW*|MSYS_NT*)
+ GIT_WORK_TREE="$(cygpath -aw "$PWD" | sed 's,\\,/,g')"
+ ;;
+*)
+ GIT_WORK_TREE="$PWD"
+ ;;
+esac
+
+# The big win for using fsmonitor is the elimination of the need to scan
+# the working directory looking for changed files and untracked files. If
+# the file information is all cached in RAM, the benefits are reduced.
+
+flush_disk_cache () {
+ case "$(uname -s)" in
+ MINGW*|MSYS_NT*)
+ sync && test-drop-caches
+ ;;
+ *)
+ sudo sync && echo 3 | sudo tee /proc/sys/vm/drop_caches
+ ;;
+ esac
+
+}
+
+test_lazy_prereq UNTRACKED_CACHE '
+ { git update-index --test-untracked-cache; ret=$?; } &&
+ test $ret -ne 1
+'
+
+test_expect_success "setup" '
+ # Maybe set untrackedCache & splitIndex depending on the environment
+ if test -n "$GIT_PERF_7519_UNTRACKED_CACHE"
+ then
+ git config core.untrackedCache "$GIT_PERF_7519_UNTRACKED_CACHE"
+ else
+ if test_have_prereq UNTRACKED_CACHE
+ then
+ git config core.untrackedCache true
+ else
+ git config core.untrackedCache false
+ fi
+ fi &&
+
+ if test -n "$GIT_PERF_7519_SPLIT_INDEX"
+ then
+ git config core.splitIndex "$GIT_PERF_7519_SPLIT_INDEX"
+ fi &&
+
+ # Hook scaffolding
+ mkdir .git/hooks &&
+ cp ../../../templates/hooks--query-fsmonitor.sample .git/hooks/query-fsmonitor &&
+
+ # have Watchman monitor the test folder
+ watchman watch "$GIT_WORK_TREE" &&
+ watchman watch-list | grep -q -F "$GIT_WORK_TREE"
+'
+
+# Worst case without fsmonitor
+test_expect_success "clear fs cache" '
+ git config core.fsmonitor false &&
+ flush_disk_cache
+'
+test_perf "status (fsmonitor=false, cold fs cache)" '
+ git status
+'
+
+# Best case without fsmonitor
+test_perf "status (fsmonitor=false, warm fs cache)" '
+ git status
+'
+
+# Let's see if -uno & -uall make any difference
+test_expect_success "clear fs cache" '
+ flush_disk_cache
+'
+test_perf "status -uno (fsmonitor=false, cold fs cache)" '
+ git status -uno
+'
+
+test_expect_success "clear fs cache" '
+ flush_disk_cache
+'
+test_perf "status -uall (fsmonitor=false, cold fs cache)" '
+ git status -uall
+'
+
+# The first run with core.fsmonitor=true has to do a normal scan and write
+# out the index extension.
+test_expect_success "populate extension" '
+ # core.preloadIndex defeats the benefits of core.fsMonitor as it
+ # calls lstat for the index entries. Turn it off as _not_ doing
+ # the work is faster than doing the work across multiple threads.
+ git config core.fsmonitor true &&
+ git config core.preloadIndex false &&
+ git status
+'
+
+# Worst case with fsmonitor
+test_expect_success "shutdown fsmonitor, clear fs cache" '
+ watchman shutdown-server &&
+ flush_disk_cache
+'
+test_perf "status (fsmonitor=true, cold fs cache, cold fsmonitor)" '
+ git status
+'
+
+# Best case with fsmonitor
+test_perf "status (fsmonitor=true, warm fs cache, warm fsmonitor)" '
+ git status
+'
+
+# Best improved with fsmonitor (compare to worst case without fsmonitor)
+test_expect_success "clear fs cache" '
+ flush_disk_cache
+'
+test_perf "status (fsmonitor=true, cold fs cache, warm fsmonitor)" '
+ git status
+'
+
+# Let's see if -uno & -uall make any difference
+test_expect_success "clear fs cache" '
+ flush_disk_cache
+'
+test_perf "status -uno (fsmonitor=true, cold fs cache)" '
+ git status -uno
+'
+
+test_expect_success "clear fs cache" '
+ flush_disk_cache
+'
+test_perf "status -uall (fsmonitor=true, cold fs cache)" '
+ git status -uall
+'
+
+test_expect_success "cleanup" '
+ watchman watch-del "$GIT_WORK_TREE" &&
+ watchman shutdown-server
+'
+
+test_done
--
2.13.0
next prev parent reply other threads:[~2017-06-10 13:41 UTC|newest]
Thread overview: 137+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-06-10 13:40 [PATCH v5 0/7] Fast git status via a file system watcher Ben Peart
2017-06-10 13:40 ` [PATCH v5 1/7] bswap: add 64 bit endianness helper get_be64 Ben Peart
2017-06-10 13:40 ` [PATCH v5 2/7] dir: make lookup_untracked() available outside of dir.c Ben Peart
2017-06-10 13:40 ` [PATCH v5 3/7] fsmonitor: teach git to optionally utilize a file system monitor to speed up detecting new or changed files Ben Peart
2017-06-27 15:43 ` Christian Couder
2017-07-03 21:25 ` Ben Peart
2017-06-10 13:40 ` [PATCH v5 4/7] fsmonitor: add test cases for fsmonitor extension Ben Peart
2017-06-27 16:20 ` Christian Couder
2017-07-07 18:50 ` Ben Peart
2017-06-10 13:40 ` [PATCH v5 5/7] fsmonitor: add documentation for the " Ben Peart
2017-06-10 13:40 ` [PATCH v5 6/7] fsmonitor: add a sample query-fsmonitor hook script for Watchman Ben Peart
2017-06-10 13:40 ` Ben Peart [this message]
2017-06-10 14:04 ` [PATCH v5 7/7] fsmonitor: add a performance test Ben Peart
2017-06-12 22:04 ` Junio C Hamano
2017-06-14 14:12 ` Ben Peart
2017-06-14 18:36 ` Junio C Hamano
2017-07-07 18:14 ` Ben Peart
2017-07-07 18:35 ` Junio C Hamano
2017-07-07 19:07 ` Ben Peart
2017-07-07 19:33 ` David Turner
2017-07-08 7:19 ` Christian Couder
2017-06-28 5:11 ` [PATCH v5 0/7] Fast git status via a file system watcher Christian Couder
2017-07-10 13:36 ` Ben Peart
2017-07-10 14:40 ` Ben Peart
2017-09-15 19:20 ` [PATCH v6 00/12] " Ben Peart
2017-09-15 19:20 ` [PATCH v6 01/12] bswap: add 64 bit endianness helper get_be64 Ben Peart
2017-09-15 19:20 ` [PATCH v6 02/12] preload-index: add override to enable testing preload-index Ben Peart
2017-09-15 19:20 ` [PATCH v6 03/12] update-index: add a new --force-write-index option Ben Peart
2017-09-15 19:20 ` [PATCH v6 04/12] fsmonitor: teach git to optionally utilize a file system monitor to speed up detecting new or changed files Ben Peart
2017-09-15 21:35 ` David Turner
2017-09-18 13:07 ` Ben Peart
2017-09-18 13:32 ` David Turner
2017-09-18 13:49 ` Ben Peart
2017-09-15 19:20 ` [PATCH v6 05/12] fsmonitor: add documentation for the fsmonitor extension Ben Peart
2017-09-15 19:43 ` David Turner
2017-09-18 13:27 ` Ben Peart
2017-09-17 8:03 ` Junio C Hamano
2017-09-18 13:29 ` Ben Peart
2017-09-15 19:20 ` [PATCH v6 06/12] ls-files: Add support in ls-files to display the fsmonitor valid bit Ben Peart
2017-09-15 20:34 ` David Turner
2017-09-15 19:20 ` [PATCH v6 07/12] update-index: add fsmonitor support to update-index Ben Peart
2017-09-15 19:20 ` [PATCH v6 08/12] fsmonitor: add a test tool to dump the index extension Ben Peart
2017-09-17 8:02 ` Junio C Hamano
2017-09-18 13:38 ` Ben Peart
2017-09-18 15:43 ` Torsten Bögershausen
2017-09-18 16:28 ` Ben Peart
2017-09-19 14:16 ` Torsten Bögershausen
2017-09-19 15:36 ` Ben Peart
2017-09-15 19:20 ` [PATCH v6 09/12] split-index: disable the fsmonitor extension when running the split index test Ben Peart
2017-09-19 20:43 ` Jonathan Nieder
2017-09-20 17:11 ` Ben Peart
2017-09-20 17:46 ` Jonathan Nieder
2017-09-21 0:05 ` Ben Peart
2017-09-15 19:20 ` [PATCH v6 10/12] fsmonitor: add test cases for fsmonitor extension Ben Peart
2017-09-15 22:00 ` David Turner
2017-09-19 19:32 ` David Turner
2017-09-19 20:30 ` Ben Peart
2017-09-16 15:27 ` Torsten Bögershausen
2017-09-17 5:43 ` [PATCH v1 1/1] test-lint: echo -e (or -E) is not portable tboegi
2017-09-19 20:37 ` Jonathan Nieder
2017-09-20 13:49 ` Torsten Bögershausen
2017-09-22 1:04 ` Junio C Hamano
2017-09-18 14:06 ` [PATCH v6 10/12] fsmonitor: add test cases for fsmonitor extension Ben Peart
2017-09-17 4:47 ` Junio C Hamano
2017-09-18 15:25 ` Ben Peart
2017-09-19 20:34 ` Jonathan Nieder
2017-09-15 19:20 ` [PATCH v6 11/12] fsmonitor: add a sample integration script for Watchman Ben Peart
2017-09-15 19:20 ` [PATCH v6 12/12] fsmonitor: add a performance test Ben Peart
2017-09-15 21:56 ` David Turner
2017-09-18 14:24 ` Johannes Schindelin
2017-09-18 18:19 ` Ben Peart
2017-09-19 15:28 ` Johannes Schindelin
2017-09-19 19:27 ` [PATCH v7 00/12] Fast git status via a file system watcher Ben Peart
2017-09-19 19:27 ` [PATCH v7 01/12] bswap: add 64 bit endianness helper get_be64 Ben Peart
2017-09-19 19:27 ` [PATCH v7 02/12] preload-index: add override to enable testing preload-index Ben Peart
2017-09-20 22:06 ` Stefan Beller
2017-09-21 0:02 ` Ben Peart
2017-09-21 0:44 ` Stefan Beller
2017-09-19 19:27 ` [PATCH v7 03/12] update-index: add a new --force-write-index option Ben Peart
2017-09-20 5:47 ` Junio C Hamano
2017-09-20 14:58 ` Ben Peart
2017-09-21 1:46 ` Junio C Hamano
2017-09-21 2:06 ` Ben Peart
2017-09-21 2:18 ` Junio C Hamano
2017-09-21 2:32 ` Junio C Hamano
2017-09-19 19:27 ` [PATCH v7 04/12] fsmonitor: teach git to optionally utilize a file system monitor to speed up detecting new or changed files Ben Peart
2017-09-20 2:28 ` Junio C Hamano
2017-09-20 16:19 ` Ben Peart
2017-09-21 2:00 ` Junio C Hamano
2017-09-21 2:24 ` Ben Peart
2017-09-21 14:35 ` Ben Peart
2017-09-22 1:02 ` Junio C Hamano
2017-09-20 6:23 ` Junio C Hamano
2017-09-20 16:29 ` Ben Peart
2017-09-19 19:27 ` [PATCH v7 05/12] fsmonitor: add documentation for the fsmonitor extension Ben Peart
2017-09-20 10:00 ` Martin Ågren
2017-09-20 17:02 ` Ben Peart
2017-09-20 17:11 ` Martin Ågren
2017-09-19 19:27 ` [PATCH v7 06/12] ls-files: Add support in ls-files to display the fsmonitor valid bit Ben Peart
2017-09-19 19:46 ` David Turner
2017-09-19 20:44 ` Ben Peart
2017-09-19 21:27 ` David Turner
2017-09-19 22:44 ` Ben Peart
2017-09-19 19:27 ` [PATCH v7 07/12] update-index: add fsmonitor support to update-index Ben Peart
2017-09-19 19:27 ` [PATCH v7 08/12] fsmonitor: add a test tool to dump the index extension Ben Peart
2017-09-19 19:27 ` [PATCH v7 09/12] split-index: disable the fsmonitor extension when running the split index test Ben Peart
2017-09-19 19:27 ` [PATCH v7 10/12] fsmonitor: add test cases for fsmonitor extension Ben Peart
2017-09-19 19:27 ` [PATCH v7 11/12] fsmonitor: add a sample integration script for Watchman Ben Peart
2017-09-19 19:27 ` [PATCH v7 12/12] fsmonitor: add a performance test Ben Peart
2017-09-22 16:35 ` [PATCH v8 00/12] Fast git status via a file system watcher Ben Peart
2017-09-22 16:35 ` [PATCH v8 01/12] bswap: add 64 bit endianness helper get_be64 Ben Peart
2017-09-22 23:37 ` Martin Ågren
2017-09-23 23:31 ` Ben Peart
2017-09-24 3:51 ` Jeff King
2017-09-24 3:52 ` Junio C Hamano
2017-09-22 16:35 ` [PATCH v8 02/12] preload-index: add override to enable testing preload-index Ben Peart
2017-09-22 16:35 ` [PATCH v8 03/12] update-index: add a new --force-write-index option Ben Peart
2017-09-22 16:35 ` [PATCH v8 04/12] fsmonitor: teach git to optionally utilize a file system monitor to speed up detecting new or changed files Ben Peart
2017-09-22 16:35 ` [PATCH v8 05/12] fsmonitor: add documentation for the fsmonitor extension Ben Peart
2017-09-22 16:35 ` [PATCH v8 06/12] ls-files: Add support in ls-files to display the fsmonitor valid bit Ben Peart
2017-09-22 16:35 ` [PATCH v8 07/12] update-index: add fsmonitor support to update-index Ben Peart
2017-09-22 16:35 ` [PATCH v8 08/12] fsmonitor: add a test tool to dump the index extension Ben Peart
2017-09-22 23:37 ` Martin Ågren
2017-09-23 23:33 ` Ben Peart
2017-09-24 3:51 ` Junio C Hamano
2017-09-22 16:35 ` [PATCH v8 09/12] split-index: disable the fsmonitor extension when running the split index test Ben Peart
2017-09-22 16:35 ` [PATCH v8 10/12] fsmonitor: add test cases for fsmonitor extension Ben Peart
2017-09-22 16:35 ` [PATCH v8 11/12] fsmonitor: add a sample integration script for Watchman Ben Peart
2017-09-22 16:35 ` [PATCH v8 12/12] fsmonitor: add a performance test Ben Peart
2017-09-29 2:20 ` [PATCH v8 00/12] Fast git status via a file system watcher Junio C Hamano
2017-09-29 12:07 ` Ben Peart
2017-10-01 8:24 ` Junio C Hamano
2017-10-03 19:48 ` Ben Peart
2017-10-04 2:09 ` Junio C Hamano
2017-10-04 6:38 ` Alex Vandiver
2017-10-04 12:48 ` Ben Peart
2017-10-04 12:27 ` Ben Peart
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20170610134026.104552-8-benpeart@microsoft.com \
--to=peartben@gmail$(echo .)com \
--cc=David.Turner@twosigma$(echo .)com \
--cc=avarab@gmail$(echo .)com \
--cc=benpeart@microsoft$(echo .)com \
--cc=christian.couder@gmail$(echo .)com \
--cc=git@vger$(echo .)kernel.org \
--cc=gitster@pobox$(echo .)com \
--cc=johannes.schindelin@gmx$(echo .)de \
--cc=pclouds@gmail$(echo .)com \
--cc=peff@peff$(echo .)net \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox