diff options
507 files changed, 25352 insertions, 2937 deletions
diff --git a/.gitignore b/.gitignore index 054249b20a..e81de1063a 100644 --- a/.gitignore +++ b/.gitignore @@ -72,11 +72,13 @@ /git-format-patch /git-fsck /git-fsck-objects +/git-fsmonitor--daemon /git-gc /git-get-tar-commit-id /git-grep /git-hash-object /git-help +/git-hook /git-http-backend /git-http-fetch /git-http-push diff --git a/Documentation/.gitignore b/Documentation/.gitignore index 9022d48355..1c3771e7d7 100644 --- a/Documentation/.gitignore +++ b/Documentation/.gitignore @@ -14,4 +14,5 @@ manpage-base-url.xsl SubmittingPatches.txt tmp-doc-diff/ GIT-ASCIIDOCFLAGS +/.build/ /GIT-EXCLUDED-PROGRAMS diff --git a/Documentation/Makefile b/Documentation/Makefile index 2021568cd5..ed656db2ae 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -226,6 +226,7 @@ endif ifneq ($(findstring $(MAKEFLAGS),s),s) ifndef V + QUIET = @ QUIET_ASCIIDOC = @echo ' ' ASCIIDOC $@; QUIET_XMLTO = @echo ' ' XMLTO $@; QUIET_DB2TEXI = @echo ' ' DB2TEXI $@; @@ -233,11 +234,15 @@ ifndef V QUIET_DBLATEX = @echo ' ' DBLATEX $@; QUIET_XSLTPROC = @echo ' ' XSLTPROC $@; QUIET_GEN = @echo ' ' GEN $@; - QUIET_LINT = @echo ' ' LINT $@; QUIET_STDERR = 2> /dev/null QUIET_SUBDIR0 = +@subdir= QUIET_SUBDIR1 = ;$(NO_SUBDIR) echo ' ' SUBDIR $$subdir; \ $(MAKE) $(PRINT_DIR) -C $$subdir + + QUIET_LINT_GITLINK = @echo ' ' LINT GITLINK $<; + QUIET_LINT_MANSEC = @echo ' ' LINT MAN SEC $<; + QUIET_LINT_MANEND = @echo ' ' LINT MAN END $<; + export V endif endif @@ -285,7 +290,7 @@ install-html: html ../GIT-VERSION-FILE: FORCE $(QUIET_SUBDIR0)../ $(QUIET_SUBDIR1) GIT-VERSION-FILE -ifneq ($(MAKECMDGOALS),clean) +ifneq ($(filter-out lint-docs clean,$(MAKECMDGOALS)),) -include ../GIT-VERSION-FILE endif @@ -344,6 +349,7 @@ GIT-ASCIIDOCFLAGS: FORCE fi clean: + $(RM) -rf .build/ $(RM) *.xml *.xml+ *.html *.html+ *.1 *.5 *.7 $(RM) *.texi *.texi+ *.texi++ git.info gitman.info $(RM) *.pdf @@ -457,14 +463,61 @@ quick-install-html: require-htmlrepo print-man1: @for i in $(MAN1_TXT); do echo $$i; done -lint-docs:: - $(QUIET_LINT)$(PERL_PATH) lint-gitlink.perl \ +## Lint: Common +.build: + $(QUIET)mkdir $@ +.build/lint-docs: | .build + $(QUIET)mkdir $@ + +## Lint: gitlink +.build/lint-docs/gitlink: | .build/lint-docs + $(QUIET)mkdir $@ +.build/lint-docs/gitlink/howto: | .build/lint-docs/gitlink + $(QUIET)mkdir $@ +.build/lint-docs/gitlink/config: | .build/lint-docs/gitlink + $(QUIET)mkdir $@ +LINT_DOCS_GITLINK = $(patsubst %.txt,.build/lint-docs/gitlink/%.ok,$(HOWTO_TXT) $(DOC_DEP_TXT)) +$(LINT_DOCS_GITLINK): | .build/lint-docs/gitlink +$(LINT_DOCS_GITLINK): | .build/lint-docs/gitlink/howto +$(LINT_DOCS_GITLINK): | .build/lint-docs/gitlink/config +$(LINT_DOCS_GITLINK): lint-gitlink.perl +$(LINT_DOCS_GITLINK): .build/lint-docs/gitlink/%.ok: %.txt + $(QUIET_LINT_GITLINK)$(PERL_PATH) lint-gitlink.perl \ + $< \ $(HOWTO_TXT) $(DOC_DEP_TXT) \ --section=1 $(MAN1_TXT) \ --section=5 $(MAN5_TXT) \ - --section=7 $(MAN7_TXT); \ - $(PERL_PATH) lint-man-end-blurb.perl $(MAN_TXT); \ - $(PERL_PATH) lint-man-section-order.perl $(MAN_TXT); + --section=7 $(MAN7_TXT) >$@ +.PHONY: lint-docs-gitlink +lint-docs-gitlink: $(LINT_DOCS_GITLINK) + +## Lint: man-end-blurb +.build/lint-docs/man-end-blurb: | .build/lint-docs + $(QUIET)mkdir $@ +LINT_DOCS_MAN_END_BLURB = $(patsubst %.txt,.build/lint-docs/man-end-blurb/%.ok,$(MAN_TXT)) +$(LINT_DOCS_MAN_END_BLURB): | .build/lint-docs/man-end-blurb +$(LINT_DOCS_MAN_END_BLURB): lint-man-end-blurb.perl +$(LINT_DOCS_MAN_END_BLURB): .build/lint-docs/man-end-blurb/%.ok: %.txt + $(QUIET_LINT_MANEND)$(PERL_PATH) lint-man-end-blurb.perl $< >$@ +.PHONY: lint-docs-man-end-blurb +lint-docs-man-end-blurb: $(LINT_DOCS_MAN_END_BLURB) + +## Lint: man-section-order +.build/lint-docs/man-section-order: | .build/lint-docs + $(QUIET)mkdir $@ +LINT_DOCS_MAN_SECTION_ORDER = $(patsubst %.txt,.build/lint-docs/man-section-order/%.ok,$(MAN_TXT)) +$(LINT_DOCS_MAN_SECTION_ORDER): | .build/lint-docs/man-section-order +$(LINT_DOCS_MAN_SECTION_ORDER): lint-man-section-order.perl +$(LINT_DOCS_MAN_SECTION_ORDER): .build/lint-docs/man-section-order/%.ok: %.txt + $(QUIET_LINT_MANSEC)$(PERL_PATH) lint-man-section-order.perl $< >$@ +.PHONY: lint-docs-man-section-order +lint-docs-man-section-order: $(LINT_DOCS_MAN_SECTION_ORDER) + +## Lint: list of targets above +.PHONY: lint-docs +lint-docs: lint-docs-gitlink +lint-docs: lint-docs-man-end-blurb +lint-docs: lint-docs-man-section-order ifeq ($(wildcard po/Makefile),po/Makefile) doc-l10n install-l10n:: diff --git a/Documentation/MyFirstContribution.txt b/Documentation/MyFirstContribution.txt index b20bc8e914..63a2ef5449 100644 --- a/Documentation/MyFirstContribution.txt +++ b/Documentation/MyFirstContribution.txt @@ -905,19 +905,34 @@ Sending emails with Git is a two-part process; before you can prepare the emails themselves, you'll need to prepare the patches. Luckily, this is pretty simple: ---- -$ git format-patch --cover-letter -o psuh/ master..psuh ----- - -The `--cover-letter` parameter tells `format-patch` to create a cover letter -template for you. You will need to fill in the template before you're ready -to send - but for now, the template will be next to your other patches. - -The `-o psuh/` parameter tells `format-patch` to place the patch files into a -directory. This is useful because `git send-email` can take a directory and -send out all the patches from there. - -`master..psuh` tells `format-patch` to generate patches for the difference -between `master` and `psuh`. It will make one patch file per commit. After you +$ git format-patch --cover-letter -o psuh/ --base=auto psuh@{u}..psuh +---- + + . The `--cover-letter` option tells `format-patch` to create a + cover letter template for you. You will need to fill in the + template before you're ready to send - but for now, the template + will be next to your other patches. + + . The `-o psuh/` option tells `format-patch` to place the patch + files into a directory. This is useful because `git send-email` + can take a directory and send out all the patches from there. + + . The `--base=auto` option tells the command to record the "base + commit", on which the recipient is expected to apply the patch + series. The `auto` value will cause `format-patch` to compute + the base commit automatically, which is the merge base of tip + commit of the remote-tracking branch and the specified revision + range. + + . The `psuh@{u}..psuh` option tells `format-patch` to generate + patches for the commits you created on the `psuh` branch since it + forked from its upstream (which is `origin/master` if you + followed the example in the "Set up your workspace" section). If + you are already on the `psuh` branch, you can just say `@{u}`, + which means "commits on the current branch since it forked from + its upstream", which is the same thing. + +The command will make one patch file per commit. After you run, you can go have a look at each of the patches with your favorite text editor and make sure everything looks alright; however, it's not recommended to make code fixups via the patch file. It's a better idea to make the change the diff --git a/Documentation/RelNotes/2.34.0.txt b/Documentation/RelNotes/2.34.0.txt index c85385dc03..fdf7992193 100644 --- a/Documentation/RelNotes/2.34.0.txt +++ b/Documentation/RelNotes/2.34.0.txt @@ -74,6 +74,11 @@ UI, Workflows & Features * "git repack" has been taught to generate multi-pack reachability bitmaps. + * "git fsck" has been taught to report mismatch between expected and + actual types of an object better. + + * Use ssh public crypto for object and push-cert signing. + Performance, Internal Implementation, Development Support etc. @@ -173,6 +178,11 @@ Performance, Internal Implementation, Development Support etc. * Prevent "make sparse" from running for the source files that haven't been modified. + * The codepath to write a new version of .midx multi-pack index files + has learned to release the mmaped memory holding the current + version of .midx before removing them from the disk, as some + platforms do not allow removal of a file that still has mapping. + Fixes since v2.33 ----------------- @@ -334,6 +344,30 @@ Fixes since v2.33 to be grabbed, which can cause the build&test to fail. Tighten it. (merge 7491ef6198 js/windows-ci-path-fix later to maint). + * Avoid performance measurements from getting ruined by gc and other + housekeeping pauses interfering in the middle. + (merge be79131a53 rs/disable-gc-during-perf-tests later to maint). + + * Stop "git add --dry-run" from creating new blob and tree objects. + (merge e578d0311d rs/add-dry-run-without-objects later to maint). + + * "git commit" gave duplicated error message when the object store + was unwritable, which has been corrected. + (merge 4ef91a2d79 ab/fix-commit-error-message-upon-unwritable-object-store later to maint). + + * Recent sparse-index addition, namely any use of index_name_pos(), + can expand sparse index entries and breaks any code that walks + cache-tree or existing index entries. One such instance of such a + breakage has been corrected. + + * The xxdiff difftool backend can exit with status 128, which the + difftool-helper that launches the backend takes as a significant + failure, when it is not significant at all. Work it around. + (merge 571f4348dd da/mergetools-special-case-xxdiff-exit-128 later to maint). + + * Improve test framework around unwritable directories. + (merge 5d22e18965 ab/test-cleanly-recreate-trash-directory later to maint). + * Other code cleanup, docfix, build fix, etc. (merge f188160be9 ab/bundle-remove-verbose-option later to maint). (merge 8c6b4332b4 rs/close-pack-leakfix later to maint). @@ -343,3 +377,7 @@ Fixes since v2.33 (merge 100c2da2d3 rs/p3400-lose-tac later to maint). (merge 76f3b69896 tb/aggregate-ignore-leading-whitespaces later to maint). (merge 6e4fd8bfcd tz/doc-link-to-bundle-format-fix later to maint). + (merge f6c013dfa1 jc/doc-commit-header-continuation-line later to maint). + (merge ec9a37d69b ab/pkt-line-cleanup later to maint). + (merge 8650c6298c ab/fix-make-lint-docs later to maint). + (merge 1c720357ce ab/test-lib-diff-cleanup later to maint). diff --git a/Documentation/SubmittingPatches b/Documentation/SubmittingPatches index e409022d93..aea01bf36b 100644 --- a/Documentation/SubmittingPatches +++ b/Documentation/SubmittingPatches @@ -19,8 +19,11 @@ change is relevant to. base your work on the tip of the topic. * A new feature should be based on `master` in general. If the new - feature depends on a topic that is in `seen`, but not in `master`, - base your work on the tip of that topic. + feature depends on other topics that are in `next`, but not in + `master`, fork a branch from the tip of `master`, merge these topics + to the branch, and work on that branch. You can remind yourself of + how you prepared the base with `git log --first-parent master..` + easily by doing so. * Corrections and enhancements to a topic not yet in `master` should be based on the tip of that topic. If the topic has not been merged @@ -28,10 +31,10 @@ change is relevant to. into the series. * In the exceptional case that a new feature depends on several topics - not in `master`, start working on `next` or `seen` privately and send - out patches for discussion. Before the final merge, you may have to - wait until some of the dependent topics graduate to `master`, and - rebase your work. + not in `master`, start working on `next` or `seen` privately and + send out patches only for discussion. Once your new feature starts + to stabilize, you would have to rebase it (see the "depends on other + topics" above). * Some parts of the system have dedicated maintainers with their own repositories (see the section "Subsystems" below). Changes to @@ -71,8 +74,13 @@ Make sure that you have tests for the bug you are fixing. See [[tests]] When adding a new feature, make sure that you have new tests to show the feature triggers the new behavior when it should, and to show the -feature does not trigger when it shouldn't. After any code change, make -sure that the entire test suite passes. +feature does not trigger when it shouldn't. After any code change, +make sure that the entire test suite passes. When fixing a bug, make +sure you have new tests that breaks if somebody else breaks what you +fixed by accident to avoid regression. Also, try merging your work to +'next' and 'seen' and make sure the tests still pass; topics by others +that are still in flight may have unexpected interactions with what +you are trying to do in your topic. Pushing to a fork of https://github.com/git/git will use their CI integration to test your changes on Linux, Mac and Windows. See the @@ -144,8 +152,21 @@ without external resources. Instead of giving a URL to a mailing list archive, summarize the relevant points of the discussion. [[commit-reference]] -If you want to reference a previous commit in the history of a stable -branch, use the format "abbreviated hash (subject, date)", like this: + +There are a few reasons why you may want to refer to another commit in +the "more stable" part of the history (i.e. on branches like `maint`, +`master`, and `next`): + +. A commit that introduced the root cause of a bug you are fixing. + +. A commit that introduced a feature that is what you are enhancing. + +. A commit that conflicts with your work when you made a trial merge + of your work into `next` and `seen` for testing. + +When you reference a commit on a more stable branch (like `master`, +`maint` and `next`), use the format "abbreviated hash (subject, +date)", like this: .... Commit f86a374 (pack-bitmap.c: fix a memleak, 2015-03-30) @@ -259,9 +280,11 @@ Please make sure your patch does not add commented out debugging code, or include any extra files which do not relate to what your patch is trying to achieve. Make sure to review your patch after generating it, to ensure accuracy. Before -sending out, please make sure it cleanly applies to the `master` -branch head. If you are preparing a work based on "next" branch, -that is fine, but please mark it as such. +sending out, please make sure it cleanly applies to the base you +have chosen in the "Decide what to base your work on" section, +and unless it targets the `master` branch (which is the default), +mark your patches as such. + [[send-patches]] === Sending your patches. @@ -365,7 +388,10 @@ Security mailing list{security-ml-ref}. Send your patch with "To:" set to the mailing list, with "cc:" listing people who are involved in the area you are touching (the `git contacts` command in `contrib/contacts/` can help to -identify them), to solicit comments and reviews. +identify them), to solicit comments and reviews. Also, when you made +trial merges of your topic to `next` and `seen`, you may have noticed +work by others conflicting with your changes. There is a good possibility +that these people may know the area you are touching well. :current-maintainer: footnote:[The current maintainer: gitster@pobox.com] :git-ml: footnote:[The mailing list: git@vger.kernel.org] diff --git a/Documentation/config.txt b/Documentation/config.txt index 0c0e6b859f..b168f02dc3 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -262,11 +262,19 @@ color:: colors (at most two, one for foreground and one for background) and attributes (as many as you want), separated by spaces. + -The basic colors accepted are `normal`, `black`, `red`, `green`, `yellow`, -`blue`, `magenta`, `cyan` and `white`. The first color given is the -foreground; the second is the background. All the basic colors except -`normal` have a bright variant that can be specified by prefixing the -color with `bright`, like `brightred`. +The basic colors accepted are `normal`, `black`, `red`, `green`, +`yellow`, `blue`, `magenta`, `cyan`, `white` and `default`. The first +color given is the foreground; the second is the background. All the +basic colors except `normal` and `default` have a bright variant that can +be specified by prefixing the color with `bright`, like `brightred`. ++ +The color `normal` makes no change to the color. It is the same as an +empty string, but can be used as the foreground color when specifying a +background color alone (for example, "normal red"). ++ +The color `default` explicitly resets the color to the terminal default, +for example to specify a cleared background. Although it varies between +terminals, this is usually not the same as setting to "white black". + Colors may also be given as numbers between 0 and 255; these use ANSI 256-color mode (but note that not all terminals may support this). If @@ -280,6 +288,11 @@ The position of any attributes with respect to the colors be turned off by prefixing them with `no` or `no-` (e.g., `noreverse`, `no-ul`, etc). + +The pseudo-attribute `reset` resets all colors and attributes before +applying the specified coloring. For example, `reset green` will result +in a green foreground and default background without any active +attributes. ++ An empty color string produces no color effect at all. This can be used to avoid coloring specific elements without disabling color entirely. + @@ -304,7 +317,7 @@ path relative to Git's "runtime prefix", i.e. relative to the location where Git itself was installed. For example, `%(prefix)/bin/` refers to the directory in which the Git executable itself lives. If Git was compiled without runtime prefix support, the compiled-in prefix will be -subsituted instead. In the unlikely event that a literal path needs to +substituted instead. In the unlikely event that a literal path needs to be specified that should _not_ be expanded, it needs to be prefixed by `./`, like so: `./%(prefix)/bin`. diff --git a/Documentation/config/branch.txt b/Documentation/config/branch.txt index d323d7327f..1e0c7af014 100644 --- a/Documentation/config/branch.txt +++ b/Documentation/config/branch.txt @@ -7,7 +7,8 @@ branch.autoSetupMerge:: automatic setup is done; `true` -- automatic setup is done when the starting point is a remote-tracking branch; `always` -- automatic setup is done when the starting point is either a - local branch or remote-tracking + local branch or remote-tracking branch; `inherit` -- if the starting point + has a tracking configuration, it is copied to the new branch. This option defaults to true. branch.autoSetupRebase:: diff --git a/Documentation/config/color.txt b/Documentation/config/color.txt index dd2d2e0d84..1795b2d16b 100644 --- a/Documentation/config/color.txt +++ b/Documentation/config/color.txt @@ -17,11 +17,9 @@ date settings, starting and ending with a color, the dates should be set from oldest to newest. The metadata will be colored with the specified colors if the line was introduced before the given timestamp, overwriting older timestamped colors. - + Instead of an absolute timestamp relative timestamps work as well, e.g. `2.weeks.ago` is valid to address anything older than 2 weeks. - + It defaults to `blue,12 month ago,white,1 month ago,red`, which colors everything older than one year blue, recent changes between @@ -107,9 +105,12 @@ color.grep.<slot>:: `matchContext`;; matching text in context lines `matchSelected`;; - matching text in selected lines + matching text in selected lines. Also, used to customize the following + linkgit:git-log[1] subcommands: `--grep`, `--author` and `--committer`. `selected`;; - non-matching text in selected lines + non-matching text in selected lines. Also, used to customize the + following linkgit:git-log[1] subcommands: `--grep`, `--author` and + `--committer`. `separator`;; separators between fields on a line (`:`, `-`, and `=`) and between hunks (`--`) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index c04f62a54a..c9c2e29b73 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -62,22 +62,50 @@ core.protectNTFS:: Defaults to `true` on Windows, and `false` elsewhere. core.fsmonitor:: - If set, the value of this variable is used as a command which - will identify all files that may have changed since the - requested date/time. This information is used to speed up git by - avoiding unnecessary processing of files that have not changed. - See the "fsmonitor-watchman" section of linkgit:githooks[5]. + If set, this variable contains the pathname of the "fsmonitor" + hook command. ++ +This hook command is used to identify all files that may have changed +since the requested date/time. This information is used to speed up +git by avoiding unnecessary scanning of files that have not changed. ++ +See the "fsmonitor-watchman" section of linkgit:githooks[5]. ++ +Note: The value of this config setting is ignored if the +built-in file system monitor is enabled (see `core.useBuiltinFSMonitor`). core.fsmonitorHookVersion:: - Sets the version of hook that is to be used when calling fsmonitor. - There are currently versions 1 and 2. When this is not set, - version 2 will be tried first and if it fails then version 1 - will be tried. Version 1 uses a timestamp as input to determine - which files have changes since that time but some monitors - like watchman have race conditions when used with a timestamp. - Version 2 uses an opaque string so that the monitor can return - something that can be used to determine what files have changed - without race conditions. + Sets the protocol version to be used when invoking the + "fsmonitor" hook. ++ +There are currently versions 1 and 2. When this is not set, +version 2 will be tried first and if it fails then version 1 +will be tried. Version 1 uses a timestamp as input to determine +which files have changes since that time but some monitors +like Watchman have race conditions when used with a timestamp. +Version 2 uses an opaque string so that the monitor can return +something that can be used to determine what files have changed +without race conditions. ++ +Note: The value of this config setting is ignored if the +built-in file system monitor is enabled (see `core.useBuiltinFSMonitor`). + +core.useBuiltinFSMonitor:: + If set to true, enable the built-in file system monitor + daemon for this working directory (linkgit:git-fsmonitor--daemon[1]). ++ +Like hook-based file system monitors, the built-in file system monitor +can speed up Git commands that need to refresh the Git index +(e.g. `git status`) in a working directory with many files. The +built-in monitor eliminates the need to install and maintain an +external third-party tool. ++ +The built-in file system monitor is currently available only on a +limited set of supported platforms. Currently, this includes Windows +and MacOS. ++ +Note: if this config setting is set to `true`, the values of +`core.fsmonitor` and `core.fsmonitorHookVersion` are ignored. core.trustctime:: If false, the ctime differences between the index and the @@ -548,12 +576,29 @@ core.whitespace:: errors. The default tab width is 8. Allowed values are 1 to 63. core.fsyncObjectFiles:: - This boolean will enable 'fsync()' when writing object files. -+ -This is a total waste of time and effort on a filesystem that orders -data writes properly, but can be useful for filesystems that do not use -journalling (traditional UNIX filesystems) or that only journal metadata -and not file contents (OS X's HFS+, or Linux ext3 with "data=writeback"). + A value indicating the level of effort Git will expend in + trying to make objects added to the repo durable in the event + of an unclean system shutdown. This setting currently only + controls loose objects in the object store, so updates to any + refs or the index may not be equally durable. ++ +* `false` allows data to remain in file system caches according to + operating system policy, whence it may be lost if the system loses power + or crashes. +* `true` triggers a data integrity flush for each loose object added to the + object store. This is the safest setting that is likely to ensure durability + across all operating systems and file systems that honor the 'fsync' system + call. However, this setting comes with a significant performance cost on + common hardware. Git does not currently fsync parent directories for + newly-added files, so some filesystems may still allow data to be lost on + system crash. +* `batch` enables an experimental mode that uses interfaces available in some + operating systems to write loose object data with a minimal set of FLUSH + CACHE (or equivalent) commands sent to the storage controller. If the + operating system interfaces are not available, this mode behaves the same as + `true`. This mode is expected to be as safe as `true` on macOS for repos + stored on HFS+ or APFS filesystems and on Windows for repos stored on NTFS or + ReFS. core.preloadIndex:: Enable parallel index preload for operations like 'git diff' diff --git a/Documentation/config/gpg.txt b/Documentation/config/gpg.txt index d94025cb36..c9be554c73 100644 --- a/Documentation/config/gpg.txt +++ b/Documentation/config/gpg.txt @@ -11,13 +11,13 @@ gpg.program:: gpg.format:: Specifies which key format to use when signing with `--gpg-sign`. - Default is "openpgp" and another possible value is "x509". + Default is "openpgp". Other possible values are "x509", "ssh". gpg.<format>.program:: Use this to customize the program used for the signing format you chose. (see `gpg.program` and `gpg.format`) `gpg.program` can still be used as a legacy synonym for `gpg.openpgp.program`. The default - value for `gpg.x509.program` is "gpgsm". + value for `gpg.x509.program` is "gpgsm" and `gpg.ssh.program` is "ssh-keygen". gpg.minTrustLevel:: Specifies a minimum trust level for signature verification. If @@ -33,3 +33,47 @@ gpg.minTrustLevel:: * `marginal` * `fully` * `ultimate` + +gpg.ssh.defaultKeyCommand: + This command that will be run when user.signingkey is not set and a ssh + signature is requested. On successful exit a valid ssh public key is + expected in the first line of its output. To automatically use the first + available key from your ssh-agent set this to "ssh-add -L". + +gpg.ssh.allowedSignersFile:: + A file containing ssh public keys which you are willing to trust. + The file consists of one or more lines of principals followed by an ssh + public key. + e.g.: user1@example.com,user2@example.com ssh-rsa AAAAX1... + See ssh-keygen(1) "ALLOWED SIGNERS" for details. + The principal is only used to identify the key and is available when + verifying a signature. ++ +SSH has no concept of trust levels like gpg does. To be able to differentiate +between valid signatures and trusted signatures the trust level of a signature +verification is set to `fully` when the public key is present in the allowedSignersFile. +Otherwise the trust level is `undefined` and git verify-commit/tag will fail. ++ +This file can be set to a location outside of the repository and every developer +maintains their own trust store. A central repository server could generate this +file automatically from ssh keys with push access to verify the code against. +In a corporate setting this file is probably generated at a global location +from automation that already handles developer ssh keys. ++ +A repository that only allows signed commits can store the file +in the repository itself using a path relative to the top-level of the working tree. +This way only committers with an already valid key can add or change keys in the keyring. ++ +Since OpensSSH 8.8 this file allows specifying a key lifetime using valid-after & +valid-before options. Git will mark signatures as valid if the signing key was +valid at the time of the signatures creation. This allows users to change a +signing key without invalidating all previously made signatures. ++ +Using a SSH CA key with the cert-authority option +(see ssh-keygen(1) "CERTIFICATES") is also valid. + +gpg.ssh.revocationFile:: + Either a SSH KRL or a list of revoked public keys (without the principal prefix). + See ssh-keygen(1) for details. + If a public key is found in this file then it will always be treated + as having trust level "never" and signatures will show as invalid. diff --git a/Documentation/config/merge.txt b/Documentation/config/merge.txt index e27cc63944..99e83dd36e 100644 --- a/Documentation/config/merge.txt +++ b/Documentation/config/merge.txt @@ -4,7 +4,14 @@ merge.conflictStyle:: shows a `<<<<<<<` conflict marker, changes made by one side, a `=======` marker, changes made by the other side, and then a `>>>>>>>` marker. An alternate style, "diff3", adds a `|||||||` - marker and the original text before the `=======` marker. + marker and the original text before the `=======` marker. The + "merge" style tends to produce smaller conflict regions than diff3, + both because of the exclusion of the original text, and because + when a subset of lines match on the two sides they are just pulled + out of the conflict region. Another alternate style, "zdiff3", is + similar to diff3 but removes matching lines on the two sides from + the conflict region when those matching lines appear near either + the beginning or end of a conflict region. merge.defaultToUpstream:: If merge is called without any commit argument, merge the upstream diff --git a/Documentation/config/submodule.txt b/Documentation/config/submodule.txt index ee454f8126..c10aeb7fdc 100644 --- a/Documentation/config/submodule.txt +++ b/Documentation/config/submodule.txt @@ -91,3 +91,15 @@ submodule.alternateErrorStrategy:: `ignore`, `info`, `die`. Default is `die`. Note that if set to `ignore` or `info`, and if there is an error with the computed alternate, the clone proceeds as if no alternate was specified. + +submodule.superprojectGitDir:: + The relative path from the submodule's gitdir to its superproject's + gitdir. When Git is run in a repository, it usually makes no difference + whether this repository is standalone or a submodule, but if this + configuration variable is present, additional behavior may be possible, + such as "git status" printing additional information about this + submodule's status with respect to its superproject. This config should + only be present in projects which are submodules, but is not guaranteed + to be present in every submodule, so only optional value-added behavior + should be linked to it. It is set automatically during + submodule creation. diff --git a/Documentation/config/user.txt b/Documentation/config/user.txt index 59aec7c3ae..ad78dce9ec 100644 --- a/Documentation/config/user.txt +++ b/Documentation/config/user.txt @@ -36,3 +36,10 @@ user.signingKey:: commit, you can override the default selection with this variable. This option is passed unchanged to gpg's --local-user parameter, so you may specify a key using any method that gpg supports. + If gpg.format is set to "ssh" this can contain the literal ssh public + key (e.g.: "ssh-rsa XXXXXX identifier") or a file which contains it and + corresponds to the private key used for signing. The private key + needs to be available via ssh-agent. Alternatively it can be set to + a file containing a private key directly. If not set git will call + gpg.ssh.defaultKeyCommand (e.g.: "ssh-add -L") and try to use the first + key available. diff --git a/Documentation/date-formats.txt b/Documentation/date-formats.txt index 99c455f51c..67645cae64 100644 --- a/Documentation/date-formats.txt +++ b/Documentation/date-formats.txt @@ -5,9 +5,9 @@ The `GIT_AUTHOR_DATE` and `GIT_COMMITTER_DATE` environment variables support the following date formats: Git internal format:: - It is `<unix timestamp> <time zone offset>`, where `<unix - timestamp>` is the number of seconds since the UNIX epoch. - `<time zone offset>` is a positive or negative offset from UTC. + It is `<unix-timestamp> <time-zone-offset>`, where + `<unix-timestamp>` is the number of seconds since the UNIX epoch. + `<time-zone-offset>` is a positive or negative offset from UTC. For example CET (which is 1 hour ahead of UTC) is `+0100`. RFC 2822:: diff --git a/Documentation/diff-options.txt b/Documentation/diff-options.txt index c89d530d3d..b05f1c9f1c 100644 --- a/Documentation/diff-options.txt +++ b/Documentation/diff-options.txt @@ -64,6 +64,14 @@ ifdef::git-log[] each of the parents. Separate log entry and diff is generated for each parent. + +--diff-merges=remerge::: +--diff-merges=r::: +--remerge-diff::: + With this option, two-parent merge commits are remerged to + create a temporary tree object -- potentially containing files + with conflict markers and such. A diff is then shown between + that temporary tree and the actual merge commit. ++ --diff-merges=combined::: --diff-merges=c::: -c::: diff --git a/Documentation/git-archimport.txt b/Documentation/git-archimport.txt index a595a0ffee..847777fd17 100644 --- a/Documentation/git-archimport.txt +++ b/Documentation/git-archimport.txt @@ -9,14 +9,14 @@ git-archimport - Import a GNU Arch repository into Git SYNOPSIS -------- [verse] -'git archimport' [-h] [-v] [-o] [-a] [-f] [-T] [-D depth] [-t tempdir] - <archive/branch>[:<git-branch>] ... +'git archimport' [-h] [-v] [-o] [-a] [-f] [-T] [-D <depth>] [-t <tempdir>] + <archive>/<branch>[:<git-branch>]... DESCRIPTION ----------- Imports a project from one or more GNU Arch repositories. It will follow branches -and repositories within the namespaces defined by the <archive/branch> +and repositories within the namespaces defined by the <archive>/<branch> parameters supplied. If it cannot find the remote branch a merge comes from it will just import it as a regular commit. If it can find it, it will mark it as a merge whenever possible (see discussion below). @@ -27,7 +27,7 @@ import new branches within the provided roots. It expects to be dealing with one project only. If it sees branches that have different roots, it will refuse to run. In that case, -edit your <archive/branch> parameters to define clearly the scope of the +edit your <archive>/<branch> parameters to define clearly the scope of the import. 'git archimport' uses `tla` extensively in the background to access the @@ -42,7 +42,7 @@ incremental imports. While 'git archimport' will try to create sensible branch names for the archives that it imports, it is also possible to specify Git branch names -manually. To do so, write a Git branch name after each <archive/branch> +manually. To do so, write a Git branch name after each <archive>/<branch> parameter, separated by a colon. This way, you can shorten the Arch branch names and convert Arch jargon to Git jargon, for example mapping a "PROJECT{litdd}devo{litdd}VERSION" branch to "master". @@ -104,8 +104,8 @@ OPTIONS Override the default tempdir. -<archive/branch>:: - Archive/branch identifier in a format that `tla log` understands. +<archive>/<branch>:: + <archive>/<branch> identifier in a format that `tla log` understands. GIT diff --git a/Documentation/git-archive.txt b/Documentation/git-archive.txt index 9f8172828d..bc4e76a783 100644 --- a/Documentation/git-archive.txt +++ b/Documentation/git-archive.txt @@ -93,12 +93,19 @@ BACKEND EXTRA OPTIONS zip ~~~ --0:: - Store the files instead of deflating them. --9:: - Highest and slowest compression level. You can specify any - number from 1 to 9 to adjust compression speed and ratio. +-<digit>:: + Specify compression level. Larger values allow the command + to spend more time to compress to smaller size. Supported + values are from `-0` (store-only) to `-9` (best ratio). + Default is `-6` if not given. +tar +~~~ +-<number>:: + Specify compression level. The value will be passed to the + compression command configured in `tar.<format>.command`. See + manual page of the configured command for the list of supported + levels and the default level if this option isn't specified. CONFIGURATION ------------- diff --git a/Documentation/git-branch.txt b/Documentation/git-branch.txt index 5449767121..e8a8facf85 100644 --- a/Documentation/git-branch.txt +++ b/Documentation/git-branch.txt @@ -16,7 +16,7 @@ SYNOPSIS [--points-at <object>] [--format=<format>] [(-r | --remotes) | (-a | --all)] [--list] [<pattern>...] -'git branch' [--track | --no-track] [-f] <branchname> [<start-point>] +'git branch' [--track [direct|inherit] | --no-track] [-f] <branchname> [<start-point>] 'git branch' (--set-upstream-to=<upstream> | -u <upstream>) [<branchname>] 'git branch' --unset-upstream [<branchname>] 'git branch' (-m | -M) [<oldbranch>] <newbranch> @@ -125,14 +125,14 @@ OPTIONS -m:: --move:: - Move/rename a branch and the corresponding reflog. + Move/rename a branch, together with its config and reflog. -M:: Shortcut for `--move --force`. -c:: --copy:: - Copy a branch and the corresponding reflog. + Copy a branch, together with its config and reflog. -C:: Shortcut for `--copy --force`. @@ -206,24 +206,31 @@ This option is only applicable in non-verbose mode. Display the full sha1s in the output listing rather than abbreviating them. -t:: ---track:: +--track [inherit|direct]:: When creating a new branch, set up `branch.<name>.remote` and - `branch.<name>.merge` configuration entries to mark the - start-point branch as "upstream" from the new branch. This + `branch.<name>.merge` configuration entries to set "upstream" tracking + configuration for the new branch. This configuration will tell git to show the relationship between the two branches in `git status` and `git branch -v`. Furthermore, it directs `git pull` without arguments to pull from the upstream when the new branch is checked out. + -This behavior is the default when the start point is a remote-tracking branch. +The exact upstream branch is chosen depending on the optional argument: +`--track` or `--track direct` means to use the start-point branch itself as the +upstream; `--track inherit` means to copy the upstream configuration of the +start-point branch. ++ +`--track direct` is the default when the start point is a remote-tracking branch. Set the branch.autoSetupMerge configuration variable to `false` if you want `git switch`, `git checkout` and `git branch` to always behave as if `--no-track` were given. Set it to `always` if you want this behavior when the -start-point is either a local or remote-tracking branch. +start-point is either a local or remote-tracking branch. Set it to +`inherit` if you want to copy the tracking configuration from the +branch point. --no-track:: Do not set up "upstream" configuration, even if the - branch.autoSetupMerge configuration variable is true. + branch.autoSetupMerge configuration variable is set. --set-upstream:: As this option had confusing syntax, it is no longer supported. diff --git a/Documentation/git-bundle.txt b/Documentation/git-bundle.txt index 71b5ecabd1..72ab813905 100644 --- a/Documentation/git-bundle.txt +++ b/Documentation/git-bundle.txt @@ -51,10 +51,10 @@ using the `--thin` option to linkgit:git-pack-objects[1], and unbundled using the `--fix-thin` option to linkgit:git-index-pack[1]. There is no option to create a "thick pack" when using revision -exclusions, users should not be concerned about the difference. By -using "thin packs" bundles created using exclusions are smaller in +exclusions, and users should not be concerned about the difference. By +using "thin packs", bundles created using exclusions are smaller in size. That they're "thin" under the hood is merely noted here as a -curiosity, and as a reference to other documentation +curiosity, and as a reference to other documentation. See link:technical/bundle-format.html[the `bundle-format` documentation] for more details and the discussion of "thin pack" in @@ -144,7 +144,7 @@ unbundle <file>:: SPECIFYING REFERENCES --------------------- -Revisions must accompanied by reference names to be packaged in a +Revisions must be accompanied by reference names to be packaged in a bundle. More than one reference may be packaged, and more than one set of prerequisite objects can diff --git a/Documentation/git-checkout.txt b/Documentation/git-checkout.txt index d473c9bf38..2a90ea6cd0 100644 --- a/Documentation/git-checkout.txt +++ b/Documentation/git-checkout.txt @@ -11,7 +11,7 @@ SYNOPSIS 'git checkout' [-q] [-f] [-m] [<branch>] 'git checkout' [-q] [-f] [-m] --detach [<branch>] 'git checkout' [-q] [-f] [-m] [--detach] <commit> -'git checkout' [-q] [-f] [-m] [[-b|-B|--orphan] <new_branch>] [<start_point>] +'git checkout' [-q] [-f] [-m] [[-b|-B|--orphan] <new-branch>] [<start-point>] 'git checkout' [-f|--ours|--theirs|-m|--conflict=<style>] [<tree-ish>] [--] <pathspec>... 'git checkout' [-f|--ours|--theirs|-m|--conflict=<style>] [<tree-ish>] --pathspec-from-file=<file> [--pathspec-file-nul] 'git checkout' (-p|--patch) [<tree-ish>] [--] [<pathspec>...] @@ -43,7 +43,7 @@ You could omit `<branch>`, in which case the command degenerates to rather expensive side-effects to show only the tracking information, if exists, for the current branch. -'git checkout' -b|-B <new_branch> [<start point>]:: +'git checkout' -b|-B <new-branch> [<start-point>]:: Specifying `-b` causes a new branch to be created as if linkgit:git-branch[1] were called and then checked out. In @@ -52,11 +52,11 @@ if exists, for the current branch. `--track` without `-b` implies branch creation; see the description of `--track` below. + -If `-B` is given, `<new_branch>` is created if it doesn't exist; otherwise, it +If `-B` is given, `<new-branch>` is created if it doesn't exist; otherwise, it is reset. This is the transactional equivalent of + ------------ -$ git branch -f <branch> [<start point>] +$ git branch -f <branch> [<start-point>] $ git checkout <branch> ------------ + @@ -145,18 +145,18 @@ as `ours` (i.e. "our shared canonical history"), while what you did on your side branch as `theirs` (i.e. "one contributor's work on top of it"). --b <new_branch>:: - Create a new branch named `<new_branch>` and start it at - `<start_point>`; see linkgit:git-branch[1] for details. +-b <new-branch>:: + Create a new branch named `<new-branch>` and start it at + `<start-point>`; see linkgit:git-branch[1] for details. --B <new_branch>:: - Creates the branch `<new_branch>` and start it at `<start_point>`; - if it already exists, then reset it to `<start_point>`. This is +-B <new-branch>:: + Creates the branch `<new-branch>` and start it at `<start-point>`; + if it already exists, then reset it to `<start-point>`. This is equivalent to running "git branch" with "-f"; see linkgit:git-branch[1] for details. -t:: ---track:: +--track [direct|inherit]:: When creating a new branch, set up "upstream" configuration. See "--track" in linkgit:git-branch[1] for details. + @@ -210,16 +210,16 @@ variable. `<commit>` is not a branch name. See the "DETACHED HEAD" section below for details. ---orphan <new_branch>:: - Create a new 'orphan' branch, named `<new_branch>`, started from - `<start_point>` and switch to it. The first commit made on this +--orphan <new-branch>:: + Create a new 'orphan' branch, named `<new-branch>`, started from + `<start-point>` and switch to it. The first commit made on this new branch will have no parents and it will be the root of a new history totally disconnected from all the other branches and commits. + The index and the working tree are adjusted as if you had previously run -`git checkout <start_point>`. This allows you to start a new history -that records a set of paths similar to `<start_point>` by easily running +`git checkout <start-point>`. This allows you to start a new history +that records a set of paths similar to `<start-point>` by easily running `git commit -a` to make the root commit. + This can be useful when you want to publish the tree from a commit @@ -229,7 +229,7 @@ whose full history contains proprietary or otherwise encumbered bits of code. + If you want to start a disconnected history that records a set of paths -that is totally different from the one of `<start_point>`, then you should +that is totally different from the one of `<start-point>`, then you should clear the index and the working tree right after creating the orphan branch by running `git rm -rf .` from the top level of the working tree. Afterwards you will be ready to prepare your new files, repopulating the @@ -266,8 +266,7 @@ When switching branches with `--merge`, staged changes may be lost. The same as `--merge` option above, but changes the way the conflicting hunks are presented, overriding the `merge.conflictStyle` configuration variable. Possible values are - "merge" (default) and "diff3" (in addition to what is shown by - "merge" style, shows the original contents). + "merge" (default), "diff3", and "zdiff3". -p:: --patch:: @@ -341,10 +340,10 @@ As a special case, you may use `A...B` as a shortcut for the merge base of `A` and `B` if there is exactly one merge base. You can leave out at most one of `A` and `B`, in which case it defaults to `HEAD`. -<new_branch>:: +<new-branch>:: Name for the new branch. -<start_point>:: +<start-point>:: The name of a commit at which to start the new branch; see linkgit:git-branch[1] for details. Defaults to `HEAD`. + diff --git a/Documentation/git-cherry-pick.txt b/Documentation/git-cherry-pick.txt index 5d750314b2..78dcc9171f 100644 --- a/Documentation/git-cherry-pick.txt +++ b/Documentation/git-cherry-pick.txt @@ -8,7 +8,7 @@ git-cherry-pick - Apply the changes introduced by some existing commits SYNOPSIS -------- [verse] -'git cherry-pick' [--edit] [-n] [-m parent-number] [-s] [-x] [--ff] +'git cherry-pick' [--edit] [-n] [-m <parent-number>] [-s] [-x] [--ff] [-S[<keyid>]] <commit>... 'git cherry-pick' (--continue | --skip | --abort | --quit) @@ -81,8 +81,8 @@ OPTIONS described above, and `-r` was to disable it. Now the default is not to do `-x` so this option is a no-op. --m parent-number:: ---mainline parent-number:: +-m <parent-number>:: +--mainline <parent-number>:: Usually you cannot cherry-pick a merge because you do not know which side of the merge should be considered the mainline. This option specifies the parent number (starting from 1) of diff --git a/Documentation/git-clone.txt b/Documentation/git-clone.txt index 3fe3810f1c..9685ea0691 100644 --- a/Documentation/git-clone.txt +++ b/Documentation/git-clone.txt @@ -9,10 +9,10 @@ git-clone - Clone a repository into a new directory SYNOPSIS -------- [verse] -'git clone' [--template=<template_directory>] +'git clone' [--template=<template-directory>] [-l] [-s] [--no-hardlinks] [-q] [-n] [--bare] [--mirror] [-o <name>] [-b <name>] [-u <upload-pack>] [--reference <repository>] - [--dissociate] [--separate-git-dir <git dir>] + [--dissociate] [--separate-git-dir <git-dir>] [--depth <depth>] [--[no-]single-branch] [--no-tags] [--recurse-submodules[=<pathspec>]] [--[no-]shallow-submodules] [--[no-]remote-submodules] [--jobs <n>] [--sparse] [--[no-]reject-shallow] @@ -211,7 +211,7 @@ objects from the source repository into a pack in the cloned repository. via ssh, this specifies a non-default path for the command run on the other end. ---template=<template_directory>:: +--template=<template-directory>:: Specify the directory from which templates will be used; (See the "TEMPLATE DIRECTORY" section of linkgit:git-init[1].) @@ -294,7 +294,7 @@ or `--mirror` is given) superproject's recorded SHA-1. Equivalent to passing `--remote` to `git submodule update`. ---separate-git-dir=<git dir>:: +--separate-git-dir=<git-dir>:: Instead of placing the cloned repository where it is supposed to be, place the cloned repository at the specified directory, then make a filesystem-agnostic Git symbolic link to there. diff --git a/Documentation/git-commit.txt b/Documentation/git-commit.txt index 95fec5f069..2b3e4651b4 100644 --- a/Documentation/git-commit.txt +++ b/Documentation/git-commit.txt @@ -213,9 +213,14 @@ include::signoff-option.txt[] -n:: --no-verify:: - This option bypasses the pre-commit and commit-msg hooks. + By default, pre-commit and commit-msg hooks are run. When one of these + options is given, the hooks will be bypassed. See also linkgit:githooks[5]. +--verify:: + This option re-enables running of the pre-commit and commit-msg hooks + after an earlier `-n` or `--no-verify`. + --allow-empty:: Usually recording a commit that has the exact same tree as its sole parent commit is a mistake, and the command prevents you diff --git a/Documentation/git-config.txt b/Documentation/git-config.txt index 992225f612..4f5c830995 100644 --- a/Documentation/git-config.txt +++ b/Documentation/git-config.txt @@ -9,20 +9,20 @@ git-config - Get and set repository or global options SYNOPSIS -------- [verse] -'git config' [<file-option>] [--type=<type>] [--fixed-value] [--show-origin] [--show-scope] [-z|--null] name [value [value-pattern]] -'git config' [<file-option>] [--type=<type>] --add name value -'git config' [<file-option>] [--type=<type>] [--fixed-value] --replace-all name value [value-pattern] -'git config' [<file-option>] [--type=<type>] [--show-origin] [--show-scope] [-z|--null] [--fixed-value] --get name [value-pattern] -'git config' [<file-option>] [--type=<type>] [--show-origin] [--show-scope] [-z|--null] [--fixed-value] --get-all name [value-pattern] -'git config' [<file-option>] [--type=<type>] [--show-origin] [--show-scope] [-z|--null] [--fixed-value] [--name-only] --get-regexp name_regex [value-pattern] -'git config' [<file-option>] [--type=<type>] [-z|--null] --get-urlmatch name URL -'git config' [<file-option>] [--fixed-value] --unset name [value-pattern] -'git config' [<file-option>] [--fixed-value] --unset-all name [value-pattern] -'git config' [<file-option>] --rename-section old_name new_name -'git config' [<file-option>] --remove-section name +'git config' [<file-option>] [--type=<type>] [--fixed-value] [--show-origin] [--show-scope] [-z|--null] <name> [<value> [<value-pattern>]] +'git config' [<file-option>] [--type=<type>] --add <name> <value> +'git config' [<file-option>] [--type=<type>] [--fixed-value] --replace-all <name> <value> [<value-pattern>] +'git config' [<file-option>] [--type=<type>] [--show-origin] [--show-scope] [-z|--null] [--fixed-value] --get <name> [<value-pattern>] +'git config' [<file-option>] [--type=<type>] [--show-origin] [--show-scope] [-z|--null] [--fixed-value] --get-all <name> [<value-pattern>] +'git config' [<file-option>] [--type=<type>] [--show-origin] [--show-scope] [-z|--null] [--fixed-value] [--name-only] --get-regexp <name-regex> [<value-pattern>] +'git config' [<file-option>] [--type=<type>] [-z|--null] --get-urlmatch <name> <URL> +'git config' [<file-option>] [--fixed-value] --unset <name> [<value-pattern>] +'git config' [<file-option>] [--fixed-value] --unset-all <name> [<value-pattern>] +'git config' [<file-option>] --rename-section <old-name> <new-name> +'git config' [<file-option>] --remove-section <name> 'git config' [<file-option>] [--show-origin] [--show-scope] [-z|--null] [--name-only] -l | --list -'git config' [<file-option>] --get-color name [default] -'git config' [<file-option>] --get-colorbool name [stdout-is-tty] +'git config' [<file-option>] --get-color <name> [<default>] +'git config' [<file-option>] --get-colorbool <name> [<stdout-is-tty>] 'git config' [<file-option>] -e | --edit DESCRIPTION @@ -102,9 +102,9 @@ OPTIONS in which section and variable names are lowercased, but subsection names are not. ---get-urlmatch name URL:: +--get-urlmatch <name> <URL>:: When given a two-part name section.key, the value for - section.<url>.key whose <url> part matches the best to the + section.<URL>.key whose <URL> part matches the best to the given URL is returned (if no such key exists, the value for section.key is used as a fallback). When given just the section as name, do so for all the keys in the section and @@ -145,8 +145,8 @@ See also <<FILES>>. read from or written to if `extensions.worktreeConfig` is present. If not it's the same as `--local`. --f config-file:: ---file config-file:: +-f <config-file>:: +--file <config-file>:: For writing options: write to the specified file rather than the repository `.git/config`. + @@ -155,7 +155,7 @@ available files. + See also <<FILES>>. ---blob blob:: +--blob <blob>:: Similar to `--file` but use the given blob instead of a file. E.g. you can use 'master:.gitmodules' to read values from the file '.gitmodules' in the master branch. See "SPECIFYING REVISIONS" @@ -246,18 +246,18 @@ Valid `<type>`'s include: all queried config options with the scope of that value (local, global, system, command). ---get-colorbool name [stdout-is-tty]:: +--get-colorbool <name> [<stdout-is-tty>]:: - Find the color setting for `name` (e.g. `color.diff`) and output - "true" or "false". `stdout-is-tty` should be either "true" or + Find the color setting for `<name>` (e.g. `color.diff`) and output + "true" or "false". `<stdout-is-tty>` should be either "true" or "false", and is taken into account when configuration says - "auto". If `stdout-is-tty` is missing, then checks the standard + "auto". If `<stdout-is-tty>` is missing, then checks the standard output of the command itself, and exits with status 0 if color is to be used, or exits with status 1 otherwise. When the color setting for `name` is undefined, the command uses `color.ui` as fallback. ---get-color name [default]:: +--get-color <name> [<default>]:: Find the color configured for `name` (e.g. `color.diff.new`) and output it as the ANSI color escape sequence to the standard @@ -281,7 +281,7 @@ Valid `<type>`'s include: --default <value>:: When using `--get`, and the requested variable is not found, behave as if - <value> were the value assigned to the that variable. + `<value>` were the value assigned to the that variable. CONFIGURATION ------------- diff --git a/Documentation/git-credential.txt b/Documentation/git-credential.txt index 206e3c5f40..f18673017f 100644 --- a/Documentation/git-credential.txt +++ b/Documentation/git-credential.txt @@ -8,7 +8,7 @@ git-credential - Retrieve and store user credentials SYNOPSIS -------- ------------------ -git credential <fill|approve|reject> +'git credential' (fill|approve|reject) ------------------ DESCRIPTION diff --git a/Documentation/git-cvsexportcommit.txt b/Documentation/git-cvsexportcommit.txt index 00154b6c85..41c8a8a05c 100644 --- a/Documentation/git-cvsexportcommit.txt +++ b/Documentation/git-cvsexportcommit.txt @@ -9,8 +9,8 @@ git-cvsexportcommit - Export a single commit to a CVS checkout SYNOPSIS -------- [verse] -'git cvsexportcommit' [-h] [-u] [-v] [-c] [-P] [-p] [-a] [-d cvsroot] - [-w cvsworkdir] [-W] [-f] [-m msgprefix] [PARENTCOMMIT] COMMITID +'git cvsexportcommit' [-h] [-u] [-v] [-c] [-P] [-p] [-a] [-d <cvsroot>] + [-w <cvs-workdir>] [-W] [-f] [-m <msgprefix>] [<parent-commit>] <commit-id> DESCRIPTION diff --git a/Documentation/git-cvsimport.txt b/Documentation/git-cvsimport.txt index de1ebed67d..b3f27671a0 100644 --- a/Documentation/git-cvsimport.txt +++ b/Documentation/git-cvsimport.txt @@ -11,9 +11,9 @@ SYNOPSIS [verse] 'git cvsimport' [-o <branch-for-HEAD>] [-h] [-v] [-d <CVSROOT>] [-A <author-conv-file>] [-p <options-for-cvsps>] [-P <file>] - [-C <git_repository>] [-z <fuzz>] [-i] [-k] [-u] [-s <subst>] - [-a] [-m] [-M <regex>] [-S <regex>] [-L <commitlimit>] - [-r <remote>] [-R] [<CVS_module>] + [-C <git-repository>] [-z <fuzz>] [-i] [-k] [-u] [-s <subst>] + [-a] [-m] [-M <regex>] [-S <regex>] [-L <commit-limit>] + [-r <remote>] [-R] [<CVS-module>] DESCRIPTION @@ -59,7 +59,7 @@ OPTIONS from `CVS/Root`. If no such file exists, it checks for the `CVSROOT` environment variable. -<CVS_module>:: +<CVS-module>:: The CVS module you want to import. Relative to <CVSROOT>. If not given, 'git cvsimport' tries to read it from `CVS/Repository`. diff --git a/Documentation/git-diff-files.txt b/Documentation/git-diff-files.txt index 906774f0f7..bf1febb9ae 100644 --- a/Documentation/git-diff-files.txt +++ b/Documentation/git-diff-files.txt @@ -9,7 +9,7 @@ git-diff-files - Compares files in the working tree and the index SYNOPSIS -------- [verse] -'git diff-files' [-q] [-0|-1|-2|-3|-c|--cc] [<common diff options>] [<path>...] +'git diff-files' [-q] [-0|-1|-2|-3|-c|--cc] [<common-diff-options>] [<path>...] DESCRIPTION ----------- diff --git a/Documentation/git-diff-index.txt b/Documentation/git-diff-index.txt index 27acb31cbf..679cae27d9 100644 --- a/Documentation/git-diff-index.txt +++ b/Documentation/git-diff-index.txt @@ -9,7 +9,7 @@ git-diff-index - Compare a tree to the working tree or index SYNOPSIS -------- [verse] -'git diff-index' [-m] [--cached] [--merge-base] [<common diff options>] <tree-ish> [<path>...] +'git diff-index' [-m] [--cached] [--merge-base] [<common-diff-options>] <tree-ish> [<path>...] DESCRIPTION ----------- diff --git a/Documentation/git-diff-tree.txt b/Documentation/git-diff-tree.txt index 2fc24c542f..274d5eaba9 100644 --- a/Documentation/git-diff-tree.txt +++ b/Documentation/git-diff-tree.txt @@ -11,7 +11,7 @@ SYNOPSIS [verse] 'git diff-tree' [--stdin] [-m] [-s] [-v] [--no-commit-id] [--pretty] [-t] [-r] [-c | --cc] [--combined-all-paths] [--root] [--merge-base] - [<common diff options>] <tree-ish> [<tree-ish>] [<path>...] + [<common-diff-options>] <tree-ish> [<tree-ish>] [<path>...] DESCRIPTION ----------- diff --git a/Documentation/git-format-patch.txt b/Documentation/git-format-patch.txt index fe2f69d36e..113eabc107 100644 --- a/Documentation/git-format-patch.txt +++ b/Documentation/git-format-patch.txt @@ -689,10 +689,10 @@ You can also use `git format-patch --base=P -3 C` to generate patches for A, B and C, and the identifiers for P, X, Y, Z are appended at the end of the first message. -If set `--base=auto` in cmdline, it will track base commit automatically, -the base commit will be the merge base of tip commit of the remote-tracking +If set `--base=auto` in cmdline, it will automatically compute +the base commit as the merge base of tip commit of the remote-tracking branch and revision-range specified in cmdline. -For a local branch, you need to track a remote branch by `git branch +For a local branch, you need to make it to track a remote branch by `git branch --set-upstream-to` before using this option. EXAMPLES diff --git a/Documentation/git-fsck.txt b/Documentation/git-fsck.txt index bd596619c0..5088783dcc 100644 --- a/Documentation/git-fsck.txt +++ b/Documentation/git-fsck.txt @@ -12,7 +12,7 @@ SYNOPSIS 'git fsck' [--tags] [--root] [--unreachable] [--cache] [--no-reflogs] [--[no-]full] [--strict] [--verbose] [--lost-found] [--[no-]dangling] [--[no-]progress] [--connectivity-only] - [--[no-]name-objects] [<object>*] + [--[no-]name-objects] [<object>...] DESCRIPTION ----------- diff --git a/Documentation/git-fsmonitor--daemon.txt b/Documentation/git-fsmonitor--daemon.txt new file mode 100644 index 0000000000..154e7684da --- /dev/null +++ b/Documentation/git-fsmonitor--daemon.txt @@ -0,0 +1,75 @@ +git-fsmonitor--daemon(1) +======================== + +NAME +---- +git-fsmonitor--daemon - A Built-in File System Monitor + +SYNOPSIS +-------- +[verse] +'git fsmonitor--daemon' start +'git fsmonitor--daemon' run +'git fsmonitor--daemon' stop +'git fsmonitor--daemon' status + +DESCRIPTION +----------- + +A daemon to watch the working directory for file and directory +changes using platform-specific file system notification facilities. + +This daemon communicates directly with commands like `git status` +using the link:technical/api-simple-ipc.html[simple IPC] interface +instead of the slower linkgit:githooks[5] interface. + +This daemon is built into Git so that no third-party tools are +required. + +OPTIONS +------- + +start:: + Starts a daemon in the background. + +run:: + Runs a daemon in the foreground. + +stop:: + Stops the daemon running in the current working + directory, if present. + +status:: + Exits with zero status if a daemon is watching the + current working directory. + +REMARKS +------- + +This daemon is a long running process used to watch a single working +directory and maintain a list of the recently changed files and +directories. Performance of commands such as `git status` can be +increased if they just ask for a summary of changes to the working +directory and can avoid scanning the disk. + +When `core.useBuiltinFSMonitor` is set to `true` (see +linkgit:git-config[1]) commands, such as `git status`, will ask the +daemon for changes and automatically start it (if necessary). + +For more information see the "File System Monitor" section in +linkgit:git-update-index[1]. + +CAVEATS +------- + +The fsmonitor daemon does not currently know about submodules and does +not know to filter out file system events that happen within a +submodule. If fsmonitor daemon is watching a super repo and a file is +modified within the working directory of a submodule, it will report +the change (as happening against the super repo). However, the client +will properly ignore these extra events, so performance may be affected +but it will not cause an incorrect result. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/git-gui.txt b/Documentation/git-gui.txt index c9d7e96214..e8f3ccb433 100644 --- a/Documentation/git-gui.txt +++ b/Documentation/git-gui.txt @@ -8,7 +8,7 @@ git-gui - A portable graphical interface to Git SYNOPSIS -------- [verse] -'git gui' [<command>] [arguments] +'git gui' [<command>] [<arguments>] DESCRIPTION ----------- diff --git a/Documentation/git-help.txt b/Documentation/git-help.txt index 96d5f598b4..44ea63cc6d 100644 --- a/Documentation/git-help.txt +++ b/Documentation/git-help.txt @@ -9,14 +9,14 @@ SYNOPSIS -------- [verse] 'git help' [-a|--all [--[no-]verbose]] - [[-i|--info] [-m|--man] [-w|--web]] [COMMAND|GUIDE] + [[-i|--info] [-m|--man] [-w|--web]] [<command>|<guide>] 'git help' [-g|--guides] 'git help' [-c|--config] DESCRIPTION ----------- -With no options and no COMMAND or GUIDE given, the synopsis of the 'git' +With no options and no '<command>' or '<guide>' given, the synopsis of the 'git' command and a list of the most commonly used Git commands are printed on the standard output. @@ -33,7 +33,7 @@ variables. If an alias is given, git shows the definition of the alias on standard output. To get the manual page for the aliased command, use -`git COMMAND --help`. +`git <command> --help`. Note that `git --help ...` is identical to `git help ...` because the former is internally converted into the latter. diff --git a/Documentation/git-hook.txt b/Documentation/git-hook.txt new file mode 100644 index 0000000000..77c3a8ad90 --- /dev/null +++ b/Documentation/git-hook.txt @@ -0,0 +1,45 @@ +git-hook(1) +=========== + +NAME +---- +git-hook - Run git hooks + +SYNOPSIS +-------- +[verse] +'git hook' run [--ignore-missing] <hook-name> [-- <hook-args>] + +DESCRIPTION +----------- + +A command interface to running git hooks (see linkgit:githooks[5]), +for use by other scripted git commands. + +SUBCOMMANDS +----------- + +run:: + Run the `<hook-name>` hook. See linkgit:githooks[5] for + supported hook names. ++ + +Any positional arguments to the hook should be passed after a +mandatory `--` (or `--end-of-options`, see linkgit:gitcli[7]). See +linkgit:githooks[5] for arguments hooks might expect (if any). + +OPTIONS +------- + +--ignore-missing:: + Ignore any missing hook by quietly returning zero. Used for + tools that want to do a blind one-shot run of a hook that may + or may not be present. + +SEE ALSO +-------- +linkgit:githooks[5] + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/git-http-fetch.txt b/Documentation/git-http-fetch.txt index 9fa17b60e4..319062c021 100644 --- a/Documentation/git-http-fetch.txt +++ b/Documentation/git-http-fetch.txt @@ -9,7 +9,7 @@ git-http-fetch - Download from a remote Git repository via HTTP SYNOPSIS -------- [verse] -'git http-fetch' [-c] [-t] [-a] [-d] [-v] [-w filename] [--recover] [--stdin | --packfile=<hash> | <commit>] <url> +'git http-fetch' [-c] [-t] [-a] [-d] [-v] [-w <filename>] [--recover] [--stdin | --packfile=<hash> | <commit>] <URL> DESCRIPTION ----------- diff --git a/Documentation/git-http-push.txt b/Documentation/git-http-push.txt index ea03a4eeb0..7c6a6dd7f6 100644 --- a/Documentation/git-http-push.txt +++ b/Documentation/git-http-push.txt @@ -9,7 +9,7 @@ git-http-push - Push objects over HTTP/DAV to another repository SYNOPSIS -------- [verse] -'git http-push' [--all] [--dry-run] [--force] [--verbose] <url> <ref> [<ref>...] +'git http-push' [--all] [--dry-run] [--force] [--verbose] <URL> <ref> [<ref>...] DESCRIPTION ----------- @@ -63,16 +63,15 @@ of such patterns separated by a colon ":" (this means that a ref name cannot have a colon in it). A single pattern '<name>' is just a shorthand for '<name>:<name>'. -Each pattern pair consists of the source side (before the colon) -and the destination side (after the colon). The ref to be -pushed is determined by finding a match that matches the source -side, and where it is pushed is determined by using the -destination side. +Each pattern pair '<src>:<dst>' consists of the source side (before +the colon) and the destination side (after the colon). The ref to be +pushed is determined by finding a match that matches the source side, +and where it is pushed is determined by using the destination side. - - It is an error if <src> does not match exactly one of the + - It is an error if '<src>' does not match exactly one of the local refs. - - If <dst> does not match any remote ref, either + - If '<dst>' does not match any remote ref, either * it has to start with "refs/"; <dst> is used as the destination literally in this case. diff --git a/Documentation/git-init-db.txt b/Documentation/git-init-db.txt index 648a6cd78a..18bf1a3c8c 100644 --- a/Documentation/git-init-db.txt +++ b/Documentation/git-init-db.txt @@ -9,7 +9,7 @@ git-init-db - Creates an empty Git repository SYNOPSIS -------- [verse] -'git init-db' [-q | --quiet] [--bare] [--template=<template_directory>] [--separate-git-dir <git dir>] [--shared[=<permissions>]] +'git init-db' [-q | --quiet] [--bare] [--template=<template-directory>] [--separate-git-dir <git-dir>] [--shared[=<permissions>]] DESCRIPTION diff --git a/Documentation/git-init.txt b/Documentation/git-init.txt index b611d80697..42268ada22 100644 --- a/Documentation/git-init.txt +++ b/Documentation/git-init.txt @@ -9,10 +9,10 @@ git-init - Create an empty Git repository or reinitialize an existing one SYNOPSIS -------- [verse] -'git init' [-q | --quiet] [--bare] [--template=<template_directory>] - [--separate-git-dir <git dir>] [--object-format=<format>] +'git init' [-q | --quiet] [--bare] [--template=<template-directory>] + [--separate-git-dir <git-dir>] [--object-format=<format>] [-b <branch-name> | --initial-branch=<branch-name>] - [--shared[=<permissions>]] [directory] + [--shared[=<permissions>]] [<directory>] DESCRIPTION @@ -57,12 +57,12 @@ values are 'sha1' and (if enabled) 'sha256'. 'sha1' is the default. + include::object-format-disclaimer.txt[] ---template=<template_directory>:: +--template=<template-directory>:: Specify the directory from which templates will be used. (See the "TEMPLATE DIRECTORY" section below.) ---separate-git-dir=<git dir>:: +--separate-git-dir=<git-dir>:: Instead of initializing the repository as a directory to either `$GIT_DIR` or `./.git/`, create a text file there containing the path to the actual @@ -79,7 +79,7 @@ repository. If not specified, fall back to the default name (currently `master`, but this is subject to change in the future; the name can be customized via the `init.defaultBranch` configuration variable). ---shared[=(false|true|umask|group|all|world|everybody|0xxx)]:: +--shared[=(false|true|umask|group|all|world|everybody|<perm>)]:: Specify that the Git repository is to be shared amongst several users. This allows users belonging to the same group to push into that @@ -110,13 +110,16 @@ the repository permissions. Same as 'group', but make the repository readable by all users. -'0xxx':: +'<perm>':: -'0xxx' is an octal number and each file will have mode '0xxx'. '0xxx' will -override users' umask(2) value (and not only loosen permissions as 'group' and -'all' does). '0640' will create a repository which is group-readable, but not -group-writable or accessible to others. '0660' will create a repo that is -readable and writable to the current user and group, but inaccessible to others. +'<perm>' is an 3-digit octal number prefixed with `0` and each file +will have mode '<perm>'. '<perm>' will override users' umask(2) +value (and not only loosen permissions as 'group' and 'all' +does). '0640' will create a repository which is group-readable, but +not group-writable or accessible to others. '0660' will create a repo +that is readable and writable to the current user and group, but +inaccessible to others (directories and executable files get their +`x` bit from the `r` bit for corresponding classes of uses). -- By default, the configuration flag `receive.denyNonFastForwards` is enabled diff --git a/Documentation/git-log.txt b/Documentation/git-log.txt index 0498e7bacb..20e87cecf4 100644 --- a/Documentation/git-log.txt +++ b/Documentation/git-log.txt @@ -9,7 +9,7 @@ git-log - Show commit logs SYNOPSIS -------- [verse] -'git log' [<options>] [<revision range>] [[--] <path>...] +'git log' [<options>] [<revision-range>] [[--] <path>...] DESCRIPTION ----------- @@ -81,13 +81,13 @@ produced by `--stat`, etc. include::line-range-options.txt[] -<revision range>:: +<revision-range>:: Show only commits in the specified revision range. When no - <revision range> is specified, it defaults to `HEAD` (i.e. the + <revision-range> is specified, it defaults to `HEAD` (i.e. the whole history leading to the current commit). `origin..HEAD` specifies all the commits reachable from the current commit (i.e. `HEAD`), but not from `origin`. For a complete list of - ways to spell <revision range>, see the 'Specifying Ranges' + ways to spell <revision-range>, see the 'Specifying Ranges' section of linkgit:gitrevisions[7]. [--] <path>...:: diff --git a/Documentation/git-ls-files.txt b/Documentation/git-ls-files.txt index 6d11ab506b..2e3d695fa2 100644 --- a/Documentation/git-ls-files.txt +++ b/Documentation/git-ls-files.txt @@ -10,9 +10,9 @@ SYNOPSIS -------- [verse] 'git ls-files' [-z] [-t] [-v] [-f] - (--[cached|deleted|others|ignored|stage|unmerged|killed|modified])* - (-[c|d|o|i|s|u|k|m])* - [--eol] + [-c|--cached] [-d|--deleted] [-o|--others] [-i|--|ignored] + [-s|--stage] [-u|--unmerged] [-k|--|killed] [-m|--modified] + [--directory [--no-empty-directory]] [--eol] [--deduplicate] [-x <pattern>|--exclude=<pattern>] [-X <file>|--exclude-from=<file>] diff --git a/Documentation/git-merge-file.txt b/Documentation/git-merge-file.txt index f856032613..7e9093fab6 100644 --- a/Documentation/git-merge-file.txt +++ b/Documentation/git-merge-file.txt @@ -70,6 +70,9 @@ OPTIONS --diff3:: Show conflicts in "diff3" style. +--zdiff3:: + Show conflicts in "zdiff3" style. + --ours:: --theirs:: --union:: diff --git a/Documentation/git-merge-index.txt b/Documentation/git-merge-index.txt index 2ab84a91e5..eea56b3154 100644 --- a/Documentation/git-merge-index.txt +++ b/Documentation/git-merge-index.txt @@ -9,7 +9,7 @@ git-merge-index - Run a merge for files needing merging SYNOPSIS -------- [verse] -'git merge-index' [-o] [-q] <merge-program> (-a | [--] <file>*) +'git merge-index' [-o] [-q] <merge-program> (-a | ( [--] <file>...) ) DESCRIPTION ----------- diff --git a/Documentation/git-merge.txt b/Documentation/git-merge.txt index e4f3352eb5..e8cecf5a51 100644 --- a/Documentation/git-merge.txt +++ b/Documentation/git-merge.txt @@ -240,7 +240,8 @@ from the RCS suite to present such a conflicted hunk, like this: ------------ Here are lines that are either unchanged from the common -ancestor, or cleanly resolved because only one side changed. +ancestor, or cleanly resolved because only one side changed, +or cleanly resolved because both sides changed the same way. <<<<<<< yours:sample.txt Conflict resolution is hard; let's go shopping. @@ -261,16 +262,37 @@ side wants to say it is hard and you'd prefer to go shopping, while the other side wants to claim it is easy. An alternative style can be used by setting the "merge.conflictStyle" -configuration variable to "diff3". In "diff3" style, the above conflict -may look like this: +configuration variable to either "diff3" or "zdiff3". In "diff3" +style, the above conflict may look like this: ------------ Here are lines that are either unchanged from the common -ancestor, or cleanly resolved because only one side changed. +ancestor, or cleanly resolved because only one side changed, <<<<<<< yours:sample.txt +or cleanly resolved because both sides changed the same way. Conflict resolution is hard; let's go shopping. -||||||| +||||||| base:sample.txt +or cleanly resolved because both sides changed identically. +Conflict resolution is hard. +======= +or cleanly resolved because both sides changed the same way. +Git makes conflict resolution easy. +>>>>>>> theirs:sample.txt +And here is another line that is cleanly resolved or unmodified. +------------ + +while in "zdiff3" style, it may look like this: + +------------ +Here are lines that are either unchanged from the common +ancestor, or cleanly resolved because only one side changed, +or cleanly resolved because both sides changed the same way. +<<<<<<< yours:sample.txt +Conflict resolution is hard; +let's go shopping. +||||||| base:sample.txt +or cleanly resolved because both sides changed identically. Conflict resolution is hard. ======= Git makes conflict resolution easy. diff --git a/Documentation/git-multi-pack-index.txt b/Documentation/git-multi-pack-index.txt index b008ce2850..c588fb91af 100644 --- a/Documentation/git-multi-pack-index.txt +++ b/Documentation/git-multi-pack-index.txt @@ -99,13 +99,13 @@ associated `.keep` file will not be selected for the batch to repack. EXAMPLES -------- -* Write a MIDX file for the packfiles in the current .git folder. +* Write a MIDX file for the packfiles in the current `.git` directory. + ----------------------------------------------- $ git multi-pack-index write ----------------------------------------------- -* Write a MIDX file for the packfiles in the current .git folder with a +* Write a MIDX file for the packfiles in the current `.git` directory with a corresponding bitmap. + ------------------------------------------------------------- @@ -118,7 +118,7 @@ $ git multi-pack-index write --preferred-pack=<pack> --bitmap $ git multi-pack-index --object-dir <alt> write ----------------------------------------------- -* Verify the MIDX file for the packfiles in the current .git folder. +* Verify the MIDX file for the packfiles in the current `.git` directory. + ----------------------------------------------- $ git multi-pack-index verify diff --git a/Documentation/git-p4.txt b/Documentation/git-p4.txt index 38e5257b2a..e21fcd8f71 100644 --- a/Documentation/git-p4.txt +++ b/Documentation/git-p4.txt @@ -9,10 +9,10 @@ git-p4 - Import from and submit to Perforce repositories SYNOPSIS -------- [verse] -'git p4 clone' [<sync options>] [<clone options>] <p4 depot path>... -'git p4 sync' [<sync options>] [<p4 depot path>...] +'git p4 clone' [<sync-options>] [<clone-options>] <p4-depot-path>... +'git p4 sync' [<sync-options>] [<p4-depot-path>...] 'git p4 rebase' -'git p4 submit' [<submit options>] [<master branch name>] +'git p4 submit' [<submit-options>] [<master-branch-name>] DESCRIPTION @@ -361,7 +361,7 @@ These options can be used to modify 'git p4 submit' behavior. p4/master. See the "Sync options" section above for more information. ---commit <sha1>|<sha1..sha1>:: +--commit (<sha1>|<sha1>..<sha1>):: Submit only the specified commit or range of commits, instead of the full list of changes that are in the current Git branch. diff --git a/Documentation/git-pack-objects.txt b/Documentation/git-pack-objects.txt index dbfd1f9017..f8344e1e5b 100644 --- a/Documentation/git-pack-objects.txt +++ b/Documentation/git-pack-objects.txt @@ -13,8 +13,8 @@ SYNOPSIS [--no-reuse-delta] [--delta-base-offset] [--non-empty] [--local] [--incremental] [--window=<n>] [--depth=<n>] [--revs [--unpacked | --all]] [--keep-pack=<pack-name>] - [--stdout [--filter=<filter-spec>] | base-name] - [--shallow] [--keep-true-parents] [--[no-]sparse] < object-list + [--stdout [--filter=<filter-spec>] | <base-name>] + [--shallow] [--keep-true-parents] [--[no-]sparse] < <object-list> DESCRIPTION diff --git a/Documentation/git-pack-redundant.txt b/Documentation/git-pack-redundant.txt index f2869da572..ee7034b5e5 100644 --- a/Documentation/git-pack-redundant.txt +++ b/Documentation/git-pack-redundant.txt @@ -9,7 +9,7 @@ git-pack-redundant - Find redundant pack files SYNOPSIS -------- [verse] -'git pack-redundant' [ --verbose ] [ --alt-odb ] < --all | .pack filename ... > +'git pack-redundant' [ --verbose ] [ --alt-odb ] ( --all | <pack-filename>... ) DESCRIPTION ----------- diff --git a/Documentation/git-rebase.txt b/Documentation/git-rebase.txt index a1af21fcef..9da4647061 100644 --- a/Documentation/git-rebase.txt +++ b/Documentation/git-rebase.txt @@ -714,9 +714,9 @@ information about the rebased commits and their parents (and instead generates new fake commits based off limited information in the generated patches), those commits cannot be identified; instead it has to fall back to a commit summary. Also, when merge.conflictStyle is -set to diff3, the apply backend will use "constructed merge base" to -label the content from the merge base, and thus provide no information -about the merge base commit whatsoever. +set to diff3 or zdiff3, the apply backend will use "constructed merge +base" to label the content from the merge base, and thus provide no +information about the merge base commit whatsoever. The merge backend works with the full commits on both sides of history and thus has no such limitations. diff --git a/Documentation/git-reflog.txt b/Documentation/git-reflog.txt index ff487ff77d..5ced7ad4f8 100644 --- a/Documentation/git-reflog.txt +++ b/Documentation/git-reflog.txt @@ -17,12 +17,12 @@ The command takes various subcommands, and different options depending on the subcommand: [verse] -'git reflog' ['show'] [log-options] [<ref>] +'git reflog' ['show'] [<log-options>] [<ref>] 'git reflog expire' [--expire=<time>] [--expire-unreachable=<time>] [--rewrite] [--updateref] [--stale-fix] [--dry-run | -n] [--verbose] [--all [--single-worktree] | <refs>...] 'git reflog delete' [--rewrite] [--updateref] - [--dry-run | -n] [--verbose] ref@\{specifier\}... + [--dry-run | -n] [--verbose] <ref>@\{<specifier>\}... 'git reflog exists' <ref> Reference logs, or "reflogs", record when the tips of branches and diff --git a/Documentation/git-remote.txt b/Documentation/git-remote.txt index 31c29c9b31..2bebc32566 100644 --- a/Documentation/git-remote.txt +++ b/Documentation/git-remote.txt @@ -10,7 +10,7 @@ SYNOPSIS -------- [verse] 'git remote' [-v | --verbose] -'git remote add' [-t <branch>] [-m <master>] [-f] [--[no-]tags] [--mirror=(fetch|push)] <name> <url> +'git remote add' [-t <branch>] [-m <master>] [-f] [--[no-]tags] [--mirror=(fetch|push)] <name> <URL> 'git remote rename' <old> <new> 'git remote remove' <name> 'git remote set-head' <name> (-a | --auto | -d | --delete | <branch>) @@ -18,7 +18,7 @@ SYNOPSIS 'git remote get-url' [--push] [--all] <name> 'git remote set-url' [--push] <name> <newurl> [<oldurl>] 'git remote set-url --add' [--push] <name> <newurl> -'git remote set-url --delete' [--push] <name> <url> +'git remote set-url --delete' [--push] <name> <URL> 'git remote' [-v | --verbose] 'show' [-n] <name>... 'git remote prune' [-n | --dry-run] <name>... 'git remote' [-v | --verbose] 'update' [-p | --prune] [(<group> | <remote>)...] @@ -47,7 +47,7 @@ subcommands are available to perform operations on the remotes. 'add':: Add a remote named <name> for the repository at -<url>. The command `git fetch <name>` can then be used to create and +<URL>. The command `git fetch <name>` can then be used to create and update remote-tracking branches <name>/<branch>. + With `-f` option, `git fetch <name>` is run immediately after @@ -152,7 +152,7 @@ With `--push`, push URLs are manipulated instead of fetch URLs. With `--add`, instead of changing existing URLs, new URL is added. + With `--delete`, instead of changing existing URLs, all URLs matching -regex <url> are deleted for remote <name>. Trying to delete all +regex <URL> are deleted for remote <name>. Trying to delete all non-push URLs is an error. + Note that the push URL and the fetch URL, even though they can diff --git a/Documentation/git-request-pull.txt b/Documentation/git-request-pull.txt index 4d4392d0f8..fa5a426709 100644 --- a/Documentation/git-request-pull.txt +++ b/Documentation/git-request-pull.txt @@ -8,7 +8,7 @@ git-request-pull - Generates a summary of pending changes SYNOPSIS -------- [verse] -'git request-pull' [-p] <start> <url> [<end>] +'git request-pull' [-p] <start> <URL> [<end>] DESCRIPTION ----------- @@ -21,7 +21,7 @@ the changes and indicates from where they can be pulled. The upstream project is expected to have the commit named by `<start>` and the output asks it to integrate the changes you made since that commit, up to the commit named by `<end>`, by visiting -the repository named by `<url>`. +the repository named by `<URL>`. OPTIONS @@ -33,14 +33,14 @@ OPTIONS Commit to start at. This names a commit that is already in the upstream history. -<url>:: +<URL>:: The repository URL to be pulled from. <end>:: Commit to end at (defaults to HEAD). This names the commit at the tip of the history you are asking to be pulled. + -When the repository named by `<url>` has the commit at a tip of a +When the repository named by `<URL>` has the commit at a tip of a ref that is different from the ref you have locally, you can use the `<local>:<remote>` syntax, to have its local name, a colon `:`, and its remote name. diff --git a/Documentation/git-restore.txt b/Documentation/git-restore.txt index 55bde91ef9..5964810caa 100644 --- a/Documentation/git-restore.txt +++ b/Documentation/git-restore.txt @@ -92,8 +92,7 @@ in linkgit:git-checkout[1] for details. The same as `--merge` option above, but changes the way the conflicting hunks are presented, overriding the `merge.conflictStyle` configuration variable. Possible values - are "merge" (default) and "diff3" (in addition to what is - shown by "merge" style, shows the original contents). + are "merge" (default), "diff3", and "zdiff3". --ignore-unmerged:: When restoring files on the working tree from the index, do diff --git a/Documentation/git-send-email.txt b/Documentation/git-send-email.txt index 3db4eab4ba..41cd8cb424 100644 --- a/Documentation/git-send-email.txt +++ b/Documentation/git-send-email.txt @@ -9,7 +9,8 @@ git-send-email - Send a collection of patches as emails SYNOPSIS -------- [verse] -'git send-email' [<options>] <file|directory|rev-list options>... +'git send-email' [<options>] <file|directory>... +'git send-email' [<options>] <format-patch options> 'git send-email' --dump-aliases @@ -19,7 +20,8 @@ Takes the patches given on the command line and emails them out. Patches can be specified as files, directories (which will send all files in the directory), or directly as a revision list. In the last case, any format accepted by linkgit:git-format-patch[1] can -be passed to git send-email. +be passed to git send-email, as well as options understood by +linkgit:git-format-patch[1]. The header of the email is configurable via command-line options. If not specified on the command line, the user will be prompted with a ReadLine diff --git a/Documentation/git-shortlog.txt b/Documentation/git-shortlog.txt index c9c7f3065c..f64e77047b 100644 --- a/Documentation/git-shortlog.txt +++ b/Documentation/git-shortlog.txt @@ -8,7 +8,7 @@ git-shortlog - Summarize 'git log' output SYNOPSIS -------- [verse] -'git shortlog' [<options>] [<revision range>] [[--] <path>...] +'git shortlog' [<options>] [<revision-range>] [[--] <path>...] git log --pretty=short | 'git shortlog' [<options>] DESCRIPTION @@ -89,13 +89,13 @@ counts both authors and co-authors. If width is `0` (zero) then indent the lines of the output without wrapping them. -<revision range>:: +<revision-range>:: Show only commits in the specified revision range. When no - <revision range> is specified, it defaults to `HEAD` (i.e. the + <revision-range> is specified, it defaults to `HEAD` (i.e. the whole history leading to the current commit). `origin..HEAD` specifies all the commits reachable from the current commit (i.e. `HEAD`), but not from `origin`. For a complete list of - ways to spell <revision range>, see the "Specifying Ranges" + ways to spell <revision-range>, see the "Specifying Ranges" section of linkgit:gitrevisions[7]. [--] <path>...:: diff --git a/Documentation/git-sparse-checkout.txt b/Documentation/git-sparse-checkout.txt index 42056ee9ff..9a78dd721e 100644 --- a/Documentation/git-sparse-checkout.txt +++ b/Documentation/git-sparse-checkout.txt @@ -11,7 +11,7 @@ given by a list of patterns. SYNOPSIS -------- [verse] -'git sparse-checkout <subcommand> [options]' +'git sparse-checkout <subcommand> [<options>]' DESCRIPTION diff --git a/Documentation/git-stage.txt b/Documentation/git-stage.txt index 25bcda936d..2f6aaa75b9 100644 --- a/Documentation/git-stage.txt +++ b/Documentation/git-stage.txt @@ -9,7 +9,7 @@ git-stage - Add file contents to the staging area SYNOPSIS -------- [verse] -'git stage' args... +'git stage' <arg>... DESCRIPTION diff --git a/Documentation/git-stash.txt b/Documentation/git-stash.txt index be6084ccef..6e15f47525 100644 --- a/Documentation/git-stash.txt +++ b/Documentation/git-stash.txt @@ -13,7 +13,7 @@ SYNOPSIS 'git stash' drop [-q|--quiet] [<stash>] 'git stash' ( pop | apply ) [--index] [-q|--quiet] [<stash>] 'git stash' branch <branchname> [<stash>] -'git stash' [push [-p|--patch] [-k|--[no-]keep-index] [-q|--quiet] +'git stash' [push [-p|--patch] [-S|--staged] [-k|--[no-]keep-index] [-q|--quiet] [-u|--include-untracked] [-a|--all] [-m|--message <message>] [--pathspec-from-file=<file> [--pathspec-file-nul]] [--] [<pathspec>...]] @@ -47,7 +47,7 @@ stash index (e.g. the integer `n` is equivalent to `stash@{n}`). COMMANDS -------- -push [-p|--patch] [-k|--[no-]keep-index] [-u|--include-untracked] [-a|--all] [-q|--quiet] [-m|--message <message>] [--pathspec-from-file=<file> [--pathspec-file-nul]] [--] [<pathspec>...]:: +push [-p|--patch] [-S|--staged] [-k|--[no-]keep-index] [-u|--include-untracked] [-a|--all] [-q|--quiet] [-m|--message <message>] [--pathspec-from-file=<file> [--pathspec-file-nul]] [--] [<pathspec>...]:: Save your local modifications to a new 'stash entry' and roll them back to HEAD (in the working tree and in the index). @@ -60,7 +60,7 @@ subcommand from making an unwanted stash entry. The two exceptions to this are `stash -p` which acts as alias for `stash push -p` and pathspec elements, which are allowed after a double hyphen `--` for disambiguation. -save [-p|--patch] [-k|--[no-]keep-index] [-u|--include-untracked] [-a|--all] [-q|--quiet] [<message>]:: +save [-p|--patch] [-S|--staged] [-k|--[no-]keep-index] [-u|--include-untracked] [-a|--all] [-q|--quiet] [<message>]:: This option is deprecated in favour of 'git stash push'. It differs from "stash push" in that it cannot take pathspec. @@ -205,6 +205,16 @@ to learn how to operate the `--patch` mode. The `--patch` option implies `--keep-index`. You can use `--no-keep-index` to override this. +-S:: +--staged:: + This option is only valid for `push` and `save` commands. ++ +Stash only the changes that are currently staged. This is similar to +basic `git commit` except the state is committed to the stash instead +of current branch. ++ +The `--patch` option has priority over this one. + --pathspec-from-file=<file>:: This option is only valid for `push` command. + @@ -341,6 +351,24 @@ $ edit/build/test remaining parts $ git commit foo -m 'Remaining parts' ---------------------------------------------------------------- +Saving unrelated changes for future use:: + +When you are in the middle of massive changes and you find some +unrelated issue that you don't want to forget to fix, you can do the +change(s), stage them, and use `git stash push --staged` to stash them +out for future use. This is similar to committing the staged changes, +only the commit ends-up being in the stash and not on the current branch. ++ +---------------------------------------------------------------- +# ... hack hack hack ... +$ git add --patch foo # add unrelated changes to the index +$ git stash push --staged # save these changes to the stash +# ... hack hack hack, finish curent changes ... +$ git commit -m 'Massive' # commit fully tested changes +$ git switch fixup-branch # switch to another branch +$ git stash pop # to finish work on the saved changes +---------------------------------------------------------------- + Recovering stash entries that were cleared/dropped erroneously:: If you mistakenly drop or clear stash entries, they cannot be recovered diff --git a/Documentation/git-status.txt b/Documentation/git-status.txt index 4a2c3e0408..54a4b29b47 100644 --- a/Documentation/git-status.txt +++ b/Documentation/git-status.txt @@ -314,6 +314,14 @@ Line Notes ------------------------------------------------------------ .... +Stash Information +^^^^^^^^^^^^^^^^^ + +If `--show-stash` is given, one line is printed showing the number of stash +entries if non-zero: + + # stash <N> + Changed Tracked Entries ^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Documentation/git-svn.txt b/Documentation/git-svn.txt index 222b556d7a..4e92308e85 100644 --- a/Documentation/git-svn.txt +++ b/Documentation/git-svn.txt @@ -575,7 +575,7 @@ OPTIONS ------- --shared[=(false|true|umask|group|all|world|everybody)]:: ---template=<template_directory>:: +--template=<template-directory>:: Only used with the 'init' command. These are passed directly to 'git init'. diff --git a/Documentation/git-switch.txt b/Documentation/git-switch.txt index 5c438cd505..bbcbdceb45 100644 --- a/Documentation/git-switch.txt +++ b/Documentation/git-switch.txt @@ -137,8 +137,7 @@ should result in deletion of the path). The same as `--merge` option above, but changes the way the conflicting hunks are presented, overriding the `merge.conflictStyle` configuration variable. Possible values are - "merge" (default) and "diff3" (in addition to what is shown by - "merge" style, shows the original contents). + "merge" (default), "diff3", and "zdiff3". -q:: --quiet:: @@ -152,7 +151,7 @@ should result in deletion of the path). attached to a terminal, regardless of `--quiet`. -t:: ---track:: +--track [direct|inherit]:: When creating a new branch, set up "upstream" configuration. `-c` is implied. See `--track` in linkgit:git-branch[1] for details. diff --git a/Documentation/git-update-index.txt b/Documentation/git-update-index.txt index 2853f168d9..c7c31b3fcf 100644 --- a/Documentation/git-update-index.txt +++ b/Documentation/git-update-index.txt @@ -498,7 +498,9 @@ FILE SYSTEM MONITOR This feature is intended to speed up git operations for repos that have large working directories. -It enables git to work together with a file system monitor (see the +It enables git to work together with a file system monitor (see +linkgit:git-fsmonitor--daemon[1] +and the "fsmonitor-watchman" section of linkgit:githooks[5]) that can inform it as to what files have been modified. This enables git to avoid having to lstat() every file to find modified files. @@ -508,17 +510,18 @@ performance by avoiding the cost of scanning the entire working directory looking for new files. If you want to enable (or disable) this feature, it is easier to use -the `core.fsmonitor` configuration variable (see -linkgit:git-config[1]) than using the `--fsmonitor` option to -`git update-index` in each repository, especially if you want to do so -across all repositories you use, because you can set the configuration -variable in your `$HOME/.gitconfig` just once and have it affect all -repositories you touch. - -When the `core.fsmonitor` configuration variable is changed, the -file system monitor is added to or removed from the index the next time -a command reads the index. When `--[no-]fsmonitor` are used, the file -system monitor is immediately added to or removed from the index. +the `core.fsmonitor` or `core.useBuiltinFSMonitor` configuration +variable (see linkgit:git-config[1]) than using the `--fsmonitor` +option to `git update-index` in each repository, especially if you +want to do so across all repositories you use, because you can set the +configuration variable in your `$HOME/.gitconfig` just once and have +it affect all repositories you touch. + +When the `core.fsmonitor` or `core.useBuiltinFSMonitor` configuration +variable is changed, the file system monitor is added to or removed +from the index the next time a command reads the index. When +`--[no-]fsmonitor` are used, the file system monitor is immediately +added to or removed from the index. CONFIGURATION ------------- diff --git a/Documentation/git-web--browse.txt b/Documentation/git-web--browse.txt index 8d162b56c5..f2f996cbe1 100644 --- a/Documentation/git-web--browse.txt +++ b/Documentation/git-web--browse.txt @@ -8,7 +8,7 @@ git-web--browse - Git helper script to launch a web browser SYNOPSIS -------- [verse] -'git web{litdd}browse' [<options>] <url|file>... +'git web{litdd}browse' [<options>] (<URL>|<file>)... DESCRIPTION ----------- diff --git a/Documentation/git.txt b/Documentation/git.txt index d63c65e67d..13f83a2a3a 100644 --- a/Documentation/git.txt +++ b/Documentation/git.txt @@ -42,7 +42,7 @@ OPTIONS --version:: Prints the Git suite version that the 'git' program came from. + -This option is internaly converted to `git version ...` and accepts +This option is internally converted to `git version ...` and accepts the same options as the linkgit:git-version[1] command. If `--help` is also given, it takes precedence over `--version`. @@ -832,8 +832,9 @@ for full details. `GIT_TRACE_REDACT`:: By default, when tracing is activated, Git redacts the values of - cookies, the "Authorization:" header, and the "Proxy-Authorization:" - header. Set this variable to `0` to prevent this redaction. + cookies, the "Authorization:" header, the "Proxy-Authorization:" + header and packfile URIs. Set this variable to `0` to prevent this + redaction. `GIT_LITERAL_PATHSPECS`:: Setting this variable to `1` will cause Git to treat all diff --git a/Documentation/gitattributes.txt b/Documentation/gitattributes.txt index 83fd4e19a4..9e486f3e8d 100644 --- a/Documentation/gitattributes.txt +++ b/Documentation/gitattributes.txt @@ -382,6 +382,14 @@ sign `$` upon checkout. Any byte sequence that begins with `$Id:` and ends with `$` in the worktree file is replaced with `$Id$` upon check-in. +The `ident` attribute can also provide an optional value, +which if supplied is going to be used for expansion instead of +the string `Id`. + +------------------------ +*.[ch] ident=FreeBSD +------------------------ + `filter` ^^^^^^^^ diff --git a/Documentation/gitcredentials.txt b/Documentation/gitcredentials.txt index 758bf39ba3..80517b4eb2 100644 --- a/Documentation/gitcredentials.txt +++ b/Documentation/gitcredentials.txt @@ -132,7 +132,7 @@ because the hostnames differ. Nor would it match `foo.example.com`; Git compares hostnames exactly, without considering whether two hosts are part of the same domain. Likewise, a config entry for `http://example.com` would not match: Git compares the protocols exactly. However, you may use wildcards in -the domain name and other pattern matching techniques as with the `http.<url>.*` +the domain name and other pattern matching techniques as with the `http.<URL>.*` options. If the "pattern" URL does include a path component, then this too must match @@ -147,7 +147,7 @@ CONFIGURATION OPTIONS Options for a credential context can be configured either in `credential.*` (which applies to all credentials), or -`credential.<url>.*`, where <url> matches the context as described +`credential.<URL>.*`, where <URL> matches the context as described above. The following options are available in either location: diff --git a/Documentation/githooks.txt b/Documentation/githooks.txt index b51959ff94..e7ed1b9464 100644 --- a/Documentation/githooks.txt +++ b/Documentation/githooks.txt @@ -593,7 +593,8 @@ fsmonitor-watchman This hook is invoked when the configuration option `core.fsmonitor` is set to `.git/hooks/fsmonitor-watchman` or `.git/hooks/fsmonitor-watchmanv2` -depending on the version of the hook to use. +depending on the version of the hook to use, unless overridden via +`core.useBuiltinFSMonitor` (see linkgit:git-config[1]). Version 1 takes two arguments, a version (1) and the time in elapsed nanoseconds since midnight, January 1, 1970. @@ -698,6 +699,10 @@ and "0" meaning they were not. Only one parameter should be set to "1" when the hook runs. The hook running passing "1", "1" should not be possible. +SEE ALSO +-------- +linkgit:git-hook[1] + GIT --- Part of the linkgit:git[1] suite diff --git a/Documentation/gitignore.txt b/Documentation/gitignore.txt index f8a1fc2014..f2738b10db 100644 --- a/Documentation/gitignore.txt +++ b/Documentation/gitignore.txt @@ -155,7 +155,7 @@ accessed from the index or a tree versus from the filesystem. EXAMPLES -------- - - The pattern `hello.*` matches any file or folder + - The pattern `hello.*` matches any file or directory whose name begins with `hello.`. If one wants to restrict this only to the directory and not in its subdirectories, one can prepend the pattern with a slash, i.e. `/hello.*`; diff --git a/Documentation/gitsubmodules.txt b/Documentation/gitsubmodules.txt index 891c8da4fd..941858a6ec 100644 --- a/Documentation/gitsubmodules.txt +++ b/Documentation/gitsubmodules.txt @@ -226,7 +226,7 @@ Workflow for a third party library ---------------------------------- # Add a submodule - git submodule add <url> <path> + git submodule add <URL> <path> # Occasionally update the submodule to a new version: git -C <path> checkout <new version> diff --git a/Documentation/gitweb.txt b/Documentation/gitweb.txt index 3cc9b034c4..7cee9d3689 100644 --- a/Documentation/gitweb.txt +++ b/Documentation/gitweb.txt @@ -547,7 +547,7 @@ like this: # make the front page an internal rewrite to the gitweb script RewriteRule ^/$ /cgi-bin/gitweb.cgi [QSA,L,PT] - # look for a public_git folder in unix users' home + # look for a public_git directory in unix users' home # http://git.example.org/~<user>/ RewriteRule ^/\~([^\/]+)(/|/gitweb.cgi)?$ /cgi-bin/gitweb.cgi \ [QSA,E=GITWEB_PROJECTROOT:/home/$1/public_git/,L,PT] diff --git a/Documentation/gitworkflows.txt b/Documentation/gitworkflows.txt index 47cf97f9be..59305265c5 100644 --- a/Documentation/gitworkflows.txt +++ b/Documentation/gitworkflows.txt @@ -394,7 +394,7 @@ request to do so by mail. Such a request looks like ------------------------------------- Please pull from - <url> <branch> + <URL> <branch> ------------------------------------- In that case, 'git pull' can do the fetch and merge in one go, as @@ -403,7 +403,7 @@ follows. .Push/pull: Merging remote topics [caption="Recipe: "] ===================================== -`git pull <url> <branch>` +`git pull <URL> <branch>` ===================================== Occasionally, the maintainer may get merge conflicts when they try to @@ -440,7 +440,7 @@ merge because you cannot format-patch merges): .format-patch/am: Keeping topics up to date [caption="Recipe: "] ===================================== -`git pull --rebase <url> <branch>` +`git pull --rebase <URL> <branch>` ===================================== You can then fix the conflicts during the rebase. Presumably you have diff --git a/Documentation/lint-gitlink.perl b/Documentation/lint-gitlink.perl index b22a367844..1c61dd9512 100755 --- a/Documentation/lint-gitlink.perl +++ b/Documentation/lint-gitlink.perl @@ -5,11 +5,12 @@ use warnings; # Parse arguments, a simple state machine for input like: # -# howto/*.txt config/*.txt --section=1 git.txt git-add.txt [...] --to-lint git-add.txt a-file.txt [...] +# <file-to-check.txt> <valid-files-to-link-to> --section=1 git.txt git-add.txt [...] --to-lint git-add.txt a-file.txt [...] my %TXT; my %SECTION; my $section; my $lint_these = 0; +my $to_check = shift @ARGV; for my $arg (@ARGV) { if (my ($sec) = $arg =~ /^--section=(\d+)$/s) { $section = $sec; @@ -30,13 +31,14 @@ sub report { my ($pos, $line, $target, $msg) = @_; substr($line, $pos) = "' <-- HERE"; $line =~ s/^\s+//; - print "$ARGV:$.: error: $target: $msg, shown with 'HERE' below:\n"; - print "$ARGV:$.:\t'$line\n"; + print STDERR "$ARGV:$.: error: $target: $msg, shown with 'HERE' below:\n"; + print STDERR "$ARGV:$.:\t'$line\n"; $exit_code = 1; } @ARGV = sort values %TXT; -die "BUG: Nothing to process!" unless @ARGV; +die "BUG: No list of valid linkgit:* files given" unless @ARGV; +@ARGV = $to_check; while (<>) { my $line = $_; while ($line =~ m/linkgit:((.*?)\[(\d)\])/g) { diff --git a/Documentation/lint-man-end-blurb.perl b/Documentation/lint-man-end-blurb.perl index d69312e5db..6bdb13ad9f 100755 --- a/Documentation/lint-man-end-blurb.perl +++ b/Documentation/lint-man-end-blurb.perl @@ -6,7 +6,7 @@ use warnings; my $exit_code = 0; sub report { my ($target, $msg) = @_; - print "error: $target: $msg\n"; + print STDERR "error: $target: $msg\n"; $exit_code = 1; } diff --git a/Documentation/lint-man-section-order.perl b/Documentation/lint-man-section-order.perl index b05f9156dd..425377dfeb 100755 --- a/Documentation/lint-man-section-order.perl +++ b/Documentation/lint-man-section-order.perl @@ -46,7 +46,7 @@ my $SECTION_RX = do { my $exit_code = 0; sub report { my ($msg) = @_; - print "$ARGV:$.: $msg\n"; + print STDERR "$ARGV:$.: $msg\n"; $exit_code = 1; } diff --git a/Documentation/merge-options.txt b/Documentation/merge-options.txt index 61ec157c2f..d8f7cd7ca0 100644 --- a/Documentation/merge-options.txt +++ b/Documentation/merge-options.txt @@ -132,8 +132,9 @@ ifdef::git-pull[] Only useful when merging. endif::git-pull[] ---no-verify:: - This option bypasses the pre-merge and commit-msg hooks. +--[no-]verify:: + By default, the pre-merge and commit-msg hooks are run. + When `--no-verify` is given, these are bypassed. See also linkgit:githooks[5]. ifdef::git-pull[] Only useful when merging. diff --git a/Documentation/pretty-formats.txt b/Documentation/pretty-formats.txt index 317c1382b5..aebbebbce6 100644 --- a/Documentation/pretty-formats.txt +++ b/Documentation/pretty-formats.txt @@ -20,7 +20,7 @@ built-in formats: * 'oneline' - <hash> <title line> + <hash> <title-line> + This is designed to be as compact as possible. @@ -29,17 +29,17 @@ This is designed to be as compact as possible. commit <hash> Author: <author> - <title line> + <title-line> * 'medium' commit <hash> Author: <author> - Date: <author date> + Date: <author-date> - <title line> + <title-line> - <full commit message> + <full-commit-message> * 'full' @@ -47,25 +47,25 @@ This is designed to be as compact as possible. Author: <author> Commit: <committer> - <title line> + <title-line> - <full commit message> + <full-commit-message> * 'fuller' commit <hash> Author: <author> - AuthorDate: <author date> + AuthorDate: <author-date> Commit: <committer> - CommitDate: <committer date> + CommitDate: <committer-date> - <title line> + <title-line> - <full commit message> + <full-commit-message> * 'reference' - <abbrev hash> (<title line>, <short author date>) + <abbrev-hash> (<title-line>, <short-author-date>) + This format is used to refer to another commit in a commit message and is the same as `--pretty='format:%C(auto)%h (%s, %ad)'`. By default, @@ -78,10 +78,10 @@ placeholders, its output is not affected by other options like From <hash> <date> From: <author> - Date: <author date> - Subject: [PATCH] <title line> + Date: <author-date> + Subject: [PATCH] <title-line> - <full commit message> + <full-commit-message> * 'mboxrd' + @@ -101,9 +101,9 @@ commits are displayed, but not the way the diff is shown e.g. with `git log --raw`. To get full object names in a raw diff format, use `--no-abbrev`. -* 'format:<string>' +* 'format:<format-string>' + -The 'format:<string>' format allows you to specify which information +The 'format:<format-string>' format allows you to specify which information you want to show. It works a little bit like printf format, with the notable exception that you get a newline with '%n' instead of '\n'. @@ -220,8 +220,8 @@ The placeholders are: inconsistent when tags are added or removed at the same time. + -** 'tags[=<BOOL>]': Also consider lightweight tags. -** 'abbrev=<N>': Instead of using the default number of hexadecimal digits +** 'tags[=<bool-value>]': Also consider lightweight tags. +** 'abbrev=<number>': Instead of using the default number of hexadecimal digits (which will vary according to the number of objects in the repository with a default of 7) of the abbreviated object name, use <n> digits, or as many digits as needed to form a unique object name. @@ -278,7 +278,7 @@ endif::git-rev-list[] If any option is provided multiple times the last occurrence wins. + -** 'key=<K>': only show trailers with specified key. Matching is done +** 'key=<key>': only show trailers with specified <key>. Matching is done case-insensitively and trailing colon is optional. If option is given multiple times trailer lines matching any of the keys are shown. This option automatically enables the `only` option so that @@ -286,25 +286,25 @@ endif::git-rev-list[] desired it can be disabled with `only=false`. E.g., `%(trailers:key=Reviewed-by)` shows trailer lines with key `Reviewed-by`. -** 'only[=<BOOL>]': select whether non-trailer lines from the trailer +** 'only[=<bool-value>]': select whether non-trailer lines from the trailer block should be included. -** 'separator=<SEP>': specify a separator inserted between trailer +** 'separator=<sep>': specify a separator inserted between trailer lines. When this option is not given each trailer line is - terminated with a line feed character. The string SEP may contain + terminated with a line feed character. The string <sep> may contain the literal formatting codes described above. To use comma as separator one must use `%x2C` as it would otherwise be parsed as next option. E.g., `%(trailers:key=Ticket,separator=%x2C )` shows all trailer lines whose key is "Ticket" separated by a comma and a space. -** 'unfold[=<BOOL>]': make it behave as if interpret-trailer's `--unfold` +** 'unfold[=<bool-value>]': make it behave as if interpret-trailer's `--unfold` option was given. E.g., `%(trailers:only,unfold=true)` unfolds and shows all trailer lines. -** 'keyonly[=<BOOL>]': only show the key part of the trailer. -** 'valueonly[=<BOOL>]': only show the value part of the trailer. -** 'key_value_separator=<SEP>': specify a separator inserted between +** 'keyonly[=<bool-value>]': only show the key part of the trailer. +** 'valueonly[=<bool-value>]': only show the value part of the trailer. +** 'key_value_separator=<sep>': specify a separator inserted between trailer lines. When this option is not given each trailer key-value pair is separated by ": ". Otherwise it shares the same semantics - as 'separator=<SEP>' above. + as 'separator=<sep>' above. NOTE: Some placeholders may depend on other options given to the revision traversal engine. For example, the `%g*` reflog options will @@ -313,7 +313,7 @@ insert an empty string unless we are traversing reflog entries (e.g., by decoration format if `--decorate` was not already provided on the command line. -The boolean options accept an optional value `[=<BOOL>]`. The values +The boolean options accept an optional value `[=<bool-value>]`. The values `true`, `false`, `on`, `off` etc. are all accepted. See the "boolean" sub-section in "EXAMPLES" in linkgit:git-config[1]. If a boolean option is given with no value, it's enabled. diff --git a/Documentation/technical/rerere.txt b/Documentation/technical/rerere.txt index af5f9fc24f..35d4541433 100644 --- a/Documentation/technical/rerere.txt +++ b/Documentation/technical/rerere.txt @@ -14,9 +14,9 @@ conflicts before writing them to the rerere database. Different conflict styles and branch names are normalized by stripping the labels from the conflict markers, and removing the common ancestor -version from the `diff3` conflict style. Branches that are merged -in different order are normalized by sorting the conflict hunks. More -on each of those steps in the following sections. +version from the `diff3` or `zdiff3` conflict styles. Branches that +are merged in different order are normalized by sorting the conflict +hunks. More on each of those steps in the following sections. Once these two normalization operations are applied, a conflict ID is calculated based on the normalized conflict, which is later used by @@ -42,8 +42,8 @@ get a conflict like the following: >>>>>>> AC Doing the analogous with AC2 (forking a branch ABAC2 off of branch AB -and then merging branch AC2 into it), using the diff3 conflict style, -we get a conflict like the following: +and then merging branch AC2 into it), using the diff3 or zdiff3 +conflict style, we get a conflict like the following: <<<<<<< HEAD B diff --git a/Documentation/technical/signature-format.txt b/Documentation/technical/signature-format.txt index 2c9406a56a..166721be6f 100644 --- a/Documentation/technical/signature-format.txt +++ b/Documentation/technical/signature-format.txt @@ -13,6 +13,22 @@ Signatures always begin with `-----BEGIN PGP SIGNATURE-----` and end with `-----END PGP SIGNATURE-----`, unless gpg is told to produce RFC1991 signatures which use `MESSAGE` instead of `SIGNATURE`. +Signatures sometimes appear as a part of the normal payload +(e.g. a signed tag has the signature block appended after the payload +that the signature applies to), and sometimes appear in the value of +an object header (e.g. a merge commit that merged a signed tag would +have the entire tag contents on its "mergetag" header). In the case +of the latter, the usual multi-line formatting rule for object +headers applies. I.e. the second and subsequent lines are prefixed +with a SP to signal that the line is continued from the previous +line. + +This is even true for an originally empty line. In the following +examples, the end of line that ends with a whitespace letter is +highlighted with a `$` sign; if you are trying to recreate these +example by hand, do not cut and paste them---they are there +primarily to highlight extra whitespace at the end of some lines. + The signed payload and the way the signature is embedded depends on the type of the object resp. transaction. @@ -78,7 +94,7 @@ author A U Thor <author@example.com> 1465981137 +0000 committer C O Mitter <committer@example.com> 1465981137 +0000 gpgsig -----BEGIN PGP SIGNATURE----- Version: GnuPG v1 - + $ iQEcBAABAgAGBQJXYRjRAAoJEGEJLoW3InGJ3IwIAIY4SA6GxY3BjL60YyvsJPh/ HRCJwH+w7wt3Yc/9/bW2F+gF72kdHOOs2jfv+OZhq0q4OAN6fvVSczISY/82LpS7 DVdMQj2/YcHDT4xrDNBnXnviDO9G7am/9OE77kEbXrp7QPxvhjkicHNwy2rEflAA @@ -128,13 +144,13 @@ mergetag object 04b871796dc0420f8e7561a895b52484b701d51a type commit tag signedtag tagger C O Mitter <committer@example.com> 1465981006 +0000 - + $ signed tag - + $ signed tag message body -----BEGIN PGP SIGNATURE----- Version: GnuPG v1 - + $ iQEcBAABAgAGBQJXYRhOAAoJEGEJLoW3InGJklkIAIcnhL7RwEb/+QeX9enkXhxn rxfdqrvWd1K80sl2TOt8Bg/NYwrUBw/RWJ+sg/hhHp4WtvE1HDGHlkEz3y11Lkuh 8tSxS3qKTxXUGozyPGuE90sJfExhZlW4knIQ1wt/yWqM+33E9pN4hzPqLwyrdods diff --git a/Documentation/urls-remotes.txt b/Documentation/urls-remotes.txt index bd184cd653..86d0008f94 100644 --- a/Documentation/urls-remotes.txt +++ b/Documentation/urls-remotes.txt @@ -26,14 +26,14 @@ config file would appear like this: ------------ [remote "<name>"] - url = <url> + url = <URL> pushurl = <pushurl> push = <refspec> fetch = <refspec> ------------ The `<pushurl>` is used for pushes only. It is optional and defaults -to `<url>`. +to `<URL>`. Named file in `$GIT_DIR/remotes` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -67,10 +67,10 @@ This file should have the following format: ------------ - <url>#<head> + <URL>#<head> ------------ -`<url>` is required; `#<head>` is optional. +`<URL>` is required; `#<head>` is optional. Depending on the operation, git will use one of the following refspecs, if you don't provide one on the command line. @@ -256,6 +256,8 @@ all:: # # Define NO_DEFLATE_BOUND if your zlib does not have deflateBound. # +# Define NO_UNCOMPRESS2 if your zlib does not have uncompress2. +# # Define NO_NORETURN if using buggy versions of gcc 4.6+ and profile feedback, # as the compiler can crash (http://gcc.gnu.org/bugzilla/show_bug.cgi?id=49299) # @@ -305,9 +307,6 @@ all:: # # Define NO_TCLTK if you do not want Tcl/Tk GUI. # -# Define SANE_TEXT_GREP to "-a" if you use recent versions of GNU grep -# and egrep that are pickier when their input contains non-ASCII data. -# # The TCL_PATH variable governs the location of the Tcl interpreter # used to optimize git-gui for your system. Only used if NO_TCLTK # is not set. Defaults to the bare 'tclsh'. @@ -406,6 +405,8 @@ all:: # # Define HAVE_CLOCK_MONOTONIC if your platform has CLOCK_MONOTONIC. # +# Define HAVE_SYNC_FILE_RANGE if your platform has sync_file_range. +# # Define NEEDS_LIBRT if your platform requires linking with librt (glibc version # before 2.17) for clock_gettime and CLOCK_MONOTONIC. # @@ -465,6 +466,11 @@ all:: # directory, and the JSON compilation database 'compile_commands.json' will be # created at the root of the repository. # +# If your platform supports a built-in fsmonitor backend, set +# FSMONITOR_DAEMON_BACKEND to the "<name>" of the corresponding +# `compat/fsmonitor/fsm-listen-<name>.c` that implements the +# `fsm_listen__*()` routines. +# # Define DEVELOPER to enable more compiler warnings. Compiler version # and family are auto detected, but could be overridden by defining # COMPILER_FEATURES (see config.mak.dev). You can still set @@ -705,6 +711,7 @@ TEST_BUILTINS_OBJS += test-dump-split-index.o TEST_BUILTINS_OBJS += test-dump-untracked-cache.o TEST_BUILTINS_OBJS += test-example-decorate.o TEST_BUILTINS_OBJS += test-fast-rebase.o +TEST_BUILTINS_OBJS += test-fsmonitor-client.o TEST_BUILTINS_OBJS += test-genrandom.o TEST_BUILTINS_OBJS += test-genzeros.o TEST_BUILTINS_OBJS += test-getcwd.o @@ -735,6 +742,7 @@ TEST_BUILTINS_OBJS += test-read-cache.o TEST_BUILTINS_OBJS += test-read-graph.o TEST_BUILTINS_OBJS += test-read-midx.o TEST_BUILTINS_OBJS += test-ref-store.o +TEST_BUILTINS_OBJS += test-reftable.o TEST_BUILTINS_OBJS += test-regex.o TEST_BUILTINS_OBJS += test-repository.o TEST_BUILTINS_OBJS += test-revision-walking.o @@ -813,6 +821,8 @@ TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a XDIFF_LIB = xdiff/lib.a +REFTABLE_LIB = reftable/libreftable.a +REFTABLE_TEST_LIB = reftable/libreftable_test.a GENERATED_H += command-list.h GENERATED_H += config-list.h @@ -897,6 +907,8 @@ LIB_OBJS += fetch-pack.o LIB_OBJS += fmt-merge-msg.o LIB_OBJS += fsck.o LIB_OBJS += fsmonitor.o +LIB_OBJS += fsmonitor-ipc.o +LIB_OBJS += fsmonitor-settings.o LIB_OBJS += gettext.o LIB_OBJS += gpg-interface.o LIB_OBJS += graph.o @@ -1102,11 +1114,13 @@ BUILTIN_OBJS += builtin/fmt-merge-msg.o BUILTIN_OBJS += builtin/for-each-ref.o BUILTIN_OBJS += builtin/for-each-repo.o BUILTIN_OBJS += builtin/fsck.o +BUILTIN_OBJS += builtin/fsmonitor--daemon.o BUILTIN_OBJS += builtin/gc.o BUILTIN_OBJS += builtin/get-tar-commit-id.o BUILTIN_OBJS += builtin/grep.o BUILTIN_OBJS += builtin/hash-object.o BUILTIN_OBJS += builtin/help.o +BUILTIN_OBJS += builtin/hook.o BUILTIN_OBJS += builtin/index-pack.o BUILTIN_OBJS += builtin/init-db.o BUILTIN_OBJS += builtin/interpret-trailers.o @@ -1192,7 +1206,7 @@ THIRD_PARTY_SOURCES += compat/regex/% THIRD_PARTY_SOURCES += sha1collisiondetection/% THIRD_PARTY_SOURCES += sha1dc/% -GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) +GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) EXTLIBS = GIT_USER_AGENT = git/$(GIT_VERSION) @@ -1723,6 +1737,11 @@ ifdef NO_DEFLATE_BOUND BASIC_CFLAGS += -DNO_DEFLATE_BOUND endif +ifdef NO_UNCOMPRESS2 + BASIC_CFLAGS += -DNO_UNCOMPRESS2 + REFTABLE_OBJS += compat/zlib-uncompress2.o +endif + ifdef NO_POSIX_GOODIES BASIC_CFLAGS += -DNO_POSIX_GOODIES endif @@ -1884,6 +1903,10 @@ ifdef HAVE_CLOCK_MONOTONIC BASIC_CFLAGS += -DHAVE_CLOCK_MONOTONIC endif +ifdef HAVE_SYNC_FILE_RANGE + BASIC_CFLAGS += -DHAVE_SYNC_FILE_RANGE +endif + ifdef NEEDS_LIBRT EXTLIBS += -lrt endif @@ -1927,6 +1950,11 @@ ifdef NEED_ACCESS_ROOT_HANDLER COMPAT_OBJS += compat/access.o endif +ifdef FSMONITOR_DAEMON_BACKEND + COMPAT_CFLAGS += -DHAVE_FSMONITOR_DAEMON_BACKEND + COMPAT_OBJS += compat/fsmonitor/fsm-listen-$(FSMONITOR_DAEMON_BACKEND).o +endif + ifeq ($(TCLTK_PATH),) NO_TCLTK = NoThanks endif @@ -2252,33 +2280,30 @@ command-list.h: $(wildcard Documentation/git*.txt) hook-list.h: generate-hooklist.sh Documentation/githooks.txt $(QUIET_GEN)$(SHELL_PATH) ./generate-hooklist.sh >$@ -SCRIPT_DEFINES = $(SHELL_PATH_SQ):$(DIFF_SQ):$(GIT_VERSION):\ - $(localedir_SQ):$(NO_CURL):$(USE_GETTEXT_SCHEME):$(SANE_TOOL_PATH_SQ):\ - $(gitwebdir_SQ):$(PERL_PATH_SQ):$(SANE_TEXT_GREP):$(PAGER_ENV):\ +SCRIPT_DEFINES = $(SHELL_PATH_SQ):$(DIFF_SQ):\ + $(localedir_SQ):$(USE_GETTEXT_SCHEME):$(SANE_TOOL_PATH_SQ):\ + $(gitwebdir_SQ):$(PERL_PATH_SQ):$(PAGER_ENV):\ $(perllibdir_SQ) +GIT-SCRIPT-DEFINES: FORCE + @FLAGS='$(SCRIPT_DEFINES)'; \ + if test x"$$FLAGS" != x"`cat $@ 2>/dev/null`" ; then \ + echo >&2 " * new script parameters"; \ + echo "$$FLAGS" >$@; \ + fi + define cmd_munge_script sed -e '1s|#!.*/sh|#!$(SHELL_PATH_SQ)|' \ -e 's|@SHELL_PATH@|$(SHELL_PATH_SQ)|' \ -e 's|@@DIFF@@|$(DIFF_SQ)|' \ -e 's|@@LOCALEDIR@@|$(localedir_SQ)|g' \ - -e 's/@@NO_CURL@@/$(NO_CURL)/g' \ -e 's/@@USE_GETTEXT_SCHEME@@/$(USE_GETTEXT_SCHEME)/g' \ -e $(BROKEN_PATH_FIX) \ -e 's|@@GITWEBDIR@@|$(gitwebdir_SQ)|g' \ -e 's|@@PERL@@|$(PERL_PATH_SQ)|g' \ - -e 's|@@SANE_TEXT_GREP@@|$(SANE_TEXT_GREP)|g' \ -e 's|@@PAGER_ENV@@|$(PAGER_ENV_SQ)|g' \ $@.sh >$@+ endef -GIT-SCRIPT-DEFINES: FORCE - @FLAGS='$(SCRIPT_DEFINES)'; \ - if test x"$$FLAGS" != x"`cat $@ 2>/dev/null`" ; then \ - echo >&2 " * new script parameters"; \ - echo "$$FLAGS" >$@; \ - fi - - $(SCRIPT_SH_GEN) : % : %.sh GIT-SCRIPT-DEFINES $(QUIET_GEN)$(cmd_munge_script) && \ chmod +x $@+ && \ @@ -2437,7 +2462,36 @@ XDIFF_OBJS += xdiff/xutils.o .PHONY: xdiff-objs xdiff-objs: $(XDIFF_OBJS) +REFTABLE_OBJS += reftable/basics.o +REFTABLE_OBJS += reftable/error.o +REFTABLE_OBJS += reftable/block.o +REFTABLE_OBJS += reftable/blocksource.o +REFTABLE_OBJS += reftable/iter.o +REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/merged.o +REFTABLE_OBJS += reftable/pq.o +REFTABLE_OBJS += reftable/reader.o +REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/refname.o +REFTABLE_OBJS += reftable/generic.o +REFTABLE_OBJS += reftable/stack.o +REFTABLE_OBJS += reftable/tree.o +REFTABLE_OBJS += reftable/writer.o + +REFTABLE_TEST_OBJS += reftable/basics_test.o +REFTABLE_TEST_OBJS += reftable/block_test.o +REFTABLE_TEST_OBJS += reftable/dump.o +REFTABLE_TEST_OBJS += reftable/merged_test.o +REFTABLE_TEST_OBJS += reftable/pq_test.o +REFTABLE_TEST_OBJS += reftable/record_test.o +REFTABLE_TEST_OBJS += reftable/readwrite_test.o +REFTABLE_TEST_OBJS += reftable/refname_test.o +REFTABLE_TEST_OBJS += reftable/stack_test.o +REFTABLE_TEST_OBJS += reftable/test_framework.o +REFTABLE_TEST_OBJS += reftable/tree_test.o + TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) + .PHONY: test-objs test-objs: $(TEST_OBJS) @@ -2453,9 +2507,16 @@ OBJECTS += $(PROGRAM_OBJS) OBJECTS += $(TEST_OBJS) OBJECTS += $(XDIFF_OBJS) OBJECTS += $(FUZZ_OBJS) +OBJECTS += $(REFTABLE_OBJS) $(REFTABLE_TEST_OBJS) + ifndef NO_CURL OBJECTS += http.o http-walker.o remote-curl.o endif + +SCALAR_SOURCES := contrib/scalar/scalar.c +SCALAR_OBJECTS := $(SCALAR_SOURCES:c=o) +OBJECTS += $(SCALAR_OBJECTS) + .PHONY: objects objects: $(OBJECTS) @@ -2589,12 +2650,22 @@ $(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o GIT-LDFLAGS $(GITLIBS $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS) +contrib/scalar/scalar$X: $(SCALAR_OBJECTS) GIT-LDFLAGS $(GITLIBS) + $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) \ + $(filter %.o,$^) $(LIBS) + $(LIB_FILE): $(LIB_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ $(XDIFF_LIB): $(XDIFF_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ +$(REFTABLE_LIB): $(REFTABLE_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + +$(REFTABLE_TEST_LIB): $(REFTABLE_TEST_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + export DEFAULT_EDITOR DEFAULT_PAGER Documentation/GIT-EXCLUDED-PROGRAMS: FORCE @@ -2802,6 +2873,9 @@ GIT-BUILD-OPTIONS: FORCE @echo DC_SHA1=\''$(subst ','\'',$(subst ','\'',$(DC_SHA1)))'\' >>$@+ @echo SANITIZE_LEAK=\''$(subst ','\'',$(subst ','\'',$(SANITIZE_LEAK)))'\' >>$@+ @echo X=\'$(X)\' >>$@+ +ifdef FSMONITOR_DAEMON_BACKEND + @echo FSMONITOR_DAEMON_BACKEND=\''$(subst ','\'',$(subst ','\'',$(FSMONITOR_DAEMON_BACKEND)))'\' >>$@+ +endif ifdef TEST_OUTPUT_DIRECTORY @echo TEST_OUTPUT_DIRECTORY=\''$(subst ','\'',$(subst ','\'',$(TEST_OUTPUT_DIRECTORY)))'\' >>$@+ endif @@ -2893,7 +2967,7 @@ perf: all t/helper/test-tool$X: $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) -t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS) +t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS) $(REFTABLE_TEST_LIB) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(filter %.a,$^) $(LIBS) check-sha1:: t/helper/test-tool$X @@ -2901,7 +2975,7 @@ check-sha1:: t/helper/test-tool$X SP_OBJ = $(patsubst %.o,%.sp,$(C_OBJ)) -$(SP_OBJ): %.sp: %.c %.o GIT-CFLAGS +$(SP_OBJ): %.sp: %.c %.o $(QUIET_SP)cgcc -no-compile $(ALL_CFLAGS) $(EXTRA_CPPFLAGS) \ -Wsparse-error \ $(SPARSE_FLAGS) $(SP_EXTRA_FLAGS) $< && \ @@ -3231,7 +3305,7 @@ cocciclean: clean: profile-clean coverage-clean cocciclean $(RM) *.res $(RM) $(OBJECTS) - $(RM) $(LIB_FILE) $(XDIFF_LIB) + $(RM) $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) $(REFTABLE_TEST_LIB) $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X $(RM) $(TEST_PROGRAMS) $(RM) $(FUZZ_PROGRAMS) @@ -126,6 +126,39 @@ out_err: return -1; } +static int inherit_tracking(struct tracking *tracking, const char *orig_ref) +{ + struct strbuf key = STRBUF_INIT; + char *remote; + const char *bare_ref; + + bare_ref = orig_ref; + skip_prefix(orig_ref, "refs/heads/", &bare_ref); + + strbuf_addf(&key, "branch.%s.remote", bare_ref); + if (git_config_get_string(key.buf, &remote)) { + warning(_("asked to inherit tracking from %s, but could not find %s"), + bare_ref, key.buf); + strbuf_release(&key); + return -1; + } + + strbuf_reset(&key); + strbuf_addf(&key, "branch.%s.merge", bare_ref); + if (git_config_get_string(key.buf, &tracking->src)) { + warning(_("asked to inherit tracking from %s, but could not find %s"), + bare_ref, key.buf); + strbuf_release(&key); + free(remote); + return -1; + } + + tracking->remote = remote; + tracking->matches = 1; + strbuf_release(&key); + return 0; +} + /* * This is called when new_ref is branched off of orig_ref, and tries * to infer the settings for branch.<new_ref>.{remote,merge} from the @@ -139,7 +172,9 @@ static void setup_tracking(const char *new_ref, const char *orig_ref, memset(&tracking, 0, sizeof(tracking)); tracking.spec.dst = (char *)orig_ref; - if (for_each_remote(find_tracked_branch, &tracking)) + if (track != BRANCH_TRACK_INHERIT) { + for_each_remote(find_tracked_branch, &tracking); + } else if (inherit_tracking(&tracking, orig_ref)) return; if (!tracking.matches) @@ -10,7 +10,8 @@ enum branch_track { BRANCH_TRACK_REMOTE, BRANCH_TRACK_ALWAYS, BRANCH_TRACK_EXPLICIT, - BRANCH_TRACK_OVERRIDE + BRANCH_TRACK_OVERRIDE, + BRANCH_TRACK_INHERIT }; extern enum branch_track git_branch_track; @@ -159,11 +159,13 @@ int cmd_for_each_ref(int argc, const char **argv, const char *prefix); int cmd_for_each_repo(int argc, const char **argv, const char *prefix); int cmd_format_patch(int argc, const char **argv, const char *prefix); int cmd_fsck(int argc, const char **argv, const char *prefix); +int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix); int cmd_gc(int argc, const char **argv, const char *prefix); int cmd_get_tar_commit_id(int argc, const char **argv, const char *prefix); int cmd_grep(int argc, const char **argv, const char *prefix); int cmd_hash_object(int argc, const char **argv, const char *prefix); int cmd_help(int argc, const char **argv, const char *prefix); +int cmd_hook(int argc, const char **argv, const char *prefix); int cmd_index_pack(int argc, const char **argv, const char *prefix); int cmd_init_db(int argc, const char **argv, const char *prefix); int cmd_interpret_trailers(int argc, const char **argv, const char *prefix); diff --git a/builtin/am.c b/builtin/am.c index 8677ea2348..5528f131a8 100644 --- a/builtin/am.c +++ b/builtin/am.c @@ -446,9 +446,11 @@ static void am_destroy(const struct am_state *state) static int run_applypatch_msg_hook(struct am_state *state) { int ret; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; assert(state->msg); - ret = run_hook_le(NULL, "applypatch-msg", am_path(state, "final-commit"), NULL); + strvec_push(&opt.args, am_path(state, "final-commit")); + ret = run_hooks_oneshot("applypatch-msg", &opt); if (!ret) { FREE_AND_NULL(state->msg); @@ -1609,7 +1611,7 @@ static void do_commit(const struct am_state *state) const char *reflog_msg, *author, *committer = NULL; struct strbuf sb = STRBUF_INIT; - if (run_hook_le(NULL, "pre-applypatch", NULL)) + if (run_hooks_oneshot("pre-applypatch", NULL)) exit(1); if (write_cache_as_tree(&tree, 0, NULL)) @@ -1661,7 +1663,7 @@ static void do_commit(const struct am_state *state) fclose(fp); } - run_hook_le(NULL, "post-applypatch", NULL); + run_hooks_oneshot("post-applypatch", NULL); strbuf_release(&sb); } diff --git a/builtin/bisect--helper.c b/builtin/bisect--helper.c index 28a2e6a575..1727cb051f 100644 --- a/builtin/bisect--helper.c +++ b/builtin/bisect--helper.c @@ -830,7 +830,7 @@ static int bisect_autostart(struct bisect_terms *terms) fprintf_ln(stderr, _("You need to start by \"git bisect " "start\"\n")); - if (!isatty(STDIN_FILENO)) + if (!isatty(1)) return -1; /* diff --git a/builtin/blame.c b/builtin/blame.c index 1c31a99640..0fe0d70dba 100644 --- a/builtin/blame.c +++ b/builtin/blame.c @@ -898,6 +898,9 @@ int cmd_blame(int argc, const char **argv, const char *prefix) long anchor; const int hexsz = the_hash_algo->hexsz; + prepare_repo_settings(the_repository); + the_repository->settings.command_requires_full_index = 0; + setup_default_color_by_age(); git_config(git_blame_config, &output_option); repo_init_revisions(the_repository, &revs, NULL); @@ -913,6 +916,9 @@ int cmd_blame(int argc, const char **argv, const char *prefix) PARSE_OPT_KEEP_DASHDASH | PARSE_OPT_KEEP_ARGV0); for (;;) { switch (parse_options_step(&ctx, options, blame_opt_usage)) { + case PARSE_OPT_NON_OPTION: + case PARSE_OPT_UNKNOWN: + break; case PARSE_OPT_HELP: case PARSE_OPT_ERROR: exit(129); diff --git a/builtin/branch.c b/builtin/branch.c index 0b7ed82654..1fb4b57ca9 100644 --- a/builtin/branch.c +++ b/builtin/branch.c @@ -77,12 +77,11 @@ define_list_config_array(color_branch_slots); static int git_branch_config(const char *var, const char *value, void *cb) { const char *slot_name; - struct ref_sorting **sorting_tail = (struct ref_sorting **)cb; if (!strcmp(var, "branch.sort")) { if (!value) return config_error_nonbool(var); - parse_ref_sorting(sorting_tail, value); + string_list_append(cb, value); return 0; } @@ -407,7 +406,8 @@ static char *build_format(struct ref_filter *filter, int maxwidth, const char *r return strbuf_detach(&fmt, NULL); } -static void print_ref_list(struct ref_filter *filter, struct ref_sorting *sorting, struct ref_format *format) +static void print_ref_list(struct ref_filter *filter, struct ref_sorting *sorting, + struct ref_format *format, struct string_list *output) { int i; struct ref_array array; @@ -449,7 +449,7 @@ static void print_ref_list(struct ref_filter *filter, struct ref_sorting *sortin if (column_active(colopts)) { assert(!filter->verbose && "--column and --verbose are incompatible"); /* format to a string_list to let print_columns() do its job */ - string_list_append(&output, out.buf); + string_list_append(output, out.buf); } else { fwrite(out.buf, 1, out.len, stdout); putchar('\n'); @@ -624,7 +624,8 @@ int cmd_branch(int argc, const char **argv, const char *prefix) enum branch_track track; struct ref_filter filter; int icase = 0; - static struct ref_sorting *sorting = NULL, **sorting_tail = &sorting; + static struct ref_sorting *sorting; + struct string_list sorting_options = STRING_LIST_INIT_DUP; struct ref_format format = REF_FORMAT_INIT; struct option options[] = { @@ -632,8 +633,10 @@ int cmd_branch(int argc, const char **argv, const char *prefix) OPT__VERBOSE(&filter.verbose, N_("show hash and subject, give twice for upstream branch")), OPT__QUIET(&quiet, N_("suppress informational messages")), - OPT_SET_INT('t', "track", &track, N_("set up tracking mode (see git-pull(1))"), - BRANCH_TRACK_EXPLICIT), + OPT_CALLBACK_F('t', "track", &track, "direct|inherit", + N_("set up tracking mode (see git-pull(1))"), + PARSE_OPT_OPTARG | PARSE_OPT_LITERAL_ARGHELP, + parse_opt_tracking_mode), OPT_SET_INT_F(0, "set-upstream", &track, N_("do not use"), BRANCH_TRACK_OVERRIDE, PARSE_OPT_HIDDEN), OPT_STRING('u', "set-upstream-to", &new_upstream, N_("upstream"), N_("change the upstream info")), @@ -665,7 +668,7 @@ int cmd_branch(int argc, const char **argv, const char *prefix) OPT_MERGED(&filter, N_("print only branches that are merged")), OPT_NO_MERGED(&filter, N_("print only branches that are not merged")), OPT_COLUMN(0, "column", &colopts, N_("list branches in columns")), - OPT_REF_SORT(sorting_tail), + OPT_REF_SORT(&sorting_options), OPT_CALLBACK(0, "points-at", &filter.points_at, N_("object"), N_("print only branches of the object"), parse_opt_object_name), OPT_BOOL('i', "ignore-case", &icase, N_("sorting and filtering are case insensitive")), @@ -682,7 +685,7 @@ int cmd_branch(int argc, const char **argv, const char *prefix) if (argc == 2 && !strcmp(argv[1], "-h")) usage_with_options(builtin_branch_usage, options); - git_config(git_branch_config, sorting_tail); + git_config(git_branch_config, &sorting_options); track = git_branch_track; @@ -748,14 +751,14 @@ int cmd_branch(int argc, const char **argv, const char *prefix) * local branches 'refs/heads/...' and finally remote-tracking * branches 'refs/remotes/...'. */ - if (!sorting) - sorting = ref_default_sorting(); + sorting = ref_sorting_options(&sorting_options); ref_sorting_set_sort_flags_all(sorting, REF_SORTING_ICASE, icase); ref_sorting_set_sort_flags_all( sorting, REF_SORTING_DETACHED_HEAD_FIRST, 1); - print_ref_list(&filter, sorting, &format); + print_ref_list(&filter, sorting, &format, &output); print_columns(&output, colopts, NULL); string_list_clear(&output, 0); + ref_sorting_release(sorting); return 0; } else if (edit_description) { const char *branch_name; diff --git a/builtin/bundle.c b/builtin/bundle.c index 5a85d7cd0f..df69c65175 100644 --- a/builtin/bundle.c +++ b/builtin/bundle.c @@ -56,7 +56,7 @@ static int parse_options_cmd_bundle(int argc, static int cmd_bundle_create(int argc, const char **argv, const char *prefix) { int all_progress_implied = 0; - int progress = isatty(STDERR_FILENO); + int progress = isatty(2); struct strvec pack_opts; int version = -1; int ret; diff --git a/builtin/checkout--worker.c b/builtin/checkout--worker.c index fb9fd13b73..bb325b1b1a 100644 --- a/builtin/checkout--worker.c +++ b/builtin/checkout--worker.c @@ -9,7 +9,7 @@ static void packet_to_pc_item(const char *buffer, int len, struct parallel_checkout_item *pc_item) { const struct pc_item_fixed_portion *fixed_portion; - const char *variant; + const char *variant, *ident_action; char *encoding; if (len < sizeof(struct pc_item_fixed_portion)) @@ -19,7 +19,8 @@ static void packet_to_pc_item(const char *buffer, int len, fixed_portion = (struct pc_item_fixed_portion *)buffer; if (len - sizeof(struct pc_item_fixed_portion) != - fixed_portion->name_len + fixed_portion->working_tree_encoding_len) + fixed_portion->name_len + fixed_portion->working_tree_encoding_len + + fixed_portion->ident_action_len) BUG("checkout worker received corrupted item"); variant = buffer + sizeof(struct pc_item_fixed_portion); @@ -43,11 +44,21 @@ static void packet_to_pc_item(const char *buffer, int len, pc_item->ce->ce_namelen = fixed_portion->name_len; pc_item->ce->ce_mode = fixed_portion->ce_mode; memcpy(pc_item->ce->name, variant, pc_item->ce->ce_namelen); + variant += pc_item->ce->ce_namelen; oidcpy(&pc_item->ce->oid, &fixed_portion->oid); + if (fixed_portion->ident_action_len) { + ident_action = xmemdupz(variant, + fixed_portion->ident_action_len); + variant += fixed_portion->ident_action_len; + } else { + ident_action = NULL; + } + pc_item->id = fixed_portion->id; pc_item->ca.crlf_action = fixed_portion->crlf_action; - pc_item->ca.ident = fixed_portion->ident; + pc_item->ca.ident_action.id = ident_action; + pc_item->ca.ident_action.id_len = fixed_portion->ident_action_len; pc_item->ca.working_tree_encoding = encoding; } @@ -82,8 +93,8 @@ static void worker_loop(struct checkout *state) size_t i, nr = 0, alloc = 0; while (1) { - int len = packet_read(0, NULL, NULL, packet_buffer, - sizeof(packet_buffer), 0); + int len = packet_read(0, packet_buffer, sizeof(packet_buffer), + 0); if (len < 0) BUG("packet_read() returned negative value"); diff --git a/builtin/checkout.c b/builtin/checkout.c index cbf73b8c9f..75dc4934af 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -9,6 +9,7 @@ #include "config.h" #include "diff.h" #include "dir.h" +#include "hook.h" #include "ll-merge.h" #include "lockfile.h" #include "merge-recursive.h" @@ -106,13 +107,16 @@ struct branch_info { static int post_checkout_hook(struct commit *old_commit, struct commit *new_commit, int changed) { - return run_hook_le(NULL, "post-checkout", - oid_to_hex(old_commit ? &old_commit->object.oid : null_oid()), - oid_to_hex(new_commit ? &new_commit->object.oid : null_oid()), - changed ? "1" : "0", NULL); + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + /* "new_commit" can be NULL when checking out from the index before a commit exists. */ - + strvec_pushl(&opt.args, + oid_to_hex(old_commit ? &old_commit->object.oid : null_oid()), + oid_to_hex(new_commit ? &new_commit->object.oid : null_oid()), + changed ? "1" : "0", + NULL); + return run_hooks_oneshot("post-checkout", &opt); } static int update_some(const struct object_id *oid, struct strbuf *base, @@ -1517,7 +1521,7 @@ static struct option *add_common_options(struct checkout_opts *opts, OPT_BOOL(0, "progress", &opts->show_progress, N_("force progress reporting")), OPT_BOOL('m', "merge", &opts->merge, N_("perform a 3-way merge with the new branch")), OPT_STRING(0, "conflict", &opts->conflict_style, N_("style"), - N_("conflict style (merge or diff3)")), + N_("conflict style (merge, diff3, or zdiff3)")), OPT_END() }; struct option *newopts = parse_options_concat(prevopts, options); @@ -1530,8 +1534,10 @@ static struct option *add_common_switch_branch_options( { struct option options[] = { OPT_BOOL('d', "detach", &opts->force_detach, N_("detach HEAD at named commit")), - OPT_SET_INT('t', "track", &opts->track, N_("set upstream info for new branch"), - BRANCH_TRACK_EXPLICIT), + OPT_CALLBACK_F('t', "track", &opts->track, "direct|inherit", + N_("set up tracking mode (see git-pull(1))"), + PARSE_OPT_OPTARG | PARSE_OPT_LITERAL_ARGHELP, + parse_opt_tracking_mode), OPT__FORCE(&opts->force, N_("force checkout (throw away local modifications)"), PARSE_OPT_NOCOMPLETE), OPT_STRING(0, "orphan", &opts->new_orphan_branch, N_("new-branch"), N_("new unparented branch")), diff --git a/builtin/clone.c b/builtin/clone.c index 559acf9e03..39e4aade8d 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -32,6 +32,7 @@ #include "connected.h" #include "packfile.h" #include "list-objects-filter-options.h" +#include "hook.h" /* * Overall FIXMEs: @@ -659,6 +660,7 @@ static int checkout(int submodule_progress) struct tree *tree; struct tree_desc t; int err = 0; + struct run_hooks_opt hook_opt = RUN_HOOKS_OPT_INIT; if (option_no_checkout) return 0; @@ -705,8 +707,9 @@ static int checkout(int submodule_progress) if (write_locked_index(&the_index, &lock_file, COMMIT_LOCK)) die(_("unable to write new index file")); - err |= run_hook_le(NULL, "post-checkout", oid_to_hex(null_oid()), - oid_to_hex(&oid), "1", NULL); + strvec_pushl(&hook_opt.args, oid_to_hex(null_oid()), oid_to_hex(&oid), + "1", NULL); + err |= run_hooks_oneshot("post-checkout", &hook_opt); if (!err && (option_recurse_submodules.nr > 0)) { struct strvec args = STRVEC_INIT; @@ -1040,8 +1043,10 @@ int cmd_clone(int argc, const char **argv, const char *prefix) init_db(git_dir, real_git_dir, option_template, GIT_HASH_UNKNOWN, NULL, INIT_DB_QUIET); - if (real_git_dir) + if (real_git_dir) { + free((char *)git_dir); git_dir = real_git_dir; + } /* * additional config can be injected with -c, make sure it's included diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index 3c3de3a156..4247fbde95 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -172,8 +172,8 @@ static int write_option_max_new_filters(const struct option *opt, const char *s; *to = strtol(arg, (char **)&s, 10); if (*s) - return error(_("%s expects a numerical value"), - optname(opt, opt->flags)); + return error(_("option `%s' expects a numerical value"), + "max-new-filters"); } return 0; } @@ -263,7 +263,6 @@ static int graph_write(int argc, const char **argv) git_env_bool(GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS, 0)) flags |= COMMIT_GRAPH_WRITE_BLOOM_FILTERS; - read_replace_refs = 0; odb = find_odb(the_repository, opts.obj_dir); if (opts.reachable) { @@ -318,6 +317,7 @@ int cmd_commit_graph(int argc, const char **argv, const char *prefix) if (!argc) goto usage; + read_replace_refs = 0; save_commit_buffer = 0; if (!strcmp(argv[0], "verify")) diff --git a/builtin/credential.c b/builtin/credential.c index d75dcdc64a..d7b304fa08 100644 --- a/builtin/credential.c +++ b/builtin/credential.c @@ -4,7 +4,7 @@ #include "config.h" static const char usage_msg[] = - "git credential [fill|approve|reject]"; + "git credential (fill|approve|reject)"; int cmd_credential(int argc, const char **argv, const char *prefix) { diff --git a/builtin/diff.c b/builtin/diff.c index dd8ce688ba..cbf7b51c7c 100644 --- a/builtin/diff.c +++ b/builtin/diff.c @@ -437,6 +437,9 @@ int cmd_diff(int argc, const char **argv, const char *prefix) prefix = setup_git_directory_gently(&nongit); + prepare_repo_settings(the_repository); + the_repository->settings.command_requires_full_index = 0; + if (!no_index) { /* * Treat git diff with at least one path outside of the diff --git a/builtin/fast-export.c b/builtin/fast-export.c index 95e8e89e81..8e2caf7281 100644 --- a/builtin/fast-export.c +++ b/builtin/fast-export.c @@ -312,7 +312,7 @@ static void export_blob(const struct object_id *oid) if (!buf) die("could not read blob %s", oid_to_hex(oid)); if (check_object_signature(the_repository, oid, buf, size, - type_name(type)) < 0) + type_name(type), NULL) < 0) die("oid mismatch in blob %s", oid_to_hex(oid)); object = parse_object_buffer(the_repository, oid, type, size, buf, &eaten); diff --git a/builtin/fetch.c b/builtin/fetch.c index f7abbc31ff..d683b18cb8 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -1617,6 +1617,10 @@ static int do_fetch(struct transport *transport, struct ref *rm; struct ref *source_ref = NULL; + if (!branch) { + warning(_("not on a branch to use --set-upstream with")); + goto skip; + } /* * We're setting the upstream configuration for the * current branch. The relevant upstream is the diff --git a/builtin/for-each-ref.c b/builtin/for-each-ref.c index 642b4b888f..6f62f40d12 100644 --- a/builtin/for-each-ref.c +++ b/builtin/for-each-ref.c @@ -17,7 +17,8 @@ static char const * const for_each_ref_usage[] = { int cmd_for_each_ref(int argc, const char **argv, const char *prefix) { int i; - struct ref_sorting *sorting = NULL, **sorting_tail = &sorting; + struct ref_sorting *sorting; + struct string_list sorting_options = STRING_LIST_INIT_DUP; int maxcount = 0, icase = 0; struct ref_array array; struct ref_filter filter; @@ -39,7 +40,7 @@ int cmd_for_each_ref(int argc, const char **argv, const char *prefix) OPT_INTEGER( 0 , "count", &maxcount, N_("show only <n> matched refs")), OPT_STRING( 0 , "format", &format.format, N_("format"), N_("format to use for the output")), OPT__COLOR(&format.use_color, N_("respect format colors")), - OPT_REF_SORT(sorting_tail), + OPT_REF_SORT(&sorting_options), OPT_CALLBACK(0, "points-at", &filter.points_at, N_("object"), N_("print only refs which points at the given object"), parse_opt_object_name), @@ -70,8 +71,7 @@ int cmd_for_each_ref(int argc, const char **argv, const char *prefix) if (verify_ref_format(&format)) usage_with_options(for_each_ref_usage, opts); - if (!sorting) - sorting = ref_default_sorting(); + sorting = ref_sorting_options(&sorting_options); ref_sorting_set_sort_flags_all(sorting, REF_SORTING_ICASE, icase); filter.ignore_case = icase; @@ -96,6 +96,6 @@ int cmd_for_each_ref(int argc, const char **argv, const char *prefix) ref_array_clear(&array); free_commit_list(filter.with_commit); free_commit_list(filter.no_commit); - UNLEAK(sorting); + ref_sorting_release(sorting); return 0; } diff --git a/builtin/fsck.c b/builtin/fsck.c index b42b6fe21f..d87c28a1cc 100644 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@ -593,18 +593,43 @@ static void get_default_heads(void) } } +struct for_each_loose_cb +{ + struct progress *progress; + struct strbuf obj_type; +}; + static int fsck_loose(const struct object_id *oid, const char *path, void *data) { + struct for_each_loose_cb *cb_data = data; struct object *obj; - enum object_type type; + enum object_type type = OBJ_NONE; unsigned long size; void *contents; int eaten; + struct object_info oi = OBJECT_INFO_INIT; + struct object_id real_oid = *null_oid(); + int err = 0; - if (read_loose_object(path, oid, &type, &size, &contents) < 0) { + strbuf_reset(&cb_data->obj_type); + oi.type_name = &cb_data->obj_type; + oi.sizep = &size; + oi.typep = &type; + + if (read_loose_object(path, oid, &real_oid, &contents, &oi) < 0) { + if (contents && !oideq(&real_oid, oid)) + err = error(_("%s: hash-path mismatch, found at: %s"), + oid_to_hex(&real_oid), path); + else + err = error(_("%s: object corrupt or missing: %s"), + oid_to_hex(oid), path); + } + if (type != OBJ_NONE && type < 0) + err = error(_("%s: object is of unknown type '%s': %s"), + oid_to_hex(&real_oid), cb_data->obj_type.buf, + path); + if (err < 0) { errors_found |= ERROR_OBJECT; - error(_("%s: object corrupt or missing: %s"), - oid_to_hex(oid), path); return 0; /* keep checking other objects */ } @@ -640,8 +665,10 @@ static int fsck_cruft(const char *basename, const char *path, void *data) return 0; } -static int fsck_subdir(unsigned int nr, const char *path, void *progress) +static int fsck_subdir(unsigned int nr, const char *path, void *data) { + struct for_each_loose_cb *cb_data = data; + struct progress *progress = cb_data->progress; display_progress(progress, nr + 1); return 0; } @@ -649,6 +676,10 @@ static int fsck_subdir(unsigned int nr, const char *path, void *progress) static void fsck_object_dir(const char *path) { struct progress *progress = NULL; + struct for_each_loose_cb cb_data = { + .obj_type = STRBUF_INIT, + .progress = progress, + }; if (verbose) fprintf_ln(stderr, _("Checking object directory")); @@ -657,9 +688,10 @@ static void fsck_object_dir(const char *path) progress = start_progress(_("Checking object directories"), 256); for_each_loose_file_in_objdir(path, fsck_loose, fsck_cruft, fsck_subdir, - progress); + &cb_data); display_progress(progress, 256); stop_progress(&progress); + strbuf_release(&cb_data.obj_type); } static int fsck_head_link(const char *head_ref_name, @@ -803,6 +835,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix) fsck_enable_object_names(&fsck_walk_options); git_config(git_fsck_config, &fsck_obj_options); + prepare_repo_settings(the_repository); if (connectivity_only) { for_each_loose_object(mark_loose_for_connectivity, NULL, 0); @@ -908,7 +941,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix) check_connectivity(); - if (!git_config_get_bool("core.commitgraph", &i) && i) { + if (the_repository->settings.core_commit_graph) { struct child_process commit_graph_verify = CHILD_PROCESS_INIT; const char *verify_argv[] = { "commit-graph", "verify", NULL, NULL, NULL }; @@ -924,7 +957,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix) } } - if (!git_config_get_bool("core.multipackindex", &i) && i) { + if (the_repository->settings.core_multi_pack_index) { struct child_process midx_verify = CHILD_PROCESS_INIT; const char *midx_argv[] = { "multi-pack-index", "verify", NULL, NULL, NULL }; diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c new file mode 100644 index 0000000000..6011fe42ee --- /dev/null +++ b/builtin/fsmonitor--daemon.c @@ -0,0 +1,1454 @@ +#include "builtin.h" +#include "config.h" +#include "parse-options.h" +#include "fsmonitor.h" +#include "fsmonitor-ipc.h" +#include "compat/fsmonitor/fsm-listen.h" +#include "fsmonitor--daemon.h" +#include "simple-ipc.h" +#include "khash.h" +#include "pkt-line.h" + +static const char * const builtin_fsmonitor__daemon_usage[] = { + N_("git fsmonitor--daemon start [<options>]"), + N_("git fsmonitor--daemon run [<options>]"), + N_("git fsmonitor--daemon stop"), + N_("git fsmonitor--daemon status"), + NULL +}; + +#ifdef HAVE_FSMONITOR_DAEMON_BACKEND +/* + * Global state loaded from config. + */ +#define FSMONITOR__IPC_THREADS "fsmonitor.ipcthreads" +static int fsmonitor__ipc_threads = 8; + +#define FSMONITOR__START_TIMEOUT "fsmonitor.starttimeout" +static int fsmonitor__start_timeout_sec = 60; + +static int fsmonitor_config(const char *var, const char *value, void *cb) +{ + if (!strcmp(var, FSMONITOR__IPC_THREADS)) { + int i = git_config_int(var, value); + if (i < 1) + return error(_("value of '%s' out of range: %d"), + FSMONITOR__IPC_THREADS, i); + fsmonitor__ipc_threads = i; + return 0; + } + + if (!strcmp(var, FSMONITOR__START_TIMEOUT)) { + int i = git_config_int(var, value); + if (i < 0) + return error(_("value of '%s' out of range: %d"), + FSMONITOR__START_TIMEOUT, i); + fsmonitor__start_timeout_sec = i; + return 0; + } + + return git_default_config(var, value, cb); +} + +/* + * Acting as a CLIENT. + * + * Send a "quit" command to the `git-fsmonitor--daemon` (if running) + * and wait for it to shutdown. + */ +static int do_as_client__send_stop(void) +{ + struct strbuf answer = STRBUF_INIT; + int ret; + + ret = fsmonitor_ipc__send_command("quit", &answer); + + /* The quit command does not return any response data. */ + strbuf_release(&answer); + + if (ret) + return ret; + + trace2_region_enter("fsm_client", "polling-for-daemon-exit", NULL); + while (fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) + sleep_millisec(50); + trace2_region_leave("fsm_client", "polling-for-daemon-exit", NULL); + + return 0; +} + +static int do_as_client__status(void) +{ + enum ipc_active_state state = fsmonitor_ipc__get_state(); + + switch (state) { + case IPC_STATE__LISTENING: + printf(_("fsmonitor-daemon is watching '%s'\n"), + the_repository->worktree); + return 0; + + default: + printf(_("fsmonitor-daemon is not watching '%s'\n"), + the_repository->worktree); + return 1; + } +} + +enum fsmonitor_cookie_item_result { + FCIR_ERROR = -1, /* could not create cookie file ? */ + FCIR_INIT = 0, + FCIR_SEEN, + FCIR_ABORT, +}; + +struct fsmonitor_cookie_item { + struct hashmap_entry entry; + const char *name; + enum fsmonitor_cookie_item_result result; +}; + +static int cookies_cmp(const void *data, const struct hashmap_entry *he1, + const struct hashmap_entry *he2, const void *keydata) +{ + const struct fsmonitor_cookie_item *a = + container_of(he1, const struct fsmonitor_cookie_item, entry); + const struct fsmonitor_cookie_item *b = + container_of(he2, const struct fsmonitor_cookie_item, entry); + + return strcmp(a->name, keydata ? keydata : b->name); +} + +static enum fsmonitor_cookie_item_result with_lock__wait_for_cookie( + struct fsmonitor_daemon_state *state) +{ + /* assert current thread holding state->main_lock */ + + int fd; + struct fsmonitor_cookie_item *cookie; + struct strbuf cookie_pathname = STRBUF_INIT; + struct strbuf cookie_filename = STRBUF_INIT; + enum fsmonitor_cookie_item_result result; + int my_cookie_seq; + + CALLOC_ARRAY(cookie, 1); + + my_cookie_seq = state->cookie_seq++; + + strbuf_addf(&cookie_filename, "%i-%i", getpid(), my_cookie_seq); + + strbuf_addbuf(&cookie_pathname, &state->path_cookie_prefix); + strbuf_addbuf(&cookie_pathname, &cookie_filename); + + cookie->name = strbuf_detach(&cookie_filename, NULL); + cookie->result = FCIR_INIT; + hashmap_entry_init(&cookie->entry, strhash(cookie->name)); + + hashmap_add(&state->cookies, &cookie->entry); + + trace_printf_key(&trace_fsmonitor, "cookie-wait: '%s' '%s'", + cookie->name, cookie_pathname.buf); + + /* + * Create the cookie file on disk and then wait for a notification + * that the listener thread has seen it. + */ + fd = open(cookie_pathname.buf, O_WRONLY | O_CREAT | O_EXCL, 0600); + if (fd >= 0) { + close(fd); + unlink(cookie_pathname.buf); + + /* + * NEEDSWORK: This is an infinite wait (well, unless another + * thread sends us an abort). I'd like to change this to + * use `pthread_cond_timedwait()` and return an error/timeout + * and let the caller do the trivial response thing. + */ + while (cookie->result == FCIR_INIT) + pthread_cond_wait(&state->cookies_cond, + &state->main_lock); + } else { + error_errno(_("could not create fsmonitor cookie '%s'"), + cookie->name); + + cookie->result = FCIR_ERROR; + } + + hashmap_remove(&state->cookies, &cookie->entry, NULL); + + result = cookie->result; + + free((char*)cookie->name); + free(cookie); + strbuf_release(&cookie_pathname); + + return result; +} + +/* + * Mark these cookies as _SEEN and wake up the corresponding client threads. + */ +static void with_lock__mark_cookies_seen(struct fsmonitor_daemon_state *state, + const struct string_list *cookie_names) +{ + /* assert current thread holding state->main_lock */ + + int k; + int nr_seen = 0; + + for (k = 0; k < cookie_names->nr; k++) { + struct fsmonitor_cookie_item key; + struct fsmonitor_cookie_item *cookie; + + key.name = cookie_names->items[k].string; + hashmap_entry_init(&key.entry, strhash(key.name)); + + cookie = hashmap_get_entry(&state->cookies, &key, entry, NULL); + if (cookie) { + trace_printf_key(&trace_fsmonitor, "cookie-seen: '%s'", + cookie->name); + cookie->result = FCIR_SEEN; + nr_seen++; + } + } + + if (nr_seen) + pthread_cond_broadcast(&state->cookies_cond); +} + +/* + * Set _ABORT on all pending cookies and wake up all client threads. + */ +static void with_lock__abort_all_cookies(struct fsmonitor_daemon_state *state) +{ + /* assert current thread holding state->main_lock */ + + struct hashmap_iter iter; + struct fsmonitor_cookie_item *cookie; + int nr_aborted = 0; + + hashmap_for_each_entry(&state->cookies, &iter, cookie, entry) { + trace_printf_key(&trace_fsmonitor, "cookie-abort: '%s'", + cookie->name); + cookie->result = FCIR_ABORT; + nr_aborted++; + } + + if (nr_aborted) + pthread_cond_broadcast(&state->cookies_cond); +} + +/* + * Requests to and from a FSMonitor Protocol V2 provider use an opaque + * "token" as a virtual timestamp. Clients can request a summary of all + * created/deleted/modified files relative to a token. In the response, + * clients receive a new token for the next (relative) request. + * + * + * Token Format + * ============ + * + * The contents of the token are private and provider-specific. + * + * For the built-in fsmonitor--daemon, we define a token as follows: + * + * "builtin" ":" <token_id> ":" <sequence_nr> + * + * The "builtin" prefix is used as a namespace to avoid conflicts + * with other providers (such as Watchman). + * + * The <token_id> is an arbitrary OPAQUE string, such as a GUID, + * UUID, or {timestamp,pid}. It is used to group all filesystem + * events that happened while the daemon was monitoring (and in-sync + * with the filesystem). + * + * Unlike FSMonitor Protocol V1, it is not defined as a timestamp + * and does not define less-than/greater-than relationships. + * (There are too many race conditions to rely on file system + * event timestamps.) + * + * The <sequence_nr> is a simple integer incremented whenever the + * daemon needs to make its state public. For example, if 1000 file + * system events come in, but no clients have requested the data, + * the daemon can continue to accumulate file changes in the same + * bin and does not need to advance the sequence number. However, + * as soon as a client does arrive, the daemon needs to start a new + * bin and increment the sequence number. + * + * The sequence number serves as the boundary between 2 sets + * of bins -- the older ones that the client has already seen + * and the newer ones that it hasn't. + * + * When a new <token_id> is created, the <sequence_nr> is reset to + * zero. + * + * + * About Token Ids + * =============== + * + * A new token_id is created: + * + * [1] each time the daemon is started. + * + * [2] any time that the daemon must re-sync with the filesystem + * (such as when the kernel drops or we miss events on a very + * active volume). + * + * [3] in response to a client "flush" command (for dropped event + * testing). + * + * When a new token_id is created, the daemon is free to discard all + * cached filesystem events associated with any previous token_ids. + * Events associated with a non-current token_id will never be sent + * to a client. A token_id change implicitly means that the daemon + * has gap in its event history. + * + * Therefore, clients that present a token with a stale (non-current) + * token_id will always be given a trivial response. + */ +struct fsmonitor_token_data { + struct strbuf token_id; + struct fsmonitor_batch *batch_head; + struct fsmonitor_batch *batch_tail; + uint64_t client_ref_count; +}; + +struct fsmonitor_batch { + struct fsmonitor_batch *next; + uint64_t batch_seq_nr; + const char **interned_paths; + size_t nr, alloc; + time_t pinned_time; +}; + +static struct fsmonitor_token_data *fsmonitor_new_token_data(void) +{ + static int test_env_value = -1; + static uint64_t flush_count = 0; + struct fsmonitor_token_data *token; + struct fsmonitor_batch *batch; + + CALLOC_ARRAY(token, 1); + batch = fsmonitor_batch__new(); + + strbuf_init(&token->token_id, 0); + token->batch_head = batch; + token->batch_tail = batch; + token->client_ref_count = 0; + + if (test_env_value < 0) + test_env_value = git_env_bool("GIT_TEST_FSMONITOR_TOKEN", 0); + + if (!test_env_value) { + struct timeval tv; + struct tm tm; + time_t secs; + + gettimeofday(&tv, NULL); + secs = tv.tv_sec; + gmtime_r(&secs, &tm); + + strbuf_addf(&token->token_id, + "%"PRIu64".%d.%4d%02d%02dT%02d%02d%02d.%06ldZ", + flush_count++, + getpid(), + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec, + (long)tv.tv_usec); + } else { + strbuf_addf(&token->token_id, "test_%08x", test_env_value++); + } + + /* + * We created a new <token_id> and are starting a new series + * of tokens with a zero <seq_nr>. + * + * Since clients cannot guess our new (non test) <token_id> + * they will always receive a trivial response (because of the + * mismatch on the <token_id>). The trivial response will + * tell them our new <token_id> so that subsequent requests + * will be relative to our new series. (And when sending that + * response, we pin the current head of the batch list.) + * + * Even if the client correctly guesses the <token_id>, their + * request of "builtin:<token_id>:0" asks for all changes MORE + * RECENT than batch/bin 0. + * + * This implies that it is a waste to accumulate paths in the + * initial batch/bin (because they will never be transmitted). + * + * So the daemon could be running for days and watching the + * file system, but doesn't need to actually accumulate any + * paths UNTIL we need to set a reference point for a later + * relative request. + * + * However, it is very useful for testing to always have a + * reference point set. Pin batch 0 to force early file system + * events to accumulate. + */ + if (test_env_value) + batch->pinned_time = time(NULL); + + return token; +} + +struct fsmonitor_batch *fsmonitor_batch__new(void) +{ + struct fsmonitor_batch *batch; + + CALLOC_ARRAY(batch, 1); + + return batch; +} + +void fsmonitor_batch__free_list(struct fsmonitor_batch *batch) +{ + while (batch) { + struct fsmonitor_batch *next = batch->next; + + /* + * The actual strings within the array of this batch + * are interned, so we don't own them. We only own + * the array. + */ + free(batch->interned_paths); + free(batch); + + batch = next; + } +} + +void fsmonitor_batch__add_path(struct fsmonitor_batch *batch, + const char *path) +{ + const char *interned_path = strintern(path); + + trace_printf_key(&trace_fsmonitor, "event: %s", interned_path); + + ALLOC_GROW(batch->interned_paths, batch->nr + 1, batch->alloc); + batch->interned_paths[batch->nr++] = interned_path; +} + +static void fsmonitor_batch__combine(struct fsmonitor_batch *batch_dest, + const struct fsmonitor_batch *batch_src) +{ + size_t k; + + ALLOC_GROW(batch_dest->interned_paths, + batch_dest->nr + batch_src->nr + 1, + batch_dest->alloc); + + for (k = 0; k < batch_src->nr; k++) + batch_dest->interned_paths[batch_dest->nr++] = + batch_src->interned_paths[k]; +} + +/* + * To keep the batch list from growing unbounded in response to filesystem + * activity, we try to truncate old batches from the end of the list as + * they become irrelevant. + * + * We assume that the .git/index will be updated with the most recent token + * any time the index is updated. And future commands will only ask for + * recent changes *since* that new token. So as tokens advance into the + * future, older batch items will never be requested/needed. So we can + * truncate them without loss of functionality. + * + * However, multiple commands may be talking to the daemon concurrently + * or perform a slow command, so a little "token skew" is possible. + * Therefore, we want this to be a little bit lazy and have a generous + * delay. + * + * The current reader thread walked backwards in time from `token->batch_head` + * back to `batch_marker` somewhere in the middle of the batch list. + * + * Let's walk backwards in time from that marker an arbitrary delay + * and truncate the list there. Note that these timestamps are completely + * artificial (based on when we pinned the batch item) and not on any + * filesystem activity. + * + * Return the obsolete portion of the list after we have removed it from + * the official list so that the caller can free it after leaving the lock. + */ +#define MY_TIME_DELAY_SECONDS (5 * 60) /* seconds */ + +static struct fsmonitor_batch *with_lock__truncate_old_batches( + struct fsmonitor_daemon_state *state, + const struct fsmonitor_batch *batch_marker) +{ + /* assert current thread holding state->main_lock */ + + const struct fsmonitor_batch *batch; + struct fsmonitor_batch *remainder; + + if (!batch_marker) + return NULL; + + trace_printf_key(&trace_fsmonitor, "Truncate: mark (%"PRIu64",%"PRIu64")", + batch_marker->batch_seq_nr, + (uint64_t)batch_marker->pinned_time); + + for (batch = batch_marker; batch; batch = batch->next) { + time_t t; + + if (!batch->pinned_time) /* an overflow batch */ + continue; + + t = batch->pinned_time + MY_TIME_DELAY_SECONDS; + if (t > batch_marker->pinned_time) /* too close to marker */ + continue; + + goto truncate_past_here; + } + + return NULL; + +truncate_past_here: + state->current_token_data->batch_tail = (struct fsmonitor_batch *)batch; + + remainder = ((struct fsmonitor_batch *)batch)->next; + ((struct fsmonitor_batch *)batch)->next = NULL; + + return remainder; +} + +static void fsmonitor_free_token_data(struct fsmonitor_token_data *token) +{ + if (!token) + return; + + assert(token->client_ref_count == 0); + + strbuf_release(&token->token_id); + + fsmonitor_batch__free_list(token->batch_head); + + free(token); +} + +/* + * Flush all of our cached data about the filesystem. Call this if we + * lose sync with the filesystem and miss some notification events. + * + * [1] If we are missing events, then we no longer have a complete + * history of the directory (relative to our current start token). + * We should create a new token and start fresh (as if we just + * booted up). + * + * [2] Some of those lost events may have been for cookie files. We + * should assume the worst and abort them rather letting them starve. + * + * If there are no concurrent threads readering the current token data + * series, we can free it now. Otherwise, let the last reader free + * it. + * + * Either way, the old token data series is no longer associated with + * our state data. + */ +static void with_lock__do_force_resync(struct fsmonitor_daemon_state *state) +{ + /* assert current thread holding state->main_lock */ + + struct fsmonitor_token_data *free_me = NULL; + struct fsmonitor_token_data *new_one = NULL; + + new_one = fsmonitor_new_token_data(); + + if (state->current_token_data->client_ref_count == 0) + free_me = state->current_token_data; + state->current_token_data = new_one; + + fsmonitor_free_token_data(free_me); + + with_lock__abort_all_cookies(state); +} + +void fsmonitor_force_resync(struct fsmonitor_daemon_state *state) +{ + pthread_mutex_lock(&state->main_lock); + with_lock__do_force_resync(state); + pthread_mutex_unlock(&state->main_lock); +} + +/* + * Format an opaque token string to send to the client. + */ +static void with_lock__format_response_token( + struct strbuf *response_token, + const struct strbuf *response_token_id, + const struct fsmonitor_batch *batch) +{ + /* assert current thread holding state->main_lock */ + + strbuf_reset(response_token); + strbuf_addf(response_token, "builtin:%s:%"PRIu64, + response_token_id->buf, batch->batch_seq_nr); +} + +/* + * Parse an opaque token from the client. + * Returns -1 on error. + */ +static int fsmonitor_parse_client_token(const char *buf_token, + struct strbuf *requested_token_id, + uint64_t *seq_nr) +{ + const char *p; + char *p_end; + + strbuf_reset(requested_token_id); + *seq_nr = 0; + + if (!skip_prefix(buf_token, "builtin:", &p)) + return -1; + + while (*p && *p != ':') + strbuf_addch(requested_token_id, *p++); + if (!*p++) + return -1; + + *seq_nr = (uint64_t)strtoumax(p, &p_end, 10); + if (*p_end) + return -1; + + return 0; +} + +KHASH_INIT(str, const char *, int, 0, kh_str_hash_func, kh_str_hash_equal) + +static int do_handle_client(struct fsmonitor_daemon_state *state, + const char *command, + ipc_server_reply_cb *reply, + struct ipc_server_reply_data *reply_data) +{ + struct fsmonitor_token_data *token_data = NULL; + struct strbuf response_token = STRBUF_INIT; + struct strbuf requested_token_id = STRBUF_INIT; + struct strbuf payload = STRBUF_INIT; + uint64_t requested_oldest_seq_nr = 0; + uint64_t total_response_len = 0; + const char *p; + const struct fsmonitor_batch *batch_head; + const struct fsmonitor_batch *batch; + struct fsmonitor_batch *remainder = NULL; + intmax_t count = 0, duplicates = 0; + kh_str_t *shown; + int hash_ret; + int do_trivial = 0; + int do_flush = 0; + int do_cookie = 0; + enum fsmonitor_cookie_item_result cookie_result; + + /* + * We expect `command` to be of the form: + * + * <command> := quit NUL + * | flush NUL + * | <V1-time-since-epoch-ns> NUL + * | <V2-opaque-fsmonitor-token> NUL + */ + + if (!strcmp(command, "quit")) { + /* + * A client has requested over the socket/pipe that the + * daemon shutdown. + * + * Tell the IPC thread pool to shutdown (which completes + * the await in the main thread (which can stop the + * fsmonitor listener thread)). + * + * There is no reply to the client. + */ + return SIMPLE_IPC_QUIT; + + } else if (!strcmp(command, "flush")) { + /* + * Flush all of our cached data and generate a new token + * just like if we lost sync with the filesystem. + * + * Then send a trivial response using the new token. + */ + do_flush = 1; + do_trivial = 1; + + } else if (!skip_prefix(command, "builtin:", &p)) { + /* assume V1 timestamp or garbage */ + + char *p_end; + + strtoumax(command, &p_end, 10); + trace_printf_key(&trace_fsmonitor, + ((*p_end) ? + "fsmonitor: invalid command line '%s'" : + "fsmonitor: unsupported V1 protocol '%s'"), + command); + do_trivial = 1; + + } else { + /* We have "builtin:*" */ + if (fsmonitor_parse_client_token(command, &requested_token_id, + &requested_oldest_seq_nr)) { + trace_printf_key(&trace_fsmonitor, + "fsmonitor: invalid V2 protocol token '%s'", + command); + do_trivial = 1; + + } else { + /* + * We have a V2 valid token: + * "builtin:<token_id>:<seq_nr>" + */ + do_cookie = 1; + } + } + + pthread_mutex_lock(&state->main_lock); + + if (!state->current_token_data) + BUG("fsmonitor state does not have a current token"); + + /* + * Write a cookie file inside the directory being watched in + * an effort to flush out existing filesystem events that we + * actually care about. Suspend this client thread until we + * see the filesystem events for this cookie file. + * + * Creating the cookie lets us guarantee that our FS listener + * thread has drained the kernel queue and we are caught up + * with the kernel. + * + * If we cannot create the cookie (or otherwise guarantee that + * we are caught up), we send a trivial response. We have to + * assume that there might be some very, very recent activity + * on the FS still in flight. + */ + if (do_cookie) { + cookie_result = with_lock__wait_for_cookie(state); + if (cookie_result != FCIR_SEEN) { + error(_("fsmonitor: cookie_result '%d' != SEEN"), + cookie_result); + do_trivial = 1; + } + } + + if (do_flush) + with_lock__do_force_resync(state); + + /* + * We mark the current head of the batch list as "pinned" so + * that the listener thread will treat this item as read-only + * (and prevent any more paths from being added to it) from + * now on. + */ + token_data = state->current_token_data; + batch_head = token_data->batch_head; + ((struct fsmonitor_batch *)batch_head)->pinned_time = time(NULL); + + /* + * FSMonitor Protocol V2 requires that we send a response header + * with a "new current token" and then all of the paths that changed + * since the "requested token". We send the seq_nr of the just-pinned + * head batch so that future requests from a client will be relative + * to it. + */ + with_lock__format_response_token(&response_token, + &token_data->token_id, batch_head); + + reply(reply_data, response_token.buf, response_token.len + 1); + total_response_len += response_token.len + 1; + + trace2_data_string("fsmonitor", the_repository, "response/token", + response_token.buf); + trace_printf_key(&trace_fsmonitor, "response token: %s", + response_token.buf); + + if (!do_trivial) { + if (strcmp(requested_token_id.buf, token_data->token_id.buf)) { + /* + * The client last spoke to a different daemon + * instance -OR- the daemon had to resync with + * the filesystem (and lost events), so reject. + */ + trace2_data_string("fsmonitor", the_repository, + "response/token", "different"); + do_trivial = 1; + + } else if (requested_oldest_seq_nr < + token_data->batch_tail->batch_seq_nr) { + /* + * The client wants older events than we have for + * this token_id. This means that the end of our + * batch list was truncated and we cannot give the + * client a complete snapshot relative to their + * request. + */ + trace_printf_key(&trace_fsmonitor, + "client requested truncated data"); + do_trivial = 1; + } + } + + if (do_trivial) { + pthread_mutex_unlock(&state->main_lock); + + reply(reply_data, "/", 2); + + trace2_data_intmax("fsmonitor", the_repository, + "response/trivial", 1); + + strbuf_release(&response_token); + strbuf_release(&requested_token_id); + return 0; + } + + /* + * We're going to hold onto a pointer to the current + * token-data while we walk the list of batches of files. + * During this time, we will NOT be under the lock. + * So we ref-count it. + * + * This allows the listener thread to continue prepending + * new batches of items to the token-data (which we'll ignore). + * + * AND it allows the listener thread to do a token-reset + * (and install a new `current_token_data`). + */ + token_data->client_ref_count++; + + pthread_mutex_unlock(&state->main_lock); + + /* + * The client request is relative to the token that they sent, + * so walk the batch list backwards from the current head back + * to the batch (sequence number) they named. + * + * We use khash to de-dup the list of pathnames. + * + * NEEDSWORK: each batch contains a list of interned strings, + * so we only need to do pointer comparisons here to build the + * hash table. Currently, we're still comparing the string + * values. + */ + shown = kh_init_str(); + for (batch = batch_head; + batch && batch->batch_seq_nr > requested_oldest_seq_nr; + batch = batch->next) { + size_t k; + + for (k = 0; k < batch->nr; k++) { + const char *s = batch->interned_paths[k]; + size_t s_len; + + if (kh_get_str(shown, s) != kh_end(shown)) + duplicates++; + else { + kh_put_str(shown, s, &hash_ret); + + trace_printf_key(&trace_fsmonitor, + "send[%"PRIuMAX"]: %s", + count, s); + + /* Each path gets written with a trailing NUL */ + s_len = strlen(s) + 1; + + if (payload.len + s_len >= + LARGE_PACKET_DATA_MAX) { + reply(reply_data, payload.buf, + payload.len); + total_response_len += payload.len; + strbuf_reset(&payload); + } + + strbuf_add(&payload, s, s_len); + count++; + } + } + } + + if (payload.len) { + reply(reply_data, payload.buf, payload.len); + total_response_len += payload.len; + } + + kh_release_str(shown); + + pthread_mutex_lock(&state->main_lock); + + if (token_data->client_ref_count > 0) + token_data->client_ref_count--; + + if (token_data->client_ref_count == 0) { + if (token_data != state->current_token_data) { + /* + * The listener thread did a token-reset while we were + * walking the batch list. Therefore, this token is + * stale and can be discarded completely. If we are + * the last reader thread using this token, we own + * that work. + */ + fsmonitor_free_token_data(token_data); + } else if (batch) { + /* + * We are holding the lock and are the only + * reader of the ref-counted portion of the + * list, so we get the honor of seeing if the + * list can be truncated to save memory. + * + * The main loop did not walk to the end of the + * list, so this batch is the first item in the + * batch-list that is older than the requested + * end-point sequence number. See if the tail + * end of the list is obsolete. + */ + remainder = with_lock__truncate_old_batches(state, + batch); + } + } + + pthread_mutex_unlock(&state->main_lock); + + if (remainder) + fsmonitor_batch__free_list(remainder); + + trace2_data_intmax("fsmonitor", the_repository, "response/length", total_response_len); + trace2_data_intmax("fsmonitor", the_repository, "response/count/files", count); + trace2_data_intmax("fsmonitor", the_repository, "response/count/duplicates", duplicates); + + strbuf_release(&response_token); + strbuf_release(&requested_token_id); + strbuf_release(&payload); + + return 0; +} + +static ipc_server_application_cb handle_client; + +static int handle_client(void *data, + const char *command, size_t command_len, + ipc_server_reply_cb *reply, + struct ipc_server_reply_data *reply_data) +{ + struct fsmonitor_daemon_state *state = data; + int result; + + /* + * The Simple IPC API now supports {char*, len} arguments, but + * FSMonitor always uses proper null-terminated strings, so + * we can ignore the command_len argument. (Trust, but verify.) + */ + if (command_len != strlen(command)) + BUG("FSMonitor assumes text messages"); + + trace_printf_key(&trace_fsmonitor, "requested token: %s", command); + + trace2_region_enter("fsmonitor", "handle_client", the_repository); + trace2_data_string("fsmonitor", the_repository, "request", command); + + result = do_handle_client(state, command, reply, reply_data); + + trace2_region_leave("fsmonitor", "handle_client", the_repository); + + return result; +} + +#define FSMONITOR_DIR "fsmonitor--daemon" +#define FSMONITOR_COOKIE_DIR "cookies" +#define FSMONITOR_COOKIE_PREFIX (FSMONITOR_DIR "/" FSMONITOR_COOKIE_DIR "/") + +enum fsmonitor_path_type fsmonitor_classify_path_workdir_relative( + const char *rel) +{ + if (fspathncmp(rel, ".git", 4)) + return IS_WORKDIR_PATH; + rel += 4; + + if (!*rel) + return IS_DOT_GIT; + if (*rel != '/') + return IS_WORKDIR_PATH; /* e.g. .gitignore */ + rel++; + + if (!fspathncmp(rel, FSMONITOR_COOKIE_PREFIX, + strlen(FSMONITOR_COOKIE_PREFIX))) + return IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX; + + return IS_INSIDE_DOT_GIT; +} + +enum fsmonitor_path_type fsmonitor_classify_path_gitdir_relative( + const char *rel) +{ + if (!fspathncmp(rel, FSMONITOR_COOKIE_PREFIX, + strlen(FSMONITOR_COOKIE_PREFIX))) + return IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX; + + return IS_INSIDE_GITDIR; +} + +static enum fsmonitor_path_type try_classify_workdir_abs_path( + struct fsmonitor_daemon_state *state, + const char *path) +{ + const char *rel; + + if (fspathncmp(path, state->path_worktree_watch.buf, + state->path_worktree_watch.len)) + return IS_OUTSIDE_CONE; + + rel = path + state->path_worktree_watch.len; + + if (!*rel) + return IS_WORKDIR_PATH; /* it is the root dir exactly */ + if (*rel != '/') + return IS_OUTSIDE_CONE; + rel++; + + return fsmonitor_classify_path_workdir_relative(rel); +} + +enum fsmonitor_path_type fsmonitor_classify_path_absolute( + struct fsmonitor_daemon_state *state, + const char *path) +{ + const char *rel; + enum fsmonitor_path_type t; + + t = try_classify_workdir_abs_path(state, path); + if (state->nr_paths_watching == 1) + return t; + if (t != IS_OUTSIDE_CONE) + return t; + + if (fspathncmp(path, state->path_gitdir_watch.buf, + state->path_gitdir_watch.len)) + return IS_OUTSIDE_CONE; + + rel = path + state->path_gitdir_watch.len; + + if (!*rel) + return IS_GITDIR; /* it is the <gitdir> exactly */ + if (*rel != '/') + return IS_OUTSIDE_CONE; + rel++; + + return fsmonitor_classify_path_gitdir_relative(rel); +} + +/* + * We try to combine small batches at the front of the batch-list to avoid + * having a long list. This hopefully makes it a little easier when we want + * to truncate and maintain the list. However, we don't want the paths array + * to just keep growing and growing with realloc, so we insert an arbitrary + * limit. + */ +#define MY_COMBINE_LIMIT (1024) + +void fsmonitor_publish(struct fsmonitor_daemon_state *state, + struct fsmonitor_batch *batch, + const struct string_list *cookie_names) +{ + if (!batch && !cookie_names->nr) + return; + + pthread_mutex_lock(&state->main_lock); + + if (batch) { + struct fsmonitor_batch *head; + + head = state->current_token_data->batch_head; + if (!head) { + BUG("token does not have batch"); + } else if (head->pinned_time) { + /* + * We cannot alter the current batch list + * because: + * + * [a] it is being transmitted to at least one + * client and the handle_client() thread has a + * ref-count, but not a lock on the batch list + * starting with this item. + * + * [b] it has been transmitted in the past to + * at least one client such that future + * requests are relative to this head batch. + * + * So, we can only prepend a new batch onto + * the front of the list. + */ + batch->batch_seq_nr = head->batch_seq_nr + 1; + batch->next = head; + state->current_token_data->batch_head = batch; + } else if (!head->batch_seq_nr) { + /* + * Batch 0 is unpinned. See the note in + * `fsmonitor_new_token_data()` about why we + * don't need to accumulate these paths. + */ + fsmonitor_batch__free_list(batch); + } else if (head->nr + batch->nr > MY_COMBINE_LIMIT) { + /* + * The head batch in the list has never been + * transmitted to a client, but folding the + * contents of the new batch onto it would + * exceed our arbitrary limit, so just prepend + * the new batch onto the list. + */ + batch->batch_seq_nr = head->batch_seq_nr + 1; + batch->next = head; + state->current_token_data->batch_head = batch; + } else { + /* + * We are free to add the paths in the given + * batch onto the end of the current head batch. + */ + fsmonitor_batch__combine(head, batch); + fsmonitor_batch__free_list(batch); + } + } + + if (cookie_names->nr) + with_lock__mark_cookies_seen(state, cookie_names); + + pthread_mutex_unlock(&state->main_lock); +} + +static void *fsm_listen__thread_proc(void *_state) +{ + struct fsmonitor_daemon_state *state = _state; + + trace2_thread_start("fsm-listen"); + + trace_printf_key(&trace_fsmonitor, "Watching: worktree '%s'", + state->path_worktree_watch.buf); + if (state->nr_paths_watching > 1) + trace_printf_key(&trace_fsmonitor, "Watching: gitdir '%s'", + state->path_gitdir_watch.buf); + + fsm_listen__loop(state); + + pthread_mutex_lock(&state->main_lock); + if (state->current_token_data && + state->current_token_data->client_ref_count == 0) + fsmonitor_free_token_data(state->current_token_data); + state->current_token_data = NULL; + pthread_mutex_unlock(&state->main_lock); + + trace2_thread_exit(); + return NULL; +} + +static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) +{ + struct ipc_server_opts ipc_opts = { + .nr_threads = fsmonitor__ipc_threads, + + /* + * We know that there are no other active threads yet, + * so we can let the IPC layer temporarily chdir() if + * it needs to when creating the server side of the + * Unix domain socket. + */ + .uds_disallow_chdir = 0 + }; + + /* + * Start the IPC thread pool before the we've started the file + * system event listener thread so that we have the IPC handle + * before we need it. + */ + if (ipc_server_run_async(&state->ipc_server_data, + fsmonitor_ipc__get_path(), &ipc_opts, + handle_client, state)) + return error_errno( + _("could not start IPC thread pool on '%s'"), + fsmonitor_ipc__get_path()); + + /* + * Start the fsmonitor listener thread to collect filesystem + * events. + */ + if (pthread_create(&state->listener_thread, NULL, + fsm_listen__thread_proc, state) < 0) { + ipc_server_stop_async(state->ipc_server_data); + ipc_server_await(state->ipc_server_data); + + return error(_("could not start fsmonitor listener thread")); + } + + /* + * The daemon is now fully functional in background threads. + * Wait for the IPC thread pool to shutdown (whether by client + * request or from filesystem activity). + */ + ipc_server_await(state->ipc_server_data); + + /* + * The fsmonitor listener thread may have received a shutdown + * event from the IPC thread pool, but it doesn't hurt to tell + * it again. And wait for it to shutdown. + */ + fsm_listen__stop_async(state); + pthread_join(state->listener_thread, NULL); + + return state->error_code; +} + +static int fsmonitor_run_daemon(void) +{ + struct fsmonitor_daemon_state state; + int err; + + memset(&state, 0, sizeof(state)); + + hashmap_init(&state.cookies, cookies_cmp, NULL, 0); + pthread_mutex_init(&state.main_lock, NULL); + pthread_cond_init(&state.cookies_cond, NULL); + state.error_code = 0; + state.current_token_data = fsmonitor_new_token_data(); + + /* Prepare to (recursively) watch the <worktree-root> directory. */ + strbuf_init(&state.path_worktree_watch, 0); + strbuf_addstr(&state.path_worktree_watch, absolute_path(get_git_work_tree())); + state.nr_paths_watching = 1; + + /* + * We create and delete cookie files somewhere inside the .git + * directory to help us keep sync with the file system. If + * ".git" is not a directory, then <gitdir> is not inside the + * cone of <worktree-root>, so set up a second watch to watch + * the <gitdir> so that we get events for the cookie files. + */ + strbuf_init(&state.path_gitdir_watch, 0); + strbuf_addbuf(&state.path_gitdir_watch, &state.path_worktree_watch); + strbuf_addstr(&state.path_gitdir_watch, "/.git"); + if (!is_directory(state.path_gitdir_watch.buf)) { + strbuf_reset(&state.path_gitdir_watch); + strbuf_addstr(&state.path_gitdir_watch, absolute_path(get_git_dir())); + state.nr_paths_watching = 2; + } + + /* + * We will write filesystem syncing cookie files into + * <gitdir>/<fsmonitor-dir>/<cookie-dir>/<pid>-<seq>. + * + * The extra layers of subdirectories here keep us from + * changing the mtime on ".git/" or ".git/foo/" when we create + * or delete cookie files. + * + * There have been problems with some IDEs that do a + * non-recursive watch of the ".git/" directory and run a + * series of commands any time something happens. + * + * For example, if we place our cookie files directly in + * ".git/" or ".git/foo/" then a `git status` (or similar + * command) from the IDE will cause a cookie file to be + * created in one of those dirs. This causes the mtime of + * those dirs to change. This triggers the IDE's watch + * notification. This triggers the IDE to run those commands + * again. And the process repeats and the machine never goes + * idle. + * + * Adding the extra layers of subdirectories prevents the + * mtime of ".git/" and ".git/foo" from changing when a + * cookie file is created. + */ + strbuf_init(&state.path_cookie_prefix, 0); + strbuf_addbuf(&state.path_cookie_prefix, &state.path_gitdir_watch); + + strbuf_addch(&state.path_cookie_prefix, '/'); + strbuf_addstr(&state.path_cookie_prefix, FSMONITOR_DIR); + mkdir(state.path_cookie_prefix.buf, 0777); + + strbuf_addch(&state.path_cookie_prefix, '/'); + strbuf_addstr(&state.path_cookie_prefix, FSMONITOR_COOKIE_DIR); + mkdir(state.path_cookie_prefix.buf, 0777); + + strbuf_addch(&state.path_cookie_prefix, '/'); + + /* + * Confirm that we can create platform-specific resources for the + * filesystem listener before we bother starting all the threads. + */ + if (fsm_listen__ctor(&state)) { + err = error(_("could not initialize listener thread")); + goto done; + } + + err = fsmonitor_run_daemon_1(&state); + +done: + pthread_cond_destroy(&state.cookies_cond); + pthread_mutex_destroy(&state.main_lock); + fsm_listen__dtor(&state); + + ipc_server_free(state.ipc_server_data); + + strbuf_release(&state.path_worktree_watch); + strbuf_release(&state.path_gitdir_watch); + strbuf_release(&state.path_cookie_prefix); + + /* + * NEEDSWORK: Consider "rm -rf <gitdir>/<fsmonitor-dir>" + */ + + return err; +} + +static int try_to_run_foreground_daemon(int free_console) +{ + /* + * Technically, we don't need to probe for an existing daemon + * process, since we could just call `fsmonitor_run_daemon()` + * and let it fail if the pipe/socket is busy. + * + * However, this method gives us a nicer error message for a + * common error case. + */ + if (fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) + die("fsmonitor--daemon is already running '%s'", + the_repository->worktree); + + printf(_("running fsmonitor-daemon in '%s'\n"), + the_repository->worktree); + fflush(stdout); + +#ifdef GIT_WINDOWS_NATIVE + if (free_console) + FreeConsole(); +#endif + + return !!fsmonitor_run_daemon(); +} + +static start_bg_wait_cb bg_wait_cb; + +static int bg_wait_cb(const struct child_process *cp, void *cb_data) +{ + enum ipc_active_state s = fsmonitor_ipc__get_state(); + + switch (s) { + case IPC_STATE__LISTENING: + /* child is "ready" */ + return 0; + + case IPC_STATE__NOT_LISTENING: + case IPC_STATE__PATH_NOT_FOUND: + /* give child more time */ + return 1; + + default: + case IPC_STATE__INVALID_PATH: + case IPC_STATE__OTHER_ERROR: + /* all the time in world won't help */ + return -1; + } +} + +static int try_to_start_background_daemon(void) +{ + struct child_process cp = CHILD_PROCESS_INIT; + enum start_bg_result sbgr; + + /* + * Before we try to create a background daemon process, see + * if a daemon process is already listening. This makes it + * easier for us to report an already-listening error to the + * console, since our spawn/daemon can only report the success + * of creating the background process (and not whether it + * immediately exited). + */ + if (fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) + die("fsmonitor--daemon is already running '%s'", + the_repository->worktree); + + printf(_("starting fsmonitor-daemon in '%s'\n"), + the_repository->worktree); + fflush(stdout); + + cp.git_cmd = 1; + + strvec_push(&cp.args, "fsmonitor--daemon"); + strvec_push(&cp.args, "run"); + strvec_push(&cp.args, "--free-console"); + strvec_pushf(&cp.args, "--ipc-threads=%d", fsmonitor__ipc_threads); + + cp.no_stdin = 1; + cp.no_stdout = 1; + cp.no_stderr = 1; + + sbgr = start_bg_command(&cp, bg_wait_cb, NULL, + fsmonitor__start_timeout_sec); + + switch (sbgr) { + case SBGR_READY: + return 0; + + default: + case SBGR_ERROR: + case SBGR_CB_ERROR: + return error("daemon failed to start"); + + case SBGR_TIMEOUT: + return error("daemon not online yet"); + + case SBGR_DIED: + return error("daemon terminated"); + } +} + +int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) +{ + const char *subcmd; + int free_console = 0; + + struct option options[] = { + OPT_BOOL(0, "free-console", &free_console, N_("free console")), + OPT_INTEGER(0, "ipc-threads", + &fsmonitor__ipc_threads, + N_("use <n> ipc worker threads")), + OPT_INTEGER(0, "start-timeout", + &fsmonitor__start_timeout_sec, + N_("Max seconds to wait for background daemon startup")), + + OPT_END() + }; + + git_config(fsmonitor_config, NULL); + + argc = parse_options(argc, argv, prefix, options, + builtin_fsmonitor__daemon_usage, 0); + if (argc != 1) + usage_with_options(builtin_fsmonitor__daemon_usage, options); + subcmd = argv[0]; + + if (fsmonitor__ipc_threads < 1) + die(_("invalid 'ipc-threads' value (%d)"), + fsmonitor__ipc_threads); + + if (!strcmp(subcmd, "start")) + return !!try_to_start_background_daemon(); + + if (!strcmp(subcmd, "run")) + return !!try_to_run_foreground_daemon(free_console); + + if (!strcmp(subcmd, "stop")) + return !!do_as_client__send_stop(); + + if (!strcmp(subcmd, "status")) + return !!do_as_client__status(); + + die(_("Unhandled subcommand '%s'"), subcmd); +} + +#else +int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) +{ + struct option options[] = { + OPT_END() + }; + + if (argc == 2 && !strcmp(argv[1], "-h")) + usage_with_options(builtin_fsmonitor__daemon_usage, options); + + die(_("fsmonitor--daemon not supported on this platform")); +} +#endif diff --git a/builtin/gc.c b/builtin/gc.c index 6b3de3dd51..5057f7db29 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -32,6 +32,7 @@ #include "remote.h" #include "object-store.h" #include "exec-cmd.h" +#include "hook.h" #define FAILED_RUN "failed to run %s" @@ -394,7 +395,7 @@ static int need_to_gc(void) else return 0; - if (run_hook_le(NULL, "pre-auto-gc", NULL)) + if (run_hooks_oneshot("pre-auto-gc", NULL)) return 0; return 1; } @@ -1049,12 +1050,11 @@ static int maintenance_task_loose_objects(struct maintenance_run_opts *opts) static int incremental_repack_auto_condition(void) { struct packed_git *p; - int enabled; int incremental_repack_auto_limit = 10; int count = 0; - if (git_config_get_bool("core.multiPackIndex", &enabled) || - !enabled) + prepare_repo_settings(the_repository); + if (!the_repository->settings.core_multi_pack_index) return 0; git_config_get_int("maintenance.incremental-repack.auto", diff --git a/builtin/grep.c b/builtin/grep.c index 8af5249a7b..9e34a820ad 100644 --- a/builtin/grep.c +++ b/builtin/grep.c @@ -199,8 +199,8 @@ static void *run(void *arg) grep_source_clear_data(&w->source); work_done(w); } - free_grep_patterns(arg); - free(arg); + free_grep_patterns(opt); + free(opt); return (void*) (intptr_t) hit; } @@ -401,7 +401,7 @@ static void append_path(struct grep_opt *opt, const void *data, size_t len) if (len == 1 && *(const char *)data == '\0') return; - string_list_append(path_list, xstrndup(data, len)); + string_list_append_nodup(path_list, xstrndup(data, len)); } static void run_pager(struct grep_opt *opt, const char *prefix) @@ -839,7 +839,7 @@ int cmd_grep(int argc, const char **argv, const char *prefix) struct grep_opt opt; struct object_array list = OBJECT_ARRAY_INIT; struct pathspec pathspec; - struct string_list path_list = STRING_LIST_INIT_NODUP; + struct string_list path_list = STRING_LIST_INIT_DUP; int i; int dummy; int use_index = 1; @@ -1159,8 +1159,8 @@ int cmd_grep(int argc, const char **argv, const char *prefix) strbuf_addf(&buf, "+/%s%s", strcmp("less", pager) ? "" : "*", opt.pattern_list->pattern); - string_list_append(&path_list, - strbuf_detach(&buf, NULL)); + string_list_append_nodup(&path_list, + strbuf_detach(&buf, NULL)); } } @@ -1195,7 +1195,9 @@ int cmd_grep(int argc, const char **argv, const char *prefix) if (hit && show_in_pager) run_pager(&opt, prefix); clear_pathspec(&pathspec); + string_list_clear(&path_list, 0); free_grep_patterns(&opt); + object_array_clear(&list); free_repos(); return !hit; } diff --git a/builtin/hook.c b/builtin/hook.c new file mode 100644 index 0000000000..fa26454990 --- /dev/null +++ b/builtin/hook.c @@ -0,0 +1,90 @@ +#include "cache.h" +#include "builtin.h" +#include "config.h" +#include "hook.h" +#include "parse-options.h" +#include "strbuf.h" +#include "strvec.h" + +#define BUILTIN_HOOK_RUN_USAGE \ + N_("git hook run [--ignore-missing] <hook-name> [-- <hook-args>]") + +static const char * const builtin_hook_usage[] = { + BUILTIN_HOOK_RUN_USAGE, + NULL +}; + +static const char * const builtin_hook_run_usage[] = { + BUILTIN_HOOK_RUN_USAGE, + NULL +}; + +static int run(int argc, const char **argv, const char *prefix) +{ + int i; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + int ignore_missing = 0; + const char *hook_name; + const char *hook_path; + struct option run_options[] = { + OPT_BOOL(0, "ignore-missing", &ignore_missing, + N_("silently ignore missing requested <hook-name>")), + OPT_END(), + }; + int ret; + + argc = parse_options(argc, argv, prefix, run_options, + builtin_hook_run_usage, + PARSE_OPT_KEEP_DASHDASH); + + if (!argc) + goto usage; + + /* + * Having a -- for "run" when providing <hook-args> is + * mandatory. + */ + if (argc > 1 && strcmp(argv[1], "--") && + strcmp(argv[1], "--end-of-options")) + goto usage; + + /* Add our arguments, start after -- */ + for (i = 2 ; i < argc; i++) + strvec_push(&opt.args, argv[i]); + + /* Need to take into account core.hooksPath */ + git_config(git_default_config, NULL); + + hook_name = argv[0]; + if (ignore_missing) + return run_hooks_oneshot(hook_name, &opt); + hook_path = find_hook(hook_name); + if (!hook_path) { + error("cannot find a hook named %s", hook_name); + return 1; + } + + ret = run_hooks(hook_name, hook_path, &opt); + run_hooks_opt_clear(&opt); + return ret; +usage: + usage_with_options(builtin_hook_run_usage, run_options); +} + +int cmd_hook(int argc, const char **argv, const char *prefix) +{ + struct option builtin_hook_options[] = { + OPT_END(), + }; + + argc = parse_options(argc, argv, NULL, builtin_hook_options, + builtin_hook_usage, PARSE_OPT_STOP_AT_NON_OPTION); + if (!argc) + goto usage; + + if (!strcmp(argv[0], "run")) + return run(argc, argv, prefix); + +usage: + usage_with_options(builtin_hook_usage, builtin_hook_options); +} diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 7ce69c087e..15ae406e6b 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -1415,7 +1415,7 @@ static void fix_unresolved_deltas(struct hashfile *f) if (check_object_signature(the_repository, &d->oid, data, size, - type_name(type))) + type_name(type), NULL)) die(_("local object %s is corrupt"), oid_to_hex(&d->oid)); /* diff --git a/builtin/log.c b/builtin/log.c index f75d87e8d7..2b51d8b6aa 100644 --- a/builtin/log.c +++ b/builtin/log.c @@ -35,6 +35,8 @@ #include "repository.h" #include "commit-reach.h" #include "range-diff.h" +#include "dir.h" +#include "tmp-objdir.h" #define MAIL_DEFAULT_WRAP 72 #define COVER_FROM_AUTO_MAX_SUBJECT_LEN 100 @@ -407,6 +409,13 @@ static int cmd_log_walk(struct rev_info *rev) int saved_nrl = 0; int saved_dcctc = 0; + if (rev->remerge_diff) { + rev->remerge_objdir = tmp_objdir_create("remerge-diff"); + if (!rev->remerge_objdir) + die(_("unable to create temporary object directory")); + tmp_objdir_replace_primary_odb(rev->remerge_objdir, 1); + } + if (rev->early_output) setup_early_output(); @@ -449,6 +458,11 @@ static int cmd_log_walk(struct rev_info *rev) rev->diffopt.no_free = 0; diff_free(&rev->diffopt); + if (rev->remerge_diff) { + tmp_objdir_destroy(rev->remerge_objdir); + rev->remerge_objdir = NULL; + } + if (rev->diffopt.output_format & DIFF_FORMAT_CHECKDIFF && rev->diffopt.flags.check_failed) { return 02; @@ -1943,6 +1957,8 @@ int cmd_format_patch(int argc, const char **argv, const char *prefix) die(_("--name-status does not make sense")); if (rev.diffopt.output_format & DIFF_FORMAT_CHECKDIFF) die(_("--check does not make sense")); + if (rev.remerge_diff) + die(_("--remerge_diff does not make sense")); if (!use_patch_format && (!rev.diffopt.output_format || diff --git a/builtin/ls-files.c b/builtin/ls-files.c index a2000ed6bf..031fef1bca 100644 --- a/builtin/ls-files.c +++ b/builtin/ls-files.c @@ -672,6 +672,7 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix) N_("suppress duplicate entries")), OPT_END() }; + int ret = 0; if (argc == 2 && !strcmp(argv[1], "-h")) usage_with_options(ls_files_usage, builtin_ls_files_options); @@ -775,16 +776,13 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix) if (show_resolve_undo) show_ru_info(the_repository->index); - if (ps_matched) { - int bad; - bad = report_path_error(ps_matched, &pathspec); - if (bad) - fprintf(stderr, "Did you forget to 'git add'?\n"); - - return bad ? 1 : 0; + if (ps_matched && report_path_error(ps_matched, &pathspec)) { + fprintf(stderr, "Did you forget to 'git add'?\n"); + ret = 1; } + string_list_clear(&exclude_list, 0); dir_clear(&dir); free(max_prefix); - return 0; + return ret; } diff --git a/builtin/ls-remote.c b/builtin/ls-remote.c index 318949c3d7..44448fa61d 100644 --- a/builtin/ls-remote.c +++ b/builtin/ls-remote.c @@ -54,7 +54,7 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) struct transport *transport; const struct ref *ref; struct ref_array ref_array; - static struct ref_sorting *sorting = NULL, **sorting_tail = &sorting; + struct string_list sorting_options = STRING_LIST_INIT_DUP; struct option options[] = { OPT__QUIET(&quiet, N_("do not print remote URL")), @@ -68,7 +68,7 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) OPT_BIT(0, "refs", &flags, N_("do not show peeled tags"), REF_NORMAL), OPT_BOOL(0, "get-url", &get_url, N_("take url.<base>.insteadOf into account")), - OPT_REF_SORT(sorting_tail), + OPT_REF_SORT(&sorting_options), OPT_SET_INT_F(0, "exit-code", &status, N_("exit with exit code 2 if no matching refs are found"), 2, PARSE_OPT_NOCOMPLETE), @@ -86,8 +86,6 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) packet_trace_identity("ls-remote"); - UNLEAK(sorting); - if (argc > 1) { int i; CALLOC_ARRAY(pattern, argc); @@ -139,8 +137,13 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) item->symref = xstrdup_or_null(ref->symref); } - if (sorting) + if (sorting_options.nr) { + struct ref_sorting *sorting; + + sorting = ref_sorting_options(&sorting_options); ref_array_sort(sorting, &ref_array); + ref_sorting_release(sorting); + } for (i = 0; i < ref_array.nr; i++) { const struct ref_array_item *ref = ref_array.items[i]; diff --git a/builtin/merge-file.c b/builtin/merge-file.c index 06a2f90c48..e695867ee5 100644 --- a/builtin/merge-file.c +++ b/builtin/merge-file.c @@ -34,6 +34,8 @@ int cmd_merge_file(int argc, const char **argv, const char *prefix) struct option options[] = { OPT_BOOL('p', "stdout", &to_stdout, N_("send results to standard output")), OPT_SET_INT(0, "diff3", &xmp.style, N_("use a diff3 based merge"), XDL_MERGE_DIFF3), + OPT_SET_INT(0, "zdiff3", &xmp.style, N_("use a zealous diff3 based merge"), + XDL_MERGE_ZEALOUS_DIFF3), OPT_SET_INT(0, "ours", &xmp.favor, N_("for conflicts, use our version"), XDL_MERGE_FAVOR_OURS), OPT_SET_INT(0, "theirs", &xmp.favor, N_("for conflicts, use their version"), diff --git a/builtin/merge.c b/builtin/merge.c index cc4a910c69..f061eb1166 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -448,6 +448,7 @@ static void finish(struct commit *head_commit, const struct object_id *new_head, const char *msg) { struct strbuf reflog_message = STRBUF_INIT; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; const struct object_id *head = &head_commit->object.oid; if (!msg) @@ -488,7 +489,8 @@ static void finish(struct commit *head_commit, } /* Run a post-merge hook */ - run_hook_le(NULL, "post-merge", squash ? "1" : "0", NULL); + strvec_push(&opt.args, squash ? "1" : "0"); + run_hooks_oneshot("post-merge", &opt); apply_autostash(git_path_merge_autostash(the_repository)); strbuf_release(&reflog_message); @@ -1565,8 +1567,7 @@ int cmd_merge(int argc, const char **argv, const char *prefix) if (autostash) create_autostash(the_repository, - git_path_merge_autostash(the_repository), - "merge"); + git_path_merge_autostash(the_repository)); if (checkout_fast_forward(the_repository, &head_commit->object.oid, &commit->object.oid, @@ -1578,6 +1579,7 @@ int cmd_merge(int argc, const char **argv, const char *prefix) finish(head_commit, remoteheads, &commit->object.oid, msg.buf); remove_merge_branch_state(the_repository); + strbuf_release(&msg); goto done; } else if (!remoteheads->next && common->next) ; @@ -1636,8 +1638,7 @@ int cmd_merge(int argc, const char **argv, const char *prefix) if (autostash) create_autostash(the_repository, - git_path_merge_autostash(the_repository), - "merge"); + git_path_merge_autostash(the_repository)); /* We are going to make a new commit. */ git_committer_info(IDENT_STRICT); @@ -1748,6 +1749,7 @@ int cmd_merge(int argc, const char **argv, const char *prefix) ret = suggest_conflicts(); done: + strbuf_release(&buf); free(branch_to_free); return ret; } diff --git a/builtin/mktag.c b/builtin/mktag.c index dddcccdd36..3b2dbbb37e 100644 --- a/builtin/mktag.c +++ b/builtin/mktag.c @@ -62,7 +62,8 @@ static int verify_object_in_tag(struct object_id *tagged_oid, int *tagged_type) repl = lookup_replace_object(the_repository, tagged_oid); ret = check_object_signature(the_repository, repl, - buffer, size, type_name(*tagged_type)); + buffer, size, type_name(*tagged_type), + NULL); free(buffer); return ret; diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 075d15d706..4480ba3982 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -167,6 +167,8 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv) usage_with_options(builtin_multi_pack_index_verify_usage, options); + FREE_AND_NULL(options); + return verify_midx_file(the_repository, opts.object_dir, opts.flags); } @@ -191,6 +193,8 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv) usage_with_options(builtin_multi_pack_index_expire_usage, options); + FREE_AND_NULL(options); + return expire_midx_packs(the_repository, opts.object_dir, opts.flags); } diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 1a3dd445f8..857be7826f 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -4148,11 +4148,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) read_packs_list_from_stdin(); if (rev_list_unpacked) add_unreachable_loose_objects(); - } else if (!use_internal_rev_list) + } else if (!use_internal_rev_list) { read_object_list_from_stdin(); - else { + } else { get_object_list(rp.nr, rp.v); - strvec_clear(&rp); } cleanup_preferred_base(); if (include_tag && nr_result) @@ -4162,7 +4161,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) the_repository); if (non_empty && !nr_result) - return 0; + goto cleanup; if (nr_result) { trace2_region_enter("pack-objects", "prepare-pack", the_repository); @@ -4183,5 +4182,9 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) " pack-reused %"PRIu32), written, written_delta, reused, reused_delta, reuse_packfile_objects); + +cleanup: + strvec_clear(&rp); + return 0; } diff --git a/builtin/prune.c b/builtin/prune.c index 485c9a3c56..a76e6a5f0e 100644 --- a/builtin/prune.c +++ b/builtin/prune.c @@ -18,6 +18,7 @@ static int show_only; static int verbose; static timestamp_t expire; static int show_progress = -1; +static struct strbuf remove_dir_buf = STRBUF_INIT; static int prune_tmp_file(const char *fullpath) { @@ -26,10 +27,20 @@ static int prune_tmp_file(const char *fullpath) return error("Could not stat '%s'", fullpath); if (st.st_mtime > expire) return 0; - if (show_only || verbose) - printf("Removing stale temporary file %s\n", fullpath); - if (!show_only) - unlink_or_warn(fullpath); + if (S_ISDIR(st.st_mode)) { + if (show_only || verbose) + printf("Removing stale temporary directory %s\n", fullpath); + if (!show_only) { + strbuf_reset(&remove_dir_buf); + strbuf_addstr(&remove_dir_buf, fullpath); + remove_dir_recursively(&remove_dir_buf, 0); + } + } else { + if (show_only || verbose) + printf("Removing stale temporary file %s\n", fullpath); + if (!show_only) + unlink_or_warn(fullpath); + } return 0; } @@ -97,6 +108,9 @@ static int prune_cruft(const char *basename, const char *path, void *data) static int prune_subdir(unsigned int nr, const char *path, void *data) { + if (verbose) + printf("Removing directory %s\n", path); + if (!show_only) rmdir(path); return 0; @@ -184,5 +198,6 @@ int cmd_prune(int argc, const char **argv, const char *prefix) prune_shallow(show_only ? PRUNE_SHOW_ONLY : 0); } + strbuf_release(&remove_dir_buf); return 0; } diff --git a/builtin/pull.c b/builtin/pull.c index ae9f5bd7cc..127798ba84 100644 --- a/builtin/pull.c +++ b/builtin/pull.c @@ -84,6 +84,7 @@ static char *opt_edit; static char *cleanup_arg; static char *opt_ff; static char *opt_verify_signatures; +static char *opt_verify; static int opt_autostash = -1; static int config_autostash; static int check_trust_level = 1; @@ -160,6 +161,9 @@ static struct option pull_options[] = { OPT_PASSTHRU(0, "ff-only", &opt_ff, NULL, N_("abort if fast-forward is not possible"), PARSE_OPT_NOARG | PARSE_OPT_NONEG), + OPT_PASSTHRU(0, "verify", &opt_verify, NULL, + N_("control use of pre-merge-commit and commit-msg hooks"), + PARSE_OPT_NOARG), OPT_PASSTHRU(0, "verify-signatures", &opt_verify_signatures, NULL, N_("verify that the named commit has a valid GPG signature"), PARSE_OPT_NOARG), @@ -675,6 +679,8 @@ static int run_merge(void) strvec_pushf(&args, "--cleanup=%s", cleanup_arg); if (opt_ff) strvec_push(&args, opt_ff); + if (opt_verify) + strvec_push(&args, opt_verify); if (opt_verify_signatures) strvec_push(&args, opt_verify_signatures); strvec_pushv(&args, opt_strategies.v); @@ -931,6 +937,33 @@ static int get_can_ff(struct object_id *orig_head, return ret; } +/* + * Is orig_head a descendant of _all_ merge_heads? + * Unfortunately is_descendant_of() cannot be used as it asks + * if orig_head is a descendant of at least one of them. + */ +static int already_up_to_date(struct object_id *orig_head, + struct oid_array *merge_heads) +{ + int i; + struct commit *ours; + + ours = lookup_commit_reference(the_repository, orig_head); + for (i = 0; i < merge_heads->nr; i++) { + struct commit_list *list = NULL; + struct commit *theirs; + int ok; + + theirs = lookup_commit_reference(the_repository, &merge_heads->oid[i]); + commit_list_insert(theirs, &list); + ok = repo_is_descendant_of(the_repository, ours, list); + free_commit_list(list); + if (!ok) + return 0; + } + return 1; +} + static void show_advice_pull_non_ff(void) { advise(_("You have divergent branches and need to specify how to reconcile them.\n" @@ -1072,7 +1105,7 @@ int cmd_pull(int argc, const char **argv, const char *prefix) /* ff-only takes precedence over rebase */ if (opt_ff && !strcmp(opt_ff, "--ff-only")) { - if (!can_ff) + if (!can_ff && !already_up_to_date(&orig_head, &merge_heads)) die_ff_impossible(); opt_rebase = REBASE_FALSE; } diff --git a/builtin/rebase.c b/builtin/rebase.c index 34b4744e5f..3bf8031882 100644 --- a/builtin/rebase.c +++ b/builtin/rebase.c @@ -28,6 +28,7 @@ #include "sequencer.h" #include "rebase-interactive.h" #include "reset.h" +#include "hook.h" #define DEFAULT_REFLOG_ACTION "rebase" @@ -570,7 +571,8 @@ static int finish_rebase(struct rebase_options *opts) static int move_to_original_branch(struct rebase_options *opts) { - struct strbuf orig_head_reflog = STRBUF_INIT, head_reflog = STRBUF_INIT; + struct strbuf branch_reflog = STRBUF_INIT, head_reflog = STRBUF_INIT; + struct reset_head_opts ropts = { 0 }; int ret; if (!opts->head_name) @@ -579,16 +581,17 @@ static int move_to_original_branch(struct rebase_options *opts) if (!opts->onto) BUG("move_to_original_branch without onto"); - strbuf_addf(&orig_head_reflog, "rebase finished: %s onto %s", + strbuf_addf(&branch_reflog, "rebase finished: %s onto %s", opts->head_name, oid_to_hex(&opts->onto->object.oid)); strbuf_addf(&head_reflog, "rebase finished: returning to %s", opts->head_name); - ret = reset_head(the_repository, NULL, "", opts->head_name, - RESET_HEAD_REFS_ONLY, - orig_head_reflog.buf, head_reflog.buf, - DEFAULT_REFLOG_ACTION); + ropts.branch = opts->head_name; + ropts.flags = RESET_HEAD_REFS_ONLY; + ropts.branch_msg = branch_reflog.buf; + ropts.head_msg = head_reflog.buf; + ret = reset_head(the_repository, &ropts); - strbuf_release(&orig_head_reflog); + strbuf_release(&branch_reflog); strbuf_release(&head_reflog); return ret; } @@ -670,13 +673,15 @@ static int run_am(struct rebase_options *opts) status = run_command(&format_patch); if (status) { + struct reset_head_opts ropts = { 0 }; unlink(rebased_patches); free(rebased_patches); strvec_clear(&am.args); - reset_head(the_repository, &opts->orig_head, "checkout", - opts->head_name, 0, - "HEAD", NULL, DEFAULT_REFLOG_ACTION); + ropts.oid = &opts->orig_head; + ropts.branch = opts->head_name; + ropts.default_reflog_action = DEFAULT_REFLOG_ACTION; + reset_head(the_repository, &ropts); error(_("\ngit encountered an error while preparing the " "patches to replay\n" "these revisions:\n" @@ -812,6 +817,26 @@ static int rebase_config(const char *var, const char *value, void *data) return git_default_config(var, value, data); } +static int checkout_up_to_date(struct rebase_options *options) +{ + struct strbuf buf = STRBUF_INIT; + struct reset_head_opts ropts = { 0 }; + int ret = 0; + + strbuf_addf(&buf, "%s: checkout %s", + getenv(GIT_REFLOG_ACTION_ENVIRONMENT), + options->switch_to); + ropts.oid = &options->orig_head; + ropts.branch = options->head_name; + ropts.flags = RESET_HEAD_RUN_POST_CHECKOUT_HOOK; + ropts.head_msg = buf.buf; + if (reset_head(the_repository, &ropts) < 0) + ret = error(_("could not switch to %s"), options->switch_to); + strbuf_release(&buf); + + return ret; +} + /* * Determines whether the commits in from..to are linear, i.e. contain * no merge commits. This function *expects* `from` to be an ancestor of @@ -1017,6 +1042,8 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) int reschedule_failed_exec = -1; int allow_preemptive_ff = 1; int preserve_merges_selected = 0; + struct reset_head_opts ropts = { 0 }; + struct run_hooks_opt hook_opt = RUN_HOOKS_OPT_INIT; struct option builtin_rebase_options[] = { OPT_STRING(0, "onto", &options.onto_name, N_("revision"), @@ -1254,9 +1281,8 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) rerere_clear(the_repository, &merge_rr); string_list_clear(&merge_rr, 1); - - if (reset_head(the_repository, NULL, "reset", NULL, RESET_HEAD_HARD, - NULL, NULL, DEFAULT_REFLOG_ACTION) < 0) + ropts.flags = RESET_HEAD_HARD; + if (reset_head(the_repository, &ropts) < 0) die(_("could not discard worktree changes")); remove_branch_state(the_repository, 0); if (read_basic_state(&options)) @@ -1273,9 +1299,11 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) if (read_basic_state(&options)) exit(1); - if (reset_head(the_repository, &options.orig_head, "reset", - options.head_name, RESET_HEAD_HARD, - NULL, NULL, DEFAULT_REFLOG_ACTION) < 0) + ropts.oid = &options.orig_head; + ropts.branch = options.head_name; + ropts.flags = RESET_HEAD_HARD; + ropts.default_reflog_action = DEFAULT_REFLOG_ACTION; + if (reset_head(the_repository, &ropts) < 0) die(_("could not move back to %s"), oid_to_hex(&options.orig_head)); remove_branch_state(the_repository, 0); @@ -1642,8 +1670,8 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) die(_("could not read index")); if (options.autostash) { - create_autostash(the_repository, state_dir_path("autostash", &options), - DEFAULT_REFLOG_ACTION); + create_autostash(the_repository, + state_dir_path("autostash", &options)); } if (require_clean_work_tree(the_repository, "rebase", @@ -1673,21 +1701,9 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) if (!(options.flags & REBASE_FORCE)) { /* Lazily switch to the target branch if needed... */ if (options.switch_to) { - strbuf_reset(&buf); - strbuf_addf(&buf, "%s: checkout %s", - getenv(GIT_REFLOG_ACTION_ENVIRONMENT), - options.switch_to); - if (reset_head(the_repository, - &options.orig_head, "checkout", - options.head_name, - RESET_HEAD_RUN_POST_CHECKOUT_HOOK, - NULL, buf.buf, - DEFAULT_REFLOG_ACTION) < 0) { - ret = error(_("could not switch to " - "%s"), - options.switch_to); + ret = checkout_up_to_date(&options); + if (ret) goto cleanup; - } } if (!(options.flags & REBASE_NO_QUIET)) @@ -1711,9 +1727,11 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) } /* If a hook exists, give it a chance to interrupt*/ + strvec_push(&hook_opt.args, options.upstream_arg); + if (argc) + strvec_push(&hook_opt.args, argv[0]); if (!ok_to_skip_pre_rebase && - run_hook_le(NULL, "pre-rebase", options.upstream_arg, - argc ? argv[0] : NULL, NULL)) + run_hooks_oneshot("pre-rebase", &hook_opt)) die(_("The pre-rebase hook refused to rebase.")); if (options.flags & REBASE_DIFFSTAT) { @@ -1754,10 +1772,13 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) strbuf_addf(&msg, "%s: checkout %s", getenv(GIT_REFLOG_ACTION_ENVIRONMENT), options.onto_name); - if (reset_head(the_repository, &options.onto->object.oid, "checkout", NULL, - RESET_HEAD_DETACH | RESET_ORIG_HEAD | - RESET_HEAD_RUN_POST_CHECKOUT_HOOK, - NULL, msg.buf, DEFAULT_REFLOG_ACTION)) + ropts.oid = &options.onto->object.oid; + ropts.orig_head = &options.orig_head, + ropts.flags = RESET_HEAD_DETACH | RESET_ORIG_HEAD | + RESET_HEAD_RUN_POST_CHECKOUT_HOOK; + ropts.head_msg = msg.buf; + ropts.default_reflog_action = DEFAULT_REFLOG_ACTION; + if (reset_head(the_repository, &ropts)) die(_("Could not detach HEAD")); strbuf_release(&msg); @@ -1772,9 +1793,11 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) strbuf_addf(&msg, "rebase finished: %s onto %s", options.head_name ? options.head_name : "detached HEAD", oid_to_hex(&options.onto->object.oid)); - reset_head(the_repository, NULL, "Fast-forwarded", options.head_name, - RESET_HEAD_REFS_ONLY, "HEAD", msg.buf, - DEFAULT_REFLOG_ACTION); + memset(&ropts, 0, sizeof(ropts)); + ropts.branch = options.head_name; + ropts.flags = RESET_HEAD_REFS_ONLY; + ropts.head_msg = msg.buf; + reset_head(the_repository, &ropts); strbuf_release(&msg); ret = finish_rebase(&options); goto cleanup; diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index 25cc0c907e..878fca74af 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -135,6 +135,10 @@ static int receive_pack_config(const char *var, const char *value, void *cb) if (status) return status; + status = git_gpg_config(var, value, NULL); + if (status) + return status; + if (strcmp(var, "receive.denydeletes") == 0) { deny_deletes = git_config_bool(var, value); return 0; @@ -765,8 +769,10 @@ static void prepare_push_cert_sha1(struct child_process *proc) memset(&sigcheck, '\0', sizeof(sigcheck)); bogs = parse_signed_buffer(push_cert.buf, push_cert.len); - check_signature(push_cert.buf, bogs, push_cert.buf + bogs, - push_cert.len - bogs, &sigcheck); + sigcheck.payload = xmemdupz(push_cert.buf, bogs); + sigcheck.payload_len = bogs; + check_signature(&sigcheck, push_cert.buf + bogs, + push_cert.len - bogs); nonce_status = check_nonce(push_cert.buf, bogs); } @@ -1435,9 +1441,12 @@ static const char *push_to_checkout(unsigned char *hash, struct strvec *env, const char *work_tree) { + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + strvec_pushf(env, "GIT_WORK_TREE=%s", absolute_path(work_tree)); - if (run_hook_le(env->v, push_to_checkout_hook, - hash_to_hex(hash), NULL)) + strvec_pushv(&opt.env, env->v); + strvec_push(&opt.args, hash_to_hex(hash)); + if (run_hooks_oneshot(push_to_checkout_hook, &opt)) return "push-to-checkout hook declined"; else return NULL; @@ -2209,7 +2218,7 @@ static const char *unpack(int err_fd, struct shallow_info *si) strvec_push(&child.args, alt_shallow_file); } - tmp_objdir = tmp_objdir_create(); + tmp_objdir = tmp_objdir_create("incoming"); if (!tmp_objdir) { if (err_fd > 0) close(err_fd); diff --git a/builtin/reflog.c b/builtin/reflog.c index bd4c669918..175c83e7cc 100644 --- a/builtin/reflog.c +++ b/builtin/reflog.c @@ -653,6 +653,7 @@ static int cmd_reflog_expire(int argc, const char **argv, const char *prefix) should_expire_reflog_ent, reflog_expiry_cleanup, &cb); + free(ref); } return status; } diff --git a/builtin/repack.c b/builtin/repack.c index 0b2d1e5d82..9b74e0d468 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -258,9 +258,11 @@ static void repack_promisor_objects(const struct pack_objects_args *args, for_each_packed_object(write_oid, &cmd, FOR_EACH_OBJECT_PROMISOR_ONLY); - if (cmd.in == -1) + if (cmd.in == -1) { /* No packed objects; cmd was never started */ + child_process_clear(&cmd); return; + } close(cmd.in); diff --git a/builtin/reset.c b/builtin/reset.c index 7393595349..507704dd77 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -25,6 +25,7 @@ #include "cache-tree.h" #include "submodule.h" #include "submodule-config.h" +#include "dir.h" #define REFRESH_INDEX_DELAY_WARNING_IN_MS (2 * 1000) @@ -136,21 +137,36 @@ static void update_index_from_diff(struct diff_queue_struct *q, int intent_to_add = *(int *)data; for (i = 0; i < q->nr; i++) { + int pos; struct diff_filespec *one = q->queue[i]->one; - int is_missing = !(one->mode && !is_null_oid(&one->oid)); + int is_in_reset_tree = one->mode && !is_null_oid(&one->oid); struct cache_entry *ce; - if (is_missing && !intent_to_add) { + if (!is_in_reset_tree && !intent_to_add) { remove_file_from_cache(one->path); continue; } ce = make_cache_entry(&the_index, one->mode, &one->oid, one->path, 0, 0); + + /* + * If the file 1) corresponds to an existing index entry with + * skip-worktree set, or 2) does not exist in the index but is + * outside the sparse checkout definition, add a skip-worktree bit + * to the new index entry. Note that a sparse index will be expanded + * if this entry is outside the sparse cone - this is necessary + * to properly construct the reset sparse directory. + */ + pos = cache_name_pos(one->path, strlen(one->path)); + if ((pos >= 0 && ce_skip_worktree(active_cache[pos])) || + (pos < 0 && !path_in_sparse_checkout(one->path, &the_index))) + ce->ce_flags |= CE_SKIP_WORKTREE; + if (!ce) die(_("make_cache_entry failed for path '%s'"), one->path); - if (is_missing) { + if (!is_in_reset_tree) { ce->ce_flags |= CE_INTENT_TO_ADD; set_object_name_for_intent_to_add_entry(ce); } @@ -158,6 +174,73 @@ static void update_index_from_diff(struct diff_queue_struct *q, } } +static int pathspec_needs_expanded_index(const struct pathspec *pathspec) +{ + unsigned int i, pos; + int res = 0; + char *skip_worktree_seen = NULL; + + /* + * When using a magic pathspec, assume for the sake of simplicity that + * the index needs to be expanded to match all matchable files. + */ + if (pathspec->magic) + return 1; + + for (i = 0; i < pathspec->nr; i++) { + struct pathspec_item item = pathspec->items[i]; + + /* + * If the pathspec item has a wildcard, the index should be expanded + * if the pathspec has the possibility of matching a subset of entries inside + * of a sparse directory (but not the entire directory). + * + * If the pathspec item is a literal path, the index only needs to be expanded + * if a) the pathspec isn't in the sparse checkout cone (to make sure we don't + * expand for in-cone files) and b) it doesn't match any sparse directories + * (since we can reset whole sparse directories without expanding them). + */ + if (item.nowildcard_len < item.len) { + for (pos = 0; pos < active_nr; pos++) { + struct cache_entry *ce = active_cache[pos]; + + if (!S_ISSPARSEDIR(ce->ce_mode)) + continue; + + /* + * If the pre-wildcard length is longer than the sparse + * directory name and the sparse directory is the first + * component of the pathspec, need to expand the index. + */ + if (item.nowildcard_len > ce_namelen(ce) && + !strncmp(item.original, ce->name, ce_namelen(ce))) { + res = 1; + break; + } + + /* + * If the pre-wildcard length is shorter than the sparse + * directory and the pathspec does not match the whole + * directory, need to expand the index. + */ + if (!strncmp(item.original, ce->name, item.nowildcard_len) && + wildmatch(item.original, ce->name, 0)) { + res = 1; + break; + } + } + } else if (!path_in_cone_mode_sparse_checkout(item.original, &the_index) && + !matches_skip_worktree(pathspec, i, &skip_worktree_seen)) + res = 1; + + if (res > 0) + break; + } + + free(skip_worktree_seen); + return res; +} + static int read_from_tree(const struct pathspec *pathspec, struct object_id *tree_oid, int intent_to_add) @@ -170,7 +253,13 @@ static int read_from_tree(const struct pathspec *pathspec, opt.format_callback = update_index_from_diff; opt.format_callback_data = &intent_to_add; opt.flags.override_submodule_config = 1; + opt.flags.recursive = 1; opt.repo = the_repository; + opt.change = diff_change; + opt.add_remove = diff_addremove; + + if (pathspec->nr && the_index.sparse_index && pathspec_needs_expanded_index(pathspec)) + ensure_full_index(&the_index); if (do_diff_cache(tree_oid, &opt)) return 1; @@ -249,9 +338,6 @@ static void parse_args(struct pathspec *pathspec, } *rev_ret = rev; - if (read_cache() < 0) - die(_("index file corrupt")); - parse_pathspec(pathspec, 0, PATHSPEC_PREFER_FULL | (patch_mode ? PATHSPEC_PREFIX_ORIGIN : 0), @@ -397,6 +483,12 @@ int cmd_reset(int argc, const char **argv, const char *prefix) if (intent_to_add && reset_type != MIXED) die(_("-N can only be used with --mixed")); + prepare_repo_settings(the_repository); + the_repository->settings.command_requires_full_index = 0; + + if (read_cache() < 0) + die(_("index file corrupt")); + /* Soft reset does not touch the index file nor the working tree * at all, but requires them in a good order. Other resets reset * the index file to the tree object we are switching to. */ diff --git a/builtin/send-pack.c b/builtin/send-pack.c index 8932142312..69c432ef1a 100644 --- a/builtin/send-pack.c +++ b/builtin/send-pack.c @@ -87,6 +87,10 @@ static void print_helper_status(struct ref *ref) break; case REF_STATUS_EXPECTING_REPORT: + res = "error"; + msg = "expecting report"; + break; + default: continue; } diff --git a/builtin/shortlog.c b/builtin/shortlog.c index 3e7ab1ca82..e7f7af5de3 100644 --- a/builtin/shortlog.c +++ b/builtin/shortlog.c @@ -374,6 +374,9 @@ int cmd_shortlog(int argc, const char **argv, const char *prefix) for (;;) { switch (parse_options_step(&ctx, options, shortlog_usage)) { + case PARSE_OPT_NON_OPTION: + case PARSE_OPT_UNKNOWN: + break; case PARSE_OPT_HELP: case PARSE_OPT_ERROR: exit(129); diff --git a/builtin/stash.c b/builtin/stash.c index a0ccc8654d..18c812bbe0 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -27,11 +27,11 @@ static const char * const git_stash_usage[] = { N_("git stash ( pop | apply ) [--index] [-q|--quiet] [<stash>]"), N_("git stash branch <branchname> [<stash>]"), "git stash clear", - N_("git stash [push [-p|--patch] [-k|--[no-]keep-index] [-q|--quiet]\n" + N_("git stash [push [-p|--patch] [-S|--staged] [-k|--[no-]keep-index] [-q|--quiet]\n" " [-u|--include-untracked] [-a|--all] [-m|--message <message>]\n" " [--pathspec-from-file=<file> [--pathspec-file-nul]]\n" " [--] [<pathspec>...]]"), - N_("git stash save [-p|--patch] [-k|--[no-]keep-index] [-q|--quiet]\n" + N_("git stash save [-p|--patch] [-S|--staged] [-k|--[no-]keep-index] [-q|--quiet]\n" " [-u|--include-untracked] [-a|--all] [<message>]"), NULL }; @@ -1132,6 +1132,38 @@ done: return ret; } +static int stash_staged(struct stash_info *info, struct strbuf *out_patch, + int quiet) +{ + int ret = 0; + struct child_process cp_diff_tree = CHILD_PROCESS_INIT; + struct index_state istate = { NULL }; + + if (write_index_as_tree(&info->w_tree, &istate, the_repository->index_file, + 0, NULL)) { + ret = -1; + goto done; + } + + cp_diff_tree.git_cmd = 1; + strvec_pushl(&cp_diff_tree.args, "diff-tree", "-p", "-U1", "HEAD", + oid_to_hex(&info->w_tree), "--", NULL); + if (pipe_command(&cp_diff_tree, NULL, 0, out_patch, 0, NULL, 0)) { + ret = -1; + goto done; + } + + if (!out_patch->len) { + if (!quiet) + fprintf_ln(stderr, _("No staged changes")); + ret = 1; + } + +done: + discard_index(&istate); + return ret; +} + static int stash_patch(struct stash_info *info, const struct pathspec *ps, struct strbuf *out_patch, int quiet) { @@ -1258,7 +1290,7 @@ done: } static int do_create_stash(const struct pathspec *ps, struct strbuf *stash_msg_buf, - int include_untracked, int patch_mode, + int include_untracked, int patch_mode, int only_staged, struct stash_info *info, struct strbuf *patch, int quiet) { @@ -1337,6 +1369,16 @@ static int do_create_stash(const struct pathspec *ps, struct strbuf *stash_msg_b } else if (ret > 0) { goto done; } + } else if (only_staged) { + ret = stash_staged(info, patch, quiet); + if (ret < 0) { + if (!quiet) + fprintf_ln(stderr, _("Cannot save the current " + "staged state")); + goto done; + } else if (ret > 0) { + goto done; + } } else { if (stash_working_tree(info, ps)) { if (!quiet) @@ -1395,7 +1437,7 @@ static int create_stash(int argc, const char **argv, const char *prefix) if (!check_changes_tracked_files(&ps)) return 0; - ret = do_create_stash(&ps, &stash_msg_buf, 0, 0, &info, + ret = do_create_stash(&ps, &stash_msg_buf, 0, 0, 0, &info, NULL, 0); if (!ret) printf_ln("%s", oid_to_hex(&info.w_commit)); @@ -1405,7 +1447,7 @@ static int create_stash(int argc, const char **argv, const char *prefix) } static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int quiet, - int keep_index, int patch_mode, int include_untracked) + int keep_index, int patch_mode, int include_untracked, int only_staged) { int ret = 0; struct stash_info info; @@ -1423,6 +1465,17 @@ static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int q goto done; } + /* --patch overrides --staged */ + if (patch_mode) + only_staged = 0; + + if (only_staged && include_untracked) { + fprintf_ln(stderr, _("Can't use --staged and --include-untracked" + " or --all at the same time")); + ret = -1; + goto done; + } + read_cache_preload(NULL); if (!include_untracked && ps->nr) { int i; @@ -1463,7 +1516,7 @@ static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int q if (stash_msg) strbuf_addstr(&stash_msg_buf, stash_msg); - if (do_create_stash(ps, &stash_msg_buf, include_untracked, patch_mode, + if (do_create_stash(ps, &stash_msg_buf, include_untracked, patch_mode, only_staged, &info, &patch, quiet)) { ret = -1; goto done; @@ -1480,7 +1533,7 @@ static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int q printf_ln(_("Saved working directory and index state %s"), stash_msg_buf.buf); - if (!patch_mode) { + if (!(patch_mode || only_staged)) { if (include_untracked && !ps->nr) { struct child_process cp = CHILD_PROCESS_INIT; @@ -1598,6 +1651,7 @@ static int push_stash(int argc, const char **argv, const char *prefix, { int force_assume = 0; int keep_index = -1; + int only_staged = 0; int patch_mode = 0; int include_untracked = 0; int quiet = 0; @@ -1608,6 +1662,8 @@ static int push_stash(int argc, const char **argv, const char *prefix, struct option options[] = { OPT_BOOL('k', "keep-index", &keep_index, N_("keep index")), + OPT_BOOL('S', "staged", &only_staged, + N_("stash staged changes only")), OPT_BOOL('p', "patch", &patch_mode, N_("stash in patch mode")), OPT__QUIET(&quiet, N_("quiet mode")), @@ -1646,6 +1702,9 @@ static int push_stash(int argc, const char **argv, const char *prefix, if (patch_mode) die(_("--pathspec-from-file is incompatible with --patch")); + if (only_staged) + die(_("--pathspec-from-file is incompatible with --staged")); + if (ps.nr) die(_("--pathspec-from-file is incompatible with pathspec arguments")); @@ -1657,12 +1716,13 @@ static int push_stash(int argc, const char **argv, const char *prefix, } return do_push_stash(&ps, stash_msg, quiet, keep_index, patch_mode, - include_untracked); + include_untracked, only_staged); } static int save_stash(int argc, const char **argv, const char *prefix) { int keep_index = -1; + int only_staged = 0; int patch_mode = 0; int include_untracked = 0; int quiet = 0; @@ -1673,6 +1733,8 @@ static int save_stash(int argc, const char **argv, const char *prefix) struct option options[] = { OPT_BOOL('k', "keep-index", &keep_index, N_("keep index")), + OPT_BOOL('S', "staged", &only_staged, + N_("stash staged changes only")), OPT_BOOL('p', "patch", &patch_mode, N_("stash in patch mode")), OPT__QUIET(&quiet, N_("quiet mode")), @@ -1694,7 +1756,7 @@ static int save_stash(int argc, const char **argv, const char *prefix) memset(&ps, 0, sizeof(ps)); ret = do_push_stash(&ps, stash_msg, quiet, keep_index, - patch_mode, include_untracked); + patch_mode, include_untracked, only_staged); strbuf_release(&stash_msg_buf); return ret; diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index 6298cbdd4e..ff28ced877 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -1503,16 +1503,24 @@ static void deinit_submodule(const char *path, const char *prefix, struct strbuf sb_rm = STRBUF_INIT; const char *format; - /* - * protect submodules containing a .git directory - * NEEDSWORK: instead of dying, automatically call - * absorbgitdirs and (possibly) warn. - */ - if (is_directory(sub_git_dir)) - die(_("Submodule work tree '%s' contains a .git " - "directory (use 'rm -rf' if you really want " - "to remove it including all of its history)"), - displaypath); + if (is_directory(sub_git_dir)) { + if (!(flags & OPT_FORCE)) + die(_("Submodule work tree '%s' contains a " + ".git directory.\nUse --force if you want " + "to move its contents to superproject's " + "module directory and convert .git to a file " + "and then proceed with deinit."), + displaypath); + + if (!(flags & OPT_QUIET)) + warning(_("Submodule work tree '%s' contains a .git " + "directory. This will be replaced with a " + ".git file by using absorbgitdirs."), + displaypath); + + absorb_git_dir_into_superproject(displaypath, flags); + + } if (!(flags & OPT_FORCE)) { struct child_process cp_rm = CHILD_PROCESS_INIT; @@ -1838,6 +1846,10 @@ static int clone_submodule(struct module_clone_data *clone_data) git_config_set_in_file(p, "submodule.alternateErrorStrategy", error_strategy); + git_config_set_in_file(p, "submodule.superprojectGitdir", + relative_path(absolute_path(get_git_dir()), + sm_gitdir, &sb)); + free(sm_alternate); free(error_strategy); @@ -2999,7 +3011,7 @@ struct add_data { }; #define ADD_DATA_INIT { .depth = -1 } -static void show_fetch_remotes(FILE *output, const char *git_dir_path) +static void append_fetch_remotes(struct strbuf *msg, const char *git_dir_path) { struct child_process cp_remote = CHILD_PROCESS_INIT; struct strbuf sb_remote_out = STRBUF_INIT; @@ -3015,7 +3027,7 @@ static void show_fetch_remotes(FILE *output, const char *git_dir_path) while ((next_line = strchr(line, '\n')) != NULL) { size_t len = next_line - line; if (strip_suffix_mem(line, &len, " (fetch)")) - fprintf(output, " %.*s\n", (int)len, line); + strbuf_addf(msg, " %.*s\n", (int)len, line); line = next_line + 1; } } @@ -3047,19 +3059,27 @@ static int add_submodule(const struct add_data *add_data) if (is_directory(submod_gitdir_path)) { if (!add_data->force) { - fprintf(stderr, _("A git directory for '%s' is found " - "locally with remote(s):"), - add_data->sm_name); - show_fetch_remotes(stderr, submod_gitdir_path); + struct strbuf msg = STRBUF_INIT; + char *die_msg; + + strbuf_addf(&msg, _("A git directory for '%s' is found " + "locally with remote(s):\n"), + add_data->sm_name); + + append_fetch_remotes(&msg, submod_gitdir_path); free(submod_gitdir_path); - die(_("If you want to reuse this local git " - "directory instead of cloning again from\n" - " %s\n" - "use the '--force' option. If the local git " - "directory is not the correct repo\n" - "or if you are unsure what this means, choose " - "another name with the '--name' option.\n"), - add_data->realrepo); + + strbuf_addf(&msg, _("If you want to reuse this local git " + "directory instead of cloning again from\n" + " %s\n" + "use the '--force' option. If the local git " + "directory is not the correct repo\n" + "or you are unsure what this means choose " + "another name with the '--name' option."), + add_data->realrepo); + + die_msg = strbuf_detach(&msg, NULL); + die("%s", die_msg); } else { printf(_("Reactivating local git directory for " "submodule '%s'\n"), add_data->sm_name); @@ -3220,6 +3240,7 @@ static void die_on_index_match(const char *path, int force) } free(ps_matched); } + clear_pathspec(&ps); } static void die_on_repo_without_commits(const char *path) @@ -3231,6 +3252,7 @@ static void die_on_repo_without_commits(const char *path) if (resolve_gitlink_ref(path, "HEAD", &oid) < 0) die(_("'%s' does not have a commit checked out"), path); } + strbuf_release(&sb); } static int module_add(int argc, const char **argv, const char *prefix) diff --git a/builtin/tag.c b/builtin/tag.c index 6535ed27ee..41863c5ab7 100644 --- a/builtin/tag.c +++ b/builtin/tag.c @@ -178,7 +178,6 @@ static const char tag_template_nocleanup[] = static int git_tag_config(const char *var, const char *value, void *cb) { int status; - struct ref_sorting **sorting_tail = (struct ref_sorting **)cb; if (!strcmp(var, "tag.gpgsign")) { config_sign_tag = git_config_bool(var, value); @@ -188,7 +187,7 @@ static int git_tag_config(const char *var, const char *value, void *cb) if (!strcmp(var, "tag.sort")) { if (!value) return config_error_nonbool(var); - parse_ref_sorting(sorting_tail, value); + string_list_append(cb, value); return 0; } @@ -432,11 +431,12 @@ int cmd_tag(int argc, const char **argv, const char *prefix) int annotate = 0, force = 0; int cmdmode = 0, create_tag_object = 0; const char *msgfile = NULL, *keyid = NULL; - struct msg_arg msg = { 0, STRBUF_INIT }; + struct msg_arg msg = { .buf = STRBUF_INIT }; struct ref_transaction *transaction; struct strbuf err = STRBUF_INIT; struct ref_filter filter; - static struct ref_sorting *sorting = NULL, **sorting_tail = &sorting; + struct ref_sorting *sorting; + struct string_list sorting_options = STRING_LIST_INIT_DUP; struct ref_format format = REF_FORMAT_INIT; int icase = 0; int edit_flag = 0; @@ -470,7 +470,7 @@ int cmd_tag(int argc, const char **argv, const char *prefix) OPT_WITHOUT(&filter.no_commit, N_("print only tags that don't contain the commit")), OPT_MERGED(&filter, N_("print only tags that are merged")), OPT_NO_MERGED(&filter, N_("print only tags that are not merged")), - OPT_REF_SORT(sorting_tail), + OPT_REF_SORT(&sorting_options), { OPTION_CALLBACK, 0, "points-at", &filter.points_at, N_("object"), N_("print only tags of the object"), PARSE_OPT_LASTARG_DEFAULT, @@ -482,10 +482,11 @@ int cmd_tag(int argc, const char **argv, const char *prefix) OPT_BOOL('i', "ignore-case", &icase, N_("sorting and filtering are case insensitive")), OPT_END() }; + int ret = 0; setup_ref_filter_porcelain_msg(); - git_config(git_tag_config, sorting_tail); + git_config(git_tag_config, &sorting_options); memset(&opt, 0, sizeof(opt)); memset(&filter, 0, sizeof(filter)); @@ -524,12 +525,10 @@ int cmd_tag(int argc, const char **argv, const char *prefix) die(_("--column and -n are incompatible")); colopts = 0; } - if (!sorting) - sorting = ref_default_sorting(); + sorting = ref_sorting_options(&sorting_options); ref_sorting_set_sort_flags_all(sorting, REF_SORTING_ICASE, icase); filter.ignore_case = icase; if (cmdmode == 'l') { - int ret; if (column_active(colopts)) { struct column_options copts; memset(&copts, 0, sizeof(copts)); @@ -540,7 +539,7 @@ int cmd_tag(int argc, const char **argv, const char *prefix) ret = list_tags(&filter, sorting, &format); if (column_active(colopts)) stop_column_filter(); - return ret; + goto cleanup; } if (filter.lines != -1) die(_("-n option is only allowed in list mode")); @@ -552,12 +551,15 @@ int cmd_tag(int argc, const char **argv, const char *prefix) die(_("--points-at option is only allowed in list mode")); if (filter.reachable_from || filter.unreachable_from) die(_("--merged and --no-merged options are only allowed in list mode")); - if (cmdmode == 'd') - return delete_tags(argv); + if (cmdmode == 'd') { + ret = delete_tags(argv); + goto cleanup; + } if (cmdmode == 'v') { if (format.format && verify_ref_format(&format)) usage_with_options(git_tag_usage, options); - return for_each_tag_name(argv, verify_tag, &format); + ret = for_each_tag_name(argv, verify_tag, &format); + goto cleanup; } if (msg.given || msgfile) { @@ -626,10 +628,12 @@ int cmd_tag(int argc, const char **argv, const char *prefix) printf(_("Updated tag '%s' (was %s)\n"), tag, find_unique_abbrev(&prev, DEFAULT_ABBREV)); - UNLEAK(buf); - UNLEAK(ref); - UNLEAK(reflog_msg); - UNLEAK(msg); - UNLEAK(err); - return 0; +cleanup: + ref_sorting_release(sorting); + strbuf_release(&buf); + strbuf_release(&ref); + strbuf_release(&reflog_msg); + strbuf_release(&msg.buf); + strbuf_release(&err); + return ret; } diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index 4a9466295b..51eb4f7b53 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -1,5 +1,6 @@ #include "builtin.h" #include "cache.h" +#include "bulk-checkin.h" #include "config.h" #include "object-store.h" #include "object.h" @@ -503,10 +504,12 @@ static void unpack_all(void) if (!quiet) progress = start_progress(_("Unpacking objects"), nr_objects); CALLOC_ARRAY(obj_list, nr_objects); + plug_bulk_checkin(); for (i = 0; i < nr_objects; i++) { unpack_one(i); display_progress(progress, i + 1); } + unplug_bulk_checkin(); stop_progress(&progress); if (delta_list) diff --git a/builtin/update-index.c b/builtin/update-index.c index 187203e8bb..5b92876d3f 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -5,6 +5,7 @@ */ #define USE_THE_INDEX_COMPATIBILITY_MACROS #include "cache.h" +#include "bulk-checkin.h" #include "config.h" #include "lockfile.h" #include "quote.h" @@ -1088,6 +1089,9 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) the_index.updated_skipworktree = 1; + /* we might be adding many objects to the object database */ + plug_bulk_checkin(); + /* * Custom copy of parse_options() because we want to handle * filename arguments as they come. @@ -1168,6 +1172,8 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) strbuf_release(&buf); } + /* by now we must have added all of the new objects */ + unplug_bulk_checkin(); if (split_index > 0) { if (git_config_get_split_index() == 0) warning(_("core.splitIndex is set to false; " @@ -1214,14 +1220,25 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) } if (fsmonitor > 0) { - if (git_config_get_fsmonitor() == 0) + enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); + + if (fsm_mode == FSMONITOR_MODE_DISABLED) { + warning(_("core.useBuiltinFSMonitor is unset; " + "set it if you really want to enable the " + "builtin fsmonitor")); warning(_("core.fsmonitor is unset; " - "set it if you really want to " - "enable fsmonitor")); + "set it if you really want to enable the " + "hook-based fsmonitor")); + } add_fsmonitor(&the_index); report(_("fsmonitor enabled")); } else if (!fsmonitor) { - if (git_config_get_fsmonitor() == 1) + enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); + if (fsm_mode == FSMONITOR_MODE_IPC) + warning(_("core.useBuiltinFSMonitor is set; " + "remove it if you really want to " + "disable fsmonitor")); + if (fsm_mode == FSMONITOR_MODE_HOOK) warning(_("core.fsmonitor is set; " "remove it if you really want to " "disable fsmonitor")); diff --git a/builtin/worktree.c b/builtin/worktree.c index d22ece93e1..330867c19b 100644 --- a/builtin/worktree.c +++ b/builtin/worktree.c @@ -382,22 +382,18 @@ done: * is_junk is cleared, but do return appropriate code when hook fails. */ if (!ret && opts->checkout) { - const char *hook = find_hook("post-checkout"); - if (hook) { - const char *env[] = { "GIT_DIR", "GIT_WORK_TREE", NULL }; - cp.git_cmd = 0; - cp.no_stdin = 1; - cp.stdout_to_stderr = 1; - cp.dir = path; - cp.env = env; - cp.argv = NULL; - cp.trace2_hook_name = "post-checkout"; - strvec_pushl(&cp.args, absolute_path(hook), - oid_to_hex(null_oid()), - oid_to_hex(&commit->object.oid), - "1", NULL); - ret = run_command(&cp); - } + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + + strvec_pushl(&opt.env, "GIT_DIR", "GIT_WORK_TREE", NULL); + strvec_pushl(&opt.args, + oid_to_hex(null_oid()), + oid_to_hex(&commit->object.oid), + "1", + NULL); + opt.dir = path; + opt.absolute_path = 1; + + ret = run_hooks_oneshot("post-checkout", &opt); } strvec_clear(&child_env); diff --git a/bulk-checkin.c b/bulk-checkin.c index 8785b2ac80..4deee1af46 100644 --- a/bulk-checkin.c +++ b/bulk-checkin.c @@ -3,16 +3,22 @@ */ #include "cache.h" #include "bulk-checkin.h" +#include "lockfile.h" #include "repository.h" #include "csum-file.h" #include "pack.h" #include "strbuf.h" +#include "string-list.h" +#include "tmp-objdir.h" #include "packfile.h" #include "object-store.h" -static struct bulk_checkin_state { - unsigned plugged:1; +static int bulk_checkin_plugged; +static int needs_batch_fsync; + +static struct tmp_objdir *bulk_fsync_objdir; +static struct bulk_checkin_state { char *pack_tmp_name; struct hashfile *f; off_t offset; @@ -21,7 +27,7 @@ static struct bulk_checkin_state { struct pack_idx_entry **written; uint32_t alloc_written; uint32_t nr_written; -} state; +} bulk_checkin_state; static void finish_tmp_packfile(struct strbuf *basename, const char *pack_tmp_name, @@ -79,6 +85,34 @@ clear_exit: reprepare_packed_git(the_repository); } +/* + * Cleanup after batch-mode fsync_object_files. + */ +static void do_batch_fsync(void) +{ + /* + * Issue a full hardware flush against a temporary file to ensure + * that all objects are durable before any renames occur. The code in + * fsync_loose_object_bulk_checkin has already issued a writeout + * request, but it has not flushed any writeback cache in the storage + * hardware. + */ + + if (needs_batch_fsync) { + struct strbuf temp_path = STRBUF_INIT; + struct tempfile *temp; + + strbuf_addf(&temp_path, "%s/bulk_fsync_XXXXXX", get_object_directory()); + temp = xmks_tempfile(temp_path.buf); + fsync_or_die(get_tempfile_fd(temp), get_tempfile_path(temp)); + delete_tempfile(&temp); + strbuf_release(&temp_path); + } + + if (bulk_fsync_objdir) + tmp_objdir_migrate(bulk_fsync_objdir); +} + static int already_written(struct bulk_checkin_state *state, struct object_id *oid) { int i; @@ -273,25 +307,61 @@ static int deflate_to_pack(struct bulk_checkin_state *state, return 0; } +void fsync_loose_object_bulk_checkin(int fd) +{ + assert(fsync_object_files == FSYNC_OBJECT_FILES_BATCH); + + /* + * If we have a plugged bulk checkin, we issue a call that + * cleans the filesystem page cache but avoids a hardware flush + * command. Later on we will issue a single hardware flush + * before as part of do_batch_fsync. + */ + if (bulk_checkin_plugged && + git_fsync(fd, FSYNC_WRITEOUT_ONLY) >= 0) { + if (!needs_batch_fsync) + needs_batch_fsync = 1; + } else { + fsync_or_die(fd, "loose object file"); + } +} + int index_bulk_checkin(struct object_id *oid, int fd, size_t size, enum object_type type, const char *path, unsigned flags) { - int status = deflate_to_pack(&state, oid, fd, size, type, + int status = deflate_to_pack(&bulk_checkin_state, oid, fd, size, type, path, flags); - if (!state.plugged) - finish_bulk_checkin(&state); + if (!bulk_checkin_plugged) + finish_bulk_checkin(&bulk_checkin_state); return status; } void plug_bulk_checkin(void) { - state.plugged = 1; + assert(!bulk_checkin_plugged); + + /* + * A temporary object directory is used to hold the files + * while they are not fsynced. + */ + if (fsync_object_files == FSYNC_OBJECT_FILES_BATCH) { + bulk_fsync_objdir = tmp_objdir_create("bulk-fsync"); + if (!bulk_fsync_objdir) + die(_("Could not create temporary object directory for core.fsyncobjectfiles=batch")); + + tmp_objdir_replace_primary_odb(bulk_fsync_objdir, 0); + } + + bulk_checkin_plugged = 1; } void unplug_bulk_checkin(void) { - state.plugged = 0; - if (state.f) - finish_bulk_checkin(&state); + assert(bulk_checkin_plugged); + bulk_checkin_plugged = 0; + if (bulk_checkin_state.f) + finish_bulk_checkin(&bulk_checkin_state); + + do_batch_fsync(); } diff --git a/bulk-checkin.h b/bulk-checkin.h index b26f3dc3b7..08f292379b 100644 --- a/bulk-checkin.h +++ b/bulk-checkin.h @@ -6,6 +6,8 @@ #include "cache.h" +void fsync_loose_object_bulk_checkin(int fd); + int index_bulk_checkin(struct object_id *oid, int fd, size_t size, enum object_type type, const char *path, unsigned flags); diff --git a/cache-tree.c b/cache-tree.c index 90919f9e34..65ca993361 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -440,8 +440,9 @@ static int update_one(struct cache_tree *it, } else if (dryrun) { hash_object_file(the_hash_algo, buffer.buf, buffer.len, tree_type, &it->oid); - } else if (write_object_file(buffer.buf, buffer.len, tree_type, - &it->oid)) { + } else if (write_object_file_flags(buffer.buf, buffer.len, tree_type, + &it->oid, flags & WRITE_TREE_SILENT + ? HASH_SILENT : 0)) { strbuf_release(&buffer); return -1; } @@ -740,15 +741,26 @@ out: return ret; } +static void prime_cache_tree_sparse_dir(struct cache_tree *it, + struct tree *tree) +{ + + oidcpy(&it->oid, &tree->object.oid); + it->entry_count = 1; +} + static void prime_cache_tree_rec(struct repository *r, struct cache_tree *it, - struct tree *tree) + struct tree *tree, + struct strbuf *tree_path) { struct tree_desc desc; struct name_entry entry; int cnt; + int base_path_len = tree_path->len; oidcpy(&it->oid, &tree->object.oid); + init_tree_desc(&desc, tree->buffer, tree->size); cnt = 0; while (tree_entry(&desc, &entry)) { @@ -757,14 +769,40 @@ static void prime_cache_tree_rec(struct repository *r, else { struct cache_tree_sub *sub; struct tree *subtree = lookup_tree(r, &entry.oid); + if (!subtree->object.parsed) parse_tree(subtree); sub = cache_tree_sub(it, entry.path); sub->cache_tree = cache_tree(); - prime_cache_tree_rec(r, sub->cache_tree, subtree); + + /* + * Recursively-constructed subtree path is only needed when working + * in a sparse index (where it's used to determine whether the + * subtree is a sparse directory in the index). + */ + if (r->index->sparse_index) { + strbuf_setlen(tree_path, base_path_len); + strbuf_grow(tree_path, base_path_len + entry.pathlen + 1); + strbuf_add(tree_path, entry.path, entry.pathlen); + strbuf_addch(tree_path, '/'); + } + + /* + * If a sparse index is in use, the directory being processed may be + * sparse. To confirm that, we can check whether an entry with that + * exact name exists in the index. If it does, the created subtree + * should be sparse. Otherwise, cache tree expansion should continue + * as normal. + */ + if (r->index->sparse_index && + index_entry_exists(r->index, tree_path->buf, tree_path->len)) + prime_cache_tree_sparse_dir(sub->cache_tree, subtree); + else + prime_cache_tree_rec(r, sub->cache_tree, subtree, tree_path); cnt += sub->cache_tree->entry_count; } } + it->entry_count = cnt; } @@ -772,11 +810,14 @@ void prime_cache_tree(struct repository *r, struct index_state *istate, struct tree *tree) { + struct strbuf tree_path = STRBUF_INIT; + trace2_region_enter("cache-tree", "prime_cache_tree", the_repository); cache_tree_free(&istate->cache_tree); istate->cache_tree = cache_tree(); - prime_cache_tree_rec(r, istate->cache_tree, tree); + prime_cache_tree_rec(r, istate->cache_tree, tree, &tree_path); + strbuf_release(&tree_path); istate->cache_changed |= CACHE_TREE_CHANGED; trace2_region_leave("cache-tree", "prime_cache_tree", the_repository); } @@ -826,10 +867,17 @@ static void verify_one_sparse(struct repository *r, path->buf); } -static void verify_one(struct repository *r, - struct index_state *istate, - struct cache_tree *it, - struct strbuf *path) +/* + * Returns: + * 0 - Verification completed. + * 1 - Restart verification - a call to ensure_full_index() freed the cache + * tree that is being verified and verification needs to be restarted from + * the new toplevel cache tree. + */ +static int verify_one(struct repository *r, + struct index_state *istate, + struct cache_tree *it, + struct strbuf *path) { int i, pos, len = path->len; struct strbuf tree_buf = STRBUF_INIT; @@ -837,21 +885,30 @@ static void verify_one(struct repository *r, for (i = 0; i < it->subtree_nr; i++) { strbuf_addf(path, "%s/", it->down[i]->name); - verify_one(r, istate, it->down[i]->cache_tree, path); + if (verify_one(r, istate, it->down[i]->cache_tree, path)) + return 1; strbuf_setlen(path, len); } if (it->entry_count < 0 || /* no verification on tests (t7003) that replace trees */ lookup_replace_object(r, &it->oid) != &it->oid) - return; + return 0; if (path->len) { + /* + * If the index is sparse and the cache tree is not + * index_name_pos() may trigger ensure_full_index() which will + * free the tree that is being verified. + */ + int is_sparse = istate->sparse_index; pos = index_name_pos(istate, path->buf, path->len); + if (is_sparse && !istate->sparse_index) + return 1; if (pos >= 0) { verify_one_sparse(r, istate, it, path, pos); - return; + return 0; } pos = -pos - 1; @@ -899,6 +956,7 @@ static void verify_one(struct repository *r, oid_to_hex(&new_oid), oid_to_hex(&it->oid)); strbuf_setlen(path, len); strbuf_release(&tree_buf); + return 0; } void cache_tree_verify(struct repository *r, struct index_state *istate) @@ -907,6 +965,10 @@ void cache_tree_verify(struct repository *r, struct index_state *istate) if (!istate->cache_tree) return; - verify_one(r, istate, istate->cache_tree, &path); + if (verify_one(r, istate, istate->cache_tree, &path)) { + strbuf_reset(&path); + if (verify_one(r, istate, istate->cache_tree, &path)) + BUG("ensure_full_index() called twice while verifying cache tree"); + } strbuf_release(&path); } @@ -817,6 +817,16 @@ struct cache_entry *index_file_exists(struct index_state *istate, const char *na int index_name_pos(struct index_state *, const char *name, int namelen); /* + * Determines whether an entry with the given name exists within the + * given index. The return value is 1 if an exact match is found, otherwise + * it is 0. Note that, unlike index_name_pos, this function does not expand + * the index if it is sparse. If an item exists within the full index but it + * is contained within a sparse directory (and not in the sparse index), 0 is + * returned. + */ +int index_entry_exists(struct index_state *, const char *name, int namelen); + +/* * Some functions return the negative complement of an insert position when a * precise match was not found but a position was found where the entry would * need to be inserted. This helper protects that logic from any integer @@ -887,6 +897,7 @@ int ie_modified(struct index_state *, const struct cache_entry *, struct stat *, #define HASH_WRITE_OBJECT 1 #define HASH_FORMAT_CHECK 2 #define HASH_RENORMALIZE 4 +#define HASH_SILENT 8 int index_fd(struct index_state *istate, struct object_id *oid, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags); int index_path(struct index_state *istate, struct object_id *oid, const char *path, struct stat *st, unsigned flags); @@ -984,12 +995,18 @@ void reset_shared_repository(void); extern int read_replace_refs; extern char *git_replace_ref_base; -extern int fsync_object_files; +enum fsync_object_files_mode { + FSYNC_OBJECT_FILES_OFF, + FSYNC_OBJECT_FILES_ON, + FSYNC_OBJECT_FILES_BATCH +}; + +extern enum fsync_object_files_mode fsync_object_files; +extern int use_fsync; extern int core_preload_index; extern int precomposed_unicode; extern int protect_hfs; extern int protect_ntfs; -extern const char *core_fsmonitor; extern int core_apply_sparse_checkout; extern int core_sparse_checkout_cone; @@ -1268,11 +1285,50 @@ char *xdg_cache_home(const char *filename); int git_open_cloexec(const char *name, int flags); #define git_open(name) git_open_cloexec(name, O_RDONLY) -int unpack_loose_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz); -int parse_loose_header(const char *hdr, unsigned long *sizep); + +/** + * unpack_loose_header() initializes the data stream needed to unpack + * a loose object header. + * + * Returns: + * + * - ULHR_OK on success + * - ULHR_BAD on error + * - ULHR_TOO_LONG if the header was too long + * + * It will only parse up to MAX_HEADER_LEN bytes unless an optional + * "hdrbuf" argument is non-NULL. This is intended for use with + * OBJECT_INFO_ALLOW_UNKNOWN_TYPE to extract the bad type for (error) + * reporting. The full header will be extracted to "hdrbuf" for use + * with parse_loose_header(), ULHR_TOO_LONG will still be returned + * from this function to indicate that the header was too long. + */ +enum unpack_loose_header_result { + ULHR_OK, + ULHR_BAD, + ULHR_TOO_LONG, +}; +enum unpack_loose_header_result unpack_loose_header(git_zstream *stream, + unsigned char *map, + unsigned long mapsize, + void *buffer, + unsigned long bufsiz, + struct strbuf *hdrbuf); + +/** + * parse_loose_header() parses the starting "<type> <len>\0" of an + * object. If it doesn't follow that format -1 is returned. To check + * the validity of the <type> populate the "typep" in the "struct + * object_info". It will be OBJ_BAD if the object type is unknown. The + * parsed <len> can be retrieved via "oi->sizep", and from there + * passed to unpack_loose_rest(). + */ +struct object_info; +int parse_loose_header(const char *hdr, struct object_info *oi); int check_object_signature(struct repository *r, const struct object_id *oid, - void *buf, unsigned long size, const char *type); + void *buf, unsigned long size, const char *type, + struct object_id *real_oidp); int finalize_object_file(const char *tmpfile, const char *filename); @@ -224,6 +224,7 @@ linux-gcc-default) ;; Linux32) CC=gcc + MAKEFLAGS="$MAKEFLAGS NO_UNCOMPRESS2=1" ;; linux-musl) CC=gcc @@ -40,7 +40,7 @@ struct color { enum { COLOR_UNSPECIFIED = 0, COLOR_NORMAL, - COLOR_ANSI, /* basic 0-7 ANSI colors */ + COLOR_ANSI, /* basic 0-7 ANSI colors + "default" (value = 9) */ COLOR_256, COLOR_RGB } type; @@ -83,6 +83,27 @@ static int parse_ansi_color(struct color *out, const char *name, int len) int i; int color_offset = COLOR_FOREGROUND_ANSI; + if (match_word(name, len, "default")) { + /* + * Restores to the terminal's default color, which may not be + * the same as explicitly setting "white" or "black". + * + * ECMA-48 - Control Functions \ + * for Coded Character Sets, 5th edition (June 1991): + * > 39 default display colour (implementation-defined) + * > 49 default background colour (implementation-defined) + * + * Although not supported /everywhere/--according to terminfo, + * some terminals define "op" (original pair) as a blunt + * "set to white on black", or even "send full SGR reset"-- + * it's standard and well-supported enough that if a user + * asks for it in their config this will do the right thing. + */ + out->type = COLOR_ANSI; + out->value = 9 + color_offset; + return 0; + } + if (strncasecmp(name, "bright", 6) == 0) { color_offset = COLOR_FOREGROUND_BRIGHT_ANSI; name += 6; @@ -234,6 +255,7 @@ int color_parse_mem(const char *value, int value_len, char *dst) const char *ptr = value; int len = value_len; char *end = dst + COLOR_MAXLEN; + unsigned int has_reset = 0; unsigned int attr = 0; struct color fg = { COLOR_UNSPECIFIED }; struct color bg = { COLOR_UNSPECIFIED }; @@ -248,12 +270,7 @@ int color_parse_mem(const char *value, int value_len, char *dst) return 0; } - if (!strncasecmp(ptr, "reset", len)) { - xsnprintf(dst, end - dst, GIT_COLOR_RESET); - return 0; - } - - /* [fg [bg]] [attr]... */ + /* [reset] [fg [bg]] [attr]... */ while (len > 0) { const char *word = ptr; struct color c = { COLOR_UNSPECIFIED }; @@ -270,6 +287,11 @@ int color_parse_mem(const char *value, int value_len, char *dst) len--; } + if (match_word(word, wordlen, "reset")) { + has_reset = 1; + continue; + } + if (!parse_color(&c, word, wordlen)) { if (fg.type == COLOR_UNSPECIFIED) { fg = c; @@ -295,13 +317,16 @@ int color_parse_mem(const char *value, int value_len, char *dst) *dst++ = (x); \ } while(0) - if (attr || !color_empty(&fg) || !color_empty(&bg)) { + if (has_reset || attr || !color_empty(&fg) || !color_empty(&bg)) { int sep = 0; int i; OUT('\033'); OUT('['); + if (has_reset) + sep++; + for (i = 0; attr; i++) { unsigned bit = (1 << i); if (!(attr & bit)) @@ -6,6 +6,7 @@ struct strbuf; /* * The maximum length of ANSI color sequence we would generate: * - leading ESC '[' 2 + * - reset ';' .................1 * - attr + ';' 2 * num_attr (e.g. "1;") * - no-attr + ';' 3 * num_attr (e.g. "22;") * - fg color + ';' 17 (e.g. "38;2;255;255;255;") @@ -24,30 +25,42 @@ struct strbuf; #define GIT_COLOR_NORMAL "" #define GIT_COLOR_RESET "\033[m" #define GIT_COLOR_BOLD "\033[1m" +#define GIT_COLOR_BLACK "\033[30m" #define GIT_COLOR_RED "\033[31m" #define GIT_COLOR_GREEN "\033[32m" #define GIT_COLOR_YELLOW "\033[33m" #define GIT_COLOR_BLUE "\033[34m" #define GIT_COLOR_MAGENTA "\033[35m" #define GIT_COLOR_CYAN "\033[36m" +#define GIT_COLOR_WHITE "\033[37m" +#define GIT_COLOR_DEFAULT "\033[39m" +#define GIT_COLOR_BOLD_BLACK "\033[1;30m" #define GIT_COLOR_BOLD_RED "\033[1;31m" #define GIT_COLOR_BOLD_GREEN "\033[1;32m" #define GIT_COLOR_BOLD_YELLOW "\033[1;33m" #define GIT_COLOR_BOLD_BLUE "\033[1;34m" #define GIT_COLOR_BOLD_MAGENTA "\033[1;35m" #define GIT_COLOR_BOLD_CYAN "\033[1;36m" +#define GIT_COLOR_BOLD_WHITE "\033[1;37m" +#define GIT_COLOR_BOLD_DEFAULT "\033[1;39m" +#define GIT_COLOR_FAINT_BLACK "\033[2;30m" #define GIT_COLOR_FAINT_RED "\033[2;31m" #define GIT_COLOR_FAINT_GREEN "\033[2;32m" #define GIT_COLOR_FAINT_YELLOW "\033[2;33m" #define GIT_COLOR_FAINT_BLUE "\033[2;34m" #define GIT_COLOR_FAINT_MAGENTA "\033[2;35m" #define GIT_COLOR_FAINT_CYAN "\033[2;36m" +#define GIT_COLOR_FAINT_WHITE "\033[2;37m" +#define GIT_COLOR_FAINT_DEFAULT "\033[2;39m" +#define GIT_COLOR_BG_BLACK "\033[40m" #define GIT_COLOR_BG_RED "\033[41m" #define GIT_COLOR_BG_GREEN "\033[42m" #define GIT_COLOR_BG_YELLOW "\033[43m" #define GIT_COLOR_BG_BLUE "\033[44m" #define GIT_COLOR_BG_MAGENTA "\033[45m" #define GIT_COLOR_BG_CYAN "\033[46m" +#define GIT_COLOR_BG_WHITE "\033[47m" +#define GIT_COLOR_BG_DEFAULT "\033[49m" #define GIT_COLOR_FAINT "\033[2m" #define GIT_COLOR_FAINT_ITALIC "\033[2;3m" #define GIT_COLOR_REVERSE "\033[7m" diff --git a/command-list.txt b/command-list.txt index a289f09ed6..e42c35a1c2 100644 --- a/command-list.txt +++ b/command-list.txt @@ -60,9 +60,9 @@ git-cat-file plumbinginterrogators git-check-attr purehelpers git-check-ignore purehelpers git-check-mailmap purehelpers +git-check-ref-format purehelpers git-checkout mainporcelain git-checkout-index plumbingmanipulators -git-check-ref-format purehelpers git-cherry plumbinginterrogators complete git-cherry-pick mainporcelain git-citool mainporcelain @@ -103,6 +103,7 @@ git-grep mainporcelain info git-gui mainporcelain git-hash-object plumbingmanipulators git-help ancillaryinterrogators complete +git-hook purehelpers git-http-backend synchingrepositories git-http-fetch synchelpers git-http-push synchelpers @@ -111,7 +112,6 @@ git-index-pack plumbingmanipulators git-init mainporcelain init git-instaweb ancillaryinterrogators complete git-interpret-trailers purehelpers -gitk mainporcelain git-log mainporcelain info git-ls-files plumbinginterrogators git-ls-remote plumbinginterrogators @@ -124,11 +124,11 @@ git-merge-base plumbinginterrogators git-merge-file plumbingmanipulators git-merge-index plumbingmanipulators git-merge-one-file purehelpers -git-mergetool ancillarymanipulators complete git-merge-tree ancillaryinterrogators -git-multi-pack-index plumbingmanipulators +git-mergetool ancillarymanipulators complete git-mktag plumbingmanipulators git-mktree plumbingmanipulators +git-multi-pack-index plumbingmanipulators git-mv mainporcelain worktree git-name-rev plumbinginterrogators git-notes mainporcelain @@ -154,23 +154,23 @@ git-request-pull foreignscminterface complete git-rerere ancillaryinterrogators git-reset mainporcelain history git-restore mainporcelain worktree -git-revert mainporcelain git-rev-list plumbinginterrogators git-rev-parse plumbinginterrogators +git-revert mainporcelain git-rm mainporcelain worktree git-send-email foreignscminterface complete git-send-pack synchingrepositories +git-sh-i18n purehelpers +git-sh-setup purehelpers git-shell synchelpers git-shortlog mainporcelain git-show mainporcelain info git-show-branch ancillaryinterrogators complete git-show-index plumbinginterrogators git-show-ref plumbinginterrogators -git-sh-i18n purehelpers -git-sh-setup purehelpers -git-sparse-checkout mainporcelain worktree -git-stash mainporcelain +git-sparse-checkout mainporcelain git-stage complete +git-stash mainporcelain git-status mainporcelain info git-stripspace purehelpers git-submodule mainporcelain @@ -189,10 +189,11 @@ git-var plumbinginterrogators git-verify-commit ancillaryinterrogators git-verify-pack plumbinginterrogators git-verify-tag ancillaryinterrogators -gitweb ancillaryinterrogators git-whatchanged ancillaryinterrogators complete git-worktree mainporcelain git-write-tree plumbingmanipulators +gitk mainporcelain +gitweb ancillaryinterrogators gitattributes guide gitcli guide gitcore-tutorial guide @@ -211,6 +212,6 @@ gitremote-helpers guide gitrepository-layout guide gitrevisions guide gitsubmodules guide -gittutorial-2 guide gittutorial guide +gittutorial-2 guide gitworkflows guide @@ -21,6 +21,7 @@ #include "commit-reach.h" #include "run-command.h" #include "shallow.h" +#include "hook.h" static struct commit_extra_header *read_commit_extra_header_lines(const char *buf, size_t len, const char **); @@ -1212,8 +1213,10 @@ int check_commit_signature(const struct commit *commit, struct signature_check * if (parse_signed_commit(commit, &payload, &signature, the_hash_algo) <= 0) goto out; - ret = check_signature(payload.buf, payload.len, signature.buf, - signature.len, sigc); + + sigc->payload_type = SIGNATURE_PAYLOAD_COMMIT; + sigc->payload = strbuf_detach(&payload, &sigc->payload_len); + ret = check_signature(sigc, signature.buf, signature.len); out: strbuf_release(&payload); @@ -1700,22 +1703,22 @@ size_t ignore_non_trailer(const char *buf, size_t len) int run_commit_hook(int editor_is_used, const char *index_file, const char *name, ...) { - struct strvec hook_env = STRVEC_INIT; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; va_list args; - int ret; + const char *arg; - strvec_pushf(&hook_env, "GIT_INDEX_FILE=%s", index_file); + strvec_pushf(&opt.env, "GIT_INDEX_FILE=%s", index_file); /* * Let the hook know that no editor will be launched. */ if (!editor_is_used) - strvec_push(&hook_env, "GIT_EDITOR=:"); + strvec_push(&opt.env, "GIT_EDITOR=:"); va_start(args, name); - ret = run_hook_ve(hook_env.v, name, args); + while ((arg = va_arg(args, const char *))) + strvec_push(&opt.args, arg); va_end(args); - strvec_clear(&hook_env); - return ret; + return run_hooks_oneshot(name, &opt); } diff --git a/compat/.gitattributes b/compat/.gitattributes new file mode 100644 index 0000000000..40dbfb170d --- /dev/null +++ b/compat/.gitattributes @@ -0,0 +1 @@ +/zlib-uncompress2.c whitespace=-indent-with-non-tab,-trailing-space diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c new file mode 100644 index 0000000000..2aefdc14d8 --- /dev/null +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -0,0 +1,496 @@ +#ifndef __clang__ +/* + * It is possible to #include CoreFoundation/CoreFoundation.h when compiling + * with clang, but not with GCC as of time of writing. + * + * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93082 for details. + */ +typedef unsigned int FSEventStreamCreateFlags; +#define kFSEventStreamEventFlagNone 0x00000000 +#define kFSEventStreamEventFlagMustScanSubDirs 0x00000001 +#define kFSEventStreamEventFlagUserDropped 0x00000002 +#define kFSEventStreamEventFlagKernelDropped 0x00000004 +#define kFSEventStreamEventFlagEventIdsWrapped 0x00000008 +#define kFSEventStreamEventFlagHistoryDone 0x00000010 +#define kFSEventStreamEventFlagRootChanged 0x00000020 +#define kFSEventStreamEventFlagMount 0x00000040 +#define kFSEventStreamEventFlagUnmount 0x00000080 +#define kFSEventStreamEventFlagItemCreated 0x00000100 +#define kFSEventStreamEventFlagItemRemoved 0x00000200 +#define kFSEventStreamEventFlagItemInodeMetaMod 0x00000400 +#define kFSEventStreamEventFlagItemRenamed 0x00000800 +#define kFSEventStreamEventFlagItemModified 0x00001000 +#define kFSEventStreamEventFlagItemFinderInfoMod 0x00002000 +#define kFSEventStreamEventFlagItemChangeOwner 0x00004000 +#define kFSEventStreamEventFlagItemXattrMod 0x00008000 +#define kFSEventStreamEventFlagItemIsFile 0x00010000 +#define kFSEventStreamEventFlagItemIsDir 0x00020000 +#define kFSEventStreamEventFlagItemIsSymlink 0x00040000 +#define kFSEventStreamEventFlagOwnEvent 0x00080000 +#define kFSEventStreamEventFlagItemIsHardlink 0x00100000 +#define kFSEventStreamEventFlagItemIsLastHardlink 0x00200000 +#define kFSEventStreamEventFlagItemCloned 0x00400000 + +typedef struct __FSEventStream *FSEventStreamRef; +typedef const FSEventStreamRef ConstFSEventStreamRef; + +typedef unsigned int CFStringEncoding; +#define kCFStringEncodingUTF8 0x08000100 + +typedef const struct __CFString *CFStringRef; +typedef const struct __CFArray *CFArrayRef; +typedef const struct __CFRunLoop *CFRunLoopRef; + +struct FSEventStreamContext { + long long version; + void *cb_data, *retain, *release, *copy_description; +}; + +typedef struct FSEventStreamContext FSEventStreamContext; +typedef unsigned int FSEventStreamEventFlags; +#define kFSEventStreamCreateFlagNoDefer 0x02 +#define kFSEventStreamCreateFlagWatchRoot 0x04 +#define kFSEventStreamCreateFlagFileEvents 0x10 + +typedef unsigned long long FSEventStreamEventId; +#define kFSEventStreamEventIdSinceNow 0xFFFFFFFFFFFFFFFFULL + +typedef void (*FSEventStreamCallback)(ConstFSEventStreamRef streamRef, + void *context, + __SIZE_TYPE__ num_of_events, + void *event_paths, + const FSEventStreamEventFlags event_flags[], + const FSEventStreamEventId event_ids[]); +typedef double CFTimeInterval; +FSEventStreamRef FSEventStreamCreate(void *allocator, + FSEventStreamCallback callback, + FSEventStreamContext *context, + CFArrayRef paths_to_watch, + FSEventStreamEventId since_when, + CFTimeInterval latency, + FSEventStreamCreateFlags flags); +CFStringRef CFStringCreateWithCString(void *allocator, const char *string, + CFStringEncoding encoding); +CFArrayRef CFArrayCreate(void *allocator, const void **items, long long count, + void *callbacks); +void CFRunLoopRun(void); +void CFRunLoopStop(CFRunLoopRef run_loop); +CFRunLoopRef CFRunLoopGetCurrent(void); +extern CFStringRef kCFRunLoopDefaultMode; +void FSEventStreamScheduleWithRunLoop(FSEventStreamRef stream, + CFRunLoopRef run_loop, + CFStringRef run_loop_mode); +unsigned char FSEventStreamStart(FSEventStreamRef stream); +void FSEventStreamStop(FSEventStreamRef stream); +void FSEventStreamInvalidate(FSEventStreamRef stream); +void FSEventStreamRelease(FSEventStreamRef stream); +#else +/* + * Let Apple's headers declare `isalnum()` first, before + * Git's headers override it via a constant + */ +#include <string.h> +#include <CoreFoundation/CoreFoundation.h> +#include <CoreServices/CoreServices.h> +#endif + +#include "cache.h" +#include "fsmonitor.h" +#include "fsm-listen.h" +#include "fsmonitor--daemon.h" + +struct fsmonitor_daemon_backend_data +{ + CFStringRef cfsr_worktree_path; + CFStringRef cfsr_gitdir_path; + + CFArrayRef cfar_paths_to_watch; + int nr_paths_watching; + + FSEventStreamRef stream; + + CFRunLoopRef rl; + + enum shutdown_style { + SHUTDOWN_EVENT = 0, + FORCE_SHUTDOWN, + FORCE_ERROR_STOP, + } shutdown_style; + + unsigned int stream_scheduled:1; + unsigned int stream_started:1; +}; + +static void log_flags_set(const char *path, const FSEventStreamEventFlags flag) +{ + struct strbuf msg = STRBUF_INIT; + + if (flag & kFSEventStreamEventFlagMustScanSubDirs) + strbuf_addstr(&msg, "MustScanSubDirs|"); + if (flag & kFSEventStreamEventFlagUserDropped) + strbuf_addstr(&msg, "UserDropped|"); + if (flag & kFSEventStreamEventFlagKernelDropped) + strbuf_addstr(&msg, "KernelDropped|"); + if (flag & kFSEventStreamEventFlagEventIdsWrapped) + strbuf_addstr(&msg, "EventIdsWrapped|"); + if (flag & kFSEventStreamEventFlagHistoryDone) + strbuf_addstr(&msg, "HistoryDone|"); + if (flag & kFSEventStreamEventFlagRootChanged) + strbuf_addstr(&msg, "RootChanged|"); + if (flag & kFSEventStreamEventFlagMount) + strbuf_addstr(&msg, "Mount|"); + if (flag & kFSEventStreamEventFlagUnmount) + strbuf_addstr(&msg, "Unmount|"); + if (flag & kFSEventStreamEventFlagItemChangeOwner) + strbuf_addstr(&msg, "ItemChangeOwner|"); + if (flag & kFSEventStreamEventFlagItemCreated) + strbuf_addstr(&msg, "ItemCreated|"); + if (flag & kFSEventStreamEventFlagItemFinderInfoMod) + strbuf_addstr(&msg, "ItemFinderInfoMod|"); + if (flag & kFSEventStreamEventFlagItemInodeMetaMod) + strbuf_addstr(&msg, "ItemInodeMetaMod|"); + if (flag & kFSEventStreamEventFlagItemIsDir) + strbuf_addstr(&msg, "ItemIsDir|"); + if (flag & kFSEventStreamEventFlagItemIsFile) + strbuf_addstr(&msg, "ItemIsFile|"); + if (flag & kFSEventStreamEventFlagItemIsHardlink) + strbuf_addstr(&msg, "ItemIsHardlink|"); + if (flag & kFSEventStreamEventFlagItemIsLastHardlink) + strbuf_addstr(&msg, "ItemIsLastHardlink|"); + if (flag & kFSEventStreamEventFlagItemIsSymlink) + strbuf_addstr(&msg, "ItemIsSymlink|"); + if (flag & kFSEventStreamEventFlagItemModified) + strbuf_addstr(&msg, "ItemModified|"); + if (flag & kFSEventStreamEventFlagItemRemoved) + strbuf_addstr(&msg, "ItemRemoved|"); + if (flag & kFSEventStreamEventFlagItemRenamed) + strbuf_addstr(&msg, "ItemRenamed|"); + if (flag & kFSEventStreamEventFlagItemXattrMod) + strbuf_addstr(&msg, "ItemXattrMod|"); + if (flag & kFSEventStreamEventFlagOwnEvent) + strbuf_addstr(&msg, "OwnEvent|"); + if (flag & kFSEventStreamEventFlagItemCloned) + strbuf_addstr(&msg, "ItemCloned|"); + + trace_printf_key(&trace_fsmonitor, "fsevent: '%s', flags=%u %s", + path, flag, msg.buf); + + strbuf_release(&msg); +} + +static int ef_is_root_delete(const FSEventStreamEventFlags ef) +{ + return (ef & kFSEventStreamEventFlagItemIsDir && + ef & kFSEventStreamEventFlagItemRemoved); +} + +static int ef_is_root_renamed(const FSEventStreamEventFlags ef) +{ + return (ef & kFSEventStreamEventFlagItemIsDir && + ef & kFSEventStreamEventFlagItemRenamed); +} + +static int ef_is_dropped(const FSEventStreamEventFlags ef) +{ + return (ef & kFSEventStreamEventFlagMustScanSubDirs || + ef & kFSEventStreamEventFlagKernelDropped || + ef & kFSEventStreamEventFlagUserDropped); +} + +static void fsevent_callback(ConstFSEventStreamRef streamRef, + void *ctx, + size_t num_of_events, + void *event_paths, + const FSEventStreamEventFlags event_flags[], + const FSEventStreamEventId event_ids[]) +{ + struct fsmonitor_daemon_state *state = ctx; + struct fsmonitor_daemon_backend_data *data = state->backend_data; + char **paths = (char **)event_paths; + struct fsmonitor_batch *batch = NULL; + struct string_list cookie_list = STRING_LIST_INIT_DUP; + const char *path_k; + const char *slash; + int k; + struct strbuf tmp = STRBUF_INIT; + + /* + * Build a list of all filesystem changes into a private/local + * list and without holding any locks. + */ + for (k = 0; k < num_of_events; k++) { + /* + * On Mac, we receive an array of absolute paths. + */ + path_k = paths[k]; + + /* + * If you want to debug FSEvents, log them to GIT_TRACE_FSMONITOR. + * Please don't log them to Trace2. + * + * trace_printf_key(&trace_fsmonitor, "Path: '%s'", path_k); + */ + + /* + * If event[k] is marked as dropped, we assume that we have + * lost sync with the filesystem and should flush our cached + * data. We need to: + * + * [1] Abort/wake any client threads waiting for a cookie and + * flush the cached state data (the current token), and + * create a new token. + * + * [2] Discard the batch that we were locally building (since + * they are conceptually relative to the just flushed + * token). + */ + if (ef_is_dropped(event_flags[k])) { + if (trace_pass_fl(&trace_fsmonitor)) + log_flags_set(path_k, event_flags[k]); + + fsmonitor_force_resync(state); + fsmonitor_batch__free_list(batch); + string_list_clear(&cookie_list, 0); + + /* + * We assume that any events that we received + * in this callback after this dropped event + * may still be valid, so we continue rather + * than break. (And just in case there is a + * delete of ".git" hiding in there.) + */ + continue; + } + + switch (fsmonitor_classify_path_absolute(state, path_k)) { + + case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: + case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: + /* special case cookie files within .git or gitdir */ + + /* Use just the filename of the cookie file. */ + slash = find_last_dir_sep(path_k); + string_list_append(&cookie_list, + slash ? slash + 1 : path_k); + break; + + case IS_INSIDE_DOT_GIT: + case IS_INSIDE_GITDIR: + /* ignore all other paths inside of .git or gitdir */ + break; + + case IS_DOT_GIT: + case IS_GITDIR: + /* + * If .git directory is deleted or renamed away, + * we have to quit. + */ + if (ef_is_root_delete(event_flags[k])) { + trace_printf_key(&trace_fsmonitor, + "event: gitdir removed"); + goto force_shutdown; + } + if (ef_is_root_renamed(event_flags[k])) { + trace_printf_key(&trace_fsmonitor, + "event: gitdir renamed"); + goto force_shutdown; + } + break; + + case IS_WORKDIR_PATH: + /* try to queue normal pathnames */ + + if (trace_pass_fl(&trace_fsmonitor)) + log_flags_set(path_k, event_flags[k]); + + /* + * Because of the implicit "binning" (the + * kernel calls us at a given frequency) and + * de-duping (the kernel is free to combine + * multiple events for a given pathname), an + * individual fsevent could be marked as both + * a file and directory. Add it to the queue + * with both spellings so that the client will + * know how much to invalidate/refresh. + */ + + if (event_flags[k] & kFSEventStreamEventFlagItemIsFile) { + const char *rel = path_k + + state->path_worktree_watch.len + 1; + + if (!batch) + batch = fsmonitor_batch__new(); + fsmonitor_batch__add_path(batch, rel); + } + + if (event_flags[k] & kFSEventStreamEventFlagItemIsDir) { + const char *rel = path_k + + state->path_worktree_watch.len + 1; + + strbuf_reset(&tmp); + strbuf_addstr(&tmp, rel); + strbuf_addch(&tmp, '/'); + + if (!batch) + batch = fsmonitor_batch__new(); + fsmonitor_batch__add_path(batch, tmp.buf); + } + + break; + + case IS_OUTSIDE_CONE: + default: + trace_printf_key(&trace_fsmonitor, + "ignoring '%s'", path_k); + break; + } + } + + fsmonitor_publish(state, batch, &cookie_list); + string_list_clear(&cookie_list, 0); + strbuf_release(&tmp); + return; + +force_shutdown: + fsmonitor_batch__free_list(batch); + string_list_clear(&cookie_list, 0); + + data->shutdown_style = FORCE_SHUTDOWN; + CFRunLoopStop(data->rl); + strbuf_release(&tmp); + return; +} + +/* + * NEEDSWORK: Investigate the proper value for the `latency` argument + * in the call to `FSEventStreamCreate()`. I'm not sure that this + * needs to be a config setting or just something that we tune after + * some testing. + * + * With a latency of 0.1, I was seeing lots of dropped events during + * the "touch 100000" files test within t/perf/p7519, but with a + * latency of 0.001 I did not see any dropped events. So the + * "correct" value may be somewhere in between. + * + * https://developer.apple.com/documentation/coreservices/1443980-fseventstreamcreate + */ + +int fsm_listen__ctor(struct fsmonitor_daemon_state *state) +{ + FSEventStreamCreateFlags flags = kFSEventStreamCreateFlagNoDefer | + kFSEventStreamCreateFlagWatchRoot | + kFSEventStreamCreateFlagFileEvents; + FSEventStreamContext ctx = { + 0, + state, + NULL, + NULL, + NULL + }; + struct fsmonitor_daemon_backend_data *data; + const void *dir_array[2]; + + CALLOC_ARRAY(data, 1); + state->backend_data = data; + + data->cfsr_worktree_path = CFStringCreateWithCString( + NULL, state->path_worktree_watch.buf, kCFStringEncodingUTF8); + dir_array[data->nr_paths_watching++] = data->cfsr_worktree_path; + + if (state->nr_paths_watching > 1) { + data->cfsr_gitdir_path = CFStringCreateWithCString( + NULL, state->path_gitdir_watch.buf, + kCFStringEncodingUTF8); + dir_array[data->nr_paths_watching++] = data->cfsr_gitdir_path; + } + + data->cfar_paths_to_watch = CFArrayCreate(NULL, dir_array, + data->nr_paths_watching, + NULL); + data->stream = FSEventStreamCreate(NULL, fsevent_callback, &ctx, + data->cfar_paths_to_watch, + kFSEventStreamEventIdSinceNow, + 0.001, flags); + if (data->stream == NULL) + goto failed; + + /* + * `data->rl` needs to be set inside the listener thread. + */ + + return 0; + +failed: + error("Unable to create FSEventStream."); + + FREE_AND_NULL(state->backend_data); + return -1; +} + +void fsm_listen__dtor(struct fsmonitor_daemon_state *state) +{ + struct fsmonitor_daemon_backend_data *data; + + if (!state || !state->backend_data) + return; + + data = state->backend_data; + + if (data->stream) { + if (data->stream_started) + FSEventStreamStop(data->stream); + if (data->stream_scheduled) + FSEventStreamInvalidate(data->stream); + FSEventStreamRelease(data->stream); + } + + FREE_AND_NULL(state->backend_data); +} + +void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) +{ + struct fsmonitor_daemon_backend_data *data; + + data = state->backend_data; + data->shutdown_style = SHUTDOWN_EVENT; + + CFRunLoopStop(data->rl); +} + +void fsm_listen__loop(struct fsmonitor_daemon_state *state) +{ + struct fsmonitor_daemon_backend_data *data; + + data = state->backend_data; + + data->rl = CFRunLoopGetCurrent(); + + FSEventStreamScheduleWithRunLoop(data->stream, data->rl, kCFRunLoopDefaultMode); + data->stream_scheduled = 1; + + if (!FSEventStreamStart(data->stream)) { + error("Failed to start the FSEventStream"); + goto force_error_stop_without_loop; + } + data->stream_started = 1; + + CFRunLoopRun(); + + switch (data->shutdown_style) { + case FORCE_ERROR_STOP: + state->error_code = -1; + /* fall thru */ + case FORCE_SHUTDOWN: + ipc_server_stop_async(state->ipc_server_data); + /* fall thru */ + case SHUTDOWN_EVENT: + default: + break; + } + return; + +force_error_stop_without_loop: + state->error_code = -1; + ipc_server_stop_async(state->ipc_server_data); + return; +} diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c new file mode 100644 index 0000000000..c2d11acbc1 --- /dev/null +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -0,0 +1,586 @@ +#include "cache.h" +#include "config.h" +#include "fsmonitor.h" +#include "fsm-listen.h" +#include "fsmonitor--daemon.h" + +/* + * The documentation of ReadDirectoryChangesW() states that the maximum + * buffer size is 64K when the monitored directory is remote. + * + * Larger buffers may be used when the monitored directory is local and + * will help us receive events faster from the kernel and avoid dropped + * events. + * + * So we try to use a very large buffer and silently fallback to 64K if + * we get an error. + */ +#define MAX_RDCW_BUF_FALLBACK (65536) +#define MAX_RDCW_BUF (65536 * 8) + +struct one_watch +{ + char buffer[MAX_RDCW_BUF]; + DWORD buf_len; + DWORD count; + + struct strbuf path; + HANDLE hDir; + HANDLE hEvent; + OVERLAPPED overlapped; + + /* + * Is there an active ReadDirectoryChangesW() call pending. If so, we + * need to later call GetOverlappedResult() and possibly CancelIoEx(). + */ + BOOL is_active; +}; + +struct fsmonitor_daemon_backend_data +{ + struct one_watch *watch_worktree; + struct one_watch *watch_gitdir; + + HANDLE hEventShutdown; + + HANDLE hListener[3]; /* we don't own these handles */ +#define LISTENER_SHUTDOWN 0 +#define LISTENER_HAVE_DATA_WORKTREE 1 +#define LISTENER_HAVE_DATA_GITDIR 2 + int nr_listener_handles; +}; + +/* + * Convert the WCHAR path from the notification into UTF8 and + * then normalize it. + */ +static int normalize_path_in_utf8(FILE_NOTIFY_INFORMATION *info, + struct strbuf *normalized_path) +{ + int reserve; + int len = 0; + + strbuf_reset(normalized_path); + if (!info->FileNameLength) + goto normalize; + + /* + * Pre-reserve enough space in the UTF8 buffer for + * each Unicode WCHAR character to be mapped into a + * sequence of 2 UTF8 characters. That should let us + * avoid ERROR_INSUFFICIENT_BUFFER 99.9+% of the time. + */ + reserve = info->FileNameLength + 1; + strbuf_grow(normalized_path, reserve); + + for (;;) { + len = WideCharToMultiByte(CP_UTF8, 0, info->FileName, + info->FileNameLength / sizeof(WCHAR), + normalized_path->buf, + strbuf_avail(normalized_path) - 1, + NULL, NULL); + if (len > 0) + goto normalize; + if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { + error("[GLE %ld] could not convert path to UTF-8: '%.*ls'", + GetLastError(), + (int)(info->FileNameLength / sizeof(WCHAR)), + info->FileName); + return -1; + } + + strbuf_grow(normalized_path, + strbuf_avail(normalized_path) + reserve); + } + +normalize: + strbuf_setlen(normalized_path, len); + return strbuf_normalize_path(normalized_path); +} + +void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) +{ + SetEvent(state->backend_data->hListener[LISTENER_SHUTDOWN]); +} + +static struct one_watch *create_watch(struct fsmonitor_daemon_state *state, + const char *path) +{ + struct one_watch *watch = NULL; + DWORD desired_access = FILE_LIST_DIRECTORY; + DWORD share_mode = + FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE; + HANDLE hDir; + wchar_t wpath[MAX_PATH]; + + if (xutftowcs_path(wpath, path) < 0) { + error(_("could not convert to wide characters: '%s'"), path); + return NULL; + } + + hDir = CreateFileW(wpath, + desired_access, share_mode, NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, + NULL); + if (hDir == INVALID_HANDLE_VALUE) { + error(_("[GLE %ld] could not watch '%s'"), + GetLastError(), path); + return NULL; + } + + CALLOC_ARRAY(watch, 1); + + watch->buf_len = sizeof(watch->buffer); /* assume full MAX_RDCW_BUF */ + + strbuf_init(&watch->path, 0); + strbuf_addstr(&watch->path, path); + + watch->hDir = hDir; + watch->hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + + return watch; +} + +static void destroy_watch(struct one_watch *watch) +{ + if (!watch) + return; + + strbuf_release(&watch->path); + if (watch->hDir != INVALID_HANDLE_VALUE) + CloseHandle(watch->hDir); + if (watch->hEvent != INVALID_HANDLE_VALUE) + CloseHandle(watch->hEvent); + + free(watch); +} + +static int start_rdcw_watch(struct fsmonitor_daemon_backend_data *data, + struct one_watch *watch) +{ + DWORD dwNotifyFilter = + FILE_NOTIFY_CHANGE_FILE_NAME | + FILE_NOTIFY_CHANGE_DIR_NAME | + FILE_NOTIFY_CHANGE_ATTRIBUTES | + FILE_NOTIFY_CHANGE_SIZE | + FILE_NOTIFY_CHANGE_LAST_WRITE | + FILE_NOTIFY_CHANGE_CREATION; + + ResetEvent(watch->hEvent); + + memset(&watch->overlapped, 0, sizeof(watch->overlapped)); + watch->overlapped.hEvent = watch->hEvent; + + /* + * Queue an async call using Overlapped IO. This returns immediately. + * Our event handle will be signalled when the real result is available. + * + * The return value here just means that we successfully queued it. + * We won't know if the Read...() actually produces data until later. + */ + watch->is_active = ReadDirectoryChangesW( + watch->hDir, watch->buffer, watch->buf_len, TRUE, + dwNotifyFilter, &watch->count, &watch->overlapped, NULL); + + if (watch->is_active) + return 0; + + error("ReadDirectoryChangedW failed on '%s' [GLE %ld]", + watch->path.buf, GetLastError()); + return -1; +} + +static int recv_rdcw_watch(struct one_watch *watch) +{ + DWORD gle; + + watch->is_active = FALSE; + + /* + * The overlapped result is ready. If the Read...() was successful + * we finally receive the actual result into our buffer. + */ + if (GetOverlappedResult(watch->hDir, &watch->overlapped, &watch->count, + TRUE)) + return 0; + + gle = GetLastError(); + if (gle == ERROR_INVALID_PARAMETER && + /* + * The kernel throws an invalid parameter error when our + * buffer is too big and we are pointed at a remote + * directory (and possibly for other reasons). Quietly + * set it down and try again. + * + * See note about MAX_RDCW_BUF at the top. + */ + watch->buf_len > MAX_RDCW_BUF_FALLBACK) { + watch->buf_len = MAX_RDCW_BUF_FALLBACK; + return -2; + } + + /* + * NEEDSWORK: If an external <gitdir> is deleted, the above + * returns an error. I'm not sure that there's anything that + * we can do here other than failing -- the <worktree>/.git + * link file would be broken anyway. We might try to check + * for that and return a better error message, but I'm not + * sure it is worth it. + */ + + error("GetOverlappedResult failed on '%s' [GLE %ld]", + watch->path.buf, gle); + return -1; +} + +static void cancel_rdcw_watch(struct one_watch *watch) +{ + DWORD count; + + if (!watch || !watch->is_active) + return; + + /* + * The calls to ReadDirectoryChangesW() and GetOverlappedResult() + * form a "pair" (my term) where we queue an IO and promise to + * hang around and wait for the kernel to give us the result. + * + * If for some reason after we queue the IO, we have to quit + * or otherwise not stick around for the second half, we must + * tell the kernel to abort the IO. This prevents the kernel + * from writing to our buffer and/or signalling our event + * after we free them. + * + * (Ask me how much fun it was to track that one down). + */ + CancelIoEx(watch->hDir, &watch->overlapped); + GetOverlappedResult(watch->hDir, &watch->overlapped, &count, TRUE); + watch->is_active = FALSE; +} + +/* + * Process filesystem events that happen anywhere (recursively) under the + * <worktree> root directory. For a normal working directory, this includes + * both version controlled files and the contents of the .git/ directory. + * + * If <worktree>/.git is a file, then we only see events for the file + * itself. + */ +static int process_worktree_events(struct fsmonitor_daemon_state *state) +{ + struct fsmonitor_daemon_backend_data *data = state->backend_data; + struct one_watch *watch = data->watch_worktree; + struct strbuf path = STRBUF_INIT; + struct string_list cookie_list = STRING_LIST_INIT_DUP; + struct fsmonitor_batch *batch = NULL; + const char *p = watch->buffer; + + /* + * If the kernel gets more events than will fit in the kernel + * buffer associated with our RDCW handle, it drops them and + * returns a count of zero. + * + * Yes, the call returns WITHOUT error and with length zero. + * This is the documented behavior. (My testing has confirmed + * that it also sets the last error to ERROR_NOTIFY_ENUM_DIR, + * but we do not rely on that since the function did not + * return an error and it is not documented.) + * + * (The "overflow" case is not ambiguous with the "no data" case + * because we did an INFINITE wait.) + * + * This means we have a gap in coverage. Tell the daemon layer + * to resync. + */ + if (!watch->count) { + trace2_data_string("fsmonitor", NULL, "fsm-listen/kernel", + "overflow"); + fsmonitor_force_resync(state); + return LISTENER_HAVE_DATA_WORKTREE; + } + + /* + * On Windows, `info` contains an "array" of paths that are + * relative to the root of whichever directory handle received + * the event. + */ + for (;;) { + FILE_NOTIFY_INFORMATION *info = (void *)p; + const char *slash; + enum fsmonitor_path_type t; + + strbuf_reset(&path); + if (normalize_path_in_utf8(info, &path) == -1) + goto skip_this_path; + + t = fsmonitor_classify_path_workdir_relative(path.buf); + + switch (t) { + case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: + /* special case cookie files within .git */ + + /* Use just the filename of the cookie file. */ + slash = find_last_dir_sep(path.buf); + string_list_append(&cookie_list, + slash ? slash + 1 : path.buf); + break; + + case IS_INSIDE_DOT_GIT: + /* ignore everything inside of "<worktree>/.git/" */ + break; + + case IS_DOT_GIT: + /* "<worktree>/.git" was deleted (or renamed away) */ + if ((info->Action == FILE_ACTION_REMOVED) || + (info->Action == FILE_ACTION_RENAMED_OLD_NAME)) { + trace2_data_string("fsmonitor", NULL, + "fsm-listen/dotgit", + "removed"); + goto force_shutdown; + } + break; + + case IS_WORKDIR_PATH: + /* queue normal pathname */ + if (!batch) + batch = fsmonitor_batch__new(); + fsmonitor_batch__add_path(batch, path.buf); + break; + + case IS_GITDIR: + case IS_INSIDE_GITDIR: + case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: + default: + BUG("unexpected path classification '%d' for '%s'", + t, path.buf); + } + +skip_this_path: + if (!info->NextEntryOffset) + break; + p += info->NextEntryOffset; + } + + fsmonitor_publish(state, batch, &cookie_list); + batch = NULL; + string_list_clear(&cookie_list, 0); + strbuf_release(&path); + return LISTENER_HAVE_DATA_WORKTREE; + +force_shutdown: + fsmonitor_batch__free_list(batch); + string_list_clear(&cookie_list, 0); + strbuf_release(&path); + return LISTENER_SHUTDOWN; +} + +/* + * Process filesystem events that happened anywhere (recursively) under the + * external <gitdir> (such as non-primary worktrees or submodules). + * We only care about cookie files that our client threads created here. + * + * Note that we DO NOT get filesystem events on the external <gitdir> + * itself (it is not inside something that we are watching). In particular, + * we do not get an event if the external <gitdir> is deleted. + */ +static int process_gitdir_events(struct fsmonitor_daemon_state *state) +{ + struct fsmonitor_daemon_backend_data *data = state->backend_data; + struct one_watch *watch = data->watch_gitdir; + struct strbuf path = STRBUF_INIT; + struct string_list cookie_list = STRING_LIST_INIT_DUP; + const char *p = watch->buffer; + + if (!watch->count) { + trace2_data_string("fsmonitor", NULL, "fsm-listen/kernel", + "overflow"); + fsmonitor_force_resync(state); + return LISTENER_HAVE_DATA_GITDIR; + } + + for (;;) { + FILE_NOTIFY_INFORMATION *info = (void *)p; + const char *slash; + enum fsmonitor_path_type t; + + strbuf_reset(&path); + if (normalize_path_in_utf8(info, &path) == -1) + goto skip_this_path; + + t = fsmonitor_classify_path_gitdir_relative(path.buf); + + switch (t) { + case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: + /* special case cookie files within gitdir */ + + /* Use just the filename of the cookie file. */ + slash = find_last_dir_sep(path.buf); + string_list_append(&cookie_list, + slash ? slash + 1 : path.buf); + break; + + case IS_INSIDE_GITDIR: + goto skip_this_path; + + default: + BUG("unexpected path classification '%d' for '%s'", + t, path.buf); + } + +skip_this_path: + if (!info->NextEntryOffset) + break; + p += info->NextEntryOffset; + } + + fsmonitor_publish(state, NULL, &cookie_list); + string_list_clear(&cookie_list, 0); + strbuf_release(&path); + return LISTENER_HAVE_DATA_GITDIR; +} + +void fsm_listen__loop(struct fsmonitor_daemon_state *state) +{ + struct fsmonitor_daemon_backend_data *data = state->backend_data; + DWORD dwWait; + int result; + + state->error_code = 0; + + if (start_rdcw_watch(data, data->watch_worktree) == -1) + goto force_error_stop; + + if (data->watch_gitdir && + start_rdcw_watch(data, data->watch_gitdir) == -1) + goto force_error_stop; + + for (;;) { + dwWait = WaitForMultipleObjects(data->nr_listener_handles, + data->hListener, + FALSE, INFINITE); + + if (dwWait == WAIT_OBJECT_0 + LISTENER_HAVE_DATA_WORKTREE) { + result = recv_rdcw_watch(data->watch_worktree); + if (result == -1) { + /* hard error */ + goto force_error_stop; + } + if (result == -2) { + /* retryable error */ + if (start_rdcw_watch(data, data->watch_worktree) == -1) + goto force_error_stop; + continue; + } + + /* have data */ + if (process_worktree_events(state) == LISTENER_SHUTDOWN) + goto force_shutdown; + if (start_rdcw_watch(data, data->watch_worktree) == -1) + goto force_error_stop; + continue; + } + + if (dwWait == WAIT_OBJECT_0 + LISTENER_HAVE_DATA_GITDIR) { + result = recv_rdcw_watch(data->watch_gitdir); + if (result == -1) { + /* hard error */ + goto force_error_stop; + } + if (result == -2) { + /* retryable error */ + if (start_rdcw_watch(data, data->watch_gitdir) == -1) + goto force_error_stop; + continue; + } + + /* have data */ + if (process_gitdir_events(state) == LISTENER_SHUTDOWN) + goto force_shutdown; + if (start_rdcw_watch(data, data->watch_gitdir) == -1) + goto force_error_stop; + continue; + } + + if (dwWait == WAIT_OBJECT_0 + LISTENER_SHUTDOWN) + goto clean_shutdown; + + error(_("could not read directory changes [GLE %ld]"), + GetLastError()); + goto force_error_stop; + } + +force_error_stop: + state->error_code = -1; + +force_shutdown: + /* + * Tell the IPC thead pool to stop (which completes the await + * in the main thread (which will also signal this thread (if + * we are still alive))). + */ + ipc_server_stop_async(state->ipc_server_data); + +clean_shutdown: + cancel_rdcw_watch(data->watch_worktree); + cancel_rdcw_watch(data->watch_gitdir); +} + +int fsm_listen__ctor(struct fsmonitor_daemon_state *state) +{ + struct fsmonitor_daemon_backend_data *data; + + CALLOC_ARRAY(data, 1); + + data->hEventShutdown = CreateEvent(NULL, TRUE, FALSE, NULL); + + data->watch_worktree = create_watch(state, + state->path_worktree_watch.buf); + if (!data->watch_worktree) + goto failed; + + if (state->nr_paths_watching > 1) { + data->watch_gitdir = create_watch(state, + state->path_gitdir_watch.buf); + if (!data->watch_gitdir) + goto failed; + } + + data->hListener[LISTENER_SHUTDOWN] = data->hEventShutdown; + data->nr_listener_handles++; + + data->hListener[LISTENER_HAVE_DATA_WORKTREE] = + data->watch_worktree->hEvent; + data->nr_listener_handles++; + + if (data->watch_gitdir) { + data->hListener[LISTENER_HAVE_DATA_GITDIR] = + data->watch_gitdir->hEvent; + data->nr_listener_handles++; + } + + state->backend_data = data; + return 0; + +failed: + CloseHandle(data->hEventShutdown); + destroy_watch(data->watch_worktree); + destroy_watch(data->watch_gitdir); + + return -1; +} + +void fsm_listen__dtor(struct fsmonitor_daemon_state *state) +{ + struct fsmonitor_daemon_backend_data *data; + + if (!state || !state->backend_data) + return; + + data = state->backend_data; + + CloseHandle(data->hEventShutdown); + destroy_watch(data->watch_worktree); + destroy_watch(data->watch_gitdir); + + FREE_AND_NULL(state->backend_data); +} diff --git a/compat/fsmonitor/fsm-listen.h b/compat/fsmonitor/fsm-listen.h new file mode 100644 index 0000000000..f0539349ba --- /dev/null +++ b/compat/fsmonitor/fsm-listen.h @@ -0,0 +1,49 @@ +#ifndef FSM_LISTEN_H +#define FSM_LISTEN_H + +/* This needs to be implemented by each backend */ + +#ifdef HAVE_FSMONITOR_DAEMON_BACKEND + +struct fsmonitor_daemon_state; + +/* + * Initialize platform-specific data for the fsmonitor listener thread. + * This will be called from the main thread PRIOR to staring the + * fsmonitor_fs_listener thread. + * + * Returns 0 if successful. + * Returns -1 otherwise. + */ +int fsm_listen__ctor(struct fsmonitor_daemon_state *state); + +/* + * Cleanup platform-specific data for the fsmonitor listener thread. + * This will be called from the main thread AFTER joining the listener. + */ +void fsm_listen__dtor(struct fsmonitor_daemon_state *state); + +/* + * The main body of the platform-specific event loop to watch for + * filesystem events. This will run in the fsmonitor_fs_listen thread. + * + * It should call `ipc_server_stop_async()` if the listener thread + * prematurely terminates (because of a filesystem error or if it + * detects that the .git directory has been deleted). (It should NOT + * do so if the listener thread receives a normal shutdown signal from + * the IPC layer.) + * + * It should set `state->error_code` to -1 if the daemon should exit + * with an error. + */ +void fsm_listen__loop(struct fsmonitor_daemon_state *state); + +/* + * Gently request that the fsmonitor listener thread shutdown. + * It does not wait for it to stop. The caller should do a JOIN + * to wait for it. + */ +void fsm_listen__stop_async(struct fsmonitor_daemon_state *state); + +#endif /* HAVE_FSMONITOR_DAEMON_BACKEND */ +#endif /* FSM_LISTEN_H */ diff --git a/compat/mingw.c b/compat/mingw.c index 9e0cd1e097..0f545c1a7d 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2374,7 +2374,7 @@ int mingw_raise(int sig) switch (sig) { case SIGALRM: if (timer_fn == SIG_DFL) { - if (isatty(STDERR_FILENO)) + if (isatty(2)) fputs("Alarm clock\n", stderr); exit(128 + SIGALRM); } else if (timer_fn != SIG_IGN) diff --git a/compat/mingw.h b/compat/mingw.h index c9a52ad64a..6074a3d3ce 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -329,6 +329,9 @@ int mingw_getpagesize(void); #define getpagesize mingw_getpagesize #endif +int win32_fsync_no_flush(int fd); +#define fsync_no_flush win32_fsync_no_flush + struct rlimit { unsigned int rlim_cur; }; diff --git a/compat/win32/flush.c b/compat/win32/flush.c new file mode 100644 index 0000000000..75324c24ee --- /dev/null +++ b/compat/win32/flush.c @@ -0,0 +1,28 @@ +#include "../../git-compat-util.h" +#include <winternl.h> +#include "lazyload.h" + +int win32_fsync_no_flush(int fd) +{ + IO_STATUS_BLOCK io_status; + +#define FLUSH_FLAGS_FILE_DATA_ONLY 1 + + DECLARE_PROC_ADDR(ntdll.dll, NTSTATUS, NtFlushBuffersFileEx, + HANDLE FileHandle, ULONG Flags, PVOID Parameters, ULONG ParameterSize, + PIO_STATUS_BLOCK IoStatusBlock); + + if (!INIT_PROC_ADDR(NtFlushBuffersFileEx)) { + errno = ENOSYS; + return -1; + } + + memset(&io_status, 0, sizeof(io_status)); + if (NtFlushBuffersFileEx((HANDLE)_get_osfhandle(fd), FLUSH_FLAGS_FILE_DATA_ONLY, + NULL, 0, &io_status)) { + errno = EINVAL; + return -1; + } + + return 0; +} diff --git a/compat/zlib-uncompress2.c b/compat/zlib-uncompress2.c new file mode 100644 index 0000000000..722610b971 --- /dev/null +++ b/compat/zlib-uncompress2.c @@ -0,0 +1,95 @@ +/* taken from zlib's uncompr.c + + commit cacf7f1d4e3d44d871b605da3b647f07d718623f + Author: Mark Adler <madler@alumni.caltech.edu> + Date: Sun Jan 15 09:18:46 2017 -0800 + + zlib 1.2.11 + +*/ + +#include "../reftable/system.h" +#define z_const + +/* + * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include <zlib.h> + +/* clang-format off */ + +/* =========================================================================== + Decompresses the source buffer into the destination buffer. *sourceLen is + the byte length of the source buffer. Upon entry, *destLen is the total size + of the destination buffer, which must be large enough to hold the entire + uncompressed data. (The size of the uncompressed data must have been saved + previously by the compressor and transmitted to the decompressor by some + mechanism outside the scope of this compression library.) Upon exit, + *destLen is the size of the decompressed data and *sourceLen is the number + of source bytes consumed. Upon return, source + *sourceLen points to the + first unused input byte. + + uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_BUF_ERROR if there was not enough room in the output buffer, or + Z_DATA_ERROR if the input data was corrupted, including if the input data is + an incomplete zlib stream. +*/ +int ZEXPORT uncompress2 ( + Bytef *dest, + uLongf *destLen, + const Bytef *source, + uLong *sourceLen) { + z_stream stream; + int err; + const uInt max = (uInt)-1; + uLong len, left; + Byte buf[1]; /* for detection of incomplete stream when *destLen == 0 */ + + len = *sourceLen; + if (*destLen) { + left = *destLen; + *destLen = 0; + } + else { + left = 1; + dest = buf; + } + + stream.next_in = (z_const Bytef *)source; + stream.avail_in = 0; + stream.zalloc = (alloc_func)0; + stream.zfree = (free_func)0; + stream.opaque = (voidpf)0; + + err = inflateInit(&stream); + if (err != Z_OK) return err; + + stream.next_out = dest; + stream.avail_out = 0; + + do { + if (stream.avail_out == 0) { + stream.avail_out = left > (uLong)max ? max : (uInt)left; + left -= stream.avail_out; + } + if (stream.avail_in == 0) { + stream.avail_in = len > (uLong)max ? max : (uInt)len; + len -= stream.avail_in; + } + err = inflate(&stream, Z_NO_FLUSH); + } while (err == Z_OK); + + *sourceLen -= len + stream.avail_in; + if (dest != buf) + *destLen = stream.total_out; + else if (stream.total_out && err == Z_BUF_ERROR) + left = 1; + + inflateEnd(&stream); + return err == Z_STREAM_END ? Z_OK : + err == Z_NEED_DICT ? Z_DATA_ERROR : + err == Z_BUF_ERROR && left + stream.avail_out ? Z_DATA_ERROR : + err; +} @@ -148,8 +148,10 @@ static int handle_path_include(const char *path, struct config_include_data *inc if (!is_absolute_path(path)) { char *slash; - if (!cf || !cf->path) - return error(_("relative config includes must come from files")); + if (!cf || !cf->path) { + ret = error(_("relative config includes must come from files")); + goto cleanup; + } slash = find_last_dir_sep(cf->path); if (slash) @@ -167,6 +169,7 @@ static int handle_path_include(const char *path, struct config_include_data *inc ret = git_config_from_file(git_config_include, path, inc); inc->depth--; } +cleanup: strbuf_release(&buf); free(expanded); return ret; @@ -1488,7 +1491,12 @@ static int git_default_core_config(const char *var, const char *value, void *cb) } if (!strcmp(var, "core.fsyncobjectfiles")) { - fsync_object_files = git_config_bool(var, value); + if (value && !strcmp(value, "batch")) + fsync_object_files = FSYNC_OBJECT_FILES_BATCH; + else if (git_config_bool(var, value)) + fsync_object_files = FSYNC_OBJECT_FILES_ON; + else + fsync_object_files = FSYNC_OBJECT_FILES_OFF; return 0; } @@ -1559,6 +1567,9 @@ static int git_default_branch_config(const char *var, const char *value) if (value && !strcasecmp(value, "always")) { git_branch_track = BRANCH_TRACK_ALWAYS; return 0; + } else if (value && !strcasecmp(value, "inherit")) { + git_branch_track = BRANCH_TRACK_INHERIT; + return 0; } git_branch_track = git_config_bool(var, value); return 0; @@ -2502,20 +2513,6 @@ int git_config_get_max_percent_split_change(void) return -1; /* default value */ } -int git_config_get_fsmonitor(void) -{ - if (git_config_get_pathname("core.fsmonitor", &core_fsmonitor)) - core_fsmonitor = getenv("GIT_TEST_FSMONITOR"); - - if (core_fsmonitor && !*core_fsmonitor) - core_fsmonitor = NULL; - - if (core_fsmonitor) - return 1; - - return 0; -} - int git_config_get_index_threads(int *dest) { int is_bool, val; @@ -610,7 +610,6 @@ int git_config_get_pathname(const char *key, const char **dest); int git_config_get_index_threads(int *dest); int git_config_get_split_index(void); int git_config_get_max_percent_split_change(void); -int git_config_get_fsmonitor(void); /* This dies if the configured or default date is in the future */ int git_config_get_expiry(const char *key, const char **output); diff --git a/config.mak.uname b/config.mak.uname index 3236a4918a..bd99642c72 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -57,8 +57,8 @@ ifeq ($(uname_S),Linux) HAVE_CLOCK_MONOTONIC = YesPlease # -lrt is needed for clock_gettime on glibc <= 2.16 NEEDS_LIBRT = YesPlease + HAVE_SYNC_FILE_RANGE = YesPlease HAVE_GETDELIM = YesPlease - SANE_TEXT_GREP=-a FREAD_READS_DIRECTORIES = UnfortunatelyYes BASIC_CFLAGS += -DHAVE_SYSINFO PROCFS_EXECUTABLE_PATH = /proc/self/exe @@ -153,6 +153,16 @@ ifeq ($(uname_S),Darwin) MSGFMT = /usr/local/opt/gettext/bin/msgfmt endif endif + + # The builtin FSMonitor on MacOS builds upon Simple-IPC. Both require + # Unix domain sockets and PThreads. + ifndef NO_PTHREADS + ifndef NO_UNIX_SOCKETS + FSMONITOR_DAEMON_BACKEND = darwin + endif + endif + + BASIC_LDFLAGS += -framework CoreServices endif ifeq ($(uname_S),SunOS) NEEDS_SOCKET = YesPlease @@ -262,6 +272,10 @@ ifeq ($(uname_S),FreeBSD) FILENO_IS_A_MACRO = UnfortunatelyYes endif ifeq ($(uname_S),OpenBSD) + # Versions < 7.0 need compatibility layer + ifeq ($(shell expr "$(uname_R)" : "[1-6]\."),2) + NO_UNCOMPRESS2 = UnfortunatelyYes + endif NO_STRCASESTR = YesPlease NO_MEMMEM = YesPlease USE_ST_TIMESPEC = YesPlease @@ -426,6 +440,11 @@ ifeq ($(uname_S),Windows) # so we don't need this: # # SNPRINTF_RETURNS_BOGUS = YesPlease + + # The builtin FSMonitor requires Named Pipes and Threads on Windows. + # These are always available, so we do not have to conditionally + # support it. + FSMONITOR_DAEMON_BACKEND = win32 NO_SVN_TESTS = YesPlease RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo @@ -454,6 +473,7 @@ endif CFLAGS = BASIC_CFLAGS = -nologo -I. -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE COMPAT_OBJS = compat/msvc.o compat/winansi.o \ + compat/win32/flush.o \ compat/win32/path-utils.o \ compat/win32/pthread.o compat/win32/syslog.o \ compat/win32/trace2_win32_process_info.o \ @@ -517,6 +537,7 @@ ifeq ($(uname_S),Interix) endif endif ifeq ($(uname_S),Minix) + NO_UNCOMPRESS2 = YesPlease NO_IPV6 = YesPlease NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease NO_NSEC = YesPlease @@ -609,6 +630,11 @@ ifeq ($(uname_S),MINGW) NO_STRTOUMAX = YesPlease NO_MKDTEMP = YesPlease NO_SVN_TESTS = YesPlease + + # The builtin FSMonitor requires Named Pipes and Threads on Windows. + # These are always available, so we do not have to conditionally + # support it. + FSMONITOR_DAEMON_BACKEND = win32 RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease @@ -629,6 +655,7 @@ ifeq ($(uname_S),MINGW) COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/winansi.o \ compat/win32/trace2_win32_process_info.o \ + compat/win32/flush.o \ compat/win32/path-utils.o \ compat/win32/pthread.o compat/win32/syslog.o \ compat/win32/dirent.o diff --git a/configure.ac b/configure.ac index 031e8d3fee..660b91f90b 100644 --- a/configure.ac +++ b/configure.ac @@ -507,14 +507,6 @@ if test -n "$ASCIIDOC"; then esac fi -if grep -a ascii configure.ac >/dev/null; then - AC_MSG_RESULT([Using 'grep -a' for sane_grep]) - SANE_TEXT_GREP=-a -else - SANE_TEXT_GREP= -fi -GIT_CONF_SUBST([SANE_TEXT_GREP]) - ## Checks for libraries. AC_MSG_NOTICE([CHECKS for libraries]) # @@ -672,9 +664,22 @@ AC_LINK_IFELSE([ZLIBTEST_SRC], NO_DEFLATE_BOUND=yes]) LIBS="$old_LIBS" +AC_DEFUN([ZLIBTEST_UNCOMPRESS2_SRC], [ +AC_LANG_PROGRAM([#include <zlib.h>], + [uncompress2(NULL,NULL,NULL,NULL);])]) +AC_MSG_CHECKING([for uncompress2 in -lz]) +old_LIBS="$LIBS" +LIBS="$LIBS -lz" +AC_LINK_IFELSE([ZLIBTEST_UNCOMPRESS2_SRC], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + NO_UNCOMPRESS2=yes]) +LIBS="$old_LIBS" + GIT_UNSTASH_FLAGS($ZLIB_PATH) GIT_CONF_SUBST([NO_DEFLATE_BOUND]) +GIT_CONF_SUBST([NO_UNCOMPRESS2]) # # Define NEEDS_SOCKET if linking with libc is not enough (SunOS, @@ -1090,6 +1095,14 @@ AC_COMPILE_IFELSE([CLOCK_MONOTONIC_SRC], [AC_MSG_RESULT([no]) HAVE_CLOCK_MONOTONIC=]) GIT_CONF_SUBST([HAVE_CLOCK_MONOTONIC]) + +# +# Define HAVE_SYNC_FILE_RANGE=YesPlease if sync_file_range is available. +GIT_CHECK_FUNC(sync_file_range, + [HAVE_SYNC_FILE_RANGE=YesPlease], + [HAVE_SYNC_FILE_RANGE]) +GIT_CONF_SUBST([HAVE_SYNC_FILE_RANGE]) + # # Define NO_SETITIMER if you don't have setitimer. GIT_CHECK_FUNC(setitimer, diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index fd1399c440..7b95c860d2 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -261,7 +261,8 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows") NOGDI OBJECT_CREATION_MODE=1 __USE_MINGW_ANSI_STDIO=0 USE_NED_ALLOCATOR OVERRIDE_STRDUP MMAP_PREVENTS_DELETE USE_WIN32_MMAP UNICODE _UNICODE HAVE_WPGMPTR ENSURE_MSYSTEM_IS_SET) - list(APPEND compat_SOURCES compat/mingw.c compat/winansi.c compat/win32/path-utils.c + list(APPEND compat_SOURCES compat/mingw.c compat/winansi.c + compat/win32/flush.c compat/win32/path-utils.c compat/win32/pthread.c compat/win32mmap.c compat/win32/syslog.c compat/win32/trace2_win32_process_info.c compat/win32/dirent.c compat/nedmalloc/nedmalloc.c compat/strdup.c) @@ -285,6 +286,16 @@ else() endif() endif() +if(SUPPORTS_SIMPLE_IPC) + if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) + list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-win32.c) + elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) + list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-darwin.c) + endif() +endif() + set(EXE_EXTENSION ${CMAKE_EXECUTABLE_SUFFIX}) #header checks @@ -647,6 +658,12 @@ parse_makefile_for_sources(libxdiff_SOURCES "XDIFF_OBJS") list(TRANSFORM libxdiff_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") add_library(xdiff STATIC ${libxdiff_SOURCES}) +#reftable +parse_makefile_for_sources(reftable_SOURCES "REFTABLE_OBJS") + +list(TRANSFORM reftable_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") +add_library(reftable STATIC ${reftable_SOURCES}) + if(WIN32) if(NOT MSVC)#use windres when compiling with gcc and clang add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/git.res @@ -669,7 +686,7 @@ endif() #link all required libraries to common-main add_library(common-main OBJECT ${CMAKE_SOURCE_DIR}/common-main.c) -target_link_libraries(common-main libgit xdiff ${ZLIB_LIBRARIES}) +target_link_libraries(common-main libgit xdiff reftable ${ZLIB_LIBRARIES}) if(Intl_FOUND) target_link_libraries(common-main ${Intl_LIBRARIES}) endif() @@ -781,7 +798,6 @@ foreach(script ${git_shell_scripts}) string(REPLACE "@@USE_GETTEXT_SCHEME@@" "" content "${content}") string(REPLACE "# @@BROKEN_PATH_FIX@@" "" content "${content}") string(REPLACE "@@PERL@@" "${PERL_PATH}" content "${content}") - string(REPLACE "@@SANE_TEXT_GREP@@" "-a" content "${content}") string(REPLACE "@@PAGER_ENV@@" "LESS=FRX LV=-c" content "${content}") file(WRITE ${CMAKE_BINARY_DIR}/${script} ${content}) endforeach() @@ -909,11 +925,15 @@ if(BUILD_TESTING) add_executable(test-fake-ssh ${CMAKE_SOURCE_DIR}/t/helper/test-fake-ssh.c) target_link_libraries(test-fake-ssh common-main) +#reftable-tests +parse_makefile_for_sources(test-reftable_SOURCES "REFTABLE_TEST_OBJS") +list(TRANSFORM test-reftable_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") + #test-tool parse_makefile_for_sources(test-tool_SOURCES "TEST_BUILTINS_OBJS") list(TRANSFORM test-tool_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/t/helper/") -add_executable(test-tool ${CMAKE_SOURCE_DIR}/t/helper/test-tool.c ${test-tool_SOURCES}) +add_executable(test-tool ${CMAKE_SOURCE_DIR}/t/helper/test-tool.c ${test-tool_SOURCES} ${test-reftable_SOURCES}) target_link_libraries(test-tool common-main) set_target_properties(test-fake-ssh test-tool diff --git a/contrib/buildsystems/Generators/Vcxproj.pm b/contrib/buildsystems/Generators/Vcxproj.pm index d2584450ba..1a25789d28 100644 --- a/contrib/buildsystems/Generators/Vcxproj.pm +++ b/contrib/buildsystems/Generators/Vcxproj.pm @@ -77,7 +77,7 @@ sub createProject { my $libs_release = "\n "; my $libs_debug = "\n "; if (!$static_library) { - $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); + $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib|reftable\/libreftable\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); $libs_debug = $libs_release; $libs_debug =~ s/zlib\.lib/zlibd\.lib/g; $libs_debug =~ s/libexpat\.lib/libexpatd\.lib/g; @@ -232,6 +232,7 @@ EOM EOM if (!$static_library || $target =~ 'vcs-svn' || $target =~ 'xdiff') { my $uuid_libgit = $$build_structure{"LIBS_libgit_GUID"}; + my $uuid_libreftable = $$build_structure{"LIBS_reftable/libreftable_GUID"}; my $uuid_xdiff_lib = $$build_structure{"LIBS_xdiff/lib_GUID"}; print F << "EOM"; @@ -241,6 +242,14 @@ EOM <ReferenceOutputAssembly>false</ReferenceOutputAssembly> </ProjectReference> EOM + if (!($name =~ /xdiff|libreftable/)) { + print F << "EOM"; + <ProjectReference Include="$cdup\\reftable\\libreftable\\libreftable.vcxproj"> + <Project>$uuid_libreftable</Project> + <ReferenceOutputAssembly>false</ReferenceOutputAssembly> + </ProjectReference> +EOM + } if (!($name =~ 'xdiff')) { print F << "EOM"; <ProjectReference Include="$cdup\\xdiff\\lib\\xdiff_lib.vcxproj"> diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index eb5fd4783d..3bfb4c8892 100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -515,7 +515,7 @@ __gitcomp_file () # argument, and using the options specified in the second argument. __git_ls_files_helper () { - if [ "$2" == "--committable" ]; then + if [ "$2" = "--committable" ]; then __git -C "$1" -c core.quotePath=false diff-index \ --name-only --relative HEAD -- "${3//\\/\\\\}*" else @@ -1566,7 +1566,7 @@ _git_checkout () case "$cur" in --conflict=*) - __gitcomp "diff3 merge" "" "${cur##--conflict=}" + __gitcomp "diff3 merge zdiff3" "" "${cur##--conflict=}" ;; --*) __gitcomp_builtin checkout @@ -2359,16 +2359,7 @@ _git_send_email () return ;; --*) - __gitcomp_builtin send-email "--annotate --bcc --cc --cc-cmd --chain-reply-to - --compose --confirm= --dry-run --envelope-sender - --from --identity - --in-reply-to --no-chain-reply-to --no-signed-off-by-cc - --no-suppress-from --no-thread --quiet --reply-to - --signed-off-by-cc --smtp-pass --smtp-server - --smtp-server-port --smtp-encryption= --smtp-user - --subject --suppress-cc= --suppress-from --thread --to - --validate --no-validate - $__git_format_patch_extra_options" + __gitcomp_builtin send-email "$__git_format_patch_extra_options" return ;; esac @@ -2446,7 +2437,7 @@ _git_switch () case "$cur" in --conflict=*) - __gitcomp "diff3 merge" "" "${cur##--conflict=}" + __gitcomp "diff3 merge zdiff3" "" "${cur##--conflict=}" ;; --*) __gitcomp_builtin switch @@ -2886,7 +2877,7 @@ _git_restore () case "$cur" in --conflict=*) - __gitcomp "diff3 merge" "" "${cur##--conflict=}" + __gitcomp "diff3 merge zdiff3" "" "${cur##--conflict=}" ;; --source=*) __git_complete_refs --cur="${cur##--source=}" diff --git a/contrib/scalar/.gitignore b/contrib/scalar/.gitignore new file mode 100644 index 0000000000..00441073f5 --- /dev/null +++ b/contrib/scalar/.gitignore @@ -0,0 +1,5 @@ +/*.xml +/*.1 +/*.html +/*.exe +/scalar diff --git a/contrib/scalar/Makefile b/contrib/scalar/Makefile new file mode 100644 index 0000000000..44796572ef --- /dev/null +++ b/contrib/scalar/Makefile @@ -0,0 +1,57 @@ +QUIET_SUBDIR0 = +$(MAKE) -C # space to separate -C and subdir +QUIET_SUBDIR1 = + +ifneq ($(findstring s,$(MAKEFLAGS)),s) +ifndef V + QUIET_GEN = @echo ' ' GEN $@; + QUIET_SUBDIR0 = +@subdir= + QUIET_SUBDIR1 = ;$(NO_SUBDIR) echo ' ' SUBDIR $$subdir; \ + $(MAKE) $(PRINT_DIR) -C $$subdir + QUIET = @ +else + export V +endif +endif + +all: + +include ../../config.mak.uname +-include ../../config.mak.autogen +-include ../../config.mak + +TARGETS = scalar$(X) scalar.o +GITLIBS = ../../common-main.o ../../libgit.a ../../xdiff/lib.a + +all: scalar$(X) ../../bin-wrappers/scalar + +$(GITLIBS): + $(QUIET_SUBDIR0)../.. $(QUIET_SUBDIR1) $(subst ../../,,$@) + +$(TARGETS): $(GITLIBS) scalar.c + $(QUIET_SUBDIR0)../.. $(QUIET_SUBDIR1) $(patsubst %,contrib/scalar/%,$@) + +clean: + $(RM) $(TARGETS) ../../bin-wrappers/scalar + $(RM) scalar.1 scalar.html scalar.xml + +../../bin-wrappers/scalar: ../../wrap-for-bin.sh Makefile + @mkdir -p ../../bin-wrappers + $(QUIET_GEN)sed -e '1s|#!.*/sh|#!$(SHELL_PATH_SQ)|' \ + -e 's|@@BUILD_DIR@@|$(shell cd ../.. && pwd)|' \ + -e 's|@@PROG@@|contrib/scalar/scalar$(X)|' < $< > $@ && \ + chmod +x $@ + +test: all + $(MAKE) -C t + +docs: scalar.html scalar.1 + +scalar.html: | scalar.1 # prevent them from trying to build `doc.dep` in parallel + +scalar.html scalar.1: scalar.txt + $(QUIET_SUBDIR0)../../Documentation$(QUIET_SUBDIR1) \ + MAN_TXT=../contrib/scalar/scalar.txt \ + ../contrib/scalar/$@ + $(QUIET)test scalar.1 != "$@" || mv ../../Documentation/$@ . + +.PHONY: $(GITLIBS) all clean docs test FORCE diff --git a/contrib/scalar/scalar.c b/contrib/scalar/scalar.c new file mode 100644 index 0000000000..6c496318bd --- /dev/null +++ b/contrib/scalar/scalar.c @@ -0,0 +1,845 @@ +/* + * The Scalar command-line interface. + */ + +#include "cache.h" +#include "gettext.h" +#include "parse-options.h" +#include "config.h" +#include "run-command.h" +#include "refs.h" +#include "dir.h" +#include "packfile.h" +#include "help.h" + +/* + * Remove the deepest subdirectory in the provided path string. Path must not + * include a trailing path separator. Returns 1 if parent directory found, + * otherwise 0. + */ +static int strbuf_parent_directory(struct strbuf *buf) +{ + size_t len = buf->len; + size_t offset = offset_1st_component(buf->buf); + char *path_sep = find_last_dir_sep(buf->buf + offset); + strbuf_setlen(buf, path_sep ? path_sep - buf->buf : offset); + + return buf->len < len; +} + +static void setup_enlistment_directory(int argc, const char **argv, + const char * const *usagestr, + const struct option *options, + struct strbuf *enlistment_root) +{ + struct strbuf path = STRBUF_INIT; + char *root; + int enlistment_found = 0; + + if (startup_info->have_repository) + BUG("gitdir already set up?!?"); + + if (argc > 1) + usage_with_options(usagestr, options); + + /* find the worktree, determine its corresponding root */ + if (argc == 1) + strbuf_add_absolute_path(&path, argv[0]); + else if (strbuf_getcwd(&path) < 0) + die(_("need a working directory")); + + strbuf_trim_trailing_dir_sep(&path); + do { + const size_t len = path.len; + + /* check if currently in enlistment root with src/ workdir */ + strbuf_addstr(&path, "/src"); + if (is_nonbare_repository_dir(&path)) { + if (enlistment_root) + strbuf_add(enlistment_root, path.buf, len); + + enlistment_found = 1; + break; + } + + /* reset to original path */ + strbuf_setlen(&path, len); + + /* check if currently in workdir */ + if (is_nonbare_repository_dir(&path)) { + if (enlistment_root) { + /* + * If the worktree's directory's name is `src`, the enlistment is the + * parent directory, otherwise it is identical to the worktree. + */ + root = strip_path_suffix(path.buf, "src"); + strbuf_addstr(enlistment_root, root ? root : path.buf); + free(root); + } + + enlistment_found = 1; + break; + } + } while (strbuf_parent_directory(&path)); + + if (!enlistment_found) + die(_("could not find enlistment root")); + + if (chdir(path.buf) < 0) + die_errno(_("could not switch to '%s'"), path.buf); + + strbuf_release(&path); + setup_git_directory(); +} + +static int run_git(const char *arg, ...) +{ + struct strvec argv = STRVEC_INIT; + va_list args; + const char *p; + int res; + + va_start(args, arg); + strvec_push(&argv, arg); + while ((p = va_arg(args, const char *))) + strvec_push(&argv, p); + va_end(args); + + res = run_command_v_opt(argv.v, RUN_GIT_CMD); + + strvec_clear(&argv); + return res; +} + +static int set_recommended_config(int reconfigure) +{ + struct { + const char *key; + const char *value; + int overwrite_on_reconfigure; + } config[] = { + /* Required */ + { "am.keepCR", "true", 1 }, + { "core.FSCache", "true", 1 }, + { "core.multiPackIndex", "true", 1 }, + { "core.preloadIndex", "true", 1 }, +#ifndef WIN32 + { "core.untrackedCache", "true", 1 }, +#else + /* + * Unfortunately, Scalar's Functional Tests demonstrated + * that the untracked cache feature is unreliable on Windows + * (which is a bummer because that platform would benefit the + * most from it). For some reason, freshly created files seem + * not to update the directory's `lastModified` time + * immediately, but the untracked cache would need to rely on + * that. + * + * Therefore, with a sad heart, we disable this very useful + * feature on Windows. + */ + { "core.untrackedCache", "false", 1 }, +#endif + { "core.logAllRefUpdates", "true", 1 }, + { "credential.https://dev.azure.com.useHttpPath", "true", 1 }, + { "credential.validate", "false", 1 }, /* GCM4W-only */ + { "gc.auto", "0", 1 }, + { "gui.GCWarning", "false", 1 }, + { "index.threads", "true", 1 }, + { "index.version", "4", 1 }, + { "merge.stat", "false", 1 }, + { "merge.renames", "false", 1 }, + { "pack.useBitmaps", "false", 1 }, + { "pack.useSparse", "true", 1 }, + { "receive.autoGC", "false", 1 }, + { "reset.quiet", "true", 1 }, + { "feature.manyFiles", "false", 1 }, + { "feature.experimental", "false", 1 }, + { "fetch.unpackLimit", "1", 1 }, + { "fetch.writeCommitGraph", "false", 1 }, +#ifdef WIN32 + { "http.sslBackend", "schannel", 1 }, +#endif + /* Optional */ + { "status.aheadBehind", "false" }, + { "commitGraph.generationVersion", "1" }, + { "core.autoCRLF", "false" }, + { "core.safeCRLF", "false" }, + { NULL, NULL }, + }; + int i; + char *value; + + for (i = 0; config[i].key; i++) { + if ((reconfigure && config[i].overwrite_on_reconfigure) || + git_config_get_string(config[i].key, &value)) { + trace2_data_string("scalar", the_repository, config[i].key, "created"); + if (git_config_set_gently(config[i].key, + config[i].value) < 0) + return error(_("could not configure %s=%s"), + config[i].key, config[i].value); + } else { + trace2_data_string("scalar", the_repository, config[i].key, "exists"); + free(value); + } + } + + /* + * The `log.excludeDecoration` setting is special because it allows + * for multiple values. + */ + if (git_config_get_string("log.excludeDecoration", &value)) { + trace2_data_string("scalar", the_repository, + "log.excludeDecoration", "created"); + if (git_config_set_multivar_gently("log.excludeDecoration", + "refs/prefetch/*", + CONFIG_REGEX_NONE, 0)) + return error(_("could not configure " + "log.excludeDecoration")); + } else { + trace2_data_string("scalar", the_repository, + "log.excludeDecoration", "exists"); + free(value); + } + + return 0; +} + +static int toggle_maintenance(int enable) +{ + return run_git("maintenance", enable ? "start" : "unregister", NULL); +} + +static int add_or_remove_enlistment(int add) +{ + int res; + + if (!the_repository->worktree) + die(_("Scalar enlistments require a worktree")); + + res = run_git("config", "--global", "--get", "--fixed-value", + "scalar.repo", the_repository->worktree, NULL); + + /* + * If we want to add and the setting is already there, then do nothing. + * If we want to remove and the setting is not there, then do nothing. + */ + if ((add && !res) || (!add && res)) + return 0; + + return run_git("config", "--global", add ? "--add" : "--unset", + add ? "--no-fixed-value" : "--fixed-value", + "scalar.repo", the_repository->worktree, NULL); +} + +static int register_dir(void) +{ + int res = add_or_remove_enlistment(1); + + if (!res) + res = set_recommended_config(0); + + if (!res) + res = toggle_maintenance(1); + + return res; +} + +static int unregister_dir(void) +{ + int res = 0; + + if (toggle_maintenance(0) < 0) + res = -1; + + if (add_or_remove_enlistment(0) < 0) + res = -1; + + return res; +} + +/* printf-style interface, expects `<key>=<value>` argument */ +static int set_config(const char *fmt, ...) +{ + struct strbuf buf = STRBUF_INIT; + char *value; + int res; + va_list args; + + va_start(args, fmt); + strbuf_vaddf(&buf, fmt, args); + va_end(args); + + value = strchr(buf.buf, '='); + if (value) + *(value++) = '\0'; + res = git_config_set_gently(buf.buf, value); + strbuf_release(&buf); + + return res; +} + +static char *remote_default_branch(const char *url) +{ + struct child_process cp = CHILD_PROCESS_INIT; + struct strbuf out = STRBUF_INIT; + + cp.git_cmd = 1; + strvec_pushl(&cp.args, "ls-remote", "--symref", url, "HEAD", NULL); + if (!pipe_command(&cp, NULL, 0, &out, 0, NULL, 0)) { + const char *line = out.buf; + + while (*line) { + const char *eol = strchrnul(line, '\n'), *p; + size_t len = eol - line; + char *branch; + + if (!skip_prefix(line, "ref: ", &p) || + !strip_suffix_mem(line, &len, "\tHEAD")) { + line = eol + (*eol == '\n'); + continue; + } + + eol = line + len; + if (skip_prefix(p, "refs/heads/", &p)) { + branch = xstrndup(p, eol - p); + strbuf_release(&out); + return branch; + } + + error(_("remote HEAD is not a branch: '%.*s'"), + (int)(eol - p), p); + strbuf_release(&out); + return NULL; + } + } + warning(_("failed to get default branch name from remote; " + "using local default")); + strbuf_reset(&out); + + child_process_init(&cp); + cp.git_cmd = 1; + strvec_pushl(&cp.args, "symbolic-ref", "--short", "HEAD", NULL); + if (!pipe_command(&cp, NULL, 0, &out, 0, NULL, 0)) { + strbuf_trim(&out); + return strbuf_detach(&out, NULL); + } + + strbuf_release(&out); + error(_("failed to get default branch name")); + return NULL; +} + +static int delete_enlistment(struct strbuf *enlistment) +{ +#ifdef WIN32 + struct strbuf parent = STRBUF_INIT; +#endif + + if (unregister_dir()) + die(_("failed to unregister repository")); + +#ifdef WIN32 + /* + * Change the current directory to one outside of the enlistment so + * that we may delete everything underneath it. + */ + strbuf_addbuf(&parent, enlistment); + strbuf_parent_directory(&parent); + if (chdir(parent.buf) < 0) + die_errno(_("could not switch to '%s'"), parent.buf); + strbuf_release(&parent); +#endif + + if (remove_dir_recursively(enlistment, 0)) + die(_("failed to delete enlistment directory")); + + return 0; +} + +/* + * Dummy implementation; Using `get_version_info()` would cause a link error + * without this. + */ +void load_builtin_commands(const char *prefix, struct cmdnames *cmds) +{ + die("not implemented"); +} + +static int cmd_clone(int argc, const char **argv) +{ + const char *branch = NULL; + int full_clone = 0, single_branch = 0; + struct option clone_options[] = { + OPT_STRING('b', "branch", &branch, N_("<branch>"), + N_("branch to checkout after clone")), + OPT_BOOL(0, "full-clone", &full_clone, + N_("when cloning, create full working directory")), + OPT_BOOL(0, "single-branch", &single_branch, + N_("only download metadata for the branch that will " + "be checked out")), + OPT_END(), + }; + const char * const clone_usage[] = { + N_("scalar clone [<options>] [--] <repo> [<dir>]"), + NULL + }; + const char *url; + char *enlistment = NULL, *dir = NULL; + struct strbuf buf = STRBUF_INIT; + int res; + + argc = parse_options(argc, argv, NULL, clone_options, clone_usage, 0); + + if (argc == 2) { + url = argv[0]; + enlistment = xstrdup(argv[1]); + } else if (argc == 1) { + url = argv[0]; + + strbuf_addstr(&buf, url); + /* Strip trailing slashes, if any */ + while (buf.len > 0 && is_dir_sep(buf.buf[buf.len - 1])) + strbuf_setlen(&buf, buf.len - 1); + /* Strip suffix `.git`, if any */ + strbuf_strip_suffix(&buf, ".git"); + + enlistment = find_last_dir_sep(buf.buf); + if (!enlistment) { + die(_("cannot deduce worktree name from '%s'"), url); + } + enlistment = xstrdup(enlistment + 1); + } else { + usage_msg_opt(_("You must specify a repository to clone."), + clone_usage, clone_options); + } + + if (is_directory(enlistment)) + die(_("directory '%s' exists already"), enlistment); + + dir = xstrfmt("%s/src", enlistment); + + strbuf_reset(&buf); + if (branch) + strbuf_addf(&buf, "init.defaultBranch=%s", branch); + else { + char *b = repo_default_branch_name(the_repository, 1); + strbuf_addf(&buf, "init.defaultBranch=%s", b); + free(b); + } + + if ((res = run_git("-c", buf.buf, "init", "--", dir, NULL))) + goto cleanup; + + if (chdir(dir) < 0) { + res = error_errno(_("could not switch to '%s'"), dir); + goto cleanup; + } + + setup_git_directory(); + + /* common-main already logs `argv` */ + trace2_def_repo(the_repository); + + if (!branch && !(branch = remote_default_branch(url))) { + res = error(_("failed to get default branch for '%s'"), url); + goto cleanup; + } + + if (set_config("remote.origin.url=%s", url) || + set_config("remote.origin.fetch=" + "+refs/heads/%s:refs/remotes/origin/%s", + single_branch ? branch : "*", + single_branch ? branch : "*") || + set_config("remote.origin.promisor=true") || + set_config("remote.origin.partialCloneFilter=blob:none")) { + res = error(_("could not configure remote in '%s'"), dir); + goto cleanup; + } + + if (!full_clone && + (res = run_git("sparse-checkout", "init", "--cone", NULL))) + goto cleanup; + + if (set_recommended_config(0)) + return error(_("could not configure '%s'"), dir); + + if ((res = run_git("fetch", "--quiet", "origin", NULL))) { + warning(_("partial clone failed; attempting full clone")); + + if (set_config("remote.origin.promisor") || + set_config("remote.origin.partialCloneFilter")) { + res = error(_("could not configure for full clone")); + goto cleanup; + } + + if ((res = run_git("fetch", "--quiet", "origin", NULL))) + goto cleanup; + } + + if ((res = set_config("branch.%s.remote=origin", branch))) + goto cleanup; + if ((res = set_config("branch.%s.merge=refs/heads/%s", + branch, branch))) + goto cleanup; + + strbuf_reset(&buf); + strbuf_addf(&buf, "origin/%s", branch); + res = run_git("checkout", "-f", "-t", buf.buf, NULL); + if (res) + goto cleanup; + + res = register_dir(); + +cleanup: + free(enlistment); + free(dir); + strbuf_release(&buf); + return res; +} + +static int cmd_list(int argc, const char **argv) +{ + if (argc != 1) + die(_("`scalar list` does not take arguments")); + + if (run_git("config", "--global", "--get-all", "scalar.repo", NULL) < 0) + return -1; + return 0; +} + +static int cmd_register(int argc, const char **argv) +{ + struct option options[] = { + OPT_END(), + }; + const char * const usage[] = { + N_("scalar register [<enlistment>]"), + NULL + }; + + argc = parse_options(argc, argv, NULL, options, + usage, 0); + + setup_enlistment_directory(argc, argv, usage, options, NULL); + + return register_dir(); +} + +static int get_scalar_repos(const char *key, const char *value, void *data) +{ + struct string_list *list = data; + + if (!strcmp(key, "scalar.repo")) + string_list_append(list, value); + + return 0; +} + +static int cmd_reconfigure(int argc, const char **argv) +{ + int all = 0; + struct option options[] = { + OPT_BOOL('a', "all", &all, + N_("reconfigure all registered enlistments")), + OPT_END(), + }; + const char * const usage[] = { + N_("scalar reconfigure [--all | <enlistment>]"), + NULL + }; + struct string_list scalar_repos = STRING_LIST_INIT_DUP; + int i, res = 0; + struct repository r = { NULL }; + struct strbuf commondir = STRBUF_INIT, gitdir = STRBUF_INIT; + + argc = parse_options(argc, argv, NULL, options, + usage, 0); + + if (!all) { + setup_enlistment_directory(argc, argv, usage, options, NULL); + + return set_recommended_config(1); + } + + if (argc > 0) + usage_msg_opt(_("--all or <enlistment>, but not both"), + usage, options); + + git_config(get_scalar_repos, &scalar_repos); + + for (i = 0; i < scalar_repos.nr; i++) { + const char *dir = scalar_repos.items[i].string; + + strbuf_reset(&commondir); + strbuf_reset(&gitdir); + + if (chdir(dir) < 0) { + warning_errno(_("could not switch to '%s'"), dir); + res = -1; + } else if (discover_git_directory(&commondir, &gitdir) < 0) { + warning_errno(_("git repository gone in '%s'"), dir); + res = -1; + } else { + git_config_clear(); + + the_repository = &r; + r.commondir = commondir.buf; + r.gitdir = gitdir.buf; + + if (set_recommended_config(1) < 0) + res = -1; + } + } + + string_list_clear(&scalar_repos, 1); + strbuf_release(&commondir); + strbuf_release(&gitdir); + + return res; +} + +static int cmd_run(int argc, const char **argv) +{ + struct option options[] = { + OPT_END(), + }; + struct { + const char *arg, *task; + } tasks[] = { + { "config", NULL }, + { "commit-graph", "commit-graph" }, + { "fetch", "prefetch" }, + { "loose-objects", "loose-objects" }, + { "pack-files", "incremental-repack" }, + { NULL, NULL } + }; + struct strbuf buf = STRBUF_INIT; + const char *usagestr[] = { NULL, NULL }; + int i; + + strbuf_addstr(&buf, N_("scalar run <task> [<enlistment>]\nTasks:\n")); + for (i = 0; tasks[i].arg; i++) + strbuf_addf(&buf, "\t%s\n", tasks[i].arg); + usagestr[0] = buf.buf; + + argc = parse_options(argc, argv, NULL, options, + usagestr, 0); + + if (!argc) + usage_with_options(usagestr, options); + + if (!strcmp("all", argv[0])) { + i = -1; + } else { + for (i = 0; tasks[i].arg && strcmp(tasks[i].arg, argv[0]); i++) + ; /* keep looking for the task */ + + if (i > 0 && !tasks[i].arg) { + error(_("no such task: '%s'"), argv[0]); + usage_with_options(usagestr, options); + } + } + + argc--; + argv++; + setup_enlistment_directory(argc, argv, usagestr, options, NULL); + strbuf_release(&buf); + + if (i == 0) + return register_dir(); + + if (i > 0) + return run_git("maintenance", "run", + "--task", tasks[i].task, NULL); + + if (register_dir()) + return -1; + for (i = 1; tasks[i].arg; i++) + if (run_git("maintenance", "run", + "--task", tasks[i].task, NULL)) + return -1; + return 0; +} + +static int remove_deleted_enlistment(struct strbuf *path) +{ + int res = 0; + strbuf_realpath_forgiving(path, path->buf, 1); + + if (run_git("config", "--global", + "--unset", "--fixed-value", + "scalar.repo", path->buf, NULL) < 0) + res = -1; + + if (run_git("config", "--global", + "--unset", "--fixed-value", + "maintenance.repo", path->buf, NULL) < 0) + res = -1; + + return res; +} + +static int cmd_unregister(int argc, const char **argv) +{ + struct option options[] = { + OPT_END(), + }; + const char * const usage[] = { + N_("scalar unregister [<enlistment>]"), + NULL + }; + + argc = parse_options(argc, argv, NULL, options, + usage, 0); + + /* + * Be forgiving when the enlistment or worktree does not even exist any + * longer; This can be the case if a user deleted the worktree by + * mistake and _still_ wants to unregister the thing. + */ + if (argc == 1) { + struct strbuf src_path = STRBUF_INIT, workdir_path = STRBUF_INIT; + + strbuf_addf(&src_path, "%s/src/.git", argv[0]); + strbuf_addf(&workdir_path, "%s/.git", argv[0]); + if (!is_directory(src_path.buf) && !is_directory(workdir_path.buf)) { + /* remove possible matching registrations */ + int res = -1; + + strbuf_strip_suffix(&src_path, "/.git"); + res = remove_deleted_enlistment(&src_path) && res; + + strbuf_strip_suffix(&workdir_path, "/.git"); + res = remove_deleted_enlistment(&workdir_path) && res; + + strbuf_release(&src_path); + strbuf_release(&workdir_path); + return res; + } + strbuf_release(&src_path); + strbuf_release(&workdir_path); + } + + setup_enlistment_directory(argc, argv, usage, options, NULL); + + return unregister_dir(); +} + +static int cmd_delete(int argc, const char **argv) +{ + char *cwd = xgetcwd(); + struct option options[] = { + OPT_END(), + }; + const char * const usage[] = { + N_("scalar delete <enlistment>"), + NULL + }; + struct strbuf enlistment = STRBUF_INIT; + int res = 0; + + argc = parse_options(argc, argv, NULL, options, + usage, 0); + + if (argc != 1) + usage_with_options(usage, options); + + setup_enlistment_directory(argc, argv, usage, options, &enlistment); + + if (dir_inside_of(cwd, enlistment.buf) >= 0) + res = error(_("refusing to delete current working directory")); + else { + close_object_store(the_repository->objects); + res = delete_enlistment(&enlistment); + } + strbuf_release(&enlistment); + free(cwd); + + return res; +} + +static int cmd_version(int argc, const char **argv) +{ + int verbose = 0, build_options = 0; + struct option options[] = { + OPT__VERBOSE(&verbose, N_("include Git version")), + OPT_BOOL(0, "build-options", &build_options, + N_("include Git's build options")), + OPT_END(), + }; + const char * const usage[] = { + N_("scalar verbose [-v | --verbose] [--build-options]"), + NULL + }; + struct strbuf buf = STRBUF_INIT; + + argc = parse_options(argc, argv, NULL, options, + usage, 0); + + if (argc != 0) + usage_with_options(usage, options); + + get_version_info(&buf, build_options); + fprintf(stderr, "%s\n", buf.buf); + strbuf_release(&buf); + + return 0; +} + +static struct { + const char *name; + int (*fn)(int, const char **); +} builtins[] = { + { "clone", cmd_clone }, + { "list", cmd_list }, + { "register", cmd_register }, + { "unregister", cmd_unregister }, + { "run", cmd_run }, + { "reconfigure", cmd_reconfigure }, + { "delete", cmd_delete }, + { "version", cmd_version }, + { NULL, NULL}, +}; + +int cmd_main(int argc, const char **argv) +{ + struct strbuf scalar_usage = STRBUF_INIT; + int i; + + while (argc > 1 && *argv[1] == '-') { + if (!strcmp(argv[1], "-C")) { + if (argc < 3) + die(_("-C requires a <directory>")); + if (chdir(argv[2]) < 0) + die_errno(_("could not change to '%s'"), + argv[2]); + argc -= 2; + argv += 2; + } else if (!strcmp(argv[1], "-c")) { + if (argc < 3) + die(_("-c requires a <key>=<value> argument")); + git_config_push_parameter(argv[2]); + argc -= 2; + argv += 2; + } else + break; + } + + if (argc > 1) { + argv++; + argc--; + + for (i = 0; builtins[i].name; i++) + if (!strcmp(builtins[i].name, argv[0])) + return !!builtins[i].fn(argc, argv); + } + + strbuf_addstr(&scalar_usage, + N_("scalar [-C <directory>] [-c <key>=<value>] " + "<command> [<options>]\n\nCommands:\n")); + for (i = 0; builtins[i].name; i++) + strbuf_addf(&scalar_usage, "\t%s\n", builtins[i].name); + + usage(scalar_usage.buf); +} diff --git a/contrib/scalar/scalar.txt b/contrib/scalar/scalar.txt new file mode 100644 index 0000000000..cf4e5b889c --- /dev/null +++ b/contrib/scalar/scalar.txt @@ -0,0 +1,155 @@ +scalar(1) +========= + +NAME +---- +scalar - an opinionated repository management tool + +SYNOPSIS +-------- +[verse] +scalar clone [--single-branch] [--branch <main-branch>] [--full-clone] <url> [<enlistment>] +scalar list +scalar register [<enlistment>] +scalar unregister [<enlistment>] +scalar run ( all | config | commit-graph | fetch | loose-objects | pack-files ) [<enlistment>] +scalar reconfigure [ --all | <enlistment> ] +scalar delete <enlistment> + +DESCRIPTION +----------- + +Scalar is an opinionated repository management tool. By creating new +repositories or registering existing repositories with Scalar, your Git +experience will speed up. Scalar sets advanced Git config settings, +maintains your repositories in the background, and helps reduce data sent +across the network. + +An important Scalar concept is the enlistment: this is the top-level directory +of the project. It usually contains the subdirectory `src/` which is a Git +worktree. This encourages the separation between tracked files (inside `src/`) +and untracked files, such as build artifacts (outside `src/`). When registering +an existing Git worktree with Scalar whose name is not `src`, the enlistment +will be identical to the worktree. + +The `scalar` command implements various subcommands, and different options +depending on the subcommand. With the exception of `clone`, `list` and +`reconfigure --all`, all subcommands expect to be run in an enlistment. + +The following options can be specified _before_ the subcommand: + +-C <directory>:: + Before running the subcommand, change the working directory. This + option imitates the same option of linkgit:git[1]. + +-c <key>=<value>:: + For the duration of running the specified subcommand, configure this + setting. This option imitates the same option of linkgit:git[1]. + +COMMANDS +-------- + +Clone +~~~~~ + +clone [<options>] <url> [<enlistment>]:: + Clones the specified repository, similar to linkgit:git-clone[1]. By + default, only commit and tree objects are cloned. Once finished, the + worktree is located at `<enlistment>/src`. ++ +The sparse-checkout feature is enabled (except when run with `--full-clone`) +and the only files present are those in the top-level directory. Use +`git sparse-checkout set` to expand the set of directories you want to see, +or `git sparse-checkout disable` to expand to all files (see +linkgit:git-sparse-checkout[1] for more details). You can explore the +subdirectories outside your sparse-checkout by using `git ls-tree +HEAD[:<directory>]`. + +-b <name>:: +--branch <name>:: + Instead of checking out the branch pointed to by the cloned + repository's HEAD, check out the `<name>` branch instead. + +--[no-]single-branch:: + Clone only the history leading to the tip of a single branch, either + specified by the `--branch` option or the primary branch remote's + `HEAD` points at. ++ +Further fetches into the resulting repository will only update the +remote-tracking branch for the branch this option was used for the initial +cloning. If the HEAD at the remote did not point at any branch when +`--single-branch` clone was made, no remote-tracking branch is created. + +--[no-]full-clone:: + A sparse-checkout is initialized by default. This behavior can be + turned off via `--full-clone`. + +List +~~~~ + +list:: + List enlistments that are currently registered by Scalar. This + subcommand does not need to be run inside an enlistment. + +Register +~~~~~~~~ + +register [<enlistment>]:: + Adds the enlistment's repository to the list of registered repositories + and starts background maintenance. If `<enlistment>` is not provided, + then the enlistment associated with the current working directory is + registered. ++ +Note: when this subcommand is called in a worktree that is called `src/`, its +parent directory is considered to be the Scalar enlistment. If the worktree is +_not_ called `src/`, it itself will be considered to be the Scalar enlistment. + +Unregister +~~~~~~~~~~ + +unregister [<enlistment>]:: + Remove the specified repository from the list of repositories + registered with Scalar and stop the scheduled background maintenance. + +Run +~~~ + +scalar run ( all | config | commit-graph | fetch | loose-objects | pack-files ) [<enlistment>]:: + Run the given maintenance task (or all tasks, if `all` was specified). + Except for `all` and `config`, this subcommand simply hands off to + linkgit:git-maintenance[1] (mapping `fetch` to `prefetch` and + `pack-files` to `incremental-repack`). ++ +These tasks are run automatically as part of the scheduled maintenance, +as soon as the repository is registered with Scalar. It should therefore +not be necessary to run this subcommand manually. ++ +The `config` task is specific to Scalar and configures all those +opinionated default settings that make Git work more efficiently with +large repositories. As this task is run as part of `scalar clone` +automatically, explicit invocations of this task are rarely needed. + +Reconfigure +~~~~~~~~~~~ + +After a Scalar upgrade, or when the configuration of a Scalar enlistment +was somehow corrupted or changed by mistake, this subcommand allows to +reconfigure the enlistment. + +With the `--all` option, all enlistments currently registered with Scalar +will be reconfigured. Use this option after each Scalar upgrade. + +Delete +~~~~~~ + +delete <enlistment>:: + This subcommand lets you delete an existing Scalar enlistment from your + local file system, unregistering the repository. + +SEE ALSO +-------- +linkgit:git-clone[1], linkgit:git-maintenance[1]. + +Scalar +--- +Associated with the linkgit:git[1] suite diff --git a/contrib/scalar/t/Makefile b/contrib/scalar/t/Makefile new file mode 100644 index 0000000000..6170672bb3 --- /dev/null +++ b/contrib/scalar/t/Makefile @@ -0,0 +1,78 @@ +# Run scalar tests +# +# Copyright (c) 2005,2021 Junio C Hamano, Johannes Schindelin +# + +-include ../../../config.mak.autogen +-include ../../../config.mak + +SHELL_PATH ?= $(SHELL) +PERL_PATH ?= /usr/bin/perl +RM ?= rm -f +PROVE ?= prove +DEFAULT_TEST_TARGET ?= test +TEST_LINT ?= test-lint + +ifdef TEST_OUTPUT_DIRECTORY +TEST_RESULTS_DIRECTORY = $(TEST_OUTPUT_DIRECTORY)/test-results +else +TEST_RESULTS_DIRECTORY = ../../../t/test-results +endif + +# Shell quote; +SHELL_PATH_SQ = $(subst ','\'',$(SHELL_PATH)) +PERL_PATH_SQ = $(subst ','\'',$(PERL_PATH)) +TEST_RESULTS_DIRECTORY_SQ = $(subst ','\'',$(TEST_RESULTS_DIRECTORY)) + +T = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)) + +all: $(DEFAULT_TEST_TARGET) + +test: $(TEST_LINT) + $(MAKE) aggregate-results-and-cleanup + +prove: $(TEST_LINT) + @echo "*** prove ***"; GIT_CONFIG=.git/config $(PROVE) --exec '$(SHELL_PATH_SQ)' $(GIT_PROVE_OPTS) $(T) :: $(GIT_TEST_OPTS) + $(MAKE) clean-except-prove-cache + +$(T): + @echo "*** $@ ***"; GIT_CONFIG=.git/config '$(SHELL_PATH_SQ)' $@ $(GIT_TEST_OPTS) + +clean-except-prove-cache: + $(RM) -r 'trash directory'.* '$(TEST_RESULTS_DIRECTORY_SQ)' + $(RM) -r valgrind/bin + +clean: clean-except-prove-cache + $(RM) .prove + +test-lint: test-lint-duplicates test-lint-executable test-lint-shell-syntax + +test-lint-duplicates: + @dups=`echo $(T) | tr ' ' '\n' | sed 's/-.*//' | sort | uniq -d` && \ + test -z "$$dups" || { \ + echo >&2 "duplicate test numbers:" $$dups; exit 1; } + +test-lint-executable: + @bad=`for i in $(T); do test -x "$$i" || echo $$i; done` && \ + test -z "$$bad" || { \ + echo >&2 "non-executable tests:" $$bad; exit 1; } + +test-lint-shell-syntax: + @'$(PERL_PATH_SQ)' ../../../t/check-non-portable-shell.pl $(T) + +aggregate-results-and-cleanup: $(T) + $(MAKE) aggregate-results + $(MAKE) clean + +aggregate-results: + for f in '$(TEST_RESULTS_DIRECTORY_SQ)'/t*-*.counts; do \ + echo "$$f"; \ + done | '$(SHELL_PATH_SQ)' ../../../t/aggregate-results.sh + +valgrind: + $(MAKE) GIT_TEST_OPTS="$(GIT_TEST_OPTS) --valgrind" + +test-results: + mkdir -p test-results + +.PHONY: $(T) aggregate-results clean valgrind diff --git a/contrib/scalar/t/t9099-scalar.sh b/contrib/scalar/t/t9099-scalar.sh new file mode 100755 index 0000000000..7e8771d0ef --- /dev/null +++ b/contrib/scalar/t/t9099-scalar.sh @@ -0,0 +1,88 @@ +#!/bin/sh + +test_description='test the `scalar` command' + +TEST_DIRECTORY=$PWD/../../../t +export TEST_DIRECTORY + +# Make it work with --no-bin-wrappers +PATH=$PWD/..:$PATH + +. ../../../t/test-lib.sh + +GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab ../cron.txt" +export GIT_TEST_MAINT_SCHEDULER + +test_expect_success 'scalar shows a usage' ' + test_expect_code 129 scalar -h +' + +test_expect_success 'scalar unregister' ' + git init vanish/src && + scalar register vanish/src && + git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/vanish/src" && + scalar list >scalar.repos && + grep -F "$(pwd)/vanish/src" scalar.repos && + rm -rf vanish/src/.git && + scalar unregister vanish && + test_must_fail git config --get --global --fixed-value \ + maintenance.repo "$(pwd)/vanish/src" && + scalar list >scalar.repos && + ! grep -F "$(pwd)/vanish/src" scalar.repos +' + +test_expect_success 'set up repository to clone' ' + test_commit first && + test_commit second && + test_commit third && + git switch -c parallel first && + mkdir -p 1/2 && + test_commit 1/2/3 && + git config uploadPack.allowFilter true && + git config uploadPack.allowAnySHA1InWant true +' + +test_expect_success 'scalar clone' ' + second=$(git rev-parse --verify second:second.t) && + scalar clone "file://$(pwd)" cloned --single-branch && + ( + cd cloned/src && + + git config --get --global --fixed-value maintenance.repo \ + "$(pwd)" && + + git for-each-ref --format="%(refname)" refs/remotes/origin/ >actual && + echo "refs/remotes/origin/parallel" >expect && + test_cmp expect actual && + + test_path_is_missing 1/2 && + test_must_fail git rev-list --missing=print $second && + git rev-list $second && + git cat-file blob $second >actual && + echo "second" >expect && + test_cmp expect actual + ) +' + +test_expect_success 'scalar reconfigure' ' + git init one/src && + scalar register one && + git -C one/src config core.preloadIndex false && + scalar reconfigure one && + test true = "$(git -C one/src config core.preloadIndex)" && + git -C one/src config core.preloadIndex false && + scalar reconfigure -a && + test true = "$(git -C one/src config core.preloadIndex)" +' + +test_expect_success 'scalar delete without enlistment shows a usage' ' + test_expect_code 129 scalar delete +' + +test_expect_success 'scalar delete with enlistment' ' + scalar delete cloned && + test_path_is_missing cloned +' + +test_done @@ -1056,7 +1056,10 @@ static int read_convert_config(const char *var, const char *value, void *cb) return 0; } -static int count_ident(const char *cp, unsigned long size) +#define ID_STR_DEFAULT "Id" + +static int count_ident(const char *cp, unsigned long size, + const struct ident_action *idact) { /* * "$Id: 0000000000000000000000000000000000000000 $" <=> "$Id$" @@ -1069,13 +1072,13 @@ static int count_ident(const char *cp, unsigned long size) size--; if (ch != '$') continue; - if (size < 3) + if (size < idact->id_len + 1) break; - if (memcmp("Id", cp, 2)) + if (memcmp(idact->id, cp, idact->id_len)) continue; - ch = cp[2]; - cp += 3; - size -= 3; + ch = cp[idact->id_len]; + cp += idact->id_len + 1; + size -= idact->id_len + 1; if (ch == '$') cnt++; /* $Id$ */ if (ch != ':') @@ -1099,11 +1102,11 @@ static int count_ident(const char *cp, unsigned long size) } static int ident_to_git(const char *src, size_t len, - struct strbuf *buf, int ident) + struct strbuf *buf, const struct ident_action *idact) { char *dst, *dollar; - if (!ident || (src && !count_ident(src, len))) + if (!idact->id || (src && !count_ident(src, len, idact))) return 0; if (!buf) @@ -1122,17 +1125,18 @@ static int ident_to_git(const char *src, size_t len, len -= dollar + 1 - src; src = dollar + 1; - if (len > 3 && !memcmp(src, "Id:", 3)) { - dollar = memchr(src + 3, '$', len - 3); + if (len > idact->id_len + 1 && !memcmp(src, idact->id, idact->id_len) && src[idact->id_len] == ':') { + dollar = memchr(src + idact->id_len + 1, '$', len - (idact->id_len + 1)); if (!dollar) break; - if (memchr(src + 3, '\n', dollar - src - 3)) { + if (memchr(src + idact->id_len + 1, '\n', dollar - src - (idact->id_len + 1))) { /* Line break before the next dollar. */ continue; } - memcpy(dst, "Id$", 3); - dst += 3; + memcpy(dst, idact->id, idact->id_len); + dst[idact->id_len] = '$'; + dst += idact->id_len + 1; len -= dollar + 1 - src; src = dollar + 1; } @@ -1143,16 +1147,16 @@ static int ident_to_git(const char *src, size_t len, } static int ident_to_worktree(const char *src, size_t len, - struct strbuf *buf, int ident) + struct strbuf *buf, const struct ident_action *idact) { struct object_id oid; char *to_free = NULL, *dollar, *spc; int cnt; - if (!ident) + if (!idact->id) return 0; - cnt = count_ident(src, len); + cnt = count_ident(src, len, idact); if (!cnt) return 0; @@ -1161,7 +1165,7 @@ static int ident_to_worktree(const char *src, size_t len, to_free = strbuf_detach(buf, NULL); hash_object_file(the_hash_algo, src, len, "blob", &oid); - strbuf_grow(buf, len + cnt * (the_hash_algo->hexsz + 3)); + strbuf_grow(buf, len + cnt * (the_hash_algo->hexsz + idact->id_len + 1)); for (;;) { /* step 1: run to the next '$' */ dollar = memchr(src, '$', len); @@ -1172,14 +1176,14 @@ static int ident_to_worktree(const char *src, size_t len, src = dollar + 1; /* step 2: does it looks like a bit like Id:xxx$ or Id$ ? */ - if (len < 3 || memcmp("Id", src, 2)) + if (len < idact->id_len + 1 || memcmp(idact->id, src, idact->id_len)) continue; /* step 3: skip over Id$ or Id:xxxxx$ */ - if (src[2] == '$') { - src += 3; - len -= 3; - } else if (src[2] == ':') { + if (src[idact->id_len] == '$') { + src += idact->id_len + 1; + len -= idact->id_len + 1; + } else if (src[idact->id_len] == ':') { /* * It's possible that an expanded Id has crept its way into the * repository, we cope with that by stripping the expansion out. @@ -1187,18 +1191,18 @@ static int ident_to_worktree(const char *src, size_t len, * on checkout, which won't go away by stash, but let's keep it * for git-style ids. */ - dollar = memchr(src + 3, '$', len - 3); + dollar = memchr(src + idact->id_len + 1, '$', len - (idact->id_len + 1)); if (!dollar) { /* incomplete keyword, no more '$', so just quit the loop */ break; } - if (memchr(src + 3, '\n', dollar - src - 3)) { + if (memchr(src + idact->id_len + 1, '\n', dollar - src - (idact->id_len + 1))) { /* Line break before the next dollar. */ continue; } - spc = memchr(src + 4, ' ', dollar - src - 4); + spc = memchr(src + idact->id_len + 2, ' ', dollar - src - (idact->id_len + 2)); if (spc && spc < dollar-1) { /* There are spaces in unexpected places. * This is probably an id from some other @@ -1215,7 +1219,8 @@ static int ident_to_worktree(const char *src, size_t len, } /* step 4: substitute */ - strbuf_addstr(buf, "Id: "); + strbuf_addstr(buf, idact->id); + strbuf_addstr(buf, ": "); strbuf_addstr(buf, oid_to_hex(&oid)); strbuf_addstr(buf, " $"); } @@ -1286,11 +1291,19 @@ static struct convert_driver *git_path_check_convert(struct attr_check_item *che return NULL; } -static int git_path_check_ident(struct attr_check_item *check) +static struct ident_action git_path_check_ident(struct attr_check_item *check) { + struct ident_action idact = {.id = NULL, .id_len = 0}; const char *value = check->value; - return !!ATTR_TRUE(value); + if (!ATTR_UNSET(value) && !ATTR_FALSE(value)) { + if (ATTR_TRUE(value)) + idact.id = ID_STR_DEFAULT; + else + idact.id = value; + idact.id_len = strlen(idact.id); + } + return idact; } static struct attr_check *check; @@ -1313,7 +1326,7 @@ void convert_attrs(struct index_state *istate, ca->crlf_action = git_path_check_crlf(ccheck + 4); if (ca->crlf_action == CRLF_UNDEFINED) ca->crlf_action = git_path_check_crlf(ccheck + 0); - ca->ident = git_path_check_ident(ccheck + 1); + ca->ident_action = git_path_check_ident(ccheck + 1); ca->drv = git_path_check_convert(ccheck + 2); if (ca->crlf_action != CRLF_BINARY) { enum eol eol_attr = git_path_check_eol(ccheck + 3); @@ -1433,7 +1446,7 @@ int convert_to_git(struct index_state *istate, len = dst->len; } } - return ret | ident_to_git(src, len, dst, ca.ident); + return ret | ident_to_git(src, len, dst, &ca.ident_action); } void convert_to_git_filter_fd(struct index_state *istate, @@ -1450,7 +1463,7 @@ void convert_to_git_filter_fd(struct index_state *istate, encode_to_git(path, dst->buf, dst->len, dst, ca.working_tree_encoding, conv_flags); crlf_to_git(istate, path, dst->buf, dst->len, dst, ca.crlf_action, conv_flags); - ident_to_git(dst->buf, dst->len, dst, ca.ident); + ident_to_git(dst->buf, dst->len, dst, &ca.ident_action); } static int convert_to_working_tree_ca_internal(const struct conv_attrs *ca, @@ -1462,7 +1475,7 @@ static int convert_to_working_tree_ca_internal(const struct conv_attrs *ca, { int ret = 0, ret_filter = 0; - ret |= ident_to_worktree(src, len, dst, ca->ident); + ret |= ident_to_worktree(src, len, dst, &(ca->ident_action)); if (ret) { src = dst->buf; len = dst->len; @@ -1810,14 +1823,17 @@ struct ident_filter { struct stream_filter filter; struct strbuf left; int state; + const struct ident_action *idact; char ident[GIT_MAX_HEXSZ + 5]; /* ": x40 $" */ }; -static int is_foreign_ident(const char *str) +static int is_foreign_ident(const struct ident_action *idact, const char *str) { int i; - if (!skip_prefix(str, "$Id: ", &str)) + if (str[0] != '$' || strlen(str) < idact->id_len + 3 || + memcmp(str + 1, idact->id, idact->id_len) != 0 || + !skip_prefix(str + 1 + idact->id_len, ": ", &str)) return 0; for (i = 0; str[i]; i++) { if (isspace(str[i]) && str[i+1] != '$') @@ -1847,13 +1863,16 @@ static int ident_filter_fn(struct stream_filter *filter, char *output, size_t *osize_p) { struct ident_filter *ident = (struct ident_filter *)filter; - static const char head[] = "$Id"; + const struct ident_action *idact = ident->idact; if (!input) { /* drain upon eof */ switch (ident->state) { default: - strbuf_add(&ident->left, head, ident->state); + if (ident->state > 0) + strbuf_addch(&ident->left, '$'); + if (ident->state > 1) + strbuf_add(&ident->left, idact->id, ident->state - 1); /* fallthrough */ case IDENT_SKIPPING: /* fallthrough */ @@ -1884,23 +1903,27 @@ static int ident_filter_fn(struct stream_filter *filter, strbuf_addch(&ident->left, ch); if (ch != '\n' && ch != '$') continue; - if (ch == '$' && !is_foreign_ident(ident->left.buf)) { - strbuf_setlen(&ident->left, sizeof(head) - 1); + if (ch == '$' && !is_foreign_ident(idact, ident->left.buf)) { + strbuf_setlen(&ident->left, idact->id_len + 1); strbuf_addstr(&ident->left, ident->ident); } ident->state = IDENT_DRAINING; continue; } - if (ident->state < sizeof(head) && - head[ident->state] == ch) { + if ((ident->state == 0 && ch == '$') || + (ident->state > 0 && ident->state < idact->id_len + 1 && + idact->id[ident->state - 1] == ch)) { ident->state++; continue; } - if (ident->state) - strbuf_add(&ident->left, head, ident->state); - if (ident->state == sizeof(head) - 1) { + if (ident->state) { + strbuf_addch(&ident->left, '$'); + if (ident->state > 1) + strbuf_add(&ident->left, idact->id, ident->state - 1); + } + if (ident->state == idact->id_len + 1) { if (ch != ':' && ch != '$') { strbuf_addch(&ident->left, ch); ident->state = 0; @@ -1935,7 +1958,7 @@ static struct stream_filter_vtbl ident_vtbl = { ident_free_fn, }; -static struct stream_filter *ident_filter(const struct object_id *oid) +static struct stream_filter *ident_filter(const struct object_id *oid, const struct ident_action *idact) { struct ident_filter *ident = xmalloc(sizeof(*ident)); @@ -1944,6 +1967,7 @@ static struct stream_filter *ident_filter(const struct object_id *oid) strbuf_init(&ident->left, 0); ident->filter.vtbl = &ident_vtbl; ident->state = 0; + ident->idact = idact; return (struct stream_filter *)ident; } @@ -1963,8 +1987,8 @@ struct stream_filter *get_stream_filter_ca(const struct conv_attrs *ca, if (classify_conv_attrs(ca) != CA_CLASS_STREAMABLE) return NULL; - if (ca->ident) - filter = ident_filter(oid); + if (ca->ident_action.id) + filter = ident_filter(oid, &(ca->ident_action)); if (output_eol(ca->crlf_action) == EOL_CRLF) filter = cascade_filter(filter, lf_to_crlf_filter()); @@ -76,11 +76,16 @@ enum convert_crlf_action { struct convert_driver; +struct ident_action { + const char *id; + size_t id_len; +}; + struct conv_attrs { struct convert_driver *drv; enum convert_crlf_action attr_action; /* What attr says */ enum convert_crlf_action crlf_action; /* When no attr is set, use core.autocrlf */ - int ident; + struct ident_action ident_action; /* What ident says */ const char *working_tree_encoding; /* Supported encoding or default encoding if NULL */ }; @@ -765,7 +765,7 @@ static int execute(void) set_keep_alive(0); alarm(init_timeout ? init_timeout : timeout); - pktlen = packet_read(0, NULL, NULL, packet_buffer, sizeof(packet_buffer), 0); + pktlen = packet_read(0, packet_buffer, sizeof(packet_buffer), 0); alarm(0); len = strlen(line); diff --git a/diff-merges.c b/diff-merges.c index 5060ccd890..0af4b3f919 100644 --- a/diff-merges.c +++ b/diff-merges.c @@ -17,6 +17,7 @@ static void suppress(struct rev_info *revs) revs->combined_all_paths = 0; revs->merges_imply_patch = 0; revs->merges_need_diff = 0; + revs->remerge_diff = 0; } static void set_separate(struct rev_info *revs) @@ -45,6 +46,12 @@ static void set_dense_combined(struct rev_info *revs) revs->dense_combined_merges = 1; } +static void set_remerge_diff(struct rev_info *revs) +{ + suppress(revs); + revs->remerge_diff = 1; +} + static diff_merges_setup_func_t func_by_opt(const char *optarg) { if (!strcmp(optarg, "off") || !strcmp(optarg, "none")) @@ -57,6 +64,8 @@ static diff_merges_setup_func_t func_by_opt(const char *optarg) return set_combined; else if (!strcmp(optarg, "cc") || !strcmp(optarg, "dense-combined")) return set_dense_combined; + else if (!strcmp(optarg, "r") || !strcmp(optarg, "remerge")) + return set_remerge_diff; else if (!strcmp(optarg, "m") || !strcmp(optarg, "on")) return set_to_default; return NULL; @@ -110,6 +119,9 @@ int diff_merges_parse_opts(struct rev_info *revs, const char **argv) } else if (!strcmp(arg, "--cc")) { set_dense_combined(revs); revs->merges_imply_patch = 1; + } else if (!strcmp(arg, "--remerge-diff")) { + set_remerge_diff(revs); + revs->merges_imply_patch = 1; } else if (!strcmp(arg, "--no-diff-merges")) { suppress(revs); } else if (!strcmp(arg, "--combined-all-paths")) { @@ -18,6 +18,7 @@ #include "submodule-config.h" #include "submodule.h" #include "hashmap.h" +#include "mem-pool.h" #include "ll-merge.h" #include "string-list.h" #include "strvec.h" @@ -773,6 +774,7 @@ struct emitted_diff_symbol { int flags; int indent_off; /* Offset to first non-whitespace character */ int indent_width; /* The visual width of the indentation */ + unsigned id; enum diff_symbol s; }; #define EMITTED_DIFF_SYMBOL_INIT { 0 } @@ -798,9 +800,9 @@ static void append_emitted_diff_symbol(struct diff_options *o, } struct moved_entry { - struct hashmap_entry ent; const struct emitted_diff_symbol *es; struct moved_entry *next_line; + struct moved_entry *next_match; }; struct moved_block { @@ -808,11 +810,6 @@ struct moved_block { int wsd; /* The whitespace delta of this block */ }; -static void moved_block_clear(struct moved_block *b) -{ - memset(b, 0, sizeof(*b)); -} - #define INDENT_BLANKLINE INT_MIN static void fill_es_indent_data(struct emitted_diff_symbol *es) @@ -856,79 +853,41 @@ static void fill_es_indent_data(struct emitted_diff_symbol *es) } static int compute_ws_delta(const struct emitted_diff_symbol *a, - const struct emitted_diff_symbol *b, - int *out) -{ - int a_len = a->len, - b_len = b->len, - a_off = a->indent_off, - a_width = a->indent_width, - b_off = b->indent_off, + const struct emitted_diff_symbol *b) +{ + int a_width = a->indent_width, b_width = b->indent_width; - int delta; - if (a_width == INDENT_BLANKLINE && b_width == INDENT_BLANKLINE) { - *out = INDENT_BLANKLINE; - return 1; - } - - if (a->s == DIFF_SYMBOL_PLUS) - delta = a_width - b_width; - else - delta = b_width - a_width; - - if (a_len - a_off != b_len - b_off || - memcmp(a->line + a_off, b->line + b_off, a_len - a_off)) - return 0; + if (a_width == INDENT_BLANKLINE && b_width == INDENT_BLANKLINE) + return INDENT_BLANKLINE; - *out = delta; - - return 1; + return a_width - b_width; } -static int cmp_in_block_with_wsd(const struct diff_options *o, - const struct moved_entry *cur, - const struct moved_entry *match, - struct moved_block *pmb, - int n) -{ - struct emitted_diff_symbol *l = &o->emitted_symbols->buf[n]; - int al = cur->es->len, bl = match->es->len, cl = l->len; - const char *a = cur->es->line, - *b = match->es->line, - *c = l->line; - int a_off = cur->es->indent_off, - a_width = cur->es->indent_width, - c_off = l->indent_off, - c_width = l->indent_width; +static int cmp_in_block_with_wsd(const struct moved_entry *cur, + const struct emitted_diff_symbol *l, + struct moved_block *pmb) +{ + int a_width = cur->es->indent_width, b_width = l->indent_width; int delta; - /* - * We need to check if 'cur' is equal to 'match'. As those - * are from the same (+/-) side, we do not need to adjust for - * indent changes. However these were found using fuzzy - * matching so we do have to check if they are equal. Here we - * just check the lengths. We delay calling memcmp() to check - * the contents until later as if the length comparison for a - * and c fails we can avoid the call all together. - */ - if (al != bl) + /* The text of each line must match */ + if (cur->es->id != l->id) return 1; - /* If 'l' and 'cur' are both blank then they match. */ - if (a_width == INDENT_BLANKLINE && c_width == INDENT_BLANKLINE) + /* + * If 'l' and 'cur' are both blank then we don't need to check the + * indent. We only need to check cur as we know the strings match. + * */ + if (a_width == INDENT_BLANKLINE) return 0; /* * The indent changes of the block are known and stored in pmb->wsd; * however we need to check if the indent changes of the current line - * match those of the current block and that the text of 'l' and 'cur' - * after the indentation match. + * match those of the current block. */ - if (cur->es->s == DIFF_SYMBOL_PLUS) - delta = a_width - c_width; - else - delta = c_width - a_width; + delta = b_width - a_width; /* * If the previous lines of this block were all blank then set its @@ -937,166 +896,165 @@ static int cmp_in_block_with_wsd(const struct diff_options *o, if (pmb->wsd == INDENT_BLANKLINE) pmb->wsd = delta; - return !(delta == pmb->wsd && al - a_off == cl - c_off && - !memcmp(a, b, al) && ! - memcmp(a + a_off, c + c_off, al - a_off)); + return delta != pmb->wsd; } -static int moved_entry_cmp(const void *hashmap_cmp_fn_data, - const struct hashmap_entry *eptr, - const struct hashmap_entry *entry_or_key, - const void *keydata) +struct interned_diff_symbol { + struct hashmap_entry ent; + struct emitted_diff_symbol *es; +}; + +static int interned_diff_symbol_cmp(const void *hashmap_cmp_fn_data, + const struct hashmap_entry *eptr, + const struct hashmap_entry *entry_or_key, + const void *keydata) { const struct diff_options *diffopt = hashmap_cmp_fn_data; - const struct moved_entry *a, *b; + const struct emitted_diff_symbol *a, *b; unsigned flags = diffopt->color_moved_ws_handling & XDF_WHITESPACE_FLAGS; - a = container_of(eptr, const struct moved_entry, ent); - b = container_of(entry_or_key, const struct moved_entry, ent); - - if (diffopt->color_moved_ws_handling & - COLOR_MOVED_WS_ALLOW_INDENTATION_CHANGE) - /* - * As there is not specific white space config given, - * we'd need to check for a new block, so ignore all - * white space. The setup of the white space - * configuration for the next block is done else where - */ - flags |= XDF_IGNORE_WHITESPACE; + a = container_of(eptr, const struct interned_diff_symbol, ent)->es; + b = container_of(entry_or_key, const struct interned_diff_symbol, ent)->es; - return !xdiff_compare_lines(a->es->line, a->es->len, - b->es->line, b->es->len, - flags); + return !xdiff_compare_lines(a->line + a->indent_off, + a->len - a->indent_off, + b->line + b->indent_off, + b->len - b->indent_off, flags); } -static struct moved_entry *prepare_entry(struct diff_options *o, - int line_no) +static void prepare_entry(struct diff_options *o, struct emitted_diff_symbol *l, + struct interned_diff_symbol *s) { - struct moved_entry *ret = xmalloc(sizeof(*ret)); - struct emitted_diff_symbol *l = &o->emitted_symbols->buf[line_no]; unsigned flags = o->color_moved_ws_handling & XDF_WHITESPACE_FLAGS; - unsigned int hash = xdiff_hash_string(l->line, l->len, flags); - - hashmap_entry_init(&ret->ent, hash); - ret->es = l; - ret->next_line = NULL; + unsigned int hash = xdiff_hash_string(l->line + l->indent_off, + l->len - l->indent_off, flags); - return ret; + hashmap_entry_init(&s->ent, hash); + s->es = l; } -static void add_lines_to_move_detection(struct diff_options *o, - struct hashmap *add_lines, - struct hashmap *del_lines) +struct moved_entry_list { + struct moved_entry *add, *del; +}; + +static struct moved_entry_list *add_lines_to_move_detection(struct diff_options *o, + struct mem_pool *entry_mem_pool) { struct moved_entry *prev_line = NULL; - + struct mem_pool interned_pool; + struct hashmap interned_map; + struct moved_entry_list *entry_list = NULL; + size_t entry_list_alloc = 0; + unsigned id = 0; int n; + + hashmap_init(&interned_map, interned_diff_symbol_cmp, o, 8096); + mem_pool_init(&interned_pool, 1024 * 1024); + for (n = 0; n < o->emitted_symbols->nr; n++) { - struct hashmap *hm; - struct moved_entry *key; + struct interned_diff_symbol key; + struct emitted_diff_symbol *l = &o->emitted_symbols->buf[n]; + struct interned_diff_symbol *s; + struct moved_entry *entry; - switch (o->emitted_symbols->buf[n].s) { - case DIFF_SYMBOL_PLUS: - hm = add_lines; - break; - case DIFF_SYMBOL_MINUS: - hm = del_lines; - break; - default: + if (l->s != DIFF_SYMBOL_PLUS && l->s != DIFF_SYMBOL_MINUS) { prev_line = NULL; continue; } if (o->color_moved_ws_handling & COLOR_MOVED_WS_ALLOW_INDENTATION_CHANGE) - fill_es_indent_data(&o->emitted_symbols->buf[n]); - key = prepare_entry(o, n); - if (prev_line && prev_line->es->s == o->emitted_symbols->buf[n].s) - prev_line->next_line = key; + fill_es_indent_data(l); - hashmap_add(hm, &key->ent); - prev_line = key; + prepare_entry(o, l, &key); + s = hashmap_get_entry(&interned_map, &key, ent, &key.ent); + if (s) { + l->id = s->es->id; + } else { + l->id = id; + ALLOC_GROW_BY(entry_list, id, 1, entry_list_alloc); + hashmap_add(&interned_map, + memcpy(mem_pool_alloc(&interned_pool, + sizeof(key)), + &key, sizeof(key))); + } + entry = mem_pool_alloc(entry_mem_pool, sizeof(*entry)); + entry->es = l; + entry->next_line = NULL; + if (prev_line && prev_line->es->s == l->s) + prev_line->next_line = entry; + prev_line = entry; + if (l->s == DIFF_SYMBOL_PLUS) { + entry->next_match = entry_list[l->id].add; + entry_list[l->id].add = entry; + } else { + entry->next_match = entry_list[l->id].del; + entry_list[l->id].del = entry; + } } + + hashmap_clear(&interned_map); + mem_pool_discard(&interned_pool, 0); + + return entry_list; } static void pmb_advance_or_null(struct diff_options *o, - struct moved_entry *match, - struct hashmap *hm, + struct emitted_diff_symbol *l, struct moved_block *pmb, - int pmb_nr) + int *pmb_nr) { - int i; - for (i = 0; i < pmb_nr; i++) { + int i, j; + + for (i = 0, j = 0; i < *pmb_nr; i++) { + int match; struct moved_entry *prev = pmb[i].match; struct moved_entry *cur = (prev && prev->next_line) ? prev->next_line : NULL; - if (cur && !hm->cmpfn(o, &cur->ent, &match->ent, NULL)) { - pmb[i].match = cur; - } else { - pmb[i].match = NULL; - } - } -} -static void pmb_advance_or_null_multi_match(struct diff_options *o, - struct moved_entry *match, - struct hashmap *hm, - struct moved_block *pmb, - int pmb_nr, int n) -{ - int i; - char *got_match = xcalloc(1, pmb_nr); - - hashmap_for_each_entry_from(hm, match, ent) { - for (i = 0; i < pmb_nr; i++) { - struct moved_entry *prev = pmb[i].match; - struct moved_entry *cur = (prev && prev->next_line) ? - prev->next_line : NULL; - if (!cur) - continue; - if (!cmp_in_block_with_wsd(o, cur, match, &pmb[i], n)) - got_match[i] |= 1; - } - } + if (o->color_moved_ws_handling & + COLOR_MOVED_WS_ALLOW_INDENTATION_CHANGE) + match = cur && + !cmp_in_block_with_wsd(cur, l, &pmb[i]); + else + match = cur && cur->es->id == l->id; - for (i = 0; i < pmb_nr; i++) { - if (got_match[i]) { - /* Advance to the next line */ - pmb[i].match = pmb[i].match->next_line; - } else { - moved_block_clear(&pmb[i]); + if (match) { + pmb[j] = pmb[i]; + pmb[j++].match = cur; } } - - free(got_match); + *pmb_nr = j; } -static int shrink_potential_moved_blocks(struct moved_block *pmb, - int pmb_nr) -{ - int lp, rp; - - /* Shrink the set of potential block to the remaining running */ - for (lp = 0, rp = pmb_nr - 1; lp <= rp;) { - while (lp < pmb_nr && pmb[lp].match) - lp++; - /* lp points at the first NULL now */ +static void fill_potential_moved_blocks(struct diff_options *o, + struct moved_entry *match, + struct emitted_diff_symbol *l, + struct moved_block **pmb_p, + int *pmb_alloc_p, int *pmb_nr_p) - while (rp > -1 && !pmb[rp].match) - rp--; - /* rp points at the last non-NULL */ +{ + struct moved_block *pmb = *pmb_p; + int pmb_alloc = *pmb_alloc_p, pmb_nr = *pmb_nr_p; - if (lp < pmb_nr && rp > -1 && lp < rp) { - pmb[lp] = pmb[rp]; - memset(&pmb[rp], 0, sizeof(pmb[rp])); - rp--; - lp++; - } + /* + * The current line is the start of a new block. + * Setup the set of potential blocks. + */ + for (; match; match = match->next_match) { + ALLOC_GROW(pmb, pmb_nr + 1, pmb_alloc); + if (o->color_moved_ws_handling & + COLOR_MOVED_WS_ALLOW_INDENTATION_CHANGE) + pmb[pmb_nr].wsd = compute_ws_delta(l, match->es); + else + pmb[pmb_nr].wsd = 0; + pmb[pmb_nr++].match = match; } - /* Remember the number of running sets */ - return rp + 1; + *pmb_p = pmb; + *pmb_alloc_p = pmb_alloc; + *pmb_nr_p = pmb_nr; } /* @@ -1115,6 +1073,8 @@ static int shrink_potential_moved_blocks(struct moved_block *pmb, * NEEDSWORK: This uses the same heuristic as blame_entry_score() in blame.c. * Think of a way to unify them. */ +#define DIFF_SYMBOL_MOVED_LINE_ZEBRA_MASK \ + (DIFF_SYMBOL_MOVED_LINE | DIFF_SYMBOL_MOVED_LINE_ALT) static int adjust_last_block(struct diff_options *o, int n, int block_length) { int i, alnum_count = 0; @@ -1131,95 +1091,85 @@ static int adjust_last_block(struct diff_options *o, int n, int block_length) } } for (i = 1; i < block_length + 1; i++) - o->emitted_symbols->buf[n - i].flags &= ~DIFF_SYMBOL_MOVED_LINE; + o->emitted_symbols->buf[n - i].flags &= ~DIFF_SYMBOL_MOVED_LINE_ZEBRA_MASK; return 0; } /* Find blocks of moved code, delegate actual coloring decision to helper */ static void mark_color_as_moved(struct diff_options *o, - struct hashmap *add_lines, - struct hashmap *del_lines) + struct moved_entry_list *entry_list) { struct moved_block *pmb = NULL; /* potentially moved blocks */ int pmb_nr = 0, pmb_alloc = 0; int n, flipped_block = 0, block_length = 0; + enum diff_symbol moved_symbol = DIFF_SYMBOL_BINARY_DIFF_HEADER; for (n = 0; n < o->emitted_symbols->nr; n++) { - struct hashmap *hm = NULL; - struct moved_entry *key; struct moved_entry *match = NULL; struct emitted_diff_symbol *l = &o->emitted_symbols->buf[n]; - enum diff_symbol last_symbol = 0; switch (l->s) { case DIFF_SYMBOL_PLUS: - hm = del_lines; - key = prepare_entry(o, n); - match = hashmap_get_entry(hm, key, ent, NULL); - free(key); + match = entry_list[l->id].del; break; case DIFF_SYMBOL_MINUS: - hm = add_lines; - key = prepare_entry(o, n); - match = hashmap_get_entry(hm, key, ent, NULL); - free(key); + match = entry_list[l->id].add; break; default: flipped_block = 0; } - if (!match) { - int i; - - adjust_last_block(o, n, block_length); - for(i = 0; i < pmb_nr; i++) - moved_block_clear(&pmb[i]); + if (pmb_nr && (!match || l->s != moved_symbol)) { + if (!adjust_last_block(o, n, block_length) && + block_length > 1) { + /* + * Rewind in case there is another match + * starting at the second line of the block + */ + match = NULL; + n -= block_length; + } pmb_nr = 0; block_length = 0; flipped_block = 0; - last_symbol = l->s; + } + if (!match) { + moved_symbol = DIFF_SYMBOL_BINARY_DIFF_HEADER; continue; } if (o->color_moved == COLOR_MOVED_PLAIN) { - last_symbol = l->s; l->flags |= DIFF_SYMBOL_MOVED_LINE; continue; } - if (o->color_moved_ws_handling & - COLOR_MOVED_WS_ALLOW_INDENTATION_CHANGE) - pmb_advance_or_null_multi_match(o, match, hm, pmb, pmb_nr, n); - else - pmb_advance_or_null(o, match, hm, pmb, pmb_nr); - - pmb_nr = shrink_potential_moved_blocks(pmb, pmb_nr); + pmb_advance_or_null(o, l, pmb, &pmb_nr); if (pmb_nr == 0) { - /* - * The current line is the start of a new block. - * Setup the set of potential blocks. - */ - hashmap_for_each_entry_from(hm, match, ent) { - ALLOC_GROW(pmb, pmb_nr + 1, pmb_alloc); - if (o->color_moved_ws_handling & - COLOR_MOVED_WS_ALLOW_INDENTATION_CHANGE) { - if (compute_ws_delta(l, match->es, - &pmb[pmb_nr].wsd)) - pmb[pmb_nr++].match = match; - } else { - pmb[pmb_nr].wsd = 0; - pmb[pmb_nr++].match = match; - } - } + int contiguous = adjust_last_block(o, n, block_length); + + if (!contiguous && block_length > 1) + /* + * Rewind in case there is another match + * starting at the second line of the block + */ + n -= block_length; + else + fill_potential_moved_blocks(o, match, l, + &pmb, &pmb_alloc, + &pmb_nr); - if (adjust_last_block(o, n, block_length) && - pmb_nr && last_symbol != l->s) + if (contiguous && pmb_nr && moved_symbol == l->s) flipped_block = (flipped_block + 1) % 2; else flipped_block = 0; + if (pmb_nr) + moved_symbol = l->s; + else + moved_symbol = DIFF_SYMBOL_BINARY_DIFF_HEADER; + block_length = 0; } @@ -1229,17 +1179,12 @@ static void mark_color_as_moved(struct diff_options *o, if (flipped_block && o->color_moved != COLOR_MOVED_BLOCKS) l->flags |= DIFF_SYMBOL_MOVED_LINE_ALT; } - last_symbol = l->s; } adjust_last_block(o, n, block_length); - for(n = 0; n < pmb_nr; n++) - moved_block_clear(&pmb[n]); free(pmb); } -#define DIFF_SYMBOL_MOVED_LINE_ZEBRA_MASK \ - (DIFF_SYMBOL_MOVED_LINE | DIFF_SYMBOL_MOVED_LINE_ALT) static void dim_moved_lines(struct diff_options *o) { int n; @@ -1573,7 +1518,9 @@ static void emit_diff_symbol_from_struct(struct diff_options *o, static void emit_diff_symbol(struct diff_options *o, enum diff_symbol s, const char *line, int len, unsigned flags) { - struct emitted_diff_symbol e = {line, len, flags, 0, 0, s}; + struct emitted_diff_symbol e = { + .line = line, .len = len, .flags = flags, .s = s + }; if (o->emitted_symbols) append_emitted_diff_symbol(o, &e); @@ -6345,24 +6292,18 @@ static void diff_flush_patch_all_file_pairs(struct diff_options *o) if (o->emitted_symbols) { if (o->color_moved) { - struct hashmap add_lines, del_lines; - - if (o->color_moved_ws_handling & - COLOR_MOVED_WS_ALLOW_INDENTATION_CHANGE) - o->color_moved_ws_handling |= XDF_IGNORE_WHITESPACE; - - hashmap_init(&del_lines, moved_entry_cmp, o, 0); - hashmap_init(&add_lines, moved_entry_cmp, o, 0); + struct mem_pool entry_pool; + struct moved_entry_list *entry_list; - add_lines_to_move_detection(o, &add_lines, &del_lines); - mark_color_as_moved(o, &add_lines, &del_lines); + mem_pool_init(&entry_pool, 1024 * 1024); + entry_list = add_lines_to_move_detection(o, + &entry_pool); + mark_color_as_moved(o, entry_list); if (o->color_moved == COLOR_MOVED_ZEBRA_DIM) dim_moved_lines(o); - hashmap_clear_and_free(&add_lines, struct moved_entry, - ent); - hashmap_clear_and_free(&del_lines, struct moved_entry, - ent); + mem_pool_discard(&entry_pool, 0); + free(entry_list); } for (i = 0; i < esm.nr; i++) @@ -1504,8 +1504,9 @@ static int path_in_sparse_checkout_1(const char *path, struct index_state *istate, int require_cone_mode) { - const char *base; int dtype = DT_REG; + enum pattern_match_result match = UNDECIDED; + const char *end, *slash; /* * We default to accepting a path if there are no patterns or @@ -1516,11 +1517,27 @@ static int path_in_sparse_checkout_1(const char *path, !istate->sparse_checkout_patterns->use_cone_patterns)) return 1; - base = strrchr(path, '/'); - return path_matches_pattern_list(path, strlen(path), base ? base + 1 : path, - &dtype, - istate->sparse_checkout_patterns, - istate) > 0; + /* + * If UNDECIDED, use the match from the parent dir (recursively), or + * fall back to NOT_MATCHED at the topmost level. Note that cone mode + * never returns UNDECIDED, so we will execute only one iteration in + * this case. + */ + for (end = path + strlen(path); + end > path && match == UNDECIDED; + end = slash) { + + for (slash = end - 1; slash > path && *slash != '/'; slash--) + ; /* do nothing */ + + match = path_matches_pattern_list(path, end - path, + slash > path ? slash + 1 : path, &dtype, + istate->sparse_checkout_patterns, istate); + + /* We are going to match the parent dir now */ + dtype = DT_DIR; + } + return match > 0; } int path_in_sparse_checkout(const char *path, diff --git a/environment.c b/environment.c index 9da7f3c1a1..114c70958a 100644 --- a/environment.c +++ b/environment.c @@ -17,6 +17,7 @@ #include "commit.h" #include "strvec.h" #include "object-store.h" +#include "tmp-objdir.h" #include "chdir-notify.h" #include "shallow.h" @@ -41,7 +42,8 @@ const char *git_attributes_file; const char *git_hooks_path; int zlib_compression_level = Z_BEST_SPEED; int pack_compression_level = Z_DEFAULT_COMPRESSION; -int fsync_object_files; +enum fsync_object_files_mode fsync_object_files; +int use_fsync = -1; size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE; size_t packed_git_limit = DEFAULT_PACKED_GIT_LIMIT; size_t delta_base_cache_limit = 96 * 1024 * 1024; @@ -82,7 +84,6 @@ int protect_hfs = PROTECT_HFS_DEFAULT; #define PROTECT_NTFS_DEFAULT 1 #endif int protect_ntfs = PROTECT_NTFS_DEFAULT; -const char *core_fsmonitor; /* * The character that begins a commented line in user-editable file @@ -168,6 +169,10 @@ void setup_git_env(const char *git_dir) args.graft_file = getenv_safe(&to_free, GRAFT_ENVIRONMENT); args.index_file = getenv_safe(&to_free, INDEX_ENVIRONMENT); args.alternate_db = getenv_safe(&to_free, ALTERNATE_DB_ENVIRONMENT); + if (getenv(GIT_QUARANTINE_ENVIRONMENT)) { + args.disable_ref_updates = 1; + } + repo_set_gitdir(the_repository, git_dir, &args); strvec_clear(&to_free); @@ -331,10 +336,14 @@ static void update_relative_gitdir(const char *name, void *data) { char *path = reparent_relative_path(old_cwd, new_cwd, get_git_dir()); + struct tmp_objdir *tmp_objdir = tmp_objdir_unapply_primary_odb(); trace_printf_key(&trace_setup_key, "setup: move $GIT_DIR to '%s'", path); + set_git_dir_1(path); + if (tmp_objdir) + tmp_objdir_reapply_primary_odb(tmp_objdir, old_cwd, new_cwd); free(path); } diff --git a/fetch-pack.c b/fetch-pack.c index a9604f35a3..62ea90541c 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1653,8 +1653,12 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, receive_wanted_refs(&reader, sought, nr_sought); /* get the pack(s) */ + if (git_env_bool("GIT_TRACE_REDACT", 1)) + reader.options |= PACKET_READ_REDACT_URI_PATH; if (process_section_header(&reader, "packfile-uris", 1)) receive_packfile_uris(&reader, &packfile_uris); + reader.options &= ~PACKET_READ_REDACT_URI_PATH; + process_section_header(&reader, "packfile", 0); /* diff --git a/fmt-merge-msg.c b/fmt-merge-msg.c index b969dc6ebb..e4f7810be2 100644 --- a/fmt-merge-msg.c +++ b/fmt-merge-msg.c @@ -9,6 +9,7 @@ #include "branch.h" #include "fmt-merge-msg.h" #include "commit-reach.h" +#include "gpg-interface.h" static int use_branch_desc; static int suppress_dest_pattern_seen; @@ -16,6 +17,8 @@ static struct string_list suppress_dest_patterns = STRING_LIST_INIT_DUP; int fmt_merge_msg_config(const char *key, const char *value, void *cb) { + int status = 0; + if (!strcmp(key, "merge.log") || !strcmp(key, "merge.summary")) { int is_bool; merge_log_config = git_config_bool_or_int(key, value, &is_bool); @@ -34,6 +37,9 @@ int fmt_merge_msg_config(const char *key, const char *value, void *cb) string_list_append(&suppress_dest_patterns, value); suppress_dest_pattern_seen = 1; } else { + status = git_gpg_config(key, value, NULL); + if (status) + return status; return git_default_config(key, value, cb); } return 0; @@ -527,12 +533,13 @@ static void fmt_merge_msg_sigs(struct strbuf *out) else { buf = payload.buf; len = payload.len; - if (check_signature(payload.buf, payload.len, sig.buf, - sig.len, &sigc) && - !sigc.gpg_output) + sigc.payload_type = SIGNATURE_PAYLOAD_TAG; + sigc.payload = strbuf_detach(&payload, &sigc.payload_len); + if (check_signature(&sigc, sig.buf, sig.len) && + !sigc.output) strbuf_addstr(&sig, "gpg verification failed.\n"); else - strbuf_addstr(&sig, sigc.gpg_output); + strbuf_addstr(&sig, sigc.output); } signature_check_clear(&sigc); diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h new file mode 100644 index 0000000000..c16ef09568 --- /dev/null +++ b/fsmonitor--daemon.h @@ -0,0 +1,140 @@ +#ifndef FSMONITOR_DAEMON_H +#define FSMONITOR_DAEMON_H + +#ifdef HAVE_FSMONITOR_DAEMON_BACKEND + +#include "cache.h" +#include "dir.h" +#include "run-command.h" +#include "simple-ipc.h" +#include "thread-utils.h" + +struct fsmonitor_batch; +struct fsmonitor_token_data; + +/* + * Create a new batch of path(s). The returned batch is considered + * private and not linked into the fsmonitor daemon state. The caller + * should fill this batch with one or more paths and then publish it. + */ +struct fsmonitor_batch *fsmonitor_batch__new(void); + +/* + * Free the list of batches starting with this one. + */ +void fsmonitor_batch__free_list(struct fsmonitor_batch *batch); + +/* + * Add this path to this batch of modified files. + * + * The batch should be private and NOT (yet) linked into the fsmonitor + * daemon state and therefore not yet visible to worker threads and so + * no locking is required. + */ +void fsmonitor_batch__add_path(struct fsmonitor_batch *batch, const char *path); + +struct fsmonitor_daemon_backend_data; /* opaque platform-specific data */ + +struct fsmonitor_daemon_state { + pthread_t listener_thread; + pthread_mutex_t main_lock; + + struct strbuf path_worktree_watch; + struct strbuf path_gitdir_watch; + int nr_paths_watching; + + struct fsmonitor_token_data *current_token_data; + + struct strbuf path_cookie_prefix; + pthread_cond_t cookies_cond; + int cookie_seq; + struct hashmap cookies; + + int error_code; + struct fsmonitor_daemon_backend_data *backend_data; + + struct ipc_server_data *ipc_server_data; +}; + +/* + * Pathname classifications. + * + * The daemon classifies the pathnames that it receives from file + * system notification events into the following categories and uses + * that to decide whether clients are told about them. (And to watch + * for file system synchronization events.) + * + * The client should only care about paths within the working + * directory proper (inside the working directory and not ".git" nor + * inside of ".git/"). That is, the client has read the index and is + * asking for a list of any paths in the working directory that have + * been modified since the last token. The client does not care about + * file system changes within the .git directory (such as new loose + * objects or packfiles). So the client will only receive paths that + * are classified as IS_WORKDIR_PATH. + * + * The daemon uses the IS_DOT_GIT and IS_GITDIR internally to mean the + * exact ".git" directory or GITDIR. If the daemon receives a delete + * event for either of these directories, it will automatically + * shutdown, for example. + * + * Note that the daemon DOES NOT explicitly watch nor special case the + * ".git/index" file. The daemon does not read the index and does not + * have any internal index-relative state. The daemon only collects + * the set of modified paths within the working directory. + */ +enum fsmonitor_path_type { + IS_WORKDIR_PATH = 0, + + IS_DOT_GIT, + IS_INSIDE_DOT_GIT, + IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX, + + IS_GITDIR, + IS_INSIDE_GITDIR, + IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX, + + IS_OUTSIDE_CONE, +}; + +/* + * Classify a pathname relative to the root of the working directory. + */ +enum fsmonitor_path_type fsmonitor_classify_path_workdir_relative( + const char *relative_path); + +/* + * Classify a pathname relative to a <gitdir> that is external to the + * worktree directory. + */ +enum fsmonitor_path_type fsmonitor_classify_path_gitdir_relative( + const char *relative_path); + +/* + * Classify an absolute pathname received from a filesystem event. + */ +enum fsmonitor_path_type fsmonitor_classify_path_absolute( + struct fsmonitor_daemon_state *state, + const char *path); + +/* + * Prepend the this batch of path(s) onto the list of batches associated + * with the current token. This makes the batch visible to worker threads. + * + * The caller no longer owns the batch and must not free it. + * + * Wake up the client threads waiting on these cookies. + */ +void fsmonitor_publish(struct fsmonitor_daemon_state *state, + struct fsmonitor_batch *batch, + const struct string_list *cookie_names); + +/* + * If the platform-specific layer loses sync with the filesystem, + * it should call this to invalidate cached data and abort waiting + * threads. + */ +void fsmonitor_force_resync(struct fsmonitor_daemon_state *state); + +#endif /* HAVE_FSMONITOR_DAEMON_BACKEND */ +#endif /* FSMONITOR_DAEMON_H */ diff --git a/fsmonitor-ipc.c b/fsmonitor-ipc.c new file mode 100644 index 0000000000..ccc32d2a17 --- /dev/null +++ b/fsmonitor-ipc.c @@ -0,0 +1,176 @@ +#include "cache.h" +#include "fsmonitor.h" +#include "simple-ipc.h" +#include "fsmonitor-ipc.h" +#include "run-command.h" +#include "strbuf.h" +#include "trace2.h" + +#ifdef HAVE_FSMONITOR_DAEMON_BACKEND + +int fsmonitor_ipc__is_supported(void) +{ + return 1; +} + +GIT_PATH_FUNC(fsmonitor_ipc__get_path, "fsmonitor--daemon.ipc") + +enum ipc_active_state fsmonitor_ipc__get_state(void) +{ + return ipc_get_active_state(fsmonitor_ipc__get_path()); +} + +static int spawn_daemon(void) +{ + const char *args[] = { "fsmonitor--daemon", "start", NULL }; + + return run_command_v_opt_tr2(args, RUN_COMMAND_NO_STDIN | RUN_GIT_CMD, + "fsmonitor"); +} + +int fsmonitor_ipc__send_query(const char *since_token, + struct strbuf *answer) +{ + int ret = -1; + int tried_to_spawn = 0; + enum ipc_active_state state = IPC_STATE__OTHER_ERROR; + struct ipc_client_connection *connection = NULL; + struct ipc_client_connect_options options + = IPC_CLIENT_CONNECT_OPTIONS_INIT; + const char *tok = since_token ? since_token : ""; + size_t tok_len = since_token ? strlen(since_token) : 0; + + options.wait_if_busy = 1; + options.wait_if_not_found = 0; + + trace2_region_enter("fsm_client", "query", NULL); + trace2_data_string("fsm_client", NULL, "query/command", tok); + +try_again: + state = ipc_client_try_connect(fsmonitor_ipc__get_path(), &options, + &connection); + + switch (state) { + case IPC_STATE__LISTENING: + ret = ipc_client_send_command_to_connection( + connection, tok, tok_len, answer); + ipc_client_close_connection(connection); + + trace2_data_intmax("fsm_client", NULL, + "query/response-length", answer->len); + + if (fsmonitor_is_trivial_response(answer)) + trace2_data_intmax("fsm_client", NULL, + "query/trivial-response", 1); + + goto done; + + case IPC_STATE__NOT_LISTENING: + case IPC_STATE__PATH_NOT_FOUND: + if (tried_to_spawn) + goto done; + + tried_to_spawn++; + if (spawn_daemon()) + goto done; + + /* + * Try again, but this time give the daemon a chance to + * actually create the pipe/socket. + * + * Granted, the daemon just started so it can't possibly have + * any FS cached yet, so we'll always get a trivial answer. + * BUT the answer should include a new token that can serve + * as the basis for subsequent requests. + */ + options.wait_if_not_found = 1; + goto try_again; + + case IPC_STATE__INVALID_PATH: + ret = error(_("fsmonitor_ipc__send_query: invalid path '%s'"), + fsmonitor_ipc__get_path()); + goto done; + + case IPC_STATE__OTHER_ERROR: + default: + ret = error(_("fsmonitor_ipc__send_query: unspecified error on '%s'"), + fsmonitor_ipc__get_path()); + goto done; + } + +done: + trace2_region_leave("fsm_client", "query", NULL); + + return ret; +} + +int fsmonitor_ipc__send_command(const char *command, + struct strbuf *answer) +{ + struct ipc_client_connection *connection = NULL; + struct ipc_client_connect_options options + = IPC_CLIENT_CONNECT_OPTIONS_INIT; + int ret; + enum ipc_active_state state; + const char *c = command ? command : ""; + size_t c_len = command ? strlen(command) : 0; + + strbuf_reset(answer); + + options.wait_if_busy = 1; + options.wait_if_not_found = 0; + + state = ipc_client_try_connect(fsmonitor_ipc__get_path(), &options, + &connection); + if (state != IPC_STATE__LISTENING) { + die("fsmonitor--daemon is not running"); + return -1; + } + + ret = ipc_client_send_command_to_connection(connection, c, c_len, + answer); + ipc_client_close_connection(connection); + + if (ret == -1) { + die("could not send '%s' command to fsmonitor--daemon", c); + return -1; + } + + return 0; +} + +#else + +/* + * A trivial implementation of the fsmonitor_ipc__ API for unsupported + * platforms. + */ + +int fsmonitor_ipc__is_supported(void) +{ + return 0; +} + +const char *fsmonitor_ipc__get_path(void) +{ + return NULL; +} + +enum ipc_active_state fsmonitor_ipc__get_state(void) +{ + return IPC_STATE__OTHER_ERROR; +} + +int fsmonitor_ipc__send_query(const char *since_token, + struct strbuf *answer) +{ + return -1; +} + +int fsmonitor_ipc__send_command(const char *command, + struct strbuf *answer) +{ + return -1; +} + +#endif diff --git a/fsmonitor-ipc.h b/fsmonitor-ipc.h new file mode 100644 index 0000000000..b6a7067c3a --- /dev/null +++ b/fsmonitor-ipc.h @@ -0,0 +1,48 @@ +#ifndef FSMONITOR_IPC_H +#define FSMONITOR_IPC_H + +#include "simple-ipc.h" + +/* + * Returns true if built-in file system monitor daemon is defined + * for this platform. + */ +int fsmonitor_ipc__is_supported(void); + +/* + * Returns the pathname to the IPC named pipe or Unix domain socket + * where a `git-fsmonitor--daemon` process will listen. This is a + * per-worktree value. + * + * Returns NULL if the daemon is not supported on this platform. + */ +const char *fsmonitor_ipc__get_path(void); + +/* + * Try to determine whether there is a `git-fsmonitor--daemon` process + * listening on the IPC pipe/socket. + */ +enum ipc_active_state fsmonitor_ipc__get_state(void); + +/* + * Connect to a `git-fsmonitor--daemon` process via simple-ipc + * and ask for the set of changed files since the given token. + * + * Spawn a daemon process in the background if necessary. + * + * Returns -1 on error; 0 on success. + */ +int fsmonitor_ipc__send_query(const char *since_token, + struct strbuf *answer); + +/* + * Connect to a `git-fsmonitor--daemon` process via simple-ipc and + * send a command verb. If no daemon is available, we DO NOT try to + * start one. + * + * Returns -1 on error; 0 on success. + */ +int fsmonitor_ipc__send_command(const char *command, + struct strbuf *answer); + +#endif /* FSMONITOR_IPC_H */ diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c new file mode 100644 index 0000000000..2770266f5e --- /dev/null +++ b/fsmonitor-settings.c @@ -0,0 +1,97 @@ +#include "cache.h" +#include "config.h" +#include "repository.h" +#include "fsmonitor-settings.h" + +/* + * We keep this structure defintion private and have getters + * for all fields so that we can lazy load it as needed. + */ +struct fsmonitor_settings { + enum fsmonitor_mode mode; + char *hook_path; +}; + +void fsm_settings__set_ipc(struct repository *r) +{ + struct fsmonitor_settings *s = r->settings.fsmonitor; + + s->mode = FSMONITOR_MODE_IPC; +} + +void fsm_settings__set_hook(struct repository *r, const char *path) +{ + struct fsmonitor_settings *s = r->settings.fsmonitor; + + s->mode = FSMONITOR_MODE_HOOK; + s->hook_path = strdup(path); +} + +void fsm_settings__set_disabled(struct repository *r) +{ + struct fsmonitor_settings *s = r->settings.fsmonitor; + + s->mode = FSMONITOR_MODE_DISABLED; + FREE_AND_NULL(s->hook_path); +} + +static int check_for_ipc(struct repository *r) +{ + int value; + + if (!repo_config_get_bool(r, "core.usebuiltinfsmonitor", &value) && + value) { + fsm_settings__set_ipc(r); + return 1; + } + + return 0; +} + +static int check_for_hook(struct repository *r) +{ + const char *const_str; + + if (repo_config_get_pathname(r, "core.fsmonitor", &const_str)) + const_str = getenv("GIT_TEST_FSMONITOR"); + + if (const_str && *const_str) { + fsm_settings__set_hook(r, const_str); + return 1; + } + + return 0; +} + +static void lookup_fsmonitor_settings(struct repository *r) +{ + struct fsmonitor_settings *s; + + CALLOC_ARRAY(s, 1); + + r->settings.fsmonitor = s; + + if (check_for_ipc(r)) + return; + + if (check_for_hook(r)) + return; + + fsm_settings__set_disabled(r); +} + +enum fsmonitor_mode fsm_settings__get_mode(struct repository *r) +{ + if (!r->settings.fsmonitor) + lookup_fsmonitor_settings(r); + + return r->settings.fsmonitor->mode; +} + +const char *fsm_settings__get_hook_path(struct repository *r) +{ + if (!r->settings.fsmonitor) + lookup_fsmonitor_settings(r); + + return r->settings.fsmonitor->hook_path; +} diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h new file mode 100644 index 0000000000..50b2923461 --- /dev/null +++ b/fsmonitor-settings.h @@ -0,0 +1,21 @@ +#ifndef FSMONITOR_SETTINGS_H +#define FSMONITOR_SETTINGS_H + +struct repository; + +enum fsmonitor_mode { + FSMONITOR_MODE_DISABLED = 0, + FSMONITOR_MODE_HOOK = 1, /* core.fsmonitor */ + FSMONITOR_MODE_IPC = 2, /* core.useBuiltinFSMonitor */ +}; + +void fsm_settings__set_ipc(struct repository *r); +void fsm_settings__set_hook(struct repository *r, const char *path); +void fsm_settings__set_disabled(struct repository *r); + +enum fsmonitor_mode fsm_settings__get_mode(struct repository *r); +const char *fsm_settings__get_hook_path(struct repository *r); + +struct fsmonitor_settings; + +#endif /* FSMONITOR_SETTINGS_H */ diff --git a/fsmonitor.c b/fsmonitor.c index ab9bfc60b3..2befde1ffd 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -3,6 +3,7 @@ #include "dir.h" #include "ewah/ewok.h" #include "fsmonitor.h" +#include "fsmonitor-ipc.h" #include "run-command.h" #include "strbuf.h" @@ -148,15 +149,18 @@ void write_fsmonitor_extension(struct strbuf *sb, struct index_state *istate) /* * Call the query-fsmonitor hook passing the last update token of the saved results. */ -static int query_fsmonitor(int version, const char *last_update, struct strbuf *query_result) +static int query_fsmonitor_hook(struct repository *r, + int version, + const char *last_update, + struct strbuf *query_result) { struct child_process cp = CHILD_PROCESS_INIT; int result; - if (!core_fsmonitor) + if (fsm_settings__get_mode(r) != FSMONITOR_MODE_HOOK) return -1; - strvec_push(&cp.args, core_fsmonitor); + strvec_push(&cp.args, fsm_settings__get_hook_path(r)); strvec_pushf(&cp.args, "%d", version); strvec_pushf(&cp.args, "%s", last_update); cp.use_shell = 1; @@ -229,6 +233,45 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) untracked_cache_invalidate_path(istate, name, 0); } +/* + * The number of pathnames that we need to receive from FSMonitor + * before we force the index to be updated. + * + * Note that any pathname within the set of received paths MAY cause + * cache-entry or istate flag bits to be updated and thus cause the + * index to be updated on disk. + * + * However, the response may contain many paths (such as ignored + * paths) that will not update any flag bits. And thus not force the + * index to be updated. (This is fine and normal.) It also means + * that the token will not be updated in the FSMonitor index + * extension. So the next Git command will find the same token in the + * index, make the same token-relative request, and receive the same + * response (plus any newly changed paths). If this response is large + * (and continues to grow), performance could be impacted. + * + * For example, if the user runs a build and it writes 100K object + * files but doesn't modify any source files, the index would not need + * to be updated. The FSMonitor response (after the build and + * relative to a pre-build token) might be 5MB. Each subsequent Git + * command will receive that same 100K/5MB response until something + * causes the index to be updated. And `refresh_fsmonitor()` will + * have to iterate over those 100K paths each time. + * + * Performance could be improved if we optionally force update the + * index after a very large response and get an updated token into + * the FSMonitor index extension. This should allow subsequent + * commands to get smaller and more current responses. + * + * The value chosen here does not need to be precise. The index + * will be updated automatically the first time the user touches + * a tracked file and causes a command like `git status` to + * update an mtime to be updated and/or set a flag bit. + * + * NEEDSWORK: Does this need to be a config value? + */ +static int fsmonitor_force_update_threshold = 100; + void refresh_fsmonitor(struct index_state *istate) { struct strbuf query_result = STRBUF_INIT; @@ -238,17 +281,57 @@ void refresh_fsmonitor(struct index_state *istate) struct strbuf last_update_token = STRBUF_INIT; char *buf; unsigned int i; + struct repository *r = istate->repo ? istate->repo : the_repository; + enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); - if (!core_fsmonitor || istate->fsmonitor_has_run_once) + if (fsm_mode <= FSMONITOR_MODE_DISABLED || + istate->fsmonitor_has_run_once) return; - hook_version = fsmonitor_hook_version(); - istate->fsmonitor_has_run_once = 1; trace_printf_key(&trace_fsmonitor, "refresh fsmonitor"); + + if (fsm_mode == FSMONITOR_MODE_IPC) { + query_success = !fsmonitor_ipc__send_query( + istate->fsmonitor_last_update ? + istate->fsmonitor_last_update : "builtin:fake", + &query_result); + if (query_success) { + /* + * The response contains a series of nul terminated + * strings. The first is the new token. + * + * Use `char *buf` as an interlude to trick the CI + * static analysis to let us use `strbuf_addstr()` + * here (and only copy the token) rather than + * `strbuf_addbuf()`. + */ + buf = query_result.buf; + strbuf_addstr(&last_update_token, buf); + bol = last_update_token.len + 1; + } else { + /* + * The builtin daemon is not available on this + * platform -OR- we failed to get a response. + * + * Generate a fake token (rather than a V1 + * timestamp) for the index extension. (If + * they switch back to the hook API, we don't + * want ambiguous state.) + */ + strbuf_addstr(&last_update_token, "builtin:fake"); + } + + goto apply_results; + } + + assert(fsm_mode == FSMONITOR_MODE_HOOK); + + hook_version = fsmonitor_hook_version(); + /* - * This could be racy so save the date/time now and query_fsmonitor + * This could be racy so save the date/time now and query_fsmonitor_hook * should be inclusive to ensure we don't miss potential changes. */ last_update = getnanotime(); @@ -256,13 +339,14 @@ void refresh_fsmonitor(struct index_state *istate) strbuf_addf(&last_update_token, "%"PRIu64"", last_update); /* - * If we have a last update token, call query_fsmonitor for the set of + * If we have a last update token, call query_fsmonitor_hook for the set of * changes since that token, else assume everything is possibly dirty * and check it all. */ if (istate->fsmonitor_last_update) { if (hook_version == -1 || hook_version == HOOK_INTERFACE_VERSION2) { - query_success = !query_fsmonitor(HOOK_INTERFACE_VERSION2, + query_success = !query_fsmonitor_hook( + r, HOOK_INTERFACE_VERSION2, istate->fsmonitor_last_update, &query_result); if (query_success) { @@ -292,37 +376,76 @@ void refresh_fsmonitor(struct index_state *istate) } if (hook_version == HOOK_INTERFACE_VERSION1) { - query_success = !query_fsmonitor(HOOK_INTERFACE_VERSION1, + query_success = !query_fsmonitor_hook( + r, HOOK_INTERFACE_VERSION1, istate->fsmonitor_last_update, &query_result); } - trace_performance_since(last_update, "fsmonitor process '%s'", core_fsmonitor); - trace_printf_key(&trace_fsmonitor, "fsmonitor process '%s' returned %s", - core_fsmonitor, query_success ? "success" : "failure"); + trace_performance_since(last_update, "fsmonitor process '%s'", + fsm_settings__get_hook_path(r)); + trace_printf_key(&trace_fsmonitor, + "fsmonitor process '%s' returned %s", + fsm_settings__get_hook_path(r), + query_success ? "success" : "failure"); } - /* a fsmonitor process can return '/' to indicate all entries are invalid */ +apply_results: + /* + * The response from FSMonitor (excluding the header token) is + * either: + * + * [a] a (possibly empty) list of NUL delimited relative + * pathnames of changed paths. This list can contain + * files and directories. Directories have a trailing + * slash. + * + * [b] a single '/' to indicate the provider had no + * information and that we should consider everything + * invalid. We call this a trivial response. + */ + trace2_region_enter("fsmonitor", "apply_results", istate->repo); + if (query_success && query_result.buf[bol] != '/') { - /* Mark all entries returned by the monitor as dirty */ + /* + * Mark all pathnames returned by the monitor as dirty. + * + * This updates both the cache-entries and the untracked-cache. + */ + int count = 0; + buf = query_result.buf; for (i = bol; i < query_result.len; i++) { if (buf[i] != '\0') continue; fsmonitor_refresh_callback(istate, buf + bol); bol = i + 1; + count++; } - if (bol < query_result.len) + if (bol < query_result.len) { fsmonitor_refresh_callback(istate, buf + bol); + count++; + } /* Now mark the untracked cache for fsmonitor usage */ if (istate->untracked) istate->untracked->use_fsmonitor = 1; - } else { - /* We only want to run the post index changed hook if we've actually changed entries, so keep track - * if we actually changed entries or not */ + if (count > fsmonitor_force_update_threshold) + istate->cache_changed |= FSMONITOR_CHANGED; + + trace2_data_intmax("fsmonitor", istate->repo, "apply_count", + count); + + } else { + /* + * We received a trivial response, so invalidate everything. + * + * We only want to run the post index changed hook if + * we've actually changed entries, so keep track if we + * actually changed entries or not. + */ int is_cache_changed = 0; - /* Mark all entries invalid */ + for (i = 0; i < istate->cache_nr; i++) { if (istate->cache[i]->ce_flags & CE_FSMONITOR_VALID) { is_cache_changed = 1; @@ -330,13 +453,18 @@ void refresh_fsmonitor(struct index_state *istate) } } - /* If we're going to check every file, ensure we save the results */ + /* + * If we're going to check every file, ensure we save + * the results. + */ if (is_cache_changed) istate->cache_changed |= FSMONITOR_CHANGED; if (istate->untracked) istate->untracked->use_fsmonitor = 0; } + trace2_region_leave("fsmonitor", "apply_results", istate->repo); + strbuf_release(&query_result); /* Now that we've updated istate, save the last_update_token */ @@ -411,7 +539,8 @@ void remove_fsmonitor(struct index_state *istate) void tweak_fsmonitor(struct index_state *istate) { unsigned int i; - int fsmonitor_enabled = git_config_get_fsmonitor(); + struct repository *r = istate->repo ? istate->repo : the_repository; + int fsmonitor_enabled = (fsm_settings__get_mode(r) > FSMONITOR_MODE_DISABLED); if (istate->fsmonitor_dirty) { if (fsmonitor_enabled) { @@ -431,16 +560,8 @@ void tweak_fsmonitor(struct index_state *istate) istate->fsmonitor_dirty = NULL; } - switch (fsmonitor_enabled) { - case -1: /* keep: do nothing */ - break; - case 0: /* false */ - remove_fsmonitor(istate); - break; - case 1: /* true */ + if (fsmonitor_enabled) add_fsmonitor(istate); - break; - default: /* unknown value: do nothing */ - break; - } + else + remove_fsmonitor(istate); } diff --git a/fsmonitor.h b/fsmonitor.h index f20d72631d..f9201411aa 100644 --- a/fsmonitor.h +++ b/fsmonitor.h @@ -3,6 +3,7 @@ #include "cache.h" #include "dir.h" +#include "fsmonitor-settings.h" extern struct trace_key trace_fsmonitor; @@ -57,7 +58,11 @@ int fsmonitor_is_trivial_response(const struct strbuf *query_result); */ static inline int is_fsmonitor_refreshed(const struct index_state *istate) { - return !core_fsmonitor || istate->fsmonitor_has_run_once; + struct repository *r = istate->repo ? istate->repo : the_repository; + enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); + + return fsm_mode <= FSMONITOR_MODE_DISABLED || + istate->fsmonitor_has_run_once; } /* @@ -67,7 +72,11 @@ static inline int is_fsmonitor_refreshed(const struct index_state *istate) */ static inline void mark_fsmonitor_valid(struct index_state *istate, struct cache_entry *ce) { - if (core_fsmonitor && !(ce->ce_flags & CE_FSMONITOR_VALID)) { + struct repository *r = istate->repo ? istate->repo : the_repository; + enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); + + if (fsm_mode > FSMONITOR_MODE_DISABLED && + !(ce->ce_flags & CE_FSMONITOR_VALID)) { istate->cache_changed = 1; ce->ce_flags |= CE_FSMONITOR_VALID; trace_printf_key(&trace_fsmonitor, "mark_fsmonitor_clean '%s'", ce->name); @@ -83,7 +92,10 @@ static inline void mark_fsmonitor_valid(struct index_state *istate, struct cache */ static inline void mark_fsmonitor_invalid(struct index_state *istate, struct cache_entry *ce) { - if (core_fsmonitor) { + struct repository *r = istate->repo ? istate->repo : the_repository; + enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); + + if (fsm_mode > FSMONITOR_MODE_DISABLED) { ce->ce_flags &= ~CE_FSMONITOR_VALID; untracked_cache_invalidate_path(istate, ce->name, 1); trace_printf_key(&trace_fsmonitor, "mark_fsmonitor_invalid '%s'", ce->name); diff --git a/generate-cmdlist.sh b/generate-cmdlist.sh index 9dbbb08e70..394443c66d 100755 --- a/generate-cmdlist.sh +++ b/generate-cmdlist.sh @@ -6,30 +6,32 @@ die () { } command_list () { - eval "grep -ve '^#' $exclude_programs" <"$1" -} - -get_categories () { - tr ' ' '\012'| - grep -v '^$' | - sort | - uniq + while read cmd rest + do + case "$cmd" in + "#"*) + continue; + ;; + *) + case "$exclude_programs" in + *":$cmd:"*) + ;; + *) + if test -n "$1" + then + printf "%s\n" $rest + else + echo "$cmd $rest" + fi + ;; + esac + esac + done } category_list () { - command_list "$1" | - cut -c 40- | - get_categories -} - -get_synopsis () { - sed -n ' - /^NAME/,/'"$1"'/H - ${ - x - s/.*'"$1"' - \(.*\)/N_("\1")/ - p - }' "Documentation/$1.txt" + command_list --no-cat <"$1" | + LC_ALL=C sort -u } define_categories () { @@ -63,24 +65,32 @@ define_category_names () { print_command_list () { echo "static struct cmdname_help command_list[] = {" - command_list "$1" | + command_list <"$1" | while read cmd rest do - printf " { \"$cmd\", $(get_synopsis $cmd), 0" - for cat in $(echo "$rest" | get_categories) + synopsis= + while read line do - printf " | CAT_$cat" - done + case "$line" in + "$cmd - "*) + synopsis=${line#$cmd - } + break + ;; + esac + done <"Documentation/$cmd.txt" + + printf '\t{ "%s", N_("%s"), 0' "$cmd" "$synopsis" + printf " | CAT_%s" $rest echo " }," done echo "};" } -exclude_programs= +exclude_programs=: while test "--exclude-program" = "$1" do shift - exclude_programs="$exclude_programs -e \"^$1 \"" + exclude_programs="$exclude_programs$1:" shift done diff --git a/git-compat-util.h b/git-compat-util.h index 141bb86351..5eebe92811 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1215,6 +1215,13 @@ __attribute__((format (printf, 1, 2))) NORETURN void BUG(const char *fmt, ...); #endif +enum fsync_action { + FSYNC_WRITEOUT_ONLY, + FSYNC_HARDWARE_FLUSH +}; + +int git_fsync(int fd, enum fsync_action action); + /* * Preserves errno, prints a message, but gives no warning for ENOENT. * Returns 0 on success, which includes trying to unlink an object that does diff --git a/git-cvsserver.perl b/git-cvsserver.perl index 64319bed43..4c8118010a 100755 --- a/git-cvsserver.perl +++ b/git-cvsserver.perl @@ -3607,6 +3607,22 @@ package GITCVS::updater; use strict; use warnings; use DBI; +our $_use_fsync; + +# n.b. consider using Git.pm +sub use_fsync { + if (!defined($_use_fsync)) { + my $x = $ENV{GIT_TEST_FSYNC}; + if (defined $x) { + local $ENV{GIT_CONFIG}; + delete $ENV{GIT_CONFIG}; + my $v = ::safe_pipe_capture('git', '-c', "test.fsync=$x", + qw(config --type=bool test.fsync)); + $_use_fsync = defined($v) ? ($v eq "true\n") : 1; + } + } + $_use_fsync; +} =head1 METHODS @@ -3676,6 +3692,9 @@ sub new $self->{dbuser}, $self->{dbpass}); die "Error connecting to database\n" unless defined $self->{dbh}; + if ($self->{dbdriver} eq 'SQLite' && !use_fsync()) { + $self->{dbh}->do('PRAGMA synchronous = OFF'); + } $self->{tables} = {}; foreach my $table ( keys %{$self->{dbh}->table_info(undef,undef,undef,'TABLE')->fetchall_hashref('TABLE_NAME')} ) diff --git a/git-filter-branch.sh b/git-filter-branch.sh index cb89372813..3a51d4507c 100755 --- a/git-filter-branch.sh +++ b/git-filter-branch.sh @@ -579,7 +579,7 @@ if [ "$filter_tag_name" ]; then git hash-object -t tag -w --stdin) || die "Could not create new tag object for $ref" if git cat-file tag "$ref" | \ - sane_grep '^-----BEGIN PGP SIGNATURE-----' >/dev/null 2>&1 + grep '^-----BEGIN PGP SIGNATURE-----' >/dev/null 2>&1 then warn "gpg signature stripped from tag object $sha1t" fi diff --git a/git-instaweb.sh b/git-instaweb.sh index 7c55229773..4349566c89 100755 --- a/git-instaweb.sh +++ b/git-instaweb.sh @@ -49,7 +49,7 @@ resolve_full_httpd () { *apache2*|*lighttpd*|*httpd*) # yes, *httpd* covers *lighttpd* above, but it is there for clarity # ensure that the apache2/lighttpd command ends with "-f" - if ! echo "$httpd" | sane_grep -- '-f *$' >/dev/null 2>&1 + if ! echo "$httpd" | grep -- '-f *$' >/dev/null 2>&1 then httpd="$httpd -f" fi @@ -380,10 +380,7 @@ TypesConfig "$fqgitdir/mime.types" DirectoryIndex gitweb.cgi EOF - # check to see if Dennis Stosberg's mod_perl compatibility patch - # (<20060621130708.Gcbc6e5c@leonov.stosberg.net>) has been applied - if test -f "$module_path/mod_perl.so" && - sane_grep 'MOD_PERL' "$root/gitweb.cgi" >/dev/null + if test -f "$module_path/mod_perl.so" then # favor mod_perl if available cat >> "$conf" <<EOF @@ -402,7 +399,7 @@ EOF # plain-old CGI resolve_full_httpd list_mods=$(echo "$full_httpd" | sed 's/-f$/-l/') - $list_mods | sane_grep 'mod_cgi\.c' >/dev/null 2>&1 || \ + $list_mods | grep 'mod_cgi\.c' >/dev/null 2>&1 || \ if test -f "$module_path/mod_cgi.so" then echo "LoadModule cgi_module $module_path/mod_cgi.so" >> "$conf" @@ -208,70 +208,12 @@ def decode_path(path): def run_git_hook(cmd, param=[]): """Execute a hook if the hook exists.""" - if verbose: - sys.stderr.write("Looking for hook: %s\n" % cmd) - sys.stderr.flush() - - hooks_path = gitConfig("core.hooksPath") - if len(hooks_path) <= 0: - hooks_path = os.path.join(os.environ["GIT_DIR"], "hooks") - - if not isinstance(param, list): - param=[param] - - # resolve hook file name, OS depdenent - hook_file = os.path.join(hooks_path, cmd) - if platform.system() == 'Windows': - if not os.path.isfile(hook_file): - # look for the file with an extension - files = glob.glob(hook_file + ".*") - if not files: - return True - files.sort() - hook_file = files.pop() - while hook_file.upper().endswith(".SAMPLE"): - # The file is a sample hook. We don't want it - if len(files) > 0: - hook_file = files.pop() - else: - return True - - if not os.path.isfile(hook_file) or not os.access(hook_file, os.X_OK): - return True - - return run_hook_command(hook_file, param) == 0 - -def run_hook_command(cmd, param): - """Executes a git hook command - cmd = the command line file to be executed. This can be - a file that is run by OS association. - - param = a list of parameters to pass to the cmd command - - On windows, the extension is checked to see if it should - be run with the Git for Windows Bash shell. If there - is no file extension, the file is deemed a bash shell - and will be handed off to sh.exe. Otherwise, Windows - will be called with the shell to handle the file assocation. - - For non Windows operating systems, the file is called - as an executable. - """ - cli = [cmd] + param - use_shell = False - if platform.system() == 'Windows': - (root,ext) = os.path.splitext(cmd) - if ext == "": - exe_path = os.environ.get("EXEPATH") - if exe_path is None: - exe_path = "" - else: - exe_path = os.path.join(exe_path, "bin") - cli = [os.path.join(exe_path, "SH.EXE")] + cli - else: - use_shell = True - return subprocess.call(cli, shell=use_shell) - + args = ['git', 'hook', 'run', '--ignore-missing', cmd] + if param: + args.append("--") + for p in param: + args.append(p) + return subprocess.call(args) == 0 def write_pipe(c, stdin): if verbose: diff --git a/git-send-email.perl b/git-send-email.perl index 5262d88ee3..a98460bdb9 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -40,7 +40,8 @@ package main; sub usage { print <<EOT; -git send-email [options] <file | directory | rev-list options > +git send-email' [<options>] <file|directory> +git send-email' [<options>] <format-patch options> git send-email --dump-aliases Composing: @@ -113,9 +114,38 @@ EOT exit(1); } +sub uniq { + my %seen; + grep !$seen{$_}++, @_; +} + sub completion_helper { - print Git::command('format-patch', '--git-completion-helper'); - exit(0); + my ($original_opts) = @_; + my %not_for_completion = ( + "git-completion-helper" => undef, + "h" => undef, + ); + my @send_email_opts = (); + + foreach my $key (keys %$original_opts) { + unless (exists $not_for_completion{$key}) { + $key =~ s/!$//; + + if ($key =~ /[:=][si]$/) { + $key =~ s/[:=][si]$//; + push (@send_email_opts, "--$_=") foreach (split (/\|/, $key)); + } else { + push (@send_email_opts, "--$_") foreach (split (/\|/, $key)); + } + } + } + + my @format_patch_opts = split(/ /, Git::command('format-patch', '--git-completion-helper')); + my @opts = (@send_email_opts, @format_patch_opts); + @opts = uniq (grep !/^$/, @opts); + # There's an implicit '\n' here already, no need to add an explicit one. + print "@opts"; + exit(0); } # most mail servers generate the Date: header, but not all... @@ -195,13 +225,13 @@ my $multiedit; my $editor; sub system_or_msg { - my ($args, $msg) = @_; + my ($args, $msg, $cmd_name) = @_; system(@$args); my $signalled = $? & 127; my $exit_code = $? >> 8; return unless $signalled or $exit_code; - my @sprintf_args = ($args->[0], $exit_code); + my @sprintf_args = ($cmd_name ? $cmd_name : $args->[0], $exit_code); if (defined $msg) { # Quiet the 'redundant' warning category, except we # need to support down to Perl 5.8, so we can't do a @@ -425,10 +455,11 @@ my %known_config_keys; my $key = "sendemail.identity"; $identity = Git::config(@repo, $key) if exists $known_config_keys{$key}; } -my $rc = GetOptions( +my %identity_options = ( "identity=s" => \$identity, "no-identity" => \$no_identity, ); +my $rc = GetOptions(%identity_options); usage() unless $rc; undef $identity if $no_identity; @@ -444,14 +475,17 @@ undef $identity if $no_identity; my $help; my $git_completion_helper; -$rc = GetOptions("h" => \$help, - "dump-aliases" => \$dump_aliases); +my %dump_aliases_options = ( + "h" => \$help, + "dump-aliases" => \$dump_aliases, +); +$rc = GetOptions(%dump_aliases_options); usage() unless $rc; die __("--dump-aliases incompatible with other options\n") if !$help and $dump_aliases and @ARGV; -$rc = GetOptions( +my %options = ( "sender|from=s" => \$sender, - "in-reply-to=s" => \$initial_in_reply_to, + "in-reply-to=s" => \$initial_in_reply_to, "reply-to=s" => \$reply_to, "subject=s" => \$initial_subject, "to=s" => \@getopt_to, @@ -508,7 +542,8 @@ $rc = GetOptions( "batch-size=i" => \$batch_size, "relogin-delay=i" => \$relogin_delay, "git-completion-helper" => \$git_completion_helper, - ); +); +$rc = GetOptions(%options); # Munge any "either config or getopt, not both" variables my @initial_to = @getopt_to ? @getopt_to : ($no_to ? () : @config_to); @@ -516,7 +551,8 @@ my @initial_cc = @getopt_cc ? @getopt_cc : ($no_cc ? () : @config_cc); my @initial_bcc = @getopt_bcc ? @getopt_bcc : ($no_bcc ? () : @config_bcc); usage() if $help; -completion_helper() if $git_completion_helper; +my %all_options = (%options, %dump_aliases_options, %identity_options); +completion_helper(\%all_options) if $git_completion_helper; unless ($rc) { usage(); } @@ -2039,10 +2075,10 @@ sub validate_patch { my ($fn, $xfer_encoding) = @_; if ($repo) { + my $hook_name = 'sendemail-validate'; my $hooks_path = $repo->command_oneline('rev-parse', '--git-path', 'hooks'); require File::Spec; - my $validate_hook = File::Spec->catfile($hooks_path, - 'sendemail-validate'); + my $validate_hook = File::Spec->catfile($hooks_path, $hook_name); my $hook_error; if (-x $validate_hook) { require Cwd; @@ -2052,13 +2088,19 @@ sub validate_patch { chdir($repo->wc_path() or $repo->repo_path()) or die("chdir: $!"); local $ENV{"GIT_DIR"} = $repo->repo_path(); - $hook_error = system_or_msg([$validate_hook, $target]); + my @cmd = ("git", "hook", "run", "--ignore-missing", + $hook_name, "--"); + my @cmd_msg = (@cmd, "<patch>"); + my @cmd_run = (@cmd, $target); + $hook_error = system_or_msg(\@cmd_run, undef, "@cmd_msg"); chdir($cwd_save) or die("chdir: $!"); } if ($hook_error) { - die sprintf(__("fatal: %s: rejected by sendemail-validate hook\n" . - "%s\n" . - "warning: no patches were sent\n"), $fn, $hook_error); + $hook_error = sprintf(__("fatal: %s: rejected by %s hook\n" . + $hook_error . "\n" . + "warning: no patches were sent\n"), + $fn, $hook_name); + die $hook_error; } } diff --git a/git-sh-i18n.sh b/git-sh-i18n.sh index e3d9f4836d..a15c0620db 100644 --- a/git-sh-i18n.sh +++ b/git-sh-i18n.sh @@ -51,12 +51,6 @@ gettext_without_eval_gettext) ) } - eval_ngettext () { - ngettext "$1" "$2" "$3" | ( - export PATH $(git sh-i18n--envsubst --variables "$2"); - git sh-i18n--envsubst "$2" - ) - } ;; *) gettext () { @@ -70,12 +64,6 @@ gettext_without_eval_gettext) ) } - eval_ngettext () { - (test "$3" = 1 && printf "%s" "$1" || printf "%s" "$2") | ( - export PATH $(git sh-i18n--envsubst --variables "$2"); - git sh-i18n--envsubst "$2" - ) - } ;; esac diff --git a/git-sh-setup.sh b/git-sh-setup.sh index cee053cdc3..b93f39288c 100644 --- a/git-sh-setup.sh +++ b/git-sh-setup.sh @@ -173,14 +173,6 @@ git_pager() { eval "$GIT_PAGER" '"$@"' } -sane_grep () { - GREP_OPTIONS= LC_ALL=C grep @@SANE_TEXT_GREP@@ "$@" -} - -sane_egrep () { - GREP_OPTIONS= LC_ALL=C egrep @@SANE_TEXT_GREP@@ "$@" -} - is_bare_repository () { git rev-parse --is-bare-repository } @@ -217,9 +209,6 @@ require_clean_work_tree () { then action=$1 case "$action" in - rebase) - gettextln "Cannot rebase: You have unstaged changes." >&2 - ;; "rewrite branches") gettextln "Cannot rewrite branches: You have unstaged changes." >&2 ;; @@ -235,14 +224,7 @@ require_clean_work_tree () { if test $err = 0 then action=$1 - case "$action" in - rebase) - gettextln "Cannot rebase: Your index contains uncommitted changes." >&2 - ;; - *) - eval_gettextln "Cannot \$action: Your index contains uncommitted changes." >&2 - ;; - esac + eval_gettextln "Cannot \$action: Your index contains uncommitted changes." >&2 else gettextln "Additionally, your index contains uncommitted changes." >&2 fi diff --git a/git-submodule.sh b/git-submodule.sh index 652861aa66..ec02eb5971 100755 --- a/git-submodule.sh +++ b/git-submodule.sh @@ -449,6 +449,20 @@ cmd_update() ;; esac + # Cache a pointer to the superproject's gitdir. This may have + # changed, unless it's a fresh clone. Writes it to worktree + # if applicable, otherwise to local. + if test -z "$just_cloned" + then + sm_gitdir="$(git -C "$sm_path" rev-parse --absolute-git-dir)" + relative_gitdir="$(git rev-parse --path-format=relative \ + --prefix "${sm_gitdir}" \ + --git-dir)" + + git -C "$sm_path" config --worktree \ + submodule.superprojectgitdir "$relative_gitdir" + fi + if test -n "$recursive" then ( @@ -533,11 +533,13 @@ static struct cmd_struct commands[] = { { "format-patch", cmd_format_patch, RUN_SETUP }, { "fsck", cmd_fsck, RUN_SETUP }, { "fsck-objects", cmd_fsck, RUN_SETUP }, + { "fsmonitor--daemon", cmd_fsmonitor__daemon, RUN_SETUP }, { "gc", cmd_gc, RUN_SETUP }, { "get-tar-commit-id", cmd_get_tar_commit_id, NO_PARSEOPT }, { "grep", cmd_grep, RUN_SETUP_GENTLY }, { "hash-object", cmd_hash_object }, { "help", cmd_help }, + { "hook", cmd_hook, RUN_SETUP }, { "index-pack", cmd_index_pack, RUN_SETUP_GENTLY | NO_PARSEOPT }, { "init", cmd_init_db }, { "init-db", cmd_init_db }, diff --git a/gpg-interface.c b/gpg-interface.c index 127aecfc2b..bb2d5b2c67 100644 --- a/gpg-interface.c +++ b/gpg-interface.c @@ -3,11 +3,14 @@ #include "config.h" #include "run-command.h" #include "strbuf.h" +#include "dir.h" #include "gpg-interface.h" #include "sigchain.h" #include "tempfile.h" +#include "alias.h" static char *configured_signing_key; +static const char *ssh_default_key_command, *ssh_allowed_signers, *ssh_revocation_file; static enum signature_trust_level configured_min_trust_level = TRUST_UNDEFINED; struct gpg_format { @@ -15,6 +18,14 @@ struct gpg_format { const char *program; const char **verify_args; const char **sigs; + int (*verify_signed_buffer)(struct signature_check *sigc, + struct gpg_format *fmt, + const char *signature, + size_t signature_size); + int (*sign_buffer)(struct strbuf *buffer, struct strbuf *signature, + const char *signing_key); + const char *(*get_default_key)(void); + const char *(*get_key_id)(void); }; static const char *openpgp_verify_args[] = { @@ -35,14 +46,59 @@ static const char *x509_sigs[] = { NULL }; +static const char *ssh_verify_args[] = { NULL }; +static const char *ssh_sigs[] = { + "-----BEGIN SSH SIGNATURE-----", + NULL +}; + +static int verify_gpg_signed_buffer(struct signature_check *sigc, + struct gpg_format *fmt, + const char *signature, + size_t signature_size); +static int verify_ssh_signed_buffer(struct signature_check *sigc, + struct gpg_format *fmt, + const char *signature, + size_t signature_size); +static int sign_buffer_gpg(struct strbuf *buffer, struct strbuf *signature, + const char *signing_key); +static int sign_buffer_ssh(struct strbuf *buffer, struct strbuf *signature, + const char *signing_key); + +static const char *get_default_ssh_signing_key(void); + +static const char *get_ssh_key_id(void); + static struct gpg_format gpg_format[] = { - { .name = "openpgp", .program = "gpg", - .verify_args = openpgp_verify_args, - .sigs = openpgp_sigs + { + .name = "openpgp", + .program = "gpg", + .verify_args = openpgp_verify_args, + .sigs = openpgp_sigs, + .verify_signed_buffer = verify_gpg_signed_buffer, + .sign_buffer = sign_buffer_gpg, + .get_default_key = NULL, + .get_key_id = NULL, }, - { .name = "x509", .program = "gpgsm", - .verify_args = x509_verify_args, - .sigs = x509_sigs + { + .name = "x509", + .program = "gpgsm", + .verify_args = x509_verify_args, + .sigs = x509_sigs, + .verify_signed_buffer = verify_gpg_signed_buffer, + .sign_buffer = sign_buffer_gpg, + .get_default_key = NULL, + .get_key_id = NULL, + }, + { + .name = "ssh", + .program = "ssh-keygen", + .verify_args = ssh_verify_args, + .sigs = ssh_sigs, + .verify_signed_buffer = verify_ssh_signed_buffer, + .sign_buffer = sign_buffer_ssh, + .get_default_key = get_default_ssh_signing_key, + .get_key_id = get_ssh_key_id, }, }; @@ -72,7 +128,7 @@ static struct gpg_format *get_format_by_sig(const char *sig) void signature_check_clear(struct signature_check *sigc) { FREE_AND_NULL(sigc->payload); - FREE_AND_NULL(sigc->gpg_output); + FREE_AND_NULL(sigc->output); FREE_AND_NULL(sigc->gpg_status); FREE_AND_NULL(sigc->signer); FREE_AND_NULL(sigc->key); @@ -257,16 +313,16 @@ error: FREE_AND_NULL(sigc->key); } -static int verify_signed_buffer(const char *payload, size_t payload_size, - const char *signature, size_t signature_size, - struct strbuf *gpg_output, - struct strbuf *gpg_status) +static int verify_gpg_signed_buffer(struct signature_check *sigc, + struct gpg_format *fmt, + const char *signature, + size_t signature_size) { struct child_process gpg = CHILD_PROCESS_INIT; - struct gpg_format *fmt; struct tempfile *temp; int ret; - struct strbuf buf = STRBUF_INIT; + struct strbuf gpg_stdout = STRBUF_INIT; + struct strbuf gpg_stderr = STRBUF_INIT; temp = mks_tempfile_t(".git_vtag_tmpXXXXXX"); if (!temp) @@ -279,10 +335,6 @@ static int verify_signed_buffer(const char *payload, size_t payload_size, return -1; } - fmt = get_format_by_sig(signature); - if (!fmt) - BUG("bad signature '%s'", signature); - strvec_push(&gpg.args, fmt->program); strvec_pushv(&gpg.args, fmt->verify_args); strvec_pushl(&gpg.args, @@ -290,57 +342,303 @@ static int verify_signed_buffer(const char *payload, size_t payload_size, "--verify", temp->filename.buf, "-", NULL); - if (!gpg_status) - gpg_status = &buf; - sigchain_push(SIGPIPE, SIG_IGN); - ret = pipe_command(&gpg, payload, payload_size, - gpg_status, 0, gpg_output, 0); + ret = pipe_command(&gpg, sigc->payload, sigc->payload_len, &gpg_stdout, 0, + &gpg_stderr, 0); sigchain_pop(SIGPIPE); delete_tempfile(&temp); - ret |= !strstr(gpg_status->buf, "\n[GNUPG:] GOODSIG "); - strbuf_release(&buf); /* no matter it was used or not */ + ret |= !strstr(gpg_stdout.buf, "\n[GNUPG:] GOODSIG "); + sigc->output = strbuf_detach(&gpg_stderr, NULL); + sigc->gpg_status = strbuf_detach(&gpg_stdout, NULL); + + parse_gpg_output(sigc); + + strbuf_release(&gpg_stdout); + strbuf_release(&gpg_stderr); return ret; } -int check_signature(const char *payload, size_t plen, const char *signature, - size_t slen, struct signature_check *sigc) +static void parse_ssh_output(struct signature_check *sigc) { - struct strbuf gpg_output = STRBUF_INIT; - struct strbuf gpg_status = STRBUF_INIT; + const char *line, *principal, *search; + char *to_free; + char *key = NULL; + + /* + * ssh-keygen output should be: + * Good "git" signature for PRINCIPAL with RSA key SHA256:FINGERPRINT + * + * or for valid but unknown keys: + * Good "git" signature with RSA key SHA256:FINGERPRINT + * + * Note that "PRINCIPAL" can contain whitespace, "RSA" and + * "SHA256" part could be a different token that names of + * the algorithms used, and "FINGERPRINT" is a hexadecimal + * string. By finding the last occurence of " with ", we can + * reliably parse out the PRINCIPAL. + */ + sigc->result = 'B'; + sigc->trust_level = TRUST_NEVER; + + line = to_free = xmemdupz(sigc->output, strcspn(sigc->output, "\n")); + + if (skip_prefix(line, "Good \"git\" signature for ", &line)) { + /* Valid signature and known principal */ + sigc->result = 'G'; + sigc->trust_level = TRUST_FULLY; + + /* Search for the last "with" to get the full principal */ + principal = line; + do { + search = strstr(line, " with "); + if (search) + line = search + 1; + } while (search != NULL); + sigc->signer = xmemdupz(principal, line - principal - 1); + } else if (skip_prefix(line, "Good \"git\" signature with ", &line)) { + /* Valid signature, but key unknown */ + sigc->result = 'G'; + sigc->trust_level = TRUST_UNDEFINED; + } else { + goto cleanup; + } + + key = strstr(line, "key"); + if (key) { + sigc->fingerprint = xstrdup(strstr(line, "key") + 4); + sigc->key = xstrdup(sigc->fingerprint); + } else { + /* + * Output did not match what we expected + * Treat the signature as bad + */ + sigc->result = 'B'; + } + +cleanup: + free(to_free); +} + +static int verify_ssh_signed_buffer(struct signature_check *sigc, + struct gpg_format *fmt, + const char *signature, + size_t signature_size) +{ + struct child_process ssh_keygen = CHILD_PROCESS_INIT; + struct tempfile *buffer_file; + int ret = -1; + const char *line; + size_t trust_size; + char *principal; + struct strbuf ssh_principals_out = STRBUF_INIT; + struct strbuf ssh_principals_err = STRBUF_INIT; + struct strbuf ssh_keygen_out = STRBUF_INIT; + struct strbuf ssh_keygen_err = STRBUF_INIT; + struct strbuf verify_time = STRBUF_INIT; + const struct date_mode verify_date_mode = { + .type = DATE_STRFTIME, + .strftime_fmt = "%Y%m%d%H%M%S", + /* SSH signing key validity has no timezone information - Use the local timezone */ + .local = 1, + }; + + if (!ssh_allowed_signers) { + error(_("gpg.ssh.allowedSignersFile needs to be configured and exist for ssh signature verification")); + return -1; + } + + buffer_file = mks_tempfile_t(".git_vtag_tmpXXXXXX"); + if (!buffer_file) + return error_errno(_("could not create temporary file")); + if (write_in_full(buffer_file->fd, signature, signature_size) < 0 || + close_tempfile_gently(buffer_file) < 0) { + error_errno(_("failed writing detached signature to '%s'"), + buffer_file->filename.buf); + delete_tempfile(&buffer_file); + return -1; + } + + if (sigc->payload_timestamp) + strbuf_addf(&verify_time, "-Overify-time=%s", + show_date(sigc->payload_timestamp, 0, &verify_date_mode)); + + /* Find the principal from the signers */ + strvec_pushl(&ssh_keygen.args, fmt->program, + "-Y", "find-principals", + "-f", ssh_allowed_signers, + "-s", buffer_file->filename.buf, + verify_time.buf, + NULL); + ret = pipe_command(&ssh_keygen, NULL, 0, &ssh_principals_out, 0, + &ssh_principals_err, 0); + if (ret && strstr(ssh_principals_err.buf, "usage:")) { + error(_("ssh-keygen -Y find-principals/verify is needed for ssh signature verification (available in openssh version 8.2p1+)")); + goto out; + } + if (ret || !ssh_principals_out.len) { + /* + * We did not find a matching principal in the allowedSigners + * Check without validation + */ + child_process_init(&ssh_keygen); + strvec_pushl(&ssh_keygen.args, fmt->program, + "-Y", "check-novalidate", + "-n", "git", + "-s", buffer_file->filename.buf, + verify_time.buf, + NULL); + pipe_command(&ssh_keygen, sigc->payload, sigc->payload_len, + &ssh_keygen_out, 0, &ssh_keygen_err, 0); + + /* + * Fail on unknown keys + * we still call check-novalidate to display the signature info + */ + ret = -1; + } else { + /* Check every principal we found (one per line) */ + for (line = ssh_principals_out.buf; *line; + line = strchrnul(line + 1, '\n')) { + while (*line == '\n') + line++; + if (!*line) + break; + + trust_size = strcspn(line, "\n"); + principal = xmemdupz(line, trust_size); + + child_process_init(&ssh_keygen); + strbuf_release(&ssh_keygen_out); + strbuf_release(&ssh_keygen_err); + strvec_push(&ssh_keygen.args, fmt->program); + /* + * We found principals + * Try with each until we find a match + */ + strvec_pushl(&ssh_keygen.args, "-Y", "verify", + "-n", "git", + "-f", ssh_allowed_signers, + "-I", principal, + "-s", buffer_file->filename.buf, + verify_time.buf, + NULL); + + if (ssh_revocation_file) { + if (file_exists(ssh_revocation_file)) { + strvec_pushl(&ssh_keygen.args, "-r", + ssh_revocation_file, NULL); + } else { + warning(_("ssh signing revocation file configured but not found: %s"), + ssh_revocation_file); + } + } + + sigchain_push(SIGPIPE, SIG_IGN); + ret = pipe_command(&ssh_keygen, sigc->payload, sigc->payload_len, + &ssh_keygen_out, 0, &ssh_keygen_err, 0); + sigchain_pop(SIGPIPE); + + FREE_AND_NULL(principal); + + if (!ret) + ret = !starts_with(ssh_keygen_out.buf, "Good"); + + if (!ret) + break; + } + } + + strbuf_stripspace(&ssh_keygen_out, 0); + strbuf_stripspace(&ssh_keygen_err, 0); + /* Add stderr outputs to show the user actual ssh-keygen errors */ + strbuf_add(&ssh_keygen_out, ssh_principals_err.buf, ssh_principals_err.len); + strbuf_add(&ssh_keygen_out, ssh_keygen_err.buf, ssh_keygen_err.len); + sigc->output = strbuf_detach(&ssh_keygen_out, NULL); + sigc->gpg_status = xstrdup(sigc->output); + + parse_ssh_output(sigc); + +out: + if (buffer_file) + delete_tempfile(&buffer_file); + strbuf_release(&ssh_principals_out); + strbuf_release(&ssh_principals_err); + strbuf_release(&ssh_keygen_out); + strbuf_release(&ssh_keygen_err); + strbuf_release(&verify_time); + + return ret; +} + +static int parse_payload_metadata(struct signature_check *sigc) +{ + const char *ident_line = NULL; + size_t ident_len; + struct ident_split ident; + const char *signer_header; + + switch(sigc->payload_type) { + case SIGNATURE_PAYLOAD_COMMIT: + signer_header = "committer"; + break; + case SIGNATURE_PAYLOAD_TAG: + signer_header = "tagger"; + break; + default: + /* Ignore unknown payload types */ + return 0; + } + + ident_line = find_commit_header(sigc->payload, signer_header, &ident_len); + if (!ident_line || !ident_len) + return 1; + + if (split_ident_line(&ident, ident_line, ident_len)) + return 1; + + if (!sigc->payload_timestamp && ident.date_begin && ident.date_end) + sigc->payload_timestamp = parse_timestamp(ident.date_begin, NULL, 10); + + return 0; +} + +int check_signature(struct signature_check *sigc, + const char *signature, size_t slen) +{ + struct gpg_format *fmt; int status; sigc->result = 'N'; sigc->trust_level = -1; - status = verify_signed_buffer(payload, plen, signature, slen, - &gpg_output, &gpg_status); - if (status && !gpg_output.len) - goto out; - sigc->payload = xmemdupz(payload, plen); - sigc->gpg_output = strbuf_detach(&gpg_output, NULL); - sigc->gpg_status = strbuf_detach(&gpg_status, NULL); - parse_gpg_output(sigc); + fmt = get_format_by_sig(signature); + if (!fmt) + die(_("bad/incompatible signature '%s'"), signature); + + if (parse_payload_metadata(sigc)) + return 1; + + status = fmt->verify_signed_buffer(sigc, fmt, signature, slen); + + if (status && !sigc->output) + return !!status; + status |= sigc->result != 'G'; status |= sigc->trust_level < configured_min_trust_level; - out: - strbuf_release(&gpg_status); - strbuf_release(&gpg_output); - return !!status; } void print_signature_buffer(const struct signature_check *sigc, unsigned flags) { - const char *output = flags & GPG_VERIFY_RAW ? - sigc->gpg_status : sigc->gpg_output; + const char *output = flags & GPG_VERIFY_RAW ? sigc->gpg_status : + sigc->output; if (flags & GPG_VERIFY_VERBOSE && sigc->payload) - fputs(sigc->payload, stdout); + fwrite(sigc->payload, 1, sigc->payload_len, stdout); if (output) fputs(output, stderr); @@ -419,12 +717,33 @@ int git_gpg_config(const char *var, const char *value, void *cb) return 0; } + if (!strcmp(var, "gpg.ssh.defaultkeycommand")) { + if (!value) + return config_error_nonbool(var); + return git_config_string(&ssh_default_key_command, var, value); + } + + if (!strcmp(var, "gpg.ssh.allowedsignersfile")) { + if (!value) + return config_error_nonbool(var); + return git_config_pathname(&ssh_allowed_signers, var, value); + } + + if (!strcmp(var, "gpg.ssh.revocationfile")) { + if (!value) + return config_error_nonbool(var); + return git_config_pathname(&ssh_revocation_file, var, value); + } + if (!strcmp(var, "gpg.program") || !strcmp(var, "gpg.openpgp.program")) fmtname = "openpgp"; if (!strcmp(var, "gpg.x509.program")) fmtname = "x509"; + if (!strcmp(var, "gpg.ssh.program")) + fmtname = "ssh"; + if (fmtname) { fmt = get_format_by_name(fmtname); return git_config_string(&fmt->program, var, value); @@ -433,18 +752,148 @@ int git_gpg_config(const char *var, const char *value, void *cb) return 0; } +static char *get_ssh_key_fingerprint(const char *signing_key) +{ + struct child_process ssh_keygen = CHILD_PROCESS_INIT; + int ret = -1; + struct strbuf fingerprint_stdout = STRBUF_INIT; + struct strbuf **fingerprint; + char *fingerprint_ret; + + /* + * With SSH Signing this can contain a filename or a public key + * For textual representation we usually want a fingerprint + */ + if (starts_with(signing_key, "ssh-")) { + strvec_pushl(&ssh_keygen.args, "ssh-keygen", "-lf", "-", NULL); + ret = pipe_command(&ssh_keygen, signing_key, + strlen(signing_key), &fingerprint_stdout, 0, + NULL, 0); + } else { + strvec_pushl(&ssh_keygen.args, "ssh-keygen", "-lf", + configured_signing_key, NULL); + ret = pipe_command(&ssh_keygen, NULL, 0, &fingerprint_stdout, 0, + NULL, 0); + } + + if (!!ret) + die_errno(_("failed to get the ssh fingerprint for key '%s'"), + signing_key); + + fingerprint = strbuf_split_max(&fingerprint_stdout, ' ', 3); + if (!fingerprint[1]) + die_errno(_("failed to get the ssh fingerprint for key '%s'"), + signing_key); + + fingerprint_ret = strbuf_detach(fingerprint[1], NULL); + strbuf_list_free(fingerprint); + strbuf_release(&fingerprint_stdout); + return fingerprint_ret; +} + +/* Returns the first public key from an ssh-agent to use for signing */ +static const char *get_default_ssh_signing_key(void) +{ + struct child_process ssh_default_key = CHILD_PROCESS_INIT; + int ret = -1; + struct strbuf key_stdout = STRBUF_INIT, key_stderr = STRBUF_INIT; + struct strbuf **keys; + char *key_command = NULL; + const char **argv; + int n; + char *default_key = NULL; + + if (!ssh_default_key_command) + die(_("either user.signingkey or gpg.ssh.defaultKeyCommand needs to be configured")); + + key_command = xstrdup(ssh_default_key_command); + n = split_cmdline(key_command, &argv); + + if (n < 0) + die("malformed build-time gpg.ssh.defaultKeyCommand: %s", + split_cmdline_strerror(n)); + + strvec_pushv(&ssh_default_key.args, argv); + ret = pipe_command(&ssh_default_key, NULL, 0, &key_stdout, 0, + &key_stderr, 0); + + if (!ret) { + keys = strbuf_split_max(&key_stdout, '\n', 2); + if (keys[0] && starts_with(keys[0]->buf, "ssh-")) { + default_key = strbuf_detach(keys[0], NULL); + } else { + warning(_("gpg.ssh.defaultKeycommand succeeded but returned no keys: %s %s"), + key_stderr.buf, key_stdout.buf); + } + + strbuf_list_free(keys); + } else { + warning(_("gpg.ssh.defaultKeyCommand failed: %s %s"), + key_stderr.buf, key_stdout.buf); + } + + free(key_command); + free(argv); + strbuf_release(&key_stdout); + + return default_key; +} + +static const char *get_ssh_key_id(void) { + return get_ssh_key_fingerprint(get_signing_key()); +} + +/* Returns a textual but unique representation of the signing key */ +const char *get_signing_key_id(void) +{ + if (use_format->get_key_id) { + return use_format->get_key_id(); + } + + /* GPG/GPGSM only store a key id on this variable */ + return get_signing_key(); +} + const char *get_signing_key(void) { if (configured_signing_key) return configured_signing_key; - return git_committer_info(IDENT_STRICT|IDENT_NO_DATE); + if (use_format->get_default_key) { + return use_format->get_default_key(); + } + + return git_committer_info(IDENT_STRICT | IDENT_NO_DATE); } int sign_buffer(struct strbuf *buffer, struct strbuf *signature, const char *signing_key) { + return use_format->sign_buffer(buffer, signature, signing_key); +} + +/* + * Strip CR from the line endings, in case we are on Windows. + * NEEDSWORK: make it trim only CRs before LFs and rename + */ +static void remove_cr_after(struct strbuf *buffer, size_t offset) +{ + size_t i, j; + + for (i = j = offset; i < buffer->len; i++) { + if (buffer->buf[i] != '\r') { + if (i != j) + buffer->buf[j] = buffer->buf[i]; + j++; + } + } + strbuf_setlen(buffer, j); +} + +static int sign_buffer_gpg(struct strbuf *buffer, struct strbuf *signature, + const char *signing_key) +{ struct child_process gpg = CHILD_PROCESS_INIT; int ret; - size_t i, j, bottom; + size_t bottom; struct strbuf gpg_status = STRBUF_INIT; strvec_pushl(&gpg.args, @@ -470,13 +919,98 @@ int sign_buffer(struct strbuf *buffer, struct strbuf *signature, const char *sig return error(_("gpg failed to sign the data")); /* Strip CR from the line endings, in case we are on Windows. */ - for (i = j = bottom; i < signature->len; i++) - if (signature->buf[i] != '\r') { - if (i != j) - signature->buf[j] = signature->buf[i]; - j++; - } - strbuf_setlen(signature, j); + remove_cr_after(signature, bottom); return 0; } + +static int sign_buffer_ssh(struct strbuf *buffer, struct strbuf *signature, + const char *signing_key) +{ + struct child_process signer = CHILD_PROCESS_INIT; + int ret = -1; + size_t bottom, keylen; + struct strbuf signer_stderr = STRBUF_INIT; + struct tempfile *key_file = NULL, *buffer_file = NULL; + char *ssh_signing_key_file = NULL; + struct strbuf ssh_signature_filename = STRBUF_INIT; + + if (!signing_key || signing_key[0] == '\0') + return error( + _("user.signingkey needs to be set for ssh signing")); + + if (starts_with(signing_key, "ssh-")) { + /* A literal ssh key */ + key_file = mks_tempfile_t(".git_signing_key_tmpXXXXXX"); + if (!key_file) + return error_errno( + _("could not create temporary file")); + keylen = strlen(signing_key); + if (write_in_full(key_file->fd, signing_key, keylen) < 0 || + close_tempfile_gently(key_file) < 0) { + error_errno(_("failed writing ssh signing key to '%s'"), + key_file->filename.buf); + goto out; + } + ssh_signing_key_file = strbuf_detach(&key_file->filename, NULL); + } else { + /* We assume a file */ + ssh_signing_key_file = expand_user_path(signing_key, 1); + } + + buffer_file = mks_tempfile_t(".git_signing_buffer_tmpXXXXXX"); + if (!buffer_file) { + error_errno(_("could not create temporary file")); + goto out; + } + + if (write_in_full(buffer_file->fd, buffer->buf, buffer->len) < 0 || + close_tempfile_gently(buffer_file) < 0) { + error_errno(_("failed writing ssh signing key buffer to '%s'"), + buffer_file->filename.buf); + goto out; + } + + strvec_pushl(&signer.args, use_format->program, + "-Y", "sign", + "-n", "git", + "-f", ssh_signing_key_file, + buffer_file->filename.buf, + NULL); + + sigchain_push(SIGPIPE, SIG_IGN); + ret = pipe_command(&signer, NULL, 0, NULL, 0, &signer_stderr, 0); + sigchain_pop(SIGPIPE); + + if (ret) { + if (strstr(signer_stderr.buf, "usage:")) + error(_("ssh-keygen -Y sign is needed for ssh signing (available in openssh version 8.2p1+)")); + + error("%s", signer_stderr.buf); + goto out; + } + + bottom = signature->len; + + strbuf_addbuf(&ssh_signature_filename, &buffer_file->filename); + strbuf_addstr(&ssh_signature_filename, ".sig"); + if (strbuf_read_file(signature, ssh_signature_filename.buf, 0) < 0) { + error_errno( + _("failed reading ssh signing data buffer from '%s'"), + ssh_signature_filename.buf); + } + unlink_or_warn(ssh_signature_filename.buf); + + /* Strip CR from the line endings, in case we are on Windows. */ + remove_cr_after(signature, bottom); + +out: + if (key_file) + delete_tempfile(&key_file); + if (buffer_file) + delete_tempfile(&buffer_file); + strbuf_release(&signer_stderr); + strbuf_release(&ssh_signature_filename); + FREE_AND_NULL(ssh_signing_key_file); + return ret; +} diff --git a/gpg-interface.h b/gpg-interface.h index 80567e4894..b30cbdcd3d 100644 --- a/gpg-interface.h +++ b/gpg-interface.h @@ -15,9 +15,19 @@ enum signature_trust_level { TRUST_ULTIMATE, }; +enum payload_type { + SIGNATURE_PAYLOAD_UNDEFINED, + SIGNATURE_PAYLOAD_COMMIT, + SIGNATURE_PAYLOAD_TAG, + SIGNATURE_PAYLOAD_PUSH_CERT, +}; + struct signature_check { char *payload; - char *gpg_output; + size_t payload_len; + enum payload_type payload_type; + timestamp_t payload_timestamp; + char *output; char *gpg_status; /* @@ -64,9 +74,14 @@ int sign_buffer(struct strbuf *buffer, struct strbuf *signature, int git_gpg_config(const char *, const char *, void *); void set_signing_key(const char *); const char *get_signing_key(void); -int check_signature(const char *payload, size_t plen, - const char *signature, size_t slen, - struct signature_check *sigc); + +/* + * Returns a textual unique representation of the signing key in use + * Either a GPG KeyID or a SSH Key Fingerprint + */ +const char *get_signing_key_id(void); +int check_signature(struct signature_check *sigc, + const char *signature, size_t slen); void print_signature_buffer(const struct signature_check *sigc, unsigned flags); @@ -382,8 +382,10 @@ static void compile_pcre2_pattern(struct grep_pat *p, const struct grep_opt *opt } options |= PCRE2_CASELESS; } - if (!opt->ignore_locale && is_utf8_locale() && has_non_ascii(p->pattern) && - !(!opt->ignore_case && (p->fixed || p->is_fixed))) + if ((!opt->ignore_locale && !has_non_ascii(p->pattern)) || + (!opt->ignore_locale && is_utf8_locale() && + has_non_ascii(p->pattern) && !(!opt->ignore_case && + (p->fixed || p->is_fixed)))) options |= (PCRE2_UTF | PCRE2_MATCH_INVALID_UTF); #ifdef GIT_PCRE2_VERSION_10_36_OR_HIGHER @@ -944,10 +946,10 @@ static struct { { "reflog ", 7 }, }; -static int match_one_pattern(struct grep_pat *p, - const char *bol, const char *eol, - enum grep_context ctx, - regmatch_t *pmatch, int eflags) +static int headerless_match_one_pattern(struct grep_pat *p, + const char *bol, const char *eol, + enum grep_context ctx, + regmatch_t *pmatch, int eflags) { int hit = 0; const char *start = bol; @@ -956,25 +958,6 @@ static int match_one_pattern(struct grep_pat *p, ((p->token == GREP_PATTERN_HEAD) != (ctx == GREP_CONTEXT_HEAD))) return 0; - if (p->token == GREP_PATTERN_HEAD) { - const char *field; - size_t len; - assert(p->field < ARRAY_SIZE(header_field)); - field = header_field[p->field].field; - len = header_field[p->field].len; - if (strncmp(bol, field, len)) - return 0; - bol += len; - switch (p->field) { - case GREP_HEADER_AUTHOR: - case GREP_HEADER_COMMITTER: - strip_timestamp(bol, &eol); - break; - default: - break; - } - } - again: hit = patmatch(p, bol, eol, pmatch, eflags); @@ -1025,6 +1008,36 @@ static int match_one_pattern(struct grep_pat *p, return hit; } +static int match_one_pattern(struct grep_pat *p, + const char *bol, const char *eol, + enum grep_context ctx, regmatch_t *pmatch, + int eflags) +{ + const char *field; + size_t len; + + if (p->token == GREP_PATTERN_HEAD) { + assert(p->field < ARRAY_SIZE(header_field)); + field = header_field[p->field].field; + len = header_field[p->field].len; + if (strncmp(bol, field, len)) + return 0; + bol += len; + + switch (p->field) { + case GREP_HEADER_AUTHOR: + case GREP_HEADER_COMMITTER: + strip_timestamp(bol, &eol); + break; + default: + break; + } + } + + return headerless_match_one_pattern(p, bol, eol, ctx, pmatch, eflags); +} + + static int match_expr_eval(struct grep_opt *opt, struct grep_expr *x, const char *bol, const char *eol, enum grep_context ctx, ssize_t *col, @@ -1143,7 +1156,7 @@ static int match_next_pattern(struct grep_pat *p, { regmatch_t match; - if (!match_one_pattern(p, bol, eol, ctx, &match, eflags)) + if (!headerless_match_one_pattern(p, bol, eol, ctx, &match, eflags)) return 0; if (match.rm_so < 0 || match.rm_eo < 0) return 0; @@ -1158,19 +1171,26 @@ static int match_next_pattern(struct grep_pat *p, return 1; } -static int next_match(struct grep_opt *opt, - const char *bol, const char *eol, - enum grep_context ctx, regmatch_t *pmatch, int eflags) +int grep_next_match(struct grep_opt *opt, + const char *bol, const char *eol, + enum grep_context ctx, regmatch_t *pmatch, + enum grep_header_field field, int eflags) { struct grep_pat *p; int hit = 0; pmatch->rm_so = pmatch->rm_eo = -1; if (bol < eol) { - for (p = opt->pattern_list; p; p = p->next) { + for (p = ((ctx == GREP_CONTEXT_HEAD) + ? opt->header_list : opt->pattern_list); + p; p = p->next) { switch (p->token) { - case GREP_PATTERN: /* atom */ case GREP_PATTERN_HEAD: + if ((field != GREP_HEADER_FIELD_MAX) && + (p->field != field)) + continue; + /* fall thru */ + case GREP_PATTERN: /* atom */ case GREP_PATTERN_BODY: hit |= match_next_pattern(p, bol, eol, ctx, pmatch, eflags); @@ -1261,7 +1281,8 @@ static void show_line(struct grep_opt *opt, else if (sign == '=') line_color = opt->colors[GREP_COLOR_FUNCTION]; } - while (next_match(opt, bol, eol, ctx, &match, eflags)) { + while (grep_next_match(opt, bol, eol, ctx, &match, + GREP_HEADER_FIELD_MAX, eflags)) { if (match.rm_so == match.rm_eo) break; @@ -191,6 +191,15 @@ void compile_grep_patterns(struct grep_opt *opt); void free_grep_patterns(struct grep_opt *opt); int grep_buffer(struct grep_opt *opt, const char *buf, unsigned long size); +/* The field parameter is only used to filter header patterns + * (where appropriate). If filtering isn't desirable + * GREP_HEADER_FIELD_MAX should be supplied. + */ +int grep_next_match(struct grep_opt *opt, + const char *bol, const char *eol, + enum grep_context ctx, regmatch_t *pmatch, + enum grep_header_field field, int eflags); + struct grep_source { char *name; @@ -95,12 +95,18 @@ static inline void git_SHA256_Clone(git_SHA256_CTX *dst, const git_SHA256_CTX *s /* Number of algorithms supported (including unknown). */ #define GIT_HASH_NALGOS (GIT_HASH_SHA256 + 1) +/* "sha1", big-endian */ +#define GIT_SHA1_FORMAT_ID 0x73686131 + /* The length in bytes and in hex digits of an object name (SHA-1 value). */ #define GIT_SHA1_RAWSZ 20 #define GIT_SHA1_HEXSZ (2 * GIT_SHA1_RAWSZ) /* The block size of SHA-1. */ #define GIT_SHA1_BLKSZ 64 +/* "s256", big-endian */ +#define GIT_SHA256_FORMAT_ID 0x73323536 + /* The length in bytes and in hex digits of an object name (SHA-256 value). */ #define GIT_SHA256_RAWSZ 32 #define GIT_SHA256_HEXSZ (2 * GIT_SHA256_RAWSZ) @@ -12,6 +12,7 @@ #include "refs.h" #include "parse-options.h" #include "prompt.h" +#include "fsmonitor-ipc.h" struct category_description { uint32_t category; @@ -695,6 +696,9 @@ void get_version_info(struct strbuf *buf, int show_build_options) strbuf_addf(buf, "sizeof-size_t: %d\n", (int)sizeof(size_t)); strbuf_addf(buf, "shell-path: %s\n", SHELL_PATH); /* NEEDSWORK: also save and output GIT-BUILD_OPTIONS? */ + + if (fsmonitor_ipc__is_supported()) + strbuf_addstr(buf, "feature: fsmonitor--daemon\n"); } } @@ -1,6 +1,7 @@ #include "cache.h" #include "hook.h" #include "run-command.h" +#include "config.h" const char *find_hook(const char *name) { @@ -40,3 +41,123 @@ int hook_exists(const char *name) { return !!find_hook(name); } + +void run_hooks_opt_clear(struct run_hooks_opt *o) +{ + strvec_clear(&o->env); + strvec_clear(&o->args); +} + +static int pick_next_hook(struct child_process *cp, + struct strbuf *out, + void *pp_cb, + void **pp_task_cb) +{ + struct hook_cb_data *hook_cb = pp_cb; + const char *hook_path = hook_cb->hook_path; + + if (!hook_path) + return 0; + + cp->no_stdin = 1; + cp->env = hook_cb->options->env.v; + cp->stdout_to_stderr = 1; + cp->trace2_hook_name = hook_cb->hook_name; + cp->dir = hook_cb->options->dir; + + strvec_push(&cp->args, hook_path); + strvec_pushv(&cp->args, hook_cb->options->args.v); + + /* Provide context for errors if necessary */ + *pp_task_cb = (char *)hook_path; + + /* + * This pick_next_hook() will be called again, we're only + * running one hook, so indicate that no more work will be + * done. + */ + hook_cb->hook_path = NULL; + + return 1; +} + +static int notify_start_failure(struct strbuf *out, + void *pp_cb, + void *pp_task_cp) +{ + struct hook_cb_data *hook_cb = pp_cb; + const char *hook_path = pp_task_cp; + + hook_cb->rc |= 1; + + strbuf_addf(out, _("Couldn't start hook '%s'\n"), + hook_path); + + return 1; +} + +static int notify_hook_finished(int result, + struct strbuf *out, + void *pp_cb, + void *pp_task_cb) +{ + struct hook_cb_data *hook_cb = pp_cb; + + hook_cb->rc |= result; + + return 0; +} + +int run_hooks(const char *hook_name, const char *hook_path, + struct run_hooks_opt *options) +{ + struct strbuf abs_path = STRBUF_INIT; + struct hook_cb_data cb_data = { + .rc = 0, + .hook_name = hook_name, + .options = options, + }; + int jobs = 1; + + if (!options) + BUG("a struct run_hooks_opt must be provided to run_hooks"); + + if (options->absolute_path) { + strbuf_add_absolute_path(&abs_path, hook_path); + hook_path = abs_path.buf; + } + cb_data.hook_path = hook_path; + + run_processes_parallel_tr2(jobs, + pick_next_hook, + notify_start_failure, + notify_hook_finished, + &cb_data, + "hook", + hook_name); + + if (options->absolute_path) + strbuf_release(&abs_path); + + return cb_data.rc; +} + +int run_hooks_oneshot(const char *hook_name, struct run_hooks_opt *options) +{ + const char *hook_path; + struct run_hooks_opt hook_opt_scratch = RUN_HOOKS_OPT_INIT; + int ret = 0; + + if (!options) + options = &hook_opt_scratch; + + hook_path = find_hook(hook_name); + if (!hook_path) + goto cleanup; + + ret = run_hooks(hook_name, hook_path, options); +cleanup: + run_hooks_opt_clear(options); + + return ret; +} @@ -1,5 +1,37 @@ #ifndef HOOK_H #define HOOK_H +#include "strvec.h" + +struct run_hooks_opt +{ + /* Environment vars to be set for each hook */ + struct strvec env; + + /* Args to be passed to each hook */ + struct strvec args; + + /* + * Resolve and run the "absolute_path(hook)" instead of + * "hook". Used for "git worktree" hooks + */ + int absolute_path; + + /* Path to initial working directory for subprocess */ + const char *dir; +}; + +#define RUN_HOOKS_OPT_INIT { \ + .env = STRVEC_INIT, \ + .args = STRVEC_INIT, \ +} + +struct hook_cb_data { + /* rc reflects the cumulative failure state */ + int rc; + const char *hook_name; + const char *hook_path; + struct run_hooks_opt *options; +}; /* * Returns the path to the hook file, or NULL if the hook is missing @@ -13,4 +45,28 @@ const char *find_hook(const char *name); */ int hook_exists(const char *hookname); +/** + * Clear data from an initialized "struct run_hooks_opt". + */ +void run_hooks_opt_clear(struct run_hooks_opt *o); + +/** + * Takes an already resolved hook found via find_hook() and runs + * it. Does not call run_hooks_opt_clear() for you. + * + * See run_hooks_oneshot() for the simpler one-shot API. + */ +int run_hooks(const char *hookname, const char *hook_path, + struct run_hooks_opt *options); + +/** + * Calls find_hook() on your "hook_name" and runs the hooks (if any) + * with run_hooks(). + * + * If "options" is provided calls run_hooks_opt_clear() on it for + * you. If "options" is NULL the default options from + * RUN_HOOKS_OPT_INIT will be used. + */ +int run_hooks_oneshot(const char *hook_name, struct run_hooks_opt *options); + #endif diff --git a/http-backend.c b/http-backend.c index e7c0eeab23..3d6e2ff17f 100644 --- a/http-backend.c +++ b/http-backend.c @@ -466,9 +466,7 @@ static void run_service(const char **argv, int buffer_input) struct child_process cld = CHILD_PROCESS_INIT; ssize_t req_len = get_content_length(); - if (encoding && !strcmp(encoding, "gzip")) - gzipped_request = 1; - else if (encoding && !strcmp(encoding, "x-gzip")) + if (encoding && (!strcmp(encoding, "gzip") || !strcmp(encoding, "x-gzip"))) gzipped_request = 1; if (!user || !*user) diff --git a/http-fetch.c b/http-fetch.c index fa642462a9..c7c7d391ac 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -4,6 +4,7 @@ #include "http.h" #include "walker.h" #include "strvec.h" +#include "urlmatch.h" static const char http_fetch_usage[] = "git http-fetch " "[-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin | --packfile=hash | commit-id] url"; @@ -63,8 +64,17 @@ static void fetch_single_packfile(struct object_id *packfile_hash, if (start_active_slot(preq->slot)) { run_active_slot(preq->slot); if (results.curl_result != CURLE_OK) { - die("Unable to get pack file %s\n%s", preq->url, - curl_errorstr); + struct url_info url; + char *nurl = url_normalize(preq->url, &url); + if (!nurl || !git_env_bool("GIT_TRACE_REDACT", 1)) { + die("unable to get pack file '%s'\n%s", preq->url, + curl_errorstr); + } else { + die("failed to get '%.*s' url from '%.*s' " + "(full URL redacted due to GIT_TRACE_REDACT setting)\n%s", + (int)url.scheme_len, url.url, + (int)url.host_len, &url.url[url.host_off], curl_errorstr); + } } } else { die("Unable to start request"); diff --git a/ll-merge.c b/ll-merge.c index 261657578c..4d5bdc1246 100644 --- a/ll-merge.c +++ b/ll-merge.c @@ -11,11 +11,13 @@ #include "run-command.h" #include "ll-merge.h" #include "quote.h" +#include "strbuf.h" struct ll_merge_driver; typedef int (*ll_merge_fn)(const struct ll_merge_driver *, mmbuffer_t *result, + struct strbuf *warnings, const char *path, mmfile_t *orig, const char *orig_name, mmfile_t *src1, const char *name1, @@ -51,6 +53,7 @@ void reset_merge_attributes(void) */ static int ll_binary_merge(const struct ll_merge_driver *drv_unused, mmbuffer_t *result, + struct strbuf *warnings, const char *path, mmfile_t *orig, const char *orig_name, mmfile_t *src1, const char *name1, @@ -59,6 +62,7 @@ static int ll_binary_merge(const struct ll_merge_driver *drv_unused, int marker_size) { mmfile_t *stolen; + const char *msg = "Cannot merge binary files: %s (%s vs. %s)"; assert(opts); /* @@ -71,8 +75,11 @@ static int ll_binary_merge(const struct ll_merge_driver *drv_unused, } else { switch (opts->variant) { default: - warning("Cannot merge binary files: %s (%s vs. %s)", - path, name1, name2); + if (warnings) { + strbuf_addstr(warnings, "Warning: "); + strbuf_addf(warnings, msg, path, name1, name2); + } else + warning(msg, path, name1, name2); /* fallthru */ case XDL_MERGE_FAVOR_OURS: stolen = src1; @@ -98,6 +105,7 @@ static int ll_binary_merge(const struct ll_merge_driver *drv_unused, static int ll_xdl_merge(const struct ll_merge_driver *drv_unused, mmbuffer_t *result, + struct strbuf *warnings, const char *path, mmfile_t *orig, const char *orig_name, mmfile_t *src1, const char *name1, @@ -114,7 +122,7 @@ static int ll_xdl_merge(const struct ll_merge_driver *drv_unused, buffer_is_binary(orig->ptr, orig->size) || buffer_is_binary(src1->ptr, src1->size) || buffer_is_binary(src2->ptr, src2->size)) { - return ll_binary_merge(drv_unused, result, + return ll_binary_merge(drv_unused, result, warnings, path, orig, orig_name, src1, name1, @@ -138,6 +146,7 @@ static int ll_xdl_merge(const struct ll_merge_driver *drv_unused, static int ll_union_merge(const struct ll_merge_driver *drv_unused, mmbuffer_t *result, + struct strbuf *warnings, const char *path, mmfile_t *orig, const char *orig_name, mmfile_t *src1, const char *name1, @@ -150,7 +159,7 @@ static int ll_union_merge(const struct ll_merge_driver *drv_unused, assert(opts); o = *opts; o.variant = XDL_MERGE_FAVOR_UNION; - return ll_xdl_merge(drv_unused, result, path, + return ll_xdl_merge(drv_unused, result, warnings, path, orig, orig_name, src1, name1, src2, name2, &o, marker_size); } @@ -180,6 +189,7 @@ static void create_temp(mmfile_t *src, char *path, size_t len) */ static int ll_ext_merge(const struct ll_merge_driver *fn, mmbuffer_t *result, + struct strbuf *warnings, const char *path, mmfile_t *orig, const char *orig_name, mmfile_t *src1, const char *name1, @@ -362,13 +372,14 @@ static void normalize_file(mmfile_t *mm, const char *path, struct index_state *i } } -int ll_merge(mmbuffer_t *result_buf, - const char *path, - mmfile_t *ancestor, const char *ancestor_label, - mmfile_t *ours, const char *our_label, - mmfile_t *theirs, const char *their_label, - struct index_state *istate, - const struct ll_merge_options *opts) +int ll_merge_with_warnings(mmbuffer_t *result_buf, + struct strbuf *warnings, + const char *path, + mmfile_t *ancestor, const char *ancestor_label, + mmfile_t *ours, const char *our_label, + mmfile_t *theirs, const char *their_label, + struct index_state *istate, + const struct ll_merge_options *opts) { struct attr_check *check = load_merge_attributes(); static const struct ll_merge_options default_opts; @@ -401,11 +412,27 @@ int ll_merge(mmbuffer_t *result_buf, if (opts->extra_marker_size) { marker_size += opts->extra_marker_size; } - return driver->fn(driver, result_buf, path, ancestor, ancestor_label, + return driver->fn(driver, result_buf, warnings, path, + ancestor, ancestor_label, ours, our_label, theirs, their_label, opts, marker_size); } +int ll_merge(mmbuffer_t *result_buf, + const char *path, + mmfile_t *ancestor, const char *ancestor_label, + mmfile_t *ours, const char *our_label, + mmfile_t *theirs, const char *their_label, + struct index_state *istate, + const struct ll_merge_options *opts) +{ + return ll_merge_with_warnings(result_buf, NULL, path, + ancestor, ancestor_label, + ours, our_label, + theirs, their_label, + istate, opts); +} + int ll_merge_marker_size(struct index_state *istate, const char *path) { static struct attr_check *check; diff --git a/ll-merge.h b/ll-merge.h index aceb1b2413..a5469918ad 100644 --- a/ll-merge.h +++ b/ll-merge.h @@ -96,6 +96,15 @@ int ll_merge(mmbuffer_t *result_buf, struct index_state *istate, const struct ll_merge_options *opts); +int ll_merge_with_warnings(mmbuffer_t *result_buf, + struct strbuf *warnings, + const char *path, + mmfile_t *ancestor, const char *ancestor_label, + mmfile_t *ours, const char *our_label, + mmfile_t *theirs, const char *their_label, + struct index_state *istate, + const struct ll_merge_options *opts); + int ll_merge_marker_size(struct index_state *istate, const char *path); void reset_merge_attributes(void); diff --git a/log-tree.c b/log-tree.c index 6dc4412268..9bee36f259 100644 --- a/log-tree.c +++ b/log-tree.c @@ -1,12 +1,15 @@ #include "cache.h" +#include "commit-reach.h" #include "config.h" #include "diff.h" #include "object-store.h" #include "repository.h" +#include "tmp-objdir.h" #include "commit.h" #include "tag.h" #include "graph.h" #include "log-tree.h" +#include "merge-ort.h" #include "reflog-walk.h" #include "refs.h" #include "string-list.h" @@ -16,6 +19,7 @@ #include "line-log.h" #include "help.h" #include "range-diff.h" +#include "dir.h" static struct decoration name_decoration = { "object names" }; static int decoration_loaded; @@ -513,12 +517,13 @@ static void show_signature(struct rev_info *opt, struct commit *commit) if (parse_signed_commit(commit, &payload, &signature, the_hash_algo) <= 0) goto out; - status = check_signature(payload.buf, payload.len, signature.buf, - signature.len, &sigc); - if (status && !sigc.gpg_output) + sigc.payload_type = SIGNATURE_PAYLOAD_COMMIT; + sigc.payload = strbuf_detach(&payload, &sigc.payload_len); + status = check_signature(&sigc, signature.buf, signature.len); + if (status && !sigc.output) show_sig_lines(opt, status, "No signature\n"); else - show_sig_lines(opt, status, sigc.gpg_output); + show_sig_lines(opt, status, sigc.output); signature_check_clear(&sigc); out: @@ -583,10 +588,11 @@ static int show_one_mergetag(struct commit *commit, status = -1; if (parse_signature(extra->value, extra->len, &payload, &signature)) { /* could have a good signature */ - status = check_signature(payload.buf, payload.len, - signature.buf, signature.len, &sigc); - if (sigc.gpg_output) - strbuf_addstr(&verify_message, sigc.gpg_output); + sigc.payload_type = SIGNATURE_PAYLOAD_TAG; + sigc.payload = strbuf_detach(&payload, &sigc.payload_len); + status = check_signature(&sigc, signature.buf, signature.len); + if (sigc.output) + strbuf_addstr(&verify_message, sigc.output); else strbuf_addstr(&verify_message, "No signature\n"); signature_check_clear(&sigc); @@ -902,6 +908,58 @@ static int do_diff_combined(struct rev_info *opt, struct commit *commit) return !opt->loginfo; } +static int do_remerge_diff(struct rev_info *opt, + struct commit_list *parents, + struct object_id *oid, + struct commit *commit) +{ + struct merge_options o; + struct commit_list *bases; + struct merge_result res; + struct pretty_print_context ctx = {0}; + struct strbuf commit1 = STRBUF_INIT; + struct strbuf commit2 = STRBUF_INIT; + + /* Setup merge options */ + init_merge_options(&o, the_repository); + memset(&res, 0, sizeof(res)); + o.show_rename_progress = 0; + + ctx.abbrev = DEFAULT_ABBREV; + format_commit_message(parents->item, "%h (%s)", &commit1, &ctx); + format_commit_message(parents->next->item, "%h (%s)", &commit2, &ctx); + o.branch1 = commit1.buf; + o.branch2 = commit2.buf; + o.record_conflict_msgs_in_tree = 1; + + /* Parse the relevant commits and get the merge bases */ + parse_commit_or_die(parents->item); + parse_commit_or_die(parents->next->item); + bases = get_merge_bases(parents->item, parents->next->item); + + /* Re-merge the parents */ + merge_incore_recursive(&o, + bases, parents->item, parents->next->item, + &res); + + /* Show the diff */ + diff_tree_oid(&res.tree->object.oid, oid, "", &opt->diffopt); + log_tree_diff_flush(opt); + + /* Cleanup */ + strbuf_release(&commit1); + strbuf_release(&commit2); + merge_finalize(&o, &res); + + /* Clean up the temporary object directory */ + if (opt->remerge_objdir != NULL) + tmp_objdir_discard_contents(opt->remerge_objdir); + else + BUG("unable to remove temporary object directory"); + + return !opt->loginfo; +} + /* * Show the diff of a commit. * @@ -936,6 +994,18 @@ static int log_tree_diff(struct rev_info *opt, struct commit *commit, struct log } if (is_merge) { + int octopus = (parents->next->next != NULL); + + if (opt->remerge_diff) { + if (octopus) { + show_log(opt); + fprintf(opt->diffopt.file, + "diff: warning: Skipping remerge-diff " + "for octopus merges.\n"); + return 1; + } + return do_remerge_diff(opt, parents, oid, commit); + } if (opt->combine_merges) return do_diff_combined(opt, commit); if (opt->separate_merges) { diff --git a/merge-ort.c b/merge-ort.c index e5456f4722..da140e0315 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -609,6 +609,7 @@ static int err(struct merge_options *opt, const char *err, ...) static void format_commit(struct strbuf *sb, int indent, + struct repository *repo, struct commit *commit) { struct merge_remote_desc *desc; @@ -622,7 +623,7 @@ static void format_commit(struct strbuf *sb, return; } - format_commit_message(commit, "%h %s", sb, &ctx); + repo_format_commit_message(repo, commit, "%h %s", sb, &ctx); strbuf_addch(sb, '\n'); } @@ -633,7 +634,11 @@ static void path_msg(struct merge_options *opt, const char *fmt, ...) { va_list ap; - struct strbuf *sb = strmap_get(&opt->priv->output, path); + struct strbuf *sb; + + if (opt->record_conflict_msgs_in_tree && omittable_hint) + return; /* Do not record mere hints in tree */ + sb = strmap_get(&opt->priv->output, path); if (!sb) { sb = xmalloc(sizeof(*sb)); strbuf_init(sb, 0); @@ -1578,17 +1583,6 @@ static int merge_submodule(struct merge_options *opt, if (is_null_oid(b)) return 0; - /* - * NEEDSWORK: Remove this when all submodule object accesses are - * through explicitly specified repositores. - */ - if (add_submodule_odb(path)) { - path_msg(opt, path, 0, - _("Failed to merge submodule %s (not checked out)"), - path); - return 0; - } - if (repo_submodule_init(&subrepo, opt->repo, path, null_oid())) { path_msg(opt, path, 0, _("Failed to merge submodule %s (not checked out)"), @@ -1653,7 +1647,7 @@ static int merge_submodule(struct merge_options *opt, break; case 1: - format_commit(&sb, 4, + format_commit(&sb, 4, &subrepo, (struct commit *)merges.objects[0].item); path_msg(opt, path, 0, _("Failed to merge submodule %s, but a possible merge " @@ -1670,7 +1664,7 @@ static int merge_submodule(struct merge_options *opt, break; default: for (i = 0; i < merges.nr; i++) - format_commit(&sb, 4, + format_commit(&sb, 4, &subrepo, (struct commit *)merges.objects[i].item); path_msg(opt, path, 0, _("Failed to merge submodule %s, but multiple " @@ -1754,6 +1748,7 @@ static int merge_3way(struct merge_options *opt, struct ll_merge_options ll_opts = {0}; char *base, *name1, *name2; int merge_status; + struct strbuf warnings = STRBUF_INIT; if (!opt->priv->attr_index.initialized) initialize_attr_index(opt); @@ -1794,10 +1789,14 @@ static int merge_3way(struct merge_options *opt, read_mmblob(&src1, a); read_mmblob(&src2, b); - merge_status = ll_merge(result_buf, path, &orig, base, - &src1, name1, &src2, name2, - &opt->priv->attr_index, &ll_opts); + merge_status = ll_merge_with_warnings(result_buf, &warnings, path, + &orig, base, + &src1, name1, &src2, name2, + &opt->priv->attr_index, &ll_opts); + if (warnings.len > 0) + path_msg(opt, path, 0, "%s", warnings.buf); + strbuf_release(&warnings); free(base); free(name1); free(name2); @@ -2426,7 +2425,7 @@ static void apply_directory_rename_modifications(struct merge_options *opt, */ ci->path_conflict = 1; if (pair->status == 'A') - path_msg(opt, new_path, 0, + path_msg(opt, new_path, 1, _("CONFLICT (file location): %s added in %s " "inside a directory that was renamed in %s, " "suggesting it should perhaps be moved to " @@ -2434,7 +2433,7 @@ static void apply_directory_rename_modifications(struct merge_options *opt, old_path, branch_with_new_path, branch_with_dir_rename, new_path); else - path_msg(opt, new_path, 0, + path_msg(opt, new_path, 1, _("CONFLICT (file location): %s renamed to %s " "in %s, inside a directory that was renamed " "in %s, suggesting it should perhaps be " @@ -3548,6 +3547,74 @@ static void write_completed_directory(struct merge_options *opt, info->last_directory_len = strlen(info->last_directory); } +static void put_path_msgs_in_file(struct merge_options *opt, + const char *path, + struct merged_info *mi, + struct directory_versions *dir_metadata) +{ + struct strbuf tmp = STRBUF_INIT, new_path_contents = STRBUF_INIT; + char *new_path; + int new_path_basic_len, unique_counter; + struct merged_info *new_mi; + char final; + struct strbuf *sb = strmap_get(&opt->priv->output, path); + + assert(opt->record_conflict_msgs_in_tree); + if (!sb) + return; + + /* + * Determine a pathname for recording conflict messages. We'd like it + * to sort just before path, but have a name very similar to what path + * has. + */ + strbuf_addstr(&tmp, path); + final = tmp.buf[tmp.len-1]; + strbuf_setlen(&tmp, tmp.len-1); + strbuf_addf(&tmp, " %c.conflict_msg", final); + + /* + * In extremely unlikely event this filename is not unique, modify it + * with ".<integer>" suffixes until it is. + */ + new_path_basic_len = tmp.len; + unique_counter = 0; + while (strmap_contains(&opt->priv->paths, tmp.buf)) { + strbuf_setlen(&tmp, new_path_basic_len); + strbuf_addf(&tmp, ".%d", ++unique_counter); + } + + /* Now that we have a unique string, move it to our pool */ + new_path = mem_pool_strdup(&opt->priv->pool, tmp.buf); + strbuf_release(&tmp); + + /* Set up contents we want to place in the file. */ + strbuf_addf(&new_path_contents, "== Conflict notices for %s ==\n", + path); + strbuf_addbuf(&new_path_contents, sb); + + /* Set up new_mi */ + new_mi = mem_pool_calloc(&opt->priv->pool, 1, sizeof(*new_mi)); + new_mi->result.mode = 0100644; + new_mi->is_null = 0; + new_mi->clean = 1; + new_mi->basename_offset = mi->basename_offset; + new_mi->directory_name = mi->directory_name; + if (write_object_file(new_path_contents.buf, new_path_contents.len, + blob_type, &new_mi->result.oid)) + die(_("Unable to add %s to database"), new_path); + + /* + * Save new_mi in opt->priv->path (so that something will deallocate + * it later), and record the entry for it. + */ + strmap_put(&opt->priv->paths, new_path, new_mi); + record_entry_for_tree(dir_metadata, new_path, new_mi); + + /* Cleanup */ + strbuf_release(&new_path_contents); +} + /* Per entry merge function */ static void process_entry(struct merge_options *opt, const char *path, @@ -3831,7 +3898,7 @@ static void process_entry(struct merge_options *opt, reason = _("add/add"); if (S_ISGITLINK(merged_file.mode)) reason = _("submodule"); - path_msg(opt, path, 0, + path_msg(opt, path, 1, _("CONFLICT (%s): Merge conflict in %s"), reason, path); } @@ -4008,6 +4075,8 @@ static void process_entries(struct merge_options *opt, struct conflict_info *ci = (struct conflict_info *)mi; process_entry(opt, path, ci, &dir_metadata); } + if (!opt->priv->call_depth && opt->record_conflict_msgs_in_tree) + put_path_msgs_in_file(opt, path, mi, &dir_metadata); } trace2_region_leave("merge", "processing", opt->repo); @@ -4252,6 +4321,9 @@ void merge_switch_to_result(struct merge_options *opt, struct string_list olist = STRING_LIST_INIT_NODUP; int i; + if (opt->record_conflict_msgs_in_tree) + BUG("Either display conflict messages or record them in tree, not both"); + trace2_region_enter("merge", "display messages", opt->repo); /* Hack to pre-allocate olist to the desired size */ diff --git a/merge-recursive.c b/merge-recursive.c index c553751889..53cbf31d98 100644 --- a/merge-recursive.c +++ b/merge-recursive.c @@ -334,7 +334,9 @@ static void output(struct merge_options *opt, int v, const char *fmt, ...) flush_output(opt); } -static void output_commit_title(struct merge_options *opt, struct commit *commit) +static void repo_output_commit_title(struct merge_options *opt, + struct repository *repo, + struct commit *commit) { struct merge_remote_desc *desc; @@ -343,23 +345,29 @@ static void output_commit_title(struct merge_options *opt, struct commit *commit if (desc) strbuf_addf(&opt->obuf, "virtual %s\n", desc->name); else { - strbuf_add_unique_abbrev(&opt->obuf, &commit->object.oid, - DEFAULT_ABBREV); + strbuf_repo_add_unique_abbrev(&opt->obuf, repo, + &commit->object.oid, + DEFAULT_ABBREV); strbuf_addch(&opt->obuf, ' '); - if (parse_commit(commit) != 0) + if (repo_parse_commit(repo, commit) != 0) strbuf_addstr(&opt->obuf, _("(bad commit)\n")); else { const char *title; - const char *msg = get_commit_buffer(commit, NULL); + const char *msg = repo_get_commit_buffer(repo, commit, NULL); int len = find_commit_subject(msg, &title); if (len) strbuf_addf(&opt->obuf, "%.*s\n", len, title); - unuse_commit_buffer(commit, msg); + repo_unuse_commit_buffer(repo, commit, msg); } } flush_output(opt); } +static void output_commit_title(struct merge_options *opt, struct commit *commit) +{ + repo_output_commit_title(opt, the_repository, commit); +} + static int add_cacheinfo(struct merge_options *opt, const struct diff_filespec *blob, const char *path, int stage, int refresh, int options) @@ -1152,14 +1160,14 @@ static int find_first_merges(struct repository *repo, return result->nr; } -static void print_commit(struct commit *commit) +static void print_commit(struct repository *repo, struct commit *commit) { struct strbuf sb = STRBUF_INIT; struct pretty_print_context ctx = {0}; ctx.date_mode.type = DATE_NORMAL; /* FIXME: Merge this with output_commit_title() */ assert(!merge_remote_util(commit)); - format_commit_message(commit, " %h: %m %s", &sb, &ctx); + repo_format_commit_message(repo, commit, " %h: %m %s", &sb, &ctx); fprintf(stderr, "%s\n", sb.buf); strbuf_release(&sb); } @@ -1199,15 +1207,6 @@ static int merge_submodule(struct merge_options *opt, if (is_null_oid(b)) return 0; - /* - * NEEDSWORK: Remove this when all submodule object accesses are - * through explicitly specified repositores. - */ - if (add_submodule_odb(path)) { - output(opt, 1, _("Failed to merge submodule %s (not checked out)"), path); - return 0; - } - if (repo_submodule_init(&subrepo, opt->repo, path, null_oid())) { output(opt, 1, _("Failed to merge submodule %s (not checked out)"), path); return 0; @@ -1232,7 +1231,7 @@ static int merge_submodule(struct merge_options *opt, oidcpy(result, b); if (show(opt, 3)) { output(opt, 3, _("Fast-forwarding submodule %s to the following commit:"), path); - output_commit_title(opt, commit_b); + repo_output_commit_title(opt, &subrepo, commit_b); } else if (show(opt, 2)) output(opt, 2, _("Fast-forwarding submodule %s"), path); else @@ -1245,7 +1244,7 @@ static int merge_submodule(struct merge_options *opt, oidcpy(result, a); if (show(opt, 3)) { output(opt, 3, _("Fast-forwarding submodule %s to the following commit:"), path); - output_commit_title(opt, commit_a); + repo_output_commit_title(opt, &subrepo, commit_a); } else if (show(opt, 2)) output(opt, 2, _("Fast-forwarding submodule %s"), path); else @@ -1277,7 +1276,7 @@ static int merge_submodule(struct merge_options *opt, case 1: output(opt, 1, _("Failed to merge submodule %s (not fast-forward)"), path); output(opt, 2, _("Found a possible merge resolution for the submodule:\n")); - print_commit((struct commit *) merges.objects[0].item); + print_commit(&subrepo, (struct commit *) merges.objects[0].item); output(opt, 2, _( "If this is correct simply add it to the index " "for example\n" @@ -1290,7 +1289,7 @@ static int merge_submodule(struct merge_options *opt, default: output(opt, 1, _("Failed to merge submodule %s (multiple merges found)"), path); for (i = 0; i < merges.nr; i++) - print_commit((struct commit *) merges.objects[i].item); + print_commit(&subrepo, (struct commit *) merges.objects[i].item); } object_array_clear(&merges); @@ -3712,6 +3711,9 @@ static int merge_start(struct merge_options *opt, struct tree *head) assert(opt->priv == NULL); + /* Not supported; option specific to merge-ort */ + assert(!opt->record_conflict_msgs_in_tree); + /* Sanity check on repo state; index must match head */ if (repo_index_has_changes(opt->repo, head, &sb)) { err(opt, _("Your local changes to the following files would be overwritten by merge:\n %s"), diff --git a/merge-recursive.h b/merge-recursive.h index 0795a1d3ec..2e2fab37f4 100644 --- a/merge-recursive.h +++ b/merge-recursive.h @@ -46,6 +46,7 @@ struct merge_options { /* miscellaneous control options */ const char *subtree_shift; unsigned renormalize : 1; + unsigned record_conflict_msgs_in_tree : 1; /* internal fields used by the implementation */ struct merge_options_internal *priv; diff --git a/mergetools/xxdiff b/mergetools/xxdiff index ce5b8e9f29..d5ce467995 100644 --- a/mergetools/xxdiff +++ b/mergetools/xxdiff @@ -3,6 +3,13 @@ diff_cmd () { -R 'Accel.Search: "Ctrl+F"' \ -R 'Accel.SearchForward: "Ctrl+G"' \ "$LOCAL" "$REMOTE" + + # xxdiff can segfault on binary files which are often uninteresting. + # Do not allow segfaults to stop us from continuing on to the next file. + if test $? = 128 + then + return 1 + fi } merge_cmd () { @@ -57,15 +57,15 @@ const unsigned char *get_midx_checksum(struct multi_pack_index *m) return m->data + m->data_len - the_hash_algo->rawsz; } -char *get_midx_filename(const char *object_dir) +void get_midx_filename(struct strbuf *out, const char *object_dir) { - return xstrfmt("%s/pack/multi-pack-index", object_dir); + strbuf_addf(out, "%s/pack/multi-pack-index", object_dir); } -char *get_midx_rev_filename(struct multi_pack_index *m) +void get_midx_rev_filename(struct strbuf *out, struct multi_pack_index *m) { - return xstrfmt("%s/pack/multi-pack-index-%s.rev", - m->object_dir, hash_to_hex(get_midx_checksum(m))); + get_midx_filename(out, m->object_dir); + strbuf_addf(out, "-%s.rev", hash_to_hex(get_midx_checksum(m))); } static int midx_read_oid_fanout(const unsigned char *chunk_start, @@ -89,28 +89,30 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local size_t midx_size; void *midx_map = NULL; uint32_t hash_version; - char *midx_name = get_midx_filename(object_dir); + struct strbuf midx_name = STRBUF_INIT; uint32_t i; const char *cur_pack_name; struct chunkfile *cf = NULL; - fd = git_open(midx_name); + get_midx_filename(&midx_name, object_dir); + + fd = git_open(midx_name.buf); if (fd < 0) goto cleanup_fail; if (fstat(fd, &st)) { - error_errno(_("failed to read %s"), midx_name); + error_errno(_("failed to read %s"), midx_name.buf); goto cleanup_fail; } midx_size = xsize_t(st.st_size); if (midx_size < MIDX_MIN_SIZE) { - error(_("multi-pack-index file %s is too small"), midx_name); + error(_("multi-pack-index file %s is too small"), midx_name.buf); goto cleanup_fail; } - FREE_AND_NULL(midx_name); + strbuf_release(&midx_name); midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); @@ -179,12 +181,13 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local trace2_data_intmax("midx", the_repository, "load/num_packs", m->num_packs); trace2_data_intmax("midx", the_repository, "load/num_objects", m->num_objects); + free_chunkfile(cf); return m; cleanup_fail: free(m); - free(midx_name); - free(cf); + strbuf_release(&midx_name); + free_chunkfile(cf); if (midx_map) munmap(midx_map, midx_size); if (0 <= fd) @@ -1107,6 +1110,22 @@ cleanup: return ret; } +static struct multi_pack_index *lookup_multi_pack_index(struct repository *r, + const char *object_dir) +{ + struct multi_pack_index *cur; + + /* Ensure the given object_dir is local, or a known alternate. */ + find_odb(r, object_dir); + + for (cur = get_multi_pack_index(r); cur; cur = cur->next) { + if (!strcmp(object_dir, cur->object_dir)) + return cur; + } + + return NULL; +} + static int write_midx_internal(const char *object_dir, struct string_list *packs_to_include, struct string_list *packs_to_drop, @@ -1114,25 +1133,21 @@ static int write_midx_internal(const char *object_dir, const char *refs_snapshot, unsigned flags) { - char *midx_name; + struct strbuf midx_name = STRBUF_INIT; unsigned char midx_hash[GIT_MAX_RAWSZ]; uint32_t i; struct hashfile *f = NULL; struct lock_file lk; struct write_midx_context ctx = { 0 }; - struct multi_pack_index *cur; int pack_name_concat_len = 0; int dropped_packs = 0; int result = 0; struct chunkfile *cf; - /* Ensure the given object_dir is local, or a known alternate. */ - find_odb(the_repository, object_dir); - - midx_name = get_midx_filename(object_dir); - if (safe_create_leading_directories(midx_name)) + get_midx_filename(&midx_name, object_dir); + if (safe_create_leading_directories(midx_name.buf)) die_errno(_("unable to create leading directories of %s"), - midx_name); + midx_name.buf); if (!packs_to_include) { /* @@ -1140,12 +1155,7 @@ static int write_midx_internal(const char *object_dir, * packs to include, since all packs and objects are copied * blindly from an existing MIDX if one is present. */ - for (cur = get_multi_pack_index(the_repository); cur; cur = cur->next) { - if (!strcmp(object_dir, cur->object_dir)) { - ctx.m = cur; - break; - } - } + ctx.m = lookup_multi_pack_index(the_repository, object_dir); } if (ctx.m && !midx_checksum_valid(ctx.m)) { @@ -1366,7 +1376,7 @@ static int write_midx_internal(const char *object_dir, pack_name_concat_len += MIDX_CHUNK_ALIGNMENT - (pack_name_concat_len % MIDX_CHUNK_ALIGNMENT); - hold_lock_file_for_update(&lk, midx_name, LOCK_DIE_ON_ERROR); + hold_lock_file_for_update(&lk, midx_name.buf, LOCK_DIE_ON_ERROR); f = hashfd(get_lock_file_fd(&lk), get_lock_file_path(&lk)); if (ctx.nr - dropped_packs == 0) { @@ -1403,9 +1413,9 @@ static int write_midx_internal(const char *object_dir, ctx.pack_order = midx_pack_order(&ctx); if (flags & MIDX_WRITE_REV_INDEX) - write_midx_reverse_index(midx_name, midx_hash, &ctx); + write_midx_reverse_index(midx_name.buf, midx_hash, &ctx); if (flags & MIDX_WRITE_BITMAP) { - if (write_midx_bitmap(midx_name, midx_hash, &ctx, + if (write_midx_bitmap(midx_name.buf, midx_hash, &ctx, refs_snapshot, flags) < 0) { error(_("could not write multi-pack bitmap")); result = 1; @@ -1416,7 +1426,8 @@ static int write_midx_internal(const char *object_dir, if (ctx.m) close_object_store(the_repository->objects); - commit_lock_file(&lk); + if (commit_lock_file(&lk) < 0) + die_errno(_("could not write multi-pack-index")); clear_midx_files_ext(object_dir, ".bitmap", midx_hash); clear_midx_files_ext(object_dir, ".rev", midx_hash); @@ -1434,7 +1445,7 @@ cleanup: free(ctx.entries); free(ctx.pack_perm); free(ctx.pack_order); - free(midx_name); + strbuf_release(&midx_name); return result; } @@ -1498,20 +1509,22 @@ static void clear_midx_files_ext(const char *object_dir, const char *ext, void clear_midx_file(struct repository *r) { - char *midx = get_midx_filename(r->objects->odb->path); + struct strbuf midx = STRBUF_INIT; + + get_midx_filename(&midx, r->objects->odb->path); if (r->objects && r->objects->multi_pack_index) { close_midx(r->objects->multi_pack_index); r->objects->multi_pack_index = NULL; } - if (remove_path(midx)) - die(_("failed to clear multi-pack-index at %s"), midx); + if (remove_path(midx.buf)) + die(_("failed to clear multi-pack-index at %s"), midx.buf); clear_midx_files_ext(r->objects->odb->path, ".bitmap", NULL); clear_midx_files_ext(r->objects->odb->path, ".rev", NULL); - free(midx); + strbuf_release(&midx); } static int verify_midx_error; @@ -1564,12 +1577,15 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag if (!m) { int result = 0; struct stat sb; - char *filename = get_midx_filename(object_dir); - if (!stat(filename, &sb)) { + struct strbuf filename = STRBUF_INIT; + + get_midx_filename(&filename, object_dir); + + if (!stat(filename.buf, &sb)) { error(_("multi-pack-index file exists, but failed to parse")); result = 1; } - free(filename); + strbuf_release(&filename); return result; } @@ -1602,7 +1618,7 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag * Remaining tests assume that we have objects, so we can * return here. */ - return verify_midx_error; + goto cleanup; } if (flags & MIDX_PROGRESS) @@ -1680,7 +1696,9 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag } stop_progress(&progress); +cleanup: free(pairs); + close_midx(m); return verify_midx_error; } @@ -1689,7 +1707,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla { uint32_t i, *count, result = 0; struct string_list packs_to_drop = STRING_LIST_INIT_DUP; - struct multi_pack_index *m = load_multi_pack_index(object_dir, 1); + struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir); struct progress *progress = NULL; if (!m) @@ -1734,12 +1752,11 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla free(count); - if (packs_to_drop.nr) { + if (packs_to_drop.nr) result = write_midx_internal(object_dir, NULL, &packs_to_drop, NULL, NULL, flags); - m = NULL; - } string_list_clear(&packs_to_drop, 0); + return result; } @@ -1855,7 +1872,7 @@ int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, struct child_process cmd = CHILD_PROCESS_INIT; FILE *cmd_in; struct strbuf base_name = STRBUF_INIT; - struct multi_pack_index *m = load_multi_pack_index(object_dir, 1); + struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir); /* * When updating the default for these configuration @@ -1927,11 +1944,8 @@ int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, } result = write_midx_internal(object_dir, NULL, NULL, NULL, NULL, flags); - m = NULL; cleanup: - if (m) - close_midx(m); free(include_pack); return result; } @@ -48,8 +48,8 @@ struct multi_pack_index { #define MIDX_WRITE_BITMAP_HASH_CACHE (1 << 3) const unsigned char *get_midx_checksum(struct multi_pack_index *m); -char *get_midx_filename(const char *object_dir); -char *get_midx_rev_filename(struct multi_pack_index *m); +void get_midx_filename(struct strbuf *out, const char *object_dir); +void get_midx_rev_filename(struct strbuf *out, struct multi_pack_index *m); struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local); int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t pack_int_id); diff --git a/object-file.c b/object-file.c index 112d9b4bad..ebbb130430 100644 --- a/object-file.c +++ b/object-file.c @@ -165,7 +165,6 @@ static void git_hash_unknown_final_oid(struct object_id *oid, git_hash_ctx *ctx) BUG("trying to finalize unknown hash"); } - const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = { { NULL, @@ -184,8 +183,7 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = { }, { "sha1", - /* "sha1", big-endian */ - 0x73686131, + GIT_SHA1_FORMAT_ID, GIT_SHA1_RAWSZ, GIT_SHA1_HEXSZ, GIT_SHA1_BLKSZ, @@ -200,8 +198,7 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = { }, { "sha256", - /* "s256", big-endian */ - 0x73323536, + GIT_SHA256_FORMAT_ID, GIT_SHA256_RAWSZ, GIT_SHA256_HEXSZ, GIT_SHA256_BLKSZ, @@ -683,6 +680,49 @@ void add_to_alternates_memory(const char *reference) '\n', NULL, 0); } +struct object_directory *set_temporary_primary_odb(const char *dir, int will_destroy) +{ + struct object_directory *new_odb; + + /* + * Make sure alternates are initialized, or else our entry may be + * overwritten when they are. + */ + prepare_alt_odb(the_repository); + + /* + * Make a new primary odb and link the old primary ODB in as an + * alternate + */ + new_odb = xcalloc(1, sizeof(*new_odb)); + new_odb->path = xstrdup(dir); + + /* + * Disable ref updates while a temporary odb is active, since + * the objects in the database may roll back. + */ + new_odb->disable_ref_updates = 1; + new_odb->will_destroy = will_destroy; + new_odb->next = the_repository->objects->odb; + the_repository->objects->odb = new_odb; + return new_odb->next; +} + +void restore_primary_odb(struct object_directory *restore_odb, const char *old_path) +{ + struct object_directory *cur_odb = the_repository->objects->odb; + + if (strcmp(old_path, cur_odb->path)) + BUG("expected %s as primary object store; found %s", + old_path, cur_odb->path); + + if (cur_odb->next != restore_odb) + BUG("we expect the old primary object store to be the first alternate"); + + the_repository->objects->odb = restore_odb; + free_object_directory(cur_odb); +} + /* * Compute the exact path an alternate is at and returns it. In case of * error NULL is returned and the human readable error is added to `err` @@ -1016,9 +1056,11 @@ void *xmmap(void *start, size_t length, * the streaming interface and rehash it to do the same. */ int check_object_signature(struct repository *r, const struct object_id *oid, - void *map, unsigned long size, const char *type) + void *map, unsigned long size, const char *type, + struct object_id *real_oidp) { - struct object_id real_oid; + struct object_id tmp; + struct object_id *real_oid = real_oidp ? real_oidp : &tmp; enum object_type obj_type; struct git_istream *st; git_hash_ctx c; @@ -1026,8 +1068,8 @@ int check_object_signature(struct repository *r, const struct object_id *oid, int hdrlen; if (map) { - hash_object_file(r->hash_algo, map, size, type, &real_oid); - return !oideq(oid, &real_oid) ? -1 : 0; + hash_object_file(r->hash_algo, map, size, type, real_oid); + return !oideq(oid, real_oid) ? -1 : 0; } st = open_istream(r, oid, &obj_type, &size, NULL); @@ -1052,9 +1094,9 @@ int check_object_signature(struct repository *r, const struct object_id *oid, break; r->hash_algo->update_fn(&c, buf, readlen); } - r->hash_algo->final_oid_fn(&real_oid, &c); + r->hash_algo->final_oid_fn(real_oid, &c); close_istream(st); - return !oideq(oid, &real_oid) ? -1 : 0; + return !oideq(oid, real_oid) ? -1 : 0; } int git_open_cloexec(const char *name, int flags) @@ -1187,11 +1229,14 @@ void *map_loose_object(struct repository *r, return map_loose_object_1(r, NULL, oid, size); } -static int unpack_loose_short_header(git_zstream *stream, - unsigned char *map, unsigned long mapsize, - void *buffer, unsigned long bufsiz) +enum unpack_loose_header_result unpack_loose_header(git_zstream *stream, + unsigned char *map, + unsigned long mapsize, + void *buffer, + unsigned long bufsiz, + struct strbuf *header) { - int ret; + int status; /* Get the data stream */ memset(stream, 0, sizeof(*stream)); @@ -1202,43 +1247,24 @@ static int unpack_loose_short_header(git_zstream *stream, git_inflate_init(stream); obj_read_unlock(); - ret = git_inflate(stream, 0); + status = git_inflate(stream, 0); obj_read_lock(); - - return ret; -} - -int unpack_loose_header(git_zstream *stream, - unsigned char *map, unsigned long mapsize, - void *buffer, unsigned long bufsiz) -{ - int status = unpack_loose_short_header(stream, map, mapsize, - buffer, bufsiz); - if (status < Z_OK) - return status; - - /* Make sure we have the terminating NUL */ - if (!memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer)) - return -1; - return 0; -} - -static int unpack_loose_header_to_strbuf(git_zstream *stream, unsigned char *map, - unsigned long mapsize, void *buffer, - unsigned long bufsiz, struct strbuf *header) -{ - int status; - - status = unpack_loose_short_header(stream, map, mapsize, buffer, bufsiz); - if (status < Z_OK) - return -1; + return ULHR_BAD; /* * Check if entire header is unpacked in the first iteration. */ if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer)) - return 0; + return ULHR_OK; + + /* + * We have a header longer than MAX_HEADER_LEN. The "header" + * here is only non-NULL when we run "cat-file + * --allow-unknown-type". + */ + if (!header) + return ULHR_TOO_LONG; /* * buffer[0..bufsiz] was not large enough. Copy the partial @@ -1259,7 +1285,7 @@ static int unpack_loose_header_to_strbuf(git_zstream *stream, unsigned char *map stream->next_out = buffer; stream->avail_out = bufsiz; } while (status != Z_STREAM_END); - return -1; + return ULHR_TOO_LONG; } static void *unpack_loose_rest(git_zstream *stream, @@ -1317,8 +1343,7 @@ static void *unpack_loose_rest(git_zstream *stream, * too permissive for what we want to check. So do an anal * object header parse by hand. */ -static int parse_loose_header_extended(const char *hdr, struct object_info *oi, - unsigned int flags) +int parse_loose_header(const char *hdr, struct object_info *oi) { const char *type_buf = hdr; unsigned long size; @@ -1340,15 +1365,6 @@ static int parse_loose_header_extended(const char *hdr, struct object_info *oi, type = type_from_string_gently(type_buf, type_len, 1); if (oi->type_name) strbuf_add(oi->type_name, type_buf, type_len); - /* - * Set type to 0 if its an unknown object and - * we're obtaining the type using '--allow-unknown-type' - * option. - */ - if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE) && (type < 0)) - type = 0; - else if (type < 0) - die(_("invalid object type")); if (oi->typep) *oi->typep = type; @@ -1375,15 +1391,14 @@ static int parse_loose_header_extended(const char *hdr, struct object_info *oi, /* * The length must be followed by a zero byte */ - return *hdr ? -1 : type; -} - -int parse_loose_header(const char *hdr, unsigned long *sizep) -{ - struct object_info oi = OBJECT_INFO_INIT; + if (*hdr) + return -1; - oi.sizep = sizep; - return parse_loose_header_extended(hdr, &oi, 0); + /* + * The format is valid, but the type may still be bogus. The + * Caller needs to check its oi->typep. + */ + return 0; } static int loose_object_info(struct repository *r, @@ -1397,6 +1412,8 @@ static int loose_object_info(struct repository *r, char hdr[MAX_HEADER_LEN]; struct strbuf hdrbuf = STRBUF_INIT; unsigned long size_scratch; + enum object_type type_scratch; + int allow_unknown = flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE; if (oi->delta_base_oid) oidclr(oi->delta_base_oid); @@ -1427,43 +1444,48 @@ static int loose_object_info(struct repository *r, if (!oi->sizep) oi->sizep = &size_scratch; + if (!oi->typep) + oi->typep = &type_scratch; if (oi->disk_sizep) *oi->disk_sizep = mapsize; - if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE)) { - if (unpack_loose_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0) - status = error(_("unable to unpack %s header with --allow-unknown-type"), - oid_to_hex(oid)); - } else if (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0) + + switch (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr), + allow_unknown ? &hdrbuf : NULL)) { + case ULHR_OK: + if (parse_loose_header(hdrbuf.len ? hdrbuf.buf : hdr, oi) < 0) + status = error(_("unable to parse %s header"), oid_to_hex(oid)); + else if (!allow_unknown && *oi->typep < 0) + die(_("invalid object type")); + + if (!oi->contentp) + break; + *oi->contentp = unpack_loose_rest(&stream, hdr, *oi->sizep, oid); + if (*oi->contentp) + goto cleanup; + + status = -1; + break; + case ULHR_BAD: status = error(_("unable to unpack %s header"), oid_to_hex(oid)); - if (status < 0) - ; /* Do nothing */ - else if (hdrbuf.len) { - if ((status = parse_loose_header_extended(hdrbuf.buf, oi, flags)) < 0) - status = error(_("unable to parse %s header with --allow-unknown-type"), - oid_to_hex(oid)); - } else if ((status = parse_loose_header_extended(hdr, oi, flags)) < 0) - status = error(_("unable to parse %s header"), oid_to_hex(oid)); - - if (status >= 0 && oi->contentp) { - *oi->contentp = unpack_loose_rest(&stream, hdr, - *oi->sizep, oid); - if (!*oi->contentp) { - git_inflate_end(&stream); - status = -1; - } - } else - git_inflate_end(&stream); + break; + case ULHR_TOO_LONG: + status = error(_("header for %s too long, exceeds %d bytes"), + oid_to_hex(oid), MAX_HEADER_LEN); + break; + } + git_inflate_end(&stream); +cleanup: munmap(map, mapsize); - if (status && oi->typep) - *oi->typep = status; if (oi->sizep == &size_scratch) oi->sizep = NULL; strbuf_release(&hdrbuf); + if (oi->typep == &type_scratch) + oi->typep = NULL; oi->whence = OI_LOOSE; - return (status < 0) ? status : 0; + return status; } int obj_read_use_lock = 0; @@ -1546,7 +1568,14 @@ static int do_oid_object_info_extended(struct repository *r, break; } - if (register_all_submodule_odb_as_alternates()) + /* + * If r is the_repository, this might be an attempt at + * accessing a submodule object as if it were in the_repository + * (having called add_submodule_odb() on that submodule's ODB). + * If any such ODBs exist, register them and try again. + */ + if (r == the_repository && + register_all_submodule_odb_as_alternates()) /* We added some alternates; retry */ continue; @@ -1820,8 +1849,21 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf, /* Finalize a file on disk, and close it. */ static void close_loose_object(int fd) { - if (fsync_object_files) - fsync_or_die(fd, "loose object file"); + if (!the_repository->objects->odb->will_destroy) { + switch (fsync_object_files) { + case FSYNC_OBJECT_FILES_OFF: + break; + case FSYNC_OBJECT_FILES_ON: + fsync_or_die(fd, "loose object file"); + break; + case FSYNC_OBJECT_FILES_BATCH: + fsync_loose_object_bulk_checkin(fd); + break; + default: + BUG("Invalid fsync_object_files mode."); + } + } + if (close(fd) != 0) die_errno(_("error when closing loose object file")); } @@ -1873,7 +1915,7 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename) static int write_loose_object(const struct object_id *oid, char *hdr, int hdrlen, const void *buf, unsigned long len, - time_t mtime) + time_t mtime, unsigned flags) { int fd, ret; unsigned char compressed[4096]; @@ -1887,7 +1929,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr, fd = create_tmpfile(&tmp_file, filename.buf); if (fd < 0) { - if (errno == EACCES) + if (flags & HASH_SILENT) + return -1; + else if (errno == EACCES) return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory()); else return error_errno(_("unable to create temporary file")); @@ -1937,7 +1981,8 @@ static int write_loose_object(const struct object_id *oid, char *hdr, struct utimbuf utb; utb.actime = mtime; utb.modtime = mtime; - if (utime(tmp_file.buf, &utb) < 0) + if (utime(tmp_file.buf, &utb) < 0 && + !(flags & HASH_SILENT)) warning_errno(_("failed utime() on %s"), tmp_file.buf); } @@ -1962,8 +2007,9 @@ static int freshen_packed_object(const struct object_id *oid) return 1; } -int write_object_file(const void *buf, unsigned long len, const char *type, - struct object_id *oid) +int write_object_file_flags(const void *buf, unsigned long len, + const char *type, struct object_id *oid, + unsigned flags) { char hdr[MAX_HEADER_LEN]; int hdrlen = sizeof(hdr); @@ -1975,7 +2021,7 @@ int write_object_file(const void *buf, unsigned long len, const char *type, &hdrlen); if (freshen_packed_object(oid) || freshen_loose_object(oid)) return 0; - return write_loose_object(oid, hdr, hdrlen, buf, len, 0); + return write_loose_object(oid, hdr, hdrlen, buf, len, 0, flags); } int hash_object_file_literally(const void *buf, unsigned long len, @@ -1995,7 +2041,7 @@ int hash_object_file_literally(const void *buf, unsigned long len, goto cleanup; if (freshen_packed_object(oid) || freshen_loose_object(oid)) goto cleanup; - status = write_loose_object(oid, header, hdrlen, buf, len, 0); + status = write_loose_object(oid, header, hdrlen, buf, len, 0, 0); cleanup: free(header); @@ -2017,7 +2063,7 @@ int force_object_loose(const struct object_id *oid, time_t mtime) if (!buf) return error(_("cannot read object for %s"), oid_to_hex(oid)); hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(type), (uintmax_t)len) + 1; - ret = write_loose_object(oid, hdr, hdrlen, buf, len, mtime); + ret = write_loose_object(oid, hdr, hdrlen, buf, len, mtime, 0); free(buf); return ret; @@ -2524,17 +2570,16 @@ static int check_stream_oid(git_zstream *stream, int read_loose_object(const char *path, const struct object_id *expected_oid, - enum object_type *type, - unsigned long *size, - void **contents) + struct object_id *real_oid, + void **contents, + struct object_info *oi) { int ret = -1; void *map = NULL; unsigned long mapsize; git_zstream stream; char hdr[MAX_HEADER_LEN]; - - *contents = NULL; + unsigned long *size = oi->sizep; map = map_loose_object_1(the_repository, path, NULL, &mapsize); if (!map) { @@ -2542,19 +2587,19 @@ int read_loose_object(const char *path, goto out; } - if (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0) { + if (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr), + NULL) < 0) { error(_("unable to unpack header of %s"), path); goto out; } - *type = parse_loose_header(hdr, size); - if (*type < 0) { + if (parse_loose_header(hdr, oi) < 0) { error(_("unable to parse header of %s"), path); git_inflate_end(&stream); goto out; } - if (*type == OBJ_BLOB && *size > big_file_threshold) { + if (*oi->typep == OBJ_BLOB && *size > big_file_threshold) { if (check_stream_oid(&stream, hdr, *size, path, expected_oid) < 0) goto out; } else { @@ -2565,10 +2610,7 @@ int read_loose_object(const char *path, goto out; } if (check_object_signature(the_repository, expected_oid, - *contents, *size, - type_name(*type))) { - error(_("hash mismatch for %s (expected %s)"), path, - oid_to_hex(expected_oid)); + *contents, *size, oi->type_name->buf, real_oid)) { free(*contents); goto out; } diff --git a/object-store.h b/object-store.h index 1e647a5be3..9ae9262c34 100644 --- a/object-store.h +++ b/object-store.h @@ -28,6 +28,18 @@ struct object_directory { struct oidtree *loose_objects_cache; /* + * This is a temporary object store created by the tmp_objdir + * facility. Disable ref updates since the objects in the store + * might be discarded on rollback. + */ + unsigned int disable_ref_updates : 1; + + /* + * This object store is ephemeral, so there is no need to fsync. + */ + unsigned int will_destroy : 1; + + /* * Path to the alternative object store. If this is a relative path, * it is relative to the current working directory. */ @@ -59,6 +71,17 @@ void add_to_alternates_file(const char *dir); void add_to_alternates_memory(const char *dir); /* + * Replace the current writable object directory with the specified temporary + * object directory; returns the former primary object directory. + */ +struct object_directory *set_temporary_primary_odb(const char *dir, int will_destroy); + +/* + * Restore a previous ODB replaced by set_temporary_main_odb. + */ +void restore_primary_odb(struct object_directory *restore_odb, const char *old_path); + +/* * Populate and return the loose object cache array corresponding to the * given object ID. */ @@ -68,6 +91,9 @@ struct oidtree *odb_loose_cache(struct object_directory *odb, /* Empty the loose object cache for the specified object directory. */ void odb_clear_loose_cache(struct object_directory *odb); +/* Clear and free the specified object directory */ +void free_object_directory(struct object_directory *odb); + struct packed_git { struct hashmap_entry packmap_ent; struct packed_git *next; @@ -223,8 +249,14 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf, unsigned long len, const char *type, struct object_id *oid); -int write_object_file(const void *buf, unsigned long len, - const char *type, struct object_id *oid); +int write_object_file_flags(const void *buf, unsigned long len, + const char *type, struct object_id *oid, + unsigned flags); +static inline int write_object_file(const void *buf, unsigned long len, + const char *type, struct object_id *oid) +{ + return write_object_file_flags(buf, len, type, oid, 0); +} int hash_object_file_literally(const void *buf, unsigned long len, const char *type, struct object_id *oid, @@ -245,6 +277,7 @@ int force_object_loose(const struct object_id *oid, time_t mtime); /* * Open the loose object at path, check its hash, and return the contents, + * use the "oi" argument to assert things about the object, or e.g. populate its * type, and size. If the object is a blob, then "contents" may return NULL, * to allow streaming of large blobs. * @@ -252,9 +285,9 @@ int force_object_loose(const struct object_id *oid, time_t mtime); */ int read_loose_object(const char *path, const struct object_id *expected_oid, - enum object_type *type, - unsigned long *size, - void **contents); + struct object_id *real_oid, + void **contents, + struct object_info *oi); /* Retry packed storage after checking packed and loose storage */ #define HAS_OBJECT_RECHECK_PACKED 1 @@ -279,7 +279,7 @@ struct object *parse_object(struct repository *r, const struct object_id *oid) if ((obj && obj->type == OBJ_BLOB && repo_has_object_file(r, oid)) || (!obj && repo_has_object_file(r, oid) && oid_object_info(r, oid, NULL) == OBJ_BLOB)) { - if (check_object_signature(r, repl, NULL, 0, NULL) < 0) { + if (check_object_signature(r, repl, NULL, 0, NULL, NULL) < 0) { error(_("hash mismatch %s"), oid_to_hex(oid)); return NULL; } @@ -290,7 +290,7 @@ struct object *parse_object(struct repository *r, const struct object_id *oid) buffer = repo_read_object_file(r, oid, &type, &size); if (buffer) { if (check_object_signature(r, repl, buffer, size, - type_name(type)) < 0) { + type_name(type), NULL) < 0) { free(buffer); error(_("hash mismatch %s"), oid_to_hex(repl)); return NULL; @@ -513,7 +513,7 @@ struct raw_object_store *raw_object_store_new(void) return o; } -static void free_object_directory(struct object_directory *odb) +void free_object_directory(struct object_directory *odb) { free(odb->path); odb_clear_loose_cache(odb); diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c index 9c55c1531e..cab3eaa2ac 100644 --- a/pack-bitmap-write.c +++ b/pack-bitmap-write.c @@ -575,15 +575,15 @@ void bitmap_writer_select_commits(struct commit **indexed_commits, QSORT(indexed_commits, indexed_commits_nr, date_compare); - if (writer.show_progress) - writer.progress = start_progress("Selecting bitmap commits", 0); - if (indexed_commits_nr < 100) { for (i = 0; i < indexed_commits_nr; ++i) push_bitmapped_commit(indexed_commits[i]); return; } + if (writer.show_progress) + writer.progress = start_progress("Selecting bitmap commits", 0); + for (;;) { struct commit *chosen = NULL; diff --git a/pack-bitmap.c b/pack-bitmap.c index f47a0a7db4..a56ceb9441 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -292,9 +292,12 @@ static int load_bitmap_entries_v1(struct bitmap_index *index) char *midx_bitmap_filename(struct multi_pack_index *midx) { - return xstrfmt("%s-%s.bitmap", - get_midx_filename(midx->object_dir), - hash_to_hex(get_midx_checksum(midx))); + struct strbuf buf = STRBUF_INIT; + + get_midx_filename(&buf, midx->object_dir); + strbuf_addf(&buf, "-%s.bitmap", hash_to_hex(get_midx_checksum(midx))); + + return strbuf_detach(&buf, NULL); } char *pack_bitmap_filename(struct packed_git *p) @@ -324,10 +327,12 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, } if (bitmap_git->pack || bitmap_git->midx) { + struct strbuf buf = STRBUF_INIT; + get_midx_filename(&buf, midx->object_dir); /* ignore extra bitmap file; we can only handle one */ - warning("ignoring extra bitmap file: %s", - get_midx_filename(midx->object_dir)); + warning("ignoring extra bitmap file: %s", buf.buf); close(fd); + strbuf_release(&buf); return -1; } @@ -1721,6 +1726,12 @@ void test_bitmap_walk(struct rev_info *revs) else die("mismatch in bitmap results"); + bitmap_free(result); + bitmap_free(tdata.base); + bitmap_free(tdata.commits); + bitmap_free(tdata.trees); + bitmap_free(tdata.blobs); + bitmap_free(tdata.tags); free_bitmap_index(bitmap_git); } @@ -1848,9 +1859,17 @@ void free_bitmap_index(struct bitmap_index *b) ewah_pool_free(b->trees); ewah_pool_free(b->blobs); ewah_pool_free(b->tags); + if (b->bitmaps) { + struct stored_bitmap *sb; + kh_foreach_value(b->bitmaps, sb, { + ewah_pool_free(sb->root); + free(sb); + }); + } kh_destroy_oid_map(b->bitmaps); free(b->ext_index.objects); free(b->ext_index.hashes); + kh_destroy_oid_pos(b->ext_index.positions); bitmap_free(b->result); bitmap_free(b->haves); if (bitmap_is_midx(b)) { diff --git a/pack-check.c b/pack-check.c index c8e560d71a..3f418e3a6a 100644 --- a/pack-check.c +++ b/pack-check.c @@ -142,7 +142,8 @@ static int verify_packfile(struct repository *r, err = error("cannot unpack %s from %s at offset %"PRIuMAX"", oid_to_hex(&oid), p->pack_name, (uintmax_t)entries[i].offset); - else if (check_object_signature(r, &oid, data, size, type_name(type))) + else if (check_object_signature(r, &oid, data, size, + type_name(type), NULL)) err = error("packed %s from %s is corrupt", oid_to_hex(&oid), p->pack_name); else if (fn) { diff --git a/pack-revindex.c b/pack-revindex.c index 0e4a31d9db..70d0fbafcb 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -296,14 +296,14 @@ int load_pack_revindex(struct packed_git *p) int load_midx_revindex(struct multi_pack_index *m) { - char *revindex_name; + struct strbuf revindex_name = STRBUF_INIT; int ret; if (m->revindex_data) return 0; - revindex_name = get_midx_rev_filename(m); + get_midx_rev_filename(&revindex_name, m); - ret = load_revindex_from_disk(revindex_name, + ret = load_revindex_from_disk(revindex_name.buf, m->num_objects, &m->revindex_map, &m->revindex_len); @@ -313,7 +313,7 @@ int load_midx_revindex(struct multi_pack_index *m) m->revindex_data = (const uint32_t *)((const char *)m->revindex_map + RIDX_HEADER_SIZE); cleanup: - free(revindex_name); + strbuf_release(&revindex_name); return ret; } diff --git a/parallel-checkout.c b/parallel-checkout.c index ddc0ff3c06..3a6c5de3dd 100644 --- a/parallel-checkout.c +++ b/parallel-checkout.c @@ -91,7 +91,8 @@ static int is_eligible_for_parallel_checkout(const struct cache_entry *ce, return 0; packed_item_size = sizeof(struct pc_item_fixed_portion) + ce->ce_namelen + - (ca->working_tree_encoding ? strlen(ca->working_tree_encoding) : 0); + (ca->working_tree_encoding ? strlen(ca->working_tree_encoding) : 0) + + ca->ident_action.id_len; /* * The amount of data we send to the workers per checkout item is @@ -403,13 +404,15 @@ static void send_one_item(int fd, struct parallel_checkout_item *pc_item) size_t name_len = pc_item->ce->ce_namelen; size_t working_tree_encoding_len = working_tree_encoding ? strlen(working_tree_encoding) : 0; + const char *ident_action_id = pc_item->ca.ident_action.id; + size_t ident_action_len = pc_item->ca.ident_action.id_len; /* * Any changes in the calculation of the message size must also be made * in is_eligible_for_parallel_checkout(). */ len_data = sizeof(struct pc_item_fixed_portion) + name_len + - working_tree_encoding_len; + working_tree_encoding_len + ident_action_len; data = xmalloc(len_data); @@ -417,7 +420,7 @@ static void send_one_item(int fd, struct parallel_checkout_item *pc_item) fixed_portion->id = pc_item->id; fixed_portion->ce_mode = pc_item->ce->ce_mode; fixed_portion->crlf_action = pc_item->ca.crlf_action; - fixed_portion->ident = pc_item->ca.ident; + fixed_portion->ident_action_len = ident_action_len; fixed_portion->name_len = name_len; fixed_portion->working_tree_encoding_len = working_tree_encoding_len; /* @@ -434,6 +437,11 @@ static void send_one_item(int fd, struct parallel_checkout_item *pc_item) variant += working_tree_encoding_len; } memcpy(variant, pc_item->ce->name, name_len); + variant += name_len; + if (ident_action_len) { + memcpy(variant, ident_action_id, ident_action_len); + variant += ident_action_len; + } packet_write(fd, data, len_data); @@ -603,8 +611,7 @@ static void gather_results_from_workers(struct pc_worker *workers, continue; if (pfd->revents & POLLIN) { - int len = packet_read(pfd->fd, NULL, NULL, - packet_buffer, + int len = packet_read(pfd->fd, packet_buffer, sizeof(packet_buffer), 0); if (len < 0) { diff --git a/parallel-checkout.h b/parallel-checkout.h index 80f539bcb7..c3c282f516 100644 --- a/parallel-checkout.h +++ b/parallel-checkout.h @@ -76,9 +76,9 @@ struct parallel_checkout_item { /* * The fixed-size portion of `struct parallel_checkout_item` that is sent to the - * workers. Following this will be 2 strings: ca.working_tree_encoding and - * ce.name; These are NOT null terminated, since we have the size in the fixed - * portion. + * workers. Following this will be 3 strings: ca.working_tree_encoding, ca.name + * and ca.ident_action.id; These are NOT null terminated, since we have the size + * in the fixed portion. * * Note that not all fields of conv_attrs and cache_entry are passed, only the * ones that will be required by the workers to smudge and write the entry. @@ -88,7 +88,7 @@ struct pc_item_fixed_portion { struct object_id oid; unsigned int ce_mode; enum convert_crlf_action crlf_action; - int ident; + size_t ident_action_len; size_t working_tree_encoding_len; size_t name_len; }; diff --git a/parse-options-cb.c b/parse-options-cb.c index 3c811e1e4a..b1a1ab797e 100644 --- a/parse-options-cb.c +++ b/parse-options-cb.c @@ -1,5 +1,6 @@ #include "git-compat-util.h" #include "parse-options.h" +#include "branch.h" #include "cache.h" #include "commit.h" #include "color.h" @@ -293,3 +294,17 @@ int parse_opt_passthru_argv(const struct option *opt, const char *arg, int unset return 0; } + +int parse_opt_tracking_mode(const struct option *opt, const char *arg, int unset) { + if (unset) + *(enum branch_track *)opt->value = BRANCH_TRACK_NEVER; + else if (!arg || !strcmp(arg, "direct")) + *(enum branch_track *)opt->value = BRANCH_TRACK_EXPLICIT; + else if (!strcmp(arg, "inherit")) + *(enum branch_track *)opt->value = BRANCH_TRACK_INHERIT; + else + return error(_("option `%s' expects \"direct\" or \"inherit\""), + opt->long_name); + + return 0; +} diff --git a/parse-options.c b/parse-options.c index 6e0535bdaa..9a0484c883 100644 --- a/parse-options.c +++ b/parse-options.c @@ -8,10 +8,13 @@ static int disallow_abbreviated_options; -#define OPT_SHORT 1 -#define OPT_UNSET 2 +enum opt_parsed { + OPT_LONG = 0, + OPT_SHORT = 1<<0, + OPT_UNSET = 1<<1, +}; -int optbug(const struct option *opt, const char *reason) +static int optbug(const struct option *opt, const char *reason) { if (opt->long_name) { if (opt->short_name) @@ -22,9 +25,26 @@ int optbug(const struct option *opt, const char *reason) return error("BUG: switch '%c' %s", opt->short_name, reason); } +static const char *optname(const struct option *opt, enum opt_parsed flags) +{ + static struct strbuf sb = STRBUF_INIT; + + strbuf_reset(&sb); + if (flags & OPT_SHORT) + strbuf_addf(&sb, "switch `%c'", opt->short_name); + else if (flags & OPT_UNSET) + strbuf_addf(&sb, "option `no-%s'", opt->long_name); + else if (flags == OPT_LONG) + strbuf_addf(&sb, "option `%s'", opt->long_name); + else + BUG("optname() got unknown flags %d", flags); + + return sb.buf; +} + static enum parse_opt_result get_arg(struct parse_opt_ctx_t *p, const struct option *opt, - int flags, const char **arg) + enum opt_parsed flags, const char **arg) { if (p->opt) { *arg = p->opt; @@ -50,7 +70,7 @@ static void fix_filename(const char *prefix, const char **file) static enum parse_opt_result opt_command_mode_error( const struct option *opt, const struct option *all_opts, - int flags) + enum opt_parsed flags) { const struct option *that; struct strbuf that_name = STRBUF_INIT; @@ -82,7 +102,7 @@ static enum parse_opt_result opt_command_mode_error( static enum parse_opt_result get_value(struct parse_opt_ctx_t *p, const struct option *opt, const struct option *all_opts, - int flags) + enum opt_parsed flags) { const char *s, *arg; const int unset = flags & OPT_UNSET; @@ -298,11 +318,11 @@ static enum parse_opt_result parse_long_opt( const struct option *all_opts = options; const char *arg_end = strchrnul(arg, '='); const struct option *abbrev_option = NULL, *ambiguous_option = NULL; - int abbrev_flags = 0, ambiguous_flags = 0; + enum opt_parsed abbrev_flags = OPT_LONG, ambiguous_flags = OPT_LONG; for (; options->type != OPTION_END; options++) { const char *rest, *long_name = options->long_name; - int flags = 0, opt_flags = 0; + enum opt_parsed flags = OPT_LONG, opt_flags = OPT_LONG; if (!long_name) continue; @@ -481,7 +501,8 @@ static void parse_options_check(const struct option *opts) static void parse_options_start_1(struct parse_opt_ctx_t *ctx, int argc, const char **argv, const char *prefix, - const struct option *options, int flags) + const struct option *options, + enum parse_opt_flags flags) { ctx->argc = argc; ctx->argv = argv; @@ -506,7 +527,8 @@ static void parse_options_start_1(struct parse_opt_ctx_t *ctx, void parse_options_start(struct parse_opt_ctx_t *ctx, int argc, const char **argv, const char *prefix, - const struct option *options, int flags) + const struct option *options, + enum parse_opt_flags flags) { memset(ctx, 0, sizeof(*ctx)); parse_options_start_1(ctx, argc, argv, prefix, options, flags); @@ -697,13 +719,14 @@ static void free_preprocessed_options(struct option *options) free(options); } -static int usage_with_options_internal(struct parse_opt_ctx_t *, - const char * const *, - const struct option *, int, int); +static enum parse_opt_result usage_with_options_internal(struct parse_opt_ctx_t *, + const char * const *, + const struct option *, + int, int); -int parse_options_step(struct parse_opt_ctx_t *ctx, - const struct option *options, - const char * const usagestr[]) +enum parse_opt_result parse_options_step(struct parse_opt_ctx_t *ctx, + const struct option *options, + const char * const usagestr[]) { int internal_help = !(ctx->flags & PARSE_OPT_NO_INTERNAL_HELP); @@ -837,9 +860,11 @@ int parse_options_end(struct parse_opt_ctx_t *ctx) return ctx->cpidx + ctx->argc; } -int parse_options(int argc, const char **argv, const char *prefix, - const struct option *options, const char * const usagestr[], - int flags) +enum parse_opt_result parse_options(int argc, const char **argv, + const char *prefix, + const struct option *options, + const char * const usagestr[], + enum parse_opt_flags flags) { struct parse_opt_ctx_t ctx; struct option *real_options; @@ -861,7 +886,7 @@ int parse_options(int argc, const char **argv, const char *prefix, case PARSE_OPT_NON_OPTION: case PARSE_OPT_DONE: break; - default: /* PARSE_OPT_UNKNOWN */ + case PARSE_OPT_UNKNOWN: if (ctx.argv[0][1] == '-') { error(_("unknown option `%s'"), ctx.argv[0] + 2); } else if (isascii(*ctx.opt)) { @@ -897,9 +922,10 @@ static int usage_argh(const struct option *opts, FILE *outfile) #define USAGE_OPTS_WIDTH 24 #define USAGE_GAP 2 -static int usage_with_options_internal(struct parse_opt_ctx_t *ctx, - const char * const *usagestr, - const struct option *opts, int full, int err) +static enum parse_opt_result usage_with_options_internal(struct parse_opt_ctx_t *ctx, + const char * const *usagestr, + const struct option *opts, + int full, int err) { FILE *outfile = err ? stderr : stdout; int need_newline; @@ -1052,18 +1078,3 @@ void NORETURN usage_msg_opt(const char *msg, fprintf(stderr, "fatal: %s\n\n", msg); usage_with_options(usagestr, options); } - -const char *optname(const struct option *opt, int flags) -{ - static struct strbuf sb = STRBUF_INIT; - - strbuf_reset(&sb); - if (flags & OPT_SHORT) - strbuf_addf(&sb, "switch `%c'", opt->short_name); - else if (flags & OPT_UNSET) - strbuf_addf(&sb, "option `no-%s'", opt->long_name); - else - strbuf_addf(&sb, "option `%s'", opt->long_name); - - return sb.buf; -} diff --git a/parse-options.h b/parse-options.h index 13405472ee..b6f39b4396 100644 --- a/parse-options.h +++ b/parse-options.h @@ -33,6 +33,7 @@ enum parse_opt_flags { PARSE_OPT_KEEP_UNKNOWN = 1 << 3, PARSE_OPT_NO_INTERNAL_HELP = 1 << 4, PARSE_OPT_ONE_SHOT = 1 << 5, + PARSE_OPT_SHELL_EVAL = 1 << 6, }; enum parse_opt_option_flags { @@ -44,7 +45,6 @@ enum parse_opt_option_flags { PARSE_OPT_NODASH = 1 << 5, PARSE_OPT_LITERAL_ARGHELP = 1 << 6, PARSE_OPT_FROM_ALIAS = 1 << 7, - PARSE_OPT_SHELL_EVAL = 1 << 8, PARSE_OPT_NOCOMPLETE = 1 << 9, PARSE_OPT_COMP_ARG = 1 << 10, PARSE_OPT_CMDMODE = 1 << 11, @@ -134,7 +134,7 @@ struct option { const char *argh; const char *help; - int flags; + enum parse_opt_option_flags flags; parse_opt_cb *callback; intptr_t defval; parse_opt_ll_cb *ll_callback; @@ -213,9 +213,11 @@ struct option { * untouched and parse_options() returns the number of options * processed. */ -int parse_options(int argc, const char **argv, const char *prefix, - const struct option *options, - const char * const usagestr[], int flags); +enum parse_opt_result parse_options(int argc, const char **argv, + const char *prefix, + const struct option *options, + const char * const usagestr[], + enum parse_opt_flags flags); NORETURN void usage_with_options(const char * const *usagestr, const struct option *options); @@ -224,9 +226,6 @@ NORETURN void usage_msg_opt(const char *msg, const char * const *usagestr, const struct option *options); -int optbug(const struct option *opt, const char *reason); -const char *optname(const struct option *opt, int flags); - /* * Use these assertions for callbacks that expect to be called with NONEG and * NOARG respectively, and do not otherwise handle the "unset" and "arg" @@ -264,7 +263,7 @@ struct parse_opt_ctx_t { const char **out; int argc, cpidx, total; const char *opt; - int flags; + enum parse_opt_flags flags; const char *prefix; const char **alias_groups; /* must be in groups of 3 elements! */ struct option *updated_options; @@ -272,11 +271,12 @@ struct parse_opt_ctx_t { void parse_options_start(struct parse_opt_ctx_t *ctx, int argc, const char **argv, const char *prefix, - const struct option *options, int flags); + const struct option *options, + enum parse_opt_flags flags); -int parse_options_step(struct parse_opt_ctx_t *ctx, - const struct option *options, - const char * const usagestr[]); +enum parse_opt_result parse_options_step(struct parse_opt_ctx_t *ctx, + const struct option *options, + const char * const usagestr[]); int parse_options_end(struct parse_opt_ctx_t *ctx); @@ -302,6 +302,8 @@ enum parse_opt_result parse_opt_unknown_cb(struct parse_opt_ctx_t *ctx, const char *, int); int parse_opt_passthru(const struct option *, const char *, int); int parse_opt_passthru_argv(const struct option *, const char *, int); +/* value is enum branch_track* */ +int parse_opt_tracking_mode(const struct option *, const char *, int); #define OPT__VERBOSE(var, h) OPT_COUNTUP('v', "verbose", (var), (h)) #define OPT__QUIET(var, h) OPT_COUNTUP('q', "quiet", (var), (h)) diff --git a/perl/Git/SVN.pm b/perl/Git/SVN.pm index 35ff5a6896..df5a87a151 100644 --- a/perl/Git/SVN.pm +++ b/perl/Git/SVN.pm @@ -6,7 +6,7 @@ use constant rev_map_fmt => 'NH*'; use vars qw/$_no_metadata $_repack $_repack_flags $_use_svm_props $_head $_use_svnsync_props $no_reuse_existing - $_use_log_author $_add_author_from $_localtime/; + $_use_log_author $_add_author_from $_localtime $_use_fsync/; use Carp qw/croak/; use File::Path qw/mkpath/; use IPC::Open3; @@ -2269,6 +2269,19 @@ sub mkfile { } } +# TODO: move this to Git.pm? +sub use_fsync { + if (!defined($_use_fsync)) { + my $x = $ENV{GIT_TEST_FSYNC}; + if (defined $x) { + my $v = command_oneline('git', '-c', "test.fsync=$x", + qw(config --type=bool test.fsync)); + $_use_fsync = defined($v) ? ($v eq "true\n") : 1; + } + } + $_use_fsync; +} + sub rev_map_set { my ($self, $rev, $commit, $update_ref, $uuid) = @_; defined $commit or die "missing arg3\n"; @@ -2290,7 +2303,7 @@ sub rev_map_set { my $sync; # both of these options make our .rev_db file very, very important # and we can't afford to lose it because rebuild() won't work - if ($self->use_svm_props || $self->no_metadata) { + if (($self->use_svm_props || $self->no_metadata) && use_fsync()) { require File::Copy; $sync = 1; File::Copy::copy($db, $db_lock) or die "rev_map_set(@_): ", diff --git a/pkt-line.c b/pkt-line.c index de4a94b437..06013d2a54 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -289,22 +289,6 @@ void packet_buf_write(struct strbuf *buf, const char *fmt, ...) va_end(args); } -void packet_buf_write_len(struct strbuf *buf, const char *data, size_t len) -{ - size_t orig_len, n; - - orig_len = buf->len; - strbuf_addstr(buf, "0000"); - strbuf_add(buf, data, len); - n = buf->len - orig_len; - - if (n > LARGE_PACKET_MAX) - die(_("protocol error: impossibly long line")); - - set_packet_header(&buf->buf[orig_len], n); - packet_trace(data, len, 1); -} - int write_packetized_from_fd_no_flush(int fd_in, int fd_out) { char *buf = xmalloc(LARGE_PACKET_DATA_MAX); @@ -386,6 +370,31 @@ int packet_length(const char lenbuf_hex[4]) return (val < 0) ? val : (val << 8) | hex2chr(lenbuf_hex + 2); } +static char *find_packfile_uri_path(const char *buffer) +{ + const char *URI_MARK = "://"; + char *path; + int len; + + /* First char is sideband mark */ + buffer += 1; + + len = strspn(buffer, "0123456789abcdefABCDEF"); + if (!(len == 40 || len == 64) || buffer[len] != ' ') + return NULL; /* required "<hash>SP" not seen */ + + path = strstr(buffer + len + 1, URI_MARK); + if (!path) + return NULL; + + path = strchr(path + strlen(URI_MARK), '/'); + if (!path || !*(path + 1)) + return NULL; + + /* position after '/' */ + return ++path; +} + enum packet_read_status packet_read_with_status(int fd, char **src_buffer, size_t *src_len, char *buffer, unsigned size, int *pktlen, @@ -393,6 +402,7 @@ enum packet_read_status packet_read_with_status(int fd, char **src_buffer, { int len; char linelen[4]; + char *uri_path_start; if (get_packet_data(fd, src_buffer, src_len, linelen, 4, options) < 0) { *pktlen = -1; @@ -443,7 +453,18 @@ enum packet_read_status packet_read_with_status(int fd, char **src_buffer, len--; buffer[len] = 0; - packet_trace(buffer, len, 0); + if (options & PACKET_READ_REDACT_URI_PATH && + (uri_path_start = find_packfile_uri_path(buffer))) { + const char *redacted = "<redacted>"; + struct strbuf tracebuf = STRBUF_INIT; + strbuf_insert(&tracebuf, 0, buffer, len); + strbuf_splice(&tracebuf, uri_path_start - buffer, + strlen(uri_path_start), redacted, strlen(redacted)); + packet_trace(tracebuf.buf, tracebuf.len, 0); + strbuf_release(&tracebuf); + } else { + packet_trace(buffer, len, 0); + } if ((options & PACKET_READ_DIE_ON_ERR_PACKET) && starts_with(buffer, "ERR ")) @@ -453,38 +474,28 @@ enum packet_read_status packet_read_with_status(int fd, char **src_buffer, return PACKET_READ_NORMAL; } -int packet_read(int fd, char **src_buffer, size_t *src_len, - char *buffer, unsigned size, int options) +int packet_read(int fd, char *buffer, unsigned size, int options) { int pktlen = -1; - packet_read_with_status(fd, src_buffer, src_len, buffer, size, - &pktlen, options); + packet_read_with_status(fd, NULL, NULL, buffer, size, &pktlen, + options); return pktlen; } -static char *packet_read_line_generic(int fd, - char **src, size_t *src_len, - int *dst_len) +char *packet_read_line(int fd, int *dst_len) { - int len = packet_read(fd, src, src_len, - packet_buffer, sizeof(packet_buffer), + int len = packet_read(fd, packet_buffer, sizeof(packet_buffer), PACKET_READ_CHOMP_NEWLINE); if (dst_len) *dst_len = len; return (len > 0) ? packet_buffer : NULL; } -char *packet_read_line(int fd, int *len_p) -{ - return packet_read_line_generic(fd, NULL, NULL, len_p); -} - int packet_read_line_gently(int fd, int *dst_len, char **dst_line) { - int len = packet_read(fd, NULL, NULL, - packet_buffer, sizeof(packet_buffer), + int len = packet_read(fd, packet_buffer, sizeof(packet_buffer), PACKET_READ_CHOMP_NEWLINE|PACKET_READ_GENTLE_ON_EOF); if (dst_len) *dst_len = len; @@ -493,11 +504,6 @@ int packet_read_line_gently(int fd, int *dst_len, char **dst_line) return len; } -char *packet_read_line_buf(char **src, size_t *src_len, int *dst_len) -{ - return packet_read_line_generic(-1, src, src_len, dst_len); -} - ssize_t read_packetized_to_strbuf(int fd_in, struct strbuf *sb_out, int options) { int packet_len; @@ -507,7 +513,7 @@ ssize_t read_packetized_to_strbuf(int fd_in, struct strbuf *sb_out, int options) for (;;) { strbuf_grow(sb_out, LARGE_PACKET_DATA_MAX); - packet_len = packet_read(fd_in, NULL, NULL, + packet_len = packet_read(fd_in, /* strbuf_grow() above always allocates one extra byte to * store a '\0' at the end of the string. packet_read() * writes a '\0' extra byte at the end, too. Let it know diff --git a/pkt-line.h b/pkt-line.h index 82b95e4bdd..6d2a63db23 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -29,7 +29,6 @@ void packet_buf_delim(struct strbuf *buf); void set_packet_header(char *buf, int size); void packet_write(int fd_out, const char *buf, size_t size); void packet_buf_write(struct strbuf *buf, const char *fmt, ...) __attribute__((format (printf, 2, 3))); -void packet_buf_write_len(struct strbuf *buf, const char *data, size_t len); int packet_flush_gently(int fd); int packet_write_fmt_gently(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3))); int write_packetized_from_fd_no_flush(int fd_in, int fd_out); @@ -88,8 +87,8 @@ void packet_fflush(FILE *f); #define PACKET_READ_CHOMP_NEWLINE (1u<<1) #define PACKET_READ_DIE_ON_ERR_PACKET (1u<<2) #define PACKET_READ_GENTLE_ON_READ_ERROR (1u<<3) -int packet_read(int fd, char **src_buffer, size_t *src_len, char - *buffer, unsigned size, int options); +#define PACKET_READ_REDACT_URI_PATH (1u<<4) +int packet_read(int fd, char *buffer, unsigned size, int options); /* * Convert a four hex digit packet line length header into its numeric @@ -139,12 +138,6 @@ char *packet_read_line(int fd, int *size); int packet_read_line_gently(int fd, int *size, char **dst_line); /* - * Same as packet_read_line, but read from a buf rather than a descriptor; - * see packet_read for details on how src_* is used. - */ -char *packet_read_line_buf(char **src_buf, size_t *src_len, int *size); - -/* * Reads a stream of variable sized packets until a flush packet is detected. */ ssize_t read_packetized_to_strbuf(int fd_in, struct strbuf *sb_out, int options); @@ -431,6 +431,52 @@ const char *show_ident_date(const struct ident_split *ident, return show_date(date, tz, mode); } +static inline void strbuf_add_with_color(struct strbuf *sb, const char *color, + const char *buf, size_t buflen) +{ + strbuf_addstr(sb, color); + strbuf_add(sb, buf, buflen); + if (*color) + strbuf_addstr(sb, GIT_COLOR_RESET); +} + +static void append_line_with_color(struct strbuf *sb, struct grep_opt *opt, + const char *line, size_t linelen, + int color, enum grep_context ctx, + enum grep_header_field field) +{ + const char *buf, *eol, *line_color, *match_color; + regmatch_t match; + int eflags = 0; + + buf = line; + eol = buf + linelen; + + if (!opt || !want_color(color) || opt->invert) + goto end; + + line_color = opt->colors[GREP_COLOR_SELECTED]; + match_color = opt->colors[GREP_COLOR_MATCH_SELECTED]; + + while (grep_next_match(opt, buf, eol, ctx, &match, field, eflags)) { + if (match.rm_so == match.rm_eo) + break; + + strbuf_add_with_color(sb, line_color, buf, match.rm_so); + strbuf_add_with_color(sb, match_color, buf + match.rm_so, + match.rm_eo - match.rm_so); + buf += match.rm_eo; + eflags = REG_NOTBOL; + } + + if (eflags) + strbuf_add_with_color(sb, line_color, buf, eol - buf); + else { +end: + strbuf_add(sb, buf, eol - buf); + } +} + void pp_user_info(struct pretty_print_context *pp, const char *what, struct strbuf *sb, const char *line, const char *encoding) @@ -496,9 +542,26 @@ void pp_user_info(struct pretty_print_context *pp, strbuf_addch(sb, '\n'); strbuf_addf(sb, " <%.*s>\n", (int)maillen, mailbuf); } else { - strbuf_addf(sb, "%s: %.*s%.*s <%.*s>\n", what, - (pp->fmt == CMIT_FMT_FULLER) ? 4 : 0, " ", - (int)namelen, namebuf, (int)maillen, mailbuf); + struct strbuf id = STRBUF_INIT; + enum grep_header_field field = GREP_HEADER_FIELD_MAX; + struct grep_opt *opt = pp->rev ? &pp->rev->grep_filter : NULL; + + if (!strcmp(what, "Author")) + field = GREP_HEADER_AUTHOR; + else if (!strcmp(what, "Commit")) + field = GREP_HEADER_COMMITTER; + + strbuf_addf(sb, "%s: ", what); + if (pp->fmt == CMIT_FMT_FULLER) + strbuf_addchars(sb, ' ', 4); + + strbuf_addf(&id, "%.*s <%.*s>", (int)namelen, namebuf, + (int)maillen, mailbuf); + + append_line_with_color(sb, opt, id.buf, id.len, pp->color, + GREP_CONTEXT_HEAD, field); + strbuf_addch(sb, '\n'); + strbuf_release(&id); } switch (pp->fmt) { @@ -1451,8 +1514,8 @@ static size_t format_commit_one(struct strbuf *sb, /* in UTF-8 */ check_commit_signature(c->commit, &(c->signature_check)); switch (placeholder[1]) { case 'G': - if (c->signature_check.gpg_output) - strbuf_addstr(sb, c->signature_check.gpg_output); + if (c->signature_check.output) + strbuf_addstr(sb, c->signature_check.output); break; case '?': switch (c->signature_check.result) { @@ -1954,8 +2017,9 @@ static int pp_utf8_width(const char *start, const char *end) return width; } -static void strbuf_add_tabexpand(struct strbuf *sb, int tabwidth, - const char *line, int linelen) +static void strbuf_add_tabexpand(struct strbuf *sb, struct grep_opt *opt, + int color, int tabwidth, const char *line, + int linelen) { const char *tab; @@ -1972,7 +2036,9 @@ static void strbuf_add_tabexpand(struct strbuf *sb, int tabwidth, break; /* Output the data .. */ - strbuf_add(sb, line, tab - line); + append_line_with_color(sb, opt, line, tab - line, color, + GREP_CONTEXT_BODY, + GREP_HEADER_FIELD_MAX); /* .. and the de-tabified tab */ strbuf_addchars(sb, ' ', tabwidth - (width % tabwidth)); @@ -1987,7 +2053,8 @@ static void strbuf_add_tabexpand(struct strbuf *sb, int tabwidth, * worrying about width - there's nothing more to * align. */ - strbuf_add(sb, line, linelen); + append_line_with_color(sb, opt, line, linelen, color, GREP_CONTEXT_BODY, + GREP_HEADER_FIELD_MAX); } /* @@ -1999,11 +2066,16 @@ static void pp_handle_indent(struct pretty_print_context *pp, struct strbuf *sb, int indent, const char *line, int linelen) { + struct grep_opt *opt = pp->rev ? &pp->rev->grep_filter : NULL; + strbuf_addchars(sb, ' ', indent); if (pp->expand_tabs_in_log) - strbuf_add_tabexpand(sb, pp->expand_tabs_in_log, line, linelen); + strbuf_add_tabexpand(sb, opt, pp->color, pp->expand_tabs_in_log, + line, linelen); else - strbuf_add(sb, line, linelen); + append_line_with_color(sb, opt, line, linelen, pp->color, + GREP_CONTEXT_BODY, + GREP_HEADER_FIELD_MAX); } static int is_mboxrd_from(const char *line, int len) @@ -2021,7 +2093,9 @@ void pp_remainder(struct pretty_print_context *pp, struct strbuf *sb, int indent) { + struct grep_opt *opt = pp->rev ? &pp->rev->grep_filter : NULL; int first = 1; + for (;;) { const char *line = *msg_p; int linelen = get_one_line(line); @@ -2042,14 +2116,17 @@ void pp_remainder(struct pretty_print_context *pp, if (indent) pp_handle_indent(pp, sb, indent, line, linelen); else if (pp->expand_tabs_in_log) - strbuf_add_tabexpand(sb, pp->expand_tabs_in_log, - line, linelen); + strbuf_add_tabexpand(sb, opt, pp->color, + pp->expand_tabs_in_log, line, + linelen); else { if (pp->fmt == CMIT_FMT_MBOXRD && is_mboxrd_from(line, linelen)) strbuf_addch(sb, '>'); - strbuf_add(sb, line, linelen); + append_line_with_color(sb, opt, line, linelen, + pp->color, GREP_CONTEXT_BODY, + GREP_HEADER_FIELD_MAX); } strbuf_addch(sb, '\n'); } diff --git a/progress.c b/progress.c index 680c6a8bf9..7483aec2e2 100644 --- a/progress.c +++ b/progress.c @@ -46,6 +46,7 @@ struct progress { }; static volatile sig_atomic_t progress_update; +static struct progress *global_progress; /* * These are only intended for testing the progress output, i.e. exclusively @@ -249,6 +250,14 @@ void display_progress(struct progress *progress, uint64_t n) display(progress, n, NULL); } +static void set_global_progress(struct progress *progress) +{ + if (global_progress) + BUG("'%s' progress still active when trying to start '%s'", + global_progress->title, progress->title); + global_progress = progress; +} + static struct progress *start_progress_delay(const char *title, uint64_t total, unsigned delay, unsigned sparse) { @@ -264,6 +273,7 @@ static struct progress *start_progress_delay(const char *title, uint64_t total, strbuf_init(&progress->counters_sb, 0); progress->title_len = utf8_strwidth(title); progress->split = 0; + set_global_progress(progress); set_progress_signal(); trace2_region_enter("progress", title, the_repository); return progress; @@ -325,20 +335,28 @@ void stop_progress(struct progress **p_progress) finish_if_sparse(*p_progress); if (*p_progress) { + struct progress *progress = *p_progress; trace2_data_intmax("progress", the_repository, "total_objects", (*p_progress)->total); if ((*p_progress)->throughput) trace2_data_intmax("progress", the_repository, "total_bytes", - (*p_progress)->throughput->curr_total); + progress->throughput->curr_total); - trace2_region_leave("progress", (*p_progress)->title, the_repository); + trace2_region_leave("progress", progress->title, the_repository); } stop_progress_msg(p_progress, _("done")); } +static void unset_global_progress(void) +{ + if (!global_progress) + BUG("should have active global_progress when cleaning up"); + global_progress = NULL; +} + void stop_progress_msg(struct progress **p_progress, const char *msg) { struct progress *progress; @@ -368,6 +386,7 @@ void stop_progress_msg(struct progress **p_progress, const char *msg) free(buf); } clear_progress_signal(); + unset_global_progress(); strbuf_release(&progress->counters_sb); if (progress->throughput) strbuf_release(&progress->throughput->display); diff --git a/read-cache.c b/read-cache.c index 8a50ff66b3..0e09a04793 100644 --- a/read-cache.c +++ b/read-cache.c @@ -28,6 +28,7 @@ #include "sparse-index.h" #include "csum-file.h" #include "promisor-remote.h" +#include "hook.h" /* Mask for the name length in ce_flags in the on-disk index */ @@ -68,6 +69,11 @@ */ #define CACHE_ENTRY_PATH_LENGTH 80 +enum index_search_mode { + NO_EXPAND_SPARSE = 0, + EXPAND_SPARSE = 1 +}; + static inline struct cache_entry *mem_pool__ce_alloc(struct mem_pool *mem_pool, size_t len) { struct cache_entry *ce; @@ -551,7 +557,10 @@ int cache_name_stage_compare(const char *name1, int len1, int stage1, const char return 0; } -static int index_name_stage_pos(struct index_state *istate, const char *name, int namelen, int stage) +static int index_name_stage_pos(struct index_state *istate, + const char *name, int namelen, + int stage, + enum index_search_mode search_mode) { int first, last; @@ -570,7 +579,7 @@ static int index_name_stage_pos(struct index_state *istate, const char *name, in first = next+1; } - if (istate->sparse_index && + if (search_mode == EXPAND_SPARSE && istate->sparse_index && first > 0) { /* Note: first <= istate->cache_nr */ struct cache_entry *ce = istate->cache[first - 1]; @@ -586,7 +595,7 @@ static int index_name_stage_pos(struct index_state *istate, const char *name, in ce_namelen(ce) < namelen && !strncmp(name, ce->name, ce_namelen(ce))) { ensure_full_index(istate); - return index_name_stage_pos(istate, name, namelen, stage); + return index_name_stage_pos(istate, name, namelen, stage, search_mode); } } @@ -595,7 +604,12 @@ static int index_name_stage_pos(struct index_state *istate, const char *name, in int index_name_pos(struct index_state *istate, const char *name, int namelen) { - return index_name_stage_pos(istate, name, namelen, 0); + return index_name_stage_pos(istate, name, namelen, 0, EXPAND_SPARSE); +} + +int index_entry_exists(struct index_state *istate, const char *name, int namelen) +{ + return index_name_stage_pos(istate, name, namelen, 0, NO_EXPAND_SPARSE) >= 0; } int remove_index_entry_at(struct index_state *istate, int pos) @@ -738,7 +752,7 @@ int add_to_index(struct index_state *istate, const char *path, struct stat *st, int intent_only = flags & ADD_CACHE_INTENT; int add_option = (ADD_CACHE_OK_TO_ADD|ADD_CACHE_OK_TO_REPLACE| (intent_only ? ADD_CACHE_NEW_ONLY : 0)); - int hash_flags = HASH_WRITE_OBJECT; + unsigned hash_flags = pretend ? 0 : HASH_WRITE_OBJECT; struct object_id oid; if (flags & ADD_CACHE_RENORMALIZE) @@ -1237,7 +1251,7 @@ static int has_dir_name(struct index_state *istate, */ } - pos = index_name_stage_pos(istate, name, len, stage); + pos = index_name_stage_pos(istate, name, len, stage, EXPAND_SPARSE); if (pos >= 0) { /* * Found one, but not so fast. This could @@ -1337,7 +1351,7 @@ static int add_index_entry_with_check(struct index_state *istate, struct cache_e strcmp(ce->name, istate->cache[istate->cache_nr - 1]->name) > 0) pos = index_pos_to_insert_pos(istate->cache_nr); else - pos = index_name_stage_pos(istate, ce->name, ce_namelen(ce), ce_stage(ce)); + pos = index_name_stage_pos(istate, ce->name, ce_namelen(ce), ce_stage(ce), EXPAND_SPARSE); /* existing match? Just replace it. */ if (pos >= 0) { @@ -1372,7 +1386,7 @@ static int add_index_entry_with_check(struct index_state *istate, struct cache_e if (!ok_to_replace) return error(_("'%s' appears as both a file and as a directory"), ce->name); - pos = index_name_stage_pos(istate, ce->name, ce_namelen(ce), ce_stage(ce)); + pos = index_name_stage_pos(istate, ce->name, ce_namelen(ce), ce_stage(ce), EXPAND_SPARSE); pos = -pos-1; } return pos + 1; @@ -2352,9 +2366,17 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) if (!istate->repo) istate->repo = the_repository; + + /* + * If the command explicitly requires a full index, force it + * to be full. Otherwise, correct the sparsity based on repository + * settings and other properties of the index (if necessary). + */ prepare_repo_settings(istate->repo); if (istate->repo->settings.command_requires_full_index) ensure_full_index(istate); + else + ensure_correct_sparsity(istate); return istate->cache_nr; @@ -3101,6 +3123,7 @@ static int do_write_locked_index(struct index_state *istate, struct lock_file *l { int ret; int was_full = !istate->sparse_index; + struct run_hooks_opt hook_opt = RUN_HOOKS_OPT_INIT; ret = convert_to_sparse(istate, 0); @@ -3129,9 +3152,12 @@ static int do_write_locked_index(struct index_state *istate, struct lock_file *l else ret = close_lock_file_gently(lock); - run_hook_le(NULL, "post-index-change", - istate->updated_workdir ? "1" : "0", - istate->updated_skipworktree ? "1" : "0", NULL); + strvec_pushl(&hook_opt.args, + istate->updated_workdir ? "1" : "0", + istate->updated_skipworktree ? "1" : "0", + NULL); + run_hooks_oneshot("post-index-change", &hook_opt); + istate->updated_workdir = 0; istate->updated_skipworktree = 0; diff --git a/ref-filter.c b/ref-filter.c index add429be79..afed03472f 100644 --- a/ref-filter.c +++ b/ref-filter.c @@ -2470,6 +2470,12 @@ static int memcasecmp(const void *vs1, const void *vs2, size_t n) return 0; } +struct ref_sorting { + struct ref_sorting *next; + int atom; /* index into used_atom array (internal) */ + enum ref_sorting_order sort_flags; +}; + static int cmp_ref_sorting(struct ref_sorting *s, struct ref_array_item *a, struct ref_array_item *b) { struct atom_value *va, *vb; @@ -2663,7 +2669,7 @@ static int parse_sorting_atom(const char *atom) } /* If no sorting option is given, use refname to sort as default */ -struct ref_sorting *ref_default_sorting(void) +static struct ref_sorting *ref_default_sorting(void) { static const char cstr_name[] = "refname"; @@ -2674,7 +2680,7 @@ struct ref_sorting *ref_default_sorting(void) return sorting; } -void parse_ref_sorting(struct ref_sorting **sorting_tail, const char *arg) +static void parse_ref_sorting(struct ref_sorting **sorting_tail, const char *arg) { struct ref_sorting *s; @@ -2692,17 +2698,34 @@ void parse_ref_sorting(struct ref_sorting **sorting_tail, const char *arg) s->atom = parse_sorting_atom(arg); } -int parse_opt_ref_sorting(const struct option *opt, const char *arg, int unset) +struct ref_sorting *ref_sorting_options(struct string_list *options) { + struct string_list_item *item; + struct ref_sorting *sorting = NULL, **tail = &sorting; + + if (!options->nr) { + sorting = ref_default_sorting(); + } else { + for_each_string_list_item(item, options) + parse_ref_sorting(tail, item->string); + } + /* - * NEEDSWORK: We should probably clear the list in this case, but we've - * already munged the global used_atoms list, which would need to be - * undone. + * From here on, the ref_sorting list should be used to talk + * about the sort order used for the output. The caller + * should not touch the string form anymore. */ - BUG_ON_OPT_NEG(unset); + string_list_clear(options, 0); + return sorting; +} - parse_ref_sorting(opt->value, arg); - return 0; +void ref_sorting_release(struct ref_sorting *sorting) +{ + while (sorting) { + struct ref_sorting *next = sorting->next; + free(sorting); + sorting = next; + } } int parse_opt_merge_filter(const struct option *opt, const char *arg, int unset) diff --git a/ref-filter.h b/ref-filter.h index b636f4389d..aa0eea4ecf 100644 --- a/ref-filter.h +++ b/ref-filter.h @@ -23,16 +23,13 @@ #define FILTER_REFS_KIND_MASK (FILTER_REFS_ALL | FILTER_REFS_DETACHED_HEAD) struct atom_value; +struct ref_sorting; -struct ref_sorting { - struct ref_sorting *next; - int atom; /* index into used_atom array (internal) */ - enum { - REF_SORTING_REVERSE = 1<<0, - REF_SORTING_ICASE = 1<<1, - REF_SORTING_VERSION = 1<<2, - REF_SORTING_DETACHED_HEAD_FIRST = 1<<3, - } sort_flags; +enum ref_sorting_order { + REF_SORTING_REVERSE = 1<<0, + REF_SORTING_ICASE = 1<<1, + REF_SORTING_VERSION = 1<<2, + REF_SORTING_DETACHED_HEAD_FIRST = 1<<3, }; struct ref_array_item { @@ -97,9 +94,8 @@ struct ref_format { #define OPT_NO_MERGED(f, h) _OPT_MERGED_NO_MERGED("no-merged", f, h) #define OPT_REF_SORT(var) \ - OPT_CALLBACK_F(0, "sort", (var), \ - N_("key"), N_("field name to sort on"), \ - PARSE_OPT_NONEG, parse_opt_ref_sorting) + OPT_STRING_LIST(0, "sort", (var), \ + N_("key"), N_("field name to sort on")) /* * API for filtering a set of refs. Based on the type of refs the user @@ -121,12 +117,10 @@ int format_ref_array_item(struct ref_array_item *info, struct ref_format *format, struct strbuf *final_buf, struct strbuf *error_buf); -/* Parse a single sort specifier and add it to the list */ -void parse_ref_sorting(struct ref_sorting **sorting_tail, const char *atom); -/* Callback function for parsing the sort option */ -int parse_opt_ref_sorting(const struct option *opt, const char *arg, int unset); -/* Default sort option based on refname */ -struct ref_sorting *ref_default_sorting(void); +/* Release a "struct ref_sorting" */ +void ref_sorting_release(struct ref_sorting *); +/* Convert list of sort options into ref_sorting */ +struct ref_sorting *ref_sorting_options(struct string_list *); /* Function to parse --merged and --no-merged options */ int parse_opt_merge_filter(const struct option *opt, const char *arg, int unset); /* Get the current HEAD's description */ @@ -251,12 +251,13 @@ int refname_is_safe(const char *refname) * does not exist, emit a warning and return false. */ int ref_resolves_to_object(const char *refname, + struct repository *repo, const struct object_id *oid, unsigned int flags) { if (flags & REF_ISBROKEN) return 0; - if (!has_object_file(oid)) { + if (!repo_has_object_file(repo, oid)) { error(_("%s does not point to a valid object!"), refname); return 0; } @@ -268,9 +269,10 @@ char *refs_resolve_refdup(struct ref_store *refs, struct object_id *oid, int *flags) { const char *result; + int ignore_errno; result = refs_resolve_ref_unsafe(refs, refname, resolve_flags, - oid, flags); + oid, flags, &ignore_errno); return xstrdup_or_null(result); } @@ -290,20 +292,17 @@ struct ref_filter { void *cb_data; }; -int refs_read_ref_full(struct ref_store *refs, const char *refname, - int resolve_flags, struct object_id *oid, int *flags) +int read_ref_full(const char *refname, int resolve_flags, struct object_id *oid, int *flags) { - if (refs_resolve_ref_unsafe(refs, refname, resolve_flags, oid, flags)) + int ignore_errno; + struct ref_store *refs = get_main_ref_store(the_repository); + + if (refs_resolve_ref_unsafe(refs, refname, resolve_flags, + oid, flags, &ignore_errno)) return 0; return -1; } -int read_ref_full(const char *refname, int resolve_flags, struct object_id *oid, int *flags) -{ - return refs_read_ref_full(get_main_ref_store(the_repository), refname, - resolve_flags, oid, flags); -} - int read_ref(const char *refname, struct object_id *oid) { return read_ref_full(refname, RESOLVE_REF_READING, oid, NULL); @@ -311,7 +310,9 @@ int read_ref(const char *refname, struct object_id *oid) int refs_ref_exists(struct ref_store *refs, const char *refname) { - return !!refs_resolve_ref_unsafe(refs, refname, RESOLVE_REF_READING, NULL, NULL); + int ignore_errno; + return !!refs_resolve_ref_unsafe(refs, refname, RESOLVE_REF_READING, + NULL, NULL, &ignore_errno); } int ref_exists(const char *refname) @@ -654,13 +655,16 @@ int expand_ref(struct repository *repo, const char *str, int len, struct object_id oid_from_ref; struct object_id *this_result; int flag; + struct ref_store *refs = get_main_ref_store(repo); + int ignore_errno; this_result = refs_found ? &oid_from_ref : oid; strbuf_reset(&fullref); strbuf_addf(&fullref, *p, len, str); - r = refs_resolve_ref_unsafe(get_main_ref_store(repo), - fullref.buf, RESOLVE_REF_READING, - this_result, &flag); + r = refs_resolve_ref_unsafe(refs, fullref.buf, + RESOLVE_REF_READING, + this_result, &flag, + &ignore_errno); if (r) { if (!refs_found++) *ref = xstrdup(r); @@ -689,12 +693,14 @@ int repo_dwim_log(struct repository *r, const char *str, int len, for (p = ref_rev_parse_rules; *p; p++) { struct object_id hash; const char *ref, *it; + int ignore_errno; strbuf_reset(&path); strbuf_addf(&path, *p, len, str); ref = refs_resolve_ref_unsafe(refs, path.buf, RESOLVE_REF_READING, - oid ? &hash : NULL, NULL); + oid ? &hash : NULL, NULL, + &ignore_errno); if (!ref) continue; if (refs_reflog_exists(refs, path.buf)) @@ -1372,32 +1378,14 @@ const char *find_descendant_ref(const char *dirname, return NULL; } -int refs_rename_ref_available(struct ref_store *refs, - const char *old_refname, - const char *new_refname) -{ - struct string_list skip = STRING_LIST_INIT_NODUP; - struct strbuf err = STRBUF_INIT; - int ok; - - string_list_insert(&skip, old_refname); - ok = !refs_verify_refname_available(refs, new_refname, - NULL, &skip, &err); - if (!ok) - error("%s", err.buf); - - string_list_clear(&skip, 0); - strbuf_release(&err); - return ok; -} - int refs_head_ref(struct ref_store *refs, each_ref_fn fn, void *cb_data) { struct object_id oid; int flag; + int ignore_errno; - if (!refs_read_ref_full(refs, "HEAD", RESOLVE_REF_READING, - &oid, &flag)) + if (refs_resolve_ref_unsafe(refs, "HEAD", RESOLVE_REF_READING, + &oid, &flag, &ignore_errno)) return fn("HEAD", &oid, flag, cb_data); return 0; @@ -1648,7 +1636,8 @@ int for_each_fullref_in_prefixes(const char *namespace, static int refs_read_special_head(struct ref_store *ref_store, const char *refname, struct object_id *oid, - struct strbuf *referent, unsigned int *type) + struct strbuf *referent, unsigned int *type, + int *failure_errno) { struct strbuf full_path = STRBUF_INIT; struct strbuf content = STRBUF_INIT; @@ -1658,7 +1647,8 @@ static int refs_read_special_head(struct ref_store *ref_store, if (strbuf_read_file(&content, full_path.buf, 0) < 0) goto done; - result = parse_loose_ref_contents(content.buf, oid, referent, type); + result = parse_loose_ref_contents(content.buf, oid, referent, type, + failure_errno); done: strbuf_release(&full_path); @@ -1666,30 +1656,33 @@ done: return result; } -int refs_read_raw_ref(struct ref_store *ref_store, - const char *refname, struct object_id *oid, - struct strbuf *referent, unsigned int *type) +int refs_read_raw_ref(struct ref_store *ref_store, const char *refname, + struct object_id *oid, struct strbuf *referent, + unsigned int *type, int *failure_errno) { + assert(failure_errno); if (!strcmp(refname, "FETCH_HEAD") || !strcmp(refname, "MERGE_HEAD")) { return refs_read_special_head(ref_store, refname, oid, referent, - type); + type, failure_errno); } return ref_store->be->read_raw_ref(ref_store, refname, oid, referent, - type, &errno); + type, failure_errno); } -/* This function needs to return a meaningful errno on failure */ const char *refs_resolve_ref_unsafe(struct ref_store *refs, const char *refname, int resolve_flags, - struct object_id *oid, int *flags) + struct object_id *oid, + int *flags, int *failure_errno) { static struct strbuf sb_refname = STRBUF_INIT; struct object_id unused_oid; int unused_flags; int symref_count; + assert(failure_errno); + if (!oid) oid = &unused_oid; if (!flags) @@ -1700,7 +1693,7 @@ const char *refs_resolve_ref_unsafe(struct ref_store *refs, if (check_refname_format(refname, REFNAME_ALLOW_ONELEVEL)) { if (!(resolve_flags & RESOLVE_REF_ALLOW_BAD_NAME) || !refname_is_safe(refname)) { - errno = EINVAL; + *failure_errno = EINVAL; return NULL; } @@ -1718,9 +1711,11 @@ const char *refs_resolve_ref_unsafe(struct ref_store *refs, for (symref_count = 0; symref_count < SYMREF_MAXDEPTH; symref_count++) { unsigned int read_flags = 0; - if (refs_read_raw_ref(refs, refname, - oid, &sb_refname, &read_flags)) { + if (refs_read_raw_ref(refs, refname, oid, &sb_refname, + &read_flags, failure_errno)) { *flags |= read_flags; + if (errno) + *failure_errno = errno; /* In reading mode, refs must eventually resolve */ if (resolve_flags & RESOLVE_REF_READING) @@ -1731,9 +1726,9 @@ const char *refs_resolve_ref_unsafe(struct ref_store *refs, * may show errors besides ENOENT if there are * similarly-named refs. */ - if (errno != ENOENT && - errno != EISDIR && - errno != ENOTDIR) + if (*failure_errno != ENOENT && + *failure_errno != EISDIR && + *failure_errno != ENOTDIR) return NULL; oidclr(oid); @@ -1760,7 +1755,7 @@ const char *refs_resolve_ref_unsafe(struct ref_store *refs, if (check_refname_format(refname, REFNAME_ALLOW_ONELEVEL)) { if (!(resolve_flags & RESOLVE_REF_ALLOW_BAD_NAME) || !refname_is_safe(refname)) { - errno = EINVAL; + *failure_errno = EINVAL; return NULL; } @@ -1768,7 +1763,7 @@ const char *refs_resolve_ref_unsafe(struct ref_store *refs, } } - errno = ELOOP; + *failure_errno = ELOOP; return NULL; } @@ -1783,8 +1778,10 @@ int refs_init_db(struct strbuf *err) const char *resolve_ref_unsafe(const char *refname, int resolve_flags, struct object_id *oid, int *flags) { + int ignore_errno; + return refs_resolve_ref_unsafe(get_main_ref_store(the_repository), refname, - resolve_flags, oid, flags); + resolve_flags, oid, flags, &ignore_errno); } int resolve_gitlink_ref(const char *submodule, const char *refname, @@ -1792,14 +1789,15 @@ int resolve_gitlink_ref(const char *submodule, const char *refname, { struct ref_store *refs; int flags; + int ignore_errno; refs = get_submodule_ref_store(submodule); if (!refs) return -1; - if (!refs_resolve_ref_unsafe(refs, refname, 0, oid, &flags) || - is_null_oid(oid)) + if (!refs_resolve_ref_unsafe(refs, refname, 0, oid, &flags, + &ignore_errno) || is_null_oid(oid)) return -1; return 0; } @@ -1870,7 +1868,8 @@ static struct ref_store *lookup_ref_store_map(struct hashmap *map, * Create, record, and return a ref_store instance for the specified * gitdir. */ -static struct ref_store *ref_store_init(const char *gitdir, +static struct ref_store *ref_store_init(struct repository *repo, + const char *gitdir, unsigned int flags) { const char *be_name = "files"; @@ -1880,7 +1879,7 @@ static struct ref_store *ref_store_init(const char *gitdir, if (!be) BUG("reference backend %s is unknown", be_name); - refs = be->init(gitdir, flags); + refs = be->init(repo, gitdir, flags); return refs; } @@ -1892,7 +1891,7 @@ struct ref_store *get_main_ref_store(struct repository *r) if (!r->gitdir) BUG("attempting to get main_ref_store outside of repository"); - r->refs_private = ref_store_init(r->gitdir, REF_STORE_ALL_CAPS); + r->refs_private = ref_store_init(r, r->gitdir, REF_STORE_ALL_CAPS); r->refs_private = maybe_debug_wrap_ref_store(r->gitdir, r->refs_private); return r->refs_private; } @@ -1922,6 +1921,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule) struct ref_store *refs; char *to_free = NULL; size_t len; + struct repository *subrepo; if (!submodule) return NULL; @@ -1947,8 +1947,19 @@ struct ref_store *get_submodule_ref_store(const char *submodule) if (submodule_to_gitdir(&submodule_sb, submodule)) goto done; - /* assume that add_submodule_odb() has been called */ - refs = ref_store_init(submodule_sb.buf, + subrepo = xmalloc(sizeof(*subrepo)); + /* + * NEEDSWORK: Make get_submodule_ref_store() work with arbitrary + * superprojects other than the_repository. This probably should be + * done by making it take a struct repository * parameter instead of a + * submodule path. + */ + if (repo_submodule_init(subrepo, the_repository, submodule, + null_oid())) { + free(subrepo); + goto done; + } + refs = ref_store_init(subrepo, submodule_sb.buf, REF_STORE_READ | REF_STORE_ODB); register_ref_store_map(&submodule_ref_stores, "submodule", refs, submodule); @@ -1974,10 +1985,12 @@ struct ref_store *get_worktree_ref_store(const struct worktree *wt) return refs; if (wt->id) - refs = ref_store_init(git_common_path("worktrees/%s", wt->id), + refs = ref_store_init(the_repository, + git_common_path("worktrees/%s", wt->id), REF_STORE_ALL_CAPS); else - refs = ref_store_init(get_git_common_dir(), + refs = ref_store_init(the_repository, + get_git_common_dir(), REF_STORE_ALL_CAPS); if (refs) @@ -2086,8 +2099,11 @@ static int run_transaction_hook(struct ref_transaction *transaction, update->refname); if (write_in_full(proc.in, buf.buf, buf.len) < 0) { - if (errno != EPIPE) + if (errno != EPIPE) { + /* Don't leak errno outside this API */ + errno = 0; ret = -1; + } break; } } @@ -2121,7 +2137,7 @@ int ref_transaction_prepare(struct ref_transaction *transaction, break; } - if (getenv(GIT_QUARANTINE_ENVIRONMENT)) { + if (the_repository->objects->odb->disable_ref_updates) { strbuf_addstr(err, _("ref updates forbidden inside quarantine environment")); return -1; @@ -2222,6 +2238,13 @@ int refs_verify_refname_available(struct ref_store *refs, strbuf_grow(&dirname, strlen(refname) + 1); for (slash = strchr(refname, '/'); slash; slash = strchr(slash + 1, '/')) { + /* + * Just saying "Is a directory" when we e.g. can't + * lock some multi-level ref isn't very informative, + * the user won't be told *what* is a directory, so + * let's not use strerror() below. + */ + int ignore_errno; /* Expand dirname to the new prefix, not including the trailing slash: */ strbuf_add(&dirname, refname + dirname.len, slash - refname - dirname.len); @@ -2233,7 +2256,8 @@ int refs_verify_refname_available(struct ref_store *refs, if (skip && string_list_has_string(skip, dirname.buf)) continue; - if (!refs_read_raw_ref(refs, dirname.buf, &oid, &referent, &type)) { + if (!refs_read_raw_ref(refs, dirname.buf, &oid, &referent, + &type, &ignore_errno)) { strbuf_addf(err, _("'%s' exists; cannot create '%s'"), dirname.buf, refname); goto cleanup; @@ -58,6 +58,11 @@ struct worktree; * resolved. The function returns NULL for such ref names. * Caps and underscores refers to the special refs, such as HEAD, * FETCH_HEAD and friends, that all live outside of the refs/ directory. + * + * Callers should not inspect "errno" on failure, but rather pass in a + * "failure_errno" parameter, on failure the "errno" will indicate the + * type of failure encountered, but not necessarily one that came from + * a syscall. We might have faked it up. */ #define RESOLVE_REF_READING 0x01 #define RESOLVE_REF_NO_RECURSE 0x02 @@ -67,7 +72,8 @@ const char *refs_resolve_ref_unsafe(struct ref_store *refs, const char *refname, int resolve_flags, struct object_id *oid, - int *flags); + int *flags, int *failure_errno); + const char *resolve_ref_unsafe(const char *refname, int resolve_flags, struct object_id *oid, int *flags); @@ -77,8 +83,6 @@ char *refs_resolve_refdup(struct ref_store *refs, char *resolve_refdup(const char *refname, int resolve_flags, struct object_id *oid, int *flags); -int refs_read_ref_full(struct ref_store *refs, const char *refname, - int resolve_flags, struct object_id *oid, int *flags); int read_ref_full(const char *refname, int resolve_flags, struct object_id *oid, int *flags); int read_ref(const char *refname, struct object_id *oid); diff --git a/refs/files-backend.c b/refs/files-backend.c index 6a6ead0b99..4b14f30d48 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -79,13 +79,15 @@ static void clear_loose_ref_cache(struct files_ref_store *refs) * Create a new submodule ref cache and add it to the internal * set of caches. */ -static struct ref_store *files_ref_store_create(const char *gitdir, +static struct ref_store *files_ref_store_create(struct repository *repo, + const char *gitdir, unsigned int flags) { struct files_ref_store *refs = xcalloc(1, sizeof(*refs)); struct ref_store *ref_store = (struct ref_store *)refs; struct strbuf sb = STRBUF_INIT; + ref_store->repo = repo; ref_store->gitdir = xstrdup(gitdir); base_ref_store_init(ref_store, &refs_be_files); refs->store_flags = flags; @@ -93,7 +95,7 @@ static struct ref_store *files_ref_store_create(const char *gitdir, get_common_dir_noenv(&sb, gitdir); refs->gitcommondir = strbuf_detach(&sb, NULL); strbuf_addf(&sb, "%s/packed-refs", refs->gitcommondir); - refs->packed_ref_store = packed_ref_store_create(sb.buf, flags); + refs->packed_ref_store = packed_ref_store_create(repo, sb.buf, flags); strbuf_release(&sb); chdir_notify_reparent("files-backend $GIT_DIR", &refs->base.gitdir); @@ -280,10 +282,11 @@ static void loose_fill_ref_dir(struct ref_store *ref_store, create_dir_entry(dir->cache, refname.buf, refname.len)); } else { + int ignore_errno; if (!refs_resolve_ref_unsafe(&refs->base, refname.buf, RESOLVE_REF_READING, - &oid, &flag)) { + &oid, &flag, &ignore_errno)) { oidclr(&oid); flag |= REF_ISBROKEN; } else if (is_null_oid(&oid)) { @@ -355,6 +358,7 @@ static int files_read_raw_ref(struct ref_store *ref_store, const char *refname, int fd; int ret = -1; int remaining_retries = 3; + int myerr = 0; *type = 0; strbuf_reset(&sb_path); @@ -381,11 +385,14 @@ stat_ref: goto out; if (lstat(path, &st) < 0) { - if (errno != ENOENT) + int ignore_errno; + myerr = errno; + errno = 0; + if (myerr != ENOENT) goto out; - if (refs_read_raw_ref(refs->packed_ref_store, refname, - oid, referent, type)) { - errno = ENOENT; + if (refs_read_raw_ref(refs->packed_ref_store, refname, oid, + referent, type, &ignore_errno)) { + myerr = ENOENT; goto out; } ret = 0; @@ -396,7 +403,9 @@ stat_ref: if (S_ISLNK(st.st_mode)) { strbuf_reset(&sb_contents); if (strbuf_readlink(&sb_contents, path, st.st_size) < 0) { - if (errno == ENOENT || errno == EINVAL) + myerr = errno; + errno = 0; + if (myerr == ENOENT || myerr == EINVAL) /* inconsistent with lstat; retry */ goto stat_ref; else @@ -418,14 +427,15 @@ stat_ref: /* Is it a directory? */ if (S_ISDIR(st.st_mode)) { + int ignore_errno; /* * Even though there is a directory where the loose * ref is supposed to be, there could still be a * packed ref: */ - if (refs_read_raw_ref(refs->packed_ref_store, refname, - oid, referent, type)) { - errno = EISDIR; + if (refs_read_raw_ref(refs->packed_ref_store, refname, oid, + referent, type, &ignore_errno)) { + myerr = EISDIR; goto out; } ret = 0; @@ -438,7 +448,8 @@ stat_ref: */ fd = open(path, O_RDONLY); if (fd < 0) { - if (errno == ENOENT && !S_ISLNK(st.st_mode)) + myerr = errno; + if (myerr == ENOENT && !S_ISLNK(st.st_mode)) /* inconsistent with lstat; retry */ goto stat_ref; else @@ -446,26 +457,29 @@ stat_ref: } strbuf_reset(&sb_contents); if (strbuf_read(&sb_contents, fd, 256) < 0) { - int save_errno = errno; + myerr = errno; close(fd); - errno = save_errno; goto out; } close(fd); strbuf_rtrim(&sb_contents); buf = sb_contents.buf; - ret = parse_loose_ref_contents(buf, oid, referent, type); + ret = parse_loose_ref_contents(buf, oid, referent, type, &myerr); out: - *failure_errno = errno; + if (ret && !myerr) + BUG("returning non-zero %d, should have set myerr!", ret); + *failure_errno = myerr; + strbuf_release(&sb_path); strbuf_release(&sb_contents); return ret; } int parse_loose_ref_contents(const char *buf, struct object_id *oid, - struct strbuf *referent, unsigned int *type) + struct strbuf *referent, unsigned int *type, + int *failure_errno) { const char *p; if (skip_prefix(buf, "ref:", &buf)) { @@ -484,7 +498,7 @@ int parse_loose_ref_contents(const char *buf, struct object_id *oid, if (parse_oid_hex(buf, oid, &p) || (*p != '\0' && !isspace(*p))) { *type |= REF_ISBROKEN; - errno = EINVAL; + *failure_errno = EINVAL; return -1; } return 0; @@ -730,6 +744,7 @@ struct files_ref_iterator { struct ref_iterator base; struct ref_iterator *iter0; + struct repository *repo; unsigned int flags; }; @@ -751,6 +766,7 @@ static int files_ref_iterator_advance(struct ref_iterator *ref_iterator) if (!(iter->flags & DO_FOR_EACH_INCLUDE_BROKEN) && !ref_resolves_to_object(iter->iter0->refname, + iter->repo, iter->iter0->oid, iter->iter0->flags)) continue; @@ -829,7 +845,7 @@ static struct ref_iterator *files_ref_iterator_begin( */ loose_iter = cache_ref_iterator_begin(get_loose_ref_cache(refs), - prefix, 1); + prefix, ref_store->repo, 1); /* * The packed-refs file might contain broken references, for @@ -853,6 +869,7 @@ static struct ref_iterator *files_ref_iterator_begin( base_ref_iterator_init(ref_iterator, &files_ref_iterator_vtable, overlay_iter->ordered); iter->iter0 = overlay_iter; + iter->repo = ref_store->repo; iter->flags = flags; return ref_iterator; @@ -990,11 +1007,12 @@ static int create_reflock(const char *path, void *cb) * Locks a ref returning the lock on success and NULL on failure. */ static struct ref_lock *lock_ref_oid_basic(struct files_ref_store *refs, - const char *refname, int *type, + const char *refname, struct strbuf *err) { struct strbuf ref_file = STRBUF_INIT; struct ref_lock *lock; + int ignore_errno; files_assert_main_repository(refs, "lock_ref_oid_basic"); assert(err); @@ -1002,16 +1020,6 @@ static struct ref_lock *lock_ref_oid_basic(struct files_ref_store *refs, CALLOC_ARRAY(lock, 1); files_ref_path(refs, &ref_file, refname); - if (!refs_resolve_ref_unsafe(&refs->base, refname, - RESOLVE_REF_NO_RECURSE, - &lock->old_oid, type)) { - if (!refs_verify_refname_available(&refs->base, refname, - NULL, NULL, err)) - strbuf_addf(err, "unable to resolve reference '%s': %s", - refname, strerror(errno)); - - goto error_return; - } /* * If the ref did not exist and we are creating it, make sure @@ -1031,9 +1039,8 @@ static struct ref_lock *lock_ref_oid_basic(struct files_ref_store *refs, goto error_return; } - if (refs_read_ref_full(&refs->base, lock->ref_name, - 0, - &lock->old_oid, NULL)) + if (!refs_resolve_ref_unsafe(&refs->base, lock->ref_name, 0, + &lock->old_oid, NULL, &ignore_errno)) oidclr(&lock->old_oid); goto out; @@ -1169,7 +1176,7 @@ static int should_pack_ref(const char *refname, return 0; /* Do not pack broken refs: */ - if (!ref_resolves_to_object(refname, oid, ref_flags)) + if (!ref_resolves_to_object(refname, the_repository, oid, ref_flags)) return 0; return 1; @@ -1192,7 +1199,8 @@ static int files_pack_refs(struct ref_store *ref_store, unsigned int flags) packed_refs_lock(refs->packed_ref_store, LOCK_DIE_ON_ERROR, &err); - iter = cache_ref_iterator_begin(get_loose_ref_cache(refs), NULL, 0); + iter = cache_ref_iterator_begin(get_loose_ref_cache(refs), NULL, + the_repository, 0); while ((ok = ref_iterator_advance(iter)) == ITER_OK) { /* * If the loose reference can be packed, add an entry @@ -1352,6 +1360,35 @@ static int commit_ref_update(struct files_ref_store *refs, const struct object_id *oid, const char *logmsg, struct strbuf *err); +/* + * Emit a better error message than lockfile.c's + * unable_to_lock_message() would in case there is a D/F conflict with + * another existing reference. If there would be a conflict, emit an error + * message and return false; otherwise, return true. + * + * Note that this function is not safe against all races with other + * processes, and that's not its job. We'll emit a more verbose error on D/f + * conflicts if we get past it into lock_ref_oid_basic(). + */ +static int refs_rename_ref_available(struct ref_store *refs, + const char *old_refname, + const char *new_refname) +{ + struct string_list skip = STRING_LIST_INIT_NODUP; + struct strbuf err = STRBUF_INIT; + int ok; + + string_list_insert(&skip, old_refname); + ok = !refs_verify_refname_available(refs, new_refname, + NULL, &skip, &err); + if (!ok) + error("%s", err.buf); + + string_list_clear(&skip, 0); + strbuf_release(&err); + return ok; +} + static int files_copy_or_rename_ref(struct ref_store *ref_store, const char *oldrefname, const char *newrefname, const char *logmsg, int copy) @@ -1367,6 +1404,7 @@ static int files_copy_or_rename_ref(struct ref_store *ref_store, struct strbuf tmp_renamed_log = STRBUF_INIT; int log, ret; struct strbuf err = STRBUF_INIT; + int ignore_errno; files_reflog_path(refs, &sb_oldref, oldrefname); files_reflog_path(refs, &sb_newref, newrefname); @@ -1380,7 +1418,7 @@ static int files_copy_or_rename_ref(struct ref_store *ref_store, if (!refs_resolve_ref_unsafe(&refs->base, oldrefname, RESOLVE_REF_READING | RESOLVE_REF_NO_RECURSE, - &orig_oid, &flag)) { + &orig_oid, &flag, &ignore_errno)) { ret = error("refname %s not found", oldrefname); goto out; } @@ -1424,9 +1462,9 @@ static int files_copy_or_rename_ref(struct ref_store *ref_store, * the safety anyway; we want to delete the reference whatever * its current value. */ - if (!copy && !refs_read_ref_full(&refs->base, newrefname, - RESOLVE_REF_READING | RESOLVE_REF_NO_RECURSE, - NULL, NULL) && + if (!copy && refs_resolve_ref_unsafe(&refs->base, newrefname, + RESOLVE_REF_READING | RESOLVE_REF_NO_RECURSE, + NULL, NULL, &ignore_errno) && refs_delete_ref(&refs->base, NULL, newrefname, NULL, REF_NO_DEREF)) { if (errno == EISDIR) { @@ -1452,7 +1490,7 @@ static int files_copy_or_rename_ref(struct ref_store *ref_store, logmoved = log; - lock = lock_ref_oid_basic(refs, newrefname, NULL, &err); + lock = lock_ref_oid_basic(refs, newrefname, &err); if (!lock) { if (copy) error("unable to copy '%s' to '%s': %s", oldrefname, newrefname, err.buf); @@ -1474,7 +1512,7 @@ static int files_copy_or_rename_ref(struct ref_store *ref_store, goto out; rollback: - lock = lock_ref_oid_basic(refs, oldrefname, NULL, &err); + lock = lock_ref_oid_basic(refs, oldrefname, &err); if (!lock) { error("unable to lock %s for rollback: %s", oldrefname, err.buf); strbuf_release(&err); @@ -1791,10 +1829,12 @@ static int commit_ref_update(struct files_ref_store *refs, */ int head_flag; const char *head_ref; + int ignore_errno; head_ref = refs_resolve_ref_unsafe(&refs->base, "HEAD", RESOLVE_REF_READING, - NULL, &head_flag); + NULL, &head_flag, + &ignore_errno); if (head_ref && (head_flag & REF_ISSYMREF) && !strcmp(head_ref, lock->ref_name)) { struct strbuf log_err = STRBUF_INIT; @@ -1838,9 +1878,12 @@ static void update_symref_reflog(struct files_ref_store *refs, { struct strbuf err = STRBUF_INIT; struct object_id new_oid; + int ignore_errno; + if (logmsg && - !refs_read_ref_full(&refs->base, target, - RESOLVE_REF_READING, &new_oid, NULL) && + refs_resolve_ref_unsafe(&refs->base, target, + RESOLVE_REF_READING, &new_oid, NULL, + &ignore_errno) && files_log_ref_write(refs, refname, &lock->old_oid, &new_oid, logmsg, 0, &err)) { error("%s", err.buf); @@ -1881,7 +1924,7 @@ static int files_create_symref(struct ref_store *ref_store, struct ref_lock *lock; int ret; - lock = lock_ref_oid_basic(refs, refname, NULL, &err); + lock = lock_ref_oid_basic(refs, refname, &err); if (!lock) { error("%s", err.buf); strbuf_release(&err); @@ -2114,6 +2157,7 @@ static int files_reflog_iterator_advance(struct ref_iterator *ref_iterator) (struct files_reflog_iterator *)ref_iterator; struct dir_iterator *diter = iter->dir_iterator; int ok; + int ignore_errno; while ((ok = dir_iterator_advance(diter)) == ITER_OK) { int flags; @@ -2125,9 +2169,10 @@ static int files_reflog_iterator_advance(struct ref_iterator *ref_iterator) if (ends_with(diter->basename, ".lock")) continue; - if (refs_read_ref_full(iter->ref_store, - diter->relative_path, 0, - &iter->oid, &flags)) { + if (!refs_resolve_ref_unsafe(iter->ref_store, + diter->relative_path, 0, + &iter->oid, &flags, + &ignore_errno)) { error("bad ref for %s", diter->path.buf); continue; } @@ -2471,9 +2516,11 @@ static int lock_ref_for_update(struct files_ref_store *refs, * the transaction, so we have to read it here * to record and possibly check old_oid: */ - if (refs_read_ref_full(&refs->base, - referent.buf, 0, - &lock->old_oid, NULL)) { + int ignore_errno; + if (!refs_resolve_ref_unsafe(&refs->base, + referent.buf, 0, + &lock->old_oid, NULL, + &ignore_errno)) { if (update->flags & REF_HAVE_OLD) { strbuf_addf(err, "cannot lock ref '%s': " "error reading reference", @@ -3085,7 +3132,6 @@ static int files_reflog_expire(struct ref_store *ref_store, struct strbuf log_file_sb = STRBUF_INIT; char *log_file; int status = 0; - int type; struct strbuf err = STRBUF_INIT; const struct object_id *oid; @@ -3099,7 +3145,7 @@ static int files_reflog_expire(struct ref_store *ref_store, * reference itself, plus we might need to update the * reference if --updateref was specified: */ - lock = lock_ref_oid_basic(refs, refname, &type, &err); + lock = lock_ref_oid_basic(refs, refname, &err); if (!lock) { error("cannot lock ref '%s': %s", refname, err.buf); strbuf_release(&err); @@ -3161,9 +3207,20 @@ static int files_reflog_expire(struct ref_store *ref_store, * a reference if there are no remaining reflog * entries. */ - int update = (flags & EXPIRE_REFLOGS_UPDATE_REF) && - !(type & REF_ISSYMREF) && - !is_null_oid(&cb.last_kept_oid); + int update = 0; + + if ((flags & EXPIRE_REFLOGS_UPDATE_REF) && + !is_null_oid(&cb.last_kept_oid)) { + int ignore_errno; + int type; + const char *ref; + + ref = refs_resolve_ref_unsafe(&refs->base, refname, + RESOLVE_REF_NO_RECURSE, + NULL, &type, + &ignore_errno); + update = !!(ref && !(type & REF_ISSYMREF)); + } if (close_lock_file_gently(&reflog_lock)) { status |= error("couldn't write %s: %s", log_file, diff --git a/refs/packed-backend.c b/refs/packed-backend.c index 47247a1491..9da932a540 100644 --- a/refs/packed-backend.c +++ b/refs/packed-backend.c @@ -193,13 +193,15 @@ static int release_snapshot(struct snapshot *snapshot) } } -struct ref_store *packed_ref_store_create(const char *path, +struct ref_store *packed_ref_store_create(struct repository *repo, + const char *path, unsigned int store_flags) { struct packed_ref_store *refs = xcalloc(1, sizeof(*refs)); struct ref_store *ref_store = (struct ref_store *)refs; base_ref_store_init(ref_store, &refs_be_packed); + ref_store->repo = repo; ref_store->gitdir = xstrdup(path); refs->store_flags = store_flags; @@ -776,6 +778,7 @@ struct packed_ref_iterator { struct object_id oid, peeled; struct strbuf refname_buf; + struct repository *repo; unsigned int flags; }; @@ -864,8 +867,8 @@ static int packed_ref_iterator_advance(struct ref_iterator *ref_iterator) continue; if (!(iter->flags & DO_FOR_EACH_INCLUDE_BROKEN) && - !ref_resolves_to_object(iter->base.refname, &iter->oid, - iter->flags)) + !ref_resolves_to_object(iter->base.refname, iter->repo, + &iter->oid, iter->flags)) continue; return ITER_OK; @@ -883,6 +886,9 @@ static int packed_ref_iterator_peel(struct ref_iterator *ref_iterator, struct packed_ref_iterator *iter = (struct packed_ref_iterator *)ref_iterator; + if (iter->repo != the_repository) + BUG("peeling for non-the_repository is not supported"); + if ((iter->base.flags & REF_KNOWS_PEELED)) { oidcpy(peeled, &iter->peeled); return is_null_oid(&iter->peeled) ? -1 : 0; @@ -954,6 +960,7 @@ static struct ref_iterator *packed_ref_iterator_begin( iter->base.oid = &iter->oid; + iter->repo = ref_store->repo; iter->flags = flags; if (prefix && *prefix) @@ -1347,6 +1354,7 @@ int is_packed_transaction_needed(struct ref_store *ref_store, ret = 0; for (i = 0; i < transaction->nr; i++) { struct ref_update *update = transaction->updates[i]; + int failure_errno; unsigned int type; struct object_id oid; @@ -1357,9 +1365,9 @@ int is_packed_transaction_needed(struct ref_store *ref_store, */ continue; - if (!refs_read_raw_ref(ref_store, update->refname, - &oid, &referent, &type) || - errno != ENOENT) { + if (!refs_read_raw_ref(ref_store, update->refname, &oid, + &referent, &type, &failure_errno) || + failure_errno != ENOENT) { /* * We have to actually delete that reference * -> this transaction is needed. diff --git a/refs/packed-backend.h b/refs/packed-backend.h index a01a0aff9c..f61a73ec25 100644 --- a/refs/packed-backend.h +++ b/refs/packed-backend.h @@ -1,6 +1,7 @@ #ifndef REFS_PACKED_BACKEND_H #define REFS_PACKED_BACKEND_H +struct repository; struct ref_transaction; /* @@ -12,7 +13,8 @@ struct ref_transaction; * even among packed refs. */ -struct ref_store *packed_ref_store_create(const char *path, +struct ref_store *packed_ref_store_create(struct repository *repo, + const char *path, unsigned int store_flags); /* diff --git a/refs/ref-cache.c b/refs/ref-cache.c index a5ad8a39fb..be4aa5e098 100644 --- a/refs/ref-cache.c +++ b/refs/ref-cache.c @@ -378,6 +378,8 @@ struct cache_ref_iterator { * on from there.) */ struct cache_ref_iterator_level *levels; + + struct repository *repo; }; static int cache_ref_iterator_advance(struct ref_iterator *ref_iterator) @@ -434,6 +436,11 @@ static int cache_ref_iterator_advance(struct ref_iterator *ref_iterator) static int cache_ref_iterator_peel(struct ref_iterator *ref_iterator, struct object_id *peeled) { + struct cache_ref_iterator *iter = + (struct cache_ref_iterator *)ref_iterator; + + if (iter->repo != the_repository) + BUG("peeling for non-the_repository is not supported"); return peel_object(ref_iterator->oid, peeled) ? -1 : 0; } @@ -456,6 +463,7 @@ static struct ref_iterator_vtable cache_ref_iterator_vtable = { struct ref_iterator *cache_ref_iterator_begin(struct ref_cache *cache, const char *prefix, + struct repository *repo, int prime_dir) { struct ref_dir *dir; @@ -490,5 +498,7 @@ struct ref_iterator *cache_ref_iterator_begin(struct ref_cache *cache, level->prefix_state = PREFIX_CONTAINS_DIR; } + iter->repo = repo; + return ref_iterator; } diff --git a/refs/ref-cache.h b/refs/ref-cache.h index 5c042ae718..850d9d3744 100644 --- a/refs/ref-cache.h +++ b/refs/ref-cache.h @@ -214,6 +214,7 @@ struct ref_entry *find_ref_entry(struct ref_dir *dir, const char *refname); */ struct ref_iterator *cache_ref_iterator_begin(struct ref_cache *cache, const char *prefix, + struct repository *repo, int prime_dir); #endif /* REFS_REF_CACHE_H */ diff --git a/refs/refs-internal.h b/refs/refs-internal.h index 72746407fc..fb2c58ce3b 100644 --- a/refs/refs-internal.h +++ b/refs/refs-internal.h @@ -66,6 +66,7 @@ int refname_is_safe(const char *refname); * referred-to object does not exist, emit a warning and return false. */ int ref_resolves_to_object(const char *refname, + struct repository *repo, const struct object_id *oid, unsigned int flags); @@ -149,9 +150,9 @@ struct ref_update { const char refname[FLEX_ARRAY]; }; -int refs_read_raw_ref(struct ref_store *ref_store, - const char *refname, struct object_id *oid, - struct strbuf *referent, unsigned int *type); +int refs_read_raw_ref(struct ref_store *ref_store, const char *refname, + struct object_id *oid, struct strbuf *referent, + unsigned int *type, int *failure_errno); /* * Write an error to `err` and return a nonzero value iff the same @@ -228,20 +229,6 @@ const char *find_descendant_ref(const char *dirname, const struct string_list *extras, const struct string_list *skip); -/* - * Check whether an attempt to rename old_refname to new_refname would - * cause a D/F conflict with any existing reference (other than - * possibly old_refname). If there would be a conflict, emit an error - * message and return false; otherwise, return true. - * - * Note that this function is not safe against all races with other - * processes (though rename_ref() catches some races that might get by - * this check). - */ -int refs_rename_ref_available(struct ref_store *refs, - const char *old_refname, - const char *new_refname); - /* We allow "recursive" symbolic refs. Only within reason, though */ #define SYMREF_MAXDEPTH 5 @@ -539,7 +526,8 @@ struct ref_store; * should call base_ref_store_init() to initialize the shared part of * the ref_store and to record the ref_store for later lookup. */ -typedef struct ref_store *ref_store_init_fn(const char *gitdir, +typedef struct ref_store *ref_store_init_fn(struct repository *repo, + const char *gitdir, unsigned int flags); typedef int ref_init_db_fn(struct ref_store *refs, struct strbuf *err); @@ -701,15 +689,22 @@ struct ref_store { /* The backend describing this ref_store's storage scheme: */ const struct ref_storage_be *be; - /* The gitdir that this ref_store applies to: */ + struct repository *repo; + + /* + * The gitdir that this ref_store applies to. Note that this is not + * necessarily repo->gitdir if the repo has multiple worktrees. + */ char *gitdir; }; /* - * Parse contents of a loose ref file. + * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for + * invalid contents. */ int parse_loose_ref_contents(const char *buf, struct object_id *oid, - struct strbuf *referent, unsigned int *type); + struct strbuf *referent, unsigned int *type, + int *failure_errno); /* * Fill in the generic part of refs and add it to our collection of diff --git a/reftable/LICENSE b/reftable/LICENSE new file mode 100644 index 0000000000..402e0f9356 --- /dev/null +++ b/reftable/LICENSE @@ -0,0 +1,31 @@ +BSD License + +Copyright (c) 2020, Google LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Google LLC nor the names of its contributors may +be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/reftable/basics.c b/reftable/basics.c new file mode 100644 index 0000000000..f761e48028 --- /dev/null +++ b/reftable/basics.c @@ -0,0 +1,128 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "basics.h" + +void put_be24(uint8_t *out, uint32_t i) +{ + out[0] = (uint8_t)((i >> 16) & 0xff); + out[1] = (uint8_t)((i >> 8) & 0xff); + out[2] = (uint8_t)(i & 0xff); +} + +uint32_t get_be24(uint8_t *in) +{ + return (uint32_t)(in[0]) << 16 | (uint32_t)(in[1]) << 8 | + (uint32_t)(in[2]); +} + +void put_be16(uint8_t *out, uint16_t i) +{ + out[0] = (uint8_t)((i >> 8) & 0xff); + out[1] = (uint8_t)(i & 0xff); +} + +int binsearch(size_t sz, int (*f)(size_t k, void *args), void *args) +{ + size_t lo = 0; + size_t hi = sz; + + /* Invariants: + * + * (hi == sz) || f(hi) == true + * (lo == 0 && f(0) == true) || fi(lo) == false + */ + while (hi - lo > 1) { + size_t mid = lo + (hi - lo) / 2; + + if (f(mid, args)) + hi = mid; + else + lo = mid; + } + + if (lo) + return hi; + + return f(0, args) ? 0 : 1; +} + +void free_names(char **a) +{ + char **p; + if (!a) { + return; + } + for (p = a; *p; p++) { + reftable_free(*p); + } + reftable_free(a); +} + +int names_length(char **names) +{ + char **p = names; + for (; *p; p++) { + /* empty */ + } + return p - names; +} + +void parse_names(char *buf, int size, char ***namesp) +{ + char **names = NULL; + size_t names_cap = 0; + size_t names_len = 0; + + char *p = buf; + char *end = buf + size; + while (p < end) { + char *next = strchr(p, '\n'); + if (next && next < end) { + *next = 0; + } else { + next = end; + } + if (p < next) { + if (names_len == names_cap) { + names_cap = 2 * names_cap + 1; + names = reftable_realloc( + names, names_cap * sizeof(*names)); + } + names[names_len++] = xstrdup(p); + } + p = next + 1; + } + + names = reftable_realloc(names, (names_len + 1) * sizeof(*names)); + names[names_len] = NULL; + *namesp = names; +} + +int names_equal(char **a, char **b) +{ + int i = 0; + for (; a[i] && b[i]; i++) { + if (strcmp(a[i], b[i])) { + return 0; + } + } + + return a[i] == b[i]; +} + +int common_prefix_size(struct strbuf *a, struct strbuf *b) +{ + int p = 0; + for (; p < a->len && p < b->len; p++) { + if (a->buf[p] != b->buf[p]) + break; + } + + return p; +} diff --git a/reftable/basics.h b/reftable/basics.h new file mode 100644 index 0000000000..096b36862b --- /dev/null +++ b/reftable/basics.h @@ -0,0 +1,60 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BASICS_H +#define BASICS_H + +/* + * miscellaneous utilities that are not provided by Git. + */ + +#include "system.h" + +/* Bigendian en/decoding of integers */ + +void put_be24(uint8_t *out, uint32_t i); +uint32_t get_be24(uint8_t *in); +void put_be16(uint8_t *out, uint16_t i); + +/* + * find smallest index i in [0, sz) at which f(i) is true, assuming + * that f is ascending. Return sz if f(i) is false for all indices. + * + * Contrary to bsearch(3), this returns something useful if the argument is not + * found. + */ +int binsearch(size_t sz, int (*f)(size_t k, void *args), void *args); + +/* + * Frees a NULL terminated array of malloced strings. The array itself is also + * freed. + */ +void free_names(char **a); + +/* parse a newline separated list of names. `size` is the length of the buffer, + * without terminating '\0'. Empty names are discarded. */ +void parse_names(char *buf, int size, char ***namesp); + +/* compares two NULL-terminated arrays of strings. */ +int names_equal(char **a, char **b); + +/* returns the array size of a NULL-terminated array of strings. */ +int names_length(char **names); + +/* Allocation routines; they invoke the functions set through + * reftable_set_alloc() */ +void *reftable_malloc(size_t sz); +void *reftable_realloc(void *p, size_t sz); +void reftable_free(void *p); +void *reftable_calloc(size_t sz); + +/* Find the longest shared prefix size of `a` and `b` */ +struct strbuf; +int common_prefix_size(struct strbuf *a, struct strbuf *b); + +#endif diff --git a/reftable/basics_test.c b/reftable/basics_test.c new file mode 100644 index 0000000000..1fcd229725 --- /dev/null +++ b/reftable/basics_test.c @@ -0,0 +1,98 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" + +#include "basics.h" +#include "test_framework.h" +#include "reftable-tests.h" + +struct binsearch_args { + int key; + int *arr; +}; + +static int binsearch_func(size_t i, void *void_args) +{ + struct binsearch_args *args = void_args; + + return args->key < args->arr[i]; +} + +static void test_binsearch(void) +{ + int arr[] = { 2, 4, 6, 8, 10 }; + size_t sz = ARRAY_SIZE(arr); + struct binsearch_args args = { + .arr = arr, + }; + + int i = 0; + for (i = 1; i < 11; i++) { + int res; + args.key = i; + res = binsearch(sz, &binsearch_func, &args); + + if (res < sz) { + EXPECT(args.key < arr[res]); + if (res > 0) { + EXPECT(args.key >= arr[res - 1]); + } + } else { + EXPECT(args.key == 10 || args.key == 11); + } + } +} + +static void test_names_length(void) +{ + char *a[] = { "a", "b", NULL }; + EXPECT(names_length(a) == 2); +} + +static void test_parse_names_normal(void) +{ + char in[] = "a\nb\n"; + char **out = NULL; + parse_names(in, strlen(in), &out); + EXPECT(!strcmp(out[0], "a")); + EXPECT(!strcmp(out[1], "b")); + EXPECT(!out[2]); + free_names(out); +} + +static void test_parse_names_drop_empty(void) +{ + char in[] = "a\n\n"; + char **out = NULL; + parse_names(in, strlen(in), &out); + EXPECT(!strcmp(out[0], "a")); + EXPECT(!out[1]); + free_names(out); +} + +static void test_common_prefix(void) +{ + struct strbuf s1 = STRBUF_INIT; + struct strbuf s2 = STRBUF_INIT; + strbuf_addstr(&s1, "abcdef"); + strbuf_addstr(&s2, "abc"); + EXPECT(common_prefix_size(&s1, &s2) == 3); + strbuf_release(&s1); + strbuf_release(&s2); +} + +int basics_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_common_prefix); + RUN_TEST(test_parse_names_normal); + RUN_TEST(test_parse_names_drop_empty); + RUN_TEST(test_binsearch); + RUN_TEST(test_names_length); + return 0; +} diff --git a/reftable/block.c b/reftable/block.c new file mode 100644 index 0000000000..855e3f5c94 --- /dev/null +++ b/reftable/block.c @@ -0,0 +1,437 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "block.h" + +#include "blocksource.h" +#include "constants.h" +#include "record.h" +#include "reftable-error.h" +#include "system.h" +#include <zlib.h> + +int header_size(int version) +{ + switch (version) { + case 1: + return 24; + case 2: + return 28; + } + abort(); +} + +int footer_size(int version) +{ + switch (version) { + case 1: + return 68; + case 2: + return 72; + } + abort(); +} + +static int block_writer_register_restart(struct block_writer *w, int n, + int is_restart, struct strbuf *key) +{ + int rlen = w->restart_len; + if (rlen >= MAX_RESTARTS) { + is_restart = 0; + } + + if (is_restart) { + rlen++; + } + if (2 + 3 * rlen + n > w->block_size - w->next) + return -1; + if (is_restart) { + if (w->restart_len == w->restart_cap) { + w->restart_cap = w->restart_cap * 2 + 1; + w->restarts = reftable_realloc( + w->restarts, sizeof(uint32_t) * w->restart_cap); + } + + w->restarts[w->restart_len++] = w->next; + } + + w->next += n; + + strbuf_reset(&w->last_key); + strbuf_addbuf(&w->last_key, key); + w->entries++; + return 0; +} + +void block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *buf, + uint32_t block_size, uint32_t header_off, int hash_size) +{ + bw->buf = buf; + bw->hash_size = hash_size; + bw->block_size = block_size; + bw->header_off = header_off; + bw->buf[header_off] = typ; + bw->next = header_off + 4; + bw->restart_interval = 16; + bw->entries = 0; + bw->restart_len = 0; + bw->last_key.len = 0; +} + +uint8_t block_writer_type(struct block_writer *bw) +{ + return bw->buf[bw->header_off]; +} + +/* adds the reftable_record to the block. Returns -1 if it does not fit, 0 on + success */ +int block_writer_add(struct block_writer *w, struct reftable_record *rec) +{ + struct strbuf empty = STRBUF_INIT; + struct strbuf last = + w->entries % w->restart_interval == 0 ? empty : w->last_key; + struct string_view out = { + .buf = w->buf + w->next, + .len = w->block_size - w->next, + }; + + struct string_view start = out; + + int is_restart = 0; + struct strbuf key = STRBUF_INIT; + int n = 0; + + reftable_record_key(rec, &key); + n = reftable_encode_key(&is_restart, out, last, key, + reftable_record_val_type(rec)); + if (n < 0) + goto done; + string_view_consume(&out, n); + + n = reftable_record_encode(rec, out, w->hash_size); + if (n < 0) + goto done; + string_view_consume(&out, n); + + if (block_writer_register_restart(w, start.len - out.len, is_restart, + &key) < 0) + goto done; + + strbuf_release(&key); + return 0; + +done: + strbuf_release(&key); + return -1; +} + +int block_writer_finish(struct block_writer *w) +{ + int i; + for (i = 0; i < w->restart_len; i++) { + put_be24(w->buf + w->next, w->restarts[i]); + w->next += 3; + } + + put_be16(w->buf + w->next, w->restart_len); + w->next += 2; + put_be24(w->buf + 1 + w->header_off, w->next); + + if (block_writer_type(w) == BLOCK_TYPE_LOG) { + int block_header_skip = 4 + w->header_off; + uLongf src_len = w->next - block_header_skip; + uLongf dest_cap = src_len * 1.001 + 12; + + uint8_t *compressed = reftable_malloc(dest_cap); + while (1) { + uLongf out_dest_len = dest_cap; + int zresult = compress2(compressed, &out_dest_len, + w->buf + block_header_skip, + src_len, 9); + if (zresult == Z_BUF_ERROR && dest_cap < LONG_MAX) { + dest_cap *= 2; + compressed = + reftable_realloc(compressed, dest_cap); + if (compressed) + continue; + } + + if (Z_OK != zresult) { + reftable_free(compressed); + return REFTABLE_ZLIB_ERROR; + } + + memcpy(w->buf + block_header_skip, compressed, + out_dest_len); + w->next = out_dest_len + block_header_skip; + reftable_free(compressed); + break; + } + } + return w->next; +} + +uint8_t block_reader_type(struct block_reader *r) +{ + return r->block.data[r->header_off]; +} + +int block_reader_init(struct block_reader *br, struct reftable_block *block, + uint32_t header_off, uint32_t table_block_size, + int hash_size) +{ + uint32_t full_block_size = table_block_size; + uint8_t typ = block->data[header_off]; + uint32_t sz = get_be24(block->data + header_off + 1); + + uint16_t restart_count = 0; + uint32_t restart_start = 0; + uint8_t *restart_bytes = NULL; + + if (!reftable_is_block_type(typ)) + return REFTABLE_FORMAT_ERROR; + + if (typ == BLOCK_TYPE_LOG) { + int block_header_skip = 4 + header_off; + uLongf dst_len = sz - block_header_skip; /* total size of dest + buffer. */ + uLongf src_len = block->len - block_header_skip; + /* Log blocks specify the *uncompressed* size in their header. + */ + uint8_t *uncompressed = reftable_malloc(sz); + + /* Copy over the block header verbatim. It's not compressed. */ + memcpy(uncompressed, block->data, block_header_skip); + + /* Uncompress */ + if (Z_OK != + uncompress2(uncompressed + block_header_skip, &dst_len, + block->data + block_header_skip, &src_len)) { + reftable_free(uncompressed); + return REFTABLE_ZLIB_ERROR; + } + + if (dst_len + block_header_skip != sz) + return REFTABLE_FORMAT_ERROR; + + /* We're done with the input data. */ + reftable_block_done(block); + block->data = uncompressed; + block->len = sz; + block->source = malloc_block_source(); + full_block_size = src_len + block_header_skip; + } else if (full_block_size == 0) { + full_block_size = sz; + } else if (sz < full_block_size && sz < block->len && + block->data[sz] != 0) { + /* If the block is smaller than the full block size, it is + padded (data followed by '\0') or the next block is + unaligned. */ + full_block_size = sz; + } + + restart_count = get_be16(block->data + sz - 2); + restart_start = sz - 2 - 3 * restart_count; + restart_bytes = block->data + restart_start; + + /* transfer ownership. */ + br->block = *block; + block->data = NULL; + block->len = 0; + + br->hash_size = hash_size; + br->block_len = restart_start; + br->full_block_size = full_block_size; + br->header_off = header_off; + br->restart_count = restart_count; + br->restart_bytes = restart_bytes; + + return 0; +} + +static uint32_t block_reader_restart_offset(struct block_reader *br, int i) +{ + return get_be24(br->restart_bytes + 3 * i); +} + +void block_reader_start(struct block_reader *br, struct block_iter *it) +{ + it->br = br; + strbuf_reset(&it->last_key); + it->next_off = br->header_off + 4; +} + +struct restart_find_args { + int error; + struct strbuf key; + struct block_reader *r; +}; + +static int restart_key_less(size_t idx, void *args) +{ + struct restart_find_args *a = args; + uint32_t off = block_reader_restart_offset(a->r, idx); + struct string_view in = { + .buf = a->r->block.data + off, + .len = a->r->block_len - off, + }; + + /* the restart key is verbatim in the block, so this could avoid the + alloc for decoding the key */ + struct strbuf rkey = STRBUF_INIT; + struct strbuf last_key = STRBUF_INIT; + uint8_t unused_extra; + int n = reftable_decode_key(&rkey, &unused_extra, last_key, in); + int result; + if (n < 0) { + a->error = 1; + return -1; + } + + result = strbuf_cmp(&a->key, &rkey); + strbuf_release(&rkey); + return result; +} + +void block_iter_copy_from(struct block_iter *dest, struct block_iter *src) +{ + dest->br = src->br; + dest->next_off = src->next_off; + strbuf_reset(&dest->last_key); + strbuf_addbuf(&dest->last_key, &src->last_key); +} + +int block_iter_next(struct block_iter *it, struct reftable_record *rec) +{ + struct string_view in = { + .buf = it->br->block.data + it->next_off, + .len = it->br->block_len - it->next_off, + }; + struct string_view start = in; + struct strbuf key = STRBUF_INIT; + uint8_t extra = 0; + int n = 0; + + if (it->next_off >= it->br->block_len) + return 1; + + n = reftable_decode_key(&key, &extra, it->last_key, in); + if (n < 0) + return -1; + + string_view_consume(&in, n); + n = reftable_record_decode(rec, key, extra, in, it->br->hash_size); + if (n < 0) + return -1; + string_view_consume(&in, n); + + strbuf_reset(&it->last_key); + strbuf_addbuf(&it->last_key, &key); + it->next_off += start.len - in.len; + strbuf_release(&key); + return 0; +} + +int block_reader_first_key(struct block_reader *br, struct strbuf *key) +{ + struct strbuf empty = STRBUF_INIT; + int off = br->header_off + 4; + struct string_view in = { + .buf = br->block.data + off, + .len = br->block_len - off, + }; + + uint8_t extra = 0; + int n = reftable_decode_key(key, &extra, empty, in); + if (n < 0) + return n; + + return 0; +} + +int block_iter_seek(struct block_iter *it, struct strbuf *want) +{ + return block_reader_seek(it->br, it, want); +} + +void block_iter_close(struct block_iter *it) +{ + strbuf_release(&it->last_key); +} + +int block_reader_seek(struct block_reader *br, struct block_iter *it, + struct strbuf *want) +{ + struct restart_find_args args = { + .key = *want, + .r = br, + }; + struct reftable_record rec = reftable_new_record(block_reader_type(br)); + struct strbuf key = STRBUF_INIT; + int err = 0; + struct block_iter next = { + .last_key = STRBUF_INIT, + }; + + int i = binsearch(br->restart_count, &restart_key_less, &args); + if (args.error) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + it->br = br; + if (i > 0) { + i--; + it->next_off = block_reader_restart_offset(br, i); + } else { + it->next_off = br->header_off + 4; + } + + /* We're looking for the last entry less/equal than the wanted key, so + we have to go one entry too far and then back up. + */ + while (1) { + block_iter_copy_from(&next, it); + err = block_iter_next(&next, &rec); + if (err < 0) + goto done; + + reftable_record_key(&rec, &key); + if (err > 0 || strbuf_cmp(&key, want) >= 0) { + err = 0; + goto done; + } + + block_iter_copy_from(it, &next); + } + +done: + strbuf_release(&key); + strbuf_release(&next.last_key); + reftable_record_destroy(&rec); + + return err; +} + +void block_writer_release(struct block_writer *bw) +{ + FREE_AND_NULL(bw->restarts); + strbuf_release(&bw->last_key); + /* the block is not owned. */ +} + +void reftable_block_done(struct reftable_block *blockp) +{ + struct reftable_block_source source = blockp->source; + if (blockp && source.ops) + source.ops->return_block(source.arg, blockp); + blockp->data = NULL; + blockp->len = 0; + blockp->source.ops = NULL; + blockp->source.arg = NULL; +} diff --git a/reftable/block.h b/reftable/block.h new file mode 100644 index 0000000000..e207706a64 --- /dev/null +++ b/reftable/block.h @@ -0,0 +1,127 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BLOCK_H +#define BLOCK_H + +#include "basics.h" +#include "record.h" +#include "reftable-blocksource.h" + +/* + * Writes reftable blocks. The block_writer is reused across blocks to minimize + * allocation overhead. + */ +struct block_writer { + uint8_t *buf; + uint32_t block_size; + + /* Offset ofof the global header. Nonzero in the first block only. */ + uint32_t header_off; + + /* How often to restart keys. */ + int restart_interval; + int hash_size; + + /* Offset of next uint8_t to write. */ + uint32_t next; + uint32_t *restarts; + uint32_t restart_len; + uint32_t restart_cap; + + struct strbuf last_key; + int entries; +}; + +/* + * initializes the blockwriter to write `typ` entries, using `buf` as temporary + * storage. `buf` is not owned by the block_writer. */ +void block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *buf, + uint32_t block_size, uint32_t header_off, int hash_size); + +/* returns the block type (eg. 'r' for ref records. */ +uint8_t block_writer_type(struct block_writer *bw); + +/* appends the record, or -1 if it doesn't fit. */ +int block_writer_add(struct block_writer *w, struct reftable_record *rec); + +/* appends the key restarts, and compress the block if necessary. */ +int block_writer_finish(struct block_writer *w); + +/* clears out internally allocated block_writer members. */ +void block_writer_release(struct block_writer *bw); + +/* Read a block. */ +struct block_reader { + /* offset of the block header; nonzero for the first block in a + * reftable. */ + uint32_t header_off; + + /* the memory block */ + struct reftable_block block; + int hash_size; + + /* size of the data, excluding restart data. */ + uint32_t block_len; + uint8_t *restart_bytes; + uint16_t restart_count; + + /* size of the data in the file. For log blocks, this is the compressed + * size. */ + uint32_t full_block_size; +}; + +/* Iterate over entries in a block */ +struct block_iter { + /* offset within the block of the next entry to read. */ + uint32_t next_off; + struct block_reader *br; + + /* key for last entry we read. */ + struct strbuf last_key; +}; + +/* initializes a block reader. */ +int block_reader_init(struct block_reader *br, struct reftable_block *bl, + uint32_t header_off, uint32_t table_block_size, + int hash_size); + +/* Position `it` at start of the block */ +void block_reader_start(struct block_reader *br, struct block_iter *it); + +/* Position `it` to the `want` key in the block */ +int block_reader_seek(struct block_reader *br, struct block_iter *it, + struct strbuf *want); + +/* Returns the block type (eg. 'r' for refs) */ +uint8_t block_reader_type(struct block_reader *r); + +/* Decodes the first key in the block */ +int block_reader_first_key(struct block_reader *br, struct strbuf *key); + +void block_iter_copy_from(struct block_iter *dest, struct block_iter *src); + +/* return < 0 for error, 0 for OK, > 0 for EOF. */ +int block_iter_next(struct block_iter *it, struct reftable_record *rec); + +/* Seek to `want` with in the block pointed to by `it` */ +int block_iter_seek(struct block_iter *it, struct strbuf *want); + +/* deallocate memory for `it`. The block reader and its block is left intact. */ +void block_iter_close(struct block_iter *it); + +/* size of file header, depending on format version */ +int header_size(int version); + +/* size of file footer, depending on format version */ +int footer_size(int version); + +/* returns a block to its source. */ +void reftable_block_done(struct reftable_block *ret); + +#endif diff --git a/reftable/block_test.c b/reftable/block_test.c new file mode 100644 index 0000000000..4b3ea262dc --- /dev/null +++ b/reftable/block_test.c @@ -0,0 +1,120 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "block.h" + +#include "system.h" +#include "blocksource.h" +#include "basics.h" +#include "constants.h" +#include "record.h" +#include "test_framework.h" +#include "reftable-tests.h" + +static void test_block_read_write(void) +{ + const int header_off = 21; /* random */ + char *names[30]; + const int N = ARRAY_SIZE(names); + const int block_size = 1024; + struct reftable_block block = { NULL }; + struct block_writer bw = { + .last_key = STRBUF_INIT, + }; + struct reftable_ref_record ref = { NULL }; + struct reftable_record rec = { NULL }; + int i = 0; + int n; + struct block_reader br = { 0 }; + struct block_iter it = { .last_key = STRBUF_INIT }; + int j = 0; + struct strbuf want = STRBUF_INIT; + + block.data = reftable_calloc(block_size); + block.len = block_size; + block.source = malloc_block_source(); + block_writer_init(&bw, BLOCK_TYPE_REF, block.data, block_size, + header_off, hash_size(GIT_SHA1_FORMAT_ID)); + reftable_record_from_ref(&rec, &ref); + + for (i = 0; i < N; i++) { + char name[100]; + uint8_t hash[GIT_SHA1_RAWSZ]; + snprintf(name, sizeof(name), "branch%02d", i); + memset(hash, i, sizeof(hash)); + + ref.refname = name; + ref.value_type = REFTABLE_REF_VAL1; + ref.value.val1 = hash; + + names[i] = xstrdup(name); + n = block_writer_add(&bw, &rec); + ref.refname = NULL; + ref.value_type = REFTABLE_REF_DELETION; + EXPECT(n == 0); + } + + n = block_writer_finish(&bw); + EXPECT(n > 0); + + block_writer_release(&bw); + + block_reader_init(&br, &block, header_off, block_size, GIT_SHA1_RAWSZ); + + block_reader_start(&br, &it); + + while (1) { + int r = block_iter_next(&it, &rec); + EXPECT(r >= 0); + if (r > 0) { + break; + } + EXPECT_STREQ(names[j], ref.refname); + j++; + } + + reftable_record_release(&rec); + block_iter_close(&it); + + for (i = 0; i < N; i++) { + struct block_iter it = { .last_key = STRBUF_INIT }; + strbuf_reset(&want); + strbuf_addstr(&want, names[i]); + + n = block_reader_seek(&br, &it, &want); + EXPECT(n == 0); + + n = block_iter_next(&it, &rec); + EXPECT(n == 0); + + EXPECT_STREQ(names[i], ref.refname); + + want.len--; + n = block_reader_seek(&br, &it, &want); + EXPECT(n == 0); + + n = block_iter_next(&it, &rec); + EXPECT(n == 0); + EXPECT_STREQ(names[10 * (i / 10)], ref.refname); + + block_iter_close(&it); + } + + reftable_record_release(&rec); + reftable_block_done(&br.block); + strbuf_release(&want); + for (i = 0; i < N; i++) { + reftable_free(names[i]); + } +} + +int block_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_block_read_write); + return 0; +} diff --git a/reftable/blocksource.c b/reftable/blocksource.c new file mode 100644 index 0000000000..0044eecd9a --- /dev/null +++ b/reftable/blocksource.c @@ -0,0 +1,148 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" + +#include "basics.h" +#include "blocksource.h" +#include "reftable-blocksource.h" +#include "reftable-error.h" + +static void strbuf_return_block(void *b, struct reftable_block *dest) +{ + memset(dest->data, 0xff, dest->len); + reftable_free(dest->data); +} + +static void strbuf_close(void *b) +{ +} + +static int strbuf_read_block(void *v, struct reftable_block *dest, uint64_t off, + uint32_t size) +{ + struct strbuf *b = v; + assert(off + size <= b->len); + dest->data = reftable_calloc(size); + memcpy(dest->data, b->buf + off, size); + dest->len = size; + return size; +} + +static uint64_t strbuf_size(void *b) +{ + return ((struct strbuf *)b)->len; +} + +static struct reftable_block_source_vtable strbuf_vtable = { + .size = &strbuf_size, + .read_block = &strbuf_read_block, + .return_block = &strbuf_return_block, + .close = &strbuf_close, +}; + +void block_source_from_strbuf(struct reftable_block_source *bs, + struct strbuf *buf) +{ + assert(!bs->ops); + bs->ops = &strbuf_vtable; + bs->arg = buf; +} + +static void malloc_return_block(void *b, struct reftable_block *dest) +{ + memset(dest->data, 0xff, dest->len); + reftable_free(dest->data); +} + +static struct reftable_block_source_vtable malloc_vtable = { + .return_block = &malloc_return_block, +}; + +static struct reftable_block_source malloc_block_source_instance = { + .ops = &malloc_vtable, +}; + +struct reftable_block_source malloc_block_source(void) +{ + return malloc_block_source_instance; +} + +struct file_block_source { + int fd; + uint64_t size; +}; + +static uint64_t file_size(void *b) +{ + return ((struct file_block_source *)b)->size; +} + +static void file_return_block(void *b, struct reftable_block *dest) +{ + memset(dest->data, 0xff, dest->len); + reftable_free(dest->data); +} + +static void file_close(void *b) +{ + int fd = ((struct file_block_source *)b)->fd; + if (fd > 0) { + close(fd); + ((struct file_block_source *)b)->fd = 0; + } + + reftable_free(b); +} + +static int file_read_block(void *v, struct reftable_block *dest, uint64_t off, + uint32_t size) +{ + struct file_block_source *b = v; + assert(off + size <= b->size); + dest->data = reftable_malloc(size); + if (pread(b->fd, dest->data, size, off) != size) + return -1; + dest->len = size; + return size; +} + +static struct reftable_block_source_vtable file_vtable = { + .size = &file_size, + .read_block = &file_read_block, + .return_block = &file_return_block, + .close = &file_close, +}; + +int reftable_block_source_from_file(struct reftable_block_source *bs, + const char *name) +{ + struct stat st = { 0 }; + int err = 0; + int fd = open(name, O_RDONLY); + struct file_block_source *p = NULL; + if (fd < 0) { + if (errno == ENOENT) { + return REFTABLE_NOT_EXIST_ERROR; + } + return -1; + } + + err = fstat(fd, &st); + if (err < 0) + return -1; + + p = reftable_calloc(sizeof(struct file_block_source)); + p->size = st.st_size; + p->fd = fd; + + assert(!bs->ops); + bs->ops = &file_vtable; + bs->arg = p; + return 0; +} diff --git a/reftable/blocksource.h b/reftable/blocksource.h new file mode 100644 index 0000000000..072e2727ad --- /dev/null +++ b/reftable/blocksource.h @@ -0,0 +1,22 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BLOCKSOURCE_H +#define BLOCKSOURCE_H + +#include "system.h" + +struct reftable_block_source; + +/* Create an in-memory block source for reading reftables */ +void block_source_from_strbuf(struct reftable_block_source *bs, + struct strbuf *buf); + +struct reftable_block_source malloc_block_source(void); + +#endif diff --git a/reftable/constants.h b/reftable/constants.h new file mode 100644 index 0000000000..5eee72c4c1 --- /dev/null +++ b/reftable/constants.h @@ -0,0 +1,21 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef CONSTANTS_H +#define CONSTANTS_H + +#define BLOCK_TYPE_LOG 'g' +#define BLOCK_TYPE_INDEX 'i' +#define BLOCK_TYPE_REF 'r' +#define BLOCK_TYPE_OBJ 'o' +#define BLOCK_TYPE_ANY 0 + +#define MAX_RESTARTS ((1 << 16) - 1) +#define DEFAULT_BLOCK_SIZE 4096 + +#endif diff --git a/reftable/dump.c b/reftable/dump.c new file mode 100644 index 0000000000..155953d1b8 --- /dev/null +++ b/reftable/dump.c @@ -0,0 +1,107 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "git-compat-util.h" +#include "hash.h" + +#include "reftable-blocksource.h" +#include "reftable-error.h" +#include "reftable-merged.h" +#include "reftable-record.h" +#include "reftable-tests.h" +#include "reftable-writer.h" +#include "reftable-iterator.h" +#include "reftable-reader.h" +#include "reftable-stack.h" +#include "reftable-generic.h" + +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> + +static int compact_stack(const char *stackdir) +{ + struct reftable_stack *stack = NULL; + struct reftable_write_options cfg = { 0 }; + + int err = reftable_new_stack(&stack, stackdir, cfg); + if (err < 0) + goto done; + + err = reftable_stack_compact_all(stack, NULL); + if (err < 0) + goto done; +done: + if (stack) { + reftable_stack_destroy(stack); + } + return err; +} + +static void print_help(void) +{ + printf("usage: dump [-cst] arg\n\n" + "options: \n" + " -c compact\n" + " -t dump table\n" + " -s dump stack\n" + " -6 sha256 hash format\n" + " -h this help\n" + "\n"); +} + +int reftable_dump_main(int argc, char *const *argv) +{ + int err = 0; + int opt_dump_table = 0; + int opt_dump_stack = 0; + int opt_compact = 0; + uint32_t opt_hash_id = GIT_SHA1_FORMAT_ID; + const char *arg = NULL, *argv0 = argv[0]; + + for (; argc > 1; argv++, argc--) + if (*argv[1] != '-') + break; + else if (!strcmp("-t", argv[1])) + opt_dump_table = 1; + else if (!strcmp("-6", argv[1])) + opt_hash_id = GIT_SHA256_FORMAT_ID; + else if (!strcmp("-s", argv[1])) + opt_dump_stack = 1; + else if (!strcmp("-c", argv[1])) + opt_compact = 1; + else if (!strcmp("-?", argv[1]) || !strcmp("-h", argv[1])) { + print_help(); + return 2; + } + + if (argc != 2) { + fprintf(stderr, "need argument\n"); + print_help(); + return 2; + } + + arg = argv[1]; + + if (opt_dump_table) { + err = reftable_reader_print_file(arg); + } else if (opt_dump_stack) { + err = reftable_stack_print_directory(arg, opt_hash_id); + } else if (opt_compact) { + err = compact_stack(arg); + } + + if (err < 0) { + fprintf(stderr, "%s: %s: %s\n", argv0, arg, + reftable_error_str(err)); + return 1; + } + return 0; +} diff --git a/reftable/error.c b/reftable/error.c new file mode 100644 index 0000000000..f6f16def92 --- /dev/null +++ b/reftable/error.c @@ -0,0 +1,41 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "reftable-error.h" + +#include <stdio.h> + +const char *reftable_error_str(int err) +{ + static char buf[250]; + switch (err) { + case REFTABLE_IO_ERROR: + return "I/O error"; + case REFTABLE_FORMAT_ERROR: + return "corrupt reftable file"; + case REFTABLE_NOT_EXIST_ERROR: + return "file does not exist"; + case REFTABLE_LOCK_ERROR: + return "data is outdated"; + case REFTABLE_API_ERROR: + return "misuse of the reftable API"; + case REFTABLE_ZLIB_ERROR: + return "zlib failure"; + case REFTABLE_NAME_CONFLICT: + return "file/directory conflict"; + case REFTABLE_EMPTY_TABLE_ERROR: + return "wrote empty table"; + case REFTABLE_REFNAME_ERROR: + return "invalid refname"; + case -1: + return "general error"; + default: + snprintf(buf, sizeof(buf), "unknown error code %d", err); + return buf; + } +} diff --git a/reftable/generic.c b/reftable/generic.c new file mode 100644 index 0000000000..7a8a738d86 --- /dev/null +++ b/reftable/generic.c @@ -0,0 +1,169 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "basics.h" +#include "record.h" +#include "generic.h" +#include "reftable-iterator.h" +#include "reftable-generic.h" + +int reftable_table_seek_ref(struct reftable_table *tab, + struct reftable_iterator *it, const char *name) +{ + struct reftable_ref_record ref = { + .refname = (char *)name, + }; + struct reftable_record rec = { NULL }; + reftable_record_from_ref(&rec, &ref); + return tab->ops->seek_record(tab->table_arg, it, &rec); +} + +int reftable_table_seek_log(struct reftable_table *tab, + struct reftable_iterator *it, const char *name) +{ + struct reftable_log_record log = { + .refname = (char *)name, + .update_index = ~((uint64_t)0), + }; + struct reftable_record rec = { NULL }; + reftable_record_from_log(&rec, &log); + return tab->ops->seek_record(tab->table_arg, it, &rec); +} + +int reftable_table_read_ref(struct reftable_table *tab, const char *name, + struct reftable_ref_record *ref) +{ + struct reftable_iterator it = { NULL }; + int err = reftable_table_seek_ref(tab, &it, name); + if (err) + goto done; + + err = reftable_iterator_next_ref(&it, ref); + if (err) + goto done; + + if (strcmp(ref->refname, name) || + reftable_ref_record_is_deletion(ref)) { + reftable_ref_record_release(ref); + err = 1; + goto done; + } + +done: + reftable_iterator_destroy(&it); + return err; +} + +int reftable_table_print(struct reftable_table *tab) { + struct reftable_iterator it = { NULL }; + struct reftable_ref_record ref = { NULL }; + struct reftable_log_record log = { NULL }; + uint32_t hash_id = reftable_table_hash_id(tab); + int err = reftable_table_seek_ref(tab, &it, ""); + if (err < 0) { + return err; + } + + while (1) { + err = reftable_iterator_next_ref(&it, &ref); + if (err > 0) { + break; + } + if (err < 0) { + return err; + } + reftable_ref_record_print(&ref, hash_id); + } + reftable_iterator_destroy(&it); + reftable_ref_record_release(&ref); + + err = reftable_table_seek_log(tab, &it, ""); + if (err < 0) { + return err; + } + while (1) { + err = reftable_iterator_next_log(&it, &log); + if (err > 0) { + break; + } + if (err < 0) { + return err; + } + reftable_log_record_print(&log, hash_id); + } + reftable_iterator_destroy(&it); + reftable_log_record_release(&log); + return 0; +} + +uint64_t reftable_table_max_update_index(struct reftable_table *tab) +{ + return tab->ops->max_update_index(tab->table_arg); +} + +uint64_t reftable_table_min_update_index(struct reftable_table *tab) +{ + return tab->ops->min_update_index(tab->table_arg); +} + +uint32_t reftable_table_hash_id(struct reftable_table *tab) +{ + return tab->ops->hash_id(tab->table_arg); +} + +void reftable_iterator_destroy(struct reftable_iterator *it) +{ + if (!it->ops) { + return; + } + it->ops->close(it->iter_arg); + it->ops = NULL; + FREE_AND_NULL(it->iter_arg); +} + +int reftable_iterator_next_ref(struct reftable_iterator *it, + struct reftable_ref_record *ref) +{ + struct reftable_record rec = { NULL }; + reftable_record_from_ref(&rec, ref); + return iterator_next(it, &rec); +} + +int reftable_iterator_next_log(struct reftable_iterator *it, + struct reftable_log_record *log) +{ + struct reftable_record rec = { NULL }; + reftable_record_from_log(&rec, log); + return iterator_next(it, &rec); +} + +int iterator_next(struct reftable_iterator *it, struct reftable_record *rec) +{ + return it->ops->next(it->iter_arg, rec); +} + +static int empty_iterator_next(void *arg, struct reftable_record *rec) +{ + return 1; +} + +static void empty_iterator_close(void *arg) +{ +} + +static struct reftable_iterator_vtable empty_vtable = { + .next = &empty_iterator_next, + .close = &empty_iterator_close, +}; + +void iterator_set_empty(struct reftable_iterator *it) +{ + assert(!it->ops); + it->iter_arg = NULL; + it->ops = &empty_vtable; +} diff --git a/reftable/generic.h b/reftable/generic.h new file mode 100644 index 0000000000..98886a0640 --- /dev/null +++ b/reftable/generic.h @@ -0,0 +1,32 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef GENERIC_H +#define GENERIC_H + +#include "record.h" +#include "reftable-generic.h" + +/* generic interface to reftables */ +struct reftable_table_vtable { + int (*seek_record)(void *tab, struct reftable_iterator *it, + struct reftable_record *); + uint32_t (*hash_id)(void *tab); + uint64_t (*min_update_index)(void *tab); + uint64_t (*max_update_index)(void *tab); +}; + +struct reftable_iterator_vtable { + int (*next)(void *iter_arg, struct reftable_record *rec); + void (*close)(void *iter_arg); +}; + +void iterator_set_empty(struct reftable_iterator *it); +int iterator_next(struct reftable_iterator *it, struct reftable_record *rec); + +#endif diff --git a/reftable/iter.c b/reftable/iter.c new file mode 100644 index 0000000000..93d04f735b --- /dev/null +++ b/reftable/iter.c @@ -0,0 +1,194 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "iter.h" + +#include "system.h" + +#include "block.h" +#include "generic.h" +#include "constants.h" +#include "reader.h" +#include "reftable-error.h" + +int iterator_is_null(struct reftable_iterator *it) +{ + return !it->ops; +} + +static void filtering_ref_iterator_close(void *iter_arg) +{ + struct filtering_ref_iterator *fri = iter_arg; + strbuf_release(&fri->oid); + reftable_iterator_destroy(&fri->it); +} + +static int filtering_ref_iterator_next(void *iter_arg, + struct reftable_record *rec) +{ + struct filtering_ref_iterator *fri = iter_arg; + struct reftable_ref_record *ref = rec->data; + int err = 0; + while (1) { + err = reftable_iterator_next_ref(&fri->it, ref); + if (err != 0) { + break; + } + + if (fri->double_check) { + struct reftable_iterator it = { NULL }; + + err = reftable_table_seek_ref(&fri->tab, &it, + ref->refname); + if (err == 0) { + err = reftable_iterator_next_ref(&it, ref); + } + + reftable_iterator_destroy(&it); + + if (err < 0) { + break; + } + + if (err > 0) { + continue; + } + } + + if (ref->value_type == REFTABLE_REF_VAL2 && + (!memcmp(fri->oid.buf, ref->value.val2.target_value, + fri->oid.len) || + !memcmp(fri->oid.buf, ref->value.val2.value, + fri->oid.len))) + return 0; + + if (ref->value_type == REFTABLE_REF_VAL1 && + !memcmp(fri->oid.buf, ref->value.val1, fri->oid.len)) { + return 0; + } + } + + reftable_ref_record_release(ref); + return err; +} + +static struct reftable_iterator_vtable filtering_ref_iterator_vtable = { + .next = &filtering_ref_iterator_next, + .close = &filtering_ref_iterator_close, +}; + +void iterator_from_filtering_ref_iterator(struct reftable_iterator *it, + struct filtering_ref_iterator *fri) +{ + assert(!it->ops); + it->iter_arg = fri; + it->ops = &filtering_ref_iterator_vtable; +} + +static void indexed_table_ref_iter_close(void *p) +{ + struct indexed_table_ref_iter *it = p; + block_iter_close(&it->cur); + reftable_block_done(&it->block_reader.block); + reftable_free(it->offsets); + strbuf_release(&it->oid); +} + +static int indexed_table_ref_iter_next_block(struct indexed_table_ref_iter *it) +{ + uint64_t off; + int err = 0; + if (it->offset_idx == it->offset_len) { + it->is_finished = 1; + return 1; + } + + reftable_block_done(&it->block_reader.block); + + off = it->offsets[it->offset_idx++]; + err = reader_init_block_reader(it->r, &it->block_reader, off, + BLOCK_TYPE_REF); + if (err < 0) { + return err; + } + if (err > 0) { + /* indexed block does not exist. */ + return REFTABLE_FORMAT_ERROR; + } + block_reader_start(&it->block_reader, &it->cur); + return 0; +} + +static int indexed_table_ref_iter_next(void *p, struct reftable_record *rec) +{ + struct indexed_table_ref_iter *it = p; + struct reftable_ref_record *ref = rec->data; + + while (1) { + int err = block_iter_next(&it->cur, rec); + if (err < 0) { + return err; + } + + if (err > 0) { + err = indexed_table_ref_iter_next_block(it); + if (err < 0) { + return err; + } + + if (it->is_finished) { + return 1; + } + continue; + } + /* BUG */ + if (!memcmp(it->oid.buf, ref->value.val2.target_value, + it->oid.len) || + !memcmp(it->oid.buf, ref->value.val2.value, it->oid.len)) { + return 0; + } + } +} + +int new_indexed_table_ref_iter(struct indexed_table_ref_iter **dest, + struct reftable_reader *r, uint8_t *oid, + int oid_len, uint64_t *offsets, int offset_len) +{ + struct indexed_table_ref_iter empty = INDEXED_TABLE_REF_ITER_INIT; + struct indexed_table_ref_iter *itr = + reftable_calloc(sizeof(struct indexed_table_ref_iter)); + int err = 0; + + *itr = empty; + itr->r = r; + strbuf_add(&itr->oid, oid, oid_len); + + itr->offsets = offsets; + itr->offset_len = offset_len; + + err = indexed_table_ref_iter_next_block(itr); + if (err < 0) { + reftable_free(itr); + } else { + *dest = itr; + } + return err; +} + +static struct reftable_iterator_vtable indexed_table_ref_iter_vtable = { + .next = &indexed_table_ref_iter_next, + .close = &indexed_table_ref_iter_close, +}; + +void iterator_from_indexed_table_ref_iter(struct reftable_iterator *it, + struct indexed_table_ref_iter *itr) +{ + assert(!it->ops); + it->iter_arg = itr; + it->ops = &indexed_table_ref_iter_vtable; +} diff --git a/reftable/iter.h b/reftable/iter.h new file mode 100644 index 0000000000..09eb0cbfa5 --- /dev/null +++ b/reftable/iter.h @@ -0,0 +1,69 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef ITER_H +#define ITER_H + +#include "system.h" +#include "block.h" +#include "record.h" + +#include "reftable-iterator.h" +#include "reftable-generic.h" + +/* Returns true for a zeroed out iterator, such as the one returned from + * iterator_destroy. */ +int iterator_is_null(struct reftable_iterator *it); + +/* iterator that produces only ref records that point to `oid` */ +struct filtering_ref_iterator { + int double_check; + struct reftable_table tab; + struct strbuf oid; + struct reftable_iterator it; +}; +#define FILTERING_REF_ITERATOR_INIT \ + { \ + .oid = STRBUF_INIT \ + } + +void iterator_from_filtering_ref_iterator(struct reftable_iterator *, + struct filtering_ref_iterator *); + +/* iterator that produces only ref records that point to `oid`, + * but using the object index. + */ +struct indexed_table_ref_iter { + struct reftable_reader *r; + struct strbuf oid; + + /* mutable */ + uint64_t *offsets; + + /* Points to the next offset to read. */ + int offset_idx; + int offset_len; + struct block_reader block_reader; + struct block_iter cur; + int is_finished; +}; + +#define INDEXED_TABLE_REF_ITER_INIT \ + { \ + .cur = { .last_key = STRBUF_INIT }, .oid = STRBUF_INIT, \ + } + +void iterator_from_indexed_table_ref_iter(struct reftable_iterator *it, + struct indexed_table_ref_iter *itr); + +/* Takes ownership of `offsets` */ +int new_indexed_table_ref_iter(struct indexed_table_ref_iter **dest, + struct reftable_reader *r, uint8_t *oid, + int oid_len, uint64_t *offsets, int offset_len); + +#endif diff --git a/reftable/merged.c b/reftable/merged.c new file mode 100644 index 0000000000..e5b53da6db --- /dev/null +++ b/reftable/merged.c @@ -0,0 +1,362 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "merged.h" + +#include "constants.h" +#include "iter.h" +#include "pq.h" +#include "reader.h" +#include "record.h" +#include "generic.h" +#include "reftable-merged.h" +#include "reftable-error.h" +#include "system.h" + +static int merged_iter_init(struct merged_iter *mi) +{ + int i = 0; + for (i = 0; i < mi->stack_len; i++) { + struct reftable_record rec = reftable_new_record(mi->typ); + int err = iterator_next(&mi->stack[i], &rec); + if (err < 0) { + return err; + } + + if (err > 0) { + reftable_iterator_destroy(&mi->stack[i]); + reftable_record_destroy(&rec); + } else { + struct pq_entry e = { + .rec = rec, + .index = i, + }; + merged_iter_pqueue_add(&mi->pq, e); + } + } + + return 0; +} + +static void merged_iter_close(void *p) +{ + struct merged_iter *mi = p; + int i = 0; + merged_iter_pqueue_release(&mi->pq); + for (i = 0; i < mi->stack_len; i++) { + reftable_iterator_destroy(&mi->stack[i]); + } + reftable_free(mi->stack); +} + +static int merged_iter_advance_nonnull_subiter(struct merged_iter *mi, + size_t idx) +{ + struct reftable_record rec = reftable_new_record(mi->typ); + struct pq_entry e = { + .rec = rec, + .index = idx, + }; + int err = iterator_next(&mi->stack[idx], &rec); + if (err < 0) + return err; + + if (err > 0) { + reftable_iterator_destroy(&mi->stack[idx]); + reftable_record_destroy(&rec); + return 0; + } + + merged_iter_pqueue_add(&mi->pq, e); + return 0; +} + +static int merged_iter_advance_subiter(struct merged_iter *mi, size_t idx) +{ + if (iterator_is_null(&mi->stack[idx])) + return 0; + return merged_iter_advance_nonnull_subiter(mi, idx); +} + +static int merged_iter_next_entry(struct merged_iter *mi, + struct reftable_record *rec) +{ + struct strbuf entry_key = STRBUF_INIT; + struct pq_entry entry = { 0 }; + int err = 0; + + if (merged_iter_pqueue_is_empty(mi->pq)) + return 1; + + entry = merged_iter_pqueue_remove(&mi->pq); + err = merged_iter_advance_subiter(mi, entry.index); + if (err < 0) + return err; + + /* + One can also use reftable as datacenter-local storage, where the ref + database is maintained in globally consistent database (eg. + CockroachDB or Spanner). In this scenario, replication delays together + with compaction may cause newer tables to contain older entries. In + such a deployment, the loop below must be changed to collect all + entries for the same key, and return new the newest one. + */ + reftable_record_key(&entry.rec, &entry_key); + while (!merged_iter_pqueue_is_empty(mi->pq)) { + struct pq_entry top = merged_iter_pqueue_top(mi->pq); + struct strbuf k = STRBUF_INIT; + int err = 0, cmp = 0; + + reftable_record_key(&top.rec, &k); + + cmp = strbuf_cmp(&k, &entry_key); + strbuf_release(&k); + + if (cmp > 0) { + break; + } + + merged_iter_pqueue_remove(&mi->pq); + err = merged_iter_advance_subiter(mi, top.index); + if (err < 0) { + return err; + } + reftable_record_destroy(&top.rec); + } + + reftable_record_copy_from(rec, &entry.rec, hash_size(mi->hash_id)); + reftable_record_destroy(&entry.rec); + strbuf_release(&entry_key); + return 0; +} + +static int merged_iter_next(struct merged_iter *mi, struct reftable_record *rec) +{ + while (1) { + int err = merged_iter_next_entry(mi, rec); + if (err == 0 && mi->suppress_deletions && + reftable_record_is_deletion(rec)) { + continue; + } + + return err; + } +} + +static int merged_iter_next_void(void *p, struct reftable_record *rec) +{ + struct merged_iter *mi = p; + if (merged_iter_pqueue_is_empty(mi->pq)) + return 1; + + return merged_iter_next(mi, rec); +} + +static struct reftable_iterator_vtable merged_iter_vtable = { + .next = &merged_iter_next_void, + .close = &merged_iter_close, +}; + +static void iterator_from_merged_iter(struct reftable_iterator *it, + struct merged_iter *mi) +{ + assert(!it->ops); + it->iter_arg = mi; + it->ops = &merged_iter_vtable; +} + +int reftable_new_merged_table(struct reftable_merged_table **dest, + struct reftable_table *stack, int n, + uint32_t hash_id) +{ + struct reftable_merged_table *m = NULL; + uint64_t last_max = 0; + uint64_t first_min = 0; + int i = 0; + for (i = 0; i < n; i++) { + uint64_t min = reftable_table_min_update_index(&stack[i]); + uint64_t max = reftable_table_max_update_index(&stack[i]); + + if (reftable_table_hash_id(&stack[i]) != hash_id) { + return REFTABLE_FORMAT_ERROR; + } + if (i == 0 || min < first_min) { + first_min = min; + } + if (i == 0 || max > last_max) { + last_max = max; + } + } + + m = reftable_calloc(sizeof(struct reftable_merged_table)); + m->stack = stack; + m->stack_len = n; + m->min = first_min; + m->max = last_max; + m->hash_id = hash_id; + *dest = m; + return 0; +} + +/* clears the list of subtable, without affecting the readers themselves. */ +void merged_table_release(struct reftable_merged_table *mt) +{ + FREE_AND_NULL(mt->stack); + mt->stack_len = 0; +} + +void reftable_merged_table_free(struct reftable_merged_table *mt) +{ + if (!mt) { + return; + } + merged_table_release(mt); + reftable_free(mt); +} + +uint64_t +reftable_merged_table_max_update_index(struct reftable_merged_table *mt) +{ + return mt->max; +} + +uint64_t +reftable_merged_table_min_update_index(struct reftable_merged_table *mt) +{ + return mt->min; +} + +static int reftable_table_seek_record(struct reftable_table *tab, + struct reftable_iterator *it, + struct reftable_record *rec) +{ + return tab->ops->seek_record(tab->table_arg, it, rec); +} + +static int merged_table_seek_record(struct reftable_merged_table *mt, + struct reftable_iterator *it, + struct reftable_record *rec) +{ + struct reftable_iterator *iters = reftable_calloc( + sizeof(struct reftable_iterator) * mt->stack_len); + struct merged_iter merged = { + .stack = iters, + .typ = reftable_record_type(rec), + .hash_id = mt->hash_id, + .suppress_deletions = mt->suppress_deletions, + }; + int n = 0; + int err = 0; + int i = 0; + for (i = 0; i < mt->stack_len && err == 0; i++) { + int e = reftable_table_seek_record(&mt->stack[i], &iters[n], + rec); + if (e < 0) { + err = e; + } + if (e == 0) { + n++; + } + } + if (err < 0) { + int i = 0; + for (i = 0; i < n; i++) { + reftable_iterator_destroy(&iters[i]); + } + reftable_free(iters); + return err; + } + + merged.stack_len = n; + err = merged_iter_init(&merged); + if (err < 0) { + merged_iter_close(&merged); + return err; + } else { + struct merged_iter *p = + reftable_malloc(sizeof(struct merged_iter)); + *p = merged; + iterator_from_merged_iter(it, p); + } + return 0; +} + +int reftable_merged_table_seek_ref(struct reftable_merged_table *mt, + struct reftable_iterator *it, + const char *name) +{ + struct reftable_ref_record ref = { + .refname = (char *)name, + }; + struct reftable_record rec = { NULL }; + reftable_record_from_ref(&rec, &ref); + return merged_table_seek_record(mt, it, &rec); +} + +int reftable_merged_table_seek_log_at(struct reftable_merged_table *mt, + struct reftable_iterator *it, + const char *name, uint64_t update_index) +{ + struct reftable_log_record log = { + .refname = (char *)name, + .update_index = update_index, + }; + struct reftable_record rec = { NULL }; + reftable_record_from_log(&rec, &log); + return merged_table_seek_record(mt, it, &rec); +} + +int reftable_merged_table_seek_log(struct reftable_merged_table *mt, + struct reftable_iterator *it, + const char *name) +{ + uint64_t max = ~((uint64_t)0); + return reftable_merged_table_seek_log_at(mt, it, name, max); +} + +uint32_t reftable_merged_table_hash_id(struct reftable_merged_table *mt) +{ + return mt->hash_id; +} + +static int reftable_merged_table_seek_void(void *tab, + struct reftable_iterator *it, + struct reftable_record *rec) +{ + return merged_table_seek_record(tab, it, rec); +} + +static uint32_t reftable_merged_table_hash_id_void(void *tab) +{ + return reftable_merged_table_hash_id(tab); +} + +static uint64_t reftable_merged_table_min_update_index_void(void *tab) +{ + return reftable_merged_table_min_update_index(tab); +} + +static uint64_t reftable_merged_table_max_update_index_void(void *tab) +{ + return reftable_merged_table_max_update_index(tab); +} + +static struct reftable_table_vtable merged_table_vtable = { + .seek_record = reftable_merged_table_seek_void, + .hash_id = reftable_merged_table_hash_id_void, + .min_update_index = reftable_merged_table_min_update_index_void, + .max_update_index = reftable_merged_table_max_update_index_void, +}; + +void reftable_table_from_merged_table(struct reftable_table *tab, + struct reftable_merged_table *merged) +{ + assert(!tab->ops); + tab->ops = &merged_table_vtable; + tab->table_arg = merged; +} diff --git a/reftable/merged.h b/reftable/merged.h new file mode 100644 index 0000000000..7d9f95d27e --- /dev/null +++ b/reftable/merged.h @@ -0,0 +1,38 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef MERGED_H +#define MERGED_H + +#include "pq.h" + +struct reftable_merged_table { + struct reftable_table *stack; + size_t stack_len; + uint32_t hash_id; + + /* If unset, produce deletions. This is useful for compaction. For the + * full stack, deletions should be produced. */ + int suppress_deletions; + + uint64_t min; + uint64_t max; +}; + +struct merged_iter { + struct reftable_iterator *stack; + uint32_t hash_id; + size_t stack_len; + uint8_t typ; + int suppress_deletions; + struct merged_iter_pqueue pq; +}; + +void merged_table_release(struct reftable_merged_table *mt); + +#endif diff --git a/reftable/merged_test.c b/reftable/merged_test.c new file mode 100644 index 0000000000..24461e8a80 --- /dev/null +++ b/reftable/merged_test.c @@ -0,0 +1,468 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "merged.h" + +#include "system.h" + +#include "basics.h" +#include "blocksource.h" +#include "constants.h" +#include "reader.h" +#include "record.h" +#include "test_framework.h" +#include "reftable-merged.h" +#include "reftable-tests.h" +#include "reftable-generic.h" +#include "reftable-writer.h" + +static void write_test_table(struct strbuf *buf, + struct reftable_ref_record refs[], int n) +{ + int min = 0xffffffff; + int max = 0; + int i = 0; + int err; + + struct reftable_write_options opts = { + .block_size = 256, + }; + struct reftable_writer *w = NULL; + for (i = 0; i < n; i++) { + uint64_t ui = refs[i].update_index; + if (ui > max) { + max = ui; + } + if (ui < min) { + min = ui; + } + } + + w = reftable_new_writer(&strbuf_add_void, buf, &opts); + reftable_writer_set_limits(w, min, max); + + for (i = 0; i < n; i++) { + uint64_t before = refs[i].update_index; + int n = reftable_writer_add_ref(w, &refs[i]); + EXPECT(n == 0); + EXPECT(before == refs[i].update_index); + } + + err = reftable_writer_close(w); + EXPECT_ERR(err); + + reftable_writer_free(w); +} + +static void write_test_log_table(struct strbuf *buf, + struct reftable_log_record logs[], int n, + uint64_t update_index) +{ + int i = 0; + int err; + + struct reftable_write_options opts = { + .block_size = 256, + .exact_log_message = 1, + }; + struct reftable_writer *w = NULL; + w = reftable_new_writer(&strbuf_add_void, buf, &opts); + reftable_writer_set_limits(w, update_index, update_index); + + for (i = 0; i < n; i++) { + int err = reftable_writer_add_log(w, &logs[i]); + EXPECT_ERR(err); + } + + err = reftable_writer_close(w); + EXPECT_ERR(err); + + reftable_writer_free(w); +} + +static struct reftable_merged_table * +merged_table_from_records(struct reftable_ref_record **refs, + struct reftable_block_source **source, + struct reftable_reader ***readers, int *sizes, + struct strbuf *buf, int n) +{ + int i = 0; + struct reftable_merged_table *mt = NULL; + int err; + struct reftable_table *tabs = + reftable_calloc(n * sizeof(struct reftable_table)); + *readers = reftable_calloc(n * sizeof(struct reftable_reader *)); + *source = reftable_calloc(n * sizeof(**source)); + for (i = 0; i < n; i++) { + write_test_table(&buf[i], refs[i], sizes[i]); + block_source_from_strbuf(&(*source)[i], &buf[i]); + + err = reftable_new_reader(&(*readers)[i], &(*source)[i], + "name"); + EXPECT_ERR(err); + reftable_table_from_reader(&tabs[i], (*readers)[i]); + } + + err = reftable_new_merged_table(&mt, tabs, n, GIT_SHA1_FORMAT_ID); + EXPECT_ERR(err); + return mt; +} + +static void readers_destroy(struct reftable_reader **readers, size_t n) +{ + int i = 0; + for (; i < n; i++) + reftable_reader_free(readers[i]); + reftable_free(readers); +} + +static void test_merged_between(void) +{ + uint8_t hash1[GIT_SHA1_RAWSZ] = { 1, 2, 3, 0 }; + + struct reftable_ref_record r1[] = { { + .refname = "b", + .update_index = 1, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = hash1, + } }; + struct reftable_ref_record r2[] = { { + .refname = "a", + .update_index = 2, + .value_type = REFTABLE_REF_DELETION, + } }; + + struct reftable_ref_record *refs[] = { r1, r2 }; + int sizes[] = { 1, 1 }; + struct strbuf bufs[2] = { STRBUF_INIT, STRBUF_INIT }; + struct reftable_block_source *bs = NULL; + struct reftable_reader **readers = NULL; + struct reftable_merged_table *mt = + merged_table_from_records(refs, &bs, &readers, sizes, bufs, 2); + int i; + struct reftable_ref_record ref = { NULL }; + struct reftable_iterator it = { NULL }; + int err = reftable_merged_table_seek_ref(mt, &it, "a"); + EXPECT_ERR(err); + + err = reftable_iterator_next_ref(&it, &ref); + EXPECT_ERR(err); + EXPECT(ref.update_index == 2); + reftable_ref_record_release(&ref); + reftable_iterator_destroy(&it); + readers_destroy(readers, 2); + reftable_merged_table_free(mt); + for (i = 0; i < ARRAY_SIZE(bufs); i++) { + strbuf_release(&bufs[i]); + } + reftable_free(bs); +} + +static void test_merged(void) +{ + uint8_t hash1[GIT_SHA1_RAWSZ] = { 1 }; + uint8_t hash2[GIT_SHA1_RAWSZ] = { 2 }; + struct reftable_ref_record r1[] = { + { + .refname = "a", + .update_index = 1, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = hash1, + }, + { + .refname = "b", + .update_index = 1, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = hash1, + }, + { + .refname = "c", + .update_index = 1, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = hash1, + } + }; + struct reftable_ref_record r2[] = { { + .refname = "a", + .update_index = 2, + .value_type = REFTABLE_REF_DELETION, + } }; + struct reftable_ref_record r3[] = { + { + .refname = "c", + .update_index = 3, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = hash2, + }, + { + .refname = "d", + .update_index = 3, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = hash1, + }, + }; + + struct reftable_ref_record want[] = { + r2[0], + r1[1], + r3[0], + r3[1], + }; + + struct reftable_ref_record *refs[] = { r1, r2, r3 }; + int sizes[3] = { 3, 1, 2 }; + struct strbuf bufs[3] = { STRBUF_INIT, STRBUF_INIT, STRBUF_INIT }; + struct reftable_block_source *bs = NULL; + struct reftable_reader **readers = NULL; + struct reftable_merged_table *mt = + merged_table_from_records(refs, &bs, &readers, sizes, bufs, 3); + + struct reftable_iterator it = { NULL }; + int err = reftable_merged_table_seek_ref(mt, &it, "a"); + struct reftable_ref_record *out = NULL; + size_t len = 0; + size_t cap = 0; + int i = 0; + + EXPECT_ERR(err); + EXPECT(reftable_merged_table_hash_id(mt) == GIT_SHA1_FORMAT_ID); + EXPECT(reftable_merged_table_min_update_index(mt) == 1); + + while (len < 100) { /* cap loops/recursion. */ + struct reftable_ref_record ref = { NULL }; + int err = reftable_iterator_next_ref(&it, &ref); + if (err > 0) { + break; + } + if (len == cap) { + cap = 2 * cap + 1; + out = reftable_realloc( + out, sizeof(struct reftable_ref_record) * cap); + } + out[len++] = ref; + } + reftable_iterator_destroy(&it); + + EXPECT(ARRAY_SIZE(want) == len); + for (i = 0; i < len; i++) { + EXPECT(reftable_ref_record_equal(&want[i], &out[i], + GIT_SHA1_RAWSZ)); + } + for (i = 0; i < len; i++) { + reftable_ref_record_release(&out[i]); + } + reftable_free(out); + + for (i = 0; i < 3; i++) { + strbuf_release(&bufs[i]); + } + readers_destroy(readers, 3); + reftable_merged_table_free(mt); + reftable_free(bs); +} + +static struct reftable_merged_table * +merged_table_from_log_records(struct reftable_log_record **logs, + struct reftable_block_source **source, + struct reftable_reader ***readers, int *sizes, + struct strbuf *buf, int n) +{ + int i = 0; + struct reftable_merged_table *mt = NULL; + int err; + struct reftable_table *tabs = + reftable_calloc(n * sizeof(struct reftable_table)); + *readers = reftable_calloc(n * sizeof(struct reftable_reader *)); + *source = reftable_calloc(n * sizeof(**source)); + for (i = 0; i < n; i++) { + write_test_log_table(&buf[i], logs[i], sizes[i], i + 1); + block_source_from_strbuf(&(*source)[i], &buf[i]); + + err = reftable_new_reader(&(*readers)[i], &(*source)[i], + "name"); + EXPECT_ERR(err); + reftable_table_from_reader(&tabs[i], (*readers)[i]); + } + + err = reftable_new_merged_table(&mt, tabs, n, GIT_SHA1_FORMAT_ID); + EXPECT_ERR(err); + return mt; +} + +static void test_merged_logs(void) +{ + uint8_t hash1[GIT_SHA1_RAWSZ] = { 1 }; + uint8_t hash2[GIT_SHA1_RAWSZ] = { 2 }; + uint8_t hash3[GIT_SHA1_RAWSZ] = { 3 }; + struct reftable_log_record r1[] = { + { + .refname = "a", + .update_index = 2, + .value_type = REFTABLE_LOG_UPDATE, + .value.update = { + .old_hash = hash2, + /* deletion */ + .name = "jane doe", + .email = "jane@invalid", + .message = "message2", + } + }, + { + .refname = "a", + .update_index = 1, + .value_type = REFTABLE_LOG_UPDATE, + .value.update = { + .old_hash = hash1, + .new_hash = hash2, + .name = "jane doe", + .email = "jane@invalid", + .message = "message1", + } + }, + }; + struct reftable_log_record r2[] = { + { + .refname = "a", + .update_index = 3, + .value_type = REFTABLE_LOG_UPDATE, + .value.update = { + .new_hash = hash3, + .name = "jane doe", + .email = "jane@invalid", + .message = "message3", + } + }, + }; + struct reftable_log_record r3[] = { + { + .refname = "a", + .update_index = 2, + .value_type = REFTABLE_LOG_DELETION, + }, + }; + struct reftable_log_record want[] = { + r2[0], + r3[0], + r1[1], + }; + + struct reftable_log_record *logs[] = { r1, r2, r3 }; + int sizes[3] = { 2, 1, 1 }; + struct strbuf bufs[3] = { STRBUF_INIT, STRBUF_INIT, STRBUF_INIT }; + struct reftable_block_source *bs = NULL; + struct reftable_reader **readers = NULL; + struct reftable_merged_table *mt = merged_table_from_log_records( + logs, &bs, &readers, sizes, bufs, 3); + + struct reftable_iterator it = { NULL }; + int err = reftable_merged_table_seek_log(mt, &it, "a"); + struct reftable_log_record *out = NULL; + size_t len = 0; + size_t cap = 0; + int i = 0; + + EXPECT_ERR(err); + EXPECT(reftable_merged_table_hash_id(mt) == GIT_SHA1_FORMAT_ID); + EXPECT(reftable_merged_table_min_update_index(mt) == 1); + + while (len < 100) { /* cap loops/recursion. */ + struct reftable_log_record log = { NULL }; + int err = reftable_iterator_next_log(&it, &log); + if (err > 0) { + break; + } + if (len == cap) { + cap = 2 * cap + 1; + out = reftable_realloc( + out, sizeof(struct reftable_log_record) * cap); + } + out[len++] = log; + } + reftable_iterator_destroy(&it); + + EXPECT(ARRAY_SIZE(want) == len); + for (i = 0; i < len; i++) { + EXPECT(reftable_log_record_equal(&want[i], &out[i], + GIT_SHA1_RAWSZ)); + } + + err = reftable_merged_table_seek_log_at(mt, &it, "a", 2); + EXPECT_ERR(err); + reftable_log_record_release(&out[0]); + err = reftable_iterator_next_log(&it, &out[0]); + EXPECT_ERR(err); + EXPECT(reftable_log_record_equal(&out[0], &r3[0], GIT_SHA1_RAWSZ)); + reftable_iterator_destroy(&it); + + for (i = 0; i < len; i++) { + reftable_log_record_release(&out[i]); + } + reftable_free(out); + + for (i = 0; i < 3; i++) { + strbuf_release(&bufs[i]); + } + readers_destroy(readers, 3); + reftable_merged_table_free(mt); + reftable_free(bs); +} + +static void test_default_write_opts(void) +{ + struct reftable_write_options opts = { 0 }; + struct strbuf buf = STRBUF_INIT; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, &buf, &opts); + + struct reftable_ref_record rec = { + .refname = "master", + .update_index = 1, + }; + int err; + struct reftable_block_source source = { NULL }; + struct reftable_table *tab = reftable_calloc(sizeof(*tab) * 1); + uint32_t hash_id; + struct reftable_reader *rd = NULL; + struct reftable_merged_table *merged = NULL; + + reftable_writer_set_limits(w, 1, 1); + + err = reftable_writer_add_ref(w, &rec); + EXPECT_ERR(err); + + err = reftable_writer_close(w); + EXPECT_ERR(err); + reftable_writer_free(w); + + block_source_from_strbuf(&source, &buf); + + err = reftable_new_reader(&rd, &source, "filename"); + EXPECT_ERR(err); + + hash_id = reftable_reader_hash_id(rd); + EXPECT(hash_id == GIT_SHA1_FORMAT_ID); + + reftable_table_from_reader(&tab[0], rd); + err = reftable_new_merged_table(&merged, tab, 1, GIT_SHA1_FORMAT_ID); + EXPECT_ERR(err); + + reftable_reader_free(rd); + reftable_merged_table_free(merged); + strbuf_release(&buf); +} + +/* XXX test refs_for(oid) */ + +int merged_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_merged_logs); + RUN_TEST(test_merged_between); + RUN_TEST(test_merged); + RUN_TEST(test_default_write_opts); + return 0; +} diff --git a/reftable/pq.c b/reftable/pq.c new file mode 100644 index 0000000000..efc474017a --- /dev/null +++ b/reftable/pq.c @@ -0,0 +1,105 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "pq.h" + +#include "reftable-record.h" +#include "system.h" +#include "basics.h" + +int pq_less(struct pq_entry *a, struct pq_entry *b) +{ + struct strbuf ak = STRBUF_INIT; + struct strbuf bk = STRBUF_INIT; + int cmp = 0; + reftable_record_key(&a->rec, &ak); + reftable_record_key(&b->rec, &bk); + + cmp = strbuf_cmp(&ak, &bk); + + strbuf_release(&ak); + strbuf_release(&bk); + + if (cmp == 0) + return a->index > b->index; + + return cmp < 0; +} + +struct pq_entry merged_iter_pqueue_top(struct merged_iter_pqueue pq) +{ + return pq.heap[0]; +} + +int merged_iter_pqueue_is_empty(struct merged_iter_pqueue pq) +{ + return pq.len == 0; +} + +struct pq_entry merged_iter_pqueue_remove(struct merged_iter_pqueue *pq) +{ + int i = 0; + struct pq_entry e = pq->heap[0]; + pq->heap[0] = pq->heap[pq->len - 1]; + pq->len--; + + i = 0; + while (i < pq->len) { + int min = i; + int j = 2 * i + 1; + int k = 2 * i + 2; + if (j < pq->len && pq_less(&pq->heap[j], &pq->heap[i])) { + min = j; + } + if (k < pq->len && pq_less(&pq->heap[k], &pq->heap[min])) { + min = k; + } + + if (min == i) { + break; + } + + SWAP(pq->heap[i], pq->heap[min]); + i = min; + } + + return e; +} + +void merged_iter_pqueue_add(struct merged_iter_pqueue *pq, struct pq_entry e) +{ + int i = 0; + if (pq->len == pq->cap) { + pq->cap = 2 * pq->cap + 1; + pq->heap = reftable_realloc(pq->heap, + pq->cap * sizeof(struct pq_entry)); + } + + pq->heap[pq->len++] = e; + i = pq->len - 1; + while (i > 0) { + int j = (i - 1) / 2; + if (pq_less(&pq->heap[j], &pq->heap[i])) { + break; + } + + SWAP(pq->heap[j], pq->heap[i]); + + i = j; + } +} + +void merged_iter_pqueue_release(struct merged_iter_pqueue *pq) +{ + int i = 0; + for (i = 0; i < pq->len; i++) { + reftable_record_destroy(&pq->heap[i].rec); + } + FREE_AND_NULL(pq->heap); + pq->len = pq->cap = 0; +} diff --git a/reftable/pq.h b/reftable/pq.h new file mode 100644 index 0000000000..56fc1b6d87 --- /dev/null +++ b/reftable/pq.h @@ -0,0 +1,33 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef PQ_H +#define PQ_H + +#include "record.h" + +struct pq_entry { + int index; + struct reftable_record rec; +}; + +struct merged_iter_pqueue { + struct pq_entry *heap; + size_t len; + size_t cap; +}; + +struct pq_entry merged_iter_pqueue_top(struct merged_iter_pqueue pq); +int merged_iter_pqueue_is_empty(struct merged_iter_pqueue pq); +void merged_iter_pqueue_check(struct merged_iter_pqueue pq); +struct pq_entry merged_iter_pqueue_remove(struct merged_iter_pqueue *pq); +void merged_iter_pqueue_add(struct merged_iter_pqueue *pq, struct pq_entry e); +void merged_iter_pqueue_release(struct merged_iter_pqueue *pq); +int pq_less(struct pq_entry *a, struct pq_entry *b); + +#endif diff --git a/reftable/pq_test.c b/reftable/pq_test.c new file mode 100644 index 0000000000..c9bb05e37b --- /dev/null +++ b/reftable/pq_test.c @@ -0,0 +1,82 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" + +#include "basics.h" +#include "constants.h" +#include "pq.h" +#include "record.h" +#include "reftable-tests.h" +#include "test_framework.h" + +void merged_iter_pqueue_check(struct merged_iter_pqueue pq) +{ + int i; + for (i = 1; i < pq.len; i++) { + int parent = (i - 1) / 2; + + EXPECT(pq_less(&pq.heap[parent], &pq.heap[i])); + } +} + +static void test_pq(void) +{ + char *names[54] = { NULL }; + int N = ARRAY_SIZE(names) - 1; + + struct merged_iter_pqueue pq = { NULL }; + const char *last = NULL; + + int i = 0; + for (i = 0; i < N; i++) { + char name[100]; + snprintf(name, sizeof(name), "%02d", i); + names[i] = xstrdup(name); + } + + i = 1; + do { + struct reftable_record rec = + reftable_new_record(BLOCK_TYPE_REF); + struct pq_entry e = { 0 }; + + reftable_record_as_ref(&rec)->refname = names[i]; + e.rec = rec; + merged_iter_pqueue_add(&pq, e); + merged_iter_pqueue_check(pq); + i = (i * 7) % N; + } while (i != 1); + + while (!merged_iter_pqueue_is_empty(pq)) { + struct pq_entry e = merged_iter_pqueue_remove(&pq); + struct reftable_ref_record *ref = + reftable_record_as_ref(&e.rec); + + merged_iter_pqueue_check(pq); + + if (last) { + EXPECT(strcmp(last, ref->refname) < 0); + } + last = ref->refname; + ref->refname = NULL; + reftable_free(ref); + } + + for (i = 0; i < N; i++) { + reftable_free(names[i]); + } + + merged_iter_pqueue_release(&pq); +} + +int pq_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_pq); + return 0; +} diff --git a/reftable/publicbasics.c b/reftable/publicbasics.c new file mode 100644 index 0000000000..0ad7d5c0ff --- /dev/null +++ b/reftable/publicbasics.c @@ -0,0 +1,65 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "reftable-malloc.h" + +#include "basics.h" +#include "system.h" + +static void *(*reftable_malloc_ptr)(size_t sz); +static void *(*reftable_realloc_ptr)(void *, size_t); +static void (*reftable_free_ptr)(void *); + +void *reftable_malloc(size_t sz) +{ + if (reftable_malloc_ptr) + return (*reftable_malloc_ptr)(sz); + return malloc(sz); +} + +void *reftable_realloc(void *p, size_t sz) +{ + if (reftable_realloc_ptr) + return (*reftable_realloc_ptr)(p, sz); + return realloc(p, sz); +} + +void reftable_free(void *p) +{ + if (reftable_free_ptr) + reftable_free_ptr(p); + else + free(p); +} + +void *reftable_calloc(size_t sz) +{ + void *p = reftable_malloc(sz); + memset(p, 0, sz); + return p; +} + +void reftable_set_alloc(void *(*malloc)(size_t), + void *(*realloc)(void *, size_t), void (*free)(void *)) +{ + reftable_malloc_ptr = malloc; + reftable_realloc_ptr = realloc; + reftable_free_ptr = free; +} + +int hash_size(uint32_t id) +{ + switch (id) { + case 0: + case GIT_SHA1_FORMAT_ID: + return GIT_SHA1_RAWSZ; + case GIT_SHA256_FORMAT_ID: + return GIT_SHA256_RAWSZ; + } + abort(); +} diff --git a/reftable/reader.c b/reftable/reader.c new file mode 100644 index 0000000000..006709a645 --- /dev/null +++ b/reftable/reader.c @@ -0,0 +1,801 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "reader.h" + +#include "system.h" +#include "block.h" +#include "constants.h" +#include "generic.h" +#include "iter.h" +#include "record.h" +#include "reftable-error.h" +#include "reftable-generic.h" +#include "tree.h" + +uint64_t block_source_size(struct reftable_block_source *source) +{ + return source->ops->size(source->arg); +} + +int block_source_read_block(struct reftable_block_source *source, + struct reftable_block *dest, uint64_t off, + uint32_t size) +{ + int result = source->ops->read_block(source->arg, dest, off, size); + dest->source = *source; + return result; +} + +void block_source_close(struct reftable_block_source *source) +{ + if (!source->ops) { + return; + } + + source->ops->close(source->arg); + source->ops = NULL; +} + +static struct reftable_reader_offsets * +reader_offsets_for(struct reftable_reader *r, uint8_t typ) +{ + switch (typ) { + case BLOCK_TYPE_REF: + return &r->ref_offsets; + case BLOCK_TYPE_LOG: + return &r->log_offsets; + case BLOCK_TYPE_OBJ: + return &r->obj_offsets; + } + abort(); +} + +static int reader_get_block(struct reftable_reader *r, + struct reftable_block *dest, uint64_t off, + uint32_t sz) +{ + if (off >= r->size) + return 0; + + if (off + sz > r->size) { + sz = r->size - off; + } + + return block_source_read_block(&r->source, dest, off, sz); +} + +uint32_t reftable_reader_hash_id(struct reftable_reader *r) +{ + return r->hash_id; +} + +const char *reader_name(struct reftable_reader *r) +{ + return r->name; +} + +static int parse_footer(struct reftable_reader *r, uint8_t *footer, + uint8_t *header) +{ + uint8_t *f = footer; + uint8_t first_block_typ; + int err = 0; + uint32_t computed_crc; + uint32_t file_crc; + + if (memcmp(f, "REFT", 4)) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + f += 4; + + if (memcmp(footer, header, header_size(r->version))) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + f++; + r->block_size = get_be24(f); + + f += 3; + r->min_update_index = get_be64(f); + f += 8; + r->max_update_index = get_be64(f); + f += 8; + + if (r->version == 1) { + r->hash_id = GIT_SHA1_FORMAT_ID; + } else { + r->hash_id = get_be32(f); + switch (r->hash_id) { + case GIT_SHA1_FORMAT_ID: + break; + case GIT_SHA256_FORMAT_ID: + break; + default: + err = REFTABLE_FORMAT_ERROR; + goto done; + } + f += 4; + } + + r->ref_offsets.index_offset = get_be64(f); + f += 8; + + r->obj_offsets.offset = get_be64(f); + f += 8; + + r->object_id_len = r->obj_offsets.offset & ((1 << 5) - 1); + r->obj_offsets.offset >>= 5; + + r->obj_offsets.index_offset = get_be64(f); + f += 8; + r->log_offsets.offset = get_be64(f); + f += 8; + r->log_offsets.index_offset = get_be64(f); + f += 8; + + computed_crc = crc32(0, footer, f - footer); + file_crc = get_be32(f); + f += 4; + if (computed_crc != file_crc) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + first_block_typ = header[header_size(r->version)]; + r->ref_offsets.is_present = (first_block_typ == BLOCK_TYPE_REF); + r->ref_offsets.offset = 0; + r->log_offsets.is_present = (first_block_typ == BLOCK_TYPE_LOG || + r->log_offsets.offset > 0); + r->obj_offsets.is_present = r->obj_offsets.offset > 0; + err = 0; +done: + return err; +} + +int init_reader(struct reftable_reader *r, struct reftable_block_source *source, + const char *name) +{ + struct reftable_block footer = { NULL }; + struct reftable_block header = { NULL }; + int err = 0; + uint64_t file_size = block_source_size(source); + + /* Need +1 to read type of first block. */ + uint32_t read_size = header_size(2) + 1; /* read v2 because it's larger. */ + memset(r, 0, sizeof(struct reftable_reader)); + + if (read_size > file_size) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + err = block_source_read_block(source, &header, 0, read_size); + if (err != read_size) { + err = REFTABLE_IO_ERROR; + goto done; + } + + if (memcmp(header.data, "REFT", 4)) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + r->version = header.data[4]; + if (r->version != 1 && r->version != 2) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + r->size = file_size - footer_size(r->version); + r->source = *source; + r->name = xstrdup(name); + r->hash_id = 0; + + err = block_source_read_block(source, &footer, r->size, + footer_size(r->version)); + if (err != footer_size(r->version)) { + err = REFTABLE_IO_ERROR; + goto done; + } + + err = parse_footer(r, footer.data, header.data); +done: + reftable_block_done(&footer); + reftable_block_done(&header); + return err; +} + +struct table_iter { + struct reftable_reader *r; + uint8_t typ; + uint64_t block_off; + struct block_iter bi; + int is_finished; +}; +#define TABLE_ITER_INIT \ + { \ + .bi = {.last_key = STRBUF_INIT } \ + } + +static void table_iter_copy_from(struct table_iter *dest, + struct table_iter *src) +{ + dest->r = src->r; + dest->typ = src->typ; + dest->block_off = src->block_off; + dest->is_finished = src->is_finished; + block_iter_copy_from(&dest->bi, &src->bi); +} + +static int table_iter_next_in_block(struct table_iter *ti, + struct reftable_record *rec) +{ + int res = block_iter_next(&ti->bi, rec); + if (res == 0 && reftable_record_type(rec) == BLOCK_TYPE_REF) { + ((struct reftable_ref_record *)rec->data)->update_index += + ti->r->min_update_index; + } + + return res; +} + +static void table_iter_block_done(struct table_iter *ti) +{ + if (!ti->bi.br) { + return; + } + reftable_block_done(&ti->bi.br->block); + FREE_AND_NULL(ti->bi.br); + + ti->bi.last_key.len = 0; + ti->bi.next_off = 0; +} + +static int32_t extract_block_size(uint8_t *data, uint8_t *typ, uint64_t off, + int version) +{ + int32_t result = 0; + + if (off == 0) { + data += header_size(version); + } + + *typ = data[0]; + if (reftable_is_block_type(*typ)) { + result = get_be24(data + 1); + } + return result; +} + +int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br, + uint64_t next_off, uint8_t want_typ) +{ + int32_t guess_block_size = r->block_size ? r->block_size : + DEFAULT_BLOCK_SIZE; + struct reftable_block block = { NULL }; + uint8_t block_typ = 0; + int err = 0; + uint32_t header_off = next_off ? 0 : header_size(r->version); + int32_t block_size = 0; + + if (next_off >= r->size) + return 1; + + err = reader_get_block(r, &block, next_off, guess_block_size); + if (err < 0) + return err; + + block_size = extract_block_size(block.data, &block_typ, next_off, + r->version); + if (block_size < 0) + return block_size; + + if (want_typ != BLOCK_TYPE_ANY && block_typ != want_typ) { + reftable_block_done(&block); + return 1; + } + + if (block_size > guess_block_size) { + reftable_block_done(&block); + err = reader_get_block(r, &block, next_off, block_size); + if (err < 0) { + return err; + } + } + + return block_reader_init(br, &block, header_off, r->block_size, + hash_size(r->hash_id)); +} + +static int table_iter_next_block(struct table_iter *dest, + struct table_iter *src) +{ + uint64_t next_block_off = src->block_off + src->bi.br->full_block_size; + struct block_reader br = { 0 }; + int err = 0; + + dest->r = src->r; + dest->typ = src->typ; + dest->block_off = next_block_off; + + err = reader_init_block_reader(src->r, &br, next_block_off, src->typ); + if (err > 0) { + dest->is_finished = 1; + return 1; + } + if (err != 0) + return err; + else { + struct block_reader *brp = + reftable_malloc(sizeof(struct block_reader)); + *brp = br; + + dest->is_finished = 0; + block_reader_start(brp, &dest->bi); + } + return 0; +} + +static int table_iter_next(struct table_iter *ti, struct reftable_record *rec) +{ + if (reftable_record_type(rec) != ti->typ) + return REFTABLE_API_ERROR; + + while (1) { + struct table_iter next = TABLE_ITER_INIT; + int err = 0; + if (ti->is_finished) { + return 1; + } + + err = table_iter_next_in_block(ti, rec); + if (err <= 0) { + return err; + } + + err = table_iter_next_block(&next, ti); + if (err != 0) { + ti->is_finished = 1; + } + table_iter_block_done(ti); + if (err != 0) { + return err; + } + table_iter_copy_from(ti, &next); + block_iter_close(&next.bi); + } +} + +static int table_iter_next_void(void *ti, struct reftable_record *rec) +{ + return table_iter_next(ti, rec); +} + +static void table_iter_close(void *p) +{ + struct table_iter *ti = p; + table_iter_block_done(ti); + block_iter_close(&ti->bi); +} + +static struct reftable_iterator_vtable table_iter_vtable = { + .next = &table_iter_next_void, + .close = &table_iter_close, +}; + +static void iterator_from_table_iter(struct reftable_iterator *it, + struct table_iter *ti) +{ + assert(!it->ops); + it->iter_arg = ti; + it->ops = &table_iter_vtable; +} + +static int reader_table_iter_at(struct reftable_reader *r, + struct table_iter *ti, uint64_t off, + uint8_t typ) +{ + struct block_reader br = { 0 }; + struct block_reader *brp = NULL; + + int err = reader_init_block_reader(r, &br, off, typ); + if (err != 0) + return err; + + brp = reftable_malloc(sizeof(struct block_reader)); + *brp = br; + ti->r = r; + ti->typ = block_reader_type(brp); + ti->block_off = off; + block_reader_start(brp, &ti->bi); + return 0; +} + +static int reader_start(struct reftable_reader *r, struct table_iter *ti, + uint8_t typ, int index) +{ + struct reftable_reader_offsets *offs = reader_offsets_for(r, typ); + uint64_t off = offs->offset; + if (index) { + off = offs->index_offset; + if (off == 0) { + return 1; + } + typ = BLOCK_TYPE_INDEX; + } + + return reader_table_iter_at(r, ti, off, typ); +} + +static int reader_seek_linear(struct reftable_reader *r, struct table_iter *ti, + struct reftable_record *want) +{ + struct reftable_record rec = + reftable_new_record(reftable_record_type(want)); + struct strbuf want_key = STRBUF_INIT; + struct strbuf got_key = STRBUF_INIT; + struct table_iter next = TABLE_ITER_INIT; + int err = -1; + + reftable_record_key(want, &want_key); + + while (1) { + err = table_iter_next_block(&next, ti); + if (err < 0) + goto done; + + if (err > 0) { + break; + } + + err = block_reader_first_key(next.bi.br, &got_key); + if (err < 0) + goto done; + + if (strbuf_cmp(&got_key, &want_key) > 0) { + table_iter_block_done(&next); + break; + } + + table_iter_block_done(ti); + table_iter_copy_from(ti, &next); + } + + err = block_iter_seek(&ti->bi, &want_key); + if (err < 0) + goto done; + err = 0; + +done: + block_iter_close(&next.bi); + reftable_record_destroy(&rec); + strbuf_release(&want_key); + strbuf_release(&got_key); + return err; +} + +static int reader_seek_indexed(struct reftable_reader *r, + struct reftable_iterator *it, + struct reftable_record *rec) +{ + struct reftable_index_record want_index = { .last_key = STRBUF_INIT }; + struct reftable_record want_index_rec = { NULL }; + struct reftable_index_record index_result = { .last_key = STRBUF_INIT }; + struct reftable_record index_result_rec = { NULL }; + struct table_iter index_iter = TABLE_ITER_INIT; + struct table_iter next = TABLE_ITER_INIT; + int err = 0; + + reftable_record_key(rec, &want_index.last_key); + reftable_record_from_index(&want_index_rec, &want_index); + reftable_record_from_index(&index_result_rec, &index_result); + + err = reader_start(r, &index_iter, reftable_record_type(rec), 1); + if (err < 0) + goto done; + + err = reader_seek_linear(r, &index_iter, &want_index_rec); + while (1) { + err = table_iter_next(&index_iter, &index_result_rec); + table_iter_block_done(&index_iter); + if (err != 0) + goto done; + + err = reader_table_iter_at(r, &next, index_result.offset, 0); + if (err != 0) + goto done; + + err = block_iter_seek(&next.bi, &want_index.last_key); + if (err < 0) + goto done; + + if (next.typ == reftable_record_type(rec)) { + err = 0; + break; + } + + if (next.typ != BLOCK_TYPE_INDEX) { + err = REFTABLE_FORMAT_ERROR; + break; + } + + table_iter_copy_from(&index_iter, &next); + } + + if (err == 0) { + struct table_iter empty = TABLE_ITER_INIT; + struct table_iter *malloced = + reftable_calloc(sizeof(struct table_iter)); + *malloced = empty; + table_iter_copy_from(malloced, &next); + iterator_from_table_iter(it, malloced); + } +done: + block_iter_close(&next.bi); + table_iter_close(&index_iter); + reftable_record_release(&want_index_rec); + reftable_record_release(&index_result_rec); + return err; +} + +static int reader_seek_internal(struct reftable_reader *r, + struct reftable_iterator *it, + struct reftable_record *rec) +{ + struct reftable_reader_offsets *offs = + reader_offsets_for(r, reftable_record_type(rec)); + uint64_t idx = offs->index_offset; + struct table_iter ti = TABLE_ITER_INIT; + int err = 0; + if (idx > 0) + return reader_seek_indexed(r, it, rec); + + err = reader_start(r, &ti, reftable_record_type(rec), 0); + if (err < 0) + return err; + err = reader_seek_linear(r, &ti, rec); + if (err < 0) + return err; + else { + struct table_iter *p = + reftable_malloc(sizeof(struct table_iter)); + *p = ti; + iterator_from_table_iter(it, p); + } + + return 0; +} + +static int reader_seek(struct reftable_reader *r, struct reftable_iterator *it, + struct reftable_record *rec) +{ + uint8_t typ = reftable_record_type(rec); + + struct reftable_reader_offsets *offs = reader_offsets_for(r, typ); + if (!offs->is_present) { + iterator_set_empty(it); + return 0; + } + + return reader_seek_internal(r, it, rec); +} + +int reftable_reader_seek_ref(struct reftable_reader *r, + struct reftable_iterator *it, const char *name) +{ + struct reftable_ref_record ref = { + .refname = (char *)name, + }; + struct reftable_record rec = { NULL }; + reftable_record_from_ref(&rec, &ref); + return reader_seek(r, it, &rec); +} + +int reftable_reader_seek_log_at(struct reftable_reader *r, + struct reftable_iterator *it, const char *name, + uint64_t update_index) +{ + struct reftable_log_record log = { + .refname = (char *)name, + .update_index = update_index, + }; + struct reftable_record rec = { NULL }; + reftable_record_from_log(&rec, &log); + return reader_seek(r, it, &rec); +} + +int reftable_reader_seek_log(struct reftable_reader *r, + struct reftable_iterator *it, const char *name) +{ + uint64_t max = ~((uint64_t)0); + return reftable_reader_seek_log_at(r, it, name, max); +} + +void reader_close(struct reftable_reader *r) +{ + block_source_close(&r->source); + FREE_AND_NULL(r->name); +} + +int reftable_new_reader(struct reftable_reader **p, + struct reftable_block_source *src, char const *name) +{ + struct reftable_reader *rd = + reftable_calloc(sizeof(struct reftable_reader)); + int err = init_reader(rd, src, name); + if (err == 0) { + *p = rd; + } else { + block_source_close(src); + reftable_free(rd); + } + return err; +} + +void reftable_reader_free(struct reftable_reader *r) +{ + reader_close(r); + reftable_free(r); +} + +static int reftable_reader_refs_for_indexed(struct reftable_reader *r, + struct reftable_iterator *it, + uint8_t *oid) +{ + struct reftable_obj_record want = { + .hash_prefix = oid, + .hash_prefix_len = r->object_id_len, + }; + struct reftable_record want_rec = { NULL }; + struct reftable_iterator oit = { NULL }; + struct reftable_obj_record got = { NULL }; + struct reftable_record got_rec = { NULL }; + int err = 0; + struct indexed_table_ref_iter *itr = NULL; + + /* Look through the reverse index. */ + reftable_record_from_obj(&want_rec, &want); + err = reader_seek(r, &oit, &want_rec); + if (err != 0) + goto done; + + /* read out the reftable_obj_record */ + reftable_record_from_obj(&got_rec, &got); + err = iterator_next(&oit, &got_rec); + if (err < 0) + goto done; + + if (err > 0 || + memcmp(want.hash_prefix, got.hash_prefix, r->object_id_len)) { + /* didn't find it; return empty iterator */ + iterator_set_empty(it); + err = 0; + goto done; + } + + err = new_indexed_table_ref_iter(&itr, r, oid, hash_size(r->hash_id), + got.offsets, got.offset_len); + if (err < 0) + goto done; + got.offsets = NULL; + iterator_from_indexed_table_ref_iter(it, itr); + +done: + reftable_iterator_destroy(&oit); + reftable_record_release(&got_rec); + return err; +} + +static int reftable_reader_refs_for_unindexed(struct reftable_reader *r, + struct reftable_iterator *it, + uint8_t *oid) +{ + struct table_iter ti_empty = TABLE_ITER_INIT; + struct table_iter *ti = reftable_calloc(sizeof(struct table_iter)); + struct filtering_ref_iterator *filter = NULL; + struct filtering_ref_iterator empty = FILTERING_REF_ITERATOR_INIT; + int oid_len = hash_size(r->hash_id); + int err; + + *ti = ti_empty; + err = reader_start(r, ti, BLOCK_TYPE_REF, 0); + if (err < 0) { + reftable_free(ti); + return err; + } + + filter = reftable_malloc(sizeof(struct filtering_ref_iterator)); + *filter = empty; + + strbuf_add(&filter->oid, oid, oid_len); + reftable_table_from_reader(&filter->tab, r); + filter->double_check = 0; + iterator_from_table_iter(&filter->it, ti); + + iterator_from_filtering_ref_iterator(it, filter); + return 0; +} + +int reftable_reader_refs_for(struct reftable_reader *r, + struct reftable_iterator *it, uint8_t *oid) +{ + if (r->obj_offsets.is_present) + return reftable_reader_refs_for_indexed(r, it, oid); + return reftable_reader_refs_for_unindexed(r, it, oid); +} + +uint64_t reftable_reader_max_update_index(struct reftable_reader *r) +{ + return r->max_update_index; +} + +uint64_t reftable_reader_min_update_index(struct reftable_reader *r) +{ + return r->min_update_index; +} + +/* generic table interface. */ + +static int reftable_reader_seek_void(void *tab, struct reftable_iterator *it, + struct reftable_record *rec) +{ + return reader_seek(tab, it, rec); +} + +static uint32_t reftable_reader_hash_id_void(void *tab) +{ + return reftable_reader_hash_id(tab); +} + +static uint64_t reftable_reader_min_update_index_void(void *tab) +{ + return reftable_reader_min_update_index(tab); +} + +static uint64_t reftable_reader_max_update_index_void(void *tab) +{ + return reftable_reader_max_update_index(tab); +} + +static struct reftable_table_vtable reader_vtable = { + .seek_record = reftable_reader_seek_void, + .hash_id = reftable_reader_hash_id_void, + .min_update_index = reftable_reader_min_update_index_void, + .max_update_index = reftable_reader_max_update_index_void, +}; + +void reftable_table_from_reader(struct reftable_table *tab, + struct reftable_reader *reader) +{ + assert(!tab->ops); + tab->ops = &reader_vtable; + tab->table_arg = reader; +} + + +int reftable_reader_print_file(const char *tablename) +{ + struct reftable_block_source src = { NULL }; + int err = reftable_block_source_from_file(&src, tablename); + struct reftable_reader *r = NULL; + struct reftable_table tab = { NULL }; + if (err < 0) + goto done; + + err = reftable_new_reader(&r, &src, tablename); + if (err < 0) + goto done; + + reftable_table_from_reader(&tab, r); + err = reftable_table_print(&tab); +done: + reftable_reader_free(r); + return err; +} diff --git a/reftable/reader.h b/reftable/reader.h new file mode 100644 index 0000000000..e869165f23 --- /dev/null +++ b/reftable/reader.h @@ -0,0 +1,64 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef READER_H +#define READER_H + +#include "block.h" +#include "record.h" +#include "reftable-iterator.h" +#include "reftable-reader.h" + +uint64_t block_source_size(struct reftable_block_source *source); + +int block_source_read_block(struct reftable_block_source *source, + struct reftable_block *dest, uint64_t off, + uint32_t size); +void block_source_close(struct reftable_block_source *source); + +/* metadata for a block type */ +struct reftable_reader_offsets { + int is_present; + uint64_t offset; + uint64_t index_offset; +}; + +/* The state for reading a reftable file. */ +struct reftable_reader { + /* for convience, associate a name with the instance. */ + char *name; + struct reftable_block_source source; + + /* Size of the file, excluding the footer. */ + uint64_t size; + + /* 'sha1' for SHA1, 's256' for SHA-256 */ + uint32_t hash_id; + + uint32_t block_size; + uint64_t min_update_index; + uint64_t max_update_index; + /* Length of the OID keys in the 'o' section */ + int object_id_len; + int version; + + struct reftable_reader_offsets ref_offsets; + struct reftable_reader_offsets obj_offsets; + struct reftable_reader_offsets log_offsets; +}; + +int init_reader(struct reftable_reader *r, struct reftable_block_source *source, + const char *name); +void reader_close(struct reftable_reader *r); +const char *reader_name(struct reftable_reader *r); + +/* initialize a block reader to read from `r` */ +int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br, + uint64_t next_off, uint8_t want_typ); + +#endif diff --git a/reftable/readwrite_test.c b/reftable/readwrite_test.c new file mode 100644 index 0000000000..5f6bcc2f77 --- /dev/null +++ b/reftable/readwrite_test.c @@ -0,0 +1,652 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" + +#include "basics.h" +#include "block.h" +#include "blocksource.h" +#include "constants.h" +#include "reader.h" +#include "record.h" +#include "test_framework.h" +#include "reftable-tests.h" +#include "reftable-writer.h" + +static const int update_index = 5; + +static void test_buffer(void) +{ + struct strbuf buf = STRBUF_INIT; + struct reftable_block_source source = { NULL }; + struct reftable_block out = { NULL }; + int n; + uint8_t in[] = "hello"; + strbuf_add(&buf, in, sizeof(in)); + block_source_from_strbuf(&source, &buf); + EXPECT(block_source_size(&source) == 6); + n = block_source_read_block(&source, &out, 0, sizeof(in)); + EXPECT(n == sizeof(in)); + EXPECT(!memcmp(in, out.data, n)); + reftable_block_done(&out); + + n = block_source_read_block(&source, &out, 1, 2); + EXPECT(n == 2); + EXPECT(!memcmp(out.data, "el", 2)); + + reftable_block_done(&out); + block_source_close(&source); + strbuf_release(&buf); +} + +static void write_table(char ***names, struct strbuf *buf, int N, + int block_size, uint32_t hash_id) +{ + struct reftable_write_options opts = { + .block_size = block_size, + .hash_id = hash_id, + }; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, buf, &opts); + struct reftable_ref_record ref = { NULL }; + int i = 0, n; + struct reftable_log_record log = { NULL }; + const struct reftable_stats *stats = NULL; + *names = reftable_calloc(sizeof(char *) * (N + 1)); + reftable_writer_set_limits(w, update_index, update_index); + for (i = 0; i < N; i++) { + uint8_t hash[GIT_SHA256_RAWSZ] = { 0 }; + char name[100]; + int n; + + set_test_hash(hash, i); + + snprintf(name, sizeof(name), "refs/heads/branch%02d", i); + + ref.refname = name; + ref.update_index = update_index; + ref.value_type = REFTABLE_REF_VAL1; + ref.value.val1 = hash; + (*names)[i] = xstrdup(name); + + n = reftable_writer_add_ref(w, &ref); + EXPECT(n == 0); + } + + for (i = 0; i < N; i++) { + uint8_t hash[GIT_SHA256_RAWSZ] = { 0 }; + char name[100]; + int n; + + set_test_hash(hash, i); + + snprintf(name, sizeof(name), "refs/heads/branch%02d", i); + + log.refname = name; + log.update_index = update_index; + log.value_type = REFTABLE_LOG_UPDATE; + log.value.update.new_hash = hash; + log.value.update.message = "message"; + + n = reftable_writer_add_log(w, &log); + EXPECT(n == 0); + } + + n = reftable_writer_close(w); + EXPECT(n == 0); + + stats = writer_stats(w); + for (i = 0; i < stats->ref_stats.blocks; i++) { + int off = i * opts.block_size; + if (off == 0) { + off = header_size( + (hash_id == GIT_SHA256_FORMAT_ID) ? 2 : 1); + } + EXPECT(buf->buf[off] == 'r'); + } + + EXPECT(stats->log_stats.blocks > 0); + reftable_writer_free(w); +} + +static void test_log_buffer_size(void) +{ + struct strbuf buf = STRBUF_INIT; + struct reftable_write_options opts = { + .block_size = 4096, + }; + int err; + int i; + struct reftable_log_record + log = { .refname = "refs/heads/master", + .update_index = 0xa, + .value_type = REFTABLE_LOG_UPDATE, + .value = { .update = { + .name = "Han-Wen Nienhuys", + .email = "hanwen@google.com", + .tz_offset = 100, + .time = 0x5e430672, + .message = "commit: 9\n", + } } }; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, &buf, &opts); + + /* This tests buffer extension for log compression. Must use a random + hash, to ensure that the compressed part is larger than the original. + */ + uint8_t hash1[GIT_SHA1_RAWSZ], hash2[GIT_SHA1_RAWSZ]; + for (i = 0; i < GIT_SHA1_RAWSZ; i++) { + hash1[i] = (uint8_t)(rand() % 256); + hash2[i] = (uint8_t)(rand() % 256); + } + log.value.update.old_hash = hash1; + log.value.update.new_hash = hash2; + reftable_writer_set_limits(w, update_index, update_index); + err = reftable_writer_add_log(w, &log); + EXPECT_ERR(err); + err = reftable_writer_close(w); + EXPECT_ERR(err); + reftable_writer_free(w); + strbuf_release(&buf); +} + +static void test_log_write_read(void) +{ + int N = 2; + char **names = reftable_calloc(sizeof(char *) * (N + 1)); + int err; + struct reftable_write_options opts = { + .block_size = 256, + }; + struct reftable_ref_record ref = { NULL }; + int i = 0; + struct reftable_log_record log = { NULL }; + int n; + struct reftable_iterator it = { NULL }; + struct reftable_reader rd = { NULL }; + struct reftable_block_source source = { NULL }; + struct strbuf buf = STRBUF_INIT; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, &buf, &opts); + const struct reftable_stats *stats = NULL; + reftable_writer_set_limits(w, 0, N); + for (i = 0; i < N; i++) { + char name[256]; + struct reftable_ref_record ref = { NULL }; + snprintf(name, sizeof(name), "b%02d%0*d", i, 130, 7); + names[i] = xstrdup(name); + ref.refname = name; + ref.update_index = i; + + err = reftable_writer_add_ref(w, &ref); + EXPECT_ERR(err); + } + for (i = 0; i < N; i++) { + uint8_t hash1[GIT_SHA1_RAWSZ], hash2[GIT_SHA1_RAWSZ]; + struct reftable_log_record log = { NULL }; + set_test_hash(hash1, i); + set_test_hash(hash2, i + 1); + + log.refname = names[i]; + log.update_index = i; + log.value_type = REFTABLE_LOG_UPDATE; + log.value.update.old_hash = hash1; + log.value.update.new_hash = hash2; + + err = reftable_writer_add_log(w, &log); + EXPECT_ERR(err); + } + + n = reftable_writer_close(w); + EXPECT(n == 0); + + stats = writer_stats(w); + EXPECT(stats->log_stats.blocks > 0); + reftable_writer_free(w); + w = NULL; + + block_source_from_strbuf(&source, &buf); + + err = init_reader(&rd, &source, "file.log"); + EXPECT_ERR(err); + + err = reftable_reader_seek_ref(&rd, &it, names[N - 1]); + EXPECT_ERR(err); + + err = reftable_iterator_next_ref(&it, &ref); + EXPECT_ERR(err); + + /* end of iteration. */ + err = reftable_iterator_next_ref(&it, &ref); + EXPECT(0 < err); + + reftable_iterator_destroy(&it); + reftable_ref_record_release(&ref); + + err = reftable_reader_seek_log(&rd, &it, ""); + EXPECT_ERR(err); + + i = 0; + while (1) { + int err = reftable_iterator_next_log(&it, &log); + if (err > 0) { + break; + } + + EXPECT_ERR(err); + EXPECT_STREQ(names[i], log.refname); + EXPECT(i == log.update_index); + i++; + reftable_log_record_release(&log); + } + + EXPECT(i == N); + reftable_iterator_destroy(&it); + + /* cleanup. */ + strbuf_release(&buf); + free_names(names); + reader_close(&rd); +} + +static void test_table_read_write_sequential(void) +{ + char **names; + struct strbuf buf = STRBUF_INIT; + int N = 50; + struct reftable_iterator it = { NULL }; + struct reftable_block_source source = { NULL }; + struct reftable_reader rd = { NULL }; + int err = 0; + int j = 0; + + write_table(&names, &buf, N, 256, GIT_SHA1_FORMAT_ID); + + block_source_from_strbuf(&source, &buf); + + err = init_reader(&rd, &source, "file.ref"); + EXPECT_ERR(err); + + err = reftable_reader_seek_ref(&rd, &it, ""); + EXPECT_ERR(err); + + while (1) { + struct reftable_ref_record ref = { NULL }; + int r = reftable_iterator_next_ref(&it, &ref); + EXPECT(r >= 0); + if (r > 0) { + break; + } + EXPECT(0 == strcmp(names[j], ref.refname)); + EXPECT(update_index == ref.update_index); + + j++; + reftable_ref_record_release(&ref); + } + EXPECT(j == N); + reftable_iterator_destroy(&it); + strbuf_release(&buf); + free_names(names); + + reader_close(&rd); +} + +static void test_table_write_small_table(void) +{ + char **names; + struct strbuf buf = STRBUF_INIT; + int N = 1; + write_table(&names, &buf, N, 4096, GIT_SHA1_FORMAT_ID); + EXPECT(buf.len < 200); + strbuf_release(&buf); + free_names(names); +} + +static void test_table_read_api(void) +{ + char **names; + struct strbuf buf = STRBUF_INIT; + int N = 50; + struct reftable_reader rd = { NULL }; + struct reftable_block_source source = { NULL }; + int err; + int i; + struct reftable_log_record log = { NULL }; + struct reftable_iterator it = { NULL }; + + write_table(&names, &buf, N, 256, GIT_SHA1_FORMAT_ID); + + block_source_from_strbuf(&source, &buf); + + err = init_reader(&rd, &source, "file.ref"); + EXPECT_ERR(err); + + err = reftable_reader_seek_ref(&rd, &it, names[0]); + EXPECT_ERR(err); + + err = reftable_iterator_next_log(&it, &log); + EXPECT(err == REFTABLE_API_ERROR); + + strbuf_release(&buf); + for (i = 0; i < N; i++) { + reftable_free(names[i]); + } + reftable_iterator_destroy(&it); + reftable_free(names); + reader_close(&rd); + strbuf_release(&buf); +} + +static void test_table_read_write_seek(int index, int hash_id) +{ + char **names; + struct strbuf buf = STRBUF_INIT; + int N = 50; + struct reftable_reader rd = { NULL }; + struct reftable_block_source source = { NULL }; + int err; + int i = 0; + + struct reftable_iterator it = { NULL }; + struct strbuf pastLast = STRBUF_INIT; + struct reftable_ref_record ref = { NULL }; + + write_table(&names, &buf, N, 256, hash_id); + + block_source_from_strbuf(&source, &buf); + + err = init_reader(&rd, &source, "file.ref"); + EXPECT_ERR(err); + EXPECT(hash_id == reftable_reader_hash_id(&rd)); + + if (!index) { + rd.ref_offsets.index_offset = 0; + } else { + EXPECT(rd.ref_offsets.index_offset > 0); + } + + for (i = 1; i < N; i++) { + int err = reftable_reader_seek_ref(&rd, &it, names[i]); + EXPECT_ERR(err); + err = reftable_iterator_next_ref(&it, &ref); + EXPECT_ERR(err); + EXPECT(0 == strcmp(names[i], ref.refname)); + EXPECT(REFTABLE_REF_VAL1 == ref.value_type); + EXPECT(i == ref.value.val1[0]); + + reftable_ref_record_release(&ref); + reftable_iterator_destroy(&it); + } + + strbuf_addstr(&pastLast, names[N - 1]); + strbuf_addstr(&pastLast, "/"); + + err = reftable_reader_seek_ref(&rd, &it, pastLast.buf); + if (err == 0) { + struct reftable_ref_record ref = { NULL }; + int err = reftable_iterator_next_ref(&it, &ref); + EXPECT(err > 0); + } else { + EXPECT(err > 0); + } + + strbuf_release(&pastLast); + reftable_iterator_destroy(&it); + + strbuf_release(&buf); + for (i = 0; i < N; i++) { + reftable_free(names[i]); + } + reftable_free(names); + reader_close(&rd); +} + +static void test_table_read_write_seek_linear(void) +{ + test_table_read_write_seek(0, GIT_SHA1_FORMAT_ID); +} + +static void test_table_read_write_seek_linear_sha256(void) +{ + test_table_read_write_seek(0, GIT_SHA256_FORMAT_ID); +} + +static void test_table_read_write_seek_index(void) +{ + test_table_read_write_seek(1, GIT_SHA1_FORMAT_ID); +} + +static void test_table_refs_for(int indexed) +{ + int N = 50; + char **want_names = reftable_calloc(sizeof(char *) * (N + 1)); + int want_names_len = 0; + uint8_t want_hash[GIT_SHA1_RAWSZ]; + + struct reftable_write_options opts = { + .block_size = 256, + }; + struct reftable_ref_record ref = { NULL }; + int i = 0; + int n; + int err; + struct reftable_reader rd; + struct reftable_block_source source = { NULL }; + + struct strbuf buf = STRBUF_INIT; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, &buf, &opts); + + struct reftable_iterator it = { NULL }; + int j; + + set_test_hash(want_hash, 4); + + for (i = 0; i < N; i++) { + uint8_t hash[GIT_SHA1_RAWSZ]; + char fill[51] = { 0 }; + char name[100]; + uint8_t hash1[GIT_SHA1_RAWSZ]; + uint8_t hash2[GIT_SHA1_RAWSZ]; + struct reftable_ref_record ref = { NULL }; + + memset(hash, i, sizeof(hash)); + memset(fill, 'x', 50); + /* Put the variable part in the start */ + snprintf(name, sizeof(name), "br%02d%s", i, fill); + name[40] = 0; + ref.refname = name; + + set_test_hash(hash1, i / 4); + set_test_hash(hash2, 3 + i / 4); + ref.value_type = REFTABLE_REF_VAL2; + ref.value.val2.value = hash1; + ref.value.val2.target_value = hash2; + + /* 80 bytes / entry, so 3 entries per block. Yields 17 + */ + /* blocks. */ + n = reftable_writer_add_ref(w, &ref); + EXPECT(n == 0); + + if (!memcmp(hash1, want_hash, GIT_SHA1_RAWSZ) || + !memcmp(hash2, want_hash, GIT_SHA1_RAWSZ)) { + want_names[want_names_len++] = xstrdup(name); + } + } + + n = reftable_writer_close(w); + EXPECT(n == 0); + + reftable_writer_free(w); + w = NULL; + + block_source_from_strbuf(&source, &buf); + + err = init_reader(&rd, &source, "file.ref"); + EXPECT_ERR(err); + if (!indexed) { + rd.obj_offsets.is_present = 0; + } + + err = reftable_reader_seek_ref(&rd, &it, ""); + EXPECT_ERR(err); + reftable_iterator_destroy(&it); + + err = reftable_reader_refs_for(&rd, &it, want_hash); + EXPECT_ERR(err); + + j = 0; + while (1) { + int err = reftable_iterator_next_ref(&it, &ref); + EXPECT(err >= 0); + if (err > 0) { + break; + } + + EXPECT(j < want_names_len); + EXPECT(0 == strcmp(ref.refname, want_names[j])); + j++; + reftable_ref_record_release(&ref); + } + EXPECT(j == want_names_len); + + strbuf_release(&buf); + free_names(want_names); + reftable_iterator_destroy(&it); + reader_close(&rd); +} + +static void test_table_refs_for_no_index(void) +{ + test_table_refs_for(0); +} + +static void test_table_refs_for_obj_index(void) +{ + test_table_refs_for(1); +} + +static void test_write_empty_table(void) +{ + struct reftable_write_options opts = { 0 }; + struct strbuf buf = STRBUF_INIT; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, &buf, &opts); + struct reftable_block_source source = { NULL }; + struct reftable_reader *rd = NULL; + struct reftable_ref_record rec = { NULL }; + struct reftable_iterator it = { NULL }; + int err; + + reftable_writer_set_limits(w, 1, 1); + + err = reftable_writer_close(w); + EXPECT(err == REFTABLE_EMPTY_TABLE_ERROR); + reftable_writer_free(w); + + EXPECT(buf.len == header_size(1) + footer_size(1)); + + block_source_from_strbuf(&source, &buf); + + err = reftable_new_reader(&rd, &source, "filename"); + EXPECT_ERR(err); + + err = reftable_reader_seek_ref(rd, &it, ""); + EXPECT_ERR(err); + + err = reftable_iterator_next_ref(&it, &rec); + EXPECT(err > 0); + + reftable_iterator_destroy(&it); + reftable_reader_free(rd); + strbuf_release(&buf); +} + +static void test_write_key_order(void) +{ + struct reftable_write_options opts = { 0 }; + struct strbuf buf = STRBUF_INIT; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, &buf, &opts); + struct reftable_ref_record refs[2] = { + { + .refname = "b", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value = { + .symref = "target", + }, + }, { + .refname = "a", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value = { + .symref = "target", + }, + } + }; + int err; + + reftable_writer_set_limits(w, 1, 1); + err = reftable_writer_add_ref(w, &refs[0]); + EXPECT_ERR(err); + err = reftable_writer_add_ref(w, &refs[1]); + printf("%d\n", err); + EXPECT(err == REFTABLE_API_ERROR); + reftable_writer_close(w); + reftable_writer_free(w); + strbuf_release(&buf); +} + +static void test_corrupt_table_empty(void) +{ + struct strbuf buf = STRBUF_INIT; + struct reftable_block_source source = { NULL }; + struct reftable_reader rd = { NULL }; + int err; + + block_source_from_strbuf(&source, &buf); + err = init_reader(&rd, &source, "file.log"); + EXPECT(err == REFTABLE_FORMAT_ERROR); +} + +static void test_corrupt_table(void) +{ + uint8_t zeros[1024] = { 0 }; + struct strbuf buf = STRBUF_INIT; + struct reftable_block_source source = { NULL }; + struct reftable_reader rd = { NULL }; + int err; + strbuf_add(&buf, zeros, sizeof(zeros)); + + block_source_from_strbuf(&source, &buf); + err = init_reader(&rd, &source, "file.log"); + EXPECT(err == REFTABLE_FORMAT_ERROR); + strbuf_release(&buf); +} + +int readwrite_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_corrupt_table); + RUN_TEST(test_corrupt_table_empty); + RUN_TEST(test_log_write_read); + RUN_TEST(test_write_key_order); + RUN_TEST(test_table_read_write_seek_linear_sha256); + RUN_TEST(test_log_buffer_size); + RUN_TEST(test_table_write_small_table); + RUN_TEST(test_buffer); + RUN_TEST(test_table_read_api); + RUN_TEST(test_table_read_write_sequential); + RUN_TEST(test_table_read_write_seek_linear); + RUN_TEST(test_table_read_write_seek_index); + RUN_TEST(test_table_refs_for_no_index); + RUN_TEST(test_table_refs_for_obj_index); + RUN_TEST(test_write_empty_table); + return 0; +} diff --git a/reftable/record.c b/reftable/record.c new file mode 100644 index 0000000000..6a5dac32dc --- /dev/null +++ b/reftable/record.c @@ -0,0 +1,1212 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +/* record.c - methods for different types of records. */ + +#include "record.h" + +#include "system.h" +#include "constants.h" +#include "reftable-error.h" +#include "basics.h" + +int get_var_int(uint64_t *dest, struct string_view *in) +{ + int ptr = 0; + uint64_t val; + + if (in->len == 0) + return -1; + val = in->buf[ptr] & 0x7f; + + while (in->buf[ptr] & 0x80) { + ptr++; + if (ptr > in->len) { + return -1; + } + val = (val + 1) << 7 | (uint64_t)(in->buf[ptr] & 0x7f); + } + + *dest = val; + return ptr + 1; +} + +int put_var_int(struct string_view *dest, uint64_t val) +{ + uint8_t buf[10] = { 0 }; + int i = 9; + int n = 0; + buf[i] = (uint8_t)(val & 0x7f); + i--; + while (1) { + val >>= 7; + if (!val) { + break; + } + val--; + buf[i] = 0x80 | (uint8_t)(val & 0x7f); + i--; + } + + n = sizeof(buf) - i - 1; + if (dest->len < n) + return -1; + memcpy(dest->buf, &buf[i + 1], n); + return n; +} + +int reftable_is_block_type(uint8_t typ) +{ + switch (typ) { + case BLOCK_TYPE_REF: + case BLOCK_TYPE_LOG: + case BLOCK_TYPE_OBJ: + case BLOCK_TYPE_INDEX: + return 1; + } + return 0; +} + +uint8_t *reftable_ref_record_val1(struct reftable_ref_record *rec) +{ + switch (rec->value_type) { + case REFTABLE_REF_VAL1: + return rec->value.val1; + case REFTABLE_REF_VAL2: + return rec->value.val2.value; + default: + return NULL; + } +} + +uint8_t *reftable_ref_record_val2(struct reftable_ref_record *rec) +{ + switch (rec->value_type) { + case REFTABLE_REF_VAL2: + return rec->value.val2.target_value; + default: + return NULL; + } +} + +static int decode_string(struct strbuf *dest, struct string_view in) +{ + int start_len = in.len; + uint64_t tsize = 0; + int n = get_var_int(&tsize, &in); + if (n <= 0) + return -1; + string_view_consume(&in, n); + if (in.len < tsize) + return -1; + + strbuf_reset(dest); + strbuf_add(dest, in.buf, tsize); + string_view_consume(&in, tsize); + + return start_len - in.len; +} + +static int encode_string(char *str, struct string_view s) +{ + struct string_view start = s; + int l = strlen(str); + int n = put_var_int(&s, l); + if (n < 0) + return -1; + string_view_consume(&s, n); + if (s.len < l) + return -1; + memcpy(s.buf, str, l); + string_view_consume(&s, l); + + return start.len - s.len; +} + +int reftable_encode_key(int *restart, struct string_view dest, + struct strbuf prev_key, struct strbuf key, + uint8_t extra) +{ + struct string_view start = dest; + int prefix_len = common_prefix_size(&prev_key, &key); + uint64_t suffix_len = key.len - prefix_len; + int n = put_var_int(&dest, (uint64_t)prefix_len); + if (n < 0) + return -1; + string_view_consume(&dest, n); + + *restart = (prefix_len == 0); + + n = put_var_int(&dest, suffix_len << 3 | (uint64_t)extra); + if (n < 0) + return -1; + string_view_consume(&dest, n); + + if (dest.len < suffix_len) + return -1; + memcpy(dest.buf, key.buf + prefix_len, suffix_len); + string_view_consume(&dest, suffix_len); + + return start.len - dest.len; +} + +int reftable_decode_key(struct strbuf *key, uint8_t *extra, + struct strbuf last_key, struct string_view in) +{ + int start_len = in.len; + uint64_t prefix_len = 0; + uint64_t suffix_len = 0; + int n = get_var_int(&prefix_len, &in); + if (n < 0) + return -1; + string_view_consume(&in, n); + + if (prefix_len > last_key.len) + return -1; + + n = get_var_int(&suffix_len, &in); + if (n <= 0) + return -1; + string_view_consume(&in, n); + + *extra = (uint8_t)(suffix_len & 0x7); + suffix_len >>= 3; + + if (in.len < suffix_len) + return -1; + + strbuf_reset(key); + strbuf_add(key, last_key.buf, prefix_len); + strbuf_add(key, in.buf, suffix_len); + string_view_consume(&in, suffix_len); + + return start_len - in.len; +} + +static void reftable_ref_record_key(const void *r, struct strbuf *dest) +{ + const struct reftable_ref_record *rec = + (const struct reftable_ref_record *)r; + strbuf_reset(dest); + strbuf_addstr(dest, rec->refname); +} + +static void reftable_ref_record_copy_from(void *rec, const void *src_rec, + int hash_size) +{ + struct reftable_ref_record *ref = rec; + const struct reftable_ref_record *src = src_rec; + assert(hash_size > 0); + + /* This is simple and correct, but we could probably reuse the hash + * fields. */ + reftable_ref_record_release(ref); + if (src->refname) { + ref->refname = xstrdup(src->refname); + } + ref->update_index = src->update_index; + ref->value_type = src->value_type; + switch (src->value_type) { + case REFTABLE_REF_DELETION: + break; + case REFTABLE_REF_VAL1: + ref->value.val1 = reftable_malloc(hash_size); + memcpy(ref->value.val1, src->value.val1, hash_size); + break; + case REFTABLE_REF_VAL2: + ref->value.val2.value = reftable_malloc(hash_size); + memcpy(ref->value.val2.value, src->value.val2.value, hash_size); + ref->value.val2.target_value = reftable_malloc(hash_size); + memcpy(ref->value.val2.target_value, + src->value.val2.target_value, hash_size); + break; + case REFTABLE_REF_SYMREF: + ref->value.symref = xstrdup(src->value.symref); + break; + } +} + +static char hexdigit(int c) +{ + if (c <= 9) + return '0' + c; + return 'a' + (c - 10); +} + +static void hex_format(char *dest, uint8_t *src, int hash_size) +{ + assert(hash_size > 0); + if (src) { + int i = 0; + for (i = 0; i < hash_size; i++) { + dest[2 * i] = hexdigit(src[i] >> 4); + dest[2 * i + 1] = hexdigit(src[i] & 0xf); + } + dest[2 * hash_size] = 0; + } +} + +void reftable_ref_record_print(struct reftable_ref_record *ref, + uint32_t hash_id) +{ + char hex[2 * GIT_SHA256_RAWSZ + 1] = { 0 }; /* BUG */ + printf("ref{%s(%" PRIu64 ") ", ref->refname, ref->update_index); + switch (ref->value_type) { + case REFTABLE_REF_SYMREF: + printf("=> %s", ref->value.symref); + break; + case REFTABLE_REF_VAL2: + hex_format(hex, ref->value.val2.value, hash_size(hash_id)); + printf("val 2 %s", hex); + hex_format(hex, ref->value.val2.target_value, + hash_size(hash_id)); + printf("(T %s)", hex); + break; + case REFTABLE_REF_VAL1: + hex_format(hex, ref->value.val1, hash_size(hash_id)); + printf("val 1 %s", hex); + break; + case REFTABLE_REF_DELETION: + printf("delete"); + break; + } + printf("}\n"); +} + +static void reftable_ref_record_release_void(void *rec) +{ + reftable_ref_record_release(rec); +} + +void reftable_ref_record_release(struct reftable_ref_record *ref) +{ + switch (ref->value_type) { + case REFTABLE_REF_SYMREF: + reftable_free(ref->value.symref); + break; + case REFTABLE_REF_VAL2: + reftable_free(ref->value.val2.target_value); + reftable_free(ref->value.val2.value); + break; + case REFTABLE_REF_VAL1: + reftable_free(ref->value.val1); + break; + case REFTABLE_REF_DELETION: + break; + default: + abort(); + } + + reftable_free(ref->refname); + memset(ref, 0, sizeof(struct reftable_ref_record)); +} + +static uint8_t reftable_ref_record_val_type(const void *rec) +{ + const struct reftable_ref_record *r = + (const struct reftable_ref_record *)rec; + return r->value_type; +} + +static int reftable_ref_record_encode(const void *rec, struct string_view s, + int hash_size) +{ + const struct reftable_ref_record *r = + (const struct reftable_ref_record *)rec; + struct string_view start = s; + int n = put_var_int(&s, r->update_index); + assert(hash_size > 0); + if (n < 0) + return -1; + string_view_consume(&s, n); + + switch (r->value_type) { + case REFTABLE_REF_SYMREF: + n = encode_string(r->value.symref, s); + if (n < 0) { + return -1; + } + string_view_consume(&s, n); + break; + case REFTABLE_REF_VAL2: + if (s.len < 2 * hash_size) { + return -1; + } + memcpy(s.buf, r->value.val2.value, hash_size); + string_view_consume(&s, hash_size); + memcpy(s.buf, r->value.val2.target_value, hash_size); + string_view_consume(&s, hash_size); + break; + case REFTABLE_REF_VAL1: + if (s.len < hash_size) { + return -1; + } + memcpy(s.buf, r->value.val1, hash_size); + string_view_consume(&s, hash_size); + break; + case REFTABLE_REF_DELETION: + break; + default: + abort(); + } + + return start.len - s.len; +} + +static int reftable_ref_record_decode(void *rec, struct strbuf key, + uint8_t val_type, struct string_view in, + int hash_size) +{ + struct reftable_ref_record *r = rec; + struct string_view start = in; + uint64_t update_index = 0; + int n = get_var_int(&update_index, &in); + if (n < 0) + return n; + string_view_consume(&in, n); + + reftable_ref_record_release(r); + + assert(hash_size > 0); + + r->refname = reftable_realloc(r->refname, key.len + 1); + memcpy(r->refname, key.buf, key.len); + r->update_index = update_index; + r->refname[key.len] = 0; + r->value_type = val_type; + switch (val_type) { + case REFTABLE_REF_VAL1: + if (in.len < hash_size) { + return -1; + } + + r->value.val1 = reftable_malloc(hash_size); + memcpy(r->value.val1, in.buf, hash_size); + string_view_consume(&in, hash_size); + break; + + case REFTABLE_REF_VAL2: + if (in.len < 2 * hash_size) { + return -1; + } + + r->value.val2.value = reftable_malloc(hash_size); + memcpy(r->value.val2.value, in.buf, hash_size); + string_view_consume(&in, hash_size); + + r->value.val2.target_value = reftable_malloc(hash_size); + memcpy(r->value.val2.target_value, in.buf, hash_size); + string_view_consume(&in, hash_size); + break; + + case REFTABLE_REF_SYMREF: { + struct strbuf dest = STRBUF_INIT; + int n = decode_string(&dest, in); + if (n < 0) { + return -1; + } + string_view_consume(&in, n); + r->value.symref = dest.buf; + } break; + + case REFTABLE_REF_DELETION: + break; + default: + abort(); + break; + } + + return start.len - in.len; +} + +static int reftable_ref_record_is_deletion_void(const void *p) +{ + return reftable_ref_record_is_deletion( + (const struct reftable_ref_record *)p); +} + +static struct reftable_record_vtable reftable_ref_record_vtable = { + .key = &reftable_ref_record_key, + .type = BLOCK_TYPE_REF, + .copy_from = &reftable_ref_record_copy_from, + .val_type = &reftable_ref_record_val_type, + .encode = &reftable_ref_record_encode, + .decode = &reftable_ref_record_decode, + .release = &reftable_ref_record_release_void, + .is_deletion = &reftable_ref_record_is_deletion_void, +}; + +static void reftable_obj_record_key(const void *r, struct strbuf *dest) +{ + const struct reftable_obj_record *rec = + (const struct reftable_obj_record *)r; + strbuf_reset(dest); + strbuf_add(dest, rec->hash_prefix, rec->hash_prefix_len); +} + +static void reftable_obj_record_release(void *rec) +{ + struct reftable_obj_record *obj = rec; + FREE_AND_NULL(obj->hash_prefix); + FREE_AND_NULL(obj->offsets); + memset(obj, 0, sizeof(struct reftable_obj_record)); +} + +static void reftable_obj_record_copy_from(void *rec, const void *src_rec, + int hash_size) +{ + struct reftable_obj_record *obj = rec; + const struct reftable_obj_record *src = + (const struct reftable_obj_record *)src_rec; + + reftable_obj_record_release(obj); + *obj = *src; + obj->hash_prefix = reftable_malloc(obj->hash_prefix_len); + memcpy(obj->hash_prefix, src->hash_prefix, obj->hash_prefix_len); + + obj->offsets = reftable_malloc(obj->offset_len * sizeof(uint64_t)); + COPY_ARRAY(obj->offsets, src->offsets, obj->offset_len); +} + +static uint8_t reftable_obj_record_val_type(const void *rec) +{ + const struct reftable_obj_record *r = rec; + if (r->offset_len > 0 && r->offset_len < 8) + return r->offset_len; + return 0; +} + +static int reftable_obj_record_encode(const void *rec, struct string_view s, + int hash_size) +{ + const struct reftable_obj_record *r = rec; + struct string_view start = s; + int i = 0; + int n = 0; + uint64_t last = 0; + if (r->offset_len == 0 || r->offset_len >= 8) { + n = put_var_int(&s, r->offset_len); + if (n < 0) { + return -1; + } + string_view_consume(&s, n); + } + if (r->offset_len == 0) + return start.len - s.len; + n = put_var_int(&s, r->offsets[0]); + if (n < 0) + return -1; + string_view_consume(&s, n); + + last = r->offsets[0]; + for (i = 1; i < r->offset_len; i++) { + int n = put_var_int(&s, r->offsets[i] - last); + if (n < 0) { + return -1; + } + string_view_consume(&s, n); + last = r->offsets[i]; + } + return start.len - s.len; +} + +static int reftable_obj_record_decode(void *rec, struct strbuf key, + uint8_t val_type, struct string_view in, + int hash_size) +{ + struct string_view start = in; + struct reftable_obj_record *r = rec; + uint64_t count = val_type; + int n = 0; + uint64_t last; + int j; + r->hash_prefix = reftable_malloc(key.len); + memcpy(r->hash_prefix, key.buf, key.len); + r->hash_prefix_len = key.len; + + if (val_type == 0) { + n = get_var_int(&count, &in); + if (n < 0) { + return n; + } + + string_view_consume(&in, n); + } + + r->offsets = NULL; + r->offset_len = 0; + if (count == 0) + return start.len - in.len; + + r->offsets = reftable_malloc(count * sizeof(uint64_t)); + r->offset_len = count; + + n = get_var_int(&r->offsets[0], &in); + if (n < 0) + return n; + string_view_consume(&in, n); + + last = r->offsets[0]; + j = 1; + while (j < count) { + uint64_t delta = 0; + int n = get_var_int(&delta, &in); + if (n < 0) { + return n; + } + string_view_consume(&in, n); + + last = r->offsets[j] = (delta + last); + j++; + } + return start.len - in.len; +} + +static int not_a_deletion(const void *p) +{ + return 0; +} + +static struct reftable_record_vtable reftable_obj_record_vtable = { + .key = &reftable_obj_record_key, + .type = BLOCK_TYPE_OBJ, + .copy_from = &reftable_obj_record_copy_from, + .val_type = &reftable_obj_record_val_type, + .encode = &reftable_obj_record_encode, + .decode = &reftable_obj_record_decode, + .release = &reftable_obj_record_release, + .is_deletion = not_a_deletion, +}; + +void reftable_log_record_print(struct reftable_log_record *log, + uint32_t hash_id) +{ + char hex[GIT_SHA256_RAWSZ + 1] = { 0 }; + + switch (log->value_type) { + case REFTABLE_LOG_DELETION: + printf("log{%s(%" PRIu64 ") delete", log->refname, + log->update_index); + break; + case REFTABLE_LOG_UPDATE: + printf("log{%s(%" PRIu64 ") %s <%s> %" PRIu64 " %04d\n", + log->refname, log->update_index, log->value.update.name, + log->value.update.email, log->value.update.time, + log->value.update.tz_offset); + hex_format(hex, log->value.update.old_hash, hash_size(hash_id)); + printf("%s => ", hex); + hex_format(hex, log->value.update.new_hash, hash_size(hash_id)); + printf("%s\n\n%s\n}\n", hex, log->value.update.message); + break; + } +} + +static void reftable_log_record_key(const void *r, struct strbuf *dest) +{ + const struct reftable_log_record *rec = + (const struct reftable_log_record *)r; + int len = strlen(rec->refname); + uint8_t i64[8]; + uint64_t ts = 0; + strbuf_reset(dest); + strbuf_add(dest, (uint8_t *)rec->refname, len + 1); + + ts = (~ts) - rec->update_index; + put_be64(&i64[0], ts); + strbuf_add(dest, i64, sizeof(i64)); +} + +static void reftable_log_record_copy_from(void *rec, const void *src_rec, + int hash_size) +{ + struct reftable_log_record *dst = rec; + const struct reftable_log_record *src = + (const struct reftable_log_record *)src_rec; + + reftable_log_record_release(dst); + *dst = *src; + if (dst->refname) { + dst->refname = xstrdup(dst->refname); + } + switch (dst->value_type) { + case REFTABLE_LOG_DELETION: + break; + case REFTABLE_LOG_UPDATE: + if (dst->value.update.email) { + dst->value.update.email = + xstrdup(dst->value.update.email); + } + if (dst->value.update.name) { + dst->value.update.name = + xstrdup(dst->value.update.name); + } + if (dst->value.update.message) { + dst->value.update.message = + xstrdup(dst->value.update.message); + } + + if (dst->value.update.new_hash) { + dst->value.update.new_hash = reftable_malloc(hash_size); + memcpy(dst->value.update.new_hash, + src->value.update.new_hash, hash_size); + } + if (dst->value.update.old_hash) { + dst->value.update.old_hash = reftable_malloc(hash_size); + memcpy(dst->value.update.old_hash, + src->value.update.old_hash, hash_size); + } + break; + } +} + +static void reftable_log_record_release_void(void *rec) +{ + struct reftable_log_record *r = rec; + reftable_log_record_release(r); +} + +void reftable_log_record_release(struct reftable_log_record *r) +{ + reftable_free(r->refname); + switch (r->value_type) { + case REFTABLE_LOG_DELETION: + break; + case REFTABLE_LOG_UPDATE: + reftable_free(r->value.update.new_hash); + reftable_free(r->value.update.old_hash); + reftable_free(r->value.update.name); + reftable_free(r->value.update.email); + reftable_free(r->value.update.message); + break; + } + memset(r, 0, sizeof(struct reftable_log_record)); +} + +static uint8_t reftable_log_record_val_type(const void *rec) +{ + const struct reftable_log_record *log = + (const struct reftable_log_record *)rec; + + return reftable_log_record_is_deletion(log) ? 0 : 1; +} + +static uint8_t zero[GIT_SHA256_RAWSZ] = { 0 }; + +static int reftable_log_record_encode(const void *rec, struct string_view s, + int hash_size) +{ + const struct reftable_log_record *r = rec; + struct string_view start = s; + int n = 0; + uint8_t *oldh = NULL; + uint8_t *newh = NULL; + if (reftable_log_record_is_deletion(r)) + return 0; + + oldh = r->value.update.old_hash; + newh = r->value.update.new_hash; + if (!oldh) { + oldh = zero; + } + if (!newh) { + newh = zero; + } + + if (s.len < 2 * hash_size) + return -1; + + memcpy(s.buf, oldh, hash_size); + memcpy(s.buf + hash_size, newh, hash_size); + string_view_consume(&s, 2 * hash_size); + + n = encode_string(r->value.update.name ? r->value.update.name : "", s); + if (n < 0) + return -1; + string_view_consume(&s, n); + + n = encode_string(r->value.update.email ? r->value.update.email : "", + s); + if (n < 0) + return -1; + string_view_consume(&s, n); + + n = put_var_int(&s, r->value.update.time); + if (n < 0) + return -1; + string_view_consume(&s, n); + + if (s.len < 2) + return -1; + + put_be16(s.buf, r->value.update.tz_offset); + string_view_consume(&s, 2); + + n = encode_string( + r->value.update.message ? r->value.update.message : "", s); + if (n < 0) + return -1; + string_view_consume(&s, n); + + return start.len - s.len; +} + +static int reftable_log_record_decode(void *rec, struct strbuf key, + uint8_t val_type, struct string_view in, + int hash_size) +{ + struct string_view start = in; + struct reftable_log_record *r = rec; + uint64_t max = 0; + uint64_t ts = 0; + struct strbuf dest = STRBUF_INIT; + int n; + + if (key.len <= 9 || key.buf[key.len - 9] != 0) + return REFTABLE_FORMAT_ERROR; + + r->refname = reftable_realloc(r->refname, key.len - 8); + memcpy(r->refname, key.buf, key.len - 8); + ts = get_be64(key.buf + key.len - 8); + + r->update_index = (~max) - ts; + + if (val_type != r->value_type) { + switch (r->value_type) { + case REFTABLE_LOG_UPDATE: + FREE_AND_NULL(r->value.update.old_hash); + FREE_AND_NULL(r->value.update.new_hash); + FREE_AND_NULL(r->value.update.message); + FREE_AND_NULL(r->value.update.email); + FREE_AND_NULL(r->value.update.name); + break; + case REFTABLE_LOG_DELETION: + break; + } + } + + r->value_type = val_type; + if (val_type == REFTABLE_LOG_DELETION) + return 0; + + if (in.len < 2 * hash_size) + return REFTABLE_FORMAT_ERROR; + + r->value.update.old_hash = + reftable_realloc(r->value.update.old_hash, hash_size); + r->value.update.new_hash = + reftable_realloc(r->value.update.new_hash, hash_size); + + memcpy(r->value.update.old_hash, in.buf, hash_size); + memcpy(r->value.update.new_hash, in.buf + hash_size, hash_size); + + string_view_consume(&in, 2 * hash_size); + + n = decode_string(&dest, in); + if (n < 0) + goto done; + string_view_consume(&in, n); + + r->value.update.name = + reftable_realloc(r->value.update.name, dest.len + 1); + memcpy(r->value.update.name, dest.buf, dest.len); + r->value.update.name[dest.len] = 0; + + strbuf_reset(&dest); + n = decode_string(&dest, in); + if (n < 0) + goto done; + string_view_consume(&in, n); + + r->value.update.email = + reftable_realloc(r->value.update.email, dest.len + 1); + memcpy(r->value.update.email, dest.buf, dest.len); + r->value.update.email[dest.len] = 0; + + ts = 0; + n = get_var_int(&ts, &in); + if (n < 0) + goto done; + string_view_consume(&in, n); + r->value.update.time = ts; + if (in.len < 2) + goto done; + + r->value.update.tz_offset = get_be16(in.buf); + string_view_consume(&in, 2); + + strbuf_reset(&dest); + n = decode_string(&dest, in); + if (n < 0) + goto done; + string_view_consume(&in, n); + + r->value.update.message = + reftable_realloc(r->value.update.message, dest.len + 1); + memcpy(r->value.update.message, dest.buf, dest.len); + r->value.update.message[dest.len] = 0; + + strbuf_release(&dest); + return start.len - in.len; + +done: + strbuf_release(&dest); + return REFTABLE_FORMAT_ERROR; +} + +static int null_streq(char *a, char *b) +{ + char *empty = ""; + if (!a) + a = empty; + + if (!b) + b = empty; + + return 0 == strcmp(a, b); +} + +static int zero_hash_eq(uint8_t *a, uint8_t *b, int sz) +{ + if (!a) + a = zero; + + if (!b) + b = zero; + + return !memcmp(a, b, sz); +} + +int reftable_log_record_equal(struct reftable_log_record *a, + struct reftable_log_record *b, int hash_size) +{ + if (!(null_streq(a->refname, b->refname) && + a->update_index == b->update_index && + a->value_type == b->value_type)) + return 0; + + switch (a->value_type) { + case REFTABLE_LOG_DELETION: + return 1; + case REFTABLE_LOG_UPDATE: + return null_streq(a->value.update.name, b->value.update.name) && + a->value.update.time == b->value.update.time && + a->value.update.tz_offset == b->value.update.tz_offset && + null_streq(a->value.update.email, + b->value.update.email) && + null_streq(a->value.update.message, + b->value.update.message) && + zero_hash_eq(a->value.update.old_hash, + b->value.update.old_hash, hash_size) && + zero_hash_eq(a->value.update.new_hash, + b->value.update.new_hash, hash_size); + } + + abort(); +} + +static int reftable_log_record_is_deletion_void(const void *p) +{ + return reftable_log_record_is_deletion( + (const struct reftable_log_record *)p); +} + +static struct reftable_record_vtable reftable_log_record_vtable = { + .key = &reftable_log_record_key, + .type = BLOCK_TYPE_LOG, + .copy_from = &reftable_log_record_copy_from, + .val_type = &reftable_log_record_val_type, + .encode = &reftable_log_record_encode, + .decode = &reftable_log_record_decode, + .release = &reftable_log_record_release_void, + .is_deletion = &reftable_log_record_is_deletion_void, +}; + +struct reftable_record reftable_new_record(uint8_t typ) +{ + struct reftable_record rec = { NULL }; + switch (typ) { + case BLOCK_TYPE_REF: { + struct reftable_ref_record *r = + reftable_calloc(sizeof(struct reftable_ref_record)); + reftable_record_from_ref(&rec, r); + return rec; + } + + case BLOCK_TYPE_OBJ: { + struct reftable_obj_record *r = + reftable_calloc(sizeof(struct reftable_obj_record)); + reftable_record_from_obj(&rec, r); + return rec; + } + case BLOCK_TYPE_LOG: { + struct reftable_log_record *r = + reftable_calloc(sizeof(struct reftable_log_record)); + reftable_record_from_log(&rec, r); + return rec; + } + case BLOCK_TYPE_INDEX: { + struct reftable_index_record empty = { .last_key = + STRBUF_INIT }; + struct reftable_index_record *r = + reftable_calloc(sizeof(struct reftable_index_record)); + *r = empty; + reftable_record_from_index(&rec, r); + return rec; + } + } + abort(); + return rec; +} + +/* clear out the record, yielding the reftable_record data that was + * encapsulated. */ +static void *reftable_record_yield(struct reftable_record *rec) +{ + void *p = rec->data; + rec->data = NULL; + return p; +} + +void reftable_record_destroy(struct reftable_record *rec) +{ + reftable_record_release(rec); + reftable_free(reftable_record_yield(rec)); +} + +static void reftable_index_record_key(const void *r, struct strbuf *dest) +{ + const struct reftable_index_record *rec = r; + strbuf_reset(dest); + strbuf_addbuf(dest, &rec->last_key); +} + +static void reftable_index_record_copy_from(void *rec, const void *src_rec, + int hash_size) +{ + struct reftable_index_record *dst = rec; + const struct reftable_index_record *src = src_rec; + + strbuf_reset(&dst->last_key); + strbuf_addbuf(&dst->last_key, &src->last_key); + dst->offset = src->offset; +} + +static void reftable_index_record_release(void *rec) +{ + struct reftable_index_record *idx = rec; + strbuf_release(&idx->last_key); +} + +static uint8_t reftable_index_record_val_type(const void *rec) +{ + return 0; +} + +static int reftable_index_record_encode(const void *rec, struct string_view out, + int hash_size) +{ + const struct reftable_index_record *r = + (const struct reftable_index_record *)rec; + struct string_view start = out; + + int n = put_var_int(&out, r->offset); + if (n < 0) + return n; + + string_view_consume(&out, n); + + return start.len - out.len; +} + +static int reftable_index_record_decode(void *rec, struct strbuf key, + uint8_t val_type, struct string_view in, + int hash_size) +{ + struct string_view start = in; + struct reftable_index_record *r = rec; + int n = 0; + + strbuf_reset(&r->last_key); + strbuf_addbuf(&r->last_key, &key); + + n = get_var_int(&r->offset, &in); + if (n < 0) + return n; + + string_view_consume(&in, n); + return start.len - in.len; +} + +static struct reftable_record_vtable reftable_index_record_vtable = { + .key = &reftable_index_record_key, + .type = BLOCK_TYPE_INDEX, + .copy_from = &reftable_index_record_copy_from, + .val_type = &reftable_index_record_val_type, + .encode = &reftable_index_record_encode, + .decode = &reftable_index_record_decode, + .release = &reftable_index_record_release, + .is_deletion = ¬_a_deletion, +}; + +void reftable_record_key(struct reftable_record *rec, struct strbuf *dest) +{ + rec->ops->key(rec->data, dest); +} + +uint8_t reftable_record_type(struct reftable_record *rec) +{ + return rec->ops->type; +} + +int reftable_record_encode(struct reftable_record *rec, struct string_view dest, + int hash_size) +{ + return rec->ops->encode(rec->data, dest, hash_size); +} + +void reftable_record_copy_from(struct reftable_record *rec, + struct reftable_record *src, int hash_size) +{ + assert(src->ops->type == rec->ops->type); + + rec->ops->copy_from(rec->data, src->data, hash_size); +} + +uint8_t reftable_record_val_type(struct reftable_record *rec) +{ + return rec->ops->val_type(rec->data); +} + +int reftable_record_decode(struct reftable_record *rec, struct strbuf key, + uint8_t extra, struct string_view src, int hash_size) +{ + return rec->ops->decode(rec->data, key, extra, src, hash_size); +} + +void reftable_record_release(struct reftable_record *rec) +{ + rec->ops->release(rec->data); +} + +int reftable_record_is_deletion(struct reftable_record *rec) +{ + return rec->ops->is_deletion(rec->data); +} + +void reftable_record_from_ref(struct reftable_record *rec, + struct reftable_ref_record *ref_rec) +{ + assert(!rec->ops); + rec->data = ref_rec; + rec->ops = &reftable_ref_record_vtable; +} + +void reftable_record_from_obj(struct reftable_record *rec, + struct reftable_obj_record *obj_rec) +{ + assert(!rec->ops); + rec->data = obj_rec; + rec->ops = &reftable_obj_record_vtable; +} + +void reftable_record_from_index(struct reftable_record *rec, + struct reftable_index_record *index_rec) +{ + assert(!rec->ops); + rec->data = index_rec; + rec->ops = &reftable_index_record_vtable; +} + +void reftable_record_from_log(struct reftable_record *rec, + struct reftable_log_record *log_rec) +{ + assert(!rec->ops); + rec->data = log_rec; + rec->ops = &reftable_log_record_vtable; +} + +struct reftable_ref_record *reftable_record_as_ref(struct reftable_record *rec) +{ + assert(reftable_record_type(rec) == BLOCK_TYPE_REF); + return rec->data; +} + +struct reftable_log_record *reftable_record_as_log(struct reftable_record *rec) +{ + assert(reftable_record_type(rec) == BLOCK_TYPE_LOG); + return rec->data; +} + +static int hash_equal(uint8_t *a, uint8_t *b, int hash_size) +{ + if (a && b) + return !memcmp(a, b, hash_size); + + return a == b; +} + +int reftable_ref_record_equal(struct reftable_ref_record *a, + struct reftable_ref_record *b, int hash_size) +{ + assert(hash_size > 0); + if (!(0 == strcmp(a->refname, b->refname) && + a->update_index == b->update_index && + a->value_type == b->value_type)) + return 0; + + switch (a->value_type) { + case REFTABLE_REF_SYMREF: + return !strcmp(a->value.symref, b->value.symref); + case REFTABLE_REF_VAL2: + return hash_equal(a->value.val2.value, b->value.val2.value, + hash_size) && + hash_equal(a->value.val2.target_value, + b->value.val2.target_value, hash_size); + case REFTABLE_REF_VAL1: + return hash_equal(a->value.val1, b->value.val1, hash_size); + case REFTABLE_REF_DELETION: + return 1; + default: + abort(); + } +} + +int reftable_ref_record_compare_name(const void *a, const void *b) +{ + return strcmp(((struct reftable_ref_record *)a)->refname, + ((struct reftable_ref_record *)b)->refname); +} + +int reftable_ref_record_is_deletion(const struct reftable_ref_record *ref) +{ + return ref->value_type == REFTABLE_REF_DELETION; +} + +int reftable_log_record_compare_key(const void *a, const void *b) +{ + const struct reftable_log_record *la = a; + const struct reftable_log_record *lb = b; + + int cmp = strcmp(la->refname, lb->refname); + if (cmp) + return cmp; + if (la->update_index > lb->update_index) + return -1; + return (la->update_index < lb->update_index) ? 1 : 0; +} + +int reftable_log_record_is_deletion(const struct reftable_log_record *log) +{ + return (log->value_type == REFTABLE_LOG_DELETION); +} + +void string_view_consume(struct string_view *s, int n) +{ + s->buf += n; + s->len -= n; +} diff --git a/reftable/record.h b/reftable/record.h new file mode 100644 index 0000000000..498e8c50bf --- /dev/null +++ b/reftable/record.h @@ -0,0 +1,139 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef RECORD_H +#define RECORD_H + +#include "system.h" + +#include <stdint.h> + +#include "reftable-record.h" + +/* + * A substring of existing string data. This structure takes no responsibility + * for the lifetime of the data it points to. + */ +struct string_view { + uint8_t *buf; + size_t len; +}; + +/* Advance `s.buf` by `n`, and decrease length. */ +void string_view_consume(struct string_view *s, int n); + +/* utilities for de/encoding varints */ + +int get_var_int(uint64_t *dest, struct string_view *in); +int put_var_int(struct string_view *dest, uint64_t val); + +/* Methods for records. */ +struct reftable_record_vtable { + /* encode the key of to a uint8_t strbuf. */ + void (*key)(const void *rec, struct strbuf *dest); + + /* The record type of ('r' for ref). */ + uint8_t type; + + void (*copy_from)(void *dest, const void *src, int hash_size); + + /* a value of [0..7], indicating record subvariants (eg. ref vs. symref + * vs ref deletion) */ + uint8_t (*val_type)(const void *rec); + + /* encodes rec into dest, returning how much space was used. */ + int (*encode)(const void *rec, struct string_view dest, int hash_size); + + /* decode data from `src` into the record. */ + int (*decode)(void *rec, struct strbuf key, uint8_t extra, + struct string_view src, int hash_size); + + /* deallocate and null the record. */ + void (*release)(void *rec); + + /* is this a tombstone? */ + int (*is_deletion)(const void *rec); +}; + +/* record is a generic wrapper for different types of records. */ +struct reftable_record { + void *data; + struct reftable_record_vtable *ops; +}; + +/* returns true for recognized block types. Block start with the block type. */ +int reftable_is_block_type(uint8_t typ); + +/* creates a malloced record of the given type. Dispose with record_destroy */ +struct reftable_record reftable_new_record(uint8_t typ); + +/* Encode `key` into `dest`. Sets `is_restart` to indicate a restart. Returns + * number of bytes written. */ +int reftable_encode_key(int *is_restart, struct string_view dest, + struct strbuf prev_key, struct strbuf key, + uint8_t extra); + +/* Decode into `key` and `extra` from `in` */ +int reftable_decode_key(struct strbuf *key, uint8_t *extra, + struct strbuf last_key, struct string_view in); + +/* reftable_index_record are used internally to speed up lookups. */ +struct reftable_index_record { + uint64_t offset; /* Offset of block */ + struct strbuf last_key; /* Last key of the block. */ +}; + +/* reftable_obj_record stores an object ID => ref mapping. */ +struct reftable_obj_record { + uint8_t *hash_prefix; /* leading bytes of the object ID */ + int hash_prefix_len; /* number of leading bytes. Constant + * across a single table. */ + uint64_t *offsets; /* a vector of file offsets. */ + int offset_len; +}; + +/* see struct record_vtable */ + +void reftable_record_key(struct reftable_record *rec, struct strbuf *dest); +uint8_t reftable_record_type(struct reftable_record *rec); +void reftable_record_copy_from(struct reftable_record *rec, + struct reftable_record *src, int hash_size); +uint8_t reftable_record_val_type(struct reftable_record *rec); +int reftable_record_encode(struct reftable_record *rec, struct string_view dest, + int hash_size); +int reftable_record_decode(struct reftable_record *rec, struct strbuf key, + uint8_t extra, struct string_view src, + int hash_size); +int reftable_record_is_deletion(struct reftable_record *rec); + +/* zeroes out the embedded record */ +void reftable_record_release(struct reftable_record *rec); + +/* clear and deallocate embedded record, and zero `rec`. */ +void reftable_record_destroy(struct reftable_record *rec); + +/* initialize generic records from concrete records. The generic record should + * be zeroed out. */ +void reftable_record_from_obj(struct reftable_record *rec, + struct reftable_obj_record *objrec); +void reftable_record_from_index(struct reftable_record *rec, + struct reftable_index_record *idxrec); +void reftable_record_from_ref(struct reftable_record *rec, + struct reftable_ref_record *refrec); +void reftable_record_from_log(struct reftable_record *rec, + struct reftable_log_record *logrec); +struct reftable_ref_record *reftable_record_as_ref(struct reftable_record *ref); +struct reftable_log_record *reftable_record_as_log(struct reftable_record *ref); + +/* for qsort. */ +int reftable_ref_record_compare_name(const void *a, const void *b); + +/* for qsort. */ +int reftable_log_record_compare_key(const void *a, const void *b); + +#endif diff --git a/reftable/record_test.c b/reftable/record_test.c new file mode 100644 index 0000000000..f4ad7cace4 --- /dev/null +++ b/reftable/record_test.c @@ -0,0 +1,412 @@ +/* + Copyright 2020 Google LLC + + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file or at + https://developers.google.com/open-source/licenses/bsd +*/ + +#include "record.h" + +#include "system.h" +#include "basics.h" +#include "constants.h" +#include "test_framework.h" +#include "reftable-tests.h" + +static void test_copy(struct reftable_record *rec) +{ + struct reftable_record copy = + reftable_new_record(reftable_record_type(rec)); + reftable_record_copy_from(©, rec, GIT_SHA1_RAWSZ); + /* do it twice to catch memory leaks */ + reftable_record_copy_from(©, rec, GIT_SHA1_RAWSZ); + switch (reftable_record_type(©)) { + case BLOCK_TYPE_REF: + EXPECT(reftable_ref_record_equal(reftable_record_as_ref(©), + reftable_record_as_ref(rec), + GIT_SHA1_RAWSZ)); + break; + case BLOCK_TYPE_LOG: + EXPECT(reftable_log_record_equal(reftable_record_as_log(©), + reftable_record_as_log(rec), + GIT_SHA1_RAWSZ)); + break; + } + reftable_record_destroy(©); +} + +static void test_varint_roundtrip(void) +{ + uint64_t inputs[] = { 0, + 1, + 27, + 127, + 128, + 257, + 4096, + ((uint64_t)1 << 63), + ((uint64_t)1 << 63) + ((uint64_t)1 << 63) - 1 }; + int i = 0; + for (i = 0; i < ARRAY_SIZE(inputs); i++) { + uint8_t dest[10]; + + struct string_view out = { + .buf = dest, + .len = sizeof(dest), + }; + uint64_t in = inputs[i]; + int n = put_var_int(&out, in); + uint64_t got = 0; + + EXPECT(n > 0); + out.len = n; + n = get_var_int(&got, &out); + EXPECT(n > 0); + + EXPECT(got == in); + } +} + +static void test_common_prefix(void) +{ + struct { + const char *a, *b; + int want; + } cases[] = { + { "abc", "ab", 2 }, + { "", "abc", 0 }, + { "abc", "abd", 2 }, + { "abc", "pqr", 0 }, + }; + + int i = 0; + for (i = 0; i < ARRAY_SIZE(cases); i++) { + struct strbuf a = STRBUF_INIT; + struct strbuf b = STRBUF_INIT; + strbuf_addstr(&a, cases[i].a); + strbuf_addstr(&b, cases[i].b); + EXPECT(common_prefix_size(&a, &b) == cases[i].want); + + strbuf_release(&a); + strbuf_release(&b); + } +} + +static void set_hash(uint8_t *h, int j) +{ + int i = 0; + for (i = 0; i < hash_size(GIT_SHA1_FORMAT_ID); i++) { + h[i] = (j >> i) & 0xff; + } +} + +static void test_reftable_ref_record_roundtrip(void) +{ + int i = 0; + + for (i = REFTABLE_REF_DELETION; i < REFTABLE_NR_REF_VALUETYPES; i++) { + struct reftable_ref_record in = { NULL }; + struct reftable_ref_record out = { NULL }; + struct reftable_record rec_out = { NULL }; + struct strbuf key = STRBUF_INIT; + struct reftable_record rec = { NULL }; + uint8_t buffer[1024] = { 0 }; + struct string_view dest = { + .buf = buffer, + .len = sizeof(buffer), + }; + + int n, m; + + in.value_type = i; + switch (i) { + case REFTABLE_REF_DELETION: + break; + case REFTABLE_REF_VAL1: + in.value.val1 = reftable_malloc(GIT_SHA1_RAWSZ); + set_hash(in.value.val1, 1); + break; + case REFTABLE_REF_VAL2: + in.value.val2.value = reftable_malloc(GIT_SHA1_RAWSZ); + set_hash(in.value.val2.value, 1); + in.value.val2.target_value = + reftable_malloc(GIT_SHA1_RAWSZ); + set_hash(in.value.val2.target_value, 2); + break; + case REFTABLE_REF_SYMREF: + in.value.symref = xstrdup("target"); + break; + } + in.refname = xstrdup("refs/heads/master"); + + reftable_record_from_ref(&rec, &in); + test_copy(&rec); + + EXPECT(reftable_record_val_type(&rec) == i); + + reftable_record_key(&rec, &key); + n = reftable_record_encode(&rec, dest, GIT_SHA1_RAWSZ); + EXPECT(n > 0); + + /* decode into a non-zero reftable_record to test for leaks. */ + + reftable_record_from_ref(&rec_out, &out); + m = reftable_record_decode(&rec_out, key, i, dest, + GIT_SHA1_RAWSZ); + EXPECT(n == m); + + EXPECT(reftable_ref_record_equal(&in, &out, GIT_SHA1_RAWSZ)); + reftable_record_release(&rec_out); + + strbuf_release(&key); + reftable_ref_record_release(&in); + } +} + +static void test_reftable_log_record_equal(void) +{ + struct reftable_log_record in[2] = { + { + .refname = xstrdup("refs/heads/master"), + .update_index = 42, + }, + { + .refname = xstrdup("refs/heads/master"), + .update_index = 22, + } + }; + + EXPECT(!reftable_log_record_equal(&in[0], &in[1], GIT_SHA1_RAWSZ)); + in[1].update_index = in[0].update_index; + EXPECT(reftable_log_record_equal(&in[0], &in[1], GIT_SHA1_RAWSZ)); + reftable_log_record_release(&in[0]); + reftable_log_record_release(&in[1]); +} + +static void test_reftable_log_record_roundtrip(void) +{ + int i; + struct reftable_log_record in[2] = { + { + .refname = xstrdup("refs/heads/master"), + .update_index = 42, + .value_type = REFTABLE_LOG_UPDATE, + .value = { + .update = { + .old_hash = reftable_malloc(GIT_SHA1_RAWSZ), + .new_hash = reftable_malloc(GIT_SHA1_RAWSZ), + .name = xstrdup("han-wen"), + .email = xstrdup("hanwen@google.com"), + .message = xstrdup("test"), + .time = 1577123507, + .tz_offset = 100, + }, + } + }, + { + .refname = xstrdup("refs/heads/master"), + .update_index = 22, + .value_type = REFTABLE_LOG_DELETION, + } + }; + set_test_hash(in[0].value.update.new_hash, 1); + set_test_hash(in[0].value.update.old_hash, 2); + for (i = 0; i < ARRAY_SIZE(in); i++) { + struct reftable_record rec = { NULL }; + struct strbuf key = STRBUF_INIT; + uint8_t buffer[1024] = { 0 }; + struct string_view dest = { + .buf = buffer, + .len = sizeof(buffer), + }; + /* populate out, to check for leaks. */ + struct reftable_log_record out = { + .refname = xstrdup("old name"), + .value_type = REFTABLE_LOG_UPDATE, + .value = { + .update = { + .new_hash = reftable_calloc(GIT_SHA1_RAWSZ), + .old_hash = reftable_calloc(GIT_SHA1_RAWSZ), + .name = xstrdup("old name"), + .email = xstrdup("old@email"), + .message = xstrdup("old message"), + }, + }, + }; + struct reftable_record rec_out = { NULL }; + int n, m, valtype; + + reftable_record_from_log(&rec, &in[i]); + + test_copy(&rec); + + reftable_record_key(&rec, &key); + + n = reftable_record_encode(&rec, dest, GIT_SHA1_RAWSZ); + EXPECT(n >= 0); + reftable_record_from_log(&rec_out, &out); + valtype = reftable_record_val_type(&rec); + m = reftable_record_decode(&rec_out, key, valtype, dest, + GIT_SHA1_RAWSZ); + EXPECT(n == m); + + EXPECT(reftable_log_record_equal(&in[i], &out, GIT_SHA1_RAWSZ)); + reftable_log_record_release(&in[i]); + strbuf_release(&key); + reftable_record_release(&rec_out); + } +} + +static void test_u24_roundtrip(void) +{ + uint32_t in = 0x112233; + uint8_t dest[3]; + uint32_t out; + put_be24(dest, in); + out = get_be24(dest); + EXPECT(in == out); +} + +static void test_key_roundtrip(void) +{ + uint8_t buffer[1024] = { 0 }; + struct string_view dest = { + .buf = buffer, + .len = sizeof(buffer), + }; + struct strbuf last_key = STRBUF_INIT; + struct strbuf key = STRBUF_INIT; + struct strbuf roundtrip = STRBUF_INIT; + int restart; + uint8_t extra; + int n, m; + uint8_t rt_extra; + + strbuf_addstr(&last_key, "refs/heads/master"); + strbuf_addstr(&key, "refs/tags/bla"); + extra = 6; + n = reftable_encode_key(&restart, dest, last_key, key, extra); + EXPECT(!restart); + EXPECT(n > 0); + + m = reftable_decode_key(&roundtrip, &rt_extra, last_key, dest); + EXPECT(n == m); + EXPECT(0 == strbuf_cmp(&key, &roundtrip)); + EXPECT(rt_extra == extra); + + strbuf_release(&last_key); + strbuf_release(&key); + strbuf_release(&roundtrip); +} + +static void test_reftable_obj_record_roundtrip(void) +{ + uint8_t testHash1[GIT_SHA1_RAWSZ] = { 1, 2, 3, 4, 0 }; + uint64_t till9[] = { 1, 2, 3, 4, 500, 600, 700, 800, 9000 }; + struct reftable_obj_record recs[3] = { { + .hash_prefix = testHash1, + .hash_prefix_len = 5, + .offsets = till9, + .offset_len = 3, + }, + { + .hash_prefix = testHash1, + .hash_prefix_len = 5, + .offsets = till9, + .offset_len = 9, + }, + { + .hash_prefix = testHash1, + .hash_prefix_len = 5, + } }; + int i = 0; + for (i = 0; i < ARRAY_SIZE(recs); i++) { + struct reftable_obj_record in = recs[i]; + uint8_t buffer[1024] = { 0 }; + struct string_view dest = { + .buf = buffer, + .len = sizeof(buffer), + }; + struct reftable_record rec = { NULL }; + struct strbuf key = STRBUF_INIT; + struct reftable_obj_record out = { NULL }; + struct reftable_record rec_out = { NULL }; + int n, m; + uint8_t extra; + + reftable_record_from_obj(&rec, &in); + test_copy(&rec); + reftable_record_key(&rec, &key); + n = reftable_record_encode(&rec, dest, GIT_SHA1_RAWSZ); + EXPECT(n > 0); + extra = reftable_record_val_type(&rec); + reftable_record_from_obj(&rec_out, &out); + m = reftable_record_decode(&rec_out, key, extra, dest, + GIT_SHA1_RAWSZ); + EXPECT(n == m); + + EXPECT(in.hash_prefix_len == out.hash_prefix_len); + EXPECT(in.offset_len == out.offset_len); + + EXPECT(!memcmp(in.hash_prefix, out.hash_prefix, + in.hash_prefix_len)); + EXPECT(0 == memcmp(in.offsets, out.offsets, + sizeof(uint64_t) * in.offset_len)); + strbuf_release(&key); + reftable_record_release(&rec_out); + } +} + +static void test_reftable_index_record_roundtrip(void) +{ + struct reftable_index_record in = { + .offset = 42, + .last_key = STRBUF_INIT, + }; + uint8_t buffer[1024] = { 0 }; + struct string_view dest = { + .buf = buffer, + .len = sizeof(buffer), + }; + struct strbuf key = STRBUF_INIT; + struct reftable_record rec = { NULL }; + struct reftable_index_record out = { .last_key = STRBUF_INIT }; + struct reftable_record out_rec = { NULL }; + int n, m; + uint8_t extra; + + strbuf_addstr(&in.last_key, "refs/heads/master"); + reftable_record_from_index(&rec, &in); + reftable_record_key(&rec, &key); + test_copy(&rec); + + EXPECT(0 == strbuf_cmp(&key, &in.last_key)); + n = reftable_record_encode(&rec, dest, GIT_SHA1_RAWSZ); + EXPECT(n > 0); + + extra = reftable_record_val_type(&rec); + reftable_record_from_index(&out_rec, &out); + m = reftable_record_decode(&out_rec, key, extra, dest, GIT_SHA1_RAWSZ); + EXPECT(m == n); + + EXPECT(in.offset == out.offset); + + reftable_record_release(&out_rec); + strbuf_release(&key); + strbuf_release(&in.last_key); +} + +int record_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_reftable_log_record_equal); + RUN_TEST(test_reftable_log_record_roundtrip); + RUN_TEST(test_reftable_ref_record_roundtrip); + RUN_TEST(test_varint_roundtrip); + RUN_TEST(test_key_roundtrip); + RUN_TEST(test_common_prefix); + RUN_TEST(test_reftable_obj_record_roundtrip); + RUN_TEST(test_reftable_index_record_roundtrip); + RUN_TEST(test_u24_roundtrip); + return 0; +} diff --git a/reftable/refname.c b/reftable/refname.c new file mode 100644 index 0000000000..9573496932 --- /dev/null +++ b/reftable/refname.c @@ -0,0 +1,209 @@ +/* + Copyright 2020 Google LLC + + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file or at + https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" +#include "reftable-error.h" +#include "basics.h" +#include "refname.h" +#include "reftable-iterator.h" + +struct find_arg { + char **names; + const char *want; +}; + +static int find_name(size_t k, void *arg) +{ + struct find_arg *f_arg = arg; + return strcmp(f_arg->names[k], f_arg->want) >= 0; +} + +static int modification_has_ref(struct modification *mod, const char *name) +{ + struct reftable_ref_record ref = { NULL }; + int err = 0; + + if (mod->add_len > 0) { + struct find_arg arg = { + .names = mod->add, + .want = name, + }; + int idx = binsearch(mod->add_len, find_name, &arg); + if (idx < mod->add_len && !strcmp(mod->add[idx], name)) { + return 0; + } + } + + if (mod->del_len > 0) { + struct find_arg arg = { + .names = mod->del, + .want = name, + }; + int idx = binsearch(mod->del_len, find_name, &arg); + if (idx < mod->del_len && !strcmp(mod->del[idx], name)) { + return 1; + } + } + + err = reftable_table_read_ref(&mod->tab, name, &ref); + reftable_ref_record_release(&ref); + return err; +} + +static void modification_release(struct modification *mod) +{ + /* don't delete the strings themselves; they're owned by ref records. + */ + FREE_AND_NULL(mod->add); + FREE_AND_NULL(mod->del); + mod->add_len = 0; + mod->del_len = 0; +} + +static int modification_has_ref_with_prefix(struct modification *mod, + const char *prefix) +{ + struct reftable_iterator it = { NULL }; + struct reftable_ref_record ref = { NULL }; + int err = 0; + + if (mod->add_len > 0) { + struct find_arg arg = { + .names = mod->add, + .want = prefix, + }; + int idx = binsearch(mod->add_len, find_name, &arg); + if (idx < mod->add_len && + !strncmp(prefix, mod->add[idx], strlen(prefix))) + goto done; + } + err = reftable_table_seek_ref(&mod->tab, &it, prefix); + if (err) + goto done; + + while (1) { + err = reftable_iterator_next_ref(&it, &ref); + if (err) + goto done; + + if (mod->del_len > 0) { + struct find_arg arg = { + .names = mod->del, + .want = ref.refname, + }; + int idx = binsearch(mod->del_len, find_name, &arg); + if (idx < mod->del_len && + !strcmp(ref.refname, mod->del[idx])) { + continue; + } + } + + if (strncmp(ref.refname, prefix, strlen(prefix))) { + err = 1; + goto done; + } + err = 0; + goto done; + } + +done: + reftable_ref_record_release(&ref); + reftable_iterator_destroy(&it); + return err; +} + +static int validate_refname(const char *name) +{ + while (1) { + char *next = strchr(name, '/'); + if (!*name) { + return REFTABLE_REFNAME_ERROR; + } + if (!next) { + return 0; + } + if (next - name == 0 || (next - name == 1 && *name == '.') || + (next - name == 2 && name[0] == '.' && name[1] == '.')) + return REFTABLE_REFNAME_ERROR; + name = next + 1; + } + return 0; +} + +int validate_ref_record_addition(struct reftable_table tab, + struct reftable_ref_record *recs, size_t sz) +{ + struct modification mod = { + .tab = tab, + .add = reftable_calloc(sizeof(char *) * sz), + .del = reftable_calloc(sizeof(char *) * sz), + }; + int i = 0; + int err = 0; + for (; i < sz; i++) { + if (reftable_ref_record_is_deletion(&recs[i])) { + mod.del[mod.del_len++] = recs[i].refname; + } else { + mod.add[mod.add_len++] = recs[i].refname; + } + } + + err = modification_validate(&mod); + modification_release(&mod); + return err; +} + +static void strbuf_trim_component(struct strbuf *sl) +{ + while (sl->len > 0) { + int is_slash = (sl->buf[sl->len - 1] == '/'); + strbuf_setlen(sl, sl->len - 1); + if (is_slash) + break; + } +} + +int modification_validate(struct modification *mod) +{ + struct strbuf slashed = STRBUF_INIT; + int err = 0; + int i = 0; + for (; i < mod->add_len; i++) { + err = validate_refname(mod->add[i]); + if (err) + goto done; + strbuf_reset(&slashed); + strbuf_addstr(&slashed, mod->add[i]); + strbuf_addstr(&slashed, "/"); + + err = modification_has_ref_with_prefix(mod, slashed.buf); + if (err == 0) { + err = REFTABLE_NAME_CONFLICT; + goto done; + } + if (err < 0) + goto done; + + strbuf_reset(&slashed); + strbuf_addstr(&slashed, mod->add[i]); + while (slashed.len) { + strbuf_trim_component(&slashed); + err = modification_has_ref(mod, slashed.buf); + if (err == 0) { + err = REFTABLE_NAME_CONFLICT; + goto done; + } + if (err < 0) + goto done; + } + } + err = 0; +done: + strbuf_release(&slashed); + return err; +} diff --git a/reftable/refname.h b/reftable/refname.h new file mode 100644 index 0000000000..a24b40fcb4 --- /dev/null +++ b/reftable/refname.h @@ -0,0 +1,29 @@ +/* + Copyright 2020 Google LLC + + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file or at + https://developers.google.com/open-source/licenses/bsd +*/ +#ifndef REFNAME_H +#define REFNAME_H + +#include "reftable-record.h" +#include "reftable-generic.h" + +struct modification { + struct reftable_table tab; + + char **add; + size_t add_len; + + char **del; + size_t del_len; +}; + +int validate_ref_record_addition(struct reftable_table tab, + struct reftable_ref_record *recs, size_t sz); + +int modification_validate(struct modification *mod); + +#endif diff --git a/reftable/refname_test.c b/reftable/refname_test.c new file mode 100644 index 0000000000..8645cd93bb --- /dev/null +++ b/reftable/refname_test.c @@ -0,0 +1,102 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "basics.h" +#include "block.h" +#include "blocksource.h" +#include "constants.h" +#include "reader.h" +#include "record.h" +#include "refname.h" +#include "reftable-error.h" +#include "reftable-writer.h" +#include "system.h" + +#include "test_framework.h" +#include "reftable-tests.h" + +struct testcase { + char *add; + char *del; + int error_code; +}; + +static void test_conflict(void) +{ + struct reftable_write_options opts = { 0 }; + struct strbuf buf = STRBUF_INIT; + struct reftable_writer *w = + reftable_new_writer(&strbuf_add_void, &buf, &opts); + struct reftable_ref_record rec = { + .refname = "a/b", + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "destination", /* make sure it's not a symref. + */ + .update_index = 1, + }; + int err; + int i; + struct reftable_block_source source = { NULL }; + struct reftable_reader *rd = NULL; + struct reftable_table tab = { NULL }; + struct testcase cases[] = { + { "a/b/c", NULL, REFTABLE_NAME_CONFLICT }, + { "b", NULL, 0 }, + { "a", NULL, REFTABLE_NAME_CONFLICT }, + { "a", "a/b", 0 }, + + { "p/", NULL, REFTABLE_REFNAME_ERROR }, + { "p//q", NULL, REFTABLE_REFNAME_ERROR }, + { "p/./q", NULL, REFTABLE_REFNAME_ERROR }, + { "p/../q", NULL, REFTABLE_REFNAME_ERROR }, + + { "a/b/c", "a/b", 0 }, + { NULL, "a//b", 0 }, + }; + reftable_writer_set_limits(w, 1, 1); + + err = reftable_writer_add_ref(w, &rec); + EXPECT_ERR(err); + + err = reftable_writer_close(w); + EXPECT_ERR(err); + reftable_writer_free(w); + + block_source_from_strbuf(&source, &buf); + err = reftable_new_reader(&rd, &source, "filename"); + EXPECT_ERR(err); + + reftable_table_from_reader(&tab, rd); + + for (i = 0; i < ARRAY_SIZE(cases); i++) { + struct modification mod = { + .tab = tab, + }; + + if (cases[i].add) { + mod.add = &cases[i].add; + mod.add_len = 1; + } + if (cases[i].del) { + mod.del = &cases[i].del; + mod.del_len = 1; + } + + err = modification_validate(&mod); + EXPECT(err == cases[i].error_code); + } + + reftable_reader_free(rd); + strbuf_release(&buf); +} + +int refname_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_conflict); + return 0; +} diff --git a/reftable/reftable-blocksource.h b/reftable/reftable-blocksource.h new file mode 100644 index 0000000000..5aa3990a57 --- /dev/null +++ b/reftable/reftable-blocksource.h @@ -0,0 +1,49 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_BLOCKSOURCE_H +#define REFTABLE_BLOCKSOURCE_H + +#include <stdint.h> + +/* block_source is a generic wrapper for a seekable readable file. + */ +struct reftable_block_source { + struct reftable_block_source_vtable *ops; + void *arg; +}; + +/* a contiguous segment of bytes. It keeps track of its generating block_source + * so it can return itself into the pool. */ +struct reftable_block { + uint8_t *data; + int len; + struct reftable_block_source source; +}; + +/* block_source_vtable are the operations that make up block_source */ +struct reftable_block_source_vtable { + /* returns the size of a block source */ + uint64_t (*size)(void *source); + + /* reads a segment from the block source. It is an error to read + beyond the end of the block */ + int (*read_block)(void *source, struct reftable_block *dest, + uint64_t off, uint32_t size); + /* mark the block as read; may return the data back to malloc */ + void (*return_block)(void *source, struct reftable_block *blockp); + + /* release all resources associated with the block source */ + void (*close)(void *source); +}; + +/* opens a file on the file system as a block_source */ +int reftable_block_source_from_file(struct reftable_block_source *block_src, + const char *name); + +#endif diff --git a/reftable/reftable-error.h b/reftable/reftable-error.h new file mode 100644 index 0000000000..6f89bedf1a --- /dev/null +++ b/reftable/reftable-error.h @@ -0,0 +1,62 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_ERROR_H +#define REFTABLE_ERROR_H + +/* + * Errors in reftable calls are signaled with negative integer return values. 0 + * means success. + */ +enum reftable_error { + /* Unexpected file system behavior */ + REFTABLE_IO_ERROR = -2, + + /* Format inconsistency on reading data */ + REFTABLE_FORMAT_ERROR = -3, + + /* File does not exist. Returned from block_source_from_file(), because + * it needs special handling in stack. + */ + REFTABLE_NOT_EXIST_ERROR = -4, + + /* Trying to write out-of-date data. */ + REFTABLE_LOCK_ERROR = -5, + + /* Misuse of the API: + * - on writing a record with NULL refname. + * - on writing a reftable_ref_record outside the table limits + * - on writing a ref or log record before the stack's + * next_update_inde*x + * - on writing a log record with multiline message with + * exact_log_message unset + * - on reading a reftable_ref_record from log iterator, or vice versa. + * + * When a call misuses the API, the internal state of the library is + * kept unchanged. + */ + REFTABLE_API_ERROR = -6, + + /* Decompression error */ + REFTABLE_ZLIB_ERROR = -7, + + /* Wrote a table without blocks. */ + REFTABLE_EMPTY_TABLE_ERROR = -8, + + /* Dir/file conflict. */ + REFTABLE_NAME_CONFLICT = -9, + + /* Invalid ref name. */ + REFTABLE_REFNAME_ERROR = -10, +}; + +/* convert the numeric error code to a string. The string should not be + * deallocated. */ +const char *reftable_error_str(int err); + +#endif diff --git a/reftable/reftable-generic.h b/reftable/reftable-generic.h new file mode 100644 index 0000000000..d239751a77 --- /dev/null +++ b/reftable/reftable-generic.h @@ -0,0 +1,47 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_GENERIC_H +#define REFTABLE_GENERIC_H + +#include "reftable-iterator.h" + +struct reftable_table_vtable; + +/* + * Provides a unified API for reading tables, either merged tables, or single + * readers. */ +struct reftable_table { + struct reftable_table_vtable *ops; + void *table_arg; +}; + +int reftable_table_seek_log(struct reftable_table *tab, + struct reftable_iterator *it, const char *name); + +int reftable_table_seek_ref(struct reftable_table *tab, + struct reftable_iterator *it, const char *name); + +/* returns the hash ID from a generic reftable_table */ +uint32_t reftable_table_hash_id(struct reftable_table *tab); + +/* returns the max update_index covered by this table. */ +uint64_t reftable_table_max_update_index(struct reftable_table *tab); + +/* returns the min update_index covered by this table. */ +uint64_t reftable_table_min_update_index(struct reftable_table *tab); + +/* convenience function to read a single ref. Returns < 0 for error, 0 + for success, and 1 if ref not found. */ +int reftable_table_read_ref(struct reftable_table *tab, const char *name, + struct reftable_ref_record *ref); + +/* dump table contents onto stdout for debugging */ +int reftable_table_print(struct reftable_table *tab); + +#endif diff --git a/reftable/reftable-iterator.h b/reftable/reftable-iterator.h new file mode 100644 index 0000000000..d3eee7af35 --- /dev/null +++ b/reftable/reftable-iterator.h @@ -0,0 +1,39 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_ITERATOR_H +#define REFTABLE_ITERATOR_H + +#include "reftable-record.h" + +struct reftable_iterator_vtable; + +/* iterator is the generic interface for walking over data stored in a + * reftable. + */ +struct reftable_iterator { + struct reftable_iterator_vtable *ops; + void *iter_arg; +}; + +/* reads the next reftable_ref_record. Returns < 0 for error, 0 for OK and > 0: + * end of iteration. + */ +int reftable_iterator_next_ref(struct reftable_iterator *it, + struct reftable_ref_record *ref); + +/* reads the next reftable_log_record. Returns < 0 for error, 0 for OK and > 0: + * end of iteration. + */ +int reftable_iterator_next_log(struct reftable_iterator *it, + struct reftable_log_record *log); + +/* releases resources associated with an iterator. */ +void reftable_iterator_destroy(struct reftable_iterator *it); + +#endif diff --git a/reftable/reftable-malloc.h b/reftable/reftable-malloc.h new file mode 100644 index 0000000000..5f2185f1f3 --- /dev/null +++ b/reftable/reftable-malloc.h @@ -0,0 +1,18 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_H +#define REFTABLE_H + +#include <stddef.h> + +/* Overrides the functions to use for memory management. */ +void reftable_set_alloc(void *(*malloc)(size_t), + void *(*realloc)(void *, size_t), void (*free)(void *)); + +#endif diff --git a/reftable/reftable-merged.h b/reftable/reftable-merged.h new file mode 100644 index 0000000000..1a6d16915a --- /dev/null +++ b/reftable/reftable-merged.h @@ -0,0 +1,72 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_MERGED_H +#define REFTABLE_MERGED_H + +#include "reftable-iterator.h" + +/* + * Merged tables + * + * A ref database kept in a sequence of table files. The merged_table presents a + * unified view to reading (seeking, iterating) a sequence of immutable tables. + * + * The merged tables are on purpose kept disconnected from their actual storage + * (eg. files on disk), because it is useful to merge tables aren't files. For + * example, the per-workspace and global ref namespace can be implemented as a + * merged table of two stacks of file-backed reftables. + */ + +/* A merged table is implements seeking/iterating over a stack of tables. */ +struct reftable_merged_table; + +/* A generic reftable; see below. */ +struct reftable_table; + +/* reftable_new_merged_table creates a new merged table. It takes ownership of + the stack array. +*/ +int reftable_new_merged_table(struct reftable_merged_table **dest, + struct reftable_table *stack, int n, + uint32_t hash_id); + +/* returns an iterator positioned just before 'name' */ +int reftable_merged_table_seek_ref(struct reftable_merged_table *mt, + struct reftable_iterator *it, + const char *name); + +/* returns an iterator for log entry, at given update_index */ +int reftable_merged_table_seek_log_at(struct reftable_merged_table *mt, + struct reftable_iterator *it, + const char *name, uint64_t update_index); + +/* like reftable_merged_table_seek_log_at but look for the newest entry. */ +int reftable_merged_table_seek_log(struct reftable_merged_table *mt, + struct reftable_iterator *it, + const char *name); + +/* returns the max update_index covered by this merged table. */ +uint64_t +reftable_merged_table_max_update_index(struct reftable_merged_table *mt); + +/* returns the min update_index covered by this merged table. */ +uint64_t +reftable_merged_table_min_update_index(struct reftable_merged_table *mt); + +/* releases memory for the merged_table */ +void reftable_merged_table_free(struct reftable_merged_table *m); + +/* return the hash ID of the merged table. */ +uint32_t reftable_merged_table_hash_id(struct reftable_merged_table *m); + +/* create a generic table from reftable_merged_table */ +void reftable_table_from_merged_table(struct reftable_table *tab, + struct reftable_merged_table *table); + +#endif diff --git a/reftable/reftable-reader.h b/reftable/reftable-reader.h new file mode 100644 index 0000000000..4a4bc2fdf8 --- /dev/null +++ b/reftable/reftable-reader.h @@ -0,0 +1,101 @@ +/* + Copyright 2020 Google LLC + + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file or at + https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_READER_H +#define REFTABLE_READER_H + +#include "reftable-iterator.h" +#include "reftable-blocksource.h" + +/* + * Reading single tables + * + * The follow routines are for reading single files. For an + * application-level interface, skip ahead to struct + * reftable_merged_table and struct reftable_stack. + */ + +/* The reader struct is a handle to an open reftable file. */ +struct reftable_reader; + +/* Generic table. */ +struct reftable_table; + +/* reftable_new_reader opens a reftable for reading. If successful, + * returns 0 code and sets pp. The name is used for creating a + * stack. Typically, it is the basename of the file. The block source + * `src` is owned by the reader, and is closed on calling + * reftable_reader_destroy(). On error, the block source `src` is + * closed as well. + */ +int reftable_new_reader(struct reftable_reader **pp, + struct reftable_block_source *src, const char *name); + +/* reftable_reader_seek_ref returns an iterator where 'name' would be inserted + in the table. To seek to the start of the table, use name = "". + + example: + + struct reftable_reader *r = NULL; + int err = reftable_new_reader(&r, &src, "filename"); + if (err < 0) { ... } + struct reftable_iterator it = {0}; + err = reftable_reader_seek_ref(r, &it, "refs/heads/master"); + if (err < 0) { ... } + struct reftable_ref_record ref = {0}; + while (1) { + err = reftable_iterator_next_ref(&it, &ref); + if (err > 0) { + break; + } + if (err < 0) { + ..error handling.. + } + ..found.. + } + reftable_iterator_destroy(&it); + reftable_ref_record_release(&ref); +*/ +int reftable_reader_seek_ref(struct reftable_reader *r, + struct reftable_iterator *it, const char *name); + +/* returns the hash ID used in this table. */ +uint32_t reftable_reader_hash_id(struct reftable_reader *r); + +/* seek to logs for the given name, older than update_index. To seek to the + start of the table, use name = "". +*/ +int reftable_reader_seek_log_at(struct reftable_reader *r, + struct reftable_iterator *it, const char *name, + uint64_t update_index); + +/* seek to newest log entry for given name. */ +int reftable_reader_seek_log(struct reftable_reader *r, + struct reftable_iterator *it, const char *name); + +/* closes and deallocates a reader. */ +void reftable_reader_free(struct reftable_reader *); + +/* return an iterator for the refs pointing to `oid`. */ +int reftable_reader_refs_for(struct reftable_reader *r, + struct reftable_iterator *it, uint8_t *oid); + +/* return the max_update_index for a table */ +uint64_t reftable_reader_max_update_index(struct reftable_reader *r); + +/* return the min_update_index for a table */ +uint64_t reftable_reader_min_update_index(struct reftable_reader *r); + +/* creates a generic table from a file reader. */ +void reftable_table_from_reader(struct reftable_table *tab, + struct reftable_reader *reader); + +/* print table onto stdout for debugging. */ +int reftable_reader_print_file(const char *tablename); + +#endif diff --git a/reftable/reftable-record.h b/reftable/reftable-record.h new file mode 100644 index 0000000000..5370d2288c --- /dev/null +++ b/reftable/reftable-record.h @@ -0,0 +1,114 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_RECORD_H +#define REFTABLE_RECORD_H + +#include <stdint.h> + +/* + * Basic data types + * + * Reftables store the state of each ref in struct reftable_ref_record, and they + * store a sequence of reflog updates in struct reftable_log_record. + */ + +/* reftable_ref_record holds a ref database entry target_value */ +struct reftable_ref_record { + char *refname; /* Name of the ref, malloced. */ + uint64_t update_index; /* Logical timestamp at which this value is + * written */ + + enum { + /* tombstone to hide deletions from earlier tables */ + REFTABLE_REF_DELETION = 0x0, + + /* a simple ref */ + REFTABLE_REF_VAL1 = 0x1, + /* a tag, plus its peeled hash */ + REFTABLE_REF_VAL2 = 0x2, + + /* a symbolic reference */ + REFTABLE_REF_SYMREF = 0x3, +#define REFTABLE_NR_REF_VALUETYPES 4 + } value_type; + union { + uint8_t *val1; /* malloced hash. */ + struct { + uint8_t *value; /* first value, malloced hash */ + uint8_t *target_value; /* second value, malloced hash */ + } val2; + char *symref; /* referent, malloced 0-terminated string */ + } value; +}; + +/* Returns the first hash, or NULL if `rec` is not of type + * REFTABLE_REF_VAL1 or REFTABLE_REF_VAL2. */ +uint8_t *reftable_ref_record_val1(struct reftable_ref_record *rec); + +/* Returns the second hash, or NULL if `rec` is not of type + * REFTABLE_REF_VAL2. */ +uint8_t *reftable_ref_record_val2(struct reftable_ref_record *rec); + +/* returns whether 'ref' represents a deletion */ +int reftable_ref_record_is_deletion(const struct reftable_ref_record *ref); + +/* prints a reftable_ref_record onto stdout. Useful for debugging. */ +void reftable_ref_record_print(struct reftable_ref_record *ref, + uint32_t hash_id); + +/* frees and nulls all pointer values inside `ref`. */ +void reftable_ref_record_release(struct reftable_ref_record *ref); + +/* returns whether two reftable_ref_records are the same. Useful for testing. */ +int reftable_ref_record_equal(struct reftable_ref_record *a, + struct reftable_ref_record *b, int hash_size); + +/* reftable_log_record holds a reflog entry */ +struct reftable_log_record { + char *refname; + uint64_t update_index; /* logical timestamp of a transactional update. + */ + + enum { + /* tombstone to hide deletions from earlier tables */ + REFTABLE_LOG_DELETION = 0x0, + + /* a simple update */ + REFTABLE_LOG_UPDATE = 0x1, +#define REFTABLE_NR_LOG_VALUETYPES 2 + } value_type; + + union { + struct { + uint8_t *new_hash; + uint8_t *old_hash; + char *name; + char *email; + uint64_t time; + int16_t tz_offset; + char *message; + } update; + } value; +}; + +/* returns whether 'ref' represents the deletion of a log record. */ +int reftable_log_record_is_deletion(const struct reftable_log_record *log); + +/* frees and nulls all pointer values. */ +void reftable_log_record_release(struct reftable_log_record *log); + +/* returns whether two records are equal. Useful for testing. */ +int reftable_log_record_equal(struct reftable_log_record *a, + struct reftable_log_record *b, int hash_size); + +/* dumps a reftable_log_record on stdout, for debugging/testing. */ +void reftable_log_record_print(struct reftable_log_record *log, + uint32_t hash_id); + +#endif diff --git a/reftable/reftable-stack.h b/reftable/reftable-stack.h new file mode 100644 index 0000000000..1b602dda58 --- /dev/null +++ b/reftable/reftable-stack.h @@ -0,0 +1,128 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_STACK_H +#define REFTABLE_STACK_H + +#include "reftable-writer.h" + +/* + * The stack presents an interface to a mutable sequence of reftables. + + * A stack can be mutated by pushing a table to the top of the stack. + + * The reftable_stack automatically compacts files on disk to ensure good + * amortized performance. + * + * For windows and other platforms that cannot have open files as rename + * destinations, concurrent access from multiple processes needs the rand() + * random seed to be randomized. + */ +struct reftable_stack; + +/* open a new reftable stack. The tables along with the table list will be + * stored in 'dir'. Typically, this should be .git/reftables. + */ +int reftable_new_stack(struct reftable_stack **dest, const char *dir, + struct reftable_write_options config); + +/* returns the update_index at which a next table should be written. */ +uint64_t reftable_stack_next_update_index(struct reftable_stack *st); + +/* holds a transaction to add tables at the top of a stack. */ +struct reftable_addition; + +/* + * returns a new transaction to add reftables to the given stack. As a side + * effect, the ref database is locked. + */ +int reftable_stack_new_addition(struct reftable_addition **dest, + struct reftable_stack *st); + +/* Adds a reftable to transaction. */ +int reftable_addition_add(struct reftable_addition *add, + int (*write_table)(struct reftable_writer *wr, + void *arg), + void *arg); + +/* Commits the transaction, releasing the lock. After calling this, + * reftable_addition_destroy should still be called. + */ +int reftable_addition_commit(struct reftable_addition *add); + +/* Release all non-committed data from the transaction, and deallocate the + * transaction. Releases the lock if held. */ +void reftable_addition_destroy(struct reftable_addition *add); + +/* add a new table to the stack. The write_table function must call + * reftable_writer_set_limits, add refs and return an error value. */ +int reftable_stack_add(struct reftable_stack *st, + int (*write_table)(struct reftable_writer *wr, + void *write_arg), + void *write_arg); + +/* returns the merged_table for seeking. This table is valid until the + * next write or reload, and should not be closed or deleted. + */ +struct reftable_merged_table * +reftable_stack_merged_table(struct reftable_stack *st); + +/* frees all resources associated with the stack. */ +void reftable_stack_destroy(struct reftable_stack *st); + +/* Reloads the stack if necessary. This is very cheap to run if the stack was up + * to date */ +int reftable_stack_reload(struct reftable_stack *st); + +/* Policy for expiring reflog entries. */ +struct reftable_log_expiry_config { + /* Drop entries older than this timestamp */ + uint64_t time; + + /* Drop older entries */ + uint64_t min_update_index; +}; + +/* compacts all reftables into a giant table. Expire reflog entries if config is + * non-NULL */ +int reftable_stack_compact_all(struct reftable_stack *st, + struct reftable_log_expiry_config *config); + +/* heuristically compact unbalanced table stack. */ +int reftable_stack_auto_compact(struct reftable_stack *st); + +/* delete stale .ref tables. */ +int reftable_stack_clean(struct reftable_stack *st); + +/* convenience function to read a single ref. Returns < 0 for error, 0 for + * success, and 1 if ref not found. */ +int reftable_stack_read_ref(struct reftable_stack *st, const char *refname, + struct reftable_ref_record *ref); + +/* convenience function to read a single log. Returns < 0 for error, 0 for + * success, and 1 if ref not found. */ +int reftable_stack_read_log(struct reftable_stack *st, const char *refname, + struct reftable_log_record *log); + +/* statistics on past compactions. */ +struct reftable_compaction_stats { + uint64_t bytes; /* total number of bytes written */ + uint64_t entries_written; /* total number of entries written, including + failures. */ + int attempts; /* how often we tried to compact */ + int failures; /* failures happen on concurrent updates */ +}; + +/* return statistics for compaction up till now. */ +struct reftable_compaction_stats * +reftable_stack_compaction_stats(struct reftable_stack *st); + +/* print the entire stack represented by the directory */ +int reftable_stack_print_directory(const char *stackdir, uint32_t hash_id); + +#endif diff --git a/reftable/reftable-tests.h b/reftable/reftable-tests.h new file mode 100644 index 0000000000..0019cbcfa4 --- /dev/null +++ b/reftable/reftable-tests.h @@ -0,0 +1,23 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_TESTS_H +#define REFTABLE_TESTS_H + +int basics_test_main(int argc, const char **argv); +int block_test_main(int argc, const char **argv); +int merged_test_main(int argc, const char **argv); +int pq_test_main(int argc, const char **argv); +int record_test_main(int argc, const char **argv); +int refname_test_main(int argc, const char **argv); +int readwrite_test_main(int argc, const char **argv); +int stack_test_main(int argc, const char **argv); +int tree_test_main(int argc, const char **argv); +int reftable_dump_main(int argc, char *const *argv); + +#endif diff --git a/reftable/reftable-writer.h b/reftable/reftable-writer.h new file mode 100644 index 0000000000..af36462ced --- /dev/null +++ b/reftable/reftable-writer.h @@ -0,0 +1,148 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_WRITER_H +#define REFTABLE_WRITER_H + +#include "reftable-record.h" + +#include <stdint.h> +#include <unistd.h> /* ssize_t */ + +/* Writing single reftables */ + +/* reftable_write_options sets options for writing a single reftable. */ +struct reftable_write_options { + /* boolean: do not pad out blocks to block size. */ + unsigned unpadded : 1; + + /* the blocksize. Should be less than 2^24. */ + uint32_t block_size; + + /* boolean: do not generate a SHA1 => ref index. */ + unsigned skip_index_objects : 1; + + /* how often to write complete keys in each block. */ + int restart_interval; + + /* 4-byte identifier ("sha1", "s256") of the hash. + * Defaults to SHA1 if unset + */ + uint32_t hash_id; + + /* boolean: do not check ref names for validity or dir/file conflicts. + */ + unsigned skip_name_check : 1; + + /* boolean: copy log messages exactly. If unset, check that the message + * is a single line, and add '\n' if missing. + */ + unsigned exact_log_message : 1; +}; + +/* reftable_block_stats holds statistics for a single block type */ +struct reftable_block_stats { + /* total number of entries written */ + int entries; + /* total number of key restarts */ + int restarts; + /* total number of blocks */ + int blocks; + /* total number of index blocks */ + int index_blocks; + /* depth of the index */ + int max_index_level; + + /* offset of the first block for this type */ + uint64_t offset; + /* offset of the top level index block for this type, or 0 if not + * present */ + uint64_t index_offset; +}; + +/* stats holds overall statistics for a single reftable */ +struct reftable_stats { + /* total number of blocks written. */ + int blocks; + /* stats for ref data */ + struct reftable_block_stats ref_stats; + /* stats for the SHA1 to ref map. */ + struct reftable_block_stats obj_stats; + /* stats for index blocks */ + struct reftable_block_stats idx_stats; + /* stats for log blocks */ + struct reftable_block_stats log_stats; + + /* disambiguation length of shortened object IDs. */ + int object_id_len; +}; + +/* reftable_new_writer creates a new writer */ +struct reftable_writer * +reftable_new_writer(ssize_t (*writer_func)(void *, const void *, size_t), + void *writer_arg, struct reftable_write_options *opts); + +/* Set the range of update indices for the records we will add. When writing a + table into a stack, the min should be at least + reftable_stack_next_update_index(), or REFTABLE_API_ERROR is returned. + + For transactional updates to a stack, typically min==max, and the + update_index can be obtained by inspeciting the stack. When converting an + existing ref database into a single reftable, this would be a range of + update-index timestamps. + */ +void reftable_writer_set_limits(struct reftable_writer *w, uint64_t min, + uint64_t max); + +/* + Add a reftable_ref_record. The record should have names that come after + already added records. + + The update_index must be within the limits set by + reftable_writer_set_limits(), or REFTABLE_API_ERROR is returned. It is an + REFTABLE_API_ERROR error to write a ref record after a log record. +*/ +int reftable_writer_add_ref(struct reftable_writer *w, + struct reftable_ref_record *ref); + +/* + Convenience function to add multiple reftable_ref_records; the function sorts + the records before adding them, reordering the records array passed in. +*/ +int reftable_writer_add_refs(struct reftable_writer *w, + struct reftable_ref_record *refs, int n); + +/* + adds reftable_log_records. Log records are keyed by (refname, decreasing + update_index). The key for the record added must come after the already added + log records. +*/ +int reftable_writer_add_log(struct reftable_writer *w, + struct reftable_log_record *log); + +/* + Convenience function to add multiple reftable_log_records; the function sorts + the records before adding them, reordering records array passed in. +*/ +int reftable_writer_add_logs(struct reftable_writer *w, + struct reftable_log_record *logs, int n); + +/* reftable_writer_close finalizes the reftable. The writer is retained so + * statistics can be inspected. */ +int reftable_writer_close(struct reftable_writer *w); + +/* writer_stats returns the statistics on the reftable being written. + + This struct becomes invalid when the writer is freed. + */ +const struct reftable_stats *writer_stats(struct reftable_writer *w); + +/* reftable_writer_free deallocates memory for the writer */ +void reftable_writer_free(struct reftable_writer *w); + +#endif diff --git a/reftable/reftable.c b/reftable/reftable.c new file mode 100644 index 0000000000..0e4607a7cd --- /dev/null +++ b/reftable/reftable.c @@ -0,0 +1,115 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "basics.h" +#include "record.h" +#include "generic.h" +#include "reftable-iterator.h" +#include "reftable-generic.h" + +int reftable_table_seek_ref(struct reftable_table *tab, + struct reftable_iterator *it, const char *name) +{ + struct reftable_ref_record ref = { + .refname = (char *)name, + }; + struct reftable_record rec = { NULL }; + reftable_record_from_ref(&rec, &ref); + return tab->ops->seek_record(tab->table_arg, it, &rec); +} + +int reftable_table_read_ref(struct reftable_table *tab, const char *name, + struct reftable_ref_record *ref) +{ + struct reftable_iterator it = { NULL }; + int err = reftable_table_seek_ref(tab, &it, name); + if (err) + goto done; + + err = reftable_iterator_next_ref(&it, ref); + if (err) + goto done; + + if (strcmp(ref->refname, name) || + reftable_ref_record_is_deletion(ref)) { + reftable_ref_record_release(ref); + err = 1; + goto done; + } + +done: + reftable_iterator_destroy(&it); + return err; +} + +uint64_t reftable_table_max_update_index(struct reftable_table *tab) +{ + return tab->ops->max_update_index(tab->table_arg); +} + +uint64_t reftable_table_min_update_index(struct reftable_table *tab) +{ + return tab->ops->min_update_index(tab->table_arg); +} + +uint32_t reftable_table_hash_id(struct reftable_table *tab) +{ + return tab->ops->hash_id(tab->table_arg); +} + +void reftable_iterator_destroy(struct reftable_iterator *it) +{ + if (!it->ops) { + return; + } + it->ops->close(it->iter_arg); + it->ops = NULL; + FREE_AND_NULL(it->iter_arg); +} + +int reftable_iterator_next_ref(struct reftable_iterator *it, + struct reftable_ref_record *ref) +{ + struct reftable_record rec = { NULL }; + reftable_record_from_ref(&rec, ref); + return iterator_next(it, &rec); +} + +int reftable_iterator_next_log(struct reftable_iterator *it, + struct reftable_log_record *log) +{ + struct reftable_record rec = { NULL }; + reftable_record_from_log(&rec, log); + return iterator_next(it, &rec); +} + +int iterator_next(struct reftable_iterator *it, struct reftable_record *rec) +{ + return it->ops->next(it->iter_arg, rec); +} + +static int empty_iterator_next(void *arg, struct reftable_record *rec) +{ + return 1; +} + +static void empty_iterator_close(void *arg) +{ +} + +static struct reftable_iterator_vtable empty_vtable = { + .next = &empty_iterator_next, + .close = &empty_iterator_close, +}; + +void iterator_set_empty(struct reftable_iterator *it) +{ + assert(!it->ops); + it->iter_arg = NULL; + it->ops = &empty_vtable; +} diff --git a/reftable/stack.c b/reftable/stack.c new file mode 100644 index 0000000000..df5021ebf0 --- /dev/null +++ b/reftable/stack.c @@ -0,0 +1,1396 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "stack.h" + +#include "system.h" +#include "merged.h" +#include "reader.h" +#include "refname.h" +#include "reftable-error.h" +#include "reftable-record.h" +#include "reftable-merged.h" +#include "writer.h" + +static int stack_try_add(struct reftable_stack *st, + int (*write_table)(struct reftable_writer *wr, + void *arg), + void *arg); +static int stack_write_compact(struct reftable_stack *st, + struct reftable_writer *wr, int first, int last, + struct reftable_log_expiry_config *config); +static int stack_check_addition(struct reftable_stack *st, + const char *new_tab_name); +static void reftable_addition_close(struct reftable_addition *add); +static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st, + int reuse_open); + +static void stack_filename(struct strbuf *dest, struct reftable_stack *st, + const char *name) +{ + strbuf_reset(dest); + strbuf_addstr(dest, st->reftable_dir); + strbuf_addstr(dest, "/"); + strbuf_addstr(dest, name); +} + +static ssize_t reftable_fd_write(void *arg, const void *data, size_t sz) +{ + int *fdp = (int *)arg; + return write(*fdp, data, sz); +} + +int reftable_new_stack(struct reftable_stack **dest, const char *dir, + struct reftable_write_options config) +{ + struct reftable_stack *p = + reftable_calloc(sizeof(struct reftable_stack)); + struct strbuf list_file_name = STRBUF_INIT; + int err = 0; + + if (config.hash_id == 0) { + config.hash_id = GIT_SHA1_FORMAT_ID; + } + + *dest = NULL; + + strbuf_reset(&list_file_name); + strbuf_addstr(&list_file_name, dir); + strbuf_addstr(&list_file_name, "/tables.list"); + + p->list_file = strbuf_detach(&list_file_name, NULL); + p->reftable_dir = xstrdup(dir); + p->config = config; + + err = reftable_stack_reload_maybe_reuse(p, 1); + if (err < 0) { + reftable_stack_destroy(p); + } else { + *dest = p; + } + return err; +} + +static int fd_read_lines(int fd, char ***namesp) +{ + off_t size = lseek(fd, 0, SEEK_END); + char *buf = NULL; + int err = 0; + if (size < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + err = lseek(fd, 0, SEEK_SET); + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + + buf = reftable_malloc(size + 1); + if (read(fd, buf, size) != size) { + err = REFTABLE_IO_ERROR; + goto done; + } + buf[size] = 0; + + parse_names(buf, size, namesp); + +done: + reftable_free(buf); + return err; +} + +int read_lines(const char *filename, char ***namesp) +{ + int fd = open(filename, O_RDONLY); + int err = 0; + if (fd < 0) { + if (errno == ENOENT) { + *namesp = reftable_calloc(sizeof(char *)); + return 0; + } + + return REFTABLE_IO_ERROR; + } + err = fd_read_lines(fd, namesp); + close(fd); + return err; +} + +struct reftable_merged_table * +reftable_stack_merged_table(struct reftable_stack *st) +{ + return st->merged; +} + +static int has_name(char **names, const char *name) +{ + while (*names) { + if (!strcmp(*names, name)) + return 1; + names++; + } + return 0; +} + +/* Close and free the stack */ +void reftable_stack_destroy(struct reftable_stack *st) +{ + char **names = NULL; + int err = 0; + if (st->merged) { + reftable_merged_table_free(st->merged); + st->merged = NULL; + } + + err = read_lines(st->list_file, &names); + if (err < 0) { + FREE_AND_NULL(names); + } + + if (st->readers) { + int i = 0; + struct strbuf filename = STRBUF_INIT; + for (i = 0; i < st->readers_len; i++) { + const char *name = reader_name(st->readers[i]); + strbuf_reset(&filename); + if (names && !has_name(names, name)) { + stack_filename(&filename, st, name); + } + reftable_reader_free(st->readers[i]); + + if (filename.len) { + /* On Windows, can only unlink after closing. */ + unlink(filename.buf); + } + } + strbuf_release(&filename); + st->readers_len = 0; + FREE_AND_NULL(st->readers); + } + FREE_AND_NULL(st->list_file); + FREE_AND_NULL(st->reftable_dir); + reftable_free(st); + free_names(names); +} + +static struct reftable_reader **stack_copy_readers(struct reftable_stack *st, + int cur_len) +{ + struct reftable_reader **cur = + reftable_calloc(sizeof(struct reftable_reader *) * cur_len); + int i = 0; + for (i = 0; i < cur_len; i++) { + cur[i] = st->readers[i]; + } + return cur; +} + +static int reftable_stack_reload_once(struct reftable_stack *st, char **names, + int reuse_open) +{ + int cur_len = !st->merged ? 0 : st->merged->stack_len; + struct reftable_reader **cur = stack_copy_readers(st, cur_len); + int err = 0; + int names_len = names_length(names); + struct reftable_reader **new_readers = + reftable_calloc(sizeof(struct reftable_reader *) * names_len); + struct reftable_table *new_tables = + reftable_calloc(sizeof(struct reftable_table) * names_len); + int new_readers_len = 0; + struct reftable_merged_table *new_merged = NULL; + int i; + + while (*names) { + struct reftable_reader *rd = NULL; + char *name = *names++; + + /* this is linear; we assume compaction keeps the number of + tables under control so this is not quadratic. */ + int j = 0; + for (j = 0; reuse_open && j < cur_len; j++) { + if (cur[j] && 0 == strcmp(cur[j]->name, name)) { + rd = cur[j]; + cur[j] = NULL; + break; + } + } + + if (!rd) { + struct reftable_block_source src = { NULL }; + struct strbuf table_path = STRBUF_INIT; + stack_filename(&table_path, st, name); + + err = reftable_block_source_from_file(&src, + table_path.buf); + strbuf_release(&table_path); + + if (err < 0) + goto done; + + err = reftable_new_reader(&rd, &src, name); + if (err < 0) + goto done; + } + + new_readers[new_readers_len] = rd; + reftable_table_from_reader(&new_tables[new_readers_len], rd); + new_readers_len++; + } + + /* success! */ + err = reftable_new_merged_table(&new_merged, new_tables, + new_readers_len, st->config.hash_id); + if (err < 0) + goto done; + + new_tables = NULL; + st->readers_len = new_readers_len; + if (st->merged) { + merged_table_release(st->merged); + reftable_merged_table_free(st->merged); + } + if (st->readers) { + reftable_free(st->readers); + } + st->readers = new_readers; + new_readers = NULL; + new_readers_len = 0; + + new_merged->suppress_deletions = 1; + st->merged = new_merged; + for (i = 0; i < cur_len; i++) { + if (cur[i]) { + const char *name = reader_name(cur[i]); + struct strbuf filename = STRBUF_INIT; + stack_filename(&filename, st, name); + + reader_close(cur[i]); + reftable_reader_free(cur[i]); + + /* On Windows, can only unlink after closing. */ + unlink(filename.buf); + + strbuf_release(&filename); + } + } + +done: + for (i = 0; i < new_readers_len; i++) { + reader_close(new_readers[i]); + reftable_reader_free(new_readers[i]); + } + reftable_free(new_readers); + reftable_free(new_tables); + reftable_free(cur); + return err; +} + +/* return negative if a before b. */ +static int tv_cmp(struct timeval *a, struct timeval *b) +{ + time_t diff = a->tv_sec - b->tv_sec; + int udiff = a->tv_usec - b->tv_usec; + + if (diff != 0) + return diff; + + return udiff; +} + +static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st, + int reuse_open) +{ + struct timeval deadline = { 0 }; + int err = gettimeofday(&deadline, NULL); + int64_t delay = 0; + int tries = 0; + if (err < 0) + return err; + + deadline.tv_sec += 3; + while (1) { + char **names = NULL; + char **names_after = NULL; + struct timeval now = { 0 }; + int err = gettimeofday(&now, NULL); + int err2 = 0; + if (err < 0) { + return err; + } + + /* Only look at deadlines after the first few times. This + simplifies debugging in GDB */ + tries++; + if (tries > 3 && tv_cmp(&now, &deadline) >= 0) { + break; + } + + err = read_lines(st->list_file, &names); + if (err < 0) { + free_names(names); + return err; + } + err = reftable_stack_reload_once(st, names, reuse_open); + if (err == 0) { + free_names(names); + break; + } + if (err != REFTABLE_NOT_EXIST_ERROR) { + free_names(names); + return err; + } + + /* err == REFTABLE_NOT_EXIST_ERROR can be caused by a concurrent + writer. Check if there was one by checking if the name list + changed. + */ + err2 = read_lines(st->list_file, &names_after); + if (err2 < 0) { + free_names(names); + return err2; + } + + if (names_equal(names_after, names)) { + free_names(names); + free_names(names_after); + return err; + } + free_names(names); + free_names(names_after); + + delay = delay + (delay * rand()) / RAND_MAX + 1; + sleep_millisec(delay); + } + + return 0; +} + +/* -1 = error + 0 = up to date + 1 = changed. */ +static int stack_uptodate(struct reftable_stack *st) +{ + char **names = NULL; + int err = read_lines(st->list_file, &names); + int i = 0; + if (err < 0) + return err; + + for (i = 0; i < st->readers_len; i++) { + if (!names[i]) { + err = 1; + goto done; + } + + if (strcmp(st->readers[i]->name, names[i])) { + err = 1; + goto done; + } + } + + if (names[st->merged->stack_len]) { + err = 1; + goto done; + } + +done: + free_names(names); + return err; +} + +int reftable_stack_reload(struct reftable_stack *st) +{ + int err = stack_uptodate(st); + if (err > 0) + return reftable_stack_reload_maybe_reuse(st, 1); + return err; +} + +int reftable_stack_add(struct reftable_stack *st, + int (*write)(struct reftable_writer *wr, void *arg), + void *arg) +{ + int err = stack_try_add(st, write, arg); + if (err < 0) { + if (err == REFTABLE_LOCK_ERROR) { + /* Ignore error return, we want to propagate + REFTABLE_LOCK_ERROR. + */ + reftable_stack_reload(st); + } + return err; + } + + if (!st->disable_auto_compact) + return reftable_stack_auto_compact(st); + + return 0; +} + +static void format_name(struct strbuf *dest, uint64_t min, uint64_t max) +{ + char buf[100]; + uint32_t rnd = (uint32_t)rand(); + snprintf(buf, sizeof(buf), "0x%012" PRIx64 "-0x%012" PRIx64 "-%08x", + min, max, rnd); + strbuf_reset(dest); + strbuf_addstr(dest, buf); +} + +struct reftable_addition { + int lock_file_fd; + struct strbuf lock_file_name; + struct reftable_stack *stack; + + char **new_tables; + int new_tables_len; + uint64_t next_update_index; +}; + +#define REFTABLE_ADDITION_INIT \ + { \ + .lock_file_name = STRBUF_INIT \ + } + +static int reftable_stack_init_addition(struct reftable_addition *add, + struct reftable_stack *st) +{ + int err = 0; + add->stack = st; + + strbuf_reset(&add->lock_file_name); + strbuf_addstr(&add->lock_file_name, st->list_file); + strbuf_addstr(&add->lock_file_name, ".lock"); + + add->lock_file_fd = open(add->lock_file_name.buf, + O_EXCL | O_CREAT | O_WRONLY, 0644); + if (add->lock_file_fd < 0) { + if (errno == EEXIST) { + err = REFTABLE_LOCK_ERROR; + } else { + err = REFTABLE_IO_ERROR; + } + goto done; + } + err = stack_uptodate(st); + if (err < 0) + goto done; + + if (err > 1) { + err = REFTABLE_LOCK_ERROR; + goto done; + } + + add->next_update_index = reftable_stack_next_update_index(st); +done: + if (err) { + reftable_addition_close(add); + } + return err; +} + +static void reftable_addition_close(struct reftable_addition *add) +{ + int i = 0; + struct strbuf nm = STRBUF_INIT; + for (i = 0; i < add->new_tables_len; i++) { + stack_filename(&nm, add->stack, add->new_tables[i]); + unlink(nm.buf); + reftable_free(add->new_tables[i]); + add->new_tables[i] = NULL; + } + reftable_free(add->new_tables); + add->new_tables = NULL; + add->new_tables_len = 0; + + if (add->lock_file_fd > 0) { + close(add->lock_file_fd); + add->lock_file_fd = 0; + } + if (add->lock_file_name.len > 0) { + unlink(add->lock_file_name.buf); + strbuf_release(&add->lock_file_name); + } + + strbuf_release(&nm); +} + +void reftable_addition_destroy(struct reftable_addition *add) +{ + if (!add) { + return; + } + reftable_addition_close(add); + reftable_free(add); +} + +int reftable_addition_commit(struct reftable_addition *add) +{ + struct strbuf table_list = STRBUF_INIT; + int i = 0; + int err = 0; + if (add->new_tables_len == 0) + goto done; + + for (i = 0; i < add->stack->merged->stack_len; i++) { + strbuf_addstr(&table_list, add->stack->readers[i]->name); + strbuf_addstr(&table_list, "\n"); + } + for (i = 0; i < add->new_tables_len; i++) { + strbuf_addstr(&table_list, add->new_tables[i]); + strbuf_addstr(&table_list, "\n"); + } + + err = write(add->lock_file_fd, table_list.buf, table_list.len); + strbuf_release(&table_list); + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + + err = close(add->lock_file_fd); + add->lock_file_fd = 0; + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + + err = rename(add->lock_file_name.buf, add->stack->list_file); + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + + /* success, no more state to clean up. */ + strbuf_release(&add->lock_file_name); + for (i = 0; i < add->new_tables_len; i++) { + reftable_free(add->new_tables[i]); + } + reftable_free(add->new_tables); + add->new_tables = NULL; + add->new_tables_len = 0; + + err = reftable_stack_reload(add->stack); +done: + reftable_addition_close(add); + return err; +} + +int reftable_stack_new_addition(struct reftable_addition **dest, + struct reftable_stack *st) +{ + int err = 0; + struct reftable_addition empty = REFTABLE_ADDITION_INIT; + *dest = reftable_calloc(sizeof(**dest)); + **dest = empty; + err = reftable_stack_init_addition(*dest, st); + if (err) { + reftable_free(*dest); + *dest = NULL; + } + return err; +} + +static int stack_try_add(struct reftable_stack *st, + int (*write_table)(struct reftable_writer *wr, + void *arg), + void *arg) +{ + struct reftable_addition add = REFTABLE_ADDITION_INIT; + int err = reftable_stack_init_addition(&add, st); + if (err < 0) + goto done; + if (err > 0) { + err = REFTABLE_LOCK_ERROR; + goto done; + } + + err = reftable_addition_add(&add, write_table, arg); + if (err < 0) + goto done; + + err = reftable_addition_commit(&add); +done: + reftable_addition_close(&add); + return err; +} + +int reftable_addition_add(struct reftable_addition *add, + int (*write_table)(struct reftable_writer *wr, + void *arg), + void *arg) +{ + struct strbuf temp_tab_file_name = STRBUF_INIT; + struct strbuf tab_file_name = STRBUF_INIT; + struct strbuf next_name = STRBUF_INIT; + struct reftable_writer *wr = NULL; + int err = 0; + int tab_fd = 0; + + strbuf_reset(&next_name); + format_name(&next_name, add->next_update_index, add->next_update_index); + + stack_filename(&temp_tab_file_name, add->stack, next_name.buf); + strbuf_addstr(&temp_tab_file_name, ".temp.XXXXXX"); + + tab_fd = mkstemp(temp_tab_file_name.buf); + if (tab_fd < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + + wr = reftable_new_writer(reftable_fd_write, &tab_fd, + &add->stack->config); + err = write_table(wr, arg); + if (err < 0) + goto done; + + err = reftable_writer_close(wr); + if (err == REFTABLE_EMPTY_TABLE_ERROR) { + err = 0; + goto done; + } + if (err < 0) + goto done; + + err = close(tab_fd); + tab_fd = 0; + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + + err = stack_check_addition(add->stack, temp_tab_file_name.buf); + if (err < 0) + goto done; + + if (wr->min_update_index < add->next_update_index) { + err = REFTABLE_API_ERROR; + goto done; + } + + format_name(&next_name, wr->min_update_index, wr->max_update_index); + strbuf_addstr(&next_name, ".ref"); + + stack_filename(&tab_file_name, add->stack, next_name.buf); + + /* + On windows, this relies on rand() picking a unique destination name. + Maybe we should do retry loop as well? + */ + err = rename(temp_tab_file_name.buf, tab_file_name.buf); + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + + add->new_tables = reftable_realloc(add->new_tables, + sizeof(*add->new_tables) * + (add->new_tables_len + 1)); + add->new_tables[add->new_tables_len] = strbuf_detach(&next_name, NULL); + add->new_tables_len++; +done: + if (tab_fd > 0) { + close(tab_fd); + tab_fd = 0; + } + if (temp_tab_file_name.len > 0) { + unlink(temp_tab_file_name.buf); + } + + strbuf_release(&temp_tab_file_name); + strbuf_release(&tab_file_name); + strbuf_release(&next_name); + reftable_writer_free(wr); + return err; +} + +uint64_t reftable_stack_next_update_index(struct reftable_stack *st) +{ + int sz = st->merged->stack_len; + if (sz > 0) + return reftable_reader_max_update_index(st->readers[sz - 1]) + + 1; + return 1; +} + +static int stack_compact_locked(struct reftable_stack *st, int first, int last, + struct strbuf *temp_tab, + struct reftable_log_expiry_config *config) +{ + struct strbuf next_name = STRBUF_INIT; + int tab_fd = -1; + struct reftable_writer *wr = NULL; + int err = 0; + + format_name(&next_name, + reftable_reader_min_update_index(st->readers[first]), + reftable_reader_max_update_index(st->readers[last])); + + stack_filename(temp_tab, st, next_name.buf); + strbuf_addstr(temp_tab, ".temp.XXXXXX"); + + tab_fd = mkstemp(temp_tab->buf); + wr = reftable_new_writer(reftable_fd_write, &tab_fd, &st->config); + + err = stack_write_compact(st, wr, first, last, config); + if (err < 0) + goto done; + err = reftable_writer_close(wr); + if (err < 0) + goto done; + + err = close(tab_fd); + tab_fd = 0; + +done: + reftable_writer_free(wr); + if (tab_fd > 0) { + close(tab_fd); + tab_fd = 0; + } + if (err != 0 && temp_tab->len > 0) { + unlink(temp_tab->buf); + strbuf_release(temp_tab); + } + strbuf_release(&next_name); + return err; +} + +static int stack_write_compact(struct reftable_stack *st, + struct reftable_writer *wr, int first, int last, + struct reftable_log_expiry_config *config) +{ + int subtabs_len = last - first + 1; + struct reftable_table *subtabs = reftable_calloc( + sizeof(struct reftable_table) * (last - first + 1)); + struct reftable_merged_table *mt = NULL; + int err = 0; + struct reftable_iterator it = { NULL }; + struct reftable_ref_record ref = { NULL }; + struct reftable_log_record log = { NULL }; + + uint64_t entries = 0; + + int i = 0, j = 0; + for (i = first, j = 0; i <= last; i++) { + struct reftable_reader *t = st->readers[i]; + reftable_table_from_reader(&subtabs[j++], t); + st->stats.bytes += t->size; + } + reftable_writer_set_limits(wr, st->readers[first]->min_update_index, + st->readers[last]->max_update_index); + + err = reftable_new_merged_table(&mt, subtabs, subtabs_len, + st->config.hash_id); + if (err < 0) { + reftable_free(subtabs); + goto done; + } + + err = reftable_merged_table_seek_ref(mt, &it, ""); + if (err < 0) + goto done; + + while (1) { + err = reftable_iterator_next_ref(&it, &ref); + if (err > 0) { + err = 0; + break; + } + if (err < 0) { + break; + } + + if (first == 0 && reftable_ref_record_is_deletion(&ref)) { + continue; + } + + err = reftable_writer_add_ref(wr, &ref); + if (err < 0) { + break; + } + entries++; + } + reftable_iterator_destroy(&it); + + err = reftable_merged_table_seek_log(mt, &it, ""); + if (err < 0) + goto done; + + while (1) { + err = reftable_iterator_next_log(&it, &log); + if (err > 0) { + err = 0; + break; + } + if (err < 0) { + break; + } + if (first == 0 && reftable_log_record_is_deletion(&log)) { + continue; + } + + if (config && config->min_update_index > 0 && + log.update_index < config->min_update_index) { + continue; + } + + if (config && config->time > 0 && + log.value.update.time < config->time) { + continue; + } + + err = reftable_writer_add_log(wr, &log); + if (err < 0) { + break; + } + entries++; + } + +done: + reftable_iterator_destroy(&it); + if (mt) { + merged_table_release(mt); + reftable_merged_table_free(mt); + } + reftable_ref_record_release(&ref); + reftable_log_record_release(&log); + st->stats.entries_written += entries; + return err; +} + +/* < 0: error. 0 == OK, > 0 attempt failed; could retry. */ +static int stack_compact_range(struct reftable_stack *st, int first, int last, + struct reftable_log_expiry_config *expiry) +{ + struct strbuf temp_tab_file_name = STRBUF_INIT; + struct strbuf new_table_name = STRBUF_INIT; + struct strbuf lock_file_name = STRBUF_INIT; + struct strbuf ref_list_contents = STRBUF_INIT; + struct strbuf new_table_path = STRBUF_INIT; + int err = 0; + int have_lock = 0; + int lock_file_fd = 0; + int compact_count = last - first + 1; + char **listp = NULL; + char **delete_on_success = + reftable_calloc(sizeof(char *) * (compact_count + 1)); + char **subtable_locks = + reftable_calloc(sizeof(char *) * (compact_count + 1)); + int i = 0; + int j = 0; + int is_empty_table = 0; + + if (first > last || (!expiry && first == last)) { + err = 0; + goto done; + } + + st->stats.attempts++; + + strbuf_reset(&lock_file_name); + strbuf_addstr(&lock_file_name, st->list_file); + strbuf_addstr(&lock_file_name, ".lock"); + + lock_file_fd = + open(lock_file_name.buf, O_EXCL | O_CREAT | O_WRONLY, 0644); + if (lock_file_fd < 0) { + if (errno == EEXIST) { + err = 1; + } else { + err = REFTABLE_IO_ERROR; + } + goto done; + } + /* Don't want to write to the lock for now. */ + close(lock_file_fd); + lock_file_fd = 0; + + have_lock = 1; + err = stack_uptodate(st); + if (err != 0) + goto done; + + for (i = first, j = 0; i <= last; i++) { + struct strbuf subtab_file_name = STRBUF_INIT; + struct strbuf subtab_lock = STRBUF_INIT; + int sublock_file_fd = -1; + + stack_filename(&subtab_file_name, st, + reader_name(st->readers[i])); + + strbuf_reset(&subtab_lock); + strbuf_addbuf(&subtab_lock, &subtab_file_name); + strbuf_addstr(&subtab_lock, ".lock"); + + sublock_file_fd = open(subtab_lock.buf, + O_EXCL | O_CREAT | O_WRONLY, 0644); + if (sublock_file_fd > 0) { + close(sublock_file_fd); + } else if (sublock_file_fd < 0) { + if (errno == EEXIST) { + err = 1; + } else { + err = REFTABLE_IO_ERROR; + } + } + + subtable_locks[j] = subtab_lock.buf; + delete_on_success[j] = subtab_file_name.buf; + j++; + + if (err != 0) + goto done; + } + + err = unlink(lock_file_name.buf); + if (err < 0) + goto done; + have_lock = 0; + + err = stack_compact_locked(st, first, last, &temp_tab_file_name, + expiry); + /* Compaction + tombstones can create an empty table out of non-empty + * tables. */ + is_empty_table = (err == REFTABLE_EMPTY_TABLE_ERROR); + if (is_empty_table) { + err = 0; + } + if (err < 0) + goto done; + + lock_file_fd = + open(lock_file_name.buf, O_EXCL | O_CREAT | O_WRONLY, 0644); + if (lock_file_fd < 0) { + if (errno == EEXIST) { + err = 1; + } else { + err = REFTABLE_IO_ERROR; + } + goto done; + } + have_lock = 1; + + format_name(&new_table_name, st->readers[first]->min_update_index, + st->readers[last]->max_update_index); + strbuf_addstr(&new_table_name, ".ref"); + + stack_filename(&new_table_path, st, new_table_name.buf); + + if (!is_empty_table) { + /* retry? */ + err = rename(temp_tab_file_name.buf, new_table_path.buf); + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + } + + for (i = 0; i < first; i++) { + strbuf_addstr(&ref_list_contents, st->readers[i]->name); + strbuf_addstr(&ref_list_contents, "\n"); + } + if (!is_empty_table) { + strbuf_addbuf(&ref_list_contents, &new_table_name); + strbuf_addstr(&ref_list_contents, "\n"); + } + for (i = last + 1; i < st->merged->stack_len; i++) { + strbuf_addstr(&ref_list_contents, st->readers[i]->name); + strbuf_addstr(&ref_list_contents, "\n"); + } + + err = write(lock_file_fd, ref_list_contents.buf, ref_list_contents.len); + if (err < 0) { + err = REFTABLE_IO_ERROR; + unlink(new_table_path.buf); + goto done; + } + err = close(lock_file_fd); + lock_file_fd = 0; + if (err < 0) { + err = REFTABLE_IO_ERROR; + unlink(new_table_path.buf); + goto done; + } + + err = rename(lock_file_name.buf, st->list_file); + if (err < 0) { + err = REFTABLE_IO_ERROR; + unlink(new_table_path.buf); + goto done; + } + have_lock = 0; + + /* Reload the stack before deleting. On windows, we can only delete the + files after we closed them. + */ + err = reftable_stack_reload_maybe_reuse(st, first < last); + + listp = delete_on_success; + while (*listp) { + if (strcmp(*listp, new_table_path.buf)) { + unlink(*listp); + } + listp++; + } + +done: + free_names(delete_on_success); + + listp = subtable_locks; + while (*listp) { + unlink(*listp); + listp++; + } + free_names(subtable_locks); + if (lock_file_fd > 0) { + close(lock_file_fd); + lock_file_fd = 0; + } + if (have_lock) { + unlink(lock_file_name.buf); + } + strbuf_release(&new_table_name); + strbuf_release(&new_table_path); + strbuf_release(&ref_list_contents); + strbuf_release(&temp_tab_file_name); + strbuf_release(&lock_file_name); + return err; +} + +int reftable_stack_compact_all(struct reftable_stack *st, + struct reftable_log_expiry_config *config) +{ + return stack_compact_range(st, 0, st->merged->stack_len - 1, config); +} + +static int stack_compact_range_stats(struct reftable_stack *st, int first, + int last, + struct reftable_log_expiry_config *config) +{ + int err = stack_compact_range(st, first, last, config); + if (err > 0) { + st->stats.failures++; + } + return err; +} + +static int segment_size(struct segment *s) +{ + return s->end - s->start; +} + +int fastlog2(uint64_t sz) +{ + int l = 0; + if (sz == 0) + return 0; + for (; sz; sz /= 2) { + l++; + } + return l - 1; +} + +struct segment *sizes_to_segments(int *seglen, uint64_t *sizes, int n) +{ + struct segment *segs = reftable_calloc(sizeof(struct segment) * n); + int next = 0; + struct segment cur = { 0 }; + int i = 0; + + if (n == 0) { + *seglen = 0; + return segs; + } + for (i = 0; i < n; i++) { + int log = fastlog2(sizes[i]); + if (cur.log != log && cur.bytes > 0) { + struct segment fresh = { + .start = i, + }; + + segs[next++] = cur; + cur = fresh; + } + + cur.log = log; + cur.end = i + 1; + cur.bytes += sizes[i]; + } + segs[next++] = cur; + *seglen = next; + return segs; +} + +struct segment suggest_compaction_segment(uint64_t *sizes, int n) +{ + int seglen = 0; + struct segment *segs = sizes_to_segments(&seglen, sizes, n); + struct segment min_seg = { + .log = 64, + }; + int i = 0; + for (i = 0; i < seglen; i++) { + if (segment_size(&segs[i]) == 1) { + continue; + } + + if (segs[i].log < min_seg.log) { + min_seg = segs[i]; + } + } + + while (min_seg.start > 0) { + int prev = min_seg.start - 1; + if (fastlog2(min_seg.bytes) < fastlog2(sizes[prev])) { + break; + } + + min_seg.start = prev; + min_seg.bytes += sizes[prev]; + } + + reftable_free(segs); + return min_seg; +} + +static uint64_t *stack_table_sizes_for_compaction(struct reftable_stack *st) +{ + uint64_t *sizes = + reftable_calloc(sizeof(uint64_t) * st->merged->stack_len); + int version = (st->config.hash_id == GIT_SHA1_FORMAT_ID) ? 1 : 2; + int overhead = header_size(version) - 1; + int i = 0; + for (i = 0; i < st->merged->stack_len; i++) { + sizes[i] = st->readers[i]->size - overhead; + } + return sizes; +} + +int reftable_stack_auto_compact(struct reftable_stack *st) +{ + uint64_t *sizes = stack_table_sizes_for_compaction(st); + struct segment seg = + suggest_compaction_segment(sizes, st->merged->stack_len); + reftable_free(sizes); + if (segment_size(&seg) > 0) + return stack_compact_range_stats(st, seg.start, seg.end - 1, + NULL); + + return 0; +} + +struct reftable_compaction_stats * +reftable_stack_compaction_stats(struct reftable_stack *st) +{ + return &st->stats; +} + +int reftable_stack_read_ref(struct reftable_stack *st, const char *refname, + struct reftable_ref_record *ref) +{ + struct reftable_table tab = { NULL }; + reftable_table_from_merged_table(&tab, reftable_stack_merged_table(st)); + return reftable_table_read_ref(&tab, refname, ref); +} + +int reftable_stack_read_log(struct reftable_stack *st, const char *refname, + struct reftable_log_record *log) +{ + struct reftable_iterator it = { NULL }; + struct reftable_merged_table *mt = reftable_stack_merged_table(st); + int err = reftable_merged_table_seek_log(mt, &it, refname); + if (err) + goto done; + + err = reftable_iterator_next_log(&it, log); + if (err) + goto done; + + if (strcmp(log->refname, refname) || + reftable_log_record_is_deletion(log)) { + err = 1; + goto done; + } + +done: + if (err) { + reftable_log_record_release(log); + } + reftable_iterator_destroy(&it); + return err; +} + +static int stack_check_addition(struct reftable_stack *st, + const char *new_tab_name) +{ + int err = 0; + struct reftable_block_source src = { NULL }; + struct reftable_reader *rd = NULL; + struct reftable_table tab = { NULL }; + struct reftable_ref_record *refs = NULL; + struct reftable_iterator it = { NULL }; + int cap = 0; + int len = 0; + int i = 0; + + if (st->config.skip_name_check) + return 0; + + err = reftable_block_source_from_file(&src, new_tab_name); + if (err < 0) + goto done; + + err = reftable_new_reader(&rd, &src, new_tab_name); + if (err < 0) + goto done; + + err = reftable_reader_seek_ref(rd, &it, ""); + if (err > 0) { + err = 0; + goto done; + } + if (err < 0) + goto done; + + while (1) { + struct reftable_ref_record ref = { NULL }; + err = reftable_iterator_next_ref(&it, &ref); + if (err > 0) { + break; + } + if (err < 0) + goto done; + + if (len >= cap) { + cap = 2 * cap + 1; + refs = reftable_realloc(refs, cap * sizeof(refs[0])); + } + + refs[len++] = ref; + } + + reftable_table_from_merged_table(&tab, reftable_stack_merged_table(st)); + + err = validate_ref_record_addition(tab, refs, len); + +done: + for (i = 0; i < len; i++) { + reftable_ref_record_release(&refs[i]); + } + + free(refs); + reftable_iterator_destroy(&it); + reftable_reader_free(rd); + return err; +} + +static int is_table_name(const char *s) +{ + const char *dot = strrchr(s, '.'); + return dot && !strcmp(dot, ".ref"); +} + +static void remove_maybe_stale_table(struct reftable_stack *st, uint64_t max, + const char *name) +{ + int err = 0; + uint64_t update_idx = 0; + struct reftable_block_source src = { NULL }; + struct reftable_reader *rd = NULL; + struct strbuf table_path = STRBUF_INIT; + stack_filename(&table_path, st, name); + + err = reftable_block_source_from_file(&src, table_path.buf); + if (err < 0) + goto done; + + err = reftable_new_reader(&rd, &src, name); + if (err < 0) + goto done; + + update_idx = reftable_reader_max_update_index(rd); + reftable_reader_free(rd); + + if (update_idx <= max) { + unlink(table_path.buf); + } +done: + strbuf_release(&table_path); +} + +static int reftable_stack_clean_locked(struct reftable_stack *st) +{ + uint64_t max = reftable_merged_table_max_update_index( + reftable_stack_merged_table(st)); + DIR *dir = opendir(st->reftable_dir); + struct dirent *d = NULL; + if (!dir) { + return REFTABLE_IO_ERROR; + } + + while ((d = readdir(dir))) { + int i = 0; + int found = 0; + if (!is_table_name(d->d_name)) + continue; + + for (i = 0; !found && i < st->readers_len; i++) { + found = !strcmp(reader_name(st->readers[i]), d->d_name); + } + if (found) + continue; + + remove_maybe_stale_table(st, max, d->d_name); + } + + closedir(dir); + return 0; +} + +int reftable_stack_clean(struct reftable_stack *st) +{ + struct reftable_addition *add = NULL; + int err = reftable_stack_new_addition(&add, st); + if (err < 0) { + goto done; + } + + err = reftable_stack_reload(st); + if (err < 0) { + goto done; + } + + err = reftable_stack_clean_locked(st); + +done: + reftable_addition_destroy(add); + return err; +} + +int reftable_stack_print_directory(const char *stackdir, uint32_t hash_id) +{ + struct reftable_stack *stack = NULL; + struct reftable_write_options cfg = { .hash_id = hash_id }; + struct reftable_merged_table *merged = NULL; + struct reftable_table table = { NULL }; + + int err = reftable_new_stack(&stack, stackdir, cfg); + if (err < 0) + goto done; + + merged = reftable_stack_merged_table(stack); + reftable_table_from_merged_table(&table, merged); + err = reftable_table_print(&table); +done: + if (stack) + reftable_stack_destroy(stack); + return err; +} diff --git a/reftable/stack.h b/reftable/stack.h new file mode 100644 index 0000000000..f57005846e --- /dev/null +++ b/reftable/stack.h @@ -0,0 +1,41 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef STACK_H +#define STACK_H + +#include "system.h" +#include "reftable-writer.h" +#include "reftable-stack.h" + +struct reftable_stack { + char *list_file; + char *reftable_dir; + int disable_auto_compact; + + struct reftable_write_options config; + + struct reftable_reader **readers; + size_t readers_len; + struct reftable_merged_table *merged; + struct reftable_compaction_stats stats; +}; + +int read_lines(const char *filename, char ***lines); + +struct segment { + int start, end; + int log; + uint64_t bytes; +}; + +int fastlog2(uint64_t sz); +struct segment *sizes_to_segments(int *seglen, uint64_t *sizes, int n); +struct segment suggest_compaction_segment(uint64_t *sizes, int n); + +#endif diff --git a/reftable/stack_test.c b/reftable/stack_test.c new file mode 100644 index 0000000000..eb0b7228b0 --- /dev/null +++ b/reftable/stack_test.c @@ -0,0 +1,953 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "stack.h" + +#include "system.h" + +#include "reftable-reader.h" +#include "merged.h" +#include "basics.h" +#include "constants.h" +#include "record.h" +#include "test_framework.h" +#include "reftable-tests.h" + +#include <sys/types.h> +#include <dirent.h> + +static void clear_dir(const char *dirname) +{ + struct strbuf path = STRBUF_INIT; + strbuf_addstr(&path, dirname); + remove_dir_recursively(&path, 0); + strbuf_release(&path); +} + +static int count_dir_entries(const char *dirname) +{ + DIR *dir = opendir(dirname); + int len = 0; + struct dirent *d; + if (dir == NULL) + return 0; + + while ((d = readdir(dir))) { + if (!strcmp(d->d_name, "..") || !strcmp(d->d_name, ".")) + continue; + len++; + } + closedir(dir); + return len; +} + +/* + * Work linenumber into the tempdir, so we can see which tests forget to + * cleanup. + */ +static char *get_tmp_template(int linenumber) +{ + const char *tmp = getenv("TMPDIR"); + static char template[1024]; + snprintf(template, sizeof(template) - 1, "%s/stack_test-%d.XXXXXX", + tmp ? tmp : "/tmp", linenumber); + return template; +} + +static char *get_tmp_dir(int linenumber) +{ + char *dir = get_tmp_template(linenumber); + EXPECT(mkdtemp(dir)); + return dir; +} + +static void test_read_file(void) +{ + char *fn = get_tmp_template(__LINE__); + int fd = mkstemp(fn); + char out[1024] = "line1\n\nline2\nline3"; + int n, err; + char **names = NULL; + char *want[] = { "line1", "line2", "line3" }; + int i = 0; + + EXPECT(fd > 0); + n = write(fd, out, strlen(out)); + EXPECT(n == strlen(out)); + err = close(fd); + EXPECT(err >= 0); + + err = read_lines(fn, &names); + EXPECT_ERR(err); + + for (i = 0; names[i]; i++) { + EXPECT(0 == strcmp(want[i], names[i])); + } + free_names(names); + remove(fn); +} + +static void test_parse_names(void) +{ + char buf[] = "line\n"; + char **names = NULL; + parse_names(buf, strlen(buf), &names); + + EXPECT(NULL != names[0]); + EXPECT(0 == strcmp(names[0], "line")); + EXPECT(NULL == names[1]); + free_names(names); +} + +static void test_names_equal(void) +{ + char *a[] = { "a", "b", "c", NULL }; + char *b[] = { "a", "b", "d", NULL }; + char *c[] = { "a", "b", NULL }; + + EXPECT(names_equal(a, a)); + EXPECT(!names_equal(a, b)); + EXPECT(!names_equal(a, c)); +} + +static int write_test_ref(struct reftable_writer *wr, void *arg) +{ + struct reftable_ref_record *ref = arg; + reftable_writer_set_limits(wr, ref->update_index, ref->update_index); + return reftable_writer_add_ref(wr, ref); +} + +struct write_log_arg { + struct reftable_log_record *log; + uint64_t update_index; +}; + +static int write_test_log(struct reftable_writer *wr, void *arg) +{ + struct write_log_arg *wla = arg; + + reftable_writer_set_limits(wr, wla->update_index, wla->update_index); + return reftable_writer_add_log(wr, wla->log); +} + +static void test_reftable_stack_add_one(void) +{ + char *dir = get_tmp_dir(__LINE__); + + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err; + struct reftable_ref_record ref = { + .refname = "HEAD", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + struct reftable_ref_record dest = { NULL }; + + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_add(st, &write_test_ref, &ref); + EXPECT_ERR(err); + + err = reftable_stack_read_ref(st, ref.refname, &dest); + EXPECT_ERR(err); + EXPECT(0 == strcmp("master", dest.value.symref)); + + printf("testing print functionality:\n"); + err = reftable_stack_print_directory(dir, GIT_SHA1_FORMAT_ID); + EXPECT_ERR(err); + + err = reftable_stack_print_directory(dir, GIT_SHA256_FORMAT_ID); + EXPECT(err == REFTABLE_FORMAT_ERROR); + + reftable_ref_record_release(&dest); + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void test_reftable_stack_uptodate(void) +{ + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st1 = NULL; + struct reftable_stack *st2 = NULL; + char *dir = get_tmp_dir(__LINE__); + + int err; + struct reftable_ref_record ref1 = { + .refname = "HEAD", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + struct reftable_ref_record ref2 = { + .refname = "branch2", + .update_index = 2, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + + + /* simulate multi-process access to the same stack + by creating two stacks for the same directory. + */ + err = reftable_new_stack(&st1, dir, cfg); + EXPECT_ERR(err); + + err = reftable_new_stack(&st2, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_add(st1, &write_test_ref, &ref1); + EXPECT_ERR(err); + + err = reftable_stack_add(st2, &write_test_ref, &ref2); + EXPECT(err == REFTABLE_LOCK_ERROR); + + err = reftable_stack_reload(st2); + EXPECT_ERR(err); + + err = reftable_stack_add(st2, &write_test_ref, &ref2); + EXPECT_ERR(err); + reftable_stack_destroy(st1); + reftable_stack_destroy(st2); + clear_dir(dir); +} + +static void test_reftable_stack_transaction_api(void) +{ + char *dir = get_tmp_dir(__LINE__); + + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err; + struct reftable_addition *add = NULL; + + struct reftable_ref_record ref = { + .refname = "HEAD", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + struct reftable_ref_record dest = { NULL }; + + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + reftable_addition_destroy(add); + + err = reftable_stack_new_addition(&add, st); + EXPECT_ERR(err); + + err = reftable_addition_add(add, &write_test_ref, &ref); + EXPECT_ERR(err); + + err = reftable_addition_commit(add); + EXPECT_ERR(err); + + reftable_addition_destroy(add); + + err = reftable_stack_read_ref(st, ref.refname, &dest); + EXPECT_ERR(err); + EXPECT(REFTABLE_REF_SYMREF == dest.value_type); + EXPECT(0 == strcmp("master", dest.value.symref)); + + reftable_ref_record_release(&dest); + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void test_reftable_stack_validate_refname(void) +{ + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err; + char *dir = get_tmp_dir(__LINE__); + + int i; + struct reftable_ref_record ref = { + .refname = "a/b", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + char *additions[] = { "a", "a/b/c" }; + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_add(st, &write_test_ref, &ref); + EXPECT_ERR(err); + + for (i = 0; i < ARRAY_SIZE(additions); i++) { + struct reftable_ref_record ref = { + .refname = additions[i], + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + + err = reftable_stack_add(st, &write_test_ref, &ref); + EXPECT(err == REFTABLE_NAME_CONFLICT); + } + + reftable_stack_destroy(st); + clear_dir(dir); +} + +static int write_error(struct reftable_writer *wr, void *arg) +{ + return *((int *)arg); +} + +static void test_reftable_stack_update_index_check(void) +{ + char *dir = get_tmp_dir(__LINE__); + + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err; + struct reftable_ref_record ref1 = { + .refname = "name1", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + struct reftable_ref_record ref2 = { + .refname = "name2", + .update_index = 1, + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_add(st, &write_test_ref, &ref1); + EXPECT_ERR(err); + + err = reftable_stack_add(st, &write_test_ref, &ref2); + EXPECT(err == REFTABLE_API_ERROR); + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void test_reftable_stack_lock_failure(void) +{ + char *dir = get_tmp_dir(__LINE__); + + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err, i; + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + for (i = -1; i != REFTABLE_EMPTY_TABLE_ERROR; i--) { + err = reftable_stack_add(st, &write_error, &i); + EXPECT(err == i); + } + + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void test_reftable_stack_add(void) +{ + int i = 0; + int err = 0; + struct reftable_write_options cfg = { + .exact_log_message = 1, + }; + struct reftable_stack *st = NULL; + char *dir = get_tmp_dir(__LINE__); + + struct reftable_ref_record refs[2] = { { NULL } }; + struct reftable_log_record logs[2] = { { NULL } }; + int N = ARRAY_SIZE(refs); + + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + st->disable_auto_compact = 1; + + for (i = 0; i < N; i++) { + char buf[256]; + snprintf(buf, sizeof(buf), "branch%02d", i); + refs[i].refname = xstrdup(buf); + refs[i].update_index = i + 1; + refs[i].value_type = REFTABLE_REF_VAL1; + refs[i].value.val1 = reftable_malloc(GIT_SHA1_RAWSZ); + set_test_hash(refs[i].value.val1, i); + + logs[i].refname = xstrdup(buf); + logs[i].update_index = N + i + 1; + logs[i].value_type = REFTABLE_LOG_UPDATE; + + logs[i].value.update.new_hash = reftable_malloc(GIT_SHA1_RAWSZ); + logs[i].value.update.email = xstrdup("identity@invalid"); + set_test_hash(logs[i].value.update.new_hash, i); + } + + for (i = 0; i < N; i++) { + int err = reftable_stack_add(st, &write_test_ref, &refs[i]); + EXPECT_ERR(err); + } + + for (i = 0; i < N; i++) { + struct write_log_arg arg = { + .log = &logs[i], + .update_index = reftable_stack_next_update_index(st), + }; + int err = reftable_stack_add(st, &write_test_log, &arg); + EXPECT_ERR(err); + } + + err = reftable_stack_compact_all(st, NULL); + EXPECT_ERR(err); + + for (i = 0; i < N; i++) { + struct reftable_ref_record dest = { NULL }; + + int err = reftable_stack_read_ref(st, refs[i].refname, &dest); + EXPECT_ERR(err); + EXPECT(reftable_ref_record_equal(&dest, refs + i, + GIT_SHA1_RAWSZ)); + reftable_ref_record_release(&dest); + } + + for (i = 0; i < N; i++) { + struct reftable_log_record dest = { NULL }; + int err = reftable_stack_read_log(st, refs[i].refname, &dest); + EXPECT_ERR(err); + EXPECT(reftable_log_record_equal(&dest, logs + i, + GIT_SHA1_RAWSZ)); + reftable_log_record_release(&dest); + } + + /* cleanup */ + reftable_stack_destroy(st); + for (i = 0; i < N; i++) { + reftable_ref_record_release(&refs[i]); + reftable_log_record_release(&logs[i]); + } + clear_dir(dir); +} + +static void test_reftable_stack_log_normalize(void) +{ + int err = 0; + struct reftable_write_options cfg = { + 0, + }; + struct reftable_stack *st = NULL; + char *dir = get_tmp_dir(__LINE__); + + uint8_t h1[GIT_SHA1_RAWSZ] = { 0x01 }, h2[GIT_SHA1_RAWSZ] = { 0x02 }; + + struct reftable_log_record input = { .refname = "branch", + .update_index = 1, + .value_type = REFTABLE_LOG_UPDATE, + .value = { .update = { + .new_hash = h1, + .old_hash = h2, + } } }; + struct reftable_log_record dest = { + .update_index = 0, + }; + struct write_log_arg arg = { + .log = &input, + .update_index = 1, + }; + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + input.value.update.message = "one\ntwo"; + err = reftable_stack_add(st, &write_test_log, &arg); + EXPECT(err == REFTABLE_API_ERROR); + + input.value.update.message = "one"; + err = reftable_stack_add(st, &write_test_log, &arg); + EXPECT_ERR(err); + + err = reftable_stack_read_log(st, input.refname, &dest); + EXPECT_ERR(err); + EXPECT(0 == strcmp(dest.value.update.message, "one\n")); + + input.value.update.message = "two\n"; + arg.update_index = 2; + err = reftable_stack_add(st, &write_test_log, &arg); + EXPECT_ERR(err); + err = reftable_stack_read_log(st, input.refname, &dest); + EXPECT_ERR(err); + EXPECT(0 == strcmp(dest.value.update.message, "two\n")); + + /* cleanup */ + reftable_stack_destroy(st); + reftable_log_record_release(&dest); + clear_dir(dir); +} + +static void test_reftable_stack_tombstone(void) +{ + int i = 0; + char *dir = get_tmp_dir(__LINE__); + + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err; + struct reftable_ref_record refs[2] = { { NULL } }; + struct reftable_log_record logs[2] = { { NULL } }; + int N = ARRAY_SIZE(refs); + struct reftable_ref_record dest = { NULL }; + struct reftable_log_record log_dest = { NULL }; + + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + /* even entries add the refs, odd entries delete them. */ + for (i = 0; i < N; i++) { + const char *buf = "branch"; + refs[i].refname = xstrdup(buf); + refs[i].update_index = i + 1; + if (i % 2 == 0) { + refs[i].value_type = REFTABLE_REF_VAL1; + refs[i].value.val1 = reftable_malloc(GIT_SHA1_RAWSZ); + set_test_hash(refs[i].value.val1, i); + } + + logs[i].refname = xstrdup(buf); + /* update_index is part of the key. */ + logs[i].update_index = 42; + if (i % 2 == 0) { + logs[i].value_type = REFTABLE_LOG_UPDATE; + logs[i].value.update.new_hash = + reftable_malloc(GIT_SHA1_RAWSZ); + set_test_hash(logs[i].value.update.new_hash, i); + logs[i].value.update.email = + xstrdup("identity@invalid"); + } + } + for (i = 0; i < N; i++) { + int err = reftable_stack_add(st, &write_test_ref, &refs[i]); + EXPECT_ERR(err); + } + + for (i = 0; i < N; i++) { + struct write_log_arg arg = { + .log = &logs[i], + .update_index = reftable_stack_next_update_index(st), + }; + int err = reftable_stack_add(st, &write_test_log, &arg); + EXPECT_ERR(err); + } + + err = reftable_stack_read_ref(st, "branch", &dest); + EXPECT(err == 1); + reftable_ref_record_release(&dest); + + err = reftable_stack_read_log(st, "branch", &log_dest); + EXPECT(err == 1); + reftable_log_record_release(&log_dest); + + err = reftable_stack_compact_all(st, NULL); + EXPECT_ERR(err); + + err = reftable_stack_read_ref(st, "branch", &dest); + EXPECT(err == 1); + + err = reftable_stack_read_log(st, "branch", &log_dest); + EXPECT(err == 1); + reftable_ref_record_release(&dest); + reftable_log_record_release(&log_dest); + + /* cleanup */ + reftable_stack_destroy(st); + for (i = 0; i < N; i++) { + reftable_ref_record_release(&refs[i]); + reftable_log_record_release(&logs[i]); + } + clear_dir(dir); +} + +static void test_reftable_stack_hash_id(void) +{ + char *dir = get_tmp_dir(__LINE__); + + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err; + + struct reftable_ref_record ref = { + .refname = "master", + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "target", + .update_index = 1, + }; + struct reftable_write_options cfg32 = { .hash_id = GIT_SHA256_FORMAT_ID }; + struct reftable_stack *st32 = NULL; + struct reftable_write_options cfg_default = { 0 }; + struct reftable_stack *st_default = NULL; + struct reftable_ref_record dest = { NULL }; + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_add(st, &write_test_ref, &ref); + EXPECT_ERR(err); + + /* can't read it with the wrong hash ID. */ + err = reftable_new_stack(&st32, dir, cfg32); + EXPECT(err == REFTABLE_FORMAT_ERROR); + + /* check that we can read it back with default config too. */ + err = reftable_new_stack(&st_default, dir, cfg_default); + EXPECT_ERR(err); + + err = reftable_stack_read_ref(st_default, "master", &dest); + EXPECT_ERR(err); + + EXPECT(reftable_ref_record_equal(&ref, &dest, GIT_SHA1_RAWSZ)); + reftable_ref_record_release(&dest); + reftable_stack_destroy(st); + reftable_stack_destroy(st_default); + clear_dir(dir); +} + +static void test_log2(void) +{ + EXPECT(1 == fastlog2(3)); + EXPECT(2 == fastlog2(4)); + EXPECT(2 == fastlog2(5)); +} + +static void test_sizes_to_segments(void) +{ + uint64_t sizes[] = { 2, 3, 4, 5, 7, 9 }; + /* .................0 1 2 3 4 5 */ + + int seglen = 0; + struct segment *segs = + sizes_to_segments(&seglen, sizes, ARRAY_SIZE(sizes)); + EXPECT(segs[2].log == 3); + EXPECT(segs[2].start == 5); + EXPECT(segs[2].end == 6); + + EXPECT(segs[1].log == 2); + EXPECT(segs[1].start == 2); + EXPECT(segs[1].end == 5); + reftable_free(segs); +} + +static void test_sizes_to_segments_empty(void) +{ + int seglen = 0; + struct segment *segs = sizes_to_segments(&seglen, NULL, 0); + EXPECT(seglen == 0); + reftable_free(segs); +} + +static void test_sizes_to_segments_all_equal(void) +{ + uint64_t sizes[] = { 5, 5 }; + + int seglen = 0; + struct segment *segs = + sizes_to_segments(&seglen, sizes, ARRAY_SIZE(sizes)); + EXPECT(seglen == 1); + EXPECT(segs[0].start == 0); + EXPECT(segs[0].end == 2); + reftable_free(segs); +} + +static void test_suggest_compaction_segment(void) +{ + uint64_t sizes[] = { 128, 64, 17, 16, 9, 9, 9, 16, 16 }; + /* .................0 1 2 3 4 5 6 */ + struct segment min = + suggest_compaction_segment(sizes, ARRAY_SIZE(sizes)); + EXPECT(min.start == 2); + EXPECT(min.end == 7); +} + +static void test_suggest_compaction_segment_nothing(void) +{ + uint64_t sizes[] = { 64, 32, 16, 8, 4, 2 }; + struct segment result = + suggest_compaction_segment(sizes, ARRAY_SIZE(sizes)); + EXPECT(result.start == result.end); +} + +static void test_reflog_expire(void) +{ + char *dir = get_tmp_dir(__LINE__); + + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + struct reftable_log_record logs[20] = { { NULL } }; + int N = ARRAY_SIZE(logs) - 1; + int i = 0; + int err; + struct reftable_log_expiry_config expiry = { + .time = 10, + }; + struct reftable_log_record log = { NULL }; + + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + for (i = 1; i <= N; i++) { + char buf[256]; + snprintf(buf, sizeof(buf), "branch%02d", i); + + logs[i].refname = xstrdup(buf); + logs[i].update_index = i; + logs[i].value_type = REFTABLE_LOG_UPDATE; + logs[i].value.update.time = i; + logs[i].value.update.new_hash = reftable_malloc(GIT_SHA1_RAWSZ); + logs[i].value.update.email = xstrdup("identity@invalid"); + set_test_hash(logs[i].value.update.new_hash, i); + } + + for (i = 1; i <= N; i++) { + struct write_log_arg arg = { + .log = &logs[i], + .update_index = reftable_stack_next_update_index(st), + }; + int err = reftable_stack_add(st, &write_test_log, &arg); + EXPECT_ERR(err); + } + + err = reftable_stack_compact_all(st, NULL); + EXPECT_ERR(err); + + err = reftable_stack_compact_all(st, &expiry); + EXPECT_ERR(err); + + err = reftable_stack_read_log(st, logs[9].refname, &log); + EXPECT(err == 1); + + err = reftable_stack_read_log(st, logs[11].refname, &log); + EXPECT_ERR(err); + + expiry.min_update_index = 15; + err = reftable_stack_compact_all(st, &expiry); + EXPECT_ERR(err); + + err = reftable_stack_read_log(st, logs[14].refname, &log); + EXPECT(err == 1); + + err = reftable_stack_read_log(st, logs[16].refname, &log); + EXPECT_ERR(err); + + /* cleanup */ + reftable_stack_destroy(st); + for (i = 0; i <= N; i++) { + reftable_log_record_release(&logs[i]); + } + clear_dir(dir); + reftable_log_record_release(&log); +} + +static int write_nothing(struct reftable_writer *wr, void *arg) +{ + reftable_writer_set_limits(wr, 1, 1); + return 0; +} + +static void test_empty_add(void) +{ + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + int err; + char *dir = get_tmp_dir(__LINE__); + + struct reftable_stack *st2 = NULL; + + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_add(st, &write_nothing, NULL); + EXPECT_ERR(err); + + err = reftable_new_stack(&st2, dir, cfg); + EXPECT_ERR(err); + clear_dir(dir); + reftable_stack_destroy(st); + reftable_stack_destroy(st2); +} + +static void test_reftable_stack_auto_compaction(void) +{ + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st = NULL; + char *dir = get_tmp_dir(__LINE__); + + int err, i; + int N = 100; + + err = reftable_new_stack(&st, dir, cfg); + EXPECT_ERR(err); + + st->disable_auto_compact = 1; /* call manually below for coverage. */ + for (i = 0; i < N; i++) { + char name[100]; + struct reftable_ref_record ref = { + .refname = name, + .update_index = reftable_stack_next_update_index(st), + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + snprintf(name, sizeof(name), "branch%04d", i); + + err = reftable_stack_add(st, &write_test_ref, &ref); + EXPECT_ERR(err); + + err = reftable_stack_auto_compact(st); + EXPECT(i < 3 || st->merged->stack_len < 2 * fastlog2(i)); + } + + EXPECT(reftable_stack_compaction_stats(st)->entries_written < + (uint64_t)(N * fastlog2(N))); + + reftable_stack_destroy(st); + clear_dir(dir); +} + +static void test_reftable_stack_compaction_concurrent(void) +{ + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st1 = NULL, *st2 = NULL; + char *dir = get_tmp_dir(__LINE__); + + int err, i; + int N = 3; + + err = reftable_new_stack(&st1, dir, cfg); + EXPECT_ERR(err); + + for (i = 0; i < N; i++) { + char name[100]; + struct reftable_ref_record ref = { + .refname = name, + .update_index = reftable_stack_next_update_index(st1), + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + snprintf(name, sizeof(name), "branch%04d", i); + + err = reftable_stack_add(st1, &write_test_ref, &ref); + EXPECT_ERR(err); + } + + err = reftable_new_stack(&st2, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_compact_all(st1, NULL); + EXPECT_ERR(err); + + reftable_stack_destroy(st1); + reftable_stack_destroy(st2); + + EXPECT(count_dir_entries(dir) == 2); + clear_dir(dir); +} + +static void unclean_stack_close(struct reftable_stack *st) +{ + /* break abstraction boundary to simulate unclean shutdown. */ + int i = 0; + for (; i < st->readers_len; i++) { + reftable_reader_free(st->readers[i]); + } + st->readers_len = 0; + FREE_AND_NULL(st->readers); +} + +static void test_reftable_stack_compaction_concurrent_clean(void) +{ + struct reftable_write_options cfg = { 0 }; + struct reftable_stack *st1 = NULL, *st2 = NULL, *st3 = NULL; + char *dir = get_tmp_dir(__LINE__); + + int err, i; + int N = 3; + + err = reftable_new_stack(&st1, dir, cfg); + EXPECT_ERR(err); + + for (i = 0; i < N; i++) { + char name[100]; + struct reftable_ref_record ref = { + .refname = name, + .update_index = reftable_stack_next_update_index(st1), + .value_type = REFTABLE_REF_SYMREF, + .value.symref = "master", + }; + snprintf(name, sizeof(name), "branch%04d", i); + + err = reftable_stack_add(st1, &write_test_ref, &ref); + EXPECT_ERR(err); + } + + err = reftable_new_stack(&st2, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_compact_all(st1, NULL); + EXPECT_ERR(err); + + unclean_stack_close(st1); + unclean_stack_close(st2); + + err = reftable_new_stack(&st3, dir, cfg); + EXPECT_ERR(err); + + err = reftable_stack_clean(st3); + EXPECT_ERR(err); + EXPECT(count_dir_entries(dir) == 2); + + reftable_stack_destroy(st1); + reftable_stack_destroy(st2); + reftable_stack_destroy(st3); + + clear_dir(dir); +} + +int stack_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_empty_add); + RUN_TEST(test_log2); + RUN_TEST(test_names_equal); + RUN_TEST(test_parse_names); + RUN_TEST(test_read_file); + RUN_TEST(test_reflog_expire); + RUN_TEST(test_reftable_stack_add); + RUN_TEST(test_reftable_stack_add_one); + RUN_TEST(test_reftable_stack_auto_compaction); + RUN_TEST(test_reftable_stack_compaction_concurrent); + RUN_TEST(test_reftable_stack_compaction_concurrent_clean); + RUN_TEST(test_reftable_stack_hash_id); + RUN_TEST(test_reftable_stack_lock_failure); + RUN_TEST(test_reftable_stack_log_normalize); + RUN_TEST(test_reftable_stack_tombstone); + RUN_TEST(test_reftable_stack_transaction_api); + RUN_TEST(test_reftable_stack_update_index_check); + RUN_TEST(test_reftable_stack_uptodate); + RUN_TEST(test_reftable_stack_validate_refname); + RUN_TEST(test_sizes_to_segments); + RUN_TEST(test_sizes_to_segments_all_equal); + RUN_TEST(test_sizes_to_segments_empty); + RUN_TEST(test_suggest_compaction_segment); + RUN_TEST(test_suggest_compaction_segment_nothing); + return 0; +} diff --git a/reftable/system.h b/reftable/system.h new file mode 100644 index 0000000000..4907306c0c --- /dev/null +++ b/reftable/system.h @@ -0,0 +1,32 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef SYSTEM_H +#define SYSTEM_H + +/* This header glues the reftable library to the rest of Git */ + +#include "git-compat-util.h" +#include "strbuf.h" +#include "hash.h" /* hash ID, sizes.*/ +#include "dir.h" /* remove_dir_recursively, for tests.*/ + +#include <zlib.h> + +#ifdef NO_UNCOMPRESS2 +/* + * This is uncompress2, which is only available in zlib >= 1.2.9 + * (released as of early 2017) + */ +int uncompress2(Bytef *dest, uLongf *destLen, const Bytef *source, + uLong *sourceLen); +#endif + +int hash_size(uint32_t id); + +#endif diff --git a/reftable/test_framework.c b/reftable/test_framework.c new file mode 100644 index 0000000000..84ac972cad --- /dev/null +++ b/reftable/test_framework.c @@ -0,0 +1,23 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" +#include "test_framework.h" + +#include "basics.h" + +void set_test_hash(uint8_t *p, int i) +{ + memset(p, (uint8_t)i, hash_size(GIT_SHA1_FORMAT_ID)); +} + +ssize_t strbuf_add_void(void *b, const void *data, size_t sz) +{ + strbuf_add(b, data, sz); + return sz; +} diff --git a/reftable/test_framework.h b/reftable/test_framework.h new file mode 100644 index 0000000000..774cb275bf --- /dev/null +++ b/reftable/test_framework.h @@ -0,0 +1,53 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef TEST_FRAMEWORK_H +#define TEST_FRAMEWORK_H + +#include "system.h" +#include "reftable-error.h" + +#define EXPECT_ERR(c) \ + if (c != 0) { \ + fflush(stderr); \ + fflush(stdout); \ + fprintf(stderr, "%s: %d: error == %d (%s), want 0\n", \ + __FILE__, __LINE__, c, reftable_error_str(c)); \ + abort(); \ + } + +#define EXPECT_STREQ(a, b) \ + if (strcmp(a, b)) { \ + fflush(stderr); \ + fflush(stdout); \ + fprintf(stderr, "%s:%d: %s (%s) != %s (%s)\n", __FILE__, \ + __LINE__, #a, a, #b, b); \ + abort(); \ + } + +#define EXPECT(c) \ + if (!(c)) { \ + fflush(stderr); \ + fflush(stdout); \ + fprintf(stderr, "%s: %d: failed assertion %s\n", __FILE__, \ + __LINE__, #c); \ + abort(); \ + } + +#define RUN_TEST(f) \ + fprintf(stderr, "running %s\n", #f); \ + fflush(stderr); \ + f(); + +void set_test_hash(uint8_t *p, int i); + +/* Like strbuf_add, but suitable for passing to reftable_new_writer + */ +ssize_t strbuf_add_void(void *b, const void *data, size_t sz); + +#endif diff --git a/reftable/tree.c b/reftable/tree.c new file mode 100644 index 0000000000..82db7995dd --- /dev/null +++ b/reftable/tree.c @@ -0,0 +1,63 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "tree.h" + +#include "basics.h" +#include "system.h" + +struct tree_node *tree_search(void *key, struct tree_node **rootp, + int (*compare)(const void *, const void *), + int insert) +{ + int res; + if (*rootp == NULL) { + if (!insert) { + return NULL; + } else { + struct tree_node *n = + reftable_calloc(sizeof(struct tree_node)); + n->key = key; + *rootp = n; + return *rootp; + } + } + + res = compare(key, (*rootp)->key); + if (res < 0) + return tree_search(key, &(*rootp)->left, compare, insert); + else if (res > 0) + return tree_search(key, &(*rootp)->right, compare, insert); + return *rootp; +} + +void infix_walk(struct tree_node *t, void (*action)(void *arg, void *key), + void *arg) +{ + if (t->left) { + infix_walk(t->left, action, arg); + } + action(arg, t->key); + if (t->right) { + infix_walk(t->right, action, arg); + } +} + +void tree_free(struct tree_node *t) +{ + if (t == NULL) { + return; + } + if (t->left) { + tree_free(t->left); + } + if (t->right) { + tree_free(t->right); + } + reftable_free(t); +} diff --git a/reftable/tree.h b/reftable/tree.h new file mode 100644 index 0000000000..fbdd002e23 --- /dev/null +++ b/reftable/tree.h @@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef TREE_H +#define TREE_H + +/* tree_node is a generic binary search tree. */ +struct tree_node { + void *key; + struct tree_node *left, *right; +}; + +/* looks for `key` in `rootp` using `compare` as comparison function. If insert + * is set, insert the key if it's not found. Else, return NULL. + */ +struct tree_node *tree_search(void *key, struct tree_node **rootp, + int (*compare)(const void *, const void *), + int insert); + +/* performs an infix walk of the tree. */ +void infix_walk(struct tree_node *t, void (*action)(void *arg, void *key), + void *arg); + +/* + * deallocates the tree nodes recursively. Keys should be deallocated separately + * by walking over the tree. */ +void tree_free(struct tree_node *t); + +#endif diff --git a/reftable/tree_test.c b/reftable/tree_test.c new file mode 100644 index 0000000000..cbff125588 --- /dev/null +++ b/reftable/tree_test.c @@ -0,0 +1,61 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "tree.h" + +#include "basics.h" +#include "record.h" +#include "test_framework.h" +#include "reftable-tests.h" + +static int test_compare(const void *a, const void *b) +{ + return (char *)a - (char *)b; +} + +struct curry { + void *last; +}; + +static void check_increasing(void *arg, void *key) +{ + struct curry *c = arg; + if (c->last) { + EXPECT(test_compare(c->last, key) < 0); + } + c->last = key; +} + +static void test_tree(void) +{ + struct tree_node *root = NULL; + + void *values[11] = { NULL }; + struct tree_node *nodes[11] = { NULL }; + int i = 1; + struct curry c = { NULL }; + do { + nodes[i] = tree_search(values + i, &root, &test_compare, 1); + i = (i * 7) % 11; + } while (i != 1); + + for (i = 1; i < ARRAY_SIZE(nodes); i++) { + EXPECT(values + i == nodes[i]->key); + EXPECT(nodes[i] == + tree_search(values + i, &root, &test_compare, 0)); + } + + infix_walk(root, check_increasing, &c); + tree_free(root); +} + +int tree_test_main(int argc, const char *argv[]) +{ + RUN_TEST(test_tree); + return 0; +} diff --git a/reftable/writer.c b/reftable/writer.c new file mode 100644 index 0000000000..3ca721e9f6 --- /dev/null +++ b/reftable/writer.c @@ -0,0 +1,690 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "writer.h" + +#include "system.h" + +#include "block.h" +#include "constants.h" +#include "record.h" +#include "tree.h" +#include "reftable-error.h" + +/* finishes a block, and writes it to storage */ +static int writer_flush_block(struct reftable_writer *w); + +/* deallocates memory related to the index */ +static void writer_clear_index(struct reftable_writer *w); + +/* finishes writing a 'r' (refs) or 'g' (reflogs) section */ +static int writer_finish_public_section(struct reftable_writer *w); + +static struct reftable_block_stats * +writer_reftable_block_stats(struct reftable_writer *w, uint8_t typ) +{ + switch (typ) { + case 'r': + return &w->stats.ref_stats; + case 'o': + return &w->stats.obj_stats; + case 'i': + return &w->stats.idx_stats; + case 'g': + return &w->stats.log_stats; + } + abort(); + return NULL; +} + +/* write data, queuing the padding for the next write. Returns negative for + * error. */ +static int padded_write(struct reftable_writer *w, uint8_t *data, size_t len, + int padding) +{ + int n = 0; + if (w->pending_padding > 0) { + uint8_t *zeroed = reftable_calloc(w->pending_padding); + int n = w->write(w->write_arg, zeroed, w->pending_padding); + if (n < 0) + return n; + + w->pending_padding = 0; + reftable_free(zeroed); + } + + w->pending_padding = padding; + n = w->write(w->write_arg, data, len); + if (n < 0) + return n; + n += padding; + return 0; +} + +static void options_set_defaults(struct reftable_write_options *opts) +{ + if (opts->restart_interval == 0) { + opts->restart_interval = 16; + } + + if (opts->hash_id == 0) { + opts->hash_id = GIT_SHA1_FORMAT_ID; + } + if (opts->block_size == 0) { + opts->block_size = DEFAULT_BLOCK_SIZE; + } +} + +static int writer_version(struct reftable_writer *w) +{ + return (w->opts.hash_id == 0 || w->opts.hash_id == GIT_SHA1_FORMAT_ID) ? + 1 : + 2; +} + +static int writer_write_header(struct reftable_writer *w, uint8_t *dest) +{ + memcpy(dest, "REFT", 4); + + dest[4] = writer_version(w); + + put_be24(dest + 5, w->opts.block_size); + put_be64(dest + 8, w->min_update_index); + put_be64(dest + 16, w->max_update_index); + if (writer_version(w) == 2) { + put_be32(dest + 24, w->opts.hash_id); + } + return header_size(writer_version(w)); +} + +static void writer_reinit_block_writer(struct reftable_writer *w, uint8_t typ) +{ + int block_start = 0; + if (w->next == 0) { + block_start = header_size(writer_version(w)); + } + + strbuf_release(&w->last_key); + block_writer_init(&w->block_writer_data, typ, w->block, + w->opts.block_size, block_start, + hash_size(w->opts.hash_id)); + w->block_writer = &w->block_writer_data; + w->block_writer->restart_interval = w->opts.restart_interval; +} + +static struct strbuf reftable_empty_strbuf = STRBUF_INIT; + +struct reftable_writer * +reftable_new_writer(ssize_t (*writer_func)(void *, const void *, size_t), + void *writer_arg, struct reftable_write_options *opts) +{ + struct reftable_writer *wp = + reftable_calloc(sizeof(struct reftable_writer)); + strbuf_init(&wp->block_writer_data.last_key, 0); + options_set_defaults(opts); + if (opts->block_size >= (1 << 24)) { + /* TODO - error return? */ + abort(); + } + wp->last_key = reftable_empty_strbuf; + wp->block = reftable_calloc(opts->block_size); + wp->write = writer_func; + wp->write_arg = writer_arg; + wp->opts = *opts; + writer_reinit_block_writer(wp, BLOCK_TYPE_REF); + + return wp; +} + +void reftable_writer_set_limits(struct reftable_writer *w, uint64_t min, + uint64_t max) +{ + w->min_update_index = min; + w->max_update_index = max; +} + +void reftable_writer_free(struct reftable_writer *w) +{ + reftable_free(w->block); + reftable_free(w); +} + +struct obj_index_tree_node { + struct strbuf hash; + uint64_t *offsets; + size_t offset_len; + size_t offset_cap; +}; + +#define OBJ_INDEX_TREE_NODE_INIT \ + { \ + .hash = STRBUF_INIT \ + } + +static int obj_index_tree_node_compare(const void *a, const void *b) +{ + return strbuf_cmp(&((const struct obj_index_tree_node *)a)->hash, + &((const struct obj_index_tree_node *)b)->hash); +} + +static void writer_index_hash(struct reftable_writer *w, struct strbuf *hash) +{ + uint64_t off = w->next; + + struct obj_index_tree_node want = { .hash = *hash }; + + struct tree_node *node = tree_search(&want, &w->obj_index_tree, + &obj_index_tree_node_compare, 0); + struct obj_index_tree_node *key = NULL; + if (node == NULL) { + struct obj_index_tree_node empty = OBJ_INDEX_TREE_NODE_INIT; + key = reftable_malloc(sizeof(struct obj_index_tree_node)); + *key = empty; + + strbuf_reset(&key->hash); + strbuf_addbuf(&key->hash, hash); + tree_search((void *)key, &w->obj_index_tree, + &obj_index_tree_node_compare, 1); + } else { + key = node->key; + } + + if (key->offset_len > 0 && key->offsets[key->offset_len - 1] == off) { + return; + } + + if (key->offset_len == key->offset_cap) { + key->offset_cap = 2 * key->offset_cap + 1; + key->offsets = reftable_realloc( + key->offsets, sizeof(uint64_t) * key->offset_cap); + } + + key->offsets[key->offset_len++] = off; +} + +static int writer_add_record(struct reftable_writer *w, + struct reftable_record *rec) +{ + struct strbuf key = STRBUF_INIT; + int err = -1; + reftable_record_key(rec, &key); + if (strbuf_cmp(&w->last_key, &key) >= 0) { + err = REFTABLE_API_ERROR; + goto done; + } + + strbuf_reset(&w->last_key); + strbuf_addbuf(&w->last_key, &key); + if (w->block_writer == NULL) { + writer_reinit_block_writer(w, reftable_record_type(rec)); + } + + assert(block_writer_type(w->block_writer) == reftable_record_type(rec)); + + if (block_writer_add(w->block_writer, rec) == 0) { + err = 0; + goto done; + } + + err = writer_flush_block(w); + if (err < 0) { + goto done; + } + + writer_reinit_block_writer(w, reftable_record_type(rec)); + err = block_writer_add(w->block_writer, rec); + if (err < 0) { + goto done; + } + + err = 0; +done: + strbuf_release(&key); + return err; +} + +int reftable_writer_add_ref(struct reftable_writer *w, + struct reftable_ref_record *ref) +{ + struct reftable_record rec = { NULL }; + struct reftable_ref_record copy = *ref; + int err = 0; + + if (ref->refname == NULL) + return REFTABLE_API_ERROR; + if (ref->update_index < w->min_update_index || + ref->update_index > w->max_update_index) + return REFTABLE_API_ERROR; + + reftable_record_from_ref(&rec, ©); + copy.update_index -= w->min_update_index; + + err = writer_add_record(w, &rec); + if (err < 0) + return err; + + if (!w->opts.skip_index_objects && reftable_ref_record_val1(ref)) { + struct strbuf h = STRBUF_INIT; + strbuf_add(&h, (char *)reftable_ref_record_val1(ref), + hash_size(w->opts.hash_id)); + writer_index_hash(w, &h); + strbuf_release(&h); + } + + if (!w->opts.skip_index_objects && reftable_ref_record_val2(ref)) { + struct strbuf h = STRBUF_INIT; + strbuf_add(&h, reftable_ref_record_val2(ref), + hash_size(w->opts.hash_id)); + writer_index_hash(w, &h); + strbuf_release(&h); + } + return 0; +} + +int reftable_writer_add_refs(struct reftable_writer *w, + struct reftable_ref_record *refs, int n) +{ + int err = 0; + int i = 0; + QSORT(refs, n, reftable_ref_record_compare_name); + for (i = 0; err == 0 && i < n; i++) { + err = reftable_writer_add_ref(w, &refs[i]); + } + return err; +} + +static int reftable_writer_add_log_verbatim(struct reftable_writer *w, + struct reftable_log_record *log) +{ + struct reftable_record rec = { NULL }; + if (w->block_writer && + block_writer_type(w->block_writer) == BLOCK_TYPE_REF) { + int err = writer_finish_public_section(w); + if (err < 0) + return err; + } + + w->next -= w->pending_padding; + w->pending_padding = 0; + + reftable_record_from_log(&rec, log); + return writer_add_record(w, &rec); +} + +int reftable_writer_add_log(struct reftable_writer *w, + struct reftable_log_record *log) +{ + char *input_log_message = NULL; + struct strbuf cleaned_message = STRBUF_INIT; + int err = 0; + + if (log->value_type == REFTABLE_LOG_DELETION) + return reftable_writer_add_log_verbatim(w, log); + + if (log->refname == NULL) + return REFTABLE_API_ERROR; + + input_log_message = log->value.update.message; + if (!w->opts.exact_log_message && log->value.update.message) { + strbuf_addstr(&cleaned_message, log->value.update.message); + while (cleaned_message.len && + cleaned_message.buf[cleaned_message.len - 1] == '\n') + strbuf_setlen(&cleaned_message, + cleaned_message.len - 1); + if (strchr(cleaned_message.buf, '\n')) { + /* multiple lines not allowed. */ + err = REFTABLE_API_ERROR; + goto done; + } + strbuf_addstr(&cleaned_message, "\n"); + log->value.update.message = cleaned_message.buf; + } + + err = reftable_writer_add_log_verbatim(w, log); + log->value.update.message = input_log_message; +done: + strbuf_release(&cleaned_message); + return err; +} + +int reftable_writer_add_logs(struct reftable_writer *w, + struct reftable_log_record *logs, int n) +{ + int err = 0; + int i = 0; + QSORT(logs, n, reftable_log_record_compare_key); + + for (i = 0; err == 0 && i < n; i++) { + err = reftable_writer_add_log(w, &logs[i]); + } + return err; +} + +static int writer_finish_section(struct reftable_writer *w) +{ + uint8_t typ = block_writer_type(w->block_writer); + uint64_t index_start = 0; + int max_level = 0; + int threshold = w->opts.unpadded ? 1 : 3; + int before_blocks = w->stats.idx_stats.blocks; + int err = writer_flush_block(w); + int i = 0; + struct reftable_block_stats *bstats = NULL; + if (err < 0) + return err; + + while (w->index_len > threshold) { + struct reftable_index_record *idx = NULL; + int idx_len = 0; + + max_level++; + index_start = w->next; + writer_reinit_block_writer(w, BLOCK_TYPE_INDEX); + + idx = w->index; + idx_len = w->index_len; + + w->index = NULL; + w->index_len = 0; + w->index_cap = 0; + for (i = 0; i < idx_len; i++) { + struct reftable_record rec = { NULL }; + reftable_record_from_index(&rec, idx + i); + if (block_writer_add(w->block_writer, &rec) == 0) { + continue; + } + + err = writer_flush_block(w); + if (err < 0) + return err; + + writer_reinit_block_writer(w, BLOCK_TYPE_INDEX); + + err = block_writer_add(w->block_writer, &rec); + if (err != 0) { + /* write into fresh block should always succeed + */ + abort(); + } + } + for (i = 0; i < idx_len; i++) { + strbuf_release(&idx[i].last_key); + } + reftable_free(idx); + } + + writer_clear_index(w); + + err = writer_flush_block(w); + if (err < 0) + return err; + + bstats = writer_reftable_block_stats(w, typ); + bstats->index_blocks = w->stats.idx_stats.blocks - before_blocks; + bstats->index_offset = index_start; + bstats->max_index_level = max_level; + + /* Reinit lastKey, as the next section can start with any key. */ + w->last_key.len = 0; + + return 0; +} + +struct common_prefix_arg { + struct strbuf *last; + int max; +}; + +static void update_common(void *void_arg, void *key) +{ + struct common_prefix_arg *arg = void_arg; + struct obj_index_tree_node *entry = key; + if (arg->last) { + int n = common_prefix_size(&entry->hash, arg->last); + if (n > arg->max) { + arg->max = n; + } + } + arg->last = &entry->hash; +} + +struct write_record_arg { + struct reftable_writer *w; + int err; +}; + +static void write_object_record(void *void_arg, void *key) +{ + struct write_record_arg *arg = void_arg; + struct obj_index_tree_node *entry = key; + struct reftable_obj_record obj_rec = { + .hash_prefix = (uint8_t *)entry->hash.buf, + .hash_prefix_len = arg->w->stats.object_id_len, + .offsets = entry->offsets, + .offset_len = entry->offset_len, + }; + struct reftable_record rec = { NULL }; + if (arg->err < 0) + goto done; + + reftable_record_from_obj(&rec, &obj_rec); + arg->err = block_writer_add(arg->w->block_writer, &rec); + if (arg->err == 0) + goto done; + + arg->err = writer_flush_block(arg->w); + if (arg->err < 0) + goto done; + + writer_reinit_block_writer(arg->w, BLOCK_TYPE_OBJ); + arg->err = block_writer_add(arg->w->block_writer, &rec); + if (arg->err == 0) + goto done; + obj_rec.offset_len = 0; + arg->err = block_writer_add(arg->w->block_writer, &rec); + + /* Should be able to write into a fresh block. */ + assert(arg->err == 0); + +done:; +} + +static void object_record_free(void *void_arg, void *key) +{ + struct obj_index_tree_node *entry = key; + + FREE_AND_NULL(entry->offsets); + strbuf_release(&entry->hash); + reftable_free(entry); +} + +static int writer_dump_object_index(struct reftable_writer *w) +{ + struct write_record_arg closure = { .w = w }; + struct common_prefix_arg common = { NULL }; + if (w->obj_index_tree) { + infix_walk(w->obj_index_tree, &update_common, &common); + } + w->stats.object_id_len = common.max + 1; + + writer_reinit_block_writer(w, BLOCK_TYPE_OBJ); + + if (w->obj_index_tree) { + infix_walk(w->obj_index_tree, &write_object_record, &closure); + } + + if (closure.err < 0) + return closure.err; + return writer_finish_section(w); +} + +static int writer_finish_public_section(struct reftable_writer *w) +{ + uint8_t typ = 0; + int err = 0; + + if (w->block_writer == NULL) + return 0; + + typ = block_writer_type(w->block_writer); + err = writer_finish_section(w); + if (err < 0) + return err; + if (typ == BLOCK_TYPE_REF && !w->opts.skip_index_objects && + w->stats.ref_stats.index_blocks > 0) { + err = writer_dump_object_index(w); + if (err < 0) + return err; + } + + if (w->obj_index_tree) { + infix_walk(w->obj_index_tree, &object_record_free, NULL); + tree_free(w->obj_index_tree); + w->obj_index_tree = NULL; + } + + w->block_writer = NULL; + return 0; +} + +int reftable_writer_close(struct reftable_writer *w) +{ + uint8_t footer[72]; + uint8_t *p = footer; + int err = writer_finish_public_section(w); + int empty_table = w->next == 0; + if (err != 0) + goto done; + w->pending_padding = 0; + if (empty_table) { + /* Empty tables need a header anyway. */ + uint8_t header[28]; + int n = writer_write_header(w, header); + err = padded_write(w, header, n, 0); + if (err < 0) + goto done; + } + + p += writer_write_header(w, footer); + put_be64(p, w->stats.ref_stats.index_offset); + p += 8; + put_be64(p, (w->stats.obj_stats.offset) << 5 | w->stats.object_id_len); + p += 8; + put_be64(p, w->stats.obj_stats.index_offset); + p += 8; + + put_be64(p, w->stats.log_stats.offset); + p += 8; + put_be64(p, w->stats.log_stats.index_offset); + p += 8; + + put_be32(p, crc32(0, footer, p - footer)); + p += 4; + + err = padded_write(w, footer, footer_size(writer_version(w)), 0); + if (err < 0) + goto done; + + if (empty_table) { + err = REFTABLE_EMPTY_TABLE_ERROR; + goto done; + } + +done: + /* free up memory. */ + block_writer_release(&w->block_writer_data); + writer_clear_index(w); + strbuf_release(&w->last_key); + return err; +} + +static void writer_clear_index(struct reftable_writer *w) +{ + int i = 0; + for (i = 0; i < w->index_len; i++) { + strbuf_release(&w->index[i].last_key); + } + + FREE_AND_NULL(w->index); + w->index_len = 0; + w->index_cap = 0; +} + +static const int debug = 0; + +static int writer_flush_nonempty_block(struct reftable_writer *w) +{ + uint8_t typ = block_writer_type(w->block_writer); + struct reftable_block_stats *bstats = + writer_reftable_block_stats(w, typ); + uint64_t block_typ_off = (bstats->blocks == 0) ? w->next : 0; + int raw_bytes = block_writer_finish(w->block_writer); + int padding = 0; + int err = 0; + struct reftable_index_record ir = { .last_key = STRBUF_INIT }; + if (raw_bytes < 0) + return raw_bytes; + + if (!w->opts.unpadded && typ != BLOCK_TYPE_LOG) { + padding = w->opts.block_size - raw_bytes; + } + + if (block_typ_off > 0) { + bstats->offset = block_typ_off; + } + + bstats->entries += w->block_writer->entries; + bstats->restarts += w->block_writer->restart_len; + bstats->blocks++; + w->stats.blocks++; + + if (debug) { + fprintf(stderr, "block %c off %" PRIu64 " sz %d (%d)\n", typ, + w->next, raw_bytes, + get_be24(w->block + w->block_writer->header_off + 1)); + } + + if (w->next == 0) { + writer_write_header(w, w->block); + } + + err = padded_write(w, w->block, raw_bytes, padding); + if (err < 0) + return err; + + if (w->index_cap == w->index_len) { + w->index_cap = 2 * w->index_cap + 1; + w->index = reftable_realloc( + w->index, + sizeof(struct reftable_index_record) * w->index_cap); + } + + ir.offset = w->next; + strbuf_reset(&ir.last_key); + strbuf_addbuf(&ir.last_key, &w->block_writer->last_key); + w->index[w->index_len] = ir; + + w->index_len++; + w->next += padding + raw_bytes; + w->block_writer = NULL; + return 0; +} + +static int writer_flush_block(struct reftable_writer *w) +{ + if (w->block_writer == NULL) + return 0; + if (w->block_writer->entries == 0) + return 0; + return writer_flush_nonempty_block(w); +} + +const struct reftable_stats *writer_stats(struct reftable_writer *w) +{ + return &w->stats; +} diff --git a/reftable/writer.h b/reftable/writer.h new file mode 100644 index 0000000000..09b88673d9 --- /dev/null +++ b/reftable/writer.h @@ -0,0 +1,50 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef WRITER_H +#define WRITER_H + +#include "basics.h" +#include "block.h" +#include "tree.h" +#include "reftable-writer.h" + +struct reftable_writer { + ssize_t (*write)(void *, const void *, size_t); + void *write_arg; + int pending_padding; + struct strbuf last_key; + + /* offset of next block to write. */ + uint64_t next; + uint64_t min_update_index, max_update_index; + struct reftable_write_options opts; + + /* memory buffer for writing */ + uint8_t *block; + + /* writer for the current section. NULL or points to + * block_writer_data */ + struct block_writer *block_writer; + + struct block_writer block_writer_data; + + /* pending index records for the current section */ + struct reftable_index_record *index; + size_t index_len; + size_t index_cap; + + /* + * tree for use with tsearch; used to populate the 'o' inverse OID + * map */ + struct tree_node *obj_index_tree; + + struct reftable_stats stats; +}; + +#endif diff --git a/remote-curl.c b/remote-curl.c index 5975103b96..d69156312b 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -1088,7 +1088,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads, rpc->protocol_header = NULL; while (!err) { - int n = packet_read(rpc->out, NULL, NULL, rpc->buf, rpc->alloc, 0); + int n = packet_read(rpc->out, rpc->buf, rpc->alloc, 0); if (!n) break; rpc->pos = 0; @@ -21,33 +21,6 @@ struct counted_string { size_t len; const char *s; }; -struct rewrite { - const char *base; - size_t baselen; - struct counted_string *instead_of; - int instead_of_nr; - int instead_of_alloc; -}; -struct rewrites { - struct rewrite **rewrite; - int rewrite_alloc; - int rewrite_nr; -}; - -static struct remote **remotes; -static int remotes_alloc; -static int remotes_nr; -static struct hashmap remotes_hash; - -static struct branch **branches; -static int branches_alloc; -static int branches_nr; - -static struct branch *current_branch; -static const char *pushremote_name; - -static struct rewrites rewrites; -static struct rewrites rewrites_push; static int valid_remote(const struct remote *remote) { @@ -92,17 +65,19 @@ static void add_pushurl(struct remote *remote, const char *pushurl) remote->pushurl[remote->pushurl_nr++] = pushurl; } -static void add_pushurl_alias(struct remote *remote, const char *url) +static void add_pushurl_alias(struct remote_state *remote_state, + struct remote *remote, const char *url) { - const char *pushurl = alias_url(url, &rewrites_push); + const char *pushurl = alias_url(url, &remote_state->rewrites_push); if (pushurl != url) add_pushurl(remote, pushurl); } -static void add_url_alias(struct remote *remote, const char *url) +static void add_url_alias(struct remote_state *remote_state, + struct remote *remote, const char *url) { - add_url(remote, alias_url(url, &rewrites)); - add_pushurl_alias(remote, url); + add_url(remote, alias_url(url, &remote_state->rewrites)); + add_pushurl_alias(remote_state, remote, url); } struct remotes_hash_key { @@ -127,13 +102,8 @@ static int remotes_hash_cmp(const void *unused_cmp_data, return strcmp(a->name, b->name); } -static inline void init_remotes_hash(void) -{ - if (!remotes_hash.cmpfn) - hashmap_init(&remotes_hash, remotes_hash_cmp, NULL, 0); -} - -static struct remote *make_remote(const char *name, int len) +static struct remote *make_remote(struct remote_state *remote_state, + const char *name, int len) { struct remote *ret; struct remotes_hash_key lookup; @@ -142,12 +112,11 @@ static struct remote *make_remote(const char *name, int len) if (!len) len = strlen(name); - init_remotes_hash(); lookup.str = name; lookup.len = len; hashmap_entry_init(&lookup_entry, memhash(name, len)); - e = hashmap_get(&remotes_hash, &lookup_entry, &lookup); + e = hashmap_get(&remote_state->remotes_hash, &lookup_entry, &lookup); if (e) return container_of(e, struct remote, ent); @@ -158,15 +127,38 @@ static struct remote *make_remote(const char *name, int len) refspec_init(&ret->push, REFSPEC_PUSH); refspec_init(&ret->fetch, REFSPEC_FETCH); - ALLOC_GROW(remotes, remotes_nr + 1, remotes_alloc); - remotes[remotes_nr++] = ret; + ALLOC_GROW(remote_state->remotes, remote_state->remotes_nr + 1, + remote_state->remotes_alloc); + remote_state->remotes[remote_state->remotes_nr++] = ret; hashmap_entry_init(&ret->ent, lookup_entry.hash); - if (hashmap_put_entry(&remotes_hash, ret, ent)) + if (hashmap_put_entry(&remote_state->remotes_hash, ret, ent)) BUG("hashmap_put overwrote entry after hashmap_get returned NULL"); return ret; } +static void remote_clear(struct remote *remote) +{ + int i; + + free((char *)remote->name); + free((char *)remote->foreign_vcs); + + for (i = 0; i < remote->url_nr; i++) { + free((char *)remote->url[i]); + } + FREE_AND_NULL(remote->pushurl); + + for (i = 0; i < remote->pushurl_nr; i++) { + free((char *)remote->pushurl[i]); + } + FREE_AND_NULL(remote->pushurl); + free((char *)remote->receivepack); + free((char *)remote->uploadpack); + FREE_AND_NULL(remote->http_proxy); + FREE_AND_NULL(remote->http_proxy_authmethod); +} + static void add_merge(struct branch *branch, const char *name) { ALLOC_GROW(branch->merge_name, branch->merge_nr + 1, @@ -174,23 +166,77 @@ static void add_merge(struct branch *branch, const char *name) branch->merge_name[branch->merge_nr++] = name; } -static struct branch *make_branch(const char *name, size_t len) +struct branches_hash_key { + const char *str; + int len; +}; + +static int branches_hash_cmp(const void *unused_cmp_data, + const struct hashmap_entry *eptr, + const struct hashmap_entry *entry_or_key, + const void *keydata) +{ + const struct branch *a, *b; + const struct branches_hash_key *key = keydata; + + a = container_of(eptr, const struct branch, ent); + b = container_of(entry_or_key, const struct branch, ent); + + if (key) + return strncmp(a->name, key->str, key->len) || + a->name[key->len]; + else + return strcmp(a->name, b->name); +} + +static struct branch *find_branch(struct remote_state *remote_state, + const char *name, size_t len) +{ + struct branches_hash_key lookup; + struct hashmap_entry lookup_entry, *e; + + if (!len) + len = strlen(name); + + lookup.str = name; + lookup.len = len; + hashmap_entry_init(&lookup_entry, memhash(name, len)); + + e = hashmap_get(&remote_state->branches_hash, &lookup_entry, &lookup); + if (e) + return container_of(e, struct branch, ent); + + return NULL; +} + +static void die_on_missing_branch(struct repository *repo, + struct branch *branch) +{ + /* branch == NULL is always valid because it represents detached HEAD. */ + if (branch && + branch != find_branch(repo->remote_state, branch->name, 0)) + die("branch %s was not found in the repository", branch->name); +} + +static struct branch *make_branch(struct remote_state *remote_state, + const char *name, size_t len) { struct branch *ret; - int i; - for (i = 0; i < branches_nr; i++) { - if (!strncmp(name, branches[i]->name, len) && - !branches[i]->name[len]) - return branches[i]; - } + ret = find_branch(remote_state, name, len); + if (ret) + return ret; - ALLOC_GROW(branches, branches_nr + 1, branches_alloc); + ALLOC_GROW(remote_state->branches, remote_state->branches_nr + 1, + remote_state->branches_alloc); CALLOC_ARRAY(ret, 1); - branches[branches_nr++] = ret; + remote_state->branches[remote_state->branches_nr++] = ret; ret->name = xstrndup(name, len); ret->refname = xstrfmt("refs/heads/%s", ret->name); + hashmap_entry_init(&ret->ent, memhash(name, len)); + if (hashmap_put_entry(&remote_state->branches_hash, ret, ent)) + BUG("hashmap_put overwrote entry after hashmap_get returned NULL"); return ret; } @@ -229,7 +275,8 @@ static const char *skip_spaces(const char *s) return s; } -static void read_remotes_file(struct remote *remote) +static void read_remotes_file(struct remote_state *remote_state, + struct remote *remote) { struct strbuf buf = STRBUF_INIT; FILE *f = fopen_or_warn(git_path("remotes/%s", remote->name), "r"); @@ -244,7 +291,8 @@ static void read_remotes_file(struct remote *remote) strbuf_rtrim(&buf); if (skip_prefix(buf.buf, "URL:", &v)) - add_url_alias(remote, xstrdup(skip_spaces(v))); + add_url_alias(remote_state, remote, + xstrdup(skip_spaces(v))); else if (skip_prefix(buf.buf, "Push:", &v)) refspec_append(&remote->push, skip_spaces(v)); else if (skip_prefix(buf.buf, "Pull:", &v)) @@ -254,7 +302,8 @@ static void read_remotes_file(struct remote *remote) fclose(f); } -static void read_branches_file(struct remote *remote) +static void read_branches_file(struct remote_state *remote_state, + struct remote *remote) { char *frag; struct strbuf buf = STRBUF_INIT; @@ -286,7 +335,7 @@ static void read_branches_file(struct remote *remote) else frag = (char *)git_default_branch_name(0); - add_url_alias(remote, strbuf_detach(&buf, NULL)); + add_url_alias(remote_state, remote, strbuf_detach(&buf, NULL)); refspec_appendf(&remote->fetch, "refs/heads/%s:refs/heads/%s", frag, remote->name); @@ -305,10 +354,12 @@ static int handle_config(const char *key, const char *value, void *cb) const char *subkey; struct remote *remote; struct branch *branch; + struct remote_state *remote_state = cb; + if (parse_config_key(key, "branch", &name, &namelen, &subkey) >= 0) { if (!name) return 0; - branch = make_branch(name, namelen); + branch = make_branch(remote_state, name, namelen); if (!strcmp(subkey, "remote")) { return git_config_string(&branch->remote_name, key, value); } else if (!strcmp(subkey, "pushremote")) { @@ -327,12 +378,14 @@ static int handle_config(const char *key, const char *value, void *cb) if (!strcmp(subkey, "insteadof")) { if (!value) return config_error_nonbool(key); - rewrite = make_rewrite(&rewrites, name, namelen); + rewrite = make_rewrite(&remote_state->rewrites, name, + namelen); add_instead_of(rewrite, xstrdup(value)); } else if (!strcmp(subkey, "pushinsteadof")) { if (!value) return config_error_nonbool(key); - rewrite = make_rewrite(&rewrites_push, name, namelen); + rewrite = make_rewrite(&remote_state->rewrites_push, + name, namelen); add_instead_of(rewrite, xstrdup(value)); } } @@ -342,7 +395,8 @@ static int handle_config(const char *key, const char *value, void *cb) /* Handle remote.* variables */ if (!name && !strcmp(subkey, "pushdefault")) - return git_config_string(&pushremote_name, key, value); + return git_config_string(&remote_state->pushremote_name, key, + value); if (!name) return 0; @@ -352,7 +406,7 @@ static int handle_config(const char *key, const char *value, void *cb) name); return 0; } - remote = make_remote(name, namelen); + remote = make_remote(remote_state, name, namelen); remote->origin = REMOTE_CONFIG; if (current_config_scope() == CONFIG_SCOPE_LOCAL || current_config_scope() == CONFIG_SCOPE_WORKTREE) @@ -422,44 +476,52 @@ static int handle_config(const char *key, const char *value, void *cb) return 0; } -static void alias_all_urls(void) +static void alias_all_urls(struct remote_state *remote_state) { int i, j; - for (i = 0; i < remotes_nr; i++) { + for (i = 0; i < remote_state->remotes_nr; i++) { int add_pushurl_aliases; - if (!remotes[i]) + if (!remote_state->remotes[i]) continue; - for (j = 0; j < remotes[i]->pushurl_nr; j++) { - remotes[i]->pushurl[j] = alias_url(remotes[i]->pushurl[j], &rewrites); + for (j = 0; j < remote_state->remotes[i]->pushurl_nr; j++) { + remote_state->remotes[i]->pushurl[j] = + alias_url(remote_state->remotes[i]->pushurl[j], + &remote_state->rewrites); } - add_pushurl_aliases = remotes[i]->pushurl_nr == 0; - for (j = 0; j < remotes[i]->url_nr; j++) { + add_pushurl_aliases = remote_state->remotes[i]->pushurl_nr == 0; + for (j = 0; j < remote_state->remotes[i]->url_nr; j++) { if (add_pushurl_aliases) - add_pushurl_alias(remotes[i], remotes[i]->url[j]); - remotes[i]->url[j] = alias_url(remotes[i]->url[j], &rewrites); + add_pushurl_alias( + remote_state, remote_state->remotes[i], + remote_state->remotes[i]->url[j]); + remote_state->remotes[i]->url[j] = + alias_url(remote_state->remotes[i]->url[j], + &remote_state->rewrites); } } } -static void read_config(void) +static void read_config(struct repository *repo) { - static int loaded; int flag; - if (loaded) + if (repo->remote_state->initialized) return; - loaded = 1; + repo->remote_state->initialized = 1; - current_branch = NULL; + repo->remote_state->current_branch = NULL; if (startup_info->have_repository) { - const char *head_ref = resolve_ref_unsafe("HEAD", 0, NULL, &flag); + int ignore_errno; + const char *head_ref = refs_resolve_ref_unsafe( + get_main_ref_store(repo), "HEAD", 0, NULL, &flag, &ignore_errno); if (head_ref && (flag & REF_ISSYMREF) && skip_prefix(head_ref, "refs/heads/", &head_ref)) { - current_branch = make_branch(head_ref, strlen(head_ref)); + repo->remote_state->current_branch = make_branch( + repo->remote_state, head_ref, strlen(head_ref)); } } - git_config(handle_config, NULL); - alias_all_urls(); + repo_config(repo, handle_config, repo->remote_state); + alias_all_urls(repo->remote_state); } static int valid_remote_nick(const char *name) @@ -474,7 +536,9 @@ static int valid_remote_nick(const char *name) return 1; } -const char *remote_for_branch(struct branch *branch, int *explicit) +static const char *remotes_remote_for_branch(struct remote_state *remote_state, + struct branch *branch, + int *explicit) { if (branch && branch->remote_name) { if (explicit) @@ -486,32 +550,61 @@ const char *remote_for_branch(struct branch *branch, int *explicit) return "origin"; } -const char *pushremote_for_branch(struct branch *branch, int *explicit) +const char *repo_remote_for_branch(struct repository *repo, struct branch *branch, int *explicit) +{ + read_config(repo); + die_on_missing_branch(repo, branch); + + return remotes_remote_for_branch(repo->remote_state, branch, + explicit); +} + +static const char * +remotes_pushremote_for_branch(struct remote_state *remote_state, + struct branch *branch, int *explicit) { if (branch && branch->pushremote_name) { if (explicit) *explicit = 1; return branch->pushremote_name; } - if (pushremote_name) { + if (remote_state->pushremote_name) { if (explicit) *explicit = 1; - return pushremote_name; + return remote_state->pushremote_name; } - return remote_for_branch(branch, explicit); + return remotes_remote_for_branch(remote_state, branch, explicit); } -const char *remote_ref_for_branch(struct branch *branch, int for_push) +const char *repo_pushremote_for_branch(struct repository *repo, struct branch *branch, int *explicit) { + read_config(repo); + die_on_missing_branch(repo, branch); + + return remotes_pushremote_for_branch(repo->remote_state, + branch, explicit); +} + +static struct remote *remotes_remote_get(struct remote_state *remote_state, + const char *name); + +const char *repo_remote_ref_for_branch(struct repository *repo, struct branch *branch, int for_push) +{ + read_config(repo); + die_on_missing_branch(repo, branch); + if (branch) { if (!for_push) { if (branch->merge_nr) { return branch->merge_name[0]; } } else { - const char *dst, *remote_name = - pushremote_for_branch(branch, NULL); - struct remote *remote = remote_get(remote_name); + const char *dst, + *remote_name = remotes_pushremote_for_branch( + repo->remote_state, branch, + NULL); + struct remote *remote = remotes_remote_get( + repo->remote_state, remote_name); if (remote && remote->push.nr && (dst = apply_refspecs(&remote->push, @@ -523,41 +616,58 @@ const char *remote_ref_for_branch(struct branch *branch, int for_push) return NULL; } -static struct remote *remote_get_1(const char *name, - const char *(*get_default)(struct branch *, int *)) +static struct remote * +remotes_remote_get_1(struct remote_state *remote_state, const char *name, + const char *(*get_default)(struct remote_state *, + struct branch *, int *)) { struct remote *ret; int name_given = 0; - read_config(); - if (name) name_given = 1; else - name = get_default(current_branch, &name_given); + name = get_default(remote_state, remote_state->current_branch, + &name_given); - ret = make_remote(name, 0); + ret = make_remote(remote_state, name, 0); if (valid_remote_nick(name) && have_git_dir()) { if (!valid_remote(ret)) - read_remotes_file(ret); + read_remotes_file(remote_state, ret); if (!valid_remote(ret)) - read_branches_file(ret); + read_branches_file(remote_state, ret); } if (name_given && !valid_remote(ret)) - add_url_alias(ret, name); + add_url_alias(remote_state, ret, name); if (!valid_remote(ret)) return NULL; return ret; } -struct remote *remote_get(const char *name) +static inline struct remote * +remotes_remote_get(struct remote_state *remote_state, const char *name) { - return remote_get_1(name, remote_for_branch); + return remotes_remote_get_1(remote_state, name, + remotes_remote_for_branch); } -struct remote *pushremote_get(const char *name) +struct remote *repo_remote_get(struct repository *repo, const char *name) { - return remote_get_1(name, pushremote_for_branch); + read_config(repo); + return remotes_remote_get(repo->remote_state, name); +} + +static inline struct remote * +remotes_pushremote_get(struct remote_state *remote_state, const char *name) +{ + return remotes_remote_get_1(remote_state, name, + remotes_pushremote_for_branch); +} + +struct remote *repo_pushremote_get(struct repository *repo, const char *name) +{ + read_config(repo); + return remotes_pushremote_get(repo->remote_state, name); } int remote_is_configured(struct remote *remote, int in_repo) @@ -569,15 +679,17 @@ int remote_is_configured(struct remote *remote, int in_repo) return !!remote->origin; } -int for_each_remote(each_remote_fn fn, void *priv) +int repo_for_each_remote(struct repository *repo, each_remote_fn fn, void *priv) { int i, result = 0; - read_config(); - for (i = 0; i < remotes_nr && !result; i++) { - struct remote *r = remotes[i]; - if (!r) + read_config(repo); + for (i = 0; i < repo->remote_state->remotes_nr && !result; + i++) { + struct remote *remote = + repo->remote_state->remotes[i]; + if (!remote) continue; - result = fn(r, priv); + result = fn(remote, priv); } return result; } @@ -1642,7 +1754,7 @@ void set_ref_status_for_push(struct ref *remote_refs, int send_mirror, } } -static void set_merge(struct branch *ret) +static void set_merge(struct remote_state *remote_state, struct branch *ret) { struct remote *remote; char *ref; @@ -1662,7 +1774,7 @@ static void set_merge(struct branch *ret) return; } - remote = remote_get(ret->remote_name); + remote = remotes_remote_get(remote_state, ret->remote_name); CALLOC_ARRAY(ret->merge, ret->merge_nr); for (i = 0; i < ret->merge_nr; i++) { @@ -1679,16 +1791,16 @@ static void set_merge(struct branch *ret) } } -struct branch *branch_get(const char *name) +struct branch *repo_branch_get(struct repository *repo, const char *name) { struct branch *ret; - read_config(); + read_config(repo); if (!name || !*name || !strcmp(name, "HEAD")) - ret = current_branch; + ret = repo->remote_state->current_branch; else - ret = make_branch(name, strlen(name)); - set_merge(ret); + ret = make_branch(repo->remote_state, name, strlen(name)); + set_merge(repo->remote_state, ret); return ret; } @@ -1759,11 +1871,14 @@ static const char *tracking_for_push_dest(struct remote *remote, return ret; } -static const char *branch_get_push_1(struct branch *branch, struct strbuf *err) +static const char *branch_get_push_1(struct remote_state *remote_state, + struct branch *branch, struct strbuf *err) { struct remote *remote; - remote = remote_get(pushremote_for_branch(branch, NULL)); + remote = remotes_remote_get( + remote_state, + remotes_pushremote_for_branch(remote_state, branch, NULL)); if (!remote) return error_buf(err, _("branch '%s' has no remote for pushing"), @@ -1819,13 +1934,17 @@ static const char *branch_get_push_1(struct branch *branch, struct strbuf *err) BUG("unhandled push situation"); } -const char *branch_get_push(struct branch *branch, struct strbuf *err) +const char *repo_branch_get_push(struct repository *repo, struct branch *branch, struct strbuf *err) { + read_config(repo); + die_on_missing_branch(repo, branch); + if (!branch) return error_buf(err, _("HEAD does not point to a branch")); if (!branch->push_tracking_ref) - branch->push_tracking_ref = branch_get_push_1(branch, err); + branch->push_tracking_ref = branch_get_push_1( + repo->remote_state, branch, err); return branch->push_tracking_ref; } @@ -2078,15 +2197,16 @@ static int stat_branch_pair(const char *branch_name, const char *base, * upstream defined, or ref does not exist). Returns 0 if the commits are * identical. Returns 1 if commits are different. */ -int stat_tracking_info(struct branch *branch, int *num_ours, int *num_theirs, - const char **tracking_name, int for_push, - enum ahead_behind_flags abf) +int repo_stat_tracking_info(struct repository *repo, struct branch *branch, + int *num_ours, int *num_theirs, + const char **tracking_name, int for_push, + enum ahead_behind_flags abf) { const char *base; /* Cannot stat unless we are marked to build on top of somebody else. */ - base = for_push ? branch_get_push(branch, NULL) : - branch_get_upstream(branch, NULL); + base = for_push ? repo_branch_get_push(repo, branch, NULL) : + branch_get_upstream(branch, NULL); if (tracking_name) *tracking_name = base; if (!base) @@ -2098,15 +2218,16 @@ int stat_tracking_info(struct branch *branch, int *num_ours, int *num_theirs, /* * Return true when there is anything to report, otherwise false. */ -int format_tracking_info(struct branch *branch, struct strbuf *sb, - enum ahead_behind_flags abf) +int repo_format_tracking_info(struct repository *repo, struct branch *branch, + struct strbuf *sb, enum ahead_behind_flags abf) { int ours, theirs, sti; const char *full_base; char *base; int upstream_is_gone = 0; - sti = stat_tracking_info(branch, &ours, &theirs, &full_base, 0, abf); + sti = repo_stat_tracking_info(repo, branch, &ours, &theirs, &full_base, + 0, abf); if (sti < 0) { if (!full_base) return 0; @@ -2585,3 +2706,35 @@ void apply_push_cas(struct push_cas_option *cas, check_if_includes_upstream(ref); } } + +struct remote_state *remote_state_new(void) +{ + struct remote_state *r = xmalloc(sizeof(*r)); + + memset(r, 0, sizeof(*r)); + + hashmap_init(&r->remotes_hash, remotes_hash_cmp, NULL, 0); + hashmap_init(&r->branches_hash, branches_hash_cmp, NULL, 0); + return r; +} + +void remote_state_clear(struct remote_state *remote_state) +{ + int i; + + for (i = 0; i < remote_state->remotes_nr; i++) { + remote_clear(remote_state->remotes[i]); + } + FREE_AND_NULL(remote_state->remotes); + remote_state->remotes_alloc = 0; + remote_state->remotes_nr = 0; + + hashmap_clear_and_free(&remote_state->remotes_hash, struct remote, ent); + + for (i = 0; i < remote_state->branches_nr; i++) { + FREE_AND_NULL(remote_state->branches[i]); + } + FREE_AND_NULL(remote_state->branches); + remote_state->branches_alloc = 0; + remote_state->branches_nr = 0; +} @@ -23,6 +23,43 @@ enum { REMOTE_BRANCHES }; +struct rewrite { + const char *base; + size_t baselen; + struct counted_string *instead_of; + int instead_of_nr; + int instead_of_alloc; +}; + +struct rewrites { + struct rewrite **rewrite; + int rewrite_alloc; + int rewrite_nr; +}; + +struct remote_state { + struct remote **remotes; + int remotes_alloc; + int remotes_nr; + struct hashmap remotes_hash; + + struct branch **branches; + int branches_alloc; + int branches_nr; + struct hashmap branches_hash; + + struct branch *current_branch; + const char *pushremote_name; + + struct rewrites rewrites; + struct rewrites rewrites_push; + + int initialized; +}; + +void remote_state_clear(struct remote_state *remote_state); +struct remote_state *remote_state_new(void); + struct remote { struct hashmap_entry ent; @@ -83,15 +120,28 @@ struct remote { * remote_get(NULL) will return the default remote, given the current branch * and configuration. */ -struct remote *remote_get(const char *name); - -struct remote *pushremote_get(const char *name); +struct remote *repo_remote_get(struct repository *repo, const char *name); +static inline struct remote *remote_get(const char *name) +{ + return repo_remote_get(the_repository, name); +} + +struct remote *repo_pushremote_get(struct repository *repo, const char *name); +static inline struct remote *pushremote_get(const char *name) +{ + return repo_pushremote_get(the_repository, name); +} int remote_is_configured(struct remote *remote, int in_repo); typedef int each_remote_fn(struct remote *remote, void *priv); /* iterate through struct remotes */ -int for_each_remote(each_remote_fn fn, void *priv); +int repo_for_each_remote(struct repository *repo, each_remote_fn fn, + void *priv); +static inline int for_each_remote(each_remote_fn fn, void *priv) +{ + return repo_for_each_remote(the_repository, fn, priv); +} int remote_has_url(struct remote *remote, const char *url); @@ -256,6 +306,7 @@ int remote_find_tracking(struct remote *remote, struct refspec_item *refspec); * branch_get(name) for "refs/heads/{name}", or with branch_get(NULL) for HEAD. */ struct branch { + struct hashmap_entry ent; /* The short name of the branch. */ const char *name; @@ -286,10 +337,29 @@ struct branch { const char *push_tracking_ref; }; -struct branch *branch_get(const char *name); -const char *remote_for_branch(struct branch *branch, int *explicit); -const char *pushremote_for_branch(struct branch *branch, int *explicit); -const char *remote_ref_for_branch(struct branch *branch, int for_push); +struct branch *repo_branch_get(struct repository *repo, const char *name); +static inline struct branch *branch_get(const char *name) +{ + return repo_branch_get(the_repository, name); +} + +const char *repo_remote_for_branch(struct repository *repo, struct branch *branch, int *explicit); +static inline const char *remote_for_branch(struct branch *branch, int *explicit) +{ + return repo_remote_for_branch(the_repository, branch, explicit); +} + +const char *repo_pushremote_for_branch(struct repository *repo, struct branch *branch, int *explicit); +static inline const char *pushremote_for_branch(struct branch *branch, int *explicit) +{ + return repo_pushremote_for_branch(the_repository, branch, explicit); +} + +const char *repo_remote_ref_for_branch(struct repository *repo, struct branch *branch, int for_push); +static inline const char *remote_ref_for_branch(struct branch *branch, int for_push) +{ + return repo_remote_ref_for_branch(the_repository, branch, for_push); +} /* returns true if the given branch has merge configuration given. */ int branch_has_merge_config(struct branch *branch); @@ -313,7 +383,11 @@ const char *branch_get_upstream(struct branch *branch, struct strbuf *err); * * The return value and `err` conventions match those of `branch_get_upstream`. */ -const char *branch_get_push(struct branch *branch, struct strbuf *err); +const char *repo_branch_get_push(struct repository *repo, struct branch *branch, struct strbuf *err); +static inline const char *branch_get_push(struct branch *branch, struct strbuf *err) +{ + return repo_branch_get_push(the_repository, branch, err); +} /* Flags to match_refs. */ enum match_refs_flags { @@ -332,11 +406,27 @@ enum ahead_behind_flags { }; /* Reporting of tracking info */ -int stat_tracking_info(struct branch *branch, int *num_ours, int *num_theirs, - const char **upstream_name, int for_push, - enum ahead_behind_flags abf); -int format_tracking_info(struct branch *branch, struct strbuf *sb, - enum ahead_behind_flags abf); +int repo_stat_tracking_info(struct repository *repo, struct branch *branch, + int *num_ours, int *num_theirs, + const char **upstream_name, int for_push, + enum ahead_behind_flags abf); +static inline int stat_tracking_info(struct branch *branch, int *num_ours, + int *num_theirs, + const char **upstream_name, int for_push, + enum ahead_behind_flags abf) +{ + return repo_stat_tracking_info(the_repository, branch, num_ours, + num_theirs, upstream_name, for_push, + abf); +} + +int repo_format_tracking_info(struct repository *repo, struct branch *branch, + struct strbuf *sb, enum ahead_behind_flags abf); +static inline int format_tracking_info(struct branch *branch, struct strbuf *sb, + enum ahead_behind_flags abf) +{ + return repo_format_tracking_info(the_repository, branch, sb, abf); +} struct ref *get_local_heads(void); /* diff --git a/repo-settings.c b/repo-settings.c index b93e91a212..3ed763ad9c 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -2,6 +2,7 @@ #include "config.h" #include "repository.h" #include "midx.h" +#include "compat/fsmonitor/fsm-listen.h" static void repo_cfg_bool(struct repository *r, const char *key, int *dest, int def) diff --git a/repository.c b/repository.c index c5b90ba93e..34610c5a33 100644 --- a/repository.c +++ b/repository.c @@ -9,6 +9,7 @@ #include "config.h" #include "object.h" #include "lockfile.h" +#include "remote.h" #include "submodule-config.h" #include "sparse-index.h" #include "promisor-remote.h" @@ -24,6 +25,7 @@ void initialize_the_repository(void) the_repo.index = &the_index; the_repo.objects = raw_object_store_new(); + the_repo.remote_state = remote_state_new(); the_repo.parsed_objects = parsed_object_pool_new(); repo_set_hash_algo(&the_repo, GIT_HASH_SHA1); @@ -80,6 +82,8 @@ void repo_set_gitdir(struct repository *repo, expand_base_dir(&repo->objects->odb->path, o->object_dir, repo->commondir, "objects"); + repo->objects->odb->disable_ref_updates = o->disable_ref_updates; + free(repo->objects->alternate_db); repo->objects->alternate_db = xstrdup_or_null(o->alternate_db); expand_base_dir(&repo->graft_file, o->graft_file, @@ -164,6 +168,7 @@ int repo_init(struct repository *repo, repo->objects = raw_object_store_new(); repo->parsed_objects = parsed_object_pool_new(); + repo->remote_state = remote_state_new(); if (repo_init_gitdir(repo, gitdir)) goto error; @@ -270,6 +275,11 @@ void repo_clear(struct repository *repo) promisor_remote_clear(repo->promisor_remote_config); FREE_AND_NULL(repo->promisor_remote_config); } + + if (repo->remote_state) { + remote_state_clear(repo->remote_state); + FREE_AND_NULL(repo->remote_state); + } } int repo_read_index(struct repository *repo) diff --git a/repository.h b/repository.h index a057653981..14b36cd8f0 100644 --- a/repository.h +++ b/repository.h @@ -4,6 +4,7 @@ #include "path.h" struct config_set; +struct fsmonitor_settings; struct git_hash_algo; struct index_state; struct lock_file; @@ -11,6 +12,7 @@ struct pathspec; struct raw_object_store; struct submodule_cache; struct promisor_remote_config; +struct remote_state; enum untracked_cache_setting { UNTRACKED_CACHE_KEEP, @@ -34,6 +36,8 @@ struct repo_settings { int command_requires_full_index; int sparse_index; + struct fsmonitor_settings *fsmonitor; /* lazy loaded */ + int index_version; enum untracked_cache_setting core_untracked_cache; @@ -127,6 +131,9 @@ struct repository { */ struct index_state *index; + /* Repository's remotes and associated structures. */ + struct remote_state *remote_state; + /* Repository's current hash algorithm, as serialized on disk. */ const struct git_hash_algo *hash_algo; @@ -158,6 +165,7 @@ struct set_gitdir_args { const char *graft_file; const char *index_file; const char *alternate_db; + int disable_ref_updates; }; void repo_set_gitdir(struct repository *repo, const char *root, @@ -7,32 +7,105 @@ #include "tree-walk.h" #include "tree.h" #include "unpack-trees.h" +#include "hook.h" -int reset_head(struct repository *r, struct object_id *oid, const char *action, - const char *switch_to_branch, unsigned flags, - const char *reflog_orig_head, const char *reflog_head, - const char *default_reflog_action) +static int update_refs(const struct reset_head_opts *opts, + const struct object_id *oid) { - unsigned detach_head = flags & RESET_HEAD_DETACH; - unsigned reset_hard = flags & RESET_HEAD_HARD; - unsigned run_hook = flags & RESET_HEAD_RUN_POST_CHECKOUT_HOOK; - unsigned refs_only = flags & RESET_HEAD_REFS_ONLY; - unsigned update_orig_head = flags & RESET_ORIG_HEAD; + unsigned detach_head = opts->flags & RESET_HEAD_DETACH; + unsigned run_hook = opts->flags & RESET_HEAD_RUN_POST_CHECKOUT_HOOK; + unsigned update_orig_head = opts->flags & RESET_ORIG_HEAD; + const struct object_id *orig_head = opts->orig_head; + const char *switch_to_branch = opts->branch; + const char *reflog_branch = opts->branch_msg; + const char *reflog_head = opts->head_msg; + const char *reflog_orig_head = opts->orig_head_msg; + const char *default_reflog_action = opts->default_reflog_action; + struct object_id *orig = NULL, oid_orig, *old_orig = NULL, oid_old_orig; + struct strbuf msg = STRBUF_INIT; + const char *reflog_action; + size_t prefix_len; + int ret; + + if ((update_orig_head && !reflog_orig_head) || !reflog_head) { + if (!default_reflog_action) + BUG("default_reflog_action must be given when reflog messages are omitted"); + reflog_action = getenv(GIT_REFLOG_ACTION_ENVIRONMENT); + strbuf_addf(&msg, "%s: ", reflog_action ? reflog_action : + default_reflog_action); + } + prefix_len = msg.len; + + if (update_orig_head) { + if (!get_oid("ORIG_HEAD", &oid_old_orig)) + old_orig = &oid_old_orig; + if (!get_oid("HEAD", &oid_orig)) { + orig = &oid_orig; + if (!reflog_orig_head) { + strbuf_addstr(&msg, "updating ORIG_HEAD"); + reflog_orig_head = msg.buf; + } + update_ref(reflog_orig_head, "ORIG_HEAD", + orig_head ? orig_head : orig, + old_orig, 0, UPDATE_REFS_MSG_ON_ERR); + } else if (old_orig) + delete_ref(NULL, "ORIG_HEAD", old_orig, 0); + } + + if (!reflog_head) { + strbuf_setlen(&msg, prefix_len); + strbuf_addstr(&msg, "updating HEAD"); + reflog_head = msg.buf; + } + if (!switch_to_branch) + ret = update_ref(reflog_head, "HEAD", oid, orig, + detach_head ? REF_NO_DEREF : 0, + UPDATE_REFS_MSG_ON_ERR); + else { + ret = update_ref(reflog_branch ? reflog_branch : reflog_head, + switch_to_branch, oid, NULL, 0, + UPDATE_REFS_MSG_ON_ERR); + if (!ret) + ret = create_symref("HEAD", switch_to_branch, + reflog_head); + } + if (!ret && run_hook) { + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + strvec_pushl(&opt.args, + oid_to_hex(orig ? orig : null_oid()), + oid_to_hex(oid), + "1", + NULL); + run_hooks_oneshot("post-checkout", &opt); + } + strbuf_release(&msg); + return ret; +} + +int reset_head(struct repository *r, const struct reset_head_opts *opts) +{ + const struct object_id *oid = opts->oid; + const char *switch_to_branch = opts->branch; + unsigned reset_hard = opts->flags & RESET_HEAD_HARD; + unsigned refs_only = opts->flags & RESET_HEAD_REFS_ONLY; + unsigned update_orig_head = opts->flags & RESET_ORIG_HEAD; struct object_id head_oid; struct tree_desc desc[2] = { { NULL }, { NULL } }; struct lock_file lock = LOCK_INIT; struct unpack_trees_options unpack_tree_opts = { 0 }; struct tree *tree; - const char *reflog_action; - struct strbuf msg = STRBUF_INIT; - size_t prefix_len; - struct object_id *orig = NULL, oid_orig, - *old_orig = NULL, oid_old_orig; + const char *action; int ret = 0, nr = 0; if (switch_to_branch && !starts_with(switch_to_branch, "refs/")) BUG("Not a fully qualified branch: '%s'", switch_to_branch); + if (opts->orig_head_msg && !update_orig_head) + BUG("ORIG_HEAD reflog message given without updating ORIG_HEAD"); + + if (opts->branch_msg && !opts->branch) + BUG("branch reflog message given without a branch"); + if (!refs_only && repo_hold_locked_index(r, &lock, LOCK_REPORT_ON_ERROR) < 0) { ret = -1; goto leave_reset_head; @@ -47,8 +120,9 @@ int reset_head(struct repository *r, struct object_id *oid, const char *action, oid = &head_oid; if (refs_only) - goto reset_head_refs; + return update_refs(opts, oid); + action = reset_hard ? "reset" : "checkout"; setup_unpack_trees_porcelain(&unpack_tree_opts, action); unpack_tree_opts.head_idx = 1; unpack_tree_opts.src_index = r->index; @@ -58,7 +132,7 @@ int reset_head(struct repository *r, struct object_id *oid, const char *action, unpack_tree_opts.merge = 1; unpack_tree_opts.preserve_ignored = 0; /* FIXME: !overwrite_ignore */ init_checkout_metadata(&unpack_tree_opts.meta, switch_to_branch, oid, NULL); - if (!detach_head) + if (reset_hard) unpack_tree_opts.reset = UNPACK_RESET_PROTECT_UNTRACKED; if (repo_read_index_unmerged(r) < 0) { @@ -90,49 +164,10 @@ int reset_head(struct repository *r, struct object_id *oid, const char *action, goto leave_reset_head; } -reset_head_refs: - reflog_action = getenv(GIT_REFLOG_ACTION_ENVIRONMENT); - strbuf_addf(&msg, "%s: ", reflog_action ? reflog_action : default_reflog_action); - prefix_len = msg.len; - - if (update_orig_head) { - if (!get_oid("ORIG_HEAD", &oid_old_orig)) - old_orig = &oid_old_orig; - if (!get_oid("HEAD", &oid_orig)) { - orig = &oid_orig; - if (!reflog_orig_head) { - strbuf_addstr(&msg, "updating ORIG_HEAD"); - reflog_orig_head = msg.buf; - } - update_ref(reflog_orig_head, "ORIG_HEAD", orig, - old_orig, 0, UPDATE_REFS_MSG_ON_ERR); - } else if (old_orig) - delete_ref(NULL, "ORIG_HEAD", old_orig, 0); - } - - if (!reflog_head) { - strbuf_setlen(&msg, prefix_len); - strbuf_addstr(&msg, "updating HEAD"); - reflog_head = msg.buf; - } - if (!switch_to_branch) - ret = update_ref(reflog_head, "HEAD", oid, orig, - detach_head ? REF_NO_DEREF : 0, - UPDATE_REFS_MSG_ON_ERR); - else { - ret = update_ref(reflog_head, switch_to_branch, oid, - NULL, 0, UPDATE_REFS_MSG_ON_ERR); - if (!ret) - ret = create_symref("HEAD", switch_to_branch, - reflog_head); - } - if (run_hook) - run_hook_le(NULL, "post-checkout", - oid_to_hex(orig ? orig : null_oid()), - oid_to_hex(oid), "1", NULL); + if (oid != &head_oid || update_orig_head || switch_to_branch) + ret = update_refs(opts, oid); leave_reset_head: - strbuf_release(&msg); rollback_lock_file(&lock); clear_unpack_trees_porcelain(&unpack_tree_opts); while (nr) @@ -12,9 +12,34 @@ #define RESET_HEAD_REFS_ONLY (1<<3) #define RESET_ORIG_HEAD (1<<4) -int reset_head(struct repository *r, struct object_id *oid, const char *action, - const char *switch_to_branch, unsigned flags, - const char *reflog_orig_head, const char *reflog_head, - const char *default_reflog_action); +struct reset_head_opts { + /* The oid of the commit to checkout/reset to. Defaults to HEAD */ + const struct object_id *oid; + /* Optional commit when setting ORIG_HEAD. Defaults to HEAD */ + const struct object_id *orig_head; + /* Optional branch to switch to */ + const char *branch; + /* Flags defined above */ + unsigned flags; + /* Optional reflog message for branch, defaults to head_msg. */ + const char *branch_msg; + /* + * Optional reflog message for HEAD, if this is not set then + * default_reflog_action must be. + */ + const char *head_msg; + /* + * Optional reflog message for ORIG_HEAD, if this is not set and flags + * contains RESET_ORIG_HEAD then default_reflog_action must be set. + */ + const char *orig_head_msg; + /* + * Action to use in default reflog messages, only required if a ref is + * being updated and the reflog messages above are omitted. + */ + const char *default_reflog_action; +}; + +int reset_head(struct repository *r, const struct reset_head_opts *opts); #endif diff --git a/revision.h b/revision.h index 5578bb4720..44efce3f41 100644 --- a/revision.h +++ b/revision.h @@ -195,7 +195,8 @@ struct rev_info { combine_merges:1, combined_all_paths:1, dense_combined_merges:1, - first_parent_merges:1; + first_parent_merges:1, + remerge_diff:1; /* Format info */ int show_notes; @@ -317,6 +318,9 @@ struct rev_info { /* misc. flags related to '--no-kept-objects' */ unsigned keep_pack_cache_flags; + + /* Location where temporary objects for remerge-diff are written. */ + struct tmp_objdir *remerge_objdir; }; int ref_excluded(struct string_list *, const char *path); diff --git a/run-command.c b/run-command.c index 7ef5cc712a..8a21ff525f 100644 --- a/run-command.c +++ b/run-command.c @@ -1323,38 +1323,6 @@ int async_with_fork(void) #endif } -int run_hook_ve(const char *const *env, const char *name, va_list args) -{ - struct child_process hook = CHILD_PROCESS_INIT; - const char *p; - - p = find_hook(name); - if (!p) - return 0; - - strvec_push(&hook.args, p); - while ((p = va_arg(args, const char *))) - strvec_push(&hook.args, p); - hook.env = env; - hook.no_stdin = 1; - hook.stdout_to_stderr = 1; - hook.trace2_hook_name = name; - - return run_command(&hook); -} - -int run_hook_le(const char *const *env, const char *name, ...) -{ - va_list args; - int ret; - - va_start(args, name); - ret = run_hook_ve(env, name, args); - va_end(args); - - return ret; -} - struct io_pump { /* initialized by caller */ int fd; diff --git a/run-command.h b/run-command.h index 4987826258..59e1fbff64 100644 --- a/run-command.h +++ b/run-command.h @@ -224,23 +224,6 @@ int finish_command_in_signal(struct child_process *); */ int run_command(struct child_process *); -/** - * Run a hook. - * The first argument is a pathname to an index file, or NULL - * if the hook uses the default index file or no index is needed. - * The second argument is the name of the hook. - * The further arguments correspond to the hook arguments. - * The last argument has to be NULL to terminate the arguments list. - * If the hook does not exist or is not executable, the return - * value will be zero. - * If it is executable, the hook will be executed and the exit - * status of the hook is returned. - * On execution, .stdout_to_stderr and .no_stdin will be set. - */ -LAST_ARG_MUST_BE_NULL -int run_hook_le(const char *const *env, const char *name, ...); -int run_hook_ve(const char *const *env, const char *name, va_list args); - /* * Trigger an auto-gc */ diff --git a/send-pack.c b/send-pack.c index b3a495b7b1..bc0fcdbb00 100644 --- a/send-pack.c +++ b/send-pack.c @@ -341,13 +341,13 @@ static int generate_push_cert(struct strbuf *req_buf, { const struct ref *ref; struct string_list_item *item; - char *signing_key = xstrdup(get_signing_key()); + char *signing_key_id = xstrdup(get_signing_key_id()); const char *cp, *np; struct strbuf cert = STRBUF_INIT; int update_seen = 0; strbuf_addstr(&cert, "certificate version 0.1\n"); - strbuf_addf(&cert, "pusher %s ", signing_key); + strbuf_addf(&cert, "pusher %s ", signing_key_id); datestamp(&cert); strbuf_addch(&cert, '\n'); if (args->url && *args->url) { @@ -374,7 +374,7 @@ static int generate_push_cert(struct strbuf *req_buf, if (!update_seen) goto free_return; - if (sign_buffer(&cert, &cert, signing_key)) + if (sign_buffer(&cert, &cert, get_signing_key())) die(_("failed to sign the push certificate")); packet_buf_write(req_buf, "push-cert%c%s", 0, cap_string); @@ -386,7 +386,7 @@ static int generate_push_cert(struct strbuf *req_buf, packet_buf_write(req_buf, "push-cert-end\n"); free_return: - free(signing_key); + free(signing_key_id); strbuf_release(&cert); return update_seen; } diff --git a/sequencer.c b/sequencer.c index 74ecb23d20..f680543532 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1282,6 +1282,8 @@ void print_commit_summary(struct repository *r, struct pretty_print_context pctx = {0}; struct strbuf author_ident = STRBUF_INIT; struct strbuf committer_ident = STRBUF_INIT; + struct ref_store *refs; + int resolve_errno; commit = lookup_commit(r, oid); if (!commit) @@ -1331,9 +1333,13 @@ void print_commit_summary(struct repository *r, rev.diffopt.break_opt = 0; diff_setup_done(&rev.diffopt); - head = resolve_ref_unsafe("HEAD", 0, NULL, NULL); - if (!head) + refs = get_main_ref_store(the_repository); + head = refs_resolve_ref_unsafe(refs, "HEAD", 0, NULL, NULL, + &resolve_errno); + if (!head) { + errno = resolve_errno; die_errno(_("unable to resolve HEAD after creating commit")); + } if (!strcmp(head, "HEAD")) head = _("detached HEAD"); else @@ -3645,9 +3651,9 @@ static int do_reset(struct repository *r, struct strbuf ref_name = STRBUF_INIT; struct object_id oid; struct lock_file lock = LOCK_INIT; - struct tree_desc desc; + struct tree_desc desc = { 0 }; struct tree *tree; - struct unpack_trees_options unpack_tree_opts; + struct unpack_trees_options unpack_tree_opts = { 0 }; int ret = 0; if (repo_hold_locked_index(r, &lock, LOCK_REPORT_ON_ERROR) < 0) @@ -3679,14 +3685,11 @@ static int do_reset(struct repository *r, strbuf_addf(&ref_name, "refs/rewritten/%.*s", len, name); if (get_oid(ref_name.buf, &oid) && get_oid(ref_name.buf + strlen("refs/rewritten/"), &oid)) { - error(_("could not read '%s'"), ref_name.buf); - rollback_lock_file(&lock); - strbuf_release(&ref_name); - return -1; + ret = error(_("could not read '%s'"), ref_name.buf); + goto cleanup; } } - memset(&unpack_tree_opts, 0, sizeof(unpack_tree_opts)); setup_unpack_trees_porcelain(&unpack_tree_opts, "reset"); unpack_tree_opts.head_idx = 1; unpack_tree_opts.src_index = r->index; @@ -3698,24 +3701,18 @@ static int do_reset(struct repository *r, init_checkout_metadata(&unpack_tree_opts.meta, name, &oid, NULL); if (repo_read_index_unmerged(r)) { - rollback_lock_file(&lock); - strbuf_release(&ref_name); - return error_resolve_conflict(_(action_name(opts))); + ret = error_resolve_conflict(_(action_name(opts))); + goto cleanup; } if (!fill_tree_descriptor(r, &desc, &oid)) { - error(_("failed to find tree of %s"), oid_to_hex(&oid)); - rollback_lock_file(&lock); - free((void *)desc.buffer); - strbuf_release(&ref_name); - return -1; + ret = error(_("failed to find tree of %s"), oid_to_hex(&oid)); + goto cleanup; } if (unpack_trees(1, &desc, &unpack_tree_opts)) { - rollback_lock_file(&lock); - free((void *)desc.buffer); - strbuf_release(&ref_name); - return -1; + ret = -1; + goto cleanup; } tree = parse_tree_indirect(&oid); @@ -3723,14 +3720,17 @@ static int do_reset(struct repository *r, if (write_locked_index(r->index, &lock, COMMIT_LOCK) < 0) ret = error(_("could not write index")); - free((void *)desc.buffer); if (!ret) ret = update_ref(reflog_message(opts, "reset", "'%.*s'", len, name), "HEAD", &oid, NULL, 0, UPDATE_REFS_MSG_ON_ERR); - +cleanup: + free((void *)desc.buffer); + if (ret < 0) + rollback_lock_file(&lock); strbuf_release(&ref_name); + clear_unpack_trees_porcelain(&unpack_tree_opts); return ret; } @@ -4098,8 +4098,8 @@ static enum todo_command peek_command(struct todo_list *todo_list, int offset) return -1; } -void create_autostash(struct repository *r, const char *path, - const char *default_reflog_action) +void create_autostash(struct repository *r, const char *path) + { struct strbuf buf = STRBUF_INIT; struct lock_file lock_file = LOCK_INIT; @@ -4114,6 +4114,7 @@ void create_autostash(struct repository *r, const char *path, if (has_unstaged_changes(r, 1) || has_uncommitted_changes(r, 1)) { struct child_process stash = CHILD_PROCESS_INIT; + struct reset_head_opts ropts = { .flags = RESET_HEAD_HARD }; struct object_id oid; strvec_pushl(&stash.args, @@ -4135,11 +4136,8 @@ void create_autostash(struct repository *r, const char *path, path); write_file(path, "%s", oid_to_hex(&oid)); printf(_("Created autostash: %s\n"), buf.buf); - if (reset_head(r, NULL, "reset --hard", - NULL, RESET_HEAD_HARD, NULL, NULL, - default_reflog_action) < 0) + if (reset_head(r, &ropts) < 0) die(_("could not reset --hard")); - if (discard_index(r->index) < 0 || repo_read_index(r) < 0) die(_("could not read index")); @@ -4224,42 +4222,26 @@ int apply_autostash_oid(const char *stash_oid) return apply_save_autostash_oid(stash_oid, 1); } -static int run_git_checkout(struct repository *r, struct replay_opts *opts, - const char *commit, const char *action) -{ - struct child_process cmd = CHILD_PROCESS_INIT; - int ret; - - cmd.git_cmd = 1; - - strvec_push(&cmd.args, "checkout"); - strvec_push(&cmd.args, commit); - strvec_pushf(&cmd.env_array, GIT_REFLOG_ACTION "=%s", action); - - if (opts->verbose) - ret = run_command(&cmd); - else - ret = run_command_silent_on_success(&cmd); - - if (!ret) - discard_index(r->index); - - return ret; -} - static int checkout_onto(struct repository *r, struct replay_opts *opts, const char *onto_name, const struct object_id *onto, const struct object_id *orig_head) { - const char *action = reflog_message(opts, "start", "checkout %s", onto_name); - - if (run_git_checkout(r, opts, oid_to_hex(onto), action)) { + struct reset_head_opts ropts = { + .oid = onto, + .orig_head = orig_head, + .flags = RESET_HEAD_DETACH | RESET_ORIG_HEAD | + RESET_HEAD_RUN_POST_CHECKOUT_HOOK, + .head_msg = reflog_message(opts, "start", "checkout %s", + onto_name), + .default_reflog_action = "rebase" + }; + if (reset_head(r, &ropts)) { apply_autostash(rebase_path_autostash()); sequencer_remove_state(opts); return error(_("could not detach HEAD")); } - return update_ref(NULL, "ORIG_HEAD", orig_head, NULL, 0, UPDATE_REFS_MSG_ON_ERR); + return 0; } static int stopped_at_head(struct repository *r) diff --git a/sequencer.h b/sequencer.h index 05a7d2ba6b..da64473636 100644 --- a/sequencer.h +++ b/sequencer.h @@ -197,8 +197,7 @@ void commit_post_rewrite(struct repository *r, const struct commit *current_head, const struct object_id *new_head); -void create_autostash(struct repository *r, const char *path, - const char *default_reflog_action); +void create_autostash(struct repository *r, const char *path); int save_autostash(const char *path); int apply_autostash(const char *path); int apply_autostash_oid(const char *stash_oid); diff --git a/sparse-index.c b/sparse-index.c index 7b7ff79e04..bc3ee358c6 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -122,17 +122,17 @@ static int index_has_unmerged_entries(struct index_state *istate) return 0; } -int convert_to_sparse(struct index_state *istate, int flags) +static int is_sparse_index_allowed(struct index_state *istate, int flags) { - int test_env; - if (istate->sparse_index || !istate->cache_nr || - !core_apply_sparse_checkout || !core_sparse_checkout_cone) + if (!core_apply_sparse_checkout || !core_sparse_checkout_cone) return 0; if (!istate->repo) istate->repo = the_repository; if (!(flags & SPARSE_INDEX_MEMORY_ONLY)) { + int test_env; + /* * The sparse index is not (yet) integrated with a split index. */ @@ -168,6 +168,19 @@ int convert_to_sparse(struct index_state *istate, int flags) if (!istate->sparse_checkout_patterns->use_cone_patterns) return 0; + return 1; +} + +int convert_to_sparse(struct index_state *istate, int flags) +{ + /* + * If the index is already sparse, empty, or otherwise + * cannot be converted to sparse, do not convert. + */ + if (istate->sparse_index || !istate->cache_nr || + !is_sparse_index_allowed(istate, flags)) + return 0; + /* * NEEDSWORK: If we have unmerged entries, then stay full. * Unmerged entries prevent the cache-tree extension from working. @@ -313,6 +326,18 @@ void ensure_full_index(struct index_state *istate) trace2_region_leave("index", "ensure_full_index", istate->repo); } +void ensure_correct_sparsity(struct index_state *istate) +{ + /* + * If the index can be sparse, make it sparse. Otherwise, + * ensure the index is full. + */ + if (is_sparse_index_allowed(istate, 0)) + convert_to_sparse(istate, 0); + else + ensure_full_index(istate); +} + /* * This static global helps avoid infinite recursion between * expand_to_path() and index_file_exists(). diff --git a/sparse-index.h b/sparse-index.h index 9f3d7bc7fa..656bd835b2 100644 --- a/sparse-index.h +++ b/sparse-index.h @@ -4,6 +4,7 @@ struct index_state; #define SPARSE_INDEX_MEMORY_ONLY (1 << 0) int convert_to_sparse(struct index_state *istate, int flags); +void ensure_correct_sparsity(struct index_state *istate); /* * Some places in the codebase expect to search for a specific path. @@ -1059,15 +1059,21 @@ void strbuf_addftime(struct strbuf *sb, const char *fmt, const struct tm *tm, strbuf_setlen(sb, sb->len + len); } -void strbuf_add_unique_abbrev(struct strbuf *sb, const struct object_id *oid, - int abbrev_len) +void strbuf_repo_add_unique_abbrev(struct strbuf *sb, struct repository *repo, + const struct object_id *oid, int abbrev_len) { int r; strbuf_grow(sb, GIT_MAX_HEXSZ + 1); - r = find_unique_abbrev_r(sb->buf + sb->len, oid, abbrev_len); + r = repo_find_unique_abbrev_r(repo, sb->buf + sb->len, oid, abbrev_len); strbuf_setlen(sb, sb->len + r); } +void strbuf_add_unique_abbrev(struct strbuf *sb, const struct object_id *oid, + int abbrev_len) +{ + strbuf_repo_add_unique_abbrev(sb, the_repository, oid, abbrev_len); +} + /* * Returns the length of a line, without trailing spaces. * @@ -634,8 +634,10 @@ void strbuf_list_free(struct strbuf **list); * Add the abbreviation, as generated by find_unique_abbrev, of `sha1` to * the strbuf `sb`. */ -void strbuf_add_unique_abbrev(struct strbuf *sb, - const struct object_id *oid, +struct repository; +void strbuf_repo_add_unique_abbrev(struct strbuf *sb, struct repository *repo, + const struct object_id *oid, int abbrev_len); +void strbuf_add_unique_abbrev(struct strbuf *sb, const struct object_id *oid, int abbrev_len); /** diff --git a/streaming.c b/streaming.c index 5f480ad50c..fe54665d86 100644 --- a/streaming.c +++ b/streaming.c @@ -223,19 +223,24 @@ static int open_istream_loose(struct git_istream *st, struct repository *r, const struct object_id *oid, enum object_type *type) { + struct object_info oi = OBJECT_INFO_INIT; + oi.sizep = &st->size; + oi.typep = type; + st->u.loose.mapped = map_loose_object(r, oid, &st->u.loose.mapsize); if (!st->u.loose.mapped) return -1; - if ((unpack_loose_header(&st->z, - st->u.loose.mapped, - st->u.loose.mapsize, - st->u.loose.hdr, - sizeof(st->u.loose.hdr)) < 0) || - (parse_loose_header(st->u.loose.hdr, &st->size) < 0)) { - git_inflate_end(&st->z); - munmap(st->u.loose.mapped, st->u.loose.mapsize); - return -1; + switch (unpack_loose_header(&st->z, st->u.loose.mapped, + st->u.loose.mapsize, st->u.loose.hdr, + sizeof(st->u.loose.hdr), NULL)) { + case ULHR_OK: + break; + case ULHR_BAD: + case ULHR_TOO_LONG: + goto error; } + if (parse_loose_header(st->u.loose.hdr, &oi) < 0 || *type < 0) + goto error; st->u.loose.hdr_used = strlen(st->u.loose.hdr) + 1; st->u.loose.hdr_avail = st->z.total_out; @@ -244,6 +249,10 @@ static int open_istream_loose(struct git_istream *st, struct repository *r, st->read = read_istream_loose; return 0; +error: + git_inflate_end(&st->z); + munmap(st->u.loose.mapped, st->u.loose.mapsize); + return -1; } diff --git a/submodule.c b/submodule.c index f3c99634a9..bcb2ec22fa 100644 --- a/submodule.c +++ b/submodule.c @@ -201,6 +201,8 @@ int register_all_submodule_odb_as_alternates(void) add_to_alternates_memory(added_submodule_odb_paths.items[i].string); if (ret) { string_list_clear(&added_submodule_odb_paths, 0); + trace2_data_intmax("submodule", the_repository, + "register_all_submodule_odb_as_alternates/registered", ret); if (git_env_bool("GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB", 0)) BUG("register_all_submodule_odb_as_alternates() called"); } @@ -928,23 +930,33 @@ struct has_commit_data { static int check_has_commit(const struct object_id *oid, void *data) { struct has_commit_data *cb = data; + struct repository subrepo; + enum object_type type; - enum object_type type = oid_object_info(cb->repo, oid, NULL); + if (repo_submodule_init(&subrepo, cb->repo, cb->path, null_oid())) { + cb->result = 0; + goto cleanup; + } + + type = oid_object_info(&subrepo, oid, NULL); switch (type) { case OBJ_COMMIT: - return 0; + goto cleanup; case OBJ_BAD: /* * Object is missing or invalid. If invalid, an error message * has already been printed. */ cb->result = 0; - return 0; + goto cleanup; default: die(_("submodule entry '%s' (%s) is a %s, not a commit"), cb->path, oid_to_hex(oid), type_name(type)); } +cleanup: + repo_clear(&subrepo); + return 0; } static int submodule_has_commits(struct repository *r, @@ -2085,6 +2097,7 @@ static void relocate_single_git_dir_into_superproject(const char *path) char *old_git_dir = NULL, *real_old_git_dir = NULL, *real_new_git_dir = NULL; struct strbuf new_gitdir = STRBUF_INIT; const struct submodule *sub; + struct strbuf config_path = STRBUF_INIT, sb = STRBUF_INIT; if (submodule_uses_worktrees(path)) die(_("relocate_gitdir for submodule '%s' with " @@ -2115,6 +2128,15 @@ static void relocate_single_git_dir_into_superproject(const char *path) relocate_gitdir(path, real_old_git_dir, real_new_git_dir); + /* cache pointer to superproject's gitdir */ + /* NEEDSWORK: this may differ if experimental.worktreeConfig is enabled */ + strbuf_addf(&config_path, "%s/config", real_new_git_dir); + git_config_set_in_file(config_path.buf, "submodule.superprojectGitdir", + relative_path(absolute_path(get_git_dir()), + real_new_git_dir, &sb)); + + strbuf_release(&config_path); + strbuf_release(&sb); free(old_git_dir); free(real_old_git_dir); free(real_new_git_dir); diff --git a/t/.gitattributes b/t/.gitattributes index dafa17c3e6..9930e28351 100644 --- a/t/.gitattributes +++ b/t/.gitattributes @@ -1,6 +1,5 @@ t[0-9][0-9][0-9][0-9]/* -whitespace /chainlint/*.expect eol=lf -/lib-diff/* eol=lf /t0110/url-* binary /t3206/* eol=lf /t3900/*.txt eol=lf @@ -405,8 +405,8 @@ every 'git commit-graph write', as if the `--changed-paths` option was passed in. GIT_TEST_FSMONITOR=$PWD/t7519/fsmonitor-all exercises the fsmonitor -code path for utilizing a file system monitor to speed up detecting -new or changed files. +code path for utilizing a (hook based) file system monitor to speed up +detecting new or changed files. GIT_TEST_INDEX_VERSION=<n> exercises the index read/write code path for the index version specified. Can be set to any valid version @@ -463,11 +463,8 @@ GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB=<boolean>, when true, makes registering submodule ODBs as alternates a fatal action. Support for this environment variable can be removed once the migration to explicitly providing repositories when accessing submodule objects is -complete (in which case we might want to replace this with a trace2 -call so that users can make it visible if accessing submodule objects -without an explicit repository still happens) or needs to be abandoned -for whatever reason (in which case the migrated codepaths still retain -their performance benefits). +complete or needs to be abandoned for whatever reason (in which case the +migrated codepaths still retain their performance benefits). Naming Tests ------------ diff --git a/t/helper/test-chmtime.c b/t/helper/test-chmtime.c index 524b55ca49..dc28890a18 100644 --- a/t/helper/test-chmtime.c +++ b/t/helper/test-chmtime.c @@ -134,6 +134,21 @@ int cmd__chmtime(int argc, const char **argv) } if (utb.modtime != sb.st_mtime && utime(argv[i], &utb) < 0) { +#ifdef GIT_WINDOWS_NATIVE + if (S_ISDIR(sb.st_mode)) { + /* + * NEEDSWORK: The Windows version of `utime()` + * (aka `mingw_utime()`) does not correctly + * handle directory arguments, since it uses + * `_wopen()`. Ignore it for now since this + * is just a test. + */ + fprintf(stderr, + ("Failed to modify time on directory %s. " + "Skipping\n"), argv[i]); + continue; + } +#endif fprintf(stderr, "Failed to modify time on %s: %s\n", argv[i], strerror(errno)); return 1; diff --git a/t/helper/test-fsmonitor-client.c b/t/helper/test-fsmonitor-client.c new file mode 100644 index 0000000000..f7a5b3a32f --- /dev/null +++ b/t/helper/test-fsmonitor-client.c @@ -0,0 +1,121 @@ +/* + * test-fsmonitor-client.c: client code to send commands/requests to + * a `git fsmonitor--daemon` daemon. + */ + +#include "test-tool.h" +#include "cache.h" +#include "parse-options.h" +#include "fsmonitor-ipc.h" + +#ifndef HAVE_FSMONITOR_DAEMON_BACKEND +int cmd__fsmonitor_client(int argc, const char **argv) +{ + die("fsmonitor--daemon not available on this platform"); +} +#else + +/* + * Read the `.git/index` to get the last token written to the + * FSMonitor Index Extension. + */ +static const char *get_token_from_index(void) +{ + struct index_state *istate = the_repository->index; + + if (do_read_index(istate, the_repository->index_file, 0) < 0) + die("unable to read index file"); + if (!istate->fsmonitor_last_update) + die("index file does not have fsmonitor extension"); + + return istate->fsmonitor_last_update; +} + +/* + * Send an IPC query to a `git-fsmonitor--daemon` daemon and + * ask for the changes since the given token or from the last + * token in the index extension. + * + * This will implicitly start a daemon process if necessary. The + * daemon process will persist after we exit. + */ +static int do_send_query(const char *token) +{ + struct strbuf answer = STRBUF_INIT; + int ret; + + if (!token || !*token) + token = get_token_from_index(); + + ret = fsmonitor_ipc__send_query(token, &answer); + if (ret < 0) + die(_("could not query fsmonitor--daemon")); + + write_in_full(1, answer.buf, answer.len); + strbuf_release(&answer); + + return 0; +} + +/* + * Send a "flush" command to the `git-fsmonitor--daemon` (if running) + * and tell it to flush its cache. + * + * This feature is primarily used by the test suite to simulate a loss of + * sync with the filesystem where we miss kernel events. + */ +static int do_send_flush(void) +{ + struct strbuf answer = STRBUF_INIT; + int ret; + + ret = fsmonitor_ipc__send_command("flush", &answer); + if (ret) + return ret; + + write_in_full(1, answer.buf, answer.len); + strbuf_release(&answer); + + return 0; +} + +int cmd__fsmonitor_client(int argc, const char **argv) +{ + const char *subcmd; + const char *token = NULL; + + const char * const fsmonitor_client_usage[] = { + N_("test-helper fsmonitor-client query [<token>]"), + N_("test-helper fsmonitor-client flush"), + NULL, + }; + + struct option options[] = { + OPT_STRING(0, "token", &token, N_("token"), + N_("command token to send to the server")), + OPT_END() + }; + + if (argc < 2) + usage_with_options(fsmonitor_client_usage, options); + + if (argc == 2 && !strcmp(argv[1], "-h")) + usage_with_options(fsmonitor_client_usage, options); + + subcmd = argv[1]; + argv--; + argc++; + + argc = parse_options(argc, argv, NULL, options, fsmonitor_client_usage, 0); + + setup_git_directory(); + + if (!strcmp(subcmd, "query")) + return !!do_send_query(token); + + if (!strcmp(subcmd, "flush")) + return !!do_send_flush(); + + die("Unhandled subcommand: '%s'", subcmd); +} +#endif diff --git a/t/helper/test-oid-array.c b/t/helper/test-oid-array.c index b16cd0b11b..d1324d086a 100644 --- a/t/helper/test-oid-array.c +++ b/t/helper/test-oid-array.c @@ -35,5 +35,9 @@ int cmd__oid_array(int argc, const char **argv) else die("unknown command: %s", line.buf); } + + strbuf_release(&line); + oid_array_clear(&array); + return 0; } diff --git a/t/helper/test-oidtree.c b/t/helper/test-oidtree.c index 180ee28dd9..d48a409f4e 100644 --- a/t/helper/test-oidtree.c +++ b/t/helper/test-oidtree.c @@ -45,5 +45,8 @@ int cmd__oidtree(int argc, const char **argv) die("unknown command: %s", line.buf); } } + + strbuf_release(&line); + return 0; } diff --git a/t/helper/test-parse-options.c b/t/helper/test-parse-options.c index a282b6ff13..48d3cf6692 100644 --- a/t/helper/test-parse-options.c +++ b/t/helper/test-parse-options.c @@ -14,7 +14,6 @@ static int dry_run = 0, quiet = 0; static char *string = NULL; static char *file = NULL; static int ambiguous; -static struct string_list list = STRING_LIST_INIT_NODUP; static struct { int called; @@ -107,6 +106,8 @@ int cmd__parse_options(int argc, const char **argv) NULL }; struct string_list expect = STRING_LIST_INIT_NODUP; + struct string_list list = STRING_LIST_INIT_NODUP; + struct option options[] = { OPT_BOOL(0, "yes", &boolean, "get a boolean"), OPT_BOOL('D', "no-doubt", &boolean, "begins with 'no-'"), @@ -185,5 +186,9 @@ int cmd__parse_options(int argc, const char **argv) for (i = 0; i < argc; i++) show(&expect, &ret, "arg %02d: %s", i, argv[i]); + expect.strdup_strings = 1; + string_list_clear(&expect, 0); + string_list_clear(&list, 0); + return ret; } diff --git a/t/helper/test-prio-queue.c b/t/helper/test-prio-queue.c index f4028442e3..133b5e6f4a 100644 --- a/t/helper/test-prio-queue.c +++ b/t/helper/test-prio-queue.c @@ -46,5 +46,7 @@ int cmd__prio_queue(int argc, const char **argv) } } + clear_prio_queue(&pq); + return 0; } diff --git a/t/helper/test-progress.c b/t/helper/test-progress.c index 5d05cbe789..1435c28e95 100644 --- a/t/helper/test-progress.c +++ b/t/helper/test-progress.c @@ -3,6 +3,9 @@ * * Reads instructions from standard input, one instruction per line: * + * "start <total>[ <title>]" - Call start_progress(title, total), + * Uses the default title of "Working hard" + * if the " <title>" is omitted. * "progress <items>" - Call display_progress() with the given item count * as parameter. * "throughput <bytes> <millis> - Call display_throughput() with the given @@ -10,6 +13,7 @@ * specify the time elapsed since the * start_progress() call. * "update" - Set the 'progress_update' flag. + * "stop" - Call stop_progress(). * * See 't0500-progress-display.sh' for examples. */ @@ -19,34 +23,52 @@ #include "parse-options.h" #include "progress.h" #include "strbuf.h" +#include "string-list.h" + +/* + * We can't use "end + 1" as an argument to start_progress() below, it + * doesn't xstrdup() its "title" argument. We need to hold onto a + * valid "char *" for it until the end. + */ +static char *dup_title(struct string_list *titles, const char *title) +{ + return string_list_insert(titles, title)->string; +} int cmd__progress(int argc, const char **argv) { - int total = 0; - const char *title; + const char *const default_title = "Working hard"; + struct string_list titles = STRING_LIST_INIT_DUP; struct strbuf line = STRBUF_INIT; - struct progress *progress; + struct progress *progress = NULL; const char *usage[] = { - "test-tool progress [--total=<n>] <progress-title>", + "test-tool progress <stdin", NULL }; struct option options[] = { - OPT_INTEGER(0, "total", &total, "total number of items"), OPT_END(), }; argc = parse_options(argc, argv, NULL, options, usage, 0); - if (argc != 1) - die("need a title for the progress output"); - title = argv[0]; + if (argc) + usage_with_options(usage, options); progress_testing = 1; - progress = start_progress(title, total); while (strbuf_getline(&line, stdin) != EOF) { char *end; - if (skip_prefix(line.buf, "progress ", (const char **) &end)) { + if (skip_prefix(line.buf, "start ", (const char **) &end)) { + uint64_t total = strtoull(end, &end, 10); + if (*end == '\0') + progress = start_progress(default_title, total); + else if (*end == ' ') + progress = start_progress(dup_title(&titles, + end + 1), + total); + else + die("invalid input: '%s'\n", line.buf); + } else if (skip_prefix(line.buf, "progress ", (const char **) &end)) { uint64_t item_count = strtoull(end, &end, 10); if (*end != '\0') die("invalid input: '%s'\n", line.buf); @@ -63,12 +85,16 @@ int cmd__progress(int argc, const char **argv) die("invalid input: '%s'\n", line.buf); progress_test_ns = test_ms * 1000 * 1000; display_throughput(progress, byte_count); - } else if (!strcmp(line.buf, "update")) + } else if (!strcmp(line.buf, "update")) { progress_test_force_update(); - else + } else if (!strcmp(line.buf, "stop")) { + stop_progress(&progress); + } else { die("invalid input: '%s'\n", line.buf); + } } - stop_progress(&progress); + strbuf_release(&line); + string_list_clear(&titles, 0); return 0; } diff --git a/t/helper/test-read-cache.c b/t/helper/test-read-cache.c index b52c174acc..0d9f08931a 100644 --- a/t/helper/test-read-cache.c +++ b/t/helper/test-read-cache.c @@ -39,8 +39,6 @@ int cmd__read_cache(int argc, const char **argv) int table = 0, expand = 0; initialize_the_repository(); - prepare_repo_settings(r); - r->settings.command_requires_full_index = 0; for (++argv, --argc; *argv && starts_with(*argv, "--"); ++argv, --argc) { if (skip_prefix(*argv, "--print-and-refresh=", &name)) @@ -56,6 +54,9 @@ int cmd__read_cache(int argc, const char **argv) setup_git_directory(); git_config(git_default_config, NULL); + prepare_repo_settings(r); + r->settings.command_requires_full_index = 0; + for (i = 0; i < cnt; i++) { repo_read_index(r); diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index 9d6fa7a377..27072ba94d 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -55,9 +55,10 @@ static int read_midx_file(const char *object_dir, int show_objects) printf("%s %"PRIu64"\t%s\n", oid_to_hex(&oid), e.offset, e.p->pack_name); } - return 0; } + close_midx(m); + return 0; } diff --git a/t/helper/test-ref-store.c b/t/helper/test-ref-store.c index b314b81a45..3986665037 100644 --- a/t/helper/test-ref-store.c +++ b/t/helper/test-ref-store.c @@ -123,9 +123,10 @@ static int cmd_resolve_ref(struct ref_store *refs, const char **argv) int resolve_flags = arg_flags(*argv++, "resolve-flags"); int flags; const char *ref; + int ignore_errno; ref = refs_resolve_ref_unsafe(refs, refname, resolve_flags, - &oid, &flags); + &oid, &flags, &ignore_errno); printf("%s %s 0x%x\n", oid_to_hex(&oid), ref ? ref : "(null)", flags); return ref ? 0 : 1; } diff --git a/t/helper/test-reftable.c b/t/helper/test-reftable.c new file mode 100644 index 0000000000..26b03d7b78 --- /dev/null +++ b/t/helper/test-reftable.c @@ -0,0 +1,21 @@ +#include "reftable/reftable-tests.h" +#include "test-tool.h" + +int cmd__reftable(int argc, const char **argv) +{ + basics_test_main(argc, argv); + block_test_main(argc, argv); + merged_test_main(argc, argv); + pq_test_main(argc, argv); + record_test_main(argc, argv); + refname_test_main(argc, argv); + readwrite_test_main(argc, argv); + stack_test_main(argc, argv); + tree_test_main(argc, argv); + return 0; +} + +int cmd__dump_reftable(int argc, const char **argv) +{ + return reftable_dump_main(argc, (char *const *)argv); +} diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index 3ce5585e53..a4c5647b2f 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -31,6 +31,7 @@ static struct test_cmd cmds[] = { { "dump-untracked-cache", cmd__dump_untracked_cache }, { "example-decorate", cmd__example_decorate }, { "fast-rebase", cmd__fast_rebase }, + { "fsmonitor-client", cmd__fsmonitor_client }, { "genrandom", cmd__genrandom }, { "genzeros", cmd__genzeros }, { "getcwd", cmd__getcwd }, @@ -53,13 +54,15 @@ static struct test_cmd cmds[] = { { "pcre2-config", cmd__pcre2_config }, { "pkt-line", cmd__pkt_line }, { "prio-queue", cmd__prio_queue }, - { "proc-receive", cmd__proc_receive}, + { "proc-receive", cmd__proc_receive }, { "progress", cmd__progress }, { "reach", cmd__reach }, { "read-cache", cmd__read_cache }, { "read-graph", cmd__read_graph }, { "read-midx", cmd__read_midx }, { "ref-store", cmd__ref_store }, + { "reftable", cmd__reftable }, + { "dump-reftable", cmd__dump_reftable }, { "regex", cmd__regex }, { "repository", cmd__repository }, { "revision-walking", cmd__revision_walking }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index 9f0f522850..fa4c936310 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -19,8 +19,10 @@ int cmd__dump_cache_tree(int argc, const char **argv); int cmd__dump_fsmonitor(int argc, const char **argv); int cmd__dump_split_index(int argc, const char **argv); int cmd__dump_untracked_cache(int argc, const char **argv); +int cmd__dump_reftable(int argc, const char **argv); int cmd__example_decorate(int argc, const char **argv); int cmd__fast_rebase(int argc, const char **argv); +int cmd__fsmonitor_client(int argc, const char **argv); int cmd__genrandom(int argc, const char **argv); int cmd__genzeros(int argc, const char **argv); int cmd__getcwd(int argc, const char **argv); @@ -49,6 +51,7 @@ int cmd__read_cache(int argc, const char **argv); int cmd__read_graph(int argc, const char **argv); int cmd__read_midx(int argc, const char **argv); int cmd__ref_store(int argc, const char **argv); +int cmd__reftable(int argc, const char **argv); int cmd__regex(int argc, const char **argv); int cmd__repository(int argc, const char **argv); int cmd__revision_walking(int argc, const char **argv); diff --git a/t/lib-diff-data.sh b/t/lib-diff-data.sh new file mode 100644 index 0000000000..c64ec18324 --- /dev/null +++ b/t/lib-diff-data.sh @@ -0,0 +1,22 @@ +COPYING_test_data () { + cat <<\EOF + + Note that the only valid version of the GPL as far as this project + is concerned is _this_ particular version of the license (ie v2, not + v2.2 or v3.x or whatever), unless explicitly otherwise stated. + + HOWEVER, in order to allow a migration to GPLv3 if that seems like + a good idea, I also ask that people involved with the project make + their preferences known. In particular, if you trust me to make that + decision, you might note so in your copyright message, ie something + like + + This file is licensed under the GPL v2, or a later version + at the discretion of Linus. + + might avoid issues. But we can also just decide to synchronize and + contact all copyright holders on record if/when the occasion arises. + + Linus Torvalds +EOF +} diff --git a/t/lib-diff.sh b/t/lib-diff.sh index 2de880f7a5..c4606bd4b7 100644 --- a/t/lib-diff.sh +++ b/t/lib-diff.sh @@ -1,3 +1,5 @@ +. "$TEST_DIRECTORY"/lib-diff-data.sh + : sanitize_diff_raw='/^:/s/ '"\($OID_REGEX\)"' '"\($OID_REGEX\)"' \([A-Z]\)[0-9]* / \1 \2 \3# /' diff --git a/t/lib-diff/COPYING b/t/lib-diff/COPYING deleted file mode 100644 index 6ff87c4664..0000000000 --- a/t/lib-diff/COPYING +++ /dev/null @@ -1,361 +0,0 @@ - - Note that the only valid version of the GPL as far as this project - is concerned is _this_ particular version of the license (ie v2, not - v2.2 or v3.x or whatever), unless explicitly otherwise stated. - - HOWEVER, in order to allow a migration to GPLv3 if that seems like - a good idea, I also ask that people involved with the project make - their preferences known. In particular, if you trust me to make that - decision, you might note so in your copyright message, ie something - like - - This file is licensed under the GPL v2, or a later version - at the discretion of Linus. - - might avoid issues. But we can also just decide to synchronize and - contact all copyright holders on record if/when the occasion arises. - - Linus Torvalds - ----------------------------------------- - - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc. - 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Library General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - <one line to give the program's name and a brief idea of what it does.> - Copyright (C) <year> <name of author> - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - <signature of Ty Coon>, 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General -Public License instead of this License. diff --git a/t/lib-diff/README b/t/lib-diff/README deleted file mode 100644 index 548142c327..0000000000 --- a/t/lib-diff/README +++ /dev/null @@ -1,46 +0,0 @@ -//////////////////////////////////////////////////////////////// - - GIT - the stupid content tracker - -//////////////////////////////////////////////////////////////// - -"git" can mean anything, depending on your mood. - - - random three-letter combination that is pronounceable, and not - actually used by any common UNIX command. The fact that it is a - mispronunciation of "get" may or may not be relevant. - - stupid. contemptible and despicable. simple. Take your pick from the - dictionary of slang. - - "global information tracker": you're in a good mood, and it actually - works for you. Angels sing, and a light suddenly fills the room. - - "goddamn idiotic truckload of sh*t": when it breaks - -Git is a fast, scalable, distributed revision control system with an -unusually rich command set that provides both high-level operations -and full access to internals. - -Git is an Open Source project covered by the GNU General Public License. -It was originally written by Linus Torvalds with help of a group of -hackers around the net. It is currently maintained by Junio C Hamano. - -Please read the file INSTALL for installation instructions. -See Documentation/tutorial.txt to get started, then see -Documentation/everyday.txt for a useful minimum set of commands, -and "man git-commandname" for documentation of each command. -CVS users may also want to read Documentation/cvs-migration.txt. - -Many Git online resources are accessible from http://git.or.cz/ -including full documentation and Git related tools. - -The user discussion and development of Git take place on the Git -mailing list -- everyone is welcome to post bug reports, feature -requests, comments and patches to git@vger.kernel.org. To subscribe -to the list, send an email with just "subscribe git" in the body to -majordomo@vger.kernel.org. The mailing list archives are available at -http://marc.theaimsgroup.com/?l=git and other archival sites. - -The messages titled "A note from the maintainer", "What's in -git.git (stable)" and "What's cooking in git.git (topics)" and -the discussion following them on the mailing list give a good -reference for project status, development direction and -remaining tasks. diff --git a/t/lib-gpg.sh b/t/lib-gpg.sh index 9fc5241228..139e7a45ea 100644 --- a/t/lib-gpg.sh +++ b/t/lib-gpg.sh @@ -87,6 +87,51 @@ test_lazy_prereq RFC1991 ' echo | gpg --homedir "${GNUPGHOME}" -b --rfc1991 >/dev/null ' +GPGSSH_KEY_PRIMARY="${GNUPGHOME}/ed25519_ssh_signing_key" +GPGSSH_KEY_SECONDARY="${GNUPGHOME}/rsa_2048_ssh_signing_key" +GPGSSH_KEY_UNTRUSTED="${GNUPGHOME}/untrusted_ssh_signing_key" +GPGSSH_KEY_EXPIRED="${GNUPGHOME}/expired_ssh_signing_key" +GPGSSH_KEY_NOTYETVALID="${GNUPGHOME}/notyetvalid_ssh_signing_key" +GPGSSH_KEY_TIMEBOXEDVALID="${GNUPGHOME}/timeboxed_valid_ssh_signing_key" +GPGSSH_KEY_TIMEBOXEDINVALID="${GNUPGHOME}/timeboxed_invalid_ssh_signing_key" +GPGSSH_KEY_WITH_PASSPHRASE="${GNUPGHOME}/protected_ssh_signing_key" +GPGSSH_KEY_PASSPHRASE="super_secret" +GPGSSH_ALLOWED_SIGNERS="${GNUPGHOME}/ssh.all_valid.allowedSignersFile" + +GPGSSH_GOOD_SIGNATURE_TRUSTED='Good "git" signature for' +GPGSSH_GOOD_SIGNATURE_UNTRUSTED='Good "git" signature with' +GPGSSH_KEY_NOT_TRUSTED="No principal matched" +GPGSSH_BAD_SIGNATURE="Signature verification failed" + +test_lazy_prereq GPGSSH ' + ssh_version=$(ssh-keygen -Y find-principals -n "git" 2>&1) + test $? != 127 || exit 1 + echo $ssh_version | grep -q "find-principals:missing signature file" + test $? = 0 || exit 1; + mkdir -p "${GNUPGHOME}" && + chmod 0700 "${GNUPGHOME}" && + ssh-keygen -t ed25519 -N "" -C "git ed25519 key" -f "${GPGSSH_KEY_PRIMARY}" >/dev/null && + echo "\"principal with number 1\" $(cat "${GPGSSH_KEY_PRIMARY}.pub")" >> "${GPGSSH_ALLOWED_SIGNERS}" && + ssh-keygen -t rsa -b 2048 -N "" -C "git rsa2048 key" -f "${GPGSSH_KEY_SECONDARY}" >/dev/null && + echo "\"principal with number 2\" $(cat "${GPGSSH_KEY_SECONDARY}.pub")" >> "${GPGSSH_ALLOWED_SIGNERS}" && + ssh-keygen -t ed25519 -N "${GPGSSH_KEY_PASSPHRASE}" -C "git ed25519 encrypted key" -f "${GPGSSH_KEY_WITH_PASSPHRASE}" >/dev/null && + echo "\"principal with number 3\" $(cat "${GPGSSH_KEY_WITH_PASSPHRASE}.pub")" >> "${GPGSSH_ALLOWED_SIGNERS}" && + ssh-keygen -t ed25519 -N "" -C "git ed25519 key" -f "${GPGSSH_KEY_UNTRUSTED}" >/dev/null +' + +test_lazy_prereq GPGSSH_VERIFYTIME ' + # Check if ssh-keygen has a verify-time option by passing an invalid date to it + ssh-keygen -Overify-time=INVALID -Y check-novalidate -s doesnotmatter 2>&1 | grep -q -F "Invalid \"verify-time\"" && + ssh-keygen -t ed25519 -N "" -C "timeboxed valid key" -f "${GPGSSH_KEY_TIMEBOXEDVALID}" >/dev/null && + echo "\"timeboxed valid key\" valid-after=\"20050407000000\",valid-before=\"200504100000\" $(cat "${GPGSSH_KEY_TIMEBOXEDVALID}.pub")" >> "${GPGSSH_ALLOWED_SIGNERS}" && + ssh-keygen -t ed25519 -N "" -C "timeboxed invalid key" -f "${GPGSSH_KEY_TIMEBOXEDINVALID}" >/dev/null && + echo "\"timeboxed invalid key\" valid-after=\"20050401000000\",valid-before=\"20050402000000\" $(cat "${GPGSSH_KEY_TIMEBOXEDINVALID}.pub")" >> "${GPGSSH_ALLOWED_SIGNERS}" && + ssh-keygen -t ed25519 -N "" -C "expired key" -f "${GPGSSH_KEY_EXPIRED}" >/dev/null && + echo "\"principal with expired key\" valid-before=\"20000101000000\" $(cat "${GPGSSH_KEY_EXPIRED}.pub")" >> "${GPGSSH_ALLOWED_SIGNERS}" && + ssh-keygen -t ed25519 -N "" -C "not yet valid key" -f "${GPGSSH_KEY_NOTYETVALID}" >/dev/null && + echo "\"principal with not yet valid key\" valid-after=\"29990101000000\" $(cat "${GPGSSH_KEY_NOTYETVALID}.pub")" >> "${GPGSSH_ALLOWED_SIGNERS}" +' + sanitize_pgp() { perl -ne ' /^-----END PGP/ and $in_pgp = 0; diff --git a/t/lib-httpd.sh b/t/lib-httpd.sh index d2edfa4c50..782891908d 100644 --- a/t/lib-httpd.sh +++ b/t/lib-httpd.sh @@ -131,6 +131,7 @@ prepare_httpd() { cp "$TEST_PATH"/passwd "$HTTPD_ROOT_PATH" install_script incomplete-length-upload-pack-v2-http.sh install_script incomplete-body-upload-pack-v2-http.sh + install_script error-no-report.sh install_script broken-smart-http.sh install_script error-smart-http.sh install_script error.sh diff --git a/t/lib-httpd/apache.conf b/t/lib-httpd/apache.conf index 180a41fe96..497b9b9d92 100644 --- a/t/lib-httpd/apache.conf +++ b/t/lib-httpd/apache.conf @@ -122,6 +122,7 @@ Alias /auth/dumb/ www/auth/dumb/ </LocationMatch> ScriptAlias /smart/incomplete_length/git-upload-pack incomplete-length-upload-pack-v2-http.sh/ ScriptAlias /smart/incomplete_body/git-upload-pack incomplete-body-upload-pack-v2-http.sh/ +ScriptAlias /smart/no_report/git-receive-pack error-no-report.sh/ ScriptAliasMatch /error_git_upload_pack/(.*)/git-upload-pack error.sh/ ScriptAliasMatch /smart_*[^/]*/(.*) ${GIT_EXEC_PATH}/git-http-backend/$1 ScriptAlias /broken_smart/ broken-smart-http.sh/ @@ -137,6 +138,9 @@ ScriptAliasMatch /one_time_perl/(.*) apply-one-time-perl.sh/$1 <Files incomplete-body-upload-pack-v2-http.sh> Options ExecCGI </Files> +<Files error-no-report.sh> + Options ExecCGI +</Files> <Files broken-smart-http.sh> Options ExecCGI </Files> diff --git a/t/lib-httpd/error-no-report.sh b/t/lib-httpd/error-no-report.sh new file mode 100644 index 0000000000..39ff75bbc4 --- /dev/null +++ b/t/lib-httpd/error-no-report.sh @@ -0,0 +1,6 @@ +echo "Content-Type: application/x-git-receive-pack-result" +echo +printf '0013\001000eunpack ok\n' +printf '0015\002skipping report\n' +printf '0009\0010000' +printf '0000' diff --git a/t/lib-unique-files.sh b/t/lib-unique-files.sh new file mode 100644 index 0000000000..a7de4ca851 --- /dev/null +++ b/t/lib-unique-files.sh @@ -0,0 +1,36 @@ +# Helper to create files with unique contents + + +# Create multiple files with unique contents. Takes the number of +# directories, the number of files in each directory, and the base +# directory. +# +# test_create_unique_files 2 3 my_dir -- Creates 2 directories with 3 files +# each in my_dir, all with unique +# contents. + +test_create_unique_files() { + test "$#" -ne 3 && BUG "3 param" + + local dirs=$1 + local files=$2 + local basedir=$3 + local counter=0 + test_tick + local basedata=$test_tick + + + rm -rf $basedir + + for i in $(test_seq $dirs) + do + local dir=$basedir/dir$i + + mkdir -p "$dir" + for j in $(test_seq $files) + do + counter=$((counter + 1)) + echo "$basedata.$counter" >"$dir/file$j.txt" + done + done +} diff --git a/t/oid-info/oid b/t/oid-info/oid index a754970523..7547d2c790 100644 --- a/t/oid-info/oid +++ b/t/oid-info/oid @@ -27,3 +27,5 @@ numeric sha1:0123456789012345678901234567890123456789 numeric sha256:0123456789012345678901234567890123456789012345678901234567890123 deadbeef sha1:deadbeefdeadbeefdeadbeefdeadbeefdeadbeef deadbeef sha256:deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef +deadbeef_short sha1:deadbeefdeadbeefdeadbeefdeadbeefdeadbee +deadbeef_short sha256:deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbee diff --git a/t/perf/config b/t/perf/config new file mode 100644 index 0000000000..b92768b039 --- /dev/null +++ b/t/perf/config @@ -0,0 +1,2 @@ +[gc] + auto = 0 diff --git a/t/perf/p2000-sparse-operations.sh b/t/perf/p2000-sparse-operations.sh index 597626276f..9ac76a049b 100755 --- a/t/perf/p2000-sparse-operations.sh +++ b/t/perf/p2000-sparse-operations.sh @@ -110,5 +110,12 @@ test_perf_on_all git add -A test_perf_on_all git add . test_perf_on_all git commit -a -m A test_perf_on_all git checkout -f - +test_perf_on_all git reset +test_perf_on_all git reset --hard +test_perf_on_all git reset -- does-not-exist +test_perf_on_all git diff +test_perf_on_all git diff --staged +test_perf_on_all git blame $SPARSE_CONE/a +test_perf_on_all git blame $SPARSE_CONE/f3/a test_done diff --git a/t/perf/p3700-add.sh b/t/perf/p3700-add.sh new file mode 100755 index 0000000000..e93c08a2e7 --- /dev/null +++ b/t/perf/p3700-add.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# +# This test measures the performance of adding new files to the object database +# and index. The test was originally added to measure the effect of the +# core.fsyncObjectFiles=batch mode, which is why we are testing different values +# of that setting explicitly and creating a lot of unique objects. + +test_description="Tests performance of add" + +. ./perf-lib.sh + +. $TEST_DIRECTORY/lib-unique-files.sh + +test_perf_default_repo +test_checkout_worktree + +dir_count=10 +files_per_dir=50 +total_files=$((dir_count * files_per_dir)) + +# We need to create the files each time we run the perf test, but +# we do not want to measure the cost of creating the files, so run +# the tet once. +if test "${GIT_PERF_REPEAT_COUNT-1}" -ne 1 +then + echo "warning: Setting GIT_PERF_REPEAT_COUNT=1" >&2 + GIT_PERF_REPEAT_COUNT=1 +fi + +for m in false true batch +do + test_expect_success "create the files for core.fsyncObjectFiles=$m" ' + git reset --hard && + # create files across directories + test_create_unique_files $dir_count $files_per_dir files + ' + + test_perf "add $total_files files (core.fsyncObjectFiles=$m)" " + git -c core.fsyncobjectfiles=$m add files + " +done + +test_done diff --git a/t/perf/p3900-stash.sh b/t/perf/p3900-stash.sh new file mode 100755 index 0000000000..c9fcd0c03e --- /dev/null +++ b/t/perf/p3900-stash.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# +# This test measures the performance of adding new files to the object database +# and index. The test was originally added to measure the effect of the +# core.fsyncObjectFiles=batch mode, which is why we are testing different values +# of that setting explicitly and creating a lot of unique objects. + +test_description="Tests performance of stash" + +. ./perf-lib.sh + +. $TEST_DIRECTORY/lib-unique-files.sh + +test_perf_default_repo +test_checkout_worktree + +dir_count=10 +files_per_dir=50 +total_files=$((dir_count * files_per_dir)) + +# We need to create the files each time we run the perf test, but +# we do not want to measure the cost of creating the files, so run +# the tet once. +if test "${GIT_PERF_REPEAT_COUNT-1}" -ne 1 +then + echo "warning: Setting GIT_PERF_REPEAT_COUNT=1" >&2 + GIT_PERF_REPEAT_COUNT=1 +fi + +for m in false true batch +do + test_expect_success "create the files for core.fsyncObjectFiles=$m" ' + git reset --hard && + # create files across directories + test_create_unique_files $dir_count $files_per_dir files + ' + + # We only stash files in the 'files' subdirectory since + # the perf test infrastructure creates files in the + # current working directory that need to be preserved + test_perf "stash 500 files (core.fsyncObjectFiles=$m)" " + git -c core.fsyncobjectfiles=$m stash push -u -- files + " +done + +test_done diff --git a/t/perf/p4002-diff-color-moved.sh b/t/perf/p4002-diff-color-moved.sh new file mode 100755 index 0000000000..ad56bcb71e --- /dev/null +++ b/t/perf/p4002-diff-color-moved.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +test_description='Tests diff --color-moved performance' +. ./perf-lib.sh + +test_perf_default_repo + +if ! git rev-parse --verify v2.29.0^{commit} >/dev/null +then + skip_all='skipping because tag v2.29.0 was not found' + test_done +fi + +GIT_PAGER_IN_USE=1 +test_export GIT_PAGER_IN_USE + +test_perf 'diff --no-color-moved --no-color-moved-ws large change' ' + git diff --no-color-moved --no-color-moved-ws v2.28.0 v2.29.0 +' + +test_perf 'diff --color-moved --no-color-moved-ws large change' ' + git diff --color-moved=zebra --no-color-moved-ws v2.28.0 v2.29.0 +' + +test_perf 'diff --color-moved-ws=allow-indentation-change large change' ' + git diff --color-moved=zebra --color-moved-ws=allow-indentation-change \ + v2.28.0 v2.29.0 +' + +test_perf 'log --no-color-moved --no-color-moved-ws' ' + git log --no-color-moved --no-color-moved-ws --no-merges --patch \ + -n1000 v2.29.0 +' + +test_perf 'log --color-moved --no-color-moved-ws' ' + git log --color-moved=zebra --no-color-moved-ws --no-merges --patch \ + -n1000 v2.29.0 +' + +test_perf 'log --color-moved-ws=allow-indentation-change' ' + git log --color-moved=zebra --color-moved-ws=allow-indentation-change \ + --no-merges --patch -n1000 v2.29.0 +' + +test_done diff --git a/t/perf/p7519-fsmonitor.sh b/t/perf/p7519-fsmonitor.sh index 5eb5044a10..e70252ed65 100755 --- a/t/perf/p7519-fsmonitor.sh +++ b/t/perf/p7519-fsmonitor.sh @@ -24,7 +24,8 @@ test_description="Test core.fsmonitor" # GIT_PERF_7519_SPLIT_INDEX: used to configure core.splitIndex # GIT_PERF_7519_FSMONITOR: used to configure core.fsMonitor. May be an # absolute path to an integration. May be a space delimited list of -# absolute paths to integrations. +# absolute paths to integrations. (This hook or list of hooks does not +# include the built-in fsmonitor--daemon.) # # The big win for using fsmonitor is the elimination of the need to scan the # working directory looking for changed and untracked files. If the file @@ -98,6 +99,13 @@ trace_stop() { fi } +touch_files() { + n=$1 + d="$n"_files + + (cd $d ; test_seq 1 $n | xargs touch ) +} + test_expect_success "one time repo setup" ' # set untrackedCache depending on the environment if test -n "$GIT_PERF_7519_UNTRACKED_CACHE" @@ -119,10 +127,11 @@ test_expect_success "one time repo setup" ' fi && mkdir 1_file 10_files 100_files 1000_files 10000_files && - for i in $(test_seq 1 10); do touch 10_files/$i; done && - for i in $(test_seq 1 100); do touch 100_files/$i; done && - for i in $(test_seq 1 1000); do touch 1000_files/$i; done && - for i in $(test_seq 1 10000); do touch 10000_files/$i; done && + touch_files 1 && + touch_files 10 && + touch_files 100 && + touch_files 1000 && + touch_files 10000 && git add 1_file 10_files 100_files 1000_files 10000_files && git commit -qm "Add files" && @@ -135,10 +144,16 @@ test_expect_success "one time repo setup" ' setup_for_fsmonitor() { # set INTEGRATION_SCRIPT depending on the environment - if test -n "$INTEGRATION_PATH" + if test -n "$USE_FSMONITOR_DAEMON" + then + git config core.useBuiltinFSMonitor true && + INTEGRATION_SCRIPT=false + elif test -n "$INTEGRATION_PATH" then + git config core.useBuiltinFSMonitor false && INTEGRATION_SCRIPT="$INTEGRATION_PATH" else + git config core.useBuiltinFSMonitor false && # # Choose integration script based on existence of Watchman. # Fall back to an empty integration script. @@ -174,7 +189,10 @@ test_perf_w_drop_caches () { } test_fsmonitor_suite() { - if test -n "$INTEGRATION_SCRIPT"; then + if test -n "$USE_FSMONITOR_DAEMON" + then + DESC="builtin fsmonitor--daemon" + elif test -n "$INTEGRATION_SCRIPT"; then DESC="fsmonitor=$(basename $INTEGRATION_SCRIPT)" else DESC="fsmonitor=disabled" @@ -199,15 +217,15 @@ test_fsmonitor_suite() { # Update the mtimes on upto 100k files to make status think # that they are dirty. For simplicity, omit any files with - # LFs (i.e. anything that ls-files thinks it needs to dquote). - # Then fully backslash-quote the paths to capture any - # whitespace so that they pass thru xargs properly. + # LFs (i.e. anything that ls-files thinks it needs to dquote) + # and any files with whitespace so that they pass thru xargs + # properly. # test_perf_w_drop_caches "status (dirty) ($DESC)" ' git ls-files | \ head -100000 | \ grep -v \" | \ - sed '\''s/\(.\)/\\\1/g'\'' | \ + egrep -v " ." | \ xargs test-tool chmtime -300 && git status ' @@ -285,4 +303,25 @@ test_expect_success "setup without fsmonitor" ' test_fsmonitor_suite trace_stop +# +# Run a full set of perf tests using the built-in fsmonitor--daemon. +# It does not use the Hook API, so it has a different setup. +# Explicitly start the daemon here and before we start client commands +# so that we can later add custom tracing. +# +if test_have_prereq FSMONITOR_DAEMON +then + USE_FSMONITOR_DAEMON=t + + trace_start fsmonitor--daemon--server + git fsmonitor--daemon start + + trace_start fsmonitor--daemon--client + test_expect_success "setup for fsmonitor--daemon" 'setup_for_fsmonitor' + test_fsmonitor_suite + + git fsmonitor--daemon stop + trace_stop +fi + test_done diff --git a/t/perf/perf-lib.sh b/t/perf/perf-lib.sh index f5ed092ee5..1fe04dee65 100644 --- a/t/perf/perf-lib.sh +++ b/t/perf/perf-lib.sh @@ -27,6 +27,10 @@ TEST_NO_MALLOC_CHECK=t . ../test-lib.sh +unset GIT_CONFIG_NOSYSTEM +GIT_CONFIG_SYSTEM="$TEST_DIRECTORY/perf/config" +export GIT_CONFIG_SYSTEM + if test -n "$GIT_TEST_INSTALLED" -a -z "$PERF_SET_GIT_TEST_INSTALLED" then error "Do not use GIT_TEST_INSTALLED with the perf tests. @@ -74,7 +78,7 @@ test_perf_copy_repo_contents () { for stuff in "$1"/* do case "$stuff" in - */objects|*/hooks|*/config|*/commondir|*/gitdir|*/worktrees) + */objects|*/hooks|*/config|*/commondir|*/gitdir|*/worktrees|*/fsmonitor--daemon*) ;; *) cp -R "$stuff" "$repo/.git/" || exit 1 @@ -230,6 +234,7 @@ test_perf_ () { test_ok_ "$1" fi "$TEST_DIRECTORY"/perf/min_time.perl test_time.* >"$base".result + rm test_time.* } test_perf () { diff --git a/t/t0001-init.sh b/t/t0001-init.sh index df544bb321..7603ad2f82 100755 --- a/t/t0001-init.sh +++ b/t/t0001-init.sh @@ -2,6 +2,7 @@ test_description='git init' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh check_config () { diff --git a/t/t0002-gitfile.sh b/t/t0002-gitfile.sh index 8440e6add1..76052cb562 100755 --- a/t/t0002-gitfile.sh +++ b/t/t0002-gitfile.sh @@ -7,6 +7,7 @@ Verify that plumbing commands work when .git is a file GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh objpath() { diff --git a/t/t0003-attributes.sh b/t/t0003-attributes.sh index 1e4c672b84..b9ed612af1 100755 --- a/t/t0003-attributes.sh +++ b/t/t0003-attributes.sh @@ -2,6 +2,7 @@ test_description=gitattributes +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh attr_check_basic () { diff --git a/t/t0004-unwritable.sh b/t/t0004-unwritable.sh index 37d68ef03b..2e9d652d82 100755 --- a/t/t0004-unwritable.sh +++ b/t/t0004-unwritable.sh @@ -19,27 +19,66 @@ test_expect_success setup ' test_expect_success POSIXPERM,SANITY 'write-tree should notice unwritable repository' ' test_when_finished "chmod 775 .git/objects .git/objects/??" && chmod a-w .git/objects .git/objects/?? && - test_must_fail git write-tree + test_must_fail git write-tree 2>out.write-tree +' + +test_lazy_prereq WRITE_TREE_OUT 'test -e "$TRASH_DIRECTORY"/out.write-tree' +test_expect_success WRITE_TREE_OUT 'write-tree output on unwritable repository' ' + cat >expect <<-\EOF && + error: insufficient permission for adding an object to repository database .git/objects + fatal: git-write-tree: error building trees + EOF + test_cmp expect out.write-tree ' test_expect_success POSIXPERM,SANITY,!SANITIZE_LEAK 'commit should notice unwritable repository' ' test_when_finished "chmod 775 .git/objects .git/objects/??" && chmod a-w .git/objects .git/objects/?? && - test_must_fail git commit -m second + test_must_fail git commit -m second 2>out.commit +' + +test_lazy_prereq COMMIT_OUT 'test -e "$TRASH_DIRECTORY"/out.commit' +test_expect_success COMMIT_OUT 'commit output on unwritable repository' ' + cat >expect <<-\EOF && + error: insufficient permission for adding an object to repository database .git/objects + error: Error building trees + EOF + test_cmp expect out.commit ' test_expect_success POSIXPERM,SANITY 'update-index should notice unwritable repository' ' test_when_finished "chmod 775 .git/objects .git/objects/??" && echo 6O >file && chmod a-w .git/objects .git/objects/?? && - test_must_fail git update-index file + test_must_fail git update-index file 2>out.update-index +' + +test_lazy_prereq UPDATE_INDEX_OUT 'test -e "$TRASH_DIRECTORY"/out.update-index' +test_expect_success UPDATE_INDEX_OUT 'update-index output on unwritable repository' ' + cat >expect <<-\EOF && + error: insufficient permission for adding an object to repository database .git/objects + error: file: failed to insert into database + fatal: Unable to process path file + EOF + test_cmp expect out.update-index ' test_expect_success POSIXPERM,SANITY 'add should notice unwritable repository' ' test_when_finished "chmod 775 .git/objects .git/objects/??" && echo b >file && chmod a-w .git/objects .git/objects/?? && - test_must_fail git add file + test_must_fail git add file 2>out.add +' + +test_lazy_prereq ADD_OUT 'test -e "$TRASH_DIRECTORY"/out.add' +test_expect_success ADD_OUT 'add output on unwritable repository' ' + cat >expect <<-\EOF && + error: insufficient permission for adding an object to repository database .git/objects + error: file: failed to insert into database + error: unable to index file '\''file'\'' + fatal: updating files failed + EOF + test_cmp expect out.add ' test_done diff --git a/t/t0005-signals.sh b/t/t0005-signals.sh index 4c214bd11c..a5ec6a0315 100755 --- a/t/t0005-signals.sh +++ b/t/t0005-signals.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='signals work as we expect' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh cat >expect <<EOF diff --git a/t/t0007-git-var.sh b/t/t0007-git-var.sh index 88b9ae8158..53af92d571 100755 --- a/t/t0007-git-var.sh +++ b/t/t0007-git-var.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='basic sanity checks for git var' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'get GIT_AUTHOR_IDENT' ' diff --git a/t/t0008-ignores.sh b/t/t0008-ignores.sh index a594b4aa7d..532637de88 100755 --- a/t/t0008-ignores.sh +++ b/t/t0008-ignores.sh @@ -2,6 +2,7 @@ test_description=check-ignore +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh init_vars () { diff --git a/t/t0009-prio-queue.sh b/t/t0009-prio-queue.sh index 3941ad2528..eea99107a4 100755 --- a/t/t0009-prio-queue.sh +++ b/t/t0009-prio-queue.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='basic tests for priority queue implementation' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh cat >expect <<'EOF' diff --git a/t/t0010-racy-git.sh b/t/t0010-racy-git.sh index 5657c5a87b..837c8b7228 100755 --- a/t/t0010-racy-git.sh +++ b/t/t0010-racy-git.sh @@ -2,6 +2,7 @@ test_description='racy GIT' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # This test can give false success if your machine is sufficiently diff --git a/t/t0013-sha1dc.sh b/t/t0013-sha1dc.sh index 419f31a8f7..9ad76080aa 100755 --- a/t/t0013-sha1dc.sh +++ b/t/t0013-sha1dc.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='test sha1 collision detection' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh TEST_DATA="$TEST_DIRECTORY/t0013" diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh index 33dfc9cd56..92526db275 100755 --- a/t/t0021-conversion.sh +++ b/t/t0021-conversion.sh @@ -78,25 +78,29 @@ test_expect_success setup ' { echo "*.t filter=rot13" echo "*.i ident" + echo "*.ci ident=customId" } >.gitattributes && { echo a b c d e f g h i j k l m echo n o p q r s t u v w x y z echo '\''$Id$'\'' + echo '\''$customId$'\'' } >test && cat test >test.t && cat test >test.o && cat test >test.i && - git add test test.t test.i && - rm -f test test.t test.i && - git checkout -- test test.t test.i && + cat test >test.ci && + git add test test.t test.i test.ci && + rm -f test test.t test.i test.ci && + git checkout -- test test.t test.i test.ci && echo "content-test2" >test2.o && echo "content-test3 - filename with special characters" >"test3 '\''sq'\'',\$x=.o" ' -script='s/^\$Id: \([0-9a-f]*\) \$/\1/p' +script_i='s/^\$Id: \([0-9a-f]*\) \$/\1/p' +script_ci='s/^\$customId: \([0-9a-f]*\) \$/\1/p' test_expect_success check ' @@ -106,8 +110,14 @@ test_expect_success check ' # ident should be stripped in the repository git diff --raw --exit-code :test :test.i && id=$(git rev-parse --verify :test) && - embedded=$(sed -ne "$script" test.i) && + embedded=$(sed -ne "$script_i" test.i) && + nembedded=$(sed -ne "$script_ci" test.i) && test "z$id" = "z$embedded" && + test "z" = "z$nembedded" && + embedded=$(sed -ne "$script_ci" test.ci) && + nembedded=$(sed -ne "$script_i" test.ci) && + test "z$id" = "z$embedded" && + test "z" = "z$nembedded" && git cat-file blob :test.t >test.r && @@ -115,61 +125,84 @@ test_expect_success check ' test_cmp test.r test.t ' +gen_expanded_keywords() { + local id="${1}" + echo "File with expanded keywords" + echo "\$$id\$" + echo "\$$id:\$" + echo "\$$id: 0000000000000000000000000000000000000000 \$" + echo "\$$id: NoSpaceAtEnd\$" + echo "\$$id:NoSpaceAtFront \$" + echo "\$$id:NoSpaceAtEitherEnd\$" + echo "\$$id: NoTerminatingSymbol" + echo "\$$id: Foreign Commit With Spaces \$" + printf "\$$id: NoTerminatingSymbolAtEOF" +} + +gen_expected_output_0() { + local id="${1}" + local hid="${2}" + echo "File with expanded keywords" + echo "\$$id: $hid \$" + echo "\$$id: $hid \$" + echo "\$$id: $hid \$" + echo "\$$id: $hid \$" + echo "\$$id: $hid \$" + echo "\$$id: $hid \$" + echo "\$$id: NoTerminatingSymbol" + echo "\$$id: Foreign Commit With Spaces \$" +} + +gen_expected_output() { + local id="${1}" + gen_expected_output_0 "${@}" + printf "\$$id: NoTerminatingSymbolAtEOF" +} + +gen_expected_output_crlf() { + local id="${1}" + gen_expected_output_0 "${@}" | append_cr + printf "\$$id: NoTerminatingSymbolAtEOF" +} + # If an expanded ident ever gets into the repository, we want to make sure that # it is collapsed before being expanded again on checkout test_expect_success expanded_in_repo ' - { - echo "File with expanded keywords" - echo "\$Id\$" - echo "\$Id:\$" - echo "\$Id: 0000000000000000000000000000000000000000 \$" - echo "\$Id: NoSpaceAtEnd\$" - echo "\$Id:NoSpaceAtFront \$" - echo "\$Id:NoSpaceAtEitherEnd\$" - echo "\$Id: NoTerminatingSymbol" - echo "\$Id: Foreign Commit With Spaces \$" - } >expanded-keywords.0 && - - { - cat expanded-keywords.0 && - printf "\$Id: NoTerminatingSymbolAtEOF" - } >expanded-keywords && + gen_expanded_keywords Id >expanded-keywords && + gen_expanded_keywords customId >expanded-keywords_ci && cat expanded-keywords >expanded-keywords-crlf && + cat expanded-keywords_ci >expanded-keywords-crlf_ci && git add expanded-keywords expanded-keywords-crlf && + git add expanded-keywords_ci expanded-keywords-crlf_ci && git commit -m "File with keywords expanded" && id=$(git rev-parse --verify :expanded-keywords) && + id_ci=$(git rev-parse --verify :expanded-keywords_ci) && - { - echo "File with expanded keywords" - echo "\$Id: $id \$" - echo "\$Id: $id \$" - echo "\$Id: $id \$" - echo "\$Id: $id \$" - echo "\$Id: $id \$" - echo "\$Id: $id \$" - echo "\$Id: NoTerminatingSymbol" - echo "\$Id: Foreign Commit With Spaces \$" - } >expected-output.0 && - { - cat expected-output.0 && - printf "\$Id: NoTerminatingSymbolAtEOF" - } >expected-output && - { - append_cr <expected-output.0 && - printf "\$Id: NoTerminatingSymbolAtEOF" - } >expected-output-crlf && + gen_expected_output Id $id >expected-output && + gen_expected_output customId $id_ci >expected-output_ci && + gen_expected_output_crlf Id $id >expected-output-crlf && + gen_expected_output_crlf customId $id_ci >expected-output-crlf_ci && { echo "expanded-keywords ident" + echo "expanded-keywords_ci ident=customId" echo "expanded-keywords-crlf ident text eol=crlf" + echo "expanded-keywords-crlf_ci ident=customId text eol=crlf" } >>.gitattributes && rm -f expanded-keywords expanded-keywords-crlf && + rm -f expanded-keywords_ci expanded-keywords-crlf_ci && git checkout -- expanded-keywords && test_cmp expected-output expanded-keywords && git checkout -- expanded-keywords-crlf && - test_cmp expected-output-crlf expanded-keywords-crlf + test_cmp expected-output-crlf expanded-keywords-crlf && + + git checkout -- expanded-keywords_ci && + test_cmp expected-output_ci expanded-keywords_ci && + + git checkout -- expanded-keywords-crlf_ci && + test_cmp expected-output-crlf_ci expanded-keywords-crlf_ci ' # The use of %f in a filter definition is expanded to the path to diff --git a/t/t0022-crlf-rename.sh b/t/t0022-crlf-rename.sh index 7af3fbcc7b..c1a331e9e9 100755 --- a/t/t0022-crlf-rename.sh +++ b/t/t0022-crlf-rename.sh @@ -2,6 +2,7 @@ test_description='ignore CR in CRLF sequence while computing similiarity' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t0024-crlf-archive.sh b/t/t0024-crlf-archive.sh index 4e9fa3cd68..a34de56420 100755 --- a/t/t0024-crlf-archive.sh +++ b/t/t0024-crlf-archive.sh @@ -2,6 +2,7 @@ test_description='respect crlf in git archive' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t0025-crlf-renormalize.sh b/t/t0025-crlf-renormalize.sh index e13363ade5..81447978b7 100755 --- a/t/t0025-crlf-renormalize.sh +++ b/t/t0025-crlf-renormalize.sh @@ -2,6 +2,7 @@ test_description='CRLF renormalization' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t0026-eol-config.sh b/t/t0026-eol-config.sh index c5203e232c..cdcafcdff7 100755 --- a/t/t0026-eol-config.sh +++ b/t/t0026-eol-config.sh @@ -2,6 +2,7 @@ test_description='CRLF conversion' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh has_cr() { diff --git a/t/t0029-core-unsetenvvars.sh b/t/t0029-core-unsetenvvars.sh index 24ce46a6ea..b138e1d9cb 100755 --- a/t/t0029-core-unsetenvvars.sh +++ b/t/t0029-core-unsetenvvars.sh @@ -2,6 +2,7 @@ test_description='test the Windows-only core.unsetenvvars setting' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh if ! test_have_prereq MINGW diff --git a/t/t0032-reftable-unittest.sh b/t/t0032-reftable-unittest.sh new file mode 100755 index 0000000000..0ed14971a5 --- /dev/null +++ b/t/t0032-reftable-unittest.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# +# Copyright (c) 2020 Google LLC +# + +test_description='reftable unittests' + +. ./test-lib.sh + +test_expect_success 'unittests' ' + TMPDIR=$(pwd) && export TMPDIR && + test-tool reftable +' + +test_done diff --git a/t/t0040-parse-options.sh b/t/t0040-parse-options.sh index da310ed29b..ed2fb620a9 100755 --- a/t/t0040-parse-options.sh +++ b/t/t0040-parse-options.sh @@ -5,6 +5,7 @@ test_description='our own option parser' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh cat >expect <<\EOF @@ -168,9 +169,45 @@ test_expect_success 'long options' ' ' test_expect_success 'missing required value' ' - test_expect_code 129 test-tool parse-options -s && - test_expect_code 129 test-tool parse-options --string && - test_expect_code 129 test-tool parse-options --file + cat >expect <<-\EOF && + error: switch `s'\'' requires a value + EOF + test_expect_code 129 test-tool parse-options -s 2>actual && + test_cmp expect actual && + + cat >expect <<-\EOF && + error: option `string'\'' requires a value + EOF + test_expect_code 129 test-tool parse-options --string 2>actual && + test_cmp expect actual && + + cat >expect <<-\EOF && + error: option `file'\'' requires a value + EOF + test_expect_code 129 test-tool parse-options --file 2>actual && + test_cmp expect actual +' + +test_expect_success 'superfluous value provided: boolean' ' + cat >expect <<-\EOF && + error: option `yes'\'' takes no value + EOF + test_expect_code 129 test-tool parse-options --yes=hi 2>actual && + test_cmp expect actual && + + cat >expect <<-\EOF && + error: option `no-yes'\'' takes no value + EOF + test_expect_code 129 test-tool parse-options --no-yes=hi 2>actual && + test_cmp expect actual +' + +test_expect_success 'superfluous value provided: cmdmode' ' + cat >expect <<-\EOF && + error: option `mode1'\'' takes no value + EOF + test_expect_code 129 test-tool parse-options --mode1=hi 2>actual && + test_cmp expect actual ' cat >expect <<\EOF diff --git a/t/t0052-simple-ipc.sh b/t/t0052-simple-ipc.sh index ff98be31a5..1a36a53574 100755 --- a/t/t0052-simple-ipc.sh +++ b/t/t0052-simple-ipc.sh @@ -2,6 +2,7 @@ test_description='simple command server' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test-tool simple-ipc SUPPORTS_SIMPLE_IPC || { diff --git a/t/t0055-beyond-symlinks.sh b/t/t0055-beyond-symlinks.sh index 0c6ff567a1..6bada37022 100755 --- a/t/t0055-beyond-symlinks.sh +++ b/t/t0055-beyond-symlinks.sh @@ -2,6 +2,7 @@ test_description='update-index and add refuse to add beyond symlinks' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success SYMLINKS setup ' diff --git a/t/t0061-run-command.sh b/t/t0061-run-command.sh index 7d599675e3..ee281909bc 100755 --- a/t/t0061-run-command.sh +++ b/t/t0061-run-command.sh @@ -5,6 +5,7 @@ test_description='Test run command' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh cat >hello-script <<-EOF diff --git a/t/t0064-oid-array.sh b/t/t0064-oid-array.sh index 2e5438ccda..88c89e8f48 100755 --- a/t/t0064-oid-array.sh +++ b/t/t0064-oid-array.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='basic tests for the oid array implementation' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh echoid () { diff --git a/t/t0065-strcmp-offset.sh b/t/t0065-strcmp-offset.sh index 91fa639c4a..94e34c83ed 100755 --- a/t/t0065-strcmp-offset.sh +++ b/t/t0065-strcmp-offset.sh @@ -2,6 +2,7 @@ test_description='Test strcmp_offset functionality' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh while read s1 s2 expect diff --git a/t/t0066-dir-iterator.sh b/t/t0066-dir-iterator.sh index 92910e4e6c..63a1a45cd3 100755 --- a/t/t0066-dir-iterator.sh +++ b/t/t0066-dir-iterator.sh @@ -2,6 +2,7 @@ test_description='Test the dir-iterator functionality' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t0067-parse_pathspec_file.sh b/t/t0067-parse_pathspec_file.sh index 7bab49f361..0188d0423a 100755 --- a/t/t0067-parse_pathspec_file.sh +++ b/t/t0067-parse_pathspec_file.sh @@ -2,6 +2,7 @@ test_description='Test parse_pathspec_file()' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'one item from stdin' ' diff --git a/t/t0069-oidtree.sh b/t/t0069-oidtree.sh index bfb1397d7b..74cc59bf8a 100755 --- a/t/t0069-oidtree.sh +++ b/t/t0069-oidtree.sh @@ -1,6 +1,7 @@ #!/bin/sh test_description='basic tests for the oidtree implementation' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh maxhexsz=$(test_oid hexsz) diff --git a/t/t0110-urlmatch-normalization.sh b/t/t0110-urlmatch-normalization.sh index f99529d838..4dc9fecf72 100755 --- a/t/t0110-urlmatch-normalization.sh +++ b/t/t0110-urlmatch-normalization.sh @@ -47,7 +47,7 @@ test_expect_success 'url authority' ' test-tool urlmatch-normalization "scheme://@host" && test-tool urlmatch-normalization "scheme://%00@host" && ! test-tool urlmatch-normalization "scheme://%%@host" && - ! test-tool urlmatch-normalization "scheme://host_" && + test-tool urlmatch-normalization "scheme://host_" && test-tool urlmatch-normalization "scheme://user:pass@host/" && test-tool urlmatch-normalization "scheme://@host/" && test-tool urlmatch-normalization "scheme://host/" && diff --git a/t/t0210-trace2-normal.sh b/t/t0210-trace2-normal.sh index 0cf3a63b75..37c359bd5a 100755 --- a/t/t0210-trace2-normal.sh +++ b/t/t0210-trace2-normal.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='test trace2 facility (normal target)' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # Turn off any inherited trace2 settings for this test. diff --git a/t/t0211-trace2-perf.sh b/t/t0211-trace2-perf.sh index 6ee8ee3b67..22d0845544 100755 --- a/t/t0211-trace2-perf.sh +++ b/t/t0211-trace2-perf.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='test trace2 facility (perf target)' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # Turn off any inherited trace2 settings for this test. diff --git a/t/t0212-trace2-event.sh b/t/t0212-trace2-event.sh index 1529155cf0..6d3374ff77 100755 --- a/t/t0212-trace2-event.sh +++ b/t/t0212-trace2-event.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='test trace2 facility' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # Turn off any inherited trace2 settings for this test. diff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh index bba679685f..c76485b1b6 100755 --- a/t/t0410-partial-clone.sh +++ b/t/t0410-partial-clone.sh @@ -6,6 +6,10 @@ test_description='partial clone' # missing promisor objects cause repacks which write bitmaps to fail GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0 +# When enabled, some commands will write commit-graphs. This causes fsck +# to fail when delete_object() is called because fsck will attempt to +# verify the out-of-sync commit graph. +GIT_TEST_COMMIT_GRAPH=0 delete_object () { rm $1/.git/objects/$(echo $2 | sed -e 's|^..|&/|') @@ -322,7 +326,7 @@ test_expect_success 'rev-list stops traversal at missing and promised commit' ' git -C repo config core.repositoryformatversion 1 && git -C repo config extensions.partialclone "arbitrary string" && - GIT_TEST_COMMIT_GRAPH=0 git -C repo -c core.commitGraph=false rev-list --exclude-promisor-objects --objects bar >out && + git -C repo rev-list --exclude-promisor-objects --objects bar >out && grep $(git -C repo rev-parse bar) out && ! grep $FOO out ' diff --git a/t/t0500-progress-display.sh b/t/t0500-progress-display.sh index 22058b503a..867fdace3f 100755 --- a/t/t0500-progress-display.sh +++ b/t/t0500-progress-display.sh @@ -2,6 +2,7 @@ test_description='progress display' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh show_cr () { @@ -17,6 +18,7 @@ test_expect_success 'simple progress display' ' EOF cat >in <<-\EOF && + start 0 update progress 1 update @@ -25,8 +27,9 @@ test_expect_success 'simple progress display' ' progress 4 update progress 5 + stop EOF - test-tool progress "Working hard" <in 2>stderr && + test-tool progress <in 2>stderr && show_cr <stderr >out && test_cmp expect out @@ -41,11 +44,13 @@ test_expect_success 'progress display with total' ' EOF cat >in <<-\EOF && + start 3 progress 1 progress 2 progress 3 + stop EOF - test-tool progress --total=3 "Working hard" <in 2>stderr && + test-tool progress <in 2>stderr && show_cr <stderr >out && test_cmp expect out @@ -62,14 +67,14 @@ Working hard.......2.........3.........4.........5.........6: EOF cat >in <<-\EOF && + start 100000 Working hard.......2.........3.........4.........5.........6 progress 100 progress 1000 progress 10000 progress 100000 + stop EOF - test-tool progress --total=100000 \ - "Working hard.......2.........3.........4.........5.........6" \ - <in 2>stderr && + test-tool progress <in 2>stderr && show_cr <stderr >out && test_cmp expect out @@ -88,16 +93,16 @@ Working hard.......2.........3.........4.........5.........6: EOF cat >in <<-\EOF && + start 100000 Working hard.......2.........3.........4.........5.........6 update progress 1 update progress 2 progress 10000 progress 100000 + stop EOF - test-tool progress --total=100000 \ - "Working hard.......2.........3.........4.........5.........6" \ - <in 2>stderr && + test-tool progress <in 2>stderr && show_cr <stderr >out && test_cmp expect out @@ -116,14 +121,14 @@ Working hard.......2.........3.........4.........5.........6: EOF cat >in <<-\EOF && + start 100000 Working hard.......2.........3.........4.........5.........6 progress 25000 progress 50000 progress 75000 progress 100000 + stop EOF - test-tool progress --total=100000 \ - "Working hard.......2.........3.........4.........5.........6" \ - <in 2>stderr && + test-tool progress <in 2>stderr && show_cr <stderr >out && test_cmp expect out @@ -140,14 +145,14 @@ Working hard.......2.........3.........4.........5.........6.........7.........: EOF cat >in <<-\EOF && + start 100000 Working hard.......2.........3.........4.........5.........6.........7......... progress 25000 progress 50000 progress 75000 progress 100000 + stop EOF - test-tool progress --total=100000 \ - "Working hard.......2.........3.........4.........5.........6.........7........." \ - <in 2>stderr && + test-tool progress <in 2>stderr && show_cr <stderr >out && test_cmp expect out @@ -164,12 +169,14 @@ test_expect_success 'progress shortens - crazy caller' ' EOF cat >in <<-\EOF && + start 1000 progress 100 progress 200 progress 1 progress 1000 + stop EOF - test-tool progress --total=1000 "Working hard" <in 2>stderr && + test-tool progress <in 2>stderr && show_cr <stderr >out && test_cmp expect out @@ -185,6 +192,7 @@ test_expect_success 'progress display with throughput' ' EOF cat >in <<-\EOF && + start 0 throughput 102400 1000 update progress 10 @@ -197,8 +205,9 @@ test_expect_success 'progress display with throughput' ' throughput 409600 4000 update progress 40 + stop EOF - test-tool progress "Working hard" <in 2>stderr && + test-tool progress <in 2>stderr && show_cr <stderr >out && test_cmp expect out @@ -214,6 +223,7 @@ test_expect_success 'progress display with throughput and total' ' EOF cat >in <<-\EOF && + start 40 throughput 102400 1000 progress 10 throughput 204800 2000 @@ -222,8 +232,9 @@ test_expect_success 'progress display with throughput and total' ' progress 30 throughput 409600 4000 progress 40 + stop EOF - test-tool progress --total=40 "Working hard" <in 2>stderr && + test-tool progress <in 2>stderr && show_cr <stderr >out && test_cmp expect out @@ -239,6 +250,7 @@ test_expect_success 'cover up after throughput shortens' ' EOF cat >in <<-\EOF && + start 0 throughput 409600 1000 update progress 1 @@ -251,8 +263,9 @@ test_expect_success 'cover up after throughput shortens' ' throughput 1638400 4000 update progress 4 + stop EOF - test-tool progress "Working hard" <in 2>stderr && + test-tool progress <in 2>stderr && show_cr <stderr >out && test_cmp expect out @@ -267,6 +280,7 @@ test_expect_success 'cover up after throughput shortens a lot' ' EOF cat >in <<-\EOF && + start 0 throughput 1 1000 update progress 1 @@ -276,15 +290,28 @@ test_expect_success 'cover up after throughput shortens a lot' ' throughput 3145728 3000 update progress 3 + stop EOF - test-tool progress "Working hard" <in 2>stderr && + test-tool progress <in 2>stderr && show_cr <stderr >out && test_cmp expect out ' +test_expect_success 'BUG: start two concurrent progress bars' ' + cat >in <<-\EOF && + start 0 one + start 0 two + EOF + + test_must_fail test-tool progress \ + <in 2>stderr && + grep "^BUG: .*'\''one'\'' progress still active when trying to start '\''two'\''$" stderr +' + test_expect_success 'progress generates traces' ' cat >in <<-\EOF && + start 40 throughput 102400 1000 update progress 10 @@ -297,10 +324,11 @@ test_expect_success 'progress generates traces' ' throughput 409600 4000 update progress 40 + stop EOF - GIT_TRACE2_EVENT="$(pwd)/trace.event" test-tool progress --total=40 \ - "Working hard" <in 2>stderr && + GIT_TRACE2_EVENT="$(pwd)/trace.event" test-tool progress \ + <in 2>stderr && # t0212/parse_events.perl intentionally omits regions and data. test_region progress "Working hard" trace.event && @@ -308,4 +336,39 @@ test_expect_success 'progress generates traces' ' grep "\"key\":\"total_bytes\",\"value\":\"409600\"" trace.event ' +test_expect_success 'progress generates traces: stop / start' ' + cat >in <<-\EOF && + start 0 + stop + EOF + + GIT_TRACE2_EVENT="$(pwd)/trace-startstop.event" test-tool progress \ + <in 2>stderr && + test_region progress "Working hard" trace-startstop.event +' + +test_expect_success 'progress generates traces: start without stop' ' + cat >in <<-\EOF && + start 0 + EOF + + GIT_TRACE2_EVENT="$(pwd)/trace-start.event" \ + LSAN_OPTIONS=detect_leaks=0 \ + test-tool progress \ + <in 2>stderr && + grep region_enter.*progress trace-start.event && + ! grep region_leave.*progress trace-start.event +' + +test_expect_success 'progress generates traces: stop without start' ' + cat >in <<-\EOF && + stop + EOF + + GIT_TRACE2_EVENT="$(pwd)/trace-stop.event" test-tool progress \ + <in 2>stderr && + ! grep region_enter.*progress trace-stop.event && + ! grep region_leave.*progress trace-stop.event +' + test_done diff --git a/t/t1000-read-tree-m-3way.sh b/t/t1000-read-tree-m-3way.sh index 013c5a7bc3..0e8c0dfbbe 100755 --- a/t/t1000-read-tree-m-3way.sh +++ b/t/t1000-read-tree-m-3way.sh @@ -71,6 +71,8 @@ In addition: DF: a special case, where A makes a directory and B makes a file. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-read-tree.sh . "$TEST_DIRECTORY"/lib-read-tree-m-3way.sh diff --git a/t/t1001-read-tree-m-2way.sh b/t/t1001-read-tree-m-2way.sh index 1057a96b24..d1115528cb 100755 --- a/t/t1001-read-tree-m-2way.sh +++ b/t/t1001-read-tree-m-2way.sh @@ -20,6 +20,8 @@ In the test, these paths are used: rezrov - in H, deleted in M yomin - not in H or M ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-read-tree.sh diff --git a/t/t1003-read-tree-prefix.sh b/t/t1003-read-tree-prefix.sh index b6111cd150..e0db2066f3 100755 --- a/t/t1003-read-tree-prefix.sh +++ b/t/t1003-read-tree-prefix.sh @@ -6,6 +6,7 @@ test_description='git read-tree --prefix test. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t1006-cat-file.sh b/t/t1006-cat-file.sh index 4a753705ec..658628375c 100755 --- a/t/t1006-cat-file.sh +++ b/t/t1006-cat-file.sh @@ -315,46 +315,236 @@ test_expect_success '%(deltabase) reports packed delta bases' ' } ' -bogus_type="bogus" -bogus_content="bogus" -bogus_size=$(strlen "$bogus_content") -bogus_sha1=$(echo_without_newline "$bogus_content" | git hash-object -t $bogus_type --literally -w --stdin) +test_expect_success 'setup bogus data' ' + bogus_short_type="bogus" && + bogus_short_content="bogus" && + bogus_short_size=$(strlen "$bogus_short_content") && + bogus_short_sha1=$(echo_without_newline "$bogus_short_content" | git hash-object -t $bogus_short_type --literally -w --stdin) && + + bogus_long_type="abcdefghijklmnopqrstuvwxyz1234679" && + bogus_long_content="bogus" && + bogus_long_size=$(strlen "$bogus_long_content") && + bogus_long_sha1=$(echo_without_newline "$bogus_long_content" | git hash-object -t $bogus_long_type --literally -w --stdin) +' + +for arg1 in '' --allow-unknown-type +do + for arg2 in -s -t -p + do + if test "$arg1" = "--allow-unknown-type" && test "$arg2" = "-p" + then + continue + fi + + + test_expect_success "cat-file $arg1 $arg2 error on bogus short OID" ' + cat >expect <<-\EOF && + fatal: invalid object type + EOF + + if test "$arg1" = "--allow-unknown-type" + then + git cat-file $arg1 $arg2 $bogus_short_sha1 + else + test_must_fail git cat-file $arg1 $arg2 $bogus_short_sha1 >out 2>actual && + test_must_be_empty out && + test_cmp expect actual + fi + ' + + test_expect_success "cat-file $arg1 $arg2 error on bogus full OID" ' + if test "$arg2" = "-p" + then + cat >expect <<-EOF + error: header for $bogus_long_sha1 too long, exceeds 32 bytes + fatal: Not a valid object name $bogus_long_sha1 + EOF + else + cat >expect <<-EOF + error: header for $bogus_long_sha1 too long, exceeds 32 bytes + fatal: git cat-file: could not get object info + EOF + fi && + + if test "$arg1" = "--allow-unknown-type" + then + git cat-file $arg1 $arg2 $bogus_short_sha1 + else + test_must_fail git cat-file $arg1 $arg2 $bogus_long_sha1 >out 2>actual && + test_must_be_empty out && + test_cmp expect actual + fi + ' + + test_expect_success "cat-file $arg1 $arg2 error on missing short OID" ' + cat >expect.err <<-EOF && + fatal: Not a valid object name $(test_oid deadbeef_short) + EOF + test_must_fail git cat-file $arg1 $arg2 $(test_oid deadbeef_short) >out 2>err.actual && + test_must_be_empty out + ' + + test_expect_success "cat-file $arg1 $arg2 error on missing full OID" ' + if test "$arg2" = "-p" + then + cat >expect.err <<-EOF + fatal: Not a valid object name $(test_oid deadbeef) + EOF + else + cat >expect.err <<-\EOF + fatal: git cat-file: could not get object info + EOF + fi && + test_must_fail git cat-file $arg1 $arg2 $(test_oid deadbeef) >out 2>err.actual && + test_must_be_empty out && + test_cmp expect.err err.actual + ' + done +done + +test_expect_success '-e is OK with a broken object without --allow-unknown-type' ' + git cat-file -e $bogus_short_sha1 +' + +test_expect_success '-e can not be combined with --allow-unknown-type' ' + test_expect_code 128 git cat-file -e --allow-unknown-type $bogus_short_sha1 +' + +test_expect_success '-p cannot print a broken object even with --allow-unknown-type' ' + test_must_fail git cat-file -p $bogus_short_sha1 && + test_expect_code 128 git cat-file -p --allow-unknown-type $bogus_short_sha1 +' + +test_expect_success '<type> <hash> does not work with objects of broken types' ' + cat >err.expect <<-\EOF && + fatal: invalid object type "bogus" + EOF + test_must_fail git cat-file $bogus_short_type $bogus_short_sha1 2>err.actual && + test_cmp err.expect err.actual +' + +test_expect_success 'broken types combined with --batch and --batch-check' ' + echo $bogus_short_sha1 >bogus-oid && + + cat >err.expect <<-\EOF && + fatal: invalid object type + EOF + + test_must_fail git cat-file --batch <bogus-oid 2>err.actual && + test_cmp err.expect err.actual && + + test_must_fail git cat-file --batch-check <bogus-oid 2>err.actual && + test_cmp err.expect err.actual +' + +test_expect_success 'the --batch and --batch-check options do not combine with --allow-unknown-type' ' + test_expect_code 128 git cat-file --batch --allow-unknown-type <bogus-oid && + test_expect_code 128 git cat-file --batch-check --allow-unknown-type <bogus-oid +' + +test_expect_success 'the --allow-unknown-type option does not consider replacement refs' ' + cat >expect <<-EOF && + $bogus_short_type + EOF + git cat-file -t --allow-unknown-type $bogus_short_sha1 >actual && + test_cmp expect actual && + + # Create it manually, as "git replace" will die on bogus + # types. + head=$(git rev-parse --verify HEAD) && + test_when_finished "rm -rf .git/refs/replace" && + mkdir -p .git/refs/replace && + echo $head >.git/refs/replace/$bogus_short_sha1 && + + cat >expect <<-EOF && + commit + EOF + git cat-file -t --allow-unknown-type $bogus_short_sha1 >actual && + test_cmp expect actual +' test_expect_success "Type of broken object is correct" ' - echo $bogus_type >expect && - git cat-file -t --allow-unknown-type $bogus_sha1 >actual && + echo $bogus_short_type >expect && + git cat-file -t --allow-unknown-type $bogus_short_sha1 >actual && test_cmp expect actual ' test_expect_success "Size of broken object is correct" ' - echo $bogus_size >expect && - git cat-file -s --allow-unknown-type $bogus_sha1 >actual && + echo $bogus_short_size >expect && + git cat-file -s --allow-unknown-type $bogus_short_sha1 >actual && test_cmp expect actual ' test_expect_success 'clean up broken object' ' - rm .git/objects/$(test_oid_to_path $bogus_sha1) + rm .git/objects/$(test_oid_to_path $bogus_short_sha1) ' -bogus_type="abcdefghijklmnopqrstuvwxyz1234679" -bogus_content="bogus" -bogus_size=$(strlen "$bogus_content") -bogus_sha1=$(echo_without_newline "$bogus_content" | git hash-object -t $bogus_type --literally -w --stdin) - test_expect_success "Type of broken object is correct when type is large" ' - echo $bogus_type >expect && - git cat-file -t --allow-unknown-type $bogus_sha1 >actual && + echo $bogus_long_type >expect && + git cat-file -t --allow-unknown-type $bogus_long_sha1 >actual && test_cmp expect actual ' test_expect_success "Size of large broken object is correct when type is large" ' - echo $bogus_size >expect && - git cat-file -s --allow-unknown-type $bogus_sha1 >actual && + echo $bogus_long_size >expect && + git cat-file -s --allow-unknown-type $bogus_long_sha1 >actual && test_cmp expect actual ' test_expect_success 'clean up broken object' ' - rm .git/objects/$(test_oid_to_path $bogus_sha1) + rm .git/objects/$(test_oid_to_path $bogus_long_sha1) +' + +test_expect_success 'cat-file -t and -s on corrupt loose object' ' + git init --bare corrupt-loose.git && + ( + cd corrupt-loose.git && + + # Setup and create the empty blob and its path + empty_path=$(git rev-parse --git-path objects/$(test_oid_to_path "$EMPTY_BLOB")) && + git hash-object -w --stdin </dev/null && + + # Create another blob and its path + echo other >other.blob && + other_blob=$(git hash-object -w --stdin <other.blob) && + other_path=$(git rev-parse --git-path objects/$(test_oid_to_path "$other_blob")) && + + # Before the swap the size is 0 + cat >out.expect <<-EOF && + 0 + EOF + git cat-file -s "$EMPTY_BLOB" >out.actual 2>err.actual && + test_must_be_empty err.actual && + test_cmp out.expect out.actual && + + # Swap the two to corrupt the repository + mv -f "$other_path" "$empty_path" && + test_must_fail git fsck 2>err.fsck && + grep "hash-path mismatch" err.fsck && + + # confirm that cat-file is reading the new swapped-in + # blob... + cat >out.expect <<-EOF && + blob + EOF + git cat-file -t "$EMPTY_BLOB" >out.actual 2>err.actual && + test_must_be_empty err.actual && + test_cmp out.expect out.actual && + + # ... since it has a different size now. + cat >out.expect <<-EOF && + 6 + EOF + git cat-file -s "$EMPTY_BLOB" >out.actual 2>err.actual && + test_must_be_empty err.actual && + test_cmp out.expect out.actual && + + # So far "cat-file" has been happy to spew the found + # content out as-is. Try to make it zlib-invalid. + mv -f other.blob "$empty_path" && + test_must_fail git fsck 2>err.fsck && + grep "^error: inflate: data stream error (" err.fsck + ) ' # Tests for git cat-file --follow-symlinks diff --git a/t/t1009-read-tree-new-index.sh b/t/t1009-read-tree-new-index.sh index 2935f68f8d..fc179ac5dd 100755 --- a/t/t1009-read-tree-new-index.sh +++ b/t/t1009-read-tree-new-index.sh @@ -5,6 +5,7 @@ test_description='test read-tree into a fresh index file' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t1010-mktree.sh b/t/t1010-mktree.sh index b946f87686..48bfad07ab 100755 --- a/t/t1010-mktree.sh +++ b/t/t1010-mktree.sh @@ -2,6 +2,7 @@ test_description='git mktree' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t1012-read-tree-df.sh b/t/t1012-read-tree-df.sh index 57f0770df1..cde93d22cd 100755 --- a/t/t1012-read-tree-df.sh +++ b/t/t1012-read-tree-df.sh @@ -2,6 +2,7 @@ test_description='read-tree D/F conflict corner cases' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-read-tree.sh diff --git a/t/t1014-read-tree-confusing.sh b/t/t1014-read-tree-confusing.sh index da3376b3bb..8ea8d36818 100755 --- a/t/t1014-read-tree-confusing.sh +++ b/t/t1014-read-tree-confusing.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='check that read-tree rejects confusing paths' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'create base tree' ' diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index ca91c6a67f..9531dd0d26 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -416,6 +416,46 @@ test_expect_success 'diff --staged' ' test_all_match git diff --staged ' +test_expect_success 'diff partially-staged' ' + init_repos && + + write_script edit-contents <<-\EOF && + echo text >>$1 + EOF + + # Add file within cone + test_sparse_match git sparse-checkout set deep && + run_on_all ../edit-contents deep/testfile && + test_all_match git add deep/testfile && + run_on_all ../edit-contents deep/testfile && + + test_all_match git diff && + test_all_match git diff --staged && + + # Add file outside cone + test_all_match git reset --hard && + run_on_all mkdir newdirectory && + run_on_all ../edit-contents newdirectory/testfile && + test_sparse_match git sparse-checkout set newdirectory && + test_all_match git add newdirectory/testfile && + run_on_all ../edit-contents newdirectory/testfile && + test_sparse_match git sparse-checkout set && + + test_all_match git diff && + test_all_match git diff --staged && + + # Merge conflict outside cone + # The sparse checkout will report a warning that is not in the + # full checkout, so we use `run_on_all` instead of + # `test_all_match` + run_on_all git reset --hard && + test_all_match git checkout merge-left && + test_all_match test_must_fail git merge merge-right && + + test_all_match git diff && + test_all_match git diff --staged +' + # NEEDSWORK: sparse-checkout behaves differently from full-checkout when # running this test with 'df-conflict-2' after 'df-conflict-1'. test_expect_success 'diff with renames and conflicts' ' @@ -478,43 +518,122 @@ test_expect_success 'blame with pathspec inside sparse definition' ' test_all_match git blame deep/deeper1/deepest/a ' -# TODO: blame currently does not support blaming files outside of the -# sparse definition. It complains that the file doesn't exist locally. -test_expect_failure 'blame with pathspec outside sparse definition' ' +# Blame does not support blaming files outside of the sparse +# definition, so we verify this scenario. +test_expect_success 'blame with pathspec outside sparse definition' ' init_repos && - test_all_match git blame folder1/a && - test_all_match git blame folder2/a && - test_all_match git blame deep/deeper2/a && - test_all_match git blame deep/deeper2/deepest/a + test_sparse_match git sparse-checkout set && + test_sparse_match test_must_fail git blame folder1/a && + test_sparse_match test_must_fail git blame folder2/a && + test_sparse_match test_must_fail git blame deep/deeper2/a && + test_sparse_match test_must_fail git blame deep/deeper2/deepest/a ' -# NEEDSWORK: a sparse-checkout behaves differently from a full checkout -# in this scenario, but it shouldn't. -test_expect_failure 'checkout and reset (mixed)' ' +test_expect_success 'checkout and reset (mixed)' ' init_repos && test_all_match git checkout -b reset-test update-deep && test_all_match git reset deepest && - test_all_match git reset update-folder1 && - test_all_match git reset update-folder2 + + # Because skip-worktree is preserved, resetting to update-folder1 + # will show worktree changes for folder1/a in full-checkout, but not + # in sparse-checkout or sparse-index. + git -C full-checkout reset update-folder1 >full-checkout-out && + test_sparse_match git reset update-folder1 && + grep "M folder1/a" full-checkout-out && + ! grep "M folder1/a" sparse-checkout-out && + run_on_sparse test_path_is_missing folder1 ' -# NEEDSWORK: a sparse-checkout behaves differently from a full checkout -# in this scenario, but it shouldn't. -test_expect_success 'checkout and reset (mixed) [sparse]' ' +test_expect_success 'checkout and reset (merge)' ' init_repos && - test_sparse_match git checkout -b reset-test update-deep && - test_sparse_match git reset deepest && - test_sparse_match git reset update-folder1 && - test_sparse_match git reset update-folder2 + write_script edit-contents <<-\EOF && + echo text >>$1 + EOF + + test_all_match git checkout -b reset-test update-deep && + run_on_all ../edit-contents a && + test_all_match git reset --merge deepest && + test_all_match git status --porcelain=v2 && + + test_all_match git reset --hard update-deep && + run_on_all ../edit-contents deep/a && + test_all_match test_must_fail git reset --merge deepest +' + +test_expect_success 'checkout and reset (keep)' ' + init_repos && + + write_script edit-contents <<-\EOF && + echo text >>$1 + EOF + + test_all_match git checkout -b reset-test update-deep && + run_on_all ../edit-contents a && + test_all_match git reset --keep deepest && + test_all_match git status --porcelain=v2 && + + test_all_match git reset --hard update-deep && + run_on_all ../edit-contents deep/a && + test_all_match test_must_fail git reset --keep deepest +' + +test_expect_success 'reset with pathspecs inside sparse definition' ' + init_repos && + + write_script edit-contents <<-\EOF && + echo text >>$1 + EOF + + test_all_match git checkout -b reset-test update-deep && + run_on_all ../edit-contents deep/a && + + test_all_match git reset base -- deep/a && + test_all_match git status --porcelain=v2 && + + test_all_match git reset base -- nonexistent-file && + test_all_match git status --porcelain=v2 && + + test_all_match git reset deepest -- deep && + test_all_match git status --porcelain=v2 +' + +# Although the working tree differs between full and sparse checkouts after +# reset, the state of the index is the same. +test_expect_success 'reset with pathspecs outside sparse definition' ' + init_repos && + test_all_match git checkout -b reset-test base && + + test_sparse_match git reset update-folder1 -- folder1 && + git -C full-checkout reset update-folder1 -- folder1 && + test_sparse_match git status --porcelain=v2 && + test_all_match git rev-parse HEAD:folder1 && + + test_sparse_match git reset update-folder2 -- folder2/a && + git -C full-checkout reset update-folder2 -- folder2/a && + test_sparse_match git status --porcelain=v2 && + test_all_match git rev-parse HEAD:folder2/a +' + +test_expect_success 'reset with wildcard pathspec' ' + init_repos && + + test_all_match git checkout -b reset-test update-deep && + test_all_match git reset base -- \*/a && + test_all_match git status --porcelain=v2 && + test_all_match git rev-parse HEAD:folder1/a && + + test_all_match git reset base -- folder\* && + test_all_match git status --porcelain=v2 && + test_all_match git rev-parse HEAD:folder2 ' test_expect_success 'merge, cherry-pick, and rebase' ' init_repos && - for OPERATION in "merge -m merge" cherry-pick rebase + for OPERATION in "merge -m merge" cherry-pick "rebase --apply" "rebase --merge" do test_all_match git checkout -B temp update-deep && test_all_match git $OPERATION update-folder1 && @@ -685,15 +804,50 @@ test_expect_success 'submodule handling' ' grep "160000 commit $(git -C initial-repo rev-parse HEAD) modules/sub" cache ' +# When working with a sparse index, some commands will need to expand the +# index to operate properly. If those commands also write the index back +# to disk, they need to convert the index to sparse before writing. +# This test verifies that both of these events are logged in trace2 logs. test_expect_success 'sparse-index is expanded and converted back' ' init_repos && GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ - git -C sparse-index -c core.fsmonitor="" reset --hard && + git -C sparse-index reset -- folder1/a && test_region index convert_to_sparse trace2.txt && test_region index ensure_full_index trace2.txt ' +test_expect_success 'index.sparse disabled inline uses full index' ' + init_repos && + + # When index.sparse is disabled inline with `git status`, the + # index is expanded at the beginning of the execution then never + # converted back to sparse. It is then written to disk as a full index. + rm -f trace2.txt && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ + git -C sparse-index -c index.sparse=false status && + ! test_region index convert_to_sparse trace2.txt && + test_region index ensure_full_index trace2.txt && + + # Since index.sparse is set to true at a repo level, the index + # is converted from full to sparse when read, then never expanded + # over the course of `git status`. It is written to disk as a sparse + # index. + rm -f trace2.txt && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ + git -C sparse-index status && + test_region index convert_to_sparse trace2.txt && + ! test_region index ensure_full_index trace2.txt && + + # Now that the index has been written to disk as sparse, it is not + # converted to sparse (or expanded to full) when read by `git status`. + rm -f trace2.txt && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ + git -C sparse-index status && + ! test_region index convert_to_sparse trace2.txt && + ! test_region index ensure_full_index trace2.txt +' + ensure_not_expanded () { rm -f trace2.txt && echo >>sparse-index/untracked.txt && @@ -726,9 +880,9 @@ test_expect_success 'sparse-index is not expanded' ' ensure_not_expanded checkout - && ensure_not_expanded switch rename-out-to-out && ensure_not_expanded switch - && - git -C sparse-index reset --hard && + ensure_not_expanded reset --hard && ensure_not_expanded checkout rename-out-to-out -- deep/deeper1 && - git -C sparse-index reset --hard && + ensure_not_expanded reset --hard && ensure_not_expanded restore -s rename-out-to-out -- deep/deeper1 && echo >>sparse-index/README.md && @@ -738,6 +892,39 @@ test_expect_success 'sparse-index is not expanded' ' echo >>sparse-index/untracked.txt && ensure_not_expanded add . && + for ref in update-deep update-folder1 update-folder2 update-deep + do + echo >>sparse-index/README.md && + ensure_not_expanded reset --hard $ref || return 1 + done && + + ensure_not_expanded reset --mixed base && + ensure_not_expanded reset --hard update-deep && + ensure_not_expanded reset --keep base && + ensure_not_expanded reset --merge update-deep && + ensure_not_expanded reset --hard && + + ensure_not_expanded reset base -- deep/a && + ensure_not_expanded reset base -- nonexistent-file && + ensure_not_expanded reset deepest -- deep && + + # Although folder1 is outside the sparse definition, it exists as a + # directory entry in the index, so the pathspec will not force the + # index to be expanded. + ensure_not_expanded reset deepest -- folder1 && + ensure_not_expanded reset deepest -- folder1/ && + + # Wildcard identifies only in-cone files, no index expansion + ensure_not_expanded reset deepest -- deep/\* && + + # Wildcard identifies only full sparse directories, no index expansion + ensure_not_expanded reset deepest -- folder\* && + + echo a test change >>sparse-index/README.md && + ensure_not_expanded diff && + git -C sparse-index add README.md && + ensure_not_expanded diff --staged && + ensure_not_expanded checkout -f update-deep && test_config -C sparse-index pull.twohead ort && ( @@ -767,6 +954,15 @@ test_expect_success 'sparse-index is not expanded: merge conflict in cone' ' ) ' +test_expect_success 'sparse index is not expanded: blame' ' + init_repos && + + ensure_not_expanded blame a && + ensure_not_expanded blame deep/a && + ensure_not_expanded blame deep/deeper1/a && + ensure_not_expanded blame deep/deeper1/deepest/a +' + # NEEDSWORK: a sparse-checkout behaves differently from a full checkout # in this scenario, but it shouldn't. test_expect_success 'reset mixed and checkout orphan' ' diff --git a/t/t1100-commit-tree-options.sh b/t/t1100-commit-tree-options.sh index ae66ba5bab..0f37a43fd3 100755 --- a/t/t1100-commit-tree-options.sh +++ b/t/t1100-commit-tree-options.sh @@ -12,6 +12,7 @@ Also make sure that command line parser understands the normal "flags first and then non flag arguments" command line. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh cat >expected <<EOF diff --git a/t/t1305-config-include.sh b/t/t1305-config-include.sh index ccbb116c01..5cde79ef8c 100755 --- a/t/t1305-config-include.sh +++ b/t/t1305-config-include.sh @@ -1,6 +1,7 @@ #!/bin/sh test_description='test config file include directives' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # Force setup_explicit_git_dir() to run until the end. This is needed diff --git a/t/t1417-reflog-updateref.sh b/t/t1417-reflog-updateref.sh new file mode 100755 index 0000000000..14f13b57c6 --- /dev/null +++ b/t/t1417-reflog-updateref.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +test_description='git reflog --updateref' + +TEST_PASSES_SANITIZE_LEAK=true +. ./test-lib.sh + +test_expect_success 'setup' ' + git init -b main repo && + ( + cd repo && + + test_commit A && + test_commit B && + test_commit C && + + cp .git/logs/HEAD HEAD.old && + git reset --hard HEAD~ && + cp HEAD.old .git/logs/HEAD + ) +' + +test_reflog_updateref () { + exp=$1 + shift + args="$@" + + test_expect_success REFFILES "get '$exp' with '$args'" ' + test_when_finished "rm -rf copy" && + cp -R repo copy && + + ( + cd copy && + + $args && + git rev-parse $exp >expect && + git rev-parse HEAD >actual && + + test_cmp expect actual + ) + ' +} + +test_reflog_updateref B git reflog delete --updateref HEAD@{0} +test_reflog_updateref B git reflog delete --updateref HEAD@{1} +test_reflog_updateref C git reflog delete --updateref main@{0} +test_reflog_updateref B git reflog delete --updateref main@{1} +test_reflog_updateref B git reflog delete --updateref --rewrite HEAD@{0} +test_reflog_updateref B git reflog delete --updateref --rewrite HEAD@{1} +test_reflog_updateref C git reflog delete --updateref --rewrite main@{0} +test_reflog_updateref B git reflog delete --updateref --rewrite main@{1} +test_reflog_updateref B test_must_fail git reflog expire HEAD@{0} +test_reflog_updateref B test_must_fail git reflog expire HEAD@{1} +test_reflog_updateref B test_must_fail git reflog expire main@{0} +test_reflog_updateref B test_must_fail git reflog expire main@{1} +test_reflog_updateref B test_must_fail git reflog expire --updateref HEAD@{0} +test_reflog_updateref B test_must_fail git reflog expire --updateref HEAD@{1} +test_reflog_updateref B test_must_fail git reflog expire --updateref main@{0} +test_reflog_updateref B test_must_fail git reflog expire --updateref main@{1} +test_reflog_updateref B test_must_fail git reflog expire --updateref --rewrite HEAD@{0} +test_reflog_updateref B test_must_fail git reflog expire --updateref --rewrite HEAD@{1} +test_reflog_updateref B test_must_fail git reflog expire --updateref --rewrite main@{0} +test_reflog_updateref B test_must_fail git reflog expire --updateref --rewrite main@{1} + +test_done diff --git a/t/t1430-bad-ref-name.sh b/t/t1430-bad-ref-name.sh index fa3aeb80f2..4c77cf89a6 100755 --- a/t/t1430-bad-ref-name.sh +++ b/t/t1430-bad-ref-name.sh @@ -4,6 +4,7 @@ test_description='Test handling of ref names that check-ref-format rejects' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t1450-fsck.sh b/t/t1450-fsck.sh index 5071ac63a5..6337236fd8 100755 --- a/t/t1450-fsck.sh +++ b/t/t1450-fsck.sh @@ -48,24 +48,70 @@ remove_object () { rm "$(sha1_file "$1")" } -test_expect_success 'object with bad sha1' ' - sha=$(echo blob | git hash-object -w --stdin) && - old=$(test_oid_to_path "$sha") && - new=$(dirname $old)/$(test_oid ff_2) && - sha="$(dirname $new)$(basename $new)" && - mv .git/objects/$old .git/objects/$new && - test_when_finished "remove_object $sha" && - git update-index --add --cacheinfo 100644 $sha foo && - test_when_finished "git read-tree -u --reset HEAD" && - tree=$(git write-tree) && - test_when_finished "remove_object $tree" && - cmt=$(echo bogus | git commit-tree $tree) && - test_when_finished "remove_object $cmt" && - git update-ref refs/heads/bogus $cmt && - test_when_finished "git update-ref -d refs/heads/bogus" && +test_expect_success 'object with hash mismatch' ' + git init --bare hash-mismatch && + ( + cd hash-mismatch && - test_must_fail git fsck 2>out && - test_i18ngrep "$sha.*corrupt" out + oid=$(echo blob | git hash-object -w --stdin) && + oldoid=$oid && + old=$(test_oid_to_path "$oid") && + new=$(dirname $old)/$(test_oid ff_2) && + oid="$(dirname $new)$(basename $new)" && + + mv objects/$old objects/$new && + git update-index --add --cacheinfo 100644 $oid foo && + tree=$(git write-tree) && + cmt=$(echo bogus | git commit-tree $tree) && + git update-ref refs/heads/bogus $cmt && + + test_must_fail git fsck 2>out && + grep "$oldoid: hash-path mismatch, found at: .*$new" out + ) +' + +test_expect_success 'object with hash and type mismatch' ' + git init --bare hash-type-mismatch && + ( + cd hash-type-mismatch && + + oid=$(echo blob | git hash-object -w --stdin -t garbage --literally) && + oldoid=$oid && + old=$(test_oid_to_path "$oid") && + new=$(dirname $old)/$(test_oid ff_2) && + oid="$(dirname $new)$(basename $new)" && + + mv objects/$old objects/$new && + git update-index --add --cacheinfo 100644 $oid foo && + tree=$(git write-tree) && + cmt=$(echo bogus | git commit-tree $tree) && + git update-ref refs/heads/bogus $cmt && + + + test_must_fail git fsck 2>out && + grep "^error: $oldoid: hash-path mismatch, found at: .*$new" out && + grep "^error: $oldoid: object is of unknown type '"'"'garbage'"'"'" out + ) +' + +test_expect_success POSIXPERM 'zlib corrupt loose object output ' ' + git init --bare corrupt-loose-output && + ( + cd corrupt-loose-output && + oid=$(git hash-object -w --stdin --literally </dev/null) && + oidf=objects/$(test_oid_to_path "$oid") && + chmod 755 $oidf && + echo extra garbage >>$oidf && + + cat >expect.error <<-EOF && + error: garbage at end of loose object '\''$oid'\'' + error: unable to unpack contents of ./$oidf + error: $oid: object corrupt or missing: ./$oidf + EOF + test_must_fail git fsck 2>actual && + grep ^error: actual >error && + test_cmp expect.error error + ) ' test_expect_success 'branch pointing to non-commit' ' @@ -865,4 +911,21 @@ test_expect_success 'detect corrupt index file in fsck' ' test_i18ngrep "bad index file" errors ' +test_expect_success 'fsck error and recovery on invalid object type' ' + git init --bare garbage-type && + ( + cd garbage-type && + + garbage_blob=$(git hash-object --stdin -w -t garbage --literally </dev/null) && + + cat >err.expect <<-\EOF && + fatal: invalid object type + EOF + test_must_fail git fsck >out 2>err && + grep -e "^error" -e "^fatal" err >errors && + test_line_count = 1 errors && + grep "$garbage_blob: object is of unknown type '"'"'garbage'"'"':" err + ) +' + test_done diff --git a/t/t1504-ceiling-dirs.sh b/t/t1504-ceiling-dirs.sh index 3d51615e42..0fafcf9dde 100755 --- a/t/t1504-ceiling-dirs.sh +++ b/t/t1504-ceiling-dirs.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='test GIT_CEILING_DIRECTORIES' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_prefix() { diff --git a/t/t1510-repo-setup.sh b/t/t1510-repo-setup.sh index bbfe05b8e4..591505a39c 100755 --- a/t/t1510-repo-setup.sh +++ b/t/t1510-repo-setup.sh @@ -43,6 +43,7 @@ A few rules for repo setup: # This test heavily relies on the standard error of nested function calls. test_untraceable=UnfortunatelyYes +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh here=$(pwd) diff --git a/t/t1800-hook.sh b/t/t1800-hook.sh new file mode 100755 index 0000000000..29718aa991 --- /dev/null +++ b/t/t1800-hook.sh @@ -0,0 +1,134 @@ +#!/bin/sh + +test_description='git-hook command' + +TEST_PASSES_SANITIZE_LEAK=true +. ./test-lib.sh + +test_expect_success 'git hook usage' ' + test_expect_code 129 git hook && + test_expect_code 129 git hook run && + test_expect_code 129 git hook run -h && + test_expect_code 129 git hook run --unknown 2>err && + grep "unknown option" err +' + +test_expect_success 'git hook run: nonexistent hook' ' + cat >stderr.expect <<-\EOF && + error: cannot find a hook named test-hook + EOF + test_expect_code 1 git hook run test-hook 2>stderr.actual && + test_cmp stderr.expect stderr.actual +' + +test_expect_success 'git hook run: nonexistent hook with --ignore-missing' ' + git hook run --ignore-missing does-not-exist 2>stderr.actual && + test_must_be_empty stderr.actual +' + +test_expect_success 'git hook run: basic' ' + write_script .git/hooks/test-hook <<-EOF && + echo Test hook + EOF + + cat >expect <<-\EOF && + Test hook + EOF + git hook run test-hook 2>actual && + test_cmp expect actual +' + +test_expect_success 'git hook run: stdout and stderr both write to our stderr' ' + write_script .git/hooks/test-hook <<-EOF && + echo >&1 Will end up on stderr + echo >&2 Will end up on stderr + EOF + + cat >stderr.expect <<-\EOF && + Will end up on stderr + Will end up on stderr + EOF + git hook run test-hook >stdout.actual 2>stderr.actual && + test_cmp stderr.expect stderr.actual && + test_must_be_empty stdout.actual +' + +test_expect_success 'git hook run: exit codes are passed along' ' + write_script .git/hooks/test-hook <<-EOF && + exit 1 + EOF + + test_expect_code 1 git hook run test-hook && + + write_script .git/hooks/test-hook <<-EOF && + exit 2 + EOF + + test_expect_code 2 git hook run test-hook && + + write_script .git/hooks/test-hook <<-EOF && + exit 128 + EOF + + test_expect_code 128 git hook run test-hook && + + write_script .git/hooks/test-hook <<-EOF && + exit 129 + EOF + + test_expect_code 129 git hook run test-hook +' + +test_expect_success 'git hook run arg u ments without -- is not allowed' ' + test_expect_code 129 git hook run test-hook arg u ments +' + +test_expect_success 'git hook run -- pass arguments' ' + write_script .git/hooks/test-hook <<-\EOF && + echo $1 + echo $2 + EOF + + cat >expect <<-EOF && + arg + u ments + EOF + + git hook run test-hook -- arg "u ments" 2>actual && + test_cmp expect actual +' + +test_expect_success 'git hook run -- out-of-repo runs excluded' ' + write_script .git/hooks/test-hook <<-EOF && + echo Test hook + EOF + + nongit test_must_fail git hook run test-hook +' + +test_expect_success 'git -c core.hooksPath=<PATH> hook run' ' + mkdir my-hooks && + write_script my-hooks/test-hook <<-\EOF && + echo Hook ran $1 >>actual + EOF + + cat >expect <<-\EOF && + Test hook + Hook ran one + Hook ran two + Hook ran three + Hook ran four + EOF + + # Test various ways of specifying the path. See also + # t1350-config-hooks-path.sh + >actual && + git hook run test-hook -- ignored 2>>actual && + git -c core.hooksPath=my-hooks hook run test-hook -- one 2>>actual && + git -c core.hooksPath=my-hooks/ hook run test-hook -- two 2>>actual && + git -c core.hooksPath="$PWD/my-hooks" hook run test-hook -- three 2>>actual && + git -c core.hooksPath="$PWD/my-hooks/" hook run test-hook -- four 2>>actual && + test_cmp expect actual +' + +test_done diff --git a/t/t2002-checkout-cache-u.sh b/t/t2002-checkout-cache-u.sh index 70361c806e..fc95cf9048 100755 --- a/t/t2002-checkout-cache-u.sh +++ b/t/t2002-checkout-cache-u.sh @@ -8,6 +8,7 @@ test_description='git checkout-index -u test. With -u flag, git checkout-index internally runs the equivalent of git update-index --refresh on the checked out entry.' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success \ diff --git a/t/t2003-checkout-cache-mkdir.sh b/t/t2003-checkout-cache-mkdir.sh index ff163cf675..f0fd441d81 100755 --- a/t/t2003-checkout-cache-mkdir.sh +++ b/t/t2003-checkout-cache-mkdir.sh @@ -10,6 +10,7 @@ also verifies that such leading path may contain symlinks, unlike the GIT controlled paths. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t2004-checkout-cache-temp.sh b/t/t2004-checkout-cache-temp.sh index a9352b08a8..9bb503a975 100755 --- a/t/t2004-checkout-cache-temp.sh +++ b/t/t2004-checkout-cache-temp.sh @@ -8,6 +8,7 @@ test_description='git checkout-index --temp test. With --temp flag, git checkout-index writes to temporary merge files rather than the tracked path.' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t2005-checkout-index-symlinks.sh b/t/t2005-checkout-index-symlinks.sh index 9fa5610474..112682a45a 100755 --- a/t/t2005-checkout-index-symlinks.sh +++ b/t/t2005-checkout-index-symlinks.sh @@ -8,6 +8,7 @@ test_description='git checkout-index on filesystem w/o symlinks test. This tests that git checkout-index creates a symbolic link as a plain file if core.symlinks is false.' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success \ diff --git a/t/t2017-checkout-orphan.sh b/t/t2017-checkout-orphan.sh index 88d6992a5e..31fb64c5be 100755 --- a/t/t2017-checkout-orphan.sh +++ b/t/t2017-checkout-orphan.sh @@ -64,6 +64,13 @@ test_expect_success '--orphan ignores branch.autosetupmerge' ' git checkout --orphan gamma && test -z "$(git config branch.gamma.merge)" && test refs/heads/gamma = "$(git symbolic-ref HEAD)" && + test_must_fail git rev-parse --verify HEAD^ && + git checkout main && + git config branch.autosetupmerge inherit && + git checkout --orphan eta && + test -z "$(git config branch.eta.merge)" && + test -z "$(git config branch.eta.remote)" && + test refs/heads/eta = "$(git symbolic-ref HEAD)" && test_must_fail git rev-parse --verify HEAD^ ' diff --git a/t/t2027-checkout-track.sh b/t/t2027-checkout-track.sh index 4453741b96..e87d77f629 100755 --- a/t/t2027-checkout-track.sh +++ b/t/t2027-checkout-track.sh @@ -24,4 +24,27 @@ test_expect_success 'checkout --track -b rejects an extra path argument' ' test_i18ngrep "cannot be used with updating paths" err ' +test_expect_success 'checkout --track -b overrides autoSetupMerge=inherit' ' + # Set up tracking config on main + git config branch.main.remote origin && + git config branch.main.merge refs/heads/main && + test_config branch.autoSetupMerge inherit && + # With --track=inherit, we copy the tracking config from main + git checkout --track=inherit -b b1 main && + test_cmp_config origin branch.b1.remote && + test_cmp_config refs/heads/main branch.b1.merge && + # With branch.autoSetupMerge=inherit, we do the same + git checkout -b b2 main && + test_cmp_config origin branch.b2.remote && + test_cmp_config refs/heads/main branch.b2.merge && + # But --track overrides this + git checkout --track -b b3 main && + test_cmp_config . branch.b3.remote && + test_cmp_config refs/heads/main branch.b3.merge && + # And --track=direct does as well + git checkout --track=direct -b b4 main && + test_cmp_config . branch.b4.remote && + test_cmp_config refs/heads/main branch.b4.merge +' + test_done diff --git a/t/t2050-git-dir-relative.sh b/t/t2050-git-dir-relative.sh index 21f4659a9d..1f193cde96 100755 --- a/t/t2050-git-dir-relative.sh +++ b/t/t2050-git-dir-relative.sh @@ -12,6 +12,7 @@ into the subdir while keeping the worktree location, and tries commits from the top and the subdir, checking that the commit-hook still gets called.' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh COMMIT_FILE="$(pwd)/output" diff --git a/t/t2060-switch.sh b/t/t2060-switch.sh index 9bc6a3aa5c..76e9d12e36 100755 --- a/t/t2060-switch.sh +++ b/t/t2060-switch.sh @@ -107,4 +107,32 @@ test_expect_success 'not switching when something is in progress' ' test_must_fail git switch -d @^ ' +test_expect_success 'tracking info copied with autoSetupMerge=inherit' ' + # default config does not copy tracking info + git switch -c foo-no-inherit foo && + test -z "$(git config branch.foo-no-inherit.remote)" && + test -z "$(git config branch.foo-no-inherit.merge)" && + # with --track=inherit, we copy tracking info from foo + git switch --track=inherit -c foo2 foo && + test_cmp_config origin branch.foo2.remote && + test_cmp_config refs/heads/foo branch.foo2.merge && + # with autoSetupMerge=inherit, we do the same + test_config branch.autoSetupMerge inherit && + git switch -c foo3 foo && + test_cmp_config origin branch.foo3.remote && + test_cmp_config refs/heads/foo branch.foo3.merge && + # with --track, we override autoSetupMerge + git switch --track -c foo4 foo && + test_cmp_config . branch.foo4.remote && + test_cmp_config refs/heads/foo branch.foo4.merge && + # and --track=direct does as well + git switch --track=direct -c foo5 foo && + test_cmp_config . branch.foo5.remote && + test_cmp_config refs/heads/foo branch.foo5.merge && + # no tracking info to inherit from main + git switch -c main2 main && + test -z "$(git config branch.main2.remote)" && + test -z "$(git config branch.main2.merge)" +' + test_done diff --git a/t/t2081-parallel-checkout-collisions.sh b/t/t2081-parallel-checkout-collisions.sh index f6fcfc0c1e..6acdb89d12 100755 --- a/t/t2081-parallel-checkout-collisions.sh +++ b/t/t2081-parallel-checkout-collisions.sh @@ -11,6 +11,7 @@ The tests in this file exercise parallel checkout's collision detection code in both these mechanics. " +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY/lib-parallel-checkout.sh" diff --git a/t/t2082-parallel-checkout-attributes.sh b/t/t2082-parallel-checkout-attributes.sh index 2525457961..822957a8dc 100755 --- a/t/t2082-parallel-checkout-attributes.sh +++ b/t/t2082-parallel-checkout-attributes.sh @@ -20,16 +20,19 @@ test_expect_success 'parallel-checkout with ident' ' ( cd ident && echo "A ident" >.gitattributes && + echo "C ident=MyCusomVeryLongAndWordyId" >>.gitattributes && echo "\$Id\$" >A && echo "\$Id\$" >B && + echo "\$MyCusomVeryLongAndWordyId\$" >C && git add -A && git commit -m id && - rm A B && + rm A B C && test_checkout_workers 2 git reset --hard && hexsz=$(test_oid hexsz) && grep -E "\\\$Id: [0-9a-f]{$hexsz} \\\$" A && - grep "\\\$Id\\\$" B + grep "\\\$Id\\\$" B && + grep -E "\\\$MyCusomVeryLongAndWordyId: [0-9a-f]{$hexsz} \\\$" C ) ' diff --git a/t/t2200-add-update.sh b/t/t2200-add-update.sh index 45ca35d60a..94c4cb0672 100755 --- a/t/t2200-add-update.sh +++ b/t/t2200-add-update.sh @@ -129,12 +129,15 @@ test_expect_success 'add -n -u should not add but just report' ' echo "remove '\''top'\''" ) >expect && before=$(git ls-files -s check top) && + git count-objects -v >objects_before && echo changed >>check && rm -f top && git add -n -u >actual && after=$(git ls-files -s check top) && + git count-objects -v >objects_after && test "$before" = "$after" && + test_cmp objects_before objects_after && test_cmp expect actual ' diff --git a/t/t2300-cd-to-toplevel.sh b/t/t2300-cd-to-toplevel.sh index c8de6d8a19..b40eeb263f 100755 --- a/t/t2300-cd-to-toplevel.sh +++ b/t/t2300-cd-to-toplevel.sh @@ -2,6 +2,7 @@ test_description='cd_to_toplevel' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh EXEC_PATH="$(git --exec-path)" diff --git a/t/t3000-ls-files-others.sh b/t/t3000-ls-files-others.sh index 740ce56eab..11af4552f7 100755 --- a/t/t3000-ls-files-others.sh +++ b/t/t3000-ls-files-others.sh @@ -15,6 +15,8 @@ filesystem. path3/file3 - a file in a directory path4 - an empty directory ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup ' ' diff --git a/t/t3001-ls-files-others-exclude.sh b/t/t3001-ls-files-others-exclude.sh index 516c95ea0e..48cec4e5f8 100755 --- a/t/t3001-ls-files-others-exclude.sh +++ b/t/t3001-ls-files-others-exclude.sh @@ -8,6 +8,7 @@ test_description='git ls-files --others --exclude This test runs git ls-files --others and tests --exclude patterns. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh rm -fr one three @@ -102,7 +103,7 @@ test_expect_success \ >output && test_cmp expect output' -test_expect_success 'restore gitignore' ' +test_expect_success !SANITIZE_LEAK 'restore gitignore' ' git checkout --ignore-skip-worktree-bits $allignores && rm .git/index ' @@ -125,7 +126,7 @@ cat > expect << EOF # three/ EOF -test_expect_success 'git status honors core.excludesfile' \ +test_expect_success !SANITIZE_LEAK 'git status honors core.excludesfile' \ 'test_cmp expect output' test_expect_success 'trailing slash in exclude allows directory match(1)' ' diff --git a/t/t3002-ls-files-dashpath.sh b/t/t3002-ls-files-dashpath.sh index 8704b04e1b..54d22a45df 100755 --- a/t/t3002-ls-files-dashpath.sh +++ b/t/t3002-ls-files-dashpath.sh @@ -12,6 +12,8 @@ filesystem. -foo - a file with a funny name. -- - another file with a funny name. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success \ diff --git a/t/t3003-ls-files-exclude.sh b/t/t3003-ls-files-exclude.sh index c41c4f046a..7933dff9b3 100755 --- a/t/t3003-ls-files-exclude.sh +++ b/t/t3003-ls-files-exclude.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='ls-files --exclude does not affect index files' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'create repo with file' ' diff --git a/t/t3004-ls-files-basic.sh b/t/t3004-ls-files-basic.sh index 9fd5a1f188..a16e25c79b 100755 --- a/t/t3004-ls-files-basic.sh +++ b/t/t3004-ls-files-basic.sh @@ -6,6 +6,7 @@ This test runs git ls-files with various unusual or malformed command-line arguments. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'ls-files in empty repository' ' diff --git a/t/t3005-ls-files-relative.sh b/t/t3005-ls-files-relative.sh index 727e9ae1a4..6ba8b589cd 100755 --- a/t/t3005-ls-files-relative.sh +++ b/t/t3005-ls-files-relative.sh @@ -5,6 +5,7 @@ test_description='ls-files tests with relative paths This test runs git ls-files with various relative path arguments. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'prepare' ' diff --git a/t/t3006-ls-files-long.sh b/t/t3006-ls-files-long.sh index e109c3fbfb..2aaf91ebc8 100755 --- a/t/t3006-ls-files-long.sh +++ b/t/t3006-ls-files-long.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='overly long paths' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t3008-ls-files-lazy-init-name-hash.sh b/t/t3008-ls-files-lazy-init-name-hash.sh index 85f3704958..51d3dffaa6 100755 --- a/t/t3008-ls-files-lazy-init-name-hash.sh +++ b/t/t3008-ls-files-lazy-init-name-hash.sh @@ -2,6 +2,7 @@ test_description='Test the lazy init name hash with various folder structures' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh if test 1 -eq $(test-tool online-cpus) diff --git a/t/t3020-ls-files-error-unmatch.sh b/t/t3020-ls-files-error-unmatch.sh index 124e73b8e6..2cbcbc0721 100755 --- a/t/t3020-ls-files-error-unmatch.sh +++ b/t/t3020-ls-files-error-unmatch.sh @@ -9,6 +9,8 @@ This test runs git ls-files --error-unmatch to ensure it correctly returns an error when a non-existent path is provided on the command line. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t3070-wildmatch.sh b/t/t3070-wildmatch.sh index 56ea4bda13..72d5b014d8 100755 --- a/t/t3070-wildmatch.sh +++ b/t/t3070-wildmatch.sh @@ -2,6 +2,7 @@ test_description='wildmatch tests' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # Disable expensive chain-lint tests; all of the tests in this script diff --git a/t/t3100-ls-tree-restrict.sh b/t/t3100-ls-tree-restrict.sh index 18baf49a49..436de44971 100755 --- a/t/t3100-ls-tree-restrict.sh +++ b/t/t3100-ls-tree-restrict.sh @@ -16,6 +16,8 @@ This test runs git ls-tree with the following in a tree. The new path restriction code should do the right thing for path2 and path2/baz. Also path0/ should snow nothing. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success \ diff --git a/t/t3101-ls-tree-dirname.sh b/t/t3101-ls-tree-dirname.sh index 12bf31022a..05fde64225 100755 --- a/t/t3101-ls-tree-dirname.sh +++ b/t/t3101-ls-tree-dirname.sh @@ -19,6 +19,8 @@ This test runs git ls-tree with the following in a tree. Test the handling of multiple directories which have matching file entries. Also test odd filename and missing entries handling. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t3102-ls-tree-wildcards.sh b/t/t3102-ls-tree-wildcards.sh index 1e16c6b8ea..3942db2290 100755 --- a/t/t3102-ls-tree-wildcards.sh +++ b/t/t3102-ls-tree-wildcards.sh @@ -2,6 +2,7 @@ test_description='ls-tree with(out) globs' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t3103-ls-tree-misc.sh b/t/t3103-ls-tree-misc.sh index 14520913af..d18ba1bd84 100755 --- a/t/t3103-ls-tree-misc.sh +++ b/t/t3103-ls-tree-misc.sh @@ -7,6 +7,7 @@ Miscellaneous tests for git ls-tree. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t3200-branch.sh b/t/t3200-branch.sh index e575ffb4ff..267a624671 100755 --- a/t/t3200-branch.sh +++ b/t/t3200-branch.sh @@ -731,6 +731,28 @@ test_expect_success SYMLINKS 'git branch -m u v should fail when the reflog for test_must_fail git branch -m u v ' +test_expect_success SYMLINKS 'git branch -m with symlinked .git/refs' ' + test_when_finished "rm -rf subdir" && + git init --bare subdir && + + rm -rfv subdir/refs subdir/objects subdir/packed-refs && + ln -s ../.git/refs subdir/refs && + ln -s ../.git/objects subdir/objects && + ln -s ../.git/packed-refs subdir/packed-refs && + + git -C subdir rev-parse --absolute-git-dir >subdir.dir && + git rev-parse --absolute-git-dir >our.dir && + ! test_cmp subdir.dir our.dir && + + git -C subdir log && + git -C subdir branch rename-src && + git rev-parse rename-src >expect && + git -C subdir branch -m rename-src rename-dest && + git rev-parse rename-dest >actual && + test_cmp expect actual && + git branch -D rename-dest +' + test_expect_success 'test tracking setup via --track' ' git config remote.local.url . && git config remote.local.fetch refs/heads/*:refs/remotes/local/* && @@ -1418,8 +1440,51 @@ test_expect_success 'invalid sort parameter in configuration' ' ( cd sort && git config branch.sort "v:notvalid" && - test_must_fail git branch + + # this works in the "listing" mode, so bad sort key + # is a dying offence. + test_must_fail git branch && + + # these do not need to use sorting, and should all + # succeed + git branch newone main && + git branch -c newone newerone && + git branch -m newone newestone && + git branch -d newerone newestone ) ' +test_expect_success 'tracking info copied with --track=inherit' ' + git branch --track=inherit foo2 my1 && + test_cmp_config local branch.foo2.remote && + test_cmp_config refs/heads/main branch.foo2.merge +' + +test_expect_success 'tracking info copied with autoSetupMerge=inherit' ' + test_unconfig branch.autoSetupMerge && + # default config does not copy tracking info + git branch foo-no-inherit my1 && + test -z "$(git config branch.foo-no-inherit.remote)" && + test -z "$(git config branch.foo-no-inherit.merge)" && + # with autoSetupMerge=inherit, we copy tracking info from my1 + test_config branch.autoSetupMerge inherit && + git branch foo3 my1 && + test_cmp_config local branch.foo3.remote && + test_cmp_config refs/heads/main branch.foo3.merge && + # no tracking info to inherit from main + git branch main2 main && + test -z "$(git config branch.main2.remote)" && + test -z "$(git config branch.main2.merge)" +' + +test_expect_success '--track overrides branch.autoSetupMerge' ' + test_config branch.autoSetupMerge inherit && + git branch --track=direct foo4 my1 && + test_cmp_config . branch.foo4.remote && + test_cmp_config refs/heads/my1 branch.foo4.merge && + git branch --no-track foo5 my1 && + test -z "$(git config branch.foo5.remote)" && + test -z "$(git config branch.foo5.merge)" +' + test_done diff --git a/t/t3205-branch-color.sh b/t/t3205-branch-color.sh index 08bd906173..6a521c1a3e 100755 --- a/t/t3205-branch-color.sh +++ b/t/t3205-branch-color.sh @@ -4,6 +4,7 @@ test_description='basic branch output coloring' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'set up some sample branches' ' diff --git a/t/t3211-peel-ref.sh b/t/t3211-peel-ref.sh index 37b9d26f4b..9cbc34fc58 100755 --- a/t/t3211-peel-ref.sh +++ b/t/t3211-peel-ref.sh @@ -4,6 +4,7 @@ test_description='tests for the peel_ref optimization of packed-refs' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'create annotated tag in refs/tags' ' diff --git a/t/t3300-funny-names.sh b/t/t3300-funny-names.sh index f5bf16abcd..d3ac826283 100755 --- a/t/t3300-funny-names.sh +++ b/t/t3300-funny-names.sh @@ -9,6 +9,7 @@ This test tries pathnames with funny characters in the working tree, index, and tree objects. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh HT=' ' diff --git a/t/t3406-rebase-message.sh b/t/t3406-rebase-message.sh index 77a313f62e..d17b450e81 100755 --- a/t/t3406-rebase-message.sh +++ b/t/t3406-rebase-message.sh @@ -105,6 +105,29 @@ test_expect_success 'GIT_REFLOG_ACTION' ' test_cmp expect actual ' +test_expect_success 'rebase --apply reflog' ' + git checkout -b reflog-apply start && + old_head_reflog="$(git log -g --format=%gs -1 HEAD)" && + + git rebase --apply Y && + + git log -g --format=%gs -4 HEAD >actual && + cat >expect <<-EOF && + rebase finished: returning to refs/heads/reflog-apply + rebase: Z + rebase: checkout Y + $old_head_reflog + EOF + test_cmp expect actual && + + git log -g --format=%gs -2 reflog-apply >actual && + cat >expect <<-EOF && + rebase finished: refs/heads/reflog-apply onto $(git rev-parse Y) + branch: Created from start + EOF + test_cmp expect actual +' + test_expect_success 'rebase -i onto unrelated history' ' git init unrelated && test_commit -C unrelated 1 && diff --git a/t/t3418-rebase-continue.sh b/t/t3418-rebase-continue.sh index 22eca73aa3..e67a310ad5 100755 --- a/t/t3418-rebase-continue.sh +++ b/t/t3418-rebase-continue.sh @@ -308,4 +308,30 @@ test_expect_success 'there is no --no-reschedule-failed-exec in an ongoing rebas test_expect_code 129 git rebase --edit-todo --no-reschedule-failed-exec ' +test_orig_head_helper() { + test_when_finished 'git rebase --abort && + git checkout topic && + git reset --hard commit-new-file-F2-on-topic-branch' && + git update-ref -d ORIG_HEAD && + test_must_fail git rebase "$@" && + test_cmp_rev ORIG_HEAD commit-new-file-F2-on-topic-branch +} + +test_orig_head() { + type=$1 + test_expect_success "rebase $type sets ORIG_HEAD correctly" ' + git checkout topic && + git reset --hard commit-new-file-F2-on-topic-branch && + test_orig_head_helper $type main + ' + + test_expect_success "rebase $type <upstream> <branch> sets ORIG_HEAD correctly" ' + git checkout main && + test_orig_head_helper $type main topic + ' +} + +test_orig_head --apply +test_orig_head --merge + test_done diff --git a/t/t3601-rm-pathspec-file.sh b/t/t3601-rm-pathspec-file.sh index 7de21f8bcf..b2a8db69af 100755 --- a/t/t3601-rm-pathspec-file.sh +++ b/t/t3601-rm-pathspec-file.sh @@ -2,6 +2,7 @@ test_description='rm --pathspec-from-file' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_tick diff --git a/t/t3602-rm-sparse-checkout.sh b/t/t3602-rm-sparse-checkout.sh index ecce497a9c..034ec01091 100755 --- a/t/t3602-rm-sparse-checkout.sh +++ b/t/t3602-rm-sparse-checkout.sh @@ -40,14 +40,20 @@ done test_expect_success 'recursive rm does not remove sparse entries' ' git reset --hard && git sparse-checkout set sub/dir && - test_must_fail git rm -r sub && - git rm --sparse -r sub && + git rm -r sub && git status --porcelain -uno >actual && cat >expected <<-\EOF && + D sub/dir/e + EOF + test_cmp expected actual && + + git rm --sparse -r sub && + git status --porcelain -uno >actual2 && + cat >expected2 <<-\EOF && D sub/d D sub/dir/e EOF - test_cmp expected actual + test_cmp expected2 actual2 ' test_expect_success 'recursive rm --sparse removes sparse entries' ' @@ -105,4 +111,29 @@ test_expect_success 'refuse to rm a non-skip-worktree path outside sparse cone' test_path_is_missing b ' +test_expect_success 'can remove files from non-sparse dir' ' + git reset --hard && + git sparse-checkout disable && + mkdir -p w x/y && + test_commit w/f && + test_commit x/y/f && + + git sparse-checkout set w !/x y/ && + git rm w/f.t x/y/f.t 2>stderr && + test_must_be_empty stderr +' + +test_expect_success 'refuse to remove non-skip-worktree file from sparse dir' ' + git reset --hard && + git sparse-checkout disable && + mkdir -p x/y/z && + test_commit x/y/z/f && + git sparse-checkout set !/x y/ !x/y/z && + + git update-index --no-skip-worktree x/y/z/f.t && + test_must_fail git rm x/y/z/f.t 2>stderr && + echo x/y/z/f.t | cat sparse_error_header - sparse_hint >expect && + test_cmp expect stderr +' + test_done diff --git a/t/t3700-add.sh b/t/t3700-add.sh index 4086e1ebbc..aaecefda15 100755 --- a/t/t3700-add.sh +++ b/t/t3700-add.sh @@ -5,8 +5,11 @@ test_description='Test of git add, including the -- option.' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh +. $TEST_DIRECTORY/lib-unique-files.sh + # Test the file mode "$1" of the file "$2" in the index. test_mode_in_index () { case "$(git ls-files -s "$2")" in @@ -33,6 +36,24 @@ test_expect_success \ 'Test that "git add -- -q" works' \ 'touch -- -q && git add -- -q' +test_expect_success 'git add: core.fsyncobjectfiles=batch' " + test_create_unique_files 2 4 fsync-files && + git -c core.fsyncobjectfiles=batch add -- ./fsync-files/ && + rm -f fsynced_files && + git ls-files --stage fsync-files/ > fsynced_files && + test_line_count = 8 fsynced_files && + awk -- '{print \$2}' fsynced_files | xargs -n1 git cat-file -e +" + +test_expect_success 'git update-index: core.fsyncobjectfiles=batch' " + test_create_unique_files 2 4 fsync-files2 && + find fsync-files2 ! -type d -print | xargs git -c core.fsyncobjectfiles=batch update-index --add -- && + rm -f fsynced_files2 && + git ls-files --stage fsync-files2/ > fsynced_files2 && + test_line_count = 8 fsynced_files2 && + awk -- '{print \$2}' fsynced_files2 | xargs -n1 git cat-file -e +" + test_expect_success \ 'git add: Test that executable bit is not used if core.filemode=0' \ 'git config core.filemode 0 && diff --git a/t/t3705-add-sparse-checkout.sh b/t/t3705-add-sparse-checkout.sh index 5b904988d4..f3143c9290 100755 --- a/t/t3705-add-sparse-checkout.sh +++ b/t/t3705-add-sparse-checkout.sh @@ -214,4 +214,21 @@ test_expect_success 'add allows sparse entries with --sparse' ' test_must_be_empty stderr ' +test_expect_success 'can add files from non-sparse dir' ' + git sparse-checkout set w !/x y/ && + mkdir -p w x/y && + touch w/f x/y/f && + git add w/f x/y/f 2>stderr && + test_must_be_empty stderr +' + +test_expect_success 'refuse to add non-skip-worktree file from sparse dir' ' + git sparse-checkout set !/x y/ !x/y/z && + mkdir -p x/y/z && + touch x/y/z/f && + test_must_fail git add x/y/z/f 2>stderr && + echo x/y/z/f | cat sparse_error_header - sparse_hint >expect && + test_cmp expect stderr +' + test_done diff --git a/t/t3902-quoted.sh b/t/t3902-quoted.sh index f528008c36..72a5a565e9 100755 --- a/t/t3902-quoted.sh +++ b/t/t3902-quoted.sh @@ -5,6 +5,7 @@ test_description='quoted output' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh FN='濱野' diff --git a/t/t3903-stash.sh b/t/t3903-stash.sh index f0a82be9de..59fd5057a7 100755 --- a/t/t3903-stash.sh +++ b/t/t3903-stash.sh @@ -9,6 +9,7 @@ GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh +. $TEST_DIRECTORY/lib-unique-files.sh diff_cmp () { for i in "$1" "$2" @@ -288,6 +289,17 @@ test_expect_success 'stash --no-keep-index' ' test bar,bar2 = $(cat file),$(cat file2) ' +test_expect_success 'stash --staged' ' + echo bar3 >file && + echo bar4 >file2 && + git add file2 && + git stash --staged && + test bar3,bar2 = $(cat file),$(cat file2) && + git reset --hard && + git stash pop && + test bar,bar4 = $(cat file),$(cat file2) +' + test_expect_success 'dont assume push with non-option args' ' test_must_fail git stash -q drop 2>err && test_i18ngrep -e "subcommand wasn'\''t specified; '\''push'\'' can'\''t be assumed due to unexpected token '\''drop'\''" err @@ -1293,6 +1305,19 @@ test_expect_success 'stash handles skip-worktree entries nicely' ' git rev-parse --verify refs/stash:A.t ' +test_expect_success 'stash with core.fsyncobjectfiles=batch' " + test_create_unique_files 2 4 fsync-files && + git -c core.fsyncobjectfiles=batch stash push -u -- ./fsync-files/ && + rm -f fsynced_files && + + # The files were untracked, so use the third parent, + # which contains the untracked files + git ls-tree -r stash^3 -- ./fsync-files/ > fsynced_files && + test_line_count = 8 fsynced_files && + awk -- '{print \$3}' fsynced_files | xargs -n1 git cat-file -e +" + + test_expect_success 'stash -c stash.useBuiltin=false warning ' ' expected="stash.useBuiltin support has been removed" && diff --git a/t/t4002-diff-basic.sh b/t/t4002-diff-basic.sh index 6a9f010197..ea52e5b91b 100755 --- a/t/t4002-diff-basic.sh +++ b/t/t4002-diff-basic.sh @@ -6,6 +6,8 @@ test_description='Test diff raw-output. ' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-read-tree-m-3way.sh diff --git a/t/t4003-diff-rename-1.sh b/t/t4003-diff-rename-1.sh index db07ff3eb1..f4485a87c6 100755 --- a/t/t4003-diff-rename-1.sh +++ b/t/t4003-diff-rename-1.sh @@ -11,7 +11,7 @@ test_description='More rename detection test_expect_success \ 'prepare reference tree' \ - 'cat "$TEST_DIRECTORY"/lib-diff/COPYING >COPYING && + 'COPYING_test_data >COPYING && echo frotz >rezrov && git update-index --add COPYING rezrov && tree=$(git write-tree) && @@ -99,7 +99,7 @@ test_expect_success \ test_expect_success \ 'prepare work tree once again' \ - 'cat "$TEST_DIRECTORY"/lib-diff/COPYING >COPYING && + 'COPYING_test_data >COPYING && git update-index --add --remove COPYING COPYING.1' # tree has COPYING and rezrov. work tree has COPYING and COPYING.1, diff --git a/t/t4005-diff-rename-2.sh b/t/t4005-diff-rename-2.sh index 8647906132..6f1b323f97 100755 --- a/t/t4005-diff-rename-2.sh +++ b/t/t4005-diff-rename-2.sh @@ -9,7 +9,7 @@ test_description='Same rename detection as t4003 but testing diff-raw.' . "$TEST_DIRECTORY"/lib-diff.sh ;# test-lib chdir's into trash test_expect_success 'setup reference tree' ' - cat "$TEST_DIRECTORY"/lib-diff/COPYING >COPYING && + COPYING_test_data >COPYING && echo frotz >rezrov && git update-index --add COPYING rezrov && tree=$(git write-tree) && @@ -64,7 +64,7 @@ test_expect_success 'validate output from rename/copy detection (#2)' ' # nows how to say Copy. test_expect_success 'validate output from rename/copy detection (#3)' ' - cat "$TEST_DIRECTORY"/lib-diff/COPYING >COPYING && + COPYING_test_data >COPYING && git update-index --add --remove COPYING COPYING.1 && cat <<-EOF >expected && diff --git a/t/t4007-rename-3.sh b/t/t4007-rename-3.sh index cbb9c62f53..c634653b5b 100755 --- a/t/t4007-rename-3.sh +++ b/t/t4007-rename-3.sh @@ -11,13 +11,12 @@ test_description='Rename interaction with pathspec. test_expect_success 'prepare reference tree' ' mkdir path0 path1 && - cp "$TEST_DIRECTORY"/lib-diff/COPYING path0/COPYING && + COPYING_test_data >path0/COPYING && git update-index --add path0/COPYING && tree=$(git write-tree) && - echo $tree + blob=$(git rev-parse :path0/COPYING) ' -blob=$(git hash-object "$TEST_DIRECTORY/lib-diff/COPYING") test_expect_success 'prepare work tree' ' cp path0/COPYING path1/COPYING && git update-index --add --remove path0/COPYING path1/COPYING diff --git a/t/t4008-diff-break-rewrite.sh b/t/t4008-diff-break-rewrite.sh index 2299f27511..562aaf3a2a 100755 --- a/t/t4008-diff-break-rewrite.sh +++ b/t/t4008-diff-break-rewrite.sh @@ -25,8 +25,8 @@ Further, with -B and -M together, these should turn into two renames. . "$TEST_DIRECTORY"/lib-diff.sh ;# test-lib chdir's into trash test_expect_success setup ' - cat "$TEST_DIRECTORY"/lib-diff/README >file0 && - cat "$TEST_DIRECTORY"/lib-diff/COPYING >file1 && + echo some dissimilar content >file0 && + COPYING_test_data >file1 && blob0_id=$(git hash-object file0) && blob1_id=$(git hash-object file1) && git update-index --add file0 file1 && diff --git a/t/t4009-diff-rename-4.sh b/t/t4009-diff-rename-4.sh index b1da807f16..59b7f44f05 100755 --- a/t/t4009-diff-rename-4.sh +++ b/t/t4009-diff-rename-4.sh @@ -11,7 +11,7 @@ test_description='Same rename detection as t4003 but testing diff-raw -z. test_expect_success \ 'prepare reference tree' \ - 'cat "$TEST_DIRECTORY"/lib-diff/COPYING >COPYING && + 'COPYING_test_data >COPYING && echo frotz >rezrov && git update-index --add COPYING rezrov && orig=$(git hash-object COPYING) && @@ -81,7 +81,7 @@ test_expect_success \ test_expect_success \ 'prepare work tree once again' \ - 'cat "$TEST_DIRECTORY"/lib-diff/COPYING >COPYING && + 'COPYING_test_data >COPYING && git update-index --add --remove COPYING COPYING.1' git diff-index -z -C --find-copies-harder $tree >current diff --git a/t/t4015-diff-whitespace.sh b/t/t4015-diff-whitespace.sh index 2c13b62d3c..50d0cf486b 100755 --- a/t/t4015-diff-whitespace.sh +++ b/t/t4015-diff-whitespace.sh @@ -1442,6 +1442,143 @@ test_expect_success 'detect permutations inside moved code -- dimmed-zebra' ' test_cmp expected actual ' +test_expect_success 'zebra alternate color is only used when necessary' ' + cat >old.txt <<-\EOF && + line 1A should be marked as oldMoved newMovedAlternate + line 1B should be marked as oldMoved newMovedAlternate + unchanged + line 2A should be marked as oldMoved newMovedAlternate + line 2B should be marked as oldMoved newMovedAlternate + line 3A should be marked as oldMovedAlternate newMoved + line 3B should be marked as oldMovedAlternate newMoved + unchanged + line 4A should be marked as oldMoved newMovedAlternate + line 4B should be marked as oldMoved newMovedAlternate + line 5A should be marked as oldMovedAlternate newMoved + line 5B should be marked as oldMovedAlternate newMoved + line 6A should be marked as oldMoved newMoved + line 6B should be marked as oldMoved newMoved + EOF + cat >new.txt <<-\EOF && + line 1A should be marked as oldMoved newMovedAlternate + line 1B should be marked as oldMoved newMovedAlternate + unchanged + line 3A should be marked as oldMovedAlternate newMoved + line 3B should be marked as oldMovedAlternate newMoved + line 2A should be marked as oldMoved newMovedAlternate + line 2B should be marked as oldMoved newMovedAlternate + unchanged + line 6A should be marked as oldMoved newMoved + line 6B should be marked as oldMoved newMoved + line 4A should be marked as oldMoved newMovedAlternate + line 4B should be marked as oldMoved newMovedAlternate + line 5A should be marked as oldMovedAlternate newMoved + line 5B should be marked as oldMovedAlternate newMoved + EOF + test_expect_code 1 git diff --no-index --color --color-moved=zebra \ + --color-moved-ws=allow-indentation-change \ + old.txt new.txt >output && + grep -v index output | test_decode_color >actual && + cat >expected <<-\EOF && + <BOLD>diff --git a/old.txt b/new.txt<RESET> + <BOLD>--- a/old.txt<RESET> + <BOLD>+++ b/new.txt<RESET> + <CYAN>@@ -1,14 +1,14 @@<RESET> + <BOLD;MAGENTA>-line 1A should be marked as oldMoved newMovedAlternate<RESET> + <BOLD;MAGENTA>-line 1B should be marked as oldMoved newMovedAlternate<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN> line 1A should be marked as oldMoved newMovedAlternate<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN> line 1B should be marked as oldMoved newMovedAlternate<RESET> + unchanged<RESET> + <BOLD;MAGENTA>-line 2A should be marked as oldMoved newMovedAlternate<RESET> + <BOLD;MAGENTA>-line 2B should be marked as oldMoved newMovedAlternate<RESET> + <BOLD;BLUE>-line 3A should be marked as oldMovedAlternate newMoved<RESET> + <BOLD;BLUE>-line 3B should be marked as oldMovedAlternate newMoved<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN> line 3A should be marked as oldMovedAlternate newMoved<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN> line 3B should be marked as oldMovedAlternate newMoved<RESET> + <BOLD;YELLOW>+<RESET><BOLD;YELLOW> line 2A should be marked as oldMoved newMovedAlternate<RESET> + <BOLD;YELLOW>+<RESET><BOLD;YELLOW> line 2B should be marked as oldMoved newMovedAlternate<RESET> + unchanged<RESET> + <BOLD;MAGENTA>-line 4A should be marked as oldMoved newMovedAlternate<RESET> + <BOLD;MAGENTA>-line 4B should be marked as oldMoved newMovedAlternate<RESET> + <BOLD;BLUE>-line 5A should be marked as oldMovedAlternate newMoved<RESET> + <BOLD;BLUE>-line 5B should be marked as oldMovedAlternate newMoved<RESET> + <BOLD;MAGENTA>-line 6A should be marked as oldMoved newMoved<RESET> + <BOLD;MAGENTA>-line 6B should be marked as oldMoved newMoved<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN> line 6A should be marked as oldMoved newMoved<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN> line 6B should be marked as oldMoved newMoved<RESET> + <BOLD;YELLOW>+<RESET><BOLD;YELLOW> line 4A should be marked as oldMoved newMovedAlternate<RESET> + <BOLD;YELLOW>+<RESET><BOLD;YELLOW> line 4B should be marked as oldMoved newMovedAlternate<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN> line 5A should be marked as oldMovedAlternate newMoved<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN> line 5B should be marked as oldMovedAlternate newMoved<RESET> + EOF + test_cmp expected actual +' + +test_expect_success 'short lines of opposite sign do not get marked as moved' ' + cat >old.txt <<-\EOF && + this line should be marked as moved + unchanged + unchanged + unchanged + unchanged + too short + this line should be marked as oldMoved newMoved + this line should be marked as oldMovedAlternate newMoved + unchanged 1 + unchanged 2 + unchanged 3 + unchanged 4 + this line should be marked as oldMoved newMoved/newMovedAlternate + EOF + cat >new.txt <<-\EOF && + too short + unchanged + unchanged + this line should be marked as moved + too short + unchanged + unchanged + this line should be marked as oldMoved newMoved/newMovedAlternate + unchanged 1 + unchanged 2 + this line should be marked as oldMovedAlternate newMoved + this line should be marked as oldMoved newMoved/newMovedAlternate + unchanged 3 + this line should be marked as oldMoved newMoved + unchanged 4 + EOF + test_expect_code 1 git diff --no-index --color --color-moved=zebra \ + old.txt new.txt >output && cat output && + grep -v index output | test_decode_color >actual && + cat >expect <<-\EOF && + <BOLD>diff --git a/old.txt b/new.txt<RESET> + <BOLD>--- a/old.txt<RESET> + <BOLD>+++ b/new.txt<RESET> + <CYAN>@@ -1,13 +1,15 @@<RESET> + <BOLD;MAGENTA>-this line should be marked as moved<RESET> + <GREEN>+<RESET><GREEN>too short<RESET> + unchanged<RESET> + unchanged<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN>this line should be marked as moved<RESET> + <GREEN>+<RESET><GREEN>too short<RESET> + unchanged<RESET> + unchanged<RESET> + <RED>-too short<RESET> + <BOLD;MAGENTA>-this line should be marked as oldMoved newMoved<RESET> + <BOLD;BLUE>-this line should be marked as oldMovedAlternate newMoved<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN>this line should be marked as oldMoved newMoved/newMovedAlternate<RESET> + unchanged 1<RESET> + unchanged 2<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN>this line should be marked as oldMovedAlternate newMoved<RESET> + <BOLD;YELLOW>+<RESET><BOLD;YELLOW>this line should be marked as oldMoved newMoved/newMovedAlternate<RESET> + unchanged 3<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN>this line should be marked as oldMoved newMoved<RESET> + unchanged 4<RESET> + <BOLD;MAGENTA>-this line should be marked as oldMoved newMoved/newMovedAlternate<RESET> + EOF + test_cmp expect actual +' + test_expect_success 'cmd option assumes configured colored-moved' ' test_config color.diff.oldMoved "magenta" && test_config color.diff.newMoved "cyan" && @@ -1833,6 +1970,52 @@ test_expect_success '--color-moved treats adjacent blocks as separate for MIN_AL test_cmp expected actual ' +test_expect_success '--color-moved rewinds for MIN_ALNUM_COUNT' ' + git reset --hard && + test_write_lines >file \ + A B C one two three four five six seven D E F G H I J && + git add file && + test_write_lines >file \ + one two A B C D E F G H I J two three four five six seven && + git diff --color-moved=zebra -- file && + + git diff --color-moved=zebra --color -- file >actual.raw && + grep -v "index" actual.raw | test_decode_color >actual && + cat >expected <<-\EOF && + <BOLD>diff --git a/file b/file<RESET> + <BOLD>--- a/file<RESET> + <BOLD>+++ b/file<RESET> + <CYAN>@@ -1,13 +1,8 @@<RESET> + <GREEN>+<RESET><GREEN>one<RESET> + <GREEN>+<RESET><GREEN>two<RESET> + A<RESET> + B<RESET> + C<RESET> + <RED>-one<RESET> + <BOLD;MAGENTA>-two<RESET> + <BOLD;MAGENTA>-three<RESET> + <BOLD;MAGENTA>-four<RESET> + <BOLD;MAGENTA>-five<RESET> + <BOLD;MAGENTA>-six<RESET> + <BOLD;MAGENTA>-seven<RESET> + D<RESET> + E<RESET> + F<RESET> + <CYAN>@@ -15,3 +10,9 @@<RESET> <RESET>G<RESET> + H<RESET> + I<RESET> + J<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN>two<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN>three<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN>four<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN>five<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN>six<RESET> + <BOLD;CYAN>+<RESET><BOLD;CYAN>seven<RESET> + EOF + + test_cmp expected actual +' + test_expect_success 'move detection with submodules' ' test_create_repo bananas && echo ripe >bananas/recipe && @@ -2023,10 +2206,10 @@ EMPTY='' test_expect_success 'compare mixed whitespace delta across moved blocks' ' git reset --hard && - tr Q_ "\t " <<-EOF >text.txt && - ${EMPTY} - ____too short without - ${EMPTY} + tr "^|Q_" "\f\v\t " <<-EOF >text.txt && + ^__ + |____too short without + ^ ___being grouped across blank line ${EMPTY} context @@ -2045,7 +2228,7 @@ test_expect_success 'compare mixed whitespace delta across moved blocks' ' git add text.txt && git commit -m "add text.txt" && - tr Q_ "\t " <<-EOF >text.txt && + tr "^|Q_" "\f\v\t " <<-EOF >text.txt && context lines to @@ -2056,7 +2239,7 @@ test_expect_success 'compare mixed whitespace delta across moved blocks' ' ${EMPTY} QQtoo short without ${EMPTY} - Q_______being grouped across blank line + ^Q_______being grouped across blank line ${EMPTY} Q_QThese two lines have had their indentation reduced by four spaces @@ -2068,16 +2251,16 @@ test_expect_success 'compare mixed whitespace delta across moved blocks' ' -c core.whitespace=space-before-tab \ diff --color --color-moved --ws-error-highlight=all \ --color-moved-ws=allow-indentation-change >actual.raw && - grep -v "index" actual.raw | test_decode_color >actual && + grep -v "index" actual.raw | tr "\f\v" "^|" | test_decode_color >actual && cat <<-\EOF >expected && <BOLD>diff --git a/text.txt b/text.txt<RESET> <BOLD>--- a/text.txt<RESET> <BOLD>+++ b/text.txt<RESET> <CYAN>@@ -1,16 +1,16 @@<RESET> - <BOLD;MAGENTA>-<RESET> - <BOLD;MAGENTA>-<RESET><BOLD;MAGENTA> too short without<RESET> - <BOLD;MAGENTA>-<RESET> + <BOLD;MAGENTA>-<RESET><BOLD;MAGENTA>^<RESET><BRED> <RESET> + <BOLD;MAGENTA>-<RESET><BOLD;MAGENTA>| too short without<RESET> + <BOLD;MAGENTA>-<RESET><BOLD;MAGENTA>^<RESET> <BOLD;MAGENTA>-<RESET><BOLD;MAGENTA> being grouped across blank line<RESET> <BOLD;MAGENTA>-<RESET> <RESET>context<RESET> @@ -2097,7 +2280,7 @@ test_expect_success 'compare mixed whitespace delta across moved blocks' ' <BOLD;YELLOW>+<RESET> <BOLD;YELLOW>+<RESET> <BOLD;YELLOW>too short without<RESET> <BOLD;YELLOW>+<RESET> - <BOLD;YELLOW>+<RESET> <BOLD;YELLOW> being grouped across blank line<RESET> + <BOLD;YELLOW>+<RESET><BOLD;YELLOW>^ being grouped across blank line<RESET> <BOLD;YELLOW>+<RESET> <BOLD;CYAN>+<RESET> <BRED> <RESET> <BOLD;CYAN>These two lines have had their<RESET> <BOLD;CYAN>+<RESET><BOLD;CYAN>indentation reduced by four spaces<RESET> diff --git a/t/t4016-diff-quote.sh b/t/t4016-diff-quote.sh index 876271d682..5a8d887683 100755 --- a/t/t4016-diff-quote.sh +++ b/t/t4016-diff-quote.sh @@ -6,6 +6,7 @@ test_description='Quoting paths in diff output. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh P0='pathname' diff --git a/t/t4019-diff-wserror.sh b/t/t4019-diff-wserror.sh index c6135c7548..c68729ac09 100755 --- a/t/t4019-diff-wserror.sh +++ b/t/t4019-diff-wserror.sh @@ -2,6 +2,7 @@ test_description='diff whitespace error detection' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t4022-diff-rewrite.sh b/t/t4022-diff-rewrite.sh index 6d1c3d949c..1c89050a97 100755 --- a/t/t4022-diff-rewrite.sh +++ b/t/t4022-diff-rewrite.sh @@ -3,15 +3,17 @@ test_description='rewrite diff' . ./test-lib.sh +. "$TEST_DIRECTORY"/lib-diff-data.sh test_expect_success setup ' - cat "$TEST_DIRECTORY"/../COPYING >test && + COPYING_test_data >test.data && + cp test.data test && git add test && tr \ "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" \ "nopqrstuvwxyzabcdefghijklmNOPQRSTUVWXYZABCDEFGHIJKLM" \ - <"$TEST_DIRECTORY"/../COPYING >test && + <test.data >test && echo "to be deleted" >test2 && blob=$(git hash-object test2) && blob=$(git rev-parse --short $blob) && diff --git a/t/t4023-diff-rename-typechange.sh b/t/t4023-diff-rename-typechange.sh index 8c9823765e..47d6f35dcc 100755 --- a/t/t4023-diff-rename-typechange.sh +++ b/t/t4023-diff-rename-typechange.sh @@ -3,25 +3,26 @@ test_description='typechange rename detection' . ./test-lib.sh +. "$TEST_DIRECTORY"/lib-diff.sh test_expect_success setup ' rm -f foo bar && - cat "$TEST_DIRECTORY"/../COPYING >foo && + COPYING_test_data >foo && test_ln_s_add linklink bar && git add foo && git commit -a -m Initial && git tag one && git rm -f foo bar && - cat "$TEST_DIRECTORY"/../COPYING >bar && + COPYING_test_data >bar && test_ln_s_add linklink foo && git add bar && git commit -a -m Second && git tag two && git rm -f foo bar && - cat "$TEST_DIRECTORY"/../COPYING >foo && + COPYING_test_data >foo && git add foo && git commit -a -m Third && git tag three && @@ -35,7 +36,7 @@ test_expect_success setup ' # This is purely for sanity check git rm -f foo bar && - cat "$TEST_DIRECTORY"/../COPYING >foo && + COPYING_test_data >foo && cat "$TEST_DIRECTORY"/../Makefile >bar && git add foo bar && git commit -a -m Fifth && @@ -43,7 +44,7 @@ test_expect_success setup ' git rm -f foo bar && cat "$TEST_DIRECTORY"/../Makefile >foo && - cat "$TEST_DIRECTORY"/../COPYING >bar && + COPYING_test_data >bar && git add foo bar && git commit -a -m Sixth && git tag six diff --git a/t/t4025-hunk-header.sh b/t/t4025-hunk-header.sh index 35578f2bb9..6356961de4 100755 --- a/t/t4025-hunk-header.sh +++ b/t/t4025-hunk-header.sh @@ -2,6 +2,7 @@ test_description='diff hunk header truncation' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh N='日本語' diff --git a/t/t4026-color.sh b/t/t4026-color.sh index c0b642c1ab..cc3f60d468 100755 --- a/t/t4026-color.sh +++ b/t/t4026-color.sh @@ -4,6 +4,8 @@ # test_description='Test diff/status color escape codes' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh ESC=$(printf '\033') @@ -58,6 +60,10 @@ test_expect_success 'fg bg attr...' ' color "blue bold dim ul blink reverse" "[1;2;4;5;7;34m" ' +test_expect_success 'reset fg bg attr...' ' + color "reset blue bold dim ul blink reverse" "[;1;2;4;5;7;34m" +' + # note that nobold and nodim are the same code (22) test_expect_success 'attr negation' ' color "nobold nodim noul noblink noreverse" "[22;24;25;27m" @@ -94,6 +100,18 @@ test_expect_success '24-bit colors' ' color "#ff00ff black" "[38;2;255;0;255;40m" ' +test_expect_success '"default" foreground' ' + color "default" "[39m" +' + +test_expect_success '"normal default" to clear background' ' + color "normal default" "[49m" +' + +test_expect_success '"default" can be combined with attributes' ' + color "default default no-reverse bold" "[1;27;39;49m" +' + test_expect_success '"normal" yields no color at all"' ' color "normal black" "[40m" ' diff --git a/t/t4034/cpp/expect b/t/t4034/cpp/expect index 37d1ea2587..dc500ae092 100644 --- a/t/t4034/cpp/expect +++ b/t/t4034/cpp/expect @@ -1,36 +1,35 @@ <BOLD>diff --git a/pre b/post<RESET> -<BOLD>index 23d5c8a..7e8c026 100644<RESET> +<BOLD>index a1a09b7..f1b6f3c 100644<RESET> <BOLD>--- a/pre<RESET> <BOLD>+++ b/post<RESET> -<CYAN>@@ -1,19 +1,19 @@<RESET> -Foo() : x(0<RED>&&1<RESET><GREEN>&42<RESET>) { <GREEN>bar(x);<RESET> } +<CYAN>@@ -1,30 +1,30 @@<RESET> +Foo() : x(0<RED>&&1<RESET><GREEN>&42<RESET>) { <RED>foo0<RESET><GREEN>bar<RESET>(x.<RED>find<RESET><GREEN>Find<RESET>); } cout<<"Hello World<RED>!<RESET><GREEN>?<RESET>\n"<<endl; -<GREEN>(<RESET>1<GREEN>) (<RESET>-1e10<GREEN>) (<RESET>0xabcdef<GREEN>)<RESET> '<RED>x<RESET><GREEN>y<RESET>' -[<RED>a<RESET><GREEN>x<RESET>] <RED>a<RESET><GREEN>x<RESET>-><RED>b a<RESET><GREEN>y x<RESET>.<RED>b<RESET><GREEN>y<RESET> -!<RED>a<RESET><GREEN>x<RESET> ~<RED>a a<RESET><GREEN>x x<RESET>++ <RED>a<RESET><GREEN>x<RESET>-- <RED>a<RESET><GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>&<RED>b<RESET> -<RED>a<RESET><GREEN>y<RESET> -<GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>/<RED>b a<RESET><GREEN>y x<RESET>%<RED>b<RESET> -<RED>a<RESET><GREEN>y<RESET> -<GREEN>x<RESET>+<RED>b a<RESET><GREEN>y x<RESET>-<RED>b<RESET> -<RED>a<RESET><GREEN>y<RESET> -<GREEN>x<RESET><<<RED>b a<RESET><GREEN>y x<RESET>>><RED>b<RESET> -<RED>a<RESET><GREEN>y<RESET> -<GREEN>x<RESET><<RED>b a<RESET><GREEN>y x<RESET><=<RED>b a<RESET><GREEN>y x<RESET>><RED>b a<RESET><GREEN>y x<RESET>>=<RED>b<RESET> -<RED>a<RESET><GREEN>y<RESET> -<GREEN>x<RESET>==<RED>b a<RESET><GREEN>y x<RESET>!=<RED>b<RESET> -<RED>a<RESET><GREEN>y<RESET> -<GREEN>x<RESET>&<RED>b<RESET> -<RED>a<RESET><GREEN>y<RESET> -<GREEN>x<RESET>^<RED>b<RESET> -<RED>a<RESET><GREEN>y<RESET> -<GREEN>x<RESET>|<RED>b<RESET> -<RED>a<RESET><GREEN>y<RESET> -<GREEN>x<RESET>&&<RED>b<RESET> -<RED>a<RESET><GREEN>y<RESET> -<GREEN>x<RESET>||<RED>b<RESET> -<RED>a<RESET><GREEN>y<RESET> -<GREEN>x<RESET>?<RED>b<RESET><GREEN>y<RESET>:z -<RED>a<RESET><GREEN>x<RESET>=<RED>b a<RESET><GREEN>y x<RESET>+=<RED>b a<RESET><GREEN>y x<RESET>-=<RED>b a<RESET><GREEN>y x<RESET>*=<RED>b a<RESET><GREEN>y x<RESET>/=<RED>b a<RESET><GREEN>y x<RESET>%=<RED>b a<RESET><GREEN>y x<RESET><<=<RED>b a<RESET><GREEN>y x<RESET>>>=<RED>b a<RESET><GREEN>y x<RESET>&=<RED>b a<RESET><GREEN>y x<RESET>^=<RED>b a<RESET><GREEN>y x<RESET>|=<RED>b<RESET> -<RED>a<RESET><GREEN>y<RESET> -<GREEN>x<RESET>,y -<RED>a<RESET><GREEN>x<RESET>::<RED>b<RESET><GREEN>y<RESET> +<GREEN>(<RESET>1 <RED>-<RESET><GREEN>+<RESET>1e10 0xabcdef<GREEN>)<RESET> '<RED>x<RESET><GREEN>2<RESET>' +// long double<RESET> +<RED>3.141592653e-10l<RESET><GREEN>3.141592654e+10l<RESET> +// float<RESET> +<RED>120E5f<RESET><GREEN>120E6f<RESET> +// hex<RESET> +<RED>0xdead<RESET><GREEN>0xdeaf<RESET>'1<RED>eaF<RESET><GREEN>eaf<RESET>+<RED>8ULL<RESET><GREEN>7ULL<RESET> +// octal<RESET> +<RED>01234567<RESET><GREEN>01234560<RESET> +// binary<RESET> +<RED>0b1000<RESET><GREEN>0b1100<RESET>+e1 +// expression<RESET> +1.5-e+<RED>2<RESET><GREEN>3<RESET>+f +// another one<RESET> +str.e+<RED>65<RESET><GREEN>75<RESET> +[a] b<RED>-><RESET><GREEN>->*<RESET>v d<RED>.<RESET><GREEN>.*<RESET>e +<GREEN>~<RESET>!a <GREEN>!<RESET>~b c<RED>++<RESET><GREEN>+<RESET> d<RED>--<RESET><GREEN>-<RESET> e*<GREEN>*<RESET>f g<RED>&<RESET><GREEN>&&<RESET>h +a<RED>*<RESET><GREEN>*=<RESET>b c<RED>/<RESET><GREEN>/=<RESET>d e<RED>%<RESET><GREEN>%=<RESET>f +a<RED>+<RESET><GREEN>++<RESET>b c<RED>-<RESET><GREEN>--<RESET>d +a<RED><<<RESET><GREEN><<=<RESET>b c<RED>>><RESET><GREEN>>>=<RESET>d +a<RED><<RESET><GREEN><=<RESET>b c<RED><=<RESET><GREEN><<RESET>d e<RED>><RESET><GREEN>>=<RESET>f g<RED>>=<RESET><GREEN>><RESET>h i<RED><=<RESET><GREEN><=><RESET>j +a<RED>==<RESET><GREEN>!=<RESET>b c<RED>!=<RESET><GREEN>=<RESET>d +a<RED>^<RESET><GREEN>^=<RESET>b c<RED>|<RESET><GREEN>|=<RESET>d e<RED>&&<RESET><GREEN>&=<RESET>f +a<RED>||<RESET><GREEN>|<RESET>b +a?<GREEN>:<RESET>b +a<RED>=<RESET><GREEN>==<RESET>b c<RED>+=<RESET><GREEN>+<RESET>d e<RED>-=<RESET><GREEN>-<RESET>f g<RED>*=<RESET><GREEN>*<RESET>h i<RED>/=<RESET><GREEN>/<RESET>j k<RED>%=<RESET><GREEN>%<RESET>l m<RED><<=<RESET><GREEN><<<RESET>n o<RED>>>=<RESET><GREEN>>><RESET>p q<RED>&=<RESET><GREEN>&<RESET>r s<RED>^=<RESET><GREEN>^<RESET>t u<RED>|=<RESET><GREEN>|<RESET>v +a,b<RESET> +a<RED>::<RESET><GREEN>:<RESET>b diff --git a/t/t4034/cpp/post b/t/t4034/cpp/post index 7e8c026cef..f1b6f3c228 100644 --- a/t/t4034/cpp/post +++ b/t/t4034/cpp/post @@ -1,19 +1,30 @@ -Foo() : x(0&42) { bar(x); } +Foo() : x(0&42) { bar(x.Find); } cout<<"Hello World?\n"<<endl; -(1) (-1e10) (0xabcdef) 'y' -[x] x->y x.y -!x ~x x++ x-- x*y x&y -x*y x/y x%y -x+y x-y -x<<y x>>y -x<y x<=y x>y x>=y -x==y x!=y -x&y -x^y -x|y -x&&y -x||y -x?y:z -x=y x+=y x-=y x*=y x/=y x%=y x<<=y x>>=y x&=y x^=y x|=y -x,y -x::y +(1 +1e10 0xabcdef) '2' +// long double +3.141592654e+10l +// float +120E6f +// hex +0xdeaf'1eaf+7ULL +// octal +01234560 +// binary +0b1100+e1 +// expression +1.5-e+3+f +// another one +str.e+75 +[a] b->*v d.*e +~!a !~b c+ d- e**f g&&h +a*=b c/=d e%=f +a++b c--d +a<<=b c>>=d +a<=b c<d e>=f g>h i<=>j +a!=b c=d +a^=b c|=d e&=f +a|b +a?:b +a==b c+d e-f g*h i/j k%l m<<n o>>p q&r s^t u|v +a,b +a:b diff --git a/t/t4034/cpp/pre b/t/t4034/cpp/pre index 23d5c8adf5..a1a09b7712 100644 --- a/t/t4034/cpp/pre +++ b/t/t4034/cpp/pre @@ -1,19 +1,30 @@ -Foo():x(0&&1){} +Foo():x(0&&1){ foo0( x.find); } cout<<"Hello World!\n"<<endl; 1 -1e10 0xabcdef 'x' -[a] a->b a.b -!a ~a a++ a-- a*b a&b -a*b a/b a%b -a+b a-b -a<<b a>>b -a<b a<=b a>b a>=b -a==b a!=b -a&b -a^b -a|b -a&&b +// long double +3.141592653e-10l +// float +120E5f +// hex +0xdead'1eaF+8ULL +// octal +01234567 +// binary +0b1000+e1 +// expression +1.5-e+2+f +// another one +str.e+65 +[a] b->v d.e +!a ~b c++ d-- e*f g&h +a*b c/d e%f +a+b c-d +a<<b c>>d +a<b c<=d e>f g>=h i<=j +a==b c!=d +a^b c|d e&&f a||b -a?b:z -a=b a+=b a-=b a*=b a/=b a%=b a<<=b a>>=b a&=b a^=b a|=b -a,y +a?b +a=b c+=d e-=f g*=h i/=j k%=l m<<=n o>>=p q&=r s^=t u|=v +a,b a::b diff --git a/t/t4202-log.sh b/t/t4202-log.sh index 9dfead936b..ba855ec893 100755 --- a/t/t4202-log.sh +++ b/t/t4202-log.sh @@ -449,6 +449,57 @@ test_expect_success !FAIL_PREREQS 'log with various grep.patternType configurati ) ' +test_expect_success 'log --author' ' + cat >expect <<-\EOF && + Author: <BOLD;RED>A U<RESET> Thor <author@example.com> + EOF + git log -1 --color=always --author="A U" >log && + grep Author log >actual.raw && + test_decode_color <actual.raw >actual && + test_cmp expect actual +' + +test_expect_success 'log --committer' ' + cat >expect <<-\EOF && + Commit: C O Mitter <committer@<BOLD;RED>example<RESET>.com> + EOF + git log -1 --color=always --pretty=fuller --committer="example" >log && + grep "Commit:" log >actual.raw && + test_decode_color <actual.raw >actual && + test_cmp expect actual +' + +test_expect_success 'log -i --grep with color' ' + cat >expect <<-\EOF && + <BOLD;RED>Sec<RESET>ond + <BOLD;RED>sec<RESET>ond + EOF + git log --color=always -i --grep=^sec >log && + grep -i sec log >actual.raw && + test_decode_color <actual.raw >actual && + test_cmp expect actual +' + +test_expect_success '-c color.grep.selected log --grep' ' + cat >expect <<-\EOF && + <GREEN>th<RESET><BOLD;RED>ir<RESET><GREEN>d<RESET> + EOF + git -c color.grep.selected="green" log --color=always --grep=ir >log && + grep ir log >actual.raw && + test_decode_color <actual.raw >actual && + test_cmp expect actual +' + +test_expect_success '-c color.grep.matchSelected log --grep' ' + cat >expect <<-\EOF && + <BLUE>i<RESET>n<BLUE>i<RESET>t<BLUE>i<RESET>al + EOF + git -c color.grep.matchSelected="blue" log --color=always --grep=i >log && + grep al log >actual.raw && + test_decode_color <actual.raw >actual && + test_cmp expect actual +' + cat > expect <<EOF * Second * sixth @@ -1616,6 +1667,34 @@ test_expect_success GPGSM 'setup signed branch x509' ' git commit -S -m signed_commit ' +test_expect_success GPGSSH 'setup sshkey signed branch' ' + test_config gpg.format ssh && + test_config user.signingkey "${GPGSSH_KEY_PRIMARY}" && + test_when_finished "git reset --hard && git checkout main" && + git checkout -b signed-ssh main && + echo foo >foo && + git add foo && + git commit -S -m signed_commit +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'create signed commits with keys having defined lifetimes' ' + test_config gpg.format ssh && + touch file && + git add file && + + echo expired >file && test_tick && git commit -a -m expired -S"${GPGSSH_KEY_EXPIRED}" && + git tag expired-signed && + + echo notyetvalid >file && test_tick && git commit -a -m notyetvalid -S"${GPGSSH_KEY_NOTYETVALID}" && + git tag notyetvalid-signed && + + echo timeboxedvalid >file && test_tick && git commit -a -m timeboxedvalid -S"${GPGSSH_KEY_TIMEBOXEDVALID}" && + git tag timeboxedvalid-signed && + + echo timeboxedinvalid >file && test_tick && git commit -a -m timeboxedinvalid -S"${GPGSSH_KEY_TIMEBOXEDINVALID}" && + git tag timeboxedinvalid-signed +' + test_expect_success GPGSM 'log x509 fingerprint' ' echo "F8BF62E0693D0694816377099909C779FA23FD65 | " >expect && git log -n1 --format="%GF | %GP" signed-x509 >actual && @@ -1628,6 +1707,13 @@ test_expect_success GPGSM 'log OpenPGP fingerprint' ' test_cmp expect actual ' +test_expect_success GPGSSH 'log ssh key fingerprint' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + ssh-keygen -lf "${GPGSSH_KEY_PRIMARY}" | awk "{print \$2\" | \"}" >expect && + git log -n1 --format="%GF | %GP" signed-ssh >actual && + test_cmp expect actual +' + test_expect_success GPG 'log --graph --show-signature' ' git log --graph --show-signature -n1 signed >actual && grep "^| gpg: Signature made" actual && @@ -1640,6 +1726,37 @@ test_expect_success GPGSM 'log --graph --show-signature x509' ' grep "^| gpgsm: Good signature" actual ' +test_expect_success GPGSSH 'log --graph --show-signature ssh' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git log --graph --show-signature -n1 signed-ssh >actual && + grep "${GOOD_SIGNATURE_TRUSTED}" actual +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'log shows failure on expired signature key' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git log --graph --show-signature -n1 expired-signed >actual && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'log shows failure on not yet valid signature key' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git log --graph --show-signature -n1 notyetvalid-signed >actual && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'log show success with commit date and key validity matching' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git log --graph --show-signature -n1 timeboxedvalid-signed >actual && + grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'log shows failure with commit date outside of key validity' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git log --graph --show-signature -n1 timeboxedinvalid-signed >actual && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual +' + test_expect_success GPG 'log --graph --show-signature for merged tag' ' test_when_finished "git reset --hard && git checkout main" && git checkout -b plain main && diff --git a/t/t4300-merge-tree.sh b/t/t4300-merge-tree.sh index e59601e5fe..c52c8a21fa 100755 --- a/t/t4300-merge-tree.sh +++ b/t/t4300-merge-tree.sh @@ -4,6 +4,8 @@ # test_description='git merge-tree' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t5300-pack-object.sh b/t/t5300-pack-object.sh index e13a884207..38663dc139 100755 --- a/t/t5300-pack-object.sh +++ b/t/t5300-pack-object.sh @@ -162,23 +162,23 @@ test_expect_success 'pack-objects with bogus arguments' ' check_unpack () { test_when_finished "rm -rf git2" && - git init --bare git2 && - git -C git2 unpack-objects -n <"$1".pack && - git -C git2 unpack-objects <"$1".pack && - (cd .git && find objects -type f -print) | - while read path - do - cmp git2/$path .git/$path || { - echo $path differs. - return 1 - } - done + git $2 init --bare git2 && + ( + git $2 -C git2 unpack-objects -n <"$1".pack && + git $2 -C git2 unpack-objects <"$1".pack && + git $2 -C git2 cat-file --batch-check="%(objectname)" + ) <obj-list >current && + cmp obj-list current } test_expect_success 'unpack without delta' ' check_unpack test-1-${packname_1} ' +test_expect_success 'unpack without delta (core.fsyncobjectfiles=batch)' ' + check_unpack test-1-${packname_1} "-c core.fsyncobjectfiles=batch" +' + test_expect_success 'pack with REF_DELTA' ' packname_2=$(git pack-objects --progress test-2 <obj-list 2>stderr) && check_deltas stderr -gt 0 @@ -188,6 +188,10 @@ test_expect_success 'unpack with REF_DELTA' ' check_unpack test-2-${packname_2} ' +test_expect_success 'unpack with REF_DELTA (core.fsyncobjectfiles=batch)' ' + check_unpack test-2-${packname_2} "-c core.fsyncobjectfiles=batch" +' + test_expect_success 'pack with OFS_DELTA' ' packname_3=$(git pack-objects --progress --delta-base-offset test-3 \ <obj-list 2>stderr) && @@ -198,6 +202,10 @@ test_expect_success 'unpack with OFS_DELTA' ' check_unpack test-3-${packname_3} ' +test_expect_success 'unpack with OFS_DELTA (core.fsyncobjectfiles=batch)' ' + check_unpack test-3-${packname_3} "-c core.fsyncobjectfiles=batch" +' + test_expect_success 'compare delta flavors' ' perl -e '\'' defined($_ = -s $_) or die for @ARGV; diff --git a/t/t5310-pack-bitmaps.sh b/t/t5310-pack-bitmaps.sh index 673baa5c3c..dcf03d324a 100755 --- a/t/t5310-pack-bitmaps.sh +++ b/t/t5310-pack-bitmaps.sh @@ -5,7 +5,6 @@ GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh -. "$TEST_DIRECTORY"/lib-bundle.sh . "$TEST_DIRECTORY"/lib-bitmap.sh # t5310 deals only with single-pack bitmaps, so don't write MIDX bitmaps in diff --git a/t/t5318-commit-graph.sh b/t/t5318-commit-graph.sh index 295c5bd94d..f516fda7cc 100755 --- a/t/t5318-commit-graph.sh +++ b/t/t5318-commit-graph.sh @@ -70,8 +70,8 @@ test_expect_success 'create commits and repack' ' ' graph_git_two_modes() { - git -c core.commitGraph=true $1 >output - git -c core.commitGraph=false $1 >expect + git -c core.commitGraph=true $1 >output && + git -c core.commitGraph=false $1 >expect && test_cmp expect output } @@ -385,6 +385,7 @@ test_expect_success 'replace-objects invalidates commit-graph' ' git commit-graph write --reachable && test_path_is_file .git/objects/info/commit-graph && git replace HEAD~1 HEAD~2 && + graph_git_two_modes "commit-graph verify" && git -c core.commitGraph=false log >expect && git -c core.commitGraph=true log >actual && test_cmp expect actual && @@ -693,12 +694,33 @@ test_expect_success 'detect incorrect chunk count' ' $GRAPH_CHUNK_LOOKUP_OFFSET ' -test_expect_success 'git fsck (checks commit-graph)' ' +test_expect_success 'git fsck (checks commit-graph when config set to true)' ' cd "$TRASH_DIRECTORY/full" && git fsck && corrupt_graph_and_verify $GRAPH_BYTE_FOOTER "\00" \ "incorrect checksum" && cp commit-graph-pre-write-test $objdir/info/commit-graph && + test_must_fail git -c core.commitGraph=true fsck +' + +test_expect_success 'git fsck (ignores commit-graph when config set to false)' ' + cd "$TRASH_DIRECTORY/full" && + git fsck && + corrupt_graph_and_verify $GRAPH_BYTE_FOOTER "\00" \ + "incorrect checksum" && + cp commit-graph-pre-write-test $objdir/info/commit-graph && + git -c core.commitGraph=false fsck +' + +test_expect_success 'git fsck (checks commit-graph when config unset)' ' + cd "$TRASH_DIRECTORY/full" && + test_when_finished "git config core.commitGraph true" && + + git fsck && + corrupt_graph_and_verify $GRAPH_BYTE_FOOTER "\00" \ + "incorrect checksum" && + test_unconfig core.commitGraph && + cp commit-graph-pre-write-test $objdir/info/commit-graph && test_must_fail git fsck ' diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index a3c72b68f7..3f69e43178 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -467,7 +467,10 @@ test_expect_success 'verify incorrect offset' ' test_expect_success 'git-fsck incorrect offset' ' corrupt_midx_and_verify $MIDX_BYTE_OFFSET "\377" $objdir \ "incorrect object offset" \ - "git -c core.multipackindex=true fsck" + "git -c core.multiPackIndex=true fsck" && + test_unconfig core.multiPackIndex && + test_must_fail git fsck && + git -c core.multiPackIndex=false fsck ' test_expect_success 'corrupt MIDX is not reused' ' diff --git a/t/t5324-split-commit-graph.sh b/t/t5324-split-commit-graph.sh index 587226ed10..847b809710 100755 --- a/t/t5324-split-commit-graph.sh +++ b/t/t5324-split-commit-graph.sh @@ -55,8 +55,8 @@ test_expect_success 'create commits and write commit-graph' ' ' graph_git_two_modes() { - git -c core.commitGraph=true $1 >output - git -c core.commitGraph=false $1 >expect + git ${2:+ -C "$2"} -c core.commitGraph=true $1 >output && + git ${2:+ -C "$2"} -c core.commitGraph=false $1 >expect && test_cmp expect output } @@ -64,12 +64,13 @@ graph_git_behavior() { MSG=$1 BRANCH=$2 COMPARE=$3 + DIR=$4 test_expect_success "check normal git operations: $MSG" ' - graph_git_two_modes "log --oneline $BRANCH" && - graph_git_two_modes "log --topo-order $BRANCH" && - graph_git_two_modes "log --graph $COMPARE..$BRANCH" && - graph_git_two_modes "branch -vv" && - graph_git_two_modes "merge-base -a $BRANCH $COMPARE" + graph_git_two_modes "log --oneline $BRANCH" "$DIR" && + graph_git_two_modes "log --topo-order $BRANCH" "$DIR" && + graph_git_two_modes "log --graph $COMPARE..$BRANCH" "$DIR" && + graph_git_two_modes "branch -vv" "$DIR" && + graph_git_two_modes "merge-base -a $BRANCH $COMPARE" "$DIR" ' } @@ -187,7 +188,10 @@ test_expect_success 'create fork and chain across alternate' ' ) ' -graph_git_behavior 'alternate: commit 13 vs 6' commits/13 commits/6 +if test -d fork +then + graph_git_behavior 'alternate: commit 13 vs 6' commits/13 origin/commits/6 "fork" +fi test_expect_success 'test merge stragety constants' ' git clone . merge-2 && diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index 8212ca56dc..d4c2f1b68f 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -541,6 +541,15 @@ do done +test_expect_success "push to remote with detached HEAD and config remote.*.push = src:dest" ' + mk_test testrepo heads/main && + git checkout $the_first_commit && + test_config remote.there.url testrepo && + test_config remote.there.push refs/heads/main:refs/heads/main && + git push there && + check_push_result testrepo $the_commit heads/main +' + test_expect_success 'push with remote.pushdefault' ' mk_test up_repo heads/main && mk_test down_repo heads/main && diff --git a/t/t5521-pull-options.sh b/t/t5521-pull-options.sh index 7601c919fd..66cfcb09c5 100755 --- a/t/t5521-pull-options.sh +++ b/t/t5521-pull-options.sh @@ -228,4 +228,28 @@ test_expect_success 'git pull --no-signoff flag cancels --signoff flag' ' test_must_be_empty actual ' +test_expect_success 'git pull --no-verify flag passed to merge' ' + test_when_finished "rm -fr src dst actual" && + git init src && + test_commit -C src one && + git clone src dst && + write_script dst/.git/hooks/commit-msg <<-\EOF && + false + EOF + test_commit -C src two && + git -C dst pull --no-ff --no-verify +' + +test_expect_success 'git pull --no-verify --verify passed to merge' ' + test_when_finished "rm -fr src dst actual" && + git init src && + test_commit -C src one && + git clone src dst && + write_script dst/.git/hooks/commit-msg <<-\EOF && + false + EOF + test_commit -C src two && + test_must_fail git -C dst pull --no-ff --no-verify --verify +' + test_done diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh index ed11569d8d..2dc75b80db 100755 --- a/t/t5526-fetch-submodules.sh +++ b/t/t5526-fetch-submodules.sh @@ -6,6 +6,9 @@ test_description='Recursive "git fetch" for submodules' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB=1 +export GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB + . ./test-lib.sh pwd=$(pwd) diff --git a/t/t5531-deep-submodule-push.sh b/t/t5531-deep-submodule-push.sh index d573ca496a..3f58b515ce 100755 --- a/t/t5531-deep-submodule-push.sh +++ b/t/t5531-deep-submodule-push.sh @@ -5,6 +5,9 @@ test_description='test push with submodules' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB=1 +export GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB + . ./test-lib.sh test_expect_success setup ' diff --git a/t/t5534-push-signed.sh b/t/t5534-push-signed.sh index bba768f5de..24d374adba 100755 --- a/t/t5534-push-signed.sh +++ b/t/t5534-push-signed.sh @@ -137,6 +137,53 @@ test_expect_success GPG 'signed push sends push certificate' ' test_cmp expect dst/push-cert-status ' +test_expect_success GPGSSH 'ssh signed push sends push certificate' ' + prepare_dst && + mkdir -p dst/.git/hooks && + git -C dst config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git -C dst config receive.certnonceseed sekrit && + write_script dst/.git/hooks/post-receive <<-\EOF && + # discard the update list + cat >/dev/null + # record the push certificate + if test -n "${GIT_PUSH_CERT-}" + then + git cat-file blob $GIT_PUSH_CERT >../push-cert + fi && + + cat >../push-cert-status <<E_O_F + SIGNER=${GIT_PUSH_CERT_SIGNER-nobody} + KEY=${GIT_PUSH_CERT_KEY-nokey} + STATUS=${GIT_PUSH_CERT_STATUS-nostatus} + NONCE_STATUS=${GIT_PUSH_CERT_NONCE_STATUS-nononcestatus} + NONCE=${GIT_PUSH_CERT_NONCE-nononce} + E_O_F + + EOF + + test_config gpg.format ssh && + test_config user.signingkey "${GPGSSH_KEY_PRIMARY}" && + FINGERPRINT=$(ssh-keygen -lf "${GPGSSH_KEY_PRIMARY}" | awk "{print \$2;}") && + git push --signed dst noop ff +noff && + + ( + cat <<-\EOF && + SIGNER=principal with number 1 + KEY=FINGERPRINT + STATUS=G + NONCE_STATUS=OK + EOF + sed -n -e "s/^nonce /NONCE=/p" -e "/^$/q" dst/push-cert + ) | sed -e "s|FINGERPRINT|$FINGERPRINT|" >expect && + + noop=$(git rev-parse noop) && + ff=$(git rev-parse ff) && + noff=$(git rev-parse noff) && + grep "$noop $ff refs/heads/ff" dst/push-cert && + grep "$noop $noff refs/heads/noff" dst/push-cert && + test_cmp expect dst/push-cert-status +' + test_expect_success GPG 'inconsistent push options in signed push not allowed' ' # First, invoke receive-pack with dummy input to obtain its preamble. prepare_dst && @@ -276,6 +323,60 @@ test_expect_success GPGSM 'fail without key and heed user.signingkey x509' ' test_cmp expect dst/push-cert-status ' +test_expect_success GPGSSH 'fail without key and heed user.signingkey ssh' ' + test_config gpg.format ssh && + prepare_dst && + mkdir -p dst/.git/hooks && + git -C dst config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git -C dst config receive.certnonceseed sekrit && + write_script dst/.git/hooks/post-receive <<-\EOF && + # discard the update list + cat >/dev/null + # record the push certificate + if test -n "${GIT_PUSH_CERT-}" + then + git cat-file blob $GIT_PUSH_CERT >../push-cert + fi && + + cat >../push-cert-status <<E_O_F + SIGNER=${GIT_PUSH_CERT_SIGNER-nobody} + KEY=${GIT_PUSH_CERT_KEY-nokey} + STATUS=${GIT_PUSH_CERT_STATUS-nostatus} + NONCE_STATUS=${GIT_PUSH_CERT_NONCE_STATUS-nononcestatus} + NONCE=${GIT_PUSH_CERT_NONCE-nononce} + E_O_F + + EOF + + test_config user.email hasnokey@nowhere.com && + test_config gpg.format ssh && + test_config user.signingkey "" && + ( + sane_unset GIT_COMMITTER_EMAIL && + test_must_fail git push --signed dst noop ff +noff + ) && + test_config user.signingkey "${GPGSSH_KEY_PRIMARY}" && + FINGERPRINT=$(ssh-keygen -lf "${GPGSSH_KEY_PRIMARY}" | awk "{print \$2;}") && + git push --signed dst noop ff +noff && + + ( + cat <<-\EOF && + SIGNER=principal with number 1 + KEY=FINGERPRINT + STATUS=G + NONCE_STATUS=OK + EOF + sed -n -e "s/^nonce /NONCE=/p" -e "/^$/q" dst/push-cert + ) | sed -e "s|FINGERPRINT|$FINGERPRINT|" >expect && + + noop=$(git rev-parse noop) && + ff=$(git rev-parse ff) && + noff=$(git rev-parse noff) && + grep "$noop $ff refs/heads/ff" dst/push-cert && + grep "$noop $noff refs/heads/noff" dst/push-cert && + test_cmp expect dst/push-cert-status +' + test_expect_success GPG 'failed atomic push does not execute GPG' ' prepare_dst && git -C dst config receive.certnonceseed sekrit && diff --git a/t/t5541-http-push-smart.sh b/t/t5541-http-push-smart.sh index c024fa2818..8ca50f8b18 100755 --- a/t/t5541-http-push-smart.sh +++ b/t/t5541-http-push-smart.sh @@ -509,4 +509,20 @@ test_expect_success 'colorize errors/hints' ' test_i18ngrep ! "^hint: " decoded ' +test_expect_success 'report error server does not provide ref status' ' + git init "$HTTPD_DOCUMENT_ROOT_PATH/no_report" && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/no_report" config http.receivepack true && + test_must_fail git push --porcelain \ + $HTTPD_URL_USER_PASS/smart/no_report \ + HEAD:refs/tags/will-fail >actual && + test_must_fail git -C "$HTTPD_DOCUMENT_ROOT_PATH/no_report" \ + rev-parse --verify refs/tags/will-fail && + cat >expect <<-EOF && + To $HTTPD_URL/smart/no_report + ! HEAD:refs/tags/will-fail [remote failure] (remote failed to report status) + Done + EOF + test_cmp expect actual +' + test_done diff --git a/t/t5545-push-options.sh b/t/t5545-push-options.sh index 58c7add7ee..214228349a 100755 --- a/t/t5545-push-options.sh +++ b/t/t5545-push-options.sh @@ -5,6 +5,9 @@ test_description='pushing to a repository using push options' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB=1 +export GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB + . ./test-lib.sh mk_repo_pair () { diff --git a/t/t5572-pull-submodule.sh b/t/t5572-pull-submodule.sh index 4f92a116e1..fa6b4cca65 100755 --- a/t/t5572-pull-submodule.sh +++ b/t/t5572-pull-submodule.sh @@ -2,6 +2,9 @@ test_description='pull can handle submodules' +GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB=1 +export GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB + . ./test-lib.sh . "$TEST_DIRECTORY"/lib-submodule-update.sh diff --git a/t/t5580-unc-paths.sh b/t/t5580-unc-paths.sh index cd803ae8bf..cd7604fff9 100755 --- a/t/t5580-unc-paths.sh +++ b/t/t5580-unc-paths.sh @@ -4,6 +4,7 @@ test_description='various Windows-only path tests' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh if test_have_prereq CYGWIN diff --git a/t/t5615-alternate-env.sh b/t/t5615-alternate-env.sh index b4905b822c..83513e46a3 100755 --- a/t/t5615-alternate-env.sh +++ b/t/t5615-alternate-env.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='handling of alternates in environment variables' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh check_obj () { diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index d527cf6c49..f01af2f2ed 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -1107,6 +1107,57 @@ test_expect_success 'packfile-uri with transfer.fsckobjects fails when .gitmodul test_i18ngrep "disallowed submodule name" err ' +test_expect_success 'packfile-uri path redacted in trace' ' + P="$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + rm -rf "$P" http_child log && + + git init "$P" && + git -C "$P" config "uploadpack.allowsidebandall" "true" && + + echo my-blob >"$P/my-blob" && + git -C "$P" add my-blob && + git -C "$P" commit -m x && + + git -C "$P" hash-object my-blob >objh && + git -C "$P" pack-objects "$HTTPD_DOCUMENT_ROOT_PATH/mypack" <objh >packh && + git -C "$P" config --add \ + "uploadpack.blobpackfileuri" \ + "$(cat objh) $(cat packh) $HTTPD_URL/dumb/mypack-$(cat packh).pack" && + + GIT_TRACE=1 GIT_TRACE_PACKET="$(pwd)/log" GIT_TEST_SIDEBAND_ALL=1 \ + git -c protocol.version=2 \ + -c fetch.uriprotocols=http,https \ + clone "$HTTPD_URL/smart/http_parent" http_child && + + grep -F "clone< \\1$(cat packh) $HTTPD_URL/<redacted>" log +' + +test_expect_success 'packfile-uri path not redacted in trace when GIT_TRACE_REDACT=0' ' + P="$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + rm -rf "$P" http_child log && + + git init "$P" && + git -C "$P" config "uploadpack.allowsidebandall" "true" && + + echo my-blob >"$P/my-blob" && + git -C "$P" add my-blob && + git -C "$P" commit -m x && + + git -C "$P" hash-object my-blob >objh && + git -C "$P" pack-objects "$HTTPD_DOCUMENT_ROOT_PATH/mypack" <objh >packh && + git -C "$P" config --add \ + "uploadpack.blobpackfileuri" \ + "$(cat objh) $(cat packh) $HTTPD_URL/dumb/mypack-$(cat packh).pack" && + + GIT_TRACE=1 GIT_TRACE_PACKET="$(pwd)/log" GIT_TEST_SIDEBAND_ALL=1 \ + GIT_TRACE_REDACT=0 \ + git -c protocol.version=2 \ + -c fetch.uriprotocols=http,https \ + clone "$HTTPD_URL/smart/http_parent" http_child && + + grep -F "clone< \\1$(cat packh) $HTTPD_URL/dumb/mypack-$(cat packh).pack" log +' + test_expect_success 'http:// --negotiate-only' ' SERVER="$HTTPD_DOCUMENT_ROOT_PATH/server" && URI="$HTTPD_URL/smart/server" && diff --git a/t/t6200-fmt-merge-msg.sh b/t/t6200-fmt-merge-msg.sh index 44f55d93fe..2dd2423643 100755 --- a/t/t6200-fmt-merge-msg.sh +++ b/t/t6200-fmt-merge-msg.sh @@ -81,6 +81,36 @@ test_expect_success GPG 'set up a signed tag' ' git tag -s -m signed-tag-msg signed-good-tag left ' +test_expect_success GPGSSH 'created ssh signed commit and tag' ' + test_config gpg.format ssh && + git checkout -b signed-ssh && + touch file && + git add file && + git commit -m "ssh signed" -S"${GPGSSH_KEY_PRIMARY}" && + git tag -s -u"${GPGSSH_KEY_PRIMARY}" -m signed-ssh-tag-msg signed-good-ssh-tag left && + git tag -s -u"${GPGSSH_KEY_UNTRUSTED}" -m signed-ssh-tag-msg-untrusted signed-untrusted-ssh-tag left +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'create signed tags with keys having defined lifetimes' ' + test_when_finished "test_unconfig commit.gpgsign" && + test_config gpg.format ssh && + git checkout -b signed-expiry-ssh && + touch file && + git add file && + + echo expired >file && test_tick && git commit -a -m expired -S"${GPGSSH_KEY_EXPIRED}" && + git tag -s -u "${GPGSSH_KEY_EXPIRED}" -m expired-signed expired-signed && + + echo notyetvalid >file && test_tick && git commit -a -m notyetvalid -S"${GPGSSH_KEY_NOTYETVALID}" && + git tag -s -u "${GPGSSH_KEY_NOTYETVALID}" -m notyetvalid-signed notyetvalid-signed && + + echo timeboxedvalid >file && test_tick && git commit -a -m timeboxedvalid -S"${GPGSSH_KEY_TIMEBOXEDVALID}" && + git tag -s -u "${GPGSSH_KEY_TIMEBOXEDVALID}" -m timeboxedvalid-signed timeboxedvalid-signed && + + echo timeboxedinvalid >file && test_tick && git commit -a -m timeboxedinvalid -S"${GPGSSH_KEY_TIMEBOXEDINVALID}" && + git tag -s -u "${GPGSSH_KEY_TIMEBOXEDINVALID}" -m timeboxedinvalid-signed timeboxedinvalid-signed +' + test_expect_success 'message for merging local branch' ' echo "Merge branch ${apos}left${apos}" >expected && @@ -109,6 +139,58 @@ test_expect_success GPG 'message for merging local tag signed by unknown key' ' grep -E "^# gpg: Can${apos}t check signature: (public key not found|No public key)" actual ' +test_expect_success GPGSSH 'message for merging local tag signed by good ssh key' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git checkout main && + git fetch . signed-good-ssh-tag && + git fmt-merge-msg <.git/FETCH_HEAD >actual 2>&1 && + grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual +' + +test_expect_success GPGSSH 'message for merging local tag signed by unknown ssh key' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git checkout main && + git fetch . signed-untrusted-ssh-tag && + git fmt-merge-msg <.git/FETCH_HEAD >actual 2>&1 && + grep "${GPGSSH_GOOD_SIGNATURE_UNTRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual && + grep "${GPGSSH_KEY_NOT_TRUSTED}" actual +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'message for merging local tag signed by expired ssh key' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git checkout main && + git fetch . expired-signed && + git fmt-merge-msg <.git/FETCH_HEAD >actual 2>&1 && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'message for merging local tag signed by not yet valid ssh key' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git checkout main && + git fetch . notyetvalid-signed && + git fmt-merge-msg <.git/FETCH_HEAD >actual 2>&1 && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'message for merging local tag signed by valid timeboxed ssh key' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git checkout main && + git fetch . timeboxedvalid-signed && + git fmt-merge-msg <.git/FETCH_HEAD >actual 2>&1 && + grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'message for merging local tag signed by invalid timeboxed ssh key' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git checkout main && + git fetch . timeboxedinvalid-signed && + git fmt-merge-msg <.git/FETCH_HEAD >actual 2>&1 && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual +' + test_expect_success 'message for merging external branch' ' echo "Merge branch ${apos}left${apos} of $(pwd)" >expected && diff --git a/t/t6300-for-each-ref.sh b/t/t6300-for-each-ref.sh index 80679d5e12..9f2c706c12 100755 --- a/t/t6300-for-each-ref.sh +++ b/t/t6300-for-each-ref.sh @@ -419,6 +419,11 @@ test_expect_success 'Verify descending sort' ' test_cmp expected actual ' +test_expect_success 'Give help even with invalid sort atoms' ' + test_expect_code 129 git for-each-ref --sort=bogus -h >actual 2>&1 && + grep "^usage: git for-each-ref" actual +' + cat >expected <<\EOF refs/tags/testtag refs/tags/testtag-2 @@ -1019,6 +1024,27 @@ test_expect_success 'equivalent sorts fall back on refname' ' test_cmp expected actual ' +test_expect_success '--no-sort cancels the previous sort keys' ' + cat >expected <<-\EOF && + 100000 <user1@example.com> refs/tags/multi-ref1-100000-user1 + 100000 <user2@example.com> refs/tags/multi-ref1-100000-user2 + 100000 <user1@example.com> refs/tags/multi-ref2-100000-user1 + 100000 <user2@example.com> refs/tags/multi-ref2-100000-user2 + 200000 <user1@example.com> refs/tags/multi-ref1-200000-user1 + 200000 <user2@example.com> refs/tags/multi-ref1-200000-user2 + 200000 <user1@example.com> refs/tags/multi-ref2-200000-user1 + 200000 <user2@example.com> refs/tags/multi-ref2-200000-user2 + EOF + git for-each-ref \ + --format="%(taggerdate:unix) %(taggeremail) %(refname)" \ + --sort=-refname \ + --sort=taggeremail \ + --no-sort \ + --sort=taggerdate \ + "refs/tags/multi-*" >actual && + test_cmp expected actual +' + test_expect_success 'do not dereference NULL upon %(HEAD) on unborn branch' ' test_when_finished "git checkout main" && git for-each-ref --format="%(HEAD) %(refname:short)" refs/heads/ >actual && diff --git a/t/t6404-recursive-merge.sh b/t/t6404-recursive-merge.sh index eaf48e941e..a986eec042 100755 --- a/t/t6404-recursive-merge.sh +++ b/t/t6404-recursive-merge.sh @@ -108,8 +108,14 @@ test_expect_success 'refuse to merge binary files' ' printf "\0\0" >binary-file && git add binary-file && git commit -m binary2 && - test_must_fail git merge F >merge.out 2>merge.err && - grep "Cannot merge binary files: binary-file (HEAD vs. F)" merge.err + if test "$GIT_TEST_MERGE_ALGORITHM" = ort + then + test_must_fail git merge F >merge.out && + grep "Cannot merge binary files: binary-file (HEAD vs. F)" merge.out + else + test_must_fail git merge F >merge.out 2>merge.err && + grep "Cannot merge binary files: binary-file (HEAD vs. F)" merge.err + fi ' test_expect_success 'mark rename/delete as unmerged' ' diff --git a/t/t6406-merge-attr.sh b/t/t6406-merge-attr.sh index 8494645837..75a7994988 100755 --- a/t/t6406-merge-attr.sh +++ b/t/t6406-merge-attr.sh @@ -221,8 +221,14 @@ test_expect_success 'binary files with union attribute' ' printf "two\0" >bin.txt && git commit -am two && - test_must_fail git merge bin-main 2>stderr && - grep -i "warning.*cannot merge.*HEAD vs. bin-main" stderr + if test "$GIT_TEST_MERGE_ALGORITHM" = ort + then + test_must_fail git merge bin-main >stdout && + grep -i "warning.*cannot merge.*HEAD vs. bin-main" stdout + else + test_must_fail git merge bin-main 2>stderr && + grep -i "warning.*cannot merge.*HEAD vs. bin-main" stderr + fi ' test_done diff --git a/t/t6427-diff3-conflict-markers.sh b/t/t6427-diff3-conflict-markers.sh index 25c4b720e7..dc74075718 100755 --- a/t/t6427-diff3-conflict-markers.sh +++ b/t/t6427-diff3-conflict-markers.sh @@ -211,4 +211,94 @@ test_expect_success 'rebase --apply describes fake ancestor base' ' ) ' +test_setup_zdiff3 () { + test_create_repo zdiff3 && + ( + cd zdiff3 && + + test_write_lines 1 2 3 4 5 6 7 8 9 >basic && + test_write_lines 1 2 3 AA 4 5 BB 6 7 8 >middle-common && + test_write_lines 1 2 3 4 5 6 7 8 9 >interesting && + test_write_lines 1 2 3 4 5 6 7 8 9 >evil && + + git add basic middle-common interesting evil && + git commit -m base && + + git branch left && + git branch right && + + git checkout left && + test_write_lines 1 2 3 4 A B C D E 7 8 9 >basic && + test_write_lines 1 2 3 CC 4 5 DD 6 7 8 >middle-common && + test_write_lines 1 2 3 4 A B C D E F G H I J 7 8 9 >interesting && + test_write_lines 1 2 3 4 X A B C 7 8 9 >evil && + git add -u && + git commit -m letters && + + git checkout right && + test_write_lines 1 2 3 4 A X C Y E 7 8 9 >basic && + test_write_lines 1 2 3 EE 4 5 FF 6 7 8 >middle-common && + test_write_lines 1 2 3 4 A B C 5 6 G H I J 7 8 9 >interesting && + test_write_lines 1 2 3 4 Y A B C B C 7 8 9 >evil && + git add -u && + git commit -m permuted + ) +} + +test_expect_failure 'check zdiff3 markers' ' + test_setup_zdiff3 && + ( + cd zdiff3 && + + git checkout left^0 && + + base=$(git rev-parse --short HEAD^1) && + test_must_fail git -c merge.conflictstyle=zdiff3 merge -s recursive right^0 && + + test_write_lines 1 2 3 4 A \ + "<<<<<<< HEAD" B C D \ + "||||||| $base" 5 6 \ + ======= X C Y \ + ">>>>>>> right^0" \ + E 7 8 9 \ + >expect && + test_cmp expect basic && + + test_write_lines 1 2 3 \ + "<<<<<<< HEAD" CC \ + "||||||| $base" AA \ + ======= EE \ + ">>>>>>> right^0" \ + 4 5 \ + "<<<<<<< HEAD" DD \ + "||||||| $base" BB \ + ======= FF \ + ">>>>>>> right^0" \ + 6 7 8 \ + >expect && + test_cmp expect middle-common && + + test_write_lines 1 2 3 4 A B C \ + "<<<<<<< HEAD" D E F \ + "||||||| $base" 5 6 \ + ======= 5 6 \ + ">>>>>>> right^0" \ + G H I J 7 8 9 \ + >expect && + test_cmp expect interesting && + + # Not passing this one yet; the common "B C" lines is still + # being left in the conflict blocks on the left and right + # sides. + test_write_lines 1 2 3 4 \ + "<<<<<<< HEAD" X A \ + "||||||| $base" 5 6 \ + ======= Y A B C \ + ">>>>>>> right^0" \ + B C 7 8 9 \ + >expect && + test_cmp expect evil + ) +' + test_done diff --git a/t/t6437-submodule-merge.sh b/t/t6437-submodule-merge.sh index e5e89c2045..178413c22f 100755 --- a/t/t6437-submodule-merge.sh +++ b/t/t6437-submodule-merge.sh @@ -5,6 +5,9 @@ test_description='merging with submodules' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB=1 +export GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB + . ./test-lib.sh . "$TEST_DIRECTORY"/lib-merge.sh diff --git a/t/t7001-mv.sh b/t/t7001-mv.sh index 25bb9bbb89..963356ba5f 100755 --- a/t/t7001-mv.sh +++ b/t/t7001-mv.sh @@ -2,10 +2,11 @@ test_description='git mv in subdirs' . ./test-lib.sh +. "$TEST_DIRECTORY"/lib-diff-data.sh test_expect_success 'prepare reference tree' ' mkdir path0 path1 && - cp "$TEST_DIRECTORY"/../COPYING path0/COPYING && + COPYING_test_data >path0/COPYING && git add path0/COPYING && git commit -m add -a ' @@ -107,7 +108,7 @@ test_expect_success 'clean up' ' ' test_expect_success 'adding another file' ' - cp "$TEST_DIRECTORY"/../README.md path0/README && + COPYING_test_data | tr A-Za-z N-ZA-Mn-za-m >path0/README && git add path0/README && git commit -m add2 -a ' diff --git a/t/t7002-mv-sparse-checkout.sh b/t/t7002-mv-sparse-checkout.sh index 545748949a..1d3d2aca21 100755 --- a/t/t7002-mv-sparse-checkout.sh +++ b/t/t7002-mv-sparse-checkout.sh @@ -143,8 +143,6 @@ test_expect_success 'recursive mv refuses to move (possible) sparse' ' cat >>expect <<-\EOF && sub/d sub2/d - sub/dir/e - sub2/dir/e sub/dir2/e sub2/dir2/e EOF @@ -186,4 +184,26 @@ test_expect_success 'recursive mv refuses to move sparse' ' git reset --hard HEAD~1 ' +test_expect_success 'can move files to non-sparse dir' ' + git reset --hard && + git sparse-checkout init --no-cone && + git sparse-checkout set a b c w !/x y/ && + mkdir -p w x/y && + + git mv a w/new-a 2>stderr && + git mv b x/y/new-b 2>stderr && + test_must_be_empty stderr +' + +test_expect_success 'refuse to move file to non-skip-worktree sparse path' ' + git reset --hard && + git sparse-checkout init --no-cone && + git sparse-checkout set a !/x y/ !x/y/z && + mkdir -p x/y/z && + + test_must_fail git mv a x/y/z/new-a 2>stderr && + echo x/y/z/new-a | cat sparse_error_header - sparse_hint >expect && + test_cmp expect stderr +' + test_done diff --git a/t/t7031-verify-tag-signed-ssh.sh b/t/t7031-verify-tag-signed-ssh.sh new file mode 100755 index 0000000000..1cb36b9ab8 --- /dev/null +++ b/t/t7031-verify-tag-signed-ssh.sh @@ -0,0 +1,203 @@ +#!/bin/sh + +test_description='signed tag tests' +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh +. "$TEST_DIRECTORY/lib-gpg.sh" + +test_expect_success GPGSSH 'create signed tags ssh' ' + test_when_finished "test_unconfig commit.gpgsign" && + test_config gpg.format ssh && + test_config user.signingkey "${GPGSSH_KEY_PRIMARY}" && + + echo 1 >file && git add file && + test_tick && git commit -m initial && + git tag -s -m initial initial && + git branch side && + + echo 2 >file && test_tick && git commit -a -m second && + git tag -s -m second second && + + git checkout side && + echo 3 >elif && git add elif && + test_tick && git commit -m "third on side" && + + git checkout main && + test_tick && git merge -S side && + git tag -s -m merge merge && + + echo 4 >file && test_tick && git commit -a -S -m "fourth unsigned" && + git tag -a -m fourth-unsigned fourth-unsigned && + + test_tick && git commit --amend -S -m "fourth signed" && + git tag -s -m fourth fourth-signed && + + echo 5 >file && test_tick && git commit -a -m "fifth" && + git tag fifth-unsigned && + + git config commit.gpgsign true && + echo 6 >file && test_tick && git commit -a -m "sixth" && + git tag -a -m sixth sixth-unsigned && + + test_tick && git rebase -f HEAD^^ && git tag -s -m 6th sixth-signed HEAD^ && + git tag -m seventh -s seventh-signed && + + echo 8 >file && test_tick && git commit -a -m eighth && + git tag -u"${GPGSSH_KEY_UNTRUSTED}" -m eighth eighth-signed-alt +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'create signed tags with keys having defined lifetimes' ' + test_when_finished "test_unconfig commit.gpgsign" && + test_config gpg.format ssh && + + echo expired >file && test_tick && git commit -a -m expired -S"${GPGSSH_KEY_EXPIRED}" && + git tag -s -u "${GPGSSH_KEY_EXPIRED}" -m expired-signed expired-signed && + + echo notyetvalid >file && test_tick && git commit -a -m notyetvalid -S"${GPGSSH_KEY_NOTYETVALID}" && + git tag -s -u "${GPGSSH_KEY_NOTYETVALID}" -m notyetvalid-signed notyetvalid-signed && + + echo timeboxedvalid >file && test_tick && git commit -a -m timeboxedvalid -S"${GPGSSH_KEY_TIMEBOXEDVALID}" && + git tag -s -u "${GPGSSH_KEY_TIMEBOXEDVALID}" -m timeboxedvalid-signed timeboxedvalid-signed && + + echo timeboxedinvalid >file && test_tick && git commit -a -m timeboxedinvalid -S"${GPGSSH_KEY_TIMEBOXEDINVALID}" && + git tag -s -u "${GPGSSH_KEY_TIMEBOXEDINVALID}" -m timeboxedinvalid-signed timeboxedinvalid-signed +' + +test_expect_success GPGSSH 'verify and show ssh signatures' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + ( + for tag in initial second merge fourth-signed sixth-signed seventh-signed + do + git verify-tag $tag 2>actual && + grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual && + echo $tag OK || exit 1 + done + ) && + ( + for tag in fourth-unsigned fifth-unsigned sixth-unsigned + do + test_must_fail git verify-tag $tag 2>actual && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual && + echo $tag OK || exit 1 + done + ) && + ( + for tag in eighth-signed-alt + do + test_must_fail git verify-tag $tag 2>actual && + grep "${GPGSSH_GOOD_SIGNATURE_UNTRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual && + grep "${GPGSSH_KEY_NOT_TRUSTED}" actual && + echo $tag OK || exit 1 + done + ) +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'verify-tag exits failure on expired signature key' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + test_must_fail git verify-tag expired-signed 2>actual && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'verify-tag exits failure on not yet valid signature key' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + test_must_fail git verify-tag notyetvalid-signed 2>actual && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'verify-tag succeeds with tag date and key validity matching' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git verify-tag timeboxedvalid-signed 2>actual && + grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'verify-tag failes with tag date outside of key validity' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + test_must_fail git verify-tag timeboxedinvalid-signed 2>actual && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual +' + +test_expect_success GPGSSH 'detect fudged ssh signature' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git cat-file tag seventh-signed >raw && + sed -e "/^tag / s/seventh/7th forged/" raw >forged1 && + git hash-object -w -t tag forged1 >forged1.tag && + test_must_fail git verify-tag $(cat forged1.tag) 2>actual1 && + grep "${GPGSSH_BAD_SIGNATURE}" actual1 && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual1 && + ! grep "${GPGSSH_GOOD_SIGNATURE_UNTRUSTED}" actual1 +' + +test_expect_success GPGSSH 'verify ssh signatures with --raw' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + ( + for tag in initial second merge fourth-signed sixth-signed seventh-signed + do + git verify-tag --raw $tag 2>actual && + grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual && + echo $tag OK || exit 1 + done + ) && + ( + for tag in fourth-unsigned fifth-unsigned sixth-unsigned + do + test_must_fail git verify-tag --raw $tag 2>actual && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual && + echo $tag OK || exit 1 + done + ) && + ( + for tag in eighth-signed-alt + do + test_must_fail git verify-tag --raw $tag 2>actual && + grep "${GPGSSH_GOOD_SIGNATURE_UNTRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual && + echo $tag OK || exit 1 + done + ) +' + +test_expect_success GPGSSH 'verify signatures with --raw ssh' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git verify-tag --raw sixth-signed 2>actual && + grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual && + echo sixth-signed OK +' + +test_expect_success GPGSSH 'verify multiple tags ssh' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + tags="seventh-signed sixth-signed" && + for i in $tags + do + git verify-tag -v --raw $i || return 1 + done >expect.stdout 2>expect.stderr.1 && + grep "^${GPGSSH_GOOD_SIGNATURE_TRUSTED}" <expect.stderr.1 >expect.stderr && + git verify-tag -v --raw $tags >actual.stdout 2>actual.stderr.1 && + grep "^${GPGSSH_GOOD_SIGNATURE_TRUSTED}" <actual.stderr.1 >actual.stderr && + test_cmp expect.stdout actual.stdout && + test_cmp expect.stderr actual.stderr +' + +test_expect_success GPGSSH 'verifying tag with --format - ssh' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + cat >expect <<-\EOF && + tagname : fourth-signed + EOF + git verify-tag --format="tagname : %(tag)" "fourth-signed" >actual && + test_cmp expect actual +' + +test_expect_success GPGSSH 'verifying a forged tag with --format should fail silently - ssh' ' + test_must_fail git verify-tag --format="tagname : %(tag)" $(cat forged1.tag) >actual-forged && + test_must_be_empty actual-forged +' + +test_done diff --git a/t/t7064-wtstatus-pv2.sh b/t/t7064-wtstatus-pv2.sh index eeb0534163..47fc21d962 100755 --- a/t/t7064-wtstatus-pv2.sh +++ b/t/t7064-wtstatus-pv2.sh @@ -113,6 +113,21 @@ test_expect_success 'after first commit, create unstaged changes' ' test_cmp expect actual ' +test_expect_success 'after first commit, stash existing changes' ' + cat >expect <<-EOF && + # branch.oid $H0 + # branch.head initial-branch + # stash 2 + EOF + + test_when_finished "git stash pop && git stash pop" && + + git stash -- file_x && + git stash && + git status --porcelain=v2 --branch --show-stash --untracked-files=no >actual && + test_cmp expect actual +' + test_expect_success 'after first commit but omit untracked files and branch' ' cat >expect <<-EOF && 1 .M N... 100644 100644 100644 $OID_X $OID_X file_x diff --git a/t/t7101-reset-empty-subdirs.sh b/t/t7101-reset-empty-subdirs.sh index bfce05ac5d..5530651eea 100755 --- a/t/t7101-reset-empty-subdirs.sh +++ b/t/t7101-reset-empty-subdirs.sh @@ -5,10 +5,11 @@ test_description='git reset should cull empty subdirs' . ./test-lib.sh +. "$TEST_DIRECTORY"/lib-diff-data.sh test_expect_success 'creating initial files' ' mkdir path0 && - cp "$TEST_DIRECTORY"/../COPYING path0/COPYING && + COPYING_test_data >path0/COPYING && git add path0/COPYING && git commit -m add -a ' @@ -16,10 +17,10 @@ test_expect_success 'creating initial files' ' test_expect_success 'creating second files' ' mkdir path1 && mkdir path1/path2 && - cp "$TEST_DIRECTORY"/../COPYING path1/path2/COPYING && - cp "$TEST_DIRECTORY"/../COPYING path1/COPYING && - cp "$TEST_DIRECTORY"/../COPYING COPYING && - cp "$TEST_DIRECTORY"/../COPYING path0/COPYING-TOO && + COPYING_test_data >path1/path2/COPYING && + COPYING_test_data >path1/COPYING && + COPYING_test_data >COPYING && + COPYING_test_data >path0/COPYING-TOO && git add path1/path2/COPYING && git add path1/COPYING && git add COPYING && diff --git a/t/t7102-reset.sh b/t/t7102-reset.sh index 601b2bf97f..d05426062e 100755 --- a/t/t7102-reset.sh +++ b/t/t7102-reset.sh @@ -472,6 +472,23 @@ test_expect_success '--mixed refreshes the index' ' test_cmp expect output ' +test_expect_success '--mixed preserves skip-worktree' ' + echo 123 >>file2 && + git add file2 && + git update-index --skip-worktree file2 && + git reset --mixed HEAD >output && + test_must_be_empty output && + + cat >expect <<-\EOF && + Unstaged changes after reset: + M file2 + EOF + git update-index --no-skip-worktree file2 && + git add file2 && + git reset --mixed HEAD >output && + test_cmp expect output +' + test_expect_success 'resetting specific path that is unmerged' ' git rm --cached file2 && F1=$(git rev-parse HEAD:file1) && diff --git a/t/t7104-reset-hard.sh b/t/t7104-reset-hard.sh index 7948ec392b..cf9697eba9 100755 --- a/t/t7104-reset-hard.sh +++ b/t/t7104-reset-hard.sh @@ -2,6 +2,7 @@ test_description='reset --hard unmerged' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t7201-co.sh b/t/t7201-co.sh index b7ba1c3268..3129543b09 100755 --- a/t/t7201-co.sh +++ b/t/t7201-co.sh @@ -658,4 +658,21 @@ test_expect_success 'custom merge driver with checkout -m' ' test_cmp expect arm ' +test_expect_success 'tracking info copied with autoSetupMerge=inherit' ' + git reset --hard main && + # default config does not copy tracking info + git checkout -b foo-no-inherit koala/bear && + test -z "$(git config branch.foo-no-inherit.remote)" && + test -z "$(git config branch.foo-no-inherit.merge)" && + # with autoSetupMerge=inherit, we copy tracking info from koala/bear + test_config branch.autoSetupMerge inherit && + git checkout -b foo koala/bear && + test_cmp_config origin branch.foo.remote && + test_cmp_config refs/heads/koala/bear branch.foo.merge && + # no tracking info to inherit from main + git checkout -b main2 main && + test -z "$(git config branch.main2.remote)" && + test -z "$(git config branch.main2.merge)" +' + test_done diff --git a/t/t7400-submodule-basic.sh b/t/t7400-submodule-basic.sh index cb1b8e35db..f92fa80b03 100755 --- a/t/t7400-submodule-basic.sh +++ b/t/t7400-submodule-basic.sh @@ -107,23 +107,27 @@ test_expect_success 'setup - repository to add submodules to' ' # generates, which will expand symbolic links. submodurl=$(pwd -P) -listbranches() { - git for-each-ref --format='%(refname)' 'refs/heads/*' -} - inspect() { - dir=$1 && - dotdot="${2:-..}" && + sub_dir=$1 && + super_dir=$2 && - ( - cd "$dir" && - listbranches >"$dotdot/heads" && - { git symbolic-ref HEAD || :; } >"$dotdot/head" && - git rev-parse HEAD >"$dotdot/head-sha1" && - git update-index --refresh && - git diff-files --exit-code && - git clean -n -d -x >"$dotdot/untracked" - ) + git -C "$sub_dir" for-each-ref --format='%(refname)' 'refs/heads/*' >heads && + { git -C "$sub_dir" symbolic-ref HEAD || :; } >head && + git -C "$sub_dir" rev-parse HEAD >head-sha1 && + git -C "$sub_dir" update-index --refresh && + git -C "$sub_dir" diff-files --exit-code && + + # Ensure that submodule.superprojectGitDir contains the path from the + # submodule's gitdir to the superproject's gitdir. + + super_abs_gitdir=$(git -C "$super_dir" rev-parse --absolute-git-dir) && + sub_abs_gitdir=$(git -C "$sub_dir" rev-parse --absolute-git-dir) && + + [ "$(git -C "$sub_dir" config --get submodule.superprojectGitDir)" = \ + "$(test-tool path-utils relative_path "$super_abs_gitdir" \ + "$sub_abs_gitdir")" ] && + + git -C "$sub_dir" clean -n -d -x >untracked } test_expect_success 'submodule add' ' @@ -146,7 +150,7 @@ test_expect_success 'submodule add' ' ) && rm -f heads head untracked && - inspect addtest/submod ../.. && + inspect addtest/submod addtest && test_cmp expect heads && test_cmp expect head && test_must_be_empty untracked @@ -248,7 +252,7 @@ test_expect_success 'submodule add --branch' ' ) && rm -f heads head untracked && - inspect addtest/submod-branch ../.. && + inspect addtest/submod-branch addtest && test_cmp expect-heads heads && test_cmp expect-head head && test_must_be_empty untracked @@ -264,7 +268,7 @@ test_expect_success 'submodule add with ./ in path' ' ) && rm -f heads head untracked && - inspect addtest/dotsubmod/frotz ../../.. && + inspect addtest/dotsubmod/frotz addtest && test_cmp expect heads && test_cmp expect head && test_must_be_empty untracked @@ -280,7 +284,7 @@ test_expect_success 'submodule add with /././ in path' ' ) && rm -f heads head untracked && - inspect addtest/dotslashdotsubmod/frotz ../../.. && + inspect addtest/dotslashdotsubmod/frotz addtest && test_cmp expect heads && test_cmp expect head && test_must_be_empty untracked @@ -296,7 +300,7 @@ test_expect_success 'submodule add with // in path' ' ) && rm -f heads head untracked && - inspect addtest/slashslashsubmod/frotz ../../.. && + inspect addtest/slashslashsubmod/frotz addtest && test_cmp expect heads && test_cmp expect head && test_must_be_empty untracked @@ -312,7 +316,7 @@ test_expect_success 'submodule add with /.. in path' ' ) && rm -f heads head untracked && - inspect addtest/realsubmod ../.. && + inspect addtest/realsubmod addtest && test_cmp expect heads && test_cmp expect head && test_must_be_empty untracked @@ -328,7 +332,7 @@ test_expect_success 'submodule add with ./, /.. and // in path' ' ) && rm -f heads head untracked && - inspect addtest/realsubmod2 ../.. && + inspect addtest/realsubmod2 addtest && test_cmp expect heads && test_cmp expect head && test_must_be_empty untracked @@ -359,7 +363,7 @@ test_expect_success 'submodule add in subdirectory' ' ) && rm -f heads head untracked && - inspect addtest/realsubmod3 ../.. && + inspect addtest/realsubmod3 addtest && test_cmp expect heads && test_cmp expect head && test_must_be_empty untracked @@ -500,7 +504,7 @@ test_expect_success 'update should work when path is an empty dir' ' git submodule update -q >update.out && test_must_be_empty update.out && - inspect init && + inspect init . && test_cmp expect head-sha1 ' @@ -559,7 +563,7 @@ test_expect_success 'update should checkout rev1' ' echo "$rev1" >expect && git submodule update init && - inspect init && + inspect init . && test_cmp expect head-sha1 ' @@ -1182,18 +1186,18 @@ test_expect_success 'submodule deinit is silent when used on an uninitialized su rmdir init example2 ' -test_expect_success 'submodule deinit fails when submodule has a .git directory even when forced' ' +test_expect_success 'submodule deinit fails when submodule has a .git directory unless forced' ' git submodule update --init && ( cd init && rm .git && - cp -R ../.git/modules/example .git && + mv ../.git/modules/example .git && GIT_WORK_TREE=. git config --unset core.worktree ) && test_must_fail git submodule deinit init && - test_must_fail git submodule deinit -f init && - test -d init/.git && - test -n "$(git config --get-regexp "submodule\.example\.")" + git submodule deinit -f init && + test_path_is_missing init/.git && + test -z "$(git config --get-regexp "submodule\.example\.")" ' test_expect_success 'submodule with UTF-8 name' ' diff --git a/t/t7406-submodule-update.sh b/t/t7406-submodule-update.sh index 11cccbb333..7fd5741a5a 100755 --- a/t/t7406-submodule-update.sh +++ b/t/t7406-submodule-update.sh @@ -1061,4 +1061,16 @@ test_expect_success 'submodule update --quiet passes quietness to fetch with a s ) ' +test_expect_success 'submodule update adds superproject gitdir to older repos' ' + (cd super && + git -C submodule config --unset submodule.superprojectGitdir && + git submodule update && + test-tool path-utils relative_path \ + "$(git rev-parse --absolute-git-dir)" \ + "$(git -C submodule rev-parse --absolute-git-dir)" >expect && + git -C submodule config submodule.superprojectGitdir >actual && + test_cmp expect actual + ) +' + test_done diff --git a/t/t7412-submodule-absorbgitdirs.sh b/t/t7412-submodule-absorbgitdirs.sh index 1cfa150768..9ee5ccd660 100755 --- a/t/t7412-submodule-absorbgitdirs.sh +++ b/t/t7412-submodule-absorbgitdirs.sh @@ -30,7 +30,17 @@ test_expect_success 'absorb the git dir' ' git status >actual.1 && git -C sub1 rev-parse HEAD >actual.2 && test_cmp expect.1 actual.1 && - test_cmp expect.2 actual.2 + test_cmp expect.2 actual.2 && + + # make sure the submodule cached the superproject gitdir correctly + submodule_gitdir="$(git -C sub1 rev-parse --absolute-git-dir)" && + superproject_gitdir="$(git rev-parse --absolute-git-dir)" && + + test-tool path-utils relative_path "$superproject_gitdir" \ + "$submodule_gitdir" >expect && + git -C sub1 config submodule.superprojectGitDir >actual && + + test_cmp expect actual ' test_expect_success 'absorbing does not fail for deinitialized submodules' ' @@ -61,7 +71,16 @@ test_expect_success 'absorb the git dir in a nested submodule' ' git status >actual.1 && git -C sub1/nested rev-parse HEAD >actual.2 && test_cmp expect.1 actual.1 && - test_cmp expect.2 actual.2 + test_cmp expect.2 actual.2 && + + sub1_gitdir="$(git -C sub1 rev-parse --absolute-git-dir)" && + sub1_nested_gitdir="$(git -C sub1/nested rev-parse --absolute-git-dir)" && + + test-tool path-utils relative_path "$sub1_gitdir" "$sub1_nested_gitdir" \ + >expect && + git -C sub1/nested config submodule.superprojectGitDir >actual && + + test_cmp expect actual ' test_expect_success 're-setup nested submodule' ' diff --git a/t/t7418-submodule-sparse-gitmodules.sh b/t/t7418-submodule-sparse-gitmodules.sh index 3f7f271883..f87e524d6d 100755 --- a/t/t7418-submodule-sparse-gitmodules.sh +++ b/t/t7418-submodule-sparse-gitmodules.sh @@ -12,6 +12,9 @@ The test setup uses a sparse checkout, however the same scenario can be set up also by committing .gitmodules and then just removing it from the filesystem. ' +GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB=1 +export GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB + . ./test-lib.sh test_expect_success 'sparse checkout setup which hides .gitmodules' ' diff --git a/t/t7504-commit-msg-hook.sh b/t/t7504-commit-msg-hook.sh index 4e7592522a..bba58f0480 100755 --- a/t/t7504-commit-msg-hook.sh +++ b/t/t7504-commit-msg-hook.sh @@ -133,6 +133,14 @@ test_expect_success '--no-verify with failing hook' ' ' +test_expect_success '-n followed by --verify with failing hook' ' + + echo "even more" >> file && + git add file && + test_must_fail git commit -n --verify -m "even more" + +' + test_expect_success '--no-verify with failing hook (editor)' ' echo "more stuff" >> file && diff --git a/t/t7510-signed-commit.sh b/t/t7510-signed-commit.sh index 8df5a74f1d..d65a0171f2 100755 --- a/t/t7510-signed-commit.sh +++ b/t/t7510-signed-commit.sh @@ -71,7 +71,25 @@ test_expect_success GPG 'create signed commits' ' git tag eleventh-signed $(cat oid) && echo 12 | git commit-tree --gpg-sign=B7227189 HEAD^{tree} >oid && test_line_count = 1 oid && - git tag twelfth-signed-alt $(cat oid) + git tag twelfth-signed-alt $(cat oid) && + + cat >keydetails <<-\EOF && + Key-Type: RSA + Key-Length: 2048 + Subkey-Type: RSA + Subkey-Length: 2048 + Name-Real: Unknown User + Name-Email: unknown@git.com + Expire-Date: 0 + %no-ask-passphrase + %no-protection + EOF + gpg --batch --gen-key keydetails && + echo 13 >file && git commit -a -S"unknown@git.com" -m thirteenth && + git tag thirteenth-signed && + DELETE_FINGERPRINT=$(gpg -K --with-colons --fingerprint --batch unknown@git.com | grep "^fpr" | head -n 1 | awk -F ":" "{print \$10;}") && + gpg --batch --yes --delete-secret-keys $DELETE_FINGERPRINT && + gpg --batch --yes --delete-keys unknown@git.com ' test_expect_success GPG 'verify and show signatures' ' @@ -110,6 +128,13 @@ test_expect_success GPG 'verify and show signatures' ' ) ' +test_expect_success GPG 'verify-commit exits failure on unknown signature' ' + test_must_fail git verify-commit thirteenth-signed 2>actual && + ! grep "Good signature from" actual && + ! grep "BAD signature from" actual && + grep -q -F -e "No public key" -e "public key not found" actual +' + test_expect_success GPG 'verify-commit exits success on untrusted signature' ' git verify-commit eighth-signed-alt 2>actual && grep "Good signature from" actual && @@ -338,6 +363,8 @@ test_expect_success GPG 'show double signature with custom format' ' ' +# NEEDSWORK: This test relies on the test_tick commit/author dates from the first +# 'create signed commits' test even though it creates its own test_expect_success GPG 'verify-commit verifies multiply signed commits' ' git init multiply-signed && cd multiply-signed && diff --git a/t/t7518-ident-corner-cases.sh b/t/t7518-ident-corner-cases.sh index 905957bd0a..fffdb6ff2e 100755 --- a/t/t7518-ident-corner-cases.sh +++ b/t/t7518-ident-corner-cases.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='corner cases in ident strings' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # confirm that we do not segfault _and_ that we do not say "(null)", as diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh new file mode 100755 index 0000000000..e62ec9aa3c --- /dev/null +++ b/t/t7527-builtin-fsmonitor.sh @@ -0,0 +1,607 @@ +#!/bin/sh + +test_description='built-in file system watcher' + +. ./test-lib.sh + +if ! test_have_prereq FSMONITOR_DAEMON +then + skip_all="fsmonitor--daemon is not supported on this platform" + test_done +fi + +stop_daemon_delete_repo () { + r=$1 + git -C $r fsmonitor--daemon stop >/dev/null 2>/dev/null + rm -rf $1 + return 0 +} + +start_daemon () { + case "$#" in + 1) r="-C $1";; + *) r=""; + esac + + git $r fsmonitor--daemon start || return $? + git $r fsmonitor--daemon status || return $? + + return 0 +} + +# Is a Trace2 data event present with the given catetory and key? +# We do not care what the value is. +# +have_t2_data_event () { + c=$1 + k=$2 + + grep -e '"event":"data".*"category":"'"$c"'".*"key":"'"$k"'"' +} + +test_expect_success 'explicit daemon start and stop' ' + test_when_finished "stop_daemon_delete_repo test_explicit" && + + git init test_explicit && + start_daemon test_explicit && + + git -C test_explicit fsmonitor--daemon stop && + test_must_fail git -C test_explicit fsmonitor--daemon status +' + +test_expect_success 'implicit daemon start' ' + test_when_finished "stop_daemon_delete_repo test_implicit" && + + git init test_implicit && + test_must_fail git -C test_implicit fsmonitor--daemon status && + + # query will implicitly start the daemon. + # + # for test-script simplicity, we send a V1 timestamp rather than + # a V2 token. either way, the daemon response to any query contains + # a new V2 token. (the daemon may complain that we sent a V1 request, + # but this test case is only concerned with whether the daemon was + # implicitly started.) + + GIT_TRACE2_EVENT="$(pwd)/.git/trace" \ + test-tool -C test_implicit fsmonitor-client query --token 0 >actual && + nul_to_q <actual >actual.filtered && + grep "builtin:" actual.filtered && + + # confirm that a daemon was started in the background. + # + # since the mechanism for starting the background daemon is platform + # dependent, just confirm that the foreground command received a + # response from the daemon. + + have_t2_data_event fsm_client query/response-length <.git/trace && + + git -C test_implicit fsmonitor--daemon status && + git -C test_implicit fsmonitor--daemon stop && + test_must_fail git -C test_implicit fsmonitor--daemon status +' + +test_expect_success 'implicit daemon stop (delete .git)' ' + test_when_finished "stop_daemon_delete_repo test_implicit_1" && + + git init test_implicit_1 && + + start_daemon test_implicit_1 && + + # deleting the .git directory will implicitly stop the daemon. + rm -rf test_implicit_1/.git && + + # [1] Create an empty .git directory so that the following Git + # command will stay relative to the `-C` directory. + # + # Without this, the Git command will override the requested + # -C argument and crawl out to the containing Git source tree. + # This would make the test result dependent upon whether we + # were using fsmonitor on our development worktree. + # + sleep 1 && + mkdir test_implicit_1/.git && + + test_must_fail git -C test_implicit_1 fsmonitor--daemon status +' + +test_expect_success 'implicit daemon stop (rename .git)' ' + test_when_finished "stop_daemon_delete_repo test_implicit_2" && + + git init test_implicit_2 && + + start_daemon test_implicit_2 && + + # renaming the .git directory will implicitly stop the daemon. + mv test_implicit_2/.git test_implicit_2/.xxx && + + # See [1] above. + # + sleep 1 && + mkdir test_implicit_2/.git && + + test_must_fail git -C test_implicit_2 fsmonitor--daemon status +' + +test_expect_success 'cannot start multiple daemons' ' + test_when_finished "stop_daemon_delete_repo test_multiple" && + + git init test_multiple && + + start_daemon test_multiple && + + test_must_fail git -C test_multiple fsmonitor--daemon start 2>actual && + grep "fsmonitor--daemon is already running" actual && + + git -C test_multiple fsmonitor--daemon stop && + test_must_fail git -C test_multiple fsmonitor--daemon status +' + +# These tests use the main repo in the trash directory + +test_expect_success 'setup' ' + >tracked && + >modified && + >delete && + >rename && + mkdir dir1 && + >dir1/tracked && + >dir1/modified && + >dir1/delete && + >dir1/rename && + mkdir dir2 && + >dir2/tracked && + >dir2/modified && + >dir2/delete && + >dir2/rename && + mkdir dirtorename && + >dirtorename/a && + >dirtorename/b && + + cat >.gitignore <<-\EOF && + .gitignore + expect* + actual* + flush* + trace* + EOF + + git -c core.useBuiltinFSMonitor= add . && + test_tick && + git -c core.useBuiltinFSMonitor= commit -m initial && + + git config core.useBuiltinFSMonitor true +' + +# The test already explicitly stopped (or tried to stop) the daemon. +# This is here in case something else fails first. +# +redundant_stop_daemon () { + git fsmonitor--daemon stop + return 0 +} + +test_expect_success 'update-index implicitly starts daemon' ' + test_when_finished redundant_stop_daemon && + + test_must_fail git fsmonitor--daemon status && + + GIT_TRACE2_EVENT="$(pwd)/.git/trace_implicit_1" \ + git update-index --fsmonitor && + + git fsmonitor--daemon status && + test_might_fail git fsmonitor--daemon stop && + + # Confirm that the trace2 log contains a record of the + # daemon starting. + test_subcommand git fsmonitor--daemon start <.git/trace_implicit_1 +' + +test_expect_success 'status implicitly starts daemon' ' + test_when_finished redundant_stop_daemon && + + test_must_fail git fsmonitor--daemon status && + + GIT_TRACE2_EVENT="$(pwd)/.git/trace_implicit_2" \ + git status >actual && + + git fsmonitor--daemon status && + test_might_fail git fsmonitor--daemon stop && + + # Confirm that the trace2 log contains a record of the + # daemon starting. + test_subcommand git fsmonitor--daemon start <.git/trace_implicit_2 +' + +edit_files() { + echo 1 >modified + echo 2 >dir1/modified + echo 3 >dir2/modified + >dir1/untracked +} + +delete_files() { + rm -f delete + rm -f dir1/delete + rm -f dir2/delete +} + +create_files() { + echo 1 >new + echo 2 >dir1/new + echo 3 >dir2/new +} + +rename_files() { + mv rename renamed + mv dir1/rename dir1/renamed + mv dir2/rename dir2/renamed +} + +file_to_directory() { + rm -f delete + mkdir delete + echo 1 >delete/new +} + +directory_to_file() { + rm -rf dir1 + echo 1 >dir1 +} + +verify_status() { + git status >actual && + GIT_INDEX_FILE=.git/fresh-index git read-tree master && + GIT_INDEX_FILE=.git/fresh-index git -c core.useBuiltinFSMonitor= status >expect && + test_cmp expect actual && + echo HELLO AFTER && + cat .git/trace && + echo HELLO AFTER +} + +# The next few test cases confirm that our fsmonitor daemon sees each type +# of OS filesystem notification that we care about. At this layer we just +# ensure we are getting the OS notifications and do not try to confirm what +# is reported by `git status`. +# +# We run a simple query after modifying the filesystem just to introduce +# a bit of a delay so that the trace logging from the daemon has time to +# get flushed to disk. +# +# We `reset` and `clean` at the bottom of each test (and before stopping the +# daemon) because these commands might implicitly restart the daemon. + +clean_up_repo_and_stop_daemon () { + git reset --hard HEAD + git clean -fd + git fsmonitor--daemon stop + rm -f .git/trace +} + +test_expect_success 'edit some files' ' + test_when_finished clean_up_repo_and_stop_daemon && + + ( + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && + export GIT_TRACE_FSMONITOR && + + start_daemon + ) && + + edit_files && + + test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && + + grep "^event: dir1/modified$" .git/trace && + grep "^event: dir2/modified$" .git/trace && + grep "^event: modified$" .git/trace && + grep "^event: dir1/untracked$" .git/trace +' + +test_expect_success 'create some files' ' + test_when_finished clean_up_repo_and_stop_daemon && + + ( + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && + export GIT_TRACE_FSMONITOR && + + start_daemon + ) && + + create_files && + + test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && + + grep "^event: dir1/new$" .git/trace && + grep "^event: dir2/new$" .git/trace && + grep "^event: new$" .git/trace +' + +test_expect_success 'delete some files' ' + test_when_finished clean_up_repo_and_stop_daemon && + + ( + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && + export GIT_TRACE_FSMONITOR && + + start_daemon + ) && + + delete_files && + + test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && + + grep "^event: dir1/delete$" .git/trace && + grep "^event: dir2/delete$" .git/trace && + grep "^event: delete$" .git/trace +' + +test_expect_success 'rename some files' ' + test_when_finished clean_up_repo_and_stop_daemon && + + ( + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && + export GIT_TRACE_FSMONITOR && + + start_daemon + ) && + + rename_files && + + test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && + + grep "^event: dir1/rename$" .git/trace && + grep "^event: dir2/rename$" .git/trace && + grep "^event: rename$" .git/trace && + grep "^event: dir1/renamed$" .git/trace && + grep "^event: dir2/renamed$" .git/trace && + grep "^event: renamed$" .git/trace +' + +test_expect_success 'rename directory' ' + test_when_finished clean_up_repo_and_stop_daemon && + + ( + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && + export GIT_TRACE_FSMONITOR && + + start_daemon + ) && + + mv dirtorename dirrenamed && + + test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && + + grep "^event: dirtorename/*$" .git/trace && + grep "^event: dirrenamed/*$" .git/trace +' + +test_expect_success 'file changes to directory' ' + test_when_finished clean_up_repo_and_stop_daemon && + + ( + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && + export GIT_TRACE_FSMONITOR && + + start_daemon + ) && + + file_to_directory && + + test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && + + grep "^event: delete$" .git/trace && + grep "^event: delete/new$" .git/trace +' + +test_expect_success 'directory changes to a file' ' + test_when_finished clean_up_repo_and_stop_daemon && + + ( + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace" && + export GIT_TRACE_FSMONITOR && + + start_daemon + ) && + + directory_to_file && + + test-tool fsmonitor-client query --token 0 >/dev/null 2>&1 && + + grep "^event: dir1$" .git/trace +' + +# The next few test cases exercise the token-resync code. When filesystem +# drops events (because of filesystem velocity or because the daemon isn't +# polling fast enough), we need to discard the cached data (relative to the +# current token) and start collecting events under a new token. +# +# the 'test-tool fsmonitor-client flush' command can be used to send a +# "flush" message to a running daemon and ask it to do a flush/resync. + +test_expect_success 'flush cached data' ' + test_when_finished "stop_daemon_delete_repo test_flush" && + + git init test_flush && + + ( + GIT_TEST_FSMONITOR_TOKEN=true && + export GIT_TEST_FSMONITOR_TOKEN && + + GIT_TRACE_FSMONITOR="$(pwd)/.git/trace_daemon" && + export GIT_TRACE_FSMONITOR && + + start_daemon test_flush + ) && + + # The daemon should have an initial token with no events in _0 and + # then a few (probably platform-specific number of) events in _1. + # These should both have the same <token_id>. + + test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000001:0" >actual_0 && + nul_to_q <actual_0 >actual_q0 && + + touch test_flush/file_1 && + touch test_flush/file_2 && + + test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000001:0" >actual_1 && + nul_to_q <actual_1 >actual_q1 && + + grep "file_1" actual_q1 && + + # Force a flush. This will change the <token_id>, reset the <seq_nr>, and + # flush the file data. Then create some events and ensure that the file + # again appears in the cache. It should have the new <token_id>. + + test-tool -C test_flush fsmonitor-client flush >flush_0 && + nul_to_q <flush_0 >flush_q0 && + grep "^builtin:test_00000002:0Q/Q$" flush_q0 && + + test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000002:0" >actual_2 && + nul_to_q <actual_2 >actual_q2 && + + grep "^builtin:test_00000002:0Q$" actual_q2 && + + touch test_flush/file_3 && + + test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000002:0" >actual_3 && + nul_to_q <actual_3 >actual_q3 && + + grep "file_3" actual_q3 +' + +# The next few test cases create repos where the .git directory is NOT +# inside the one of the working directory. That is, where .git is a file +# that points to a directory elsewhere. This happens for submodules and +# non-primary worktrees. + +test_expect_success 'setup worktree base' ' + git init wt-base && + echo 1 >wt-base/file1 && + git -C wt-base add file1 && + git -C wt-base commit -m "c1" +' + +test_expect_success 'worktree with .git file' ' + git -C wt-base worktree add ../wt-secondary && + + ( + GIT_TRACE2_PERF="$(pwd)/trace2_wt_secondary" && + export GIT_TRACE2_PERF && + + GIT_TRACE_FSMONITOR="$(pwd)/trace_wt_secondary" && + export GIT_TRACE_FSMONITOR && + + start_daemon wt-secondary + ) && + + git -C wt-secondary fsmonitor--daemon stop && + test_must_fail git -C wt-secondary fsmonitor--daemon status +' + +# NEEDSWORK: Repeat one of the "edit" tests on wt-secondary and +# confirm that we get the same events and behavior -- that is, that +# fsmonitor--daemon correctly watches BOTH the working directory and +# the external GITDIR directory and behaves the same as when ".git" +# is a directory inside the working directory. + +test_expect_success 'cleanup worktrees' ' + stop_daemon_delete_repo wt-secondary && + stop_daemon_delete_repo wt-base +' + +# The next few tests perform arbitrary/contrived file operations and +# confirm that status is correct. That is, that the data (or lack of +# data) from fsmonitor doesn't cause incorrect results. And doesn't +# cause incorrect results when the untracked-cache is enabled. + +test_lazy_prereq UNTRACKED_CACHE ' + { git update-index --test-untracked-cache; ret=$?; } && + test $ret -ne 1 +' + +test_expect_success 'Matrix: setup for untracked-cache,fsmonitor matrix' ' + test_might_fail git config --unset core.useBuiltinFSMonitor && + git update-index --no-fsmonitor && + test_might_fail git fsmonitor--daemon stop +' + +matrix_clean_up_repo () { + git reset --hard HEAD + git clean -fd +} + +matrix_try () { + uc=$1 + fsm=$2 + fn=$3 + + test_expect_success "Matrix[uc:$uc][fsm:$fsm] $fn" ' + matrix_clean_up_repo && + $fn && + if test $uc = false -a $fsm = false + then + git status --porcelain=v1 >.git/expect.$fn + else + git status --porcelain=v1 >.git/actual.$fn + test_cmp .git/expect.$fn .git/actual.$fn + fi + ' + + return $? +} + +uc_values="false" +test_have_prereq UNTRACKED_CACHE && uc_values="false true" +for uc_val in $uc_values +do + if test $uc_val = false + then + test_expect_success "Matrix[uc:$uc_val] disable untracked cache" ' + git config core.untrackedcache false && + git update-index --no-untracked-cache + ' + else + test_expect_success "Matrix[uc:$uc_val] enable untracked cache" ' + git config core.untrackedcache true && + git update-index --untracked-cache + ' + fi + + fsm_values="false true" + for fsm_val in $fsm_values + do + if test $fsm_val = false + then + test_expect_success "Matrix[uc:$uc_val][fsm:$fsm_val] disable fsmonitor" ' + test_might_fail git config --unset core.useBuiltinFSMonitor && + git update-index --no-fsmonitor && + test_might_fail git fsmonitor--daemon stop 2>/dev/null + ' + else + test_expect_success "Matrix[uc:$uc_val][fsm:$fsm_val] enable fsmonitor" ' + git config core.useBuiltinFSMonitor true && + git fsmonitor--daemon start && + git update-index --fsmonitor + ' + fi + + matrix_try $uc_val $fsm_val edit_files + matrix_try $uc_val $fsm_val delete_files + matrix_try $uc_val $fsm_val create_files + matrix_try $uc_val $fsm_val rename_files + matrix_try $uc_val $fsm_val file_to_directory + matrix_try $uc_val $fsm_val directory_to_file + + if test $fsm_val = true + then + test_expect_success "Matrix[uc:$uc_val][fsm:$fsm_val] disable fsmonitor at end" ' + test_might_fail git config --unset core.useBuiltinFSMonitor && + git update-index --no-fsmonitor && + test_might_fail git fsmonitor--daemon stop 2>/dev/null + ' + fi + done +done + +test_done diff --git a/t/t7528-signed-commit-ssh.sh b/t/t7528-signed-commit-ssh.sh new file mode 100755 index 0000000000..dae76ded0c --- /dev/null +++ b/t/t7528-signed-commit-ssh.sh @@ -0,0 +1,440 @@ +#!/bin/sh + +test_description='ssh signed commit tests' +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh +GNUPGHOME_NOT_USED=$GNUPGHOME +. "$TEST_DIRECTORY/lib-gpg.sh" + +test_expect_success GPGSSH 'create signed commits' ' + test_oid_cache <<-\EOF && + header sha1:gpgsig + header sha256:gpgsig-sha256 + EOF + + test_when_finished "test_unconfig commit.gpgsign" && + test_config gpg.format ssh && + test_config user.signingkey "${GPGSSH_KEY_PRIMARY}" && + + echo 1 >file && git add file && + test_tick && git commit -S -m initial && + git tag initial && + git branch side && + + echo 2 >file && test_tick && git commit -a -S -m second && + git tag second && + + git checkout side && + echo 3 >elif && git add elif && + test_tick && git commit -m "third on side" && + + git checkout main && + test_tick && git merge -S side && + git tag merge && + + echo 4 >file && test_tick && git commit -a -m "fourth unsigned" && + git tag fourth-unsigned && + + test_tick && git commit --amend -S -m "fourth signed" && + git tag fourth-signed && + + git config commit.gpgsign true && + echo 5 >file && test_tick && git commit -a -m "fifth signed" && + git tag fifth-signed && + + git config commit.gpgsign false && + echo 6 >file && test_tick && git commit -a -m "sixth" && + git tag sixth-unsigned && + + git config commit.gpgsign true && + echo 7 >file && test_tick && git commit -a -m "seventh" --no-gpg-sign && + git tag seventh-unsigned && + + test_tick && git rebase -f HEAD^^ && git tag sixth-signed HEAD^ && + git tag seventh-signed && + + echo 8 >file && test_tick && git commit -a -m eighth -S"${GPGSSH_KEY_UNTRUSTED}" && + git tag eighth-signed-alt && + + # commit.gpgsign is still on but this must not be signed + echo 9 | git commit-tree HEAD^{tree} >oid && + test_line_count = 1 oid && + git tag ninth-unsigned $(cat oid) && + # explicit -S of course must sign. + echo 10 | git commit-tree -S HEAD^{tree} >oid && + test_line_count = 1 oid && + git tag tenth-signed $(cat oid) && + + # --gpg-sign[=<key-id>] must sign. + echo 11 | git commit-tree --gpg-sign HEAD^{tree} >oid && + test_line_count = 1 oid && + git tag eleventh-signed $(cat oid) && + echo 12 | git commit-tree --gpg-sign="${GPGSSH_KEY_UNTRUSTED}" HEAD^{tree} >oid && + test_line_count = 1 oid && + git tag twelfth-signed-alt $(cat oid) +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'create signed commits with keys having defined lifetimes' ' + test_when_finished "test_unconfig commit.gpgsign" && + test_config gpg.format ssh && + + echo expired >file && test_tick && git commit -a -m expired -S"${GPGSSH_KEY_EXPIRED}" && + git tag expired-signed && + + echo notyetvalid >file && test_tick && git commit -a -m notyetvalid -S"${GPGSSH_KEY_NOTYETVALID}" && + git tag notyetvalid-signed && + + echo timeboxedvalid >file && test_tick && git commit -a -m timeboxedvalid -S"${GPGSSH_KEY_TIMEBOXEDVALID}" && + git tag timeboxedvalid-signed && + + echo timeboxedinvalid >file && test_tick && git commit -a -m timeboxedinvalid -S"${GPGSSH_KEY_TIMEBOXEDINVALID}" && + git tag timeboxedinvalid-signed +' + +test_expect_success GPGSSH 'verify and show signatures' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + test_config gpg.mintrustlevel UNDEFINED && + ( + for commit in initial second merge fourth-signed \ + fifth-signed sixth-signed seventh-signed tenth-signed \ + eleventh-signed + do + git verify-commit $commit && + git show --pretty=short --show-signature $commit >actual && + grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual && + echo $commit OK || exit 1 + done + ) && + ( + for commit in merge^2 fourth-unsigned sixth-unsigned \ + seventh-unsigned ninth-unsigned + do + test_must_fail git verify-commit $commit && + git show --pretty=short --show-signature $commit >actual && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual && + echo $commit OK || exit 1 + done + ) && + ( + for commit in eighth-signed-alt twelfth-signed-alt + do + git show --pretty=short --show-signature $commit >actual && + grep "${GPGSSH_GOOD_SIGNATURE_UNTRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual && + grep "${GPGSSH_KEY_NOT_TRUSTED}" actual && + echo $commit OK || exit 1 + done + ) +' + +test_expect_success GPGSSH 'verify-commit exits failure on untrusted signature' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + test_must_fail git verify-commit eighth-signed-alt 2>actual && + grep "${GPGSSH_GOOD_SIGNATURE_UNTRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual && + grep "${GPGSSH_KEY_NOT_TRUSTED}" actual +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'verify-commit exits failure on expired signature key' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + test_must_fail git verify-commit expired-signed 2>actual && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'verify-commit exits failure on not yet valid signature key' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + test_must_fail git verify-commit notyetvalid-signed 2>actual && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'verify-commit succeeds with commit date and key validity matching' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git verify-commit timeboxedvalid-signed 2>actual && + grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual +' + +test_expect_success GPGSSH,GPGSSH_VERIFYTIME 'verify-commit exits failure with commit date outside of key validity' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + test_must_fail git verify-commit timeboxedinvalid-signed 2>actual && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual +' + +test_expect_success GPGSSH 'verify-commit exits success with matching minTrustLevel' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + test_config gpg.minTrustLevel fully && + git verify-commit sixth-signed +' + +test_expect_success GPGSSH 'verify-commit exits success with low minTrustLevel' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + test_config gpg.minTrustLevel marginal && + git verify-commit sixth-signed +' + +test_expect_success GPGSSH 'verify-commit exits failure with high minTrustLevel' ' + test_config gpg.minTrustLevel ultimate && + test_must_fail git verify-commit eighth-signed-alt +' + +test_expect_success GPGSSH 'verify signatures with --raw' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + ( + for commit in initial second merge fourth-signed fifth-signed sixth-signed seventh-signed + do + git verify-commit --raw $commit 2>actual && + grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual && + echo $commit OK || exit 1 + done + ) && + ( + for commit in merge^2 fourth-unsigned sixth-unsigned seventh-unsigned + do + test_must_fail git verify-commit --raw $commit 2>actual && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual && + echo $commit OK || exit 1 + done + ) && + ( + for commit in eighth-signed-alt + do + test_must_fail git verify-commit --raw $commit 2>actual && + grep "${GPGSSH_GOOD_SIGNATURE_UNTRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual && + echo $commit OK || exit 1 + done + ) +' + +test_expect_success GPGSSH 'proper header is used for hash algorithm' ' + git cat-file commit fourth-signed >output && + grep "^$(test_oid header) -----BEGIN SSH SIGNATURE-----" output +' + +test_expect_success GPGSSH 'show signed commit with signature' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git show -s initial >commit && + git show -s --show-signature initial >show && + git verify-commit -v initial >verify.1 2>verify.2 && + git cat-file commit initial >cat && + grep -v -e "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" -e "Warning: " show >show.commit && + grep -e "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" -e "Warning: " show >show.gpg && + grep -v "^ " cat | grep -v "^gpgsig.* " >cat.commit && + test_cmp show.commit commit && + test_cmp show.gpg verify.2 && + test_cmp cat.commit verify.1 +' + +test_expect_success GPGSSH 'detect fudged signature' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git cat-file commit seventh-signed >raw && + sed -e "s/^seventh/7th forged/" raw >forged1 && + git hash-object -w -t commit forged1 >forged1.commit && + test_must_fail git verify-commit $(cat forged1.commit) && + git show --pretty=short --show-signature $(cat forged1.commit) >actual1 && + grep "${GPGSSH_BAD_SIGNATURE}" actual1 && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual1 && + ! grep "${GPGSSH_GOOD_SIGNATURE_UNTRUSTED}" actual1 +' + +test_expect_success GPGSSH 'detect fudged signature with NUL' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git cat-file commit seventh-signed >raw && + cat raw >forged2 && + echo Qwik | tr "Q" "\000" >>forged2 && + git hash-object -w -t commit forged2 >forged2.commit && + test_must_fail git verify-commit $(cat forged2.commit) && + git show --pretty=short --show-signature $(cat forged2.commit) >actual2 && + grep "${GPGSSH_BAD_SIGNATURE}" actual2 && + ! grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual2 +' + +test_expect_success GPGSSH 'amending already signed commit' ' + test_config gpg.format ssh && + test_config user.signingkey "${GPGSSH_KEY_PRIMARY}" && + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + git checkout fourth-signed^0 && + git commit --amend -S --no-edit && + git verify-commit HEAD && + git show -s --show-signature HEAD >actual && + grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual && + ! grep "${GPGSSH_BAD_SIGNATURE}" actual +' + +test_expect_success GPGSSH 'show good signature with custom format' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + FINGERPRINT=$(ssh-keygen -lf "${GPGSSH_KEY_PRIMARY}" | awk "{print \$2;}") && + cat >expect.tmpl <<-\EOF && + G + FINGERPRINT + principal with number 1 + FINGERPRINT + + EOF + sed "s|FINGERPRINT|$FINGERPRINT|g" expect.tmpl >expect && + git log -1 --format="%G?%n%GK%n%GS%n%GF%n%GP" sixth-signed >actual && + test_cmp expect actual +' + +test_expect_success GPGSSH 'show bad signature with custom format' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + cat >expect <<-\EOF && + B + + + + + EOF + git log -1 --format="%G?%n%GK%n%GS%n%GF%n%GP" $(cat forged1.commit) >actual && + test_cmp expect actual +' + +test_expect_success GPGSSH 'show untrusted signature with custom format' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + cat >expect.tmpl <<-\EOF && + U + FINGERPRINT + + FINGERPRINT + + EOF + git log -1 --format="%G?%n%GK%n%GS%n%GF%n%GP" eighth-signed-alt >actual && + FINGERPRINT=$(ssh-keygen -lf "${GPGSSH_KEY_UNTRUSTED}" | awk "{print \$2;}") && + sed "s|FINGERPRINT|$FINGERPRINT|g" expect.tmpl >expect && + test_cmp expect actual +' + +test_expect_success GPGSSH 'show untrusted signature with undefined trust level' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + cat >expect.tmpl <<-\EOF && + undefined + FINGERPRINT + + FINGERPRINT + + EOF + git log -1 --format="%GT%n%GK%n%GS%n%GF%n%GP" eighth-signed-alt >actual && + FINGERPRINT=$(ssh-keygen -lf "${GPGSSH_KEY_UNTRUSTED}" | awk "{print \$2;}") && + sed "s|FINGERPRINT|$FINGERPRINT|g" expect.tmpl >expect && + test_cmp expect actual +' + +test_expect_success GPGSSH 'show untrusted signature with ultimate trust level' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + cat >expect.tmpl <<-\EOF && + fully + FINGERPRINT + principal with number 1 + FINGERPRINT + + EOF + git log -1 --format="%GT%n%GK%n%GS%n%GF%n%GP" sixth-signed >actual && + FINGERPRINT=$(ssh-keygen -lf "${GPGSSH_KEY_PRIMARY}" | awk "{print \$2;}") && + sed "s|FINGERPRINT|$FINGERPRINT|g" expect.tmpl >expect && + test_cmp expect actual +' + +test_expect_success GPGSSH 'show lack of signature with custom format' ' + cat >expect <<-\EOF && + N + + + + + EOF + git log -1 --format="%G?%n%GK%n%GS%n%GF%n%GP" seventh-unsigned >actual && + test_cmp expect actual +' + +test_expect_success GPGSSH 'log.showsignature behaves like --show-signature' ' + test_config gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + test_config log.showsignature true && + git show initial >actual && + grep "${GPGSSH_GOOD_SIGNATURE_TRUSTED}" actual +' + +test_expect_success GPGSSH 'check config gpg.format values' ' + test_config gpg.format ssh && + test_config user.signingkey "${GPGSSH_KEY_PRIMARY}" && + test_config gpg.format ssh && + git commit -S --amend -m "success" && + test_config gpg.format OpEnPgP && + test_must_fail git commit -S --amend -m "fail" +' + +test_expect_failure GPGSSH 'detect fudged commit with double signature (TODO)' ' + sed -e "/gpgsig/,/END PGP/d" forged1 >double-base && + sed -n -e "/gpgsig/,/END PGP/p" forged1 | \ + sed -e "s/^$(test_oid header)//;s/^ //" | gpg --dearmor >double-sig1.sig && + gpg -o double-sig2.sig -u 29472784 --detach-sign double-base && + cat double-sig1.sig double-sig2.sig | gpg --enarmor >double-combined.asc && + sed -e "s/^\(-.*\)ARMORED FILE/\1SIGNATURE/;1s/^/$(test_oid header) /;2,\$s/^/ /" \ + double-combined.asc > double-gpgsig && + sed -e "/committer/r double-gpgsig" double-base >double-commit && + git hash-object -w -t commit double-commit >double-commit.commit && + test_must_fail git verify-commit $(cat double-commit.commit) && + git show --pretty=short --show-signature $(cat double-commit.commit) >double-actual && + grep "BAD signature from" double-actual && + grep "Good signature from" double-actual +' + +test_expect_failure GPGSSH 'show double signature with custom format (TODO)' ' + cat >expect <<-\EOF && + E + + + + + EOF + git log -1 --format="%G?%n%GK%n%GS%n%GF%n%GP" $(cat double-commit.commit) >actual && + test_cmp expect actual +' + + +test_expect_failure GPGSSH 'verify-commit verifies multiply signed commits (TODO)' ' + git init multiply-signed && + cd multiply-signed && + test_commit first && + echo 1 >second && + git add second && + tree=$(git write-tree) && + parent=$(git rev-parse HEAD^{commit}) && + git commit --gpg-sign -m second && + git cat-file commit HEAD && + # Avoid trailing whitespace. + sed -e "s/^Q//" -e "s/^Z/ /" >commit <<-EOF && + Qtree $tree + Qparent $parent + Qauthor A U Thor <author@example.com> 1112912653 -0700 + Qcommitter C O Mitter <committer@example.com> 1112912653 -0700 + Qgpgsig -----BEGIN PGP SIGNATURE----- + QZ + Q iHQEABECADQWIQRz11h0S+chaY7FTocTtvUezd5DDQUCX/uBDRYcY29tbWl0dGVy + Q QGV4YW1wbGUuY29tAAoJEBO29R7N3kMNd+8AoK1I8mhLHviPH+q2I5fIVgPsEtYC + Q AKCTqBh+VabJceXcGIZuF0Ry+udbBQ== + Q =tQ0N + Q -----END PGP SIGNATURE----- + Qgpgsig-sha256 -----BEGIN PGP SIGNATURE----- + QZ + Q iHQEABECADQWIQRz11h0S+chaY7FTocTtvUezd5DDQUCX/uBIBYcY29tbWl0dGVy + Q QGV4YW1wbGUuY29tAAoJEBO29R7N3kMN/NEAn0XO9RYSBj2dFyozi0JKSbssYMtO + Q AJwKCQ1BQOtuwz//IjU8TiS+6S4iUw== + Q =pIwP + Q -----END PGP SIGNATURE----- + Q + Qsecond + EOF + head=$(git hash-object -t commit -w commit) && + git reset --hard $head && + git verify-commit $head 2>actual && + grep "Good signature from" actual && + ! grep "BAD signature from" actual +' + +test_done diff --git a/t/t7601-merge-pull-config.sh b/t/t7601-merge-pull-config.sh index 1f652f433e..6275641b9c 100755 --- a/t/t7601-merge-pull-config.sh +++ b/t/t7601-merge-pull-config.sh @@ -2,7 +2,7 @@ test_description='git merge -Testing pull.* configuration parsing.' +Testing pull.* configuration parsing and other things.' . ./test-lib.sh @@ -387,6 +387,20 @@ test_expect_success 'pull prevents non-fast-forward with "only" in pull.ff' ' test_must_fail git pull . c3 ' +test_expect_success 'already-up-to-date pull succeeds with "only" in pull.ff' ' + git reset --hard c1 && + test_config pull.ff only && + git pull . c0 && + test "$(git rev-parse HEAD)" = "$(git rev-parse c1)" +' + +test_expect_success 'already-up-to-date pull/rebase succeeds with "only" in pull.ff' ' + git reset --hard c1 && + test_config pull.ff only && + git -c pull.rebase=true pull . c0 && + test "$(git rev-parse HEAD)" = "$(git rev-parse c1)" +' + test_expect_success 'merge c1 with c2 (ours in pull.twohead)' ' git reset --hard c1 && git config pull.twohead ours && diff --git a/t/t7604-merge-custom-message.sh b/t/t7604-merge-custom-message.sh index cd4f9607dc..eca7555101 100755 --- a/t/t7604-merge-custom-message.sh +++ b/t/t7604-merge-custom-message.sh @@ -4,6 +4,7 @@ test_description='git merge Testing merge when using a custom message for the merge commit.' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh create_merge_msgs() { diff --git a/t/t7811-grep-open.sh b/t/t7811-grep-open.sh index a98785da79..1dd07141a7 100755 --- a/t/t7811-grep-open.sh +++ b/t/t7811-grep-open.sh @@ -3,6 +3,7 @@ test_description='git grep --open-files-in-pager ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-pager.sh unset PAGER GIT_PAGER @@ -114,8 +115,8 @@ test_expect_success 'modified file' ' unrelated EOF + test_when_finished "git reset --hard" && echo "enum grep_pat_token" >unrelated && - test_when_finished "git checkout HEAD unrelated" && GIT_PAGER=./less git grep -F -O "enum grep_pat_token" >out && test_cmp expect actual && test_must_be_empty out diff --git a/t/t7812-grep-icase-non-ascii.sh b/t/t7812-grep-icase-non-ascii.sh index e5d1e4ea68..22487d90fd 100755 --- a/t/t7812-grep-icase-non-ascii.sh +++ b/t/t7812-grep-icase-non-ascii.sh @@ -53,6 +53,54 @@ test_expect_success REGEX_LOCALE 'pickaxe -i on non-ascii' ' test_cmp expected actual ' +test_expect_success GETTEXT_LOCALE,PCRE 'log --author with an ascii pattern on UTF-8 data' ' + cat >expected <<-\EOF && + Author: <BOLD;RED>À Ú Thor<RESET> <author@example.com> + EOF + test_write_lines "forth" >file4 && + git add file4 && + git commit --author="À Ú Thor <author@example.com>" -m sécond && + git log -1 --color=always --perl-regexp --author=".*Thor" >log && + grep Author log >actual.raw && + test_decode_color <actual.raw >actual && + test_cmp expected actual +' + +test_expect_success GETTEXT_LOCALE,PCRE 'log --committer with an ascii pattern on ISO-8859-1 data' ' + cat >expected <<-\EOF && + Commit: Ç<BOLD;RED> O Mîtter <committer@example.com><RESET> + EOF + test_write_lines "fifth" >file5 && + git add file5 && + GIT_COMMITTER_NAME="Ç O Mîtter" && + GIT_COMMITTER_EMAIL="committer@example.com" && + git -c i18n.commitEncoding=latin1 commit -m thïrd && + git -c i18n.logOutputEncoding=latin1 log -1 --pretty=fuller --color=always --perl-regexp --committer=" O.*" >log && + grep Commit: log >actual.raw && + test_decode_color <actual.raw >actual && + test_cmp expected actual +' + +test_expect_success GETTEXT_LOCALE,PCRE 'log --grep with an ascii pattern on UTF-8 data' ' + cat >expected <<-\EOF && + sé<BOLD;RED>con<RESET>d + EOF + git log -1 --color=always --perl-regexp --grep="con" >log && + grep con log >actual.raw && + test_decode_color <actual.raw >actual && + test_cmp expected actual +' + +test_expect_success GETTEXT_LOCALE,PCRE 'log --grep with an ascii pattern on ISO-8859-1 data' ' + cat >expected <<-\EOF && + <BOLD;RED>thïrd<RESET> + EOF + git -c i18n.logOutputEncoding=latin1 log -1 --color=always --perl-regexp --grep="th.*rd" >log && + grep "th.*rd" log >actual.raw && + test_decode_color <actual.raw >actual && + test_cmp expected actual +' + test_expect_success GETTEXT_LOCALE,LIBPCRE2 'PCRE v2: setup invalid UTF-8 data' ' printf "\\200\\n" >invalid-0x80 && echo "ævar" >expected && diff --git a/t/t7813-grep-icase-iso.sh b/t/t7813-grep-icase-iso.sh index 701e08a8e5..1227885737 100755 --- a/t/t7813-grep-icase-iso.sh +++ b/t/t7813-grep-icase-iso.sh @@ -2,6 +2,7 @@ test_description='grep icase on non-English locales' +TEST_PASSES_SANITIZE_LEAK=true . ./lib-gettext.sh test_expect_success GETTEXT_ISO_LOCALE 'setup' ' diff --git a/t/t7816-grep-binary-pattern.sh b/t/t7816-grep-binary-pattern.sh index 9d67a5fc4c..fdb2355649 100755 --- a/t/t7816-grep-binary-pattern.sh +++ b/t/t7816-grep-binary-pattern.sh @@ -2,6 +2,7 @@ test_description='git grep with a binary pattern files' +TEST_PASSES_SANITIZE_LEAK=true . ./lib-gettext.sh nul_match_internal () { diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 9b9f11a8e7..74aa638475 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -336,15 +336,15 @@ test_expect_success EXPENSIVE 'incremental-repack 2g limit' ' --no-progress --batch-size=2147483647 <run-2g.txt ' -test_expect_success 'maintenance.incremental-repack.auto' ' +run_incremental_repack_and_verify () { + test_commit A && git repack -adk && - git config core.multiPackIndex true && git multi-pack-index write && GIT_TRACE2_EVENT="$(pwd)/midx-init.txt" git \ -c maintenance.incremental-repack.auto=1 \ maintenance run --auto --task=incremental-repack 2>/dev/null && test_subcommand ! git multi-pack-index write --no-progress <midx-init.txt && - test_commit A && + test_commit B && git pack-objects --revs .git/objects/pack/pack <<-\EOF && HEAD ^HEAD~1 @@ -353,7 +353,7 @@ test_expect_success 'maintenance.incremental-repack.auto' ' -c maintenance.incremental-repack.auto=2 \ maintenance run --auto --task=incremental-repack 2>/dev/null && test_subcommand ! git multi-pack-index write --no-progress <trace-A && - test_commit B && + test_commit C && git pack-objects --revs .git/objects/pack/pack <<-\EOF && HEAD ^HEAD~1 @@ -362,6 +362,26 @@ test_expect_success 'maintenance.incremental-repack.auto' ' -c maintenance.incremental-repack.auto=2 \ maintenance run --auto --task=incremental-repack 2>/dev/null && test_subcommand git multi-pack-index write --no-progress <trace-B +} + +test_expect_success 'maintenance.incremental-repack.auto' ' + rm -rf incremental-repack-true && + git init incremental-repack-true && + ( + cd incremental-repack-true && + git config core.multiPackIndex true && + run_incremental_repack_and_verify + ) +' + +test_expect_success 'maintenance.incremental-repack.auto (when config is unset)' ' + rm -rf incremental-repack-unset && + git init incremental-repack-unset && + ( + cd incremental-repack-unset && + test_unconfig core.multiPackIndex && + run_incremental_repack_and_verify + ) ' test_expect_success 'pack-refs task' ' diff --git a/t/t9001-send-email.sh b/t/t9001-send-email.sh index aa0c20499b..84d0f40d76 100755 --- a/t/t9001-send-email.sh +++ b/t/t9001-send-email.sh @@ -539,7 +539,7 @@ test_expect_success $PREREQ "--validate respects relative core.hooksPath path" ' test_path_is_file my-hooks.ran && cat >expect <<-EOF && fatal: longline.patch: rejected by sendemail-validate hook - fatal: command '"'"'my-hooks/sendemail-validate'"'"' died with exit code 1 + fatal: command '"'"'git hook run --ignore-missing sendemail-validate -- <patch>'"'"' died with exit code 1 warning: no patches were sent EOF test_cmp expect actual @@ -558,7 +558,7 @@ test_expect_success $PREREQ "--validate respects absolute core.hooksPath path" ' test_path_is_file my-hooks.ran && cat >expect <<-EOF && fatal: longline.patch: rejected by sendemail-validate hook - fatal: command '"'"'$hooks_path/sendemail-validate'"'"' died with exit code 1 + fatal: command '"'"'git hook run --ignore-missing sendemail-validate -- <patch>'"'"' died with exit code 1 warning: no patches were sent EOF test_cmp expect actual diff --git a/t/t9902-completion.sh b/t/t9902-completion.sh index 5decc3b269..518203fbe0 100755 --- a/t/t9902-completion.sh +++ b/t/t9902-completion.sh @@ -2148,6 +2148,9 @@ test_expect_success PERL 'send-email' ' --cover-from-description=Z --cover-letter Z EOF + test_completion "git send-email --val" <<-\EOF && + --validate Z + EOF test_completion "git send-email ma" "main " ' diff --git a/t/test-lib.sh b/t/test-lib.sh index 151da80c56..e80317e271 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -489,6 +489,13 @@ then export GIT_PERL_FATAL_WARNINGS fi +case $GIT_TEST_FSYNC in +'') + GIT_TEST_FSYNC=0 + export GIT_TEST_FSYNC + ;; +esac + # Add libc MALLOC and MALLOC_PERTURB test # only if we are not executing the test with valgrind if test -n "$valgrind" || @@ -589,17 +596,33 @@ USER_TERM="$TERM" TERM=dumb export TERM USER_TERM -error () { - say_color error "error: $*" +_error_exit () { finalize_junit_xml GIT_EXIT_OK=t exit 1 } +error () { + say_color error "error: $*" + _error_exit +} + BUG () { error >&7 "bug in the test script: $*" } +BAIL_OUT () { + test $# -ne 1 && BUG "1 param" + + # Do not change "Bail out! " string. It's part of TAP syntax: + # https://testanything.org/tap-specification.html + local bail_out="Bail out! " + local message="$1" + + say_color error $bail_out "$message" + _error_exit +} + say () { say_color info "$*" } @@ -608,9 +631,7 @@ if test -n "$HARNESS_ACTIVE" then if test "$verbose" = t || test -n "$verbose_only" then - printf 'Bail out! %s\n' \ - 'verbose mode forbidden under TAP harness; try --verbose-log' - exit 1 + BAIL_OUT 'verbose mode forbidden under TAP harness; try --verbose-log' fi fi @@ -720,7 +741,7 @@ test_failure_ () { say_color error "not ok $test_count - $1" shift printf '%s\n' "$*" | sed -e 's/^/# /' - test "$immediate" = "" || { finalize_junit_xml; GIT_EXIT_OK=t; exit 1; } + test "$immediate" = "" || _error_exit } test_known_broken_ok_ () { @@ -1398,7 +1419,7 @@ then fi elif test_bool_env GIT_TEST_PASSING_SANITIZE_LEAK false then - error "GIT_TEST_PASSING_SANITIZE_LEAK=true has no effect except when compiled with SANITIZE=leak" + BAIL_OUT "GIT_TEST_PASSING_SANITIZE_LEAK=true has no effect except when compiled with SANITIZE=leak" fi # Last-minute variable setup @@ -1407,8 +1428,20 @@ HOME="$TRASH_DIRECTORY" GNUPGHOME="$HOME/gnupg-home-not-used" export HOME GNUPGHOME USER_HOME +# "rm -rf" existing trash directory, even if a previous run left it +# with bad permissions. +remove_trash_directory () { + dir="$1" + if ! rm -rf "$dir" 2>/dev/null + then + chmod -R u+rwx "$dir" + rm -rf "$dir" + fi + ! test -d "$dir" +} + # Test repository -rm -fr "$TRASH_DIRECTORY" || { +remove_trash_directory "$TRASH_DIRECTORY" || { GIT_EXIT_OK=t echo >&5 "FATAL: Cannot prepare test area" exit 1 @@ -1736,3 +1769,9 @@ test_lazy_prereq SHA1 ' # Tests that verify the scheduler integration must set this locally # to avoid errors. GIT_TEST_MAINT_SCHEDULER="none:exit 1" + +# Does this platform support `git fsmonitor--daemon` +# +test_lazy_prereq FSMONITOR_DAEMON ' + git version --build-options | grep "feature:" | grep "fsmonitor--daemon" +' @@ -25,8 +25,9 @@ static int run_gpg_verify(const char *buf, unsigned long size, unsigned flags) return error("no signature found"); } - ret = check_signature(payload.buf, payload.len, signature.buf, - signature.len, &sigc); + sigc.payload_type = SIGNATURE_PAYLOAD_TAG; + sigc.payload = strbuf_detach(&payload, &sigc.payload_len); + ret = check_signature(&sigc, signature.buf, signature.len); if (!(flags & GPG_VERIFY_OMIT_STATUS)) print_signature_buffer(&sigc, flags); diff --git a/tmp-objdir.c b/tmp-objdir.c index b8d880e362..5878ba1df2 100644 --- a/tmp-objdir.c +++ b/tmp-objdir.c @@ -1,5 +1,6 @@ #include "cache.h" #include "tmp-objdir.h" +#include "chdir-notify.h" #include "dir.h" #include "sigchain.h" #include "string-list.h" @@ -11,6 +12,8 @@ struct tmp_objdir { struct strbuf path; struct strvec env; + struct object_directory *prev_odb; + int will_destroy; }; /* @@ -38,6 +41,9 @@ static int tmp_objdir_destroy_1(struct tmp_objdir *t, int on_signal) if (t == the_tmp_objdir) the_tmp_objdir = NULL; + if (!on_signal && t->prev_odb) + restore_primary_odb(t->prev_odb, t->path.buf); + /* * This may use malloc via strbuf_grow(), but we should * have pre-grown t->path sufficiently so that this @@ -52,6 +58,7 @@ static int tmp_objdir_destroy_1(struct tmp_objdir *t, int on_signal) */ if (!on_signal) tmp_objdir_free(t); + return err; } @@ -121,7 +128,7 @@ static int setup_tmp_objdir(const char *root) return ret; } -struct tmp_objdir *tmp_objdir_create(void) +struct tmp_objdir *tmp_objdir_create(const char *prefix) { static int installed_handlers; struct tmp_objdir *t; @@ -129,11 +136,16 @@ struct tmp_objdir *tmp_objdir_create(void) if (the_tmp_objdir) BUG("only one tmp_objdir can be used at a time"); - t = xmalloc(sizeof(*t)); + t = xcalloc(1, sizeof(*t)); strbuf_init(&t->path, 0); strvec_init(&t->env); - strbuf_addf(&t->path, "%s/incoming-XXXXXX", get_object_directory()); + /* + * Use a string starting with tmp_ so that the builtin/prune.c code + * can recognize any stale objdirs left behind by a crash and delete + * them. + */ + strbuf_addf(&t->path, "%s/tmp_objdir-%s-XXXXXX", get_object_directory(), prefix); /* * Grow the strbuf beyond any filename we expect to be placed in it. @@ -170,6 +182,13 @@ struct tmp_objdir *tmp_objdir_create(void) return t; } +void tmp_objdir_discard_contents(struct tmp_objdir *t) +{ + remove_dir_recursively(&t->path, REMOVE_DIR_KEEP_TOPLEVEL); + if (setup_tmp_objdir(t->path.buf)) + die_errno(_("could not recreate objdir %s"), t->path.buf); +} + /* * Make sure we copy packfiles and their associated metafiles in the correct * order. All of these ends_with checks are slightly expensive to do in @@ -269,6 +288,13 @@ int tmp_objdir_migrate(struct tmp_objdir *t) if (!t) return 0; + if (t->prev_odb) { + if (the_repository->objects->odb->will_destroy) + BUG("migrating an ODB that was marked for destruction"); + restore_primary_odb(t->prev_odb, t->path.buf); + t->prev_odb = NULL; + } + strbuf_addbuf(&src, &t->path); strbuf_addstr(&dst, get_object_directory()); @@ -292,3 +318,33 @@ void tmp_objdir_add_as_alternate(const struct tmp_objdir *t) { add_to_alternates_memory(t->path.buf); } + +void tmp_objdir_replace_primary_odb(struct tmp_objdir *t, int will_destroy) +{ + if (t->prev_odb) + BUG("the primary object database is already replaced"); + t->prev_odb = set_temporary_primary_odb(t->path.buf, will_destroy); + t->will_destroy = will_destroy; +} + +struct tmp_objdir *tmp_objdir_unapply_primary_odb(void) +{ + if (!the_tmp_objdir || !the_tmp_objdir->prev_odb) + return NULL; + + restore_primary_odb(the_tmp_objdir->prev_odb, the_tmp_objdir->path.buf); + the_tmp_objdir->prev_odb = NULL; + return the_tmp_objdir; +} + +void tmp_objdir_reapply_primary_odb(struct tmp_objdir *t, const char *old_cwd, + const char *new_cwd) +{ + char *path; + + path = reparent_relative_path(old_cwd, new_cwd, t->path.buf); + strbuf_reset(&t->path); + strbuf_addstr(&t->path, path); + free(path); + tmp_objdir_replace_primary_odb(t, t->will_destroy); +} diff --git a/tmp-objdir.h b/tmp-objdir.h index b1e45b4c75..98f4155b6c 100644 --- a/tmp-objdir.h +++ b/tmp-objdir.h @@ -10,7 +10,7 @@ * * Example: * - * struct tmp_objdir *t = tmp_objdir_create(); + * struct tmp_objdir *t = tmp_objdir_create("incoming"); * if (!run_command_v_opt_cd_env(cmd, 0, NULL, tmp_objdir_env(t)) && * !tmp_objdir_migrate(t)) * printf("success!\n"); @@ -22,9 +22,10 @@ struct tmp_objdir; /* - * Create a new temporary object directory; returns NULL on failure. + * Create a new temporary object directory with the specified prefix; + * returns NULL on failure. */ -struct tmp_objdir *tmp_objdir_create(void); +struct tmp_objdir *tmp_objdir_create(const char *prefix); /* * Return a list of environment strings, suitable for use with @@ -34,6 +35,12 @@ struct tmp_objdir *tmp_objdir_create(void); const char **tmp_objdir_env(const struct tmp_objdir *); /* + * Discard the files in the specified temporary object directory but leave + * the directory itself around. + */ +void tmp_objdir_discard_contents(struct tmp_objdir *); + +/* * Finalize a temporary object directory by migrating its objects into the main * object database, removing the temporary directory, and freeing any * associated resources. @@ -51,4 +58,26 @@ int tmp_objdir_destroy(struct tmp_objdir *); */ void tmp_objdir_add_as_alternate(const struct tmp_objdir *); +/* + * Replaces the main object store in the current process with the temporary + * object directory and makes the former main object store an alternate. + * If will_destroy is nonzero, the object directory may not be migrated. + */ +void tmp_objdir_replace_primary_odb(struct tmp_objdir *, int will_destroy); + +/* + * If the primary object database was replaced by a temporary object directory, + * restore it to its original value while keeping the directory contents around. + * Returns NULL if the primary object database was not replaced. + */ +struct tmp_objdir *tmp_objdir_unapply_primary_odb(void); + +/* + * Reapplies the former primary temporary object database, after protentially + * changing its relative path. + */ +void tmp_objdir_reapply_primary_odb(struct tmp_objdir *, const char *old_cwd, + const char *new_cwd); + + #endif /* TMP_OBJDIR_H */ diff --git a/transport-helper.c b/transport-helper.c index e8dbdd1153..a0297b0986 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -845,6 +845,10 @@ static int push_update_ref_status(struct strbuf *buf, forced = 1; FREE_AND_NULL(msg); } + else if (!strcmp(msg, "expecting report")) { + status = REF_STATUS_EXPECTING_REPORT; + FREE_AND_NULL(msg); + } } if (state->hint) diff --git a/unpack-trees.c b/unpack-trees.c index a7e1712d23..b71fdab5e3 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -645,17 +645,24 @@ static void mark_ce_used_same_name(struct cache_entry *ce, } } -static struct cache_entry *next_cache_entry(struct unpack_trees_options *o) +static struct cache_entry *next_cache_entry(struct unpack_trees_options *o, int *hint) { const struct index_state *index = o->src_index; int pos = o->cache_bottom; + if (*hint > pos) + pos = *hint; + while (pos < index->cache_nr) { struct cache_entry *ce = index->cache[pos]; - if (!(ce->ce_flags & CE_UNPACKED)) + if (!(ce->ce_flags & CE_UNPACKED)) { + *hint = pos + 1; return ce; + } pos++; } + + *hint = pos; return NULL; } @@ -1365,12 +1372,13 @@ static int unpack_callback(int n, unsigned long mask, unsigned long dirmask, str /* Are we supposed to look at the index too? */ if (o->merge) { + int hint = -1; while (1) { int cmp; struct cache_entry *ce; if (o->diff_index_cached) - ce = next_cache_entry(o); + ce = next_cache_entry(o, &hint); else ce = find_cache_entry(info, p); @@ -1690,7 +1698,7 @@ static int verify_absent(const struct cache_entry *, int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options *o) { struct repository *repo = the_repository; - int i, ret; + int i, hint, ret; static struct cache_entry *dfc; struct pattern_list pl; int free_pattern_list = 0; @@ -1779,13 +1787,15 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options info.pathspec = o->pathspec; if (o->prefix) { + hint = -1; + /* * Unpack existing index entries that sort before the * prefix the tree is spliced into. Note that o->merge * is always true in this case. */ while (1) { - struct cache_entry *ce = next_cache_entry(o); + struct cache_entry *ce = next_cache_entry(o, &hint); if (!ce) break; if (ce_in_traverse_path(ce, &info)) @@ -1806,8 +1816,9 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options /* Any left-over entries in the index? */ if (o->merge) { + hint = -1; while (1) { - struct cache_entry *ce = next_cache_entry(o); + struct cache_entry *ce = next_cache_entry(o, &hint); if (!ce) break; if (unpack_index_entry(ce, o) < 0) @@ -2156,9 +2167,10 @@ static int verify_clean_subdirectory(const struct cache_entry *ce, if (o->dir) d.exclude_per_dir = o->dir->exclude_per_dir; i = read_directory(&d, o->src_index, pathbuf, namelen+1, NULL); + dir_clear(&d); + free(pathbuf); if (i) return add_rejected_path(o, ERROR_NOT_UPTODATE_DIR, ce->name); - free(pathbuf); return cnt; } diff --git a/urlmatch.c b/urlmatch.c index 33a2ccd306..03ad3f30a9 100644 --- a/urlmatch.c +++ b/urlmatch.c @@ -5,7 +5,7 @@ #define URL_DIGIT "0123456789" #define URL_ALPHADIGIT URL_ALPHA URL_DIGIT #define URL_SCHEME_CHARS URL_ALPHADIGIT "+.-" -#define URL_HOST_CHARS URL_ALPHADIGIT ".-[:]" /* IPv6 literals need [:] */ +#define URL_HOST_CHARS URL_ALPHADIGIT ".-_[:]" /* IPv6 literals need [:] */ #define URL_UNSAFE_CHARS " <>\"%{}|\\^`" /* plus 0x00-0x1F,0x7F-0xFF */ #define URL_GEN_RESERVED ":/?#[]@" #define URL_SUB_RESERVED "!$&'()*+,;=" diff --git a/userdiff.c b/userdiff.c index af02b1878c..8578cb0d12 100644 --- a/userdiff.c +++ b/userdiff.c @@ -64,9 +64,15 @@ PATTERNS("cpp", /* functions/methods, variables, and compounds at top level */ "^((::[[:space:]]*)?[A-Za-z_].*)$", /* -- */ + /* identifiers and keywords */ "[a-zA-Z_][a-zA-Z0-9_]*" - "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lLuU]*" - "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->\\*?|\\.\\*"), + /* decimal and octal integers as well as floatingpoint numbers */ + "|[0-9][0-9.]*([Ee][-+]?[0-9]+)?[fFlLuU]*" + /* hexadecimal and binary integers */ + "|0[xXbB][0-9a-fA-F]+[lLuU]*" + /* floatingpoint numbers that begin with a decimal point */ + "|\\.[0-9][0-9]*([Ee][-+]?[0-9]+)?[fFlL]?" + "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->\\*?|\\.\\*|<=>"), PATTERNS("csharp", /* Keywords */ "!^[ \t]*(do|while|for|if|else|instanceof|new|return|switch|case|throw|catch|using)\n" diff --git a/worktree.c b/worktree.c index 092a4f92ad..2c155b1015 100644 --- a/worktree.c +++ b/worktree.c @@ -28,11 +28,13 @@ static void add_head_info(struct worktree *wt) { int flags; const char *target; + int ignore_errno; target = refs_resolve_ref_unsafe(get_worktree_ref_store(wt), "HEAD", 0, - &wt->head_oid, &flags); + &wt->head_oid, &flags, + &ignore_errno); if (!target) return; @@ -418,6 +420,7 @@ const struct worktree *find_shared_symref(const char *symref, const char *symref_target; struct ref_store *refs; int flags; + int ignore_errno; if (wt->is_bare) continue; @@ -435,7 +438,8 @@ const struct worktree *find_shared_symref(const char *symref, refs = get_worktree_ref_store(wt); symref_target = refs_resolve_ref_unsafe(refs, symref, 0, - NULL, &flags); + NULL, &flags, + &ignore_errno); if ((flags & REF_ISSYMREF) && symref_target && !strcmp(symref_target, target)) { existing = wt; @@ -563,16 +567,17 @@ int other_head_refs(each_ref_fn fn, void *cb_data) struct worktree *wt = *p; struct object_id oid; int flag; + int ignore_errno; if (wt->is_current) continue; strbuf_reset(&refname); strbuf_worktree_ref(wt, &refname, "HEAD"); - if (!refs_read_ref_full(get_main_ref_store(the_repository), - refname.buf, - RESOLVE_REF_READING, - &oid, &flag)) + if (refs_resolve_ref_unsafe(get_main_ref_store(the_repository), + refname.buf, + RESOLVE_REF_READING, + &oid, &flag, &ignore_errno)) ret = fn(refname.buf, &oid, flag, cb_data); if (ret) break; @@ -552,6 +552,54 @@ int xmkstemp_mode(char *filename_template, int mode) return fd; } +int git_fsync(int fd, enum fsync_action action) +{ + switch (action) { + case FSYNC_WRITEOUT_ONLY: + +#ifdef __APPLE__ + /* + * on macOS, fsync just causes filesystem cache writeback but does not + * flush hardware caches. + */ + return fsync(fd); +#endif + +#ifdef HAVE_SYNC_FILE_RANGE + /* + * On linux 2.6.17 and above, sync_file_range is the way to issue + * a writeback without a hardware flush. An offset of 0 and size of 0 + * indicates writeout of the entire file and the wait flags ensure that all + * dirty data is written to the disk (potentially in a disk-side cache) + * before we continue. + */ + + return sync_file_range(fd, 0, 0, SYNC_FILE_RANGE_WAIT_BEFORE | + SYNC_FILE_RANGE_WRITE | + SYNC_FILE_RANGE_WAIT_AFTER); +#endif + +#ifdef fsync_no_flush + return fsync_no_flush(fd); +#endif + + errno = ENOSYS; + return -1; + + case FSYNC_HARDWARE_FLUSH: + +#ifdef __APPLE__ + return fcntl(fd, F_FULLFSYNC); +#else + return fsync(fd); +#endif + + default: + BUG("unexpected git_fsync(%d) call", action); + } + +} + static int warn_if_unremovable(const char *op, const char *file, int rc) { int err; diff --git a/write-or-die.c b/write-or-die.c index 0b1ec8190b..94b151f1a5 100644 --- a/write-or-die.c +++ b/write-or-die.c @@ -1,4 +1,5 @@ #include "cache.h" +#include "config.h" #include "run-command.h" /* @@ -57,7 +58,11 @@ void fprintf_or_die(FILE *f, const char *fmt, ...) void fsync_or_die(int fd, const char *msg) { - while (fsync(fd) < 0) { + if (use_fsync < 0) + use_fsync = git_env_bool("GIT_TEST_FSYNC", 1); + if (!use_fsync) + return; + while (git_fsync(fd, FSYNC_HARDWARE_FLUSH) < 0) { if (errno != EINTR) die_errno("fsync error on '%s'", msg); } diff --git a/wt-status.c b/wt-status.c index e4f29b2b4c..5d215f4e4f 100644 --- a/wt-status.c +++ b/wt-status.c @@ -948,11 +948,17 @@ static int stash_count_refs(struct object_id *ooid, struct object_id *noid, return 0; } +static int count_stash_entries(void) +{ + int n = 0; + for_each_reflog_ent("refs/stash", stash_count_refs, &n); + return n; +} + static void wt_longstatus_print_stash_summary(struct wt_status *s) { - int stash_count = 0; + int stash_count = count_stash_entries(); - for_each_reflog_ent("refs/stash", stash_count_refs, &stash_count); if (stash_count > 0) status_printf_ln(s, GIT_COLOR_NORMAL, Q_("Your stash currently has %d entry", @@ -2177,6 +2183,18 @@ static void wt_porcelain_v2_print_tracking(struct wt_status *s) } /* + * Print the stash count in a porcelain-friendly format + */ +static void wt_porcelain_v2_print_stash(struct wt_status *s) +{ + int stash_count = count_stash_entries(); + char eol = s->null_termination ? '\0' : '\n'; + + if (stash_count > 0) + fprintf(s->fp, "# stash %d%c", stash_count, eol); +} + +/* * Convert various submodule status values into a * fixed-length string of characters in the buffer provided. */ @@ -2437,6 +2455,9 @@ static void wt_porcelain_v2_print(struct wt_status *s) if (s->show_branch) wt_porcelain_v2_print_tracking(s); + if (s->show_stash) + wt_porcelain_v2_print_stash(s); + for (i = 0; i < s->change.nr; i++) { it = &(s->change.items[i]); d = it->util; diff --git a/xdiff-interface.c b/xdiff-interface.c index 75b32aef51..2e3a5a2943 100644 --- a/xdiff-interface.c +++ b/xdiff-interface.c @@ -313,6 +313,8 @@ int git_xmerge_config(const char *var, const char *value, void *cb) die("'%s' is not a boolean", var); if (!strcmp(value, "diff3")) git_xmerge_style = XDL_MERGE_DIFF3; + else if (!strcmp(value, "zdiff3")) + git_xmerge_style = XDL_MERGE_ZEALOUS_DIFF3; else if (!strcmp(value, "merge")) git_xmerge_style = 0; /* diff --git a/xdiff/xdiff.h b/xdiff/xdiff.h index b29deca5de..72e25a9ffa 100644 --- a/xdiff/xdiff.h +++ b/xdiff/xdiff.h @@ -66,6 +66,7 @@ extern "C" { /* merge output styles */ #define XDL_MERGE_DIFF3 1 +#define XDL_MERGE_ZEALOUS_DIFF3 2 typedef struct s_mmfile { char *ptr; diff --git a/xdiff/xmerge.c b/xdiff/xmerge.c index 1659edb453..df0c604177 100644 --- a/xdiff/xmerge.c +++ b/xdiff/xmerge.c @@ -230,7 +230,7 @@ static int fill_conflict_hunk(xdfenv_t *xe1, const char *name1, size += xdl_recs_copy(xe1, m->i1, m->chg1, needs_cr, 1, dest ? dest + size : NULL); - if (style == XDL_MERGE_DIFF3) { + if (style == XDL_MERGE_DIFF3 || style == XDL_MERGE_ZEALOUS_DIFF3) { /* Shared preimage */ if (!dest) { size += marker_size + 1 + needs_cr + marker3_size; @@ -327,7 +327,7 @@ static int xdl_fill_merge_buffer(xdfenv_t *xe1, const char *name1, * lines. Try hard to show only these few lines as conflicting. */ static int xdl_refine_conflicts(xdfenv_t *xe1, xdfenv_t *xe2, xdmerge_t *m, - xpparam_t const *xpp) + xpparam_t const *xpp, int style) { for (; m; m = m->next) { mmfile_t t1, t2; @@ -368,6 +368,42 @@ static int xdl_refine_conflicts(xdfenv_t *xe1, xdfenv_t *xe2, xdmerge_t *m, continue; } x = xscr; + if (style == XDL_MERGE_ZEALOUS_DIFF3) { + int advance1 = xscr->i1, advance2 = xscr->i2; + + /* + * Advance m->i1 and m->i2 so that conflict for sides + * 1 and 2 start after common region. Decrement + * m->chg[12] since there are now fewer conflict lines + * for those sides. + */ + m->i1 += advance1; + m->i2 += advance2; + m->chg1 -= advance1; + m->chg2 -= advance2; + + /* + * Splitting conflicts due to internal common regions + * on the two sides would be inappropriate since we + * are also showing the merge base and have no + * reasonable way to split the merge base text. + */ + while (xscr->next) + xscr = xscr->next; + + /* + * Lower the number of conflict lines to not include + * the final common lines, if any. Do this by setting + * number of conflict lines to + * (line offset for start of conflict in xscr) + + * (number of lines in the conflict in xscr) + */ + m->chg1 = (xscr->i1 - advance1) + (xscr->chg1); + m->chg2 = (xscr->i2 - advance2) + (xscr->chg2); + xdl_free_env(&xe); + xdl_free_script(x); + continue; + } m->i1 = xscr->i1 + i1; m->chg1 = xscr->chg1; m->i2 = xscr->i2 + i2; @@ -419,6 +455,7 @@ static int lines_contain_alnum(xdfenv_t *xe, int i, int chg) static void xdl_merge_two_conflicts(xdmerge_t *m) { xdmerge_t *next_m = m->next; + m->chg0 = next_m->i0 + next_m->chg0 - m->i0; m->chg1 = next_m->i1 + next_m->chg1 - m->i1; m->chg2 = next_m->i2 + next_m->chg2 - m->i2; m->next = next_m->next; @@ -430,12 +467,12 @@ static void xdl_merge_two_conflicts(xdmerge_t *m) * it appears simpler -- because it takes up less (or as many) lines -- * if the lines are moved into the conflicts. */ -static int xdl_simplify_non_conflicts(xdfenv_t *xe1, xdmerge_t *m, +static int xdl_simplify_non_conflicts(xdfenv_t *xe1, xdmerge_t *m, int style, int simplify_if_no_alnum) { int result = 0; - if (!m) + if (!m || style == XDL_MERGE_ZEALOUS_DIFF3) return result; for (;;) { xdmerge_t *next_m = m->next; @@ -482,6 +519,25 @@ static int xdl_do_merge(xdfenv_t *xe1, xdchange_t *xscr1, int style = xmp->style; int favor = xmp->favor; + /* + * XDL_MERGE_DIFF3 does not attempt to refine conflicts by looking + * at common areas of sides 1 & 2, because the base (side 0) does + * not match and is being shown. Similarly, simplification of + * non-conflicts is also skipped due to the skipping of conflict + * refinement. + * + * XDL_MERGE_ZEALOUS_DIFF3, on the other hand, will attempt to + * refine conflicts looking for common areas of sides 1 & 2. + * However, since the base is being shown and does not match, + * it will only look for common areas at the beginning or end + * of the conflict block. Since XDL_MERGE_ZEALOUS_DIFF3's + * conflict refinement is much more limited in this fashion, the + * conflict simplification will be skipped. + * + * See xdl_refine_conflicts() and xdl_simplify_non_conflicts() + * for more details, particularly looking for + * XDL_MERGE_ZEALOUS_DIFF3. + */ if (style == XDL_MERGE_DIFF3) { /* * "diff3 -m" output does not make sense for anything @@ -604,8 +660,8 @@ static int xdl_do_merge(xdfenv_t *xe1, xdchange_t *xscr1, changes = c; /* refine conflicts */ if (XDL_MERGE_ZEALOUS <= level && - (xdl_refine_conflicts(xe1, xe2, changes, xpp) < 0 || - xdl_simplify_non_conflicts(xe1, changes, + (xdl_refine_conflicts(xe1, xe2, changes, xpp, style) < 0 || + xdl_simplify_non_conflicts(xe1, changes, style, XDL_MERGE_ZEALOUS < level) < 0)) { xdl_cleanup_merge(changes); return -1; |