From 40fce0b2c547fe7a73267f5285fb1f596f8ff947 Mon Sep 17 00:00:00 2001 From: Thomas Schwinge Date: Mon, 13 Jun 2011 16:44:12 +0200 Subject: open_issues/git_duplicated_content: Progress? --- hurd/translator/ext2fs.mdwn | 16 ++- hurd/translator/ext2fs/filetype.mdwn | 33 ++++++ open_issues/ext2fs_file_corruption.mdwn | 199 -------------------------------- open_issues/git-core-2.mdwn | 7 ++ open_issues/git_duplicated_content.mdwn | 125 ++++++++++++++++++++ 5 files changed, 177 insertions(+), 203 deletions(-) create mode 100644 hurd/translator/ext2fs/filetype.mdwn delete mode 100644 open_issues/ext2fs_file_corruption.mdwn create mode 100644 open_issues/git_duplicated_content.mdwn diff --git a/hurd/translator/ext2fs.mdwn b/hurd/translator/ext2fs.mdwn index 305576b8..fff2e74b 100644 --- a/hurd/translator/ext2fs.mdwn +++ b/hurd/translator/ext2fs.mdwn @@ -1,4 +1,4 @@ -[[!meta copyright="Copyright © 2007, 2008, 2010 Free Software Foundation, +[[!meta copyright="Copyright © 2007, 2008, 2010, 2011 Free Software Foundation, Inc."]] [[!meta license="""[[!toggle id="license" text="GFDL 1.2+"]][[!toggleable @@ -9,14 +9,20 @@ Sections, no Front-Cover Texts, and no Back-Cover Texts. A copy of the license is included in the section entitled [[GNU Free Documentation License|/fdl]]."]]"""]] -# Large Stores +# Implementation + + * [[filetype]] option + + +## Large Stores The `ext2fs` translator from the upstream Hurd code base can only handle file systems with sizes of less than roughly 2 GiB. [[!tag open_issue_hurd]] -## Ognyan's Work + +### Ognyan's Work * Ognyan Kulev, [[*Supporting Large ext2 File Systems in the Hurd*|ogi-fosdem2005.mgp]], 2005, at FOSDEM @@ -34,4 +40,6 @@ small backend stores, like floppy devices. # Documentation - + * + + * diff --git a/hurd/translator/ext2fs/filetype.mdwn b/hurd/translator/ext2fs/filetype.mdwn new file mode 100644 index 00000000..5d85bac9 --- /dev/null +++ b/hurd/translator/ext2fs/filetype.mdwn @@ -0,0 +1,33 @@ +[[!meta copyright="Copyright © 2011 Free Software Foundation, Inc."]] + +[[!meta license="""[[!toggle id="license" text="GFDL 1.2+"]][[!toggleable +id="license" text="Permission is granted to copy, distribute and/or modify this +document under the terms of the GNU Free Documentation License, Version 1.2 or +any later version published by the Free Software Foundation; with no Invariant +Sections, no Front-Cover Texts, and no Back-Cover Texts. A copy of the license +is included in the section entitled [[GNU Free Documentation +License|/fdl]]."]]"""]] + +The *ext2fs* translator doesn't support the ext2 format's *filetype* option. + +According to *mke2fs(8)*: + +> **filetype**: Store file type information in directory entries. + +By setting directory listings' informational `d_type` field (`readdir`, etc.), +this may avoid the need for subsequent `stat` calls. + +Not all file systems can support this option. + +In `[hurd]/ext2fs/dir.c` the `EXT2_FEATURE_INCOMPAT_FILETYPE` is generally +masked out (is not even considered) when adding a node to a directory in +`diskfs_direnter_hard` and when reading in `diskfs_get_directs`. The Hurd's +ext2fs unconditionally sets this field to 0 (`EXT2_FT_UNKNOWN`). + + +# `e2fsck` + +Running `e2fsck` on a file system with the *filetype* option, will correct the +*filetype* for a lot of files (all `EXT2_FT_UNKNOWN`?) to either 1 (regular +file, `EXT2_FT_REG_FILE`), or 2 (directory, `EXT2_FT_DIR`), and likely others. +The Hurd's ext2fs will again ignore these fields, of course. diff --git a/open_issues/ext2fs_file_corruption.mdwn b/open_issues/ext2fs_file_corruption.mdwn deleted file mode 100644 index 2af35bb5..00000000 --- a/open_issues/ext2fs_file_corruption.mdwn +++ /dev/null @@ -1,199 +0,0 @@ -[[!meta copyright="Copyright © 2011 Free Software Foundation, Inc."]] - -[[!meta license="""[[!toggle id="license" text="GFDL 1.2+"]][[!toggleable -id="license" text="Permission is granted to copy, distribute and/or modify this -document under the terms of the GNU Free Documentation License, Version 1.2 or -any later version published by the Free Software Foundation; with no Invariant -Sections, no Front-Cover Texts, and no Back-Cover Texts. A copy of the license -is included in the section entitled [[GNU Free Documentation -License|/fdl]]."]]"""]] - -[[!tag open_issue_hurd]] - -[[!toc]] - - -# Log - -Create two 5 GiB partitions. - - # # Would default to 256 byte inodes otherwise. - # mke2fs -I 128 -o linux /dev/hd1s1 - [...] - # mke2fs -o hurd /dev/hd1s2 - [...] - -The only difference (apart from UUID, time stamps, hash seeds) in ext2 file -system meta data, according to `dumpe2fs -h`: - - [...] - -Filesystem features: ext_attr resize_inode dir_index filetype sparse_super large_file - +Filesystem features: ext_attr resize_inode dir_index sparse_super large_file - Filesystem flags: signed_directory_hash - Default mount options: (none) - Filesystem state: clean - Errors behavior: Continue - -Filesystem OS type: Linux - +Filesystem OS type: Hurd - [...] - -According to *mke2fs(8)*: - -> *filetype* -> Store file type information in directory entries. - - # settrans -c /media/hd1s1 /hurd/ext2fs /dev/hd1s1 - # settrans -c /media/hd1s2 /hurd/ext2fs /dev/hd1s2 - # mkdir /media/hd1s{1,2}/tmp - # chmod 1777 /media/hd1s{1,2}/tmp - - $ git-new-workdir ~/tmp/binutils/git /media/hd1s1/tmp/master master - error: unable to create file gas/testsuite/gas/arm/attr-mfpu-vfpv3-d16.d (Interrupted system call) - Checking out files: 100% (12315/12315), done. - Already on 'master' - $ cd /media/hd1s1/tmp/master - $ git status - # On branch master - # Changes not staged for commit: - # (use "git add ..." to update what will be committed) - # (use "git checkout -- ..." to discard changes in working directory) - # - # modified: gas/testsuite/gas/arm/attr-mfpu-vfpv3-d16.d - # - no changes added to commit (use "git add" and/or "git commit -a") - $ git checkout -f - $ git status - # On branch master - nothing to commit (working directory clean) - -([[Git issue|git-core-2]] is known.) - - $ git-new-workdir ~/tmp/binutils/git /media/hd1s2/tmp/master master - error: unable to create file bfd/elf32-dlx.c (Interrupted system call) - error: unable to create file bfd/sunos.c (Interrupted system call) - error: unable to create file gas/testsuite/gas/arm/attr-mfpu-vfpv3-d16.d (Interrupted system call) - error: unable to create file gas/testsuite/gas/mmix/regx-op.d (Interrupted system call) - error: unable to create file gas/testsuite/gas/tic6x/reloc-bad-4.s (Interrupted system call) - error: unable to create file gold/testsuite/script_test_2.t (Interrupted system call) - error: unable to create file ld/testsuite/ld-mmix/loc7m.d (Interrupted system call) - error: unable to create file ld/testsuite/ld-powerpc/tlsexe.g (Interrupted system call) - Checking out files: 100% (12315/12315), done. - Already on 'master' - $ cd /media/hd1s2/tmp/master - $ git status - # On branch master - # Changes not staged for commit: - # (use "git add ..." to update what will be committed) - # (use "git checkout -- ..." to discard changes in working directory) - # - # modified: bfd/elf32-dlx.c - # modified: bfd/sunos.c - # modified: gas/testsuite/gas/arm/attr-mfpu-vfpv3-d16.d - # modified: gas/testsuite/gas/mmix/regx-op.d - # modified: gas/testsuite/gas/tic6x/reloc-bad-4.s - # modified: gold/testsuite/script_test_2.t - # modified: ld/testsuite/ld-mmix/loc7m.d - # modified: ld/testsuite/ld-powerpc/tlsexe.g - # - no changes added to commit (use "git add" and/or "git commit -a") - $ git checkout -f - $ git status - # On branch master - nothing to commit (working directory clean) - -Now you'd expect these directories to have identical content, but: - - $ diff -x .git -ru /media/hd1s{1,2}/tmp/master/ > /tmp/diff - $ ls -l /tmp/diff - -rw-r--r-- 1 thomas thomas 613677 10. Jun 19:12 /tmp/diff - $ grep '^[^ @+-]' < /tmp/diff - diff -x .git -ru /media/hd1s1/tmp/master//ld/configure /media/hd1s2/tmp/master//ld/configure - -(Note that this isn't a file that Git had issues with.) - -Try again: - - $ diff -x .git -ru /media/hd1s{1,2}/tmp/master/ > /tmp/diff_ - $ ls -l /tmp/diff* - -rw-r--r-- 1 thomas thomas 613677 10. Jun 19:12 /tmp/diff - -rw-r--r-- 1 thomas thomas 613677 10. Jun 19:17 /tmp/diff_ - $ cmp /tmp/diff{,_}; echo $? - 0 - -At least it's consistent. Force a reload: - - # settrans -ag /media/hd1s1 - # settrans -ag /media/hd1s2 - -Try again: - - $ diff -x .git -ru /media/hd1s{1,2}/tmp/master/ > /tmp/diff__ - $ ls -l /tmp/diff* - -rw-r--r-- 1 thomas thomas 613677 10. Jun 19:12 /tmp/diff - -rw-r--r-- 1 thomas thomas 613677 10. Jun 19:17 /tmp/diff_ - -rw-r--r-- 1 thomas thomas 613677 10. Jun 19:30 /tmp/diff__ - $ cmp /tmp/diff{,__}; echo $? - 0 - -Consistent; thus very likely corrupt on-disk. - -After a few tries, the pattern generally is that for the files where there are -differences, once the file regularely ends, its content appears once more. -That is, the files' content appears once (regularely), and then the same again. -From three retries, I've seen this only ever happen on the `-o -linux`/`filetype` option file system. - -Some more copying: - - $ (cd /media/hd1s1/tmp/ && cp -a master master_) - $ (cd /media/hd1s2/tmp/ && cp -a master master_) - $ diff -x .git -ru /media/hd1s1/tmp/master{,_}/ > /tmp/diff1 - $ diff -x .git -ru /media/hd1s2/tmp/master{,_}/ > /tmp/diff2 - $ ls -l /tmp/diff{1,2} - -rw-r--r-- 1 thomas thomas 0 10. Jun 19:46 /tmp/diff1 - -rw-r--r-- 1 thomas thomas 0 10. Jun 19:46 /tmp/diff2 - -No further difference. - - -# Conclusion? - -The ext2 `filetype` option, as used for Linux-type file systems by default, -might cause some confusion inside [[hurd/translator/ext2fs]]? - -[[!tag open_issue_documentation]] The strange thing is that in -`[hurd]/ext2fs/dir.c` the `EXT2_FEATURE_INCOMPAT_FILETYPE` is generally masked -out (is not even considered) when adding a node to a directory in -`diskfs_direnter_hard` and when reading in `diskfs_get_directs`, so unless -setting this file system flag has some other effects on the meta data (which it -doesn't seem to have according to the Linux kernel source code), it shouldn't -make any difference. Also, this information would only be used for setting -directory listing's informational `d_type` field (`readdir`, etc.). Hrm. -Perhaps only a red herring? - - -## `e2fsck` - -Running `e2fsck` on a file system with the `filetypes` option, there are -messages that the `filetype` for a lot of files is set (to either 1 (regular -file, `EXT2_FT_REG_FILE`), or 2 (directory, `EXT2_FT_DIR`)). The Hurd's ext2fs -should unconditionally set this field to 0 (`EXT2_FT_UNKNOWN`). - - -# Next - -Start over with the `filetype` option reversed: - - # mke2fs -I 128 -o linux -O ^filetype /dev/hd1s1 - # mke2fs -o hurd -O filetype /dev/hd1s2 - -In fact, for `-o hurd`, `-O filetype` doesn't have any effect (according to -`dumpe2fs -h`), so the meta data now only differs in *Filesystem OS type*. - -Invoke `git-new-workdir` once more (for each). - -Now there is corruption in files on the `-o hurd -O filetype` file system. But -this time not caused by `git-new-workdir`, but instead when copying a binutils -checkout to a new directory. Huh. Does `-O filetype` in fact have any effect -with `-o hurd`? The `e2fsprogs` source code should tell, but is not available -right now. diff --git a/open_issues/git-core-2.mdwn b/open_issues/git-core-2.mdwn index a1fdbac1..acd82442 100644 --- a/open_issues/git-core-2.mdwn +++ b/open_issues/git-core-2.mdwn @@ -133,3 +133,10 @@ starting from an empty working directory (after an external `git push`): $ git status # On branch master nothing to commit (working directory clean) + +--- + +2011-06-13 + +This seems to be fixed after a rebuild of the Debian git 1.7.5.3-1 with a +recent toolchain. diff --git a/open_issues/git_duplicated_content.mdwn b/open_issues/git_duplicated_content.mdwn new file mode 100644 index 00000000..3b4bc44b --- /dev/null +++ b/open_issues/git_duplicated_content.mdwn @@ -0,0 +1,125 @@ +[[!meta copyright="Copyright © 2011 Free Software Foundation, Inc."]] + +[[!meta license="""[[!toggle id="license" text="GFDL 1.2+"]][[!toggleable +id="license" text="Permission is granted to copy, distribute and/or modify this +document under the terms of the GNU Free Documentation License, Version 1.2 or +any later version published by the Free Software Foundation; with no Invariant +Sections, no Front-Cover Texts, and no Back-Cover Texts. A copy of the license +is included in the section entitled [[GNU Free Documentation +License|/fdl]]."]]"""]] + +[[!tag open_issue_porting]] + + $ git-new-workdir ~/tmp/binutils/git /media/hd1s1/tmp/master master + error: unable to create file gas/testsuite/gas/arm/attr-mfpu-vfpv3-d16.d (Interrupted system call) + Checking out files: 100% (12315/12315), done. + Already on 'master' + $ cd /media/hd1s1/tmp/master + $ git status + # On branch master + # Changes not staged for commit: + # (use "git add ..." to update what will be committed) + # (use "git checkout -- ..." to discard changes in working directory) + # + # modified: gas/testsuite/gas/arm/attr-mfpu-vfpv3-d16.d + # + no changes added to commit (use "git add" and/or "git commit -a") + $ git checkout -f + $ git status + # On branch master + nothing to commit (working directory clean) + +([[Git issue|git-core-2]] is known.) + + $ git-new-workdir ~/tmp/binutils/git /media/hd1s2/tmp/master master + error: unable to create file bfd/elf32-dlx.c (Interrupted system call) + error: unable to create file bfd/sunos.c (Interrupted system call) + error: unable to create file gas/testsuite/gas/arm/attr-mfpu-vfpv3-d16.d (Interrupted system call) + error: unable to create file gas/testsuite/gas/mmix/regx-op.d (Interrupted system call) + error: unable to create file gas/testsuite/gas/tic6x/reloc-bad-4.s (Interrupted system call) + error: unable to create file gold/testsuite/script_test_2.t (Interrupted system call) + error: unable to create file ld/testsuite/ld-mmix/loc7m.d (Interrupted system call) + error: unable to create file ld/testsuite/ld-powerpc/tlsexe.g (Interrupted system call) + Checking out files: 100% (12315/12315), done. + Already on 'master' + $ cd /media/hd1s2/tmp/master + $ git status + # On branch master + # Changes not staged for commit: + # (use "git add ..." to update what will be committed) + # (use "git checkout -- ..." to discard changes in working directory) + # + # modified: bfd/elf32-dlx.c + # modified: bfd/sunos.c + # modified: gas/testsuite/gas/arm/attr-mfpu-vfpv3-d16.d + # modified: gas/testsuite/gas/mmix/regx-op.d + # modified: gas/testsuite/gas/tic6x/reloc-bad-4.s + # modified: gold/testsuite/script_test_2.t + # modified: ld/testsuite/ld-mmix/loc7m.d + # modified: ld/testsuite/ld-powerpc/tlsexe.g + # + no changes added to commit (use "git add" and/or "git commit -a") + $ git checkout -f + $ git status + # On branch master + nothing to commit (working directory clean) + +Now you'd expect these directories to have identical content, but: + + $ diff -x .git -ru /media/hd1s{1,2}/tmp/master/ > /tmp/diff + $ ls -l /tmp/diff + -rw-r--r-- 1 thomas thomas 613677 10. Jun 19:12 /tmp/diff + $ grep '^[^ @+-]' < /tmp/diff + diff -x .git -ru /media/hd1s1/tmp/master//ld/configure /media/hd1s2/tmp/master//ld/configure + +(Note that this isn't a file that Git had issues with.) + +Try again: + + $ diff -x .git -ru /media/hd1s{1,2}/tmp/master/ > /tmp/diff_ + $ ls -l /tmp/diff* + -rw-r--r-- 1 thomas thomas 613677 10. Jun 19:12 /tmp/diff + -rw-r--r-- 1 thomas thomas 613677 10. Jun 19:17 /tmp/diff_ + $ cmp /tmp/diff{,_}; echo $? + 0 + +At least it's consistent. Force a reload: + + # settrans -ag /media/hd1s1 + # settrans -ag /media/hd1s2 + +Try again: + + $ diff -x .git -ru /media/hd1s{1,2}/tmp/master/ > /tmp/diff__ + $ ls -l /tmp/diff* + -rw-r--r-- 1 thomas thomas 613677 10. Jun 19:12 /tmp/diff + -rw-r--r-- 1 thomas thomas 613677 10. Jun 19:17 /tmp/diff_ + -rw-r--r-- 1 thomas thomas 613677 10. Jun 19:30 /tmp/diff__ + $ cmp /tmp/diff{,__}; echo $? + 0 + +Consistent; thus very likely corrupt on-disk. + +After a few tries, the pattern generally is that for the files where there are +differences, once the file regularely ends, its content appears once more. +That is, the files' content appears once (regularely), and then the same again. + +Some more copying: + + $ (cd /media/hd1s1/tmp/ && cp -a master master_) + $ (cd /media/hd1s2/tmp/ && cp -a master master_) + $ diff -x .git -ru /media/hd1s1/tmp/master{,_}/ > /tmp/diff1 + $ diff -x .git -ru /media/hd1s2/tmp/master{,_}/ > /tmp/diff2 + $ ls -l /tmp/diff{1,2} + -rw-r--r-- 1 thomas thomas 0 10. Jun 19:46 /tmp/diff1 + -rw-r--r-- 1 thomas thomas 0 10. Jun 19:46 /tmp/diff2 + +No further difference. + +--- + + $ git-new-workdir git master master + $ diff -x .git -ur tar_master/ master/ > master.diff + + $ rm -rf ar_master* && (cd git/ && git archive master) | (mkdir ar_master && cd ar_master/ && tar -x) && diff -x .git -ru tar_master/ ar_master/ > ar_master.diff; ls -l ar_master.diff + $ (cd git/ && git archive master) | md5sum -- cgit v1.2.3