Skip to content

Clamp threads on extraction, and stop four ways to lose data quietly #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ValisSowilo merged 1 commit into from
Sep 23, 2026
Merged

Clamp threads on extraction, and stop four ways to lose data quietly #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 91 additions & 3 deletions gleipnir.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3189,9 +3189,50 @@ static int is_dir(const char *p) {
return (st.st_mode & S_IFMT) == S_IFDIR;
}

/* True if a and b name the same file on disk, by identity rather than by
* spelling: "a.gl", "./a.gl" and a hard link to it all count. False when
* either does not exist or cannot be asked about. */
static int same_file(const char *a, const char *b) {
#ifdef _WIN32
BY_HANDLE_FILE_INFORMATION ia, ib;
DWORD share = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE;
HANDLE ha = CreateFileA(a, 0, share, NULL, OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS, NULL);
if (ha == INVALID_HANDLE_VALUE) return 0;
HANDLE hb = CreateFileA(b, 0, share, NULL, OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS, NULL);
if (hb == INVALID_HANDLE_VALUE) { CloseHandle(ha); return 0; }
int same = GetFileInformationByHandle(ha, &ia) &&
GetFileInformationByHandle(hb, &ib) &&
ia.dwVolumeSerialNumber == ib.dwVolumeSerialNumber &&
ia.nFileIndexHigh == ib.nFileIndexHigh &&
ia.nFileIndexLow == ib.nFileIndexLow;
CloseHandle(ha); CloseHandle(hb);
return same;
#else
STAT_T sa, sb;
return STAT_F(a, &sa) == 0 && STAT_F(b, &sb) == 0 &&
sa.st_dev == sb.st_dev && sa.st_ino == sb.st_ino;
#endif
}

static void walk(const char *path, Names *out) {
if (excluded(path)) return;
if (!is_dir(path)) { nm_add(out, path); return; }
if (!is_dir(path)) {
/* Only regular files have a size that means anything. A FIFO, a
* device or a /proc entry seeks to 0, so it used to be stored as an
* empty member with a valid SHA-256 and exit 0 -- `gleipnir c a.gl
* /dev/stdin` archived nothing and reported success. A path that
* cannot be stat'd at all is kept, so fopen reports the real error. */
STAT_T st;
if (STAT_F(path, &st) == 0 && (st.st_mode & S_IFMT) != S_IFREG) {
fprintf(stderr, "gleipnir: %s: not a regular file, skipped\n", path);
EXITCODE = 1;
return;
}
nm_add(out, path);
return;
}
#ifdef _WIN32
char pat[NAME_MAX_B];
snprintf(pat, sizeof pat, "%s\\*", path);
Expand Down Expand Up @@ -3413,6 +3454,24 @@ static int do_compress(char **paths, int npath, const char *outp, int lvl) {
}
files.n = w;
}

/* The output is truncated before any input is read, so an input that is
* the output -- `gleipnir c backup.gl .` run a second time, where the walk
* finds last run's backup.gl -- would be archived as a partial copy of
* the archive being written. Drop it, and say so. */
{
size_t w = 0;
for (size_t i = 0; i < files.n; i++) {
if (same_file(files.v[i], outp)) {
if (VERBOSE)
fprintf(stderr, "gleipnir: %s: is the output archive, skipped\n",
files.v[i]);
free(files.v[i]); free(snames[i]); continue;
}
files.v[w] = files.v[i]; snames[w] = snames[i]; w++;
}
files.n = w;
}
if (!files.n) { fprintf(stderr, "gleipnir: nothing to compress\n"); return 1; }

/* Two inputs can resolve to the same stored name -- `gen c a.gen /x/data
Expand Down Expand Up @@ -3771,6 +3830,11 @@ static int arc_open(const char *path, Archive *a) {
}
uint64_t ns = rd64(&r);
if (r.bad || ns > (1u << 28)) { free(ib); goto bad; }
/* The writer never emits a segment longer than the -s it records in the
* header (floored at MINCHUNK). Allowing anything up to 1 TB let a crafted
* index ask extraction for a terabyte per thread, which surfaced as "out
* of memory", exit 1, rather than as the corrupt archive it is. */
uint64_t seglim = a->h.segmax > MINCHUNK ? a->h.segmax : MINCHUNK;
for (uint64_t i = 0; i < ns; i++) {
Seg s;
s.off = rd64(&r); s.clen = rd64(&r);
Expand All @@ -3779,7 +3843,7 @@ static int arc_open(const char *path, Archive *a) {
/* A segment must lie entirely inside the payload region. Without this
* a corrupt offset would seek anywhere in the file and decode noise. */
if (s.off < HDR_BYTES || s.clen > ixoff || s.off > ixoff - s.clen ||
s.rawlen > (1ull << 40)) { r.bad = 1; break; }
s.rawlen > seglim) { r.bad = 1; break; }
idx_adds(&a->x, s);
}
/* Recovery records are optional and were added after the segment table, so
Expand Down Expand Up @@ -3923,7 +3987,23 @@ static int run_members(Archive *a, const char *destdir, int test_only,

for (uint64_t i = 0; i < a->x.nm; i++) grand += a->x.m[i].size;

int nthr = NTHREAD > 1 ? NTHREAD : 1;
/* The same clamp compression applies. Without it `x -t0` on a -9 archive
* asked for a full model per core -- more than compression ever did, since
* decoding holds more per worker -- and died in the allocator with no hint
* that the thread count was the cause. Geometry is not known until the
* segments are parsed, so size the model for a strided one, which is the
* largest (30 contexts at -9 against 27), and put the globals back. */
int nthr = 1;
if (NTHREAD > 1) {
uint64_t segbytes = 0;
for (uint64_t i = 0; i < a->x.ns; i++)
if (a->x.s[i].rawlen > segbytes) segbytes = a->x.s[i].rawlen;
int ds = DET_STRIDE, dw = DET_WIDTH;
DET_STRIDE = 1; DET_WIDTH = 1;
set_level(a->h.lvl);
nthr = threads_for(segbytes);
DET_STRIDE = ds; DET_WIDTH = dw;
}
SegR *sr = calloc((size_t)nthr, sizeof(SegR));
Job *jb = calloc((size_t)nthr, sizeof(Job));
Job *rj = calloc((size_t)nthr, sizeof(Job));
Expand Down Expand Up @@ -4531,6 +4611,14 @@ int main(int argc, char **argv) {
fprintf(stderr, "gleipnir: r needs an output archive\n");
arc_close(&ar); return 1;
}
/* do_repair opens its output "wb" before reading a single segment, so
* `gleipnir r bad.gl bad.gl` truncated the damaged archive to nothing
* and then tried to rebuild it from the empty file. */
if (same_file(arc, argv[a])) {
fprintf(stderr, "gleipnir: r would overwrite the archive it is "
"repairing; write to a different file\n");
arc_close(&ar); return 1;
}
rc = do_repair(&ar, argv[a]);
}
else if (mode == 't') rc = deep ? run_members(&ar, NULL, 1, NULL, 0)
Expand Down
10 changes: 7 additions & 3 deletions scripts/sfuzz.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ def seg_stored(payload):


def build(path, name, blob, rawlen, seghash=0, sha=b"\0" * 32, lvl=5,
memshift=0, segoff=None, seg0=0, nseg=1):
memshift=0, segoff=None, seg0=0, nseg=1, segmax=64 << 20):
"""A complete archive: one member, one segment, with a correct header hash,
index hash and trailer. Everything arc_open verifies is right, so the
decoder is always the thing under test."""
Expand All @@ -157,7 +157,7 @@ def build(path, name, blob, rawlen, seghash=0, sha=b"\0" * 32, lvl=5,
struct.pack_into("<IHH", hdr, 0, MAGIC, VERSION, 0)
hdr[8] = lvl & 0xFF
hdr[9] = (memshift + 16) & 0xFF
struct.pack_into("<I", hdr, 12, 64 << 20) # segmax
struct.pack_into("<I", hdr, 12, segmax) # segmax
struct.pack_into("<QQQ", hdr, 16, 1, idxoff, rawlen)
struct.pack_into("<Q", hdr, 40, xxh64(bytes(hdr[:40])))

Expand Down Expand Up @@ -205,9 +205,13 @@ def add(nm, note, blob, rawlen, **kw):
add("rawlen_over_wn_packed",
"packed: 64 working bytes, index claims 256 MB of symbols",
seg_model(bps=4, sym=bytes(range(16)), wn=64, blocks=((B_STORE, 64),),
sin=b"\x5a" * 64), 256 << 20)
sin=b"\x5a" * 64), 256 << 20, segmax=512 << 20)
add("rawlen_over_wn_plain", "plain: 4096 working bytes, index claims 64 MB",
seg_model(wn=4096, blocks=store1, sin=plain), 64 << 20)
# and against the header: the writer never emits a segment over its -s,
# so a claim past segmax is refused before anything is allocated for it
add("rawlen_over_segmax", "index claims a 1 TB segment under a 64 MB -s",
seg_model(wn=4096, blocks=store1, sin=plain), 1 << 40)
add("rawlen_zero_wn_large", "index claims an empty member, blob is not",
seg_model(wn=4096, blocks=store1, sin=plain), 0)

Expand Down
Loading